RubyGems - streamworker - Versions diffs - 0.0.2 - Mend

streamworker 0.0.2

Files changed (17) hide show

data/MIT-LICENSE +20 -0
data/Rakefile +28 -0
data/app/assets/javascripts/streamworker/application.js +15 -0
data/app/assets/stylesheets/streamworker/application.css +13 -0
data/app/controllers/streamworker/application_controller.rb +5 -0
data/app/helpers/streamworker/application_helper.rb +4 -0
data/app/views/layouts/streamworker/application.html.erb +14 -0
data/config/routes.rb +2 -0
data/lib/streamworker.rb +6 -0
data/lib/streamworker/engine.rb +7 -0
data/lib/streamworker/shopify.rb +5 -0
data/lib/streamworker/version.rb +3 -0
data/lib/streamworker/worker.rb +4 -0
data/lib/streamworker/workers/shopify_worker.rb +20 -0
data/lib/streamworker/workers/worker.rb +233 -0
data/lib/tasks/streamworker_tasks.rake +4 -0
metadata +147 -0

data/MIT-LICENSE ADDED

@@ -0,0 +1,20 @@
+Copyright 2013 YOURNAME
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile ADDED

@@ -0,0 +1,28 @@
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+require 'rdoc/task'
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'Streamworker'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+# APP_RAKEFILE = File.expand_path("../spec/internal/Rakefile", __FILE__)
+# load 'rails/tasks/engine.rake'
+Bundler::GemHelper.install_tasks
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
+task :default => :spec

data/app/assets/javascripts/streamworker/application.js ADDED

@@ -0,0 +1,15 @@
+// This is a manifest file that'll be compiled into application.js, which will include all the files
+// listed below.
+//
+// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
+// or vendor/assets/javascripts of plugins, if any, can be referenced here using a relative path.
+//
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// compiled file.
+//
+// WARNING: THE FIRST BLANK LINE MARKS THE END OF WHAT'S TO BE PROCESSED, ANY BLANK LINE SHOULD
+// GO AFTER THE REQUIRES BELOW.
+//
+//= require jquery
+//= require jquery_ujs
+//= require_tree .

data/app/assets/stylesheets/streamworker/application.css ADDED

@@ -0,0 +1,13 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or vendor/assets/stylesheets of plugins, if any, can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the top of the
+ * compiled file, but it's generally better to create a new file per style scope.
+ *
+ *= require_self
+ *= require_tree .
+ */

data/app/controllers/streamworker/application_controller.rb ADDED

@@ -0,0 +1,5 @@
+require 'streamworker'
+module Streamworker
+  class ApplicationController < ActionController::Base
+  end
+end

data/app/helpers/streamworker/application_helper.rb ADDED

@@ -0,0 +1,4 @@
+module Streamworker
+  module ApplicationHelper
+  end
+end

data/app/views/layouts/streamworker/application.html.erb ADDED

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>Streamworker</title>
+  <%= stylesheet_link_tag    "streamworker/application", media: "all" %>
+  <%= javascript_include_tag "streamworker/application" %>
+  <%= csrf_meta_tags %>
+</head>
+<body>
+<%= yield %>
+</body>
+</html>

data/config/routes.rb ADDED

	@@ -0,0 +1,2 @@
1	+ Streamworker::Engine.routes.draw do
2	+ end

data/lib/streamworker.rb ADDED

@@ -0,0 +1,6 @@
+require "streamworker/engine"
+require "streamworker/workers/worker"
+module Streamworker
+end

data/lib/streamworker/engine.rb ADDED

@@ -0,0 +1,7 @@
+require 'rails'
+module Streamworker
+  class Engine < ::Rails::Engine
+    isolate_namespace Streamworker
+  end
+end

data/lib/streamworker/shopify.rb ADDED

@@ -0,0 +1,5 @@
+require 'streamworker'
+require 'streamworker/workers/shopify_worker'
+module Shopify
+end

data/lib/streamworker/version.rb ADDED

@@ -0,0 +1,3 @@
+module Streamworker
+  VERSION = "0.0.2"
+end

data/lib/streamworker/worker.rb ADDED

@@ -0,0 +1,4 @@
+module Streamworker
+  module Workers
+  end
+end

data/lib/streamworker/workers/shopify_worker.rb ADDED

@@ -0,0 +1,20 @@
+module Streamworker
+  module Workers
+    class ShopifyWorker < Worker
+      attr_accessor :credit_threshold
+      def initialize(view_context, opts={})
+        @credit_threshold = 28
+        super view_context, opts
+      end
+      def with_shopify_session
+        ShopifyAPI::Base.activate_session(opts[:session])
+        yield
+      ensure
+        ShopifyAPI::Base.clear_session
+      end
+    end
+  end
+end

data/lib/streamworker/workers/worker.rb ADDED

@@ -0,0 +1,233 @@
+module Streamworker
+  module Workers
+    class Worker
+      include Enumerable
+      QUERIES_PER_BLOCK = 500
+      TIME_PER_BLOCK = 300
+      TIMEOUT_MARGIN = 5
+      attr_accessor :view_context, :opts
+      attr_accessor :repeats
+      attr_accessor :line_num # subclasses responsible for setting this as appropriate
+      attr_accessor :title
+      attr_accessor :num_records, :num_success, :num_errors
+      def initialize(view_context, opts={})
+        @opts = opts.with_indifferent_access
+        @view_context = view_context
+        @title = "Working..."
+        @repeats = opts[:repeats] || 1
+        @repeats = @repeats.to_i
+        @fragment = false
+        @started_at = Time.now
+        @opts[:unicorn_timeout] ||= ENV['UNICORN_TIMEOUT']
+        @opts[:unicorn_timeout] ||= 30
+        @opts[:unicorn_timeout] = @opts[:unicorn_timeout].to_i
+        @num_records = opts[:num_records].to_i || 1
+        @num_success = 0
+        @num_errors = 0
+      end
+      def queries_per_record
+        1
+      end
+      def projected_queries
+        (self.num_records * self.queries_per_record).to_f
+      end
+      def num_queries
+        (self.num_success + self.num_errors) .to_f * self.queries_per_record
+      end
+      def calculate_times
+        actual_time_used = Time.now - @started_at
+        work_time_remaining = opts[:unicorn_timeout] - actual_time_used
+        theoretical_total_time = (projected_queries / QUERIES_PER_BLOCK) * TIME_PER_BLOCK
+        theoretical_time_used = (num_queries / QUERIES_PER_BLOCK) * TIME_PER_BLOCK
+        factor = actual_time_used.to_f / theoretical_time_used
+        factor = [factor, 1].max if projected_queries > QUERIES_PER_BLOCK
+        total_time = theoretical_total_time * factor
+        # puts "--------- calculate_times ---------"
+        # puts "Time.now: #{Time.now.inspect}"
+        # puts "@started_at: #{@started_at.inspect}"
+        # puts "QUERIES_PER_BLOCK: #{QUERIES_PER_BLOCK.inspect}"
+        # puts "TIME_PER_BLOCK: #{TIME_PER_BLOCK.inspect}"
+        # puts "(self.num_records * self.queries_per_record): #{(self.num_records * self.queries_per_record).inspect}"
+        # puts "opts[:unicorn_timeout] : #{opts[:unicorn_timeout] .inspect}"
+        # puts "actual_time_used: #{actual_time_used.inspect}"
+        # puts "work_time_remaining: #{work_time_remaining.inspect}"
+        # puts "theoretical_total_time: #{theoretical_total_time.inspect}"
+        # puts "theoretical_time_used: #{theoretical_time_used.inspect}"
+        # puts "factor: #{factor.inspect}"
+        # puts "total_time: #{total_time.inspect}"
+        # puts "(total_time - actual_time_used): #{(total_time - actual_time_used).inspect}"
+        # puts
+        {
+          work_time: opts[:unicorn_timeout] .to_i,
+          work_time_remaining: work_time_remaining,
+          time_used: actual_time_used,
+          time_remaining: (total_time - actual_time_used),
+          total_time: total_time
+        }
+      end
+      def imminent_timeout?
+        # puts "--------- imminent_timeout ---------"
+        # puts "work_time_remaining: #{calculate_times[:work_time_remaining].inspect}"
+        # puts "TIMEOUT_MARGIN: #{TIMEOUT_MARGIN.inspect}"
+        calculate_times[:work_time_remaining] < TIMEOUT_MARGIN
+      end
+      def report_timeout_footer(msg={})
+        msg[:work_desc] ||= "#{num_success} records"
+        msg[:how_to_finish] ||= "resubmit the last #{num_records - num_success} records."
+        times = calculate_times
+        %Q{
+          </div>
+          <hr/>
+          <div class="alert alert-error alert_block span8">
+            <h4><i class="icon-time icon-large pull-left"></i>Server Timeout!</h4>
+            <br/>
+              Unfortunately, the backend processing time is limited to #{times[:work_time]} seconds, so we have to stop processing this job after #{msg[:work_desc]}.
+            <br/><br/>
+              To finish processing, please #{msg[:how_to_finish]}.
+          </div>
+          #{scroll}
+        </div>#{self.foot}
+        }
+      end
+      def set_headers(response)
+        response.headers['Last-Modified'] = Time.now.ctime.to_s
+        response.headers.delete('Content-Length')
+        response.headers['Cache-Control'] = 'no-cache'
+        response.headers['Transfer-Encoding'] = 'chunked'
+      end
+      def header
+        repeats =  ""
+        repeats = %Q{<p class="muted">Repeating #{@repeats} times</p>} if @repeats > 1
+        header = <<-EOHTML
+        #{self.head}<div class="container">
+            #{repeats}
+            <div class="import-results">
+        EOHTML
+        # Safari waits until it gets the first 1024 bytes to start displaying
+        Rails.logger.debug header
+        header + (" " * [0, (1025 - header.length)].max)
+      end
+      def head(scroll=true)
+        scroll = scroll ? "" : %Q{ style="overflow: hidden;"}
+        %Q{
+    <!DOCTYPE html>
+    <html class="white"#{scroll}>
+      <head>
+        #{view_context.stylesheet_link_tag('application')}
+        #{view_context.javascript_include_tag('application')}
+        #{view_context.javascript_include_tag('scroller')}
+        <title>#{self.title}</title>
+      </head>
+      <body class="stream-worker-results">
+        }
+      end
+      def footer(msg)
+        <<-EOHTML
+          </div>
+          <h3>#{msg}</h3>
+          #{scroll}
+        </div>#{self.foot}
+        EOHTML
+      end
+      def foot
+        %Q{
+      </body>
+    </html>
+      }
+      end
+      def scroll
+        %Q{<script type="text/javascript">
+              scrollBottom();
+              parent.update_stream_worker_progress(#{num_records}, #{num_success}, #{num_errors});
+            </script>
+        }
+      end
+      def open_report_line(str)
+        report_line(str, close: false)
+      end
+      def fragment?
+        @fragment
+      end
+      def report_fragment(str)
+        fragment? ? str : open_report_line(str)
+      end
+      def close_report_line
+        fragment? ? report_line("", close: true) : ""
+      end
+      def report_line(str, opts={})
+        # Rails.logger.info("report_line str: #{str.inspect} opts: #{opts.inspect} fragment?: #{fragment?.inspect}")
+        opts = {close: true}.merge(opts)
+        p_class = ["report-line", opts[:class]].compact.join(" ")
+        start = fragment? ?  "" : %Q{
+            <p class="#{p_class}">}
+        @fragment = ! opts[:close]
+        close = ""
+        if opts[:close]
+          close = %Q{</p>
+              #{scroll}
+    }
+        end
+        out = %Q{#{start}#{str}#{close}}
+        # Rails.logger.info("    out: #{out.inspect}")
+        out
+      end
+      def report_error(str, list=[])
+        err = %Q{
+            #{error_line_num}
+            <div class="alert alert-error">
+                <p class="text-error"><i class="icon-warning-sign icon-large"></i>#{str}</p>
+              }
+        err << %Q{<ul class="error-list">\n} unless list.empty?
+        list.each { |e| err << %Q{              <li>#{e}</li>\n} }
+        err << %Q{              </ul>\n} unless list.empty?
+        err << %Q{           </div>}
+        err
+      end
+      def success_line_num
+        %Q{<span class="badge badge-success badge-line-num">#{line_num}</span>}
+      end
+      def error_line_num
+        %Q{<span class="badge badge-important badge-line-num">#{line_num}</span>}
+      end
+      def each
+        raise "Worker subclasses must implement each to yield their output"
+      end
+    end
+  end
+end

data/lib/tasks/streamworker_tasks.rake ADDED

@@ -0,0 +1,4 @@
+# desc "Explaining what the task does"
+# task :streamworker do
+#   # Task goes here
+# end

metadata ADDED

@@ -0,0 +1,147 @@
+--- !ruby/object:Gem::Specification
+name: streamworker
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+  prerelease:
+platform: ruby
+authors:
+- Michael Johnston
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2013-07-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: unicorn
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: slim
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.0.0.pre.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.0.0.pre.8
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-rails
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Rails Engine that provides workers implemented as http streaming requests.
+email:
+- opensource@lastobelus.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- app/assets/javascripts/streamworker/application.js
+- app/assets/stylesheets/streamworker/application.css
+- app/controllers/streamworker/application_controller.rb
+- app/helpers/streamworker/application_helper.rb
+- app/views/layouts/streamworker/application.html.erb
+- config/routes.rb
+- lib/streamworker/engine.rb
+- lib/streamworker/shopify.rb
+- lib/streamworker/version.rb
+- lib/streamworker/worker.rb
+- lib/streamworker/workers/shopify_worker.rb
+- lib/streamworker/workers/worker.rb
+- lib/streamworker.rb
+- lib/tasks/streamworker_tasks.rake
+- MIT-LICENSE
+- Rakefile
+homepage: https://github.com/lastobelus/streamworker
+licenses: []
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 1094623444961662123
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 1094623444961662123
+requirements: []
+rubyforge_project:
+rubygems_version: 1.8.25
+signing_key:
+specification_version: 3
+summary: Rails Engine that provides workers implemented as http streaming requests.
+test_files: []