RubyGems - streamworker - Versions diffs - 0.0.2 - Mend

streamworker 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (17) hide show

data/MIT-LICENSE +20 -0
data/Rakefile +28 -0
data/app/assets/javascripts/streamworker/application.js +15 -0
data/app/assets/stylesheets/streamworker/application.css +13 -0
data/app/controllers/streamworker/application_controller.rb +5 -0
data/app/helpers/streamworker/application_helper.rb +4 -0
data/app/views/layouts/streamworker/application.html.erb +14 -0
data/config/routes.rb +2 -0
data/lib/streamworker.rb +6 -0
data/lib/streamworker/engine.rb +7 -0
data/lib/streamworker/shopify.rb +5 -0
data/lib/streamworker/version.rb +3 -0
data/lib/streamworker/worker.rb +4 -0
data/lib/streamworker/workers/shopify_worker.rb +20 -0
data/lib/streamworker/workers/worker.rb +233 -0
data/lib/tasks/streamworker_tasks.rake +4 -0
metadata +147 -0

data/MIT-LICENSE ADDED

@@ -0,0 +1,20 @@
+Copyright 2013 YOURNAME
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile ADDED

@@ -0,0 +1,28 @@
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+require 'rdoc/task'
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'Streamworker'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+# APP_RAKEFILE = File.expand_path("../spec/internal/Rakefile", __FILE__)
+# load 'rails/tasks/engine.rake'
+Bundler::GemHelper.install_tasks
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
+task :default => :spec

data/app/assets/javascripts/streamworker/application.js ADDED

@@ -0,0 +1,15 @@
+// This is a manifest file that'll be compiled into application.js, which will include all the files
+// listed below.
+//
+// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
+// or vendor/assets/javascripts of plugins, if any, can be referenced here using a relative path.
+//
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// compiled file.
+//
+// WARNING: THE FIRST BLANK LINE MARKS THE END OF WHAT'S TO BE PROCESSED, ANY BLANK LINE SHOULD
+// GO AFTER THE REQUIRES BELOW.
+//
+//= require jquery
+//= require jquery_ujs
+//= require_tree .

data/app/assets/stylesheets/streamworker/application.css ADDED

@@ -0,0 +1,13 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or vendor/assets/stylesheets of plugins, if any, can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the top of the
+ * compiled file, but it's generally better to create a new file per style scope.
+ *
+ *= require_self
+ *= require_tree .
+ */

data/app/controllers/streamworker/application_controller.rb ADDED

@@ -0,0 +1,5 @@
+require 'streamworker'
+module Streamworker
+  class ApplicationController < ActionController::Base
+  end
+end

data/app/helpers/streamworker/application_helper.rb ADDED

@@ -0,0 +1,4 @@
+module Streamworker
+  module ApplicationHelper
+  end
+end

data/app/views/layouts/streamworker/application.html.erb ADDED

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>Streamworker</title>
+  <%= stylesheet_link_tag    "streamworker/application", media: "all" %>
+  <%= javascript_include_tag "streamworker/application" %>
+  <%= csrf_meta_tags %>
+</head>
+<body>
+<%= yield %>
+</body>
+</html>

data/config/routes.rb ADDED

	@@ -0,0 +1,2 @@
1	+ Streamworker::Engine.routes.draw do
2	+ end

data/lib/streamworker.rb ADDED

@@ -0,0 +1,6 @@
+require "streamworker/engine"
+require "streamworker/workers/worker"
+module Streamworker
+end

data/lib/streamworker/engine.rb ADDED

@@ -0,0 +1,7 @@
+require 'rails'
+module Streamworker
+  class Engine < ::Rails::Engine
+    isolate_namespace Streamworker
+  end
+end

data/lib/streamworker/shopify.rb ADDED

@@ -0,0 +1,5 @@
+require 'streamworker'
+require 'streamworker/workers/shopify_worker'
+module Shopify
+end

data/lib/streamworker/version.rb ADDED

@@ -0,0 +1,3 @@
+module Streamworker
+  VERSION = "0.0.2"
+end

data/lib/streamworker/worker.rb ADDED

@@ -0,0 +1,4 @@
+module Streamworker
+  module Workers
+  end
+end

data/lib/streamworker/workers/shopify_worker.rb ADDED

@@ -0,0 +1,20 @@
+module Streamworker
+  module Workers
+    class ShopifyWorker < Worker
+      attr_accessor :credit_threshold
+      def initialize(view_context, opts={})
+        @credit_threshold = 28
+        super view_context, opts
+      end
+      def with_shopify_session
+        ShopifyAPI::Base.activate_session(opts[:session])
+        yield
+      ensure
+        ShopifyAPI::Base.clear_session
+      end
+    end
+  end
+end

data/lib/streamworker/workers/worker.rb ADDED

@@ -0,0 +1,233 @@
+module Streamworker
+  module Workers
+    class Worker
+      include Enumerable
+      QUERIES_PER_BLOCK = 500
+      TIME_PER_BLOCK = 300
+      TIMEOUT_MARGIN = 5
+      attr_accessor :view_context, :opts
+      attr_accessor :repeats
+      attr_accessor :line_num # subclasses responsible for setting this as appropriate
+      attr_accessor :title
+      attr_accessor :num_records, :num_success, :num_errors
+      def initialize(view_context, opts={})
+        @opts = opts.with_indifferent_access
+        @view_context = view_context
+        @title = "Working..."
+        @repeats = opts[:repeats] || 1
+        @repeats = @repeats.to_i
+        @fragment = false
+        @started_at = Time.now
+        @opts[:unicorn_timeout] ||= ENV['UNICORN_TIMEOUT']
+        @opts[:unicorn_timeout] ||= 30
+        @opts[:unicorn_timeout] = @opts[:unicorn_timeout].to_i
+        @num_records = opts[:num_records].to_i || 1
+        @num_success = 0
+        @num_errors = 0
+      end
+      def queries_per_record
+        1
+      end
+      def projected_queries
+        (self.num_records * self.queries_per_record).to_f
+      end
+      def num_queries
+        (self.num_success + self.num_errors) .to_f * self.queries_per_record
+      end
+      def calculate_times
+        actual_time_used = Time.now - @started_at
+        work_time_remaining = opts[:unicorn_timeout] - actual_time_used
+        theoretical_total_time = (projected_queries / QUERIES_PER_BLOCK) * TIME_PER_BLOCK
+        theoretical_time_used = (num_queries / QUERIES_PER_BLOCK) * TIME_PER_BLOCK
+        factor = actual_time_used.to_f / theoretical_time_used
+        factor = [factor, 1].max if projected_queries > QUERIES_PER_BLOCK
+        total_time = theoretical_total_time * factor
+        # puts "--------- calculate_times ---------"
+        # puts "Time.now: #{Time.now.inspect}"
+        # puts "@started_at: #{@started_at.inspect}"
+        # puts "QUERIES_PER_BLOCK: #{QUERIES_PER_BLOCK.inspect}"
+        # puts "TIME_PER_BLOCK: #{TIME_PER_BLOCK.inspect}"
+        # puts "(self.num_records * self.queries_per_record): #{(self.num_records * self.queries_per_record).inspect}"
+        # puts "opts[:unicorn_timeout] : #{opts[:unicorn_timeout] .inspect}"
+        # puts "actual_time_used: #{actual_time_used.inspect}"
+        # puts "work_time_remaining: #{work_time_remaining.inspect}"
+        # puts "theoretical_total_time: #{theoretical_total_time.inspect}"
+        # puts "theoretical_time_used: #{theoretical_time_used.inspect}"
+        # puts "factor: #{factor.inspect}"
+        # puts "total_time: #{total_time.inspect}"
+        # puts "(total_time - actual_time_used): #{(total_time - actual_time_used).inspect}"
+        # puts
+        {
+          work_time: opts[:unicorn_timeout] .to_i,
+          work_time_remaining: work_time_remaining,
+          time_used: actual_time_used,
+          time_remaining: (total_time - actual_time_used),
+          total_time: total_time
+        }
+      end
+      def imminent_timeout?
+        # puts "--------- imminent_timeout ---------"
+        # puts "work_time_remaining: #{calculate_times[:work_time_remaining].inspect}"
+        # puts "TIMEOUT_MARGIN: #{TIMEOUT_MARGIN.inspect}"
+        calculate_times[:work_time_remaining] < TIMEOUT_MARGIN
+      end
+      def report_timeout_footer(msg={})
+        msg[:work_desc] ||= "#{num_success} records"
+        msg[:how_to_finish] ||= "resubmit the last #{num_records - num_success} records."
+        times = calculate_times
+        %Q{
+          </div>
+          <hr/>
+          <div class="alert alert-error alert_block span8">
+            <h4><i class="icon-time icon-large pull-left"></i>Server Timeout!</h4>
+            <br/>
+              Unfortunately, the backend processing time is limited to #{times[:work_time]} seconds, so we have to stop processing this job after #{msg[:work_desc]}.
+            <br/><br/>
+              To finish processing, please #{msg[:how_to_finish]}.
+          </div>
+          #{scroll}
+        </div>#{self.foot}
+        }
+      end
+      def set_headers(response)
+        response.headers['Last-Modified'] = Time.now.ctime.to_s
+        response.headers.delete('Content-Length')
+        response.headers['Cache-Control'] = 'no-cache'
+        response.headers['Transfer-Encoding'] = 'chunked'
+      end
+      def header
+        repeats =  ""
+        repeats = %Q{<p class="muted">Repeating #{@repeats} times</p>} if @repeats > 1
+        header = <<-EOHTML
+        #{self.head}<div class="container">
+            #{repeats}
+            <div class="import-results">
+        EOHTML
+        # Safari waits until it gets the first 1024 bytes to start displaying
+        Rails.logger.debug header
+        header + (" " * [0, (1025 - header.length)].max)
+      end
+      def head(scroll=true)
+        scroll = scroll ? "" : %Q{ style="overflow: hidden;"}
+        %Q{
+    <!DOCTYPE html>
+    <html class="white"#{scroll}>
+      <head>
+        #{view_context.stylesheet_link_tag('application')}
+        #{view_context.javascript_include_tag('application')}
+        #{view_context.javascript_include_tag('scroller')}
+        <title>#{self.title}</title>
+      </head>
+      <body class="stream-worker-results">
+        }
+      end
+      def footer(msg)
+        <<-EOHTML
+          </div>
+          <h3>#{msg}</h3>
+          #{scroll}
+        </div>#{self.foot}
+        EOHTML
+      end
+      def foot
+        %Q{
+      </body>
+    </html>
+      }
+      end
+      def scroll
+        %Q{<script type="text/javascript">
+              scrollBottom();
+              parent.update_stream_worker_progress(#{num_records}, #{num_success}, #{num_errors});
+            </script>
+        }
+      end
+      def open_report_line(str)
+        report_line(str, close: false)
+      end
+      def fragment?
+        @fragment
+      end
+      def report_fragment(str)
+        fragment? ? str : open_report_line(str)
+      end
+      def close_report_line
+        fragment? ? report_line("", close: true) : ""
+      end
+      def report_line(str, opts={})
+        # Rails.logger.info("report_line str: #{str.inspect} opts: #{opts.inspect} fragment?: #{fragment?.inspect}")
+        opts = {close: true}.merge(opts)
+        p_class = ["report-line", opts[:class]].compact.join(" ")
+        start = fragment? ?  "" : %Q{
+            <p class="#{p_class}">}
+        @fragment = ! opts[:close]
+        close = ""
+        if opts[:close]
+          close = %Q{</p>
+              #{scroll}
+    }
+        end
+        out = %Q{#{start}#{str}#{close}}
+        # Rails.logger.info("    out: #{out.inspect}")
+        out
+      end
+      def report_error(str, list=[])
+        err = %Q{
+            #{error_line_num}
+            <div class="alert alert-error">
+                <p class="text-error"><i class="icon-warning-sign icon-large"></i>#{str}</p>
+              }
+        err << %Q{<ul class="error-list">\n} unless list.empty?
+        list.each { |e| err << %Q{              <li>#{e}</li>\n} }
+        err << %Q{              </ul>\n} unless list.empty?
+        err << %Q{           </div>}
+        err
+      end
+      def success_line_num
+        %Q{<span class="badge badge-success badge-line-num">#{line_num}</span>}
+      end
+      def error_line_num
+        %Q{<span class="badge badge-important badge-line-num">#{line_num}</span>}
+      end
+      def each
+        raise "Worker subclasses must implement each to yield their output"
+      end
+    end
+  end
+end

data/lib/tasks/streamworker_tasks.rake ADDED

@@ -0,0 +1,4 @@
+# desc "Explaining what the task does"
+# task :streamworker do
+#   # Task goes here
+# end

metadata ADDED

@@ -0,0 +1,147 @@
+--- !ruby/object:Gem::Specification
+name: streamworker
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+  prerelease:
+platform: ruby
+authors:
+- Michael Johnston
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2013-07-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: unicorn
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: slim
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.0.0.pre.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.0.0.pre.8
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-rails
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Rails Engine that provides workers implemented as http streaming requests.
+email:
+- opensource@lastobelus.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- app/assets/javascripts/streamworker/application.js
+- app/assets/stylesheets/streamworker/application.css
+- app/controllers/streamworker/application_controller.rb
+- app/helpers/streamworker/application_helper.rb
+- app/views/layouts/streamworker/application.html.erb
+- config/routes.rb
+- lib/streamworker/engine.rb
+- lib/streamworker/shopify.rb
+- lib/streamworker/version.rb
+- lib/streamworker/worker.rb
+- lib/streamworker/workers/shopify_worker.rb
+- lib/streamworker/workers/worker.rb
+- lib/streamworker.rb
+- lib/tasks/streamworker_tasks.rake
+- MIT-LICENSE
+- Rakefile
+homepage: https://github.com/lastobelus/streamworker
+licenses: []
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 1094623444961662123
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 1094623444961662123
+requirements: []
+rubyforge_project:
+rubygems_version: 1.8.25
+signing_key:
+specification_version: 3
+summary: Rails Engine that provides workers implemented as http streaming requests.
+test_files: []