streamworker 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,20 @@
1
+ Copyright 2013 YOURNAME
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining
4
+ a copy of this software and associated documentation files (the
5
+ "Software"), to deal in the Software without restriction, including
6
+ without limitation the rights to use, copy, modify, merge, publish,
7
+ distribute, sublicense, and/or sell copies of the Software, and to
8
+ permit persons to whom the Software is furnished to do so, subject to
9
+ the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be
12
+ included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,28 @@
1
+ begin
2
+ require 'bundler/setup'
3
+ rescue LoadError
4
+ puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
5
+ end
6
+
7
+ require 'rdoc/task'
8
+
9
+ RDoc::Task.new(:rdoc) do |rdoc|
10
+ rdoc.rdoc_dir = 'rdoc'
11
+ rdoc.title = 'Streamworker'
12
+ rdoc.options << '--line-numbers'
13
+ rdoc.rdoc_files.include('README.rdoc')
14
+ rdoc.rdoc_files.include('lib/**/*.rb')
15
+ end
16
+
17
+ # APP_RAKEFILE = File.expand_path("../spec/internal/Rakefile", __FILE__)
18
+ # load 'rails/tasks/engine.rake'
19
+
20
+
21
+
22
+ Bundler::GemHelper.install_tasks
23
+
24
+ require 'rspec/core/rake_task'
25
+
26
+ RSpec::Core::RakeTask.new(:spec)
27
+
28
+ task :default => :spec
@@ -0,0 +1,15 @@
1
+ // This is a manifest file that'll be compiled into application.js, which will include all the files
2
+ // listed below.
3
+ //
4
+ // Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
5
+ // or vendor/assets/javascripts of plugins, if any, can be referenced here using a relative path.
6
+ //
7
+ // It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
8
+ // compiled file.
9
+ //
10
+ // WARNING: THE FIRST BLANK LINE MARKS THE END OF WHAT'S TO BE PROCESSED, ANY BLANK LINE SHOULD
11
+ // GO AFTER THE REQUIRES BELOW.
12
+ //
13
+ //= require jquery
14
+ //= require jquery_ujs
15
+ //= require_tree .
@@ -0,0 +1,13 @@
1
+ /*
2
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
3
+ * listed below.
4
+ *
5
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
6
+ * or vendor/assets/stylesheets of plugins, if any, can be referenced here using a relative path.
7
+ *
8
+ * You're free to add application-wide styles to this file and they'll appear at the top of the
9
+ * compiled file, but it's generally better to create a new file per style scope.
10
+ *
11
+ *= require_self
12
+ *= require_tree .
13
+ */
@@ -0,0 +1,5 @@
1
+ require 'streamworker'
2
+ module Streamworker
3
+ class ApplicationController < ActionController::Base
4
+ end
5
+ end
@@ -0,0 +1,4 @@
1
+ module Streamworker
2
+ module ApplicationHelper
3
+ end
4
+ end
@@ -0,0 +1,14 @@
1
+ <!DOCTYPE html>
2
+ <html>
3
+ <head>
4
+ <title>Streamworker</title>
5
+ <%= stylesheet_link_tag "streamworker/application", media: "all" %>
6
+ <%= javascript_include_tag "streamworker/application" %>
7
+ <%= csrf_meta_tags %>
8
+ </head>
9
+ <body>
10
+
11
+ <%= yield %>
12
+
13
+ </body>
14
+ </html>
@@ -0,0 +1,2 @@
1
+ Streamworker::Engine.routes.draw do
2
+ end
@@ -0,0 +1,6 @@
1
+ require "streamworker/engine"
2
+
3
+ require "streamworker/workers/worker"
4
+
5
+ module Streamworker
6
+ end
@@ -0,0 +1,7 @@
1
+ require 'rails'
2
+
3
+ module Streamworker
4
+ class Engine < ::Rails::Engine
5
+ isolate_namespace Streamworker
6
+ end
7
+ end
@@ -0,0 +1,5 @@
1
+ require 'streamworker'
2
+ require 'streamworker/workers/shopify_worker'
3
+ module Shopify
4
+ end
5
+
@@ -0,0 +1,3 @@
1
+ module Streamworker
2
+ VERSION = "0.0.2"
3
+ end
@@ -0,0 +1,4 @@
1
+ module Streamworker
2
+ module Workers
3
+ end
4
+ end
@@ -0,0 +1,20 @@
1
+ module Streamworker
2
+ module Workers
3
+ class ShopifyWorker < Worker
4
+
5
+ attr_accessor :credit_threshold
6
+
7
+ def initialize(view_context, opts={})
8
+ @credit_threshold = 28
9
+ super view_context, opts
10
+ end
11
+
12
+ def with_shopify_session
13
+ ShopifyAPI::Base.activate_session(opts[:session])
14
+ yield
15
+ ensure
16
+ ShopifyAPI::Base.clear_session
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,233 @@
1
+ module Streamworker
2
+ module Workers
3
+ class Worker
4
+ include Enumerable
5
+
6
+ QUERIES_PER_BLOCK = 500
7
+ TIME_PER_BLOCK = 300
8
+
9
+ TIMEOUT_MARGIN = 5
10
+ attr_accessor :view_context, :opts
11
+ attr_accessor :repeats
12
+ attr_accessor :line_num # subclasses responsible for setting this as appropriate
13
+ attr_accessor :title
14
+ attr_accessor :num_records, :num_success, :num_errors
15
+
16
+ def initialize(view_context, opts={})
17
+ @opts = opts.with_indifferent_access
18
+ @view_context = view_context
19
+
20
+ @title = "Working..."
21
+ @repeats = opts[:repeats] || 1
22
+ @repeats = @repeats.to_i
23
+ @fragment = false
24
+ @started_at = Time.now
25
+ @opts[:unicorn_timeout] ||= ENV['UNICORN_TIMEOUT']
26
+ @opts[:unicorn_timeout] ||= 30
27
+ @opts[:unicorn_timeout] = @opts[:unicorn_timeout].to_i
28
+ @num_records = opts[:num_records].to_i || 1
29
+ @num_success = 0
30
+ @num_errors = 0
31
+ end
32
+
33
+
34
+ def queries_per_record
35
+ 1
36
+ end
37
+
38
+ def projected_queries
39
+ (self.num_records * self.queries_per_record).to_f
40
+ end
41
+
42
+ def num_queries
43
+ (self.num_success + self.num_errors) .to_f * self.queries_per_record
44
+ end
45
+
46
+ def calculate_times
47
+ actual_time_used = Time.now - @started_at
48
+ work_time_remaining = opts[:unicorn_timeout] - actual_time_used
49
+ theoretical_total_time = (projected_queries / QUERIES_PER_BLOCK) * TIME_PER_BLOCK
50
+ theoretical_time_used = (num_queries / QUERIES_PER_BLOCK) * TIME_PER_BLOCK
51
+ factor = actual_time_used.to_f / theoretical_time_used
52
+ factor = [factor, 1].max if projected_queries > QUERIES_PER_BLOCK
53
+ total_time = theoretical_total_time * factor
54
+
55
+ # puts "--------- calculate_times ---------"
56
+ # puts "Time.now: #{Time.now.inspect}"
57
+ # puts "@started_at: #{@started_at.inspect}"
58
+ # puts "QUERIES_PER_BLOCK: #{QUERIES_PER_BLOCK.inspect}"
59
+ # puts "TIME_PER_BLOCK: #{TIME_PER_BLOCK.inspect}"
60
+ # puts "(self.num_records * self.queries_per_record): #{(self.num_records * self.queries_per_record).inspect}"
61
+ # puts "opts[:unicorn_timeout] : #{opts[:unicorn_timeout] .inspect}"
62
+ # puts "actual_time_used: #{actual_time_used.inspect}"
63
+ # puts "work_time_remaining: #{work_time_remaining.inspect}"
64
+ # puts "theoretical_total_time: #{theoretical_total_time.inspect}"
65
+ # puts "theoretical_time_used: #{theoretical_time_used.inspect}"
66
+ # puts "factor: #{factor.inspect}"
67
+ # puts "total_time: #{total_time.inspect}"
68
+ # puts "(total_time - actual_time_used): #{(total_time - actual_time_used).inspect}"
69
+ # puts
70
+ {
71
+ work_time: opts[:unicorn_timeout] .to_i,
72
+ work_time_remaining: work_time_remaining,
73
+ time_used: actual_time_used,
74
+ time_remaining: (total_time - actual_time_used),
75
+ total_time: total_time
76
+ }
77
+ end
78
+
79
+ def imminent_timeout?
80
+ # puts "--------- imminent_timeout ---------"
81
+ # puts "work_time_remaining: #{calculate_times[:work_time_remaining].inspect}"
82
+ # puts "TIMEOUT_MARGIN: #{TIMEOUT_MARGIN.inspect}"
83
+ calculate_times[:work_time_remaining] < TIMEOUT_MARGIN
84
+ end
85
+
86
+ def report_timeout_footer(msg={})
87
+ msg[:work_desc] ||= "#{num_success} records"
88
+ msg[:how_to_finish] ||= "resubmit the last #{num_records - num_success} records."
89
+ times = calculate_times
90
+ %Q{
91
+ </div>
92
+ <hr/>
93
+ <div class="alert alert-error alert_block span8">
94
+
95
+ <h4><i class="icon-time icon-large pull-left"></i>Server Timeout!</h4>
96
+ <br/>
97
+ Unfortunately, the backend processing time is limited to #{times[:work_time]} seconds, so we have to stop processing this job after #{msg[:work_desc]}.
98
+
99
+ <br/><br/>
100
+ To finish processing, please #{msg[:how_to_finish]}.
101
+
102
+ </div>
103
+ #{scroll}
104
+ </div>#{self.foot}
105
+ }
106
+ end
107
+
108
+ def set_headers(response)
109
+ response.headers['Last-Modified'] = Time.now.ctime.to_s
110
+ response.headers.delete('Content-Length')
111
+ response.headers['Cache-Control'] = 'no-cache'
112
+ response.headers['Transfer-Encoding'] = 'chunked'
113
+ end
114
+
115
+ def header
116
+ repeats = ""
117
+ repeats = %Q{<p class="muted">Repeating #{@repeats} times</p>} if @repeats > 1
118
+
119
+ header = <<-EOHTML
120
+ #{self.head}<div class="container">
121
+ #{repeats}
122
+ <div class="import-results">
123
+ EOHTML
124
+ # Safari waits until it gets the first 1024 bytes to start displaying
125
+ Rails.logger.debug header
126
+
127
+ header + (" " * [0, (1025 - header.length)].max)
128
+ end
129
+
130
+ def head(scroll=true)
131
+ scroll = scroll ? "" : %Q{ style="overflow: hidden;"}
132
+ %Q{
133
+ <!DOCTYPE html>
134
+ <html class="white"#{scroll}>
135
+ <head>
136
+ #{view_context.stylesheet_link_tag('application')}
137
+ #{view_context.javascript_include_tag('application')}
138
+ #{view_context.javascript_include_tag('scroller')}
139
+ <title>#{self.title}</title>
140
+ </head>
141
+ <body class="stream-worker-results">
142
+ }
143
+ end
144
+
145
+ def footer(msg)
146
+ <<-EOHTML
147
+ </div>
148
+ <h3>#{msg}</h3>
149
+ #{scroll}
150
+
151
+ </div>#{self.foot}
152
+ EOHTML
153
+ end
154
+
155
+ def foot
156
+ %Q{
157
+ </body>
158
+ </html>
159
+ }
160
+ end
161
+
162
+ def scroll
163
+ %Q{<script type="text/javascript">
164
+ scrollBottom();
165
+ parent.update_stream_worker_progress(#{num_records}, #{num_success}, #{num_errors});
166
+ </script>
167
+ }
168
+ end
169
+
170
+ def open_report_line(str)
171
+ report_line(str, close: false)
172
+ end
173
+
174
+ def fragment?
175
+ @fragment
176
+ end
177
+
178
+ def report_fragment(str)
179
+ fragment? ? str : open_report_line(str)
180
+ end
181
+
182
+ def close_report_line
183
+ fragment? ? report_line("", close: true) : ""
184
+ end
185
+
186
+ def report_line(str, opts={})
187
+ # Rails.logger.info("report_line str: #{str.inspect} opts: #{opts.inspect} fragment?: #{fragment?.inspect}")
188
+ opts = {close: true}.merge(opts)
189
+ p_class = ["report-line", opts[:class]].compact.join(" ")
190
+ start = fragment? ? "" : %Q{
191
+ <p class="#{p_class}">}
192
+ @fragment = ! opts[:close]
193
+ close = ""
194
+ if opts[:close]
195
+ close = %Q{</p>
196
+ #{scroll}
197
+ }
198
+ end
199
+ out = %Q{#{start}#{str}#{close}}
200
+ # Rails.logger.info(" out: #{out.inspect}")
201
+ out
202
+ end
203
+
204
+
205
+ def report_error(str, list=[])
206
+ err = %Q{
207
+ #{error_line_num}
208
+ <div class="alert alert-error">
209
+ <p class="text-error"><i class="icon-warning-sign icon-large"></i>#{str}</p>
210
+ }
211
+ err << %Q{<ul class="error-list">\n} unless list.empty?
212
+ list.each { |e| err << %Q{ <li>#{e}</li>\n} }
213
+ err << %Q{ </ul>\n} unless list.empty?
214
+ err << %Q{ </div>}
215
+
216
+ err
217
+ end
218
+
219
+ def success_line_num
220
+ %Q{<span class="badge badge-success badge-line-num">#{line_num}</span>}
221
+ end
222
+
223
+ def error_line_num
224
+ %Q{<span class="badge badge-important badge-line-num">#{line_num}</span>}
225
+ end
226
+
227
+ def each
228
+ raise "Worker subclasses must implement each to yield their output"
229
+ end
230
+
231
+ end
232
+ end
233
+ end
@@ -0,0 +1,4 @@
1
+ # desc "Explaining what the task does"
2
+ # task :streamworker do
3
+ # # Task goes here
4
+ # end
metadata ADDED
@@ -0,0 +1,147 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: streamworker
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.2
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - Michael Johnston
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2013-07-19 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: rails
16
+ requirement: !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ~>
20
+ - !ruby/object:Gem::Version
21
+ version: '3.2'
22
+ type: :runtime
23
+ prerelease: false
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ~>
28
+ - !ruby/object:Gem::Version
29
+ version: '3.2'
30
+ - !ruby/object:Gem::Dependency
31
+ name: unicorn
32
+ requirement: !ruby/object:Gem::Requirement
33
+ none: false
34
+ requirements:
35
+ - - ! '>='
36
+ - !ruby/object:Gem::Version
37
+ version: '0'
38
+ type: :runtime
39
+ prerelease: false
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ! '>='
44
+ - !ruby/object:Gem::Version
45
+ version: '0'
46
+ - !ruby/object:Gem::Dependency
47
+ name: slim
48
+ requirement: !ruby/object:Gem::Requirement
49
+ none: false
50
+ requirements:
51
+ - - ! '>='
52
+ - !ruby/object:Gem::Version
53
+ version: 2.0.0.pre.8
54
+ type: :runtime
55
+ prerelease: false
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ none: false
58
+ requirements:
59
+ - - ! '>='
60
+ - !ruby/object:Gem::Version
61
+ version: 2.0.0.pre.8
62
+ - !ruby/object:Gem::Dependency
63
+ name: sqlite3
64
+ requirement: !ruby/object:Gem::Requirement
65
+ none: false
66
+ requirements:
67
+ - - ! '>='
68
+ - !ruby/object:Gem::Version
69
+ version: '0'
70
+ type: :development
71
+ prerelease: false
72
+ version_requirements: !ruby/object:Gem::Requirement
73
+ none: false
74
+ requirements:
75
+ - - ! '>='
76
+ - !ruby/object:Gem::Version
77
+ version: '0'
78
+ - !ruby/object:Gem::Dependency
79
+ name: rspec-rails
80
+ requirement: !ruby/object:Gem::Requirement
81
+ none: false
82
+ requirements:
83
+ - - ! '>='
84
+ - !ruby/object:Gem::Version
85
+ version: '0'
86
+ type: :development
87
+ prerelease: false
88
+ version_requirements: !ruby/object:Gem::Requirement
89
+ none: false
90
+ requirements:
91
+ - - ! '>='
92
+ - !ruby/object:Gem::Version
93
+ version: '0'
94
+ description: Rails Engine that provides workers implemented as http streaming requests.
95
+ email:
96
+ - opensource@lastobelus.com
97
+ executables: []
98
+ extensions: []
99
+ extra_rdoc_files: []
100
+ files:
101
+ - app/assets/javascripts/streamworker/application.js
102
+ - app/assets/stylesheets/streamworker/application.css
103
+ - app/controllers/streamworker/application_controller.rb
104
+ - app/helpers/streamworker/application_helper.rb
105
+ - app/views/layouts/streamworker/application.html.erb
106
+ - config/routes.rb
107
+ - lib/streamworker/engine.rb
108
+ - lib/streamworker/shopify.rb
109
+ - lib/streamworker/version.rb
110
+ - lib/streamworker/worker.rb
111
+ - lib/streamworker/workers/shopify_worker.rb
112
+ - lib/streamworker/workers/worker.rb
113
+ - lib/streamworker.rb
114
+ - lib/tasks/streamworker_tasks.rake
115
+ - MIT-LICENSE
116
+ - Rakefile
117
+ homepage: https://github.com/lastobelus/streamworker
118
+ licenses: []
119
+ post_install_message:
120
+ rdoc_options: []
121
+ require_paths:
122
+ - lib
123
+ required_ruby_version: !ruby/object:Gem::Requirement
124
+ none: false
125
+ requirements:
126
+ - - ! '>='
127
+ - !ruby/object:Gem::Version
128
+ version: '0'
129
+ segments:
130
+ - 0
131
+ hash: 1094623444961662123
132
+ required_rubygems_version: !ruby/object:Gem::Requirement
133
+ none: false
134
+ requirements:
135
+ - - ! '>='
136
+ - !ruby/object:Gem::Version
137
+ version: '0'
138
+ segments:
139
+ - 0
140
+ hash: 1094623444961662123
141
+ requirements: []
142
+ rubyforge_project:
143
+ rubygems_version: 1.8.25
144
+ signing_key:
145
+ specification_version: 3
146
+ summary: Rails Engine that provides workers implemented as http streaming requests.
147
+ test_files: []