iron_worker 2.4.3 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -0
  3. data/README.md +352 -0
  4. data/iron_worker.gemspec +3 -8
  5. data/lib/iron_worker.rb +13 -51
  6. data/lib/iron_worker/api_client.rb +152 -0
  7. data/lib/iron_worker/version.rb +9 -1
  8. data/lib/iron_worker/worker_helper.rb +72 -0
  9. metadata +18 -183
  10. data/README.markdown +0 -73
  11. data/lib/generators/iron_worker/iron_worker_generator.rb +0 -13
  12. data/lib/generators/iron_worker/templates/template_worker.erb +0 -7
  13. data/lib/iron_worker/api.rb +0 -244
  14. data/lib/iron_worker/base.rb +0 -449
  15. data/lib/iron_worker/config.rb +0 -287
  16. data/lib/iron_worker/rails2_init.rb +0 -8
  17. data/lib/iron_worker/railtie.rb +0 -16
  18. data/lib/iron_worker/server/overrides.rb +0 -198
  19. data/lib/iron_worker/server/runner.rb +0 -82
  20. data/lib/iron_worker/service.rb +0 -644
  21. data/lib/iron_worker/used_in_worker.rb +0 -11
  22. data/lib/iron_worker/utils.rb +0 -11
  23. data/rails/init.rb +0 -0
  24. data/test/Gemfile +0 -36
  25. data/test/Gemfile.lock +0 -169
  26. data/test/README.md +0 -5
  27. data/test/batch_run.rb +0 -121
  28. data/test/config_sample.yml +0 -13
  29. data/test/models/cool_model.rb +0 -7
  30. data/test/models/db_model.rb +0 -6
  31. data/test/models/model_1.rb +0 -16
  32. data/test/models/model_2.rb +0 -16
  33. data/test/quick_run.rb +0 -36
  34. data/test/quick_schedule.rb +0 -18
  35. data/test/resources/something.yml +0 -4
  36. data/test/simple_script.rb +0 -11
  37. data/test/test_base.rb +0 -79
  38. data/test/test_database.rb +0 -31
  39. data/test/test_gems.rb +0 -31
  40. data/test/test_inheritance.rb +0 -25
  41. data/test/test_iron_worker.rb +0 -194
  42. data/test/test_merging.rb +0 -51
  43. data/test/test_non_worker_script.rb +0 -14
  44. data/test/test_rails.rb +0 -35
  45. data/test/test_scheduling.rb +0 -74
  46. data/test/test_scheduling_no_active_support.rb +0 -37
  47. data/test/test_service.rb +0 -28
  48. data/test/test_uploads.rb +0 -30
  49. data/test/test_webhooks.rb +0 -45
  50. data/test/v2quick.rb +0 -29
  51. data/test/workers/awesome_job.rb +0 -26
  52. data/test/workers/aws_s3_worker.rb +0 -10
  53. data/test/workers/big_data_worker.rb +0 -15
  54. data/test/workers/big_gems_worker.rb +0 -35
  55. data/test/workers/cool_worker.rb +0 -14
  56. data/test/workers/db_worker.rb +0 -27
  57. data/test/workers/fail_worker.rb +0 -10
  58. data/test/workers/gem_dependency_worker.rb +0 -20
  59. data/test/workers/hacker_worker.rb +0 -65
  60. data/test/workers/iw_test_worker.rb +0 -22
  61. data/test/workers/iw_test_worker_2.rb +0 -53
  62. data/test/workers/iw_test_worker_3.rb +0 -17
  63. data/test/workers/local_vs_remote_worker.rb +0 -9
  64. data/test/workers/merging_worker.rb +0 -40
  65. data/test/workers/mq_worker.rb +0 -21
  66. data/test/workers/one_line_worker.rb +0 -10
  67. data/test/workers/prawn_worker.rb +0 -32
  68. data/test/workers/progress_worker.rb +0 -26
  69. data/test/workers/qb_worker.rb +0 -17
  70. data/test/workers/rails_worker.rb +0 -24
  71. data/test/workers/rails_worker2.rb +0 -12
  72. data/test/workers/requiring_worker.rb +0 -4
  73. data/test/workers/running_back_worker.rb +0 -7
  74. data/test/workers/scheduled_worker.rb +0 -15
  75. data/test/workers/second_worker.rb +0 -16
  76. data/test/workers/third_worker.rb +0 -6
  77. data/test/workers/webhook_worker.rb +0 -7
  78. data/test/zip_vs_gzip.rb +0 -26
@@ -1,11 +0,0 @@
1
- # UsedInWorker can be included in classes that you are merging in order to get some of the IronWorker features into that class.
2
- # For instance, you can use the log() method and it will be logged to the IronWorker logs.
3
-
4
- module IronWorker
5
- module UsedInWorker
6
-
7
- def log(str)
8
- puts str.to_s
9
- end
10
- end
11
- end
@@ -1,11 +0,0 @@
1
- module IronWorker
2
-
3
- module Utils
4
-
5
- def self.ends_with?(s, suffix)
6
- suffix = suffix.to_s
7
- s[-suffix.length, suffix.length] == suffix
8
- end
9
- end
10
-
11
- end
data/rails/init.rb DELETED
File without changes
data/test/Gemfile DELETED
@@ -1,36 +0,0 @@
1
- # These are required for the tests
2
-
3
- source 'http://rubygems.org'
4
-
5
- gem 'iron_worker'
6
-
7
- group :test do
8
- gem 'rack'
9
- gem 'test-unit'
10
- gem 'minitest'
11
- gem 'jeweler2'
12
- gem 'rubyzip'
13
- gem 'rest'
14
- gem 'typhoeus'
15
- gem 'uber_config'
16
-
17
- gem 'simple_record'
18
- gem 'activesupport'
19
- gem 'simple_record'
20
- gem 'dropbox', '1.2.3'
21
- gem 'pusher'
22
- gem 'carrierwave'
23
- gem 'aws-s3'
24
- gem 'mechanize'
25
- gem 'mail'
26
- gem 'timecop'
27
- gem 'fog'
28
- gem 'logglier'
29
- #gem 'mysql2'
30
- gem 'mini_fb'
31
- gem 'iron_mq'
32
- gem 'prawn'
33
- gem 'pdf-reader'
34
- gem 'concur'
35
- gem 'mechanize'
36
- end
data/test/Gemfile.lock DELETED
@@ -1,169 +0,0 @@
1
- GEM
2
- remote: http://rubygems.org/
3
- specs:
4
- Ascii85 (1.0.2)
5
- activesupport (3.2.8)
6
- i18n (~> 0.6)
7
- multi_json (~> 1.0)
8
- aws (2.5.8)
9
- http_connection
10
- http_connection
11
- uuidtools
12
- uuidtools
13
- xml-simple
14
- xml-simple
15
- aws-s3 (0.6.3)
16
- builder
17
- mime-types
18
- xml-simple
19
- builder (3.1.4)
20
- carrierwave (0.5.8)
21
- activesupport (~> 3.0)
22
- concur (1.0.0)
23
- faraday
24
- faraday
25
- domain_name (0.5.4)
26
- unf (~> 0.0.3)
27
- dropbox (1.2.3)
28
- json (>= 1.2.0)
29
- mechanize (>= 1.0.0)
30
- multipart-post (>= 1.0)
31
- oauth (>= 0.3.6)
32
- ethon (0.5.3)
33
- ffi (~> 1.0.11)
34
- mime-types (~> 1.18)
35
- excon (0.16.7)
36
- faraday (0.8.4)
37
- multipart-post (~> 1.1)
38
- ffi (1.0.11)
39
- fog (1.7.0)
40
- builder
41
- excon (~> 0.14)
42
- formatador (~> 0.2.0)
43
- mime-types
44
- multi_json (~> 1.0)
45
- net-scp (~> 1.0.4)
46
- net-ssh (>= 2.1.3)
47
- nokogiri (~> 1.5.0)
48
- ruby-hmac
49
- formatador (0.2.4)
50
- git (1.2.5)
51
- hashery (2.0.1)
52
- hashie (1.2.0)
53
- http_connection (1.4.2)
54
- i18n (0.6.1)
55
- iron_core (0.4.3)
56
- rest (>= 2.0.2)
57
- iron_mq (2.1.3)
58
- iron_core (>= 0.4.2)
59
- iron_worker (2.3.15)
60
- bundler
61
- bundler
62
- rest
63
- rest
64
- rest-client
65
- rest-client
66
- zip
67
- zip
68
- jeweler2 (2.0.9)
69
- git (>= 1.2.5)
70
- json (1.7.5)
71
- logglier (0.2.8)
72
- mail (2.4.4)
73
- i18n (>= 0.4.0)
74
- mime-types (~> 1.16)
75
- treetop (~> 1.4.8)
76
- mechanize (2.5.1)
77
- domain_name (~> 0.5, >= 0.5.1)
78
- mime-types (~> 1.17, >= 1.17.2)
79
- net-http-digest_auth (~> 1.1, >= 1.1.1)
80
- net-http-persistent (~> 2.5, >= 2.5.2)
81
- nokogiri (~> 1.4)
82
- ntlm-http (~> 0.1, >= 0.1.1)
83
- webrobots (~> 0.0, >= 0.0.9)
84
- mime-types (1.19)
85
- mini_fb (1.2.0)
86
- hashie
87
- hashie
88
- rest-client
89
- rest-client
90
- minitest (4.2.0)
91
- multi_json (1.3.7)
92
- multipart-post (1.1.5)
93
- net-http-digest_auth (1.2.1)
94
- net-http-persistent (2.8)
95
- net-scp (1.0.4)
96
- net-ssh (>= 1.99.1)
97
- net-ssh (2.6.1)
98
- nokogiri (1.5.5)
99
- ntlm-http (0.1.1)
100
- oauth (0.4.7)
101
- pdf-reader (1.2.0)
102
- Ascii85 (~> 1.0.0)
103
- hashery (~> 2.0)
104
- ruby-rc4
105
- polyglot (0.3.3)
106
- prawn (0.12.0)
107
- pdf-reader (>= 0.9.0)
108
- ttfunk (~> 1.0.2)
109
- pusher (0.10.0)
110
- multi_json (~> 1.0)
111
- signature (~> 0.1.4)
112
- rack (1.4.1)
113
- rest (2.1.0)
114
- net-http-persistent
115
- rest-client (>= 0.3.0)
116
- rest-client (1.6.7)
117
- mime-types (>= 1.16)
118
- ruby-hmac (0.4.0)
119
- ruby-rc4 (0.1.5)
120
- signature (0.1.4)
121
- simple_record (2.2.0)
122
- aws
123
- concur
124
- test-unit (2.5.2)
125
- timecop (0.5.3)
126
- treetop (1.4.12)
127
- polyglot
128
- polyglot (>= 0.3.1)
129
- ttfunk (1.0.3)
130
- typhoeus (0.5.1)
131
- ethon (= 0.5.3)
132
- uber_config (1.0.5)
133
- unf (0.0.5)
134
- unf_ext
135
- unf_ext (0.0.5)
136
- uuidtools (2.1.3)
137
- webrobots (0.0.13)
138
- xml-simple (1.1.2)
139
- zip (2.0.2)
140
-
141
- PLATFORMS
142
- ruby
143
-
144
- DEPENDENCIES
145
- activesupport
146
- aws-s3
147
- carrierwave
148
- concur
149
- dropbox (= 1.2.3)
150
- fog
151
- iron_mq
152
- iron_worker
153
- jeweler2
154
- logglier
155
- mail
156
- mechanize
157
- mini_fb
158
- minitest
159
- pdf-reader
160
- prawn
161
- pusher
162
- rack
163
- rest
164
- simple_record
165
- test-unit
166
- timecop
167
- typhoeus
168
- uber_config
169
- zip
data/test/README.md DELETED
@@ -1,5 +0,0 @@
1
- When testing workers, be sure to run:
2
-
3
- sudo rake install
4
-
5
- Because the uploaded gem will be the one that is installed on the system.
data/test/batch_run.rb DELETED
@@ -1,121 +0,0 @@
1
- require_relative 'test_base'
2
- require_relative 'workers/one_line_worker'
3
- require_relative 'workers/merging_worker'
4
- require_relative 'workers/progress_worker'
5
- require_relative 'workers/mq_worker'
6
- require 'concur'
7
- #require_relative 'prawn_worker'
8
-
9
- class BatchRun < TestBase
10
-
11
- def test_concur_batch
12
-
13
- old_log_level = IronWorker.logger.level
14
- IronWorker.logger.level = Logger::INFO
15
-
16
- clz = OneLineWorker
17
- num_tasks = 100
18
-
19
- worker = clz.new
20
- worker.upload
21
-
22
- jobs = []
23
- executor = Concur::Executor.new_thread_pool_executor(20)
24
- num_tasks.times do |i|
25
- jobs << executor.execute do
26
- begin
27
- worker2 = clz.new
28
- puts "queueing #{i}"
29
- if clz == MqWorker
30
- worker2.config = {:token=>IronWorker.config.token, :project_id=>IronWorker.config.project_id}
31
- else
32
- worker2.x = "hello payload #{i}"
33
- end
34
- response_hash = worker2.queue(:priority=>(@config[:priority] || 0))
35
- puts "response_hash #{i} = " + response_hash.inspect
36
- assert response_hash["msg"]
37
- assert response_hash["status_code"]
38
- assert response_hash["tasks"]
39
- assert response_hash["status_code"] == 200
40
- assert response_hash["tasks"][0]["id"].length == 24, "length is #{response_hash["tasks"][0]["id"].length}"
41
- assert response_hash["tasks"][0]["id"] == worker2.task_id, "id in hash: #{response_hash["tasks"][0]["id"]}, task_id: #{worker2.task_id}. response was #{worker2.response.inspect}"
42
- worker2
43
- rescue => ex
44
- puts "ERROR! #{ex.class.name}: #{ex.message} -- #{ex.backtrace.inspect}"
45
- raise ex
46
- end
47
-
48
- end
49
- end
50
-
51
- sleep 10
52
-
53
- completed_count = 0
54
- errored_queuing_count = 0
55
- error_count = 0
56
- while jobs.size > 0
57
- jobs.each_with_index do |f, i|
58
- # p f
59
- begin
60
- t = f.get
61
- # p t
62
- puts i.to_s + ' task_id=' + t.task_id.to_s
63
- status_response = t.status # worker.status(t["task_id"])
64
- puts 'status ' + status_response["status"] + ' for ' + status_response.inspect
65
- puts 'msg=' + status_response["msg"].to_s
66
- if status_response["status"] == "complete" || status_response["status"] == "error"
67
- if true || status_response["status"] == "error"
68
- puts t.get_log
69
- end
70
-
71
- jobs.delete(f)
72
- completed_count += 1
73
- puts "#{completed_count} completed so far. #{jobs.size} left..."
74
- if status_response["status"] == "error"
75
- error_count += 1
76
- end
77
- end
78
- rescue => ex
79
- puts 'error! ' + ex.class.name + ' -> ' + ex.message.to_s
80
- puts ex.backtrace
81
- errored_queuing_count += 1
82
- jobs.delete(f)
83
- end
84
- end
85
- puts 'sleep'
86
- sleep 2
87
- puts 'done sleeping'
88
- end
89
-
90
- puts 'Total completed=' + completed_count.to_s
91
- puts 'Total errored while queuing=' + errored_queuing_count.to_s
92
- puts 'Total errored while running=' + error_count.to_s
93
-
94
- executor.shutdown
95
-
96
- #tasks = []
97
- #1000.times do |i|
98
- # puts "#{i}"
99
- # worker = ProgressWorker.new
100
- # #worker = OneLineWorker.new
101
- # # worker = MergingWorker.new
102
- # #worker = PrawnWorker.new
103
- # worker.queue
104
- # tasks << worker
105
- #end
106
- #
107
- #tasks.each_with_index do |task, i|
108
- # puts "#{i}"
109
- # status = task.wait_until_complete
110
- # p status
111
- # puts "\n\n\nLOG START:"
112
- # puts task.get_log
113
- # puts "LOG END\n\n\n"
114
- # assert status["status"] == "complete", "Status was not complete, it was #{status["status"]}"
115
- #end
116
- IronWorker.logger.level = old_log_level
117
-
118
- end
119
-
120
- end
121
-
@@ -1,13 +0,0 @@
1
- iron:
2
- token: MY_TOKEN
3
- project_id: MY_PROJECT_ID
4
-
5
- database:
6
- adapter: mysql2
7
- host: ec2-ABC.compute-1.amazonaws.com
8
- port: 3062
9
- database: MY_DB
10
- username: MY_USER
11
- password: MY_PASS
12
-
13
-
@@ -1,7 +0,0 @@
1
- require 'simple_record'
2
-
3
- class CoolModel < SimpleRecord::Base
4
-
5
- has_strings :name
6
-
7
- end
@@ -1,6 +0,0 @@
1
- #require 'active_record'
2
-
3
- class DbModel #< ActiveRecord::Base
4
-
5
-
6
- end
@@ -1,16 +0,0 @@
1
-
2
-
3
- class Model1
4
- attr_accessor :heidi, :ho
5
-
6
- include IronWorker::UsedInWorker
7
-
8
- def say_hello
9
- log "Hi there sir"
10
- end
11
-
12
- # testk laksdfj klasj df
13
- def test
14
- log 'test'
15
- end
16
- end
@@ -1,16 +0,0 @@
1
-
2
-
3
- class Model2
4
- attr_accessor :heidi, :ho
5
-
6
- include IronWorker::UsedInWorker
7
-
8
- def say_hello
9
- log "Hi there sir"
10
- end
11
-
12
- # testk laksdfj klasj df
13
- def test
14
- log 'test'
15
- end
16
- end
data/test/quick_run.rb DELETED
@@ -1,36 +0,0 @@
1
- require_relative 'test_base'
2
- require_relative 'workers/one_line_worker'
3
- require_relative 'workers/progress_worker'
4
- require_relative 'workers/fail_worker'
5
-
6
- class QuickRun < TestBase
7
-
8
- def test_worker
9
- tasks = []
10
- 1.times do |i|
11
- puts "Queuing #{i}"
12
- worker = OneLineWorker.new
13
- worker.x = i
14
- worker.queue
15
- tasks << worker
16
- end
17
-
18
- tasks.each_with_index do |worker, i|
19
- puts "Waiting for #{i}"
20
- status = worker.wait_until_complete
21
- puts "#{i} is complete."
22
- p status
23
- puts "error_class: #{status["error_class"]}"
24
- puts "msg: #{status["msg"]}"
25
- puts "percent: #{status["percent"]}"
26
- puts "\n\n\nLOG START:"
27
- log = worker.get_log
28
- puts log
29
- puts "LOG END\n\n\n"
30
- assert status["status"] == "complete", "Status was not complete, it was #{status["status"]}"
31
- assert log.include?("hello")
32
- end
33
- end
34
-
35
- end
36
-