mobilize-base 1.351 → 1.361

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,25 @@
1
+ ---
2
+ - path: "jobtracker"
3
+ state: working
4
+ count: 1
5
+ confirmed_ats: []
6
+ - path: "Runner_mobilize(test)/jobs"
7
+ state: working
8
+ count: 2
9
+ confirmed_ats: []
10
+ - path: "Runner_mobilize(test)/jobs/base1/stage1"
11
+ state: working
12
+ count: 1
13
+ confirmed_ats: []
14
+ - path: "Runner_mobilize(test)/jobs/base1/stage2"
15
+ state: working
16
+ count: 1
17
+ confirmed_ats: []
18
+ - path: "Runner_mobilize(test)/jobs/base2/stage1"
19
+ state: working
20
+ count: 1
21
+ confirmed_ats: []
22
+ - path: "Runner_mobilize(test)/jobs/base2/stage1"
23
+ state: failed
24
+ count: 2
25
+ confirmed_ats: []
@@ -0,0 +1,12 @@
1
+ ---
2
+ - name: base1
3
+ active: true
4
+ trigger: once
5
+ status: ""
6
+ stage1: gfile.write source:"gsheet://base1_stage1.in", target:"gfile://base1_stage1.out"
7
+ stage2: gsheet.write source:"gfile://base1_stage1.out", target:"gsheet://base1_stage2.out"
8
+ - name: base2
9
+ active: true
10
+ trigger: after base1
11
+ status: ""
12
+ stage1: gsheet.write source:"base1."_"", target:"base2_stage1.out"
@@ -0,0 +1,97 @@
1
+ ---
2
+ - name: once_active
3
+ active: true
4
+ trigger: once
5
+ status: ""
6
+ stage1: gsheet.method source:"source", target:"target"
7
+ expected: true
8
+ - name: once_inactive
9
+ active: false
10
+ trigger: once
11
+ status: ""
12
+ stage1: gfile.method source:"source", target:"target"
13
+ expected: false
14
+ - name: 1h_comp_never
15
+ active: true
16
+ trigger: every 1.hour
17
+ status: ""
18
+ stage1: gsheet.method source:"source", target:"target"
19
+ expected: true
20
+ - name: 1h_comp_0.5h_ago
21
+ active: true
22
+ trigger: every 1.hour
23
+ status: ""
24
+ stage1: gsheet.method source:"source", target:"target"
25
+ completed_at: Time.now.utc - 1800.seconds
26
+ expected: false
27
+ - name: 1h_comp_1.5h_ago
28
+ active: true
29
+ trigger: every 1.hour
30
+ status: ""
31
+ stage1: gsheet.method source:"source", target:"target"
32
+ completed_at: Time.now.utc - 5400.seconds
33
+ expected: true
34
+ - name: 1h_aft45_comp_2_45_marks_ago
35
+ active: true
36
+ trigger: 'every 1.hour after 45'
37
+ status: ""
38
+ stage1: gsheet.method source:"source", target:"target"
39
+ completed_at: Time.at_marks_ago(2,'hour', '45')
40
+ expected: true
41
+ - name: 1h_aft45_comp_aft_1_45_mark_ago
42
+ active: true
43
+ trigger: 'every 1.hour after :45'
44
+ status: ""
45
+ stage1: gsheet.method source:"source", target:"target"
46
+ completed_at: Time.at_marks_ago(1,'hour', '45')
47
+ expected: false
48
+ - name: 1d_aft0135_comp_2_0135_marks_ago
49
+ active: true
50
+ trigger: 'every 1.day after 01:35'
51
+ status: ""
52
+ stage1: gsheet.method source:"source", target:"target"
53
+ completed_at: Time.at_marks_ago(2,'day', '0135')
54
+ expected: true
55
+ - name: 1d_aft0135_comp_1_0135_mark_ago
56
+ active: true
57
+ trigger: 'every 1.day after 0135'
58
+ status: ""
59
+ stage1: gsheet.method source:"source", target:"target"
60
+ completed_at: Time.at_marks_ago(1,'day', '0135')
61
+ expected: false
62
+ - name: 5d_aft0135_comp_5_0135_mark_ago
63
+ active: true
64
+ trigger: 'every 5.day after 01:35'
65
+ status: ""
66
+ stage1: gsheet.method source:"source", target:"target"
67
+ completed_at: Time.at_marks_ago(5,'day', '0135')
68
+ expected: false
69
+ - name: 5d_aft0135_comp_6_0135_mark_ago
70
+ active: true
71
+ trigger: 'every 5.day after 01:35'
72
+ status: ""
73
+ stage1: gsheet.method source:"source", target:"target"
74
+ completed_at: Time.at_marks_ago(6,'day', '0135')
75
+ expected: true
76
+ - name: parent_comp_child_fail
77
+ active: true
78
+ trigger: 'after once_inactive'
79
+ status: ""
80
+ stage1: gsheet.method source:"source", target:"target"
81
+ parent: {completed_at: Time.now.utc - 1.second}
82
+ failed_at: Time.now.utc
83
+ expected: true
84
+ - name: parent_comp_child_comp
85
+ active: true
86
+ trigger: 'after once_inactive'
87
+ status: ""
88
+ stage1: gsheet.method source:"source", target:"target"
89
+ parent: {completed_at: Time.now.utc - 1.second}
90
+ completed_at: Time.now.utc
91
+ expected: false
92
+ - name: missing_module
93
+ active: true
94
+ trigger: once
95
+ status: ""
96
+ stage1: handler.method source:"source", target:"target"
97
+ expected: false
@@ -0,0 +1,57 @@
1
+ require 'test_helper'
2
+ describe Mobilize do
3
+
4
+ it "runs integration test" do
5
+
6
+ puts "restart test redis"
7
+ TestHelper.restart_test_redis
8
+ TestHelper.drop_test_db
9
+
10
+ puts "restart workers"
11
+ Mobilize::Jobtracker.restart_workers!
12
+
13
+ u = TestHelper.owner_user
14
+ r = u.runner
15
+ user_name = u.name
16
+ gdrive_slot = u.email
17
+
18
+ puts "build test runner"
19
+ TestHelper.build_test_runner(user_name)
20
+ assert Mobilize::Jobtracker.workers.length == Mobilize::Resque.config['max_workers'].to_i
21
+
22
+ puts "add base1_stage1.in sheet"
23
+ input_fixture_name = "base1_stage1.in"
24
+ input_target_url = "gsheet://#{r.title}/#{input_fixture_name}"
25
+ TestHelper.write_fixture(input_fixture_name, input_target_url, 'replace' => true)
26
+
27
+ puts "add jobs sheet with integration jobs"
28
+ jobs_fixture_name = "integration_jobs"
29
+ jobs_target_url = "gsheet://#{r.title}/jobs"
30
+ TestHelper.write_fixture(jobs_fixture_name, jobs_target_url, 'update' => true)
31
+
32
+ puts "wait for stages"
33
+ Mobilize::Jobtracker.start
34
+ #wait for stages to complete
35
+ expected_fixture_name = "integration_expected"
36
+ TestHelper.confirm_expected_jobs(expected_fixture_name)
37
+ #stop jobtracker
38
+ Mobilize::Jobtracker.stop!
39
+
40
+ puts "jobtracker posted test sheet data to test destination, and checksum succeeded?"
41
+ tsv_hash = {}
42
+ ["base1_stage1.in", "base1_stage2.out"].each do |sheet_name|
43
+ url = "gsheet://#{r.title}/#{sheet_name}"
44
+ data = Mobilize::Dataset.read_by_url(url,user_name,gdrive_slot)
45
+ assert TestHelper.check_output(url, 'min_length' => 10) == true
46
+ tsv_hash[sheet_name] = data
47
+ end
48
+
49
+ assert tsv_hash["base1_stage2.out"] == tsv_hash["base1_stage1.in"]
50
+
51
+ err_url = "gsheet://#{r.title}/base2_stage1.err"
52
+ err_response = "Unable to parse stage params, make sure you don't have issues with your quotes, commas, or colons."
53
+
54
+ assert TestHelper.check_output(err_url, 'match' => err_response) == true
55
+
56
+ end
57
+ end
data/test/test_helper.rb CHANGED
@@ -9,26 +9,53 @@ ENV['MOBILIZE_ENV'] = 'test'
9
9
  require 'mobilize-base'
10
10
  $TESTING = true
11
11
  module TestHelper
12
- def TestHelper.wait_for_stages(time_limit=600,stage_limit=120,wait_length=10)
13
- time = 0
14
- time_since_stage = 0
15
- #check for 10 min
16
- while time < time_limit and time_since_stage < stage_limit
17
- sleep wait_length
18
- job_classes = Mobilize::Resque.jobs.map{|j| j['class']}
19
- if job_classes.include?("Mobilize::Stage")
20
- time_since_stage = 0
21
- puts "saw stage at #{time.to_s} seconds"
22
- else
23
- time_since_stage += wait_length
24
- puts "#{time_since_stage.to_s} seconds since stage seen"
12
+ def TestHelper.confirm_expected_jobs(expected_fixture_name,time_limit=600)
13
+ jobs = {}
14
+ jobs['expected'] = TestHelper.load_fixture(expected_fixture_name)
15
+ jobs['pending'] = jobs['expected'].select{|j| j['confirmed_ats'].length < j['count']}
16
+ start_time = Time.now.utc
17
+ total_time = 0
18
+ while (jobs['pending'].length>0 or #while there are pending jobs
19
+ #or there are workers working (don't count jobtracker)
20
+ Mobilize::Resque.workers('working').select{|w| w.job['payload']['class']!='Mobilize::Jobtracker'}.length>0) and
21
+ total_time < time_limit #time limit not yet expired
22
+ #working jobs are running on the queue at this instant
23
+ jobs['working'] = Mobilize::Resque.workers('working').map{|w| w.job}.select{|j| j and j['payload'] and j['payload']['args']}
24
+ #failed jobs are in the failure queue
25
+ jobs['failed'] = Mobilize::Resque.failures.select{|j| j and j['payload'] and j['payload']['args']}
26
+
27
+ #unexpected jobs are not supposed to be run in this test, includes leftover failures
28
+ jobs['unexpected'] = {}
29
+ error_msg = ""
30
+ ['working','failed'].each do |state|
31
+ jobs['unexpected'][state] = jobs[state].reject{|j|
32
+ jobs['expected'].select{|ej|
33
+ ej['state']==state and j['payload']['args'].first == ej['path']}.first}
34
+ if jobs['unexpected'][state].length>0
35
+ error_msg += state + ": " + jobs['unexpected'][state].map{|j| j['payload']['args'].first}.join(";") + "\n"
36
+ end
37
+ end
38
+ #clear out unexpected paths or there will be failure
39
+ if error_msg.length>0
40
+ raise "Found unexpected results:\n" + error_msg
41
+ end
42
+
43
+ #now make sure pending jobs get done
44
+ jobs['expected'].each do |j|
45
+ start_confirmed_ats = j['confirmed_ats']
46
+ resque_timestamps = jobs[j['state']].select{|sj| sj['payload']['args'].first == j['path']}.map{|sj| sj['run_at'] || sj['failed_at']}
47
+ new_timestamps = (resque_timestamps - start_confirmed_ats).uniq
48
+ if new_timestamps.length>0 and j['confirmed_ats'].length < j['count']
49
+ j['confirmed_ats'] += new_timestamps
50
+ puts "#{Time.now.utc.to_s}: #{new_timestamps.length.to_s} #{j['state']} added to #{j['path']}; total #{j['confirmed_ats'].length.to_s} of #{j['count']}"
51
+ end
25
52
  end
26
- time += wait_length
27
- puts "total wait time #{time.to_s} seconds"
28
- end
29
53
 
30
- if time >= time_limit
31
- raise "Timed out before stage completion"
54
+ #figure out who's still pending
55
+ jobs['pending'] = jobs['expected'].select{|j| j['confirmed_ats'].length < j['count']}
56
+ sleep 1
57
+ total_time = Time.now.utc - start_time
58
+ puts "#{total_time.to_s} seconds elapsed" if total_time.to_s.ends_with?("0")
32
59
  end
33
60
  end
34
61
 
@@ -82,8 +109,60 @@ module TestHelper
82
109
 
83
110
  def TestHelper.owner_user
84
111
  gdrive_slot = Mobilize::Gdrive.owner_email
85
- puts "create user 'mobilize'"
86
112
  user_name = gdrive_slot.split("@").first
87
113
  return Mobilize::User.find_or_create_by_name(user_name)
88
114
  end
115
+
116
+ def TestHelper.load_fixture(name)
117
+ #assume yml, check
118
+ yml_file_path = "#{Mobilize::Base.root}/test/fixtures/#{name}.yml"
119
+ standard_file_path = "#{Mobilize::Base.root}/test/fixtures/#{name}"
120
+ if File.exists?(yml_file_path)
121
+ YAML.load_file(yml_file_path)
122
+ elsif File.exists?(standard_file_path)
123
+ File.read(standard_file_path)
124
+ else
125
+ raise "Could not find #{standard_file_path}"
126
+ end
127
+ end
128
+
129
+ def TestHelper.write_fixture(fixture_name, target_url, options={})
130
+ u = TestHelper.owner_user
131
+ fixture_raw = TestHelper.load_fixture(fixture_name)
132
+ if options['replace']
133
+ fixture_data = if fixture_raw.class == Array
134
+ fixture_raw.hash_array_to_tsv
135
+ elsif fixture_raw.class == String
136
+ fixture_raw
137
+ end
138
+ Mobilize::Dataset.write_by_url(target_url,fixture_data,u.name,u.email)
139
+ elsif options['update']
140
+ handler, sheet_path = target_url.split("://")
141
+ raise "update only works for gsheet, not #{handler}" unless handler=='gsheet'
142
+ sheet = Mobilize::Gsheet.find_or_create_by_path(sheet_path,u.email)
143
+ sheet.add_or_update_rows(fixture_raw)
144
+ else
145
+ raise "unknown options #{options.to_s}"
146
+ end
147
+ return true
148
+ end
149
+
150
+ #checks output sheet for matching string or minimum length
151
+ def TestHelper.check_output(target_url, options={})
152
+ u = TestHelper.owner_user
153
+ handler, sheet_path = target_url.split("://")
154
+ handler = nil
155
+ sheet = Mobilize::Gsheet.find_by_path(sheet_path,u.email)
156
+ raise "no output found" if sheet.nil?
157
+ output = sheet.to_tsv
158
+ if options['match']
159
+ return true if output == options['match']
160
+ elsif options['min_length']
161
+ return true if output.length >= options['min_length']
162
+ else
163
+ raise "unknown check options #{options.to_s}"
164
+ end
165
+ return true
166
+ end
167
+
89
168
  end
@@ -0,0 +1,33 @@
1
+ require 'test_helper'
2
+ class TestUnit < MiniTest::Unit::TestCase
3
+ def setup
4
+ TestHelper.restart_test_redis
5
+ TestHelper.drop_test_db
6
+ end
7
+
8
+ #this test checks that several job triggers work as expected
9
+ def test_is_due
10
+ u = TestHelper.owner_user
11
+ job_hashes = TestHelper.load_fixture("is_due")
12
+ job_hashes.each do |jh|
13
+ job_path = "#{u.runner.path}/#{jh['name']}"
14
+ j = Mobilize::Job.find_or_create_by_path(job_path)
15
+ #update job params
16
+ j.update_from_hash(jh)
17
+ #apply the completed_at, failed at, and parent attributes where appropriate
18
+ if jh['completed_at']
19
+ j.stages.last.update_attributes(:completed_at=>eval(jh['completed_at']))
20
+ end
21
+ if jh['failed_at']
22
+ j.stages.last.update_attributes(:failed_at=>eval(jh['failed_at']))
23
+ end
24
+ if jh['parent']
25
+ j.parent.stages.last.update_attributes(:completed_at=>eval(jh['parent']['completed_at'])) if jh['parent']['completed_at']
26
+ j.parent.stages.last.update_attributes(:failed_at=>eval(jh['parent']['failed_at'])) if jh['parent']['failed_at']
27
+ end
28
+ expected = jh['expected']
29
+ #check if is_due
30
+ assert expected == j.is_due?
31
+ end
32
+ end
33
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: mobilize-base
3
3
  version: !ruby/object:Gem::Version
4
- version: '1.351'
4
+ version: '1.361'
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2013-04-25 00:00:00.000000000 Z
12
+ date: 2013-05-31 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: rake
@@ -182,6 +182,7 @@ files:
182
182
  - lib/mobilize-base/extensions/resque-server/views/queues.erb
183
183
  - lib/mobilize-base/extensions/resque-server/views/working.erb
184
184
  - lib/mobilize-base/extensions/string.rb
185
+ - lib/mobilize-base/extensions/time.rb
185
186
  - lib/mobilize-base/extensions/yaml.rb
186
187
  - lib/mobilize-base/handlers/email.rb
187
188
  - lib/mobilize-base/handlers/gbook.rb
@@ -203,6 +204,7 @@ files:
203
204
  - lib/mobilize-base/tasks.rb
204
205
  - lib/mobilize-base/version.rb
205
206
  - lib/samples/gdrive.yml
207
+ - lib/samples/gfile.yml
206
208
  - lib/samples/gridfs.yml
207
209
  - lib/samples/gsheet.yml
208
210
  - lib/samples/jobtracker.yml
@@ -210,11 +212,14 @@ files:
210
212
  - lib/samples/resque.yml
211
213
  - lib/samples/resque_web.rb
212
214
  - mobilize-base.gemspec
213
- - test/base_job_rows.yml
214
- - test/mobilize-base_test.rb
215
+ - test/fixtures/base1_stage1.in.yml
216
+ - test/fixtures/integration_expected.yml
217
+ - test/fixtures/integration_jobs.yml
218
+ - test/fixtures/is_due.yml
219
+ - test/integration/mobilize-base_test.rb
215
220
  - test/redis-test.conf
216
- - test/test_base_1.yml
217
221
  - test/test_helper.rb
222
+ - test/unit/mobilize-base_test.rb
218
223
  homepage: http://github.com/ngmoco/mobilize-base
219
224
  licenses: []
220
225
  post_install_message:
@@ -229,7 +234,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
229
234
  version: '0'
230
235
  segments:
231
236
  - 0
232
- hash: 4420073828570035753
237
+ hash: 3685016355014440836
233
238
  required_rubygems_version: !ruby/object:Gem::Requirement
234
239
  none: false
235
240
  requirements:
@@ -238,7 +243,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
238
243
  version: '0'
239
244
  segments:
240
245
  - 0
241
- hash: 4420073828570035753
246
+ hash: 3685016355014440836
242
247
  requirements: []
243
248
  rubyforge_project: mobilize-base
244
249
  rubygems_version: 1.8.25
@@ -247,8 +252,11 @@ specification_version: 3
247
252
  summary: Moves datasets and schedules data transfers using MongoDB, Resque and Google
248
253
  Docs
249
254
  test_files:
250
- - test/base_job_rows.yml
251
- - test/mobilize-base_test.rb
255
+ - test/fixtures/base1_stage1.in.yml
256
+ - test/fixtures/integration_expected.yml
257
+ - test/fixtures/integration_jobs.yml
258
+ - test/fixtures/is_due.yml
259
+ - test/integration/mobilize-base_test.rb
252
260
  - test/redis-test.conf
253
- - test/test_base_1.yml
254
261
  - test/test_helper.rb
262
+ - test/unit/mobilize-base_test.rb
@@ -1,15 +0,0 @@
1
- - name: base1
2
- active: true
3
- trigger: once
4
- status: ""
5
- stage1: gsheet.write source:"gfile://test_base_1.tsv", target:base1.out
6
- - name: base2
7
- active: true
8
- trigger: after base1
9
- status: ""
10
- stage1: gsheet.write source:base1.out, target:base2.out
11
- - name: base3
12
- active: true
13
- trigger: after base2
14
- status: ""
15
- stage1: gsheet.write source:"base1."_"", target:base2.out
@@ -1,65 +0,0 @@
1
- require 'test_helper'
2
-
3
- describe "Mobilize" do
4
- it "runs integration test" do
5
-
6
- puts "restart test redis"
7
- TestHelper.restart_test_redis
8
-
9
- puts "clear out test db"
10
- TestHelper.drop_test_db
11
-
12
- puts "restart workers"
13
- Mobilize::Jobtracker.restart_workers!
14
-
15
- u = TestHelper.owner_user
16
- user_name = u.name
17
- gdrive_slot = u.email
18
-
19
- puts "build test runner"
20
- TestHelper.build_test_runner(user_name)
21
- assert Mobilize::Jobtracker.workers.length == Mobilize::Resque.config['max_workers'].to_i
22
-
23
- puts "Jobtracker created runner with 'jobs' sheet?"
24
-
25
- r = u.runner
26
- jobs_sheet_url = "gsheet://#{r.path}"
27
- jobs_sheet = Mobilize::Gsheet.find_by_path(r.path,gdrive_slot)
28
- jobs_sheet_dst = Mobilize::Dataset.find_or_create_by_url(jobs_sheet_url)
29
- jobs_sheet_tsv = jobs_sheet_dst.read(user_name,gdrive_slot)
30
-
31
- assert jobs_sheet_tsv.tsv_header_array.join.length == 53 #total header length
32
-
33
- puts "add base1 input file"
34
- test_filename = "test_base_1"
35
- file_url = "gfile://#{test_filename}.tsv"
36
- test_source_ha = ::YAML.load_file("#{Mobilize::Base.root}/test/#{test_filename}.yml")*40
37
- test_source_tsv = test_source_ha.hash_array_to_tsv
38
- Mobilize::Dataset.write_by_url(file_url,test_source_tsv,user_name)
39
- rem_tsv = Mobilize::Dataset.read_by_url(file_url,user_name)
40
- assert rem_tsv == test_source_tsv
41
-
42
- puts "add row to jobs sheet, wait for stages"
43
- test_job_rows = ::YAML.load_file("#{Mobilize::Base.root}/test/base_job_rows.yml")
44
- jobs_sheet.reload
45
- jobs_sheet.add_or_update_rows(test_job_rows)
46
- #wait for stages to complete
47
- TestHelper.wait_for_stages
48
-
49
- puts "jobtracker posted test sheet data to test destination, and checksum succeeded?"
50
- tsv_hash = {}
51
- ["base1.out", "base2.out", "base3_stage1.err"].each do |sheet_name|
52
- url = "gsheet://#{r.title}/#{sheet_name}"
53
- data = Mobilize::Dataset.read_by_url(url,user_name,gdrive_slot)
54
- tsv_hash[sheet_name] = data
55
- end
56
-
57
- assert tsv_hash["base1.out"].to_s.length>0
58
- assert tsv_hash["base2.out"] == tsv_hash["base1.out"]
59
-
60
- base3_response = tsv_hash["base3_stage1.err"].tsv_to_hash_array.first['response']
61
- assert base3_response == "Unable to parse stage params, make sure you don't have issues with your quotes, commas, or colons."
62
-
63
- Mobilize::Jobtracker.stop!
64
- end
65
- end
data/test/test_base_1.yml DELETED
@@ -1,3 +0,0 @@
1
- - {date: 2013-01-01, snake_date: 2013-02-01, camelDate: 2013-03-01}
2
- - {date: 2013-01-01, snake_date: 2013-02-02, camelDate: 2013-03-02}
3
- - {date: 2013-01-01, snake_date: 2013-02-03, camelDate: 2013-03-03}