mobilize-base 1.0.2 → 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +5 -0
- data/LICENSE.txt +202 -20
- data/README.md +219 -138
- data/Rakefile +1 -2
- data/lib/mobilize-base/extensions/google_drive/acl.rb +25 -0
- data/lib/mobilize-base/extensions/google_drive/client_login_fetcher.rb +49 -0
- data/lib/mobilize-base/extensions/google_drive/file.rb +80 -0
- data/lib/mobilize-base/extensions/{google_drive.rb → google_drive/worksheet.rb} +46 -173
- data/lib/mobilize-base/extensions/resque.rb +18 -24
- data/lib/mobilize-base/extensions/string.rb +12 -0
- data/lib/mobilize-base/handlers/gbook.rb +14 -47
- data/lib/mobilize-base/handlers/gdrive.rb +17 -18
- data/lib/mobilize-base/handlers/gfile.rb +18 -39
- data/lib/mobilize-base/handlers/gridfs.rb +43 -0
- data/lib/mobilize-base/handlers/gsheet.rb +48 -99
- data/lib/mobilize-base/jobtracker.rb +29 -15
- data/lib/mobilize-base/models/dataset.rb +33 -35
- data/lib/mobilize-base/models/job.rb +21 -168
- data/lib/mobilize-base/models/runner.rb +178 -0
- data/lib/mobilize-base/models/task.rb +137 -0
- data/lib/mobilize-base/models/user.rb +47 -0
- data/lib/mobilize-base/rakes.rb +59 -0
- data/lib/mobilize-base/version.rb +1 -1
- data/lib/mobilize-base.rb +20 -9
- data/lib/samples/gdrive.yml +12 -12
- data/lib/samples/gridfs.yml +9 -0
- data/lib/samples/gsheet.yml +6 -0
- data/lib/samples/jobtracker.yml +9 -9
- data/lib/samples/mongoid.yml +3 -3
- data/mobilize-base.gemspec +1 -1
- data/test/base1_task1.yml +3 -0
- data/test/base_job_rows.yml +13 -0
- data/test/mobilize-base_test.rb +59 -0
- metadata +20 -9
- data/lib/mobilize-base/handlers/mongodb.rb +0 -32
- data/lib/mobilize-base/models/requestor.rb +0 -232
- data/lib/mobilize-base/tasks.rb +0 -43
- data/test/mobilize_test.rb +0 -108
data/test/mobilize_test.rb
DELETED
|
@@ -1,108 +0,0 @@
|
|
|
1
|
-
require 'test_helper'
|
|
2
|
-
|
|
3
|
-
describe "Mobilize" do
|
|
4
|
-
|
|
5
|
-
def before
|
|
6
|
-
puts 'nothing before'
|
|
7
|
-
end
|
|
8
|
-
|
|
9
|
-
# enqueues 4 workers on Resque
|
|
10
|
-
it "runs integration test" do
|
|
11
|
-
|
|
12
|
-
puts "restart test redis"
|
|
13
|
-
Mobilize::Jobtracker.restart_test_redis
|
|
14
|
-
|
|
15
|
-
puts "clear out test db"
|
|
16
|
-
Mobilize::Jobtracker.drop_test_db
|
|
17
|
-
|
|
18
|
-
email = Mobilize::Gdrive.owner_email
|
|
19
|
-
|
|
20
|
-
#kill all workers
|
|
21
|
-
Mobilize::Jobtracker.kill_workers
|
|
22
|
-
|
|
23
|
-
puts 'enqueue 4 workers on Resque, wait 20s'
|
|
24
|
-
Mobilize::Jobtracker.prep_workers
|
|
25
|
-
sleep 20
|
|
26
|
-
assert Mobilize::Jobtracker.workers.length == Mobilize::Resque.config['max_workers'].to_i
|
|
27
|
-
|
|
28
|
-
puts "create requestor 'mobilize'"
|
|
29
|
-
requestor = Mobilize::Requestor.find_or_create_by_email(email)
|
|
30
|
-
assert requestor.email == email
|
|
31
|
-
|
|
32
|
-
puts "delete old books and datasets"
|
|
33
|
-
# delete any old jobspec from previous test runs
|
|
34
|
-
jobspec_title = requestor.jobspec_title
|
|
35
|
-
books = Mobilize::Gbook.find_all_by_title(jobspec_title)
|
|
36
|
-
books.each{|book| book.delete}
|
|
37
|
-
|
|
38
|
-
puts "enqueue jobtracker, wait 45s"
|
|
39
|
-
Mobilize::Jobtracker.start
|
|
40
|
-
sleep 45
|
|
41
|
-
puts "jobtracker status: #{Mobilize::Jobtracker.status}"
|
|
42
|
-
puts "status:#{Mobilize::Jobtracker.status}" #!= 'stopped'
|
|
43
|
-
|
|
44
|
-
puts "requestor created jobspec?"
|
|
45
|
-
books = Mobilize::Gbook.find_all_by_title(jobspec_title)
|
|
46
|
-
assert books.length == 1
|
|
47
|
-
|
|
48
|
-
puts "Jobtracker created jobspec with 'jobs' sheet?"
|
|
49
|
-
jobs_sheets = Mobilize::Gsheet.find_all_by_name("#{jobspec_title}/Jobs",email)
|
|
50
|
-
assert jobs_sheets.length == 1
|
|
51
|
-
|
|
52
|
-
puts "add test_source data"
|
|
53
|
-
|
|
54
|
-
test_source_rows = [
|
|
55
|
-
["test_header","test_header2","test_header3"],
|
|
56
|
-
["t1"]*3,
|
|
57
|
-
["t2"]*3
|
|
58
|
-
]
|
|
59
|
-
|
|
60
|
-
book = books.first
|
|
61
|
-
test_source_sheet = Mobilize::Gsheet.find_or_create_by_name("#{jobspec_title}/test_source",email)
|
|
62
|
-
|
|
63
|
-
test_source_tsv = test_source_rows.map{|r| r.join("\t")}.join("\n")
|
|
64
|
-
test_source_sheet.write(test_source_tsv)
|
|
65
|
-
|
|
66
|
-
puts "add row to jobs sheet, wait 120s"
|
|
67
|
-
|
|
68
|
-
jobs_sheet = jobs_sheets.first
|
|
69
|
-
|
|
70
|
-
test_job_rows = [{"name" => "test",
|
|
71
|
-
"active" => "true",
|
|
72
|
-
"schedule" => "once",
|
|
73
|
-
"status" => "",
|
|
74
|
-
"last_error" => "",
|
|
75
|
-
"destination_url" => "",
|
|
76
|
-
"tasks" => "gsheet.read, gsheet.write",
|
|
77
|
-
"datasets" => "test_source",
|
|
78
|
-
"params" => "",
|
|
79
|
-
"destination" => "test_destination"},
|
|
80
|
-
#run after the first
|
|
81
|
-
{"name" => "test2",
|
|
82
|
-
"active" => "true",
|
|
83
|
-
"schedule" => "after test",
|
|
84
|
-
"status" => "",
|
|
85
|
-
"last_error" => "",
|
|
86
|
-
"destination_url" => "",
|
|
87
|
-
"tasks" => "gsheet.read, gsheet.write",
|
|
88
|
-
"datasets" => "test_source",
|
|
89
|
-
"params" => "",
|
|
90
|
-
"destination" => "test_destination2"}
|
|
91
|
-
]
|
|
92
|
-
|
|
93
|
-
jobs_sheet.add_or_update_rows(test_job_rows)
|
|
94
|
-
|
|
95
|
-
puts "job row added, force enqueued requestor"
|
|
96
|
-
requestor.enqueue!
|
|
97
|
-
sleep 120
|
|
98
|
-
|
|
99
|
-
puts "jobtracker posted test sheet data to test destination, and checksum succeeded?"
|
|
100
|
-
test_destination_sheet = Mobilize::Gsheet.find_or_create_by_name("#{jobspec_title}/test_destination",email)
|
|
101
|
-
|
|
102
|
-
assert test_destination_sheet.to_tsv == test_source_sheet.to_tsv
|
|
103
|
-
|
|
104
|
-
puts "stop test redis"
|
|
105
|
-
Mobilize::Jobtracker.stop_test_redis
|
|
106
|
-
end
|
|
107
|
-
|
|
108
|
-
end
|