mobilize-base 1.0.2 → 1.0.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (38) hide show
  1. data/.gitignore +5 -0
  2. data/LICENSE.txt +202 -20
  3. data/README.md +219 -138
  4. data/Rakefile +1 -2
  5. data/lib/mobilize-base/extensions/google_drive/acl.rb +25 -0
  6. data/lib/mobilize-base/extensions/google_drive/client_login_fetcher.rb +49 -0
  7. data/lib/mobilize-base/extensions/google_drive/file.rb +80 -0
  8. data/lib/mobilize-base/extensions/{google_drive.rb → google_drive/worksheet.rb} +46 -173
  9. data/lib/mobilize-base/extensions/resque.rb +18 -24
  10. data/lib/mobilize-base/extensions/string.rb +12 -0
  11. data/lib/mobilize-base/handlers/gbook.rb +14 -47
  12. data/lib/mobilize-base/handlers/gdrive.rb +17 -18
  13. data/lib/mobilize-base/handlers/gfile.rb +18 -39
  14. data/lib/mobilize-base/handlers/gridfs.rb +43 -0
  15. data/lib/mobilize-base/handlers/gsheet.rb +48 -99
  16. data/lib/mobilize-base/jobtracker.rb +29 -15
  17. data/lib/mobilize-base/models/dataset.rb +33 -35
  18. data/lib/mobilize-base/models/job.rb +21 -168
  19. data/lib/mobilize-base/models/runner.rb +178 -0
  20. data/lib/mobilize-base/models/task.rb +137 -0
  21. data/lib/mobilize-base/models/user.rb +47 -0
  22. data/lib/mobilize-base/rakes.rb +59 -0
  23. data/lib/mobilize-base/version.rb +1 -1
  24. data/lib/mobilize-base.rb +20 -9
  25. data/lib/samples/gdrive.yml +12 -12
  26. data/lib/samples/gridfs.yml +9 -0
  27. data/lib/samples/gsheet.yml +6 -0
  28. data/lib/samples/jobtracker.yml +9 -9
  29. data/lib/samples/mongoid.yml +3 -3
  30. data/mobilize-base.gemspec +1 -1
  31. data/test/base1_task1.yml +3 -0
  32. data/test/base_job_rows.yml +13 -0
  33. data/test/mobilize-base_test.rb +59 -0
  34. metadata +20 -9
  35. data/lib/mobilize-base/handlers/mongodb.rb +0 -32
  36. data/lib/mobilize-base/models/requestor.rb +0 -232
  37. data/lib/mobilize-base/tasks.rb +0 -43
  38. data/test/mobilize_test.rb +0 -108
data/test/mobilize_test.rb DELETED
@@ -1,108 +0,0 @@
1
- require 'test_helper'
2
-
3
- describe "Mobilize" do
4
-
5
- def before
6
- puts 'nothing before'
7
- end
8
-
9
- # enqueues 4 workers on Resque
10
- it "runs integration test" do
11
-
12
- puts "restart test redis"
13
- Mobilize::Jobtracker.restart_test_redis
14
-
15
- puts "clear out test db"
16
- Mobilize::Jobtracker.drop_test_db
17
-
18
- email = Mobilize::Gdrive.owner_email
19
-
20
- #kill all workers
21
- Mobilize::Jobtracker.kill_workers
22
-
23
- puts 'enqueue 4 workers on Resque, wait 20s'
24
- Mobilize::Jobtracker.prep_workers
25
- sleep 20
26
- assert Mobilize::Jobtracker.workers.length == Mobilize::Resque.config['max_workers'].to_i
27
-
28
- puts "create requestor 'mobilize'"
29
- requestor = Mobilize::Requestor.find_or_create_by_email(email)
30
- assert requestor.email == email
31
-
32
- puts "delete old books and datasets"
33
- # delete any old jobspec from previous test runs
34
- jobspec_title = requestor.jobspec_title
35
- books = Mobilize::Gbook.find_all_by_title(jobspec_title)
36
- books.each{|book| book.delete}
37
-
38
- puts "enqueue jobtracker, wait 45s"
39
- Mobilize::Jobtracker.start
40
- sleep 45
41
- puts "jobtracker status: #{Mobilize::Jobtracker.status}"
42
- puts "status:#{Mobilize::Jobtracker.status}" #!= 'stopped'
43
-
44
- puts "requestor created jobspec?"
45
- books = Mobilize::Gbook.find_all_by_title(jobspec_title)
46
- assert books.length == 1
47
-
48
- puts "Jobtracker created jobspec with 'jobs' sheet?"
49
- jobs_sheets = Mobilize::Gsheet.find_all_by_name("#{jobspec_title}/Jobs",email)
50
- assert jobs_sheets.length == 1
51
-
52
- puts "add test_source data"
53
-
54
- test_source_rows = [
55
- ["test_header","test_header2","test_header3"],
56
- ["t1"]*3,
57
- ["t2"]*3
58
- ]
59
-
60
- book = books.first
61
- test_source_sheet = Mobilize::Gsheet.find_or_create_by_name("#{jobspec_title}/test_source",email)
62
-
63
- test_source_tsv = test_source_rows.map{|r| r.join("\t")}.join("\n")
64
- test_source_sheet.write(test_source_tsv)
65
-
66
- puts "add row to jobs sheet, wait 120s"
67
-
68
- jobs_sheet = jobs_sheets.first
69
-
70
- test_job_rows = [{"name" => "test",
71
- "active" => "true",
72
- "schedule" => "once",
73
- "status" => "",
74
- "last_error" => "",
75
- "destination_url" => "",
76
- "tasks" => "gsheet.read, gsheet.write",
77
- "datasets" => "test_source",
78
- "params" => "",
79
- "destination" => "test_destination"},
80
- #run after the first
81
- {"name" => "test2",
82
- "active" => "true",
83
- "schedule" => "after test",
84
- "status" => "",
85
- "last_error" => "",
86
- "destination_url" => "",
87
- "tasks" => "gsheet.read, gsheet.write",
88
- "datasets" => "test_source",
89
- "params" => "",
90
- "destination" => "test_destination2"}
91
- ]
92
-
93
- jobs_sheet.add_or_update_rows(test_job_rows)
94
-
95
- puts "job row added, force enqueued requestor"
96
- requestor.enqueue!
97
- sleep 120
98
-
99
- puts "jobtracker posted test sheet data to test destination, and checksum succeeded?"
100
- test_destination_sheet = Mobilize::Gsheet.find_or_create_by_name("#{jobspec_title}/test_destination",email)
101
-
102
- assert test_destination_sheet.to_tsv == test_source_sheet.to_tsv
103
-
104
- puts "stop test redis"
105
- Mobilize::Jobtracker.stop_test_redis
106
- end
107
-
108
- end