mobilize-base 1.2 → 1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.md +10 -11
- data/lib/mobilize-base/extensions/google_drive/file.rb +7 -7
- data/lib/mobilize-base/extensions/google_drive/worksheet.rb +7 -2
- data/lib/mobilize-base/extensions/resque-server/views/queues.erb +59 -0
- data/lib/mobilize-base/extensions/resque-server/views/working.erb +85 -0
- data/lib/mobilize-base/extensions/string.rb +12 -4
- data/lib/mobilize-base/extensions/yaml.rb +11 -7
- data/lib/mobilize-base/handlers/gbook.rb +24 -31
- data/lib/mobilize-base/handlers/gfile.rb +5 -3
- data/lib/mobilize-base/handlers/gridfs.rb +19 -24
- data/lib/mobilize-base/handlers/gsheet.rb +25 -20
- data/lib/mobilize-base/handlers/resque.rb +16 -4
- data/lib/mobilize-base/helpers/job_helper.rb +54 -0
- data/lib/mobilize-base/helpers/runner_helper.rb +83 -0
- data/lib/mobilize-base/helpers/stage_helper.rb +38 -0
- data/lib/mobilize-base/jobtracker.rb +13 -5
- data/lib/mobilize-base/models/job.rb +36 -48
- data/lib/mobilize-base/models/runner.rb +24 -123
- data/lib/mobilize-base/models/stage.rb +14 -43
- data/lib/mobilize-base/tasks.rb +16 -3
- data/lib/mobilize-base/version.rb +1 -1
- data/lib/mobilize-base.rb +5 -1
- data/lib/samples/gridfs.yml +0 -3
- data/lib/samples/gsheet.yml +4 -4
- data/mobilize-base.gemspec +4 -5
- data/test/mobilize-base_test.rb +1 -0
- metadata +21 -32
data/lib/mobilize-base/tasks.rb
CHANGED
@@ -40,17 +40,17 @@ namespace :mobilize_base do
|
|
40
40
|
Mobilize::Jobtracker.prep_workers
|
41
41
|
end
|
42
42
|
desc "Stop Jobtracker"
|
43
|
-
task :
|
43
|
+
task :stop_jobtracker do
|
44
44
|
require 'mobilize-base'
|
45
45
|
Mobilize::Jobtracker.stop!
|
46
46
|
end
|
47
47
|
desc "Start Jobtracker"
|
48
|
-
task :
|
48
|
+
task :start_jobtracker do
|
49
49
|
require 'mobilize-base'
|
50
50
|
Mobilize::Jobtracker.start
|
51
51
|
end
|
52
52
|
desc "Restart Jobtracker"
|
53
|
-
task :
|
53
|
+
task :restart_jobtracker do
|
54
54
|
require 'mobilize-base'
|
55
55
|
Mobilize::Jobtracker.restart!
|
56
56
|
end
|
@@ -66,6 +66,19 @@ namespace :mobilize_base do
|
|
66
66
|
resque_redis_port_args = if Mobilize::Base.env == 'test'
|
67
67
|
" -r localhost:#{Mobilize::Base.config('resque')['redis_port']}"
|
68
68
|
end.to_s
|
69
|
+
#determine view folder and override queues and working erbs
|
70
|
+
require 'resque/server'
|
71
|
+
view_dir = ::Resque::Server.views + "/"
|
72
|
+
old_queues_erb_path = view_dir + "queues.erb"
|
73
|
+
old_working_erb_path = view_dir + "working.erb"
|
74
|
+
gem_dir = Gem::Specification.find_by_name("mobilize-base").gem_dir
|
75
|
+
new_queues_erb_path = gem_dir + "/lib/mobilize-base/extensions/resque-server/views/queues.erb"
|
76
|
+
new_working_erb_path = gem_dir + "/lib/mobilize-base/extensions/resque-server/views/working.erb"
|
77
|
+
[old_queues_erb_path,old_working_erb_path].each{|p| File.delete(p) if File.exists?(p)}
|
78
|
+
require 'fileutils'
|
79
|
+
FileUtils.copy(new_queues_erb_path,old_queues_erb_path)
|
80
|
+
FileUtils.copy(new_working_erb_path,old_working_erb_path)
|
81
|
+
sleep 5 #give them time to die
|
69
82
|
command = "bundle exec resque-web -p #{port.to_s} #{resque_web_extension_path} #{resque_redis_port_args}"
|
70
83
|
`#{command}`
|
71
84
|
end
|
data/lib/mobilize-base.rb
CHANGED
@@ -60,14 +60,18 @@ module Mobilize
|
|
60
60
|
end
|
61
61
|
mongoid_config_path = "#{Mobilize::Base.root}/#{Mobilize::Base.config_dir}mongoid.yml"
|
62
62
|
if File.exists?(mongoid_config_path)
|
63
|
-
require 'mongo'
|
64
63
|
require 'mongoid'
|
64
|
+
require 'mongoid-grid_fs'
|
65
65
|
Mongoid.load!(mongoid_config_path, Mobilize::Base.env)
|
66
66
|
require "mobilize-base/models/dataset"
|
67
67
|
require "mobilize-base/models/user"
|
68
|
+
require "mobilize-base/helpers/runner_helper"
|
68
69
|
require "mobilize-base/models/runner"
|
70
|
+
require "mobilize-base/helpers/job_helper"
|
69
71
|
require "mobilize-base/models/job"
|
72
|
+
require "mobilize-base/helpers/stage_helper"
|
70
73
|
require "mobilize-base/models/stage"
|
74
|
+
|
71
75
|
end
|
72
76
|
require 'google_drive'
|
73
77
|
require 'resque'
|
data/lib/samples/gridfs.yml
CHANGED
@@ -1,10 +1,7 @@
|
|
1
1
|
---
|
2
2
|
development:
|
3
|
-
max_versions: 10 #number of versions of cache to keep in gridfs
|
4
3
|
max_compressed_write_size: 1000000000 #~1GB
|
5
4
|
test:
|
6
|
-
max_versions: 10 #number of versions of cache to keep in gridfs
|
7
5
|
max_compressed_write_size: 1000000000 #~1GB
|
8
6
|
production:
|
9
|
-
max_versions: 10 #number of versions of cache to keep in gridfs
|
10
7
|
max_compressed_write_size: 1000000000 #~1GB
|
data/lib/samples/gsheet.yml
CHANGED
@@ -1,17 +1,17 @@
|
|
1
1
|
---
|
2
2
|
development:
|
3
|
-
max_cells:
|
3
|
+
max_cells: 50000 #current google limit is 400k cells per book but fails earlier
|
4
4
|
read_date_format: "%Y-%m-%d" #format to record when reading sheets
|
5
5
|
sheet_date_format: "%m/%d/%Y" #format to use to parse sheets
|
6
6
|
test:
|
7
|
-
max_cells:
|
7
|
+
max_cells: 50000 #current google limit is 400k cells per book but fails earlier
|
8
8
|
read_date_format: "%Y-%m-%d" #format to record when reading sheets
|
9
9
|
sheet_date_format: "%m/%d/%Y" #format to use to parse sheets
|
10
10
|
staging:
|
11
|
-
max_cells:
|
11
|
+
max_cells: 50000 #current google limit is 400k cells per book but fails earlier
|
12
12
|
read_date_format: "%Y-%m-%d" #format to record when reading sheets
|
13
13
|
sheet_date_format: "%m/%d/%Y" #format to use to parse sheets
|
14
14
|
production:
|
15
|
-
max_cells:
|
15
|
+
max_cells: 50000 #current google limit is 400k cells per book but fails earlier
|
16
16
|
read_date_format: "%Y-%m-%d" #format to record when reading sheets
|
17
17
|
sheet_date_format: "%m/%d/%Y" #format to use to parse sheets
|
data/mobilize-base.gemspec
CHANGED
@@ -22,12 +22,11 @@ Gem::Specification.new do |s|
|
|
22
22
|
s.require_paths = ["lib"]
|
23
23
|
|
24
24
|
s.add_runtime_dependency 'rake'
|
25
|
-
s.add_runtime_dependency 'bson','1.
|
26
|
-
s.add_runtime_dependency 'bson_ext','1.
|
27
|
-
s.add_runtime_dependency 'mongo', '1.6.1'
|
25
|
+
s.add_runtime_dependency 'bson','1.8.4'
|
26
|
+
s.add_runtime_dependency 'bson_ext','1.8.4'
|
28
27
|
s.add_runtime_dependency "mongoid", "~>3.0.0"
|
29
|
-
s.add_runtime_dependency '
|
30
|
-
s.add_runtime_dependency 'resque','1.
|
28
|
+
s.add_runtime_dependency 'mongoid-grid_fs'
|
29
|
+
s.add_runtime_dependency 'resque', '1.24.0'
|
31
30
|
s.add_runtime_dependency 'google_drive','0.3.2'
|
32
31
|
s.add_runtime_dependency 'popen4','0.1.2'
|
33
32
|
s.add_runtime_dependency 'actionmailer','3.1.1'
|
data/test/mobilize-base_test.rb
CHANGED
@@ -64,6 +64,7 @@ describe "Mobilize" do
|
|
64
64
|
test_1_tsv = Mobilize::Dataset.read_by_url(test_target_sheet_1_url,user_name,gdrive_slot)
|
65
65
|
test_2_tsv = Mobilize::Dataset.read_by_url(test_target_sheet_1_url,user_name,gdrive_slot)
|
66
66
|
|
67
|
+
assert test_1_tsv.to_s.length>0
|
67
68
|
assert test_1_tsv == test_2_tsv
|
68
69
|
|
69
70
|
puts "change first job to fail, wait for stages"
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mobilize-base
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: '1.
|
4
|
+
version: '1.3'
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-
|
12
|
+
date: 2013-04-18 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rake
|
@@ -34,7 +34,7 @@ dependencies:
|
|
34
34
|
requirements:
|
35
35
|
- - '='
|
36
36
|
- !ruby/object:Gem::Version
|
37
|
-
version: 1.
|
37
|
+
version: 1.8.4
|
38
38
|
type: :runtime
|
39
39
|
prerelease: false
|
40
40
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -42,7 +42,7 @@ dependencies:
|
|
42
42
|
requirements:
|
43
43
|
- - '='
|
44
44
|
- !ruby/object:Gem::Version
|
45
|
-
version: 1.
|
45
|
+
version: 1.8.4
|
46
46
|
- !ruby/object:Gem::Dependency
|
47
47
|
name: bson_ext
|
48
48
|
requirement: !ruby/object:Gem::Requirement
|
@@ -50,7 +50,7 @@ dependencies:
|
|
50
50
|
requirements:
|
51
51
|
- - '='
|
52
52
|
- !ruby/object:Gem::Version
|
53
|
-
version: 1.
|
53
|
+
version: 1.8.4
|
54
54
|
type: :runtime
|
55
55
|
prerelease: false
|
56
56
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -58,23 +58,7 @@ dependencies:
|
|
58
58
|
requirements:
|
59
59
|
- - '='
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: 1.
|
62
|
-
- !ruby/object:Gem::Dependency
|
63
|
-
name: mongo
|
64
|
-
requirement: !ruby/object:Gem::Requirement
|
65
|
-
none: false
|
66
|
-
requirements:
|
67
|
-
- - '='
|
68
|
-
- !ruby/object:Gem::Version
|
69
|
-
version: 1.6.1
|
70
|
-
type: :runtime
|
71
|
-
prerelease: false
|
72
|
-
version_requirements: !ruby/object:Gem::Requirement
|
73
|
-
none: false
|
74
|
-
requirements:
|
75
|
-
- - '='
|
76
|
-
- !ruby/object:Gem::Version
|
77
|
-
version: 1.6.1
|
61
|
+
version: 1.8.4
|
78
62
|
- !ruby/object:Gem::Dependency
|
79
63
|
name: mongoid
|
80
64
|
requirement: !ruby/object:Gem::Requirement
|
@@ -92,21 +76,21 @@ dependencies:
|
|
92
76
|
- !ruby/object:Gem::Version
|
93
77
|
version: 3.0.0
|
94
78
|
- !ruby/object:Gem::Dependency
|
95
|
-
name:
|
79
|
+
name: mongoid-grid_fs
|
96
80
|
requirement: !ruby/object:Gem::Requirement
|
97
81
|
none: false
|
98
82
|
requirements:
|
99
|
-
- -
|
83
|
+
- - ! '>='
|
100
84
|
- !ruby/object:Gem::Version
|
101
|
-
version:
|
85
|
+
version: '0'
|
102
86
|
type: :runtime
|
103
87
|
prerelease: false
|
104
88
|
version_requirements: !ruby/object:Gem::Requirement
|
105
89
|
none: false
|
106
90
|
requirements:
|
107
|
-
- -
|
91
|
+
- - ! '>='
|
108
92
|
- !ruby/object:Gem::Version
|
109
|
-
version:
|
93
|
+
version: '0'
|
110
94
|
- !ruby/object:Gem::Dependency
|
111
95
|
name: resque
|
112
96
|
requirement: !ruby/object:Gem::Requirement
|
@@ -114,7 +98,7 @@ dependencies:
|
|
114
98
|
requirements:
|
115
99
|
- - '='
|
116
100
|
- !ruby/object:Gem::Version
|
117
|
-
version: 1.
|
101
|
+
version: 1.24.0
|
118
102
|
type: :runtime
|
119
103
|
prerelease: false
|
120
104
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -122,7 +106,7 @@ dependencies:
|
|
122
106
|
requirements:
|
123
107
|
- - '='
|
124
108
|
- !ruby/object:Gem::Version
|
125
|
-
version: 1.
|
109
|
+
version: 1.24.0
|
126
110
|
- !ruby/object:Gem::Dependency
|
127
111
|
name: google_drive
|
128
112
|
requirement: !ruby/object:Gem::Requirement
|
@@ -195,6 +179,8 @@ files:
|
|
195
179
|
- lib/mobilize-base/extensions/google_drive/worksheet.rb
|
196
180
|
- lib/mobilize-base/extensions/hash.rb
|
197
181
|
- lib/mobilize-base/extensions/object.rb
|
182
|
+
- lib/mobilize-base/extensions/resque-server/views/queues.erb
|
183
|
+
- lib/mobilize-base/extensions/resque-server/views/working.erb
|
198
184
|
- lib/mobilize-base/extensions/string.rb
|
199
185
|
- lib/mobilize-base/extensions/yaml.rb
|
200
186
|
- lib/mobilize-base/handlers/email.rb
|
@@ -204,6 +190,9 @@ files:
|
|
204
190
|
- lib/mobilize-base/handlers/gridfs.rb
|
205
191
|
- lib/mobilize-base/handlers/gsheet.rb
|
206
192
|
- lib/mobilize-base/handlers/resque.rb
|
193
|
+
- lib/mobilize-base/helpers/job_helper.rb
|
194
|
+
- lib/mobilize-base/helpers/runner_helper.rb
|
195
|
+
- lib/mobilize-base/helpers/stage_helper.rb
|
207
196
|
- lib/mobilize-base/jobtracker.rb
|
208
197
|
- lib/mobilize-base/models/dataset.rb
|
209
198
|
- lib/mobilize-base/models/job.rb
|
@@ -239,7 +228,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
239
228
|
version: '0'
|
240
229
|
segments:
|
241
230
|
- 0
|
242
|
-
hash: -
|
231
|
+
hash: -3969243247461811352
|
243
232
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
244
233
|
none: false
|
245
234
|
requirements:
|
@@ -248,10 +237,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
248
237
|
version: '0'
|
249
238
|
segments:
|
250
239
|
- 0
|
251
|
-
hash: -
|
240
|
+
hash: -3969243247461811352
|
252
241
|
requirements: []
|
253
242
|
rubyforge_project: mobilize-base
|
254
|
-
rubygems_version: 1.8.
|
243
|
+
rubygems_version: 1.8.25
|
255
244
|
signing_key:
|
256
245
|
specification_version: 3
|
257
246
|
summary: Moves datasets and schedules data transfers using MongoDB, Resque and Google
|