mobilize-base 1.1.0 → 1.1.01
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +5 -4
- data/lib/mobilize-base/extensions/yaml.rb +11 -0
- data/lib/mobilize-base/handlers/gsheet.rb +3 -2
- data/lib/mobilize-base/models/runner.rb +5 -0
- data/lib/mobilize-base/models/stage.rb +4 -11
- data/lib/mobilize-base/models/user.rb +16 -0
- data/lib/mobilize-base/tasks.rb +4 -6
- data/lib/mobilize-base/version.rb +1 -1
- data/lib/mobilize-base.rb +1 -0
- data/test/base_job_rows.yml +3 -3
- metadata +5 -4
data/README.md
CHANGED
@@ -511,13 +511,14 @@ stage. These should be of the for `<key1>: <value1>, <key2>: <value2>`, where
|
|
511
511
|
integer, an array (delimited by square braces), or a hash (delimited by
|
512
512
|
curly braces).
|
513
513
|
* For mobilize-base, the following stages are available:
|
514
|
-
* gsheet.read `source: <
|
515
|
-
* The
|
516
|
-
|
514
|
+
* gsheet.read `source: <input_gsheet_path>`, which reads the sheet.
|
515
|
+
* The gsheet_path should be of the form
|
516
|
+
`<gbook_name>/<gsheet_name>` or just `<gsheet_name>` if the target is in
|
517
|
+
the Runner itself. The test uses "base1_stage1.in".
|
517
518
|
* gsheet.write `source: <stage_name>`,`target: <target_gsheet_path>`,
|
518
519
|
which writes the specified stage output to the target_gsheet.
|
519
520
|
* The stage_name should be of the form `<stage_column>`. The test uses "stage1" for the first test
|
520
|
-
and "
|
521
|
+
and "base1.out" for the second test. The first
|
521
522
|
takes the output from the first stage and the second reads it straight
|
522
523
|
from the referenced sheet.
|
523
524
|
* The test uses "Requestor_mobilize(test)/base1.out" and
|
@@ -0,0 +1,11 @@
|
|
1
|
+
module YAML
|
2
|
+
def YAML.easy_load(string)
|
3
|
+
begin
|
4
|
+
YAML.load(s.param_string)
|
5
|
+
rescue
|
6
|
+
#replace colon w space colon, double space colons w single space
|
7
|
+
gsub_colon_string = string.gsub(":",": ").gsub(": ",": ")
|
8
|
+
YAML.load("{#{gsub_colon_string}}")
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
@@ -38,8 +38,8 @@ module Mobilize
|
|
38
38
|
return false unless gdrive_slot
|
39
39
|
s = Stage.where(:path=>stage_path).first
|
40
40
|
user = s.job.runner.user.name
|
41
|
-
|
42
|
-
out_tsv =
|
41
|
+
source_dst = s.source_dsts(gdrive_slot).first
|
42
|
+
out_tsv = source_dst.read(user)
|
43
43
|
#use Gridfs to cache result
|
44
44
|
out_url = "gridfs://#{s.path}/out"
|
45
45
|
Dataset.write_by_url(out_url,out_tsv,Gdrive.owner_name)
|
@@ -52,6 +52,7 @@ module Mobilize
|
|
52
52
|
s = Stage.where(:path=>stage_path).first
|
53
53
|
user = s.job.runner.user.name
|
54
54
|
target_path = s.params['target']
|
55
|
+
target_path = "#{s.job.runner.title}/#{target_path}" unless target_path.index("/")
|
55
56
|
source_dst = s.source_dsts(gdrive_slot).first
|
56
57
|
tsv = source_dst.read(user)
|
57
58
|
sheet_name = target_path.split("/").last
|
@@ -30,16 +30,9 @@ module Mobilize
|
|
30
30
|
|
31
31
|
def params
|
32
32
|
s = self
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
begin
|
37
|
-
YAML.load(s.param_string)
|
38
|
-
raise "Must resolve to Hash" unless result.class==Hash
|
39
|
-
rescue
|
40
|
-
sub_param_string = s.param_string.gsub(":\"",": \"").gsub(":'",": '").gsub(":[",": [").gsub(":{",": {").gsub(/(:[0-9])/,'stageparamsgsub\1').gsub('stageparamsgsub:',': ')
|
41
|
-
YAML.load("{#{sub_param_string}}")
|
42
|
-
end
|
33
|
+
p = YAML.easy_load(s.param_string)
|
34
|
+
raise "Must resolve to Hash" unless p.class==Hash
|
35
|
+
return p
|
43
36
|
end
|
44
37
|
|
45
38
|
def job
|
@@ -134,7 +127,7 @@ module Mobilize
|
|
134
127
|
else
|
135
128
|
#check sheets in runner
|
136
129
|
r = s.job.runner
|
137
|
-
runner_sheet = r.gbook.worksheet_by_title(source_path)
|
130
|
+
runner_sheet = r.gbook(gdrive_slot).worksheet_by_title(source_path)
|
138
131
|
out_tsv = if runner_sheet
|
139
132
|
runner_sheet.read(user)
|
140
133
|
else
|
@@ -27,6 +27,22 @@ module Mobilize
|
|
27
27
|
return u.runner.jobs
|
28
28
|
end
|
29
29
|
|
30
|
+
def creds(gdrive_slot)
|
31
|
+
u = self
|
32
|
+
creds_path = "#{u.runner.path.split("/").first}/creds"
|
33
|
+
begin
|
34
|
+
creds_sheet = Gsheet.find_by_path(creds_path,gdrive_slot)
|
35
|
+
cred_array = creds_sheet.read(u.name).tsv_to_hash_array.map{|h| {h['name']=>{'user'=>h['user'],'password'=>h['password']}}}
|
36
|
+
result = {}
|
37
|
+
cred_array.each do |cred|
|
38
|
+
result[cred.keys.first] = cred.values.first
|
39
|
+
end
|
40
|
+
return result
|
41
|
+
rescue
|
42
|
+
return {}
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
30
46
|
def runner_path
|
31
47
|
u = self
|
32
48
|
prefix = "Runner_"
|
data/lib/mobilize-base/tasks.rb
CHANGED
@@ -2,15 +2,13 @@ namespace :mobilize_base do
|
|
2
2
|
desc "Start a Resque worker"
|
3
3
|
task :work do
|
4
4
|
require 'mobilize-base'
|
5
|
-
|
6
|
-
|
7
|
-
Mobilize::Base.config('jobtracker')['extensions'].each do |e|
|
5
|
+
Mobilize::Base.config('jobtracker')['extensions'].each do |e|
|
6
|
+
begin
|
8
7
|
require e
|
8
|
+
rescue Exception=>exc
|
9
|
+
#do nothing
|
9
10
|
end
|
10
|
-
rescue Exception=>exc
|
11
|
-
#do nothing
|
12
11
|
end
|
13
|
-
|
14
12
|
begin
|
15
13
|
worker = Resque::Worker.new(Mobilize::Resque.config['queue_name'])
|
16
14
|
rescue Resque::NoQueueError
|
data/lib/mobilize-base.rb
CHANGED
@@ -3,6 +3,7 @@ require "mobilize-base/extensions/array"
|
|
3
3
|
require "mobilize-base/extensions/hash"
|
4
4
|
require "mobilize-base/extensions/object"
|
5
5
|
require "mobilize-base/extensions/string"
|
6
|
+
require "mobilize-base/extensions/yaml"
|
6
7
|
#this is the base of the mobilize object, any methods that should be
|
7
8
|
#made available application-wide go over here
|
8
9
|
#these also define base variables for Rails
|
data/test/base_job_rows.yml
CHANGED
@@ -2,11 +2,11 @@
|
|
2
2
|
active: true
|
3
3
|
trigger: once
|
4
4
|
status: ""
|
5
|
-
stage1: gsheet.read source:
|
6
|
-
stage2: gsheet.write source:
|
5
|
+
stage1: gsheet.read source:base1_stage1.in
|
6
|
+
stage2: gsheet.write source:stage1, target:base1.out
|
7
7
|
|
8
8
|
- name: base2
|
9
9
|
active: true
|
10
10
|
trigger: after base1
|
11
11
|
status: ""
|
12
|
-
stage1: gsheet.write source:
|
12
|
+
stage1: gsheet.write source:base1.out, target:base2.out
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mobilize-base
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.1.
|
4
|
+
version: 1.1.01
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-01-
|
12
|
+
date: 2013-01-26 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rake
|
@@ -196,6 +196,7 @@ files:
|
|
196
196
|
- lib/mobilize-base/extensions/hash.rb
|
197
197
|
- lib/mobilize-base/extensions/object.rb
|
198
198
|
- lib/mobilize-base/extensions/string.rb
|
199
|
+
- lib/mobilize-base/extensions/yaml.rb
|
199
200
|
- lib/mobilize-base/handlers/email.rb
|
200
201
|
- lib/mobilize-base/handlers/gbook.rb
|
201
202
|
- lib/mobilize-base/handlers/gdrive.rb
|
@@ -238,7 +239,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
238
239
|
version: '0'
|
239
240
|
segments:
|
240
241
|
- 0
|
241
|
-
hash:
|
242
|
+
hash: 3362582751526690476
|
242
243
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
243
244
|
none: false
|
244
245
|
requirements:
|
@@ -247,7 +248,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
247
248
|
version: '0'
|
248
249
|
segments:
|
249
250
|
- 0
|
250
|
-
hash:
|
251
|
+
hash: 3362582751526690476
|
251
252
|
requirements: []
|
252
253
|
rubyforge_project: mobilize-base
|
253
254
|
rubygems_version: 1.8.24
|