mobilize-base 1.0.2 → 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +5 -0
- data/LICENSE.txt +202 -20
- data/README.md +219 -138
- data/Rakefile +1 -2
- data/lib/mobilize-base/extensions/google_drive/acl.rb +25 -0
- data/lib/mobilize-base/extensions/google_drive/client_login_fetcher.rb +49 -0
- data/lib/mobilize-base/extensions/google_drive/file.rb +80 -0
- data/lib/mobilize-base/extensions/{google_drive.rb → google_drive/worksheet.rb} +46 -173
- data/lib/mobilize-base/extensions/resque.rb +18 -24
- data/lib/mobilize-base/extensions/string.rb +12 -0
- data/lib/mobilize-base/handlers/gbook.rb +14 -47
- data/lib/mobilize-base/handlers/gdrive.rb +17 -18
- data/lib/mobilize-base/handlers/gfile.rb +18 -39
- data/lib/mobilize-base/handlers/gridfs.rb +43 -0
- data/lib/mobilize-base/handlers/gsheet.rb +48 -99
- data/lib/mobilize-base/jobtracker.rb +29 -15
- data/lib/mobilize-base/models/dataset.rb +33 -35
- data/lib/mobilize-base/models/job.rb +21 -168
- data/lib/mobilize-base/models/runner.rb +178 -0
- data/lib/mobilize-base/models/task.rb +137 -0
- data/lib/mobilize-base/models/user.rb +47 -0
- data/lib/mobilize-base/rakes.rb +59 -0
- data/lib/mobilize-base/version.rb +1 -1
- data/lib/mobilize-base.rb +20 -9
- data/lib/samples/gdrive.yml +12 -12
- data/lib/samples/gridfs.yml +9 -0
- data/lib/samples/gsheet.yml +6 -0
- data/lib/samples/jobtracker.yml +9 -9
- data/lib/samples/mongoid.yml +3 -3
- data/mobilize-base.gemspec +1 -1
- data/test/base1_task1.yml +3 -0
- data/test/base_job_rows.yml +13 -0
- data/test/mobilize-base_test.rb +59 -0
- metadata +20 -9
- data/lib/mobilize-base/handlers/mongodb.rb +0 -32
- data/lib/mobilize-base/models/requestor.rb +0 -232
- data/lib/mobilize-base/tasks.rb +0 -43
- data/test/mobilize_test.rb +0 -108
@@ -0,0 +1,137 @@
|
|
1
|
+
module Mobilize
|
2
|
+
class Task
|
3
|
+
include Mongoid::Document
|
4
|
+
include Mongoid::Timestamps
|
5
|
+
field :path, type: String
|
6
|
+
field :handler, type: String
|
7
|
+
field :call, type: String
|
8
|
+
field :param_string, type: Array
|
9
|
+
field :status, type: String
|
10
|
+
field :last_completed_at, type: Time
|
11
|
+
field :last_run_at, type: Time
|
12
|
+
|
13
|
+
index({ path: 1})
|
14
|
+
|
15
|
+
def idx
|
16
|
+
t = self
|
17
|
+
t.path.split("/").last.gsub("task","").to_i
|
18
|
+
end
|
19
|
+
|
20
|
+
def stdout_dataset
|
21
|
+
t = self
|
22
|
+
Dataset.find_or_create_by_handler_and_path("gridfs","#{t.path}/stdout")
|
23
|
+
end
|
24
|
+
|
25
|
+
def stderr_dataset
|
26
|
+
t = self
|
27
|
+
Dataset.find_or_create_by_handler_and_path("gridfs","#{t.path}/stderr")
|
28
|
+
end
|
29
|
+
|
30
|
+
def params
|
31
|
+
t = self
|
32
|
+
t.param_string.split(",").map do |p|
|
33
|
+
ps = p.strip
|
34
|
+
ps = ps[1..-1] if ['"',"'"].include?(ps[0])
|
35
|
+
ps = ps[0..-2] if ['"',"'"].include?(ps[-1])
|
36
|
+
ps
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def job
|
41
|
+
t = self
|
42
|
+
job_path = t.path.split("/")[0..-2].join("/")
|
43
|
+
Job.where(:path=>job_path).first
|
44
|
+
end
|
45
|
+
|
46
|
+
def Task.find_or_create_by_path(path)
|
47
|
+
t = Task.where(:path=>path).first
|
48
|
+
t = Task.create(:path=>path) unless t
|
49
|
+
return t
|
50
|
+
end
|
51
|
+
|
52
|
+
def prior
|
53
|
+
t = self
|
54
|
+
j = t.job
|
55
|
+
return nil if t.idx==1
|
56
|
+
return j.tasks[t.idx-2]
|
57
|
+
end
|
58
|
+
|
59
|
+
def next
|
60
|
+
t = self
|
61
|
+
j = t.job
|
62
|
+
return nil if t.idx == j.tasks.length
|
63
|
+
return j.tasks[t.idx]
|
64
|
+
end
|
65
|
+
|
66
|
+
def Task.perform(id,*args)
|
67
|
+
t = Task.where(:path=>id).first
|
68
|
+
j = t.job
|
69
|
+
t.update_status(%{Starting at #{Time.now.utc}})
|
70
|
+
stdout, stderr = [nil,nil]
|
71
|
+
begin
|
72
|
+
stdout = "Mobilize::#{t.handler.humanize}".constantize.send("#{t.call}_by_task_path",t.path).to_s
|
73
|
+
rescue ScriptError, StandardError => exc
|
74
|
+
stderr = [exc.to_s,exc.backtrace.to_s].join("\n")
|
75
|
+
#record the failure in Job so it appears on Runner
|
76
|
+
j.update_attributes(:status=>"Failed at #{Time.now.utc.to_s}")
|
77
|
+
t.update_attributes(:status=>"Failed at #{Time.now.utc.to_s}")
|
78
|
+
raise exc
|
79
|
+
end
|
80
|
+
if stdout == false
|
81
|
+
#re-queue self if output is false
|
82
|
+
t.enqueue!
|
83
|
+
return false
|
84
|
+
end
|
85
|
+
#write output to cache
|
86
|
+
t.stdout_dataset.write_cache(stdout)
|
87
|
+
t.update_attributes(:status=>"Completed at #{Time.now.utc.to_s}")
|
88
|
+
if t.idx == j.tasks.length
|
89
|
+
j.update_attributes(:status=>"Completed at #{Time.now.utc.to_s}",:last_completed_at=>Time.now.utc)
|
90
|
+
j.update_attributes(:active=>false) if j.trigger.strip == "once"
|
91
|
+
t.update_attributes(:status=>"Completed at #{Time.now.utc.to_s}",:last_completed_at=>Time.now.utc)
|
92
|
+
#check for any dependent jobs, if there are, enqueue them
|
93
|
+
r = j.runner
|
94
|
+
dep_jobs = r.jobs.select{|dj| dj.active==true and dj.trigger=="after #{j.name}"}
|
95
|
+
#put begin/rescue so all dependencies run
|
96
|
+
dep_jobs.each{|dj| begin;dj.tasks.first.enqueue! unless dj.is_working?;rescue;end}
|
97
|
+
else
|
98
|
+
#queue up next task
|
99
|
+
t.next.enqueue!
|
100
|
+
end
|
101
|
+
return true
|
102
|
+
end
|
103
|
+
|
104
|
+
def enqueue!
|
105
|
+
t = self
|
106
|
+
::Resque::Job.create("mobilize",Task,t.path,{})
|
107
|
+
return true
|
108
|
+
end
|
109
|
+
|
110
|
+
def worker
|
111
|
+
t = self
|
112
|
+
Mobilize::Resque.find_worker_by_path(t.path)
|
113
|
+
end
|
114
|
+
|
115
|
+
def worker_args
|
116
|
+
t = self
|
117
|
+
Jobtracker.get_worker_args(t.worker)
|
118
|
+
end
|
119
|
+
|
120
|
+
def set_worker_args(args)
|
121
|
+
t = self
|
122
|
+
Jobtracker.set_worker_args(t.worker,args)
|
123
|
+
end
|
124
|
+
|
125
|
+
def update_status(msg)
|
126
|
+
t = self
|
127
|
+
t.update_attributes(:status=>msg)
|
128
|
+
Mobilize::Resque.set_worker_args_by_path(t.path,{'status'=>msg})
|
129
|
+
return true
|
130
|
+
end
|
131
|
+
|
132
|
+
def is_working?
|
133
|
+
t = self
|
134
|
+
Mobilize::Resque.active_paths.include?(t.path)
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
module Mobilize
|
2
|
+
class User
|
3
|
+
include Mongoid::Document
|
4
|
+
include Mongoid::Timestamps
|
5
|
+
field :active, type: Boolean
|
6
|
+
field :name, type: String
|
7
|
+
field :last_run, type: Time
|
8
|
+
|
9
|
+
def User.find_or_create_by_name(name)
|
10
|
+
u = User.where(:name => name).first
|
11
|
+
u = User.create(:name => name) unless u
|
12
|
+
return u
|
13
|
+
end
|
14
|
+
|
15
|
+
def email
|
16
|
+
u = self
|
17
|
+
"#{u.name}@#{Gdrive.domain}"
|
18
|
+
end
|
19
|
+
|
20
|
+
def runner
|
21
|
+
u = self
|
22
|
+
Runner.find_or_create_by_path(u.runner_path)
|
23
|
+
end
|
24
|
+
|
25
|
+
def jobs(jname=nil)
|
26
|
+
u = self
|
27
|
+
return u.runners.map{|r| r.jobs(jname)}.flatten
|
28
|
+
end
|
29
|
+
|
30
|
+
def runner_path
|
31
|
+
u = self
|
32
|
+
prefix = "Runner_"
|
33
|
+
suffix = ""
|
34
|
+
if Base.env == 'development'
|
35
|
+
suffix = "(dev)"
|
36
|
+
elsif Base.env == 'test'
|
37
|
+
suffix = "(test)"
|
38
|
+
elsif Base.env == 'production'
|
39
|
+
suffix = ""
|
40
|
+
else
|
41
|
+
raise "Invalid environment"
|
42
|
+
end
|
43
|
+
title = [prefix,u.name,suffix,"/jobs"].join
|
44
|
+
return title
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
# require 'resque/tasks'
|
2
|
+
# will give you the resque tasks
|
3
|
+
|
4
|
+
namespace :mobilize do
|
5
|
+
desc "Start a Resque worker"
|
6
|
+
task :work do
|
7
|
+
require 'resque'
|
8
|
+
begin
|
9
|
+
require 'mobilize-base'
|
10
|
+
#require specified mobilize gems
|
11
|
+
Mobilize::Base.config('jobtracker')['extensions'].each do |e|
|
12
|
+
require e
|
13
|
+
end
|
14
|
+
rescue Exception=>exc
|
15
|
+
end
|
16
|
+
|
17
|
+
begin
|
18
|
+
worker = Resque::Worker.new(Mobilize::Resque.config['queue_name'])
|
19
|
+
rescue Resque::NoQueueError
|
20
|
+
abort "set QUEUE env var, e.g. $ QUEUE=critical,high rake resque:work"
|
21
|
+
end
|
22
|
+
|
23
|
+
puts "Starting worker #{worker}"
|
24
|
+
|
25
|
+
worker.work(ENV['INTERVAL'] || 5) # interval, will block
|
26
|
+
end
|
27
|
+
end
|
28
|
+
namespace :mobilize_base do
|
29
|
+
desc "Set up config and log folders and files"
|
30
|
+
task :setup do
|
31
|
+
config_dir = (ENV['MOBILIZE_CONFIG_DIR'] ||= "config/mobilize/")
|
32
|
+
log_dir = (ENV['MOBILIZE_LOG_DIR'] ||= "log/")
|
33
|
+
sample_dir = File.dirname(__FILE__) + '/../samples/'
|
34
|
+
sample_files = Dir.entries(sample_dir)
|
35
|
+
full_config_dir = "#{ENV['PWD']}/#{config_dir}"
|
36
|
+
full_log_dir = "#{ENV['PWD']}/#{log_dir}"
|
37
|
+
unless File.exists?(full_config_dir)
|
38
|
+
puts "creating #{config_dir}"
|
39
|
+
`mkdir #{full_config_dir}`
|
40
|
+
end
|
41
|
+
unless File.exists?(full_log_dir)
|
42
|
+
puts "creating #{log_dir}"
|
43
|
+
`mkdir #{full_log_dir}`
|
44
|
+
end
|
45
|
+
sample_files.each do |fname|
|
46
|
+
unless File.exists?("#{full_config_dir}#{fname}")
|
47
|
+
puts "creating #{config_dir}#{fname}"
|
48
|
+
`cp #{sample_dir}#{fname} #{full_config_dir}#{fname}`
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
desc "create indexes for all base modelsin mongodb"
|
53
|
+
task :create_indexes do
|
54
|
+
require 'mobilize-base'
|
55
|
+
["Dataset","Job","Runner","Task","User"].each do |m|
|
56
|
+
"Mobilize::#{m}".constantize.create_indexes
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
data/lib/mobilize-base.rb
CHANGED
@@ -15,15 +15,18 @@ module Mobilize
|
|
15
15
|
ENV['PWD']
|
16
16
|
end
|
17
17
|
end
|
18
|
+
def Base.config_dir
|
19
|
+
ENV['MOBILIZE_CONFIG_DIR'] ||= "config/mobilize/"
|
20
|
+
end
|
18
21
|
def Base.config(config_name)
|
19
22
|
config_dir = begin
|
20
|
-
"#{::Rails.root}
|
23
|
+
"#{::Rails.root}/#{Base.config_dir}"
|
21
24
|
rescue
|
22
|
-
"#{Base.root}
|
25
|
+
"#{Base.root}/#{Base.config_dir}"
|
23
26
|
end
|
24
27
|
yaml_path = "#{config_dir}#{config_name}.yml"
|
25
28
|
if ::File.exists?(yaml_path)
|
26
|
-
return ::YAML.load_file(yaml_path)
|
29
|
+
return ::YAML.load_file(yaml_path)[Base.env]
|
27
30
|
else
|
28
31
|
raise "Could not find #{config_name}.yml in #{config_dir}"
|
29
32
|
end
|
@@ -36,11 +39,14 @@ module Mobilize
|
|
36
39
|
ENV['MOBILIZE_ENV'] || "development"
|
37
40
|
end
|
38
41
|
end
|
42
|
+
def Base.log_dir
|
43
|
+
ENV['MOBILIZE_LOG_DIR'] ||= "log/"
|
44
|
+
end
|
39
45
|
def Base.log_path(log_name)
|
40
46
|
log_dir = begin
|
41
|
-
"#{::Rails.root}
|
47
|
+
"#{::Rails.root}/#{Base.log_dir}"
|
42
48
|
rescue
|
43
|
-
"#{Base.root}
|
49
|
+
"#{Base.root}/#{Base.log_dir}"
|
44
50
|
end
|
45
51
|
log_path = "#{log_dir}#{log_name}.log"
|
46
52
|
if ::File.exists?(log_dir)
|
@@ -51,14 +57,16 @@ module Mobilize
|
|
51
57
|
end
|
52
58
|
end
|
53
59
|
end
|
54
|
-
mongoid_config_path = "#{Mobilize::Base.root}
|
60
|
+
mongoid_config_path = "#{Mobilize::Base.root}/#{Mobilize::Base.config_dir}mongoid.yml"
|
55
61
|
if File.exists?(mongoid_config_path)
|
56
62
|
require 'mongo'
|
57
63
|
require 'mongoid'
|
58
64
|
Mongoid.load!(mongoid_config_path, Mobilize::Base.env)
|
59
65
|
require "mobilize-base/models/dataset"
|
60
|
-
require "mobilize-base/models/
|
66
|
+
require "mobilize-base/models/user"
|
67
|
+
require "mobilize-base/models/runner"
|
61
68
|
require "mobilize-base/models/job"
|
69
|
+
require "mobilize-base/models/task"
|
62
70
|
end
|
63
71
|
require 'google_drive'
|
64
72
|
require 'resque'
|
@@ -71,6 +79,9 @@ require "mobilize-base/handlers/gdrive"
|
|
71
79
|
require "mobilize-base/handlers/gfile"
|
72
80
|
require "mobilize-base/handlers/gbook"
|
73
81
|
require "mobilize-base/handlers/gsheet"
|
74
|
-
require "mobilize-base/extensions/google_drive"
|
75
|
-
require "mobilize-base/
|
82
|
+
require "mobilize-base/extensions/google_drive/acl"
|
83
|
+
require "mobilize-base/extensions/google_drive/client_login_fetcher"
|
84
|
+
require "mobilize-base/extensions/google_drive/file"
|
85
|
+
require "mobilize-base/extensions/google_drive/worksheet"
|
86
|
+
require "mobilize-base/handlers/gridfs"
|
76
87
|
require "mobilize-base/handlers/email"
|
data/lib/samples/gdrive.yml
CHANGED
@@ -1,30 +1,30 @@
|
|
1
1
|
development:
|
2
2
|
domain: 'host.com'
|
3
3
|
owner:
|
4
|
-
|
4
|
+
name: 'owner_development'
|
5
5
|
pw: "google_drive_password"
|
6
6
|
admins:
|
7
|
-
- {
|
7
|
+
- {name: 'admin'}
|
8
8
|
workers:
|
9
|
-
- {
|
10
|
-
- {
|
9
|
+
- {name: 'worker_development001', pw: "worker001_google_drive_password"}
|
10
|
+
- {name: 'worker_development002', pw: "worker002_google_drive_password"}
|
11
11
|
test:
|
12
12
|
domain: 'host.com'
|
13
13
|
owner:
|
14
|
-
|
14
|
+
name: 'owner_test'
|
15
15
|
pw: "google_drive_password"
|
16
16
|
admins:
|
17
|
-
- {
|
17
|
+
- {name: 'admin'}
|
18
18
|
workers:
|
19
|
-
- {
|
20
|
-
- {
|
19
|
+
- {name: 'worker_test001', pw: "worker001_google_drive_password"}
|
20
|
+
- {name: 'worker_test002', pw: "worker002_google_drive_password"}
|
21
21
|
production:
|
22
22
|
domain: 'host.com'
|
23
23
|
owner:
|
24
|
-
|
24
|
+
name: 'owner_production'
|
25
25
|
pw: "google_drive_password"
|
26
26
|
admins:
|
27
|
-
- {
|
27
|
+
- {name: 'admin'}
|
28
28
|
workers:
|
29
|
-
- {
|
30
|
-
- {
|
29
|
+
- {name: 'worker_production001', pw: "worker001_google_drive_password"}
|
30
|
+
- {name: 'worker_production002', pw: "worker002_google_drive_password"}
|
@@ -0,0 +1,9 @@
|
|
1
|
+
development:
|
2
|
+
max_versions: 10 #number of versions of cache to keep in gridfs
|
3
|
+
max_compressed_write_size: 1000000000 #~1GB
|
4
|
+
test:
|
5
|
+
max_versions: 10 #number of versions of cache to keep in gridfs
|
6
|
+
max_compressed_write_size: 1000000000 #~1GB
|
7
|
+
production:
|
8
|
+
max_versions: 10 #number of versions of cache to keep in gridfs
|
9
|
+
max_compressed_write_size: 1000000000 #~1GB
|
data/lib/samples/jobtracker.yml
CHANGED
@@ -1,24 +1,24 @@
|
|
1
1
|
development:
|
2
|
-
#time between Jobtracker sweeps
|
3
|
-
cycle_freq: 10
|
2
|
+
cycle_freq: 10 #time between Jobtracker sweeps
|
4
3
|
notification_freq: 3600 #1 hour between failure/timeout notifications
|
5
|
-
|
4
|
+
runner_read_freq: 300 #5 min between runner reads
|
6
5
|
max_run_time: 14400 # if a job runs for 4h+, notification will be sent
|
6
|
+
extensions: [] #additional Mobilize modules to load workers with
|
7
7
|
admins: #emails to send notifications to
|
8
8
|
- {'email': 'admin@host.com'}
|
9
9
|
test:
|
10
|
-
#time between Jobtracker sweeps
|
11
|
-
cycle_freq: 10
|
10
|
+
cycle_freq: 10 #time between Jobtracker sweeps
|
12
11
|
notification_freq: 3600 #1 hour between failure/timeout notifications
|
13
|
-
|
12
|
+
runner_read_freq: 300 #5 min between runner reads
|
14
13
|
max_run_time: 14400 # if a job runs for 4h+, notification will be sent
|
14
|
+
extensions: [] #additional Mobilize modules to load workers with
|
15
15
|
admins: #emails to send notifications to
|
16
16
|
- {'email': 'admin@host.com'}
|
17
17
|
production:
|
18
|
-
#time between Jobtracker sweeps
|
19
|
-
cycle_freq: 10
|
18
|
+
cycle_freq: 10 #time between Jobtracker sweeps
|
20
19
|
notification_freq: 3600 #1 hour between failure/timeout notifications
|
21
|
-
|
20
|
+
runner_read_freq: 300 #5 min between runner reads
|
22
21
|
max_run_time: 14400 # if a job runs for 4h+, notification will be sent
|
22
|
+
extensions: [] #additional Mobilize modules to load workers with
|
23
23
|
admins: #emails to send notifications to
|
24
24
|
- {'email': 'admin@host.com'}
|
data/lib/samples/mongoid.yml
CHANGED
@@ -1,21 +1,21 @@
|
|
1
1
|
development:
|
2
2
|
sessions:
|
3
3
|
default:
|
4
|
-
database:
|
4
|
+
database: mobilize_development
|
5
5
|
persist_in_safe_mode: true
|
6
6
|
hosts:
|
7
7
|
- 127.0.0.1:27017
|
8
8
|
test:
|
9
9
|
sessions:
|
10
10
|
default:
|
11
|
-
database:
|
11
|
+
database: mobilize_test
|
12
12
|
persist_in_safe_mode: true
|
13
13
|
hosts:
|
14
14
|
- 127.0.0.1:27017
|
15
15
|
production:
|
16
16
|
sessions:
|
17
17
|
default:
|
18
|
-
database:
|
18
|
+
database: mobilize_production
|
19
19
|
persist_in_safe_mode: true
|
20
20
|
hosts:
|
21
21
|
- 127.0.0.1:27017
|