mobilize-base 1.0.2 → 1.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. data/.gitignore +5 -0
  2. data/LICENSE.txt +202 -20
  3. data/README.md +219 -138
  4. data/Rakefile +1 -2
  5. data/lib/mobilize-base/extensions/google_drive/acl.rb +25 -0
  6. data/lib/mobilize-base/extensions/google_drive/client_login_fetcher.rb +49 -0
  7. data/lib/mobilize-base/extensions/google_drive/file.rb +80 -0
  8. data/lib/mobilize-base/extensions/{google_drive.rb → google_drive/worksheet.rb} +46 -173
  9. data/lib/mobilize-base/extensions/resque.rb +18 -24
  10. data/lib/mobilize-base/extensions/string.rb +12 -0
  11. data/lib/mobilize-base/handlers/gbook.rb +14 -47
  12. data/lib/mobilize-base/handlers/gdrive.rb +17 -18
  13. data/lib/mobilize-base/handlers/gfile.rb +18 -39
  14. data/lib/mobilize-base/handlers/gridfs.rb +43 -0
  15. data/lib/mobilize-base/handlers/gsheet.rb +48 -99
  16. data/lib/mobilize-base/jobtracker.rb +29 -15
  17. data/lib/mobilize-base/models/dataset.rb +33 -35
  18. data/lib/mobilize-base/models/job.rb +21 -168
  19. data/lib/mobilize-base/models/runner.rb +178 -0
  20. data/lib/mobilize-base/models/task.rb +137 -0
  21. data/lib/mobilize-base/models/user.rb +47 -0
  22. data/lib/mobilize-base/rakes.rb +59 -0
  23. data/lib/mobilize-base/version.rb +1 -1
  24. data/lib/mobilize-base.rb +20 -9
  25. data/lib/samples/gdrive.yml +12 -12
  26. data/lib/samples/gridfs.yml +9 -0
  27. data/lib/samples/gsheet.yml +6 -0
  28. data/lib/samples/jobtracker.yml +9 -9
  29. data/lib/samples/mongoid.yml +3 -3
  30. data/mobilize-base.gemspec +1 -1
  31. data/test/base1_task1.yml +3 -0
  32. data/test/base_job_rows.yml +13 -0
  33. data/test/mobilize-base_test.rb +59 -0
  34. metadata +20 -9
  35. data/lib/mobilize-base/handlers/mongodb.rb +0 -32
  36. data/lib/mobilize-base/models/requestor.rb +0 -232
  37. data/lib/mobilize-base/tasks.rb +0 -43
  38. data/test/mobilize_test.rb +0 -108
@@ -0,0 +1,137 @@
1
+ module Mobilize
2
+ class Task
3
+ include Mongoid::Document
4
+ include Mongoid::Timestamps
5
+ field :path, type: String
6
+ field :handler, type: String
7
+ field :call, type: String
8
+ field :param_string, type: Array
9
+ field :status, type: String
10
+ field :last_completed_at, type: Time
11
+ field :last_run_at, type: Time
12
+
13
+ index({ path: 1})
14
+
15
+ def idx
16
+ t = self
17
+ t.path.split("/").last.gsub("task","").to_i
18
+ end
19
+
20
+ def stdout_dataset
21
+ t = self
22
+ Dataset.find_or_create_by_handler_and_path("gridfs","#{t.path}/stdout")
23
+ end
24
+
25
+ def stderr_dataset
26
+ t = self
27
+ Dataset.find_or_create_by_handler_and_path("gridfs","#{t.path}/stderr")
28
+ end
29
+
30
+ def params
31
+ t = self
32
+ t.param_string.split(",").map do |p|
33
+ ps = p.strip
34
+ ps = ps[1..-1] if ['"',"'"].include?(ps[0])
35
+ ps = ps[0..-2] if ['"',"'"].include?(ps[-1])
36
+ ps
37
+ end
38
+ end
39
+
40
+ def job
41
+ t = self
42
+ job_path = t.path.split("/")[0..-2].join("/")
43
+ Job.where(:path=>job_path).first
44
+ end
45
+
46
+ def Task.find_or_create_by_path(path)
47
+ t = Task.where(:path=>path).first
48
+ t = Task.create(:path=>path) unless t
49
+ return t
50
+ end
51
+
52
+ def prior
53
+ t = self
54
+ j = t.job
55
+ return nil if t.idx==1
56
+ return j.tasks[t.idx-2]
57
+ end
58
+
59
+ def next
60
+ t = self
61
+ j = t.job
62
+ return nil if t.idx == j.tasks.length
63
+ return j.tasks[t.idx]
64
+ end
65
+
66
+ def Task.perform(id,*args)
67
+ t = Task.where(:path=>id).first
68
+ j = t.job
69
+ t.update_status(%{Starting at #{Time.now.utc}})
70
+ stdout, stderr = [nil,nil]
71
+ begin
72
+ stdout = "Mobilize::#{t.handler.humanize}".constantize.send("#{t.call}_by_task_path",t.path).to_s
73
+ rescue ScriptError, StandardError => exc
74
+ stderr = [exc.to_s,exc.backtrace.to_s].join("\n")
75
+ #record the failure in Job so it appears on Runner
76
+ j.update_attributes(:status=>"Failed at #{Time.now.utc.to_s}")
77
+ t.update_attributes(:status=>"Failed at #{Time.now.utc.to_s}")
78
+ raise exc
79
+ end
80
+ if stdout == false
81
+ #re-queue self if output is false
82
+ t.enqueue!
83
+ return false
84
+ end
85
+ #write output to cache
86
+ t.stdout_dataset.write_cache(stdout)
87
+ t.update_attributes(:status=>"Completed at #{Time.now.utc.to_s}")
88
+ if t.idx == j.tasks.length
89
+ j.update_attributes(:status=>"Completed at #{Time.now.utc.to_s}",:last_completed_at=>Time.now.utc)
90
+ j.update_attributes(:active=>false) if j.trigger.strip == "once"
91
+ t.update_attributes(:status=>"Completed at #{Time.now.utc.to_s}",:last_completed_at=>Time.now.utc)
92
+ #check for any dependent jobs, if there are, enqueue them
93
+ r = j.runner
94
+ dep_jobs = r.jobs.select{|dj| dj.active==true and dj.trigger=="after #{j.name}"}
95
+ #put begin/rescue so all dependencies run
96
+ dep_jobs.each{|dj| begin;dj.tasks.first.enqueue! unless dj.is_working?;rescue;end}
97
+ else
98
+ #queue up next task
99
+ t.next.enqueue!
100
+ end
101
+ return true
102
+ end
103
+
104
+ def enqueue!
105
+ t = self
106
+ ::Resque::Job.create("mobilize",Task,t.path,{})
107
+ return true
108
+ end
109
+
110
+ def worker
111
+ t = self
112
+ Mobilize::Resque.find_worker_by_path(t.path)
113
+ end
114
+
115
+ def worker_args
116
+ t = self
117
+ Jobtracker.get_worker_args(t.worker)
118
+ end
119
+
120
+ def set_worker_args(args)
121
+ t = self
122
+ Jobtracker.set_worker_args(t.worker,args)
123
+ end
124
+
125
+ def update_status(msg)
126
+ t = self
127
+ t.update_attributes(:status=>msg)
128
+ Mobilize::Resque.set_worker_args_by_path(t.path,{'status'=>msg})
129
+ return true
130
+ end
131
+
132
+ def is_working?
133
+ t = self
134
+ Mobilize::Resque.active_paths.include?(t.path)
135
+ end
136
+ end
137
+ end
@@ -0,0 +1,47 @@
1
+ module Mobilize
2
+ class User
3
+ include Mongoid::Document
4
+ include Mongoid::Timestamps
5
+ field :active, type: Boolean
6
+ field :name, type: String
7
+ field :last_run, type: Time
8
+
9
+ def User.find_or_create_by_name(name)
10
+ u = User.where(:name => name).first
11
+ u = User.create(:name => name) unless u
12
+ return u
13
+ end
14
+
15
+ def email
16
+ u = self
17
+ "#{u.name}@#{Gdrive.domain}"
18
+ end
19
+
20
+ def runner
21
+ u = self
22
+ Runner.find_or_create_by_path(u.runner_path)
23
+ end
24
+
25
+ def jobs(jname=nil)
26
+ u = self
27
+ return u.runners.map{|r| r.jobs(jname)}.flatten
28
+ end
29
+
30
+ def runner_path
31
+ u = self
32
+ prefix = "Runner_"
33
+ suffix = ""
34
+ if Base.env == 'development'
35
+ suffix = "(dev)"
36
+ elsif Base.env == 'test'
37
+ suffix = "(test)"
38
+ elsif Base.env == 'production'
39
+ suffix = ""
40
+ else
41
+ raise "Invalid environment"
42
+ end
43
+ title = [prefix,u.name,suffix,"/jobs"].join
44
+ return title
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,59 @@
1
+ # require 'resque/tasks'
2
+ # will give you the resque tasks
3
+
4
+ namespace :mobilize do
5
+ desc "Start a Resque worker"
6
+ task :work do
7
+ require 'resque'
8
+ begin
9
+ require 'mobilize-base'
10
+ #require specified mobilize gems
11
+ Mobilize::Base.config('jobtracker')['extensions'].each do |e|
12
+ require e
13
+ end
14
+ rescue Exception=>exc
15
+ end
16
+
17
+ begin
18
+ worker = Resque::Worker.new(Mobilize::Resque.config['queue_name'])
19
+ rescue Resque::NoQueueError
20
+ abort "set QUEUE env var, e.g. $ QUEUE=critical,high rake resque:work"
21
+ end
22
+
23
+ puts "Starting worker #{worker}"
24
+
25
+ worker.work(ENV['INTERVAL'] || 5) # interval, will block
26
+ end
27
+ end
28
+ namespace :mobilize_base do
29
+ desc "Set up config and log folders and files"
30
+ task :setup do
31
+ config_dir = (ENV['MOBILIZE_CONFIG_DIR'] ||= "config/mobilize/")
32
+ log_dir = (ENV['MOBILIZE_LOG_DIR'] ||= "log/")
33
+ sample_dir = File.dirname(__FILE__) + '/../samples/'
34
+ sample_files = Dir.entries(sample_dir)
35
+ full_config_dir = "#{ENV['PWD']}/#{config_dir}"
36
+ full_log_dir = "#{ENV['PWD']}/#{log_dir}"
37
+ unless File.exists?(full_config_dir)
38
+ puts "creating #{config_dir}"
39
+ `mkdir #{full_config_dir}`
40
+ end
41
+ unless File.exists?(full_log_dir)
42
+ puts "creating #{log_dir}"
43
+ `mkdir #{full_log_dir}`
44
+ end
45
+ sample_files.each do |fname|
46
+ unless File.exists?("#{full_config_dir}#{fname}")
47
+ puts "creating #{config_dir}#{fname}"
48
+ `cp #{sample_dir}#{fname} #{full_config_dir}#{fname}`
49
+ end
50
+ end
51
+ end
52
+ desc "create indexes for all base modelsin mongodb"
53
+ task :create_indexes do
54
+ require 'mobilize-base'
55
+ ["Dataset","Job","Runner","Task","User"].each do |m|
56
+ "Mobilize::#{m}".constantize.create_indexes
57
+ end
58
+ end
59
+ end
@@ -1,5 +1,5 @@
1
1
  module Mobilize
2
2
  module Base
3
- VERSION = "1.0.2"
3
+ VERSION = "1.0.4"
4
4
  end
5
5
  end
data/lib/mobilize-base.rb CHANGED
@@ -15,15 +15,18 @@ module Mobilize
15
15
  ENV['PWD']
16
16
  end
17
17
  end
18
+ def Base.config_dir
19
+ ENV['MOBILIZE_CONFIG_DIR'] ||= "config/mobilize/"
20
+ end
18
21
  def Base.config(config_name)
19
22
  config_dir = begin
20
- "#{::Rails.root}/config/"
23
+ "#{::Rails.root}/#{Base.config_dir}"
21
24
  rescue
22
- "#{Base.root}/config/"
25
+ "#{Base.root}/#{Base.config_dir}"
23
26
  end
24
27
  yaml_path = "#{config_dir}#{config_name}.yml"
25
28
  if ::File.exists?(yaml_path)
26
- return ::YAML.load_file(yaml_path)
29
+ return ::YAML.load_file(yaml_path)[Base.env]
27
30
  else
28
31
  raise "Could not find #{config_name}.yml in #{config_dir}"
29
32
  end
@@ -36,11 +39,14 @@ module Mobilize
36
39
  ENV['MOBILIZE_ENV'] || "development"
37
40
  end
38
41
  end
42
+ def Base.log_dir
43
+ ENV['MOBILIZE_LOG_DIR'] ||= "log/"
44
+ end
39
45
  def Base.log_path(log_name)
40
46
  log_dir = begin
41
- "#{::Rails.root}/log/"
47
+ "#{::Rails.root}/#{Base.log_dir}"
42
48
  rescue
43
- "#{Base.root}/log/"
49
+ "#{Base.root}/#{Base.log_dir}"
44
50
  end
45
51
  log_path = "#{log_dir}#{log_name}.log"
46
52
  if ::File.exists?(log_dir)
@@ -51,14 +57,16 @@ module Mobilize
51
57
  end
52
58
  end
53
59
  end
54
- mongoid_config_path = "#{Mobilize::Base.root}/config/mongoid.yml"
60
+ mongoid_config_path = "#{Mobilize::Base.root}/#{Mobilize::Base.config_dir}mongoid.yml"
55
61
  if File.exists?(mongoid_config_path)
56
62
  require 'mongo'
57
63
  require 'mongoid'
58
64
  Mongoid.load!(mongoid_config_path, Mobilize::Base.env)
59
65
  require "mobilize-base/models/dataset"
60
- require "mobilize-base/models/requestor"
66
+ require "mobilize-base/models/user"
67
+ require "mobilize-base/models/runner"
61
68
  require "mobilize-base/models/job"
69
+ require "mobilize-base/models/task"
62
70
  end
63
71
  require 'google_drive'
64
72
  require 'resque'
@@ -71,6 +79,9 @@ require "mobilize-base/handlers/gdrive"
71
79
  require "mobilize-base/handlers/gfile"
72
80
  require "mobilize-base/handlers/gbook"
73
81
  require "mobilize-base/handlers/gsheet"
74
- require "mobilize-base/extensions/google_drive"
75
- require "mobilize-base/handlers/mongodb"
82
+ require "mobilize-base/extensions/google_drive/acl"
83
+ require "mobilize-base/extensions/google_drive/client_login_fetcher"
84
+ require "mobilize-base/extensions/google_drive/file"
85
+ require "mobilize-base/extensions/google_drive/worksheet"
86
+ require "mobilize-base/handlers/gridfs"
76
87
  require "mobilize-base/handlers/email"
@@ -1,30 +1,30 @@
1
1
  development:
2
2
  domain: 'host.com'
3
3
  owner:
4
- email: 'owner_development@host.com'
4
+ name: 'owner_development'
5
5
  pw: "google_drive_password"
6
6
  admins:
7
- - {email: 'admin@host.com'}
7
+ - {name: 'admin'}
8
8
  workers:
9
- - {email: 'worker_development001@host.com', pw: "worker001_google_drive_password"}
10
- - {email: 'worker_development002@host.com', pw: "worker002_google_drive_password"}
9
+ - {name: 'worker_development001', pw: "worker001_google_drive_password"}
10
+ - {name: 'worker_development002', pw: "worker002_google_drive_password"}
11
11
  test:
12
12
  domain: 'host.com'
13
13
  owner:
14
- email: 'owner_test@host.com'
14
+ name: 'owner_test'
15
15
  pw: "google_drive_password"
16
16
  admins:
17
- - {email: 'admin@host.com'}
17
+ - {name: 'admin'}
18
18
  workers:
19
- - {email: 'worker_test001@host.com', pw: "worker001_google_drive_password"}
20
- - {email: 'worker_test002@host.com', pw: "worker002_google_drive_password"}
19
+ - {name: 'worker_test001', pw: "worker001_google_drive_password"}
20
+ - {name: 'worker_test002', pw: "worker002_google_drive_password"}
21
21
  production:
22
22
  domain: 'host.com'
23
23
  owner:
24
- email: 'owner_production@host.com'
24
+ name: 'owner_production'
25
25
  pw: "google_drive_password"
26
26
  admins:
27
- - {email: 'admin@host.com'}
27
+ - {name: 'admin'}
28
28
  workers:
29
- - {email: 'worker_production001@host.com', pw: "worker001_google_drive_password"}
30
- - {email: 'worker_production002@host.com', pw: "worker002_google_drive_password"}
29
+ - {name: 'worker_production001', pw: "worker001_google_drive_password"}
30
+ - {name: 'worker_production002', pw: "worker002_google_drive_password"}
@@ -0,0 +1,9 @@
1
+ development:
2
+ max_versions: 10 #number of versions of cache to keep in gridfs
3
+ max_compressed_write_size: 1000000000 #~1GB
4
+ test:
5
+ max_versions: 10 #number of versions of cache to keep in gridfs
6
+ max_compressed_write_size: 1000000000 #~1GB
7
+ production:
8
+ max_versions: 10 #number of versions of cache to keep in gridfs
9
+ max_compressed_write_size: 1000000000 #~1GB
@@ -0,0 +1,6 @@
1
+ development:
2
+ max_cells: 400000
3
+ test:
4
+ max_cells: 400000
5
+ production:
6
+ max_cells: 400000
@@ -1,24 +1,24 @@
1
1
  development:
2
- #time between Jobtracker sweeps
3
- cycle_freq: 10
2
+ cycle_freq: 10 #time between Jobtracker sweeps
4
3
  notification_freq: 3600 #1 hour between failure/timeout notifications
5
- requestor_refresh_freq: 300 #5 min between requestor checks
4
+ runner_read_freq: 300 #5 min between runner reads
6
5
  max_run_time: 14400 # if a job runs for 4h+, notification will be sent
6
+ extensions: [] #additional Mobilize modules to load workers with
7
7
  admins: #emails to send notifications to
8
8
  - {'email': 'admin@host.com'}
9
9
  test:
10
- #time between Jobtracker sweeps
11
- cycle_freq: 10
10
+ cycle_freq: 10 #time between Jobtracker sweeps
12
11
  notification_freq: 3600 #1 hour between failure/timeout notifications
13
- requestor_refresh_freq: 300 #5 min between requestor checks
12
+ runner_read_freq: 300 #5 min between runner reads
14
13
  max_run_time: 14400 # if a job runs for 4h+, notification will be sent
14
+ extensions: [] #additional Mobilize modules to load workers with
15
15
  admins: #emails to send notifications to
16
16
  - {'email': 'admin@host.com'}
17
17
  production:
18
- #time between Jobtracker sweeps
19
- cycle_freq: 10
18
+ cycle_freq: 10 #time between Jobtracker sweeps
20
19
  notification_freq: 3600 #1 hour between failure/timeout notifications
21
- requestor_refresh_freq: 300 #5 min between requestor checks
20
+ runner_read_freq: 300 #5 min between runner reads
22
21
  max_run_time: 14400 # if a job runs for 4h+, notification will be sent
22
+ extensions: [] #additional Mobilize modules to load workers with
23
23
  admins: #emails to send notifications to
24
24
  - {'email': 'admin@host.com'}
@@ -1,21 +1,21 @@
1
1
  development:
2
2
  sessions:
3
3
  default:
4
- database: mobilize-development
4
+ database: mobilize_development
5
5
  persist_in_safe_mode: true
6
6
  hosts:
7
7
  - 127.0.0.1:27017
8
8
  test:
9
9
  sessions:
10
10
  default:
11
- database: mobilize-test
11
+ database: mobilize_test
12
12
  persist_in_safe_mode: true
13
13
  hosts:
14
14
  - 127.0.0.1:27017
15
15
  production:
16
16
  sessions:
17
17
  default:
18
- database: mobilize-production
18
+ database: mobilize_production
19
19
  persist_in_safe_mode: true
20
20
  hosts:
21
21
  - 127.0.0.1:27017