mobilize-base 1.0.2 → 1.0.4

data/lib/mobilize-base/models/task.rb

@@ -0,0 +1,137 @@
+module Mobilize
+  class Task
+    include Mongoid::Document
+    include Mongoid::Timestamps
+    field :path, type: String
+    field :handler, type: String
+    field :call, type: String
+    field :param_string, type: Array
+    field :status, type: String
+    field :last_completed_at, type: Time
+    field :last_run_at, type: Time
+
+    index({ path: 1})
+
+    def idx
+      t = self
+      t.path.split("/").last.gsub("task","").to_i
+    end
+
+    def stdout_dataset
+      t = self
+      Dataset.find_or_create_by_handler_and_path("gridfs","#{t.path}/stdout")
+    end
+
+    def stderr_dataset
+      t = self
+      Dataset.find_or_create_by_handler_and_path("gridfs","#{t.path}/stderr")
+    end
+
+    def params
+      t = self
+      t.param_string.split(",").map do |p| 
+        ps = p.strip
+        ps = ps[1..-1] if ['"',"'"].include?(ps[0])
+        ps = ps[0..-2] if ['"',"'"].include?(ps[-1])
+        ps
+      end
+    end
+
+    def job
+      t = self
+      job_path = t.path.split("/")[0..-2].join("/")
+      Job.where(:path=>job_path).first
+    end
+
+    def Task.find_or_create_by_path(path)
+      t = Task.where(:path=>path).first
+      t = Task.create(:path=>path) unless t
+      return t
+    end
+
+    def prior
+      t = self
+      j = t.job
+      return nil if t.idx==1
+      return j.tasks[t.idx-2]
+    end
+
+    def next
+      t = self
+      j = t.job
+      return nil if t.idx == j.tasks.length
+      return j.tasks[t.idx]
+    end
+
+    def Task.perform(id,*args)
+      t = Task.where(:path=>id).first
+      j = t.job
+      t.update_status(%{Starting at #{Time.now.utc}})
+      stdout, stderr = [nil,nil]
+      begin
+        stdout = "Mobilize::#{t.handler.humanize}".constantize.send("#{t.call}_by_task_path",t.path).to_s
+      rescue ScriptError, StandardError => exc
+        stderr = [exc.to_s,exc.backtrace.to_s].join("\n")
+        #record the failure in Job so it appears on Runner
+        j.update_attributes(:status=>"Failed at #{Time.now.utc.to_s}")
+        t.update_attributes(:status=>"Failed at #{Time.now.utc.to_s}")
+        raise exc
+      end
+      if stdout == false
+        #re-queue self if output is false
+        t.enqueue!
+        return false
+      end
+      #write output to cache
+      t.stdout_dataset.write_cache(stdout)
+      t.update_attributes(:status=>"Completed at #{Time.now.utc.to_s}")
+      if t.idx == j.tasks.length
+        j.update_attributes(:status=>"Completed at #{Time.now.utc.to_s}",:last_completed_at=>Time.now.utc)
+        j.update_attributes(:active=>false) if j.trigger.strip == "once"
+        t.update_attributes(:status=>"Completed at #{Time.now.utc.to_s}",:last_completed_at=>Time.now.utc)
+        #check for any dependent jobs, if there are, enqueue them
+        r = j.runner
+        dep_jobs = r.jobs.select{|dj| dj.active==true and dj.trigger=="after #{j.name}"}
+        #put begin/rescue so all dependencies run
+        dep_jobs.each{|dj| begin;dj.tasks.first.enqueue! unless dj.is_working?;rescue;end}
+      else
+        #queue up next task
+        t.next.enqueue!
+      end
+      return true
+    end
+
+    def enqueue!
+      t = self
+      ::Resque::Job.create("mobilize",Task,t.path,{})
+      return true
+    end
+
+    def worker
+      t = self
+      Mobilize::Resque.find_worker_by_path(t.path)
+    end
+
+    def worker_args
+      t = self
+      Jobtracker.get_worker_args(t.worker)
+    end
+
+    def set_worker_args(args)
+      t = self
+      Jobtracker.set_worker_args(t.worker,args)
+    end
+
+    def update_status(msg)
+      t = self
+      t.update_attributes(:status=>msg)
+      Mobilize::Resque.set_worker_args_by_path(t.path,{'status'=>msg})
+      return true
+    end
+
+    def is_working?
+      t = self
+      Mobilize::Resque.active_paths.include?(t.path)
+    end
+  end
+end

data/lib/mobilize-base/models/user.rb

@@ -0,0 +1,47 @@
+module Mobilize
+  class User
+    include Mongoid::Document
+    include Mongoid::Timestamps
+    field :active, type: Boolean
+    field :name, type: String
+    field :last_run, type: Time
+
+    def User.find_or_create_by_name(name)
+      u = User.where(:name => name).first
+      u = User.create(:name => name) unless u
+      return u
+    end
+
+    def email
+      u = self
+      "#{u.name}@#{Gdrive.domain}"
+    end
+
+    def runner
+      u = self
+      Runner.find_or_create_by_path(u.runner_path)
+    end
+
+    def jobs(jname=nil)
+      u = self
+      return u.runners.map{|r| r.jobs(jname)}.flatten
+    end
+
+    def runner_path
+      u = self
+      prefix = "Runner_"
+      suffix = ""
+      if Base.env == 'development'
+        suffix = "(dev)"
+      elsif Base.env == 'test'
+        suffix = "(test)"
+      elsif Base.env == 'production'
+        suffix = ""
+      else
+        raise "Invalid environment"
+      end
+      title = [prefix,u.name,suffix,"/jobs"].join
+      return title
+    end
+  end
+end

data/lib/mobilize-base/rakes.rb

@@ -0,0 +1,59 @@
+# require 'resque/tasks'
+# will give you the resque tasks
+
+namespace :mobilize do
+  desc "Start a Resque worker"
+  task :work do
+    require 'resque'
+    begin
+      require 'mobilize-base'
+      #require specified mobilize gems
+      Mobilize::Base.config('jobtracker')['extensions'].each do |e|
+        require e
+      end
+    rescue Exception=>exc
+    end
+
+    begin
+      worker = Resque::Worker.new(Mobilize::Resque.config['queue_name'])
+    rescue Resque::NoQueueError
+      abort "set QUEUE env var, e.g. $ QUEUE=critical,high rake resque:work"
+    end
+
+    puts "Starting worker #{worker}"
+
+    worker.work(ENV['INTERVAL'] || 5) # interval, will block
+  end
+end
+namespace :mobilize_base do
+  desc "Set up config and log folders and files"
+  task :setup do
+    config_dir = (ENV['MOBILIZE_CONFIG_DIR'] ||= "config/mobilize/")
+    log_dir = (ENV['MOBILIZE_LOG_DIR'] ||= "log/")
+    sample_dir = File.dirname(__FILE__) + '/../samples/'
+    sample_files = Dir.entries(sample_dir)
+    full_config_dir = "#{ENV['PWD']}/#{config_dir}"
+    full_log_dir = "#{ENV['PWD']}/#{log_dir}"
+    unless File.exists?(full_config_dir)
+      puts "creating #{config_dir}"
+      `mkdir #{full_config_dir}`
+    end
+    unless File.exists?(full_log_dir)
+      puts "creating #{log_dir}"
+      `mkdir #{full_log_dir}`
+    end
+    sample_files.each do |fname|
+      unless File.exists?("#{full_config_dir}#{fname}")
+        puts "creating #{config_dir}#{fname}"
+        `cp #{sample_dir}#{fname} #{full_config_dir}#{fname}`
+      end
+    end
+  end
+  desc "create indexes for all base modelsin mongodb"
+  task :create_indexes do
+    require 'mobilize-base'
+    ["Dataset","Job","Runner","Task","User"].each do |m|
+      "Mobilize::#{m}".constantize.create_indexes
+    end
+  end
+end

data/lib/mobilize-base/version.rb

@@ -1,5 +1,5 @@
 module Mobilize
   module Base
-    VERSION = "1.0.2"
+    VERSION = "1.0.4"
   end
 end

data/lib/mobilize-base.rb

@@ -15,15 +15,18 @@ module Mobilize
         ENV['PWD']
       end
     end
+    def Base.config_dir
+      ENV['MOBILIZE_CONFIG_DIR'] ||= "config/mobilize/"
+    end
     def Base.config(config_name)
       config_dir = begin
-                     "#{::Rails.root}/config/"
+                     "#{::Rails.root}/#{Base.config_dir}"
                    rescue
-                     "#{Base.root}/config/"
+                     "#{Base.root}/#{Base.config_dir}"
                    end
       yaml_path = "#{config_dir}#{config_name}.yml"
       if ::File.exists?(yaml_path)
-        return ::YAML.load_file(yaml_path)
+        return ::YAML.load_file(yaml_path)[Base.env]
       else
         raise "Could not find #{config_name}.yml in #{config_dir}"
       end
@@ -36,11 +39,14 @@ module Mobilize
         ENV['MOBILIZE_ENV'] || "development"
       end
     end
+    def Base.log_dir
+      ENV['MOBILIZE_LOG_DIR'] ||= "log/"
+    end
     def Base.log_path(log_name)
       log_dir = begin
-                  "#{::Rails.root}/log/"
+                  "#{::Rails.root}/#{Base.log_dir}"
                 rescue
-                  "#{Base.root}/log/"
+                  "#{Base.root}/#{Base.log_dir}"
                 end
       log_path = "#{log_dir}#{log_name}.log"
       if ::File.exists?(log_dir)
@@ -51,14 +57,16 @@ module Mobilize
     end
   end
 end
-mongoid_config_path = "#{Mobilize::Base.root}/config/mongoid.yml"
+mongoid_config_path = "#{Mobilize::Base.root}/#{Mobilize::Base.config_dir}mongoid.yml"
 if File.exists?(mongoid_config_path)
   require 'mongo'
   require 'mongoid'
   Mongoid.load!(mongoid_config_path, Mobilize::Base.env)
   require "mobilize-base/models/dataset"
-  require "mobilize-base/models/requestor"
+  require "mobilize-base/models/user"
+  require "mobilize-base/models/runner"
   require "mobilize-base/models/job"
+  require "mobilize-base/models/task"
 end
 require 'google_drive'
 require 'resque'
@@ -71,6 +79,9 @@ require "mobilize-base/handlers/gdrive"
 require "mobilize-base/handlers/gfile"
 require "mobilize-base/handlers/gbook"
 require "mobilize-base/handlers/gsheet"
-require "mobilize-base/extensions/google_drive"
-require "mobilize-base/handlers/mongodb"
+require "mobilize-base/extensions/google_drive/acl"
+require "mobilize-base/extensions/google_drive/client_login_fetcher"
+require "mobilize-base/extensions/google_drive/file"
+require "mobilize-base/extensions/google_drive/worksheet"
+require "mobilize-base/handlers/gridfs"
 require "mobilize-base/handlers/email"

data/lib/samples/gdrive.yml

@@ -1,30 +1,30 @@
 development:
   domain: 'host.com'
   owner:
-    email: 'owner_development@host.com'
+    name: 'owner_development'
     pw: "google_drive_password"
   admins:
-    - {email: 'admin@host.com'}
+    - {name: 'admin'}
   workers:
-    - {email: 'worker_development001@host.com', pw: "worker001_google_drive_password"}
-    - {email: 'worker_development002@host.com', pw: "worker002_google_drive_password"}
+    - {name: 'worker_development001', pw: "worker001_google_drive_password"}
+    - {name: 'worker_development002', pw: "worker002_google_drive_password"}
 test:
   domain: 'host.com'
   owner:
-    email: 'owner_test@host.com'
+    name: 'owner_test'
     pw: "google_drive_password"
   admins:
-    - {email: 'admin@host.com'}
+    - {name: 'admin'}
   workers:
-    - {email: 'worker_test001@host.com', pw: "worker001_google_drive_password"}
-    - {email: 'worker_test002@host.com', pw: "worker002_google_drive_password"}
+    - {name: 'worker_test001', pw: "worker001_google_drive_password"}
+    - {name: 'worker_test002', pw: "worker002_google_drive_password"}
 production:
   domain: 'host.com'
   owner:
-    email: 'owner_production@host.com'
+    name: 'owner_production'
     pw: "google_drive_password"
   admins:
-    - {email: 'admin@host.com'}
+    - {name: 'admin'}
   workers:
-    - {email: 'worker_production001@host.com', pw: "worker001_google_drive_password"}
-    - {email: 'worker_production002@host.com', pw: "worker002_google_drive_password"}
+    - {name: 'worker_production001', pw: "worker001_google_drive_password"}
+    - {name: 'worker_production002', pw: "worker002_google_drive_password"}

data/lib/samples/gridfs.yml

@@ -0,0 +1,9 @@
+development:
+  max_versions: 10 #number of versions of cache to keep in gridfs
+  max_compressed_write_size: 1000000000 #~1GB
+test:
+  max_versions: 10 #number of versions of cache to keep in gridfs
+  max_compressed_write_size: 1000000000 #~1GB
+production:
+  max_versions: 10 #number of versions of cache to keep in gridfs
+  max_compressed_write_size: 1000000000 #~1GB

data/lib/samples/gsheet.yml

@@ -0,0 +1,6 @@
+development:
+  max_cells: 400000
+test:
+  max_cells: 400000
+production:
+  max_cells: 400000

data/lib/samples/jobtracker.yml

@@ -1,24 +1,24 @@
 development:
-  #time between Jobtracker sweeps
-  cycle_freq: 10
+  cycle_freq: 10 #time between Jobtracker sweeps
   notification_freq: 3600 #1 hour between failure/timeout notifications
-  requestor_refresh_freq: 300 #5 min between requestor checks
+  runner_read_freq: 300 #5 min between runner reads
   max_run_time: 14400 # if a job runs for 4h+, notification will be sent
+  extensions: [] #additional Mobilize modules to load workers with
   admins: #emails to send notifications to
   - {'email': 'admin@host.com'}
 test:
-  #time between Jobtracker sweeps
-  cycle_freq: 10
+  cycle_freq: 10 #time between Jobtracker sweeps
   notification_freq: 3600 #1 hour between failure/timeout notifications
-  requestor_refresh_freq: 300 #5 min between requestor checks
+  runner_read_freq: 300 #5 min between runner reads
   max_run_time: 14400 # if a job runs for 4h+, notification will be sent
+  extensions: [] #additional Mobilize modules to load workers with
   admins: #emails to send notifications to
   - {'email': 'admin@host.com'}
 production:
-  #time between Jobtracker sweeps
-  cycle_freq: 10
+  cycle_freq: 10 #time between Jobtracker sweeps
   notification_freq: 3600 #1 hour between failure/timeout notifications
-  requestor_refresh_freq: 300 #5 min between requestor checks
+  runner_read_freq: 300 #5 min between runner reads
   max_run_time: 14400 # if a job runs for 4h+, notification will be sent
+  extensions: [] #additional Mobilize modules to load workers with
   admins: #emails to send notifications to
   - {'email': 'admin@host.com'}

data/lib/samples/mongoid.yml

@@ -1,21 +1,21 @@
 development:
   sessions:
     default:
-      database: mobilize-development
+      database: mobilize_development
       persist_in_safe_mode: true
       hosts:
         - 127.0.0.1:27017
 test:
   sessions:
     default:
-      database: mobilize-test
+      database: mobilize_test
       persist_in_safe_mode: true
       hosts:
         - 127.0.0.1:27017
 production:
   sessions:
     default:
-      database: mobilize-production
+      database: mobilize_production
       persist_in_safe_mode: true
       hosts:
         - 127.0.0.1:27017