mobilize-base 1.36 → 1.293

data/lib/mobilize-base/models/stage.rb

@@ -2,7 +2,6 @@ module Mobilize
   class Stage
     include Mongoid::Document
     include Mongoid::Timestamps
-    include Mobilize::StageHelper
     field :path, type: String
     field :handler, type: String
     field :call, type: String
@@ -17,6 +16,40 @@ module Mobilize
 
     index({ path: 1})
 
+    def idx
+      s = self
+      s.path.split("/").last.gsub("stage","").to_i
+    end
+
+    def out_dst
+      #this gives a dataset that points to the output
+      #allowing you to determine its size
+      #before committing to a read or write
+      s = self
+      Dataset.find_by_url(s.response['out_url']) if s.response and s.response['out_url']
+    end
+
+    def err_dst
+      #this gives a dataset that points to the output
+      #allowing you to determine its size
+      #before committing to a read or write
+      s = self
+      Dataset.find_by_url(s.response['err_url']) if s.response and s.response['err_url']
+    end
+
+    def params
+      s = self
+      p = YAML.easy_load(s.param_string)
+      raise "Must resolve to Hash" unless p.class==Hash
+      return p
+    end
+
+    def job
+      s = self
+      job_path = s.path.split("/")[0..-2].join("/")
+      Job.where(:path=>job_path).first
+    end
+
     def Stage.find_or_create_by_path(path)
       s = Stage.where(:path=>path).first
       s = Stage.create(:path=>path) unless s
@@ -44,14 +77,6 @@ module Mobilize
 
     def Stage.perform(id,*args)
       s = Stage.where(:path=>id).first
-      #check to make sure params are parsable
-      begin
-        param_hash = s.params
-        raise ScriptError if param_hash.class!=Hash
-      rescue StandardError, ScriptError
-        s.fail({'signal'=>500,
-                'err_str'=>"Unable to parse stage params, make sure you don't have issues with your quotes, commas, or colons."})
-      end
       s.update_attributes(:started_at=>Time.now.utc)
       s.update_status(%{Starting at #{Time.now.utc}})
       #get response by running method
@@ -82,6 +107,8 @@ module Mobilize
 
     def complete(response)
       s = self
+      s.update_attributes(:completed_at=>Time.now.utc,:response=>response)
+      s.update_status("Completed at #{Time.now.utc.to_s}")
       j = s.job
       if s.idx == j.stages.length
         #check for any dependent jobs, if there are, enqueue them
@@ -108,10 +135,6 @@ module Mobilize
         s.next.update_attributes(:retries_done=>0)
         s.next.enqueue!
       end
-      #complete after dependencies are processed
-      #to make sure it doesn't enqueue due to runner check
-      s.update_attributes(:completed_at=>Time.now.utc,:response=>response)
-      s.update_status("Completed at #{Time.now.utc.to_s}")
       true
     end
 
@@ -122,11 +145,7 @@ module Mobilize
       j = s.job
       r = j.runner
       u = r.user
-      begin
-        j.update_attributes(:active=>false) if s.params['always_on'].to_s=="false"
-      rescue StandardError, ScriptError
-        #skip due to parse error on params
-      end
+      j.update_attributes(:active=>false) unless s.params['always_on']
       s.update_attributes(:failed_at=>Time.now.utc,:response=>response)
       stage_name = "#{j.name}_stage#{s.idx.to_s}.err"
       target_path =  (r.path.split("/")[0..-2] + [stage_name]).join("/")

data/lib/mobilize-base/tasks.rb

@@ -1,7 +1,6 @@
-namespace :mobilize do
+namespace :mobilize_base do
   desc "Start a Resque worker"
-  task :work, :env do |t,args|
-    ENV['MOBILIZE_ENV']=args.env
+  task :work do
     require 'mobilize-base'
     Mobilize::Base.config('jobtracker')['extensions'].each do |e|
       begin
@@ -21,77 +20,42 @@ namespace :mobilize do
     worker.work(ENV['INTERVAL'] || 5) # interval, will block
   end
   desc "Kill all Resque workers"
-  task :kill_workers, :env do |t,args|
-    ENV['MOBILIZE_ENV']=args.env
+  task :kill_workers do
     require 'mobilize-base'
     Mobilize::Jobtracker.kill_workers
   end
   desc "Kill idle workers not in sync with repo"
-  task :kill_idle_and_stale_workers, :env do |t,args|
-    ENV['MOBILIZE_ENV']=args.env
+  task :kill_idle_and_stale_workers do
     require 'mobilize-base'
     Mobilize::Jobtracker.kill_idle_and_stale_workers
   end
-  desc "Kill idle workers"
-  task :kill_idle_workers, :env do |t,args|
-    ENV['MOBILIZE_ENV']=args.env
+    desc "Kill idle workers"
+    task :kill_idle_workers do
     require 'mobilize-base'
     Mobilize::Jobtracker.kill_idle_workers
   end
   desc "Make sure there are the correct # of workers, kill if too many"
-  task :prep_workers, :env do |t,args|
-    ENV['MOBILIZE_ENV']=args.env
+  task :prep_workers do
     require 'mobilize-base'
     Mobilize::Jobtracker.prep_workers
   end
-  desc "Restart Resque workers"
-  task :restart_workers, :env do |t,args|
-    ENV['MOBILIZE_ENV']=args.env
-    require 'mobilize-base'
-    Mobilize::Jobtracker.kill_workers
-    sleep 5
-    Mobilize::Jobtracker.prep_workers
-  end
   desc "Stop Jobtracker"
-  task :stop_jobtracker, :env do |t,args|
-    ENV['MOBILIZE_ENV']=args.env
+  task :stop_jobtracker do
     require 'mobilize-base'
     Mobilize::Jobtracker.stop!
   end
   desc "Start Jobtracker"
-  task :start_jobtracker, :env do |t,args|
-    ENV['MOBILIZE_ENV']=args.env
+  task :start_jobtracker do
     require 'mobilize-base'
     Mobilize::Jobtracker.start
   end
   desc "Restart Jobtracker"
-  task :restart_jobtracker, :env do |t,args|
-    ENV['MOBILIZE_ENV']=args.env
+  task :restart_jobtracker do
     require 'mobilize-base'
     Mobilize::Jobtracker.restart!
   end
-  desc "Add a user"
-  task :add_user, :name, :env do |t,args|
-    ENV['MOBILIZE_ENV']=args.env
-    require 'mobilize-base'
-    Mobilize::User.find_or_create_by_name(args.name)
-  end
-  desc "Enqueue a user's runner"
-  task :enqueue_user, :name, :env do |t,args|
-    ENV['MOBILIZE_ENV']=args.env
-    require 'mobilize-base'
-    Mobilize::User.where(name: args.name).first.runner.enqueue!
-  end
-  desc "Enqueue a stage"
-  task :enqueue_stage, :path, :env do |t,args|
-    ENV['MOBILIZE_ENV']=args.env
-    require 'mobilize-base'
-    user,job,stage = args.path.split("/")
-    Mobilize::Stage.where(path: "Runner_#{user}/jobs/#{job}/#{stage}").first.en
-  end
   desc "kill all old resque web processes, start new one with resque_web.rb extension file"
-  task :resque_web, :env do |t,args|
-    ENV['MOBILIZE_ENV']=args.env
+  task :resque_web do
     require 'mobilize-base'
     port = Mobilize::Base.config('resque')['web_port']
     config_dir = (ENV['MOBILIZE_CONFIG_DIR'] ||= "config/mobilize/")
@@ -119,15 +83,14 @@ namespace :mobilize do
     `#{command}`
   end
   desc "create indexes for all base models in mongodb"
-  task :create_indexes, :env do |t,args|
-    ENV['MOBILIZE_ENV']=args.env
+  task :create_indexes do
     require 'mobilize-base'
     ["Dataset","Job","Runner","Task","User"].each do |m|
       "Mobilize::#{m}".constantize.create_indexes
     end
   end
   desc "Set up config and log folders and files, populate from samples"
-  task :setup_base do
+  task :setup do
     config_dir = (ENV['MOBILIZE_CONFIG_DIR'] ||= "config/mobilize/")
     log_dir = (ENV['MOBILIZE_LOG_DIR'] ||= "log/")
     sample_dir = File.dirname(__FILE__) + '/../samples/'

data/lib/mobilize-base/version.rb

@@ -1,5 +1,5 @@
 module Mobilize
   module Base
-    VERSION = "1.36"
+    VERSION = "1.293"
   end
 end

data/lib/samples/gdrive.yml

@@ -1,16 +1,11 @@
 ---
 development:
   domain: host.com
-  max_api_retries: 10
-  file_write_retries: 5
-  file_write_retry_delay: 30
   owner:
     name: owner_development
     pw: google_drive_password
-  admin_group_name: admins_development
   admins:
     - name: admin
-  worker_group_name: workers_development
   workers:
     - name: worker_development001 
       pw: worker001_google_drive_password
@@ -18,16 +13,11 @@ development:
       pw: worker002_google_drive_password
 test:
   domain: host.com
-  max_api_retries: 10
-  file_write_retries: 5
-  file_write_retry_delay: 30
   owner:
     name: owner_test
     pw: google_drive_password
-  admin_group_name: admins_test
   admins:
     - name: admin
-  worker_group_name: workers_test
   workers:
     - name: worker_test001 
       pw: worker001_google_drive_password
@@ -35,16 +25,11 @@ test:
       pw: worker002_google_drive_password
 production:
   domain: host.com
-  max_api_retries: 10
-  file_write_retries: 5
-  file_write_retry_delay: 30
   owner:
     name: owner_production
     pw: google_drive_password
-  admin_group_name: admins_production
   admins:
     - name: admin
-  worker_group_name: workers_production
   workers:
     - name: worker_production001 
       pw: worker001_google_drive_password

data/lib/samples/gridfs.yml

@@ -1,7 +1,10 @@
 ---
 development:
+  max_versions: 10 #number of versions of cache to keep in gridfs
   max_compressed_write_size: 1000000000 #~1GB
 test:
+  max_versions: 10 #number of versions of cache to keep in gridfs
   max_compressed_write_size: 1000000000 #~1GB
 production:
+  max_versions: 10 #number of versions of cache to keep in gridfs
   max_compressed_write_size: 1000000000 #~1GB

data/lib/samples/gsheet.yml

@@ -1,17 +1,17 @@
 ---
 development:
-  max_cells: 50000 #current google limit is 400k cells per book but fails earlier
+  max_cells: 400000 #current google limit is 400k cells per book
   read_date_format: "%Y-%m-%d" #format to record when reading sheets
   sheet_date_format: "%m/%d/%Y" #format to use to parse sheets
 test:
-  max_cells: 50000 #current google limit is 400k cells per book but fails earlier
+  max_cells: 400000 #current google limit is 400k cells per book
   read_date_format: "%Y-%m-%d" #format to record when reading sheets
   sheet_date_format: "%m/%d/%Y" #format to use to parse sheets
 staging:
-  max_cells: 50000 #current google limit is 400k cells per book but fails earlier
+  max_cells: 400000 #current google limit is 400k cells per book
   read_date_format: "%Y-%m-%d" #format to record when reading sheets
   sheet_date_format: "%m/%d/%Y" #format to use to parse sheets
 production:
-  max_cells: 50000 #current google limit is 400k cells per book but fails earlier
+  max_cells: 400000 #current google limit is 400k cells per book
   read_date_format: "%Y-%m-%d" #format to record when reading sheets
   sheet_date_format: "%m/%d/%Y" #format to use to parse sheets

data/lib/samples/jobtracker.yml

@@ -5,15 +5,21 @@ development:
   runner_read_freq: 300 #5 min between runner reads
   max_run_time: 14400 # if a job runs for 4h+, notification will be sent
   extensions: [] #additional Mobilize modules to load workers with
+  admins: #emails to send notifications to
+  - email: admin@host.com
 test:
   cycle_freq: 10 #time between Jobtracker sweeps
   notification_freq: 3600 #1 hour between failure/timeout notifications
   runner_read_freq: 300 #5 min between runner reads
   max_run_time: 14400 # if a job runs for 4h+, notification will be sent
   extensions: [] #additional Mobilize modules to load workers with
+  admins: #emails to send notifications to
+  - email: admin@host.com
 production:
   cycle_freq: 10 #time between Jobtracker sweeps
   notification_freq: 3600 #1 hour between failure/timeout notifications
   runner_read_freq: 300 #5 min between runner reads
   max_run_time: 14400 # if a job runs for 4h+, notification will be sent
   extensions: [] #additional Mobilize modules to load workers with
+  admins: #emails to send notifications to
+  - email: admin@host.com

data/mobilize-base.gemspec

@@ -22,10 +22,10 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   s.add_runtime_dependency 'rake'
-  s.add_runtime_dependency 'bson','1.8.4'
-  s.add_runtime_dependency 'bson_ext','1.8.4'
+  s.add_runtime_dependency 'bson','1.6.1'
+  s.add_runtime_dependency 'bson_ext','1.6.1'
+  s.add_runtime_dependency 'mongo', '1.6.1'
   s.add_runtime_dependency "mongoid", "~>3.0.0"
-  s.add_runtime_dependency 'mongoid-grid_fs'
   s.add_runtime_dependency 'resque', '1.24.0'
   s.add_runtime_dependency 'google_drive','0.3.2'
   s.add_runtime_dependency 'popen4','0.1.2'

data/test/base_job_rows.yml

@@ -0,0 +1,11 @@
+- name: base1
+  active: true
+  trigger: once
+  status: ""
+  stage1: gsheet.write source:"gfile://test_base_1.tsv", target:base1.out
+
+- name: base2
+  active: true
+  trigger: after base1
+  status: ""
+  stage1: gsheet.write source:base1.out, target:base2.out

data/test/mobilize-base_test.rb

@@ -0,0 +1,106 @@
+require 'test_helper'
+
+describe "Mobilize" do
+
+  def before
+    puts 'nothing before'
+  end
+
+  # enqueues 4 workers on Resque
+  it "runs integration test" do
+
+    puts "restart test redis"
+    Mobilize::Jobtracker.restart_test_redis
+
+    puts "clear out test db"
+    Mobilize::Jobtracker.drop_test_db
+
+    puts "restart workers"
+    Mobilize::Jobtracker.restart_workers!
+
+    puts "build test runner"
+    gdrive_slot = Mobilize::Gdrive.owner_email
+    puts "create user 'mobilize'"
+    user_name = gdrive_slot.split("@").first
+    u = Mobilize::User.find_or_create_by_name(user_name)
+    assert u.email == gdrive_slot
+
+    Mobilize::Jobtracker.build_test_runner(user_name)
+    assert Mobilize::Jobtracker.workers.length == Mobilize::Resque.config['max_workers'].to_i
+
+    puts "Jobtracker created runner with 'jobs' sheet?"
+    r = u.runner
+    jobs_sheet_url = "gsheet://#{r.path}"
+    jobs_sheet = Mobilize::Gsheet.find_by_path(r.path,gdrive_slot)
+    jobs_sheet_dst = Mobilize::Dataset.find_or_create_by_url(jobs_sheet_url)
+    jobs_sheet_tsv = jobs_sheet_dst.read(user_name,gdrive_slot)
+    assert jobs_sheet_tsv.tsv_header_array.join.length == 53 #total header length
+
+    #stop Jobtracker, if you're doing this by queueing runners
+    #Mobilize::Jobtracker.stop!
+
+    puts "add base1 input file"
+    test_filename = "test_base_1"
+    file_url = "gfile://#{test_filename}.tsv"
+    test_source_ha = ::YAML.load_file("#{Mobilize::Base.root}/test/#{test_filename}.yml")*40
+    test_source_tsv = test_source_ha.hash_array_to_tsv
+    Mobilize::Dataset.write_by_url(file_url,test_source_tsv,user_name)
+    rem_tsv = Mobilize::Dataset.read_by_url(file_url,user_name)
+    assert rem_tsv == test_source_tsv
+
+    puts "add row to jobs sheet, wait for stages"
+    test_job_rows = ::YAML.load_file("#{Mobilize::Base.root}/test/base_job_rows.yml")
+    jobs_sheet.reload
+    jobs_sheet.add_or_update_rows(test_job_rows)
+    #wait for stages to complete
+    #r.enqueue!
+    wait_for_stages
+
+    puts "jobtracker posted test sheet data to test destination, and checksum succeeded?"
+    test_target_sheet_1_url = "gsheet://#{r.title}/base1.out"
+    test_target_sheet_2_url = "gsheet://#{r.title}/base2.out"
+    test_error_sheet_url = "gsheet://#{r.title}/base1_stage1.err"
+
+    test_1_tsv = Mobilize::Dataset.read_by_url(test_target_sheet_1_url,user_name,gdrive_slot)
+    test_2_tsv = Mobilize::Dataset.read_by_url(test_target_sheet_1_url,user_name,gdrive_slot)
+
+    assert test_1_tsv == test_2_tsv
+
+    puts "change first job to fail, wait for stages"
+    test_job_rows.first['stage1'] = %{gsheet.write source:"gfile://test_base_1.fail", target:base1.out, retries:3}
+    Mobilize::Dataset.write_by_url(test_error_sheet_url," ",user_name,gdrive_slot)
+    jobs_sheet.add_or_update_rows(test_job_rows)
+
+    #wait for stages to complete
+    wait_for_stages
+
+    test_error_sheet = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/base1_stage1.err",gdrive_slot)
+    puts "jobtracker posted failing test error to sheet "
+    error_rows = test_error_sheet.read(user_name).tsv_to_hash_array
+    assert error_rows.first['response'] == "Could not get gfile://test_base_1.fail with error: unable to find test_base_1.fail"
+    Mobilize::Jobtracker.stop!
+  end
+
+  def wait_for_stages(time_limit=600,stage_limit=120,wait_length=10)
+    time = 0
+    time_since_stage = 0
+    #check for 10 min
+    while time < time_limit and time_since_stage < stage_limit
+      sleep wait_length
+      job_classes = Mobilize::Resque.jobs.map{|j| j['class']}
+      if job_classes.include?("Mobilize::Stage")
+        time_since_stage = 0
+        puts "saw stage at #{time.to_s} seconds"
+      else
+        time_since_stage += wait_length
+        puts "#{time_since_stage.to_s} seconds since stage seen"
+      end
+      time += wait_length
+      puts "total wait time #{time.to_s} seconds"
+    end
+
+    if time >= time_limit
+      raise "Timed out before stage completion"
+    end
+  end
+end