mobilize-base 1.361 → 1.363

data/lib/mobilize-base.rb

@@ -87,13 +87,13 @@ require "mobilize-base/handlers/resque"
 Resque.redis = "127.0.0.1:#{Mobilize::Resque.config['redis_port']}"
 require 'popen4'
 require "mobilize-base/jobtracker"
-require "mobilize-base/handlers/gdrive"
-require "mobilize-base/handlers/gfile"
-require "mobilize-base/handlers/gbook"
-require "mobilize-base/handlers/gsheet"
+require "mobilize-base/handlers/google/gdrive"
+require "mobilize-base/handlers/google/gfile"
+require "mobilize-base/handlers/google/gbook"
+require "mobilize-base/handlers/google/gsheet"
+require "mobilize-base/handlers/google/gmail"
 require "mobilize-base/extensions/google_drive/acl"
 require "mobilize-base/extensions/google_drive/client_login_fetcher"
 require "mobilize-base/extensions/google_drive/file"
 require "mobilize-base/extensions/google_drive/worksheet"
 require "mobilize-base/handlers/gridfs"
-require "mobilize-base/handlers/email"

data/lib/mobilize-base/extensions/google_drive/worksheet.rb

@@ -222,7 +222,8 @@ module GoogleDrive
           true
         end
       else
-        raise "#{errcnt} errors found in checksum"
+        sheet.save
+        "#{errcnt} errors found in checksum".oputs
       end
     end
   end

data/lib/mobilize-base/extensions/string.rb

@@ -33,9 +33,12 @@ class String
     new_str.gsub!(c,"\\#{c}")}
     new_str
   end
-  def gridsafe
+  #makes everything alphanumeric
+  #except spaces, slashes, and underscores
+  #which are made into underscores
+  def alphanunderscore
     str = self
-    str.downcase.gsub(/[^a-z0-9]/,"")
+    str.gsub(/[^A-Za-z0-9_ \/]/,"").gsub(/[ \/]/,"_")
   end
   def googlesafe
     v=self

data/lib/mobilize-base/handlers/{email.rb → google/gmail.rb}

@@ -1,6 +1,6 @@
 module Mobilize
   require 'action_mailer'
-  class Email < ActionMailer::Base
+  class Gmail < ActionMailer::Base
     ActionMailer::Base.delivery_method = :smtp
     
     ActionMailer::Base.smtp_settings = {

data/lib/mobilize-base/handlers/{gsheet.rb → google/gsheet.rb}

@@ -81,9 +81,11 @@ module Mobilize
 
     def Gsheet.write_temp(target_path,gdrive_slot,tsv)
       #find and delete temp sheet, if any
-      temp_book_title = target_path.gridsafe
+      temp_book_title = target_path.downcase.alphanunderscore
       #create book and sheet
       temp_book = Gdrive.root(gdrive_slot).create_spreadsheet(temp_book_title)
+      #add admin acl so we can look at it if it fails
+      temp_book.add_admin_acl
       rows, cols = tsv.split("\n").ie{|t| [t.length,t.first.split("\t").length]}
       temp_sheet = temp_book.add_worksheet("temp",rows,cols)
       #this step has a tendency to fail; if it does,

data/lib/mobilize-base/handlers/gridfs.rb

@@ -21,11 +21,13 @@ module Mobilize
       if curr_zs != zs
         Mongoid::GridFs.delete(curr_file.id) if curr_file
         #create temp file w zstring
-        temp_file = ::Tempfile.new("#{string}#{Time.now.to_f}".to_md5)
+        temp_file = ::Tempfile.new("#{dst_path.downcase.alphanunderscore}_#{Time.now.to_f}")
         temp_file.print(zs)
         temp_file.close
         #put data in file
         Mongoid::GridFs.put(temp_file.path,:filename=>dst_path)
+        #remove temp file
+        `rm #{temp_file.path}`
       end
       return true
     end

data/lib/mobilize-base/helpers/jobtracker_helper.rb

@@ -35,6 +35,10 @@ module Mobilize
       Resque.workers(state)
     end
 
+    def Jobtracker.disabled_methods
+      Jobtracker.config['disabled_methods']
+    end
+
     def Jobtracker.status
       args = Jobtracker.get_args
       return args['status'] if args
@@ -73,10 +77,6 @@ module Mobilize
       Resque.kill_idle_workers
     end
 
-    def Jobtracker.kill_idle_and_stale_workers
-      Resque.kill_idle_and_stale_workers
-    end
-
     def Jobtracker.prep_workers
       Resque.prep_workers
     end

data/lib/mobilize-base/jobtracker.rb

@@ -60,7 +60,7 @@ module Mobilize
         #deliver each email generated
         notifs.each do |notif|
           begin
-            Email.write(notif).deliver
+            Gmail.write(notif).deliver
           rescue
             #log email on failure
             Jobtracker.update_status("Failed to deliver #{notif.to_s}")
@@ -82,7 +82,17 @@ module Mobilize
         users.sort_by{rand}.each do |u|
           r = u.runner
           Jobtracker.update_status("Checking #{r.path}")
-          if r.is_due?
+          #check for run_now file
+          run_now_dir = "/home/#{u.name}/mobilize/"
+          run_now_path = "#{run_now_dir}run_now"
+          run_now = if File.exists?(run_now_dir) and File.exists?(run_now_path)
+                      #delete user's run now file
+                      `sudo rm -rf #{run_now_path}`
+                      true
+                    else
+                      false
+                    end
+          if r.is_due? or run_now
             r.enqueue!
             Jobtracker.update_status("Enqueued #{r.path}")
           end
@@ -92,29 +102,5 @@ module Mobilize
       Jobtracker.update_status("told to stop")
       return true
     end
-
-    def Jobtracker.deployed_at
-      #assumes deploy is as of last commit, or as of last deploy time
-      #as given by the REVISION file in the root folder
-      deploy_time = begin
-                      %{git log -1 --format="%cd"}.bash
-                    rescue
-                      revision_path = "#{ENV['PWD']}/REVISION"
-                      "touch #{revision_path}".bash unless File.exists?(revision_path)
-                      revision_string = "ls -l #{revision_path}".bash
-                      revision_rows = revision_string.split("\n").map{|lss| lss.strip.split(" ")}
-                      mod_time = revision_rows.map do |lsr| 
-                        if lsr.length == 8
-                          #ubuntu
-                          lsr[5..6].join(" ")
-                        elsif lsr.length == 9
-                          #osx
-                          lsr[5..7].join(" ")
-                        end
-                      end.first
-                      mod_time
-                    end.to_s.strip
-      Time.parse(deploy_time)
-    end
   end
 end

data/lib/mobilize-base/models/job.rb

@@ -79,11 +79,10 @@ module Mobilize
         return false
       end
 
-      #if job contains handlers not loaded by jobtracker, not due
-      loaded_handlers = Jobtracker.config['extensions'].map{|m| m.split("-").last}
-      job_handlers = j.stages.map{|s| s.handler}.uniq
+      #if job contains methods in disabled_methods, not due
+      job_methods = j.stages.map{|s| "#{s.handler}.#{s.call}"}.uniq
       #base handlers are the ones in mobilize-base/handlers
-      if (job_handlers - loaded_handlers - Base.handlers).length>0
+      if (Jobtracker.disabled_methods & job_methods).length>0
         return false
       end
 

data/lib/mobilize-base/models/user.rb

@@ -4,6 +4,7 @@ module Mobilize
     include Mongoid::Timestamps
     field :active, type: Boolean
     field :name, type: String
+    field :ssh_public_key, type: String
     field :last_run, type: Time
 
     def User.find_or_create_by_name(name)
@@ -27,22 +28,6 @@ module Mobilize
       return u.runner.jobs
     end
 
-    def creds(gdrive_slot)
-      u = self
-      creds_path = "#{u.runner.path.split("/").first}/creds"
-      begin
-        creds_sheet = Gsheet.find_by_path(creds_path,gdrive_slot)
-        cred_array = creds_sheet.read(u.name).tsv_to_hash_array.map{|h| {h['name']=>{'user'=>h['user'],'password'=>h['password']}}}
-        result = {}
-        cred_array.each do |cred|
-          result[cred.keys.first] = cred.values.first
-        end
-        return result
-      rescue
-        return {}
-      end
-    end
-
     def runner_path
       u = self
       prefix = "Runner_"

data/lib/mobilize-base/tasks.rb

@@ -26,12 +26,6 @@ namespace :mobilize do
     require 'mobilize-base'
     Mobilize::Jobtracker.kill_workers
   end
-  desc "Kill idle workers not in sync with repo"
-  task :kill_idle_and_stale_workers, :env do |t,args|
-    ENV['MOBILIZE_ENV']=args.env
-    require 'mobilize-base'
-    Mobilize::Jobtracker.kill_idle_and_stale_workers
-  end
   desc "Kill idle workers"
   task :kill_idle_workers, :env do |t,args|
     ENV['MOBILIZE_ENV']=args.env
@@ -70,11 +64,11 @@ namespace :mobilize do
     require 'mobilize-base'
     Mobilize::Jobtracker.restart!
   end
-  desc "Add a user"
-  task :add_user, :name, :env do |t,args|
+  desc "Add a user and set up local files"
+  task :add_base_user, :name, :env do |t,args|
     ENV['MOBILIZE_ENV']=args.env
     require 'mobilize-base'
-    Mobilize::User.find_or_create_by_name(args.name)
+    u = Mobilize::User.find_or_create_by_name(args.name)
   end
   desc "Enqueue a user's runner"
   task :enqueue_user, :name, :env do |t,args|
@@ -87,7 +81,7 @@ namespace :mobilize do
     ENV['MOBILIZE_ENV']=args.env
     require 'mobilize-base'
     user,job,stage = args.path.split("/")
-    Mobilize::Stage.where(path: "Runner_#{user}/jobs/#{job}/#{stage}").first.en
+    Mobilize::Stage.where(path: "Runner_#{user}/jobs/#{job}/#{stage}").first.enqueue!
   end
   desc "kill all old resque web processes, start new one with resque_web.rb extension file"
   task :resque_web, :env do |t,args|
@@ -126,7 +120,7 @@ namespace :mobilize do
       "Mobilize::#{m}".constantize.create_indexes
     end
   end
-  desc "Set up config and log folders and files, populate from samples"
+  desc "Set up base config and log folders and files"
   task :setup_base do
     config_dir = (ENV['MOBILIZE_CONFIG_DIR'] ||= "config/mobilize/")
     log_dir = (ENV['MOBILIZE_LOG_DIR'] ||= "log/")

data/lib/mobilize-base/version.rb

@@ -1,5 +1,5 @@
 module Mobilize
   module Base
-    VERSION = "1.361"
+    VERSION = "1.363"
   end
 end

data/lib/samples/jobtracker.yml

@@ -5,15 +5,18 @@ development:
   runner_read_freq: 300 #5 min between runner reads
   max_run_time: 14400 # if a job runs for 4h+, notification will be sent
   extensions: [] #additional Mobilize modules to load workers with
+  disabled_methods: [] #methods for jobs that should be ignored (used for outages)
 test:
   cycle_freq: 10 #time between Jobtracker sweeps
   notification_freq: 3600 #1 hour between failure/timeout notifications
   runner_read_freq: 300 #5 min between runner reads
   max_run_time: 14400 # if a job runs for 4h+, notification will be sent
   extensions: [] #additional Mobilize modules to load workers with
+  disabled_methods: [] #methods for jobs that should be ignored (used for outages)
 production:
   cycle_freq: 10 #time between Jobtracker sweeps
   notification_freq: 3600 #1 hour between failure/timeout notifications
   runner_read_freq: 300 #5 min between runner reads
   max_run_time: 14400 # if a job runs for 4h+, notification will be sent
   extensions: [] #additional Mobilize modules to load workers with
+  disabled_methods: [] #methods for jobs that should be ignored (used for outages)

data/test/fixtures/is_due.yml

@@ -89,9 +89,3 @@
   parent: {completed_at: Time.now.utc - 1.second}
   completed_at: Time.now.utc
   expected: false
-- name: missing_module
-  active: true
-  trigger: once
-  status: ""
-  stage1: handler.method source:"source", target:"target"
-  expected: false

data/test/test_helper.rb

@@ -10,6 +10,8 @@ require 'mobilize-base'
 $TESTING = true
 module TestHelper
   def TestHelper.confirm_expected_jobs(expected_fixture_name,time_limit=600)
+    #clear all failures
+    ::Resque::Failure.clear
     jobs = {}
     jobs['expected'] = TestHelper.load_fixture(expected_fixture_name)
     jobs['pending'] = jobs['expected'].select{|j| j['confirmed_ats'].length < j['count']}

data/test/unit/mobilize-base_test.rb

@@ -27,6 +27,7 @@ class TestUnit < MiniTest::Unit::TestCase
       end
       expected = jh['expected']
       #check if is_due
+      puts "checking #{j.name}"
       assert expected == j.is_due?
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mobilize-base
 version: !ruby/object:Gem::Version
-  version: '1.361'
+  version: '1.363'
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-31 00:00:00.000000000 Z
+date: 2013-06-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -184,12 +184,12 @@ files:
 - lib/mobilize-base/extensions/string.rb
 - lib/mobilize-base/extensions/time.rb
 - lib/mobilize-base/extensions/yaml.rb
-- lib/mobilize-base/handlers/email.rb
-- lib/mobilize-base/handlers/gbook.rb
-- lib/mobilize-base/handlers/gdrive.rb
-- lib/mobilize-base/handlers/gfile.rb
+- lib/mobilize-base/handlers/google/gbook.rb
+- lib/mobilize-base/handlers/google/gdrive.rb
+- lib/mobilize-base/handlers/google/gfile.rb
+- lib/mobilize-base/handlers/google/gmail.rb
+- lib/mobilize-base/handlers/google/gsheet.rb
 - lib/mobilize-base/handlers/gridfs.rb
-- lib/mobilize-base/handlers/gsheet.rb
 - lib/mobilize-base/handlers/resque.rb
 - lib/mobilize-base/helpers/job_helper.rb
 - lib/mobilize-base/helpers/jobtracker_helper.rb
@@ -234,7 +234,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3685016355014440836
+      hash: 1540799042957557582
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -243,7 +243,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3685016355014440836
+      hash: 1540799042957557582
 requirements: []
 rubyforge_project: mobilize-base
 rubygems_version: 1.8.25