mobilize-base 1.36 → 1.293

data/test/test_base_1.yml

@@ -0,0 +1,3 @@
+- {date: 2013-01-01, snake_date: 2013-02-01, camelDate: 2013-03-01}
+- {date: 2013-01-01, snake_date: 2013-02-02, camelDate: 2013-03-02}
+- {date: 2013-01-01, snake_date: 2013-02-03, camelDate: 2013-03-03}

data/test/test_helper.rb

@@ -8,158 +8,3 @@ $dir = File.dirname(File.expand_path(__FILE__))
 ENV['MOBILIZE_ENV'] = 'test'
 require 'mobilize-base'
 $TESTING = true
-module TestHelper
-  def TestHelper.confirm_expected_jobs(expected_fixture_name,time_limit=600)
-    jobs = {}
-    jobs['expected'] = TestHelper.load_fixture(expected_fixture_name)
-    jobs['pending'] = jobs['expected'].select{|j| j['confirmed_ats'].length < j['count']}
-    start_time = Time.now.utc
-    total_time = 0
-    while (jobs['pending'].length>0 or Mobilize::Resque.workers('working').length>0) and total_time < time_limit
-      #working jobs are running on the queue at this instant
-      jobs['working'] = Mobilize::Resque.workers('working').map{|w| w.job}.select{|j| j and j['payload'] and j['payload']['args']}
-      #failed jobs are in the failure queue
-      jobs['failed'] = Mobilize::Resque.failures.select{|j| j and j['payload'] and j['payload']['args']}
-
-      #unexpected jobs are not supposed to be run in this test, includes leftover failures
-      jobs['unexpected'] = {}
-      error_msg = ""
-      ['working','failed'].each do |state|
-        jobs['unexpected'][state] = jobs[state].reject{|j|
-                                      jobs['expected'].select{|ej|
-                                        ej['state']==state and j['payload']['args'].first == ej['path']}.first}
-        if jobs['unexpected'][state].length>0
-          error_msg += state + ": " + jobs['unexpected'][state].map{|j| j['payload']['args'].first}.join(";") + "\n"
-        end
-      end
-      #clear out unexpected paths or there will be failure
-      if error_msg.length>0
-        raise "Found unexpected results:\n" + error_msg
-      end
-
-      #now make sure pending jobs get done
-      jobs['expected'].each do |j|
-        start_confirmed_ats = j['confirmed_ats']
-        resque_timestamps = jobs[j['state']].select{|sj| sj['payload']['args'].first == j['path']}.map{|sj| sj['run_at'] || sj['failed_at']}
-        new_timestamps = (resque_timestamps - start_confirmed_ats).uniq
-        if new_timestamps.length>0 and j['confirmed_ats'].length < j['count']
-          j['confirmed_ats'] += new_timestamps
-          puts "#{Time.now.utc.to_s}: #{new_timestamps.length.to_s} #{j['state']} added to #{j['path']}; total #{j['confirmed_ats'].length.to_s} of #{j['count']}"
-        end
-      end
-
-      #figure out who's still pending
-      jobs['pending'] = jobs['expected'].select{|j| j['confirmed_ats'].length < j['count']}
-      sleep 1
-      total_time = Time.now.utc - start_time
-      puts "#{total_time.to_s} seconds elapsed" if total_time.to_s.ends_with?("0")
-    end
-  end
-
-  #test methods
-  def TestHelper.restart_test_redis
-    TestHelper.stop_test_redis
-    if !system("which redis-server")
-      raise "** can't find `redis-server` in your path, you need redis to run Resque and Mobilize"
-    end
-    "redis-server #{Mobilize::Base.root}/test/redis-test.conf".bash
-  end
-
-  def TestHelper.stop_test_redis
-    processes = `ps -A -o pid,command | grep [r]edis-test`.split($/)
-    pids = processes.map { |process| process.split(" ")[0] }
-    puts "Killing test redis server..."
-    pids.each { |pid| Process.kill("TERM", pid.to_i) }
-    puts "removing redis db dump file"
-    sleep 5
-    `rm -f #{Mobilize::Base.root}/test/dump.rdb #{Mobilize::Base.root}/test/dump-cluster.rdb`
-  end
-
-  def TestHelper.set_test_env
-    ENV['MOBILIZE_ENV']='test'
-    ::Resque.redis="localhost:9736"
-    mongoid_config_path = "#{Mobilize::Base.root}/config/mobilize/mongoid.yml"
-    Mongoid.load!(mongoid_config_path, Mobilize::Base.env)
-  end
-
-  def TestHelper.drop_test_db
-    TestHelper.set_test_env
-    Mongoid.session(:default).collections.each do |collection| 
-      unless collection.name =~ /^system\./
-        collection.drop
-      end
-    end
-  end
-
-  def TestHelper.build_test_runner(user_name)
-    TestHelper.set_test_env
-    u = Mobilize::User.where(:name=>user_name).first
-    Mobilize::Jobtracker.update_status("delete old books and datasets")
-    # delete any old runner from previous test runs
-    gdrive_slot = Mobilize::Gdrive.owner_email
-    u.runner.gsheet(gdrive_slot).spreadsheet.delete
-    Mobilize::Dataset.find_by_handler_and_path('gbook',u.runner.title).delete
-    Mobilize::Jobtracker.update_status("enqueue jobtracker, wait 45s")
-    Mobilize::Jobtracker.start
-    sleep 45
-  end
-
-  def TestHelper.owner_user
-    gdrive_slot = Mobilize::Gdrive.owner_email
-    user_name = gdrive_slot.split("@").first
-    return Mobilize::User.find_or_create_by_name(user_name)
-  end
-
-  def TestHelper.load_fixture(name)
-    #assume yml, check
-    yml_file_path = "#{Mobilize::Base.root}/test/fixtures/#{name}.yml"
-    standard_file_path = "#{Mobilize::Base.root}/test/fixtures/#{name}"
-    if File.exists?(yml_file_path)
-      YAML.load_file(yml_file_path)
-    elsif File.exists?(standard_file_path)
-      File.read(standard_file_path)
-    else
-      raise "Could not find #{standard_file_path}"
-    end
-  end
-
-  def TestHelper.write_fixture(fixture_name, target_url, options={})
-    u = TestHelper.owner_user
-    fixture_raw = TestHelper.load_fixture(fixture_name)
-    if options['replace']
-      fixture_data = if fixture_raw.class == Array
-                     fixture_raw.hash_array_to_tsv
-                   elsif fixture_raw.class == String
-                     fixture_raw
-                   end
-      Mobilize::Dataset.write_by_url(target_url,fixture_data,u.name,u.email)
-    elsif options['update']
-      handler, sheet_path = target_url.split("://")
-      raise "update only works for gsheet, not #{handler}" unless handler=='gsheet'
-      sheet = Mobilize::Gsheet.find_or_create_by_path(sheet_path,u.email)
-      sheet.add_or_update_rows(fixture_raw)
-    else
-      raise "unknown options #{options.to_s}"
-    end
-    return true
-  end
-
-  #checks output sheet for matching string or minimum length
-  def TestHelper.check_output(target_url, options={})
-    u = TestHelper.owner_user
-    handler, sheet_path = target_url.split("://")
-    handler = nil
-    sheet = Mobilize::Gsheet.find_by_path(sheet_path,u.email)
-    raise "no output found" if sheet.nil?
-    output = sheet.to_tsv
-    if options['match']
-      return true if output == options['match']
-    elsif options['min_length']
-      return true if output.length >= options['min_length']
-    else
-      raise "unknown check options #{options.to_s}"
-    end
-    return true
-  end
-
-end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mobilize-base
 version: !ruby/object:Gem::Version
-  version: '1.36'
+  version: '1.293'
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-21 00:00:00.000000000 Z
+date: 2013-03-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -34,7 +34,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.8.4
+        version: 1.6.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -42,7 +42,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.8.4
+        version: 1.6.1
 - !ruby/object:Gem::Dependency
   name: bson_ext
   requirement: !ruby/object:Gem::Requirement
@@ -50,7 +50,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.8.4
+        version: 1.6.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -58,39 +58,39 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.8.4
+        version: 1.6.1
 - !ruby/object:Gem::Dependency
-  name: mongoid
+  name: mongo
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - '='
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 1.6.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - '='
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 1.6.1
 - !ruby/object:Gem::Dependency
-  name: mongoid-grid_fs
+  name: mongoid
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.0.0
 - !ruby/object:Gem::Dependency
   name: resque
   requirement: !ruby/object:Gem::Requirement
@@ -182,7 +182,6 @@ files:
 - lib/mobilize-base/extensions/resque-server/views/queues.erb
 - lib/mobilize-base/extensions/resque-server/views/working.erb
 - lib/mobilize-base/extensions/string.rb
-- lib/mobilize-base/extensions/time.rb
 - lib/mobilize-base/extensions/yaml.rb
 - lib/mobilize-base/handlers/email.rb
 - lib/mobilize-base/handlers/gbook.rb
@@ -191,10 +190,6 @@ files:
 - lib/mobilize-base/handlers/gridfs.rb
 - lib/mobilize-base/handlers/gsheet.rb
 - lib/mobilize-base/handlers/resque.rb
-- lib/mobilize-base/helpers/job_helper.rb
-- lib/mobilize-base/helpers/jobtracker_helper.rb
-- lib/mobilize-base/helpers/runner_helper.rb
-- lib/mobilize-base/helpers/stage_helper.rb
 - lib/mobilize-base/jobtracker.rb
 - lib/mobilize-base/models/dataset.rb
 - lib/mobilize-base/models/job.rb
@@ -204,7 +199,6 @@ files:
 - lib/mobilize-base/tasks.rb
 - lib/mobilize-base/version.rb
 - lib/samples/gdrive.yml
-- lib/samples/gfile.yml
 - lib/samples/gridfs.yml
 - lib/samples/gsheet.yml
 - lib/samples/jobtracker.yml
@@ -212,14 +206,11 @@ files:
 - lib/samples/resque.yml
 - lib/samples/resque_web.rb
 - mobilize-base.gemspec
-- test/fixtures/base1_stage1.in.yml
-- test/fixtures/integration_expected.yml
-- test/fixtures/integration_jobs.yml
-- test/fixtures/is_due.yml
-- test/integration/mobilize-base_test.rb
+- test/base_job_rows.yml
+- test/mobilize-base_test.rb
 - test/redis-test.conf
+- test/test_base_1.yml
 - test/test_helper.rb
-- test/unit/mobilize-base_test.rb
 homepage: http://github.com/ngmoco/mobilize-base
 licenses: []
 post_install_message: 
@@ -234,7 +225,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -579148248180029294
+      hash: 3027972477721057312
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -243,7 +234,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -579148248180029294
+      hash: 3027972477721057312
 requirements: []
 rubyforge_project: mobilize-base
 rubygems_version: 1.8.25
@@ -252,11 +243,8 @@ specification_version: 3
 summary: Moves datasets and schedules data transfers using MongoDB, Resque and Google
   Docs
 test_files:
-- test/fixtures/base1_stage1.in.yml
-- test/fixtures/integration_expected.yml
-- test/fixtures/integration_jobs.yml
-- test/fixtures/is_due.yml
-- test/integration/mobilize-base_test.rb
+- test/base_job_rows.yml
+- test/mobilize-base_test.rb
 - test/redis-test.conf
+- test/test_base_1.yml
 - test/test_helper.rb
-- test/unit/mobilize-base_test.rb

data/lib/mobilize-base/extensions/time.rb

@@ -1,20 +0,0 @@
-class Time
-  def Time.at_marks_ago(number=1, unit='day', mark='0000')
-    curr_time = Time.now.utc
-    #strip out non-numerical characters from mark, add colon
-    mark = mark.gsub(/[^0-9]/i,"").rjust(4,'0').ie{|m| [m[0..1],":",m[-2..-1]].join}
-    #if user passes in 0 for the number, make it 1
-    number = (number.to_i <= 0 ? 1 : number.to_i)
-    if unit == 'day'
-      curr_mark_time = Time.parse(curr_time.strftime("%Y-%m-%d #{mark} UTC"))
-    elsif unit == 'hour'
-      if curr_time.strftime("%H%M").to_i > mark.to_i
-        curr_mark_time = Time.parse(curr_time.strftime("%Y-%m-%d %H:#{mark[-2..-1]} UTC"))
-      end
-    end
-    #last mark time is 
-    mark_ago_increment = (curr_time > curr_mark_time ? (number-1).send(unit) : number.send(unit))
-    last_mark_time = curr_mark_time - mark_ago_increment
-    return last_mark_time
-  end
-end

data/lib/mobilize-base/helpers/job_helper.rb

@@ -1,54 +0,0 @@
-#this module adds convenience methods to the Job model
-module Mobilize
-  module JobHelper
-    def name
-      j = self
-      j.path.split("/").last
-    end
-
-    def stages
-      j = self
-      #starts with the job path, followed by a slash
-      Stage.where(:path=>/^#{j.path.escape_regex}\//).to_a.sort_by{|s| s.path}
-    end
-
-    def status
-      #last stage status
-      j = self
-      j.active_stage.status if j.active_stage
-    end
-
-    def active_stage
-      j = self
-      #latest started at or first
-      j.stages.select{|s| s.started_at}.sort_by{|s| s.started_at}.last || j.stages.first
-    end
-
-    def completed_at
-      j = self
-      j.stages.last.completed_at if j.stages.last
-    end
-
-    def failed_at
-      j = self
-      j.active_stage.failed_at if j.active_stage
-    end
-
-    def status_at
-      j = self
-      j.active_stage.status_at if j.active_stage
-    end
-
-    #convenience methods
-    def runner
-      j = self
-      runner_path = j.path.split("/")[0..-2].join("/")
-      return Runner.where(:path=>runner_path).first
-    end
-
-    def is_working?
-      j = self
-      j.stages.select{|s| s.is_working?}.compact.length>0
-    end
-  end
-end

data/lib/mobilize-base/helpers/jobtracker_helper.rb

@@ -1,143 +0,0 @@
-module Mobilize
-  module Jobtracker
-    def Jobtracker.config
-      Base.config('jobtracker')
-    end
-
-    #modify this to increase the frequency of request cycles
-    def Jobtracker.cycle_freq
-      Jobtracker.config['cycle_freq']
-    end
-
-    #frequency of notifications
-    def Jobtracker.notification_freq
-      Jobtracker.config['notification_freq']
-    end
-
-    def Jobtracker.runner_read_freq
-      Jobtracker.config['runner_read_freq']
-    end
-
-    #long running tolerance
-    def Jobtracker.max_run_time
-      Jobtracker.config['max_run_time']
-    end
-
-    def Jobtracker.admins
-      Jobtracker.config['admins']
-    end
-
-    def Jobtracker.worker
-      Resque.find_worker_by_path("jobtracker")
-    end
-
-    def Jobtracker.workers(state="all")
-      Resque.workers(state)
-    end
-
-    def Jobtracker.status
-      args = Jobtracker.get_args
-      return args['status'] if args
-      job = Resque.jobs.select{|j| j['args'].first=='jobtracker'}.first
-      return 'queued' if job
-      return 'stopped'
-    end
-
-    def Jobtracker.update_status(msg)
-      #this is to keep jobtracker from resisting stop commands
-      return false if Jobtracker.status=="stopping"
-      #Jobtracker has no persistent database state
-      Resque.set_worker_args_by_path("jobtracker",{'status'=>msg})
-      return true
-    end
-
-    def Jobtracker.restart
-      Jobtracker.stop!
-      Jobtracker.start
-    end
-
-    def Jobtracker.set_args(args)
-      Resque.set_worker_args(Jobtracker.worker,args)
-      return true
-    end
-
-    def Jobtracker.get_args
-      Resque.get_worker_args(Jobtracker.worker)
-    end
-
-    def Jobtracker.kill_workers
-      Resque.kill_workers
-    end
-
-    def Jobtracker.kill_idle_workers
-      Resque.kill_idle_workers
-    end
-
-    def Jobtracker.kill_idle_and_stale_workers
-      Resque.kill_idle_and_stale_workers
-    end
-
-    def Jobtracker.prep_workers
-      Resque.prep_workers
-    end
-
-    def Jobtracker.failures
-      Resque.failures
-    end
-
-    def Jobtracker.start
-      if Jobtracker.status!='stopped'
-        Jobtracker.update_status("Jobtracker still #{Jobtracker.status}")
-      else
-        #make sure that workers are running and at the right number
-        #Resque.prep_workers
-        #queue up the jobtracker (starts the perform method)
-        Jobtracker.enqueue!
-      end
-      return true
-    end
-
-    def Jobtracker.enqueue!
-      ::Resque::Job.create(Resque.queue_name, Jobtracker, 'jobtracker',{})
-    end
-
-    def Jobtracker.restart!
-      Jobtracker.stop!
-      Jobtracker.start
-      return true
-    end
-
-    def Jobtracker.restart_workers!
-      Jobtracker.kill_workers
-      sleep 10
-      Jobtracker.prep_workers
-      Jobtracker.update_status("put workers back on the queue")
-    end
-
-    def Jobtracker.stop!
-      #send signal for Jobtracker to check for
-      Jobtracker.update_status('stopping')
-      sleep 5
-      i=0
-      while Jobtracker.status=='stopping'
-        puts "#{Jobtracker.to_s} still on queue, waiting"
-        sleep 5
-        i+=1
-      end
-      return true
-    end
-
-    def Jobtracker.last_notification
-      return Jobtracker.get_args["last_notification"] if Jobtracker.get_args
-    end
-
-    def Jobtracker.last_notification=(time)
-      Jobtracker.set_args({"last_notification"=>time})
-    end
-
-    def Jobtracker.notif_due?
-      last_duetime = Time.now.utc - Jobtracker.notification_freq
-      return (Jobtracker.last_notification.to_s.length==0 || Jobtracker.last_notification.to_datetime < last_duetime)
-    end
-  end
-end