mobilize-base 1.351 → 1.361

data/test/fixtures/integration_expected.yml

@@ -0,0 +1,25 @@
+---
+- path: "jobtracker"
+  state: working
+  count: 1
+  confirmed_ats: []
+- path: "Runner_mobilize(test)/jobs"
+  state: working
+  count: 2
+  confirmed_ats: []
+- path: "Runner_mobilize(test)/jobs/base1/stage1"
+  state: working
+  count: 1
+  confirmed_ats: []
+- path: "Runner_mobilize(test)/jobs/base1/stage2"
+  state: working
+  count: 1
+  confirmed_ats: []
+- path: "Runner_mobilize(test)/jobs/base2/stage1"
+  state: working
+  count: 1
+  confirmed_ats: []
+- path: "Runner_mobilize(test)/jobs/base2/stage1"
+  state: failed
+  count: 2
+  confirmed_ats: []

data/test/fixtures/integration_jobs.yml

@@ -0,0 +1,12 @@
+---
+- name: base1
+  active: true
+  trigger: once
+  status: ""
+  stage1: gfile.write source:"gsheet://base1_stage1.in", target:"gfile://base1_stage1.out"
+  stage2: gsheet.write source:"gfile://base1_stage1.out", target:"gsheet://base1_stage2.out"
+- name: base2
+  active: true
+  trigger: after base1
+  status: ""
+  stage1: gsheet.write source:"base1."_"", target:"base2_stage1.out"

data/test/fixtures/is_due.yml

@@ -0,0 +1,97 @@
+---
+- name: once_active
+  active: true
+  trigger: once
+  status: ""
+  stage1: gsheet.method source:"source", target:"target"
+  expected: true
+- name: once_inactive
+  active: false
+  trigger: once
+  status: ""
+  stage1: gfile.method source:"source", target:"target"
+  expected: false
+- name: 1h_comp_never
+  active: true
+  trigger: every 1.hour
+  status: ""
+  stage1: gsheet.method source:"source", target:"target"
+  expected: true
+- name: 1h_comp_0.5h_ago
+  active: true
+  trigger: every 1.hour
+  status: ""
+  stage1: gsheet.method source:"source", target:"target"
+  completed_at: Time.now.utc - 1800.seconds
+  expected: false
+- name: 1h_comp_1.5h_ago
+  active: true
+  trigger: every 1.hour
+  status: ""
+  stage1: gsheet.method source:"source", target:"target"
+  completed_at: Time.now.utc - 5400.seconds
+  expected: true
+- name: 1h_aft45_comp_2_45_marks_ago
+  active: true
+  trigger: 'every 1.hour after 45'
+  status: ""
+  stage1: gsheet.method source:"source", target:"target"
+  completed_at: Time.at_marks_ago(2,'hour', '45')
+  expected: true
+- name: 1h_aft45_comp_aft_1_45_mark_ago
+  active: true
+  trigger: 'every 1.hour after :45'
+  status: ""
+  stage1: gsheet.method source:"source", target:"target"
+  completed_at: Time.at_marks_ago(1,'hour', '45')
+  expected: false
+- name: 1d_aft0135_comp_2_0135_marks_ago
+  active: true
+  trigger: 'every 1.day after 01:35'
+  status: ""
+  stage1: gsheet.method source:"source", target:"target"
+  completed_at: Time.at_marks_ago(2,'day', '0135')
+  expected: true
+- name: 1d_aft0135_comp_1_0135_mark_ago
+  active: true
+  trigger: 'every 1.day after 0135'
+  status: ""
+  stage1: gsheet.method source:"source", target:"target"
+  completed_at: Time.at_marks_ago(1,'day', '0135')
+  expected: false
+- name: 5d_aft0135_comp_5_0135_mark_ago
+  active: true
+  trigger: 'every 5.day after 01:35'
+  status: ""
+  stage1: gsheet.method source:"source", target:"target"
+  completed_at: Time.at_marks_ago(5,'day', '0135')
+  expected: false
+- name: 5d_aft0135_comp_6_0135_mark_ago
+  active: true
+  trigger: 'every 5.day after 01:35'
+  status: ""
+  stage1: gsheet.method source:"source", target:"target"
+  completed_at: Time.at_marks_ago(6,'day', '0135')
+  expected: true
+- name: parent_comp_child_fail
+  active: true
+  trigger: 'after once_inactive'
+  status: ""
+  stage1: gsheet.method source:"source", target:"target"
+  parent: {completed_at: Time.now.utc - 1.second}
+  failed_at: Time.now.utc
+  expected: true
+- name: parent_comp_child_comp
+  active: true
+  trigger: 'after once_inactive'
+  status: ""
+  stage1: gsheet.method source:"source", target:"target"
+  parent: {completed_at: Time.now.utc - 1.second}
+  completed_at: Time.now.utc
+  expected: false
+- name: missing_module
+  active: true
+  trigger: once
+  status: ""
+  stage1: handler.method source:"source", target:"target"
+  expected: false

data/test/integration/mobilize-base_test.rb

@@ -0,0 +1,57 @@
+require 'test_helper'
+describe Mobilize do
+
+  it "runs integration test" do
+
+    puts "restart test redis"
+    TestHelper.restart_test_redis
+    TestHelper.drop_test_db
+
+    puts "restart workers"
+    Mobilize::Jobtracker.restart_workers!
+
+    u = TestHelper.owner_user
+    r = u.runner
+    user_name = u.name
+    gdrive_slot = u.email
+
+    puts "build test runner"
+    TestHelper.build_test_runner(user_name)
+    assert Mobilize::Jobtracker.workers.length == Mobilize::Resque.config['max_workers'].to_i
+
+    puts "add base1_stage1.in sheet"
+    input_fixture_name = "base1_stage1.in"
+    input_target_url = "gsheet://#{r.title}/#{input_fixture_name}"
+    TestHelper.write_fixture(input_fixture_name, input_target_url, 'replace' => true)
+
+    puts "add jobs sheet with integration jobs"
+    jobs_fixture_name = "integration_jobs"
+    jobs_target_url = "gsheet://#{r.title}/jobs"
+    TestHelper.write_fixture(jobs_fixture_name, jobs_target_url, 'update' => true)
+
+    puts "wait for stages"
+    Mobilize::Jobtracker.start
+    #wait for stages to complete
+    expected_fixture_name = "integration_expected"
+    TestHelper.confirm_expected_jobs(expected_fixture_name)
+    #stop jobtracker
+    Mobilize::Jobtracker.stop!
+
+    puts "jobtracker posted test sheet data to test destination, and checksum succeeded?"
+    tsv_hash = {}
+    ["base1_stage1.in", "base1_stage2.out"].each do |sheet_name|
+      url = "gsheet://#{r.title}/#{sheet_name}"
+      data = Mobilize::Dataset.read_by_url(url,user_name,gdrive_slot)
+      assert TestHelper.check_output(url, 'min_length' => 10) == true
+      tsv_hash[sheet_name] = data
+    end
+
+    assert tsv_hash["base1_stage2.out"] == tsv_hash["base1_stage1.in"]
+
+    err_url = "gsheet://#{r.title}/base2_stage1.err"
+    err_response = "Unable to parse stage params, make sure you don't have issues with your quotes, commas, or colons."
+
+    assert TestHelper.check_output(err_url, 'match' => err_response) == true
+
+  end
+end

data/test/test_helper.rb

@@ -9,26 +9,53 @@ ENV['MOBILIZE_ENV'] = 'test'
 require 'mobilize-base'
 $TESTING = true
 module TestHelper
-  def TestHelper.wait_for_stages(time_limit=600,stage_limit=120,wait_length=10)
-    time = 0
-    time_since_stage = 0
-    #check for 10 min
-    while time < time_limit and time_since_stage < stage_limit
-      sleep wait_length
-      job_classes = Mobilize::Resque.jobs.map{|j| j['class']}
-      if job_classes.include?("Mobilize::Stage")
-        time_since_stage = 0
-        puts "saw stage at #{time.to_s} seconds"
-      else
-        time_since_stage += wait_length
-        puts "#{time_since_stage.to_s} seconds since stage seen"
+  def TestHelper.confirm_expected_jobs(expected_fixture_name,time_limit=600)
+    jobs = {}
+    jobs['expected'] = TestHelper.load_fixture(expected_fixture_name)
+    jobs['pending'] = jobs['expected'].select{|j| j['confirmed_ats'].length < j['count']}
+    start_time = Time.now.utc
+    total_time = 0
+    while (jobs['pending'].length>0 or #while there are pending jobs
+           #or there are workers working (don't count jobtracker)
+           Mobilize::Resque.workers('working').select{|w| w.job['payload']['class']!='Mobilize::Jobtracker'}.length>0) and
+           total_time < time_limit #time limit not yet expired
+      #working jobs are running on the queue at this instant
+      jobs['working'] = Mobilize::Resque.workers('working').map{|w| w.job}.select{|j| j and j['payload'] and j['payload']['args']}
+      #failed jobs are in the failure queue
+      jobs['failed'] = Mobilize::Resque.failures.select{|j| j and j['payload'] and j['payload']['args']}
+
+      #unexpected jobs are not supposed to be run in this test, includes leftover failures
+      jobs['unexpected'] = {}
+      error_msg = ""
+      ['working','failed'].each do |state|
+        jobs['unexpected'][state] = jobs[state].reject{|j|
+                                      jobs['expected'].select{|ej|
+                                        ej['state']==state and j['payload']['args'].first == ej['path']}.first}
+        if jobs['unexpected'][state].length>0
+          error_msg += state + ": " + jobs['unexpected'][state].map{|j| j['payload']['args'].first}.join(";") + "\n"
+        end
+      end
+      #clear out unexpected paths or there will be failure
+      if error_msg.length>0
+        raise "Found unexpected results:\n" + error_msg
+      end
+
+      #now make sure pending jobs get done
+      jobs['expected'].each do |j|
+        start_confirmed_ats = j['confirmed_ats']
+        resque_timestamps = jobs[j['state']].select{|sj| sj['payload']['args'].first == j['path']}.map{|sj| sj['run_at'] || sj['failed_at']}
+        new_timestamps = (resque_timestamps - start_confirmed_ats).uniq
+        if new_timestamps.length>0 and j['confirmed_ats'].length < j['count']
+          j['confirmed_ats'] += new_timestamps
+          puts "#{Time.now.utc.to_s}: #{new_timestamps.length.to_s} #{j['state']} added to #{j['path']}; total #{j['confirmed_ats'].length.to_s} of #{j['count']}"
+        end
       end
-      time += wait_length
-      puts "total wait time #{time.to_s} seconds"
-    end
 
-    if time >= time_limit
-      raise "Timed out before stage completion"
+      #figure out who's still pending
+      jobs['pending'] = jobs['expected'].select{|j| j['confirmed_ats'].length < j['count']}
+      sleep 1
+      total_time = Time.now.utc - start_time
+      puts "#{total_time.to_s} seconds elapsed" if total_time.to_s.ends_with?("0")
     end
   end
 
@@ -82,8 +109,60 @@ module TestHelper
 
   def TestHelper.owner_user
     gdrive_slot = Mobilize::Gdrive.owner_email
-    puts "create user 'mobilize'"
     user_name = gdrive_slot.split("@").first
     return Mobilize::User.find_or_create_by_name(user_name)
   end
+
+  def TestHelper.load_fixture(name)
+    #assume yml, check
+    yml_file_path = "#{Mobilize::Base.root}/test/fixtures/#{name}.yml"
+    standard_file_path = "#{Mobilize::Base.root}/test/fixtures/#{name}"
+    if File.exists?(yml_file_path)
+      YAML.load_file(yml_file_path)
+    elsif File.exists?(standard_file_path)
+      File.read(standard_file_path)
+    else
+      raise "Could not find #{standard_file_path}"
+    end
+  end
+
+  def TestHelper.write_fixture(fixture_name, target_url, options={})
+    u = TestHelper.owner_user
+    fixture_raw = TestHelper.load_fixture(fixture_name)
+    if options['replace']
+      fixture_data = if fixture_raw.class == Array
+                     fixture_raw.hash_array_to_tsv
+                   elsif fixture_raw.class == String
+                     fixture_raw
+                   end
+      Mobilize::Dataset.write_by_url(target_url,fixture_data,u.name,u.email)
+    elsif options['update']
+      handler, sheet_path = target_url.split("://")
+      raise "update only works for gsheet, not #{handler}" unless handler=='gsheet'
+      sheet = Mobilize::Gsheet.find_or_create_by_path(sheet_path,u.email)
+      sheet.add_or_update_rows(fixture_raw)
+    else
+      raise "unknown options #{options.to_s}"
+    end
+    return true
+  end
+
+  #checks output sheet for matching string or minimum length
+  def TestHelper.check_output(target_url, options={})
+    u = TestHelper.owner_user
+    handler, sheet_path = target_url.split("://")
+    handler = nil
+    sheet = Mobilize::Gsheet.find_by_path(sheet_path,u.email)
+    raise "no output found" if sheet.nil?
+    output = sheet.to_tsv
+    if options['match']
+      return true if output == options['match']
+    elsif options['min_length']
+      return true if output.length >= options['min_length']
+    else
+      raise "unknown check options #{options.to_s}"
+    end
+    return true
+  end
+
 end

data/test/unit/mobilize-base_test.rb

@@ -0,0 +1,33 @@
+require 'test_helper'
+class TestUnit < MiniTest::Unit::TestCase
+  def setup
+    TestHelper.restart_test_redis
+    TestHelper.drop_test_db
+  end
+
+  #this test checks that several job triggers work as expected
+  def test_is_due
+    u = TestHelper.owner_user
+    job_hashes = TestHelper.load_fixture("is_due")
+    job_hashes.each do |jh|
+      job_path = "#{u.runner.path}/#{jh['name']}"
+      j = Mobilize::Job.find_or_create_by_path(job_path)
+      #update job params
+      j.update_from_hash(jh)
+      #apply the completed_at, failed at, and parent attributes where appropriate
+      if jh['completed_at']
+        j.stages.last.update_attributes(:completed_at=>eval(jh['completed_at']))
+      end
+      if jh['failed_at']
+        j.stages.last.update_attributes(:failed_at=>eval(jh['failed_at']))
+      end
+      if jh['parent']
+        j.parent.stages.last.update_attributes(:completed_at=>eval(jh['parent']['completed_at'])) if jh['parent']['completed_at']
+        j.parent.stages.last.update_attributes(:failed_at=>eval(jh['parent']['failed_at'])) if jh['parent']['failed_at']
+      end
+      expected = jh['expected']
+      #check if is_due
+      assert expected == j.is_due?
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mobilize-base
 version: !ruby/object:Gem::Version
-  version: '1.351'
+  version: '1.361'
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-25 00:00:00.000000000 Z
+date: 2013-05-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -182,6 +182,7 @@ files:
 - lib/mobilize-base/extensions/resque-server/views/queues.erb
 - lib/mobilize-base/extensions/resque-server/views/working.erb
 - lib/mobilize-base/extensions/string.rb
+- lib/mobilize-base/extensions/time.rb
 - lib/mobilize-base/extensions/yaml.rb
 - lib/mobilize-base/handlers/email.rb
 - lib/mobilize-base/handlers/gbook.rb
@@ -203,6 +204,7 @@ files:
 - lib/mobilize-base/tasks.rb
 - lib/mobilize-base/version.rb
 - lib/samples/gdrive.yml
+- lib/samples/gfile.yml
 - lib/samples/gridfs.yml
 - lib/samples/gsheet.yml
 - lib/samples/jobtracker.yml
@@ -210,11 +212,14 @@ files:
 - lib/samples/resque.yml
 - lib/samples/resque_web.rb
 - mobilize-base.gemspec
-- test/base_job_rows.yml
-- test/mobilize-base_test.rb
+- test/fixtures/base1_stage1.in.yml
+- test/fixtures/integration_expected.yml
+- test/fixtures/integration_jobs.yml
+- test/fixtures/is_due.yml
+- test/integration/mobilize-base_test.rb
 - test/redis-test.conf
-- test/test_base_1.yml
 - test/test_helper.rb
+- test/unit/mobilize-base_test.rb
 homepage: http://github.com/ngmoco/mobilize-base
 licenses: []
 post_install_message: 
@@ -229,7 +234,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 4420073828570035753
+      hash: 3685016355014440836
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -238,7 +243,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 4420073828570035753
+      hash: 3685016355014440836
 requirements: []
 rubyforge_project: mobilize-base
 rubygems_version: 1.8.25
@@ -247,8 +252,11 @@ specification_version: 3
 summary: Moves datasets and schedules data transfers using MongoDB, Resque and Google
   Docs
 test_files:
-- test/base_job_rows.yml
-- test/mobilize-base_test.rb
+- test/fixtures/base1_stage1.in.yml
+- test/fixtures/integration_expected.yml
+- test/fixtures/integration_jobs.yml
+- test/fixtures/is_due.yml
+- test/integration/mobilize-base_test.rb
 - test/redis-test.conf
-- test/test_base_1.yml
 - test/test_helper.rb
+- test/unit/mobilize-base_test.rb

data/test/base_job_rows.yml

@@ -1,15 +0,0 @@
-- name: base1
-  active: true
-  trigger: once
-  status: ""
-  stage1: gsheet.write source:"gfile://test_base_1.tsv", target:base1.out
-- name: base2
-  active: true
-  trigger: after base1
-  status: ""
-  stage1: gsheet.write source:base1.out, target:base2.out
-- name: base3
-  active: true
-  trigger: after base2
-  status: ""
-  stage1: gsheet.write source:"base1."_"", target:base2.out

data/test/mobilize-base_test.rb

@@ -1,65 +0,0 @@
-require 'test_helper'
-
-describe "Mobilize" do
-  it "runs integration test" do
-
-    puts "restart test redis"
-    TestHelper.restart_test_redis
-
-    puts "clear out test db"
-    TestHelper.drop_test_db
-
-    puts "restart workers"
-    Mobilize::Jobtracker.restart_workers!
-
-    u = TestHelper.owner_user
-    user_name = u.name
-    gdrive_slot = u.email
-
-    puts "build test runner"
-    TestHelper.build_test_runner(user_name)
-    assert Mobilize::Jobtracker.workers.length == Mobilize::Resque.config['max_workers'].to_i
-
-    puts "Jobtracker created runner with 'jobs' sheet?"
-
-    r = u.runner
-    jobs_sheet_url = "gsheet://#{r.path}"
-    jobs_sheet = Mobilize::Gsheet.find_by_path(r.path,gdrive_slot)
-    jobs_sheet_dst = Mobilize::Dataset.find_or_create_by_url(jobs_sheet_url)
-    jobs_sheet_tsv = jobs_sheet_dst.read(user_name,gdrive_slot)
-
-    assert jobs_sheet_tsv.tsv_header_array.join.length == 53 #total header length
-
-    puts "add base1 input file"
-    test_filename = "test_base_1"
-    file_url = "gfile://#{test_filename}.tsv"
-    test_source_ha = ::YAML.load_file("#{Mobilize::Base.root}/test/#{test_filename}.yml")*40
-    test_source_tsv = test_source_ha.hash_array_to_tsv
-    Mobilize::Dataset.write_by_url(file_url,test_source_tsv,user_name)
-    rem_tsv = Mobilize::Dataset.read_by_url(file_url,user_name)
-    assert rem_tsv == test_source_tsv
-
-    puts "add row to jobs sheet, wait for stages"
-    test_job_rows = ::YAML.load_file("#{Mobilize::Base.root}/test/base_job_rows.yml")
-    jobs_sheet.reload
-    jobs_sheet.add_or_update_rows(test_job_rows)
-    #wait for stages to complete
-    TestHelper.wait_for_stages
-
-    puts "jobtracker posted test sheet data to test destination, and checksum succeeded?"
-    tsv_hash = {}
-    ["base1.out", "base2.out", "base3_stage1.err"].each do |sheet_name|
-      url = "gsheet://#{r.title}/#{sheet_name}"
-      data = Mobilize::Dataset.read_by_url(url,user_name,gdrive_slot)
-      tsv_hash[sheet_name] = data
-    end
-
-    assert tsv_hash["base1.out"].to_s.length>0
-    assert tsv_hash["base2.out"] == tsv_hash["base1.out"]
-
-    base3_response = tsv_hash["base3_stage1.err"].tsv_to_hash_array.first['response']
-    assert base3_response == "Unable to parse stage params, make sure you don't have issues with your quotes, commas, or colons."
-
-    Mobilize::Jobtracker.stop!
-  end
-end

data/test/test_base_1.yml

@@ -1,3 +0,0 @@
-- {date: 2013-01-01, snake_date: 2013-02-01, camelDate: 2013-03-01}
-- {date: 2013-01-01, snake_date: 2013-02-02, camelDate: 2013-03-02}
-- {date: 2013-01-01, snake_date: 2013-02-03, camelDate: 2013-03-03}