mobilize-base 1.36 → 1.293

data/lib/mobilize-base/helpers/runner_helper.rb

@@ -1,83 +0,0 @@
-#this module adds convenience methods to the Runner model
-module Mobilize
-  module RunnerHelper
-    def headers
-      %w{name active trigger status stage1 stage2 stage3 stage4 stage5}
-    end
-
-    def title
-      r = self
-      r.path.split("/").first
-    end
-
-    def worker
-      r = self
-      Mobilize::Resque.find_worker_by_path(r.path)
-    end
-
-    def dataset
-      r = self
-      Dataset.find_or_create_by_handler_and_path("gsheet",r.path)
-    end
-
-    def gbook(gdrive_slot)
-      r = self
-      title = r.path.split("/").first
-      Gbook.find_by_path(title,gdrive_slot)
-    end
-
-    def gsheet(gdrive_slot)
-      r = self
-      u = r.user
-      jobs_sheet = Gsheet.find_by_path(r.path,gdrive_slot)
-      #make sure the user has a runner with a jobs sheet and has write privileges on the spreadsheet
-      unless (jobs_sheet and jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="writer"})
-        #only give the user edit permissions if they're the ones
-        #creating it
-        jobs_sheet = Gsheet.find_or_create_by_path(r.path,gdrive_slot)
-        unless jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="owner"}
-          jobs_sheet.spreadsheet.update_acl(u.email)
-        end
-        jobs_sheet.add_headers(r.headers)
-        begin;jobs_sheet.delete_sheet1;rescue;end #don't care if sheet1 deletion fails
-      end
-      return jobs_sheet
-    end
-
-    def jobs(jname=nil)
-      r = self
-      js = Job.where(:path=>/^#{r.path.escape_regex}/).to_a
-      if jname
-        return js.sel{|j| j.name == jname}.first
-      else
-        return js
-      end
-    end
-
-    def user
-      r = self
-      user_name = r.path.split("_")[1..-1].join("_").split("(").first.split("/").first
-      User.where(:name=>user_name).first
-    end
-
-    def update_status(msg)
-      r = self
-      r.update_attributes(:status=>msg, :status_at=>Time.now.utc)
-      Mobilize::Resque.set_worker_args_by_path(r.path,{'status'=>msg})
-      return true
-    end
-
-    def is_working?
-      r = self
-      Mobilize::Resque.active_paths.include?(r.path)
-    end
-
-    def is_due?
-      r = self.reload
-      return false if r.is_working?
-      prev_due_time = Time.now.utc - Jobtracker.runner_read_freq
-      return true if r.started_at.nil? or r.started_at < prev_due_time
-    end
-
-  end
-end

data/lib/mobilize-base/helpers/stage_helper.rb

@@ -1,38 +0,0 @@
-#this module adds convenience methods to the Stage model
-module Mobilize
-  module StageHelper
-    def idx
-      s = self
-      s.path.split("/").last.gsub("stage","").to_i
-    end
-
-    def out_dst
-      #this gives a dataset that points to the output
-      #allowing you to determine its size
-      #before committing to a read or write
-      s = self
-      Dataset.find_by_url(s.response['out_url']) if s.response and s.response['out_url']
-    end
-
-    def err_dst
-      #this gives a dataset that points to the output
-      #allowing you to determine its size
-      #before committing to a read or write
-      s = self
-      Dataset.find_by_url(s.response['err_url']) if s.response and s.response['err_url']
-    end
-
-    def params
-      s = self
-      p = YAML.easy_load(s.param_string)
-      raise "Must resolve to Hash" unless p.class==Hash
-      return p
-    end
-
-    def job
-      s = self
-      job_path = s.path.split("/")[0..-2].join("/")
-      Job.where(:path=>job_path).first
-    end
-  end
-end

data/lib/samples/gfile.yml

@@ -1,9 +0,0 @@
----
-development:
-  max_length: 100000000 #about 100MB
-test:
-  max_length: 100000000 #about 100MB
-staging:
-  max_length: 100000000 #about 100MB
-production:
-  max_length: 100000000 #about 100MB

data/test/fixtures/base1_stage1.in.yml

@@ -1,10 +0,0 @@
----
-- date: 2013-01-01
-  snake_date: 2013-02-01
-  camelDate: 2013-03-01
-- date: 2013-01-01
-  snake_date: 2013-02-02
-  camelDate: 2013-03-02
-- date: 2013-01-01
-  snake_date: 2013-02-03
-  camelDate: 2013-03-03

data/test/fixtures/integration_expected.yml

@@ -1,25 +0,0 @@
----
-- path: "jobtracker"
-  state: working
-  count: 1
-  confirmed_ats: []
-- path: "Runner_mobilize(test)/jobs"
-  state: working
-  count: 2
-  confirmed_ats: []
-- path: "Runner_mobilize(test)/jobs/base1/stage1"
-  state: working
-  count: 1
-  confirmed_ats: []
-- path: "Runner_mobilize(test)/jobs/base1/stage2"
-  state: working
-  count: 1
-  confirmed_ats: []
-- path: "Runner_mobilize(test)/jobs/base2/stage1"
-  state: working
-  count: 1
-  confirmed_ats: []
-- path: "Runner_mobilize(test)/jobs/base2/stage1"
-  state: failed
-  count: 2
-  confirmed_ats: []

data/test/fixtures/integration_jobs.yml

@@ -1,12 +0,0 @@
----
-- name: base1
-  active: true
-  trigger: once
-  status: ""
-  stage1: gfile.write source:"gsheet://base1_stage1.in", target:"gfile://base1_stage1.out"
-  stage2: gsheet.write source:"gfile://base1_stage1.out", target:"gsheet://base1_stage2.out"
-- name: base2
-  active: true
-  trigger: after base1
-  status: ""
-  stage1: gsheet.write source:"base1."_"", target:"base2_stage1.out"

data/test/fixtures/is_due.yml

@@ -1,97 +0,0 @@
----
-- name: once_active
-  active: true
-  trigger: once
-  status: ""
-  stage1: gsheet.method source:"source", target:"target"
-  expected: true
-- name: once_inactive
-  active: false
-  trigger: once
-  status: ""
-  stage1: gfile.method source:"source", target:"target"
-  expected: false
-- name: 1h_comp_never
-  active: true
-  trigger: every 1.hour
-  status: ""
-  stage1: gsheet.method source:"source", target:"target"
-  expected: true
-- name: 1h_comp_0.5h_ago
-  active: true
-  trigger: every 1.hour
-  status: ""
-  stage1: gsheet.method source:"source", target:"target"
-  completed_at: Time.now.utc - 1800.seconds
-  expected: false
-- name: 1h_comp_1.5h_ago
-  active: true
-  trigger: every 1.hour
-  status: ""
-  stage1: gsheet.method source:"source", target:"target"
-  completed_at: Time.now.utc - 5400.seconds
-  expected: true
-- name: 1h_aft45_comp_2_45_marks_ago
-  active: true
-  trigger: 'every 1.hour after 45'
-  status: ""
-  stage1: gsheet.method source:"source", target:"target"
-  completed_at: Time.at_marks_ago(2,'hour', '45')
-  expected: true
-- name: 1h_aft45_comp_aft_1_45_mark_ago
-  active: true
-  trigger: 'every 1.hour after :45'
-  status: ""
-  stage1: gsheet.method source:"source", target:"target"
-  completed_at: Time.at_marks_ago(1,'hour', '45')
-  expected: false
-- name: 1d_aft0135_comp_2_0135_marks_ago
-  active: true
-  trigger: 'every 1.day after 01:35'
-  status: ""
-  stage1: gsheet.method source:"source", target:"target"
-  completed_at: Time.at_marks_ago(2,'day', '0135')
-  expected: true
-- name: 1d_aft0135_comp_1_0135_mark_ago
-  active: true
-  trigger: 'every 1.day after 0135'
-  status: ""
-  stage1: gsheet.method source:"source", target:"target"
-  completed_at: Time.at_marks_ago(1,'day', '0135')
-  expected: false
-- name: 5d_aft0135_comp_5_0135_mark_ago
-  active: true
-  trigger: 'every 5.day after 01:35'
-  status: ""
-  stage1: gsheet.method source:"source", target:"target"
-  completed_at: Time.at_marks_ago(5,'day', '0135')
-  expected: false
-- name: 5d_aft0135_comp_6_0135_mark_ago
-  active: true
-  trigger: 'every 5.day after 01:35'
-  status: ""
-  stage1: gsheet.method source:"source", target:"target"
-  completed_at: Time.at_marks_ago(6,'day', '0135')
-  expected: true
-- name: parent_comp_child_fail
-  active: true
-  trigger: 'after once_inactive'
-  status: ""
-  stage1: gsheet.method source:"source", target:"target"
-  parent: {completed_at: Time.now.utc - 1.second}
-  failed_at: Time.now.utc
-  expected: true
-- name: parent_comp_child_comp
-  active: true
-  trigger: 'after once_inactive'
-  status: ""
-  stage1: gsheet.method source:"source", target:"target"
-  parent: {completed_at: Time.now.utc - 1.second}
-  completed_at: Time.now.utc
-  expected: false
-- name: missing_module
-  active: true
-  trigger: once
-  status: ""
-  stage1: handler.method source:"source", target:"target"
-  expected: false

data/test/integration/mobilize-base_test.rb

@@ -1,57 +0,0 @@
-require 'test_helper'
-describe Mobilize do
-
-  it "runs integration test" do
-
-    puts "restart test redis"
-    TestHelper.restart_test_redis
-    TestHelper.drop_test_db
-
-    puts "restart workers"
-    Mobilize::Jobtracker.restart_workers!
-
-    u = TestHelper.owner_user
-    r = u.runner
-    user_name = u.name
-    gdrive_slot = u.email
-
-    puts "build test runner"
-    TestHelper.build_test_runner(user_name)
-    assert Mobilize::Jobtracker.workers.length == Mobilize::Resque.config['max_workers'].to_i
-
-    puts "add base1_stage1.in sheet"
-    input_fixture_name = "base1_stage1.in"
-    input_target_url = "gsheet://#{r.title}/#{input_fixture_name}"
-    TestHelper.write_fixture(input_fixture_name, input_target_url, 'replace' => true)
-
-    puts "add jobs sheet with integration jobs"
-    jobs_fixture_name = "integration_jobs"
-    jobs_target_url = "gsheet://#{r.title}/jobs"
-    TestHelper.write_fixture(jobs_fixture_name, jobs_target_url, 'update' => true)
-
-    puts "wait for stages"
-    Mobilize::Jobtracker.start
-    #wait for stages to complete
-    expected_fixture_name = "integration_expected"
-    TestHelper.confirm_expected_jobs(expected_fixture_name)
-    #stop jobtracker
-    Mobilize::Jobtracker.stop!
-
-    puts "jobtracker posted test sheet data to test destination, and checksum succeeded?"
-    tsv_hash = {}
-    ["base1_stage1.in", "base1_stage2.out"].each do |sheet_name|
-      url = "gsheet://#{r.title}/#{sheet_name}"
-      data = Mobilize::Dataset.read_by_url(url,user_name,gdrive_slot)
-      assert TestHelper.check_output(url, 'min_length' => 10) == true
-      tsv_hash[sheet_name] = data
-    end
-
-    assert tsv_hash["base1_stage2.out"] == tsv_hash["base1_stage1.in"]
-
-    err_url = "gsheet://#{r.title}/base2_stage1.err"
-    err_response = "Unable to parse stage params, make sure you don't have issues with your quotes, commas, or colons."
-
-    assert TestHelper.check_output(err_url, 'match' => err_response) == true
-
-  end
-end

data/test/unit/mobilize-base_test.rb

@@ -1,33 +0,0 @@
-require 'test_helper'
-class TestUnit < MiniTest::Unit::TestCase
-  def setup
-    TestHelper.restart_test_redis
-    TestHelper.drop_test_db
-  end
-
-  #this test checks that several job triggers work as expected
-  def test_is_due
-    u = TestHelper.owner_user
-    job_hashes = TestHelper.load_fixture("is_due")
-    job_hashes.each do |jh|
-      job_path = "#{u.runner.path}/#{jh['name']}"
-      j = Mobilize::Job.find_or_create_by_path(job_path)
-      #update job params
-      j.update_from_hash(jh)
-      #apply the completed_at, failed at, and parent attributes where appropriate
-      if jh['completed_at']
-        j.stages.last.update_attributes(:completed_at=>eval(jh['completed_at']))
-      end
-      if jh['failed_at']
-        j.stages.last.update_attributes(:failed_at=>eval(jh['failed_at']))
-      end
-      if jh['parent']
-        j.parent.stages.last.update_attributes(:completed_at=>eval(jh['parent']['completed_at'])) if jh['parent']['completed_at']
-        j.parent.stages.last.update_attributes(:failed_at=>eval(jh['parent']['failed_at'])) if jh['parent']['failed_at']
-      end
-      expected = jh['expected']
-      #check if is_due
-      assert expected == j.is_due?
-    end
-  end
-end