mobilize-base 1.0.2 → 1.0.4

data/mobilize-base.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |s|
   s.homepage    = ""
   s.summary     = %q{Moves datasets and schedules data transfers using MongoDB, Resque and Google Docs}
   s.description = %q{Manage your organization's workflows entirely through Google Docs and irb.
-                     Mobilize schedules jobs, queues workers, sends failure notifications, and 
+                     Mobilize schedules jobs, queues workers, sends failure notifications, and
                      integrates mobilize-hadoop, -http, -mysql, and -mongodb packages
                      to allow seamless transport of TSV and JSON data between any two endpoints. }
 

data/test/base1_task1.yml

@@ -0,0 +1,3 @@
+- {test_header: t1, test_header2: t1, test_header3: t1}
+- {test_header: t2, test_header2: t2, test_header3: t2}
+- {test_header: t3, test_header2: t3, test_header3: t3}

data/test/base_job_rows.yml

@@ -0,0 +1,13 @@
+- name: "base1"
+  active: true
+  trigger: once
+  status: ""
+  task1: 'gsheet.read "Runner_mobilize(test)/base1_task1.in"'
+  task2: 'gsheet.write "base1/task1", "Runner_mobilize(test)/base1.out"'
+
+- name: "base2"
+  active: true
+  trigger: "after base1"
+  status: ""
+  task1: 'gsheet.read "Runner_mobilize(test)/base1.out"'
+  task2: 'gsheet.write "task1", "Runner_mobilize(test)/base2.out"'

data/test/mobilize-base_test.rb

@@ -0,0 +1,59 @@
+require 'test_helper'
+
+describe "Mobilize" do
+
+  def before
+    puts 'nothing before'
+  end
+
+  # enqueues 4 workers on Resque
+  it "runs integration test" do
+
+    puts "restart test redis"
+    Mobilize::Jobtracker.restart_test_redis
+
+    puts "clear out test db"
+    Mobilize::Jobtracker.drop_test_db
+
+    puts "restart workers"
+    Mobilize::Jobtracker.restart_workers!
+
+    puts "build test runner"
+    gdrive_slot = Mobilize::Gdrive.owner_email
+    puts "create user 'mobilize'"
+    user_name = gdrive_slot.split("@").first
+    u = Mobilize::User.find_or_create_by_name(user_name)
+    assert u.email == gdrive_slot
+
+    Mobilize::Jobtracker.build_test_runner(user_name)
+    assert Mobilize::Jobtracker.workers.length == Mobilize::Resque.config['max_workers'].to_i
+
+    puts "Jobtracker created runner with 'jobs' sheet?"
+    r = u.runner
+    jobs_sheet = r.gsheet(gdrive_slot)
+    tsv = jobs_sheet.to_tsv
+    assert tsv.length == 56 #headers only
+
+    puts "add base1_task1 input sheet"
+    test_source_sheet = Mobilize::Gsheet.find_or_create_by_path("#{r.path.split("/")[0..-2].join("/")}/base1_task1.in",gdrive_slot)
+
+    test_source_ha = ::YAML.load_file("#{Mobilize::Base.root}/test/base1_task1.yml")*40
+    test_source_tsv = test_source_ha.hash_array_to_tsv
+    test_source_sheet.write(test_source_tsv)
+
+    puts "add row to jobs sheet, wait 120s"
+    test_job_rows = ::YAML.load_file("#{Mobilize::Base.root}/test/base_job_rows.yml")
+    jobs_sheet.add_or_update_rows(test_job_rows)
+
+    puts "job row added, force enqueued runner"
+    r.enqueue!
+    sleep 120
+
+    puts "jobtracker posted test sheet data to test destination, and checksum succeeded?"
+    test_target_sheet = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/base1.out",gdrive_slot)
+
+    assert test_target_sheet.to_tsv == test_source_sheet.to_tsv
+
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mobilize-base
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.0.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-28 00:00:00.000000000 Z
+date: 2012-12-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -189,7 +189,7 @@ dependencies:
         version: 3.1.1
 description: ! "Manage your organization's workflows entirely through Google Docs
   and irb.\n                     Mobilize schedules jobs, queues workers, sends failure
-  notifications, and \n                     integrates mobilize-hadoop, -http, -mysql,
+  notifications, and\n                     integrates mobilize-hadoop, -http, -mysql,
   and -mongodb packages\n                     to allow seamless transport of TSV and
   JSON data between any two endpoints. "
 email:
@@ -205,7 +205,10 @@ files:
 - Rakefile
 - lib/mobilize-base.rb
 - lib/mobilize-base/extensions/array.rb
-- lib/mobilize-base/extensions/google_drive.rb
+- lib/mobilize-base/extensions/google_drive/acl.rb
+- lib/mobilize-base/extensions/google_drive/client_login_fetcher.rb
+- lib/mobilize-base/extensions/google_drive/file.rb
+- lib/mobilize-base/extensions/google_drive/worksheet.rb
 - lib/mobilize-base/extensions/hash.rb
 - lib/mobilize-base/extensions/object.rb
 - lib/mobilize-base/extensions/resque.rb
@@ -214,21 +217,27 @@ files:
 - lib/mobilize-base/handlers/gbook.rb
 - lib/mobilize-base/handlers/gdrive.rb
 - lib/mobilize-base/handlers/gfile.rb
+- lib/mobilize-base/handlers/gridfs.rb
 - lib/mobilize-base/handlers/gsheet.rb
-- lib/mobilize-base/handlers/mongodb.rb
 - lib/mobilize-base/jobtracker.rb
 - lib/mobilize-base/models/dataset.rb
 - lib/mobilize-base/models/job.rb
-- lib/mobilize-base/models/requestor.rb
-- lib/mobilize-base/tasks.rb
+- lib/mobilize-base/models/runner.rb
+- lib/mobilize-base/models/task.rb
+- lib/mobilize-base/models/user.rb
+- lib/mobilize-base/rakes.rb
 - lib/mobilize-base/tasks/mobilize-base.rake
 - lib/mobilize-base/version.rb
 - lib/samples/gdrive.yml
+- lib/samples/gridfs.yml
+- lib/samples/gsheet.yml
 - lib/samples/jobtracker.yml
 - lib/samples/mongoid.yml
 - lib/samples/resque.yml
 - mobilize-base.gemspec
-- test/mobilize_test.rb
+- test/base1_task1.yml
+- test/base_job_rows.yml
+- test/mobilize-base_test.rb
 - test/redis-test.conf
 - test/test_helper.rb
 homepage: ''
@@ -257,7 +266,9 @@ specification_version: 3
 summary: Moves datasets and schedules data transfers using MongoDB, Resque and Google
   Docs
 test_files:
-- test/mobilize_test.rb
+- test/base1_task1.yml
+- test/base_job_rows.yml
+- test/mobilize-base_test.rb
 - test/redis-test.conf
 - test/test_helper.rb
 has_rdoc: 

data/lib/mobilize-base/handlers/mongodb.rb

@@ -1,32 +0,0 @@
-module Mobilize
-  class Mongodb
-
-    def Mongodb.grid
-      session = ::Mongoid.configure.sessions['default']
-      database_name = session['database']
-      host,port = session['hosts'].first.split(":")
-      return ::Mongo::GridFileSystem.new(::Mongo::Connection.new(host,port).db(database_name))
-    end
-
-    def Mongodb.read_by_filename(filename)
-      begin
-        zs=Mongodb.grid.open(filename,'r').read
-        return ::Zlib::Inflate.inflate(zs)
-      rescue
-        "failed Mongo read for filename #{filename}".oputs
-        return nil
-      end
-    end
-
-    def Mongodb.write_by_filename(filename,string)
-      zs = ::Zlib::Deflate.deflate(string)
-      Mongodb.grid.open(filename,'w',:delete_old => true){|f| f.write(zs)}
-      return true
-    end
-
-    def Mongodb.delete_by_filename(filename)
-      Mongodb.grid.delete(filename)
-      return true
-    end
-  end
-end

data/lib/mobilize-base/models/requestor.rb

@@ -1,232 +0,0 @@
-module Mobilize
-  class Requestor
-    include Mongoid::Document
-    include Mongoid::Timestamps
-    field :email, type: String
-    field :oauth, type: String
-    field :name, type: String
-    field :first_name, type: String
-    field :last_name, type: String
-    field :admin_role, type: String
-    field :last_run, type: Time
-    field :status, type: String
-
-    validates_presence_of :name, :message => ' cannot be blank.'
-    validates_uniqueness_of :name, :message => ' has already been used.'
-
-    before_destroy :destroy_jobs
-
-    def Requestor.find_or_create_by_name(name)
-      r = Requestor.where(:name => name).first
-      r = Requestor.create(:name => name) unless r
-      return r
-    end
-
-    def Requestor.find_or_create_by_email(email)
-      r = Requestor.where(:email => email).first
-      r = Requestor.create(:email => email) unless r
-      user_name = email.split("@").first
-      r.update_attributes(:name => user_name) unless r.name.to_s.length>0
-      return r
-    end
-
-    def Requestor.jobs_sheet_headers
-      %w{name active schedule status last_error destination_url tasks datasets params destination}
-    end
-
-    def Requestor.perform(id,*args)
-      r = Requestor.find(id.to_s)
-      #reserve email account for read
-      gdrive_email = Gdrive.get_worker_email_by_mongo_id(id)
-      unless gdrive_email
-        "no gdrive_email available for #{r.name}".oputs
-        return false
-      end
-      jobs_sheet = r.jobs_sheet(gdrive_email)
-      #write headers to sheet
-      Requestor.jobs_sheet_headers.each_with_index do |h,h_i|
-        jobs_sheet[1,h_i+1] = h
-      end
-      jobs_sheet.save
-      #read the jobs sheet 
-      #record jobs in DB
-      #deactivate jobs not in sheet
-      r.read_jobs(gdrive_email)
-      #queue up the jobs that are due and active
-      r.jobs.each do |j|
-        begin
-          if j.active and j.is_due?
-            #cache all datasets
-            j.dataset_array.each do |dst|
-             #read tsv, write to cache for job to use
-              tsv = Gsheet.find_or_create_by_name(dst.name,gdrive_email).to_tsv
-              r.update_status("caching #{dst.name}")
-              dst.write_cache(tsv)
-            end
-            j.enqueue!
-          end
-        rescue ScriptError,StandardError => exc
-          #update errors
-          j.update_attributes(:last_error=>exc.to_s,:last_trace=>exc.backtrace.to_s)
-        end
-      end
-      #write any updates to status, error, datasource_url etc.
-      r.write_jobs(gdrive_email)
-      r.update_attributes(:last_run=>Time.now.utc)
-    end
-
-    def jobs_sheet(gdrive_email)#gdrive_email to read with
-      r = self
-      r.find_or_create_gbook_by_title(r.jobspec_title,gdrive_email)
-      jobs_name = [r.jobspec_title,"Jobs"].join("/")
-      r.find_or_create_gsheet_by_name(jobs_name,gdrive_email)
-    end
-
-    def read_jobs(gdrive_email)
-      r = self
-      jobs_sheet = r.jobs_sheet(gdrive_email)
-      rem_jobs = jobs_sheet.to_tsv.tsv_to_hash_array
-      #go through each job, update relevant job with its params
-      loc_jobs = []
-      rem_jobs.each_with_index do |rj,rj_i|
-        #skip bad rows
-        next if (rj['name'].to_s.first == "#" or ['name','schedule','tasks','active'].select{|c| rj[c].to_s.strip==""}.length>0)
-        j = Job.find_or_create_by_requestor_id_and_name(r.id.to_s,rj['name'])
-        #update top line params
-        j.update_attributes(:active => rj['active'],
-                            :schedule => rj['schedule'],
-                            :tasks => rj['tasks'],
-                            :datasets => rj['datasets'],
-                            :params => rj['params'],
-                            :destination => rj['destination'])
-        #update laststatus with "Created job for" if job is due
-        j.update_status("Due and active at #{Time.now.utc}") if j.is_due? and j.active
-        #add this job to list of local ones
-        loc_jobs << j
-      end
-      #deactivate requestor jobs that are not included in sheet;
-      #this makes sure we don't run obsolete jobs
-      (r.jobs.map{|j| j.id.to_s} - loc_jobs.map{|j| j.id.to_s}).each do |rjid|
-        j = Job.find(rjid)
-        if j.active
-          j.update_attributes(:active=>false)
-          r.update_status("Deactivated job:#{r.name}=>#{j.name}")
-        end
-      end
-      r.update_status(r.name + " jobs read at #{Time.now.utc}")
-      return true
-    end
-
-    def write_jobs(gdrive_email) #gdrive_email to update with
-      r = self
-      jobs_sheet = r.jobs_sheet(gdrive_email)
-      rem_jobs = jobs_sheet.to_tsv.tsv_to_hash_array
-      #go through each job, update relevant job with its params
-      headers = Requestor.jobs_sheet_headers
-      #write headers
-      jobs_sheet.add_headers(headers)
-      #write rows
-      rem_jobs.each_with_index do |rj,rj_i|
-        #skip bad rows
-        next if (rj['name'].to_s.first == "#" or ['name','schedule','tasks','active'].select{|c| rj[c].to_s.strip==""}.length>0)
-        if j = r.jobs(rj['name'])
-          #update active to false if this was a run once
-          j.update_attributes(:active=>false) if j.schedule.to_s == 'once'
-          jobs_sheet[rj_i+2,headers.index('active')+1] = j.active.to_s
-          jobs_sheet[rj_i+2,headers.index('status')+1] = j.status.to_s.gsub("\n",";").gsub("\t"," ")
-          jobs_sheet[rj_i+2,headers.index('last_error')+1] = j.last_error.to_s.gsub("\n",";").gsub("\t"," ")
-          jobs_sheet[rj_i+2,headers.index('destination_url')+1] = j.destination_url.to_s
-        end
-      end
-      jobs_sheet.save
-      r.update_status(r.name + " jobs written")
-      return true
-    end
-
-    def jobspec_title
-      r = self
-      prefix = "Jobspec_"
-      suffix = ""
-      if Mobilize::Base.env == 'development'
-        suffix = "_dev"
-      elsif Mobilize::Base.env == 'test' or Mobilize::Base.env == 'pry_dev'
-        suffix = "_test"
-      elsif Mobilize::Base.env == 'production' or Mobilize::Base.env == 'integration'
-        suffix = ""
-      else
-        raise "Invalid environment"
-      end
-      title = prefix + r.name + suffix
-      return title
-    end
-
-    #Google doc helper methods
-
-    def find_or_create_gbook_by_title(title,gdrive_email)
-      r = self
-      book_dst = Dataset.find_or_create_by_handler_and_name('gbook',title)
-      #give dst this requestor if none
-      book_dst.update_attributes(:requestor_id=>r.id.to_s) if book_dst.requestor_id.nil?
-      book = Gbook.find_or_create_by_dst_id(book_dst.id.to_s,gdrive_email)
-      return book
-    end
-
-    def find_or_create_gsheet_by_name(name,gdrive_email)
-      r = self
-      sheet_dst = Dataset.find_or_create_by_handler_and_name('gsheet',name)
-      sheet_dst.update_attributes(:requestor_id=>r.id.to_s) if sheet_dst.requestor_id.nil?
-      sheet = Gsheet.find_or_create_by_dst_id(sheet_dst.id.to_s,gdrive_email)
-      return sheet
-    end
-
-    def jobs(jname=nil)
-      r = self
-      js = Job.find_all_by_requestor_id(r.id.to_s)
-      if jname
-        return js.sel{|j| j.name == jname}.first
-      else
-        return js
-      end
-    end
-
-    def destroy_jobs
-      r = self
-      r.jobs.each{|s| s.delete}
-    end
-
-    def gsheets
-      r = self
-      Dataset.find_all_by_handler_and_requestor_id('gsheet',r.id.to_s)
-    end
-
-    def worker
-      r = self
-      Mobilize::Resque.find_worker_by_mongo_id(r.id.to_s)
-    end
-
-    def update_status(msg)
-      r = self
-      r.update_attributes(:status=>msg)
-      Mobilize::Resque.update_job_status(r.id.to_s,msg)
-      return true
-    end
-
-    def is_working?
-      r = self
-      Mobilize::Resque.active_mongo_ids.include?(r.id.to_s)
-    end
-
-    def is_due?
-      r = self.reload
-      return false if r.is_working?
-      last_due_time = Time.now.utc - Jobtracker.requestor_refresh_freq
-      return true if r.last_run.nil? or r.last_run < last_due_time
-    end
-
-    def enqueue!
-      r = self
-      ::Resque::Job.create("mobilize",Requestor,r.id.to_s,{"name"=>r.name})
-      return true
-    end
-  end
-end

data/lib/mobilize-base/tasks.rb

@@ -1,43 +0,0 @@
-# require 'resque/tasks'
-# will give you the resque tasks
-
-namespace :mobilize do
-
-  desc "Start a Resque worker"
-  task :work do
-    require 'resque'
-    require 'mobilize-base'
-
-    begin
-      worker = Resque::Worker.new(Mobilize::Resque.config['queue_name'])
-    rescue Resque::NoQueueError
-      abort "set QUEUE env var, e.g. $ QUEUE=critical,high rake resque:work"
-    end
-
-    puts "Starting worker #{worker}"
-
-    worker.work(ENV['INTERVAL'] || 5) # interval, will block
-  end
-
-  desc "Set up config and log folders and files"
-  task :setup do
-    sample_dir = File.dirname(__FILE__) + '/../samples/'
-    sample_files = Dir.entries(sample_dir)
-    config_dir = "#{ENV['PWD']}/config/"
-    log_dir = "#{ENV['PWD']}/log/"
-    unless File.exists?(config_dir)
-      puts "creating config dir"
-      `mkdir #{config_dir}`
-    end
-    unless File.exists?(log_dir)
-      puts "creating log dir"
-      `mkdir #{log_dir}`
-    end
-    sample_files.each do |fname|
-      unless File.exists?("#{config_dir}#{fname}")
-        puts "creating config/#{fname}"
-        `cp #{sample_dir}#{fname} #{config_dir}#{fname}`
-      end
-    end
-  end
-end