mobilize-base 1.0.0

data/lib/mobilize-base/handlers/gdriver.rb

@@ -0,0 +1,309 @@
+module Mobilize
+  module Gdriver
+    def Gdriver.config
+      Base.config('gdrive')[Base.env]
+    end
+
+    def Gdriver.owner_email
+      Gdriver.config['owner']['email']
+    end
+
+    def Gdriver.password(email)
+      if email == Gdriver.owner_email
+        Gdriver.config['owner']['pw']
+      else
+        worker = Gdriver.workers(email)
+        return worker['pw'] if worker
+      end
+    end
+
+    def Gdriver.admins
+      Gdriver.config['admins']
+    end
+
+    def Gdriver.workers(email=nil)
+      if email.nil?
+        Gdriver.config['workers']
+      else
+        Gdriver.workers.select{|w| w['email'] == email}.first
+      end
+    end
+
+    def Gdriver.worker_emails
+      Gdriver.workers.map{|w| w['email']}
+    end
+
+    def Gdriver.admin_emails
+      Gdriver.admins.map{|w| w['email']}
+    end
+
+    #email management - used to make sure not too many emails get used at the same time
+    def Gdriver.get_worker_email_by_mongo_id(mongo_id)
+      active_emails = Mobilize::Resque.jobs('working').map{|j| j['email'] if j['email']}.compact
+      Gdriver.workers.sort_by{rand}.each do |w|
+        if !(active_emails.include?(w['email']))
+          Mobilize::Resque.update_job_email(mongo_id,w['email'])
+          return w['email']
+        end
+      end
+      #return false if none are available
+      return false
+    end
+
+    def Gdriver.root(email=nil)
+      email ||= Gdriver.owner_email
+      pw = Gdriver.password(email)
+      GoogleDrive.login(email,pw)
+    end
+
+    def Gdriver.files(email=nil,params={})
+      root = Gdriver.root(email)
+      root.files(params)
+    end
+
+    def Gdriver.books(email=nil,params={})
+      Gdriver.files(email,params).select{|f| f.class==GoogleDrive::Spreadsheet}
+    end
+  end
+
+  class Gfiler
+    def Gfiler.find_by_title(title,email=nil)
+      Gdriver.files(email).select{|f| f.title==title}.first
+    end
+
+    def Gfiler.find_by_dst_id(dst_id,email=nil)
+      dst = Dataset.find(dst_id)
+      Gfiler.find_by_title(dst.path,email)
+    end
+
+    def Gfiler.add_admin_acl_by_dst_id(dst_id)
+      #adds admins and workers as writers
+      file = Gfiler.find_by_dst_id(dst_id)
+      file.add_admin_acl
+      return true
+    end
+
+    def Gfiler.add_admin_acl_by_title(title)
+      file = Gfiler.find_by_title(title)
+      file.add_admin_acl
+      return true
+    end
+
+    def Gfiler.add_worker_acl_by_title(title)
+      file = Gfiler.find_by_title(title)
+      file.add_worker_acl
+      return true
+    end
+
+    def Gfiler.update_acl_by_dst_id(dst_id,email,role="writer",edit_email=nil)
+      dst = Dataset.find(dst_id)
+      Gfiler.update_acl_by_title(dst.path,email,role,edit_email)
+    end
+
+    def Gfiler.update_acl_by_title(title,email,role="writer",edit_email=nil)
+      file = Gfiler.find_by_title(title,edit_email)
+      raise "File #{title} not found" unless file
+      file.update_acl(email,role)
+    end
+  end
+
+  module Gbooker
+    def Gbooker.find_all_by_title(title,email=nil)
+      Gdriver.books(email,{"title"=>title,"title-exact"=>"true"})
+    end
+    def Gbooker.find_or_create_by_title(title,email)
+      books = Gdriver.books(email,{"title"=>title,"title-exact"=>"true"})
+      #there should only be one book with each title, otherwise we have fail
+      book = nil
+      if books.length>1
+        #some idiot process created a duplicate book.
+        #Fix by renaming all but one with dst entry's key
+        dst = Dataset.find_by_handler_and_name('gbooker',title)
+        dkey = dst.url.split("key=").last
+        books.each do |b|
+          bkey = b.resource_id.split(":").last
+          if bkey == dkey
+            book = b
+          else
+            #delete the invalid book
+            b.delete
+            ("Deleted duplicate book #{title}").oputs
+          end
+        end
+      else
+        book = books.first
+      end
+      if book.nil?
+        #add book using owner email
+        #http
+        book = Gdriver.root.create_spreadsheet(title)
+        ("Created book #{title} at #{Time.now.utc.to_s}").oputs
+      end
+      #delete Sheet1 if there are other sheets
+      #http
+      if (sheets = book.worksheets).length>1
+        sheet1 = sheets.select{|s| s.title == "Sheet1"}.first
+        #http
+        sheet1.delete if sheet1
+      end
+      #always make sure books have admin acl
+      book.add_admin_acl
+      return book
+    end
+
+    def Gbooker.find_or_create_by_dst_id(dst_id,email=nil)
+      #creates by title, updates acl, updates dataset with url
+      dst = Dataset.find(dst_id)
+      r = Requestor.find(dst.requestor_id)
+      book = nil
+      #http
+      book = Gdriver.root.spreadsheet_by_url(dst.url) if dst.url
+      #manually try 5 times to validate sheet since we can't just try again and again
+      5.times.each do
+        begin
+          book.resource_id
+          #if no error then break loop
+          break
+        rescue=>exc
+          if book.nil? or exc.to_s.index('Invalid document id')
+            book = Gbooker.find_or_create_by_title(dst.name,email)
+            #if invalid doc then update url w new book and break loop
+            dst.update_attributes(:url=>book.human_url)
+            break
+          end
+        end
+      end
+      #add requestor write access
+      book.update_acl(r.email)
+      return book
+    end
+  end
+
+  module Gsheeter
+
+    def Gsheeter.max_cells
+      400000
+    end
+
+    def Gsheeter.read(name,email=nil)
+      sheet = Gsheeter.find_or_create_by_name(name,email)
+      sheet.to_tsv
+    end
+
+    def Gsheeter.write(name,tsv,email=nil)
+      sheet = Gsheeter.find_or_create_by_name(name,email)
+      sheet.write(tsv)
+    end
+
+    def Gsheeter.find_all_by_name(name,email)
+      book_title,sheet_title = name.split("/")
+      books = Gdriver.books(email,{"title"=>book_title,"title-exact"=>"true"})
+      sheets = books.map{|b| b.worksheets}.flatten.select{|w| w.title == sheet_title }
+      sheets
+    end
+
+    def Gsheeter.find_or_create_by_name(name,email=nil,rows=100,cols=20)
+      book_title,sheet_title = name.split("/")
+      book = Gbooker.find_or_create_by_title(book_title,email)
+      #http
+      sheet = book.worksheets.select{|w| w.title==sheet_title}.first
+      if sheet.nil?
+        #http
+        sheet = book.add_worksheet(sheet_title,rows,cols)
+        ("Created sheet #{name} at #{Time.now.utc.to_s}").oputs
+      end
+      return sheet
+    end
+
+    def Gsheeter.find_or_create_by_dst_id(dst_id,email=nil)
+      #creates by title, updates acl, updates dataset with url
+      dst = Dataset.find(dst_id)
+      r = Requestor.find(dst.requestor_id)
+      name = dst.name
+      book_title,sheet_title = name.split("/")
+      #make sure book exists and is assigned to this user
+      r.find_or_create_gbook_by_title(book_title,email)
+      #add admin write access
+      sheet = Gsheeter.find_or_create_by_name(name)
+      sheet_title = nil
+      return sheet
+    end
+
+    def Gsheeter.read_by_job_id(job_id)
+      j = Job.find(job_id)
+      r = j.requestor
+      #reserve email account for read
+      email = Gdriver.get_worker_email_by_mongo_id(job_id)
+      return false unless email
+      source = j.param_source
+      book,sheet = source.split("/")
+      #assume jobspec source if none given
+      source = [r.jobspec_title,source].join("/") if sheet.nil?
+      tsv = Gsheeter.find_or_create_by_name(source,email).to_tsv
+      book = nil
+      return tsv
+    end
+
+    def Gsheeter.read_by_dst_id(dst_id,email=nil)
+      dst = Dataset.find(dst_id)
+      name = dst.name
+      sheet = Gsheeter.find_or_create_by_name(name,email)
+      output = sheet.to_tsv
+      return output
+    end
+
+    def Gsheeter.write_by_dst_id(dst_id,tsv,email=nil)
+      dst = Dataset.find(dst_id)
+      #see if this is a specific cell
+      name = dst.name
+      return false unless email
+      #create temp tab, write data to it, checksum it against the source
+      tempsheet = Gsheeter.find_or_create_by_name("#{name}_temp")
+      tempsheet.write(tsv)
+      #delete current sheet, replace it with temp one
+      sheet = Gsheeter.find_or_create_by_name(dst.name)
+      title = sheet.title
+      #http
+      sheet.delete
+      begin
+        tempsheet.rename(title)
+      rescue
+        #need this because sometimes it gets confused and tries to rename twice
+      end
+      "Write successful for #{write_name}".oputs
+      return true
+    end
+
+    def Gsheeter.write_by_job_id(job_id)
+      j = Job.find(job_id)
+      r = j.requestor
+      dest_name = if j.destination.split("/").length==1
+                    "#{r.jobspec_title}#{"/"}#{j.destination}"
+                  else
+                    j.destination
+                  end
+      sheet_dst = Dataset.find_or_create_by_handler_and_name('gsheeter',dest_name)
+      sheet_dst.update_attributes(:requestor_id=>r.id.to_s) if sheet_dst.requestor_id.nil?
+      email = Gdriver.get_worker_email_by_mongo_id(job_id)
+      #return false if there are no emails available
+      return false unless email
+      #create temp tab, write data to it, checksum it against the source
+      tempsheet_dst = Dataset.find_or_create_by_handler_and_name('gsheeter',"#{dest_name}_temp")
+      tempsheet_dst.update_attributes(:requestor_id=>r.id.to_s) if tempsheet_dst.requestor_id.nil?
+      tempsheet = Gsheeter.find_or_create_by_dst_id(tempsheet_dst.id.to_s)
+      #tsv is the second to last stage's output (the last is the write)
+      tsv = Dataset.find(j.tasks[j.prior_task]['output_dst_id']).read
+      tempsheet.write(tsv,true,job_id)
+      #delete current sheet, replace it with temp one
+      sheet = Gsheeter.find_or_create_by_name(dest_name,email)
+      title = sheet.title
+      #http
+      sheet.delete
+      tempsheet.title = title
+      tempsheet.save
+      sheet_dst.update_attributes(:url=>tempsheet.spreadsheet.human_url)
+      "Write successful for #{dest_name}".oputs
+      return true
+    end
+  end
+end

data/lib/mobilize-base/handlers/mongoer.rb

@@ -0,0 +1,32 @@
+module Mobilize
+  class Mongoer
+
+    def Mongoer.grid
+      session = ::Mongoid.configure.sessions['default']
+      database_name = session['database']
+      host,port = session['hosts'].first.split(":")
+      return ::Mongo::GridFileSystem.new(::Mongo::Connection.new(host,port).db(database_name))
+    end
+
+    def Mongoer.read_by_filename(filename)
+      begin
+        zs=Mongoer.grid.open(filename,'r').read
+        return ::Zlib::Inflate.inflate(zs)
+      rescue
+        "failed Mongo read for filename #{filename}".oputs
+        return nil
+      end
+    end
+
+    def Mongoer.write_by_filename(filename,string)
+      zs = ::Zlib::Deflate.deflate(string)
+      Mongoer.grid.open(filename,'w',:delete_old => true){|f| f.write(zs)}
+      return true
+    end
+
+    def Mongoer.delete_by_filename(filename)
+      Mongoer.grid.delete(filename)
+      return true
+    end
+  end
+end

data/lib/mobilize-base/jobtracker.rb

@@ -0,0 +1,208 @@
+module Mobilize
+  module Jobtracker
+    def Jobtracker.config
+      Base.config('jobtracker')[Base.env]
+    end
+
+    #modify this to increase the frequency of request cycles
+    def Jobtracker.cycle_freq
+      Jobtracker.config['cycle_freq']
+    end
+
+    #frequency of notifications
+    def Jobtracker.notification_freq
+      Jobtracker.config['notification_freq']
+    end
+
+    def Jobtracker.requestor_refresh_freq
+      Jobtracker.config['requestor_refresh_freq']
+    end
+
+    #long running tolerance
+    def Jobtracker.max_run_time
+      Jobtracker.config['max_run_time']
+    end
+
+    def Jobtracker.admins
+      Jobtracker.config['admins']
+    end
+
+    def Jobtracker.admin_emails
+      Jobtracker.admins.map{|a| a['email']}
+    end
+
+    def Jobtracker.worker
+      Resque.find_worker_by_mongo_id("jobtracker")
+    end
+
+    def Jobtracker.workers(state="all")
+      Resque.workers(state)
+    end
+
+    def Jobtracker.status
+      args = Jobtracker.get_args
+      return args['status'] if args
+      job = Resque.jobs.select{|j| j['args'].first=='jobtracker'}.first
+      return 'queued' if job
+      return 'stopped'
+    end
+
+    def Jobtracker.update_status(msg)
+      #Jobtracker has no persistent database state
+      Resque.update_job_status("jobtracker",msg)
+      return true
+    end
+
+    def Jobtracker.restart
+      Jobtracker.stop!
+      Jobtracker.start
+    end
+
+    def Jobtracker.set_args(args)
+      Resque.set_worker_args(Jobtracker.worker,args)
+      return true
+    end
+
+    def Jobtracker.get_args
+      Resque.get_worker_args(Jobtracker.worker)
+    end
+
+    def Jobtracker.kill_workers
+      Resque.kill_workers
+    end
+
+    def Jobtracker.kill_idle_workers
+      Resque.kill_idle_workers
+    end
+
+    def Jobtracker.prep_workers
+      Resque.prep_workers
+    end
+
+    def Jobtracker.failures
+      Resque.failures
+    end
+
+    def Jobtracker.start
+      if Jobtracker.status!='stopped'
+        raise "Jobtracker still #{Jobtracker.status}"
+      else
+        #make sure that workers are running and at the right number
+        #Resque.prep_workers
+        #queue up the jobtracker (starts the perform method)
+        Jobtracker.enqueue!
+      end
+      return true
+    end
+
+    def Jobtracker.enqueue!
+      ::Resque::Job.create(Resque.queue_name, Jobtracker, 'jobtracker',{'status'=>'working'})
+    end
+
+    def Jobtracker.restart!
+      Jobtracker.stop!
+      Jobtracker.start
+      return true
+    end
+
+    def Jobtracker.restart_workers!
+      Jobtracker.kill_workers
+      sleep 5
+      Jobtracker.prep_workers
+    end
+
+    def Jobtracker.stop!
+      #send signal for Jobtracker to check for
+      Jobtracker.update_status('stopping')
+      sleep 5
+      i=0
+      while Jobtracker.status=='stopping'
+        puts "#{Jobtracker.to_s} still on queue, waiting"
+        sleep 5
+        i+=1
+      end
+      return true
+    end
+
+    def Jobtracker.last_notification
+      return Jobtracker.get_args["last_notification"] if Jobtracker.get_args
+    end
+
+    def Jobtracker.last_notification=(time)
+      Jobtracker.set_args({"last_notification"=>time})
+    end
+
+    def Jobtracker.notif_due?
+      last_duetime = Time.now.utc - Jobtracker.notification_freq
+      return (Jobtracker.last_notification.to_s.length==0 || Jobtracker.last_notification.to_datetime < last_duetime)
+    end
+
+    def Jobtracker.max_run_time_workers
+      #return workers who have been cranking away for 6+ hours
+        workers = Jobtracker.workers('working').select do |w|
+            w.job['runat'].to_s.length>0 and 
+              (Time.now.utc - Time.parse(w.job['runat'])) > Jobtracker.max_run_time
+        end
+        return workers
+    end
+
+    def Jobtracker.start_worker(count=nil)
+      Resque.start_workers(count)
+    end
+
+    def Jobtracker.kill_workers(count=nil)
+      Resque.kill_workers(count)
+    end
+
+    def Jobtracker.set_test_env
+      ENV['MOBILIZE_ENV']='test'
+      ::Resque.redis="localhost:9736"
+      mongoid_config_path = "#{Mobilize::Base.root}/config/mongoid.yml"
+      Mongoid.load!(mongoid_config_path, Mobilize::Base.env)
+    end
+
+    def Jobtracker.run_notifications
+      if Jobtracker.notif_due?
+        notifs = []
+        if Jobtracker.failures.length>0
+          n = {}
+          jfcs = Resque.failure_report
+          n['subj'] = "#{jfcs.keys.length.to_s} failed jobs, #{jfcs.values.map{|v| v.values}.flatten.sum.to_s} failures"
+          #one row per exception type, with the job name
+          n['body'] = jfcs.map{|key,val| val.map{|b,name| [key," : ",b,", ",name," times"].join}}.flatten.join("\n\n")
+          notifs << n
+        end
+        lws = Jobtracker.max_run_time_workers
+        if lws.length>0
+          n = {}
+          n['subj'] = "#{lws.length.to_s} max run time jobs"
+          n['body'] = lws.map{|w| %{spec:#{w['spec']} stg:#{w['stg']} runat:#{w['runat'].to_s}}}.join("\n\n")
+          notifs << n
+        end
+        notifs.each do |notif|
+          Emailer.write(n['subj'],notif['body']).deliver
+          Jobtracker.last_notification=Time.now.utc.to_s
+          "Sent notification at #{Jobtracker.last_notification}".oputs
+        end
+      end
+      return true
+    end
+
+    def Jobtracker.perform(id,*args)
+      while Jobtracker.status != 'stopping'
+        requestors = Requestor.all
+        Jobtracker.run_notifications
+        requestors.each do |r|
+          Jobtracker.update_status("Running requestor #{r.name}")
+          if r.is_due?
+            r.enqueue!
+            Jobtracker.update_status("Enqueued requestor #{r.name}")
+          end
+        end
+        sleep 5
+      end
+      Jobtracker.update_status("told to stop")
+      return true
+    end
+  end
+end

data/lib/mobilize-base/models/dataset.rb

@@ -0,0 +1,70 @@
+module Mobilize
+  class Dataset
+    include Mongoid::Document
+    include Mongoid::Timestamps
+    field :requestor_id, type: String
+    field :handler, type: String
+    field :name, type: String
+    field :url, type: String
+    field :size, type: Fixnum
+    field :last_cached_at, type: Time
+    field :last_read_at, type: Time
+    field :cache_expire_at, type: Time
+
+    index({ requestor_id: 1})
+    index({ handler: 1})
+    index({ name: 1})
+
+    before_destroy :destroy_cache
+
+    def read
+      dst = self
+      if dst.last_cached_at and (dst.cache_expire_at.nil? or dst.cache_expire_at > Time.now.utc)
+        return dst.read_cache
+      else
+        return dst.handler.humanize.constantize.read_by_dst_id(dst.id.to_s)
+      end
+    end
+
+    def Dataset.find_by_handler_and_name(handler,name)
+      Dataset.where(handler: handler, name: name).first
+    end
+
+    def Dataset.find_or_create_by_handler_and_name(handler,name)
+      dst = Dataset.where(handler: handler, name: name).first
+      dst = Dataset.create(handler: handler, name: name) unless dst
+      return dst
+    end
+
+    def Dataset.find_or_create_by_requestor_id_and_handler_and_name(requestor_id,handler,name)
+      dst = Dataset.where(requestor_id: requestor_id, handler: handler, name: name).first
+      dst = Dataset.create(requestor_id: requestor_id, handler: handler, name: name) unless dst
+      return dst
+    end
+
+    def write(data)
+      dst = self
+      dst.handler.humanize.constantize.write_by_dst_id(dst.id.to_s,data)
+      dst.save!
+      return true
+    end
+
+    def read_cache
+      dst = self
+      dst.update_attributes(:last_read_at=>Time.now.utc)
+      return Mongoer.read_by_filename(dst.id.to_s)
+    end
+
+    def write_cache(string,expire_at=nil)
+      dst = self
+      Mongoer.write_by_filename(dst.id.to_s,string)
+      dst.update_attributes(:last_cached_at=>Time.now.utc,:cache_expire_at=>expire_at,:size=>string.length)
+      return true
+    end
+
+    def delete_cache
+      return Mongoer.delete_by_filename(dst.id.to_s)
+    end
+
+  end
+end