mobilize-base 1.36 → 1.293

data/lib/mobilize-base/handlers/gridfs.rb

@@ -1,38 +1,43 @@
-require 'tempfile'
 module Mobilize
   module Gridfs
     def Gridfs.config
       Base.config('gridfs')
     end
 
-    def Gridfs.read_by_dataset_path(dst_path,*args)
-      curr_file = Mongoid::GridFs::Fs::File.where(:filename=>dst_path).first
-      zs = curr_file.data if curr_file
-      return ::Zlib::Inflate.inflate(zs) if zs.to_s.length>0
+    def Gridfs.grid
+      session = ::Mongoid.configure.sessions['default']
+      database_name = session['database']
+      host,port = session['hosts'].first.split(":")
+      return ::Mongo::GridFileSystem.new(::Mongo::Connection.new(host,port).db(database_name))
     end
 
-    def Gridfs.write_by_dataset_path(dst_path,string,*args)
+    def Gridfs.read_by_dataset_path(dst_path,user_name,*args)
+      begin
+        zs=Gridfs.grid.open(dst_path,'r').read
+        return ::Zlib::Inflate.inflate(zs)
+      rescue
+        return nil
+      end
+    end
+
+    def Gridfs.write_by_dataset_path(dst_path,string,user_name,*args)
       zs = ::Zlib::Deflate.deflate(string)
       raise "compressed string too large for Gridfs write" if zs.length > Gridfs.config['max_compressed_write_size']
-      #find and delete existing file
-      curr_file = Mongoid::GridFs::Fs::File.where(:filename=>dst_path).first
-      curr_zs =  curr_file.data if curr_file
-      #overwrite when there is a change
+      curr_zs = Gridfs.read_by_dataset_path(dst_path,user_name).to_s
+      #write a new version when there is a change
       if curr_zs != zs
-        Mongoid::GridFs.delete(curr_file.id) if curr_file
-        #create temp file w zstring
-        temp_file = ::Tempfile.new("#{string}#{Time.now.to_f}".to_md5)
-        temp_file.print(zs)
-        temp_file.close
-        #put data in file
-        Mongoid::GridFs.put(temp_file.path,:filename=>dst_path)
+        Gridfs.grid.open(dst_path,'w',:versions => Gridfs.config['max_versions']){|f| f.write(zs)}
       end
       return true
     end
 
     def Gridfs.delete(dst_path)
-      curr_file = Mongoid::GridFs::Fs::File.where(:filename=>dst_path).first
-      curr_file.delete
+      begin
+        Gridfs.grid.delete(dst_path)
+        return true
+      rescue
+        return nil
+      end
     end
   end
 end

data/lib/mobilize-base/handlers/gsheet.rb

@@ -81,16 +81,15 @@ module Mobilize
 
     def Gsheet.write_temp(target_path,gdrive_slot,tsv)
       #find and delete temp sheet, if any
-      temp_book_title = target_path.gridsafe
-      #create book and sheet
-      temp_book = Gdrive.root(gdrive_slot).create_spreadsheet(temp_book_title)
-      rows, cols = tsv.split("\n").ie{|t| [t.length,t.first.split("\t").length]}
-      temp_sheet = temp_book.add_worksheet("temp",rows,cols)
+      temp_path = [target_path.gridsafe,"temp"].join("/")
+      temp_sheet = Gsheet.find_by_path(temp_path,gdrive_slot)
+      temp_sheet.delete if temp_sheet
+      #write data to temp sheet
+      temp_sheet = Gsheet.find_or_create_by_path(temp_path,gdrive_slot)
       #this step has a tendency to fail; if it does,
       #don't fail the stage, mark it as false
       begin
-        gdrive_user = gdrive_slot.split("@").first
-        temp_sheet.write(tsv,gdrive_user)
+        temp_sheet.write(tsv,Gdrive.owner_name)
       rescue
         return nil
       end
@@ -109,7 +108,7 @@ module Mobilize
         #only give the user edit permissions if they're the ones
         #creating it
         target_sheet = Gsheet.find_or_create_by_path(target_path,gdrive_slot)
-        target_sheet.spreadsheet.update_acl(u.email) unless target_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="owner"}
+        target_sheet.spreadsheet.update_acl(u.email,"writer") unless target_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="owner"}
         target_sheet.delete_sheet1
       end
       #pass it crop param to determine whether to shrink target sheet to fit data
@@ -127,43 +126,28 @@ module Mobilize
       s = Stage.where(:path=>stage_path).first
       u = s.job.runner.user
       crop = s.params['crop'] || true
-      retries = 0
-      stdout,stderr = []
-      while stdout.nil? and stderr.nil? and retries < Gdrive.max_file_write_retries
-        begin
-          #get tsv to write from stage
-          source = s.sources(gdrive_slot).first
-          raise "Need source for gsheet write" unless source
-          tsv = source.read(u.name,gdrive_slot)
-          raise "No data source found for #{source.url}" unless tsv
-          tsv_row_count = tsv.to_s.split("\n").length
-          tsv_col_count = tsv.to_s.split("\n").first.to_s.split("\t").length
-          tsv_cell_count = tsv_row_count * tsv_col_count
-          if tsv_cell_count > Gsheet.max_cells
-            raise "Too many datapoints; you have #{tsv_cell_count.to_s}, max is #{Gsheet.max_cells.to_s}"
-          end
-          stdout = if tsv_row_count == 0
-                     #soft error; no data to write. Stage will complete.
-                     "Write skipped for #{s.target.url}"
-                   else
-                     Dataset.write_by_url(s.target.url,tsv,u.name,gdrive_slot,crop)
-                     #update status
-                     "Write successful for #{s.target.url}"
-                   end
-          Gdrive.unslot_worker_by_path(stage_path)
-          stderr = nil
-          s.update_status(stdout)
-          signal = 0
-        rescue => exc
-          if retries < Gdrive.max_file_write_retries
-            retries +=1
-            sleep Gdrive.file_write_retry_delay
-          else
-            stdout = nil
-            stderr = [exc.to_s,"\n",exc.backtrace.join("\n")].join
-            signal = 500
-          end
-        end
+      begin
+        #get tsv to write from stage
+        source = s.sources(gdrive_slot).first
+        raise "Need source for gsheet write" unless source
+        tsv = source.read(u.name,gdrive_slot)
+        raise "No data source found for #{source.url}" unless tsv
+        stdout = if tsv.to_s.length == 0
+                   #soft error; no data to write. Stage will complete.
+                   "Write skipped for #{s.target.url}"
+                 else
+                   Dataset.write_by_url(s.target.url,tsv,u.name,gdrive_slot,crop)
+                   #update status
+                   "Write successful for #{s.target.url}"
+                 end
+        Gdrive.unslot_worker_by_path(stage_path)
+        stderr = nil
+        s.update_status(stdout)
+        signal = 0
+      rescue => exc
+        stdout = nil
+        stderr = [exc.to_s,"\n",exc.backtrace.join("\n")].join
+        signal = 500
       end
       return {'out_str'=>stdout, 'err_str'=>stderr, 'signal' => signal}
     end

data/lib/mobilize-base/handlers/resque.rb

@@ -19,17 +19,13 @@ module Mobilize
     def Resque.workers(state="all")
       workers = ::Resque.workers.select{|w| w.queues.first == Resque.queue_name}
       return workers if state == 'all'
-      working_workers = workers.select{|w| w.job['queue'] == Resque.queue_name}
+      working_workers = workers.select{|w| w.job['queue']== Resque.queue_name}
       return working_workers if state == 'working'
       idle_workers = workers.select{|w| w.job['queue'].nil?}
       return idle_workers if state == 'idle'
       stale_workers = workers.select{|w| Time.parse(w.started) < Jobtracker.deployed_at}
       return stale_workers if state == 'stale'
-      timeout_workers = workers.select do |w|
-                                         w.job['payload'] and
-                                         w.job['payload']['class']!='Jobtracker' and
-                                         w.job['run_at'] < (Time.now.utc - Jobtracker.max_run_time)
-                                       end
+      timeout_workers = workers.select{|w| w.job['payload'] and w.job['payload']['class']!='Jobtracker' and w.job['runat'] < (Time.now.utc - Jobtracker.max_run_time)}
       return timeout_workers if state == 'timeout'
       raise "invalid state #{state}"
     end
@@ -117,16 +113,10 @@ module Mobilize
         stage_path = f['payload']['args'].first
         email = begin
                   s = Stage.where(:path=>stage_path).first
-                  if s.params['notify'].to_s=="false"
-                    next
-                  elsif s.params['notify'].to_s.index("@")
-                    s.params['notify']
-                  else
-                    s.job.runner.user.email
-                  end
-                rescue ScriptError, StandardError
-                  #jobs without stages are sent to admins
-                  [Gdrive.admin_group_name,Gdrive.domain].join("@")
+                  s.job.runner.user.email
+                rescue
+                  #jobs without stages are sent to first admin
+                  Jobtracker.admin_emails.first
                 end
         exc_to_s = f['error']
         if fjobs[email].nil?
@@ -148,7 +138,10 @@ module Mobilize
 
     def Resque.start_workers(count=1)
       count.times do
-        "(cd #{Base.root};rake mobilize:work[#{Base.env}]) >> #{Resque.log_path} 2>&1 &".bash
+        dir_envs = "MOBILIZE_ENV=#{Base.env} " +
+                   "MOBILIZE_CONFIG_DIR=#{Base.config_dir} " +
+                   "MOBILIZE_LOG_DIR=#{Base.log_dir}"
+        "(cd #{Base.root};rake #{dir_envs} mobilize_base:work) >> #{Resque.log_path} 2>&1 &".bash
       end
     end
 

data/lib/mobilize-base/jobtracker.rb

@@ -1,13 +1,152 @@
 module Mobilize
   module Jobtracker
-    #adds convenience methods
-    require "#{File.dirname(__FILE__)}/helpers/jobtracker_helper"
+    def Jobtracker.config
+      Base.config('jobtracker')
+    end
+
+    #modify this to increase the frequency of request cycles
+    def Jobtracker.cycle_freq
+      Jobtracker.config['cycle_freq']
+    end
+
+    #frequency of notifications
+    def Jobtracker.notification_freq
+      Jobtracker.config['notification_freq']
+    end
+
+    def Jobtracker.runner_read_freq
+      Jobtracker.config['runner_read_freq']
+    end
+
+    #long running tolerance
+    def Jobtracker.max_run_time
+      Jobtracker.config['max_run_time']
+    end
+
+    def Jobtracker.admins
+      Jobtracker.config['admins']
+    end
+
+    def Jobtracker.admin_emails
+      Jobtracker.admins.map{|a| a['email'] }
+    end
+
+    def Jobtracker.worker
+      Resque.find_worker_by_path("jobtracker")
+    end
+
+    def Jobtracker.workers(state="all")
+      Resque.workers(state)
+    end
+
+    def Jobtracker.status
+      args = Jobtracker.get_args
+      return args['status'] if args
+      job = Resque.jobs.select{|j| j['args'].first=='jobtracker'}.first
+      return 'queued' if job
+      return 'stopped'
+    end
+
+    def Jobtracker.update_status(msg)
+      #Jobtracker has no persistent database state
+      Resque.set_worker_args_by_path("jobtracker",{'status'=>msg})
+      return true
+    end
+
+    def Jobtracker.restart
+      Jobtracker.stop!
+      Jobtracker.start
+    end
+
+    def Jobtracker.set_args(args)
+      Resque.set_worker_args(Jobtracker.worker,args)
+      return true
+    end
+
+    def Jobtracker.get_args
+      Resque.get_worker_args(Jobtracker.worker)
+    end
+
+    def Jobtracker.kill_workers
+      Resque.kill_workers
+    end
+
+    def Jobtracker.kill_idle_workers
+      Resque.kill_idle_workers
+    end
+
+    def Jobtracker.kill_idle_and_stale_workers
+      Resque.kill_idle_and_stale_workers
+    end
+
+    def Jobtracker.prep_workers
+      Resque.prep_workers
+    end
+
+    def Jobtracker.failures
+      Resque.failures
+    end
+
+    def Jobtracker.start
+      if Jobtracker.status!='stopped'
+        Jobtracker.update_status("Jobtracker still #{Jobtracker.status}")
+      else
+        #make sure that workers are running and at the right number
+        #Resque.prep_workers
+        #queue up the jobtracker (starts the perform method)
+        Jobtracker.enqueue!
+      end
+      return true
+    end
+
+    def Jobtracker.enqueue!
+      ::Resque::Job.create(Resque.queue_name, Jobtracker, 'jobtracker',{})
+    end
+
+    def Jobtracker.restart!
+      Jobtracker.stop!
+      Jobtracker.start
+      return true
+    end
+
+    def Jobtracker.restart_workers!
+      Jobtracker.kill_workers
+      sleep 10
+      Jobtracker.prep_workers
+      Jobtracker.update_status("put workers back on the queue")
+    end
+
+    def Jobtracker.stop!
+      #send signal for Jobtracker to check for
+      Jobtracker.update_status('stopping')
+      sleep 5
+      i=0
+      while Jobtracker.status=='stopping'
+        Jobtracker.update_status("#{Jobtracker.to_s} still on queue, waiting")
+        sleep 5
+        i+=1
+      end
+      return true
+    end
+
+    def Jobtracker.last_notification
+      return Jobtracker.get_args["last_notification"] if Jobtracker.get_args
+    end
+
+    def Jobtracker.last_notification=(time)
+      Jobtracker.set_args({"last_notification"=>time})
+    end
+
+    def Jobtracker.notif_due?
+      last_duetime = Time.now.utc - Jobtracker.notification_freq
+      return (Jobtracker.last_notification.to_s.length==0 || Jobtracker.last_notification.to_datetime < last_duetime)
+    end
 
     def Jobtracker.max_run_time_workers
       #return workers who have been cranking away for 6+ hours
         workers = Jobtracker.workers('working').select do |w|
-            w.job['run_at'].to_s.length>0 and 
-              (Time.now.utc - Time.parse(w.job['run_at'])) > Jobtracker.max_run_time
+            w.job['runat'].to_s.length>0 and 
+              (Time.now.utc - Time.parse(w.job['runat'])) > Jobtracker.max_run_time
         end
         return workers
     end
@@ -35,36 +174,24 @@ module Mobilize
                                           end
                                         end.flatten.join("\n\n")
             u = User.where(:name=>email.split("@").first).first
-            if u
-              runner_dst = Dataset.find_by_url("gsheet://#{u.runner.path}")
-              n['body'] += "\n\n#{runner_dst.http_url}" if runner_dst and runner_dst.http_url
-            end
+            runner_dst = Dataset.find_by_url("gsheet://#{u.runner.path}")
+            n['body'] += "\n\n#{runner_dst.http_url}" if runner_dst and runner_dst.http_url
             n['to'] = email
-            n['bcc'] = [Gdrive.admin_group_name,Gdrive.domain].join("@")
+            n['bcc'] = Jobtracker.admin_emails.join(",")
             notifs << n
           end
         end
         lws = Jobtracker.max_run_time_workers
         if lws.length>0
-          bod = begin
-                  lws.map{|w| w.job['payload']['args']}.first.join("\n")
-                rescue
-                  "Failed to get job names"
-                end
           n = {}
           n['subject'] = "#{lws.length.to_s} max run time jobs"
-          n['body'] = bod
-          n['to'] = [Gdrive.admin_group_name,Gdrive.domain].join("@")
+          n['body'] = lws.map{|w| %{spec:#{w['spec']} stg:#{w['stg']} runat:#{w['runat'].to_s}}}.join("\n\n")
+          n['to'] = Jobtracker.admin_emails.join(",")
           notifs << n
         end
         #deliver each email generated
         notifs.each do |notif|
-          begin
-            Email.write(notif).deliver
-          rescue
-            #log email on failure
-            Jobtracker.update_status("Failed to deliver #{notif.to_s}")
-          end
+          Email.write(notif).deliver
         end
         #update notification time so JT knows to wait a while
         Jobtracker.last_notification = Time.now.utc.to_s
@@ -116,5 +243,52 @@ module Mobilize
                     end.to_s.strip
       Time.parse(deploy_time)
     end
+
+    #test methods
+    def Jobtracker.restart_test_redis
+      Jobtracker.stop_test_redis
+      if !system("which redis-server")
+        raise "** can't find `redis-server` in your path, you need redis to run Resque and Mobilize"
+      end
+      "redis-server #{Base.root}/test/redis-test.conf".bash
+    end
+
+    def Jobtracker.stop_test_redis
+      processes = `ps -A -o pid,command | grep [r]edis-test`.split($/)
+      pids = processes.map { |process| process.split(" ")[0] }
+      puts "Killing test redis server..."
+      pids.each { |pid| Process.kill("TERM", pid.to_i) }
+      puts "removing redis db dump file"
+      sleep 5
+      `rm -f #{Base.root}/test/dump.rdb #{Base.root}/test/dump-cluster.rdb`
+    end
+
+    def Jobtracker.set_test_env
+      ENV['MOBILIZE_ENV']='test'
+      ::Resque.redis="localhost:9736"
+      mongoid_config_path = "#{Base.root}/config/mobilize/mongoid.yml"
+      Mongoid.load!(mongoid_config_path, Base.env)
+    end
+
+    def Jobtracker.drop_test_db
+      Jobtracker.set_test_env
+      Mongoid.session(:default).collections.each do |collection| 
+        unless collection.name =~ /^system\./
+          collection.drop
+        end
+      end
+    end
+
+    def Jobtracker.build_test_runner(user_name)
+      Jobtracker.set_test_env
+      u = User.where(:name=>user_name).first
+      Jobtracker.update_status("delete old books and datasets")
+      # delete any old runner from previous test runs
+      gdrive_slot = Gdrive.owner_email
+      u.runner.gsheet(gdrive_slot).spreadsheet.delete
+      Jobtracker.update_status("enqueue jobtracker, wait 45s")
+      Mobilize::Jobtracker.start
+      sleep 45
+    end
   end
 end