jobmaster 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5ba10006ad5958002049e49b7493e4a33e75d1b7
-  data.tar.gz: 959fdef93a1f534139d207ecc384d0550f22574d
+  metadata.gz: 49a526c44aa912126b20e09c67fa3ae38955822d
+  data.tar.gz: b6c53cd48c38edccd9ee8d5e853b06727693b9e6
 SHA512:
-  metadata.gz: ca906eb1d84569a70c2c419d614db1ad92459f9fae6a4baaf128f73718f6408bc9e600b5448b34fe9b034e998f9e0b5f4f990a968fd0a823ab234b2f3a67256b
-  data.tar.gz: 8e99588ac3b62b249d52627977ecabd665fc907cda2347acf19eb0d6f0809ef590146c47156d8d8d42507a3727c62c582cf42d8dcb30f6568e511722b536865c
+  metadata.gz: a422accc2acb60a84223af4e054a2defbda10e5959e7a65b01abb95ddf221147c3d1240dc6ee64d53e27633bff25c9b985d382bd8ea42e82f346f447cc63a488
+  data.tar.gz: be62c7db023ebba3cfdcdd035849cee4d28c8b99ef56358109270fd5173c8529aa6151f13a10a01b2052fdc3aeeebefe939399272a980575f69fb49fe749ef90

data/lib/job_master/job_methods.rb

@@ -0,0 +1,134 @@
+class JobMaster
+
+  def master_cycle (sleep)
+
+    @schedule = database[:job_log].where(status: 'active').order(:frequency).reverse
+    @time_log = {}
+
+    stop = false
+
+    until stop
+      @schedule.each { |job|
+      program = job[:program]
+        if job[:frequency] == 0
+          step_single(program)
+        else
+          @time_log[program] ||= Time.now
+          difference = (Time.now - @time_log[program]) / 60
+          if difference > job[:frequency]
+            step_single(program)
+            @time_log[program] = Time.now
+          end
+        end
+        sleep(sleep)
+      }
+    end
+
+  end
+
+  def start_single (prog_name , minutes = 10, sleep = 3)
+
+    @start_time = Time.now
+    set_up_job(prog_name)
+
+    unit_time = 0.0
+
+    while (Time.now - @start_time) / 60 < minutes
+
+      unless unit_time == 0.0
+        update_job(unit_time)
+      end
+
+      unit_start = Time.now
+      unit = next_unit
+
+      break if unit.nil?
+
+      @websites.first.send(prog_name.to_sym, unit)
+
+      puts unit_time = (Time.now - unit_start) / 60
+      sleep(sleep)
+      unit_time += sleep / 60
+    end
+
+  end
+
+  def step_single (prog_name)
+
+    @start_time = Time.now
+    set_up_job(prog_name)
+    unit = next_unit
+    return false if unit.nil?
+
+    @websites.first.send(prog_name.to_sym, unit)
+    puts unit_time = (Time.now - @start_time) / 60
+    update_job(unit_time)
+
+  end
+
+  def set_up_job (program)
+    last_matching = programs_db.where(program: program, @myself => Sequel.pg_array_op(:machines).any).order(:id).last
+
+    @program_id = last_matching[:id]
+    the_job = jobs_db.where(program: program, constraints: @program_id).order(:id).last
+
+    if the_job.nil?
+      units = self.send(last_matching[:method].to_sym, last_matching[:constraints].merge(program: program))
+      insert_job_details(program, units, last_matching[:machines])
+    end
+
+    @job_id = the_job[:id]
+  end
+
+  def insert_job_details (program, units, machines)
+
+    time_share = machines.inject({}) { |acc, m|
+      acc.store(m, 0)
+      acc
+    }
+
+    details = {
+        program:       program,
+        created:       Time.now,
+        constraints:   @program_id,
+        units:         Sequel.pg_array(units),
+        size:          units.size,
+        total_minutes: 0.0,
+        time_share:    Sequel.hstore(time_share),
+        unit_checkout: Sequel.hstore({:apples => :bananas})
+    }
+    jobs_db.insert(details)
+  end
+
+  def next_unit
+    checkout = jobs_db[id: @job_id][:unit_checkout].to_hash
+
+    if checkout.keys.include?(@myself)
+      checkout[@myself]
+    else
+      units = jobs_db[id: @job_id][:units]
+      return false if units.empty?
+      next_unit = units.pop
+      jobs_db.where(id: @job_id).update(
+          :units => units,
+          :unit_checkout => Sequel.hstore(Sequel.expr(:unit_checkout)).merge(@myself => next_unit))
+      next_unit
+    end
+  end
+
+  def update_job (unit_time)
+
+    time_share = jobs_db[id: @job_id][:time_share].to_hash
+    time_share.find_add!(@myself, unit_time)
+
+    unit_checkout = jobs_db[id: @job_id][:unit_checkout].to_hash
+    unit_checkout.delete!(@myself)
+
+    jobs_db.where(id: @job_id).update(
+        :total_minutes => Sequel.expr(:total_minutes) + unit_time,
+        :time_share => Sequel.hstore(time_share),
+        :unit_checkout => Sequel.hstore(unit_checkout)
+    )
+  end
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jobmaster
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Eugene Lai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-11 00:00:00.000000000 Z
+date: 2016-03-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: flex_core
@@ -46,6 +46,7 @@ extra_rdoc_files: []
 files:
 - lib/job_master.rb
 - lib/job_master/gen_methods.rb
+- lib/job_master/job_methods.rb
 - lib/job_master/stealth.rb
 homepage: 
 licenses: []
@@ -66,7 +67,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.6.1
 signing_key: 
 specification_version: 4
 summary: Job manager for multi-website automated scraping