jobmaster 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2c5405222718b1846cc2b84ce1b83d99307fba52
+  data.tar.gz: 838630db1843875775075cc9d9321a5eb0173a02
+SHA512:
+  metadata.gz: 327dc4714bb69f279dec5ef74ec7287cb4600bd543fb1183c6729a861f7f36b4d49a6c4992e4ef3ccc599868d1ae1727d3a00308b5b225862f941b19ec260fd2
+  data.tar.gz: 516b2ec4d1a7e75e2ae907e453c5a7e42357f3a2ebb344841bc9e015b72885c02237799d6a9a4e6513984255aacce5c21cee4f1da67c50225f83aceff54ae5d8

data/lib/job_master/gen_methods.rb

@@ -0,0 +1,68 @@
+require 'flex_core'
+
+class NilClass
+  def to_hash
+    {}
+  end
+end
+
+class JobMaster
+
+  def date_gen (args)
+    min = Date.parse(args[:earliest]).jd
+    max = args[:latest] == 'none' ? Date.today.jd : Date.parse(args[:latest]).jd
+    units = (min..max).to_a.shuffle
+  end
+
+  def from_table (args)
+    table_name = args[:table_name]
+    columns = args[:columns]
+    filters = args[:filters]
+
+    table_name = table_name.to_sym if table_name.is_a?(String)
+    columns = eval(columns) if columns.is_a?(String)
+    filters = eval(filters) if filters.is_a?(String)
+
+    output = database[table_name].all
+    output.map! { |row| row.keep(*columns) } unless columns.nil?
+
+    unless filters.nil?
+      output.reject! { |row|
+        rejected = false
+        filters.each { |key, value|
+          if row[key] != value
+            rejected = true
+            break
+          end
+        }
+        rejected
+      }
+    end
+
+    output.map!(&:inspect)
+    output.to_a.shuffle!
+  end
+
+  def from_csv (file, *labels)
+
+    file = File.open(file, 'r').read.gsub(/\r/, '')
+    rows = CSV.parse(file)
+    headers = rows.shift.map(&:downcase.strip)
+    end_index = headers.rindex(nil)
+    headers.slice!(0..end_index) unless end_index == -1
+    raise ArgumentError.new('Header cannot contain empty cells') if headers.include?(nil)
+    raise ArgumentError.new('Header labels must be unique.') if headers.size != headers.uniq.size
+    indexes = labels.map { |label|
+      headers.index(label)
+    }
+    rows.map{ |row|
+      new_row = row.values_at(*indexes)
+      hash = {}
+      new_row.each_with_index { |cell, index|
+        hash.store!(labels[index], cell)
+      }
+    }
+    rows
+  end
+
+end

data/lib/job_master/stealth.rb

@@ -0,0 +1,13 @@
+require 'tunnel_blick'
+
+class JobMaster
+  def stealth(max_requests, args, *websites)
+    @tunnel = TunnelBlick.new(database)
+    dom_ids = websites.map {|ws|
+      database[:websites][domain_url: ws.home_url.scan(/[^\/]+/)[1]][:id]
+    }
+    @tunnel.connect_smart(max_requests, args, dom_ids)
+    @tunnel.add_websites(*websites)
+    @tunnel
+  end
+end

data/lib/job_master.rb

@@ -0,0 +1,138 @@
+require 'job_master/gen_methods'
+require 'job_master/stealth'
+require 'flex_core'
+require 'flex_pg'
+
+class JobMaster
+  attr_reader :database, :programs_db, :jobs_db, :websites
+
+  def initialize (database)
+    @myself = ENV['USER']
+    @database = database
+    @programs_db = database[:program_constraints]
+    @jobs_db = database[:job_log]
+  end
+
+  def add_websites (*websites)
+    @websites = websites
+  end
+
+  class << self
+
+    LEGEND = {
+        date_range_gen: [:earliest, :latest],
+        from_csv: ['file', '*labels']
+    }
+
+    def date_range_gen
+
+    end
+
+  end
+
+  def start_single (prog_name , minutes = 10, sleep = 3)
+
+    @start_time = Time.now
+    set_up_job(prog_name)
+
+    unit_time = 0.0
+
+    while (Time.now - @start_time) / 60 < minutes
+
+      unless unit_time == 0.0
+        update_job(unit_time)
+      end
+
+      unit_start = Time.now
+      unit = next_unit
+
+      break if unit.nil?
+
+      @websites.first.send(prog_name.to_sym, unit)
+
+      puts unit_time = (Time.now - unit_start) / 60
+      sleep(sleep)
+      unit_time += sleep / 60
+    end
+
+  end
+
+  def step_single (prog_name)
+
+    @start_time = Time.now
+    set_up_job(prog_name)
+    unit = next_unit
+    return false if unit.nil?
+
+    @websites.first.send(prog_name.to_sym, unit)
+    puts unit_time = (Time.now - @start_time) / 60
+    update_job(unit_time)
+
+  end
+
+  def set_up_job (program)
+    last_matching = programs_db.where(program: program, @myself => Sequel.pg_array_op(:machines).any).order(:id).last
+
+    @program_id = last_matching[:id]
+    the_job = jobs_db.where(program: program, constraints: @program_id).order(:id).last
+
+    if the_job.nil?
+      units = self.send(last_matching[:method].to_sym, last_matching[:constraints].merge(program: program))
+      insert_job_details(program, units, last_matching[:machines])
+    end
+
+    @job_id = the_job[:id]
+  end
+
+  def insert_job_details (program, units, machines)
+
+    time_share = machines.inject({}) { |acc, m|
+      acc.store(m, 0)
+      acc
+    }
+
+    details = {
+        program:       program,
+        created:       Time.now,
+        constraints:   @program_id,
+        units:         Sequel.pg_array(units),
+        size:          units.size,
+        total_minutes: 0.0,
+        time_share:    Sequel.hstore(time_share),
+        unit_checkout: Sequel.hstore({:apples => :bananas})
+    }
+    jobs_db.insert(details)
+  end
+
+  def next_unit
+    checkout = jobs_db[id: @job_id][:unit_checkout].to_hash
+
+    if checkout.keys.include?(@myself)
+      checkout[@myself]
+    else
+      units = jobs_db[id: @job_id][:units]
+      return false if units.empty?
+      next_unit = units.pop
+      jobs_db.where(id: @job_id).update(
+          :units => units,
+          :unit_checkout => Sequel.hstore(Sequel.expr(:unit_checkout)).merge(@myself => next_unit))
+      next_unit
+    end
+  end
+
+  def update_job (unit_time)
+
+    time_share = jobs_db[id: @job_id][:time_share].to_hash
+    time_share.find_add!(@myself, unit_time)
+
+    unit_checkout = jobs_db[id: @job_id][:unit_checkout].to_hash
+    unit_checkout.delete!(@myself)
+
+    jobs_db.where(id: @job_id).update(
+        :total_minutes => Sequel.expr(:total_minutes) + unit_time,
+        :time_share => Sequel.hstore(time_share),
+        :unit_checkout => Sequel.hstore(unit_checkout)
+    )
+  end
+
+end

metadata

@@ -0,0 +1,73 @@
+--- !ruby/object:Gem::Specification
+name: jobmaster
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Eugene Lai
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-03-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: flex_core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: flex_pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: To do
+email: ejt.lai@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/job_master.rb
+- lib/job_master/gen_methods.rb
+- lib/job_master/stealth.rb
+homepage: 
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: Job manager for multi-website automated scraping
+test_files: []