fanforce-worker 0.5.0 → 0.10.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NGZiNDQyNDQ3MzFmYmI2ZWVhZTExZjc3MDFjNzVmODk0OGQ3MzRlZg==
+    YmY2YzQ4NDNlOTEwOWMzMWIxMGYzMTBjNDMxOTY0ODViZjVmOGM0MQ==
   data.tar.gz: !binary |-
-    YTM0NDNkN2I3OGQ5YmQ1Yzg2NmM5MzZkOGIyOWZhMzRhYjhkOGI1Ng==
+    NmU1YTU5M2Q1ZTcxZjJkNTZhNTBlZTNlN2Y1MWQ5NDlhNjVkZjIzZg==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    NzU5NTNkYTYzODViNGJiYTc4NWVmYTkyYmJlN2QyYzdiMjgzYjNiYjA3Y2Vk
-    Y2MyNjQ1ODFmMzQxZGFhYTEzOGEwMmNlMjgxNGFmYjFjMjk5M2Q4ZDVmNWMx
-    N2M5NDA3ODgxNjI2ZWMyNzYxNTM4NDJjMGIwZDk0OWMxZTU4Mzk=
+    Y2ZkOTkxMWIwMjE2MTBiOGM1ZTdkZGVmZDI1MzdjNzU4Yzk3ZWVkNjVjMTQ0
+    MmRhNDc1NjQzYTUwYjZkNGE2ZTBjNjk5NDU0M2YyZDNiYWNjZjBjNWQ2YWVl
+    YzI4NmQyZmZlOTNhZjZmNmExZmE0NjkwNDJiYThiZWM5ZjZkNDM=
   data.tar.gz: !binary |-
-    MzFjN2RkNDI3YTI0OWQ1ZjY3ZmQ3MWNjMmRjNzcyZTY5ZmI5MmM1ZTQ4NjJm
-    NzZiYWIwODI2OGZkODI1NmY5ZTNmYjM4MWM5NWI5MGYyZWViZTBkZGYwOTM4
-    NGVmN2M2YzgxMzNhMGViNzRkOGM1ZmExOGM4NDhhMzU3NDczNTk=
+    ZGJhYzM3MmZmYTljODhhZGJhODc1OWYzMmY5YmQ5OGMwYmYxYWI1YzhkYWI3
+    ZTI4Mzg3OWM3NmI2NmVmOTlhMGI5Mzc2MmRmMmJmZDg3NzVlNzY3Y2U2MGU4
+    ZTZmMGY4ZWVlNTUzYWMwNTEyYmUwYzUxOThmYTRiZDAzZTNhZjI=

data/README.md

@@ -1,4 +1,4 @@
 fanforce-worker
 ==
 
-Fanforce worker library used by addons to connect with iron_mg and iron_worker_ng
+Fanforce worker used by addons to connect with iron_mg and iron_worker_ng

data/fanforce-worker.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |gem|
   gem.name        = 'fanforce-worker'
   gem.version     = Fanforce::Worker::VERSION
   gem.date        = '2012-10-19'
-  gem.summary     = %q{Fanforce worker library used by various Fanforce addons.}
+  gem.summary     = %q{Fanforce worker used by various Fanforce addons for background processing.}
   gem.authors     = ['Caleb Clark']
   gem.email       = ['cclark@fanforce.com']
   gem.homepage    = 'http://github.com/fanforce/fanforce-worker'
@@ -17,6 +17,9 @@ Gem::Specification.new do |gem|
   gem.require_paths = ['lib']
 
   gem.add_runtime_dependency 'iron_mq', '~> 4'
-  gem.add_runtime_dependency 'typhoeus'
-  gem.add_runtime_dependency 'fanforce-api', '~> 0.9'
+  gem.add_runtime_dependency 'iron_cache', '~> 1.4'
+  gem.add_runtime_dependency 'uuidtools', '~> 2.1'
+  gem.add_runtime_dependency 'activesupport', '~> 3.2'
+
+  gem.add_runtime_dependency 'fanforce-api', '~> 0.14'
 end

data/lib/fanforce/worker.rb

@@ -1,6 +1,5 @@
 class Fanforce
   require_relative 'worker/version'
-  require_relative 'worker/workers'
   require_relative 'worker/worker'
 end
 

data/lib/fanforce/worker/version.rb

@@ -1,5 +1,5 @@
 class Fanforce
   class Worker
-    VERSION = '0.5.0'
+    VERSION = '0.10.0'
   end
 end

data/lib/fanforce/worker/workers.rb

@@ -0,0 +1,214 @@
+class Fanforce::Worker
+
+  LOAD_TIME = Time.now
+
+  def initialize(opts={})
+    @opts = opts
+  end
+
+  def iron_mq
+    require 'iron_mq'
+    @iron_mq ||= IronMQ::Client.new(:token => @opts[:token] || ENV['IRON_TOKEN'], :project_id => @opts[:project_id] || ENV['IRON_PROJECT_ID'])
+  end
+
+  def iron_cache
+    require 'iron_cache'
+    @iron_cache ||= IronCache::Client.new(:token => @opts[:token] || ENV['IRON_TOKEN'], :project_id => @opts[:project_id] || ENV['IRON_PROJECT_ID'])
+  end
+
+  def enqueue(queue_id, params, options={})
+    retries = (options[:retries].present?) ? options.delete(:retries) : 0
+    raise 'Params being sent to the queue must be a Hash' if !params.is_a?(Hash)
+    iron_mq.queue(queue_id).post({params: params, retries: retries}.to_json, options)
+  end
+
+  def add_error(queue_id, error)
+    require 'uuidtools'
+    details_id = UUIDTools::UUID.random_create.to_s
+    iron_cache.cache("#{queue_id}-ERRORS").put(details_id, error.to_json)
+    iron_mq.queue("#{queue_id}-ERRORS").post({
+        details_id: details_id,
+        exception: truncate(error[:exception]),
+        message: truncate(error[:message].to_s),
+        params: truncate(error[:params].to_json),
+        errored_at: error[:errored_at],
+        retries: error[:retries],
+        env_vars: truncate(error[:env_vars].to_json),
+        curl_command: truncate(error[:curl_command].to_s)
+    }.to_json)
+  rescue => e
+    puts '-----------------------------------------------------'
+    puts 'WORKER ERROR WHILE RECOVERING FROM JOB ERROR:'
+    puts e.message
+    puts e.backtrace
+    puts '-----------------------------------------------------'
+    puts 'JOB ERROR:'
+    puts "details_id: #{details_id}"
+    puts "exception: #{truncate(error[:exception])}"
+    puts "message: #{truncate(error[:message].to_s)}"
+    puts "params: #{truncate(error[:params].to_json)}"
+    puts "errored_at: #{error[:errored_at]}"
+    puts "retries: #{error[:retries]}"
+    puts "env_vars: #{truncate(error[:env_vars].to_json)}"
+    puts "curl_command: #{truncate(error[:curl_command].to_s)}"
+  end
+
+  def delete_error(queue_id, job_id, details_id)
+    iron_mq.queue("#{queue_id}-ERRORS").delete(job_id)
+    iron_cache.cache("#{queue_id}-ERRORS").delete(details_id)
+  end
+
+  def error_details(queue_id, details_id)
+    cache = iron_cache.cache("#{queue_id}-ERRORS").get(details_id)
+    MultiJson.load(cache.value, :symbolize_keys => true)
+  end
+
+  def retry_error(queue_id, job_id, details_id)
+    cache = iron_cache.cache("#{queue_id}-ERRORS").get(details_id)
+    cache_data = MultiJson.load(cache.value, :symbolize_keys => true)
+    enqueue(queue_id, cache_data[:params], :retries => cache_data[:retries] + 1)
+
+    cache.delete and iron_mq.queue("#{queue_id}-ERRORS").delete(job_id)
+  end
+
+  def truncate(text, length=130, truncate_string="...")
+    if text
+      l = length - truncate_string.chars.to_a.length
+      chars = text.chars.to_a
+      (chars.length > length ? chars[0...l].join('') + truncate_string : text).to_s
+    end
+  end
+
+  ##########################################################################################
+
+  def self.enqueue(queue_id, params, options={})
+    self.new.enqueue(queue_id, params, options)
+  end
+
+  def self.add_error(queue_id, error)
+    self.new.add_error(queue_id, error)
+  end
+
+  ##########################################################################################
+
+  def self.current_queue_id=(queue_id)
+    @current_queue_id = queue_id
+  end
+
+  def self.current_queue_id
+    @current_queue_id
+  end
+
+  def self.current_worker_env=(env_vars)
+    @current_worker_env = env_vars
+  end
+
+  def self.current_worker_env
+    @current_worker_env
+  end
+
+  def self.current_params=(params)
+    @current_params = params
+  end
+
+  def self.current_params
+    @current_params
+  end
+
+  def self.current_retries=(retries)
+    @current_retries = retries
+  end
+
+  def self.current_retries
+    @current_retries
+  end
+
+  def self.current_job=(job)
+    @current_job = job
+  end
+
+  def self.current_job
+    @current_job
+  end
+
+  def self.load_env
+    if File.exists?('.developmentenv.rb')
+      require '.developmentenv'
+    elsif File.exists?('.stagingenv.rb')
+      require '.stagingenv'
+    elsif File.exists?('.productionenv.rb')
+      require '.productionenv'
+    end
+  end
+
+  def self.run(worker_data, &code_block)
+    load_env
+    require 'iron_mq'
+    require 'iron_cache'
+    require 'fanforce/api'
+    require 'active_support/all'
+
+    self.current_queue_id   = worker_data['queue_id']
+    self.current_worker_env = worker_data['env_vars']
+    queue = IronMQ::Client.new.queue(current_queue_id)
+
+    job_num = 0
+    puts 'PROCESSING...'
+    while (Time.now - LOAD_TIME) < 1800 and (job = queue.get(timeout: 3600)) do
+      puts "JOB #{job_num+=1}: #{job.body}"
+      run_job job, &code_block
+      self.delete_job
+    end
+    self.delete_job
+    puts 'DONE'
+  end
+
+  def self.retry(options)
+    self.new.enqueue(current_queue_id, current_params, options.merge(retries: current_retries + 1))
+  end
+
+  def self.run_job(job, &code_block)
+    puts '----------------------------------------------------------'
+    print 'PROCESSING MESSAGE: '
+
+    task_data = Fanforce.decode_json(job.body)
+    self.current_job = job
+    self.current_params  = task_data[:params]
+    self.current_retries = task_data[:retries]
+
+    set_env_vars(current_worker_env)
+    code_block.call(task_data[:params].clone, retries: task_data[:retries], queue_id: current_queue_id)
+    self.delete_job(job)
+
+  rescue Exception => e
+    if job.nil?
+      puts 'MESSAGE IS NIL'
+      return
+    end
+    error = task_data.merge(
+      exception: e.class.name,
+      message: e.message,
+      backtrace: e.backtrace,
+      errored_at: Time.now,
+      env_vars: current_worker_env
+    )
+    error[:curl_command] = e.curl_command if e.respond_to?(:curl_command)
+
+    puts "ADDING TO ERROR CACHE: #{error.to_json}"
+    self.delete_job(job)
+    puts 'DELETED MESSAGE'
+
+    self.add_error current_queue_id, error
+  end
+
+  def self.delete_job(job=nil)
+    return if job.nil? and current_job.nil?
+    (job || current_job).delete
+    self.current_job = nil
+  end
+
+  def self.set_env_vars(vars)
+    vars.each {|k,v| ENV[k.to_s]=v }
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fanforce-worker
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.10.0
 platform: ruby
 authors:
 - Caleb Clark
@@ -25,33 +25,61 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '4'
 - !ruby/object:Gem::Dependency
-  name: typhoeus
+  name: iron_cache
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: uuidtools
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.2'
 - !ruby/object:Gem::Dependency
   name: fanforce-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: '0.14'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: '0.14'
 description: 
 email:
 - cclark@fanforce.com
@@ -65,9 +93,8 @@ files:
 - Rakefile
 - fanforce-worker.gemspec
 - lib/fanforce/worker.rb
-- lib/fanforce/worker/iron.rb
 - lib/fanforce/worker/version.rb
-- lib/fanforce/worker/worker.rb
+- lib/fanforce/worker/workers.rb
 homepage: http://github.com/fanforce/fanforce-worker
 licenses: []
 metadata: {}
@@ -90,5 +117,5 @@ rubyforge_project:
 rubygems_version: 2.0.3
 signing_key: 
 specification_version: 4
-summary: Fanforce worker library used by various Fanforce addons.
+summary: Fanforce worker used by various Fanforce addons for background processing.
 test_files: []

data/lib/fanforce/worker/iron.rb

@@ -1,17 +0,0 @@
-require 'iron_mq'
-
-class Fanforce::Worker::Iron
-
-  def initialize(opts={})
-    @mq_client = IronMQ::Client.new(:token => opts[:token] || ENV['IRON_TOKEN'], :project_id => opts[:project_id] || ENV['IRON_PROJECT_ID'])
-  end
-
-  def enqueue(queue_id, payload)
-    @mq_client.queue(queue_id).post(payload.to_json)
-  end
-
-  def client
-    @mq_client
-  end
-
-end

data/lib/fanforce/worker/worker.rb

@@ -1,83 +0,0 @@
-class Fanforce::Worker
-
-  def initialize(opts={})
-    @opts = opts
-  end
-
-  def iron
-    require_relative 'iron'
-    @iron ||= Fanforce::Worker::Iron.new(@opts)
-  end
-
-  def enqueue(queue_id, payload)
-    raise 'Payload being sent to the queue must be a Hash' if !payload.is_a?(Hash)
-    iron.enqueue queue_id, params: payload, retries: 0
-  end
-
-  def retry_error(queue_id, msg_id)
-    msg = iron.client.queue(queue_id + '-ERRORS').peek(msg_id)
-    data = MultiJson.load(msg.body, :symbolize_keys => true)
-    iron.enqueue queue_id, params: data[:params], retries: data[:retries] + 1
-    msg.delete
-  end
-
-  ##########################################################################################
-
-  def self.add_error(queue_id, payload)
-    self.new.iron.enqueue("#{queue_id}-ERRORS", payload)
-  end
-
-  def self.enqueue(queue_id, payload)
-    self.new.enqueue(queue_id, payload)
-  end
-
-  ##########################################################################################
-
-  def self.run(worker_data, &code_block)
-    require '.pluginenv'
-    require 'iron_mq'
-    require 'fanforce/api'
-
-    run_validations
-    run_worker IronMQ::Client.new.queue(worker_data['worker_id']).get(timeout: 3600), worker_data, &code_block
-  end
-
-  def self.run_validations
-
-  end
-
-  def run_worker(msg, worker_data, &code_block)
-    puts '----------------------------------------------------------'
-    print 'PROCESSING MESSAGE: '
-    task_data = Fanforce.decode_json(msg.body)
-
-    set_env_vars(worker_data['env_vars'])
-    code_block.call(task_data[:params].clone, task_data[:retries])
-    msg.delete
-
-  rescue Exception => e
-    if msg.nil?
-      puts 'MESSAGE IS NIL'
-      return
-    end
-    task_data[:last_error] = {
-        exception: e.class.name,
-        message: e.message,
-        backtrace: e.backtrace,
-        raised_at: Time.now
-    }
-    task_data[:last_error][:curl_command] = e.curl_command if e.respond_to?(:curl_command)
-    task_data[:env_vars] = worker_data['env_vars']
-
-    puts "ADDING TO ERROR QUEUE: #{task_data.to_json}"
-    self.add_error worker_data['worker_id'], task_data
-    msg.delete
-    puts 'DELETED MESSAGE'
-  end
-
-  def set_env_vars(vars)
-    vars.each {|k,v| ENV[k.to_s]=v }
-    load 'fanforce/api/ff_globals.rb'
-  end
-
-end