iron_worker 2.3.16 → 2.3.17

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: fe522253fc444a2aee1f6f474ec587d671c25e30
+  data.tar.gz: 7e71a9926f99b579e03a776a6337aa9ec46b7335
+SHA512:
+  metadata.gz: 75c288511be5d4ff6dc27a4f554435775004142da36980e01bc0c48d8a3a0000212a424b6a0726e20319e5e5d0fe02c06572348382fc5bb38e3bfd8682de27ef
+  data.tar.gz: a522f0b44e0e050aed2a697c5498f9de884fd77a72831d00262f78a8d9d4ebd9ccc8d0c773ca5c083aee952223a8a4e96be4c1a77e2d5a2744f82c0cde22018c

data/.gitignore

@@ -0,0 +1,4 @@
+.idea
+pkg
+test/config.yml
+*.sublime-*

data/Rakefile

@@ -0,0 +1,31 @@
+#require "rubygems"
+#require "bundler/setup"
+
+require 'rake/dsl_definition' # temporary I think?
+require 'rake/testtask'
+require 'jeweler2'
+
+Jeweler::Tasks.new do |gemspec|
+  gemspec.name = "iron_worker"
+  gemspec.summary = "The official IronWorker gem for IronWorker by Iron.io. http://www.iron.io"
+  gemspec.description = "The official IronWorker gem for IronWorker by Iron.io. http://www.iron.io"
+  gemspec.email = "travis@iron.io"
+  gemspec.homepage = "http://www.iron.io"
+  gemspec.authors = ["Travis Reeder"]
+  gemspec.files = FileList['VERSION.yml','init.rb', 'lib/**/*.rb', 'rails/**/*.rb']
+  gemspec.add_dependency 'zip'
+  gemspec.add_dependency 'rest-client'
+  gemspec.add_dependency 'rest'
+  gemspec.add_dependency 'bundler'
+  #gemspec.add_dependency 'typhoeus'
+  gemspec.required_ruby_version = '>= 1.9'
+end
+Jeweler::GemcutterTasks.new
+
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = FileList['test/test*.rb']
+  t.verbose = true
+end
+
+

data/iron_worker.gemspec

@@ -0,0 +1,27 @@
+require File.expand_path('../lib/iron_worker/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Travis Reeder"]
+  gem.email         = ["travis@iron.io"]
+  gem.description   = "DEPRECATED!! Use iron_worker_ng now. The official IronWorker gem for IronWorker by Iron.io. http://www.iron.io"
+  gem.summary       = "DEPRECATED!! Use iron_worker_ng now. The official IronWorker gem for IronWorker by Iron.io. http://www.iron.io"
+  gem.homepage      = "https://github.com/iron-io/iron_worker_ruby"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "iron_worker"
+  gem.require_paths = ["lib"]
+  gem.version       = IronWorker::VERSION
+
+  gem.required_rubygems_version = ">= 1.3.6"
+  gem.required_ruby_version = Gem::Requirement.new(">= 1.9")
+  gem.add_runtime_dependency "iron_core", ">= 0.5.1"
+
+  gem.add_runtime_dependency 'zip'
+  gem.add_runtime_dependency 'rest-client'
+  gem.add_runtime_dependency 'rest'
+  gem.add_runtime_dependency 'bundler'
+
+end
+

data/lib/generators/iron_worker/templates/template_worker.erb

@@ -0,0 +1,7 @@
+class <%= @camel %> < IronWorker::Base
+
+  def run
+    log "Your code goes in this bit"
+  end
+
+end

data/lib/iron_worker/service.rb

@@ -93,7 +93,7 @@ module IronWorker
 
 
     def self.heroku?
-      !Gem::GemPathSearcher.private_instance_methods.include?(:_deprecated_initialize)
+      !Gem::GemPathSearcher.private_instance_methods.include?(:_deprecated_initialize) rescue false
     end
 
     def webhook

data/lib/iron_worker/version.rb

@@ -0,0 +1,4 @@
+module IronWorker
+  VERSION = "2.3.17"
+end
+

data/test/Gemfile

@@ -0,0 +1,36 @@
+# These are required for the tests
+
+source 'http://rubygems.org'
+
+gem 'iron_worker'
+
+group :test do
+  gem 'rack'
+  gem 'test-unit'
+  gem 'minitest'
+  gem 'jeweler2'
+  gem 'zip'
+  gem 'rest'
+  gem 'typhoeus'
+  gem 'uber_config'
+
+  gem 'simple_record'
+  gem 'activesupport'
+  gem 'simple_record'
+  gem 'dropbox', '1.2.3'
+  gem 'pusher'
+  gem 'carrierwave'
+  gem 'aws-s3'
+  gem 'mechanize'
+  gem 'mail'
+  gem 'timecop'
+  gem 'fog'
+  gem 'logglier'
+#gem 'mysql2'
+  gem 'mini_fb'
+  gem 'iron_mq'
+  gem 'prawn'
+  gem 'pdf-reader'
+  gem 'concur'
+  gem 'mechanize'
+end

data/test/Gemfile.lock

@@ -0,0 +1,169 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    Ascii85 (1.0.2)
+    activesupport (3.2.8)
+      i18n (~> 0.6)
+      multi_json (~> 1.0)
+    aws (2.5.8)
+      http_connection
+      http_connection
+      uuidtools
+      uuidtools
+      xml-simple
+      xml-simple
+    aws-s3 (0.6.3)
+      builder
+      mime-types
+      xml-simple
+    builder (3.1.4)
+    carrierwave (0.5.8)
+      activesupport (~> 3.0)
+    concur (1.0.0)
+      faraday
+      faraday
+    domain_name (0.5.4)
+      unf (~> 0.0.3)
+    dropbox (1.2.3)
+      json (>= 1.2.0)
+      mechanize (>= 1.0.0)
+      multipart-post (>= 1.0)
+      oauth (>= 0.3.6)
+    ethon (0.5.3)
+      ffi (~> 1.0.11)
+      mime-types (~> 1.18)
+    excon (0.16.7)
+    faraday (0.8.4)
+      multipart-post (~> 1.1)
+    ffi (1.0.11)
+    fog (1.7.0)
+      builder
+      excon (~> 0.14)
+      formatador (~> 0.2.0)
+      mime-types
+      multi_json (~> 1.0)
+      net-scp (~> 1.0.4)
+      net-ssh (>= 2.1.3)
+      nokogiri (~> 1.5.0)
+      ruby-hmac
+    formatador (0.2.4)
+    git (1.2.5)
+    hashery (2.0.1)
+    hashie (1.2.0)
+    http_connection (1.4.2)
+    i18n (0.6.1)
+    iron_core (0.4.3)
+      rest (>= 2.0.2)
+    iron_mq (2.1.3)
+      iron_core (>= 0.4.2)
+    iron_worker (2.3.15)
+      bundler
+      bundler
+      rest
+      rest
+      rest-client
+      rest-client
+      zip
+      zip
+    jeweler2 (2.0.9)
+      git (>= 1.2.5)
+    json (1.7.5)
+    logglier (0.2.8)
+    mail (2.4.4)
+      i18n (>= 0.4.0)
+      mime-types (~> 1.16)
+      treetop (~> 1.4.8)
+    mechanize (2.5.1)
+      domain_name (~> 0.5, >= 0.5.1)
+      mime-types (~> 1.17, >= 1.17.2)
+      net-http-digest_auth (~> 1.1, >= 1.1.1)
+      net-http-persistent (~> 2.5, >= 2.5.2)
+      nokogiri (~> 1.4)
+      ntlm-http (~> 0.1, >= 0.1.1)
+      webrobots (~> 0.0, >= 0.0.9)
+    mime-types (1.19)
+    mini_fb (1.2.0)
+      hashie
+      hashie
+      rest-client
+      rest-client
+    minitest (4.2.0)
+    multi_json (1.3.7)
+    multipart-post (1.1.5)
+    net-http-digest_auth (1.2.1)
+    net-http-persistent (2.8)
+    net-scp (1.0.4)
+      net-ssh (>= 1.99.1)
+    net-ssh (2.6.1)
+    nokogiri (1.5.5)
+    ntlm-http (0.1.1)
+    oauth (0.4.7)
+    pdf-reader (1.2.0)
+      Ascii85 (~> 1.0.0)
+      hashery (~> 2.0)
+      ruby-rc4
+    polyglot (0.3.3)
+    prawn (0.12.0)
+      pdf-reader (>= 0.9.0)
+      ttfunk (~> 1.0.2)
+    pusher (0.10.0)
+      multi_json (~> 1.0)
+      signature (~> 0.1.4)
+    rack (1.4.1)
+    rest (2.1.0)
+      net-http-persistent
+      rest-client (>= 0.3.0)
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
+    ruby-hmac (0.4.0)
+    ruby-rc4 (0.1.5)
+    signature (0.1.4)
+    simple_record (2.2.0)
+      aws
+      concur
+    test-unit (2.5.2)
+    timecop (0.5.3)
+    treetop (1.4.12)
+      polyglot
+      polyglot (>= 0.3.1)
+    ttfunk (1.0.3)
+    typhoeus (0.5.1)
+      ethon (= 0.5.3)
+    uber_config (1.0.5)
+    unf (0.0.5)
+      unf_ext
+    unf_ext (0.0.5)
+    uuidtools (2.1.3)
+    webrobots (0.0.13)
+    xml-simple (1.1.2)
+    zip (2.0.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport
+  aws-s3
+  carrierwave
+  concur
+  dropbox (= 1.2.3)
+  fog
+  iron_mq
+  iron_worker
+  jeweler2
+  logglier
+  mail
+  mechanize
+  mini_fb
+  minitest
+  pdf-reader
+  prawn
+  pusher
+  rack
+  rest
+  simple_record
+  test-unit
+  timecop
+  typhoeus
+  uber_config
+  zip

data/test/README.md

@@ -0,0 +1,5 @@
+When testing workers, be sure to run:
+
+    sudo rake install
+
+Because the uploaded gem will be the one that is installed on the system.

data/test/batch_run.rb

@@ -0,0 +1,121 @@
+require_relative 'test_base'
+require_relative 'workers/one_line_worker'
+require_relative 'workers/merging_worker'
+require_relative 'workers/progress_worker'
+require_relative 'workers/mq_worker'
+require 'concur'
+#require_relative 'prawn_worker'
+
+class BatchRun < TestBase
+
+  def test_concur_batch
+
+    old_log_level = IronWorker.logger.level
+    IronWorker.logger.level = Logger::INFO
+
+    clz = OneLineWorker
+    num_tasks = 100
+
+    worker = clz.new
+    worker.upload
+
+    jobs = []
+    executor = Concur::Executor.new_thread_pool_executor(20)
+    num_tasks.times do |i|
+      jobs << executor.execute do
+        begin
+          worker2 = clz.new
+          puts "queueing #{i}"
+          if clz == MqWorker
+            worker2.config = {:token=>IronWorker.config.token, :project_id=>IronWorker.config.project_id}
+          else
+            worker2.x =  "hello payload #{i}"
+          end
+          response_hash = worker2.queue(:priority=>(@config[:priority] || 0))
+          puts "response_hash #{i} = " + response_hash.inspect
+          assert response_hash["msg"]
+          assert response_hash["status_code"]
+          assert response_hash["tasks"]
+          assert response_hash["status_code"] == 200
+          assert response_hash["tasks"][0]["id"].length == 24, "length is #{response_hash["tasks"][0]["id"].length}"
+          assert response_hash["tasks"][0]["id"] == worker2.task_id, "id in hash: #{response_hash["tasks"][0]["id"]}, task_id: #{worker2.task_id}. response was #{worker2.response.inspect}"
+          worker2
+        rescue => ex
+          puts "ERROR! #{ex.class.name}: #{ex.message} -- #{ex.backtrace.inspect}"
+          raise ex
+        end
+
+      end
+    end
+
+    sleep 10
+
+    completed_count = 0
+    errored_queuing_count = 0
+    error_count = 0
+    while jobs.size > 0
+      jobs.each_with_index do |f, i|
+#    p f
+        begin
+          t = f.get
+#      p t
+          puts i.to_s + ' task_id=' + t.task_id.to_s
+          status_response = t.status # worker.status(t["task_id"])
+          puts 'status ' + status_response["status"] + ' for ' + status_response.inspect
+          puts 'msg=' + status_response["msg"].to_s
+          if status_response["status"] == "complete" || status_response["status"] == "error"
+            if true || status_response["status"] == "error"
+              puts t.get_log
+            end
+
+            jobs.delete(f)
+            completed_count += 1
+            puts "#{completed_count} completed so far. #{jobs.size} left..."
+            if status_response["status"] == "error"
+              error_count += 1
+            end
+          end
+        rescue => ex
+          puts 'error! ' + ex.class.name + ' -> ' + ex.message.to_s
+          puts ex.backtrace
+          errored_queuing_count += 1
+          jobs.delete(f)
+        end
+      end
+      puts 'sleep'
+      sleep 2
+      puts 'done sleeping'
+    end
+
+    puts 'Total completed=' + completed_count.to_s
+    puts 'Total errored while queuing=' + errored_queuing_count.to_s
+    puts 'Total errored while running=' + error_count.to_s
+
+    executor.shutdown
+
+    #tasks = []
+    #1000.times do |i|
+    #  puts "#{i}"
+    #  worker = ProgressWorker.new
+    #  #worker = OneLineWorker.new
+    #  #    worker = MergingWorker.new
+    #  #worker = PrawnWorker.new
+    #  worker.queue
+    #  tasks << worker
+    #end
+    #
+    #tasks.each_with_index do |task, i|
+    #  puts "#{i}"
+    #  status = task.wait_until_complete
+    #  p status
+    #  puts "\n\n\nLOG START:"
+    #  puts task.get_log
+    #  puts "LOG END\n\n\n"
+    #  assert status["status"] == "complete", "Status was not complete, it was #{status["status"]}"
+    #end
+    IronWorker.logger.level = old_log_level
+
+  end
+
+end
+

data/test/config_sample.yml

@@ -0,0 +1,13 @@
+iron:
+   token: MY_TOKEN
+   project_id: MY_PROJECT_ID
+
+database:
+    adapter: mysql2
+    host: ec2-ABC.compute-1.amazonaws.com
+    port: 3062
+    database: MY_DB
+    username: MY_USER
+    password: MY_PASS
+
+

data/test/models/cool_model.rb

@@ -0,0 +1,7 @@
+require 'simple_record'
+
+class CoolModel < SimpleRecord::Base
+
+  has_strings :name
+
+end

data/test/models/db_model.rb

@@ -0,0 +1,6 @@
+#require 'active_record'
+
+class DbModel #< ActiveRecord::Base
+
+
+end

data/test/models/model_1.rb

@@ -0,0 +1,16 @@
+
+
+class Model1
+    attr_accessor :heidi, :ho
+
+    include IronWorker::UsedInWorker
+
+    def say_hello
+        log "Hi there sir"
+    end
+
+    # testk laksdfj klasj df
+    def test
+        log 'test'
+    end
+end

data/test/models/model_2.rb

@@ -0,0 +1,16 @@
+
+
+class Model2
+    attr_accessor :heidi, :ho
+
+    include IronWorker::UsedInWorker
+
+    def say_hello
+        log "Hi there sir"
+    end
+
+    # testk laksdfj klasj df
+    def test
+        log 'test'
+    end
+end

data/test/quick_run.rb

@@ -0,0 +1,36 @@
+require_relative 'test_base'
+require_relative 'workers/one_line_worker'
+require_relative 'workers/progress_worker'
+require_relative 'workers/fail_worker'
+
+class QuickRun < TestBase
+
+  def test_worker
+    tasks = []
+    1.times do |i|
+      puts "Queuing #{i}"
+      worker = OneLineWorker.new
+      worker.x = i
+      worker.queue
+      tasks << worker
+    end
+
+    tasks.each_with_index do |worker, i|
+      puts "Waiting for #{i}"
+      status = worker.wait_until_complete
+      puts "#{i} is complete."
+      p status
+      puts "error_class: #{status["error_class"]}"
+      puts "msg: #{status["msg"]}"
+      puts "percent: #{status["percent"]}"
+      puts "\n\n\nLOG START:"
+      log = worker.get_log
+      puts log
+      puts "LOG END\n\n\n"
+      assert status["status"] == "complete", "Status was not complete, it was #{status["status"]}"
+      assert log.include?("hello")
+    end
+  end
+
+end
+

data/test/quick_schedule.rb

@@ -0,0 +1,18 @@
+require_relative 'test_base'
+require_relative 'one_line_worker'
+require_relative 'merging_worker'
+require_relative 'progress_worker'
+#require_relative 'prawn_worker'
+
+class QuickRun < TestBase
+
+  def test_scheduler_quick
+    worker = TestWorker.new
+    worker.schedule(:start_at=>10.seconds.from_now)
+    status = worker.wait_until_complete
+    assert status["status"] == "complete"
+    assert status["msg"].present?
+  end
+
+end
+

data/test/resources/something.yml

@@ -0,0 +1,4 @@
+hello:
+    world: "how are you?"
+
+

data/test/simple_script.rb

@@ -0,0 +1,11 @@
+require 'json'
+IronWorker.merge_gem 'rest-client'
+
+# load data
+job_data = JSON.load(File.open(File.join(ENV['user_dir'], 'job_data.json')))
+
+puts 'x=' + job_data['x']
+
+page = RestClient.get "http://www.github.com"
+puts page
+puts 'All good'

data/test/test_base.rb

@@ -0,0 +1,79 @@
+require "rubygems"
+require "bundler/setup"
+
+gem 'test-unit'
+require 'test/unit'
+require 'yaml'
+require 'uber_config'
+begin
+  require File.join(File.dirname(__FILE__), '../lib/iron_worker')
+rescue Exception => ex
+  puts "Could NOT load current iron_worker: " + ex.message
+  raise ex
+end
+
+#IronWorker.logger.level = Logger::DEBUG
+IronWorker.service=nil
+IronWorker.config.merged_gems={}
+
+
+require_relative "workers/iw_test_worker"
+require_relative "workers/iw_test_worker_2"
+require_relative "workers/iw_test_worker_3"
+
+class TestBase < Test::Unit::TestCase
+
+  def setup
+    @config = UberConfig.load
+    puts "config: " + @config.inspect
+    raise "Config is nil! Ensure you have a config file in the proper place." if @config.nil?
+
+    @token = @config['iron']['token']
+    @project_id = @config['iron']['project_id']
+    puts "@token: #{@token}"
+    # new style
+    IronWorker.configure do |config|
+      config.token = @token
+      config.project_id = @project_id
+      config.host = @config['iron']['host'] if @config['iron']['host']
+      config.port = @config['iron']['port'] if @config['iron']['port']
+      config.scheme = @config['iron']['scheme'] if @config['iron']['scheme']
+      config.global_attributes["db_user"] = "sa"
+      config.global_attributes["db_pass"] = "pass"
+      #config.database = @config["database"]
+      config.force_upload = true
+      # config.skip_auto_dependencies = true
+    end
+  end
+
+  def puts_log(worker)
+    puts "LOG START:"
+    puts worker.get_log
+    puts ":LOG END"
+  end
+
+  def wait_for_task(params={})
+    tries = 0
+    status = nil
+    sleep 1
+    while  tries < 60
+      status = status_for(params)
+      puts "status: #{status['status']} -- " + status.inspect
+      if status["status"] == "complete" || status["status"] == "error"
+        break
+      end
+      sleep 2
+    end
+    status
+  end
+
+  def status_for(ob)
+    if ob.is_a?(Hash)
+      ob[:schedule_id] ? WORKER.schedule_status(ob[:schedule_id]) : WORKER.status(ob[:task_id])
+    else
+      ob.status
+    end
+  end
+
+
+end

data/test/test_database.rb

@@ -0,0 +1,31 @@
+require_relative 'test_base'
+require_relative 'workers/db_worker'
+require_relative 'models/db_model'
+class IronWorkerTests < TestBase
+
+  def setup
+    super
+    IronWorker.config.database = @config['database']
+  end
+
+  #def test_active_record
+  #  dbw = DbWorker.new
+  #  dbw.run_local
+  #  assert !dbw.ob.nil?
+  #  assert !dbw.ob.id.nil?
+  #
+  #  dbw.queue
+  #  # would be interesting if the object could update itself on complete. Like it would retrieve new values from
+  #  # finished job when calling status or something.
+  #
+  #  status = wait_for_task(dbw)
+  #  puts 'status: ' + status.inspect
+  #  puts "\n\n\nLOG START:"
+  #  puts dbw.get_log
+  #  puts "LOG END\n\n\n"
+  #  assert status["status"] == "complete"
+  #
+  #
+  #end
+
+end