iron_worker 2.1.0

data/lib/iron_worker/config.rb

@@ -0,0 +1,265 @@
+module IronWorker
+
+
+  # Config is used to setup the IronWorker client.
+  # You must set the access_key and secret_key.
+  #
+  # config.global_attributes allows you to specify attributes that will automatically be set on every worker,
+  #    this is good for database connection information or things that will be used across the board.
+  #
+  # config.database configures a database connection. If specified like ActiveRecord, IronWorker will automatically establish a connection
+  # for you before running your worker.
+  class Config
+    attr_accessor :token,
+                  :project_id,
+                  :scheme,
+                  :host,
+                  :port,
+                  :global_attributes,
+                  :models,
+                  :mailers,
+                  #:gems, # todo: move anything that uses this to merged_gems
+                  :database,
+                  :mailer,
+                  :extra_requires,
+                  #:auto_merge,
+                  :server_gems,
+                  :merged,
+                  :unmerged,
+                  :merged_gems,
+                  :unmerged_gems,
+                  :force_upload
+
+
+    def initialize
+      @global_attributes = {}
+      @extra_requires = []
+      @merged = {}
+      @unmerged = {}
+      @merged_gems = {}
+      @unmerged_gems = {}
+      @mailers = {}
+
+    end
+
+    def access_key=(x)
+      raise "IronWorker Config Error: access_key and secret_key are no longer used. The new IronWorker gem requires a couple of small configuration changes, please see: http://docs.IronWorker.com/ruby/new-gem-v2-update-guide for information."
+    end
+    def secret_key=(x)
+      raise "IronWorker Config Error: access_key and secret_key are no longer used. The new IronWorker gem requires a couple of small configuration changes, please see: http://docs.IronWorker.com/ruby/new-gem-v2-update-guide for information."
+    end
+
+    @gems_to_skip = ['actionmailer', 'actionpack', 'activemodel', 'activeresource', 'activesupport',
+                     'bundler',
+                     'mail',
+                     'mysql2',
+                     'rails',
+                     'tzinfo' # HUGE!
+    ]
+
+    def self.gems_to_skip
+      @gems_to_skip
+    end
+
+    def auto_merge=(b)
+      if b
+        IronWorker.logger.info "Initializing IronWorker for Rails 3..."
+        start_time = Time.now
+        IronWorker.configure do |c2|
+          models_path = File.join(Rails.root, 'app/models/*.rb')
+          models = Dir.glob(models_path)
+          c2.models = models
+          models.each { |model| c2.merge(model) }
+          mailers_path = File.join(Rails.root, 'app/mailers/*.rb')
+          Dir.glob(mailers_path).collect { |m| c2.mailers[File.basename(m)] = {:filename=>m, :name => File.basename(m), :path_to_templates=>File.join(Rails.root, "app/views/#{File.basename(m, File.extname(m))}")} }
+          c2.extra_requires += ['active_support/core_ext', 'action_mailer']
+          #puts 'DB FILE=' + File.join(Rails.root, 'config', 'database.yml').to_s
+          if defined?(ActiveRecord) && File.exist?(File.join(Rails.root, 'config', 'database.yml'))
+            c2.extra_requires += ['active_record']
+            c2.database = Rails.configuration.database_configuration[Rails.env]
+          else
+            #puts 'NOT DOING ACTIVERECORD'
+          end
+
+          if defined?(ActionMailer) && ActionMailer::Base.smtp_settings
+            c2.mailer = ActionMailer::Base.smtp_settings
+          end
+          c2.merged_gems.merge!(get_required_gems) if defined?(Bundler)
+          IronWorker.logger.debug "MODELS " + c2.models.inspect
+          IronWorker.logger.debug "MAILERS " + c2.mailers.inspect
+          IronWorker.logger.debug "DATABASE " + c2.database.inspect
+          #IronWorker.logger.debug "GEMS " + c2.gems.inspect
+        end
+        end_time = Time.now
+        IronWorker.logger.info "IronWorker initialized. Duration: #{((end_time.to_f-start_time.to_f) * 1000.0).to_i} ms"
+      end
+    end
+
+
+    def get_required_gems
+      gems_in_gemfile = Bundler.environment.dependencies.select { |d| d.groups.include?(:default) }
+      IronWorker.logger.debug 'gems in gemfile=' + gems_in_gemfile.inspect
+      gems = {}
+      specs = Bundler.load.specs
+      IronWorker.logger.debug 'Bundler specs=' + specs.inspect
+      IronWorker.logger.debug "gems_to_skip=" + self.class.gems_to_skip.inspect
+      specs.each do |spec|
+        IronWorker.logger.debug 'spec.name=' + spec.name.inspect
+        IronWorker.logger.debug 'spec=' + spec.inspect
+        if self.class.gems_to_skip.include?(spec.name)
+          IronWorker.logger.debug "Skipping #{spec.name}"
+          next
+        end
+#        next if dep.name=='rails' #monkey patch
+        gem_info = {:name=>spec.name, :version=>spec.version}
+        gem_info[:auto_merged] = true
+        gem_info[:merge] = true
+# Now find dependency in gemfile in case user set the require
+        dep = gems_in_gemfile.find { |g| g.name == gem_info[:name] }
+        if dep
+          IronWorker.logger.debug 'dep found in gemfile: ' + dep.inspect
+          IronWorker.logger.debug 'autorequire=' + dep.autorequire.inspect
+          gem_info[:require] = dep.autorequire if dep.autorequire
+#        spec = specs.find { |g| g.name==gem_info[:name] }
+        end
+        gem_info[:version] = spec.version.to_s
+        gems[gem_info[:name]] = gem_info
+        path = IronWorker::Service.get_gem_path(gem_info)
+        if path
+          gem_info[:path] = path
+          if gem_info[:require].nil? && dep
+            # see if we should try to require this in our worker
+            require_path = gem_info[:path] + "/lib/#{gem_info[:name]}.rb"
+            IronWorker.logger.debug "require_path=" + require_path
+            if File.exists?(require_path)
+              IronWorker.logger.debug "File exists for require"
+              gem_info[:require] = gem_info[:name]
+            else
+              IronWorker.logger.debug "no require"
+#              gem_info[:no_require] = true
+            end
+          end
+        else
+          IronWorker.logger.warn "Could not find '#{gem_info[:name]}' specified in Bundler, continuing anyways."
+        end
+#        else
+#          IronWorker.logger.warn "Could not find gem spec for #{gem_info[:name]}"
+#          raise "Could not find gem spec for #{gem_info[:name]}"
+#        end
+      end
+      gems
+    end
+
+    def get_server_gems
+      return []
+      # skipping this now, don't want any server dependencies if possible
+      self.server_gems = IronWorker.service.get_server_gems unless self.server_gems
+      self.server_gems
+    end
+
+    def get_atts_to_send
+      config_data = {}
+      config_data['token'] = token
+      config_data['project_id'] = project_id
+      config_data['database'] = self.database if self.database
+      config_data['mailer'] = self.mailer if self.mailer
+      config_data['global_attributes'] = self.global_attributes if self.global_attributes
+      config_data['scheme'] = self.scheme if self.scheme
+      config_data['host'] = self.host if self.host
+      config_data['port'] = self.port if self.port
+      config_data
+    end
+
+    def merge(file)
+      f2 = IronWorker::MergeHelper.check_for_file(file, caller[2])
+      fbase = f2[:basename]
+      ret = f2
+      @merged[fbase] = ret
+      ret
+    end
+
+    def unmerge(file)
+      f2 = IronWorker::MergeHelper.check_for_file(file, caller[2])
+      fbase = f2[:basename]
+      @unmerged[fbase] =f2
+      @merged.delete(fbase)
+    end
+
+    # Merge a gem globally here
+    def merge_gem(gem_name, options={})
+      merged_gems[gem_name.to_s] = IronWorker::MergeHelper.create_gem_info(gem_name, options)
+    end
+
+    # Unmerge a global gem
+    def unmerge_gem(gem_name)
+      gs = gem_name.to_s
+      gem_info = {:name=>gs}
+      unmerged_gems[gs] = gem_info
+      merged_gems.delete(gs)
+    end
+
+  end
+
+
+  class MergeHelper
+
+    # callerr is original file that is calling the merge function, ie: your worker.
+    # See Base for examples.
+    def self.check_for_file(f, callerr)
+      IronWorker.logger.debug 'Checking for ' + f.to_s
+      f = f.to_str
+      f_ext = File.extname(f)
+      if f_ext.empty?
+        f_ext = ".rb"
+        f << f_ext
+      end
+      exists = false
+      if File.exist? f
+        exists = true
+      else
+        # try relative
+        #          p caller
+        f2 = File.join(File.dirname(callerr), f)
+        puts 'f2=' + f2
+        if File.exist? f2
+          exists = true
+          f = f2
+        end
+      end
+      unless exists
+        raise "File not found: " + f
+      end
+      f = File.expand_path(f)
+      require f if f_ext == '.rb'
+      ret = {}
+      ret[:path] = f
+      ret[:extname] = f_ext
+      ret[:basename] = File.basename(f)
+      ret[:name] = ret[:basename]
+      ret
+    end
+
+    def self.create_gem_info(gem_name, options={})
+      gem_info = {:name=>gem_name, :merge=>true}
+      if options.is_a?(Hash)
+        gem_info.merge!(options)
+        if options[:include_dirs]
+          gem_info[:include_dirs] = options[:include_dirs].is_a?(Array) ? options[:include_dirs] : [options[:include_dirs]]
+        end
+      else
+        gem_info[:version] = options
+      end
+      gem_info[:require] ||= gem_name
+
+      path = IronWorker::Service.get_gem_path(gem_info)
+      IronWorker.logger.debug "Gem path=#{path}"
+      if !path
+        raise "Gem '#{gem_name}' not found."
+      end
+      gem_info[:path] = path
+      gem_info
+    end
+  end
+
+end
+

data/lib/iron_worker/rails2_init.rb

@@ -0,0 +1,8 @@
+#puts "Initializing list of Rails models..."
+IronWorker.configure do |config|
+  path = File.join(Rails.root, 'app/models/*.rb')
+#  puts 'path=' + path
+  config.models = Dir.glob(path)
+  config.extra_requires += ['active_support/core_ext', 'active_record', 'action_mailer']
+#  puts 'config.models=' + config.models.inspect
+end

data/lib/iron_worker/railtie.rb

@@ -0,0 +1,16 @@
+# see http://api.rubyonrails.org/classes/Rails/Railtie.html
+
+require 'iron_worker'
+require 'rails'
+
+module IronWorker
+  class Railtie < Rails::Railtie
+
+
+    initializer "iron_worker.configure_rails_initialization" do |app|
+
+    end
+
+
+  end
+end

data/lib/iron_worker/server/overrides.rb

@@ -0,0 +1,199 @@
+# This is used when a bad worker is uploaded.
+
+module IronWorker
+
+  class InvalidWorkerError < StandardError; end
+
+  class << self
+    def running_class=(rc)
+      @running_class = rc
+    end
+    def running_class
+      @running_class
+    end
+    def task_data=(td)
+      @task_data = td
+    end
+    def task_data
+      @task_data
+    end
+  end
+
+  def self.disable_queueing()
+    @queueing_enabled = false
+  end
+
+  def self.enable_queueing()
+    @queueing_enabled = true
+  end
+
+  def self.queueing_enabled?
+    @queueing_enabled
+  end
+
+
+  class Base
+
+    class << self
+
+      def merge(*files)
+        #files.each do |file|
+        #  file = file.to_s
+        #  unless file.end_with?(".rb")
+        #    file << ".rb"
+        #  end
+        #  #puts 'code_dir=' + code_dir.inspect
+        #  filename = File.join(code_dir, File.basename(file))
+        #  #puts "merge #{filename}"
+        #  #puts "FILENAME #{filename}"
+        #  require filename if File.exist?(filename) # for backwards compatability
+        #end
+
+      end
+
+
+      def merge_folder(path)
+        #puts "PATH=#{path}"
+        #puts "#{code_dir}/#{Digest::MD5.hexdigest(path)}/**/*.rb"
+        #Dir["#{code_dir}/#{Digest::MD5.hexdigest(path)}/**/*.rb"].each do |f|
+        #  puts "requiring #{f.inspect}"
+        #  require f if File.exist?(f)
+        #end
+      end
+
+      def unmerge(*files)
+        # ignore this here
+      end
+
+      def merge_mailer(mailer, params=nil)
+        #merge(mailer)
+      end
+
+      def merge_gem(gem, version=nil)
+        #gem_info = {:name=>gem}
+        #if version.is_a?(Hash)
+        #  gem_info.merge!(version)
+        #else
+        #  gem_info[:version] = version
+        #end
+        #gem_name =(gem.match(/^[a-zA-Z0-9\-_]+/)[0])
+        #$LOAD_PATH << File.join(code_dir, "/gems/#{gem_name}") #backwards compatibility, should be removed later
+        #$LOAD_PATH << File.join(code_dir, "/gems/#{gem_name}/lib")
+        #                                                       # what's the diff here? This one seems more common: $:.unshift File.join(File.dirname(__FILE__), "/gems/#{gem_name}")
+        #                                                       #puts 'gem_info = ' + gem_info.inspect
+        #require gem_info[:require] || gem_name
+      end
+
+      def merge_worker(file, class_name)
+
+      end
+
+    end
+
+    def is_remote?
+      true
+    end
+
+    #:todo remove this method later, when new iron_worker gem will be released
+    def is_local?
+      !is_remote?
+    end
+
+    def log(str)
+      puts str.to_s
+    end
+
+    def set_progress(hash)
+      #puts 'set_progress self=' + self.inspect
+      IronWorker.service.set_progress(self.task_id, hash)
+    end
+
+    def something
+      puts 'which class? ' + self.class.name
+    end
+
+    def user_dir
+#      puts 'user_dir=' + @context.user_dir.to_s
+      @user_dir || "./"
+    end
+
+    def sw_set_data(data)
+      if data["attr_encoded"]
+        # new way, attributes are base 64 encoded
+        data = JSON.parse(Base64.decode64(data["attr_encoded"]))
+      end
+
+      data.each_pair do |k, v|
+        next unless k[0] == "@"
+#        puts "setting instance_variable #{k}=#{v}"
+        self.instance_variable_set(k, v)
+      end
+
+    end
+
+    def upload_if_needed(options={})
+      puts "No uploading in worker service."
+    end
+
+    alias_method :orig_queue, :queue
+    alias_method :orig_schedule, :schedule
+    alias_method :orig_status, :status
+
+    def queue(options={})
+      if IronWorker.queueing_enabled? && (!same_clazz? || options[:recursive])
+        orig_queue(options)
+#        data = sw_get_data()
+#        queue_other(self.class.name, data)
+      else
+        log (IronWorker.queueing_enabled? ? "WARNING: Recursion detected in queueing, pass in :recursive=>true to bypass this." : "Queuing disabled while loading.")
+      end
+    end
+
+    def schedule(schedule)
+      if IronWorker.queueing_enabled? && (!same_clazz? || schedule[:recursive])
+        orig_schedule(schedule)
+#        data = sw_get_data()
+#        schedule_other(self.class.name, data, schedule)
+      else
+        log (IronWorker.queueing_enabled? ? "WARNING: Recursion detected in scheduling." : "Scheduling disabled while loading.")
+      end
+
+    end
+
+    def status
+      if IronWorker.queueing_enabled?
+        orig_status
+      else
+        log "Status disabled while loading."
+      end
+    end
+
+    def same_clazz?
+      IronWorker.running_class == self.class
+    end
+
+  end
+
+
+  class Service < IronWorker::Api::Client
+    def upload(filename, class_name, options={})
+      #puts "Skipping upload, We don't upload from run.rb!"
+      # don't upload, should already be here.
+    end
+
+    def add_sw_params(hash_to_send)
+      hash_to_send["token"] = self.config.token
+      hash_to_send["project_id"] = self.config.project_id
+      hash_to_send["api_version"] = IronWorker.api_version
+    end
+  end
+
+
+  module UsedInWorker
+    def log(str)
+      puts str.to_s
+    end
+  end
+
+
+end