qwirk 0.0.1

data/examples/task/publisher.rb

@@ -0,0 +1,25 @@
+require 'rumx'
+require 'qwirk'
+
+class Publisher
+  include Rumx::Bean
+
+  bean_attr_reader   :tasks, :list,    'Tasks', :list_type => :bean
+
+  bean_operation :perform_task, :void, 'Perform task which will send <count> messages to the worker and write the output to <output_file>', [
+      [ :task_id,     :string,  'Id for this task ',                         'task1'        ],
+      [ :count,       :integer, 'Count of messages',                         1000           ],
+      [ :message,     :string,  'String portion of the message to send',     'M'            ],
+      [ :sleep_time,  :float,   'Time to sleep between messages',            0.0            ],
+      [ :output_file, :string,  'Output file to write returned messages to', 'messages.out' ]
+  ]
+
+  def initialize(adapter_key)
+    @adapter_key = adapter_key
+    @tasks       = []
+  end
+
+  def perform_task(task_id, count, message, sleep_time, output_file)
+    @tasks << Task.new(@adapter_key, task_id, count, message, sleep_time, output_file)
+  end
+end

data/examples/task/qwirk_persist.yml

@@ -0,0 +1,5 @@
+--- 
+Foo: 
+  :min_count: 2
+  :max_count: 100
+  :sleep_time: 0.1

data/examples/task/task.rb

@@ -0,0 +1,36 @@
+require 'rumx'
+require 'qwirk'
+
+class Task
+  include Qwirk::Task
+
+  def initialize(adapter_key, task_id, total_count, message, sleep_time, output_file)
+    publisher = Qwirk[adapter_key].create_publisher(:queue_name => 'Foo', :marshal => :bson, :persistent => false, :response => {:time_to_live => 10000, :marshal => :string})
+    super(publisher, task_id, total_count)
+    @out = File.open(output_file, 'w')
+    (1..total_count).each do |i|
+      obj = {'count' => i, 'message' => message}
+      #puts "Publishing object: #{obj.inspect}"
+      publish(obj)
+      sleep sleep_time
+    end
+    finished_publishing
+  end
+
+  def on_response(request, response)
+    #puts "For request #{request}, got response #{response}"
+    @out.puts response
+  end
+
+  def on_exception(request, exception)
+    puts "For request #{request} got exception #{exception.message}"
+  end
+
+  def on_update()
+  end
+
+  def on_done
+    @out.close
+    puts "We're done"
+  end
+end

data/lib/qwirk.rb

@@ -0,0 +1,63 @@
+require 'rubygems'
+require 'qwirk/remote_exception'
+require 'qwirk/marshal_strategy'
+require 'qwirk/base_worker'
+require 'qwirk/worker_config'
+require 'qwirk/worker'
+require 'qwirk/request_worker'
+require 'qwirk/task'
+require 'qwirk/publisher'
+require 'qwirk/publish_handle'
+require 'qwirk/adapter'
+require 'qwirk/queue_adapter'
+#require 'qwirk/batch'
+require 'qwirk/manager'
+require 'qwirk/loggable'
+
+module Qwirk
+  extend Qwirk::Loggable
+
+  DEFAULT_NAME = 'Qwirk'
+
+  @@config = nil
+  @@hash   = {}
+
+  class MyBean
+    include Rumx::Bean
+
+    bean_attr_reader :adapters,  :hash,   'Adapters', :hash_type => :bean
+
+    def initialize(hash)
+      @adapters = hash
+    end
+  end
+
+  def self.config=(config)
+    @@config = config
+    Rumx::Bean.root.bean_add_child(DEFAULT_NAME, MyBean.new(@@hash))
+  end
+
+  def self.[](key)
+    if @@config.nil?
+      if defined?(Rails)
+        # Allow user to use JMS w/o modifying qwirk.yml which could be checked in and hose other users
+        env = ENV['QWIRK_ENV'] || Rails.env
+        self.config = YAML.load(ERB.new(File.read(Rails.root.join("config", "qwirk.yml")), nil, '-').result(binding))[env]
+        Manager.default_options = {
+            :persist_file    => Rails.root.join('log', 'qwirk_persist.yml'),
+            :worker_file     => Rails.root.join('config', 'qwirk_workers.yml'),
+            :stop_on_signal  => true,
+            :env             => env,
+        }
+      end
+    end
+    raise 'Qwirk not configured' unless @@config && @@config[key]
+    @@hash[key] ||= Qwirk::Adapter.new(@@config[key])
+  end
+
+  def self.fail_queue_name(queue_name)
+    return "#{queue_name.to_s}Fail"
+  end
+end
+
+require 'qwirk/engine' if defined?(Rails)

data/lib/qwirk/adapter.rb

@@ -0,0 +1,45 @@
+module Qwirk
+
+  # Defines the queuing strategy.  Currently, only JMS and InMem.
+  class Adapter
+    include Rumx::Bean
+
+    attr_reader :config, :log_times, :adapter_info
+
+    def initialize(config)
+      @config = config.dup
+      @log_times = config.delete(:log_times)
+
+      adapter = config.delete(:adapter)
+      raise "No adapter definition" unless adapter
+      namespace = Qwirk::QueueAdapter.const_get(adapter)
+      @adapter_info = nil
+      if namespace.respond_to?(:init)
+        @adapter_info = namespace.init(config)
+      end
+      @publisher_klass     = namespace.const_get('Publisher')
+      @worker_config_klass = namespace.const_get('WorkerConfig')
+    end
+
+    def create_publisher(options={})
+      @publisher_parent ||= Rumx::Beans::Folder.new
+      publisher = Publisher.new(self, @config.merge(options))
+      @publisher_parent.bean_add_child(publisher.to_s, publisher)
+      return publisher
+    end
+
+    def create_manager(options={})
+      manager = Manager.new(self, @config.merge(options))
+      bean_add_child(:Workers, manager)
+      return manager
+    end
+
+    def create_adapter_publisher(queue_name, topic_name, options, response_options)
+      @publisher_klass.new(self, queue_name, topic_name, options, response_options)
+    end
+
+    def create_adapter_worker_config(parent, queue_name, topic_name, options, response_options)
+      @worker_config_klass.new(self, parent, queue_name, topic_name, options, response_options)
+    end
+  end
+end

data/lib/qwirk/base_worker.rb

@@ -0,0 +1,96 @@
+require 'rumx'
+
+module Qwirk
+  # TODO: Is this necessary anymore or just put in worker.rb?  Decide when flat file queue_adapter is implemented.
+  module BaseWorker
+
+    attr_accessor :index, :thread, :config
+
+    module ClassMethods
+      def default_name
+        name = self.name.sub(/Worker$/, '')
+        name.sub(/::/, '_')
+      end
+
+      # Dynamic class create form WorkerConfig and extended through config_accessor, etc calls that will be defined in the worker
+      def config_class
+        @config_class ||= Class.new(WorkerConfig)
+      end
+
+      # Default values for all the config attributes
+      def default_config
+        @default_config ||= WorkerConfig.initial_default_config
+      end
+
+      #config_accessor :sleep_time, :float, 'Number of seconds to sleep between messages', 5
+      def config_accessor(name, type, description, default_value=nil)
+        make_bean_attr(:bean_attr_accessor, name, type, description, default_value)
+      end
+
+      def config_reader(name, type, description, default_value=nil)
+        make_bean_attr(:bean_attr_reader, name, type, description, default_value)
+      end
+
+      def config_writer(name, type, description, default_value=nil)
+        make_bean_attr(:bean_attr_writer, name, type, description, default_value)
+      end
+
+      def define_configs(configs)
+        @configs = configs
+      end
+
+      def each_config(&block)
+        # Configs are either defined with a define_configs call or default to a single instance with default_config
+        if @configs
+          @configs.each do |name, config|
+            yield name, default_config.merge(config)
+          end
+        else
+          yield default_name, default_config
+        end
+      end
+
+      #######
+      private
+      #######
+
+      def make_bean_attr(attr_method, name, type, description, default_value)
+        config_class.send(attr_method, name, type, description, :config_item => true)
+        default_config[name.to_sym] = default_value
+      end
+    end
+
+    def self.included(base)
+      Rumx::Bean.included(base)
+      base.extend(ClassMethods)
+      if base.kind_of?(Class)
+        @worker_classes ||= []
+        @worker_classes << base unless @worker_classes.include?(base)
+      end
+    end
+
+    def self.worker_classes
+      @worker_classes ||= []
+    end
+
+    def start
+      raise "Need to override start method in #{self.class.name}"
+    end
+
+    def stop
+      raise "Need to override stop method in #{self.class.name}"
+    end
+
+    def join
+      thread.join
+    end
+
+    def status
+      raise "Need to override status method in #{self.class.name}"
+    end
+
+    def to_s
+      "#{config.name}:#{index}"
+    end
+  end
+end

data/lib/qwirk/batch.rb

@@ -0,0 +1,4 @@
+require 'qwirk/batch/acquire_file_strategy'
+require 'qwirk/batch/parse_file_strategy'
+require 'qwirk/batch/file_worker'
+require 'qwirk/batch/job_status'

data/lib/qwirk/batch/acquire_file_strategy.rb

@@ -0,0 +1,47 @@
+module Qwirk
+  module Batch
+    class AcquireFileStrategy
+      def initialize(options)
+        @glob         = options[:glob]
+        raise "file_options glob value not set" unless @glob
+        @poll_time    = (options[:poll_time] || 10.0).to_f
+        # Ftp's could be in progress, make sure the file is at least 60 seconds old by default before processing
+        @age          = (options[:age] || 60).to_i
+        @stopped      = false
+      end
+
+      # Returns the next file or nil if stopped
+      def next_file
+        until @stopped
+          Dir.glob(@glob).each do |file|
+            unless file.match /\.(processing|completed)$/
+              return file if (Time.now - File.mtime(file) > @age)
+            end
+          end
+          @sleep_thread = Thread.current
+          sleep @poll_time
+        end
+        return nil
+      end
+
+      def mark_file_as_processing(file)
+        new_file = file + '.processing'
+        File.rename(file, new_file)
+        return new_file
+      end
+
+      def complete_file(file)
+        file.match(/(.*)\.processing$/) || raise("#{file} is not currently being processed")
+        new_file = $1 + '.completed'
+        File.rename(file, new_file)
+        return new_file
+      end
+
+      def stop
+        @stopped = true
+        @sleep_thread.wakeup if @sleep_thread
+      end
+
+    end
+  end
+end

data/lib/qwirk/batch/active_record.rb

@@ -0,0 +1,3 @@
+require 'qwirk/batch/active_record/batch_job'
+require 'qwirk/batch/active_record/failed_record'
+require 'qwirk/batch/active_record/outstanding_record'

data/lib/qwirk/batch/active_record/batch_job.rb

@@ -0,0 +1,111 @@
+module Qwirk::Batch::ActiveRecord
+  class BatchJob < ActiveRecord::Base
+    include Qwirk::Batch::JobStatus
+
+    set_table_name 'mt_batch_jobs'
+
+    has_many :failed_records,      :dependent => :destroy
+    has_many :outstanding_records, :dependent => :destroy
+
+    validates :file, :worker_name, :finished_count, :status, :presence => true
+      #t.string    :file,              :null => false
+      #t.string    :worker_name,       :null => false
+      #t.integer   :total_count
+      #t.integer   :finished_count,    :null => false, :default => 0
+      #t.column    :status, 'char(8)', :null => false, :default => Qwirk::Batch::JobStatus::INITED
+      #t.datetime  :created_at,        :null => false
+      #t.datetime  :updated_at,        :null => false
+
+    # Acquire this file if it hasn't already been acquired.
+    def self.acquire(file, worker_name)
+      return nil if find_by_file_and_worker_name(file, worker_name)
+      create!(:file => file, :worker_name  => worker_name)
+    rescue ActiveRecord::ActiveRecordError => e
+      Rails.logger.warn("Assuming race condition (duplicate index) for BatchJob file=#{file} worker=#{worker_name}: #{e.message}")
+      return nil
+    end
+
+    # Acquire and resume a paused job if available
+    def self.resume_paused_job(worker_name)
+      transaction do
+        job = where(:worker_name => worker_name, status => PAUSED).lock(true).first
+        return nil unless job
+        job.outstanding_records.each do |record|
+          job.start_record(record.file_position)
+          record.destroy
+        end
+        job.update_attribute(:status => RUNNING)
+      end
+    end
+
+    def initialize(opts={})
+      super
+      @outstanding_array = []
+    end
+
+    def run(total_count)
+      update_attributes(:status => RUNNING, :total_count => total_count)
+    end
+
+    def pause
+      save_outstanding_array
+      update_attribute(:status => STOPPED)
+    end
+
+    def abort
+      save_outstanding_array
+      update_attribute(:status => ABORTED)
+    end
+
+    def cancel
+      save_outstanding_array
+      update_attribute(:status => CANCELED)
+    end
+
+    def finish
+      update_attribute(:status => FINISHED)
+    end
+
+    def start_record(file_position)
+      @outstanding_array << file_position
+    end
+
+    def finish_record(file_position)
+      @outstanding_array.delete(file_position)
+      update_attribute(:finished_count => finished_count + 1)
+    end
+
+    def fail_record(file_position, message)
+      @outstanding_array.delete(file_position)
+      failed_records.create!(:file_position => file_position, :message => message)
+    end
+
+    def retry_failed_record
+      failed_record = failed_records.first
+      return nil unless failed_record
+      failed_record.destroy
+      return failed_record.file_position
+    end
+
+    def outstanding_array
+      @outstanding_array
+    end
+
+    def failed_hash
+      hash = {}
+      failed_records.each do |failed_record|
+        hash[failed_record.file_position] = failed_record.message
+      end
+      return hash
+    end
+
+    private
+
+    def save_outstanding_array
+      outstanding_records.each {|record| record.destroy}
+      @outstanding_array.each do |file_position|
+        outstanding_records.create!(:file_position => file_position)
+      end
+    end
+  end
+end