qwirk 0.0.1

data/lib/qwirk/batch/active_record/failed_record.rb

@@ -0,0 +1,5 @@
+module Qwirk::Batch::ActiveRecord
+  class FailedRecord < ActiveRecord::Base
+    set_table_name 'mt_failed_records'
+  end
+end

data/lib/qwirk/batch/active_record/outstanding_record.rb

@@ -0,0 +1,6 @@
+module Qwirk::Batch::ActiveRecord
+  class OutstandingRecord < ActiveRecord::Base
+    set_table_name 'mt_outstanding_records'
+  end
+end
+

data/lib/qwirk/batch/file_status_strategy.rb

@@ -0,0 +1,86 @@
+require 'tmpdir'
+
+module Qwirk
+  module Batch
+    class FileStatusStrategy
+
+      attr_reader :finished_count
+
+      if defined?(Rails) && defined?(Rails.root)
+        @@persist_dir = File.join(Rails.root, 'log', 'qwirk')
+      else
+        @@persist_dir = File.join(Dir.tmpdir, 'qwirk')
+      end
+
+      def self.persist_dir
+        @@persist_dir
+      end
+
+      def self.persist_dir=(dir)
+        @@persist_dir = dir
+      end
+
+      def initialize(options)
+        @worker_name = options[:worker_name]
+        @dir = options[:persist_dir] || @@persist_dir
+      end
+
+      # Resume any previous jobs that were stopped
+      def resume?
+        false
+      end
+
+      def start(file)
+        @file           = file
+        @pending_hash   = {}
+        @fail_array     = []
+        @finished_count = 0
+      end
+
+      def stop
+        return unless @file
+        save_yaml = {
+            :file           => @file,
+            :pending        => @pending_hash,
+            :fail           => @fail_array,
+            :finished_count => @finished_count
+        }
+      end
+
+      def finish
+        @file = nil
+      end
+
+      def start_record(message_id, file_pos)
+        @pending_hash[message_id] = file_pos
+      end
+
+      def finish_record(message_id)
+        @pending_hash.delete(message_id)
+        @finished_count += 1
+      end
+
+      def fail_record(message_id)
+        file_pos = @pending_hash.delete(message_id)
+        raise "Invalid message #{message_id}, not in pending_hash" unless file_pos
+        @fail_array << file_pos
+      end
+
+      def pending_count
+        @pending_hash.size
+      end
+
+      def failed_count
+        @fail_array.size
+      end
+
+      #######
+      private
+      #######
+
+      def save
+        persist_file =
+      end
+    end
+  end
+end

data/lib/qwirk/batch/file_worker.rb

@@ -0,0 +1,228 @@
+module Qwirk
+  module Batch
+
+    # Batch worker which reads records from files and queues them up for a separate worker (Qwirk::QueueAdapter::JMS::RequestWorker) to process.
+    # For instance, a worker of this type might look as follows:
+    #   class MyBatchWorker
+    #     include Qwirk::Batch::FileWorker
+    #
+    #     file :glob => '/home/batch_files/input/**', :age => 1.minute, :max_outstanding_records => 100, :fail_threshold => 0.8, :save_period => 30.seconds
+    #     marshal :string
+    #   end
+    #
+    # The following options can be used for configuring the class
+    #   file:
+    #     :glob => <glob_path>
+    #       The path where files will be processed from.  Files will be renamed with a .processing extension while they are being processed
+    #       and to a .completed extension when processing is completed.
+    #     :age => <duration>
+    #       How old a file must be before it will be processed.  This is to prevent files that are in the middle of being uploaded from begin acquired.
+    #     :poll_time => <duration>
+    #       How often the glob is queried for new files.
+    #     :max_outstanding_records => <integer>
+    #       This is how many outstanding records can be queued at a time.
+    #     :
+    module FileWorker
+      include Qwirk::BaseWorker
+
+      module ClassMethods
+        # Define the marshaling and time_to_live that will occur on the response
+        def file(options)
+          @file_options = options
+        end
+
+        def file_options
+          @file_options
+        end
+
+        def marshal(marshal_type)
+          @marshal_type = marshal_type
+        end
+
+        def marshal_type
+          @marshal_type
+        end
+
+        def queue(name, opts={})
+          @queue_name = name
+          @reply_queue_name = opts[:reply_queue]
+        end
+
+        def queue_name
+          @queue_name
+        end
+
+        def reply_queue_name
+          @reply_queue_name
+        end
+      end
+
+      def self.included(base)
+        Qwirk::BaseWorker.included(base)
+        base.extend(ClassMethods)
+      end
+
+      # Set the global default acquire_file_strategy for an organization
+      def self.default_acquire_file_strategy=(default_strategy)
+        @@default_acquire_file_strategy = default_strategy
+      end
+
+      def self.default_acquire_file_strategy
+        @@default_acquire_file_strategy
+      end
+
+      # Set the global default parse_file_strategy for an organization
+      def self.default_parse_file_strategy=(default_strategy)
+        @@default_parse_file_strategy = default_strategy
+      end
+
+      def self.default_parse_file_strategy
+        @@default_parse_file_strategy
+      end
+
+      # Set the global default process_file_strategy for an organization
+      def self.default_file_status_strategy=(default_strategy)
+        @@default_file_status_strategy = self.file_status_strategy_from_sym(default_strategy)
+      end
+
+      def self.default_file_status_strategy
+        self.file_status_strategy_to_sym(@@default_file_status_strategy)
+      end
+
+      def self.file_status_strategy_from_sym(strategy)
+        if strategy.kind_of?(Symbol)
+          if strategy == :active_record
+            require 'qwirk/batch/active_record'
+            Qwirk::Batch::ActiveRecord::BatchJob
+          elsif strategy == :mongoid
+            require 'qwirk/batch/mongoid'
+            Qwirk::Batch::Mongoid::BatchJob
+          else
+            raise "Invalid symbol for file_status_strategy=#{strategy}"
+          end
+        else
+          strategy
+        end
+      end
+
+      def self.file_status_strategy_to_sym(strategy)
+        if strategy == Qwirk::Batch::ActiveRecord::BatchJob
+          :active_record
+        elsif strategy == Qwirk::Batch::ActiveRecord::BatchJob
+          :mongoid
+        else
+          strategy
+        end
+      end
+
+      self.default_acquire_file_strategy = AcquireFileStrategy
+      self.default_parse_file_strategy = ParseFileStrategy
+      self.default_file_status_strategy = begin
+        if defined?(ActiveRecord)
+          :active_record
+        elsif defined?(Mongoid)
+          :mongoid
+        else
+          nil
+        end
+      end
+
+      def initialize(opts={})
+        super
+        @marshal_type     = (self.class.marshal_type || :ruby).to_s
+        @marshaler        = MarshalStrategy.find(@marshal_type)
+        @stopped          = false
+        @queue_name       = opts[:queue_name] || self.class.queue_name || (self.name.match(/(.*)File$/) && $1)
+        raise "queue_name not specified in #{self.class.name}" unless @queue_name
+        @reply_queue_name = opts[:reply_queue_name] || self.class.reply_queue_name || "#{@queue_name}Reply"
+
+        file_options = self.class.file_options
+        raise "file_options not set for #{self.class.name}" unless file_options
+        acquire_strategy_class = file_options.delete(:acquire_strategy) || FileWorker.default_acquire_file_strategy
+        parse_strategy_class   = file_options.delete(:parse_strategy)   || FileWorker.default_parse_file_strategy
+        status_strategy        = file_options.delete(:status_strategy)  || FileWorker.default_parse_file_strategy
+        raise 'No status_strategy defined' unless status_strategy
+        status_strategy_class  = FileWorker.file_status_strategy_from_sym(status_strategy)
+        @acquire_file_strategy = acquire_strategy_class.new(file_options)
+        @parse_file_strategy   = parse_strategy_class.new(file_options)
+        @file_status_strategy  = status_strategy_class.new(file_options)
+        @max_outstanding_records = file_options[:max_outstanding_records] || 10
+      end
+
+      def start
+        #TODO: look for current job
+        while file = @acquire_file_strategy.acquire_file do
+          @parse_file_strategy.open(file)
+          @reply_thread = Thread.new do
+            java.lang.Thread.current_thread.name = "Qwirk worker (reply): #{worker}"
+            reply_event_loop
+          end
+          begin
+            @record_total = @parse_file_strategy.record_total
+            process_file
+          ensure
+            @parse_file_strategy.close
+          end
+        end
+      end
+
+
+      def stop
+        @stopped = true
+        @acquire_file_strategy.stop
+      end
+
+      def join
+        thread.join
+      end
+
+      def status
+        raise "Need to override status method in #{self.class.name}"
+      end
+
+      #########
+      protected
+      #########
+
+      def process_file
+        while record = @parse_file_strategy.next_record
+          obj = record_to_object(record)
+        end
+      end
+
+      # Allow extenders to manipulate the file record before sticking it on the queue
+      def record_to_object(record)
+        record
+      end
+
+      # Perform any additional operations on the given responses
+      def process_response(obj)
+      end
+
+      #######
+      private
+      #######
+
+      def reply_event_loop
+        @reply_session = Qwirk::QueueAdapter::JMS::Connection.create_session
+        @consumer = @reply_session.consumer(:queue_name => @queue_name)
+        @reply_session.start
+
+        while !@stopped && message = @consumer.receive
+          @message_mutex.synchronize do
+            obj = Qwirk::QueueAdapter::JMS.parse_response(message)
+            process_response(obj)
+            message.acknowledge
+          end
+          # TODO: @time_track now in WorkerConfig
+          #Qwirk.logger.info {"#{self}::on_message (#{('%.1f' % (@time_track.last_time*1000.0))}ms)"} if Qwirk::QueueAdapter::JMS::Connection.log_times?
+        end
+        @status = 'Exited'
+        Qwirk.logger.info "#{self}: Exiting"
+      rescue Exception => e
+        @status = "Exited with exception #{e.message}"
+        Qwirk.logger.error "#{self}: Exception, thread terminating: #{e.message}\n\t#{e.backtrace.join("\n\t")}"
+      end
+    end
+  end
+end

data/lib/qwirk/batch/job_status.rb

@@ -0,0 +1,29 @@
+module Qwirk
+  module Batch
+    module JobStatus
+
+      # Note: String max is set to 8 as defined in the schema.rb file
+
+      # Job has been acquired but is not yet running
+      INITED   = 'Inited'
+
+      # Job is currently running
+      RUNNING  = 'Running'
+
+      # Job has paused because the worker has been commanded to stop
+      PAUSED  = 'Paused'
+
+      # A client has canceled the job
+      CANCELED = 'Canceled'
+
+      # The job has been aborted due to threshold constraints (too many record failures)
+      ABORTED  = 'Aborted'
+
+      # The job has finished
+      FINISHED = 'Finished'
+
+      STATUSES = [INITED, RUNNING, PAUSED, CANCELED, ABORTED, FINISHED].freeze
+
+    end
+  end
+end

data/lib/qwirk/batch/parse_file_strategy.rb

@@ -0,0 +1,48 @@
+module Qwirk
+  module Batch
+    # Default strategy for parsing a file
+    class ParseFileStrategy
+      def initialize(file_options)
+      end
+
+      # Open the file for processing
+      def open(file)
+        @file = file
+        @fin = File.open(@file, 'r')
+        @line_count = 0
+      end
+
+      # Goto a specific position in the file.  This strategy uses line_counts.  @fin.seek and @fin.tell would
+      # be faster but wouldn't allow the file to be edited if it was formatted incorrectly.
+      def file_position=(line_count)
+        if @line_count > line_count
+          @fin.seek(0)
+          @line_count = 0
+        end
+        next_record while @line_count < line_count
+      end
+
+      # Return the current position in the file
+      def file_position
+        @line_count
+      end
+
+      # Read the next record from the file
+      def next_record
+        @line_count += 1
+        @fin.gets
+      end
+
+      # Return an estimate of the total records in the file
+      def record_total
+        # Faster than reading file in ruby but won't count incomplete lines
+        %x{wc -l #{@file}}.split.first.to_i
+      end
+
+      # Close the file
+      def close
+        @fin.close
+      end
+    end
+  end
+end

data/lib/qwirk/engine.rb

@@ -0,0 +1,9 @@
+require 'rails'
+
+module Qwirk
+  class Engine < Rails::Engine
+    initializer "qwirk initialize" , :after =>"active_record.initialize_database" do
+      #config.before_configuration do
+    end
+  end
+end

data/lib/qwirk/loggable.rb

@@ -0,0 +1,23 @@
+module Qwirk
+  module Loggable
+    def logger
+      @logger ||= (rails_logger || default_logger)
+    end
+
+    def rails_logger
+      (defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger) ||
+      (defined?(RAILS_DEFAULT_LOGGER) && RAILS_DEFAULT_LOGGER.respond_to?(:debug) && RAILS_DEFAULT_LOGGER)
+    end
+
+    def default_logger
+      require 'logger'
+      l = Logger.new($stdout)
+      l.level = Logger::DEBUG
+      l
+    end
+
+    def logger=(logger)
+      @logger = logger
+    end
+  end
+end

data/lib/qwirk/manager.rb

@@ -0,0 +1,140 @@
+require 'erb'
+require 'yaml'
+require 'socket'
+require 'rumx'
+
+module Qwirk
+  class Manager
+    include Rumx::Bean
+    attr_reader   :env, :worker_configs, :name
+
+    bean_attr_accessor :poll_time, :float, 'How often the manager should poll the workers for their status for use by :idle_worker_timeout and :max_read_threshold'
+
+    @@default_options = {}
+
+    def self.default_options=(options)
+      @@default_options = options
+    end
+
+    # Constructs a manager.  Accepts a hash of config options
+    #   name         - name which this bean will be added
+    #   env          - environment being executed under.  For a rails project, this will be the value of Rails.env
+    #   worker_file  - the worker file is a hash with the environment or hostname as the primary key and a subhash with the worker names
+    #     as the keys and the config options for the value.  In this file, the env will be searched first and if that doesn't exist,
+    #     the hostname will then be searched.  Workers can be defined for development without having to specify the hostname.  For
+    #     production, a set of workers could be defined under production or specific workers for each host name.
+    #   persist_file - WorkerConfig attributes that are modified externally (via Rumx interface) will be stored in this file.  Without this
+    #     option, external config changes that are made will be lost when the Manager is restarted.
+    def initialize(queue_adapter, options={})
+      options           = @@default_options.merge(options)
+      #puts "creating qwirk manager with #{options.inspect}"
+      @stopped          = false
+      @name             = options[:name] || Qwirk::DEFAULT_NAME
+      @poll_time        = 3.0
+      @worker_configs   = []
+      @env              = options[:env]
+      @worker_options   = parse_worker_file(options[:worker_file])
+      @persist_file     = options[:persist_file]
+      @persist_options  = (@persist_file && File.exist?(@persist_file)) ? YAML.load_file(@persist_file) : {}
+
+      BaseWorker.worker_classes.each do |worker_class|
+        worker_config_class = worker_class.config_class
+        worker_class.each_config do |config_name, default_options|
+          # Least priority is config options defined in the Worker class, then the workers.yml file, highest priority is persist_file (ad-hoc changes made manually)
+          options = {}
+          options = options.merge(@worker_options[config_name]) if @worker_options[config_name]
+          options = options.merge(@persist_options[config_name]) if @persist_options[config_name]
+          worker_config = worker_config_class.new(queue_adapter, config_name, self, worker_class, default_options, options)
+          bean_add_child(config_name, worker_config)
+          @worker_configs << worker_config
+        end
+      end
+
+      start_timer_thread
+      stop_on_signal if options[:stop_on_signal]
+    end
+
+    def start_timer_thread
+      # TODO: Optionize hard-coded values
+      @timer_thread = Thread.new do
+        begin
+          while !@stopped
+            @worker_configs.each do |worker_config|
+              worker_config.periodic_call(@poll_time)
+            end
+            sleep @poll_time
+          end
+        rescue Exception => e
+          Qwirk.logger.error "Timer thread failed with exception: #{e.message}\n\t#{e.backtrace.join("\n\t")}"
+        end
+      end
+    end
+
+    def stop
+      return if @stopped
+      @stopped = true
+      @timer_thread.wakeup
+      @worker_configs.each { |worker_config| worker_config.stop }
+    end
+
+    def stop_on_signal
+      ['HUP', 'INT', 'TERM'].each do |signal_name|
+        Signal.trap(signal_name) do
+          Qwirk.logger.info "Caught #{signal_name}"
+          stop
+        end
+      end
+    end
+
+    def save_persist_state
+      return unless @persist_file
+      new_persist_options = {}
+      BaseWorker.worker_classes.each do |worker_class|
+        worker_class.each_config do |config_name, options|
+          static_options = options.merge(@worker_options[config_name] || {})
+          worker_config = self[config_name]
+          hash = {}
+          # Only store off the config values that are specifically different from default values or values set in the workers.yml file
+          # Then updates to these values will be allowed w/o being hardcoded to an old default value.
+          worker_config.bean_get_attributes do |attribute_info|
+            if attribute_info.attribute[:config_item] && attribute_info.ancestry.size == 1
+              param_name = attribute_info.ancestry[0].to_sym
+              value = attribute_info.value
+              hash[param_name] = value if static_options[param_name] != value
+            end
+          end
+          new_persist_options[config_name] = hash unless hash.empty?
+        end
+      end
+      if new_persist_options != @persist_options
+        @persist_options = new_persist_options
+        File.open(@persist_file, 'w') do |out|
+          YAML.dump(@persist_options, out )
+        end
+      end
+    end
+
+    def [](name)
+      @worker_configs.each do |worker_config|
+        return worker_config if worker_config.name == name
+      end
+      return nil
+    end
+
+    #######
+    private
+    #######
+
+    def parse_worker_file(file)
+      if file && File.exist?(file)
+        hash = YAML.load(ERB.new(File.read(file), nil, '-').result(binding))
+        options = @env && hash[@env]
+        unless options
+          host = Socket.gethostname.sub(/\..*/, '')
+          options = hash[host]
+        end
+      end
+      return options || {}
+    end
+  end
+end