pgq 0.1

data/lib/pgq/consumer.rb

@@ -0,0 +1,23 @@
+require 'pgq/consumer_base'
+
+# Cute class, for magick inserts and light consume
+
+class Pgq::Consumer < Pgq::ConsumerBase
+  
+  # == magick insert events
+ 
+  def self.method_missing(method_name, *args)
+    enqueue(method_name, *args)
+  end
+  
+  def self.add_event(method_name, *args)
+    enqueue(method_name, *args)
+  end
+
+  # == magick consume
+
+  def perform(method_name, *args)
+    self.send(method_name, *args)
+  end
+
+end

data/lib/pgq/consumer_base.rb

@@ -0,0 +1,175 @@
+require 'pgq/utils'
+require 'pgq/api'
+require 'active_support/inflector' unless ''.respond_to?(:underscore)
+
+class Pgq::ConsumerBase
+  extend Pgq::Utils
+
+  @queue_name = 'default'
+  @consumer_name = 'default'
+  
+  attr_accessor :logger, :queue_name, :consumer_name
+
+  # == connection 
+
+  def self.database
+    ActiveRecord::Base # can redefine
+  end
+  
+  def database
+    self.class.database
+  end
+  
+  def self.connection
+    database.connection
+  end
+  
+  def connection
+    self.class.connection
+  end
+
+  # == queue name
+  
+  def self.extract_queue_name
+    self.name.to_s.gsub(/^pgq/i, '').underscore.gsub('/', '-')
+  end
+  
+  def self.set_queue_name(name)
+    self.instance_variable_set('@queue_name', name.to_s)
+  end
+  
+  # magic set queue_name from class name
+  def self.inherited(subclass)
+    subclass.set_queue_name(subclass.extract_queue_name)
+    subclass.instance_variable_set('@consumer_name', self.consumer_name)
+  end
+
+  def self.consumer_name
+    @consumer_name
+  end
+
+  def self.queue_name
+    @queue_name
+  end
+
+  # this method used when insert event, possible to reuse
+  def self.next_queue_name
+    self.queue_name
+  end
+
+  # == coder
+
+  def self.coder
+    Pgq::Marshal64Coder
+  end
+  
+  def coder
+    self.class.coder
+  end
+
+  # == insert event
+  
+  def self.enqueue(method_name, *args)
+    self.database.pgq_insert_event( self.next_queue_name, method_name.to_s, coder.dump(args) )
+  end
+  
+  # == consumer part
+  
+  def initialize(logger = nil, custom_queue_name = nil, custom_consumer_name = nil)
+    self.queue_name = custom_queue_name || self.class.queue_name
+    self.consumer_name = custom_consumer_name || self.class.consumer_name
+    self.logger = logger
+    @batch_id = nil
+  end
+  
+  def perform_batch
+    events = []
+    pgq_events = get_batch_events
+
+    return 0 if pgq_events.blank?
+    
+    events = pgq_events.map{|ev| Pgq::Event.new(self, ev) }
+    size = events.size
+    log_info "=> batch(#{queue_name}): events #{size}"
+    
+    perform_events(events)
+    
+  rescue Exception => ex
+    all_events_failed(events, ex)
+ 
+  rescue => ex
+    all_events_failed(events, ex)
+    
+  ensure
+    finish_batch(events.size)
+    
+    return events.size
+  end
+  
+  def perform_events(events)
+    events.each do |event|
+      perform_event(event)
+    end
+  end
+
+  def perform_event(event)
+    type = event.type
+    data = event.data
+
+    perform(type, *data)
+
+  rescue Exception => ex
+    message = event.exception_message(ex)
+    self.log_error(message)
+    event.failed!(message)
+    
+  rescue => ex
+    message = event.exception_message(ex)
+    self.log_error(message)
+    event.failed!(message)
+  end
+
+  def perform(type, *data)
+    raise "realize me"    
+  end
+
+  def get_batch_events
+    @batch_id = database.pgq_next_batch(queue_name, consumer_name)
+    return nil if !@batch_id
+    database.pgq_get_batch_events(@batch_id)
+  end
+
+  def finish_batch(count = nil)
+    return unless @batch_id
+    database.pgq_finish_batch(@batch_id)
+    @batch_id = nil
+  end
+  
+  def event_failed(event_id, reason)
+    database.pgq_event_failed(@batch_id, event_id, reason)
+  end
+
+  def event_retry(event_id)
+    database.pgq_event_retry(@batch_id, event_id, 0)
+  end
+
+  def all_events_failed(events, ex)
+    message = Pgq::Event.exception_message(ex)    
+    log_error(message)
+    
+    events.each do |event|
+      event.failed!(message)
+    end    
+  end
+
+  # == log methods
+
+  def log_info(mes)
+    @logger.info(mes) if @logger
+  end
+
+  def log_error(mes)
+    @logger.error(mes) if @logger
+  end
+  
+end

data/lib/pgq/consumer_group.rb

@@ -0,0 +1,27 @@
+# for consuming full batch (usefull if need group events for some group processing)
+# usually group is ~500 events
+
+require 'pgq/consumer'
+
+class Pgq::ConsumerGroup < Pgq::Consumer
+
+	# {'type' => [events]}
+	def perform_group(events_hash)
+		raise "realize me"
+	end
+	
+	def perform_events(events)
+		events = sum_events(events)
+		# log_info "consume events (#{self.queue_name}): #{events.map{|k,v| [k, v.size]}.inspect}"
+		perform_group(events) if events.present?
+	end
+
+	def sum_events(events)
+		events.inject({}) do |result, event|
+			result[event.type] ||= []
+			result[event.type] << event
+			result
+		end
+	end
+
+end

data/lib/pgq/event.rb

@@ -0,0 +1,43 @@
+class Pgq::Event
+  attr_reader :type, :data, :id, :consumer
+
+  def initialize(consumer, event)
+    @id = event['ev_id']
+    @type = event['ev_type']
+    @data = consumer.coder.load(event['ev_data']) if event['ev_data']
+    @consumer = consumer
+  end
+
+  def failed!(ex = 'Something happens')
+    if ex.is_a?(String)
+      @consumer.event_failed @id, ex
+    else # exception
+      @consumer.event_failed @id, exception_message(ex)      
+    end
+  end
+  
+  def retry!
+    @consumer.event_retry(@id)
+  end
+
+  def self.exception_message(e)
+    <<-EXCEPTION
+Exception happend
+Type: #{e.class.inspect}
+Error occurs: #{e.message}
+Backtrace: #{e.backtrace.join("\n") rescue ''}
+    EXCEPTION
+  end
+
+  # Prepare string with exception details
+  def exception_message(e)
+    <<-EXCEPTION
+Exception happend
+Type: #{type.inspect} #{e.class.inspect}
+Data: #{data.inspect}
+Error occurs: #{e.message}
+Backtrace: #{e.backtrace.join("\n") rescue ''}
+    EXCEPTION
+  end
+
+end

data/lib/pgq/marshal64_coder.rb

@@ -0,0 +1,13 @@
+require 'base64'
+
+module Pgq::Marshal64Coder
+
+  def self.dump(s)
+    Base64::encode64(Marshal.dump(s))
+  end
+
+  def self.load(s)
+    Marshal.load(Base64::decode64(s))
+  end
+
+end

data/lib/pgq/railtie.rb

@@ -0,0 +1,9 @@
+if defined?(Rails) && defined?(::Rails::Engine)
+
+  class Pgq::Engine < ::Rails::Engine
+    rake_tasks do
+#      load File.dirname(__FILE__) + "/../tasks/pgq.rake"
+    end                  
+  end
+  
+end

data/lib/pgq/utils.rb

@@ -0,0 +1,127 @@
+module Pgq::Utils
+
+  # == all queues for database
+  def queues_list
+    database.pgq_get_consumer_info.map{|x| x['queue_name']}
+  end
+
+  # == methods for migrations
+  def add_queue(queue_name, consumer_name = self.consumer_name)
+    database.pgq_add_queue(queue_name, consumer_name)
+  end
+
+  def remove_queue(queue_name, consumer_name = self.consumer_name)
+    database.pgq_remove_queue(queue_name, consumer_name)
+  end
+  
+  # == inspect queue
+  # { type => events_count }
+  def inspect_queue(queue_name)
+    ticks = database.pgq_get_queue_info(queue_name)
+    table = connection.select_value("SELECT queue_data_pfx as table FROM pgq.queue where queue_name = #{database.sanitize(queue_name)}")
+    
+    result = {}
+    
+    if ticks['current_batch']
+      sql = connection.select_value("SELECT * from pgq.batch_event_sql(#{database.sanitize(ticks['current_batch'].to_i)})")
+      last_event = connection.select_value("SELECT MAX(ev_id) AS count FROM (#{sql}) AS x")
+      
+      stats = connection.select_all <<-SQL
+        SELECT count(*) as count, ev_type
+        FROM #{table}
+        WHERE ev_id > #{database.sanitize(last_event.to_i)}
+        GROUP BY ev_type
+      SQL
+      
+      stats.each do |x|
+        result["#{x['ev_type']}"] = x['count'].to_i
+      end
+      
+    else    
+      stats = connection.select_all <<-SQL
+        SELECT ev_type
+        FROM #{table}
+        GROUP BY ev_type
+      SQL
+
+      stats.each do |x|
+        result["#{x['ev_type']}"] = 0
+      end
+    end
+
+    result
+  end
+  
+  def inspect_self_queue
+    self.inspect_queue(self.queue_name)
+  end
+
+  # show hash stats, for londiste type of storage events 
+  # { type => events_count }
+  def inspect_londiste_queue(queue_name)
+    ticks = database.pgq_get_consumer_info
+    table = connection.select_value(connection.sanitize_sql_array ["SELECT queue_data_pfx as table FROM pgq.queue where queue_name = ?", queue_name])
+    
+    result = {}
+    
+    if ticks['current_batch']
+      sql = connection.select_value("SELECT * from pgq.batch_event_sql(#{database.sanitize(ticks['current_batch'].to_i)})")
+      last_event = connection.select_value("SELECT MAX(ev_id) AS count FROM (#{sql}) AS x")
+      
+      stats = connection.select_all <<-SQL
+        SELECT count(*) as count, ev_type, ev_extra1
+        FROM #{table}
+        WHERE ev_id > #{database.sanitize(last_event.to_i)}
+        GROUP BY ev_type, ev_extra1
+      SQL
+      
+      stats.each do |x|
+        result["#{x['ev_extra1']}:#{x['ev_type']}"] = x['count'].to_i
+      end
+      
+    else    
+      stats = connection.select_all <<-SQL
+        SELECT ev_type, ev_extra1
+        FROM #{table}
+        GROUP BY ev_type, ev_extra1 ORDER BY ev_extra1, ev_type
+      SQL
+
+      stats.each do |x|
+        result["#{x['ev_extra1']}:#{x['ev_type']}"] = 0
+      end
+    end
+
+    result
+  end
+  
+  
+  # == proxing method for tests
+  def proxy(method_name)
+    self.should_receive(method_name) do |*data|
+      x = self.coder.load(self.coder.dump(data))
+      self.new.send(:perform, method_name, *x)
+    end.any_number_of_times
+  end
+  
+  # == resend failed events in queue
+  def resend_failed_events(queue_name, limit = 5_000)
+    events = database.pgq_failed_event_list(queue_name, self.consumer_name, limit, nil, 'asc') || []
+
+    events.each do |event|
+      database.pgq_failed_event_retry(queue_name, self.consumer_name, event['ev_id'])
+    end
+    
+    events.length  
+  end
+  
+  def clear_failed_events(queue_name, limit = 5_000)
+    events = database.pgq_failed_event_list(queue_name, self.consumer_name, limit, nil, 'asc') || []
+       
+    events.each do |event|
+      database.pgq_failed_event_delete(queue_name, self.consumer_name, event['ev_id'])
+    end
+                      
+    events.length
+  end
+
+end

data/lib/pgq/version.rb

@@ -0,0 +1,3 @@
+module Pgq
+  VERSION = "0.1"
+end

data/lib/pgq/worker.rb

@@ -0,0 +1,83 @@
+require 'logger'
+
+class Pgq::Worker
+  attr_reader :logger, :queues, :consumers, :sleep_time, :watch_file
+  
+  def self.predict_queue_class(queue)
+    klass = nil
+    unless klass
+      queue.to_s.match(/([a-z_]+)/i)
+      klass_s = $1.to_s
+      klass_s.chop! if klass_s.size > 0 && klass_s[-1].chr == '_'
+      klass_s = "pgq_" + klass_s unless klass_s.start_with?("pgq_")
+      klass = klass_s.camelize.constantize rescue nil
+      klass = nil unless klass.is_a?(Class)
+    end    
+    klass    
+  end
+
+  def self.connection(queue)
+    klass = predict_queue_class(queue)
+    if klass
+      klass.connection
+    else
+      raise "can't find klass for queue #{queue}"
+    end
+  end
+
+  def initialize(h)
+    @logger = h[:logger] || (defined?(Rails) && Rails.logger) || Logger.new(STDOUT)
+    @consumers = []
+    
+    queues = h[:queues]
+    raise "Queue not selected" if queues.blank?
+    
+    if queues == ['all'] || queues == 'all'
+      if defined?(Rails) && File.exists?(Rails.root + "config/queues_list.yml")
+        queues = YAML.load_file(Rails.root + "config/queues_list.yml")
+      else
+        raise "You shoud create config/queues_list.yml for all queues"
+      end
+    end
+    
+    queues = queues.split(',') if queues.is_a?(String)
+    
+    queues.each do |queue|
+      klass = Pgq::Worker.predict_queue_class(queue)
+      if klass
+        @consumers << klass.new(@logger, queue)
+      else
+        raise "Unknown queue: #{queue}"
+      end
+    end
+
+    @watch_file = h[:watch_file]
+    @sleep_time = h[:sleep_time] || 0.5
+  end
+
+  def process_batch
+    process_count = 0
+
+    @consumers.each do |consumer|
+      process_count += consumer.perform_batch
+
+      if @watch_file && File.exists?(@watch_file)
+        logger.info "Found file #{@watch_file}, exiting!"
+        File.unlink(@watch_file)
+        return processed_count
+      end
+    end
+
+    process_count
+  end
+
+  def run
+    logger.info "Worker start"
+
+    loop do
+      processed_count = process_batch
+      sleep(@sleep_time) if processed_count == 0
+    end
+  end
+
+end