pgq 0.1 → 0.1.1

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pgq (0.1)
+    pgq (0.1.1)
       activerecord (>= 2.3.2)
       activesupport (>= 2.3.2)
 

data/README.md

@@ -74,7 +74,7 @@ class PgqMy < Pgq::Consumer
 end
 ```
 
-Insert event into queue like this:
+Insert event into queue:
 
     PgqMy.some_method1(1, 2, 3)
     
@@ -97,7 +97,7 @@ Start worker for queue:
     
 
 
-Also can consume manual, or write [bin_script](http://github.com/kostya/bin_script) like this:
+Also can consume manual, or write [bin_script](http://github.com/kostya/bin_script):
 ```ruby
 class PgqRunnerScript < BinScript
 
@@ -135,11 +135,11 @@ When any raise happens in consumer, its produce failed event, which can be retry
 
 Retry manual:
 
-    Pgq::Consumer.resend_failed_events(queue_name)
+    Pgq::Consumer.retry_failed_events(queue_name)
 
 Delete manual:
     
-    Pgq::Consumer.clear_failed_events(queue_name)
+    Pgq::Consumer.delete_failed_events(queue_name)
 
 
 ### Divide events between workers, for one consumer class

data/lib/pgq/api.rb

@@ -1,6 +1,8 @@
 module Pgq::Api
   # should mixin to class, which have connection
 
+  # == manage queues
+  
   def pgq_create_queue(queue_name)
     connection.select_value(sanitize_sql_array ["SELECT pgq.create_queue(?)", queue_name]).to_i
   end
@@ -16,16 +18,8 @@ module Pgq::Api
   def pgq_unregister_consumer(queue_name, consumer_id)
     connection.select_value(sanitize_sql_array ["SELECT pgq.unregister_consumer(?, ?)", queue_name, consumer_id]).to_i
   end
-
-  def pgq_add_queue(queue_name, consumer_name)
-    pgq_create_queue(queue_name.to_s)
-    pgq_register_consumer(queue_name.to_s, consumer_name.to_s)
-  end
-
-  def pgq_remove_queue(queue_name, consumer_name)
-    pgq_unregister_consumer(queue_name.to_s, consumer_name.to_s)
-    pgq_drop_queue(queue_name.to_s)
-  end
+  
+  # == insert events
 
   def pgq_insert_event(queue_name, ev_type, ev_data, ev_extra1 = nil, ev_extra2 = nil, ev_extra3 = nil, ev_extra4 = nil)
     result = connection.select_value(sanitize_sql_array ["SELECT pgq.insert_event(?, ?, ?, ?, ?, ?, ?)", 
@@ -33,6 +27,8 @@ module Pgq::Api
     result ? result.to_i : nil
   end
   
+  # == consuming
+  
   def pgq_next_batch(queue_name, consumer_id)
     result = connection.select_value(sanitize_sql_array ["SELECT pgq.next_batch(?, ?)", queue_name, consumer_id])
     result ? result.to_i : nil
@@ -42,22 +38,22 @@ module Pgq::Api
     connection.select_all(sanitize_sql_array ["SELECT * FROM pgq.get_batch_events(?)", batch_id])
   end
   
+  def pgq_finish_batch(batch_id)
+    connection.select_value(sanitize_sql_array ["SELECT pgq.finish_batch(?)", batch_id])
+  end
+  
+  # == failed/retry
+  
   def pgq_event_failed(batch_id, event_id, reason)
     connection.select_value(sanitize_sql_array ["SELECT pgq.event_failed(?, ?, ?)", batch_id, event_id, reason]).to_i
   end
-  
+          
   def pgq_event_retry(batch_id, event_id, retry_seconds)
     connection.select_value(sanitize_sql_array ["SELECT pgq.event_retry(?, ?, ?)", batch_id, event_id, retry_seconds]).to_i
-  end
-  
-  def pgq_finish_batch(batch_id)
-    connection.select_value(sanitize_sql_array ["SELECT pgq.finish_batch(?)", batch_id])
-  end
-
-  def pgq_get_queue_info(queue_name)
-    connection.select_value(sanitize_sql_array ["SELECT pgq.get_queue_info(?)", queue_name])
-  end
+  end                  
 
+  # == failed events
+  
   def pgq_failed_event_retry(queue_name, consumer, event_id)
     connection.select_value(sanitize_sql_array ["SELECT * FROM pgq.failed_event_retry(?, ?, ?)", queue_name, consumer, event_id])
   end
@@ -66,7 +62,7 @@ module Pgq::Api
     connection.select_value(sanitize_sql_array ["SELECT * FROM pgq.failed_event_delete(?, ?, ?)", queue_name, consumer, event_id])
   end
      
-  def pgq_failed_events_count(queue_name, consumer)
+  def pgq_failed_event_count(queue_name, consumer)
     res = connection.select_value(sanitize_sql_array ["SELECT * FROM pgq.failed_event_count(?, ?)", queue_name, consumer])
     res ? res.to_i : nil
   end
@@ -76,13 +72,24 @@ module Pgq::Api
     connection.select_all(sanitize_sql_array ["SELECT * FROM pgq.failed_event_list(?, ?, ?, ?) order by ev_id #{order}", queue_name, consumer, limit.to_i, offset.to_i])
   end
 
-  # queue lag in seconds
-  def pgq_queue_lag(queue_name)
-    connection.select_value(sanitize_sql_array ["SELECT Max(EXTRACT(epoch FROM lag)) FROM pgq.get_consumer_info() where queue_name = ?", queue_name]).to_f
-  end
+  # == info methods
+
+  def pgq_get_queue_info(queue_name)
+    connection.select_value(sanitize_sql_array ["SELECT pgq.get_queue_info(?)", queue_name])
+  end          
+  
+  # Get list of queues.
+  # Result: (queue_name, queue_ntables, queue_cur_table, queue_rotation_period, queue_switch_time, queue_external_ticker, queue_ticker_max_count, queue_ticker_max_lag, queue_ticker_idle_period, ticker_lag)
+  def pgq_get_queues_info
+    connection.select_values("SELECT pgq.get_queue_info()")
+  end          
   
   def pgq_get_consumer_info
-    connection.select_all("SELECT * FROM pgq.get_consumer_info()")
+    connection.select_all("SELECT *, EXTRACT(epoch FROM last_seen) AS last_seen_sec, EXTRACT(epoch FROM lag) AS lag_sec FROM pgq.get_consumer_info()")
   end
   
+  def pgq_get_consumer_queue_info(queue_name)
+    connection.select_one(sanitize_sql_array ["SELECT *, EXTRACT(epoch FROM last_seen) AS last_seen_sec, EXTRACT(epoch FROM lag) AS lag_sec FROM pgq.get_consumer_info(?)", queue_name]) || {}
+  end
+
 end

data/lib/pgq/consumer_base.rb

@@ -35,7 +35,7 @@ class Pgq::ConsumerBase
   end
   
   def self.set_queue_name(name)
-    self.instance_variable_set('@queue_name', name.to_s)
+    @queue_name = name.to_s
   end
   
   # magic set queue_name from class name
@@ -119,14 +119,12 @@ class Pgq::ConsumerBase
     perform(type, *data)
 
   rescue Exception => ex
-    message = event.exception_message(ex)
-    self.log_error(message)
-    event.failed!(message)
+    self.log_error(event.exception_message(ex))
+    event.failed!(ex)
     
   rescue => ex
-    message = event.exception_message(ex)
-    self.log_error(message)
-    event.failed!(message)
+    self.log_error(event.exception_message(ex))
+    event.failed!(ex)
   end
 
   def perform(type, *data)
@@ -149,16 +147,15 @@ class Pgq::ConsumerBase
     database.pgq_event_failed(@batch_id, event_id, reason)
   end
 
-  def event_retry(event_id)
-    database.pgq_event_retry(@batch_id, event_id, 0)
+  def event_retry(event_id, seconds = 0)
+    database.pgq_event_retry(@batch_id, event_id, seconds)
   end
 
   def all_events_failed(events, ex)
-    message = Pgq::Event.exception_message(ex)    
-    log_error(message)
+    log_error(Pgq::Event.exception_message(ex))
     
     events.each do |event|
-      event.failed!(message)
+      event.failed!(ex)
     end    
   end
 

data/lib/pgq/event.rb

@@ -8,23 +8,19 @@ class Pgq::Event
     @consumer = consumer
   end
 
-  def failed!(ex = 'Something happens')
-    if ex.is_a?(String)
-      @consumer.event_failed @id, ex
-    else # exception
-      @consumer.event_failed @id, exception_message(ex)      
-    end
+  def failed!(ex)
+    h = {:class => ex.class.to_s, :message => ex.message, :backtrace => ex.backtrace}
+    @consumer.event_failed @id, consumer.coder.dump(h)
   end
   
-  def retry!
-    @consumer.event_retry(@id)
+  def retry!(seconds = 0)
+    @consumer.event_retry(@id, seconds)
   end
 
   def self.exception_message(e)
     <<-EXCEPTION
 Exception happend
-Type: #{e.class.inspect}
-Error occurs: #{e.message}
+Error occurs: #{e.class.inspect}(#{e.message})
 Backtrace: #{e.backtrace.join("\n") rescue ''}
     EXCEPTION
   end
@@ -33,9 +29,9 @@ Backtrace: #{e.backtrace.join("\n") rescue ''}
   def exception_message(e)
     <<-EXCEPTION
 Exception happend
-Type: #{type.inspect} #{e.class.inspect}
+Type: #{type.inspect}
 Data: #{data.inspect}
-Error occurs: #{e.message}
+Error occurs: #{e.class.inspect}(#{e.message})
 Backtrace: #{e.backtrace.join("\n") rescue ''}
     EXCEPTION
   end

data/lib/pgq/utils.rb

@@ -2,34 +2,30 @@ module Pgq::Utils
 
   # == all queues for database
   def queues_list
-    database.pgq_get_consumer_info.map{|x| x['queue_name']}
+    database.pgq_get_consumer_info.map{|x| x['queue_name']}.uniq
   end
 
   # == methods for migrations
   def add_queue(queue_name, consumer_name = self.consumer_name)
-    database.pgq_add_queue(queue_name, consumer_name)
+    database.pgq_create_queue(queue_name.to_s)
+    database.pgq_register_consumer(queue_name.to_s, consumer_name.to_s)
   end
 
   def remove_queue(queue_name, consumer_name = self.consumer_name)
-    database.pgq_remove_queue(queue_name, consumer_name)
+    database.pgq_unregister_consumer(queue_name.to_s, consumer_name.to_s)
+    database.pgq_drop_queue(queue_name.to_s)
   end
   
   # == inspect queue
   # { type => events_count }
   def inspect_queue(queue_name)
-    ticks = database.pgq_get_queue_info(queue_name)
-    table = connection.select_value("SELECT queue_data_pfx as table FROM pgq.queue where queue_name = #{database.sanitize(queue_name)}")
+    table, last_event = last_event_id(queue_name)
     
-    result = {}
-    
-    if ticks['current_batch']
-      sql = connection.select_value("SELECT * from pgq.batch_event_sql(#{database.sanitize(ticks['current_batch'].to_i)})")
-      last_event = connection.select_value("SELECT MAX(ev_id) AS count FROM (#{sql}) AS x")
-      
+    if last_event
       stats = connection.select_all <<-SQL
         SELECT count(*) as count, ev_type
         FROM #{table}
-        WHERE ev_id > #{database.sanitize(last_event.to_i)}
+        WHERE ev_id > #{last_event.to_i}
         GROUP BY ev_type
       SQL
       
@@ -59,19 +55,13 @@ module Pgq::Utils
   # show hash stats, for londiste type of storage events 
   # { type => events_count }
   def inspect_londiste_queue(queue_name)
-    ticks = database.pgq_get_consumer_info
-    table = connection.select_value(connection.sanitize_sql_array ["SELECT queue_data_pfx as table FROM pgq.queue where queue_name = ?", queue_name])
+    table, last_event = last_event_id(queue_name)
     
-    result = {}
-    
-    if ticks['current_batch']
-      sql = connection.select_value("SELECT * from pgq.batch_event_sql(#{database.sanitize(ticks['current_batch'].to_i)})")
-      last_event = connection.select_value("SELECT MAX(ev_id) AS count FROM (#{sql}) AS x")
-      
+    if last_event
       stats = connection.select_all <<-SQL
         SELECT count(*) as count, ev_type, ev_extra1
         FROM #{table}
-        WHERE ev_id > #{database.sanitize(last_event.to_i)}
+        WHERE ev_id > #{last_event.to_i}
         GROUP BY ev_type, ev_extra1
       SQL
       
@@ -97,14 +87,14 @@ module Pgq::Utils
   
   # == proxing method for tests
   def proxy(method_name)
-    self.should_receive(method_name) do |*data|
+    self.should_receive(:enqueue) do |method_name, *data|
       x = self.coder.load(self.coder.dump(data))
       self.new.send(:perform, method_name, *x)
     end.any_number_of_times
   end
   
   # == resend failed events in queue
-  def resend_failed_events(queue_name, limit = 5_000)
+  def retry_failed_events(queue_name, limit = 5_000)
     events = database.pgq_failed_event_list(queue_name, self.consumer_name, limit, nil, 'asc') || []
 
     events.each do |event|
@@ -114,7 +104,7 @@ module Pgq::Utils
     events.length  
   end
   
-  def clear_failed_events(queue_name, limit = 5_000)
+  def delete_failed_events(queue_name, limit = 5_000)
     events = database.pgq_failed_event_list(queue_name, self.consumer_name, limit, nil, 'asc') || []
        
     events.each do |event|
@@ -124,4 +114,19 @@ module Pgq::Utils
     events.length
   end
 
+  def last_event_id(queue_name) 
+    ticks = database.pgq_get_consumer_queue_info(queue_name)
+    table = connection.select_value("SELECT queue_data_pfx AS table FROM pgq.queue WHERE queue_name = #{database.sanitize(queue_name)}")
+    
+    result = nil
+        
+    if ticks['current_batch']
+      sql = connection.select_value("SELECT * FROM pgq.batch_event_sql(#{database.sanitize(ticks['current_batch'].to_i)})")
+      last_event = connection.select_value("SELECT MAX(ev_id) AS count FROM (#{sql}) AS x")
+      result = last_event.to_i
+    end
+    
+    [table, result]
+  end
+
 end

data/lib/pgq/version.rb

@@ -1,3 +1,3 @@
 module Pgq
-  VERSION = "0.1"
+  VERSION = "0.1.1"
 end

data/lib/pgq/worker.rb

@@ -29,20 +29,20 @@ class Pgq::Worker
     @logger = h[:logger] || (defined?(Rails) && Rails.logger) || Logger.new(STDOUT)
     @consumers = []
     
-    queues = h[:queues]
-    raise "Queue not selected" if queues.blank?
+    @queues = h[:queues]
+    raise "Queue not selected" if @queues.blank?
     
-    if queues == ['all'] || queues == 'all'
+    if @queues == ['all'] || @queues == 'all'
       if defined?(Rails) && File.exists?(Rails.root + "config/queues_list.yml")
-        queues = YAML.load_file(Rails.root + "config/queues_list.yml")
+        @queues = YAML.load_file(Rails.root + "config/queues_list.yml")
       else
         raise "You shoud create config/queues_list.yml for all queues"
       end
     end
     
-    queues = queues.split(',') if queues.is_a?(String)
+    @queues = @queues.split(',') if @queues.is_a?(String)
     
-    queues.each do |queue|
+    @queues.each do |queue|
       klass = Pgq::Worker.predict_queue_class(queue)
       if klass
         @consumers << klass.new(@logger, queue)
@@ -72,7 +72,7 @@ class Pgq::Worker
   end
 
   def run
-    logger.info "Worker start"
+    logger.info "Worker for (#{@queues.join(",")}) started"
 
     loop do
       processed_count = process_batch

data/spec/consumer_base_spec.rb

@@ -105,8 +105,9 @@ describe Pgq::ConsumerBase do
     end
 
     it "all_events_failed" do
-      @event.should_receive(:failed!).with(an_instance_of(String))
-      @consumer.all_events_failed(@events, Exception.new('wow'))
+      ex = Exception.new('wow')
+      @event.should_receive(:failed!).with(ex)
+      @consumer.all_events_failed(@events, ex)
     end
 
     it "perform_events" do
@@ -120,8 +121,9 @@ describe Pgq::ConsumerBase do
     end
 
     it "perform_event raised" do
-      @consumer.should_receive(:perform).with('bla', *@data).and_throw(:wow)
-      @event.should_receive(:failed!).with(an_instance_of(String))
+      ex = Exception.new('wow')
+      @consumer.should_receive(:perform).with('bla', *@data).and_raise(ex)
+      @event.should_receive(:failed!).with(ex)
       @consumer.perform_event(@event)
     end
 
@@ -129,19 +131,21 @@ describe Pgq::ConsumerBase do
 
   describe "migration" do
     it "up" do
-      Pgq::ConsumerBase.database.should_receive(:pgq_add_queue).with('super', Pgq::ConsumerBase.consumer_name)
+      Pgq::ConsumerBase.database.should_receive(:pgq_create_queue).with('super')
+      Pgq::ConsumerBase.database.should_receive(:pgq_register_consumer).with('super', Pgq::ConsumerBase.consumer_name)
       Pgq::ConsumerBase.add_queue("super")
     end
 
     it "down" do
-      Pgq::ConsumerBase.database.should_receive(:pgq_remove_queue).with('super', Pgq::ConsumerBase.consumer_name)
+      Pgq::ConsumerBase.database.should_receive(:pgq_drop_queue).with('super')
+      Pgq::ConsumerBase.database.should_receive(:pgq_unregister_consumer).with('super', Pgq::ConsumerBase.consumer_name)
       Pgq::ConsumerBase.remove_queue("super")
     end
   end
   
   it "should proxy consumer" do
     PgqTata3.proxy(:ptest)
-    PgqTata3.ptest(111, 'abc').should == 10
+    PgqTata3.enqueue(:ptest, 111, 'abc').should == 10
     $a.should == 111
     $b.should == 'abc'
   end

data/spec/event_spec.rb

@@ -21,11 +21,11 @@ describe Pgq::Event do
 
   it "should failed!" do
     @consumer.should_receive(:event_failed).with(123, an_instance_of(String))
-    @ev.failed!
+    @ev.failed!(Exception.new("haha"))
   end
 
   it "should retry!" do
-    @consumer.should_receive(:event_retry).with(123)    
+    @consumer.should_receive(:event_retry).with(123, 0)
     @ev.retry!
   end
  

metadata

@@ -1,12 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: pgq
 version: !ruby/object:Gem::Version 
-  hash: 9
+  hash: 25
   prerelease: false
   segments: 
   - 0
   - 1
-  version: "0.1"
+  - 1
+  version: 0.1.1
 platform: ruby
 authors: 
 - Makarchev Konstantin
@@ -14,7 +15,7 @@ autorequire: init
 bindir: bin
 cert_chain: []
 
-date: 2012-05-09 00:00:00 +04:00
+date: 2012-05-16 00:00:00 +04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency