pgq 0.1.1 → 0.1.2

data/.gitignore

@@ -1,4 +1,4 @@
 TODO
 *.gem
 lib/serv
-README2
+README2

data/Gemfile

@@ -1,4 +1,3 @@
 source "http://rubygems.org"
 
 gemspec
-

data/Gemfile.lock

@@ -1,9 +1,10 @@
 PATH
   remote: .
   specs:
-    pgq (0.1.1)
+    pgq (0.1.2)
       activerecord (>= 2.3.2)
       activesupport (>= 2.3.2)
+      pg
 
 GEM
   remote: http://rubygems.org/
@@ -23,7 +24,8 @@ GEM
     builder (3.0.0)
     diff-lcs (1.1.3)
     i18n (0.6.0)
-    multi_json (1.3.4)
+    multi_json (1.3.5)
+    pg (0.13.2)
     rake (0.9.2.2)
     rspec (2.10.0)
       rspec-core (~> 2.10.0)

data/LICENSE

@@ -17,4 +17,4 @@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -122,24 +122,8 @@ and run:
 
 
 
-
 ### Admin interface
-Admins interface is possible like Resque, but not realized yet.
-For queues info, run in console:
-
-    > Pgq::Consumer.database.pgq_get_consumer_info
-
-
-### Failed events
-When any raise happens in consumer, its produce failed event, which can be retry, or delete.
-
-Retry manual:
-
-    Pgq::Consumer.retry_failed_events(queue_name)
-
-Delete manual:
-    
-    Pgq::Consumer.delete_failed_events(queue_name)
+Realized by gem [pgq_web](http://github.com/kostya/pgq_web).
 
 
 ### Divide events between workers, for one consumer class

data/Rakefile

@@ -6,4 +6,4 @@ Bundler::GemHelper.install_tasks
 require 'rspec/core/rake_task'
 
 task :default => :spec
-RSpec::Core::RakeTask.new(:spec)
+RSpec::Core::RakeTask.new(:spec)

data/lib/generators/pgq/add_generator.rb

@@ -33,7 +33,7 @@ module Pgq
     end
     
     def name_c
-      name.classify
+      name.camelize
     end
   end
   

data/lib/pgq/api.rb

@@ -11,12 +11,12 @@ module Pgq::Api
     connection.select_value(sanitize_sql_array ["SELECT pgq.drop_queue(?)", queue_name]).to_i
   end
 
-  def pgq_register_consumer(queue_name, consumer_id)
-    connection.select_value(sanitize_sql_array ["SELECT pgq.register_consumer(?, ?)", queue_name, consumer_id]).to_i
+  def pgq_register_consumer(queue_name, consumer_name)
+    connection.select_value(sanitize_sql_array ["SELECT pgq.register_consumer(?, ?)", queue_name, consumer_name]).to_i
   end
 
-  def pgq_unregister_consumer(queue_name, consumer_id)
-    connection.select_value(sanitize_sql_array ["SELECT pgq.unregister_consumer(?, ?)", queue_name, consumer_id]).to_i
+  def pgq_unregister_consumer(queue_name, consumer_name)
+    connection.select_value(sanitize_sql_array ["SELECT pgq.unregister_consumer(?, ?)", queue_name, consumer_name]).to_i
   end
   
   # == insert events
@@ -29,8 +29,8 @@ module Pgq::Api
   
   # == consuming
   
-  def pgq_next_batch(queue_name, consumer_id)
-    result = connection.select_value(sanitize_sql_array ["SELECT pgq.next_batch(?, ?)", queue_name, consumer_id])
+  def pgq_next_batch(queue_name, consumer_name)
+    result = connection.select_value(sanitize_sql_array ["SELECT pgq.next_batch(?, ?)", queue_name, consumer_name])
     result ? result.to_i : nil
   end
   
@@ -54,22 +54,22 @@ module Pgq::Api
 
   # == failed events
   
-  def pgq_failed_event_retry(queue_name, consumer, event_id)
-    connection.select_value(sanitize_sql_array ["SELECT * FROM pgq.failed_event_retry(?, ?, ?)", queue_name, consumer, event_id])
+  def pgq_failed_event_retry(queue_name, consumer_name, event_id)
+    connection.select_value(sanitize_sql_array ["SELECT * FROM pgq.failed_event_retry(?, ?, ?)", queue_name, consumer_name, event_id])
   end
       
-  def pgq_failed_event_delete(queue_name, consumer, event_id)
-    connection.select_value(sanitize_sql_array ["SELECT * FROM pgq.failed_event_delete(?, ?, ?)", queue_name, consumer, event_id])
+  def pgq_failed_event_delete(queue_name, consumer_name, event_id)
+    connection.select_value(sanitize_sql_array ["SELECT * FROM pgq.failed_event_delete(?, ?, ?)", queue_name, consumer_name, event_id])
   end
      
-  def pgq_failed_event_count(queue_name, consumer)
-    res = connection.select_value(sanitize_sql_array ["SELECT * FROM pgq.failed_event_count(?, ?)", queue_name, consumer])
+  def pgq_failed_event_count(queue_name, consumer_name)
+    res = connection.select_value(sanitize_sql_array ["SELECT * FROM pgq.failed_event_count(?, ?)", queue_name, consumer_name])
     res ? res.to_i : nil
   end
   
-  def pgq_failed_event_list queue_name, consumer, limit = nil, offset = nil, order = 'desc'
+  def pgq_failed_event_list queue_name, consumer_name, limit = nil, offset = nil, order = 'desc'
     order = (order.to_s == 'desc') ? order : 'asc'
-    connection.select_all(sanitize_sql_array ["SELECT * FROM pgq.failed_event_list(?, ?, ?, ?) order by ev_id #{order}", queue_name, consumer, limit.to_i, offset.to_i])
+    connection.select_all(sanitize_sql_array ["SELECT * FROM pgq.failed_event_list(?, ?, ?, ?) ORDER BY ev_id #{order.upcase}", queue_name, consumer_name, limit.to_i, offset.to_i])
   end
 
   # == info methods
@@ -92,4 +92,41 @@ module Pgq::Api
     connection.select_one(sanitize_sql_array ["SELECT *, EXTRACT(epoch FROM last_seen) AS last_seen_sec, EXTRACT(epoch FROM lag) AS lag_sec FROM pgq.get_consumer_info(?)", queue_name]) || {}
   end
 
+  # == utils
+  
+  def pgq_last_event_id(queue_name)
+    ticks = pgq_get_consumer_queue_info(queue_name)
+    table = connection.select_value("SELECT queue_data_pfx AS table FROM pgq.queue WHERE queue_name = #{sanitize(queue_name)}")
+
+    result = nil
+
+    if ticks['current_batch']
+      sql = connection.select_value("SELECT * FROM pgq.batch_event_sql(#{sanitize(ticks['current_batch'].to_i)})")
+      last_event = connection.select_value("SELECT MAX(ev_id) AS count FROM (#{sql}) AS x")
+      result = last_event.to_i
+    end
+
+    [table, result]
+  end
+  
+  def pgq_mass_retry_failed_events(queue_name, consumer_name, limit = 5_000)
+    events = pgq_failed_event_list(queue_name, consumer_name, limit, nil, 'asc') || []
+
+    events.each do |event|
+      pgq_failed_event_retry(queue_name, consumer_name, event['ev_id'])
+    end
+
+    events.length
+  end
+
+  def pgq_mass_delete_failed_events(queue_name, consumer_name, limit = 5_000)
+    events = pgq_failed_event_list(queue_name, consumer_name, limit, nil, 'asc') || []
+
+    events.each do |event|
+      pgq_failed_event_delete(queue_name, consumer_name, event['ev_id'])
+    end
+
+    events.length
+  end
+
 end

data/lib/pgq/consumer_group.rb

@@ -4,24 +4,24 @@
 require 'pgq/consumer'
 
 class Pgq::ConsumerGroup < Pgq::Consumer
+  
+  # {'type' => [events]}
+  def perform_group(events_hash)
+    raise "realize me"
+  end
+  
+  def perform_events(events)
+    events = sum_events(events)
+    # log_info "consume events (#{self.queue_name}): #{events.map{|k,v| [k, v.size]}.inspect}"
+    perform_group(events) if events.present?
+  end
 
-	# {'type' => [events]}
-	def perform_group(events_hash)
-		raise "realize me"
-	end
-	
-	def perform_events(events)
-		events = sum_events(events)
-		# log_info "consume events (#{self.queue_name}): #{events.map{|k,v| [k, v.size]}.inspect}"
-		perform_group(events) if events.present?
-	end
+  def sum_events(events)
+    events.inject({}) do |result, event|
+      result[event.type] ||= []
+      result[event.type] << event
+      result
+    end
+  end
 
-	def sum_events(events)
-		events.inject({}) do |result, event|
-			result[event.type] ||= []
-			result[event.type] << event
-			result
-		end
-	end
-
-end
+end

data/lib/pgq/utils.rb

@@ -19,31 +19,26 @@ module Pgq::Utils
   # == inspect queue
   # { type => events_count }
   def inspect_queue(queue_name)
-    table, last_event = last_event_id(queue_name)
+    table, last_event = database.pgq_last_event_id(queue_name)
     
-    if last_event
-      stats = connection.select_all <<-SQL
+    stats = if last_event
+      connection.select_all <<-SQL
         SELECT count(*) as count, ev_type
         FROM #{table}
         WHERE ev_id > #{last_event.to_i}
         GROUP BY ev_type
       SQL
-      
-      stats.each do |x|
-        result["#{x['ev_type']}"] = x['count'].to_i
-      end
-      
     else    
-      stats = connection.select_all <<-SQL
+      connection.select_all <<-SQL
         SELECT ev_type
         FROM #{table}
         GROUP BY ev_type
       SQL
-
-      stats.each do |x|
-        result["#{x['ev_type']}"] = 0
-      end
     end
+    
+    stats.each do |x|
+      result["#{x['ev_type']}"] = x['count'].to_i
+    end                
 
     result
   end
@@ -55,30 +50,25 @@ module Pgq::Utils
   # show hash stats, for londiste type of storage events 
   # { type => events_count }
   def inspect_londiste_queue(queue_name)
-    table, last_event = last_event_id(queue_name)
+    table, last_event = database.pgq_last_event_id(queue_name)
     
-    if last_event
-      stats = connection.select_all <<-SQL
+    stats = if last_event
+      connection.select_all <<-SQL
         SELECT count(*) as count, ev_type, ev_extra1
         FROM #{table}
         WHERE ev_id > #{last_event.to_i}
         GROUP BY ev_type, ev_extra1
       SQL
-      
-      stats.each do |x|
-        result["#{x['ev_extra1']}:#{x['ev_type']}"] = x['count'].to_i
-      end
-      
     else    
-      stats = connection.select_all <<-SQL
+      connection.select_all <<-SQL
         SELECT ev_type, ev_extra1
         FROM #{table}
         GROUP BY ev_type, ev_extra1 ORDER BY ev_extra1, ev_type
       SQL
-
-      stats.each do |x|
-        result["#{x['ev_extra1']}:#{x['ev_type']}"] = 0
-      end
+    end
+    
+    stats.each do |x|
+      result["#{x['ev_extra1']}:#{x['ev_type']}"] = x['count'].to_i
     end
 
     result
@@ -95,38 +85,11 @@ module Pgq::Utils
   
   # == resend failed events in queue
   def retry_failed_events(queue_name, limit = 5_000)
-    events = database.pgq_failed_event_list(queue_name, self.consumer_name, limit, nil, 'asc') || []
-
-    events.each do |event|
-      database.pgq_failed_event_retry(queue_name, self.consumer_name, event['ev_id'])
-    end
-    
-    events.length  
+    database.pgq_mass_retry_failed_events(queue_name, self.consumer_name, limit)
   end
   
   def delete_failed_events(queue_name, limit = 5_000)
-    events = database.pgq_failed_event_list(queue_name, self.consumer_name, limit, nil, 'asc') || []
-       
-    events.each do |event|
-      database.pgq_failed_event_delete(queue_name, self.consumer_name, event['ev_id'])
-    end
-                      
-    events.length
-  end
-
-  def last_event_id(queue_name) 
-    ticks = database.pgq_get_consumer_queue_info(queue_name)
-    table = connection.select_value("SELECT queue_data_pfx AS table FROM pgq.queue WHERE queue_name = #{database.sanitize(queue_name)}")
-    
-    result = nil
-        
-    if ticks['current_batch']
-      sql = connection.select_value("SELECT * FROM pgq.batch_event_sql(#{database.sanitize(ticks['current_batch'].to_i)})")
-      last_event = connection.select_value("SELECT MAX(ev_id) AS count FROM (#{sql}) AS x")
-      result = last_event.to_i
-    end
-    
-    [table, result]
+    database.pgq_mass_delete_failed_events(queue_name, self.consumer_name, limit)
   end
 
 end

data/lib/pgq/version.rb

@@ -1,3 +1,3 @@
 module Pgq
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/pgq.gemspec

@@ -22,6 +22,7 @@ Gem::Specification.new do |s|
 
   s.add_dependency 'activesupport', ">=2.3.2"  
   s.add_dependency 'activerecord', ">=2.3.2"
+  s.add_dependency 'pg'
   
   s.add_development_dependency "rspec"
   s.add_development_dependency "rake"

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: pgq
 version: !ruby/object:Gem::Version 
-  hash: 25
+  hash: 31
   prerelease: false
   segments: 
   - 0
   - 1
-  - 1
-  version: 0.1.1
+  - 2
+  version: 0.1.2
 platform: ruby
 authors: 
 - Makarchev Konstantin
@@ -15,7 +15,7 @@ autorequire: init
 bindir: bin
 cert_chain: []
 
-date: 2012-05-16 00:00:00 +04:00
+date: 2012-05-26 00:00:00 +04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -51,7 +51,7 @@ dependencies:
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: rspec
+  name: pg
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
@@ -62,10 +62,10 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  type: :development
+  type: :runtime
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
-  name: rake
+  name: rspec
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
@@ -78,6 +78,20 @@ dependencies:
         version: "0"
   type: :development
   version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: rake
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id005
 description: Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 3! only tested.
 email: kostya27@gmail.com
 executables: []