queue_classic 0.3.3.pre → 0.3.5.pre

data/lib/queue_classic.rb

@@ -1,5 +1,7 @@
-require 'json'
 require 'pg'
+
+require 'logger'
+require 'json'
 require 'uri'
 
 $: << File.expand_path(__FILE__, 'lib')
@@ -7,11 +9,13 @@ $: << File.expand_path(__FILE__, 'lib')
 require 'queue_classic/durable_array'
 require 'queue_classic/database'
 require 'queue_classic/worker'
+require 'queue_classic/logger'
 require 'queue_classic/queue'
 require 'queue_classic/job'
 
 module QC
-  VERBOSE = ENV["VERBOSE"] == true
+  VERBOSE = ENV["VERBOSE"] || ENV["QC_VERBOSE"]
+  Logger.puts("Logging enabled")
 
   def self.method_missing(sym, *args, &block)
     Queue.send(sym, *args, &block)

data/lib/queue_classic/database.rb

@@ -1,5 +1,6 @@
 module QC
   class Database
+
     @@connection = nil
 
     DATABASE_URL        = (ENV["QC_DATABASE_URL"] || ENV["DATABASE_URL"])
@@ -7,53 +8,52 @@ module QC
     NOTIFY_TIMEOUT      = (ENV["QC_NOTIFY_TIMEOUT"] || 10).to_i
     DEFAULT_QUEUE_NAME  = "queue_classic_jobs"
 
-    def self.create_queue(name)
-      db = new(name)
-      db.create_table
-      db.disconnect
-      true
-    end
-
     attr_reader :table_name
 
     def initialize(queue_name=nil)
+      log("initialized")
+
       @top_boundry = MAX_TOP_BOUND
+      log("top_boundry=#{@top_boundry}")
+
       @table_name = queue_name || DEFAULT_QUEUE_NAME
+      log("table_name=#{@table_name}")
+
       @db_params = URI.parse(DATABASE_URL)
+      log("uri=#{DATABASE_URL}")
     end
 
-    def init_db
-      drop_table
-      create_table
-      load_functions
+    def set_application_name
+      execute("SET application_name = 'queue_classic'")
     end
 
     def listen
+      log("LISTEN")
       execute("LISTEN queue_classic_jobs")
     end
 
     def unlisten
+      log("UNLISTEN")
       execute("UNLISTEN queue_classic_jobs")
     end
 
     def wait_for_notify
+      log("waiting for notify timeout=#{NOTIFY_TIMEOUT}")
       connection.wait_for_notify(NOTIFY_TIMEOUT)
+      log("done waiting for notify")
     end
 
-    def waiting_conns
-      execute("SELECT * FROM pg_stat_activity WHERE datname = '#{@name}' AND waiting = 't' AND application_name = 'queue_classic'")
-    end
-
-    def all_conns
-      execute("SELECT * FROM pg_stat_activity WHERE datname = '#{@name}' AND application_name = 'queue_classic'")
-    end
-
-    def silence_warnings
-      execute("SET client_min_messages TO 'warning'")
+    def execute(sql)
+      log("executing=#{sql}")
+      begin
+        connection.exec(sql)
+      rescue PGError => e
+        log("execute exception=#{e.inspect}")
+      end
     end
 
-    def execute(sql)
-      connection.exec(sql)
+    def connection
+      @@connection ||= connect
     end
 
     def disconnect
@@ -61,30 +61,20 @@ module QC
       @@connection = nil
     end
 
-    def connection
-      unless @@connection
-        @name = @db_params.path.gsub("/","")
-        @@connection = PGconn.connect(
-          @db_params.host,
-          @db_params.port || 5432,
-          nil, '',
-          @name,
-          @db_params.user,
-          @db_params.password
-        )
-        @@connection.exec("SET application_name = 'queue_classic'")
-        silence_warnings unless VERBOSE
+    def connect
+      log("establishing connection")
+      conn = PGconn.connect(
+        @db_params.host,
+        @db_params.port || 5432,
+        nil, '', #opts, tty
+        @db_params.path.gsub("/",""), # database name
+        @db_params.user,
+        @db_params.password
+      )
+      if conn.status != PGconn::CONNECTION_OK
+        log("connection error=#{conn.error}")
       end
-      @@connection
-    end
-
-    def drop_table
-      execute("DROP TABLE IF EXISTS #{@table_name} CASCADE")
-    end
-
-    def create_table
-      execute("CREATE TABLE #{@table_name} (id serial, details text, locked_at timestamp)")
-      execute("CREATE INDEX #{@table_name}_id_idx ON #{@table_name} (id)")
+      conn
     end
 
     def load_functions
@@ -141,5 +131,9 @@ module QC
       EOD
     end
 
+    def log(msg)
+      Logger.puts(["database", msg].join(" "))
+    end
+
   end
 end

data/lib/queue_classic/durable_array.rb

@@ -7,7 +7,7 @@ module QC
     end
 
     def <<(details)
-      execute("INSERT INTO #{@table_name} (details) VALUES ('#{details.to_json}')")
+      execute("INSERT INTO #{@table_name} (details) VALUES ('#{JSON.dump(details)}')")
     end
 
     def count

data/lib/queue_classic/logger.rb

@@ -0,0 +1,17 @@
+module QC
+  module Logger
+
+    extend self
+
+    def ruby_logger
+      @@logger ||= ::Logger.new(STDOUT)
+    end
+
+    def puts(msg)
+      if VERBOSE
+        ruby_logger.debug(msg)
+      end
+    end
+
+  end
+end

data/lib/queue_classic/queue.rb

@@ -32,29 +32,12 @@ module QC
   end
 end
 
-module QC
-  module ConnectionHelper
-
-    def connection_status
-      {:total => database.all_conns.count, :waiting => database.waiting_conns.count}
-    end
-
-    def disconnect
-      database.disconnect
-    end
-
-  end
-end
-
 module  QC
   class Queue
 
     include AbstractQueue
     extend AbstractQueue
 
-    include ConnectionHelper
-    extend ConnectionHelper
-
     def self.array
       if defined? @@array
         @@array

data/lib/queue_classic/tasks.rb

@@ -22,8 +22,4 @@ namespace :qc do
     db.disconnect
   end
 
-  task :create_queue, :name, :needs => :environment do |t,args|
-    QC::Database.create_queue(args[:name])
-  end
-
 end

data/lib/queue_classic/worker.rb

@@ -4,9 +4,16 @@ module QC
     MAX_LOCK_ATTEMPTS = (ENV["QC_MAX_LOCK_ATTEMPTS"] || 5).to_i
 
     def initialize
-      @running     = true
-      @queue       = QC::Queue.new(ENV["QUEUE"])
+      log("worker initialized")
+      log("worker running exp. backoff algorith max_attempts=#{MAX_LOCK_ATTEMPTS}")
+      @running = true
+
+      @queue = QC::Queue.new(ENV["QUEUE"])
+      log("worker table=#{@queue.database.table_name}")
+
       @fork_worker = ENV["QC_FORK_WORKER"] == "true"
+      log("worker fork=#{@fork_worker}")
+
       handle_signals
     end
 
@@ -23,6 +30,7 @@ module QC
         trap(sig) do
           if running?
             @running = false
+            log("worker running=#{@running}")
           else
             raise Interrupt
           end
@@ -30,8 +38,14 @@ module QC
       end
     end
 
+    def setup_child
+      log("forked worker running setup")
+    end
+
     def start
+      log("worker starting")
       while running?
+        log("worker running...")
         if fork_worker?
           fork_and_work
         else
@@ -41,38 +55,49 @@ module QC
     end
 
     def fork_and_work
-      @cpid = fork { work }
+      @cpid = fork { setup_child; work }
+      log("worker forked pid=#{@cpid}")
       Process.wait(@cpid)
     end
 
     def work
+      log("worker start working")
       if job = lock_job
+        log("worker locked job=#{job.id}")
         begin
           job.work
+          log("worker finished job=#{job.id}")
         rescue Object => e
+          log("worker failed job=#{job.id} exception=#{e.inspect}")
           handle_failure(job,e)
         ensure
           @queue.delete(job)
+          log("worker deleted job=#{job.id}")
         end
       end
-      job = nil
-      GC.start
     end
 
     def lock_job
+      log("worker attempting a lock")
       attempts = 0
       job = nil
       until job
         job = @queue.dequeue
         if job.nil?
+          log("worker missed lock attempt=#{attempts}")
           attempts += 1
           if attempts < MAX_LOCK_ATTEMPTS
-            sleep(2**attempts)
+            seconds = 2**attempts
+            log("worker sleeps seconds=#{seconds}")
+            sleep(seconds)
+            log("worker tries again")
             next
           else
+            log("worker reached max attempts. max=#{MAX_LOCK_ATTEMPTS}")
             break
           end
         else
+          log("worker successfully locked job")
         end
       end
       job
@@ -87,5 +112,9 @@ module QC
       puts "!"
     end
 
+    def log(msg)
+      Logger.puts(msg)
+    end
+
   end
 end

data/test/database_helpers.rb

@@ -1,11 +1,13 @@
 module DatabaseHelpers
 
-  def init_db(table_name=nil)
-    database  = QC::Database.new(table_name)
-    database.silence_warnings
-    database.init_db
+  def init_db(table_name="queue_classic_jobs")
+    database = QC::Database.new(table_name)
+    database.execute("SET client_min_messages TO 'warning'")
+    database.execute("DROP TABLE IF EXISTS #{table_name} CASCADE")
+    database.execute("CREATE TABLE #{table_name} (id serial, details text, locked_at timestamp)")
+    database.load_functions
     database.disconnect
-    true
+    database
   end
 
 end

data/test/durable_array_test.rb

@@ -3,9 +3,7 @@ require File.expand_path("../helper.rb", __FILE__)
 context "DurableArray" do
 
   setup do
-    @database = QC::Database.new
-    @database.drop_table
-    @database.init_db
+    @database = init_db
     @array = QC::DurableArray.new(@database)
   end
 

data/test/queue_test.rb

@@ -2,7 +2,7 @@ require File.expand_path("../helper.rb", __FILE__)
 
 context "Queue" do
 
-  setup { init_db }
+  setup { @database = init_db }
 
   test "Queue class responds to enqueue" do
     QC::Queue.enqueue("Klass.method")
@@ -43,10 +43,10 @@ context "Queue" do
 
   test "queue instance responds to enqueue" do
     QC::Queue.enqueue("Something.hard_to_find")
-    init_db(:custom_queue_name)
+    tmp_db = init_db(:custom_queue_name)
     @queue = QC::Queue.new(:custom_queue_name)
     @queue.enqueue "Klass.method"
-    @queue.disconnect
+    @queue.database.disconnect
   end
 
   test "queue only uses 1 connection per class" do
@@ -55,7 +55,7 @@ context "Queue" do
     QC::Queue.delete QC::Queue.dequeue
     QC::Queue.enqueue "Klass.method"
     QC::Queue.dequeue
-    assert_equal 1, QC.connection_status[:total]
+    assert_equal 1, @database.execute("SELECT count(*) from pg_stat_activity")[0]["count"].to_i
   end
 
 end

data/test/worker_test.rb

@@ -23,10 +23,14 @@ end
 context "Worker" do
 
   setup do
-    init_db
+    @database = init_db
     @worker = TestWorker.new
   end
 
+  teardown do
+    @database.disconnect
+  end
+
   test "working a job" do
     QC::Queue.enqueue "TestNotifier.deliver", {}
 
@@ -46,6 +50,7 @@ context "Worker" do
   test "only makes one connection" do
     QC.enqueue "TestNotifier.deliver", {}
     @worker.work
-    assert_equal 1, QC.connection_status[:total]
+    assert_equal 1, @database.execute("SELECT count(*) from pg_stat_activity")[0]["count"].to_i
   end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: queue_classic
 version: !ruby/object:Gem::Version
-  version: 0.3.3.pre
+  version: 0.3.5.pre
   prerelease: 6
 platform: ruby
 authors:
@@ -13,26 +13,15 @@ date: 2011-08-22 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
-  requirement: &2153629240 !ruby/object:Gem::Requirement
+  requirement: &2160828260 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 0.10.1
+        version: 0.11.0
   type: :runtime
   prerelease: false
-  version_requirements: *2153629240
-- !ruby/object:Gem::Dependency
-  name: json
-  requirement: &2153628740 !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: *2153628740
+  version_requirements: *2160828260
 description: Queue Classic (beta) is a queueing library for Ruby apps (Rails, Sinatra,
   Etc...) Queue Classic features asynchronous job polling, database maintained locks
   and no ridiculous dependencies. As a matter of fact, Queue Classic only requires
@@ -46,6 +35,7 @@ files:
 - lib/queue_classic/database.rb
 - lib/queue_classic/durable_array.rb
 - lib/queue_classic/job.rb
+- lib/queue_classic/logger.rb
 - lib/queue_classic/queue.rb
 - lib/queue_classic/tasks.rb
 - lib/queue_classic/worker.rb