queue_classic 2.0.0rc9 → 2.0.0rc10

data/lib/queue_classic/conn.rb

@@ -3,7 +3,7 @@ module QC
     extend self
 
     def execute(stmt, *params)
-      log("executing #{stmt.inspect}, #{params.inspect}")
+      log(:level => :debug, :action => "exec_sql", :sql => stmt.inspect)
       begin
         params = nil if params.empty?
         r = connection.exec(stmt, params)
@@ -11,38 +11,36 @@ module QC
         r.each {|t| result << t}
         result.length > 1 ? result : result.pop
       rescue PGError => e
-        log("execute exception=#{e.inspect}")
+        log(:error => e.inspect)
         raise
       end
     end
 
     def notify(chan)
-      log("NOTIFY")
+      log(:level => :debug, :action => "NOTIFY")
       execute('NOTIFY "' + chan + '"') #quotes matter
     end
 
     def listen(chan)
-      log("LISTEN")
+      log(:level => :debug, :action => "LISTEN")
       execute('LISTEN "' + chan + '"') #quotes matter
     end
 
     def unlisten(chan)
-      log("UNLISTEN")
+      log(:level => :debug, :action => "UNLISTEN")
       execute('UNLISTEN "' + chan + '"') #quotes matter
     end
 
     def drain_notify
       until connection.notifies.nil?
-        log("draining notifications")
+        log(:level => :debug, :action => "drain_notifications")
       end
     end
 
     def wait_for_notify(t)
-      log("waiting for notify timeout=#{t}")
       connection.wait_for_notify(t) do |event, pid, msg|
-        log("received notification #{event}")
+        log(:level => :debug, :action => "received_notification")
       end
-      log("done waiting for notify")
     end
 
     def transaction
@@ -70,7 +68,7 @@ module QC
     end
 
     def connect
-      log("establishing connection")
+      log(:level => :debug, :action => "establish_conn")
       conn = PGconn.connect(
         db_url.host,
         db_url.port || 5432,
@@ -80,7 +78,7 @@ module QC
         db_url.password
       )
       if conn.status != PGconn::CONNECTION_OK
-        log("connection error=#{conn.error}")
+        log(:level => :error, :message => conn.error)
       end
       conn
     end
@@ -90,7 +88,7 @@ module QC
     end
 
     def log(msg)
-      Log.info(msg)
+      QC.log(msg)
     end
 
   end

data/lib/queue_classic/queries.rb

@@ -3,9 +3,11 @@ module QC
     extend self
 
     def insert(q_name, method, args, chan=nil)
-      s = "INSERT INTO #{TABLE_NAME} (q_name, method, args) VALUES ($1, $2, $3)"
-      res = Conn.execute(s, q_name, method, OkJson.encode(args))
-      Conn.notify(chan) if chan
+      QC.log_yield(:action => "insert_job") do
+        s = "INSERT INTO #{TABLE_NAME} (q_name, method, args) VALUES ($1, $2, $3)"
+        res = Conn.execute(s, q_name, method, OkJson.encode(args))
+        Conn.notify(chan) if chan
+      end
     end
 
     def lock_head(q_name, top_bound)

data/lib/queue_classic/scrolls.rb

@@ -0,0 +1,127 @@
+require "thread"
+
+module Scrolls
+  extend self
+
+  def log(data, &blk)
+    Log.log(data, &blk)
+  end
+
+  def log_exception(data, e)
+    Log.log_exception(data, e)
+  end
+
+  module Log
+    extend self
+
+    LOG_LEVEL = (ENV["QC_LOG_LEVEL"] || 3).to_i
+    LOG_LEVEL_MAP = {
+      "fatal" => 0,
+      "error" => 1,
+      "warn"  => 2,
+      "info"  => 3,
+      "debug" => 4
+    }
+
+    attr_accessor :stream
+
+    def start(out = nil)
+      # This allows log_exceptions below to pick up the defined output,
+      # otherwise stream out to STDERR
+      @defined = out.nil? ? false : true
+      sync_stream(out)
+    end
+
+    def sync_stream(out = nil)
+      out = STDOUT if out.nil?
+      @stream = out
+      @stream.sync = true
+    end
+
+    def mtx
+      @mtx ||= Mutex.new
+    end
+
+    def write(data)
+      if log_level_ok?(data[:level])
+        msg = unparse(data)
+        mtx.synchronize do
+          @stream.puts(msg)
+        end
+      end
+    end
+
+    def unparse(data)
+      data.map do |(k, v)|
+        if (v == true)
+          k.to_s
+        elsif v.is_a?(Float)
+          "#{k}=#{format("%.3f", v)}"
+        elsif v.nil?
+          nil
+        else
+          v_str = v.to_s
+          if (v_str =~ /^[a-zA-z0-9\-\_\.]+$/)
+            "#{k}=#{v_str}"
+          else
+            "#{k}=\"#{v_str.sub(/".*/, "...")}\""
+          end
+        end
+      end.compact.join(" ")
+    end
+
+    def log(data, &blk)
+      unless blk
+        write(data)
+      else
+        start = Time.now
+        res = nil
+        log(data.merge(:at => :start))
+        begin
+          res = yield
+        rescue StandardError, Timeout::Error => e
+          log(data.merge(
+            :at           => :exception,
+            :reraise      => true,
+            :class        => e.class,
+            :message      => e.message,
+            :exception_id => e.object_id.abs,
+            :elapsed      => Time.now - start
+          ))
+          raise(e)
+        end
+        log(data.merge(:at => :finish, :elapsed => Time.now - start))
+        res
+      end
+    end
+
+    def log_exception(data, e)
+      sync_stream(STDERR) unless @defined
+      log(data.merge(
+        :exception    => true,
+        :class        => e.class,
+        :message      => e.message,
+        :exception_id => e.object_id.abs
+      ))
+      if e.backtrace
+        bt = e.backtrace.reverse
+        bt[0, bt.size-6].each do |line|
+          log(data.merge(
+            :exception    => true,
+            :exception_id => e.object_id.abs,
+            :site         => line.gsub(/[`'"]/, "")
+          ))
+        end
+      end
+    end
+
+    def log_level_ok?(level)
+      if level
+        LOG_LEVEL_MAP[level.to_s] <= LOG_LEVEL
+      else
+        true
+      end
+    end
+
+  end
+end

data/lib/queue_classic/worker.rb

@@ -1,26 +1,37 @@
 module QC
   class Worker
 
-    def initialize(q_name, top_bound, fork_worker, listening_worker, max_attempts)
-      log("worker initialized")
-      @running = true
+    def initialize(*args)
+      if args.length == 5
+        q_name, top_bound, fork_worker, listening_worker, max_attempts = *args
+      elsif args.length <= 1
+        opts = args.first || {}
+        q_name           = opts[:q_name]           || QC::QUEUE
+        top_bound        = opts[:top_bound]        || QC::TOP_BOUND
+        fork_worker      = opts[:fork_worker]      || QC::FORK_WORKER
+        listening_worker = opts[:listening_worker] || QC::LISTENING_WORKER
+        max_attempts     = opts[:max_attempts]     || QC::MAX_LOCK_ATTEMPTS
+      else
+        raise ArgumentError, 'wrong number of arguments (expected no args, an options hash, or 5 separate args)'
+      end
 
+      @running = true
       @queue = Queue.new(q_name, listening_worker)
-      log("worker queue=#{@queue.name}")
-
       @top_bound = top_bound
-      log("worker top_bound=#{@top_bound}")
-
       @fork_worker = fork_worker
-      log("worker fork=#{@fork_worker}")
-
       @listening_worker = listening_worker
-      log("worker listen=#{@listening_worker}")
-
       @max_attempts = max_attempts
-      log("max lock attempts =#{@max_attempts}")
-
       handle_signals
+
+      log(
+        :level => :debug,
+        :action => "worker_initialized",
+        :queue => q_name,
+        :top_bound => top_bound,
+        :fork_worker => fork_worker,
+        :listening_worker => listening_worker,
+        :max_attempts => max_attempts
+      )
     end
 
     def running?
@@ -40,7 +51,7 @@ module QC
         trap(sig) do
           if running?
             @running = false
-            log("worker running=#{@running}")
+            log(:level => :debug, :action => "handle_signal", :running => @running)
           else
             raise Interrupt
           end
@@ -52,13 +63,10 @@ module QC
     # your worker is forking and you need to
     # re-establish database connectoins
     def setup_child
-      log("forked worker running setup")
     end
 
     def start
-      log("worker starting")
       while running?
-        log("worker running...")
         if fork_worker?
           fork_and_work
         else
@@ -69,48 +77,44 @@ module QC
 
     def fork_and_work
       @cpid = fork { setup_child; work }
-      log("worker forked pid=#{@cpid}")
+      log(:level => :debug, :action => :fork, :pid => @cpid)
       Process.wait(@cpid)
     end
 
     def work
-      log("worker start working")
       if job = lock_job
-        log("worker locked job=#{job[:id]}")
-        begin
-          call(job).tap do
-            log("worker finished job=#{job[:id]}")
+        QC.log_yield(:level => :info, :action => "work_job", :job => job[:id]) do
+          begin
+            call(job)
+          rescue Object => e
+            log(:level => :debug, :action => "failed_work", :job => job[:id], :error => e.inspect)
+            handle_failure(job, e)
+          ensure
+            @queue.delete(job[:id])
+            log(:level => :debug, :action => "delete_job", :job => job[:id])
           end
-        rescue Object => e
-          log("worker failed job=#{job[:id]} exception=#{e.inspect}")
-          handle_failure(job, e)
-        ensure
-          @queue.delete(job[:id])
-          log("worker deleted job=#{job[:id]}")
         end
       end
     end
 
     def lock_job
-      log("worker attempting a lock")
+      log(:level => :debug, :action => "lock_job")
       attempts = 0
       job = nil
       until job
         job = @queue.lock(@top_bound)
         if job.nil?
-          log("worker missed lock attempt=#{attempts}")
+          log(:level => :debug, :action => "failed_lock", :attempts => attempts)
           if attempts < @max_attempts
             seconds = 2**attempts
             wait(seconds)
-            log("worker tries again")
             attempts += 1
             next
           else
-            log("worker reached max attempts. max=#{@max_attempts}")
             break
           end
         else
-          log("worker successfully locked job")
+          log(:level => :debug, :action => "finished_lock", :job => job[:id])
         end
       end
       job
@@ -125,14 +129,14 @@ module QC
 
     def wait(t)
       if can_listen?
-        log("worker waiting on LISTEN")
+        log(:level => :debug, :action => "listen_wait", :wait => t)
         Conn.listen(@queue.chan)
         Conn.wait_for_notify(t)
         Conn.unlisten(@queue.chan)
         Conn.drain_notify
-        log("worker finished LISTEN")
+        log(:level => :debug, :action => "finished_listening")
       else
-        log("worker sleeps seconds=#{t}")
+        log(:level => :debug, :action => "sleep_wait", :wait => t)
         Kernel.sleep(t)
       end
     end
@@ -146,8 +150,8 @@ module QC
       puts "!"
     end
 
-    def log(msg)
-      Log.info(msg)
+    def log(data)
+      QC.log(data)
     end
 
   end

data/lib/queue_classic.rb

@@ -1,10 +1,9 @@
 require "pg"
-
-require "logger"
 require "uri"
 
 $: << File.expand_path(__FILE__, "lib")
 
+require "queue_classic/scrolls"
 require "queue_classic/okjson"
 require "queue_classic/conn"
 require "queue_classic/queries"
@@ -12,6 +11,8 @@ require "queue_classic/queue"
 require "queue_classic/worker"
 
 module QC
+  # ENV["QC_LOG_LEVEL"] is used in Scrolls
+  Scrolls::Log.start
 
   Root = File.expand_path("..", File.dirname(__FILE__))
   SqlFunctions = File.join(QC::Root, "/sql/ddl.sql")
@@ -22,9 +23,6 @@ module QC
     ENV["DATABASE_URL"]    ||
     raise(ArgumentError, "missing QC_DATABASE_URL or DATABASE_URL")
 
-  # export QC_LOG_LEVEL=`ruby -r "logger" -e "puts Logger::ERROR"`
-  LOG_LEVEL = (ENV["QC_LOG_LEVEL"] || Logger::DEBUG).to_i
-
   # You can use the APP_NAME to query for
   # postgres related process information in the
   # pg_stat_activity table. Don't set this unless
@@ -66,12 +64,6 @@ module QC
   # as the max exponent.
   MAX_LOCK_ATTEMPTS = (ENV["QC_MAX_LOCK_ATTEMPTS"] || 5).to_i
 
-
-  # Setup the logger
-  Log = Logger.new($stdout)
-  Log.level = LOG_LEVEL
-  Log.info("program=queue_classic log=true")
-
   # Defer method calls on the QC module to the
   # default queue. This facilitates QC.enqueue()
   def self.method_missing(sym, *args, &block)
@@ -84,4 +76,21 @@ module QC
     end
   end
 
+  def self.log_yield(data)
+    begin
+      t0 = Time.now
+      yield
+    rescue => e
+      log({:level => :error, :error => e.class, :message => e.message.strip}.merge(data))
+      raise
+    ensure
+      t = Integer((Time.now - t0)*1000)
+      log(data.merge(:elapsed => t)) unless e
+    end
+  end
+
+  def self.log(data)
+    Scrolls.log({:lib => :queue_classic}.merge(data))
+  end
+
 end

data/readme.md

@@ -1,6 +1,6 @@
 # queue_classic
 
-v2.0.0rc4
+v2.0.0rc9
 
 queue_classic is a PostgreSQL-backed queueing library that is focused on
 concurrent job locking, minimizing database load & providing a simple &
@@ -14,6 +14,7 @@ queue_classic features:
 * Forking workers
 * Postgres' rock-solid locking mechanism
 * Fuzzy-FIFO support [academic paper](http://www.cs.tau.ac.il/~shanir/nir-pubs-web/Papers/Lock_Free.pdf)
+* Instrumentation via log output
 * Long term support
 
 ## Proven
@@ -387,6 +388,17 @@ a method that you can override. This method will be passed 2 arguments: the
 exception instance and the job. Here are a few examples of things you might want
 to do inside `handle_failure()`.
 
+## Instrumentation
+
+QC will log elapsed time, errors and general usage in the form of data.
+To customize the output of the log data, override `QC.log` and `QC.log_yield`.
+By default, QC uses a simple wrapper around $stdout to put the log data in k=v
+format. For instance:
+
+```
+lib=queue_classic level=info action=insert_job elapsed=16
+```
+
 ## Tips and Tricks
 
 ### Running Synchronously for tests

data/test/helper.rb

@@ -7,8 +7,6 @@ require "queue_classic"
 require "minitest/unit"
 MiniTest::Unit.autorun
 
-QC::Log.level = Logger::ERROR
-
 class QCTest < MiniTest::Unit::TestCase
 
   def setup

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: queue_classic
 version: !ruby/object:Gem::Version
-  version: 2.0.0rc9
+  version: 2.0.0rc10
   prerelease: 5
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2012-02-29 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
-  requirement: &20818900 !ruby/object:Gem::Requirement
+  requirement: &13897100 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,7 +21,7 @@ dependencies:
         version: 0.13.2
   type: :runtime
   prerelease: false
-  version_requirements: *20818900
+  version_requirements: *13897100
 description: queue_classic is a queueing library for Ruby apps. (Rails, Sinatra, Etc...)
   queue_classic features asynchronous job polling, database maintained locks and no
   ridiculous dependencies. As a matter of fact, queue_classic only requires pg.
@@ -36,6 +36,7 @@ files:
 - lib/queue_classic/conn.rb
 - lib/queue_classic/okjson.rb
 - lib/queue_classic/worker.rb
+- lib/queue_classic/scrolls.rb
 - lib/queue_classic/queue.rb
 - lib/queue_classic/tasks.rb
 - lib/queue_classic/queries.rb