logworm_client 0.6.2 → 0.7.0

data/CHANGELOG

@@ -1,3 +1,18 @@
+v0.7.0
+    Eliminated long logs. Instead, there's now a parameter (log_request_headers in Rack, lw_log_request_headers in Rails) to add
+       request and response headers to the web_log. 
+    Cleaned up fields in the web_log table
+
+    Added option to turn off automatic logging of requests (disable_request_logging in Rack, lw_disable_request_logging in Rails)
+    Added option to turn on logging even if in development mode (enable_dev_logging in Rack, lw_enable_dev_logging in Rails)
+
+    Honor filter_parameter_logging switch when used with Rails
+
+    Cleaned up log and flush
+    Cleaned up display of elapsed time for flush
+    Cleaned up call from Rack and Rails, and enforce timeout (1 sec by default)
+    Added unit tests
+    
 v0.6.2
     Added list_tables command
     

data/Manifest

@@ -11,3 +11,8 @@ lib/logworm_client/rails.rb
 lib/logworm_utils.rb
 lib/logworm_utils/compute.rb
 lib/logworm_utils/tail.rb
+spec/logger_spec.rb
+spec/rack_spec.rb
+spec/rails_spec.rb
+spec/spec.opts
+spec/spec_helper.rb

data/Rakefile

@@ -1,5 +1,5 @@
 require 'echoe'
-Echoe.new('logworm_client', '0.6.2') do |p|
+Echoe.new('logworm_client', '0.7.0') do |p|
   p.description = "logworm client utilities"
   p.url = "http://www.logworm.com"
   p.author = "Pomelo, LLC"

data/lib/logworm_client/logger.rb

@@ -4,51 +4,90 @@ module Logworm
   class LogwormException < Exception ; end
 
   class Logger
-    # Initialize
-    LW_SERVER = DB.from_config
     $lr_queue = []
     
+    ###
+    # Use connection to the backend servers specified in environment variable or config file
+    ###
+    def self.use_default_db
+       $lw_server = DB.from_config
+    end
+    
+    ###
+    # Use a connection to a manually specified server
+    ###
+    def self.use_db(db)
+      $lw_server = db
+    end
+    
+    ###
+    # Returns a reference to the current backend server
+    ###
+    def self.db
+      $lw_server
+    end
+    
+    
+    ###
+    # Starts a new cycle: sets a request_id variable, so that all entries (until flush) share that id
+    ###
     def self.start_cycle
       $request_id = "#{Thread.current.object_id}-#{(Time.now.utc.to_f * 1000).to_i}"
     end
 
+    ###
+    # Record an entry. Not sent to the servers until 'flush' is called
+    #
+    # Warning: may raise Exception if there's a problem. It's up to the called to rescue from it in order to continue the processing
+    # of a web request, for example.
+    ###
     def self.log(table, values = {})
-      begin
-        # Rename conflicting keys
-        kvalues = values.delete_if {|k,v| k.to_s == ""}.map {|k,v| [k.to_sym, v]}
-        kvalues = Hash[*kvalues.flatten]
-        [:_ts, :_ts_utc, :_request_id].each do |k|
-          kvalues["orig_#{k}".to_sym] = kvalues[k] if kvalues.has_key? k
-        end
-        # Add information
-        ts = Time.now.utc
-        log_details = {:_ts => ts.strftime("%Y-%m-%dT%H:%M:%SZ"), :_ts_utc =>  (ts.to_f * 1000).to_i}
-        log_details[:_request_id] = $request_id if $request_id
-        # Enqueue
-        payload = kvalues.merge(log_details)
-        $lr_queue << [table, payload]
-      rescue Exception => e
-        $stderr.puts "Error in Logworm::Logger.log: #{e}"
+      return unless table and (table.is_a? String or table.is_a? Symbol)
+      return unless values.is_a? Hash
+
+      # Turn keys into symbols, delete empty ones, rename conflicting ones
+      kvalues = values.delete_if {|k,v| k.to_s == ""}.map {|k,v| [k.to_sym, v]}
+      kvalues = Hash[*kvalues.flatten]
+      [:_ts, :_ts_utc, :_request_id].each do |k|
+        kvalues["orig_#{k}".to_sym] = kvalues[k] if kvalues.has_key? k
       end
+      
+      # Add information
+      ts = Time.now.utc
+      kvalues[:_ts_utc]     = (ts.to_f * 1000).to_i
+      kvalues[:_ts]         = ts.strftime("%Y-%m-%dT%H:%M:%SZ")
+      kvalues[:_request_id] = $request_id if $request_id
+      
+      # Enqueue
+      $lr_queue << [table, kvalues]
     end
 
+    ###
+    # Sends the entries to the server, if configured
+    # Returns the number of entries send, and the time it takes
+    #
+    # Warning: may raise Exception if there's a problem. It's up to the called to rescue from it in order to continue the processing
+    # of a web request, for example.
+    ###
     def self.flush
       to_send = $lr_queue.size
-      return 0 if to_send == 0 
-      unless LW_SERVER
+
+      # Return if no entries
+      return [0,0] if to_send == 0 
+
+      # Return if no server
+      unless $lw_server
         $stderr.puts "\t logworm not configured. #{to_send} entries dropped."
         $lr_queue = [] 
-        return 0
+        return [0,0]
       end
       
-      begin
-        LW_SERVER.batch_log($lr_queue.to_json)
-      rescue Exception => e
-        $stderr.puts "in Logworm::Logger.flush: #{e}"
-        $stderr.puts e.backtrace
-      end
+      startTime = Time.now
+      $lw_server.batch_log($lr_queue.to_json)
       $lr_queue = [] 
-      to_send
+      endTime = Time.now
+      
+      [to_send, (endTime - startTime)]
     end
   end
 end

data/lib/logworm_client/rack.rb

@@ -1,60 +1,62 @@
-require 'benchmark'
+require 'rack/request'
 
 module Logworm
   class Rack
 
     def initialize(app, options = {})
       @app = app
-      @log_requests_short =  options[:log_requests].nil? or options[:log_requests] != false
-      @log_requests_long  = !options[:log_requests_long].nil? and options[:log_requests_long] == true
+      
+      @log_requests = (options[:disable_request_logging].nil? or options[:disable_request_logging] != true)
+      @log_headers  = (options[:log_request_headers] and options[:log_request_headers] == true)
+      @dev_logging  = (options[:enable_dev_logging] and options[:enable_dev_logging] == true)
+      Logger.use_default_db
+      @timeout = 1
     end
 
     def call(env)
-      return @app.call(env) unless ENV['RACK_ENV'] == 'production'
+      return @app.call(env) unless (ENV['RACK_ENV'] == 'production' or (ENV['RACK_ENV'] == 'development' and @dev_logging))
 
-      startTime = Time.now 
       Logger.start_cycle
-      status, headers, body = @app.call(env) 
-      appTime = (Time.now - startTime) 
-      queue_size = env['HTTP_X_HEROKU_QUEUE_DEPTH'] ? env['HTTP_X_HEROKU_QUEUE_DEPTH'].to_i : -1
-      Logger.log(:web_log_long, {:summary => "#{env['REQUEST_METHOD']} #{env['REQUEST_URI']} - #{status} #{appTime}", 
-                             :request => env_to_log(env), 
-                             :response => {:status => status, :headers => headers}, 
-                             :profiling => appTime}) if @log_requests_long 
-      Logger.log(:web_log, {:summary => "#{env['REQUEST_METHOD']} #{env['REQUEST_URI']} - #{status} #{appTime}", 
-                            :request_path => env['REQUEST_PATH'], :request_ip => env['REMOTE_ADDR'],
-                            :request_method => env['REQUEST_METHOD'], :input => collect_input(env),
-                            :response_status => status, 
-                            :profiling => appTime,
-                            :queue_size => queue_size}) if @log_requests_short
+      begin
+        startTime = Time.now 
+        status, response_headers, body = @app.call(env) 
+        appTime = (Time.now - startTime) 
+      ensure
+        log_request(env, status, response_headers, appTime)
+        return [status, response_headers, body] 
+      end
+    end
+    
+    private 
+    def log_request(env, status, response_headers, appTime)
+      method       = env['REQUEST_METHOD']
+      path         = (env['REQUEST_PATH'].nil? or env['REQUEST_PATH'] == "")  ? "/" : env['REQUEST_PATH']
+      ip           = env['REMOTE_ADDR']
+      http_headers = env.reject {|k,v| !(k.to_s =~ /^HTTP/) }
+      queue_size   = env['HTTP_X_HEROKU_QUEUE_DEPTH'].nil? ? -1 : env['HTTP_X_HEROKU_QUEUE_DEPTH'].to_i
+
+      entry = { :summary         => "#{method} #{path} - #{status} #{appTime}", 
+                :request_method  => method,
+                :request_path    => path, 
+                :request_ip      => ip,
+                :input           => ::Rack::Request.new(env).params,
+                :response_status => status, 
+                :profiling       => appTime,
+                :queue_size      => queue_size}
+      entry[:headers] = http_headers if @log_headers
+      entry[:response_headers] = response_headers if @log_headers
+      Logger.log(:web_log, entry) if @log_requests
+
       begin 
-        Timeout::timeout(1) { 
-          sent = 0 
-          ts = Benchmark.realtime {sent = Logger.flush}  # Flushes only if there are any entries. Times out after a second 
-          env['rack.errors'].puts("\t *** logworm - logs sent in #{ts} seconds") if sent > 0
+        Timeout::timeout(@timeout) { 
+          sent, elapsed = Logger.flush
+          env['rack.errors'].puts("-- #{sent} logworm entries recorded in #{sprintf('%.4f', elapsed)} seconds") if sent > 0
         } 
       rescue Exception => e 
-        # Ignore --nothing we can do 
-        # The list of logs may (and most likely will) be preserved for the next request
-        env['rack.errors'].puts "logworm call failed: #{e}"
+        # Ignore --nothing we can do. The list of logs may (and most likely will) be preserved for the next request
+        env['rack.errors'].puts("logworm call failed: #{e}")
       end
-      [status, headers, body] 
     end
 
-    private
-      def env_to_log(env) 
-        to_log = {} 
-        env.each do |k,v| 
-          to_log[k.to_s.downcase.to_sym] = v unless (k.to_s =~ /rack/i or k.to_s =~ /\./i) 
-        end 
-        to_log.merge({:input => collect_input(env)}) 
-      end 
-
-      def collect_input(env) 
-        inputstr = "" 
-        env['rack.input'].each {|content| inputstr += content } 
-        env['rack.input'].rewind 
-        inputstr 
-      end 
   end
 end

data/lib/logworm_client/rails.rb

@@ -2,75 +2,104 @@
 # Extemds ActionController::Base and aliases the process method, to wrap it with the standard logworm request cycle
 #
 # By default it will automatically log web requests (in a short format) into web_log
-# Can also log in long format if specified.
-#
-# Configuration
-#  In ApplicationController, add
-#    logs_requests :short, :long 
-#    logs_requests :long 
-#    logs_requests :short, :long 
+# Can also log headers, if specified
 ###
 if defined?(ActionController)
 
   require 'benchmark'
 
   ActionController::Base.class_eval do
-    @@log_requests_short = true
-    @@log_requests_long  = false
+    ## Basic settings: log requests, without headers. Use default db, and timeout after 1 second
+    @@log_requests = true
+    @@log_headers  = false
+    @@dev_logging  = false
+    @@timeout = 1
+    Logworm::Logger.use_default_db
+    
+    ###
+    # Disable automatic logging of requests
+    # Use from ApplicationController
+    ###
+    def self.lw_disable_request_logging
+      @@log_requests = false
+    end
+    
+    ###
+    # Log headers with requests
+    # Use from ApplicationController
+    ###
+    def self.lw_log_request_headers
+      @@log_headers = true
+    end
     
-    def self.logs_requests(*options)
-      @@log_requests_short = (options.include?(:short) or options.include?("short"))
-      @@log_requests_long  = (options.include?(:long)  or options.include?("long"))
+    ###
+    # Turn on logging in development mode
+    ###
+    def self.lw_enable_dev_logging
+      @@dev_logging = true
     end
     
+    ###
+    # Replaces (and embeds) the default Rails processing of a request. 
+    # Call the original method, logs the request unless disabled, and flushes the logworm list
+    ###
     def process_with_logworm_log(request, response, method = :perform_action, *arguments)
-      return process_without_logworm_log(request, response, method, *arguments) unless RAILS_ENV == 'production'
+      unless (RAILS_ENV == 'production' or (RAILS_ENV == 'development' and @@dev_logging))
+        return process_without_logworm_log(request, response, method, *arguments) 
+      end
       
-      startTime = Time.now 
       Logworm::Logger.start_cycle
       begin
-        response = process_without_logworm_log(request, response, method, *arguments)
-        appTime = (Time.now - startTime) 
+        startTime = Time.now 
+        response  = process_without_logworm_log(request, response, method, *arguments)
+        appTime   = (Time.now - startTime) 
       ensure
-        env = request.env
-        status = response.status[0..2]
-        queue_size = env['HTTP_X_HEROKU_QUEUE_DEPTH'] ? env['HTTP_X_HEROKU_QUEUE_DEPTH'].to_i : -1
-        Logworm::Logger.log(:web_log_long, 
-                             {:summary => "#{env['REQUEST_METHOD']} #{env['REQUEST_URI']} - #{status} #{appTime}", 
-                               :request => env_to_log(env).merge({:input => request.request_parameters}),
-                               :response => {:status => status, :headers => response.headers}, 
-                               :profiling => appTime,
-                               :queue_size => queue_size}) if @@log_requests_long 
-        Logworm::Logger.log(:web_log, 
-                             {:summary => "#{env['REQUEST_METHOD']} #{env['REQUEST_URI']} - #{status} #{appTime}", 
-                              :request_path => env['REQUEST_URI'], :request_ip => env['REMOTE_ADDR'],
-                              :request_method => env['REQUEST_METHOD'],
-                              :input => request.request_parameters,
-                              :response_status => status, 
-                              :profiling => appTime}) if @@log_requests_short
-        begin 
-          Timeout::timeout(1) {
-            sent = 0 
-            ts = Benchmark.realtime {sent = Logworm::Logger.flush}  # Flushes only if there are any entries. Times out after a second 
-            logger.info("\t *** logworm - logs sent in #{ts} seconds") if sent > 0
-          } 
-        rescue Exception => e 
-          # Ignore --nothing we can do 
-          # The list of logs may (and most likely will) be preserved for the next request
-          logger.error("logworm call failed: #{e}") if logger
-        end
+        log_request(request, response, appTime)
         return response
       end
     end
+    alias_method_chain :process, :logworm_log
     
-    def env_to_log(env) 
-      to_log = {} 
-      env.each do |k,v| 
-        to_log[k.to_s.downcase.to_sym] = v unless (k.to_s =~ /rack/i or k.to_s =~ /\./i) 
-      end 
-    end 
+
+    private
+    def log_request(request, response, appTime)
+      method       = request.env['REQUEST_METHOD']
+      path         = request.env['REQUEST_PATH'].blank? ? "/" : request.env['REQUEST_PATH']
+      ip           = request.env['REMOTE_ADDR']
+      http_headers = request.headers.reject {|k,v| !(k.to_s =~ /^HTTP/) }
+      status       = response.status[0..2]
+      queue_size   = request.env['HTTP_X_HEROKU_QUEUE_DEPTH'].blank? ? -1 : request.env['HTTP_X_HEROKU_QUEUE_DEPTH'].to_i
+
+      entry = { :summary         => "#{method} #{path} - #{status} #{appTime}", 
+                :request_method  => method,
+                :request_path    => path, 
+                :request_ip      => ip,
+                :input           => cleaned_input(request),
+                :response_status => status, 
+                :profiling       => appTime,
+                :queue_size      => queue_size}
+      entry[:headers] = http_headers if @@log_headers
+      entry[:response_headers] = response.headers if @@log_headers
+      Logworm::Logger.log(:web_log, entry) if @@log_requests
+
+      begin 
+        Timeout::timeout(@@timeout) { 
+          sent, elapsed = Logworm::Logger.flush
+          Rails.logger.info("#{sent} logworm entries recorded in #{sprintf('%.4f', elapsed)} seconds") if sent > 0
+        } 
+      rescue Exception => e 
+        # Ignore --nothing we can do. The list of logs may (and most likely will) be preserved for the next request
+        Rails.logger.error("logworm call failed: #{e}")
+      end
+    end
+    
+    def cleaned_input(request)
+      pars = request.parameters.clone
+      pars.delete("controller")
+      pars.delete("action")
+      respond_to?(:filter_parameters) ? filter_parameters(pars) : pars
+    end
     
-    alias_method_chain :process, :logworm_log
   end
 
 end

data/logworm_client.gemspec

@@ -2,16 +2,16 @@
 
 Gem::Specification.new do |s|
   s.name = %q{logworm_client}
-  s.version = "0.6.2"
+  s.version = "0.7.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Pomelo, LLC"]
-  s.date = %q{2010-04-23}
+  s.date = %q{2010-06-23}
   s.description = %q{logworm client utilities}
   s.email = %q{schapira@pomelollc.com}
   s.executables = ["lw-compute", "lw-tail"]
   s.extra_rdoc_files = ["CHANGELOG", "README", "bin/lw-compute", "bin/lw-tail", "lib/logworm_client.rb", "lib/logworm_client/logger.rb", "lib/logworm_client/rack.rb", "lib/logworm_client/rails.rb", "lib/logworm_utils.rb", "lib/logworm_utils/compute.rb", "lib/logworm_utils/tail.rb"]
-  s.files = ["CHANGELOG", "Manifest", "README", "Rakefile", "bin/lw-compute", "bin/lw-tail", "lib/logworm_client.rb", "lib/logworm_client/logger.rb", "lib/logworm_client/rack.rb", "lib/logworm_client/rails.rb", "lib/logworm_utils.rb", "lib/logworm_utils/compute.rb", "lib/logworm_utils/tail.rb", "logworm_client.gemspec"]
+  s.files = ["CHANGELOG", "Manifest", "README", "Rakefile", "bin/lw-compute", "bin/lw-tail", "lib/logworm_client.rb", "lib/logworm_client/logger.rb", "lib/logworm_client/rack.rb", "lib/logworm_client/rails.rb", "lib/logworm_utils.rb", "lib/logworm_utils/compute.rb", "lib/logworm_utils/tail.rb", "spec/logger_spec.rb", "spec/rack_spec.rb", "spec/rails_spec.rb", "spec/spec.opts", "spec/spec_helper.rb", "logworm_client.gemspec"]
   s.homepage = %q{http://www.logworm.com}
   s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Logworm_client", "--main", "README"]
   s.require_paths = ["lib"]

data/spec/logger_spec.rb

@@ -0,0 +1,33 @@
+require 'rubygems'
+require 'webmock'
+require 'logworm'
+
+require File.dirname(__FILE__) + '/spec_helper'
+
+$: << File.dirname(__FILE__) + '/../lib'
+require 'logworm_client/logger.rb'
+
+describe Logworm::Logger, " flushing" do
+  before do
+    Logworm::Logger.use_db Logworm::DB.new("logworm://a:b@localhost/c/d/")
+  end
+  
+  it "should only record if it's been initialized" do
+    Logworm::Logger.use_db nil
+    Logworm::Logger.log(:tbl1, :a => 1)
+    Logworm::Logger.flush.should == [0,0]
+  end
+
+  it "should only record if there are entries" do
+    Logworm::Logger.flush.should == [0,0]
+  end
+  
+  it "should try to record it it's initialized and has entries, and then reset" do
+    stub_request(:post, "localhost/log").to_return(:body => {:count => 10, :inserted_at => Time.now}.to_json)
+    Logworm::Logger.log(:tbl1, :a => 1)
+    Logworm::Logger.flush[0].should == 1
+    Logworm::Logger.flush.should == [0,0]
+  end
+  
+end
+

data/spec/rack_spec.rb

@@ -0,0 +1,16 @@
+require 'rubygems'
+require 'webmock'
+require 'logworm'
+
+require File.dirname(__FILE__) + '/spec_helper'
+
+$: << File.dirname(__FILE__) + '/../lib'
+require 'logworm_client.rb'
+
+describe Logworm::Rack, " init" do
+  it "should initialize the logger with a default db" do
+    Logworm::Logger.should_receive(:use_default_db)
+    Logworm::Rack.new("xx")
+  end
+end
+

data/spec/rails_spec.rb

@@ -0,0 +1,23 @@
+require 'rubygems'
+require 'webmock'
+require 'logworm'
+
+require File.dirname(__FILE__) + '/spec_helper'
+
+$: << File.dirname(__FILE__) + '/../lib'
+
+describe Logworm, " init" do
+  it "should initialize the logger with a default db" do
+    require 'logworm_client/logger.rb'                  # ==> Get Logworm::Logger defined
+    Logworm::Logger.should_receive(:use_default_db)
+
+    require 'action_controller'                         # ==> Get ActionController defined 
+    load 'logworm_client/rails.rb'                      # ==> does class_eval of ActionController (use load to guarantee require)
+  end
+end
+
+describe Logworm, " logging" do
+  it "should honor the filter_parameter_logging directive" do
+  end
+  
+end

data/spec/spec.opts

@@ -0,0 +1,4 @@
+--colour
+--format specdoc
+--loadby mtime
+--reverse

data/spec/spec_helper.rb

@@ -0,0 +1,8 @@
+require 'rubygems'
+require 'spec'
+require 'spec/autorun'
+require 'spec/interop/test'
+require 'webmock/rspec'
+
+include WebMock
+

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
-  - 6
-  - 2
-  version: 0.6.2
+  - 7
+  - 0
+  version: 0.7.0
 platform: ruby
 authors: 
 - Pomelo, LLC
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-04-23 00:00:00 -04:00
+date: 2010-06-23 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -102,6 +102,11 @@ files:
 - lib/logworm_utils.rb
 - lib/logworm_utils/compute.rb
 - lib/logworm_utils/tail.rb
+- spec/logger_spec.rb
+- spec/rack_spec.rb
+- spec/rails_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb
 - logworm_client.gemspec
 has_rdoc: true
 homepage: http://www.logworm.com