logworm_amqp 0.8.9 → 0.9.0

data/CHANGELOG

@@ -1,3 +1,8 @@
+v0.9.0
+    Combines base and client into a single gem
+    
+base
+----
 v0.8.9
     Cleaned up pushed stats output.
 
@@ -64,4 +69,101 @@ v0.2.0
     removed app/ libraries. Added tail as a utility, lw-tail as a command
 
 v0.1.0 
-    initial version.
+    initial version.
+    
+    
+client
+------
+v0.8.8
+    Cleaned up pushed stats output.
+
+v0.8.7
+	Started logging amqp usage statistics to our servers (randomly) for better QA.
+
+v.0.8.6
+	No longer prints out amqp information when logging and should also self check for amqp url updates incase it was changed.
+
+v0.8.5
+	Small fix
+
+v0.8.0
+	Dropped http for amqp for high scalability and reduce latency
+
+v0.7.2
+    Renamed Rails and Rack configuration commands:
+        lw_disable_request_logging ==> donot_log_requests
+        lw_log_request_headers     ==> log_headers
+        lw_enable_dev_logging      ==> log_in_development
+        
+    Renamed :headers field in web_log to :request_headers
+        
+v0.7.1
+    lw-tail and lw-compute now receive an optional app parameter, for the cases where you want to call command-line tools from a directory other than the app's directory... or when you have more than one Heroku remote/app from the same directory
+
+v0.7.0
+    Eliminated long logs. Instead, there's now a parameter (log_request_headers in Rack, lw_log_request_headers in Rails) to add
+       request and response headers to the web_log. 
+    Cleaned up fields in the web_log table
+
+    Added option to turn off automatic logging of requests (disable_request_logging in Rack, lw_disable_request_logging in Rails)
+    Added option to turn on logging even if in development mode (enable_dev_logging in Rack, lw_enable_dev_logging in Rails)
+
+    Honor filter_parameter_logging switch when used with Rails
+
+    Cleaned up log and flush
+    Cleaned up display of elapsed time for flush
+    Cleaned up call from Rack and Rails, and enforce timeout (1 sec by default)
+    Added unit tests
+    
+v0.6.2
+    Added list_tables command
+    
+v0.6.1
+    Added a bit of documentation
+
+v0.6.0 
+    Add support for querying API: lw_query
+    
+v0.5.6
+    log HEROKU_QUEUE as an integer value
+    
+v0.5.5
+    Added default logging of HEROKU_QUEUE   
+
+v0.5.4
+    Removed obsolete lw-heroku application
+    
+v0.5.3
+    Show nicer error message with tail and compute if the app is not properly configured.
+
+v0.5.2
+    Fixes minor issues with in Rails logging
+
+v0.5.1
+    Fixes minor issues with logging statements sent to console
+    
+v0.5.0
+    Supports new base logworm gem, which can work as a Heroku addon
+    Configuration parameters now stored as URL
+    No error if logworm cannot be properly configured -- lw_log simply ignored
+    No logging in development mode
+    
+v0.4.1
+    Show in console time spent communicating with the server.
+    
+v0.4.0 Added support for running on Rails!
+    To use, add config.gem ‘logworm_client’ to environment.rb
+    Then you can configure in ApplicationController, via logs_requests [:short | :long]+
+    Works from views, models, or controllers.
+    
+v0.3.2 added lw_log method --fixed small issue
+
+v0.3.1 added lw_log method
+
+v0.3.0 ensure gem and dependencies live in gemcutter
+
+v0.2.0 log short requests by default in the rack
+
+v0.1.1 minor changes
+
+v0.1.0 initial version.

data/Manifest

@@ -1,14 +1,15 @@
 CHANGELOG
 Manifest
-README.md
 Rakefile
+bin/lw-compute
+bin/lw-tail
 lib/base/config.rb
 lib/base/db.rb
 lib/base/query_builder.rb
+lib/client/logger.rb
+lib/client/rack.rb
+lib/client/rails.rb
+lib/cmd/compute.rb
+lib/cmd/tail.rb
 lib/logworm_amqp.rb
-spec/base_spec.rb
-spec/builder_spec.rb
-spec/config_spec.rb
-spec/spec.opts
-spec/spec_helper.rb
-tests/builder_test.rb
+lib/logworm_utils.rb

data/Rakefile

@@ -1,10 +1,10 @@
 require 'echoe'
-Echoe.new('logworm_amqp', '0.8.9') do |p|
-  p.description = "logworm logging tool"
+Echoe.new('logworm_amqp', '0.9.0') do |p|
+  p.description = "logworm - logging service"
   p.url = "http://www.logworm.com"
   p.author = "Pomelo, LLC"
   p.email  = "schapira@pomelollc.com"
   p.ignore_pattern = ["tmp/*", "script/*"]
-  p.development_dependencies = ["memcache-client", "hpricot", "oauth", "heroku", "minion >=0.1.15"]
-  p.runtime_dependencies = ["memcache-client", "hpricot", "oauth", "heroku", "minion >=0.1.15"]
+  p.development_dependencies = ["json >=1.4.3", "ruby-hmac", "hpricot", "oauth", "heroku"]
+  p.runtime_dependencies = ["json >=1.4.3", "ruby-hmac", "memcache-client", "hpricot", "oauth", "heroku"]
 end

data/bin/lw-compute

@@ -0,0 +1,66 @@
+#!/usr/local/bin/ruby
+
+require 'rubygems'
+require 'logworm_utils'
+require 'optparse'
+
+options = {
+  :aggregate_group => [],
+  :conditions => [],
+  :start => nil,
+  :end => nil,
+  :debug => false
+}
+
+option_parser = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$0} [options] <log table> <function> <field>"
+
+  opts.on("--app app", String, "Specify the Heroku app") do |app|
+    options[:app] = app.strip 
+  end
+
+  opts.on("-g group", String, "Specify an aggregation group (e.g, hour(_ts_utc), or response.code)") do |k|
+    options[:aggregate_group] = k
+  end
+
+  opts.on("-c condition", String, "Specify a condition to match. May be used multiple times") do |c|
+    options[:conditions] << c.strip 
+  end
+
+  opts.on("-s starttime", String, "Specify the start time for the query") do |c|
+    options[:start] = c.strip 
+  end
+
+  opts.on("-e endtime", String, "Specify the end time for the query") do |c|
+    options[:end] = c.strip 
+  end
+
+  opts.on("-v", "Show debug information") do 
+    options[:debug] = true
+  end
+  
+  opts.on( '-h', '--help', 'Display this screen' ) do
+    puts option_parser.help
+    exit(1)
+  end
+  
+end
+
+# Parse, and get the required <table>
+option_parser.parse!
+if ARGV.size == 3
+  table = ARGV[0].strip
+  function = ARGV[1].strip
+  field = ARGV[2].strip
+elsif ARGV.size == 2
+  table = ARGV[0].strip
+  function = ARGV[1].strip
+else
+  puts option_parser.help
+  exit(1)
+end
+
+# and run
+LogwormCompute.new(table, function, field, options).run
+
+

data/bin/lw-tail

@@ -0,0 +1,84 @@
+#!/usr/local/bin/ruby
+
+require 'rubygems'
+require 'logworm_utils'
+require 'optparse'
+
+def help(option_parser, options)
+  puts option_parser.help
+  puts
+  LogwormTail.list(options)
+end
+
+options = {
+  :limit => 200,
+  :loop => false,
+  :frequency => 10,
+  :fields => [],
+  :conditions => [],
+  :start => nil,
+  :end => nil,
+  :debug => false,
+  :fkat => false
+}
+
+option_parser = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$0} [options] <log table>"
+
+  opts.on("--app app", String, "Specify the Heroku app") do |app|
+    options[:app] = app.strip 
+  end
+
+  opts.on("-f [secs]", Integer, 
+          "Continuously check for more data, every [secs] seconds.", "Default: #{options[:frequency]}") do |s| 
+    options[:loop] = true
+    options[:frequency] = s if s
+  end
+
+  opts.on("-r limit", Integer, "Specify how many log entries to fetch.", "Default: #{options[:limit]}") do |n|
+    options[:limit] = n
+  end
+
+  opts.on("-k fields", String, "Specify a comma-separated list of fields to retrieve.") do |k|
+    options[:fields] = k.split(",").map {|f| f.strip}
+  end
+
+  opts.on("-c condition", String, "Specify a condition to match. May be used multiple times") do |c|
+    options[:conditions] << c.strip 
+  end
+
+  opts.on("-s starttime", String, "Specify the start time for the query") do |c|
+    options[:start] = c.strip 
+  end
+
+  opts.on("-e endtime", String, "Specify the end time for the query") do |c|
+    options[:end] = c.strip 
+  end
+
+  opts.on("--flat", "Do not expand log entries when printing them") do 
+    options[:flat] = true
+  end
+
+  opts.on("-v", "Show debug information") do 
+    options[:debug] = true
+  end
+  
+  opts.on( '-h', '--help', 'Display this screen' ) do
+    options[:help] = true
+  end
+  
+end
+
+# Parse, and get the required <table>
+option_parser.parse!
+table = ARGV.pop
+if !table or options[:help]
+  help option_parser, options
+  exit(1)
+end
+table.strip!
+
+# and run
+t = LogwormTail.new(table, options).run
+
+

data/lib/base/db.rb

@@ -31,7 +31,6 @@ module Logworm
       raise ForbiddenAccessException.new("Incorrect URL Format #{url}") unless match and match.size == 6      
       @consumer_key, @consumer_secret, @host, @token, @token_secret = match[1..5]
       @connection = OAuth::AccessToken.new(OAuth::Consumer.new(@consumer_key, @consumer_secret), @token, @token_secret)
-      reset_stats
     end
   
     def self.with_tokens(token, token_secret)
@@ -115,11 +114,12 @@ module Logworm
           resp = db_call(:get, "#{host_with_protocol}/amqp_url")
           @amqp_url = resp["url"]
           @stats_freq = resp["stats_freq"]
-          $stderr.puts "logworm server acquired: #{@amqp_url}"
+          $stderr.puts "logworm server acquired: #{@amqp_url.gsub(/[^@]+@/, "amqp://")}"
           Minion.amqp_url = @amqp_url
+          reset_stats
         rescue 
           @last_attempt = Time.now
-          $stderr.puts "logworm cannot connect to server; please wait at least #{RETRY_FREQUENCY} seconds. log entries will be lost"
+          $stderr.puts "logworm cannot connect to server; waiting #{RETRY_FREQUENCY} seconds before retry. log entries will be lost"
         end
       end
       !(@amqp_url.nil?)
@@ -149,9 +149,11 @@ module Logworm
     end
 
     def push_stats()
-       to_amqp("lw.stats", {:avg => (@total_time / @tock), :max => @amqp_max, :min => @amqp_min, 
-                            :total => @total_time, :count => @tock, :sampling_length => (Time.now - @sampling_start) })
-       $stderr.puts "logworm statistics: #{@tock} messages sent in #{Time.now - @sampling_start}. Times: #{(@total_time / @tock) * 1000}/#{@amqp_min * 1000}/#{@amqp_max * 1000} (avg/min/max seconds)"
+      avg = sprintf('%.6f', (@total_time / @tock)).to_f
+      len = sprintf('%.2f', (Time.now - @sampling_start)).to_f
+      to_amqp("lw.stats", {:avg => avg , :max => @amqp_max, :min => @amqp_min, 
+                           :total => @total_time, :count => @tock, :sampling_length => len})
+      $stderr.puts "logworm statistics: #{@tock} messages sent in last #{len} secs. Times: #{avg * 1000}/#{@amqp_min * 1000}/#{@amqp_max * 1000} (avg/min/max msecs)"
        reset_stats
     end
     

data/lib/client/logger.rb

@@ -0,0 +1,93 @@
+require 'json'
+  
+module Logworm
+  class LogwormException < Exception ; end
+
+  class Logger
+    $lr_queue = []
+    
+    ###
+    # Use connection to the backend servers specified in environment variable or config file
+    ###
+    def self.use_default_db
+       $lw_server = DB.from_config
+    end
+    
+    ###
+    # Use a connection to a manually specified server
+    ###
+    def self.use_db(db)
+      $lw_server = db
+    end
+    
+    ###
+    # Returns a reference to the current backend server
+    ###
+    def self.db
+      $lw_server
+    end
+    
+    
+    ###
+    # Starts a new cycle: sets a request_id variable, so that all entries (until flush) share that id
+    ###
+    def self.start_cycle
+      $request_id = "#{Thread.current.object_id}-#{(Time.now.utc.to_f * 1000).to_i}"
+    end
+
+    ###
+    # Record an entry. Not sent to the servers until 'flush' is called
+    #
+    # Warning: may raise Exception if there's a problem. It's up to the called to rescue from it in order to continue the processing
+    # of a web request, for example.
+    ###
+    def self.log(table, values = {})
+      return unless table and (table.is_a? String or table.is_a? Symbol)
+      return unless values.is_a? Hash
+
+      # Turn keys into symbols, delete empty ones, rename conflicting ones
+      kvalues = values.delete_if {|k,v| k.to_s == ""}.map {|k,v| [k.to_sym, v]}
+      kvalues = Hash[*kvalues.flatten]
+      [:_ts, :_ts_utc, :_request_id].each do |k|
+        kvalues["orig_#{k}".to_sym] = kvalues[k] if kvalues.has_key? k
+      end
+      
+      # Add information
+      ts = Time.now.utc
+      kvalues[:_ts_utc]     = (ts.to_f * 1000).to_i
+      kvalues[:_ts]         = ts.strftime("%Y-%m-%dT%H:%M:%SZ")
+      kvalues[:_request_id] = $request_id if $request_id
+      
+      # Enqueue
+      $lr_queue << [table, kvalues]
+    end
+
+    ###
+    # Sends the entries to the server, if configured
+    # Returns the number of entries send, and the time it takes
+    #
+    # Warning: may raise Exception if there's a problem. It's up to the called to rescue from it in order to continue the processing
+    # of a web request, for example.
+    ###
+    def self.flush
+      to_send = $lr_queue.size
+
+      # Return if no entries
+      return [0,0] if to_send == 0 
+
+      # Return if no server
+      unless $lw_server
+        $stderr.puts "\t logworm not configured. #{to_send} entries dropped."
+        $lr_queue = [] 
+        return [0,0]
+      end
+      
+      startTime = Time.now
+      $lw_server.batch_log($lr_queue.to_json)
+      $lr_queue = [] 
+      endTime = Time.now
+      
+      [to_send, (endTime - startTime)]
+    end
+  end
+end

data/lib/client/rack.rb

@@ -0,0 +1,61 @@
+require 'rack/request'
+
+module Logworm
+  class Rack
+
+    def initialize(app, options = {})
+      @app = app
+      
+      @log_requests = (options[:donot_log_requests].nil? or options[:donot_log_requests] != true)
+      @log_headers  = (options[:log_headers] and options[:log_headers] == true)
+      @dev_logging  = (options[:log_in_development] and options[:log_in_development] == true)
+      Logger.use_default_db
+      @timeout = 1
+    end
+
+    def call(env)
+      return @app.call(env) unless (ENV['RACK_ENV'] == 'production' or (ENV['RACK_ENV'] == 'development' and @dev_logging))
+
+      Logger.start_cycle
+      begin
+        startTime = Time.now 
+        status, response_headers, body = @app.call(env) 
+        appTime = (Time.now - startTime) 
+      ensure
+        log_request(env, status, response_headers, appTime)
+        return [status, response_headers, body] 
+      end
+    end
+    
+    private 
+    def log_request(env, status, response_headers, appTime)
+      method       = env['REQUEST_METHOD']
+      path         = (env['REQUEST_PATH'].nil? or env['REQUEST_PATH'] == "")  ? "/" : env['REQUEST_PATH']
+      ip           = env['REMOTE_ADDR']
+      http_headers = env.reject {|k,v| !(k.to_s =~ /^HTTP/) }
+      queue_size   = env['HTTP_X_HEROKU_QUEUE_DEPTH'].nil? ? -1 : env['HTTP_X_HEROKU_QUEUE_DEPTH'].to_i
+
+      entry = { :summary         => "#{method} #{path} - #{status} #{appTime}", 
+                :request_method  => method,
+                :request_path    => path, 
+                :request_ip      => ip,
+                :input           => ::Rack::Request.new(env).params,
+                :response_status => status, 
+                :profiling       => appTime,
+                :queue_size      => queue_size}
+      entry[:request_headers] = http_headers if @log_headers
+      entry[:response_headers] = response_headers if @log_headers
+      Logger.log(:web_log, entry) if @log_requests
+
+      begin 
+        Timeout::timeout(@timeout) { 
+          sent, elapsed = Logger.flush
+        } 
+      rescue Exception => e 
+        # Ignore --nothing we can do. The list of logs may (and most likely will) be preserved for the next request
+        env['rack.errors'].puts("logworm call failed: #{e}")
+      end
+    end
+
+  end
+end