logworm_amqp 0.9.6 → 0.9.7

data/CHANGELOG

@@ -1,3 +1,11 @@
+v0.9.7
+  - lw_query now accepts a TTL parameter -- 300 seconds by default, can be lowered up to 20 seconds min
+  - lw-compute now has a standard TTL of 60 seconds
+  - no longer preserves entries from one request to the other --made more sense when there were timeouts, but with AMQP we don't 
+    expect to see so many of them. Fixes bugs with log_with_request calls as well.
+  - Adds a new optional field in the web_log table, :apache_log, a string with information about the HTTP request in 
+    the regular Apache Log format (requested by Dave Brwon @ Top Prospect)
+  
 v0.9.6
 	- Preserves the order of desired fields in a query. 
 	- Handled an exception which was being thrown when the connection to the server could not be established.

data/Rakefile

@@ -1,5 +1,5 @@
 require 'echoe'
-Echoe.new('logworm_amqp', '0.9.6') do |p|
+Echoe.new('logworm_amqp', '0.9.7') do |p|
   p.description = "logworm - logging service"
   p.url = "http://www.logworm.com"
   p.author = "Pomelo, LLC"

data/lib/base/db.rb

@@ -87,8 +87,10 @@ module Logworm
       db_call(:get, "#{host_with_protocol}/") || []
     end
   
-    def query(table, cond)
-      db_call(:post, "#{host_with_protocol}/queries", {:table => table, :query => cond})
+    def query(table, cond, ttl = nil)
+      ttl = nil if ttl and !ttl.is_a? Fixnum
+      ttl = 20  if ttl and ttl.to_i > 0 and ttl.to_i < 20
+      db_call(:post, "#{host_with_protocol}/queries", {:table => table, :query => cond, :ttl => ttl})
     end
     
     

data/lib/client/logger.rb

@@ -34,6 +34,8 @@ module Logworm
     ###
     def self.start_cycle
       $request_id = "#{Thread.current.object_id}-#{(Time.now.utc.to_f * 1000).to_i}"
+      $lr_queue = []
+      $attaches = {}
     end
 
     ###
@@ -87,7 +89,6 @@ module Logworm
       # Return if no server
       unless $lw_server
         $stderr.puts "\t logworm not configured. #{to_send} entries dropped."
-        $lr_queue = [] 
         return [0,0]
       end
       
@@ -99,12 +100,23 @@ module Logworm
       
       startTime = Time.now
       $lw_server.batch_log($lr_queue.to_json)
-      $lr_queue = [] 
       endTime = Time.now
       
       [to_send, (endTime - startTime)]
     end
     
+    ###
+    # Utility function
+    ###
+    def self.apache_log(ip, method, path, env, status, response_headers)
+      if response_headers['Content-Length']
+        response_length = response_headers['Content-Length'].to_s == '0' ? '-' : response_headers['Content-Length']
+      else
+        response_length = "-"
+      end
+      "#{ip} - #{env["REMOTE_USER"] || "-"} [#{Time.now.strftime("%d/%b/%Y:%H:%M:%S %z")}] \"#{method} #{path} #{env['HTTP_VERSION']}\" #{status} #{response_length} \"#{env['HTTP_REFERER']}\" \"#{env['HTTP_USER_AGENT']}\""
+    end
+    
     private
     # Turn keys into symbols, delete empty ones, rename conflicting ones
     def self.cleanup(values)

data/lib/client/rack.rb

@@ -8,6 +8,7 @@ module Logworm
       
       @log_requests = (options[:donot_log_requests].nil? or options[:donot_log_requests] != true)
       @log_headers  = (options[:log_headers] and options[:log_headers] == true)
+      @log_apache   = (options[:log_apache] and options[:log_apache] == true)
       @log_envs     = options[:log_environments] || ["production"]
       @log_envs << "development" if (options[:log_in_development] and options[:log_in_development] == true) # backwards compatibility
       Logger.use_default_db
@@ -44,8 +45,9 @@ module Logworm
                 :response_status => status, 
                 :profiling       => appTime,
                 :queue_size      => queue_size}
-      entry[:request_headers] = http_headers if @log_headers
+      entry[:request_headers]  = http_headers if @log_headers
       entry[:response_headers] = response_headers if @log_headers
+      entry[:apache_log]       = Logger.apache_log(ip, method, path, env, status, response_headers) if @log_apache
       Logger.log(:web_log, entry) if @log_requests
 
       begin 
@@ -54,7 +56,7 @@ module Logworm
         } 
       rescue Exception => e 
         # Ignore --nothing we can do. The list of logs may (and most likely will) be preserved for the next request
-        env['rack.errors'].puts("logworm call failed: #{e}")
+        $stderr.puts("logworm call failed: #{e}")
       end
     end
 

data/lib/client/rails.rb

@@ -12,6 +12,7 @@ if defined?(ActionController) and Rails::VERSION::STRING and Rails::VERSION::STR
     ## Basic settings: log requests, without headers. Use default db, and timeout after 1 second
     @@log_requests = true
     @@log_headers  = false
+    @@log_apache   = false
     @@log_envs     = ["production"]
     @@timeout      = (RAILS_ENV == 'production' ? 1 : 5) 
     Logworm::Logger.use_default_db
@@ -39,6 +40,15 @@ if defined?(ActionController) and Rails::VERSION::STRING and Rails::VERSION::STR
       @@log_envs << "development"
     end
     
+    ###
+    # Log a field with Apache common log
+    # Use from ApplicationController
+    ###
+    def self.log_apache
+      @@log_apache = true
+    end
+    
+    
     def self.log_environments(*envs)
       @@log_envs = envs
     end
@@ -80,8 +90,9 @@ if defined?(ActionController) and Rails::VERSION::STRING and Rails::VERSION::STR
                 :response_status => status, 
                 :profiling       => appTime,
                 :queue_size      => queue_size}
-      entry[:request_headers] = http_headers if @@log_headers
+      entry[:request_headers]  = http_headers if @@log_headers
       entry[:response_headers] = response.headers if @@log_headers
+      entry[:apache_log]       = Logworm::Logger.apache_log(ip, method, path, request.env, status, response.headers) if @@log_apache
       Logworm::Logger.log(:web_log, entry) if @@log_requests
 
       begin 

data/lib/cmd/compute.rb

@@ -21,9 +21,9 @@ class LogwormCompute
     def run
       # Create a resource for the query
       begin
-        query_data = @db.query(@table, @query.to_json)
+        query_data = @db.query(@table, @query.to_json, 60) # 60 second ttl by default
         url = query_data["results_uri"]
-        rows = @db.results(url + "?nocache=1")["results"]
+        rows = @db.results(url)["results"]
       rescue Logworm::DatabaseException, Logworm::ForbiddenAccessException  => e
         $stderr.puts "Error: #{e}"
         exit(-1)

data/lib/logworm_amqp.rb

@@ -46,10 +46,10 @@ alias :log_with_request :lw_with_log
 # raises Logworm::DatabaseException, Logworm::ForbiddenAccessException, Logworm::InvalidQueryException
 # or just a regular Exception if it cannot find the URL to the logging database
 ###
-def lw_query(logname, query = {}) 
+def lw_query(logname, query = {}, ttl = nil) 
   db = Logworm::DB.from_config_or_die                                   # Establish connection to DB
   query = Logworm::QueryBuilder.new(query).to_json if query.is_a? Hash  # Turn query into proper JSON string
-  query_data = db.query(logname, query)                                 # POST to create query
+  query_data = db.query(logname, query, ttl)                            # POST to create query
   db.results(query_data["results_uri"])                                 # GET from query's results uri
 end
 

data/logworm_amqp.gemspec

@@ -2,11 +2,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{logworm_amqp}
-  s.version = "0.9.6"
+  s.version = "0.9.7"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Pomelo, LLC"]
-  s.date = %q{2010-08-24}
+  s.date = %q{2010-09-03}
   s.description = %q{logworm - logging service}
   s.email = %q{schapira@pomelollc.com}
   s.executables = ["lw-compute", "lw-tail"]

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: logworm_amqp
 version: !ruby/object:Gem::Version 
-  hash: 55
+  hash: 53
   prerelease: false
   segments: 
   - 0
   - 9
-  - 6
-  version: 0.9.6
+  - 7
+  version: 0.9.7
 platform: ruby
 authors: 
 - Pomelo, LLC
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-08-24 00:00:00 -04:00
+date: 2010-09-03 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency