logworm_amqp 0.8.9 → 0.9.0

data/lib/client/rails.rb

@@ -0,0 +1,105 @@
+###
+# Extemds ActionController::Base and aliases the process method, to wrap it with the standard logworm request cycle
+#
+# By default it will automatically log web requests (in a short format) into web_log
+# Can also log headers, if specified
+###
+if defined?(ActionController)
+
+  require 'benchmark'
+
+  ActionController::Base.class_eval do
+    ## Basic settings: log requests, without headers. Use default db, and timeout after 1 second
+    @@log_requests = true
+    @@log_headers  = false
+    @@dev_logging  = false
+    @@timeout = 1
+    Logworm::Logger.use_default_db
+    
+    ###
+    # Disable automatic logging of requests
+    # Use from ApplicationController
+    ###
+    def self.donot_log_requests
+      @@log_requests = false
+    end
+    
+    ###
+    # Log headers with requests
+    # Use from ApplicationController
+    ###
+    def self.log_headers
+      @@log_headers = true
+    end
+    
+    ###
+    # Turn on logging in development mode
+    ###
+    def self.log_in_development
+      @@dev_logging = true
+    end
+    
+    ###
+    # Replaces (and embeds) the default Rails processing of a request. 
+    # Call the original method, logs the request unless disabled, and flushes the logworm list
+    ###
+    def process_with_logworm_log(request, response, method = :perform_action, *arguments)
+      unless (RAILS_ENV == 'production' or (RAILS_ENV == 'development' and @@dev_logging))
+        return process_without_logworm_log(request, response, method, *arguments) 
+      end
+      
+      Logworm::Logger.start_cycle
+      begin
+        startTime = Time.now 
+        response  = process_without_logworm_log(request, response, method, *arguments)
+        appTime   = (Time.now - startTime) 
+      ensure
+        log_request(request, response, appTime)
+        return response
+      end
+    end
+    alias_method_chain :process, :logworm_log
+    
+
+    private
+    def log_request(request, response, appTime)
+      method       = request.env['REQUEST_METHOD']
+      path         = request.env['REQUEST_PATH'].blank? ? "/" : request.env['REQUEST_PATH']
+      ip           = request.env['REMOTE_ADDR']
+      http_headers = request.headers.reject {|k,v| !(k.to_s =~ /^HTTP/) }
+      status       = response.status[0..2]
+      queue_size   = request.env['HTTP_X_HEROKU_QUEUE_DEPTH'].blank? ? -1 : request.env['HTTP_X_HEROKU_QUEUE_DEPTH'].to_i
+
+      entry = { :summary         => "#{method} #{path} - #{status} #{appTime}", 
+                :request_method  => method,
+                :request_path    => path, 
+                :request_ip      => ip,
+                :input           => cleaned_input(request),
+                :response_status => status, 
+                :profiling       => appTime,
+                :queue_size      => queue_size}
+      entry[:request_headers] = http_headers if @@log_headers
+      entry[:response_headers] = response.headers if @@log_headers
+      Logworm::Logger.log(:web_log, entry) if @@log_requests
+
+      begin 
+        Timeout::timeout(@@timeout) { 
+          sent, elapsed = Logworm::Logger.flush
+        } 
+      rescue Exception => e 
+        # Ignore --nothing we can do. The list of logs may (and most likely will) be preserved for the next request
+        Rails.logger.error("logworm call failed: #{e}")
+      end
+    end
+    
+    def cleaned_input(request)
+      pars = request.parameters.clone
+      pars.delete("controller")
+      pars.delete("action")
+      respond_to?(:filter_parameters) ? filter_parameters(pars) : pars
+    end
+    
+  end
+
+end
+

data/lib/cmd/compute.rb

@@ -0,0 +1,66 @@
+class LogwormCompute
+    def initialize(table, function, field, options)
+      @table    = table
+      @function = function
+      @field    = field
+      @options  = options
+
+      @valuefield = @field ? "#{@function}(#{@field})" : @function
+
+      begin
+        @db    = Logworm::DB.from_config_or_die(@options[:app])
+        spec   = {:aggregate_function => @function, :aggregate_argument => @field}.merge(@options)
+        @query = Logworm::QueryBuilder.new(spec)
+      rescue Exception => e
+        $stderr.puts "There was an error: #{e}"
+        exit(-1)
+      end
+      
+    end
+
+    def run
+      # Create a resource for the query
+      begin
+        query_data = @db.query(@table, @query.to_json)
+        url = query_data["results_uri"]
+        rows = @db.results(url + "?nocache=1")["results"]
+      rescue Logworm::DatabaseException, Logworm::ForbiddenAccessException  => e
+        $stderr.puts "Error: #{e}"
+        exit(-1)
+      rescue Logworm::InvalidQueryException => e
+        $stderr.puts "#{e}, #{@query.to_json}"  
+        exit(-1)
+      rescue Exception  => e
+        $stderr.puts "Error: #{e}"
+        exit(-1)
+      end
+
+      if @options[:debug]
+        puts "logworm query: #{@query.to_json}"
+        puts "logworm query url: #{query_data["self_uri"]}"
+        puts "logworm results url: #{url}"
+        puts
+      end
+        
+      if @query.groups.length > 0
+        results = {}
+        rows.each do |r|
+          grp = []
+          @query.groups.each do |g|
+            grp << "#{g}:#{r[g]}"
+          end
+          key = "[#{grp.join(', ')}]"
+          value = r[@function]
+          results[key] = {:value => value, :keys => r.dup}
+          results[key][:keys].delete(@function)
+        end
+        results.keys.sort.each do |k|
+          puts "#{k} \t ==> #{@valuefield} = #{results[k][:value]}"
+        end
+      else
+        puts "#{@valuefield} = #{rows.first[@function]}"
+      end
+      
+    end
+
+end

data/lib/cmd/tail.rb

@@ -0,0 +1,106 @@
+class LogwormTail
+    def initialize(table, options)
+      @table   = table
+      @options = options
+
+      begin
+        @db    = Logworm::DB.from_config_or_die(@options[:app])
+        @query = Logworm::QueryBuilder.new(@options.merge(:force_ts => true))
+      rescue Exception => e
+        $stderr.puts "Error: #{e}"
+        exit(-1)
+      end
+    end
+
+    def self.list(options = {})
+      begin
+        @db    = Logworm::DB.from_config_or_die(options[:app])
+        @tables = @db.tables
+        if @tables and @tables.size > 0
+          puts "The following are the tables that you've created thus far:"
+          @tables.sort {|x,y| x["tablename"] <=> y["tablename"]}.each do |t| 
+            puts "\t - #{t["tablename"]}, #{t["rows"]} rows, last updated on #{date_time(t["last_write"])}"
+          end
+        else
+          puts "You haven't recorded any data yet."
+        end
+      rescue Exception => e
+        $stderr.puts "Error: #{e}"
+        exit(-1)
+      end
+    end
+    
+    def run
+      # Create a resource for the query
+      begin
+        query_data = @db.query(@table, @query.to_json)
+        url = query_data["results_uri"]
+      rescue Logworm::DatabaseException, Logworm::ForbiddenAccessException  => e
+        $stderr.puts "Error: #{e}"
+        exit(-1)
+      rescue Logworm::InvalidQueryException => e
+        $stderr.puts "#{e}"  
+        exit(-1)
+      rescue Exception  => e
+        $stderr.puts "Error: #{e}"
+        exit(-1)
+      end
+
+      if @options[:debug]
+        puts "logworm query: #{@query.to_json}"
+        puts "logworm query url: #{query_data["self_uri"]}"
+        puts "logworm results url: #{url}"
+        puts "refresh frequency: #{@options[:frequency]}" if @options[:loop]
+        puts
+      end
+  
+      while true do
+        begin
+          last_printed = print_rows(@db.results(url + "?nocache=1")["results"], last_printed || nil)
+        rescue Logworm::DatabaseException, Logworm::ForbiddenAccessException  => e
+          $stderr.puts "Error: #{e}"
+          exit(-1)
+        rescue Logworm::InvalidQueryException => e
+          $stderr.puts "#{e}"  
+          exit(-1)
+        rescue Exception  => e
+          $stderr.puts "Error: #{e}"
+          exit(-1)
+        end
+        exit(0) unless @options[:loop]
+        sleep @options[:frequency]
+      end
+    end
+
+  private
+
+    def self.date_time(val)
+      val.gsub(/T/, ' @ ').gsub(/Z/, ' GMT')
+    end
+    
+    def print_rows(rows, last)
+      last = "" if last.nil?
+      rows.reverse.each do |r|
+        next unless r["_ts"]
+        if r["_ts"] > last
+          last = r["_ts"]
+          r.delete("_id") unless @options[:fields].include?("_id")
+          r.delete("_ts") unless @options[:fields].include?("_ts")
+          puts "#{LogwormTail.date_time(last)} ==> "
+          print_row(r)
+        end
+      end
+      last
+    end
+
+    def print_row(r)
+      if @options[:flat]
+        puts "\t" + r.keys.sort.map {|k| "#{k}: #{r[k].inspect}"}.join(', ')
+      else
+        r.keys.sort.each do |k|
+          puts "\t#{k}: #{r[k].inspect}"
+        end
+        puts
+      end
+    end
+end

data/lib/logworm_amqp.rb

@@ -1,3 +1,64 @@
 require File.dirname(__FILE__) + '/base/db'
 require File.dirname(__FILE__) + '/base/config'
 require File.dirname(__FILE__) + '/base/query_builder'
+
+require File.dirname(__FILE__) + '/client/logger'
+require File.dirname(__FILE__) + '/client/rack'
+require File.dirname(__FILE__) + '/client/rails'
+
+def lw_log (logname, values)
+  Logworm::Logger.log(logname, values)
+end
+
+###
+# Perform a query against the logworm server
+# 
+# Requires a log table, and a query
+# The query can be provided as a JSON string, following the syntax described in http://www.logworm.com/docs/query
+#  or as a Hash of options, with the following keys (all optional)
+#   :fields             => String with a comma-separated list of fields (quoted or not), or Array of Strings
+#   :aggregate_function => String
+#   :aggregate_argument => String
+#   :aggregate_group    => String with a comma-separated list of fields (quoted or not), or Array of Strings
+#   :conditions         => String with comma-separated conditions (in MongoDB syntax), or Array of Strings
+#   :start              => String or Integer (for year)
+#   :end                => String or Integer (for year)
+#   :limit              => String or Integer
+# 
+# See Logworm::QueryBuilder
+# 
+# Returns Hash with
+#  id              ==> id of the query
+#  query_url       ==> URL to GET information about the query
+#  results_url     ==> URL to GET the results for the query
+#  created         ==> First creation of the query
+#  updated         ==> most recent update of the query and/or its results
+#  expires         ==> until that datime, the query won't be rerun against the database
+#  execution_time  ==> time in ms to run the query
+#  results         ==> array of hashmaps. Each element corresponds to a log entry, with its fields
+#
+# raises Logworm::DatabaseException, Logworm::ForbiddenAccessException, Logworm::InvalidQueryException
+# or just a regular Exception if it cannot find the URL to the logging database
+###
+def lw_query(logname, query = {}) 
+  db = Logworm::DB.from_config_or_die                                   # Establish connection to DB
+  query = Logworm::QueryBuilder.new(query).to_json if query.is_a? Hash  # Turn query into proper JSON string
+  query_data = db.query(logname, query)                                 # POST to create query
+  db.results(query_data["results_uri"])                                 # GET from query's results uri
+end
+
+###
+# Returns an array with information about the logging tables in the database
+# Each element in the array has;
+#  :tablename   => The name of the logging table
+#  :url         => The URL for POSTing new log entries
+#  :last_write  => Datetime of last entry
+#  :rows        => Count of entries
+#
+# raises Logworm::DatabaseException, Logworm::ForbiddenAccessException, Logworm::InvalidQueryException
+# or just a regular Exception if it cannot find the URL to the logging database
+###
+def lw_list_logs
+  db = Logworm::DB.from_config_or_die                                   # Establish connection to DB
+  db.tables                                                             # Call tables command
+end

data/lib/logworm_utils.rb

@@ -0,0 +1,4 @@
+require 'logworm_amqp'
+
+require File.dirname(__FILE__) + '/cmd/tail'
+require File.dirname(__FILE__) + '/cmd/compute'

data/logworm_amqp.gemspec

@@ -2,59 +2,63 @@
 
 Gem::Specification.new do |s|
   s.name = %q{logworm_amqp}
-  s.version = "0.8.9"
+  s.version = "0.9.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Pomelo, LLC"]
-  s.date = %q{2010-08-08}
-  s.description = %q{logworm logging tool}
+  s.date = %q{2010-08-09}
+  s.description = %q{logworm - logging service}
   s.email = %q{schapira@pomelollc.com}
-  s.extra_rdoc_files = ["CHANGELOG", "README.md", "lib/base/config.rb", "lib/base/db.rb", "lib/base/query_builder.rb", "lib/logworm_amqp.rb"]
-  s.files = ["CHANGELOG", "Manifest", "README.md", "Rakefile", "lib/base/config.rb", "lib/base/db.rb", "lib/base/query_builder.rb", "lib/logworm_amqp.rb", "spec/base_spec.rb", "spec/builder_spec.rb", "spec/config_spec.rb", "spec/spec.opts", "spec/spec_helper.rb", "tests/builder_test.rb", "logworm_amqp.gemspec"]
+  s.executables = ["lw-compute", "lw-tail"]
+  s.extra_rdoc_files = ["CHANGELOG", "bin/lw-compute", "bin/lw-tail", "lib/base/config.rb", "lib/base/db.rb", "lib/base/query_builder.rb", "lib/client/logger.rb", "lib/client/rack.rb", "lib/client/rails.rb", "lib/cmd/compute.rb", "lib/cmd/tail.rb", "lib/logworm_amqp.rb", "lib/logworm_utils.rb"]
+  s.files = ["CHANGELOG", "Manifest", "Rakefile", "bin/lw-compute", "bin/lw-tail", "lib/base/config.rb", "lib/base/db.rb", "lib/base/query_builder.rb", "lib/client/logger.rb", "lib/client/rack.rb", "lib/client/rails.rb", "lib/cmd/compute.rb", "lib/cmd/tail.rb", "lib/logworm_amqp.rb", "lib/logworm_utils.rb", "logworm_amqp.gemspec"]
   s.homepage = %q{http://www.logworm.com}
-  s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Logworm_amqp", "--main", "README.md"]
+  s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Logworm_amqp"]
   s.require_paths = ["lib"]
   s.rubyforge_project = %q{logworm_amqp}
   s.rubygems_version = %q{1.3.7}
-  s.summary = %q{logworm logging tool}
+  s.summary = %q{logworm - logging service}
 
   if s.respond_to? :specification_version then
     current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<json>, [">= 1.4.3"])
+      s.add_runtime_dependency(%q<ruby-hmac>, [">= 0"])
       s.add_runtime_dependency(%q<memcache-client>, [">= 0"])
       s.add_runtime_dependency(%q<hpricot>, [">= 0"])
       s.add_runtime_dependency(%q<oauth>, [">= 0"])
       s.add_runtime_dependency(%q<heroku>, [">= 0"])
-      s.add_runtime_dependency(%q<minion>, [">= 0.1.15"])
-      s.add_development_dependency(%q<memcache-client>, [">= 0"])
+      s.add_development_dependency(%q<json>, [">= 1.4.3"])
+      s.add_development_dependency(%q<ruby-hmac>, [">= 0"])
       s.add_development_dependency(%q<hpricot>, [">= 0"])
       s.add_development_dependency(%q<oauth>, [">= 0"])
       s.add_development_dependency(%q<heroku>, [">= 0"])
-      s.add_development_dependency(%q<minion>, [">= 0.1.15"])
     else
+      s.add_dependency(%q<json>, [">= 1.4.3"])
+      s.add_dependency(%q<ruby-hmac>, [">= 0"])
       s.add_dependency(%q<memcache-client>, [">= 0"])
       s.add_dependency(%q<hpricot>, [">= 0"])
       s.add_dependency(%q<oauth>, [">= 0"])
       s.add_dependency(%q<heroku>, [">= 0"])
-      s.add_dependency(%q<minion>, [">= 0.1.15"])
-      s.add_dependency(%q<memcache-client>, [">= 0"])
+      s.add_dependency(%q<json>, [">= 1.4.3"])
+      s.add_dependency(%q<ruby-hmac>, [">= 0"])
       s.add_dependency(%q<hpricot>, [">= 0"])
       s.add_dependency(%q<oauth>, [">= 0"])
       s.add_dependency(%q<heroku>, [">= 0"])
-      s.add_dependency(%q<minion>, [">= 0.1.15"])
     end
   else
+    s.add_dependency(%q<json>, [">= 1.4.3"])
+    s.add_dependency(%q<ruby-hmac>, [">= 0"])
     s.add_dependency(%q<memcache-client>, [">= 0"])
     s.add_dependency(%q<hpricot>, [">= 0"])
     s.add_dependency(%q<oauth>, [">= 0"])
     s.add_dependency(%q<heroku>, [">= 0"])
-    s.add_dependency(%q<minion>, [">= 0.1.15"])
-    s.add_dependency(%q<memcache-client>, [">= 0"])
+    s.add_dependency(%q<json>, [">= 1.4.3"])
+    s.add_dependency(%q<ruby-hmac>, [">= 0"])
     s.add_dependency(%q<hpricot>, [">= 0"])
     s.add_dependency(%q<oauth>, [">= 0"])
     s.add_dependency(%q<heroku>, [">= 0"])
-    s.add_dependency(%q<minion>, [">= 0.1.15"])
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: logworm_amqp
 version: !ruby/object:Gem::Version 
-  hash: 45
+  hash: 59
   prerelease: false
   segments: 
   - 0
-  - 8
   - 9
-  version: 0.8.9
+  - 0
+  version: 0.9.0
 platform: ruby
 authors: 
 - Pomelo, LLC
@@ -15,25 +15,27 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-08-08 00:00:00 -04:00
+date: 2010-08-09 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  name: memcache-client
+  name: json
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
+        hash: 1
         segments: 
-        - 0
-        version: "0"
+        - 1
+        - 4
+        - 3
+        version: 1.4.3
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
-  name: hpricot
+  name: ruby-hmac
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
@@ -47,7 +49,7 @@ dependencies:
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: oauth
+  name: memcache-client
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
@@ -61,7 +63,7 @@ dependencies:
   type: :runtime
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
-  name: heroku
+  name: hpricot
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
@@ -75,23 +77,21 @@ dependencies:
   type: :runtime
   version_requirements: *id004
 - !ruby/object:Gem::Dependency 
-  name: minion
+  name: oauth
   prerelease: false
   requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 5
+        hash: 3
         segments: 
         - 0
-        - 1
-        - 15
-        version: 0.1.15
+        version: "0"
   type: :runtime
   version_requirements: *id005
 - !ruby/object:Gem::Dependency 
-  name: memcache-client
+  name: heroku
   prerelease: false
   requirement: &id006 !ruby/object:Gem::Requirement 
     none: false
@@ -102,24 +102,26 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  type: :development
+  type: :runtime
   version_requirements: *id006
 - !ruby/object:Gem::Dependency 
-  name: hpricot
+  name: json
   prerelease: false
   requirement: &id007 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
+        hash: 1
         segments: 
-        - 0
-        version: "0"
+        - 1
+        - 4
+        - 3
+        version: 1.4.3
   type: :development
   version_requirements: *id007
 - !ruby/object:Gem::Dependency 
-  name: oauth
+  name: ruby-hmac
   prerelease: false
   requirement: &id008 !ruby/object:Gem::Requirement 
     none: false
@@ -133,7 +135,7 @@ dependencies:
   type: :development
   version_requirements: *id008
 - !ruby/object:Gem::Dependency 
-  name: heroku
+  name: hpricot
   prerelease: false
   requirement: &id009 !ruby/object:Gem::Requirement 
     none: false
@@ -147,49 +149,70 @@ dependencies:
   type: :development
   version_requirements: *id009
 - !ruby/object:Gem::Dependency 
-  name: minion
+  name: oauth
   prerelease: false
   requirement: &id010 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 5
+        hash: 3
         segments: 
         - 0
-        - 1
-        - 15
-        version: 0.1.15
+        version: "0"
   type: :development
   version_requirements: *id010
-description: logworm logging tool
+- !ruby/object:Gem::Dependency 
+  name: heroku
+  prerelease: false
+  requirement: &id011 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id011
+description: logworm - logging service
 email: schapira@pomelollc.com
-executables: []
-
+executables: 
+- lw-compute
+- lw-tail
 extensions: []
 
 extra_rdoc_files: 
 - CHANGELOG
-- README.md
+- bin/lw-compute
+- bin/lw-tail
 - lib/base/config.rb
 - lib/base/db.rb
 - lib/base/query_builder.rb
+- lib/client/logger.rb
+- lib/client/rack.rb
+- lib/client/rails.rb
+- lib/cmd/compute.rb
+- lib/cmd/tail.rb
 - lib/logworm_amqp.rb
+- lib/logworm_utils.rb
 files: 
 - CHANGELOG
 - Manifest
-- README.md
 - Rakefile
+- bin/lw-compute
+- bin/lw-tail
 - lib/base/config.rb
 - lib/base/db.rb
 - lib/base/query_builder.rb
+- lib/client/logger.rb
+- lib/client/rack.rb
+- lib/client/rails.rb
+- lib/cmd/compute.rb
+- lib/cmd/tail.rb
 - lib/logworm_amqp.rb
-- spec/base_spec.rb
-- spec/builder_spec.rb
-- spec/config_spec.rb
-- spec/spec.opts
-- spec/spec_helper.rb
-- tests/builder_test.rb
+- lib/logworm_utils.rb
 - logworm_amqp.gemspec
 has_rdoc: true
 homepage: http://www.logworm.com
@@ -201,8 +224,6 @@ rdoc_options:
 - --inline-source
 - --title
 - Logworm_amqp
-- --main
-- README.md
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
@@ -230,6 +251,6 @@ rubyforge_project: logworm_amqp
 rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
-summary: logworm logging tool
+summary: logworm - logging service
 test_files: []