logworm_client 0.3.0

data/CHANGELOG

@@ -0,0 +1,7 @@
+v0.3.0 ensure gem and dependencies live in gemcutter
+
+v0.2.0 log short requests by default in the rack
+
+v0.1.1 minor changes
+
+v0.1.0 initial version.

data/Manifest

@@ -0,0 +1,14 @@
+CHANGELOG
+Manifest
+README
+Rakefile
+bin/lw-compute
+bin/lw-heroku
+bin/lw-tail
+lib/logworm_client.rb
+lib/logworm_client/logger.rb
+lib/logworm_client/rack.rb
+lib/logworm_utils.rb
+lib/logworm_utils/compute.rb
+lib/logworm_utils/heroku.rb
+lib/logworm_utils/tail.rb

data/Rakefile

@@ -0,0 +1,10 @@
+require 'echoe'
+Echoe.new('logworm_client', '0.3.0') do |p|
+  p.description = "logworm client utilities"
+  p.url = "http://www.logworm.com"
+  p.author = "Pomelo, LLC"
+  p.email  = "schapira@pomelollc.com"
+  p.ignore_pattern = ["tmp/*", "script/*"]
+  p.development_dependencies = ["logworm", "json"]
+  p.runtime_dependencies = ["logworm", "json"]
+end

data/bin/lw-compute

@@ -0,0 +1,62 @@
+#!/usr/local/bin/ruby
+
+require 'rubygems'
+require 'logworm_utils'
+require 'optparse'
+
+options = {
+  :aggregate_group => [],
+  :conditions => [],
+  :start => nil,
+  :end => nil,
+  :debug => false
+}
+
+option_parser = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$0} [options] <log table> <function> <field>"
+
+  opts.on("-g group", String, "Specify an aggregation group (e.g, hour(_ts_utc), or response.code)") do |k|
+    options[:aggregate_group] = k
+  end
+
+  opts.on("-c condition", String, "Specify a condition to match. May be used multiple times") do |c|
+    options[:conditions] << c.strip 
+  end
+
+  opts.on("-s starttime", String, "Specify the start time for the query") do |c|
+    options[:start] = c.strip 
+  end
+
+  opts.on("-e endtime", String, "Specify the end time for the query") do |c|
+    options[:end] = c.strip 
+  end
+
+  opts.on("-v", "Show debug information") do 
+    options[:debug] = true
+  end
+  
+  opts.on( '-h', '--help', 'Display this screen' ) do
+    puts option_parser.help
+    exit(1)
+  end
+  
+end
+
+# Parse, and get the required <table>
+option_parser.parse!
+if ARGV.size == 3
+  table = ARGV[0].strip
+  function = ARGV[1].strip
+  field = ARGV[2].strip
+elsif ARGV.size == 2
+  table = ARGV[0].strip
+  function = ARGV[1].strip
+else
+  puts option_parser.help
+  exit(1)
+end
+
+# and run
+LogwormCompute.new(table, function, field, options).run
+
+

data/bin/lw-heroku

@@ -0,0 +1,8 @@
+#!/usr/local/bin/ruby
+
+require 'rubygems'
+require 'logworm_utils'
+
+LogwormHeroku.new().run
+
+

data/bin/lw-tail

@@ -0,0 +1,75 @@
+#!/usr/local/bin/ruby
+
+require 'rubygems'
+require 'logworm_utils'
+require 'optparse'
+
+options = {
+  :limit => 200,
+  :loop => false,
+  :frequency => 10,
+  :fields => [],
+  :conditions => [],
+  :start => nil,
+  :end => nil,
+  :debug => false,
+  :fkat => false
+}
+
+option_parser = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$0} [options] <log table>"
+
+  opts.on("-f [secs]", Integer, 
+          "Continuously check for more data, every [secs] seconds.", "Default: #{options[:frequency]}") do |s| 
+    options[:loop] = true
+    options[:frequency] = s if s
+  end
+
+  opts.on("-r limit", Integer, "Specify how many log entries to fetch.", "Default: #{options[:limit]}") do |n|
+    options[:limit] = n
+  end
+
+  opts.on("-k fields", String, "Specify a comma-separated list of fields to retrieve.") do |k|
+    options[:fields] = k.split(",").map {|f| f.strip}
+  end
+
+  opts.on("-c condition", String, "Specify a condition to match. May be used multiple times") do |c|
+    options[:conditions] << c.strip 
+  end
+
+  opts.on("-s starttime", String, "Specify the start time for the query") do |c|
+    options[:start] = c.strip 
+  end
+
+  opts.on("-e endtime", String, "Specify the end time for the query") do |c|
+    options[:end] = c.strip 
+  end
+
+  opts.on("--flat", "Do not expand log entries when printing them") do 
+    options[:flat] = true
+  end
+
+  opts.on("-v", "Show debug information") do 
+    options[:debug] = true
+  end
+  
+  opts.on( '-h', '--help', 'Display this screen' ) do
+    puts option_parser.help
+    exit(1)
+  end
+  
+end
+
+# Parse, and get the required <table>
+option_parser.parse!
+table = ARGV.pop
+if !table
+  puts option_parser.help
+  exit(1)
+end
+table.strip!
+
+# and run
+t = LogwormTail.new(table, options).run
+
+

data/lib/logworm_client.rb

@@ -0,0 +1,4 @@
+require 'logworm'
+
+require 'logworm_client/logger'
+require 'logworm_client/rack'

data/lib/logworm_client/logger.rb

@@ -0,0 +1,46 @@
+require 'json'
+  
+module Logworm
+  class LogwormException < Exception ; end
+
+  class Logger
+    # Initialize
+    LW_SERVER = DB.new() # Find from config file, or from keys
+    $lr_queue = []
+    
+    def self.start_cycle
+      $request_id = "#{Thread.current.object_id}-#{(Time.now.utc.to_f * 1000).to_i}"
+    end
+
+    def self.log(table, values = {})
+      begin
+        # Rename conflicting keys
+        kvalues = values.delete_if {|k,v| k.to_s == ""}.map {|k,v| [k.to_sym, v]}
+        kvalues = Hash[*kvalues.flatten]
+        [:_ts, :_ts_utc, :_request_id].each do |k|
+          kvalues["orig_#{k}".to_sym] = kvalues[k] if kvalues.has_key? k
+        end
+        # Add information
+        ts = Time.now.utc
+        log_details = {:_ts => ts.strftime("%Y-%m-%dT%H:%M:%SZ"), :_ts_utc =>  (ts.to_f * 1000).to_i}
+        log_details[:_request_id] = $request_id if $request_id
+        # Enqueue
+        payload = kvalues.merge(log_details)
+        $lr_queue << [table, payload]
+      rescue Exception => e
+        $stderr.puts "Error in Logworm::Logger.log: #{e}"
+      end
+    end
+
+    def self.flush
+      if $lr_queue.size > 0
+        begin
+          LW_SERVER.batch_log($lr_queue.to_json)
+        rescue Exception => e
+          $stderr.puts "in Logworm::Logger.flush: #{e}"
+        end
+        $lr_queue = [] 
+      end
+    end
+  end
+end

data/lib/logworm_client/rack.rb

@@ -0,0 +1,52 @@
+module Logworm
+  class Rack
+
+    def initialize(app, options = {})
+      @app = app
+      @log_requests_short =  options[:log_requests].nil? or options[:log_requests] != false
+      @log_requests_long  = !options[:log_requests_long].nil? and options[:log_requests_long] == true
+    end
+
+    def call(env)
+      startTime = Time.now 
+      Logger.start_cycle
+      status, headers, body = @app.call(env) 
+      appTime = (Time.now - startTime) 
+      Logger.log(:web_log_long, {:summary => "#{env['REQUEST_METHOD']} #{env['REQUEST_URI']} - #{status} #{appTime}", 
+                             :request => env_to_log(env), 
+                             :response => {:status => status, :headers => headers}, 
+                             :profiling => appTime}) if @log_requests_long 
+      Logger.log(:web_log, {:summary => "#{env['REQUEST_METHOD']} #{env['REQUEST_URI']} - #{status} #{appTime}", 
+                            :request_path => env['REQUEST_PATH'], :request_ip => env['REMOTE_ADDR'],
+                            :request_method => env['REQUEST_METHOD'], :input => collect_input(env),
+                            :response_status => status, 
+                            :profiling => appTime}) if @log_requests_short
+      begin 
+        Timeout::timeout(1) { 
+           Logger.flush  # Flushes only if there are any entries. Times out after half second
+        } 
+      rescue Exception => e 
+        # Ignore --nothing we can do 
+        # The list of logs may (and most likely will) be preserved for the next request
+        env['rack.errors'].puts "logworm call failed: #{e}"
+      end
+      [status, headers, body] 
+    end
+
+    private
+      def env_to_log(env) 
+        to_log = {} 
+        env.each do |k,v| 
+          to_log[k.to_s.downcase.to_sym] = v unless (k.to_s =~ /rack/i or k.to_s =~ /\./i) 
+        end 
+        to_log.merge({:input => collect_input(env)}) 
+      end 
+
+      def collect_input(env) 
+        inputstr = "" 
+        env['rack.input'].each {|content| inputstr += content } 
+        env['rack.input'].rewind 
+        inputstr 
+      end 
+  end
+end

data/lib/logworm_utils.rb

@@ -0,0 +1,5 @@
+require 'logworm'
+
+require 'logworm_utils/tail'
+require 'logworm_utils/compute'
+require 'logworm_utils/heroku'

data/lib/logworm_utils/compute.rb

@@ -0,0 +1,66 @@
+class LogwormCompute
+    def initialize(table, function, field, options)
+      @table    = table
+      @function = function
+      @field    = field
+      @options  = options
+
+      @valuefield = @field ? "#{@function}(#{@field})" : @function
+
+      begin
+        @db    = Logworm::DB.new(false)
+        spec   = {:aggregate_function => @function, :aggregate_argument => @field}.merge(@options)
+        @query = Logworm::QueryBuilder.new(spec)
+      rescue Exception => e
+        $stderr.puts "Error: #{e}"
+        exit(-1)
+      end
+      
+    end
+
+    def run
+      # Create a resource for the query
+      begin
+        query_data = @db.query(@table, @query.to_json)
+        url = query_data["results_uri"]
+        rows = @db.results(url + "?nocache=1")["results"]
+      rescue Logworm::DatabaseException, Logworm::ForbiddenAccessException  => e
+        $stderr.puts "Error: #{e}"
+        exit(-1)
+      rescue Logworm::InvalidQueryException => e
+        $stderr.puts "#{e}, #{@query.to_json}"  
+        exit(-1)
+      rescue Exception  => e
+        $stderr.puts "Error: #{e}"
+        exit(-1)
+      end
+
+      if @options[:debug]
+        puts "logworm query: #{@query.to_json}"
+        puts "logworm query url: #{query_data["self_uri"]}"
+        puts "logworm results url: #{url}"
+        puts
+      end
+        
+      if @query.groups.length > 0
+        results = {}
+        rows.each do |r|
+          grp = []
+          @query.groups.each do |g|
+            grp << "#{g}:#{r[g]}"
+          end
+          key = "[#{grp.join(', ')}]"
+          value = r[@function]
+          results[key] = {:value => value, :keys => r.dup}
+          results[key][:keys].delete(@function)
+        end
+        results.keys.sort.each do |k|
+          puts "#{k} \t ==> #{@valuefield} = #{results[k][:value]}"
+        end
+      else
+        puts "#{@valuefield} = #{rows.first[@function]}"
+      end
+      
+    end
+
+end

data/lib/logworm_utils/heroku.rb

@@ -0,0 +1,17 @@
+class LogwormHeroku
+    def initialize()
+      c = Logworm::Config.instance
+      if !c.file_found?
+        $stderr.puts "Could not find .logworm file in the current directory with proper keys"
+        exit(-1)
+      end
+      @keys = c.keys
+    end
+
+    def run
+      cmd  = "heroku config:add LW_CONSUMER_KEY=\"#{@keys[0]}\" LW_CONSUMER_SECRET=\"#{@keys[1]}\" "
+      cmd << "LW_PROJECT_KEY=\"#{@keys[2]}\" LW_PROJECT_SECRET=\"#{@keys[3]}\""
+      puts "Push configuration keys to Heroku"
+      puts "There was an error --configuration files not pushed" if system(cmd) == false
+    end
+end

data/lib/logworm_utils/tail.rb

@@ -0,0 +1,84 @@
+class LogwormTail
+    def initialize(table, options)
+      @table   = table
+      @options = options
+
+      begin
+        @db    = Logworm::DB.new(false)
+        @query = Logworm::QueryBuilder.new(@options.merge(:force_ts => true))
+      rescue Exception => e
+        $stderr.puts "Error: #{e}"
+        exit(-1)
+      end
+    end
+
+    def run
+      # Create a resource for the query
+      begin
+        query_data = @db.query(@table, @query.to_json)
+        url = query_data["results_uri"]
+      rescue Logworm::DatabaseException, Logworm::ForbiddenAccessException  => e
+        $stderr.puts "Error: #{e}"
+        exit(-1)
+      rescue Logworm::InvalidQueryException => e
+        $stderr.puts "#{e}"  
+        exit(-1)
+      rescue Exception  => e
+        $stderr.puts "Error: #{e}"
+        exit(-1)
+      end
+
+      if @options[:debug]
+        puts "logworm query: #{@query.to_json}"
+        puts "logworm query url: #{query_data["self_uri"]}"
+        puts "logworm results url: #{url}"
+        puts "refresh frequency: #{@options[:frequency]}" if @options[:loop]
+        puts
+      end
+  
+      while true do
+        begin
+          last_printed = print_rows(@db.results(url + "?nocache=1")["results"], last_printed || nil)
+        rescue Logworm::DatabaseException, Logworm::ForbiddenAccessException  => e
+          $stderr.puts "Error: #{e}"
+          exit(-1)
+        rescue Logworm::InvalidQueryException => e
+          $stderr.puts "#{e}"  
+          exit(-1)
+        rescue Exception  => e
+          $stderr.puts "Error: #{e}"
+          exit(-1)
+        end
+        exit(0) unless @options[:loop]
+        sleep @options[:frequency]
+      end
+    end
+
+  private
+
+    def print_rows(rows, last)
+      last = "" if last.nil?
+      rows.reverse.each do |r|
+        next unless r["_ts"]
+        if r["_ts"] > last
+          last = r["_ts"]
+          r.delete("_id") unless @options[:fields].include?("_id")
+          r.delete("_ts") unless @options[:fields].include?("_ts")
+          puts "#{last.gsub(/T/, ' @ ').gsub(/Z/, ' GMT')} ==> "
+          print_row(r)
+        end
+      end
+      last
+    end
+
+    def print_row(r)
+      if @options[:flat]
+        r.keys.sort.map {|k| "#{k}: #{r[k].inspect}"}.join(', ')
+      else
+        r.keys.sort.each do |k|
+          puts "\t#{k}: #{r[k].inspect}"
+        end
+        puts
+      end
+    end
+end

data/logworm_client.gemspec

@@ -0,0 +1,43 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{logworm_client}
+  s.version = "0.3.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Pomelo, LLC"]
+  s.date = %q{2010-03-01}
+  s.description = %q{logworm client utilities}
+  s.email = %q{schapira@pomelollc.com}
+  s.executables = ["lw-compute", "lw-heroku", "lw-tail"]
+  s.extra_rdoc_files = ["CHANGELOG", "README", "bin/lw-compute", "bin/lw-heroku", "bin/lw-tail", "lib/logworm_client.rb", "lib/logworm_client/logger.rb", "lib/logworm_client/rack.rb", "lib/logworm_utils.rb", "lib/logworm_utils/compute.rb", "lib/logworm_utils/heroku.rb", "lib/logworm_utils/tail.rb"]
+  s.files = ["CHANGELOG", "Manifest", "README", "Rakefile", "bin/lw-compute", "bin/lw-heroku", "bin/lw-tail", "lib/logworm_client.rb", "lib/logworm_client/logger.rb", "lib/logworm_client/rack.rb", "lib/logworm_utils.rb", "lib/logworm_utils/compute.rb", "lib/logworm_utils/heroku.rb", "lib/logworm_utils/tail.rb", "logworm_client.gemspec"]
+  s.homepage = %q{http://www.logworm.com}
+  s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Logworm_client", "--main", "README"]
+  s.require_paths = ["lib"]
+  s.rubyforge_project = %q{logworm_client}
+  s.rubygems_version = %q{1.3.6}
+  s.summary = %q{logworm client utilities}
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<logworm>, [">= 0"])
+      s.add_runtime_dependency(%q<json>, [">= 0"])
+      s.add_development_dependency(%q<logworm>, [">= 0"])
+      s.add_development_dependency(%q<json>, [">= 0"])
+    else
+      s.add_dependency(%q<logworm>, [">= 0"])
+      s.add_dependency(%q<json>, [">= 0"])
+      s.add_dependency(%q<logworm>, [">= 0"])
+      s.add_dependency(%q<json>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<logworm>, [">= 0"])
+    s.add_dependency(%q<json>, [">= 0"])
+    s.add_dependency(%q<logworm>, [">= 0"])
+    s.add_dependency(%q<json>, [">= 0"])
+  end
+end

metadata

@@ -0,0 +1,142 @@
+--- !ruby/object:Gem::Specification 
+name: logworm_client
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 3
+  - 0
+  version: 0.3.0
+platform: ruby
+authors: 
+- Pomelo, LLC
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-03-01 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: logworm
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: json
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: logworm
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: json
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id004
+description: logworm client utilities
+email: schapira@pomelollc.com
+executables: 
+- lw-compute
+- lw-heroku
+- lw-tail
+extensions: []
+
+extra_rdoc_files: 
+- CHANGELOG
+- README
+- bin/lw-compute
+- bin/lw-heroku
+- bin/lw-tail
+- lib/logworm_client.rb
+- lib/logworm_client/logger.rb
+- lib/logworm_client/rack.rb
+- lib/logworm_utils.rb
+- lib/logworm_utils/compute.rb
+- lib/logworm_utils/heroku.rb
+- lib/logworm_utils/tail.rb
+files: 
+- CHANGELOG
+- Manifest
+- README
+- Rakefile
+- bin/lw-compute
+- bin/lw-heroku
+- bin/lw-tail
+- lib/logworm_client.rb
+- lib/logworm_client/logger.rb
+- lib/logworm_client/rack.rb
+- lib/logworm_utils.rb
+- lib/logworm_utils/compute.rb
+- lib/logworm_utils/heroku.rb
+- lib/logworm_utils/tail.rb
+- logworm_client.gemspec
+has_rdoc: true
+homepage: http://www.logworm.com
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --line-numbers
+- --inline-source
+- --title
+- Logworm_client
+- --main
+- README
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 1
+      - 2
+      version: "1.2"
+requirements: []
+
+rubyforge_project: logworm_client
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: logworm client utilities
+test_files: []
+