logeater 0.2.4 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ee9771f19fbb6a41d6a072d2f1a1fb8280cdcce7
-  data.tar.gz: 2b82746a2c79e407ef544a1472ad31826a9f2587
+  metadata.gz: 678d8cc0af848644fd37cf1762d97a9d7779bd5a
+  data.tar.gz: 0ac115076cd9daedc7b74d4f01dd04e2d63e56a6
 SHA512:
-  metadata.gz: da4a22896028c127870e05217d3c9125dc73ddc9e28b00afa4dea52d8dca980240ace54dba9f1aaf08d0a628aff861e423957f5080b5774a627a30db7653cdea
-  data.tar.gz: d12c10c9df5a1a26fc1bcc36adfc8fb276046c43ef23bfc469057de9c72f8e7759305c53f576678667ecdaf3a452019289001f7cf9b78b53246122a2b8993f7c
+  metadata.gz: e3d70df2e56453afba0d382252555801e17e8d7c2ec7d83da83dc147e91bc61f6fd8dc4127bfaadf9c2d71bcdce561cd8c09403a5cc83a129443031ed3ff1545
+  data.tar.gz: 0085e7c733283ecb91ec507a5de05bb53bd2a2d1b4a63a23431cfb750505cd489e2425a63772da3f4f9cb92a274743551ac0a7c112cc0570d7f3c5813cc3272c

data/.gitignore

@@ -1,6 +1,7 @@
 /.bundle/
 /.yardoc
 /Gemfile.lock
+/.ruby-version
 /_yardoc/
 /coverage/
 /doc/

data/README.md

@@ -1,6 +1,6 @@
 # Logeater
 
-Parses log files and imports them into a database
+Parses log files and imports them into a database or converts them to json
 
 ### Features
 
@@ -12,30 +12,17 @@ Parses log files and imports them into a database
 
 ### Usage
 
-Clone the gem
 
-    git clone git@github.com:concordia-publishing-house/logeater.git
-    bundle
+###### Importing log files
 
-Create the development database
+    gem install logeater
+    DATABASE_URL=<production database url> logeater import <app name> <path to logs>/*.gz
 
-    bundle exec rake db:create db:migrate
+###### Converting log files to JSON
 
-Install the gem
+    gem install logeater
+    logeater parse <app name> <path to logs>/*.gz > parsed-logs.json
 
-    bundle exec rake install
-
-Import log files
-
-    logeater my_app ~/Desktop/logs/*.gz
-
-
-### To Do
-
- - Set up databases without cloning the gem?
- - Import to a [Heroku Postgres database](https://dashboard.heroku.com/apps/logs-production)?
- - Parse other kinds of logs?
- - Collect other data from Rails logs?
 
 
 ### Contributing

data/db/migrate/20150110151439_create_requests.rb

@@ -3,7 +3,7 @@ class CreateRequests < ActiveRecord::Migration
     create_table :requests do |t|
       t.string :app, null: false
       t.string :logfile, null: false
-      
+
       t.string :uuid, null: false
       t.string :subdomain
       t.timestamp :started_at
@@ -18,10 +18,10 @@ class CreateRequests < ActiveRecord::Migration
       t.string :format
       t.integer :http_status
       t.string :http_response
-      
+
       t.timestamps
     end
-    
+
     add_index :requests, :app
     add_index :requests, :logfile
     add_index :requests, :uuid, unique: true

data/db/migrate/20160218230227_add_indexes.rb

@@ -0,0 +1,7 @@
+class AddIndexes < ActiveRecord::Migration
+  def change
+    add_index :requests, :completed_at
+    add_index :requests, :subdomain
+    add_index :requests, :params, using: :gin
+  end
+end

data/db/migrate/20160502131002_drop_http_response.rb

@@ -0,0 +1,9 @@
+class DropHttpResponse < ActiveRecord::Migration
+  def up
+    remove_column :requests, :http_response
+  end
+
+  def down
+    add_column :requests, :http_response, :string
+  end
+end

data/db/schema.rb

@@ -11,38 +11,40 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20150602022241) do
+ActiveRecord::Schema.define(version: 20160502131002) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
   create_table "requests", force: :cascade do |t|
-    t.string   "app",           limit: 255, null: false
-    t.string   "logfile",       limit: 255, null: false
-    t.string   "uuid",          limit: 255, null: false
-    t.string   "subdomain",     limit: 255
+    t.string   "app",          null: false
+    t.string   "logfile",      null: false
+    t.string   "uuid",         null: false
+    t.string   "subdomain"
     t.datetime "started_at"
     t.datetime "completed_at"
     t.integer  "duration"
-    t.string   "http_method",   limit: 255
+    t.string   "http_method"
     t.text     "path"
     t.jsonb    "params"
-    t.string   "controller",    limit: 255
-    t.string   "action",        limit: 255
-    t.string   "remote_ip",     limit: 255
-    t.string   "format",        limit: 255
+    t.string   "controller"
+    t.string   "action"
+    t.string   "remote_ip"
+    t.string   "format"
     t.integer  "http_status"
-    t.string   "http_response", limit: 255
-    t.datetime "created_at",                null: false
-    t.datetime "updated_at",                null: false
+    t.datetime "created_at"
+    t.datetime "updated_at"
     t.integer  "user_id"
     t.boolean  "tester_bar"
   end
 
   add_index "requests", ["app"], name: "index_requests_on_app", using: :btree
+  add_index "requests", ["completed_at"], name: "index_requests_on_completed_at", using: :btree
   add_index "requests", ["controller", "action"], name: "index_requests_on_controller_and_action", using: :btree
   add_index "requests", ["http_status"], name: "index_requests_on_http_status", using: :btree
   add_index "requests", ["logfile"], name: "index_requests_on_logfile", using: :btree
+  add_index "requests", ["params"], name: "index_requests_on_params", using: :gin
+  add_index "requests", ["subdomain"], name: "index_requests_on_subdomain", using: :btree
   add_index "requests", ["uuid"], name: "index_requests_on_uuid", unique: true, using: :btree
 
 end

data/lib/logeater/cli.rb

@@ -57,14 +57,12 @@ module Logeater
         reader = Logeater::Reader.new(app, file, options.slice(:progress, :verbose))
         reader.remove_existing_entries!
 
-        started_count = Logeater::Request.count
         started_at = Time.now
-        reader.import
+        count = reader.import
         finished_at = Time.now
-        finished_count = Logeater::Request.count
 
         $stderr.puts " > \e[34mImported \e[1m%d\e[0;34m requests in \e[1m%.2f\e[0;34m seconds (%d of %d)\e[0m\n\n" % [
-          finished_count - started_count,
+          count,
           finished_at - started_at,
           i + 1,
           files.length ]

data/lib/logeater/params_parser.rb

@@ -3,15 +3,15 @@ require "ripper"
 module Logeater
   class ParamsParser
     attr_reader :params
-    
+
     def initialize(params)
       @params = params
     end
-    
+
     def parse!
       identify tokenize_hash(clean(params))
     end
-    
+
     def clean(params)
       loop do
         result = params.gsub(/\#<((?:[\w]|::)+):[^<>]+>/) { "\"#{$1}\"" }
@@ -20,46 +20,46 @@ module Logeater
       end
       params
     end
-    
+
   private
-    
+
     def tokenize_hash(ruby)
       sexp = Ripper.sexp(ruby)
       raise Parser::MalformedParameters.new(ruby) unless sexp
-      
+
       # [:program, [[:hash, ... ]]]
       sexp[1][0]
     end
-    
+
     def identify(sexp)
       case sexp[0]
-      
+
       # [:string_literal, [:string_content, [:@tstring_content, "utf8", [1, 2]]]]
       # [:string_literal, [:string_content]]
       when :string_literal then sexp[1][1] ? sexp[1][1][1] : ""
-      
+
       # [:@int, "10", [1, 14]]
       when :@int then sexp[1].to_i
-      
+
       # [:@float, "10.56", [1, 14]]
       when :@float then sexp[1].to_f
-      
+
       # [:unary, :-@, [:@float, \\\"173.41\\\", [1, 17285]]]
       when :unary then
         return -identify(sexp[2]) if sexp[1] == :-@
         raise Parser::ParserNotImplemented, "Unknown unary operator: #{sexp[1].inspect}"
-      
+
       # [:var_ref, [:@kw, "true", [1, 12]]]
       when :var_ref then
         return true if sexp[1][1] == "true"
         return false if sexp[1][1] == "false"
         return nil if sexp[1][1] == "nil"
         raise Parser::ParserNotImplemented, "Unknown variable: #{sexp[1].inspect}"
-      
+
       # [:array, [[:@int, "1", [1, 9]], [:@int, "4", [1, 12]]]]
       # [:array, nil]
       when :array then sexp[1] ? sexp[1].map { |sexp| identify(sexp) } : []
-      
+
       #  [:hash,
       #    [:assoclist_from_args,
       #     [[:assoc_new,
@@ -67,7 +67,7 @@ module Logeater
       #       [:string_literal, [:string_content, [:@tstring_content, "✓", [1, 12]]]]]]]]]
       # [:hash, nil]
       when :hash then sexp[1] ? sexp[1][1].each_with_object({}) { |(_, key, value), hash| hash[identify(key)] = identify(value) } : {}
-      
+
       else
         raise Parser::ParserNotImplemented, "I don't know how to identify #{sexp.inspect}"
         nil
@@ -76,6 +76,6 @@ module Logeater
       raise Parser::ParserNotImplemented, "An exception occurred when parsing #{sexp.inspect}\n#{$!.class.name}: #{$!.message}"
       nil
     end
-    
+
   end
 end

data/lib/logeater/parser.rb

@@ -5,52 +5,52 @@ require "logeater/parser_errors"
 
 module Logeater
   class Parser
-    
-    
+
+
     LINE_MATCHER = /^
       [A-Z],\s
     \[(?<timestamp>[^\s\]]+)(?:\s[^\]]*)?\]\s+
       (?<log_level>[A-Z]+)\s+\-\-\s:\s+
       (?<message>.*)
     $/x.freeze
-    
+
     REQUEST_LINE_MATCHER = /^
     \[(?<subdomain>[^\]]+)\]\s
     \[(?<uuid>[\w\-]{36})\]\s+
  (?:\[(?:guest|user\.(?<user_id>\d+)(?<tester_bar>:cph)?)\]\s+)?
       (?<message>.*)
     $/x.freeze
-    
+
     REQUEST_STARTED_MATCHER = /^
       Started\s
       (?<http_method>[A-Z]+)\s
      "(?<path>[^"]+)"\sfor\s
       (?<remote_ip>[\d\.]+)
     /x.freeze
-    
+
     REQUEST_CONTROLLER_MATCHER = /^
       Processing\sby\s
       (?<controller>[A-Za-z0-9:]+)\#
       (?<action>[a-z_0-9]+)\sas\s
       (?<format>.*)
     /x.freeze
-    
+
     REQUEST_PARAMETERS_MATCHER = /^
       Parameters:\s
       (?<params>\{.*\})
     $/x.freeze
-    
+
     REQUEST_COMPLETED_MATCHER = /^
       Completed\s
       (?<http_status>\d\d\d)\s
       (?:(?<http_response>.*)\s)?in\s
       (?<duration>[\d\.]+)(?<units>ms)\b
     /x.freeze # optional: (Views: 0.1ms | ActiveRecord: 50.0ms)
-    
+
     def parse!(line)
       match = line.match LINE_MATCHER
       raise UnmatchedLine.new(line) unless match
-      
+
       result = {
         type: :generic,
         timestamp: match["timestamp"],
@@ -59,13 +59,13 @@ module Logeater
       }.merge(
         parse_message(match["message"]))
     end
-    
+
     def parse_message(message)
       match = message.match REQUEST_LINE_MATCHER
       return {} unless match
-      
+
       message = match["message"]
-      
+
       { subdomain: match["subdomain"],
         uuid: match["uuid"],
         type: :request_line,
@@ -75,37 +75,37 @@ module Logeater
       }.merge(
         parse_message_extra(message))
     end
-    
+
     def parse_message_extra(message)
       match = message.match(REQUEST_STARTED_MATCHER)
       return parse_request_started_message(match) if match
-      
+
       match = message.match(REQUEST_CONTROLLER_MATCHER)
       return parse_request_controller_message(match) if match
-      
+
       match = message.match(REQUEST_PARAMETERS_MATCHER)
       return parse_request_params_message(match) if match
-      
+
       match = message.match(REQUEST_COMPLETED_MATCHER)
       return parse_request_completed_message(match) if match
-      
+
       {}
     end
-    
+
     def parse_request_started_message(match)
       { type: :request_started,
         http_method: match["http_method"],
         path: parsed_uri[match["path"]],
         remote_ip: match["remote_ip"] }
     end
-    
+
     def parse_request_controller_message(match)
       { type: :request_controller,
         controller: normalized_controller_name[match["controller"]],
         action: match["action"],
         format: match["format"] }
     end
-    
+
     def parse_request_params_message(match)
       { type: :request_params,
         params: ParamsParser.new(match["params"]).parse! }
@@ -113,35 +113,35 @@ module Logeater
       log "Unable to parse parameters: #{match["params"].inspect}"
       { params: match["params"] }
     end
-    
+
     def parse_request_completed_message(match)
       { type: :request_completed,
         http_status: match["http_status"].to_i,
         http_response: match["http_response"],
         duration: match["duration"].to_i }
     end
-    
-    
-    
+
+
+
     def log(statement)
       $stderr.puts "\e[33m#{statement}\e[0m"
     end
-    
-    
-    
+
+
+
     def initialize
       @normalized_controller_name = Hash.new do |hash, controller_name|
         hash[controller_name] = controller_name.underscore.gsub(/_controller$/, "")
       end
-      
+
       @parsed_uri = Hash.new do |hash, uri|
         hash[uri] = Addressable::URI.parse(uri).path
       end
     end
-    
+
   private
     attr_reader :normalized_controller_name,
                 :parsed_uri
-    
+
   end
 end

data/lib/logeater/parser_errors.rb

@@ -1,30 +1,30 @@
 module Logeater
   class Parser
-    
+
     class Error < ::ArgumentError
       def initialize(message, input)
         super "#{message}: #{input.inspect}"
       end
     end
-    
+
     class UnmatchedLine < Error
       def initialize(input)
         super "Unmatched line", input
       end
     end
-    
+
     class MalformedTimestamp < Error
       def initialize(input)
         super "Malformed timestamp", input
       end
     end
-    
+
     class MalformedParameters < Error
       def initialize(input)
         super "Malformed parameters", input
       end
     end
-    
+
     class ParserNotImplemented < Error
       def initialize(input)
         super "Unable to parse", input

data/lib/logeater/reader.rb

@@ -6,7 +6,7 @@ require "oj"
 module Logeater
   class Reader
     attr_reader :app, :path, :filename, :batch_size
-    
+
     def initialize(app, path, options={})
       @app = app
       @path = path
@@ -15,25 +15,28 @@ module Logeater
       @show_progress = options.fetch :progress, false
       @batch_size = options.fetch :batch_size, 500
       @verbose = options.fetch :verbose, false
+      @count = 0
       @requests = {}
       @completed_requests = []
     end
-    
-    
-    
+
+
+
     def reimport
       remove_existing_entries!
       import
     end
-    
+
     def import
+      @count = 0
       each_request do |attributes|
         completed_requests.push Logeater::Request.new(attributes)
         save! if completed_requests.length >= batch_size
       end
       save!
+      @count
     end
-    
+
     def parse(to: $stdout)
       to << "["
       first = true
@@ -49,19 +52,19 @@ module Logeater
     ensure
       to << "]"
     end
-    
+
     def remove_existing_entries!
       Logeater::Request.where(app: app, logfile: filename).delete_all
     end
-    
+
     def show_progress?
       @show_progress
     end
-    
+
     def verbose?
       @verbose
     end
-    
+
     def each_line
       File.open(path) do |file|
         io = File.extname(path) == ".gz" ? Zlib::GzipReader.new(file) : file
@@ -74,7 +77,7 @@ module Logeater
       end
     end
     alias :scan :each_line
-    
+
     def each_request
       count = 0
       each_line do |line|
@@ -85,63 +88,64 @@ module Logeater
       end
       count
     end
-    
-    
-    
+
+
+
   private
     attr_reader :parser, :requests, :completed_requests
-    
+
     def process_line!(line, &block)
       attributes = parser.parse!(line)
-      
+
       return if [:generic, :request_line].member? attributes[:type]
-      
+
       if attributes[:type] == :request_started
         requests[attributes[:uuid]] = attributes
           .slice(:uuid, :subdomain, :http_method, :path, :remote_ip, :user_id, :tester_bar)
           .merge(started_at: attributes[:timestamp], logfile: filename, app: app)
         return
       end
-      
+
       request_attributes = requests[attributes[:uuid]]
       unless request_attributes
         log "Attempting to record #{attributes[:type].inspect}; but there is no request started with UUID #{attributes[:uuid].inspect}" if verbose?
         return
       end
-      
+
       case attributes[:type]
       when :request_controller
         request_attributes.merge! attributes.slice(:controller, :action, :format)
-      
+
       when :request_params
         request_attributes.merge! attributes.slice(:params)
-      
+
       when :request_completed
         request_attributes.merge! attributes
-          .slice(:http_status, :http_response, :duration)
+          .slice(:http_status, :duration)
           .merge(completed_at: attributes[:timestamp])
-        
+
         yield request_attributes
         requests.delete attributes[:uuid]
       end
-      
+
     rescue Logeater::Parser::UnmatchedLine
       $stderr.puts "\e[90m#{$!.message}\e[0m" if verbose?
     rescue Logeater::Parser::Error
       log $!.message
     end
-    
+
     def save!
       return if completed_requests.empty?
-      Logeater::Request.import(completed_requests)
+      result = Logeater::Request.import(completed_requests)
+      @count += result.num_inserts
       completed_requests.clear
     end
-    
-    
-    
+
+
+
     def log(statement)
       $stderr.puts "\e[33m#{statement}\e[0m"
     end
-    
+
   end
 end

data/lib/logeater/version.rb

@@ -1,3 +1,3 @@
 module Logeater
-  VERSION = "0.2.4"
+  VERSION = "0.3.0"
 end

data/logeater.gemspec

@@ -24,9 +24,9 @@ Gem::Specification.new do |spec|
   spec.add_dependency "standalone_migrations", "~> 4.0.0"
   spec.add_dependency "addressable"
   spec.add_dependency "ruby-progressbar"
-  spec.add_dependency "activerecord-import", "~> 0.10.0"
+  spec.add_dependency "activerecord-import", "~> 0.10"
   spec.add_dependency "thor", "~> 0.19.1"
-  spec.add_dependency "oj", "~> 2.12.14"
+  spec.add_dependency "oj", "~> 2.15"
 
   spec.add_development_dependency "bundler", "~> 1.7"
   spec.add_development_dependency "rake", "~> 10.0"

data/test/integration/logeater_test.rb

@@ -2,27 +2,27 @@ require "test_helper"
 
 class LogeaterTest < ActiveSupport::TestCase
   attr_reader :logfile
-  
-  
+
+
   context "Given the log of a single request, it" do
     setup do
       @logfile = File.expand_path("../../data/single_request.log", __FILE__)
     end
-    
+
     should "identify the name of the logfile" do
       assert_equal "single_request.log", reader.filename
     end
-    
+
     should "create an entry in the database" do
       assert_difference "Logeater::Request.count", +1 do
         reader.import
       end
     end
-    
+
     should "set all the attributes" do
       reader.import
       request = Logeater::Request.first
-      
+
       params = {"refresh_page" => "true", "id" => "1035826228"}
       assert_equal "test", request.app
       assert_equal "single_request.log", request.logfile
@@ -39,42 +39,41 @@ class LogeaterTest < ActiveSupport::TestCase
       assert_equal "71.218.222.249", request.remote_ip
       assert_equal "JS", request.format
       assert_equal 200, request.http_status
-      assert_equal "OK", request.http_response
     end
-    
+
     should "erase any entries that had already been imported with that app and filename" do
       Logeater::Request.create!(app: app, logfile: "single_request.log", uuid: "1")
       Logeater::Request.create!(app: app, logfile: "single_request.log", uuid: "2")
       Logeater::Request.create!(app: app, logfile: "single_request.log", uuid: "3")
-      
+
       assert_difference "Logeater::Request.count", -2 do
         reader.reimport
       end
     end
   end
-  
-  
+
+
   context "Given a gzipped logfile, it" do
     setup do
       @logfile = File.expand_path("../../data/single_request.gz", __FILE__)
     end
-    
+
     should "create an entry in the database" do
       assert_difference "Logeater::Request.count", +1 do
         reader.import
       end
     end
   end
-  
-  
+
+
 private
-  
+
   def app
     "test"
   end
-  
+
   def reader
     Logeater::Reader.new(app, logfile)
   end
-  
+
 end

data/test/test_helper.rb

@@ -18,13 +18,13 @@ DatabaseCleaner.clean
 DatabaseCleaner.strategy = :transaction
 
 class ActiveSupport::TestCase
-  
+
   setup do
     DatabaseCleaner.start
   end
-  
+
   teardown do
     DatabaseCleaner.clean
   end
-  
+
 end

data/test/unit/params_parser_test.rb

@@ -1,83 +1,83 @@
 require "test_helper"
 
 class ParamsParserTest < ActiveSupport::TestCase
-  
-  
+
+
   context "Given a simple hash, it" do
     should "parse it" do
       assert_parses '{"utf8"=>"✓"}' => {"utf8"=>"✓"}
     end
-    
+
     should "handle integers" do
       assert_parses '{"person_id"=>10}' => {"person_id"=>10}
     end
-    
+
     should "handle floats" do
       assert_parses '{"person_id"=>10.56}' => {"person_id"=>10.56}
     end
-    
+
     should "handle negatives" do
       assert_parses '{"person_id"=>-10}' => {"person_id"=>-10}
       assert_parses '{"person_id"=>-10.56}' => {"person_id"=>-10.56}
     end
-    
+
     should "handle booleans" do
       assert_parses '{"visible"=>true}' => {"visible"=>true}
     end
-    
+
     should "handle nil" do
       assert_parses '{"visible"=>nil}' => {"visible"=>nil}
     end
-    
+
     should "handle arrays" do
       assert_parses '{"ids"=>[1, 4]}' => {"ids"=>[1,4]}
     end
-    
-    
-    
+
+
+
     should "handle empty strings" do
       assert_parses '{"visible"=>""}' => {"visible"=>""}
     end
-    
+
     should "handle empty arrays" do
       assert_parses '{"array"=>[]}' => {"array"=>[]}
     end
-    
+
     should "handle empty hashes" do
       assert_parses '{"hash"=>{}}' => {"hash"=>{}}
     end
   end
-  
-  
+
+
   context "Given a hash with more than one key, it" do
     should "parse it" do
       assert_parses '{"utf8"=>"✓", "authenticity_token"=>"kDM07..."}' => {"utf8"=>"✓", "authenticity_token"=>"kDM07..."}
     end
   end
-  
-  
+
+
   context "Given a hash with a nested hash, it" do
     should "handle nested hashes" do
       assert_parses '{"person"=>{"name"=>"Tim"}}' => {"person"=>{"name"=>"Tim"}}
     end
-    
+
     should "handle arrays of nested hashes" do
       assert_parses '{"people"=>[{"id"=>1},{"id"=>2}]}' => {"people"=>[{"id"=>1},{"id"=>2}]}
     end
   end
-  
-  
+
+
   context "Given a hash with a serialized Ruby object, it" do
     should "parse it" do
       assert_parses '{"tempfile"=>#<Tempfile:/tmp/RackMultipart20141213-1847-1c8fpzw>}' => {"tempfile"=>"Tempfile"}
     end
-    
+
     should "ignore the object's ivars" do
       assert_parses '{"photo"=>#<ActionDispatch::Http::UploadedFile:0x007f7c685318a8 @tempfile=#<Tempfile:/tmp/RackMultipart20141213-1847-1c8fpzw>, @original_filename="Martin-008.JPG", @content_type="image/jpeg", @headers="Content-Disposition: form-data; name=\"person[photo]\"; filename=\"Martin-008.JPG\"\\r\\nContent-Type: image/jpeg\\r\\n\">}' => {"photo"=>"ActionDispatch::Http::UploadedFile"}
     end
   end
-  
-  
+
+
   context "Given an invalid input, it" do
     should "raise MalformedParameters" do
       assert_raises Logeater::Parser::MalformedParameters do
@@ -85,17 +85,17 @@ class ParamsParserTest < ActiveSupport::TestCase
       end
     end
   end
-  
-  
+
+
 private
-  
+
   def assert_parses(params)
     value, result = params.to_a[0]
     assert_equal result, parse!(value)
   end
-  
+
   def parse!(value)
     Logeater::ParamsParser.new(value).parse!
   end
-  
+
 end

data/test/unit/parser_test.rb

@@ -2,247 +2,247 @@ require "test_helper"
 
 class ParserTest < ActiveSupport::TestCase
   attr_reader :line
-  
-  
-  
+
+
+
   context "given a line that doesn't start with the generic log prefix, it" do
     setup do
       @line = "\n"
     end
-    
+
     should "raise UnmatchedLine" do
       assert_raises Logeater::Parser::UnmatchedLine do
         parse!
       end
     end
   end
-  
-  
-  
+
+
+
   context "given any generic log line, it" do
     setup do
       @line = "I, [2015-01-10T15:18:05.850839 #18070]  INFO -- : the message\n"
     end
-    
+
     should "identify the line as :generic" do
       assert_parses type: :generic
     end
-    
+
     should "identify the log level" do
       assert_parses log_level: "INFO"
     end
-    
+
     should "identify the time, including milliseconds" do
       assert_parses timestamp: "2015-01-10T15:18:05.850839"
     end
-    
+
     should "identify the remainder of the log message" do
       assert_parses message: "the message"
     end
   end
-  
-  
-  
+
+
+
   context "given a log line for a Rails request, it" do
     setup do
       @line = "I, [2015-01-10T15:18:05.850839 #18070]  INFO -- : [livingsaviorco] [2d89d962-57c4-47c9-a9e9-6a16a5f22a12] [gzip] Compress reponse by 42.2 KB (83.3%)  (1.4ms)"
     end
-    
+
     should "identify the line as :generic" do
       assert_parses type: :request_line
     end
-    
+
     should "identify the subdomain" do
       assert_parses subdomain: "livingsaviorco"
     end
-    
+
     should "identify the request's ID" do
       assert_parses uuid: "2d89d962-57c4-47c9-a9e9-6a16a5f22a12"
     end
-    
+
     should "identify the remainder of the log message" do
       assert_parses message: "[gzip] Compress reponse by 42.2 KB (83.3%)  (1.4ms)"
     end
   end
-  
-  
-  
+
+
+
   context "given a log line for a Rails request" do
     context "that indicates the current user (who was logged in with the tester bar), it" do
       setup do
         @line = "I, [2015-01-10T15:18:05.850839 #18070]  INFO -- : [livingsaviorco] [2d89d962-57c4-47c9-a9e9-6a16a5f22a12] [user.902544074:cph] [gzip] Compress reponse by 42.2 KB (83.3%)  (1.4ms)"
       end
-      
+
       should "identify the user's id" do
         assert_parses user_id: 902544074
       end
-      
+
       should "notice that the user was logged-in with the tester bar" do
         assert_parses tester_bar: true
       end
-    
+
       should "identify the remainder of the log message" do
         assert_parses message: "[gzip] Compress reponse by 42.2 KB (83.3%)  (1.4ms)"
       end
     end
-    
+
     context "that indicates the current user (who was not logged in with the tester bar), it" do
       setup do
         @line = "I, [2015-01-10T15:18:05.850839 #18070]  INFO -- : [livingsaviorco] [2d89d962-57c4-47c9-a9e9-6a16a5f22a12] [user.902544074] [gzip] Compress reponse by 42.2 KB (83.3%)  (1.4ms)"
       end
-      
+
       should "identify the user's id" do
         assert_parses user_id: 902544074
       end
-      
+
       should "notice that the user was not logged-in with the tester bar" do
         assert_parses tester_bar: false
       end
-    
+
       should "identify the remainder of the log message" do
         assert_parses message: "[gzip] Compress reponse by 42.2 KB (83.3%)  (1.4ms)"
       end
     end
-    
+
     context "that indicates that the user is logged out, it" do
       setup do
         @line = "I, [2015-01-10T15:18:05.850839 #18070]  INFO -- : [livingsaviorco] [2d89d962-57c4-47c9-a9e9-6a16a5f22a12] [guest] [gzip] Compress reponse by 42.2 KB (83.3%)  (1.4ms)"
       end
-    
+
       should "identify the remainder of the log message" do
         assert_parses message: "[gzip] Compress reponse by 42.2 KB (83.3%)  (1.4ms)"
       end
     end
   end
-  
-  
-  
+
+
+
   context "given the \"Started\" line, it" do
     setup do
       @line = "I, [2015-01-10T15:18:12.064392 #2354]  INFO -- : [livingsaviorco] [0fc5154a-c288-4bad-9c7a-de3d7e7d2496] Started GET \"/people/1035826228?refresh_page=true\" for 71.218.222.249 at 2015-01-10 15:18:12 +0000"
     end
-    
+
     should "identify the line as :request_started" do
       assert_parses type: :request_started
     end
-    
+
     should "identify the HTTP method" do
       assert_parses http_method: "GET"
     end
-    
+
     should "identify the path (without params)" do
       assert_parses path: "/people/1035826228"
     end
-    
+
     should "identify the remote client's IP address" do
       assert_parses remote_ip: "71.218.222.249"
     end
   end
-  
-  
-  
+
+
+
   context "given the \"Processing by\" line, it" do
     setup do
       @line = "I, [2015-01-10T15:18:12.067034 #2354]  INFO -- : [livingsaviorco] [0fc5154a-c288-4bad-9c7a-de3d7e7d2496] Processing by Api::V1::PeopleController#show as JS"
     end
-    
+
     should "identify the line as :request_controller" do
       assert_parses type: :request_controller
     end
-    
+
     should "identify the controller and action" do
       assert_parses controller: "api/v1/people", action: "show"
     end
-    
+
     should "identify the format requested" do
       assert_parses format: "JS"
     end
   end
-  
-  
-  
+
+
+
   context "given the \"Parameters\" line, it" do
     setup do
       @line = "I, [2015-01-10T15:18:12.067134 #2354]  INFO -- : [livingsaviorco] [0fc5154a-c288-4bad-9c7a-de3d7e7d2496]   Parameters: {\"refresh_page\"=>\"true\", \"id\"=>\"1035826228\"}"
     end
-    
+
     should "identify the line as :request_params" do
       assert_parses type: :request_params
     end
-    
+
     should "identify the params" do
       assert_parses params: {"refresh_page" => "true", "id" => "1035826228"}
     end
   end
-  
+
   context "when the \"Parameters\" line contains invalid syntax, it" do
     setup do
       @line = "I, [2015-01-10T15:18:12.067134 #2354]  INFO -- : [livingsaviorco] [0fc5154a-c288-4bad-9c7a-de3d7e7d2496]   Parameters: {\"refresh_page\"=>}"
     end
-    
+
     should "return the params unparsed" do
       assert_parses params: "{\"refresh_page\"=>}"
     end
   end
-  
-  
-  
+
+
+
   context "given the \"Completed\" line, it" do
     setup do
       @line = "I, [2015-01-10T15:18:12.262903 #2354]  INFO -- : [livingsaviorco] [0fc5154a-c288-4bad-9c7a-de3d7e7d2496] Completed 401 Unauthorized in 2ms"
     end
-    
+
     should "identify the line as :request_completed" do
       assert_parses type: :request_completed
     end
-    
+
     should "identify the HTTP response" do
       assert_parses http_status: 401, http_response: "Unauthorized"
     end
-    
+
     should "identify the duration of the request" do
       assert_parses duration: 2
     end
   end
-  
+
   context "when the \"Completed\" line doesn't have a textual description of the status, it" do
     setup do
       @line = "I, [2015-01-10T15:18:12.262903 #2354]  INFO -- : [livingsaviorco] [0fc5154a-c288-4bad-9c7a-de3d7e7d2496] Completed 307 in 0.8ms"
     end
-    
+
     should "be cool with that" do
       assert_parses http_status: 307, http_response: nil
     end
   end
-  
+
   context "when the \"Completed\" line contains a breakdown of times, it" do
     setup do
       @line = "I, [2015-01-10T15:18:12.262903 #2354]  INFO -- : [livingsaviorco] [0fc5154a-c288-4bad-9c7a-de3d7e7d2496] Completed 200 OK in 196ms (Views: 0.1ms | ActiveRecord: 50.0ms)"
     end
-    
+
     should "identify the duration of the request" do
       assert_parses duration: 196
     end
   end
-  
-  
-  
+
+
+
 private
-  
+
   def assert_parses(expectations)
     results = parse!
-    
+
     expectations.each do |key, value|
       assert_equal value, results[key]
     end
   end
-  
+
   def parse!
     parser.parse!(line)
   end
-  
+
   def parser
     Logeater::Parser.new
   end
-  
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logeater
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.3.0
 platform: ruby
 authors:
 - Bob Lail
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-03 00:00:00.000000000 Z
+date: 2016-05-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -100,14 +100,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.10.0
+        version: '0.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.10.0
+        version: '0.10'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -128,14 +128,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.12.14
+        version: '2.15'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.12.14
+        version: '2.15'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -241,6 +241,8 @@ files:
 - db/migrate/20150207183757_change_requests_params_to_jsonb.rb
 - db/migrate/20150224021844_change_requests_path_to_text.rb
 - db/migrate/20150602022241_add_user_id_and_tester_bar_to_requests.rb
+- db/migrate/20160218230227_add_indexes.rb
+- db/migrate/20160502131002_drop_http_response.rb
 - db/schema.rb
 - lib/logeater.rb
 - lib/logeater/cli.rb
@@ -277,7 +279,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Parses log files and imports them into a database