logeater 0.1.1

data/logeater.gemspec

@@ -0,0 +1,38 @@
+# coding: utf-8
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "logeater/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "logeater"
+  spec.version       = Logeater::VERSION
+  spec.authors       = ["Bob Lail"]
+  spec.email         = ["bob.lail@cph.org"]
+  spec.summary       = %q{Parses log files and imports them into a database}
+  spec.description   = %q{Parses log files and imports them into a database}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "activerecord", "~> 3.2.21"
+  spec.add_dependency "activesupport"
+  spec.add_dependency "pg"
+  spec.add_dependency "standalone_migrations"
+  spec.add_dependency "addressable"
+  spec.add_dependency "ruby-progressbar"
+  spec.add_dependency "activerecord-import", "~> 0.3.1"
+  spec.add_dependency "activerecord-postgres-json"
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rails", ">= 3.1.0", "< 5.0.0"
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "shoulda-context"
+  spec.add_development_dependency "turn"
+  spec.add_development_dependency "database_cleaner"
+
+end

data/test/data/single_request.log

@@ -0,0 +1,9 @@
+I, [2015-01-10T15:18:12.064392 #2354]  INFO -- : [livingsaviorco] [0fc5154a-c288-4bad-9c7a-de3d7e7d2496] Started GET "/people/1035826228?refresh_page=true" for 71.218.222.249 at 2015-01-10 15:18:12 +0000
+I, [2015-01-10T15:18:12.067034 #2354]  INFO -- : [livingsaviorco] [0fc5154a-c288-4bad-9c7a-de3d7e7d2496] Processing by PeopleController#show as JS
+I, [2015-01-10T15:18:12.067134 #2354]  INFO -- : [livingsaviorco] [0fc5154a-c288-4bad-9c7a-de3d7e7d2496]   Parameters: {"refresh_page"=>"true", "id"=>"1035826228"}
+I, [2015-01-10T15:18:12.156216 #2354]  INFO -- : [livingsaviorco] [0fc5154a-c288-4bad-9c7a-de3d7e7d2496]   Rendered people/_name.html.erb (2.3ms)
+I, [2015-01-10T15:18:12.159557 #2354]  INFO -- : [livingsaviorco] [0fc5154a-c288-4bad-9c7a-de3d7e7d2496]   Rendered people/_status.html.erb (3.1ms)
+I, [2015-01-10T15:18:12.177934 #2354]  INFO -- : [livingsaviorco] [0fc5154a-c288-4bad-9c7a-de3d7e7d2496]   Rendered people/_household.html.erb (18.2ms)
+I, [2015-01-10T15:18:12.184299 #2354]  INFO -- : [livingsaviorco] [0fc5154a-c288-4bad-9c7a-de3d7e7d2496]   Rendered person_tags/_tag.html.erb (1.1ms)
+I, [2015-01-10T15:18:12.262438 #2354]  INFO -- : [livingsaviorco] [0fc5154a-c288-4bad-9c7a-de3d7e7d2496] [gzip] Compress reponse by 82.5 KB (87.6%)  (2.6ms)
+I, [2015-01-10T15:18:12.262903 #2354]  INFO -- : [livingsaviorco] [0fc5154a-c288-4bad-9c7a-de3d7e7d2496] Completed 200 OK in 196ms (Views: 0.1ms | ActiveRecord: 50.0ms)

data/test/integration/logeater_test.rb

@@ -0,0 +1,80 @@
+require "test_helper"
+
+class LogeaterTest < ActiveSupport::TestCase
+  attr_reader :logfile
+  
+  
+  context "Given the log of a single request, it" do
+    setup do
+      @logfile = File.expand_path("../../data/single_request.log", __FILE__)
+    end
+    
+    should "identify the name of the logfile" do
+      assert_equal "single_request.log", reader.filename
+    end
+    
+    should "create an entry in the database" do
+      assert_difference "Logeater::Request.count", +1 do
+        reader.import
+      end
+    end
+    
+    should "set all the attributes" do
+      reader.import
+      request = Logeater::Request.first
+      
+      params = {"refresh_page" => "true", "id" => "1035826228"}
+      assert_equal "test", request.app
+      assert_equal "single_request.log", request.logfile
+      assert_equal "0fc5154a-c288-4bad-9c7a-de3d7e7d2496", request.uuid
+      assert_equal "livingsaviorco", request.subdomain
+      assert_equal Time.new(2015, 1, 10, 15, 18, BigDecimal.new("12.064392")), request.started_at
+      assert_equal Time.new(2015, 1, 10, 15, 18, BigDecimal.new("12.262903")), request.completed_at
+      assert_equal 196, request.duration
+      assert_equal "GET", request.http_method
+      assert_equal "/people/1035826228", request.path
+      assert_equal params, request.params
+      assert_equal "people", request.controller
+      assert_equal "show", request.action
+      assert_equal "71.218.222.249", request.remote_ip
+      assert_equal "JS", request.format
+      assert_equal 200, request.http_status
+      assert_equal "OK", request.http_response
+    end
+    
+    should "erase any entries that had already been imported with that app and filename" do
+      Logeater::Request.create!(app: app, logfile: "single_request.log", uuid: "1")
+      Logeater::Request.create!(app: app, logfile: "single_request.log", uuid: "2")
+      Logeater::Request.create!(app: app, logfile: "single_request.log", uuid: "3")
+      
+      assert_difference "Logeater::Request.count", -2 do
+        reader.reimport
+      end
+    end
+  end
+  
+  
+  context "Given a gzipped logfile, it" do
+    setup do
+      @logfile = File.expand_path("../../data/single_request.gz", __FILE__)
+    end
+    
+    should "create an entry in the database" do
+      assert_difference "Logeater::Request.count", +1 do
+        reader.import
+      end
+    end
+  end
+  
+  
+private
+  
+  def app
+    "test"
+  end
+  
+  def reader
+    Logeater::Reader.new(app, logfile)
+  end
+  
+end

data/test/test_helper.rb

@@ -0,0 +1,28 @@
+ENV["RAILS_ENV"] = "test"
+require "rubygems"
+require "rails"
+require "rails/test_help"
+require "pry"
+require "database_cleaner"
+require "shoulda/context"
+require "turn"
+require "logeater"
+
+load File.expand_path("../../db/schema.rb", __FILE__)
+
+DatabaseCleaner.strategy = :truncation
+DatabaseCleaner.clean
+
+DatabaseCleaner.strategy = :transaction
+
+class ActiveSupport::TestCase
+  
+  setup do
+    DatabaseCleaner.start
+  end
+  
+  teardown do
+    DatabaseCleaner.clean
+  end
+  
+end

data/test/unit/params_parser_test.rb

@@ -0,0 +1,96 @@
+require "test_helper"
+
+class ParamsParserTest < ActiveSupport::TestCase
+  
+  
+  context "Given a simple hash, it" do
+    should "parse it" do
+      assert_parses '{"utf8"=>"✓"}' => {"utf8"=>"✓"}
+    end
+    
+    should "handle integers" do
+      assert_parses '{"person_id"=>10}' => {"person_id"=>10}
+    end
+    
+    should "handle floats" do
+      assert_parses '{"person_id"=>10.56}' => {"person_id"=>10.56}
+    end
+    
+    should "handle booleans" do
+      assert_parses '{"visible"=>true}' => {"visible"=>true}
+    end
+    
+    should "handle nil" do
+      assert_parses '{"visible"=>nil}' => {"visible"=>nil}
+    end
+    
+    should "handle arrays" do
+      assert_parses '{"ids"=>[1, 4]}' => {"ids"=>[1,4]}
+    end
+    
+    
+    
+    should "handle empty strings" do
+      assert_parses '{"visible"=>""}' => {"visible"=>""}
+    end
+    
+    should "handle empty arrays" do
+      assert_parses '{"array"=>[]}' => {"array"=>[]}
+    end
+    
+    should "handle empty hashes" do
+      assert_parses '{"hash"=>{}}' => {"hash"=>{}}
+    end
+  end
+  
+  
+  context "Given a hash with more than one key, it" do
+    should "parse it" do
+      assert_parses '{"utf8"=>"✓", "authenticity_token"=>"kDM07..."}' => {"utf8"=>"✓", "authenticity_token"=>"kDM07..."}
+    end
+  end
+  
+  
+  context "Given a hash with a nested hash, it" do
+    should "handle nested hashes" do
+      assert_parses '{"person"=>{"name"=>"Tim"}}' => {"person"=>{"name"=>"Tim"}}
+    end
+    
+    should "handle arrays of nested hashes" do
+      assert_parses '{"people"=>[{"id"=>1},{"id"=>2}]}' => {"people"=>[{"id"=>1},{"id"=>2}]}
+    end
+  end
+  
+  
+  context "Given a hash with a serialized Ruby object, it" do
+    should "parse it" do
+      assert_parses '{"tempfile"=>#<Tempfile:/tmp/RackMultipart20141213-1847-1c8fpzw>}' => {"tempfile"=>"Tempfile"}
+    end
+    
+    should "ignore the object's ivars" do
+      assert_parses '{"photo"=>#<ActionDispatch::Http::UploadedFile:0x007f7c685318a8 @tempfile=#<Tempfile:/tmp/RackMultipart20141213-1847-1c8fpzw>, @original_filename="Martin-008.JPG", @content_type="image/jpeg", @headers="Content-Disposition: form-data; name=\"person[photo]\"; filename=\"Martin-008.JPG\"\\r\\nContent-Type: image/jpeg\\r\\n\">}' => {"photo"=>"ActionDispatch::Http::UploadedFile"}
+    end
+  end
+  
+  
+  context "Given an invalid input, it" do
+    should "raise MalformedParameters" do
+      assert_raises Logeater::Parser::MalformedParameters do
+        parse! '{"nope"=>}'
+      end
+    end
+  end
+  
+  
+private
+  
+  def assert_parses(params)
+    value, result = params.to_a[0]
+    assert_equal result, parse!(value)
+  end
+  
+  def parse!(value)
+    Logeater::ParamsParser.new(value).parse!
+  end
+  
+end

data/test/unit/parser_test.rb

@@ -0,0 +1,188 @@
+require "test_helper"
+
+class ParserTest < ActiveSupport::TestCase
+  attr_reader :line
+  
+  
+  
+  context "given a line that doesn't start with the generic log prefix, it" do
+    setup do
+      @line = "\n"
+    end
+    
+    should "raise UnmatchedLine" do
+      assert_raises Logeater::Parser::UnmatchedLine do
+        parse!
+      end
+    end
+  end
+  
+  
+  
+  context "given any generic log line, it" do
+    setup do
+      @line = "I, [2015-01-10T15:18:05.850839 #18070]  INFO -- : the message\n"
+    end
+    
+    should "identify the line as :generic" do
+      assert_parses type: :generic
+    end
+    
+    should "identify the log level" do
+      assert_parses log_level: "INFO"
+    end
+    
+    should "identify the time, including milliseconds" do
+      assert_parses timestamp: Time.new(2015, 1, 10, 15, 18, BigDecimal.new("5.850839"))
+    end
+    
+    should "identify the remainder of the log message" do
+      assert_parses message: "the message"
+    end
+  end
+  
+  
+  
+  context "given a log line for a Rails request, it" do
+    setup do
+      @line = "I, [2015-01-10T15:18:05.850839 #18070]  INFO -- : [livingsaviorco] [2d89d962-57c4-47c9-a9e9-6a16a5f22a12] [gzip] Compress reponse by 42.2 KB (83.3%)  (1.4ms)"
+    end
+    
+    should "identify the line as :generic" do
+      assert_parses type: :request_line
+    end
+    
+    should "identify the subdomain" do
+      assert_parses subdomain: "livingsaviorco"
+    end
+    
+    should "identify the request's ID" do
+      assert_parses uuid: "2d89d962-57c4-47c9-a9e9-6a16a5f22a12"
+    end
+    
+    should "identify the remainder of the log message" do
+      assert_parses message: "[gzip] Compress reponse by 42.2 KB (83.3%)  (1.4ms)"
+    end
+  end
+  
+  
+  
+  context "given the \"Started\" line, it" do
+    setup do
+      @line = "I, [2015-01-10T15:18:12.064392 #2354]  INFO -- : [livingsaviorco] [0fc5154a-c288-4bad-9c7a-de3d7e7d2496] Started GET \"/people/1035826228?refresh_page=true\" for 71.218.222.249 at 2015-01-10 15:18:12 +0000"
+    end
+    
+    should "identify the line as :request_started" do
+      assert_parses type: :request_started
+    end
+    
+    should "identify the HTTP method" do
+      assert_parses http_method: "GET"
+    end
+    
+    should "identify the path (without params)" do
+      assert_parses path: "/people/1035826228"
+    end
+    
+    should "identify the remote client's IP address" do
+      assert_parses remote_ip: "71.218.222.249"
+    end
+  end
+  
+  
+  
+  context "given the \"Processing by\" line, it" do
+    setup do
+      @line = "I, [2015-01-10T15:18:12.067034 #2354]  INFO -- : [livingsaviorco] [0fc5154a-c288-4bad-9c7a-de3d7e7d2496] Processing by Api::V1::PeopleController#show as JS"
+    end
+    
+    should "identify the line as :request_controller" do
+      assert_parses type: :request_controller
+    end
+    
+    should "identify the controller and action" do
+      assert_parses controller: "api/v1/people", action: "show"
+    end
+    
+    should "identify the format requested" do
+      assert_parses format: "JS"
+    end
+  end
+  
+  
+  
+  context "given the \"Parameters\" line, it" do
+    setup do
+      @line = "I, [2015-01-10T15:18:12.067134 #2354]  INFO -- : [livingsaviorco] [0fc5154a-c288-4bad-9c7a-de3d7e7d2496]   Parameters: {\"refresh_page\"=>\"true\", \"id\"=>\"1035826228\"}"
+    end
+    
+    should "identify the line as :request_params" do
+      assert_parses type: :request_params
+    end
+    
+    should "identify the params" do
+      assert_parses params: {"refresh_page" => "true", "id" => "1035826228"}
+    end
+  end
+  
+  context "when the \"Parameters\" line contains invalid syntax, it" do
+    setup do
+      @line = "I, [2015-01-10T15:18:12.067134 #2354]  INFO -- : [livingsaviorco] [0fc5154a-c288-4bad-9c7a-de3d7e7d2496]   Parameters: {\"refresh_page\"=>}"
+    end
+    
+    should "return the params unparsed" do
+      assert_parses params: "{\"refresh_page\"=>}"
+    end
+  end
+  
+  
+  
+  context "given the \"Completed\" line, it" do
+    setup do
+      @line = "I, [2015-01-10T15:18:12.262903 #2354]  INFO -- : [livingsaviorco] [0fc5154a-c288-4bad-9c7a-de3d7e7d2496] Completed 401 Unauthorized in 2ms"
+    end
+    
+    should "identify the line as :request_completed" do
+      assert_parses type: :request_completed
+    end
+    
+    should "identify the HTTP response" do
+      assert_parses http_status: 401, http_response: "Unauthorized"
+    end
+    
+    should "identify the duration of the request" do
+      assert_parses duration: 2
+    end
+  end
+  
+  context "when the \"Completed\" line contains a breakdown of times, it" do
+    setup do
+      @line = "I, [2015-01-10T15:18:12.262903 #2354]  INFO -- : [livingsaviorco] [0fc5154a-c288-4bad-9c7a-de3d7e7d2496] Completed 200 OK in 196ms (Views: 0.1ms | ActiveRecord: 50.0ms)"
+    end
+    
+    should "identify the duration of the request" do
+      assert_parses duration: 196
+    end
+  end
+  
+  
+  
+private
+  
+  def assert_parses(expectations)
+    results = parse!
+    
+    expectations.each do |key, value|
+      assert_equal value, results[key]
+    end
+  end
+  
+  def parse!
+    parser.parse!(line)
+  end
+  
+  def parser
+    Logeater::Parser.new
+  end
+  
+end