logeater 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ce5032b80ebdd86abf84a53642d881c1e61315ac
+  data.tar.gz: 215a2ba9e8e0d90dd14daa6a7df4baa7c2295fe4
+SHA512:
+  metadata.gz: f0a033a7a9e51498b21728cca217978dcd1b1145cddf2a704386558496e01e4949d1e91c0bd20bd411f6540d46ad9edd929adfc25dff2f3e484c365a0672164e
+  data.tar.gz: bd808f7f9d91d28b09b5ccc0f2e552319edf0f3f2e9e1b64495fe2b71ac7e906acbf7daa74d37ebb333823a31604877d6e10f968c1dba17f0e194cfd14f25ea6

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.irbrc

@@ -0,0 +1,2 @@
+$:.push File.join(Dir.pwd, "lib"); require "logeater"
+

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in logeater.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Bob Lail
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,47 @@
+# Logeater
+
+Parses log files and imports them into a database
+
+### Features
+
+ - Can read from plain text or gzipped log files
+ - [Parses parameters safely](https://github.com/concordia-publishing-house/logeater/blob/master/lib/logeater/params_parser.rb)
+ - Can import a batch of files at once
+ - Parses [these attributes](https://github.com/concordia-publishing-house/logeater/blob/master/db/schema.rb#L19-L32) of requests
+
+
+### Usage
+
+Clone the gem
+
+    git clone git@github.com:concordia-publishing-house/logeater.git
+    bundle
+
+Create the development database
+
+    bundle exec rake db:create db:migrate
+
+Install the gem
+
+    bundle exec rake install
+
+Import log files
+
+    logeater my_app ~/Desktop/logs/*.gz
+
+
+### To Do
+
+ - Set up databases without cloning the gem?
+ - Import to a [Heroku Postgres database](https://dashboard.heroku.com/apps/logs-production)?
+ - Parse other kinds of logs?
+ - Collect other data from Rails logs?
+
+
+### Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/logeater/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am "Add some feature"`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,12 @@
+require "bundler/gem_tasks"
+
+require "standalone_migrations"
+StandaloneMigrations::Tasks.load_tasks
+
+require "rake/testtask"
+Rake::TestTask.new(:test) do |t|
+  t.libs << "lib"
+  t.libs << "test"
+  t.pattern = "test/**/*_test.rb"
+  t.verbose = false
+end

data/bin/logeater

@@ -0,0 +1,39 @@
+#!/usr/bin/env ruby
+
+require "logeater"
+
+app = ARGV.shift
+unless app
+  puts "USAGE: logeater <app> <files...>"
+  exit
+end
+
+files = ARGV
+if files.empty?
+  puts "USAGE: logeater <app> <files...>"
+  exit
+end
+
+started_all = Time.now
+files.each_with_index do |file, i|
+  reader = Logeater::Reader.new(app, file, progress: true)
+  reader.remove_existing_entries!
+  
+  started_count = Logeater::Request.count
+  started_at = Time.now
+  reader.import
+  finished_at = Time.now
+  finished_count = Logeater::Request.count
+  
+  puts " > \e[34mImported \e[1m%d\e[0;34m requests in \e[1m%.2f\e[0;34m seconds (%d of %d)\e[0m\n\n" % [
+    finished_count - started_count,
+    finished_at - started_at,
+    i + 1,
+    files.length ]
+end
+
+finished_all = Time.now
+seconds = finished_all - started_all
+minutes = (seconds / 60).to_i
+seconds -= (minutes * 60)
+puts "Total time %d minutes, %.2f seconds" % [minutes, seconds]

data/db/config.yml

@@ -0,0 +1,13 @@
+development:
+  adapter: postgresql
+  encoding: utf8
+  database: logs_development
+  host: localhost
+  min_messages: WARNING
+
+test:
+  adapter: postgresql
+  encoding: utf8
+  database: logs_test
+  host: localhost
+  min_messages: WARNING

data/db/migrate/20150110151439_create_requests.rb

@@ -0,0 +1,31 @@
+class CreateRequests < ActiveRecord::Migration
+  def change
+    create_table :requests do |t|
+      t.string :app, null: false
+      t.string :logfile, null: false
+      
+      t.string :uuid, null: false
+      t.string :subdomain
+      t.timestamp :started_at
+      t.timestamp :completed_at
+      t.integer :duration
+      t.string :http_method
+      t.string :path
+      t.text :params
+      t.string :controller
+      t.string :action
+      t.string :remote_ip
+      t.string :format
+      t.integer :http_status
+      t.string :http_response
+      
+      t.timestamps
+    end
+    
+    add_index :requests, :app
+    add_index :requests, :logfile
+    add_index :requests, :uuid, unique: true
+    add_index :requests, [:controller, :action]
+    add_index :requests, :http_status
+  end
+end

data/db/migrate/20150122021627_change_requests_params_to_json.rb

@@ -0,0 +1,9 @@
+class ChangeRequestsParamsToJson < ActiveRecord::Migration
+  def up
+    execute "alter table requests alter column params type json using params::json"
+  end
+
+  def down
+    change_column :requests, :params, :text
+  end
+end

data/db/schema.rb

@@ -0,0 +1,43 @@
+# encoding: UTF-8
+# This file is auto-generated from the current state of the database. Instead
+# of editing this file, please use the migrations feature of Active Record to
+# incrementally modify your database, and then regenerate this schema definition.
+#
+# Note that this schema.rb definition is the authoritative source for your
+# database schema. If you need to create the application database on another
+# system, you should be using db:schema:load, not running all the migrations
+# from scratch. The latter is a flawed and unsustainable approach (the more migrations
+# you'll amass, the slower it'll run and the greater likelihood for issues).
+#
+# It's strongly recommended to check this file into your version control system.
+
+ActiveRecord::Schema.define(:version => 20150122021627) do
+
+  create_table "requests", :force => true do |t|
+    t.string   "app",           :null => false
+    t.string   "logfile",       :null => false
+    t.string   "uuid",          :null => false
+    t.string   "subdomain"
+    t.datetime "started_at"
+    t.datetime "completed_at"
+    t.integer  "duration"
+    t.string   "http_method"
+    t.string   "path"
+    t.json     "params"
+    t.string   "controller"
+    t.string   "action"
+    t.string   "remote_ip"
+    t.string   "format"
+    t.integer  "http_status"
+    t.string   "http_response"
+    t.datetime "created_at",    :null => false
+    t.datetime "updated_at",    :null => false
+  end
+
+  add_index "requests", ["app"], :name => "index_requests_on_app"
+  add_index "requests", ["controller", "action"], :name => "index_requests_on_controller_and_action"
+  add_index "requests", ["http_status"], :name => "index_requests_on_http_status"
+  add_index "requests", ["logfile"], :name => "index_requests_on_logfile"
+  add_index "requests", ["uuid"], :name => "index_requests_on_uuid", :unique => true
+
+end

data/lib/logeater/params_parser.rb

@@ -0,0 +1,76 @@
+require "ripper"
+
+module Logeater
+  class ParamsParser
+    attr_reader :params
+    
+    def initialize(params)
+      @params = params
+    end
+    
+    def parse!
+      identify tokenize_hash(clean(params))
+    end
+    
+    def clean(params)
+      loop do
+        result = params.gsub(/\#<((?:[\w]|::)+):[^<>]+>/) { "\"#{$1}\"" }
+        break if result == params
+        params = result
+      end
+      params
+    end
+    
+  private
+    
+    def tokenize_hash(ruby)
+      sexp = Ripper.sexp(ruby)
+      raise Parser::MalformedParameters.new(ruby) unless sexp
+      
+      # [:program, [[:hash, ... ]]]
+      sexp[1][0]
+    end
+    
+    def identify(sexp)
+      case sexp[0]
+      
+      # [:string_literal, [:string_content, [:@tstring_content, "utf8", [1, 2]]]]
+      # [:string_literal, [:string_content]]
+      when :string_literal then sexp[1][1] ? sexp[1][1][1] : ""
+      
+      # [:@int, "10", [1, 14]]
+      when :@int then sexp[1].to_i
+        
+      # [:@float, "10.56", [1, 14]]
+      when :@float then sexp[1].to_f
+      
+      # [:var_ref, [:@kw, "true", [1, 12]]]
+      when :var_ref then
+        return true if sexp[1][1] == "true"
+        return false if sexp[1][1] == "false"
+        return nil if sexp[1][1] == "nil"
+        raise Parser::ParserNotImplemented, "Unknown variable: #{sexp[1].inspect}"
+      
+      # [:array, [[:@int, "1", [1, 9]], [:@int, "4", [1, 12]]]]
+      # [:array, nil]
+      when :array then sexp[1] ? sexp[1].map { |sexp| identify(sexp) } : []
+        
+      #  [:hash,
+      #    [:assoclist_from_args,
+      #     [[:assoc_new,
+      #       [:string_literal, [:string_content, [:@tstring_content, "utf8", [1, 2]]]],
+      #       [:string_literal, [:string_content, [:@tstring_content, "✓", [1, 12]]]]]]]]]
+      # [:hash, nil]
+      when :hash then sexp[1] ? sexp[1][1].each_with_object({}) { |(_, key, value), hash| hash[identify(key)] = identify(value) } : {}
+      
+      else
+        raise Parser::ParserNotImplemented, "I don't know how to identify #{sexp.inspect}"
+        nil
+      end
+    rescue
+      raise Parser::ParserNotImplemented, "An exception occurred when parsing #{sexp.inspect}\n#{$!.class.name}: #{$!.message}"
+      nil
+    end
+    
+  end
+end

data/lib/logeater/parser.rb

@@ -0,0 +1,161 @@
+require "addressable/uri"
+require "active_support/inflector"
+require "logeater/params_parser"
+require "logeater/parser_errors"
+
+module Logeater
+  class Parser
+    
+    
+    LINE_MATCHER = /^
+      [A-Z],\s
+    \[(?<timestamp>[^\s\]]+)(?:\s[^\]]*)?\]\s+
+      (?<log_level>[A-Z]+)\s+\-\-\s:\s+
+      (?<message>.*)
+    $/x.freeze
+    
+    TIMESTAMP_MATCHER = /
+      (?<year>\d\d\d\d)\-
+      (?<month>\d\d)\-
+      (?<day>\d\d)T
+      (?<hours>\d\d):
+      (?<minutes>\d\d):
+      (?<seconds>\d\d(?:\.\d+))
+    /x.freeze
+    
+    HTTP_VERBS = %w{DELETE GET HEAD OPTIONS PATCH POST PUT}.freeze
+    
+    REQUEST_LINE_MATCHER = /^
+    \[(?<subdomain>[^\]]+)\]\s
+    \[(?<uuid>[\w\-]{36})\]\s+
+      (?<message>.*)
+    $/x.freeze
+    
+    REQUEST_STARTED_MATCHER = /^
+      Started\s
+      (?<http_method>[A-Z]+)\s
+     "(?<path>[^"]+)"\sfor\s
+      (?<remote_ip>[\d\.]+)
+    /x.freeze
+    
+    REQUEST_CONTROLLER_MATCHER = /^
+      Processing\sby\s
+      (?<controller>[A-Za-z0-9:]+)\#
+      (?<action>[a-z_0-9]+)\sas\s
+      (?<format>.*)
+    /x.freeze
+    
+    REQUEST_PARAMETERS_MATCHER = /^
+      Parameters:\s
+      (?<params>\{.*\})
+    $/x.freeze
+    
+    REQUEST_COMPLETED_MATCHER = /^
+      Completed\s
+      (?<http_status>\d\d\d)\s
+      (?<http_response>.*)\sin\s
+      (?<duration>[\d\.]+)(?<units>ms)\b
+    /x.freeze # optional: (Views: 0.1ms | ActiveRecord: 50.0ms)
+    
+    def parse!(line)
+      match = line.match LINE_MATCHER
+      raise UnmatchedLine.new(line) unless match
+      
+      timestamp = match["timestamp"]
+      time = timestamp.match TIMESTAMP_MATCHER
+      raise MalformedTimestamp.new(timestamp) unless time
+      time = Time.new(*time.captures[0...-1], BigDecimal.new(time["seconds"]))
+      
+      message = match["message"]
+      
+      result = {
+        type: :generic,
+        timestamp: time,
+        log_level: match["log_level"],
+        message: message }
+      
+      result.merge(parse_message(message))
+    end
+    
+    def parse_message(message)
+      match = message.match REQUEST_LINE_MATCHER
+      return {} unless match
+      
+      message = match["message"]
+      type = identify_request_line_type(message)
+      
+      { subdomain: match["subdomain"],
+        uuid: match["uuid"],
+        type: type,
+        message: message }.merge(
+        custom_attributes_for(type, message))
+    end
+    
+    def identify_request_line_type(message)
+      return :request_started if message =~ /^Started (#{HTTP_VERBS.join("|")})/
+      return :request_controller if message.start_with? "Processing by "
+      return :request_params if message.start_with? "Parameters: "
+      return :request_completed if message =~ /^Completed \d\d\d/
+      :request_line
+    end
+    
+    def custom_attributes_for(type, message)
+      attributes = send :"parse_#{type}_message", message
+      unless attributes
+        log "Unable to parse message identified as #{type.inspect}: #{message.inspect}"
+        return {}
+      end
+      attributes
+    end
+    
+    def parse_request_line_message(message)
+      {}
+    end
+    
+    def parse_request_started_message(message)
+      match = message.match(REQUEST_STARTED_MATCHER)
+      return unless match
+      uri = Addressable::URI.parse(match["path"])
+      
+      { http_method: match["http_method"],
+        path: uri.path,
+        remote_ip: match["remote_ip"] }
+    end
+    
+    def parse_request_controller_message(message)
+      match = message.match(REQUEST_CONTROLLER_MATCHER)
+      return unless match
+      
+      { controller: match["controller"].underscore.gsub(/_controller$/, ""),
+        action: match["action"],
+        format: match["format"] }
+    end
+    
+    def parse_request_params_message(message)
+      match = message.match(REQUEST_PARAMETERS_MATCHER)
+      return unless match
+      params = ParamsParser.new(match["params"])
+      
+      { params: params.parse! }
+    rescue Logeater::Parser::MalformedParameters
+      log "Unable to parse parameters: #{match["params"].inspect}"
+      { params: match["params"] }
+    end
+    
+    def parse_request_completed_message(message)
+      match = message.match(REQUEST_COMPLETED_MATCHER)
+      return unless match
+      
+      { http_status: match["http_status"].to_i,
+        http_response: match["http_response"],
+        duration: match["duration"].to_i }
+    end
+    
+    
+    
+    def log(statement)
+      $stderr.puts "\e[33m#{statement}\e[0m"
+    end
+    
+  end
+end

data/lib/logeater/parser_errors.rb

@@ -0,0 +1,34 @@
+module Logeater
+  class Parser
+    
+    class Error < ::ArgumentError
+      def initialize(message, input)
+        super "#{message}: #{input.inspect}"
+      end
+    end
+    
+    class UnmatchedLine < Error
+      def initialize(input)
+        super "Unmatched line", input
+      end
+    end
+    
+    class MalformedTimestamp < Error
+      def initialize(input)
+        super "Malformed timestamp", input
+      end
+    end
+    
+    class MalformedParameters < Error
+      def initialize(input)
+        super "Malformed parameters", input
+      end
+    end
+    
+    class ParserNotImplemented < Error
+      def initialize(input)
+        super "Unable to parse", input
+      end
+    end
+  end
+end

data/lib/logeater/reader.rb

@@ -0,0 +1,117 @@
+require "logeater/request"
+require "zlib"
+require "ruby-progressbar"
+
+module Logeater
+  class Reader
+    attr_reader :app, :path, :filename, :batch_size
+    
+    def initialize(app, path, options={})
+      @app = app
+      @path = path
+      @filename = File.basename(path)
+      @parser = Logeater::Parser.new
+      @show_progress = options.fetch :progress, false
+      @batch_size = options.fetch :batch_size, 500
+      @verbose = options.fetch :verbose, false
+      @requests = {}
+      @completed_requests = []
+    end
+    
+    
+    
+    def reimport
+      remove_existing_entries!
+      import
+    end
+      
+    def import
+      each_line(&method(:process_line!))
+      save!
+    end
+    
+    def remove_existing_entries!
+      Logeater::Request.where(app: app, logfile: filename).delete_all
+    end
+    
+    def show_progress?
+      @show_progress
+    end
+    
+    def verbose?
+      @verbose
+    end
+    
+    def each_line
+      File.open(path) do |file|
+        io = File.extname(path) == ".gz" ? Zlib::GzipReader.new(file) : file
+        pbar = ProgressBar.create(title: filename, total: file.size, autofinish: false) if show_progress?
+        io.each_line do |line|
+          yield line
+          pbar.progress = file.pos if show_progress?
+        end
+        pbar.finish if show_progress?
+      end
+    end
+    alias :scan :each_line
+    
+    
+  private
+    attr_reader :parser, :requests, :completed_requests
+    
+    def process_line!(line)
+      attributes = parser.parse!(line)
+      
+      return if [:generic, :request_line].member? attributes[:type]
+      
+      if attributes[:type] == :request_started
+        requests[attributes[:uuid]] = attributes
+          .slice(:uuid, :subdomain, :http_method, :path, :remote_ip)
+          .merge(started_at: attributes[:timestamp], logfile: filename, app: app)
+        return
+      end
+      
+      request_attributes = requests[attributes[:uuid]]
+      unless request_attributes
+        log "Attempting to record #{attributes[:type].inspect}; but there is no request started with UUID #{attributes[:uuid].inspect}"
+        return
+      end
+      
+      case attributes[:type]
+      when :request_controller
+        request_attributes.merge! attributes.slice(:controller, :action, :format)
+      
+      when :request_params
+        request_attributes.merge! attributes.slice(:params)
+      
+      when :request_completed
+        request_attributes.merge! attributes
+          .slice(:http_status, :http_response, :duration)
+          .merge(completed_at: attributes[:timestamp])
+        
+        completed_requests.push Logeater::Request.new(request_attributes)
+        requests.delete attributes[:uuid]
+        
+        save! if completed_requests.length >= batch_size
+      end
+      
+    rescue Logeater::Parser::UnmatchedLine
+      $stderr.puts "\e[90m#{$!.message}\e[0m" if verbose?
+    rescue Logeater::Parser::Error
+      log $!.message
+    end
+    
+    def save!
+      return if completed_requests.empty?
+      Logeater::Request.import(completed_requests)
+      completed_requests.clear
+    end
+    
+    
+    
+    def log(statement)
+      $stderr.puts "\e[33m#{statement}\e[0m"
+    end
+    
+  end
+end

data/lib/logeater/request.rb

@@ -0,0 +1,14 @@
+require "active_record"
+require "activerecord-import"
+require "activerecord-postgres-json"
+
+module Logeater
+  class Request < ActiveRecord::Base
+    self.table_name = "requests"
+    
+    serialize :params, ActiveRecord::Coders::JSON
+    
+    
+    
+  end
+end

data/lib/logeater/version.rb

@@ -0,0 +1,3 @@
+module Logeater
+  VERSION = "0.1.1"
+end

data/lib/logeater.rb

@@ -0,0 +1,13 @@
+require "logeater/parser"
+require "logeater/reader"
+require "logeater/request"
+require "logeater/version"
+require "yaml"
+require "erb"
+
+config_file = File.expand_path("../../db/config.yml", __FILE__)
+config = YAML.load(ERB.new(File.read(config_file)).result).with_indifferent_access
+ActiveRecord::Base.establish_connection config[ENV["RAILS_ENV"] || "development"]
+
+module Logeater
+end