logeater 0.4.0 → 0.5.0.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 35ee92c429701edcd6d744f7301a94e64431d612
-  data.tar.gz: 47e6f5817105c8f74c92b901a57c3bea53ca8f5e
+  metadata.gz: 4ccc1408222d5eb6f6af30a91d8cb63c51457c6a
+  data.tar.gz: 83254c545dba4b4217510653a6e1f54c95ebf8bf
 SHA512:
-  metadata.gz: 82025a9438a23b5337df5813c1b3f51a1ea9dcc76734d383f446d1499b44622c28411eab6705349cb77f06281c4eab4bbf0f8dc907e0574c4237a43c335eb204
-  data.tar.gz: 278d438799e865804d80fe266003b74ed5b2200ea036022cc1711bd1eb829622b824c4b3cca15a06d2d1120651852fe3fecd98351edb3ef6ee34afbcfca9fd5e
+  metadata.gz: adc84e2364b1cb49120f206e21e007c6e0595567372c69cae4448a7bcaf360f17f23f0482de706f5f82b42db8d9870188c94a5b94a947ef460cb678342c6a748
+  data.tar.gz: 06481c37537675e13b0f17cfd78b0004459a560994f6aed86ef62c0cff592ae7cf94e77de93a521bba6ee02a076bfb1f882ba474d02372ef42c900f897ec8183

data/db/migrate/20171109171953_create_events.rb

@@ -0,0 +1,26 @@
+class CreateEvents < ActiveRecord::Migration
+  def up
+    create_table :events do |t|
+      t.timestamp :emitted_at
+      t.timestamp :received_at
+      t.integer :priority
+      t.integer :syslog_version
+      t.string :hostname
+      t.string :appname
+      t.string :proc_id
+      t.string :msg_id
+      t.text :structured_data
+      t.text :message
+      t.text :original
+      t.string :ep_app
+    end
+
+    add_index :events, :ep_app
+    execute "ALTER TABLE events ALTER COLUMN received_at SET DEFAULT now()"
+  end
+
+  def down
+    remove_index :events, :ep_app
+    drop_table :events
+  end
+end

data/db/migrate/20180116193728_change_user_id_to_be_string_in_requests.rb

@@ -0,0 +1,9 @@
+class ChangeUserIdToBeStringInRequests < ActiveRecord::Migration[5.0]
+  def up
+    execute "alter table requests alter column user_id type varchar"
+  end
+
+  def down
+    execute "alter table requests alter column user_id type integer"
+  end
+end

data/db/schema.rb

@@ -1,4 +1,3 @@
-# encoding: UTF-8
 # This file is auto-generated from the current state of the database. Instead
 # of editing this file, please use the migrations feature of Active Record to
 # incrementally modify your database, and then regenerate this schema definition.
@@ -11,11 +10,27 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20160502131002) do
+ActiveRecord::Schema.define(version: 20180116193728) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
+  create_table "events", force: :cascade do |t|
+    t.datetime "emitted_at"
+    t.datetime "received_at",     default: -> { "now()" }
+    t.integer  "priority"
+    t.integer  "syslog_version"
+    t.string   "hostname"
+    t.string   "appname"
+    t.string   "proc_id"
+    t.string   "msg_id"
+    t.text     "structured_data"
+    t.text     "message"
+    t.text     "original"
+    t.string   "ep_app"
+    t.index ["ep_app"], name: "index_events_on_ep_app", using: :btree
+  end
+
   create_table "requests", force: :cascade do |t|
     t.string   "app",          null: false
     t.string   "logfile",      null: false
@@ -34,17 +49,16 @@ ActiveRecord::Schema.define(version: 20160502131002) do
     t.integer  "http_status"
     t.datetime "created_at"
     t.datetime "updated_at"
-    t.integer  "user_id"
+    t.string   "user_id"
     t.boolean  "tester_bar"
+    t.index ["app"], name: "index_requests_on_app", using: :btree
+    t.index ["completed_at"], name: "index_requests_on_completed_at", using: :btree
+    t.index ["controller", "action"], name: "index_requests_on_controller_and_action", using: :btree
+    t.index ["http_status"], name: "index_requests_on_http_status", using: :btree
+    t.index ["logfile"], name: "index_requests_on_logfile", using: :btree
+    t.index ["params"], name: "index_requests_on_params", using: :gin
+    t.index ["subdomain"], name: "index_requests_on_subdomain", using: :btree
+    t.index ["uuid"], name: "index_requests_on_uuid", unique: true, using: :btree
   end
 
-  add_index "requests", ["app"], name: "index_requests_on_app", using: :btree
-  add_index "requests", ["completed_at"], name: "index_requests_on_completed_at", using: :btree
-  add_index "requests", ["controller", "action"], name: "index_requests_on_controller_and_action", using: :btree
-  add_index "requests", ["http_status"], name: "index_requests_on_http_status", using: :btree
-  add_index "requests", ["logfile"], name: "index_requests_on_logfile", using: :btree
-  add_index "requests", ["params"], name: "index_requests_on_params", using: :gin
-  add_index "requests", ["subdomain"], name: "index_requests_on_subdomain", using: :btree
-  add_index "requests", ["uuid"], name: "index_requests_on_uuid", unique: true, using: :btree
-
 end

data/lib/logeater/cli.rb

@@ -13,7 +13,7 @@ module Logeater
 
       started_all = Time.now
       files.each_with_index do |file, i|
-        reader = Logeater::Reader.new(app, file, progress: true)
+        reader = Logeater::Reader.new(app, Logeater::Logfile.new(file), progress: true)
 
         started_at = Time.now
         requests = reader.parse
@@ -63,24 +63,23 @@ module Logeater
           i + 1,
           files.length ]
 
-        reader = Logeater::Reader.new(app, file, options.slice(:progress, :verbose))
-
-        started_at = Time.now
-        count = reader.remove_existing_entries!
-        finished_at = Time.now
+        import_file Logeater::Logfile.new(file), app, options
+      end
 
-        $stderr.puts "   \e[34mDeleted \e[1m%d\e[0;34m requests for #{reader.filename} in \e[1m%.2f\e[0;34m seconds\e[0m\n" % [
-          count,
-          finished_at - started_at ]
+      finished_all = Time.now
+      seconds = finished_all - started_all
+      minutes = (seconds / 60).to_i
+      seconds -= (minutes * 60)
+      $stderr.puts "Total time %d minutes, %.2f seconds" % [minutes, seconds]
+    end
 
-        started_at = Time.now
-        count = reader.import
-        finished_at = Time.now
+    desc "import_since APP TIMESTAMP", "imports log events since timestamp"
+    def import_since(app, timestamp)
+      started_all = Time.now
+      $stderr.puts " > \e[34mImporting events since \e[1m#{timestamp}\e[0m\n"
 
-        $stderr.puts "   \e[34mImported \e[1m%d\e[0;34m requests in \e[1m%.2f\e[0;34m seconds\e[0m\n\n" % [
-          count,
-          finished_at - started_at ]
-      end
+      events = Logeater::Event.where(app: app).since(timestamp)
+      import_file Logeater::Eventfile.new(events), app
 
       finished_all = Time.now
       seconds = finished_all - started_all
@@ -89,5 +88,26 @@ module Logeater
       $stderr.puts "Total time %d minutes, %.2f seconds" % [minutes, seconds]
     end
 
+
+    def import_file(file, app, options={})
+      reader = Logeater::Reader.new(app, file, options.slice(:progress, :verbose))
+
+      started_at = Time.now
+      count = reader.remove_existing_entries!
+      finished_at = Time.now
+
+      $stderr.puts "   \e[34mDeleted \e[1m%d\e[0;34m requests for #{reader.filename} in \e[1m%.2f\e[0;34m seconds\e[0m\n" % [
+        count,
+        finished_at - started_at ]
+
+      started_at = Time.now
+      count = reader.import
+      finished_at = Time.now
+
+      $stderr.puts "   \e[34mImported \e[1m%d\e[0;34m requests in \e[1m%.2f\e[0;34m seconds\e[0m\n\n" % [
+        count,
+        finished_at - started_at ]
+    end
+
   end
 end

data/lib/logeater/event.rb

@@ -0,0 +1,11 @@
+require "active_record"
+
+module Logeater
+  class Event < ActiveRecord::Base
+    self.table_name = "events"
+
+    def self.since(timestamp)
+      where(arel_table[:received_at].gteq(timestamp))
+    end
+  end
+end

data/lib/logeater/eventfile.rb

@@ -0,0 +1,19 @@
+module Logeater
+  class Eventfile
+    attr_reader :events, :filename
+    attr_accessor :show_progress
+    alias :show_progress? :show_progress
+
+    def initialize(events)
+      @events = events
+      @filename = "events_#{events.count}"
+    end
+
+    def each_line
+      events.pluck(:original).each do |event|
+        yield event
+      end
+    end
+
+  end
+end

data/lib/logeater/logfile.rb

@@ -0,0 +1,25 @@
+module Logeater
+  class Logfile
+    attr_reader :path, :filename
+    attr_accessor :show_progress
+    alias :show_progress? :show_progress
+
+    def initialize(path)
+      @path = path
+      @filename = File.basename(path)
+    end
+
+    def each_line
+      File.open(path) do |file|
+        io = File.extname(path) == ".gz" ? Zlib::GzipReader.new(file) : file
+        pbar = ProgressBar.create(title: filename, total: file.size, autofinish: false, output: $stderr) if show_progress?
+        io.each_line do |line|
+          yield line
+          pbar.progress = file.pos if show_progress?
+        end
+        pbar.finish if show_progress?
+      end
+    end
+
+  end
+end

data/lib/logeater/parser.rb

@@ -17,7 +17,7 @@ module Logeater
     REQUEST_LINE_MATCHER = /^
     \[(?<subdomain>[^\]]+)\]\s
     \[(?<uuid>[\w\-]{36})\]\s+
- (?:\[(?:guest|user\.(?<user_id>\d+)(?<tester_bar>:cph)?)\]\s+)?
+ (?:\[(?:guest|user\.(?<user_id>\d+)(?<tester_bar>:cph)?|(?<email>.+@[^\]]+))\]\s+)?
       (?<message>.*)
     $/x.freeze
 
@@ -69,7 +69,7 @@ module Logeater
       { subdomain: match["subdomain"],
         uuid: match["uuid"],
         type: :request_line,
-        user_id: match["user_id"] && match["user_id"].to_i,
+        user_id: user_id_for(match),
         tester_bar: !!match["tester_bar"],
         message: message
       }.merge(
@@ -121,6 +121,12 @@ module Logeater
         duration: match["duration"].to_i }
     end
 
+    def user_id_for(match)
+      return match["user_id"].to_i if match["user_id"]
+      return match["email"] if match["email"]
+      nil
+    end
+
 
 
     def log(statement)

data/lib/logeater/reader.rb

@@ -5,14 +5,13 @@ require "oj"
 
 module Logeater
   class Reader
-    attr_reader :app, :path, :filename, :batch_size
+    attr_reader :app, :file, :batch_size
 
-    def initialize(app, path, options={})
+    def initialize(app, file, options={})
       @app = app
-      @path = path
-      @filename = File.basename(path)
+      @file = file
+      @file.show_progress = options.fetch :progress, false
       @parser = Logeater::Parser.new
-      @show_progress = options.fetch :progress, false
       @batch_size = options.fetch :batch_size, 500
       @verbose = options.fetch :verbose, false
       @count = 0
@@ -54,33 +53,16 @@ module Logeater
     end
 
     def remove_existing_entries!
-      Logeater::Request.where(app: app, logfile: filename).delete_all
-    end
-
-    def show_progress?
-      @show_progress
+      Logeater::Request.where(app: app, logfile: file.filename).delete_all
     end
 
     def verbose?
       @verbose
     end
 
-    def each_line
-      File.open(path) do |file|
-        io = File.extname(path) == ".gz" ? Zlib::GzipReader.new(file) : file
-        pbar = ProgressBar.create(title: filename, total: file.size, autofinish: false, output: $stderr) if show_progress?
-        io.each_line do |line|
-          yield line
-          pbar.progress = file.pos if show_progress?
-        end
-        pbar.finish if show_progress?
-      end
-    end
-    alias :scan :each_line
-
     def each_request
       count = 0
-      each_line do |line|
+      file.each_line do |line|
         process_line! line do |request|
           yield request
           count += 1
@@ -102,7 +84,7 @@ module Logeater
       if attributes[:type] == :request_started
         requests[attributes[:uuid]] = attributes
           .slice(:uuid, :subdomain, :http_method, :path, :remote_ip, :user_id, :tester_bar)
-          .merge(started_at: attributes[:timestamp], logfile: filename, app: app)
+          .merge(started_at: attributes[:timestamp], logfile: file.filename, app: app)
         return
       end
 

data/lib/logeater/request.rb

@@ -6,14 +6,9 @@ module Logeater
     self.table_name = "requests"
 
     def self.import(values)
-      # values have to be unique by uuid
-      values = values.uniq { |request| request.uuid }
-
-      super values
-    rescue PG::UniqueViolation, ActiveRecord::RecordNotUnique
-      # try again after skipping requests with existing uuids
+      # values have to be unique by uuid so we pull already saved ones out before trying to insert
       existing_uuids = where(uuid: values.map(&:uuid)).pluck(:uuid)
-      values = values.reject { |request| existing_uuids.member?(request.uuid) }
+      values = values.uniq(&:uuid).reject { |request| existing_uuids.member?(request.uuid) }
 
       super values
     end

data/lib/logeater/version.rb

@@ -1,3 +1,3 @@
 module Logeater
-  VERSION = "0.4.0"
+  VERSION = "0.5.0.pre"
 end

data/lib/logeater.rb

@@ -1,6 +1,9 @@
 require "logeater/parser"
 require "logeater/reader"
 require "logeater/request"
+require "logeater/event"
+require "logeater/logfile"
+require "logeater/eventfile"
 require "logeater/version"
 require "yaml"
 require "erb"

data/logeater.gemspec

@@ -20,15 +20,13 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "activerecord", ">= 4.2.7", "< 5.1.0"
   spec.add_dependency "activesupport", ">= 4.2.7", "< 5.1.0"
-  # spec.add_dependency "activerecord", "~> 5.0.0"
-  # spec.add_dependency "activesupport", "~> 5.0.0"
-  spec.add_dependency "pg", "~> 0.18.3"
+  spec.add_dependency "pg", ">= 0.18.0", "< 1.0"
   spec.add_dependency "standalone_migrations", "~> 5.0.0"
   spec.add_dependency "addressable"
   spec.add_dependency "ruby-progressbar"
   spec.add_dependency "activerecord-import", "~> 0.10"
   spec.add_dependency "thor", "~> 0.19.1"
-  spec.add_dependency "oj", "~> 2.15"
+  spec.add_dependency "oj", ">= 2.15"
 
   spec.add_development_dependency "bundler", "~> 1.7"
   spec.add_development_dependency "rake", "~> 10.0"

data/test/integration/logeater_test.rb

@@ -1,7 +1,7 @@
 require "test_helper"
 
 class LogeaterTest < ActiveSupport::TestCase
-  attr_reader :logfile
+  attr_reader :logfile, :events
 
 
   context "Given the log of a single request, it" do
@@ -10,17 +10,17 @@ class LogeaterTest < ActiveSupport::TestCase
     end
 
     should "identify the name of the logfile" do
-      assert_equal "single_request.log", reader.filename
+      assert_equal "single_request.log", logfile_reader.file.filename
     end
 
     should "create an entry in the database" do
       assert_difference "Logeater::Request.count", +1 do
-        reader.import
+        logfile_reader.import
       end
     end
 
     should "set all the attributes" do
-      reader.import
+      logfile_reader.import
       request = Logeater::Request.first
 
       params = {"refresh_page" => "true", "id" => "1035826228"}
@@ -47,7 +47,7 @@ class LogeaterTest < ActiveSupport::TestCase
       Logeater::Request.create!(app: app, logfile: "single_request.log", uuid: "3")
 
       assert_difference "Logeater::Request.count", -2 do
-        reader.reimport
+        logfile_reader.reimport
       end
     end
   end
@@ -60,7 +60,62 @@ class LogeaterTest < ActiveSupport::TestCase
 
     should "create an entry in the database" do
       assert_difference "Logeater::Request.count", +1 do
-        reader.import
+        logfile_reader.import
+      end
+    end
+  end
+
+
+  context "Given an app and a timestamp, import_since" do
+    setup do
+      log_sample = File.open(File.expand_path("./test/data/single_request.log"))
+      log_sample.lines do |line|
+        Logeater::Event.create(ep_app: app, original: line)
+      end
+      @events = Logeater::Event.all
+    end
+
+    should "import events since that given timestamp" do
+      assert_difference "Logeater::Request.count", +1 do
+        eventfile_reader.import
+      end
+    end
+
+    should "not reimport events if given twice" do
+      assert_difference "Logeater::Request.count", +1 do
+        eventfile_reader.import
+      end
+
+      assert_no_difference "Logeater::Request.count" do
+        eventfile_reader.import
+      end
+    end
+  end
+
+  context "Given a partial request in one import and the rest in a subsiquent import, it" do
+    setup do
+      @lines = File.open(File.expand_path("./test/data/single_request.log")).lines.to_a
+    end
+
+    should "save the request after having the full request" do
+      # The first two lines will not be enough to describe a complete request
+      # so Logeater will not be able to create a request from them...
+      @lines[0...2].each do |line|
+        Logeater::Event.create(ep_app: app, original: line)
+      end
+      @events = Logeater::Event.all
+      assert_no_difference "Logeater::Request.count" do
+        eventfile_reader.import
+      end
+
+      # ...but if we later discover the rest of the lines that describe a complete
+      # request, it'd be good if Logeater could then recognize and import it.
+      @lines[2..-1].each do |line|
+        Logeater::Event.create(ep_app: app, original: line)
+      end
+      @events = Logeater::Event.all
+      assert_difference "Logeater::Request.count", +1 do
+        eventfile_reader.import
       end
     end
   end
@@ -72,8 +127,12 @@ private
     "test"
   end
 
-  def reader
-    Logeater::Reader.new(app, logfile)
+  def logfile_reader
+    Logeater::Reader.new(app, Logeater::Logfile.new(logfile))
+  end
+
+  def eventfile_reader
+    Logeater::Reader.new(app, Logeater::Eventfile.new(events))
   end
 
 end

data/test/unit/parser_test.rb

@@ -104,11 +104,29 @@ class ParserTest < ActiveSupport::TestCase
       end
     end
 
+    context "that indicates the current user via email, it" do
+      setup do
+        @line = "I, [2015-01-10T15:18:05.850839 #18070]  INFO -- : [livingsaviorco] [2d89d962-57c4-47c9-a9e9-6a16a5f22a12] [example@user.com] [gzip] Compress reponse by 42.2 KB (83.3%)  (1.4ms)"
+      end
+
+      should "identify the user's email" do
+        assert_parses user_id: "example@user.com"
+      end
+
+      should "identify the remainder of the log message" do
+        assert_parses message: "[gzip] Compress reponse by 42.2 KB (83.3%)  (1.4ms)"
+      end
+    end
+
     context "that indicates that the user is logged out, it" do
       setup do
         @line = "I, [2015-01-10T15:18:05.850839 #18070]  INFO -- : [livingsaviorco] [2d89d962-57c4-47c9-a9e9-6a16a5f22a12] [guest] [gzip] Compress reponse by 42.2 KB (83.3%)  (1.4ms)"
       end
 
+      should "identify the user as a guest" do
+        assert_parses user_id: nil
+      end
+
       should "identify the remainder of the log message" do
         assert_parses message: "[gzip] Compress reponse by 42.2 KB (83.3%)  (1.4ms)"
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logeater
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0.pre
 platform: ruby
 authors:
 - Bob Lail
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-15 00:00:00.000000000 Z
+date: 2018-01-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -54,16 +54,22 @@ dependencies:
   name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.18.0
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.18.3
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.18.0
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.18.3
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: standalone_migrations
   requirement: !ruby/object:Gem::Requirement
@@ -138,14 +144,14 @@ dependencies:
   name: oj
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '2.15'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '2.15'
 - !ruby/object:Gem::Dependency
@@ -269,9 +275,14 @@ files:
 - db/migrate/20150602022241_add_user_id_and_tester_bar_to_requests.rb
 - db/migrate/20160218230227_add_indexes.rb
 - db/migrate/20160502131002_drop_http_response.rb
+- db/migrate/20171109171953_create_events.rb
+- db/migrate/20180116193728_change_user_id_to_be_string_in_requests.rb
 - db/schema.rb
 - lib/logeater.rb
 - lib/logeater/cli.rb
+- lib/logeater/event.rb
+- lib/logeater/eventfile.rb
+- lib/logeater/logfile.rb
 - lib/logeater/params_parser.rb
 - lib/logeater/parser.rb
 - lib/logeater/parser_errors.rb
@@ -300,12 +311,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: Parses log files and imports them into a database