logeater 0.5.0.pre → 0.5.4.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 4ccc1408222d5eb6f6af30a91d8cb63c51457c6a
-  data.tar.gz: 83254c545dba4b4217510653a6e1f54c95ebf8bf
+SHA256:
+  metadata.gz: cce28ddbf1036163a0b743d2bbc4ba5e93ce484015149e746a7542ab2ff61e94
+  data.tar.gz: 3260cbd40856778f7f28ca9ea0708b87bb35e74f4de8598cd173acc7fef5a2c5
 SHA512:
-  metadata.gz: adc84e2364b1cb49120f206e21e007c6e0595567372c69cae4448a7bcaf360f17f23f0482de706f5f82b42db8d9870188c94a5b94a947ef460cb678342c6a748
-  data.tar.gz: 06481c37537675e13b0f17cfd78b0004459a560994f6aed86ef62c0cff592ae7cf94e77de93a521bba6ee02a076bfb1f882ba474d02372ef42c900f897ec8183
+  metadata.gz: d960f461dcb370a50ab96083e2fae80eae3e24ee39870dbf5d82aed2a78197e87b59d21d307b1830cde3145a6d1c9916cdb747676f711df4aa20cd917b96415c
+  data.tar.gz: 751dc4a8370d1e441aece26e50ed1da36760bdc9501e83729dcd474e0af1be23b5181f260a68ff6b2ca889d8f238ab31e2276806b52b708a977c73c8fe378221

data/lib/logeater/cli.rb

@@ -78,7 +78,23 @@ module Logeater
       started_all = Time.now
       $stderr.puts " > \e[34mImporting events since \e[1m#{timestamp}\e[0m\n"
 
-      events = Logeater::Event.where(app: app).since(timestamp)
+      events = Logeater::Event.where(ep_app: app).since(timestamp)
+      import_file Logeater::Eventfile.new(events), app
+
+      finished_all = Time.now
+      seconds = finished_all - started_all
+      minutes = (seconds / 60).to_i
+      seconds -= (minutes * 60)
+      $stderr.puts "Total time %d minutes, %.2f seconds" % [minutes, seconds]
+    end
+
+
+    desc "import_new APP", "imports unimported log events"
+    def import_new(app)
+      started_all = Time.now
+      $stderr.puts " > \e[34mImporting unprocessed events for \e[1m#{app}\e[0m\n"
+
+      events = Logeater::Event.where(ep_app: app).unprocessed
       import_file Logeater::Eventfile.new(events), app
 
       finished_all = Time.now

data/lib/logeater/event.rb

@@ -7,5 +7,55 @@ module Logeater
     def self.since(timestamp)
       where(arel_table[:received_at].gteq(timestamp))
     end
+
+    def self.processed
+      where.not(processed_at: nil)
+    end
+
+    def self.unprocessed
+      where(processed_at: nil)
+    end
+
+    def logger_line
+      "#{severity_id}, [#{timestamp}]  #{severity_label} -- : #{message}"
+    end
+
+    def severity_id
+      SEVERITY[log_level][:id]
+    end
+
+    def severity_label
+      SEVERITY[log_level][:label]
+    end
+
+    def timestamp
+      emitted_at.strftime("%FT%T.%6N")
+    end
+
+  private
+
+    def log_level
+      LOG_LEVELS.fetch(priority % 8)
+    end
+
+    LOG_LEVELS = {
+      0 => Logger::FATAL,   # Emergency: system is unusable
+      1 => Logger::FATAL,   # Alert: action must be taken immediately
+      2 => Logger::FATAL,   # Critical: critical conditions
+      3 => Logger::ERROR,   # Error: error conditions
+      4 => Logger::WARN,    # Warning: warning conditions
+      5 => Logger::INFO,    # Notice: normal but significant condition
+      6 => Logger::INFO,    # Informational: informational messages
+      7 => Logger::DEBUG    # Debug: debug-level messages
+    }.freeze
+
+    SEVERITY = {
+      Logger::FATAL => { id: "F", label: "FATAL" },
+      Logger::ERROR => { id: "E", label: "ERROR" },
+      Logger::WARN => { id: "W", label: "WARN" },
+      Logger::INFO => { id: "I", label: "INFO" },
+      Logger::DEBUG => { id: "D", label: "DEBUG" }
+    }.freeze
+
   end
 end

data/lib/logeater/eventfile.rb

@@ -6,14 +6,21 @@ module Logeater
 
     def initialize(events)
       @events = events
-      @filename = "events_#{events.count}"
+      @filename = "events_#{timestamp}"
     end
 
     def each_line
-      events.pluck(:original).each do |event|
-        yield event
+      events.find_each do |event|
+        yield event.logger_line
+        event.touch :processed_at
       end
     end
 
+  private
+
+    def timestamp
+      @timestamp ||= Time.now.strftime "%Y%m%d%H%M%S.%L"
+    end
+
   end
 end

data/lib/logeater/reader.rb

@@ -71,6 +71,10 @@ module Logeater
       count
     end
 
+    def filename
+      file.filename
+    end
+
 
 
   private

data/lib/logeater/version.rb

@@ -1,3 +1,3 @@
 module Logeater
-  VERSION = "0.5.0.pre"
+  VERSION = "0.5.4.pre"
 end

data/logeater.gemspec

@@ -35,5 +35,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "minitest-reporters"
   spec.add_development_dependency "minitest-reporters-turn_reporter"
   spec.add_development_dependency "database_cleaner"
+  spec.add_development_dependency "heroku-log-parser"
 
 end

data/test/data/single_heroku_request.log

@@ -0,0 +1,15 @@
+243 <190>1 2018-04-06T19:14:09.352835+00:00 host app web.2 - [www] [a0fe6527-2573-4c84-b3b7-6390939bc14e] [user@example.com] Started GET "/api/v1/holidays/2018-05-01-6189112395f7444881844271027c8c78" for 207.141.132.3 at 2018-04-06 19:14:09 +0000
+179 <190>1 2018-04-06T19:14:09.354663+00:00 host app web.2 - [www] [a0fe6527-2573-4c84-b3b7-6390939bc14e] [user@example.com] Processing by BuilderApi::HolidaysController#show as JSON
+222 <190>1 2018-04-06T19:14:09.354762+00:00 host app web.2 - [www] [a0fe6527-2573-4c84-b3b7-6390939bc14e] [user@example.com]   Parameters: {"year"=>"2018", "month"=>"05", "day"=>"01", "id"=>"6189112395f7444881844271027c8c78"}
+300 <190>1 2018-04-06T19:14:09.358456+00:00 host app web.2 - [www] [a0fe6527-2573-4c84-b3b7-6390939bc14e] [user@example.com]   \x1B[1m\x1B[36mUser Load (1.5ms)\x1B[0m  \x1B[1m\x1B[34mSELECT  "users".* FROM "users" WHERE "users"."email" = $1 LIMIT $2\x1B[0m  [["email", "user@example.com"], ["LIMIT", 1]]
+424 <190>1 2018-04-06T19:14:09.360877+00:00 host app web.2 - [www] [a0fe6527-2573-4c84-b3b7-6390939bc14e] [user@example.com]   \x1B[1m\x1B[36mUserRole Load (1.4ms)\x1B[0m  \x1B[1m\x1B[34mSELECT  "user_roles".* FROM "user_roles" WHERE "user_roles"."user_id" = $1 AND "user_roles"."tenant_id" = $2 LIMIT $3\x1B[0m  [["user_id", "ed80a088ddc04be891131be2394f56f5"], ["tenant_id", "9324b816dbc24bcebc087feef8c0b21e"], ["LIMIT", 1]]
+288 <190>1 2018-04-06T19:14:09.363353+00:00 host app web.2 - [www] [a0fe6527-2573-4c84-b3b7-6390939bc14e] [user@example.com]   \x1B[1m\x1B[36mPublisher Load (1.2ms)\x1B[0m  \x1B[1m\x1B[34mSELECT  "publishers".* FROM "publishers" ORDER BY "publishers"."id" ASC LIMIT $1\x1B[0m  [["LIMIT", 1]]
+657 <190>1 2018-04-06T19:14:09.369707+00:00 host app web.2 - [www] [a0fe6527-2573-4c84-b3b7-6390939bc14e] [user@example.com]   \x1B[1m\x1B[36mHoliday Load (1.7ms)\x1B[0m  \x1B[1m\x1B[34mSELECT  "holidays".* FROM "holidays" INNER JOIN "lectionaries" ON "holidays"."lectionary_id" = "lectionaries"."id" LEFT OUTER JOIN "categories" ON "categories"."id" = "holidays"."category_id" WHERE "lectionaries"."publisher_id" = $1 AND "holidays"."id" = $2 ORDER BY "lectionaries"."name" ASC, "categories"."position", "holidays"."position" ASC LIMIT $3\x1B[0m  [["publisher_id", "1211d48bb65e420f84a4174918638072"], ["id", "6189112395f7444881844271027c8c78"], ["LIMIT", 1]]
+318 <190>1 2018-04-06T19:14:09.373153+00:00 host app web.2 - [www] [a0fe6527-2573-4c84-b3b7-6390939bc14e] [user@example.com]   \x1B[1m\x1B[36mTenant Load (1.2ms)\x1B[0m  \x1B[1m\x1B[34mSELECT  "tenants".* FROM "tenants" WHERE "tenants"."id" = $1 LIMIT $2\x1B[0m  [["id", "9324b816dbc24bcebc087feef8c0b21e"], ["LIMIT", 1]]
+330 <190>1 2018-04-06T19:14:09.375802+00:00 host app web.2 - [www] [a0fe6527-2573-4c84-b3b7-6390939bc14e] [user@example.com]   \x1B[1m\x1B[36mPublisher Load (1.2ms)\x1B[0m  \x1B[1m\x1B[34mSELECT  "publishers".* FROM "publishers" WHERE "publishers"."id" = $1 LIMIT $2\x1B[0m  [["id", "1211d48bb65e420f84a4174918638072"], ["LIMIT", 1]]
+363 <190>1 2018-04-06T19:14:09.379364+00:00 host app web.2 - [www] [a0fe6527-2573-4c84-b3b7-6390939bc14e] [user@example.com]   \x1B[1m\x1B[36mBible Load (1.3ms)\x1B[0m  \x1B[1m\x1B[34mSELECT  "bibles".* FROM "bibles" WHERE "bibles"."publisher_id" = $1 ORDER BY "bibles"."name" ASC LIMIT $2\x1B[0m  [["publisher_id", "1211d48bb65e420f84a4174918638072"], ["LIMIT", 1]]
+491 <190>1 2018-04-06T19:14:09.383904+00:00 host app web.2 - [www] [a0fe6527-2573-4c84-b3b7-6390939bc14e] [user@example.com]   \x1B[1m\x1B[35m (1.5ms)\x1B[0m  \x1B[1m\x1B[34mSELECT COUNT(*) AS "size", MAX("propers"."updated_at") AS timestamp FROM "propers" WHERE "propers"."holiday_id" = $1 AND "propers"."year" = $2 AND ("propers"."name" NOT IN ('Prayer of Thanksgiving', 'Proper Preface', 'Proper Preface (Jacobean)'))\x1B[0m  [["holiday_id", "6189112395f7444881844271027c8c78"], ["year", 1]]
+207 <190>1 2018-04-06T19:14:09.384229+00:00 host app web.2 - [www] [a0fe6527-2573-4c84-b3b7-6390939bc14e] [user@example.com] Cache read: proper/base/query-e05e570da916b2c6b24a534e61906087-8-20171010021720928483
+212 <190>1 2018-04-06T19:14:09.385266+00:00 host app web.2 - [www] [a0fe6527-2573-4c84-b3b7-6390939bc14e] [user@example.com] Cache fetch_hit: proper/base/query-e05e570da916b2c6b24a534e61906087-8-20171010021720928483
+157 <190>1 2018-04-06T19:14:09.387137+00:00 host app web.2 - [www] [a0fe6527-2573-4c84-b3b7-6390939bc14e] [user@example.com] \x1B[33msize: \x1B[1m6.89 KB\x1B[0m
+184 <190>1 2018-04-06T19:14:09.387366+00:00 host app web.2 - [www] [a0fe6527-2573-4c84-b3b7-6390939bc14e] [user@example.com] Completed 200 OK in 32ms (Views: 0.4ms | ActiveRecord: 10.9ms)

data/test/integration/logeater_test.rb

@@ -1,4 +1,5 @@
 require "test_helper"
+require "heroku-log-parser"
 
 class LogeaterTest < ActiveSupport::TestCase
   attr_reader :logfile, :events
@@ -68,9 +69,9 @@ class LogeaterTest < ActiveSupport::TestCase
 
   context "Given an app and a timestamp, import_since" do
     setup do
-      log_sample = File.open(File.expand_path("./test/data/single_request.log"))
+      log_sample = File.open(File.expand_path("./test/data/single_heroku_request.log"))
       log_sample.lines do |line|
-        Logeater::Event.create(ep_app: app, original: line)
+        Logeater::Event.create HerokuLogParser.parse(line).first.merge(ep_app: app)
       end
       @events = Logeater::Event.all
     end
@@ -94,14 +95,14 @@ class LogeaterTest < ActiveSupport::TestCase
 
   context "Given a partial request in one import and the rest in a subsiquent import, it" do
     setup do
-      @lines = File.open(File.expand_path("./test/data/single_request.log")).lines.to_a
+      @lines = File.open(File.expand_path("./test/data/single_heroku_request.log")).lines.to_a
     end
 
     should "save the request after having the full request" do
       # The first two lines will not be enough to describe a complete request
       # so Logeater will not be able to create a request from them...
       @lines[0...2].each do |line|
-        Logeater::Event.create(ep_app: app, original: line)
+        Logeater::Event.create HerokuLogParser.parse(line).first.merge(ep_app: app)
       end
       @events = Logeater::Event.all
       assert_no_difference "Logeater::Request.count" do
@@ -111,7 +112,7 @@ class LogeaterTest < ActiveSupport::TestCase
       # ...but if we later discover the rest of the lines that describe a complete
       # request, it'd be good if Logeater could then recognize and import it.
       @lines[2..-1].each do |line|
-        Logeater::Event.create(ep_app: app, original: line)
+        Logeater::Event.create HerokuLogParser.parse(line).first.merge(ep_app: app)
       end
       @events = Logeater::Event.all
       assert_difference "Logeater::Request.count", +1 do

data/test/unit/event_test.rb

@@ -0,0 +1,19 @@
+require "test_helper"
+
+class EventTest < ActiveSupport::TestCase
+  attr_reader :event
+
+  context "Given an event formatted using RFC 5424" do
+    setup do
+      @event = Logeater::Event.new(
+        emitted_at: Time.new(2018, 4, 6, 10, 15, 30),
+        priority: 190,
+        message: "Started GET or something")
+    end
+
+    should "generate a parseable logger line" do
+      assert_match Logeater::Parser::LINE_MATCHER, event.logger_line
+    end
+  end
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logeater
 version: !ruby/object:Gem::Version
-  version: 0.5.0.pre
+  version: 0.5.4.pre
 platform: ruby
 authors:
 - Bob Lail
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-01-19 00:00:00.000000000 Z
+date: 2019-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -252,6 +252,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: heroku-log-parser
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Parses log files and imports them into a database
 email:
 - bob.lail@cph.org
@@ -290,10 +304,12 @@ files:
 - lib/logeater/request.rb
 - lib/logeater/version.rb
 - logeater.gemspec
+- test/data/single_heroku_request.log
 - test/data/single_request.gz
 - test/data/single_request.log
 - test/integration/logeater_test.rb
 - test/test_helper.rb
+- test/unit/event_test.rb
 - test/unit/params_parser_test.rb
 - test/unit/parser_test.rb
 homepage: ''
@@ -315,15 +331,16 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Parses log files and imports them into a database
 test_files:
+- test/data/single_heroku_request.log
 - test/data/single_request.gz
 - test/data/single_request.log
 - test/integration/logeater_test.rb
 - test/test_helper.rb
+- test/unit/event_test.rb
 - test/unit/params_parser_test.rb
 - test/unit/parser_test.rb