logeater 0.1.1 → 0.2.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: ce5032b80ebdd86abf84a53642d881c1e61315ac
4
- data.tar.gz: 215a2ba9e8e0d90dd14daa6a7df4baa7c2295fe4
3
+ metadata.gz: 4976cf2269f67a86ae95b9f6a182a527e15757c6
4
+ data.tar.gz: 07888882f6e46fbff69fea38a46672efcb766caa
5
5
  SHA512:
6
- metadata.gz: f0a033a7a9e51498b21728cca217978dcd1b1145cddf2a704386558496e01e4949d1e91c0bd20bd411f6540d46ad9edd929adfc25dff2f3e484c365a0672164e
7
- data.tar.gz: bd808f7f9d91d28b09b5ccc0f2e552319edf0f3f2e9e1b64495fe2b71ac7e906acbf7daa74d37ebb333823a31604877d6e10f968c1dba17f0e194cfd14f25ea6
6
+ metadata.gz: 930d7fc93409a85005d38a8529cf9f4357922894458da1a9fb02760d3d7ebebefe79d1069d165e37148500a59e0ad02be8364fc29b86a72069dd6dd985996dea
7
+ data.tar.gz: 47774aa8db964b8405673f750d7c666b98fdf4952b683ad3b1c9f37e7939b897388b01d9aaed489965445dd56a2d546d8bf1effc906c9367e22ab18ba5efec62
data/bin/logeater CHANGED
@@ -1,39 +1,5 @@
1
1
  #!/usr/bin/env ruby
2
2
 
3
- require "logeater"
3
+ require "logeater/cli"
4
4
 
5
- app = ARGV.shift
6
- unless app
7
- puts "USAGE: logeater <app> <files...>"
8
- exit
9
- end
10
-
11
- files = ARGV
12
- if files.empty?
13
- puts "USAGE: logeater <app> <files...>"
14
- exit
15
- end
16
-
17
- started_all = Time.now
18
- files.each_with_index do |file, i|
19
- reader = Logeater::Reader.new(app, file, progress: true)
20
- reader.remove_existing_entries!
21
-
22
- started_count = Logeater::Request.count
23
- started_at = Time.now
24
- reader.import
25
- finished_at = Time.now
26
- finished_count = Logeater::Request.count
27
-
28
- puts " > \e[34mImported \e[1m%d\e[0;34m requests in \e[1m%.2f\e[0;34m seconds (%d of %d)\e[0m\n\n" % [
29
- finished_count - started_count,
30
- finished_at - started_at,
31
- i + 1,
32
- files.length ]
33
- end
34
-
35
- finished_all = Time.now
36
- seconds = finished_all - started_all
37
- minutes = (seconds / 60).to_i
38
- seconds -= (minutes * 60)
39
- puts "Total time %d minutes, %.2f seconds" % [minutes, seconds]
5
+ Logeater::CLI.start(ARGV)
@@ -0,0 +1,9 @@
1
+ class ChangeRequestsParamsToJsonb < ActiveRecord::Migration
2
+ def up
3
+ execute "alter table requests alter column params type jsonb using params::jsonb"
4
+ end
5
+
6
+ def down
7
+ execute "alter table requests alter column params type json using params::json"
8
+ end
9
+ end
@@ -0,0 +1,9 @@
1
+ class ChangeRequestsPathToText < ActiveRecord::Migration
2
+ def up
3
+ execute "alter table requests alter column path type text"
4
+ end
5
+
6
+ def down
7
+ execute "alter table requests alter column path type varchar"
8
+ end
9
+ end
@@ -0,0 +1,6 @@
1
+ class AddUserIdAndTesterBarToRequests < ActiveRecord::Migration
2
+ def change
3
+ add_column :requests, :user_id, :integer
4
+ add_column :requests, :tester_bar, :boolean
5
+ end
6
+ end
data/db/schema.rb CHANGED
@@ -9,35 +9,40 @@
9
9
  # from scratch. The latter is a flawed and unsustainable approach (the more migrations
10
10
  # you'll amass, the slower it'll run and the greater likelihood for issues).
11
11
  #
12
- # It's strongly recommended to check this file into your version control system.
12
+ # It's strongly recommended that you check this file into your version control system.
13
13
 
14
- ActiveRecord::Schema.define(:version => 20150122021627) do
14
+ ActiveRecord::Schema.define(version: 20150602022241) do
15
15
 
16
- create_table "requests", :force => true do |t|
17
- t.string "app", :null => false
18
- t.string "logfile", :null => false
19
- t.string "uuid", :null => false
20
- t.string "subdomain"
16
+ # These are extensions that must be enabled in order to support this database
17
+ enable_extension "plpgsql"
18
+
19
+ create_table "requests", force: :cascade do |t|
20
+ t.string "app", limit: 255, null: false
21
+ t.string "logfile", limit: 255, null: false
22
+ t.string "uuid", limit: 255, null: false
23
+ t.string "subdomain", limit: 255
21
24
  t.datetime "started_at"
22
25
  t.datetime "completed_at"
23
26
  t.integer "duration"
24
- t.string "http_method"
25
- t.string "path"
26
- t.json "params"
27
- t.string "controller"
28
- t.string "action"
29
- t.string "remote_ip"
30
- t.string "format"
27
+ t.string "http_method", limit: 255
28
+ t.text "path"
29
+ t.jsonb "params"
30
+ t.string "controller", limit: 255
31
+ t.string "action", limit: 255
32
+ t.string "remote_ip", limit: 255
33
+ t.string "format", limit: 255
31
34
  t.integer "http_status"
32
- t.string "http_response"
33
- t.datetime "created_at", :null => false
34
- t.datetime "updated_at", :null => false
35
+ t.string "http_response", limit: 255
36
+ t.datetime "created_at", null: false
37
+ t.datetime "updated_at", null: false
38
+ t.integer "user_id"
39
+ t.boolean "tester_bar"
35
40
  end
36
41
 
37
- add_index "requests", ["app"], :name => "index_requests_on_app"
38
- add_index "requests", ["controller", "action"], :name => "index_requests_on_controller_and_action"
39
- add_index "requests", ["http_status"], :name => "index_requests_on_http_status"
40
- add_index "requests", ["logfile"], :name => "index_requests_on_logfile"
41
- add_index "requests", ["uuid"], :name => "index_requests_on_uuid", :unique => true
42
+ add_index "requests", ["app"], name: "index_requests_on_app", using: :btree
43
+ add_index "requests", ["controller", "action"], name: "index_requests_on_controller_and_action", using: :btree
44
+ add_index "requests", ["http_status"], name: "index_requests_on_http_status", using: :btree
45
+ add_index "requests", ["logfile"], name: "index_requests_on_logfile", using: :btree
46
+ add_index "requests", ["uuid"], name: "index_requests_on_uuid", unique: true, using: :btree
42
47
 
43
48
  end
@@ -0,0 +1,81 @@
1
+ require "thor"
2
+ require "logeater"
3
+
4
+ module Logeater
5
+ class CLI < Thor
6
+
7
+ desc "parse APP FILES", "parses files and writes their output to stdout"
8
+ def parse(app, *files)
9
+ if files.empty?
10
+ $stderr.puts "No files to parse"
11
+ return
12
+ end
13
+
14
+ started_all = Time.now
15
+ files.each_with_index do |file, i|
16
+ reader = Logeater::Reader.new(app, file, progress: true)
17
+
18
+ started_at = Time.now
19
+ requests = reader.parse
20
+ finished_at = Time.now
21
+
22
+ $stderr.puts " > \e[34mParsed \e[1m%d\e[0;34m requests in \e[1m%.2f\e[0;34m seconds (%d of %d)\e[0m\n\n" % [
23
+ requests,
24
+ finished_at - started_at,
25
+ i + 1,
26
+ files.length ]
27
+ end
28
+
29
+ finished_all = Time.now
30
+ seconds = finished_all - started_all
31
+ minutes = (seconds / 60).to_i
32
+ seconds -= (minutes * 60)
33
+ $stderr.puts "Total time %d minutes, %.2f seconds" % [minutes, seconds]
34
+ end
35
+
36
+ option :force, type: :boolean, desc: "Imports all specified files, overwiting previous imports if necessary"
37
+ option :progress, type: :boolean, desc: "Renders a progress bar while importing"
38
+ option :verbose, type: :boolean
39
+ desc "import APP FILES", "imports files"
40
+ def import(app, *files)
41
+ unless options[:force]
42
+ imported_files = Logeater::Request.where(app: app).pluck("DISTINCT logfile")
43
+ files.reject! { |file| imported_files.member? File.basename(files[0]) }
44
+ end
45
+
46
+ if files.empty?
47
+ $stderr.puts "No files to import"
48
+ return
49
+ end
50
+
51
+ started_all = Time.now
52
+ files.each_with_index do |file, i|
53
+ $stderr.puts " > \e[34mImporting \e[1m#{File.basename(file)}\e[0;34m (%d of %d)\e[0m\n\n" % [
54
+ i + 1,
55
+ files.length ]
56
+
57
+ reader = Logeater::Reader.new(app, file, options.slice(:progress, :verbose))
58
+ reader.remove_existing_entries!
59
+
60
+ started_count = Logeater::Request.count
61
+ started_at = Time.now
62
+ reader.import
63
+ finished_at = Time.now
64
+ finished_count = Logeater::Request.count
65
+
66
+ $stderr.puts " > \e[34mImported \e[1m%d\e[0;34m requests in \e[1m%.2f\e[0;34m seconds (%d of %d)\e[0m\n\n" % [
67
+ finished_count - started_count,
68
+ finished_at - started_at,
69
+ i + 1,
70
+ files.length ]
71
+ end
72
+
73
+ finished_all = Time.now
74
+ seconds = finished_all - started_all
75
+ minutes = (seconds / 60).to_i
76
+ seconds -= (minutes * 60)
77
+ $stderr.puts "Total time %d minutes, %.2f seconds" % [minutes, seconds]
78
+ end
79
+
80
+ end
81
+ end
@@ -40,10 +40,15 @@ module Logeater
40
40
 
41
41
  # [:@int, "10", [1, 14]]
42
42
  when :@int then sexp[1].to_i
43
-
43
+
44
44
  # [:@float, "10.56", [1, 14]]
45
45
  when :@float then sexp[1].to_f
46
46
 
47
+ # [:unary, :-@, [:@float, \\\"173.41\\\", [1, 17285]]]
48
+ when :unary then
49
+ return -identify(sexp[2]) if sexp[1] == :-@
50
+ raise Parser::ParserNotImplemented, "Unknown unary operator: #{sexp[1].inspect}"
51
+
47
52
  # [:var_ref, [:@kw, "true", [1, 12]]]
48
53
  when :var_ref then
49
54
  return true if sexp[1][1] == "true"
@@ -54,7 +59,7 @@ module Logeater
54
59
  # [:array, [[:@int, "1", [1, 9]], [:@int, "4", [1, 12]]]]
55
60
  # [:array, nil]
56
61
  when :array then sexp[1] ? sexp[1].map { |sexp| identify(sexp) } : []
57
-
62
+
58
63
  # [:hash,
59
64
  # [:assoclist_from_args,
60
65
  # [[:assoc_new,
@@ -14,20 +14,10 @@ module Logeater
14
14
  (?<message>.*)
15
15
  $/x.freeze
16
16
 
17
- TIMESTAMP_MATCHER = /
18
- (?<year>\d\d\d\d)\-
19
- (?<month>\d\d)\-
20
- (?<day>\d\d)T
21
- (?<hours>\d\d):
22
- (?<minutes>\d\d):
23
- (?<seconds>\d\d(?:\.\d+))
24
- /x.freeze
25
-
26
- HTTP_VERBS = %w{DELETE GET HEAD OPTIONS PATCH POST PUT}.freeze
27
-
28
17
  REQUEST_LINE_MATCHER = /^
29
18
  \[(?<subdomain>[^\]]+)\]\s
30
19
  \[(?<uuid>[\w\-]{36})\]\s+
20
+ (?:\[(?:guest|user\.(?<user_id>\d+)(?<tester_bar>:cph)?)\]\s+)?
31
21
  (?<message>.*)
32
22
  $/x.freeze
33
23
 
@@ -53,7 +43,7 @@ module Logeater
53
43
  REQUEST_COMPLETED_MATCHER = /^
54
44
  Completed\s
55
45
  (?<http_status>\d\d\d)\s
56
- (?<http_response>.*)\sin\s
46
+ (?:(?<http_response>.*)\s)?in\s
57
47
  (?<duration>[\d\.]+)(?<units>ms)\b
58
48
  /x.freeze # optional: (Views: 0.1ms | ActiveRecord: 50.0ms)
59
49
 
@@ -61,20 +51,13 @@ module Logeater
61
51
  match = line.match LINE_MATCHER
62
52
  raise UnmatchedLine.new(line) unless match
63
53
 
64
- timestamp = match["timestamp"]
65
- time = timestamp.match TIMESTAMP_MATCHER
66
- raise MalformedTimestamp.new(timestamp) unless time
67
- time = Time.new(*time.captures[0...-1], BigDecimal.new(time["seconds"]))
68
-
69
- message = match["message"]
70
-
71
54
  result = {
72
55
  type: :generic,
73
- timestamp: time,
56
+ timestamp: match["timestamp"],
74
57
  log_level: match["log_level"],
75
- message: message }
76
-
77
- result.merge(parse_message(message))
58
+ message: match["message"]
59
+ }.merge(
60
+ parse_message(match["message"]))
78
61
  end
79
62
 
80
63
  def parse_message(message)
@@ -82,71 +65,58 @@ module Logeater
82
65
  return {} unless match
83
66
 
84
67
  message = match["message"]
85
- type = identify_request_line_type(message)
86
68
 
87
69
  { subdomain: match["subdomain"],
88
70
  uuid: match["uuid"],
89
- type: type,
90
- message: message }.merge(
91
- custom_attributes_for(type, message))
92
- end
93
-
94
- def identify_request_line_type(message)
95
- return :request_started if message =~ /^Started (#{HTTP_VERBS.join("|")})/
96
- return :request_controller if message.start_with? "Processing by "
97
- return :request_params if message.start_with? "Parameters: "
98
- return :request_completed if message =~ /^Completed \d\d\d/
99
- :request_line
71
+ type: :request_line,
72
+ user_id: match["user_id"] && match["user_id"].to_i,
73
+ tester_bar: !!match["tester_bar"],
74
+ message: message
75
+ }.merge(
76
+ parse_message_extra(message))
100
77
  end
101
78
 
102
- def custom_attributes_for(type, message)
103
- attributes = send :"parse_#{type}_message", message
104
- unless attributes
105
- log "Unable to parse message identified as #{type.inspect}: #{message.inspect}"
106
- return {}
107
- end
108
- attributes
109
- end
110
-
111
- def parse_request_line_message(message)
79
+ def parse_message_extra(message)
80
+ match = message.match(REQUEST_STARTED_MATCHER)
81
+ return parse_request_started_message(match) if match
82
+
83
+ match = message.match(REQUEST_CONTROLLER_MATCHER)
84
+ return parse_request_controller_message(match) if match
85
+
86
+ match = message.match(REQUEST_PARAMETERS_MATCHER)
87
+ return parse_request_params_message(match) if match
88
+
89
+ match = message.match(REQUEST_COMPLETED_MATCHER)
90
+ return parse_request_completed_message(match) if match
91
+
112
92
  {}
113
93
  end
114
94
 
115
- def parse_request_started_message(message)
116
- match = message.match(REQUEST_STARTED_MATCHER)
117
- return unless match
118
- uri = Addressable::URI.parse(match["path"])
119
-
120
- { http_method: match["http_method"],
121
- path: uri.path,
95
+ def parse_request_started_message(match)
96
+ { type: :request_started,
97
+ http_method: match["http_method"],
98
+ path: parsed_uri[match["path"]],
122
99
  remote_ip: match["remote_ip"] }
123
100
  end
124
101
 
125
- def parse_request_controller_message(message)
126
- match = message.match(REQUEST_CONTROLLER_MATCHER)
127
- return unless match
128
-
129
- { controller: match["controller"].underscore.gsub(/_controller$/, ""),
102
+ def parse_request_controller_message(match)
103
+ { type: :request_controller,
104
+ controller: normalized_controller_name[match["controller"]],
130
105
  action: match["action"],
131
106
  format: match["format"] }
132
107
  end
133
108
 
134
- def parse_request_params_message(message)
135
- match = message.match(REQUEST_PARAMETERS_MATCHER)
136
- return unless match
137
- params = ParamsParser.new(match["params"])
138
-
139
- { params: params.parse! }
109
+ def parse_request_params_message(match)
110
+ { type: :request_params,
111
+ params: ParamsParser.new(match["params"]).parse! }
140
112
  rescue Logeater::Parser::MalformedParameters
141
113
  log "Unable to parse parameters: #{match["params"].inspect}"
142
114
  { params: match["params"] }
143
115
  end
144
116
 
145
- def parse_request_completed_message(message)
146
- match = message.match(REQUEST_COMPLETED_MATCHER)
147
- return unless match
148
-
149
- { http_status: match["http_status"].to_i,
117
+ def parse_request_completed_message(match)
118
+ { type: :request_completed,
119
+ http_status: match["http_status"].to_i,
150
120
  http_response: match["http_response"],
151
121
  duration: match["duration"].to_i }
152
122
  end
@@ -157,5 +127,21 @@ module Logeater
157
127
  $stderr.puts "\e[33m#{statement}\e[0m"
158
128
  end
159
129
 
130
+
131
+
132
+ def initialize
133
+ @normalized_controller_name = Hash.new do |hash, controller_name|
134
+ hash[controller_name] = controller_name.underscore.gsub(/_controller$/, "")
135
+ end
136
+
137
+ @parsed_uri = Hash.new do |hash, uri|
138
+ hash[uri] = Addressable::URI.parse(uri).path
139
+ end
140
+ end
141
+
142
+ private
143
+ attr_reader :normalized_controller_name,
144
+ :parsed_uri
145
+
160
146
  end
161
147
  end
@@ -1,6 +1,7 @@
1
1
  require "logeater/request"
2
2
  require "zlib"
3
3
  require "ruby-progressbar"
4
+ require "oj"
4
5
 
5
6
  module Logeater
6
7
  class Reader
@@ -24,12 +25,31 @@ module Logeater
24
25
  remove_existing_entries!
25
26
  import
26
27
  end
27
-
28
+
28
29
  def import
29
- each_line(&method(:process_line!))
30
+ each_request do |attributes|
31
+ completed_requests.push Logeater::Request.new(attributes)
32
+ save! if completed_requests.length >= batch_size
33
+ end
30
34
  save!
31
35
  end
32
36
 
37
+ def parse(to: $stdout)
38
+ to << "["
39
+ first = true
40
+ each_request do |attributes|
41
+ if first
42
+ first = false
43
+ else
44
+ to << ",\n"
45
+ end
46
+
47
+ to << Oj.dump(attributes, mode: :compat)
48
+ end
49
+ ensure
50
+ to << "]"
51
+ end
52
+
33
53
  def remove_existing_entries!
34
54
  Logeater::Request.where(app: app, logfile: filename).delete_all
35
55
  end
@@ -45,7 +65,7 @@ module Logeater
45
65
  def each_line
46
66
  File.open(path) do |file|
47
67
  io = File.extname(path) == ".gz" ? Zlib::GzipReader.new(file) : file
48
- pbar = ProgressBar.create(title: filename, total: file.size, autofinish: false) if show_progress?
68
+ pbar = ProgressBar.create(title: filename, total: file.size, autofinish: false, output: $stderr) if show_progress?
49
69
  io.each_line do |line|
50
70
  yield line
51
71
  pbar.progress = file.pos if show_progress?
@@ -55,25 +75,37 @@ module Logeater
55
75
  end
56
76
  alias :scan :each_line
57
77
 
78
+ def each_request
79
+ count = 0
80
+ each_line do |line|
81
+ process_line! line do |request|
82
+ yield request
83
+ count += 1
84
+ end
85
+ end
86
+ count
87
+ end
88
+
89
+
58
90
 
59
91
  private
60
92
  attr_reader :parser, :requests, :completed_requests
61
93
 
62
- def process_line!(line)
94
+ def process_line!(line, &block)
63
95
  attributes = parser.parse!(line)
64
96
 
65
97
  return if [:generic, :request_line].member? attributes[:type]
66
98
 
67
99
  if attributes[:type] == :request_started
68
100
  requests[attributes[:uuid]] = attributes
69
- .slice(:uuid, :subdomain, :http_method, :path, :remote_ip)
101
+ .slice(:uuid, :subdomain, :http_method, :path, :remote_ip, :user_id, :tester_bar)
70
102
  .merge(started_at: attributes[:timestamp], logfile: filename, app: app)
71
103
  return
72
104
  end
73
105
 
74
106
  request_attributes = requests[attributes[:uuid]]
75
107
  unless request_attributes
76
- log "Attempting to record #{attributes[:type].inspect}; but there is no request started with UUID #{attributes[:uuid].inspect}"
108
+ log "Attempting to record #{attributes[:type].inspect}; but there is no request started with UUID #{attributes[:uuid].inspect}" if verbose?
77
109
  return
78
110
  end
79
111
 
@@ -89,10 +121,8 @@ module Logeater
89
121
  .slice(:http_status, :http_response, :duration)
90
122
  .merge(completed_at: attributes[:timestamp])
91
123
 
92
- completed_requests.push Logeater::Request.new(request_attributes)
124
+ yield request_attributes
93
125
  requests.delete attributes[:uuid]
94
-
95
- save! if completed_requests.length >= batch_size
96
126
  end
97
127
 
98
128
  rescue Logeater::Parser::UnmatchedLine
@@ -1,14 +1,9 @@
1
1
  require "active_record"
2
2
  require "activerecord-import"
3
- require "activerecord-postgres-json"
4
3
 
5
4
  module Logeater
6
5
  class Request < ActiveRecord::Base
7
6
  self.table_name = "requests"
8
-
9
- serialize :params, ActiveRecord::Coders::JSON
10
-
11
-
12
-
7
+
13
8
  end
14
9
  end
@@ -1,3 +1,3 @@
1
1
  module Logeater
2
- VERSION = "0.1.1"
2
+ VERSION = "0.2.1"
3
3
  end
data/lib/logeater.rb CHANGED
@@ -5,9 +5,24 @@ require "logeater/version"
5
5
  require "yaml"
6
6
  require "erb"
7
7
 
8
- config_file = File.expand_path("../../db/config.yml", __FILE__)
9
- config = YAML.load(ERB.new(File.read(config_file)).result).with_indifferent_access
10
- ActiveRecord::Base.establish_connection config[ENV["RAILS_ENV"] || "development"]
8
+ database_url = ENV["DATABASE_URL"] unless ENV["RAILS_ENV"] == "test"
9
+ if database_url
10
+ uri = Addressable::URI.parse(database_url)
11
+ config = {
12
+ adapter: "postgresql",
13
+ encoding: "utf8",
14
+ min_messages: "WARNING",
15
+ database: uri.path[1..-1],
16
+ host: uri.host,
17
+ username: uri.user,
18
+ password: uri.password,
19
+ port: uri.port
20
+ }
21
+ else
22
+ config_file = File.expand_path("../../db/config.yml", __FILE__)
23
+ config = YAML.load(ERB.new(File.read(config_file)).result).with_indifferent_access[ENV["RAILS_ENV"] || "development"]
24
+ end
25
+ ActiveRecord::Base.establish_connection config
11
26
 
12
27
  module Logeater
13
28
  end
data/logeater.gemspec CHANGED
@@ -18,21 +18,21 @@ Gem::Specification.new do |spec|
18
18
  spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
19
19
  spec.require_paths = ["lib"]
20
20
 
21
- spec.add_dependency "activerecord", "~> 3.2.21"
22
- spec.add_dependency "activesupport"
21
+ spec.add_dependency "activerecord", ">= 3.1.0", "< 5.0.0"
22
+ spec.add_dependency "activesupport", ">= 3.1.0", "< 5.0.0"
23
23
  spec.add_dependency "pg"
24
24
  spec.add_dependency "standalone_migrations"
25
25
  spec.add_dependency "addressable"
26
26
  spec.add_dependency "ruby-progressbar"
27
- spec.add_dependency "activerecord-import", "~> 0.3.1"
28
- spec.add_dependency "activerecord-postgres-json"
27
+ spec.add_dependency "activerecord-import"
28
+ spec.add_dependency "thor"
29
+ spec.add_dependency "oj"
29
30
 
30
31
  spec.add_development_dependency "bundler", "~> 1.7"
31
32
  spec.add_development_dependency "rake", "~> 10.0"
32
- spec.add_development_dependency "rails", ">= 3.1.0", "< 5.0.0"
33
33
  spec.add_development_dependency "pry"
34
34
  spec.add_development_dependency "shoulda-context"
35
- spec.add_development_dependency "turn"
35
+ spec.add_development_dependency "minitest-reporters"
36
36
  spec.add_development_dependency "database_cleaner"
37
37
 
38
38
  end
@@ -28,8 +28,8 @@ class LogeaterTest < ActiveSupport::TestCase
28
28
  assert_equal "single_request.log", request.logfile
29
29
  assert_equal "0fc5154a-c288-4bad-9c7a-de3d7e7d2496", request.uuid
30
30
  assert_equal "livingsaviorco", request.subdomain
31
- assert_equal Time.new(2015, 1, 10, 15, 18, BigDecimal.new("12.064392")), request.started_at
32
- assert_equal Time.new(2015, 1, 10, 15, 18, BigDecimal.new("12.262903")), request.completed_at
31
+ assert_equal Time.utc(2015, 1, 10, 15, 18, BigDecimal.new("12.064392")), request.started_at
32
+ assert_equal Time.utc(2015, 1, 10, 15, 18, BigDecimal.new("12.262903")), request.completed_at
33
33
  assert_equal 196, request.duration
34
34
  assert_equal "GET", request.http_method
35
35
  assert_equal "/people/1035826228", request.path
data/test/test_helper.rb CHANGED
@@ -1,13 +1,15 @@
1
1
  ENV["RAILS_ENV"] = "test"
2
2
  require "rubygems"
3
- require "rails"
4
- require "rails/test_help"
3
+ require "active_support/testing/autorun"
4
+ require "active_support/test_case"
5
5
  require "pry"
6
6
  require "database_cleaner"
7
7
  require "shoulda/context"
8
- require "turn"
9
8
  require "logeater"
10
9
 
10
+ require "minitest/reporters"
11
+ Minitest::Reporters.use! [Minitest::Reporters::SpecReporter.new]
12
+
11
13
  load File.expand_path("../../db/schema.rb", __FILE__)
12
14
 
13
15
  DatabaseCleaner.strategy = :truncation
@@ -16,6 +16,11 @@ class ParamsParserTest < ActiveSupport::TestCase
16
16
  assert_parses '{"person_id"=>10.56}' => {"person_id"=>10.56}
17
17
  end
18
18
 
19
+ should "handle negatives" do
20
+ assert_parses '{"person_id"=>-10}' => {"person_id"=>-10}
21
+ assert_parses '{"person_id"=>-10.56}' => {"person_id"=>-10.56}
22
+ end
23
+
19
24
  should "handle booleans" do
20
25
  assert_parses '{"visible"=>true}' => {"visible"=>true}
21
26
  end
@@ -33,7 +33,7 @@ class ParserTest < ActiveSupport::TestCase
33
33
  end
34
34
 
35
35
  should "identify the time, including milliseconds" do
36
- assert_parses timestamp: Time.new(2015, 1, 10, 15, 18, BigDecimal.new("5.850839"))
36
+ assert_parses timestamp: "2015-01-10T15:18:05.850839"
37
37
  end
38
38
 
39
39
  should "identify the remainder of the log message" do
@@ -67,6 +67,56 @@ class ParserTest < ActiveSupport::TestCase
67
67
 
68
68
 
69
69
 
70
+ context "given a log line for a Rails request" do
71
+ context "that indicates the current user (who was logged in with the tester bar), it" do
72
+ setup do
73
+ @line = "I, [2015-01-10T15:18:05.850839 #18070] INFO -- : [livingsaviorco] [2d89d962-57c4-47c9-a9e9-6a16a5f22a12] [user.902544074:cph] [gzip] Compress reponse by 42.2 KB (83.3%) (1.4ms)"
74
+ end
75
+
76
+ should "identify the user's id" do
77
+ assert_parses user_id: 902544074
78
+ end
79
+
80
+ should "notice that the user was logged-in with the tester bar" do
81
+ assert_parses tester_bar: true
82
+ end
83
+
84
+ should "identify the remainder of the log message" do
85
+ assert_parses message: "[gzip] Compress reponse by 42.2 KB (83.3%) (1.4ms)"
86
+ end
87
+ end
88
+
89
+ context "that indicates the current user (who was not logged in with the tester bar), it" do
90
+ setup do
91
+ @line = "I, [2015-01-10T15:18:05.850839 #18070] INFO -- : [livingsaviorco] [2d89d962-57c4-47c9-a9e9-6a16a5f22a12] [user.902544074] [gzip] Compress reponse by 42.2 KB (83.3%) (1.4ms)"
92
+ end
93
+
94
+ should "identify the user's id" do
95
+ assert_parses user_id: 902544074
96
+ end
97
+
98
+ should "notice that the user was not logged-in with the tester bar" do
99
+ assert_parses tester_bar: false
100
+ end
101
+
102
+ should "identify the remainder of the log message" do
103
+ assert_parses message: "[gzip] Compress reponse by 42.2 KB (83.3%) (1.4ms)"
104
+ end
105
+ end
106
+
107
+ context "that indicates that the user is logged out, it" do
108
+ setup do
109
+ @line = "I, [2015-01-10T15:18:05.850839 #18070] INFO -- : [livingsaviorco] [2d89d962-57c4-47c9-a9e9-6a16a5f22a12] [guest] [gzip] Compress reponse by 42.2 KB (83.3%) (1.4ms)"
110
+ end
111
+
112
+ should "identify the remainder of the log message" do
113
+ assert_parses message: "[gzip] Compress reponse by 42.2 KB (83.3%) (1.4ms)"
114
+ end
115
+ end
116
+ end
117
+
118
+
119
+
70
120
  context "given the \"Started\" line, it" do
71
121
  setup do
72
122
  @line = "I, [2015-01-10T15:18:12.064392 #2354] INFO -- : [livingsaviorco] [0fc5154a-c288-4bad-9c7a-de3d7e7d2496] Started GET \"/people/1035826228?refresh_page=true\" for 71.218.222.249 at 2015-01-10 15:18:12 +0000"
@@ -155,6 +205,16 @@ class ParserTest < ActiveSupport::TestCase
155
205
  end
156
206
  end
157
207
 
208
+ context "when the \"Completed\" line doesn't have a textual description of the status, it" do
209
+ setup do
210
+ @line = "I, [2015-01-10T15:18:12.262903 #2354] INFO -- : [livingsaviorco] [0fc5154a-c288-4bad-9c7a-de3d7e7d2496] Completed 307 in 0.8ms"
211
+ end
212
+
213
+ should "be cool with that" do
214
+ assert_parses http_status: 307, http_response: nil
215
+ end
216
+ end
217
+
158
218
  context "when the \"Completed\" line contains a breakdown of times, it" do
159
219
  setup do
160
220
  @line = "I, [2015-01-10T15:18:12.262903 #2354] INFO -- : [livingsaviorco] [0fc5154a-c288-4bad-9c7a-de3d7e7d2496] Completed 200 OK in 196ms (Views: 0.1ms | ActiveRecord: 50.0ms)"
metadata CHANGED
@@ -1,43 +1,55 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logeater
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.1
4
+ version: 0.2.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Bob Lail
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-01-22 00:00:00.000000000 Z
11
+ date: 2015-08-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activerecord
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - "~>"
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: 3.1.0
20
+ - - "<"
18
21
  - !ruby/object:Gem::Version
19
- version: 3.2.21
22
+ version: 5.0.0
20
23
  type: :runtime
21
24
  prerelease: false
22
25
  version_requirements: !ruby/object:Gem::Requirement
23
26
  requirements:
24
- - - "~>"
27
+ - - ">="
25
28
  - !ruby/object:Gem::Version
26
- version: 3.2.21
29
+ version: 3.1.0
30
+ - - "<"
31
+ - !ruby/object:Gem::Version
32
+ version: 5.0.0
27
33
  - !ruby/object:Gem::Dependency
28
34
  name: activesupport
29
35
  requirement: !ruby/object:Gem::Requirement
30
36
  requirements:
31
37
  - - ">="
32
38
  - !ruby/object:Gem::Version
33
- version: '0'
39
+ version: 3.1.0
40
+ - - "<"
41
+ - !ruby/object:Gem::Version
42
+ version: 5.0.0
34
43
  type: :runtime
35
44
  prerelease: false
36
45
  version_requirements: !ruby/object:Gem::Requirement
37
46
  requirements:
38
47
  - - ">="
39
48
  - !ruby/object:Gem::Version
40
- version: '0'
49
+ version: 3.1.0
50
+ - - "<"
51
+ - !ruby/object:Gem::Version
52
+ version: 5.0.0
41
53
  - !ruby/object:Gem::Dependency
42
54
  name: pg
43
55
  requirement: !ruby/object:Gem::Requirement
@@ -98,18 +110,18 @@ dependencies:
98
110
  name: activerecord-import
99
111
  requirement: !ruby/object:Gem::Requirement
100
112
  requirements:
101
- - - "~>"
113
+ - - ">="
102
114
  - !ruby/object:Gem::Version
103
- version: 0.3.1
115
+ version: '0'
104
116
  type: :runtime
105
117
  prerelease: false
106
118
  version_requirements: !ruby/object:Gem::Requirement
107
119
  requirements:
108
- - - "~>"
120
+ - - ">="
109
121
  - !ruby/object:Gem::Version
110
- version: 0.3.1
122
+ version: '0'
111
123
  - !ruby/object:Gem::Dependency
112
- name: activerecord-postgres-json
124
+ name: thor
113
125
  requirement: !ruby/object:Gem::Requirement
114
126
  requirements:
115
127
  - - ">="
@@ -123,53 +135,47 @@ dependencies:
123
135
  - !ruby/object:Gem::Version
124
136
  version: '0'
125
137
  - !ruby/object:Gem::Dependency
126
- name: bundler
138
+ name: oj
127
139
  requirement: !ruby/object:Gem::Requirement
128
140
  requirements:
129
- - - "~>"
141
+ - - ">="
130
142
  - !ruby/object:Gem::Version
131
- version: '1.7'
132
- type: :development
143
+ version: '0'
144
+ type: :runtime
133
145
  prerelease: false
134
146
  version_requirements: !ruby/object:Gem::Requirement
135
147
  requirements:
136
- - - "~>"
148
+ - - ">="
137
149
  - !ruby/object:Gem::Version
138
- version: '1.7'
150
+ version: '0'
139
151
  - !ruby/object:Gem::Dependency
140
- name: rake
152
+ name: bundler
141
153
  requirement: !ruby/object:Gem::Requirement
142
154
  requirements:
143
155
  - - "~>"
144
156
  - !ruby/object:Gem::Version
145
- version: '10.0'
157
+ version: '1.7'
146
158
  type: :development
147
159
  prerelease: false
148
160
  version_requirements: !ruby/object:Gem::Requirement
149
161
  requirements:
150
162
  - - "~>"
151
163
  - !ruby/object:Gem::Version
152
- version: '10.0'
164
+ version: '1.7'
153
165
  - !ruby/object:Gem::Dependency
154
- name: rails
166
+ name: rake
155
167
  requirement: !ruby/object:Gem::Requirement
156
168
  requirements:
157
- - - ">="
158
- - !ruby/object:Gem::Version
159
- version: 3.1.0
160
- - - "<"
169
+ - - "~>"
161
170
  - !ruby/object:Gem::Version
162
- version: 5.0.0
171
+ version: '10.0'
163
172
  type: :development
164
173
  prerelease: false
165
174
  version_requirements: !ruby/object:Gem::Requirement
166
175
  requirements:
167
- - - ">="
168
- - !ruby/object:Gem::Version
169
- version: 3.1.0
170
- - - "<"
176
+ - - "~>"
171
177
  - !ruby/object:Gem::Version
172
- version: 5.0.0
178
+ version: '10.0'
173
179
  - !ruby/object:Gem::Dependency
174
180
  name: pry
175
181
  requirement: !ruby/object:Gem::Requirement
@@ -199,7 +205,7 @@ dependencies:
199
205
  - !ruby/object:Gem::Version
200
206
  version: '0'
201
207
  - !ruby/object:Gem::Dependency
202
- name: turn
208
+ name: minitest-reporters
203
209
  requirement: !ruby/object:Gem::Requirement
204
210
  requirements:
205
211
  - - ">="
@@ -244,8 +250,12 @@ files:
244
250
  - db/config.yml
245
251
  - db/migrate/20150110151439_create_requests.rb
246
252
  - db/migrate/20150122021627_change_requests_params_to_json.rb
253
+ - db/migrate/20150207183757_change_requests_params_to_jsonb.rb
254
+ - db/migrate/20150224021844_change_requests_path_to_text.rb
255
+ - db/migrate/20150602022241_add_user_id_and_tester_bar_to_requests.rb
247
256
  - db/schema.rb
248
257
  - lib/logeater.rb
258
+ - lib/logeater/cli.rb
249
259
  - lib/logeater/params_parser.rb
250
260
  - lib/logeater/parser.rb
251
261
  - lib/logeater/parser_errors.rb