rails_log_deinterleaver 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/.gitignore ADDED
@@ -0,0 +1,9 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
data/.travis.yml ADDED
@@ -0,0 +1,4 @@
1
+ language: ruby
2
+ rvm:
3
+ - 1.9.3
4
+ before_install: gem install bundler -v 1.10.2
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in rails_log_deinterleaver.gemspec
4
+ gemspec
data/Gemfile.lock ADDED
@@ -0,0 +1,28 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ rails_log_deinterleaver (0.1.0)
5
+ file-tail (~> 1.1)
6
+ trollop (~> 2.1)
7
+
8
+ GEM
9
+ remote: https://rubygems.org/
10
+ specs:
11
+ file-tail (1.1.0)
12
+ tins (~> 1.0)
13
+ minitest (5.7.0)
14
+ rake (10.4.2)
15
+ tins (1.6.0)
16
+ trollop (2.1.2)
17
+
18
+ PLATFORMS
19
+ ruby
20
+
21
+ DEPENDENCIES
22
+ bundler (~> 1.10)
23
+ minitest (~> 5.7)
24
+ rails_log_deinterleaver!
25
+ rake (~> 10.0)
26
+
27
+ BUNDLED WITH
28
+ 1.10.2
data/LICENSE.txt ADDED
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2015 Matt Fawcett
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,19 @@
1
+ # RailsLogDeinterleaver
2
+
3
+ Rails logs in production are usually written to by multiple processes. This causes the log entries for a single request to be interleaved with other requests making the logs hard to read.
4
+
5
+ This gem includes a command line script to parse the interleaved logs and output logs grouped by the request, using the pid of the server process. it acts like the tail command, and monitors a continually updating log file.
6
+
7
+ Note that I am using syslog/rails 3 so the logs will be in a slightly different format that what rails will output by default in Rails 4. But this library could be easily updated to support both.
8
+
9
+
10
+ Install
11
+ -------
12
+ gem install rails_log_deinterleaver
13
+
14
+ Use
15
+ ---
16
+ rails_log_deinterleaver [options] /path/to/interleaved/railslog.log
17
+ ### possible options
18
+ -o, --output=x : Where to write the formatted log (if not specified stdout will be used)
19
+ -b, --backward=x : Start from the bottom of the file, x lines from the end. If not specified, the whole file will be parsed.
data/Rakefile ADDED
@@ -0,0 +1,9 @@
1
+ require 'rake/testtask'
2
+ require 'bundler/gem_tasks'
3
+
4
+ Rake::TestTask.new do |t|
5
+ t.libs << 'test'
6
+ end
7
+
8
+ desc "Run tests"
9
+ task :default => :test
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'rails_log_deinterleaver'
4
+
5
+ RailsLogDeinterleaver::Cli.new
@@ -0,0 +1,19 @@
1
+ require 'trollop'
2
+
3
+ module RailsLogDeinterleaver
4
+ class Cli
5
+ def initialize
6
+ opts = Trollop::options do
7
+ opt :output, "Output file (if unspecified, output will go to stdout)", type: :string
8
+ opt :backward, "Limit how many lines to go backward (default: no limit)", type: :integer
9
+ end
10
+ input = ARGV.last
11
+ if opts[:output]
12
+ opts[:output] = File.new(opts[:output], 'w')
13
+ end
14
+ raise 'Must specify input log file' unless input && File.exist?(input)
15
+
16
+ Parser.new(input, opts).run
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,83 @@
1
+ require 'date'
2
+ require 'file-tail'
3
+ require 'thread'
4
+
5
+ module RailsLogDeinterleaver
6
+ class Parser
7
+ def initialize(filename, options={})
8
+ @filename = filename
9
+ @options = options
10
+ @options[:request_timeout] ||= 30
11
+ @options[:date_time_format] ||= "%b %d %H:%M:%S"
12
+ @options[:start_request_regex] ||= /\[\d+\]: Started/
13
+ @options[:end_request_regex] ||= /\[\d+\]: Completed/
14
+ @options[:output] ||= $stdout
15
+ @pids = {}
16
+ end
17
+
18
+ def run
19
+ Thread.new do
20
+ self.ensure_timeouts_logged
21
+ end
22
+
23
+ File.open(@filename) do |log|
24
+ log.extend(File::Tail)
25
+ log.interval = 0.1
26
+ log.backward(@options[:backward]) if @options[:backward]
27
+ log.tail do |line|
28
+ process_line line
29
+ end
30
+ end
31
+ end
32
+
33
+ def process_line(line)
34
+ pid = pid_for_line(line)
35
+
36
+ if line.match(@options[:start_request_regex])
37
+ if @pids[pid]
38
+ # Already a request started but not finished for this pid. (Should be rare)
39
+ # Log what we have to start a new request
40
+ output_logs_for_pid(pid)
41
+ end
42
+
43
+ @pids[pid] = {lines: []}
44
+ end
45
+
46
+ return unless @pids[pid]
47
+
48
+ @pids[pid][:last_seen_at] = Time.now
49
+ @pids[pid][:lines].push(line)
50
+
51
+ if line.match(@options[:end_request_regex])
52
+ output_logs_for_pid(pid)
53
+ end
54
+ end
55
+
56
+ # Output all lines of the log for a single request,
57
+ # followed by an empty blank line.
58
+ def output_logs_for_pid(pid)
59
+ @options[:output].puts (@pids[pid][:lines] << '')
60
+ @pids.delete(pid)
61
+ end
62
+
63
+ def pid_for_line(line)
64
+ line.match(/\[(\d+)\]/).captures.first.to_i
65
+ end
66
+
67
+ def time_for_line(line)
68
+ DateTime.strptime(line, @options[:date_time_format])
69
+ end
70
+
71
+ def ensure_timeouts_logged
72
+ expired_time = Time.now - @options[:request_timeout]
73
+ @pids.each do |pid, details|
74
+ if details[:last_seen_at] <= expired_time
75
+ output_logs_for_pid(pid)
76
+ end
77
+ end
78
+
79
+ sleep 1
80
+ self.ensure_timeouts_logged
81
+ end
82
+ end
83
+ end
@@ -0,0 +1,3 @@
1
+ module RailsLogDeinterleaver
2
+ VERSION = "0.1.0"
3
+ end
@@ -0,0 +1,8 @@
1
+ require 'rubygems'
2
+ require 'bundler/setup'
3
+
4
+ require 'rails_log_deinterleaver/parser'
5
+ require 'rails_log_deinterleaver/cli'
6
+
7
+ module RailsLogDeinterleaver
8
+ end
@@ -0,0 +1,29 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'rails_log_deinterleaver/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "rails_log_deinterleaver"
8
+ spec.version = RailsLogDeinterleaver::VERSION
9
+ spec.authors = ["Matt Fawcett"]
10
+ spec.email = ["fawcett@viddler.com"]
11
+
12
+ spec.summary = "Convert interleaved rails logs into a more readable format"
13
+ spec.description = %q(Multiple rails instances writing to the same log cause entries to be interleaved,
14
+ making them hard to ready. This command line script groups them by request, using the pid. )
15
+ spec.homepage = "https://github.com/mattfawcett/rails_log_deinterleaver"
16
+ spec.license = "MIT"
17
+
18
+ spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
19
+ spec.bindir = "bin"
20
+ spec.executables = "rails_log_deinterleaver"
21
+ spec.require_paths = ["lib"]
22
+
23
+ spec.add_development_dependency "bundler", "~> 1.10"
24
+ spec.add_development_dependency "rake", "~> 10.0"
25
+ spec.add_development_dependency "minitest", "~> 5.7"
26
+
27
+ spec.add_dependency "file-tail", "~> 1.1"
28
+ spec.add_dependency "trollop", "~> 2.1"
29
+ end
metadata ADDED
@@ -0,0 +1,142 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: rails_log_deinterleaver
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - Matt Fawcett
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2015-08-17 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: bundler
16
+ requirement: !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ~>
20
+ - !ruby/object:Gem::Version
21
+ version: '1.10'
22
+ type: :development
23
+ prerelease: false
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ~>
28
+ - !ruby/object:Gem::Version
29
+ version: '1.10'
30
+ - !ruby/object:Gem::Dependency
31
+ name: rake
32
+ requirement: !ruby/object:Gem::Requirement
33
+ none: false
34
+ requirements:
35
+ - - ~>
36
+ - !ruby/object:Gem::Version
37
+ version: '10.0'
38
+ type: :development
39
+ prerelease: false
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ~>
44
+ - !ruby/object:Gem::Version
45
+ version: '10.0'
46
+ - !ruby/object:Gem::Dependency
47
+ name: minitest
48
+ requirement: !ruby/object:Gem::Requirement
49
+ none: false
50
+ requirements:
51
+ - - ~>
52
+ - !ruby/object:Gem::Version
53
+ version: '5.7'
54
+ type: :development
55
+ prerelease: false
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ none: false
58
+ requirements:
59
+ - - ~>
60
+ - !ruby/object:Gem::Version
61
+ version: '5.7'
62
+ - !ruby/object:Gem::Dependency
63
+ name: file-tail
64
+ requirement: !ruby/object:Gem::Requirement
65
+ none: false
66
+ requirements:
67
+ - - ~>
68
+ - !ruby/object:Gem::Version
69
+ version: '1.1'
70
+ type: :runtime
71
+ prerelease: false
72
+ version_requirements: !ruby/object:Gem::Requirement
73
+ none: false
74
+ requirements:
75
+ - - ~>
76
+ - !ruby/object:Gem::Version
77
+ version: '1.1'
78
+ - !ruby/object:Gem::Dependency
79
+ name: trollop
80
+ requirement: !ruby/object:Gem::Requirement
81
+ none: false
82
+ requirements:
83
+ - - ~>
84
+ - !ruby/object:Gem::Version
85
+ version: '2.1'
86
+ type: :runtime
87
+ prerelease: false
88
+ version_requirements: !ruby/object:Gem::Requirement
89
+ none: false
90
+ requirements:
91
+ - - ~>
92
+ - !ruby/object:Gem::Version
93
+ version: '2.1'
94
+ description: ! "Multiple rails instances writing to the same log cause entries to
95
+ be interleaved,\n making them hard to ready. This command
96
+ line script groups them by request, using the pid. "
97
+ email:
98
+ - fawcett@viddler.com
99
+ executables:
100
+ - rails_log_deinterleaver
101
+ extensions: []
102
+ extra_rdoc_files: []
103
+ files:
104
+ - .gitignore
105
+ - .travis.yml
106
+ - Gemfile
107
+ - Gemfile.lock
108
+ - LICENSE.txt
109
+ - README.md
110
+ - Rakefile
111
+ - bin/rails_log_deinterleaver
112
+ - lib/rails_log_deinterleaver.rb
113
+ - lib/rails_log_deinterleaver/cli.rb
114
+ - lib/rails_log_deinterleaver/parser.rb
115
+ - lib/rails_log_deinterleaver/version.rb
116
+ - rails_log_deinterleaver.gemspec
117
+ homepage: https://github.com/mattfawcett/rails_log_deinterleaver
118
+ licenses:
119
+ - MIT
120
+ post_install_message:
121
+ rdoc_options: []
122
+ require_paths:
123
+ - lib
124
+ required_ruby_version: !ruby/object:Gem::Requirement
125
+ none: false
126
+ requirements:
127
+ - - ! '>='
128
+ - !ruby/object:Gem::Version
129
+ version: '0'
130
+ required_rubygems_version: !ruby/object:Gem::Requirement
131
+ none: false
132
+ requirements:
133
+ - - ! '>='
134
+ - !ruby/object:Gem::Version
135
+ version: '0'
136
+ requirements: []
137
+ rubyforge_project:
138
+ rubygems_version: 1.8.23
139
+ signing_key:
140
+ specification_version: 3
141
+ summary: Convert interleaved rails logs into a more readable format
142
+ test_files: []