rails_log_deinterleaver 0.1.0

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 1.9.3
+before_install: gem install bundler -v 1.10.2

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in rails_log_deinterleaver.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,28 @@
+PATH
+  remote: .
+  specs:
+    rails_log_deinterleaver (0.1.0)
+      file-tail (~> 1.1)
+      trollop (~> 2.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    file-tail (1.1.0)
+      tins (~> 1.0)
+    minitest (5.7.0)
+    rake (10.4.2)
+    tins (1.6.0)
+    trollop (2.1.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.10)
+  minitest (~> 5.7)
+  rails_log_deinterleaver!
+  rake (~> 10.0)
+
+BUNDLED WITH
+   1.10.2

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Matt Fawcett
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,19 @@
+# RailsLogDeinterleaver
+
+Rails logs in production are usually written to by multiple processes. This causes the log entries for a single request to be interleaved with other requests making the logs hard to read.
+
+This gem includes a command line script to parse the interleaved logs and output logs grouped by the request, using the pid of the server process. it acts like the tail command, and monitors a continually updating log file.
+
+Note that I am using syslog/rails 3 so the logs will be in a slightly different format that what rails will output by default in Rails 4. But this library could be easily updated to support both.
+
+
+Install
+-------
+gem install rails_log_deinterleaver
+
+Use
+---
+rails_log_deinterleaver [options] /path/to/interleaved/railslog.log
+### possible options
+-o, --output=x : Where to write the formatted log (if not specified stdout will be used)
+-b, --backward=x : Start from the bottom of the file, x lines from the end. If not specified, the whole file will be parsed.

data/Rakefile

@@ -0,0 +1,9 @@
+require 'rake/testtask'
+require 'bundler/gem_tasks'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+end
+
+desc "Run tests"
+task :default => :test

data/bin/rails_log_deinterleaver

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'rails_log_deinterleaver'
+
+RailsLogDeinterleaver::Cli.new

data/lib/rails_log_deinterleaver/cli.rb

@@ -0,0 +1,19 @@
+require 'trollop'
+
+module RailsLogDeinterleaver
+  class Cli
+    def initialize
+      opts = Trollop::options do
+        opt :output, "Output file (if unspecified, output will go to stdout)", type: :string
+        opt :backward, "Limit how many lines to go backward (default: no limit)", type: :integer
+      end
+      input = ARGV.last
+      if opts[:output]
+        opts[:output] = File.new(opts[:output], 'w')
+      end
+      raise 'Must specify input log file' unless input && File.exist?(input)
+
+      Parser.new(input, opts).run
+    end
+  end
+end

data/lib/rails_log_deinterleaver/parser.rb

@@ -0,0 +1,83 @@
+require 'date'
+require 'file-tail'
+require 'thread'
+
+module RailsLogDeinterleaver
+  class Parser
+    def initialize(filename, options={})
+      @filename = filename
+      @options = options
+      @options[:request_timeout] ||= 30
+      @options[:date_time_format] ||= "%b %d %H:%M:%S"
+      @options[:start_request_regex] ||= /\[\d+\]: Started/
+      @options[:end_request_regex] ||= /\[\d+\]: Completed/
+      @options[:output] ||= $stdout
+      @pids = {}
+    end
+
+    def run
+      Thread.new do
+        self.ensure_timeouts_logged
+      end
+
+      File.open(@filename) do |log|
+        log.extend(File::Tail)
+        log.interval = 0.1
+        log.backward(@options[:backward]) if @options[:backward]
+        log.tail do |line|
+          process_line line
+        end
+      end
+    end
+
+    def process_line(line)
+      pid = pid_for_line(line)
+
+      if line.match(@options[:start_request_regex])
+        if @pids[pid]
+          # Already a request started but not finished for this pid. (Should be rare)
+          # Log what we have to start a new request
+          output_logs_for_pid(pid)
+        end
+
+        @pids[pid] = {lines: []}
+      end
+
+      return unless @pids[pid]
+
+      @pids[pid][:last_seen_at] = Time.now
+      @pids[pid][:lines].push(line)
+
+      if line.match(@options[:end_request_regex])
+        output_logs_for_pid(pid)
+      end
+    end
+
+    # Output all lines of the log for a single request,
+    # followed by an empty blank line.
+    def output_logs_for_pid(pid)
+      @options[:output].puts (@pids[pid][:lines] << '')
+      @pids.delete(pid)
+    end
+
+    def pid_for_line(line)
+      line.match(/\[(\d+)\]/).captures.first.to_i
+    end
+
+    def time_for_line(line)
+      DateTime.strptime(line, @options[:date_time_format])
+    end
+
+    def ensure_timeouts_logged
+      expired_time = Time.now - @options[:request_timeout]
+      @pids.each do |pid, details|
+        if details[:last_seen_at] <= expired_time
+          output_logs_for_pid(pid)
+        end
+      end
+
+      sleep 1
+      self.ensure_timeouts_logged
+    end
+  end
+end

data/lib/rails_log_deinterleaver/version.rb

@@ -0,0 +1,3 @@
+module RailsLogDeinterleaver
+  VERSION = "0.1.0"
+end

data/lib/rails_log_deinterleaver.rb

@@ -0,0 +1,8 @@
+require 'rubygems'
+require 'bundler/setup'
+
+require 'rails_log_deinterleaver/parser'
+require 'rails_log_deinterleaver/cli'
+
+module RailsLogDeinterleaver
+end

data/rails_log_deinterleaver.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'rails_log_deinterleaver/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "rails_log_deinterleaver"
+  spec.version       = RailsLogDeinterleaver::VERSION
+  spec.authors       = ["Matt Fawcett"]
+  spec.email         = ["fawcett@viddler.com"]
+
+  spec.summary       = "Convert interleaved rails logs into a more readable format"
+  spec.description   = %q(Multiple rails instances writing to the same log cause entries to be interleaved,
+                          making them hard to ready. This command line script groups them by request, using the pid. )
+  spec.homepage      = "https://github.com/mattfawcett/rails_log_deinterleaver"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "bin"
+  spec.executables   = "rails_log_deinterleaver"
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.7"
+
+  spec.add_dependency "file-tail", "~> 1.1"
+  spec.add_dependency "trollop", "~> 2.1"
+end

metadata

@@ -0,0 +1,142 @@
+--- !ruby/object:Gem::Specification
+name: rails_log_deinterleaver
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Matt Fawcett
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-08-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '5.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '5.7'
+- !ruby/object:Gem::Dependency
+  name: file-tail
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: trollop
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.1'
+description: ! "Multiple rails instances writing to the same log cause entries to
+  be interleaved,\n                          making them hard to ready. This command
+  line script groups them by request, using the pid. "
+email:
+- fawcett@viddler.com
+executables:
+- rails_log_deinterleaver
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/rails_log_deinterleaver
+- lib/rails_log_deinterleaver.rb
+- lib/rails_log_deinterleaver/cli.rb
+- lib/rails_log_deinterleaver/parser.rb
+- lib/rails_log_deinterleaver/version.rb
+- rails_log_deinterleaver.gemspec
+homepage: https://github.com/mattfawcett/rails_log_deinterleaver
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Convert interleaved rails logs into a more readable format
+test_files: []