omloga 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0120863a25a06fc3a0f1071965b346b3e4d254c1
+  data.tar.gz: d7fa15b01f36a9a4e796e4c4e000c8d893bdb0eb
+SHA512:
+  metadata.gz: 669be8047715977a016802a9a0e6723c1868591ee59c8a1c839d6067d2027a83406aeaea4b62e75b60d0a8c2f989c02e42d85cca6db5a79b37dca970fcf0a15c
+  data.tar.gz: ab4a597930ae7f02dbf7b563838b5257e06e45c60b12fbdd43d99525148100a86b6236518386f2e1c47469ad202ba94daf32d41ef83a5b4732e54ba35aca93ef

data/README.md

@@ -0,0 +1,4 @@
+omloga
+======
+
+A Ruby on Rails log parser and stitcher.

data/bin/omloga

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'omloga'
+
+omloga(ARGV)

data/lib/omloga/request.rb

@@ -0,0 +1,48 @@
+#encoding: utf-8
+
+require 'moped'
+
+module Omloga
+  class Request
+    attr_accessor :id, :pid, :saved, :lines, :count, :complete_count, :path, :status, :created_at, :updated_at
+
+    def initialize(pid, start_line)
+      @id = Moped::BSON::ObjectId.new
+      @pid = pid
+      @saved = false
+      @count = 1
+      @complete_count = 0
+      @lines = [start_line.to_s]
+    end
+
+    def mongo_doc
+      obj = {
+        '_id' => id,
+        'pid' => pid,
+        'count' => count,
+        'lines' => lines.join("")
+      }
+
+      if saved
+        obj['updated_at'] = Time.now
+      else
+        obj['created_at'] = obj['updated_at'] = Time.now
+      end
+      obj
+    end
+
+    def id_doc
+      {'_id' => id }
+    end
+    
+    def add_line_special(line, pos)
+      line = line.to_s
+      lines.insert(line, pos.to_i)
+    rescue StandardError => e
+      puts "\nline = \n#{line} | is_last = #{is_last}"
+      puts e.message
+      puts e.backtrace.join("\n")
+      exit
+    end
+  end #__End of class Request__
+end #__End of module Omloga__

data/lib/omloga.rb

@@ -0,0 +1,148 @@
+require 'moped'
+require 'optparse'
+require 'ostruct'
+require 'omloga/request'
+
+def omloga(args)
+
+  options = OpenStruct.new
+
+  opts_parser = OptionParser.new do |opts|
+
+    opts.banner = "\nUsage : omloga -d <mongodb-uri> -c <collection-name> [OPTIONS] <log-file-path>"
+
+    opts.separator ""
+
+    opts.on('-d', '--dburi Mongodb-URI', 'MongoDB URI with username/password (if needed) and DB name') do |uri|
+      options.dburi = uri
+    end
+
+    opts.on('-c', '--collection Collection-Name', 'Name of the collection where the log lines should be stored') do |c|
+      options.collection = c
+    end
+
+    opts.on('--log-skip SKIP-FILE-PATH', 'Logs the skipped lines to the specified file') do |sf|
+      options.skip_file_path = sf
+    end
+
+    opts.on_tail('-h', '--help', 'Show this help message') do
+      puts opts
+      exit
+    end
+  end
+
+  opts_parser.parse!(args)
+
+  if args.length < 1
+    puts "Insufficient number of arguments"
+    puts opts_parser
+    exit
+  end
+
+  def db_session
+    $DB_SESSION
+  end
+
+  def logs_collection
+    $DB_LOGS_COLLECTION
+  end
+
+  def request_hash
+    $REQUEST_HASH
+  end
+
+  $DB_SESSION = Moped::Session.connect(options.dburi)
+  $DB_LOGS_COLLECTION = db_session[options.collection]
+  $REQUEST_HASH = {}
+  $LOG_FILE = args[0]
+
+  def is_start_line?(line)
+    line = line.to_s
+    return false if line.nil? or line.length < 7 # "Started" is 7 characters long. It is minimal
+
+    if line.match(/Started/).nil?
+      false
+    else
+      true
+    end
+  end
+
+  def is_complete_line?(line)
+    line = line.to_s
+    return false if line.nil? or line.length < 9 # "Completed" is 9 characters long. It is minimal
+
+    if line.match(/Completed/).nil?
+      false
+    else
+      true
+    end
+  end
+
+  def get_pid(line)
+    match_data = line.match(/#([0-9]+)\]/)
+    if match_data
+      return match_data[1]
+    else
+      return nil
+    end
+  end
+
+  line_count = 0
+  request_count = 0
+  lines_skipped = 0
+  status_str = ''
+  $stdout.sync = true
+  log_skip = false
+  if options.skip_file_path
+    skipped_lines = File.open(options.skip_file_path, 'a')
+    log_skip = true
+  end
+
+  File.foreach($LOG_FILE) do |log_line|
+    pid = get_pid(log_line)
+    req = request_hash[pid]
+
+    if is_start_line?(log_line)
+      if req
+        req.lines << log_line
+        req.count+= 1
+      else
+        req = Omloga::Request.new(pid, log_line)
+        request_hash[pid] = req
+      end
+    else
+      unless req
+        lines_skipped+= 1
+        skipped_lines.print log_line if log_skip
+        next 
+      end
+      is_complete = is_complete_line?(log_line)
+      req.lines << log_line
+
+      if is_complete
+        req.complete_count+= 1
+
+        if req.complete_count >= req.count
+          logs_collection.find(req.id_doc).upsert(req.mongo_doc)
+          request_hash.delete(pid)
+          request_count+= req.count
+        end
+      end
+    end
+    line_count+= 1
+
+    status_str.length.times { print "\b" }
+
+    status_str = "Lines Processed : #{line_count} | Requests found : #{request_count} | Lines Skipped : #{lines_skipped}"
+    print status_str
+  end
+
+  puts ""
+
+  if log_skip
+    puts "Skipped lines have been written to : #{options.skip_file_path}" 
+    skipped_lines.close
+  end
+
+  puts "\nDone..!"
+end

metadata

@@ -0,0 +1,63 @@
+--- !ruby/object:Gem::Specification
+name: omloga
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Srirang G Doddihal
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-10-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: moped
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.4.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.4.3
+description: Stitches together log lines of every request and inserts them as one
+  record in MongoDB for easier analysis
+email: om.brahmana@gmail.com
+executables:
+- omloga
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/omloga.rb
+- lib/omloga/request.rb
+- README.md
+- bin/omloga
+homepage: http://rubygems.org/gems/omloga
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: A Ruby on Rails log parser and stitcher.
+test_files: []