splog 0.0.2 → 0.0.3

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MGQ5MTc5ZGUwMGJjNGE0NDBiZWViMWIzYmY3M2JkODZhNDMzMGJiOQ==
+    N2IwOThhNGYzZTI5MzVmYTk0YjNhOGFkMzgwOGJhM2JlOTYyMzUzNw==
   data.tar.gz: !binary |-
-    NmIxNDZjZDdjYTlkMDU3NmQ2YzQzOGRjYWExNjUyY2IzNDg2YWJhNw==
+    NTkyNGQ3NWM3NWVhNWU3Nzc4NWE1MThiYjYyZDEwYjRmYmY1MTk2OA==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    NTk2OTlhOWU0ZTJlNzQxOTkxYTg4NTNmOWZiNTQyNzYyMzY5ODg1MzFjYjFl
-    YzVjMzVlMTE4YTM3N2M5ZGVmZDVkZTQ1YTRjNmYyNjRiZGMzYmYzMDZmMDlk
-    MjMxY2U5OTZlODZkN2VkZmZmYWI1MzY5ZWE4NDk2NDE0NjY0OGI=
+    ZmRkMTFlZGY2MjZkNWJmYjczOTJkYTQ1MzYzNDg4ZWMyMzlmMGE0NDA4ZDIw
+    ZTdlYzAzZjFmYzM1MzcxYzFmYmZkYmU0YjlkYTkzYTgyY2ZmMTU0YTQ4Mjk2
+    OWFiYjZlMDIxNTA3NjhhODE5ZTBmZDI3NmIyYjM1NWVjOTI1MjA=
   data.tar.gz: !binary |-
-    ZDk3ZjQxZTA3YWZlOWZkYzc1OWEzNWVhZjAxMzc4Yjg4NzE2NGQ1MGZlNjQz
-    YWMzY2NhMGM0MjBmNzY0Y2M1MDMyZGI0ZGNlMjdhMjlhYTkzMmMyYWIxNjhh
-    ZDdmOGE3OWMzNDhiZjNjYTdjMzBkNjQxMjk5MTkwN2IyMTk1YTk=
+    YjNkODBmOTQ1YzgyMjBiM2UyYjA4NTdlNjQwMzhhZWFiNmYyMTc5ZDE4NWYx
+    NjYzMjZmNDBkMTQyNWRlY2I5NGU1YzlhYjU3MmRiYWI5ZDdjYTZlYjljZDlj
+    ZjY5ZGRmMjA0NTY1Yjg5ZTliYWQ5NDhjNjBhMGQ2Mzk4YjBkZjU=

data/README.md

@@ -113,7 +113,7 @@ Persisting the log to mongo.  Set -o with no arg so that no output to stdout.  T
 ## Building Splog in Development
 
 `rake build`
-`rake release` 
+`rake release` or `gem push`
 
 #### Examples of executing the bin/splog in development
 
@@ -148,3 +148,13 @@ Parsing a 45m Apache access log:
 ## Dependencies
 
 Splog is compatible with Ruby 1.9+
+
+## Changelog
+
+#### 0.0.3 -- Oct 2013
+
+* Added verbose logging using the ruby progress bar to give clear visual progress
+
+#### 0.0.2 -- Oct 2013
+
+* Initial release

data/lib/splog/version.rb

@@ -1,3 +1,3 @@
 module Splog
-  VERSION = '0.0.2'
+  VERSION = '0.0.3'
 end

data/lib/splog.rb

@@ -7,6 +7,7 @@ require 'yaml'
 require 'json'
 require 'enumerator'
 require 'mongo'
+require 'ruby-progressbar'
 
 include Mongo
 
@@ -22,6 +23,10 @@ module Splog
       # Yaml config options
       @config = {}
 
+      # Progress bar to create if in verbose mode
+      @progress_bar = nil
+      @line_count = nil
+
       # Command line options
       @options = {
         :append => true
@@ -335,6 +340,20 @@ module Splog
       File.open(file_name).to_enum
     end
 
+    def update_progress_bar
+      if options[:verbose] and not @progress_bar
+        if @line_count.nil?
+          @progress_bar = ProgressBar.create(:starting_at => 0, :total => @line_count)
+        else
+          @progress_bar = ProgressBar.create(:title => 'Lines Read', :format => '[%a] %c Completed |%b>>%i| %p%% %t [%e]', :total => @line_count)
+        end
+      end
+
+      if @options[:verbose] and @progress_bar
+        @progress_bar.increment
+      end
+    end
+
     def cli(args=nil)
       options = {
         :append => true,
@@ -383,11 +402,18 @@ module Splog
           options[:mongo_coll] = ext || nil
         end
 
+        parser.on('--line-count N', Integer, 'If reading from STDIN (using pipes) Setting the line count will allow splog to better log th progress in verbose mode.  Ex. --line-count `wc -l some.log`') do |ext|
+          options[:line_count] = ext || nil
+        end
+
         parser.on('--[no-]md5', 'When saving to mongo md5 the hash and set that to the _id.  This means repeated parses of the same log file should be idempotent.  Otherwise there will be duplicated lines in the database.') do |ext|
-          p ext
           options[:md5] = ext  # if -m then == true
         end
 
+        parser.on('-v', 'Verbose logging, recommended in conjunction with -o without any arguments.') do |ext|
+          options[:verbose] = ext  # if -m then == true
+        end
+
         parser.on_tail('-h', '--help', '--usage', 'Show this usage message and quit.') do |setting|
           puts parser.help
           exit
@@ -422,13 +448,22 @@ module Splog
         set_pattern(options)
         set_mapping(options)
 
+        # Total line count, if file input we can easily do wc -l on the file.  If $stdin we can allow allow a user defined
+        # input from --line-count `wc -l <filename>`
+
         # Get the enum from the file
         e = nil
         if options[:file_name] and options[:pattern_name]
           e = read_log_file(options[:file_name])
+          @line_count = %x{wc -l #{options[:file_name]}}.split.first.to_i
+
+          # Set the progress bar total
+          #update_progress_bar_total(line_count)
+
         # Or stdin otherwise
         elsif not $stdin.tty?
           e = $stdin.to_enum
+          @line_count = options[:line_count]
         else
           $stderr.print 'Please either specify a -f FILENAME or pipe content to splog.'
           exit
@@ -479,6 +514,8 @@ module Splog
               if options[:db_ref_name]
                 persist_log_entry(parsed_line)
               end
+
+              update_progress_bar
             end
           rescue => detail
             nil

data/splog.gemspec

@@ -21,6 +21,7 @@ Gem::Specification.new do |spec|
   # Add runtime dependencies
   spec.add_runtime_dependency 'mongo'
   spec.add_runtime_dependency 'bson_ext'
+  spec.add_runtime_dependency 'ruby-progressbar'
 
   # Add development dependencies
   spec.add_development_dependency 'bundler', '~> 1.3'

data/test/splog_spec.rb

@@ -282,6 +282,19 @@ describe Splog::LogParser do
     pe = parser.parse(e)
     parsed_lines = pe.to_a
     parsed_lines.length.should eql(2)
+  end
+
+  it 'should output verbose logging' do
+    # Match subsequent lines and add them to a previous line
+    test_dir = Dir.pwd.match(/.*?splog$/) ? 'test/' : ''
+    dot_file_path = File.expand_path("#{test_dir}examples/apache/.splog.yml")
+    server_log_name = File.expand_path("#{test_dir}examples/apache/simple_access_log")
 
+    parser = Splog::LogParser.new
+    parser.cli(['-p', 'apache_common','-f', server_log_name, '-c', dot_file_path, '-o', '-v'])
+    e = parser.read_log_file(parser.options[:file_name])
+    # Get an enumerable from the parser
+    pe = parser.parse(e)
+    parsed_lines = pe.to_a
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: splog
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Samuel Mendenhall
@@ -38,6 +38,20 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement