rubylogparser 0.1.0 → 0.1.1

data/EXAMPLES.txt

@@ -25,10 +25,13 @@ Time) are also printed.
   lp = RubyLogParser.new
   lp.open_query('FS', 'Select Top 10 name, size INTO STDOUT from c:/win*.* ORDER BY size DESC', 'CSV', nil)
 
-  while hash = lp.read_hash do
+  while hash = lp.data_hash(lp.process_array_line(lp.readline)) do
     p "#{hash['Name'].ljust(60)} #{hash['Size'].rjust(12)}\n"
   end
 
+  lp.parse_errors_and_stats(lp.read)
+
+  p "Parse errors:\n" + lp.parse_errors.to_s + "\n\n"
   p "Processed: " + (lp.elements_processed.nil? ? "0" : "#{lp.elements_processed}") + "\n"
   p "Output: " + (lp.elements_output.nil? ? "0" : "#{lp.elements_output}") + "\n"
   p "Time: " + (lp.execution_time.nil? ? "0" : "#{lp.execution_time}") + " seconds\n"
@@ -62,10 +65,13 @@ In this example, each line of data is read into an array for further processing
   "
   
   lp.open_query('EVT', sql, 'CSV', nil)
-  while array = lp.read_array_line do
+  while array = lp.process_array_line(lp.readline) do
     p "#{array[0].ljust(35)} #{array[1].rjust(8)}\n"
   end
+  
+  lp.parse_errors_and_stats(lp.read)
 
+  p "Parse errors:\n" + lp.parse_errors.to_s + "\n\n"
   p "Processed: " + (lp.elements_processed.nil? ? "0" : "#{lp.elements_processed}") + "\n"
   p "Output: " + (lp.elements_output.nil? ? "0" : "#{lp.elements_output}") + "\n"
   p "Time: " + (lp.execution_time.nil? ? "0" : "#{lp.execution_time}") + " seconds\n"
@@ -99,7 +105,7 @@ printing the query statistics.
 
   lp.open_query('REG', sql, 'CSV', {'e' => 100})
 
-  while array = lp.read_array_line do
+  while array = lp.process_array_line(lp.readline) do
     p "#{array[0].ljust(80)} #{array[1].rjust(25)}\n"
   end
 

data/GUIDE.txt

@@ -84,13 +84,14 @@ Next we read the Log Parser output line by line into an array and format the
 output to be printed to the screen. See the examples/files.rb example for using
 a hash to retrieve output.
 
-  while array = lp.read_array_line do
+  while array = lp.process_array_line(lp.readline) do
     p "#{array[0].ljust(35)} #{array[1].rjust(8)}\n"
   end
 
 Once all output has been processed, the query statistics and any warnings/errors
-are available.
+are read from the remaining Log Parser output in the pipe and made available.
 
+  lp.parse_errors_and_stats(lp.read)
   p "Parse errors:\n" + lp.parse_errors.to_s + "\n\n"
   p "Processed: " + (lp.elements_processed.nil? ? "0" : "#{lp.elements_processed}") + "\n"
   p "Output: " + (lp.elements_output.nil? ? "0" : "#{lp.elements_output}") + "\n"

data/History.txt

@@ -1,3 +1,9 @@
+== 0.1.1 / 2008-06-06
+
+* Updated to have 100% code coverage (as reported by rcov)
+* Methods refactored to achieve the 100% testing nirvana.
+* Some minor changes in the examples
+
 == 0.1.0 / 2008-01-10
 
 * Initial Release (Pilchuck)

data/examples/event_log.rb

@@ -19,11 +19,13 @@ ORDER BY number_of_events desc
 lp.open_query('EVT', sql, 'CSV', nil)
 
 i=1
-while array = lp.read_array_line do
+while array = lp.process_array_line(lp.readline) do
   p "#{i}.".ljust(4) + " #{array[0].ljust(35)} #{array[1].rjust(8)}\n"
   i += 1
 end
 
+lp.parse_errors_and_stats(lp.read)
+
 p "Parse errors:\n" + lp.parse_errors.to_s + "\n\n"
 p "\nProcessed: " + (lp.elements_processed.nil? ? "0" : "#{lp.elements_processed}") + "\n"
 p "Output: " + (lp.elements_output.nil? ? "0" : "#{lp.elements_output}") + "\n"

data/examples/files.rb

@@ -6,14 +6,17 @@ require 'rubylogparser.rb'
 lp = RubyLogParser.new
 
 # Note: This query may take several minutes to run
-lp.open_query('FS', 'Select Top 10 name, size INTO STDOUT from c:/win*.* ORDER BY size DESC', 'CSV', nil)
+lp.open_query('FS', 'Select Top 10 name, size INTO STDOUT from c:/rubylogparser/*.* ORDER BY size DESC', 'CSV', nil)
 
 i=1
-while hash = lp.read_hash do
-  p "#{i}.".ljust(4) + " #{hash['Name'].ljust(60)} #{hash['Size'].rjust(12)}\n"
+while hash = lp.data_hash(lp.process_array_line(lp.readline)) do
+  p "#{i}. #{hash['Name'].ljust(60)} #{hash['Size'].rjust(12)}\n"
   i += 1
 end
 
+lp.parse_errors_and_stats(lp.read)
+
+p "Parse errors:\n" + lp.parse_errors.to_s + "\n\n"
 p "Processed: " + (lp.elements_processed.nil? ? "0" : "#{lp.elements_processed}") + "\n"
 p "Output: " + (lp.elements_output.nil? ? "0" : "#{lp.elements_output}") + "\n"
 p "Time: " + (lp.execution_time.nil? ? "0" : "#{lp.execution_time}") + " seconds\n"

data/examples/queryinfo.rb

@@ -15,4 +15,4 @@ ORDER BY timegenerated DESC
 
 lp = RubyLogParser.new
 lp.open_query('EVT', sql, nil, {'queryInfo' => true})
-puts "#{lp.formatted_queryInfo}"
+puts "#{lp.queryInfo}"

data/examples/registry.rb

@@ -18,11 +18,13 @@ WHERE   LastWriteTime >= SUB(SYSTEM_TIMESTAMP(), TIMESTAMP('0000-01-02', 'yyyy-M
 lp.open_query('REG', sql, 'CSV', {'e' => 100})
 
 i=1
-while array = lp.read_array_line do
-  p "#{i}.".ljust(5) + " #{array[0].ljust(80)} #{array[1].rjust(25)}\n"
+while array = lp.process_array_line(lp.readline) do
+  p "#{i}. #{array[0].ljust(80)} #{array[1].rjust(25)}\n"
   i += 1  
 end
 
+lp.parse_errors_and_stats(lp.read)
+
 p "Parse errors:\n" + lp.parse_errors.to_s + "\n\n"
 p "Statistics:\n"
 p "Processed: " + (lp.elements_processed.nil? ? "0" : "#{lp.elements_processed}") + "\n"

data/lib/rubylogparser.rb

@@ -5,7 +5,7 @@
 #
 # Author: James A. Clark (jimclark at ieee dot org)
 #
-# Copyright (c) 2007, All Rights Reserved.
+# Copyright (c) 2008, All Rights Reserved.
 #
 # This is free software.  You may modify and redistribute this freely under
 # your choice of the GNU General Public License or the Ruby License. 
@@ -16,7 +16,7 @@
 require 'csv'
 
 class RubyLogParser
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
   
   # Stores the various options that will be used to build the command line that
   # the process is eventually started with
@@ -25,6 +25,10 @@ class RubyLogParser
   # The command line string that IO.popen uses to open the LogParser parser
   attr_accessor :cmd_string
   
+  # A flag which is nil until remaining log parser output is statistics, errors
+  # or other non-data related output
+  attr_accessor :error_flag
+  
   # The pipe to the IO.popen process to read LogParser output data
   attr_accessor :logparser_pipe
 
@@ -55,9 +59,7 @@ class RubyLogParser
   # Initialize is used to check that the Log Parser executable can be found
   # and to config the various command line defaults.
   def initialize(lp_executable = nil)
-    if self.valid_executable(lp_executable) == false
-      raise "The LogParser executable file cannot be found."  
-    end
+    self.valid_executable(lp_executable)
     self.set_defaults
   end 
 
@@ -67,7 +69,7 @@ class RubyLogParser
   # If an executable path is provided, this is tried instead allowing the user to
   # pick the desired path if one or more LogParser versions are installed.
   def valid_executable(lp_executable = nil)
-    found = false
+    @logparser_exe = nil
     if lp_executable.nil? then
       file_locations = ['c:\Program Files\IIS Resources\Log Parser 2.2\LogParser.exe', 'c:\Program Files\Log Parser 2.2\LogParser.exe', 'LogParser.exe']
     else
@@ -76,11 +78,14 @@ class RubyLogParser
     
     file_locations.each {|location|
       if File::executable? location then
-        found = true
         @logparser_exe = location
       end
     }
-    return found
+    if @logparser_exe.nil?
+      raise RuntimeError, "The Log Parser executable could not be found."
+    end
+    
+    return @logparser_exe
   end
 
   # LogParser has a variety of switches to control the input, output, SQL and
@@ -163,67 +168,51 @@ class RubyLogParser
     return @cmd_string
   end
 
-  # LogParser needs a data input source, a SQL selection query, and an output
-  # location. A hash can also be passed to override default values or provide
-  # additional command line switches. An interesting exception that needs to be
-  # handled is LogParser's "-queryInfo" switch which makes LogParser evaluate
-  # the input parameters and return info on the various column datatypes.
-  def open_query(input, query, output, options)
-    @cmd_options['i'] = input
-    @cmd_options['sql'] = query
-    @cmd_options['o'] = output
-    @cmd_string = self.build_command_string(options)
-
-    self.create_process
-    
-    if @cmd_options['queryInfo'] == true then
-      @queryInfo = []
-      @logparser_pipe.each do |line|
-        @queryInfo << line
-      end
-    elsif @cmd_options['q'] == false then
-      self.field_definitions
-    end
-  end
-
   # The LogParser process is created using IO.popen and then subsequent output
   # from LogParser is read via a pipe.
   def create_process
+    @logparser_pipe = nil
+    msg = nil
     begin
       @logparser_pipe = IO.popen(@cmd_string, "rb")
     rescue Exception => e
-      puts "Exit status is: #{e.status}"
+      msg = e.to_s
+      # puts msg # when debugging
     end
+    return @logparser_pipe
   end
 
-  # Reads a single line of output from the LogParser pipe until an EOF is
-  # detected. LogParser will first output all of the query data and then a "\r\n"
-  # blank line followed by any errors or query statistics. Hence, once this
-  # method detects the blank line, all remaining LogParser output is expected
-  # to be error or statistic related. 
+  # Return everything that remains in the Log Parser pipe
+  def read
+    @logparser_pipe.read
+  end
+  
+  # Reads a single line of output from the LogParser pipe.
   def readline
-    begin
-      line = @logparser_pipe.readline
-      if line == "\r\n" then
-        line_errors_and_stats(line)
-      else
-        line.chomp!
-        return line
-      end
-    rescue EOFError
+    self.process_line(@logparser_pipe.readline)
+  end
+
+  # When Log Parser outputs the first blank line ("\r\n"), it indicates that
+  # the data output is complete and anything that will follow are the errors
+  # or query statistics.
+  def process_line(line)
+    if line == "\r\n" then
+      self.error_flag = true # remaining output should now be error
       return nil
+    else
+      return line.chomp!
     end
   end
-  
+
   # The first line of output contains the field names (by default). This
   # information is used to name the various hash keys when data is returned in
   # a hash. 
-  def field_definitions
+  def field_definitions(names_line)
     @field_names = Array.new
-    names_line = self.readline
     if !names_line.nil? then
       @field_names = names_line.split(/,/)
     end
+    return @field_names
   end
 
   # The output from LogParser is return in comma separated value (CSV) format.
@@ -232,8 +221,7 @@ class RubyLogParser
   # a field in two. In cases where the actual number of fields returned by split
   # does not match the expected number, the CSV library is called to try the
   # split. This usually fixes the problem.
-  def read_array_line
-    next_line = self.readline
+  def process_array_line(next_line)
     if !next_line.nil? then 
       array_line = next_line.split(/,/)
       if array_line.length == @field_names.length then
@@ -243,7 +231,6 @@ class RubyLogParser
           if row.length == @field_names.length then
             return row
           else
-            line_errors_and_stats(next_line)
             return nil
           end
         }
@@ -252,54 +239,56 @@ class RubyLogParser
     return nil
   end
 
-  # Once LogParser finishes outputting query data, it then outputs any errors
-  # followed by query statistics (unless specifically turned off). Errors are
-  # pushed into the @parse_errors array and statistics are added to the
-  # appropriate class variables.
-  def line_errors_and_stats(line_input)
-    begin
-      while 1 > 0
-        case line_input
-        when /Task completed with parse errors/
-          @parse_errors = Array.new
-          until ((line_input = @logparser_pipe.readline) == "\r\n")
-            @parse_errors.push(line_input)
-          end
-        when /Statistics:/
-          while line = @logparser_pipe.readline
-            case line
-            when /Elements processed:\s*([\d,]+)\n*/i
-              @elements_processed = $1
-            when /Elements output:\s*([\d,]+)\n*/i
-              @elements_output = $1
-            when /Execution time:\s*([\d\.]+) seconds\n*/i
-              @execution_time = $1
-            end
-          end
-        end
-        line_input = @logparser_pipe.readline
-      end
-    rescue EOFError
-      return nil
+  # Extract statistics and any parse errors from output
+  def parse_errors_and_stats(text_input)
+    if text_input =~ /Statistics:\r\n(.*)/ then
+      @elements_processed = $1 if text_input =~ /^Elements processed:\s*([\d,]+)\r\n/i
+      @elements_output = $1 if text_input =~ /^Elements output:\s*([\d,]+)\r\n/i
+      @execution_time = $1 if text_input =~ /^Execution time:\s*([\d\.]+) seconds\r\n/i
     end
+      
+    if text_input =~ /Task completed with parse errors.*?\r\n(.*)/ims then
+      @parse_errors = $1
+      if @parse_errors =~ /(.*?)Statistics:\r\n/ims then
+        @parse_errors = $1
+      end
+    end  
   end
   
   # To return the data in a hash, the data line is first read into an array.
   # Then the field keys (initially loaded by the field_definitions method when
   # the LogParser process was created), are matched up with the respective data
   # values.
-  def read_hash
-    data_hash = Hash.new
-    data_array = read_array_line
-    if !data_array.nil? then 
+  def data_hash(data_array)
+    data_hash = nil
+    if !data_array.nil? then
+      data_hash = Hash.new
       @field_names.each_index {|x|
         data_hash[@field_names[x]] = data_array[x]
       }
-      return data_hash
-    else
-      return nil
     end
+    return data_hash
   end
+
+  # LogParser needs a data input source, a SQL selection query, and an output
+  # location. A hash can also be passed to override default values or provide
+  # additional command line switches. An interesting exception that needs to be
+  # handled is LogParser's "-queryInfo" switch which makes LogParser evaluate
+  # the input parameters and return info on the various column datatypes.
+  def open_query(input, query, output, options)
+    @cmd_options['i'] = input
+    @cmd_options['sql'] = query
+    @cmd_options['o'] = output
+    @cmd_string = self.build_command_string(options)
+
+    self.create_process
+    
+    if @cmd_options['queryInfo'] == true then
+      @queryInfo = self.read
+    elsif @cmd_options['q'] == false then
+      self.field_definitions(self.readline)
+    end
+  end  
   
   # When data is best processed all at once, this method will return an array
   # of arrays. The array will have a header row with the field names by default
@@ -311,20 +300,20 @@ class RubyLogParser
       data_array[i] = @field_names
       i += 1
     end
-    while array = self.read_array_line do
+    while array = self.process_array_line(self.readline) do
       data_array[i] = array
       i += 1
     end
     return data_array
   end
   
-  # A simple convenience method when running LogParser using the -queryInfo
-  # switch. The resulting output is combined into a single string for easier
-  # printing.
-  def formatted_queryInfo
-    temp = String.new
-    @queryInfo.each{|line| temp << line}
-    return temp
-  end
-  
 end
+=begin
+@logparser = RubyLogParser.new
+@logparser.open_query('EVT','SELECT Top 2 SourceName INTO STDOUT From System','CSV',nil)
+temp = @logparser.read_array(true)
+p temp.length
+p "Stat info:"
+p @logparser.read_all
+
+=end

data/test/test_rubylogparser.rb

@@ -8,10 +8,19 @@ class Test_RubyLogParser < Test::Unit::TestCase
   def setup
     @logparser = RubyLogParser.new
   end
+
+  def teardown
+    if @logparser.logparser_pipe.respond_to?("readline") then
+      @logparser.logparser_pipe.read
+    end
+    @logparser = nil
+  end
   
   def test_valid_executable
-    assert_equal true, @logparser.valid_executable(nil)
-    assert_equal false, @logparser.valid_executable('c:\someotherfilename.exe')
+    assert_not_equal nil, @logparser.valid_executable(nil)
+    assert_raise RuntimeError do
+      @logparser.valid_executable('c:\someotherfilename.exe')
+    end
   end
   
   def test_set_defaults
@@ -20,13 +29,14 @@ class Test_RubyLogParser < Test::Unit::TestCase
     assert_equal false, test_defaults['queryInfo']
   end
   
-  def test_valid_input_type
+  def test_valid_input_format
     assert_equal true, @logparser.valid_input_format("CSV")
     assert_equal true, @logparser.valid_input_format("csv")
-    assert_equal false, @logparser.valid_input_format("xxx")    
+    assert_equal false, @logparser.valid_input_format("xxx")
+    assert_equal true, @logparser.valid_input_format(nil)
   end
 
-  def test_valid_output_type
+  def test_valid_output_format
     assert_equal true, @logparser.valid_output_format("CSV")
     assert_equal true, @logparser.valid_output_format("csv")
     assert_equal false, @logparser.valid_output_format("xxx")    
@@ -39,5 +49,89 @@ class Test_RubyLogParser < Test::Unit::TestCase
     @logparser.logparser_exe = 'LogParser.exe'
     assert_equal '"LogParser.exe" -i:EVT "SQL Test" -o:CSV -queryInfo', @logparser.build_command_string({'queryInfo' => true})
   end
+  
+  def test_create_process
+    @logparser.cmd_options['i'] = 'EVT'
+    @logparser.cmd_options['sql'] = 'SELECT Top 2 SourceName INTO STDOUT From System'
+    @logparser.cmd_options['o'] = 'CSV'
+    @logparser.cmd_string = @logparser.build_command_string(nil)
+    @logparser.create_process
+    assert_equal true, @logparser.logparser_pipe.respond_to?("readline")
+    @logparser.cmd_string = ' '
+    assert_equal nil, @logparser.create_process
+  end
+  
+  def test_process_line
+    assert_equal "test", @logparser.process_line("test\r\n")
+    assert_equal nil, @logparser.process_line("\r\n")
+    assert_equal true, @logparser.error_flag
+  end
+  
+  def test_field_definitions
+    assert_equal [], @logparser.field_definitions(nil)
+    assert_equal ["A"], @logparser.field_definitions("A")
+    assert_equal ["A","B"], @logparser.field_definitions("A,B")
+  end
+  
+  def test_process_array_line
+    assert_equal ["A","B","C"], @logparser.field_definitions("A,B,C")
+    assert_equal 3, @logparser.process_array_line('data1,data2,data3').length
+    assert_equal 3, @logparser.process_array_line('data1,"da,ta2",data3').length
+    assert_equal nil, @logparser.process_array_line('data1,"da,ta2",data3,"data4"')
+  end
+  
+  def test_parse_errors_and_stats
+    text = "Task completed with parse errors.\r\nHere are the errors...\r\nStatistics:\r\n-----------\r\nElements processed: 2\r\nElements output:    2\r\nExecution time:     0.01 seconds\r\n\r\n"
+    @logparser.parse_errors_and_stats(text)
+    assert_equal '2', @logparser.elements_processed
+    assert_equal '2', @logparser.elements_output
+    assert_equal '0.01', @logparser.execution_time
+    assert_match "Here are the errors\.\.\.", @logparser.parse_errors
+  end
+  
+  def test_readall
+    @logparser.cmd_options['i'] = 'EVT'
+    @logparser.cmd_options['sql'] = 'SELECT Top 2 SourceName INTO STDOUT From System'
+    @logparser.cmd_options['o'] = 'CSV'
+    @logparser.build_command_string(nil)
+    @logparser.create_process
+    assert_equal true, @logparser.logparser_pipe.respond_to?("read")
+    assert_equal true, @logparser.read.length > 100
+    assert_equal "", @logparser.read
+  end
+  
+  def test_data_hash
+    assert_equal nil, @logparser.data_hash(nil)
+    assert_equal ["A","B","C"], @logparser.field_definitions("A,B,C")
+    temp = @logparser.data_hash(['data1','data2','data3'])
+    assert_equal 3, temp.length
+    assert_equal 'data1', temp.fetch("A")
+    assert_equal 'data2', temp.fetch("B")
+    assert_equal 'data3', temp.fetch("C")    
+  end
+  
+  def test_open_query_queryInfo
+    @logparser.open_query('EVT','SELECT * INTO STDOUT From System','CSV',{'queryInfo' => true})
+    assert_match "Input format:  EVT", @logparser.queryInfo
+  end
+  
+  def test_open_query_field_names
+    @logparser.open_query('EVT','SELECT TimeGenerated INTO STDOUT From System','CSV',nil)
+    assert_equal 'TimeGenerated', @logparser.field_names[0]
+    assert_equal 1, @logparser.field_names.length
+  end
+  
+  def test_read_array_with_header
+    @logparser.open_query('EVT','SELECT Top 2 SourceName INTO STDOUT From System','CSV',nil)
+    temp = @logparser.read_array(true)
+    assert_equal temp[0][0], 'SourceName'
+    assert_equal 3, temp.length
+  end
 
+  def test_read_array_without_header
+    @logparser.open_query('EVT','SELECT Top 2 SourceName INTO STDOUT From System','CSV',nil)
+    temp = @logparser.read_array(false)
+    assert_equal 2, temp.length
+  end
+  
 end

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.4
 specification_version: 1
 name: rubylogparser
 version: !ruby/object:Gem::Version 
-  version: 0.1.0
-date: 2008-01-11 00:00:00 -08:00
+  version: 0.1.1
+date: 2008-06-06 00:00:00 -07:00
 summary: RubyLogParser provides a wrapper around Microsoft's Log Parser executable.
 require_paths: 
 - lib
@@ -66,5 +66,5 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 1.4.0
+        version: 1.5.3
     version: