rubylogparser 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/EXAMPLES.txt +137 -0
- data/GUIDE.txt +97 -0
- data/History.txt +5 -0
- data/Manifest.txt +12 -0
- data/README.txt +107 -0
- data/Rakefile +21 -0
- data/examples/event_log.rb +30 -0
- data/examples/files.rb +19 -0
- data/examples/queryinfo.rb +18 -0
- data/examples/registry.rb +30 -0
- data/lib/rubylogparser.rb +330 -0
- data/test/test_rubylogparser.rb +43 -0
- metadata +70 -0
data/EXAMPLES.txt
ADDED
@@ -0,0 +1,137 @@
|
|
1
|
+
= RubyLogParser Examples
|
2
|
+
|
3
|
+
See the examples directory for the full source of the scripts presented below.
|
4
|
+
|
5
|
+
Also, the "Log Parser Toolkit" (more info in GUIDE.txt) has many examples of
|
6
|
+
how to use Log Parser.
|
7
|
+
|
8
|
+
== File System
|
9
|
+
|
10
|
+
Log Parser has access to the file system including the various file attributes.
|
11
|
+
In this example, the top 10 largest files are found in the Windows directory
|
12
|
+
(this *should* work for systems with either C:\winnt or C:\windows but the first
|
13
|
+
directory that starts with c:\win will be used) and returned in descending order.
|
14
|
+
|
15
|
+
Note that in the paramters for the open_query method, the input is the file
|
16
|
+
system ('FS'), the query returns the results to standard output ('INTO STDOUT')
|
17
|
+
in comma separated value ('CSV') format.
|
18
|
+
|
19
|
+
The data is read into a hash and then formatted for output to the screen. Once
|
20
|
+
all the data is returned, the various query statistics (Processed, Output and
|
21
|
+
Time) are also printed.
|
22
|
+
|
23
|
+
require 'rubylogparser.rb'
|
24
|
+
|
25
|
+
lp = RubyLogParser.new
|
26
|
+
lp.open_query('FS', 'Select Top 10 name, size INTO STDOUT from c:/win*.* ORDER BY size DESC', 'CSV', nil)
|
27
|
+
|
28
|
+
while hash = lp.read_hash do
|
29
|
+
p "#{hash['Name'].ljust(60)} #{hash['Size'].rjust(12)}\n"
|
30
|
+
end
|
31
|
+
|
32
|
+
p "Processed: " + (lp.elements_processed.nil? ? "0" : "#{lp.elements_processed}") + "\n"
|
33
|
+
p "Output: " + (lp.elements_output.nil? ? "0" : "#{lp.elements_output}") + "\n"
|
34
|
+
p "Time: " + (lp.execution_time.nil? ? "0" : "#{lp.execution_time}") + " seconds\n"
|
35
|
+
|
36
|
+
== Event Log
|
37
|
+
|
38
|
+
Log Parser can be very helpful in scanning the event log. This example scans the
|
39
|
+
System event log and counts the number of entries for each source. This makes it
|
40
|
+
easy to see what events are occuring most frequently.
|
41
|
+
|
42
|
+
Note that in the paramters for the open_query method, the input is the event
|
43
|
+
log ('EVT') and the actual query specifies the System log (other options
|
44
|
+
include security, application, IE), the query returns the results to standard
|
45
|
+
output ('INTO STDOUT') in comma separated value ('CSV') format.
|
46
|
+
|
47
|
+
In this example, each line of data is read into an array for further processing
|
48
|
+
(printed to the screen) and the query statistics printed.
|
49
|
+
|
50
|
+
require 'rubylogparser.rb'
|
51
|
+
|
52
|
+
lp = RubyLogParser.new
|
53
|
+
|
54
|
+
sql = "
|
55
|
+
Select
|
56
|
+
SourceName,
|
57
|
+
count(*) AS number_of_events
|
58
|
+
INTO STDOUT
|
59
|
+
FROM System
|
60
|
+
GROUP BY SourceName
|
61
|
+
ORDER BY number_of_events desc
|
62
|
+
"
|
63
|
+
|
64
|
+
lp.open_query('EVT', sql, 'CSV', nil)
|
65
|
+
while array = lp.read_array_line do
|
66
|
+
p "#{array[0].ljust(35)} #{array[1].rjust(8)}\n"
|
67
|
+
end
|
68
|
+
|
69
|
+
p "Processed: " + (lp.elements_processed.nil? ? "0" : "#{lp.elements_processed}") + "\n"
|
70
|
+
p "Output: " + (lp.elements_output.nil? ? "0" : "#{lp.elements_output}") + "\n"
|
71
|
+
p "Time: " + (lp.execution_time.nil? ? "0" : "#{lp.execution_time}") + " seconds\n"
|
72
|
+
|
73
|
+
== Registry
|
74
|
+
|
75
|
+
Log Parser also can scan the registry.
|
76
|
+
|
77
|
+
Note that in the paramters for the open_query method, the input is the registry
|
78
|
+
('REG') and the actual query specifies the HKLM\Software keys, the query returns
|
79
|
+
the results to standard output ('INTO STDOUT') in comma separated value ('CSV')
|
80
|
+
format.
|
81
|
+
|
82
|
+
And again, like the Event Log example, the output is read line by line into an
|
83
|
+
array for processing (formatted and printed to the screen) and then the query
|
84
|
+
statistics printed. However, errors are commonly encountered due to Windows
|
85
|
+
locking registry keys from being read and these keys are displayed before
|
86
|
+
printing the query statistics.
|
87
|
+
|
88
|
+
require 'rubylogparser.rb'
|
89
|
+
|
90
|
+
lp = RubyLogParser.new
|
91
|
+
|
92
|
+
sql = "
|
93
|
+
Select Path,
|
94
|
+
ValueName
|
95
|
+
INTO STDOUT
|
96
|
+
FROM HKLM\\SOFTWARE
|
97
|
+
WHERE LastWriteTime >= SUB(SYSTEM_TIMESTAMP(), TIMESTAMP('0000-01-02', 'yyyy-MM-dd'))
|
98
|
+
"
|
99
|
+
|
100
|
+
lp.open_query('REG', sql, 'CSV', {'e' => 100})
|
101
|
+
|
102
|
+
while array = lp.read_array_line do
|
103
|
+
p "#{array[0].ljust(80)} #{array[1].rjust(25)}\n"
|
104
|
+
end
|
105
|
+
|
106
|
+
p "Parse errors:\n" + lp.parse_errors.to_s + "\n\n"
|
107
|
+
p "Statistics:\n"
|
108
|
+
p "Processed: " + (lp.elements_processed.nil? ? "0" : "#{lp.elements_processed}") + "\n"
|
109
|
+
p "Output: " + (lp.elements_output.nil? ? "0" : "#{lp.elements_output}") + "\n"
|
110
|
+
p "Time: " + (lp.execution_time.nil? ? "0" : "#{lp.execution_time}") + " seconds\n"
|
111
|
+
|
112
|
+
== The queryInfo switch
|
113
|
+
|
114
|
+
The queryInfo switch is useful to see how Log Parser will process the various
|
115
|
+
input options including the data types the fields will be returned as.
|
116
|
+
Truthfully, this switch is probably most useful when used directly from the
|
117
|
+
command line.
|
118
|
+
|
119
|
+
An additional merit of this example is to show how additional command line
|
120
|
+
options can be passed to the open_query method. In this case, the queryInfo
|
121
|
+
parameter is set to true which overrides the default false value. Another example
|
122
|
+
that would ignore warnings ('iw') and choose the queryInfo would be passed to the
|
123
|
+
open_query method as {'queryInfo' => true, 'iw' => true}.
|
124
|
+
|
125
|
+
require 'rubylogparser.rb'
|
126
|
+
lp = RubyLogParser.new
|
127
|
+
|
128
|
+
sql = "
|
129
|
+
Select Top 3 timegenerated AS EventTime, message AS Message
|
130
|
+
FROM system
|
131
|
+
WHERE eventid='6005'
|
132
|
+
ORDER BY timegenerated DESC
|
133
|
+
"
|
134
|
+
|
135
|
+
lp.open_query('EVT', sql, nil, {'queryInfo' => true})
|
136
|
+
|
137
|
+
puts "#{lp.formatted_queryInfo}"
|
data/GUIDE.txt
ADDED
@@ -0,0 +1,97 @@
|
|
1
|
+
= Getting Started With LogParser
|
2
|
+
This guide is meant to get you started using the rubylogparser gem to interface
|
3
|
+
with Microsoft's Log Parser.
|
4
|
+
|
5
|
+
A book that I highly recommend is the "Log Parser Toolkit" by Gabriele Giuseppini
|
6
|
+
and Mark Burnett (ISBN 1-932266-52-6). It is a treasure chest of examples for
|
7
|
+
analyzing log files, event logs, investigating intrusions, security auditing,
|
8
|
+
formatting, reporting, charting and so much more. It also includes a nice
|
9
|
+
reference for input and output formats including commonly used parameters. In
|
10
|
+
short, you'll get a lot more out of Log Parser if you look at how people have
|
11
|
+
been using it to do some really interesting things.
|
12
|
+
|
13
|
+
The included examples really just scratches the surface of what is possible, but
|
14
|
+
should be enough information to get you really going!
|
15
|
+
|
16
|
+
== Log Parser overview
|
17
|
+
The basic architecture of Log Parser is that it allows an input source to be
|
18
|
+
queried by an SQL statement and the results sent to a variety of output formats.
|
19
|
+
|
20
|
+
Input sources may be log files, event logs, the file system, the registry, XML
|
21
|
+
files, ETW trace logs, among others. The output formats include CSV files, log
|
22
|
+
file formats, Syslogs, XML, charts and more.
|
23
|
+
|
24
|
+
A basic use of Log Parser is to choose an input source, select and/or filter it
|
25
|
+
with an SQL statement and output it to a desired format. Hence, conversion and
|
26
|
+
filtering of data is greatly simplified.
|
27
|
+
|
28
|
+
== Functionality of rubylogparser gem
|
29
|
+
The goal of the rubylogparser gem is to scrape the Log Parser output into Ruby
|
30
|
+
arrays or hashes for further processing. For instance, the event log can be
|
31
|
+
monitored and specific events pushed to a database via DBI, email alerts sent
|
32
|
+
using NET/SMTP, Ruport for reporting, etc.
|
33
|
+
|
34
|
+
To best use this gem and populate the Ruby data structures properly, the output
|
35
|
+
from Log Parser needs to be in CSV format sent to standard output (STDOUT).
|
36
|
+
Internally, this gem opens Log Parser using an IO.popen call and output from
|
37
|
+
Log Parser is subsequently read from a pipe using readline commands. That it
|
38
|
+
why the output needs to be specified as STDOUT so the data can be read from the
|
39
|
+
pipe. The CSV line is then split into the respective fields and returned as an
|
40
|
+
array or hash with the key values being the field/column names.
|
41
|
+
|
42
|
+
If Log Parser warnings and query statistics are enabled (they are by default),
|
43
|
+
several class variables (@elements_processed, @elements_output, @execution_time)
|
44
|
+
will be populated after all the data has been read in. Any errors will be added
|
45
|
+
to an array of strings called @parse_errors.
|
46
|
+
|
47
|
+
== Let's Fetch Some Information!
|
48
|
+
First thing is first. Make sure that you've required rubylogparser and that you
|
49
|
+
instantiate a new logparser object:
|
50
|
+
|
51
|
+
require 'rubygems'
|
52
|
+
require 'rubylogparser'
|
53
|
+
|
54
|
+
lp = RubyLogParser.new
|
55
|
+
|
56
|
+
Now we'll create a SQL statement to count the number of distinct events in the
|
57
|
+
System event log and send the output 'INTO STDOUT'.
|
58
|
+
|
59
|
+
# This SQL statement orders System events by order of their SourceName
|
60
|
+
sql = "
|
61
|
+
Select
|
62
|
+
SourceName,
|
63
|
+
count(*) AS number_of_events
|
64
|
+
INTO STDOUT
|
65
|
+
FROM System
|
66
|
+
GROUP BY SourceName
|
67
|
+
ORDER BY number_of_events desc
|
68
|
+
"
|
69
|
+
|
70
|
+
Here we create the Log Parser process and specify:
|
71
|
+
- input format ('EVT' for event log)
|
72
|
+
- the SQL statement stored in the sql variable
|
73
|
+
- the output format ('CSV' for comma separated values)
|
74
|
+
|
75
|
+
If we wanted to override any of the default log parser parameters, we could
|
76
|
+
replace the nil in the method call with a hash containing the command line
|
77
|
+
parameter values such as {'iw' => true} to ignore warnings. Run the Log Parser
|
78
|
+
executable from the command line such as "logparser.exe -h" to see the various
|
79
|
+
command line switches and additional help.
|
80
|
+
|
81
|
+
lp.open_query('EVT', sql, 'CSV', nil)
|
82
|
+
|
83
|
+
Next we read the Log Parser output line by line into an array and format the
|
84
|
+
output to be printed to the screen. See the examples/files.rb example for using
|
85
|
+
a hash to retrieve output.
|
86
|
+
|
87
|
+
while array = lp.read_array_line do
|
88
|
+
p "#{array[0].ljust(35)} #{array[1].rjust(8)}\n"
|
89
|
+
end
|
90
|
+
|
91
|
+
Once all output has been processed, the query statistics and any warnings/errors
|
92
|
+
are available.
|
93
|
+
|
94
|
+
p "Parse errors:\n" + lp.parse_errors.to_s + "\n\n"
|
95
|
+
p "Processed: " + (lp.elements_processed.nil? ? "0" : "#{lp.elements_processed}") + "\n"
|
96
|
+
p "Output: " + (lp.elements_output.nil? ? "0" : "#{lp.elements_output}") + "\n"
|
97
|
+
p "Time: " + (lp.execution_time.nil? ? "0" : "#{lp.execution_time}") + " seconds\n"
|
data/History.txt
ADDED
data/Manifest.txt
ADDED
data/README.txt
ADDED
@@ -0,0 +1,107 @@
|
|
1
|
+
rubylogparser
|
2
|
+
by Jim Clark (jimclark@ieee.org)
|
3
|
+
http://rubyforge.org/projects/rubylogparser/
|
4
|
+
|
5
|
+
== DESCRIPTION:
|
6
|
+
|
7
|
+
The RubyLogParser library is used for interacting with Microsoft's Log Parser
|
8
|
+
tool (http://www.microsoft.com/downloads/results.aspx?freetext=Log%20Parser).
|
9
|
+
|
10
|
+
Log Parser is a powerful, versatile tool that provides universal query
|
11
|
+
access to text-based data such as log files, XML files and CSV files, as
|
12
|
+
well as key data sources on the Windows operating system such as the
|
13
|
+
Event Log, the Registry, the file system, and Active Directory.
|
14
|
+
|
15
|
+
The most important benefit of using this library is that the Log Parser
|
16
|
+
output can be returned in arrays or hashes for further processing in Ruby.
|
17
|
+
|
18
|
+
== FEATURES/PROBLEMS:
|
19
|
+
|
20
|
+
The rubylogparser gem allows complete control over the Log Parser executable.
|
21
|
+
Although charts, XML files, log file formats and other output formats are
|
22
|
+
possible, the main goal is to scrape the Log Parser output into Ruby data
|
23
|
+
structures for further processing. Currently, arrays and hashes are supported.
|
24
|
+
|
25
|
+
See the following synopsis for a quick code sample or for a more detailed
|
26
|
+
explanation, check out the GUIDE[link://files/GUIDE_txt.html].
|
27
|
+
Also, check out the EXAMPLES[link://files/EXAMPLES_txt.html] file.
|
28
|
+
|
29
|
+
== SYNOPSIS:
|
30
|
+
|
31
|
+
The following code will search the c:\win*.* directory (matching c:\windows or
|
32
|
+
c:\winnt on most systems) for the 10 largest files. It make take a couple of
|
33
|
+
minutes to run. The input for Log Parser is the file system ('FS') and output
|
34
|
+
is directed to standard output (STDOUT) in comma separated value ('CSV') format
|
35
|
+
which is where the rubylogparser gem scrapes the Log Parser output from. Data
|
36
|
+
is returned in a hash with the hash keys corresponding to the selected fields in
|
37
|
+
the SQL statement.
|
38
|
+
|
39
|
+
require 'rubylogparser.rb'
|
40
|
+
lp = RubyLogParser.new
|
41
|
+
lp.open_query('FS', 'Select Top 10 name, size INTO STDOUT from c:/win*.* ORDER BY size DESC', 'CSV', nil)
|
42
|
+
while hash = lp.read_hash do
|
43
|
+
p " #{hash['Name'].ljust(60)} #{hash['Size'].rjust(12)}\n"
|
44
|
+
end
|
45
|
+
|
46
|
+
== REQUIREMENTS:
|
47
|
+
|
48
|
+
Testing was done in the following environment:
|
49
|
+
|
50
|
+
* ruby 1.8.6
|
51
|
+
|
52
|
+
* Microsoft Log Parser 2.2
|
53
|
+
|
54
|
+
Other software versions may work but are completely untested. Testing with Rails
|
55
|
+
and Ruby 1.9 will be done in early 2008.
|
56
|
+
|
57
|
+
== INSTALL:
|
58
|
+
|
59
|
+
To use this library, first make sure that you have Microsoft's Log Parser tool
|
60
|
+
installed in your system. If you choose not to use the default install location,
|
61
|
+
make sure to add the directory where the logparser.exe executable is installed
|
62
|
+
in on the path.
|
63
|
+
|
64
|
+
Next, install the gem using "gem install -r rubylogparser"
|
65
|
+
|
66
|
+
RubyLogParser will automatically search for the Log Parser executable in these
|
67
|
+
locations:
|
68
|
+
|
69
|
+
* c:\Program Files\IIS Resources\Log Parser 2.2\LogParser.exe (Default install location for the IIS Resource Kit)
|
70
|
+
|
71
|
+
* c:\Program Files\Log Parser 2.2\LogParser.exe (Default location when downloaded and installed without the IIS Resource kit)
|
72
|
+
|
73
|
+
* LogParser.exe on the path somewhere
|
74
|
+
|
75
|
+
If the LogParser.exe executable cannot be found, the "test_valid_executable"
|
76
|
+
test will fail. If not installed in a default location, adding the installation
|
77
|
+
directory to the path is the easiest way to allow the executable to be found.
|
78
|
+
|
79
|
+
== ACKNOWLEDGEMENTS:
|
80
|
+
|
81
|
+
Thank you to Ryan Davis for guidance and troubleshooting and making this
|
82
|
+
library better.
|
83
|
+
|
84
|
+
== LICENSE:
|
85
|
+
|
86
|
+
(The MIT License)
|
87
|
+
|
88
|
+
Copyright (c) 2008 Jim Clark (jimclark@ieee.org)
|
89
|
+
|
90
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
91
|
+
a copy of this software and associated documentation files (the
|
92
|
+
'Software'), to deal in the Software without restriction, including
|
93
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
94
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
95
|
+
permit persons to whom the Software is furnished to do so, subject to
|
96
|
+
the following conditions:
|
97
|
+
|
98
|
+
The above copyright notice and this permission notice shall be
|
99
|
+
included in all copies or substantial portions of the Software.
|
100
|
+
|
101
|
+
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
|
102
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
103
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
104
|
+
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
105
|
+
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
106
|
+
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
107
|
+
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/Rakefile
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'hoe'
|
3
|
+
$:.unshift(File.dirname(__FILE__) + "/lib")
|
4
|
+
require 'rubylogparser'
|
5
|
+
|
6
|
+
Hoe.new('rubylogparser', RubyLogParser::VERSION) do |p|
|
7
|
+
p.rubyforge_name = 'rubylogparser'
|
8
|
+
p.author = 'Jim Clark'
|
9
|
+
p.email = 'jimclark@ieee.org'
|
10
|
+
p.summary = "RubyLogParser provides a wrapper around Microsoft's Log Parser executable."
|
11
|
+
p.description = "RubyLogParser enables the output from Microsoft's Log Parser to be processed by Ruby data structures (arrays and hashes)."
|
12
|
+
p.remote_rdoc_dir = '' # Release to root
|
13
|
+
p.url = 'http://rubyforge.org/projects/rubylogparser/'
|
14
|
+
p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
|
15
|
+
p.need_tar = false
|
16
|
+
p.need_zip = true
|
17
|
+
end
|
18
|
+
|
19
|
+
desc "Release and publish documentation"
|
20
|
+
task :repubdoc => [:release, :publish_docs]
|
21
|
+
|
@@ -0,0 +1,30 @@
|
|
1
|
+
#!c:/ruby/bin/ruby.exe
|
2
|
+
|
3
|
+
$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
|
4
|
+
require 'rubylogparser.rb'
|
5
|
+
|
6
|
+
lp = RubyLogParser.new
|
7
|
+
|
8
|
+
# This SQL statement orders System events by order of their SourceName
|
9
|
+
sql = "
|
10
|
+
Select
|
11
|
+
SourceName,
|
12
|
+
count(*) AS number_of_events
|
13
|
+
INTO STDOUT
|
14
|
+
FROM System
|
15
|
+
GROUP BY SourceName
|
16
|
+
ORDER BY number_of_events desc
|
17
|
+
"
|
18
|
+
|
19
|
+
lp.open_query('EVT', sql, 'CSV', nil)
|
20
|
+
|
21
|
+
i=1
|
22
|
+
while array = lp.read_array_line do
|
23
|
+
p "#{i}.".ljust(4) + " #{array[0].ljust(35)} #{array[1].rjust(8)}\n"
|
24
|
+
i += 1
|
25
|
+
end
|
26
|
+
|
27
|
+
p "Parse errors:\n" + lp.parse_errors.to_s + "\n\n"
|
28
|
+
p "\nProcessed: " + (lp.elements_processed.nil? ? "0" : "#{lp.elements_processed}") + "\n"
|
29
|
+
p "Output: " + (lp.elements_output.nil? ? "0" : "#{lp.elements_output}") + "\n"
|
30
|
+
p "Time: " + (lp.execution_time.nil? ? "0" : "#{lp.execution_time}") + " seconds\n"
|
data/examples/files.rb
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
#!c:/ruby/bin/ruby.exe
|
2
|
+
|
3
|
+
$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
|
4
|
+
require 'rubylogparser.rb'
|
5
|
+
|
6
|
+
lp = RubyLogParser.new
|
7
|
+
|
8
|
+
# Note: This query may take several minutes to run
|
9
|
+
lp.open_query('FS', 'Select Top 10 name, size INTO STDOUT from c:/win*.* ORDER BY size DESC', 'CSV', nil)
|
10
|
+
|
11
|
+
i=1
|
12
|
+
while hash = lp.read_hash do
|
13
|
+
p "#{i}.".ljust(4) + " #{hash['Name'].ljust(60)} #{hash['Size'].rjust(12)}\n"
|
14
|
+
i += 1
|
15
|
+
end
|
16
|
+
|
17
|
+
p "Processed: " + (lp.elements_processed.nil? ? "0" : "#{lp.elements_processed}") + "\n"
|
18
|
+
p "Output: " + (lp.elements_output.nil? ? "0" : "#{lp.elements_output}") + "\n"
|
19
|
+
p "Time: " + (lp.execution_time.nil? ? "0" : "#{lp.execution_time}") + " seconds\n"
|
@@ -0,0 +1,18 @@
|
|
1
|
+
#!c:/ruby/bin/ruby.exe
|
2
|
+
|
3
|
+
$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
|
4
|
+
require 'rubylogparser'
|
5
|
+
|
6
|
+
# This SQL selects a common event from the system log. Since the query will be
|
7
|
+
# evaluated but not actually executed, it doesn't really matter if the specific
|
8
|
+
# event will be found in the event log.
|
9
|
+
sql = "
|
10
|
+
Select Top 3 timegenerated AS EventTime, message AS Message
|
11
|
+
FROM system
|
12
|
+
WHERE eventid='6005'
|
13
|
+
ORDER BY timegenerated DESC
|
14
|
+
"
|
15
|
+
|
16
|
+
lp = RubyLogParser.new
|
17
|
+
lp.open_query('EVT', sql, nil, {'queryInfo' => true})
|
18
|
+
puts "#{lp.formatted_queryInfo}"
|
@@ -0,0 +1,30 @@
|
|
1
|
+
#!c:/ruby/bin/ruby.exe
|
2
|
+
|
3
|
+
$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
|
4
|
+
require 'rubylogparser.rb'
|
5
|
+
|
6
|
+
lp = RubyLogParser.new
|
7
|
+
|
8
|
+
# This SQL statement filters registry keys from HKLM\Software that were written
|
9
|
+
# to in the last day
|
10
|
+
sql = "
|
11
|
+
Select Path,
|
12
|
+
ValueName
|
13
|
+
INTO STDOUT
|
14
|
+
FROM HKLM\\SOFTWARE
|
15
|
+
WHERE LastWriteTime >= SUB(SYSTEM_TIMESTAMP(), TIMESTAMP('0000-01-02', 'yyyy-MM-dd'))
|
16
|
+
"
|
17
|
+
|
18
|
+
lp.open_query('REG', sql, 'CSV', {'e' => 100})
|
19
|
+
|
20
|
+
i=1
|
21
|
+
while array = lp.read_array_line do
|
22
|
+
p "#{i}.".ljust(5) + " #{array[0].ljust(80)} #{array[1].rjust(25)}\n"
|
23
|
+
i += 1
|
24
|
+
end
|
25
|
+
|
26
|
+
p "Parse errors:\n" + lp.parse_errors.to_s + "\n\n"
|
27
|
+
p "Statistics:\n"
|
28
|
+
p "Processed: " + (lp.elements_processed.nil? ? "0" : "#{lp.elements_processed}") + "\n"
|
29
|
+
p "Output: " + (lp.elements_output.nil? ? "0" : "#{lp.elements_output}") + "\n"
|
30
|
+
p "Time: " + (lp.execution_time.nil? ? "0" : "#{lp.execution_time}") + " seconds\n"
|
@@ -0,0 +1,330 @@
|
|
1
|
+
# rubylogparser.rb : Ruby wrapper around Microsoft's LogParser executable.
|
2
|
+
# Methods and variables for interacting with the LogParser process. Most of
|
3
|
+
# these methods are for retrieving data using hashes or arrays to allow
|
4
|
+
# the maximum flexibility in retrieving the LogParser output.
|
5
|
+
#
|
6
|
+
# Author: James A. Clark (jimclark at ieee dot org)
|
7
|
+
#
|
8
|
+
# Copyright (c) 2007, All Rights Reserved.
|
9
|
+
#
|
10
|
+
# This is free software. You may modify and redistribute this freely under
|
11
|
+
# your choice of the GNU General Public License or the Ruby License.
|
12
|
+
#
|
13
|
+
# See LICENSE and COPYING for details
|
14
|
+
#
|
15
|
+
|
16
|
+
require 'csv'
|
17
|
+
|
18
|
+
class RubyLogParser
|
19
|
+
VERSION = "0.1.0"
|
20
|
+
|
21
|
+
# Stores the various options that will be used to build the command line that
|
22
|
+
# the process is eventually started with
|
23
|
+
attr_accessor :cmd_options
|
24
|
+
|
25
|
+
# The command line string that IO.popen uses to open the LogParser parser
|
26
|
+
attr_accessor :cmd_string
|
27
|
+
|
28
|
+
# The pipe to the IO.popen process to read LogParser output data
|
29
|
+
attr_accessor :logparser_pipe
|
30
|
+
|
31
|
+
# Column headings on the output data are controlled with the " -q" quiet mode
|
32
|
+
# switch. When enabled (the default), the various column names are stored in
|
33
|
+
# this array. This is populated immediately after the process is created and
|
34
|
+
# before any data is read in using the self.read_hash or self.read_array
|
35
|
+
# methods.
|
36
|
+
attr_reader :field_names
|
37
|
+
|
38
|
+
# LogParser reports parse errors when it can't do something it expects to.
|
39
|
+
# One example is when access is denied to registry keys. All parse errors are
|
40
|
+
# stuffed into this array so they can be examined at the end of processing.
|
41
|
+
attr_reader :parse_errors
|
42
|
+
|
43
|
+
# Output when using the " -queryInfo" switch will be stored here
|
44
|
+
attr_reader :queryInfo
|
45
|
+
|
46
|
+
# Query statistics are turned on by default and controlled with the " -stats"
|
47
|
+
# switch. The output includes Elements Processed, Elements Output, and
|
48
|
+
# Execution time in seconds and is available in these class variables after
|
49
|
+
# all input has been read.
|
50
|
+
attr_reader :elements_processed, :elements_output, :execution_time
|
51
|
+
|
52
|
+
# Stores the location of the LogParser.exe file.
|
53
|
+
attr_writer :logparser_exe
|
54
|
+
|
55
|
+
# Initialize is used to check that the Log Parser executable can be found
|
56
|
+
# and to config the various command line defaults.
|
57
|
+
def initialize(lp_executable = nil)
|
58
|
+
if self.valid_executable(lp_executable) == false
|
59
|
+
raise "The LogParser executable file cannot be found."
|
60
|
+
end
|
61
|
+
self.set_defaults
|
62
|
+
end
|
63
|
+
|
64
|
+
# Simple test to make sure the LogParser executable can be found. By default, it
|
65
|
+
# checks the IIS Resource Kit install path, the stand-alone LogParser install path,
|
66
|
+
# and then if the LogParser.exe file is found on the path.
|
67
|
+
# If an executable path is provided, this is tried instead allowing the user to
|
68
|
+
# pick the desired path if one or more LogParser versions are installed.
|
69
|
+
def valid_executable(lp_executable = nil)
|
70
|
+
found = false
|
71
|
+
if lp_executable.nil? then
|
72
|
+
file_locations = ['c:\Program Files\IIS Resources\Log Parser 2.2\LogParser.exe', 'c:\Program Files\Log Parser 2.2\LogParser.exe', 'LogParser.exe']
|
73
|
+
else
|
74
|
+
file_locations = [ lp_executable ]
|
75
|
+
end
|
76
|
+
|
77
|
+
file_locations.each {|location|
|
78
|
+
if File::executable? location then
|
79
|
+
found = true
|
80
|
+
@logparser_exe = location
|
81
|
+
end
|
82
|
+
}
|
83
|
+
return found
|
84
|
+
end
|
85
|
+
|
86
|
+
# LogParser has a variety of switches to control the input, output, SQL and
|
87
|
+
# process options. Most can be viewed by opening up LogParser at a command
|
88
|
+
# prompt and typing "c:\Logparser> logparser -h". There are additional
|
89
|
+
# switches that are not documented in the "logparser -h" output used to for
|
90
|
+
# controlling things like chart appearance. See the self.open_query method
|
91
|
+
# for more information.
|
92
|
+
def set_defaults
|
93
|
+
@cmd_options = {
|
94
|
+
'sql' => nil, # SQL query empty to start with
|
95
|
+
'i' => nil, # input mode
|
96
|
+
'o' => 'CSV', # output mode
|
97
|
+
'q' => false, # quiet mode - when true field headings supressed
|
98
|
+
'e' => -1, # max number of errors
|
99
|
+
'iw' => false, # ignore warnings
|
100
|
+
'stats' => true, # display statistics after executing query
|
101
|
+
'c' => false, # use built-in conversion query
|
102
|
+
'multiSite' => false, # send BIN conversion output to multiple files - see help
|
103
|
+
'saveDefaults' => false, # save specificed options as default values
|
104
|
+
'restoreDefaults' => false, # restore factory defaults
|
105
|
+
'rtp' => 0, # -1 turns off "Press a key..." prompts normally found in DOS window when NAT output selected
|
106
|
+
'queryInfo' => false # display query processing info
|
107
|
+
}
|
108
|
+
return @cmd_options
|
109
|
+
end
|
110
|
+
|
111
|
+
# Input formats are checked against valid formats to catch simple errors
|
112
|
+
def valid_input_format(input)
|
113
|
+
if input.nil? then
|
114
|
+
return true
|
115
|
+
end
|
116
|
+
input_formats = %w(IISW3C NCSA IIS IISODBC BIN IISMSID HTTPERR URLSCAN
|
117
|
+
CSV TSV W3C XML EVT ETW NETMON REG ADS TEXTLINE
|
118
|
+
TEXTWORD FS COM)
|
119
|
+
input_formats.each {|format| return true if format==input.upcase}
|
120
|
+
return false
|
121
|
+
end
|
122
|
+
|
123
|
+
# Output formats are checked against valid formats to catch simple errors
|
124
|
+
def valid_output_format(output)
|
125
|
+
if output.nil? then
|
126
|
+
return true
|
127
|
+
end
|
128
|
+
output_formats = %w(CSV TSV XML DATAGRID CHART SYSLOG NEUROVIEW NAT
|
129
|
+
W3C IIS SQL TPL NULL)
|
130
|
+
output_formats.each{|format| return true if format==output.upcase}
|
131
|
+
return false
|
132
|
+
end
|
133
|
+
|
134
|
+
# The LogParser process is opened using a string formatted for a command line.
|
135
|
+
# All of the command line options can be viewed in more detail by running
|
136
|
+
# "logparser -h" to see online help. The command line string is built using
|
137
|
+
# the currently configured options including any values in the input hash
|
138
|
+
# used to override or add specific functionality.
|
139
|
+
def build_command_string(options = nil)
|
140
|
+
@cmd_string = nil
|
141
|
+
if !options.nil? then
|
142
|
+
options.each {|key, value| @cmd_options[key] = value}
|
143
|
+
end
|
144
|
+
if (valid_input_format(@cmd_options['i']) &&
|
145
|
+
valid_output_format(@cmd_options['o']) &&
|
146
|
+
@cmd_options['sql'] != nil) then
|
147
|
+
@cmd_string = "\"#{@logparser_exe}\""
|
148
|
+
@cmd_string += " -i:#{@cmd_options['i']}" unless @cmd_options['i'].nil?
|
149
|
+
@cmd_options['sql'].gsub!(/[\r\n]/, ' ') # remove any linefeeds from formatted SQL statement
|
150
|
+
@cmd_string += " \"#{@cmd_options['sql']}\""
|
151
|
+
@cmd_string += " -o:#{@cmd_options['o']}" unless @cmd_options['o'].nil?
|
152
|
+
@cmd_string += " -q:ON" if cmd_options['q']
|
153
|
+
@cmd_string += " -e:#{@cmd_options['e']}" if @cmd_options['e'] >= 0
|
154
|
+
@cmd_string += " -iw:ON" if @cmd_options['iw']
|
155
|
+
@cmd_string += " -stats:OFF" if @cmd_options['stats'] == false
|
156
|
+
@cmd_string += " -c" if @cmd_options['c']
|
157
|
+
@cmd_string += " -multiSite:ON" if @cmd_options['multiSite']
|
158
|
+
@cmd_string += " -saveDefaults" if @cmd_options['saveDefaults']
|
159
|
+
@cmd_string += " -restoreDefaults" if @cmd_options['restoreDefaults']
|
160
|
+
@cmd_string += " -rtp:#{@cmd_options['rtp']}" if @cmd_options['rtp'] != 0 # turns off "Press a key"
|
161
|
+
@cmd_string += " -queryInfo" if @cmd_options['queryInfo']
|
162
|
+
end
|
163
|
+
return @cmd_string
|
164
|
+
end
|
165
|
+
|
166
|
+
# LogParser needs a data input source, a SQL selection query, and an output
|
167
|
+
# location. A hash can also be passed to override default values or provide
|
168
|
+
# additional command line switches. An interesting exception that needs to be
|
169
|
+
# handled is LogParser's "-queryInfo" switch which makes LogParser evaluate
|
170
|
+
# the input parameters and return info on the various column datatypes.
|
171
|
+
def open_query(input, query, output, options)
|
172
|
+
@cmd_options['i'] = input
|
173
|
+
@cmd_options['sql'] = query
|
174
|
+
@cmd_options['o'] = output
|
175
|
+
@cmd_string = self.build_command_string(options)
|
176
|
+
|
177
|
+
self.create_process
|
178
|
+
|
179
|
+
if @cmd_options['queryInfo'] == true then
|
180
|
+
@queryInfo = []
|
181
|
+
@logparser_pipe.each do |line|
|
182
|
+
@queryInfo << line
|
183
|
+
end
|
184
|
+
elsif @cmd_options['q'] == false then
|
185
|
+
self.field_definitions
|
186
|
+
end
|
187
|
+
end
|
188
|
+
|
189
|
+
# The LogParser process is created using IO.popen and then subsequent output
|
190
|
+
# from LogParser is read via a pipe.
|
191
|
+
def create_process
|
192
|
+
begin
|
193
|
+
@logparser_pipe = IO.popen(@cmd_string, "rb")
|
194
|
+
rescue Exception => e
|
195
|
+
puts "Exit status is: #{e.status}"
|
196
|
+
end
|
197
|
+
end
|
198
|
+
|
199
|
+
# Reads a single line of output from the LogParser pipe until an EOF is
|
200
|
+
# detected. LogParser will first output all of the query data and then a "\r\n"
|
201
|
+
# blank line followed by any errors or query statistics. Hence, once this
|
202
|
+
# method detects the blank line, all remaining LogParser output is expected
|
203
|
+
# to be error or statistic related.
|
204
|
+
def readline
|
205
|
+
begin
|
206
|
+
line = @logparser_pipe.readline
|
207
|
+
if line == "\r\n" then
|
208
|
+
line_errors_and_stats(line)
|
209
|
+
else
|
210
|
+
line.chomp!
|
211
|
+
return line
|
212
|
+
end
|
213
|
+
rescue EOFError
|
214
|
+
return nil
|
215
|
+
end
|
216
|
+
end
|
217
|
+
|
218
|
+
# The first line of output contains the field names (by default). This
|
219
|
+
# information is used to name the various hash keys when data is returned in
|
220
|
+
# a hash.
|
221
|
+
def field_definitions
|
222
|
+
@field_names = Array.new
|
223
|
+
names_line = self.readline
|
224
|
+
if !names_line.nil? then
|
225
|
+
@field_names = names_line.split(/,/)
|
226
|
+
end
|
227
|
+
end
|
228
|
+
|
229
|
+
# The output from LogParser is return in comma separated value (CSV) format.
|
230
|
+
# Using split to determine the fields is much faster than CSV. Occasionally
|
231
|
+
# however, a field will have a comma in it which will cause split to break
|
232
|
+
# a field in two. In cases where the actual number of fields returned by split
|
233
|
+
# does not match the expected number, the CSV library is called to try the
|
234
|
+
# split. This usually fixes the problem.
|
235
|
+
def read_array_line
|
236
|
+
next_line = self.readline
|
237
|
+
if !next_line.nil? then
|
238
|
+
array_line = next_line.split(/,/)
|
239
|
+
if array_line.length == @field_names.length then
|
240
|
+
return array_line
|
241
|
+
else
|
242
|
+
CSV.parse(next_line) {|row|
|
243
|
+
if row.length == @field_names.length then
|
244
|
+
return row
|
245
|
+
else
|
246
|
+
line_errors_and_stats(next_line)
|
247
|
+
return nil
|
248
|
+
end
|
249
|
+
}
|
250
|
+
end
|
251
|
+
end
|
252
|
+
return nil
|
253
|
+
end
|
254
|
+
|
255
|
+
# Once LogParser finishes outputting query data, it then outputs any errors
|
256
|
+
# followed by query statistics (unless specifically turned off). Errors are
|
257
|
+
# pushed into the @parse_errors array and statistics are added to the
|
258
|
+
# appropriate class variables.
|
259
|
+
def line_errors_and_stats(line_input)
|
260
|
+
begin
|
261
|
+
while 1 > 0
|
262
|
+
case line_input
|
263
|
+
when /Task completed with parse errors/
|
264
|
+
@parse_errors = Array.new
|
265
|
+
until ((line_input = @logparser_pipe.readline) == "\r\n")
|
266
|
+
@parse_errors.push(line_input)
|
267
|
+
end
|
268
|
+
when /Statistics:/
|
269
|
+
while line = @logparser_pipe.readline
|
270
|
+
case line
|
271
|
+
when /Elements processed:\s*([\d,]+)\n*/i
|
272
|
+
@elements_processed = $1
|
273
|
+
when /Elements output:\s*([\d,]+)\n*/i
|
274
|
+
@elements_output = $1
|
275
|
+
when /Execution time:\s*([\d\.]+) seconds\n*/i
|
276
|
+
@execution_time = $1
|
277
|
+
end
|
278
|
+
end
|
279
|
+
end
|
280
|
+
line_input = @logparser_pipe.readline
|
281
|
+
end
|
282
|
+
rescue EOFError
|
283
|
+
return nil
|
284
|
+
end
|
285
|
+
end
|
286
|
+
|
287
|
+
# To return the data in a hash, the data line is first read into an array.
|
288
|
+
# Then the field keys (initially loaded by the field_definitions method when
|
289
|
+
# the LogParser process was created), are matched up with the respective data
|
290
|
+
# values.
|
291
|
+
def read_hash
|
292
|
+
data_hash = Hash.new
|
293
|
+
data_array = read_array_line
|
294
|
+
if !data_array.nil? then
|
295
|
+
@field_names.each_index {|x|
|
296
|
+
data_hash[@field_names[x]] = data_array[x]
|
297
|
+
}
|
298
|
+
return data_hash
|
299
|
+
else
|
300
|
+
return nil
|
301
|
+
end
|
302
|
+
end
|
303
|
+
|
304
|
+
# When data is best processed all at once, this method will return an array
|
305
|
+
# of arrays. The array will have a header row with the field names by default
|
306
|
+
# unless LogParser is started in "quiet" mode (i.e. -q:ON).
|
307
|
+
def read_array(blnHeader_row)
|
308
|
+
data_array = Array.new
|
309
|
+
i=0
|
310
|
+
if blnHeader_row
|
311
|
+
data_array[i] = @field_names
|
312
|
+
i += 1
|
313
|
+
end
|
314
|
+
while array = self.read_array_line do
|
315
|
+
data_array[i] = array
|
316
|
+
i += 1
|
317
|
+
end
|
318
|
+
return data_array
|
319
|
+
end
|
320
|
+
|
321
|
+
# A simple convenience method when running LogParser using the -queryInfo
|
322
|
+
# switch. The resulting output is combined into a single string for easier
|
323
|
+
# printing.
|
324
|
+
def formatted_queryInfo
|
325
|
+
temp = String.new
|
326
|
+
@queryInfo.each{|line| temp << line}
|
327
|
+
return temp
|
328
|
+
end
|
329
|
+
|
330
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
#!/usr/bin/env ruby -w
|
2
|
+
|
3
|
+
$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
|
4
|
+
require 'test/unit'
|
5
|
+
require 'rubylogparser.rb'
|
6
|
+
|
7
|
+
class Test_RubyLogParser < Test::Unit::TestCase
|
8
|
+
def setup
|
9
|
+
@logparser = RubyLogParser.new
|
10
|
+
end
|
11
|
+
|
12
|
+
def test_valid_executable
|
13
|
+
assert_equal true, @logparser.valid_executable(nil)
|
14
|
+
assert_equal false, @logparser.valid_executable('c:\someotherfilename.exe')
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_set_defaults
|
18
|
+
test_defaults = @logparser.set_defaults
|
19
|
+
assert_equal 'CSV', test_defaults['o']
|
20
|
+
assert_equal false, test_defaults['queryInfo']
|
21
|
+
end
|
22
|
+
|
23
|
+
def test_valid_input_type
|
24
|
+
assert_equal true, @logparser.valid_input_format("CSV")
|
25
|
+
assert_equal true, @logparser.valid_input_format("csv")
|
26
|
+
assert_equal false, @logparser.valid_input_format("xxx")
|
27
|
+
end
|
28
|
+
|
29
|
+
def test_valid_output_type
|
30
|
+
assert_equal true, @logparser.valid_output_format("CSV")
|
31
|
+
assert_equal true, @logparser.valid_output_format("csv")
|
32
|
+
assert_equal false, @logparser.valid_output_format("xxx")
|
33
|
+
end
|
34
|
+
|
35
|
+
def test_build_command_string
|
36
|
+
@logparser.cmd_options['i'] = "EVT"
|
37
|
+
@logparser.cmd_options['sql'] = "SQL Test"
|
38
|
+
@logparser.cmd_options['o'] = "CSV"
|
39
|
+
@logparser.logparser_exe = 'LogParser.exe'
|
40
|
+
assert_equal '"LogParser.exe" -i:EVT "SQL Test" -o:CSV -queryInfo', @logparser.build_command_string({'queryInfo' => true})
|
41
|
+
end
|
42
|
+
|
43
|
+
end
|
metadata
ADDED
@@ -0,0 +1,70 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
rubygems_version: 0.9.4
|
3
|
+
specification_version: 1
|
4
|
+
name: rubylogparser
|
5
|
+
version: !ruby/object:Gem::Version
|
6
|
+
version: 0.1.0
|
7
|
+
date: 2008-01-11 00:00:00 -08:00
|
8
|
+
summary: RubyLogParser provides a wrapper around Microsoft's Log Parser executable.
|
9
|
+
require_paths:
|
10
|
+
- lib
|
11
|
+
email: jimclark@ieee.org
|
12
|
+
homepage: http://rubyforge.org/projects/rubylogparser/
|
13
|
+
rubyforge_project: rubylogparser
|
14
|
+
description: RubyLogParser enables the output from Microsoft's Log Parser to be processed by Ruby data structures (arrays and hashes).
|
15
|
+
autorequire:
|
16
|
+
default_executable:
|
17
|
+
bindir: bin
|
18
|
+
has_rdoc: true
|
19
|
+
required_ruby_version: !ruby/object:Gem::Version::Requirement
|
20
|
+
requirements:
|
21
|
+
- - ">"
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: 0.0.0
|
24
|
+
version:
|
25
|
+
platform: ruby
|
26
|
+
signing_key:
|
27
|
+
cert_chain:
|
28
|
+
post_install_message:
|
29
|
+
authors:
|
30
|
+
- Jim Clark
|
31
|
+
files:
|
32
|
+
- EXAMPLES.txt
|
33
|
+
- GUIDE.txt
|
34
|
+
- History.txt
|
35
|
+
- Manifest.txt
|
36
|
+
- README.txt
|
37
|
+
- Rakefile
|
38
|
+
- examples/event_log.rb
|
39
|
+
- examples/files.rb
|
40
|
+
- examples/queryinfo.rb
|
41
|
+
- examples/registry.rb
|
42
|
+
- lib/rubylogparser.rb
|
43
|
+
- test/test_rubylogparser.rb
|
44
|
+
test_files:
|
45
|
+
- test/test_rubylogparser.rb
|
46
|
+
rdoc_options:
|
47
|
+
- --main
|
48
|
+
- README.txt
|
49
|
+
extra_rdoc_files:
|
50
|
+
- EXAMPLES.txt
|
51
|
+
- GUIDE.txt
|
52
|
+
- History.txt
|
53
|
+
- Manifest.txt
|
54
|
+
- README.txt
|
55
|
+
executables: []
|
56
|
+
|
57
|
+
extensions: []
|
58
|
+
|
59
|
+
requirements: []
|
60
|
+
|
61
|
+
dependencies:
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: hoe
|
64
|
+
version_requirement:
|
65
|
+
version_requirements: !ruby/object:Gem::Version::Requirement
|
66
|
+
requirements:
|
67
|
+
- - ">="
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: 1.4.0
|
70
|
+
version:
|