indy 0.1.1 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,39 @@
1
+ class Indy
2
+
3
+ #
4
+ # LogFormats defines the building blocks of Indy's built in log patterns.
5
+ #
6
+ # See indy/patterns.rb for the following Constants:
7
+ # Indy::COMMON_LOG_PATTERN
8
+ # Indy::COMBINED_LOG_PATTERN
9
+ # Indy::LOG4R_DEFAULT_PATTERN
10
+ #
11
+ module LogFormats
12
+
13
+ IPV4_REGEXP = '(\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3})'
14
+ SPACE_DELIM_REGEXP = '([^\s]+)'
15
+ BRACKET_DELIM_REGEXP = '\[([^\]]+)\]'
16
+ HTTP_REQUEST_REGEXP = '"([A-Z]+ [^\s]+ [^"]+)"'
17
+ HTTP_STATUS_REGEXP = '(\d{3})'
18
+ NUMBER_REGEXP = '(\d+)'
19
+ DQUOTE_DELIM_REGEXP = '"([^"]+)"'
20
+
21
+ DEFAULT_DATE_TIME = '\d{4}.\d{2}.\d{2}\s+\d{2}.\d{2}.\d{2}' #"%Y-%m-%d %H:%M:%S"
22
+ DEFAULT_SEVERITY = [:trace,:debug,:info,:warn,:error,:fatal]
23
+ DEFAULT_SEVERITY_PATTERN = "(?:#{DEFAULT_SEVERITY.map{|s| s.to_s.upcase}.join("|")})"
24
+ DEFAULT_APPLICATION = '\w+'
25
+ DEFAULT_MESSAGE = '.+'
26
+
27
+ DEFAULT_LOG_REGEXP = /^(#{DEFAULT_DATE_TIME})\s+(#{DEFAULT_SEVERITY_PATTERN})\s+(#{DEFAULT_APPLICATION})\s+-\s+(#{DEFAULT_MESSAGE})$/
28
+ DEFAULT_LOG_FIELDS = [:time,:severity,:application,:message]
29
+
30
+ COMMON_FIELDS = [:host, :ident, :authuser, :time, :request, :status, :bytes]
31
+ COMMON_REGEXP = /^#{IPV4_REGEXP} #{SPACE_DELIM_REGEXP} #{SPACE_DELIM_REGEXP} #{BRACKET_DELIM_REGEXP} #{DQUOTE_DELIM_REGEXP} #{HTTP_STATUS_REGEXP} #{NUMBER_REGEXP}$/
32
+
33
+ COMBINED_FIELDS = COMMON_FIELDS + [:referrer, :user_agent]
34
+ COMBINED_REGEXP = /^#{IPV4_REGEXP} #{SPACE_DELIM_REGEXP} #{SPACE_DELIM_REGEXP} #{BRACKET_DELIM_REGEXP} #{DQUOTE_DELIM_REGEXP} #{HTTP_STATUS_REGEXP} #{NUMBER_REGEXP} #{DQUOTE_DELIM_REGEXP} #{DQUOTE_DELIM_REGEXP}$/
35
+
36
+ LOG4R_DEFAULT_FIELDS = [:level, :application, :message]
37
+ LOG4R_DEFAULT_REGEXP = /(?:\s+)?([A-Z]+) (\S+): (.*)$/
38
+ end
39
+ end
@@ -0,0 +1,25 @@
1
+ class Indy
2
+
3
+ #
4
+ # Indy default log format @pattern
5
+ # e.g.:
6
+ # INFO 2000-09-07 MyApp - Entering APPLICATION.
7
+ #
8
+ DEFAULT_LOG_PATTERN = [LogFormats::DEFAULT_LOG_REGEXP, LogFormats::DEFAULT_LOG_FIELDS].flatten
9
+
10
+ #
11
+ # Uncustomized Log4r log @pattern
12
+ #
13
+ LOG4R_DEFAULT_PATTERN = [LogFormats::LOG4R_DEFAULT_REGEXP, LogFormats::LOG4R_DEFAULT_FIELDS].flatten
14
+
15
+ #
16
+ # NCSA Common Log Format log @pattern
17
+ #
18
+ COMMON_LOG_PATTERN = [LogFormats::COMMON_REGEXP, LogFormats::COMMON_FIELDS].flatten
19
+
20
+ #
21
+ # NCSA Combined Log Format log @pattern
22
+ #
23
+ COMBINED_LOG_PATTERN = [LogFormats::COMBINED_REGEXP, LogFormats::COMBINED_FIELDS].flatten
24
+
25
+ end
@@ -3,9 +3,9 @@ require "#{File.dirname(__FILE__)}/helper"
3
3
  describe "Search Performance" do
4
4
 
5
5
 
6
- context "with a small data set" do
6
+ context "with a 420 line log file" do
7
7
 
8
- longer_subject = [
8
+ log_content = [
9
9
  "2000-09-07 14:07:41 INFO MyApp - Entering application.\n",
10
10
  "2000-09-07 14:07:42 DEBUG MyApp - Focusing application.\n",
11
11
  "2000-09-07 14:07:43 DEBUG MyApp - Blurring application.\n",
@@ -16,16 +16,16 @@ describe "Search Performance" do
16
16
 
17
17
  profile :file => STDOUT, :printer => :flat, :min_percent => 1 do
18
18
 
19
- it "should perform well using #for(:all)" do
20
- Indy.search(longer_subject.dup).for(:all)
19
+ it "should profile code using #for(:all)" do
20
+ Indy.search(log_content.dup).for(:all)
21
21
  end
22
22
 
23
- it "should perform well using #for(:field => 'value')" do
24
- Indy.search(longer_subject.dup).for(:severity => 'INFO')
23
+ it "should profile code using #for(:field => 'value')" do
24
+ Indy.search(log_content.dup).for(:severity => 'INFO')
25
25
  end
26
26
 
27
- it "should perform well using #time()" do
28
- Indy.search(longer_subject.dup).after(:time => "2000-09-07 14:07:45").for(:all)
27
+ it "should profile code using #time()" do
28
+ Indy.search(log_content.dup).after(:time => "2000-09-07 14:07:45").for(:all)
29
29
  end
30
30
 
31
31
  end
@@ -89,16 +89,17 @@ describe Indy do
89
89
  context "treat it first like a file" do
90
90
 
91
91
  it "should attempt to open the file" do
92
- IO.should_receive(:open).with("possible_file.ext").ordered
92
+ File.should_receive(:exist?).with("possible_file.ext").ordered
93
93
  Indy.search("possible_file.ext")
94
94
  end
95
95
 
96
- it "should not throw an error for an invalid file" do
96
+ it "should not throw an error for a non-existent file" do
97
97
  lambda { Indy.search("possible_file.ext") }.should_not raise_error
98
98
  end
99
99
 
100
100
  it "should return an IO object when there is a file" do
101
- IO.should_receive(:open).with("file_exists.ext").and_return(StringIO.new("2000-09-07 14:07:41 INFO MyApp - Entering APPLICATION."))
101
+ File.should_receive(:exist?).with("file_exists.ext").and_return( true )
102
+ File.should_receive(:open).and_return(StringIO.new("2000-09-07 14:07:41 INFO MyApp - Entering APPLICATION."))
102
103
  Indy.search("file_exists.ext").for(:application => 'MyApp').length.should == 1
103
104
  end
104
105
 
@@ -111,13 +112,10 @@ describe Indy do
111
112
  context "treat it second like a string" do
112
113
 
113
114
  it "should attempt to treat it as a string" do
114
- expecting_string = mock("String With Expectation")
115
- expecting_string.should_receive(:[])
116
- expecting_string.should_receive(:to_s).and_return("2000-09-07 14:07:41 INFO MyApp - Entering APPLICATION.")
117
-
118
- IO.should_receive(:open).with(expecting_string).ordered
119
-
120
- Indy.search(expecting_string).for(:application => 'MyApp').length.should == 1
115
+ string = "2000-09-07 14:07:41 INFO MyApp - Entering APPLICATION."
116
+ string_io = StringIO.new(string)
117
+ StringIO.should_receive(:new).with(string).ordered.and_return(string_io)
118
+ Indy.search(string).for(:application => 'MyApp').length.should == 1
121
119
  end
122
120
 
123
121
  end
@@ -161,21 +159,23 @@ describe Indy do
161
159
  @indy.should respond_to(:with)
162
160
  end
163
161
 
164
- # http://log4r.rubyforge.org/rdoc/Log4r/rdoc/patternformatter.html
165
- it "with() should accept a log4r pattern string without error" do
166
- lambda { @indy.with(["(%d) (%i) (%c) - (%m)", :time, :info, :class, :message]) }.should_not raise_error
162
+ it "with() should accept the log4r default pattern const without error" do
163
+ lambda { @indy.with(Indy::LOG4R_DEFAULT_PATTERN) }.should_not raise_error
167
164
  end
168
165
 
169
- # http://logging.apache.org/log4j/1.2/apidocs/org/apache/log4j/PatternLayout.html
170
- it "with() should accept a log4j pattern string without error" do
171
- lambda { @indy.with(["(%d) (%i) (%c) - (%m)", :time, :info, :class, :message])}.should_not raise_error
166
+ it "with() should accept :default without error" do
167
+ lambda { @indy.with(:default) }.should_not raise_error
168
+ end
169
+
170
+ it "with() should accept no params without error" do
171
+ lambda { @indy.with() }.should_not raise_error
172
172
  end
173
173
 
174
174
  it "should return itself" do
175
- @indy.with(["(%d) (%i) (%c) - (%m)", :time, :info, :class, :message]).should == @indy
175
+ @indy.with(:default).should == @indy
176
176
  end
177
177
 
178
- [:for, :search, :like, :matching].each do |method|
178
+ [:for, :like, :matching].each do |method|
179
179
  it "#{method}() should exist" do
180
180
  @indy.should respond_to(method)
181
181
  end
@@ -190,10 +190,10 @@ describe Indy do
190
190
 
191
191
  end
192
192
 
193
- context "_search when given source, param and value" do
193
+ context "_search" do
194
194
 
195
195
  before(:each) do
196
- @results = @indy.send(:_search, StringIO.new("2000-09-07 14:07:41 INFO MyApp - Entering APPLICATION."),[Indy::DEFAULT_LOG_PATTERN, Indy::DEFAULT_LOG_FIELDS].flatten) {|result| result if result[:application] == "MyApp" }
196
+ @results = @indy.send(:_search) {|result| result if result[:application] == "MyApp" }
197
197
  end
198
198
 
199
199
  it "should not return nil" do
@@ -0,0 +1,58 @@
1
+ require "#{File.dirname(__FILE__)}/helper"
2
+
3
+ describe Indy do
4
+
5
+ context "common logging format" do
6
+
7
+ common_log_pattern = {
8
+ :name => 'common_log_pattern',
9
+ :source => ["127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] \"GET /apache_pb1.gif HTTP/1.0\" 200 2326",
10
+ "127.0.0.1 - louie [10/Oct/2000:13:55:37 -0700] \"GET /apache_pb2.gif HTTP/1.0\" 200 2327",
11
+ "127.0.0.1 - frank [10/Oct/2000:13:55:38 -0700] \"GET /apache_pb3.gif HTTP/1.0\" 404 300"].join("\n"),
12
+ :regexp => Indy::LogFormats::COMMON_REGEXP,
13
+ :fields => Indy::LogFormats::COMMON_FIELDS,
14
+ :test_field => :authuser
15
+ }
16
+
17
+ combined_log_pattern = {
18
+ :name => 'combined_log_pattern',
19
+ :source => ["127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] \"GET /apache_pb1.gif HTTP/1.0\" 200 2326 \"http://www.example.com/start.html\" \"Mozilla/4.08 [en] (Win98; I ;Nav)\"",
20
+ "127.0.0.1 - louie [10/Oct/2000:13:55:37 -0700] \"GET /apache_pb2.gif HTTP/1.0\" 200 2327 \"http://www.example.com/start.html\" \"Mozilla/4.08 [en] (Win98; I ;Nav)\"",
21
+ "127.0.0.1 - frank [10/Oct/2000:13:55:38 -0700] \"GET /apache_pb3.gif HTTP/1.0\" 404 300 \"http://www.example.com/start.html\" \"Mozilla/4.08 [en] (Win98; I ;Nav)\""].join("\n"),
22
+ :regexp => Indy::LogFormats::COMBINED_REGEXP,
23
+ :fields => Indy::LogFormats::COMBINED_FIELDS,
24
+ :test_field => :authuser
25
+ }
26
+
27
+ log4r_default_pattern = {
28
+ :name => 'log4r_default_pattern',
29
+ :source => ["DEBUG mylog: This is a message with level DEBUG",
30
+ " INFO mylog: This is a message with level INFO",
31
+ " WARN louie: This is a message with level WARN",
32
+ "ERROR mylog: This is a message with level ERROR",
33
+ "FATAL mylog: This is a message with level FATAL"].join("\n"),
34
+ :regexp => Indy::LogFormats::LOG4R_DEFAULT_REGEXP,
35
+ :fields => Indy::LogFormats::LOG4R_DEFAULT_FIELDS,
36
+ :test_field => :application
37
+ }
38
+
39
+ [ common_log_pattern,
40
+ combined_log_pattern,
41
+ log4r_default_pattern ].each do |format|
42
+
43
+ it "#{format[:name]} should work" do
44
+ indy = Indy.new(:source => format[:source], :pattern => [format[:regexp],format[:fields]].flatten)
45
+ result = indy.for(format[:test_field] => 'louie')
46
+ result.count.should == 1
47
+ end
48
+
49
+ it "#{format[:name]} @pattern can be set to the Indy::LogFormat const" do
50
+ indy = Indy.new(:source => format[:source], :pattern => eval('Indy::' + format[:name].upcase))
51
+ result = indy.for(format[:test_field] => 'louie')
52
+ result.count.should == 1
53
+
54
+ end
55
+ end
56
+ end
57
+
58
+ end
@@ -24,6 +24,29 @@ describe Indy do
24
24
 
25
25
  end
26
26
 
27
+ context "file operations" do
28
+
29
+ before(:all) do
30
+ end
31
+
32
+ it "should open and close file once per search" do
33
+ file_path = File.join( File.dirname(__FILE__), 'data.log')
34
+ file_io1 = File.open(file_path)
35
+ file_io2 = File.open(file_path)
36
+
37
+ File.should_receive( :exist? ).with(file_path).ordered.and_return(true)
38
+ File.should_receive( :open ).ordered.and_return( file_io1 )
39
+ file_io1.should_receive( :close ).ordered
40
+ File.should_receive( :open ).ordered.and_return( file_io2 )
41
+ file_io2.should_receive( :close ).ordered
42
+
43
+ @indy = Indy.search(file_path)
44
+ @indy.for(:all).length.should == 2
45
+ @indy.for(:all).length.should == 2
46
+ end
47
+
48
+ end
49
+
27
50
  context "search file" do
28
51
 
29
52
  before(:all) do
@@ -80,15 +103,12 @@ describe Indy do
80
103
  it "should execute cmd on each successive search" do
81
104
  @indy.for(:application => 'MyApp').length.should == 2
82
105
  @indy.for(:severity => 'INFO').length.should == 3
83
- @indy.for(:application => 'MyApp').length.should == 2
84
- end
85
106
 
86
- it "should execute cmd on each successive search" do
87
- @file.write("\n2000-09-07 14:10:55 INFO MyApp - really really Exiting APPLICATION.\n")
107
+ @file.write("\n2000-09-07 14:10:55 DEBUG MyApp - really really Exiting APPLICATION.\n")
88
108
  @file.flush
109
+
89
110
  @indy.for(:application => 'MyApp').length.should == 3
90
- @indy.for(:severity => 'INFO').length.should == 4
91
- @indy.for(:application => 'MyApp').length.should == 3
111
+ @indy.for(:severity => 'INFO').length.should == 3
92
112
  end
93
113
 
94
114
 
@@ -13,6 +13,22 @@ describe Indy do
13
13
  @indy.send(:parse_date, line_hash).class.should == DateTime
14
14
  end
15
15
 
16
+ it "should parse dates when log includes non-conforming data" do
17
+ logdata = [ "12-03-2000 message1",
18
+ "13-03-2000 message2",
19
+ "14-03-2000 ",
20
+ " message4",
21
+ "a14-03-2000 message5",
22
+ "14-03-2000 message6\n\n\n\n",
23
+ "15-03-2000 message7",
24
+ "16-03-2000 message8\r\n",
25
+ "17-03-2000 message9"].join("\n")
26
+ @indy = Indy.new(:source => logdata, :pattern => ['^(\d[^\s]+\d) (.+)$', :time, :message])
27
+ @indy.after(:time => '13-03-2000')
28
+ @indy.for(:all).count.should == 4
29
+ end
30
+
31
+
16
32
  end
17
33
 
18
34
  context "non-default time handling" do
metadata CHANGED
@@ -1,13 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: indy
3
3
  version: !ruby/object:Gem::Version
4
- hash: 25
4
+ hash: 31
5
5
  prerelease:
6
6
  segments:
7
7
  - 0
8
8
  - 1
9
- - 1
10
- version: 0.1.1
9
+ - 2
10
+ version: 0.1.2
11
11
  platform: ruby
12
12
  authors:
13
13
  - Franklin Webber
@@ -16,7 +16,7 @@ autorequire:
16
16
  bindir: bin
17
17
  cert_chain: []
18
18
 
19
- date: 2011-01-13 00:00:00 -08:00
19
+ date: 2011-01-17 00:00:00 -08:00
20
20
  default_executable:
21
21
  dependencies:
22
22
  - !ruby/object:Gem::Dependency
@@ -35,7 +35,103 @@ dependencies:
35
35
  version: 2.3.5
36
36
  type: :runtime
37
37
  version_requirements: *id001
38
- description: " Indy is a log archelogy tool that allows you to search through log files. "
38
+ - !ruby/object:Gem::Dependency
39
+ name: cucumber
40
+ prerelease: false
41
+ requirement: &id002 !ruby/object:Gem::Requirement
42
+ none: false
43
+ requirements:
44
+ - - ">="
45
+ - !ruby/object:Gem::Version
46
+ hash: 63
47
+ segments:
48
+ - 0
49
+ - 9
50
+ - 2
51
+ version: 0.9.2
52
+ type: :development
53
+ version_requirements: *id002
54
+ - !ruby/object:Gem::Dependency
55
+ name: rspec
56
+ prerelease: false
57
+ requirement: &id003 !ruby/object:Gem::Requirement
58
+ none: false
59
+ requirements:
60
+ - - ">="
61
+ - !ruby/object:Gem::Version
62
+ hash: 31
63
+ segments:
64
+ - 2
65
+ - 4
66
+ - 0
67
+ version: 2.4.0
68
+ type: :development
69
+ version_requirements: *id003
70
+ - !ruby/object:Gem::Dependency
71
+ name: rspec-mocks
72
+ prerelease: false
73
+ requirement: &id004 !ruby/object:Gem::Requirement
74
+ none: false
75
+ requirements:
76
+ - - ">="
77
+ - !ruby/object:Gem::Version
78
+ hash: 31
79
+ segments:
80
+ - 2
81
+ - 4
82
+ - 0
83
+ version: 2.4.0
84
+ type: :development
85
+ version_requirements: *id004
86
+ - !ruby/object:Gem::Dependency
87
+ name: rspec-prof
88
+ prerelease: false
89
+ requirement: &id005 !ruby/object:Gem::Requirement
90
+ none: false
91
+ requirements:
92
+ - - ">="
93
+ - !ruby/object:Gem::Version
94
+ hash: 25
95
+ segments:
96
+ - 0
97
+ - 0
98
+ - 3
99
+ version: 0.0.3
100
+ type: :development
101
+ version_requirements: *id005
102
+ - !ruby/object:Gem::Dependency
103
+ name: rcov
104
+ prerelease: false
105
+ requirement: &id006 !ruby/object:Gem::Requirement
106
+ none: false
107
+ requirements:
108
+ - - ">="
109
+ - !ruby/object:Gem::Version
110
+ hash: 41
111
+ segments:
112
+ - 0
113
+ - 9
114
+ - 9
115
+ version: 0.9.9
116
+ type: :development
117
+ version_requirements: *id006
118
+ - !ruby/object:Gem::Dependency
119
+ name: flog
120
+ prerelease: false
121
+ requirement: &id007 !ruby/object:Gem::Requirement
122
+ none: false
123
+ requirements:
124
+ - - ">="
125
+ - !ruby/object:Gem::Version
126
+ hash: 27
127
+ segments:
128
+ - 2
129
+ - 5
130
+ - 0
131
+ version: 2.5.0
132
+ type: :development
133
+ version_requirements: *id007
134
+ description: " Indy is a log archelogy tool that allows you to interact with log data like an object while you search by fields and/or time."
39
135
  email: franklin.webber@gmail.com
40
136
  executables: []
41
137
 
@@ -63,7 +159,6 @@ files:
63
159
  - features/exact_mulitple_fields.feature
64
160
  - features/exact_time.feature
65
161
  - features/file.feature
66
- - features/log_levels.feature
67
162
  - features/message.feature
68
163
  - features/multiple_fields.feature
69
164
  - features/step_definitions/find_by.steps.rb
@@ -77,12 +172,15 @@ files:
77
172
  - indy.gemspec
78
173
  - lib/indy.rb
79
174
  - lib/indy/indy.rb
175
+ - lib/indy/log_formats.rb
176
+ - lib/indy/patterns.rb
80
177
  - lib/indy/result_set.rb
81
178
  - performance/helper.rb
82
179
  - performance/profile_spec.rb
83
180
  - spec/data.log
84
181
  - spec/helper.rb
85
182
  - spec/indy_spec.rb
183
+ - spec/log_format_spec.rb
86
184
  - spec/result_set_spec.rb
87
185
  - spec/search_spec.rb
88
186
  - spec/time_spec.rb
@@ -90,7 +188,7 @@ has_rdoc: true
90
188
  homepage: http://github.com/burtlo/Indy
91
189
  licenses:
92
190
  - MIT
93
- post_install_message: "\n [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>]\n\n Thank you for installing Indy 0.1.1 / 2011-01-13.\n\n Changes:\n \n * Scope search by time (after, before, around, or within)\n * Explicit format string can be provided to parse non-standard log times\n * Reopen (or rewind) source for each #search\n * Source parameter creates an IO object (StringIO, File, or Process)\n \n\n [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>]\n\n "
191
+ post_install_message: "\n [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>]\n\n Thank you for installing Indy 0.1.2 / 2011-01-18.\n\n Changes:\n \n * Predefined log formats for NCSA Common, NCSA Combined, and Log4r (default)\n * Source IO is explicitly closed after each #_search\n * Removed instance method #search; use #for.\n * Removed instance method #severity\n \n\n [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>] [<>]\n\n "
94
192
  rdoc_options:
95
193
  - --charset=UTF-8
96
194
  require_paths: