indy 0.3.4 → 0.4.0.pre

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,115 @@
1
+ class Indy
2
+
3
+ class LogDefinition
4
+
5
+ attr_accessor :entry_regexp, :entry_fields, :time_format, :multiline
6
+
7
+ def initialize(args=:default)
8
+ case args
9
+ when :default, {}
10
+ params_hash = set_defaults
11
+ when Array, Hash
12
+ params_hash = parse_enumerable_params(args)
13
+ end
14
+ raise ArgumentError, "Values for entry_regexp and/or entry_fields were not supplied" unless (params_hash[:entry_fields] && params_hash[:entry_regexp])
15
+ if params_hash[:multiline]
16
+ @entry_regexp = Regexp.new(params_hash[:entry_regexp], Regexp::MULTILINE)
17
+ @multiline = true
18
+ else
19
+ @entry_regexp = Regexp.new(params_hash[:entry_regexp])
20
+ end
21
+ @entry_fields = params_hash[:entry_fields]
22
+ @time_format = params_hash[:time_format]
23
+ define_struct
24
+ end
25
+
26
+ def set_defaults
27
+ params_hash = {}
28
+ params_hash[:entry_regexp] = Indy::LogFormats::DEFAULT_ENTRY_REGEXP
29
+ params_hash[:entry_fields] = Indy::LogFormats::DEFAULT_ENTRY_FIELDS
30
+ params_hash
31
+ end
32
+
33
+ def parse_enumerable_params(args)
34
+ params_hash = {}
35
+ params_hash.merge!(args)
36
+ if args.keys.include? :log_format
37
+ # support 0.3.4 params
38
+ params_hash[:entry_regexp] = args[:log_format][0]
39
+ params_hash[:entry_fields] = args[:log_format][1..-1]
40
+ params_hash.delete :log_format
41
+ end
42
+ params_hash
43
+ end
44
+
45
+ #
46
+ # Return a Struct::Entry object from a hash of values from a log entry
47
+ #
48
+ # @param [Hash] entry_hash a hash of :field_name => value pairs for one log entry
49
+ #
50
+ def create_struct( entry_hash )
51
+ values = entry_hash.keys.sort_by{|entry|entry.to_s}.collect {|key| entry_hash[key]}
52
+ result = Struct::Entry.new( *values )
53
+ result
54
+ end
55
+
56
+ #
57
+ # Define Struct::Entry with the fields from @log_definition. Ignore warnings.
58
+ #
59
+ def define_struct
60
+ fields = (@entry_fields + [:raw_entry]).sort_by{|key|key.to_s}
61
+ verbose = $VERBOSE
62
+ $VERBOSE = nil
63
+ Struct.new( "Entry", *fields )
64
+ $VERBOSE = verbose
65
+ end
66
+
67
+ #
68
+ # Convert log entry into hash
69
+ #
70
+ def entry_hash(values)
71
+ assert_valid_field_list(values) unless @field_list_is_valid # just do it once
72
+ raw_entry = values.shift
73
+ hash = Hash[ *@entry_fields.zip( values ).flatten ]
74
+ hash[:raw_entry] = raw_entry.strip
75
+ hash
76
+ end
77
+
78
+
79
+ #
80
+ # Return a hash of field=>value pairs for the log entry
81
+ #
82
+ # @param [String] raw_entry The raw log entry
83
+ #
84
+ def parse_entry(raw_entry)
85
+ match_data = /#{@entry_regexp}/.match(raw_entry)
86
+ return nil unless match_data
87
+ values = match_data.captures
88
+ entry_hash([raw_entry, values].flatten)
89
+ end
90
+
91
+ #
92
+ # Return a hash of field=>value pairs for the array of captured values from a log entry
93
+ #
94
+ # @param [Array] capture_array The array of values captured by the @log_definition.entry_regexp
95
+ #
96
+ def parse_entry_captures( capture_array )
97
+ entire_entry = capture_array.shift
98
+ values = capture_array
99
+ entry_hash([entire_entry, values].flatten)
100
+ end
101
+
102
+ #
103
+ # Ensure number of fields is expected
104
+ #
105
+ def assert_valid_field_list(values)
106
+ if values.length == @entry_fields.length + 1 # values also includes raw_entry
107
+ @field_list_is_valid = true
108
+ else
109
+ raise ArgumentError, "Field mismatch between log pattern and log data. The data is: '#{values.join(':::')}'"
110
+ end
111
+ end
112
+
113
+ end
114
+
115
+ end
@@ -23,8 +23,8 @@ class Indy
23
23
  DEFAULT_APPLICATION = '\w+'
24
24
  DEFAULT_MESSAGE = '.+'
25
25
 
26
- DEFAULT_LOG_REGEXP = /^(#{DEFAULT_DATE_TIME})\s+(#{DEFAULT_SEVERITY_PATTERN})\s+(#{DEFAULT_APPLICATION})\s+-\s+(#{DEFAULT_MESSAGE})$/
27
- DEFAULT_LOG_FIELDS = [:time,:severity,:application,:message]
26
+ DEFAULT_ENTRY_FIELDS = [:time,:severity,:application,:message]
27
+ DEFAULT_ENTRY_REGEXP = /^(#{DEFAULT_DATE_TIME})\s+(#{DEFAULT_SEVERITY_PATTERN})\s+(#{DEFAULT_APPLICATION})\s+-\s+(#{DEFAULT_MESSAGE})$/
28
28
 
29
29
  COMMON_FIELDS = [:host, :ident, :authuser, :time, :request, :status, :bytes]
30
30
  COMMON_REGEXP = /^#{IPV4_REGEXP} #{SPACE_DELIM_REGEXP} #{SPACE_DELIM_REGEXP} #{BRACKET_DELIM_REGEXP} #{DQUOTE_DELIM_REGEXP} #{HTTP_STATUS_REGEXP} #{NUMBER_REGEXP}$/
@@ -33,7 +33,10 @@ class Indy
33
33
  COMBINED_REGEXP = /^#{IPV4_REGEXP} #{SPACE_DELIM_REGEXP} #{SPACE_DELIM_REGEXP} #{BRACKET_DELIM_REGEXP} #{DQUOTE_DELIM_REGEXP} #{HTTP_STATUS_REGEXP} #{NUMBER_REGEXP} #{DQUOTE_DELIM_REGEXP} #{DQUOTE_DELIM_REGEXP}$/
34
34
 
35
35
  LOG4R_DEFAULT_FIELDS = [:level, :application, :message]
36
- LOG4R_DEFAULT_REGEXP = /(?:\s+)?([A-Z]+) (\S+): (.*)$/
36
+ LOG4R_DEFAULT_REGEXP = /^(?:\s*)([A-Z]+) (\S+): (.*)$/
37
+
38
+ LOG4J_DEFAULT_FIELDS = [:message]
39
+ LOG4J_DEFAULT_REGEXP = /^(.+?)$/
37
40
  end
38
41
 
39
42
  #
@@ -41,21 +44,26 @@ class Indy
41
44
  # e.g.:
42
45
  # INFO 2000-09-07 MyApp - Entering APPLICATION.
43
46
  #
44
- DEFAULT_LOG_FORMAT = [LogFormats::DEFAULT_LOG_REGEXP, LogFormats::DEFAULT_LOG_FIELDS].flatten
47
+ DEFAULT_LOG_FORMAT = {:entry_regexp => LogFormats::DEFAULT_ENTRY_REGEXP, :entry_fields => LogFormats::DEFAULT_ENTRY_FIELDS}
45
48
 
46
49
  #
47
50
  # Uncustomized Log4r log format
48
51
  #
49
- LOG4R_DEFAULT_FORMAT = [LogFormats::LOG4R_DEFAULT_REGEXP, LogFormats::LOG4R_DEFAULT_FIELDS].flatten
52
+ LOG4R_DEFAULT_FORMAT = {:entry_regexp => LogFormats::LOG4R_DEFAULT_REGEXP, :entry_fields => LogFormats::LOG4R_DEFAULT_FIELDS}
53
+
54
+ #
55
+ # Uncustomized Log4j log format (message field only!)
56
+ #
57
+ LOG4J_DEFAULT_FORMAT = {:entry_regexp => LogFormats::LOG4J_DEFAULT_REGEXP, :entry_fields => LogFormats::LOG4J_DEFAULT_FIELDS}
50
58
 
51
59
  #
52
60
  # NCSA Common Log Format log format
53
61
  #
54
- COMMON_LOG_FORMAT = [LogFormats::COMMON_REGEXP, LogFormats::COMMON_FIELDS].flatten
62
+ COMMON_LOG_FORMAT = {:entry_regexp => LogFormats::COMMON_REGEXP, :entry_fields => LogFormats::COMMON_FIELDS}
55
63
 
56
64
  #
57
65
  # NCSA Combined Log Format log format
58
66
  #
59
- COMBINED_LOG_FORMAT = [LogFormats::COMBINED_REGEXP, LogFormats::COMBINED_FIELDS].flatten
67
+ COMBINED_LOG_FORMAT = {:entry_regexp => LogFormats::COMBINED_REGEXP, :entry_fields => LogFormats::COMBINED_FIELDS}
60
68
 
61
69
  end
@@ -0,0 +1,147 @@
1
+ class Indy
2
+
3
+ class Search
4
+
5
+ attr_accessor :source
6
+ attr_accessor :log_definition
7
+
8
+ attr_accessor :start_time, :end_time, :inclusive
9
+
10
+ def initialize(params_hash)
11
+ while (param = params_hash.shift) do
12
+ send("#{param.first}=",param.last)
13
+ end
14
+ end
15
+
16
+ #
17
+ # Helper function called by Indy#for, Indy#like and Indy#all
18
+ #
19
+ # @param [Symbol] type The symbol :for, :like or :all
20
+ #
21
+ # @param [Hash] search_criteria the field to search for as the key and the
22
+ # value to compare against the log entries.
23
+ #
24
+ def iterate_and_compare(type,search_criteria,&block)
25
+ results = []
26
+ results += search do |entry|
27
+ if type == :all || is_match?(type,entry,search_criteria)
28
+ result_struct = @log_definition.create_struct(entry)
29
+ if block_given?
30
+ block.call(result_struct)
31
+ else
32
+ result_struct
33
+ end
34
+ end
35
+ end
36
+ results.compact
37
+ end
38
+
39
+ #
40
+ # Search the @source and yield to the block the entry that was found
41
+ # with @log_definition
42
+ #
43
+ # This method is supposed to be used internally.
44
+ #
45
+ def search(&block)
46
+ if @log_definition.multiline
47
+ multiline_search(&block)
48
+ else
49
+ standard_search(&block)
50
+ end
51
+ end
52
+
53
+ #
54
+ # Performs #search for line based entries
55
+ #
56
+ def standard_search(&block)
57
+ is_time_search = use_time_criteria?
58
+ results = []
59
+ source_lines = (is_time_search ? @source.open([@start_time,@end_time]) : @source.open)
60
+ source_lines.each do |single_line|
61
+ hash = @log_definition.parse_entry(single_line)
62
+ next unless hash
63
+ next unless Indy::Time.inside_time_window?(hash[:time],@start_time,@end_time,@inclusive) if is_time_search
64
+ results << (block.call(hash) if block_given?)
65
+ end
66
+ results.compact
67
+ end
68
+
69
+ #
70
+ # Performs #search for multi-line based entries
71
+ #
72
+ def multiline_search(&block)
73
+ is_time_search = use_time_criteria?
74
+ source_io = StringIO.new( (is_time_search ? @source.open([@start_time,@end_time]) : @source.open ).join("\n") )
75
+ results = source_io.read.scan(@log_definition.entry_regexp).collect do |entry|
76
+ hash = @log_definition.parse_entry_captures(entry)
77
+ next unless hash
78
+ next unless Indy::Time.inside_time_window?(hash[:time],@start_time,@end_time,@inclusive) if is_time_search
79
+ block.call(hash) if block_given?
80
+ end
81
+ results.compact
82
+ end
83
+
84
+ #
85
+ # Return true if start or end time has been set, and a :time field exists
86
+ #
87
+ def use_time_criteria?
88
+ if @start_time || @end_time
89
+ # ensure both boundaries are set
90
+ @start_time ||= Indy::Time.forever_ago(@log_definition.time_format)
91
+ @end_time ||= Indy::Time.forever(@log_definition.time_format)
92
+ end
93
+ @start_time && @end_time
94
+ end
95
+
96
+ #
97
+ # Evaluates if field => value criteria is an exact match on entry
98
+ #
99
+ # @param [Hash] result The entry_hash
100
+ # @param [Hash] search_criteria The field => value criteria to match
101
+ #
102
+ def is_match?(type, result, search_criteria)
103
+ if type == :for
104
+ search_criteria.reject {|criteria,value| result[criteria] == value }.empty?
105
+ elsif type == :like
106
+ search_criteria.reject {|criteria,value| result[criteria] =~ /#{value}/i }.empty?
107
+ end
108
+ end
109
+
110
+ #
111
+ # Parse hash to set @start_time, @end_time and @inclusive
112
+ #
113
+ def time_scope(params_hash)
114
+ if params_hash[:time]
115
+ time_scope_from_direction(params_hash[:direction], params_hash[:span], params_hash[:time])
116
+ else
117
+ @start_time = Indy::Time.parse_date(params_hash[:start_time]) if params_hash[:start_time]
118
+ @end_time = Indy::Time.parse_date(params_hash[:end_time]) if params_hash[:end_time]
119
+ end
120
+ @inclusive = params_hash[:inclusive]
121
+ end
122
+
123
+ #
124
+ # Parse direction, span, and time to set @start_time and @end_time
125
+ #
126
+ def time_scope_from_direction(direction, span, time)
127
+ time = Indy::Time.parse_date(time)
128
+ span = (span.to_i * 60).seconds if span
129
+ if direction == :before
130
+ @end_time = time
131
+ @start_time = time - span if span
132
+ elsif direction == :after
133
+ @start_time = time
134
+ @end_time = time + span if span
135
+ end
136
+ end
137
+
138
+ #
139
+ # Clear time scope settings
140
+ #
141
+ def reset_scope
142
+ @inclusive = @start_time = @end_time = nil
143
+ end
144
+
145
+ end
146
+
147
+ end
@@ -14,6 +14,9 @@ class Indy
14
14
  # the StringIO object
15
15
  attr_reader :io
16
16
 
17
+ # log definition
18
+ attr_reader :log_definition
19
+
17
20
  # Exception raised when unable to open source
18
21
  class Invalid < Exception; end
19
22
 
@@ -22,22 +25,30 @@ class Indy
22
25
  #
23
26
  # @param [String, Hash] param The source content String, filepath String, or :cmd => 'command' Hash
24
27
  #
25
- def initialize(param)
26
- raise Indy::Source::Invalid if param.nil?
27
- if param.respond_to?(:keys)
28
- set_connection(:cmd, param[:cmd]) if param[:cmd]
29
- set_connection(:file, param[:file]) if ( param[:file] and File.size(param[:file].path) > 0 )
30
- set_connection(:string, param[:string]) if param[:string]
31
- elsif param.respond_to?(:read) and param.respond_to?(:rewind)
32
- set_connection(:file, param)
28
+ def initialize(param,log_definition=nil)
29
+ raise Indy::Source::Invalid, "No source specified." if param.nil?
30
+ @log_definition = log_definition || LogDefinition.new()
31
+ return discover_connection(param) unless param.respond_to?(:keys)
32
+ if param[:cmd]
33
+ set_connection(:cmd, param[:cmd])
34
+ elsif param[:file]
35
+ set_connection(:file, open_or_return_file(param[:file]))
36
+ elsif param[:string]
37
+ set_connection(:string, param[:string])
38
+ end
39
+ end
40
+
41
+ #
42
+ # Support source being passed in without key indicating type
43
+ #
44
+ def discover_connection(param)
45
+ if param.respond_to?(:read) and param.respond_to?(:rewind)
46
+ set_connection(:file, param)
33
47
  elsif param.respond_to?(:to_s) and param.respond_to?(:length)
34
- # fall back to source being the string passed in
35
48
  set_connection(:string, param)
36
49
  else
37
50
  raise Indy::Source::Invalid
38
51
  end
39
-
40
-
41
52
  end
42
53
 
43
54
  #
@@ -48,82 +59,164 @@ class Indy
48
59
  @connection = value
49
60
  end
50
61
 
62
+ def open_or_return_file(param)
63
+ return param if param.respond_to? :pos
64
+ file = File.open(param, 'r')
65
+ raise ArgumentError, "Unable to open file parameter: '#{file}'" unless file.respond_to? :pos
66
+ file
67
+ end
68
+
51
69
  #
52
70
  # Return a StringIO object to provide access to the underlying log source
53
71
  #
54
- def open(time_search=nil)
72
+ def open(time_boundaries=nil)
55
73
  begin
74
+ open_method = ('open_' + @type.to_s).intern
75
+ self.send(open_method)
76
+ rescue Exception => e
77
+ raise Indy::Source::Invalid, "Unable to open log source. (#{e.message})"
78
+ end
79
+ load_data
80
+ scope_by_time(time_boundaries) if time_boundaries
81
+ @entries
82
+ end
56
83
 
57
- case @type
58
- when :cmd
59
- @io = StringIO.new( exec_command(@connection).read )
60
- raise "Failed to execute command (#{@connection})" if @io.nil?
84
+ def open_cmd
85
+ @io = StringIO.new(exec_command(@connection).read)
86
+ raise "Failed to execute command (#{@connection.inspect})" if @io.nil?
87
+ end
61
88
 
62
- when :file
63
- @connection.rewind
64
- @io = StringIO.new(@connection.read)
65
- raise "Failed to open file: #{@connection}" if @io.nil?
89
+ def open_file
90
+ @connection.rewind
91
+ @io = StringIO.new(@connection.read)
92
+ raise "Failed to open file: #{@connection.inspect}" if @io.nil?
93
+ end
66
94
 
67
- when :string
68
- @io = StringIO.new( @connection )
95
+ def open_string
96
+ @io = StringIO.new(@connection)
97
+ raise "Failed to create StringIO from source (#{@connection.inspect})" if @io.nil?
98
+ end
69
99
 
70
- else
71
- raise RuntimeError, "Invalid log source type: #{@type.inspect}"
72
- end
73
100
 
74
- rescue Exception => e
75
- raise Indy::Source::Invalid, "Unable to open log source. (#{e.message})"
101
+ #
102
+ # Return entries that meet time criteria
103
+ #
104
+ def scope_by_time(time_boundaries)
105
+ start_time, end_time = time_boundaries
106
+ scope_end = num_entries - 1
107
+ # short circuit the search if possible
108
+ if (time_at(0) > end_time) or (time_at(-1) < start_time)
109
+ @entries = []
110
+ return @entries
111
+ end
112
+ scope_begin = find_first(start_time, 0, scope_end)
113
+ scope_end = find_last(end_time, scope_begin, scope_end)
114
+ @entries = @entries[scope_begin..scope_end]
115
+ end
116
+
117
+ #
118
+ # find index of first record to match value
119
+ #
120
+ def find_first(value,start,stop)
121
+ return start if time_at(start) > value
122
+ find(:first,value,start,stop)
123
+ end
124
+
125
+ #
126
+ # find index of last record to match value
127
+ #
128
+ def find_last(value,start,stop)
129
+ return stop if time_at(stop) < value
130
+ find(:last,value,start,stop)
131
+ end
132
+
133
+ #
134
+ # Find index and time at mid point
135
+ #
136
+ def find_middle(start, stop)
137
+ index = ((stop - start) / 2) + start
138
+ time = time_at(index)
139
+ [index, time]
140
+ end
141
+
142
+ #
143
+ # Step forward or backward by one, looking for the boundary of the value
144
+ #
145
+ def find_adjacent(boundary,value,start,stop,mid_index)
146
+ case boundary
147
+ when :first
148
+ (time_at(mid_index,-1) == value) ? find_first(value,start-1,stop) : mid_index
149
+ when :last
150
+ (time_at(mid_index,1) == value) ? find_last(value,start,stop+1) : mid_index
76
151
  end
152
+ end
77
153
 
78
- # scope_by_time(source_io) if time_search
154
+ #
155
+ # Return the time of a log entry index, with an optional offset
156
+ #
157
+ def time_at(index, delta=0)
158
+ ::Time.parse(@entries[index + delta])
159
+ end
79
160
 
80
- @io
161
+ #
162
+ # Binary search for a time condition
163
+ #
164
+ def find(boundary,value,start,stop)
165
+ return start if start == stop
166
+ mid_index, mid_time = find_middle(start,stop)
167
+ if mid_time == value
168
+ find_adjacent(boundary,value,start,stop,mid_index)
169
+ elsif mid_time > value
170
+ mid_index -= 1 if mid_index == stop
171
+ find(boundary, value, start, mid_index)
172
+ elsif mid_time < value
173
+ mid_index += 1 if mid_index == start
174
+ find(boundary, value, mid_index, stop)
175
+ end
81
176
  end
82
-
177
+
83
178
  #
84
179
  # Execute the source's connection string, returning an IO object
85
180
  #
86
181
  # @param [String] command_string string of command that will return log contents
87
182
  #
88
183
  def exec_command(command_string)
89
- begin
90
- io = IO.popen(command_string)
91
- return nil if io.eof?
92
- rescue
93
- nil
94
- end
184
+ io = IO.popen(command_string)
185
+ raise Indy::Source::Invalid, "No data returned from command string execution" if io.eof?
95
186
  io
96
187
  end
97
188
 
98
-
99
-
100
189
  #
101
190
  # the number of lines in the source
102
191
  #
103
- def num_lines
104
- load_data unless @num_lines
105
- @num_lines
192
+ def num_entries
193
+ load_data unless @num_entries
194
+ @num_entries
106
195
  end
107
196
 
108
197
  #
109
198
  # array of log lines from source
110
199
  #
111
- def lines
112
- load_data unless @lines
113
- @lines
200
+ def entries
201
+ load_data unless @entries
202
+ @entries
114
203
  end
115
204
 
116
205
  #
117
206
  # read source data and populate instance variables
118
207
  #
119
- # TODO: hmmm... not called when Source#open is called directly, but #load_data would call open again. :(
120
- #
121
208
  def load_data
122
- self.open
123
- @lines = @io.readlines
209
+ self.open if @io.nil?
210
+ if @log_definition.multiline
211
+ entire_log = @io.read
212
+ @entries = entire_log.scan(@log_definition.entry_regexp).map{|matchdata|matchdata[0]}
213
+ else
214
+ @entries = @io.readlines
215
+ end
124
216
  @io.rewind
125
- @num_lines = @lines.count
217
+ @entries.delete_if {|entry| entry.match(/^\s*$/)}
218
+ @num_entries = @entries.count
126
219
  end
127
220
 
128
221
  end
129
- end
222
+ end