sawmill 0.1.15 → 0.1.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. data/History.rdoc +7 -0
  2. data/Version +1 -1
  3. data/lib/sawmill.rb +6 -6
  4. data/lib/sawmill/entry.rb +108 -108
  5. data/lib/sawmill/entry_classifier.rb +19 -19
  6. data/lib/sawmill/entry_processor.rb +63 -63
  7. data/lib/sawmill/entry_processor/build_records.rb +39 -39
  8. data/lib/sawmill/entry_processor/compile_report.rb +32 -32
  9. data/lib/sawmill/entry_processor/conditionals.rb +110 -110
  10. data/lib/sawmill/entry_processor/count_entries.rb +27 -27
  11. data/lib/sawmill/entry_processor/filter_by_basic_fields.rb +32 -32
  12. data/lib/sawmill/entry_processor/filter_by_block.rb +28 -28
  13. data/lib/sawmill/entry_processor/format.rb +41 -36
  14. data/lib/sawmill/entry_processor/interpret_stats.rb +24 -24
  15. data/lib/sawmill/entry_processor/simple_queue.rb +38 -38
  16. data/lib/sawmill/errors.rb +28 -28
  17. data/lib/sawmill/interface.rb +51 -51
  18. data/lib/sawmill/level.rb +75 -75
  19. data/lib/sawmill/log_record_middleware.rb +21 -21
  20. data/lib/sawmill/logger.rb +94 -94
  21. data/lib/sawmill/multi_parser.rb +28 -28
  22. data/lib/sawmill/parser.rb +32 -32
  23. data/lib/sawmill/railtie.rb +31 -31
  24. data/lib/sawmill/record.rb +74 -74
  25. data/lib/sawmill/record_processor.rb +54 -54
  26. data/lib/sawmill/record_processor/compile_report.rb +32 -32
  27. data/lib/sawmill/record_processor/conditionals.rb +92 -92
  28. data/lib/sawmill/record_processor/count_records.rb +24 -24
  29. data/lib/sawmill/record_processor/decompose.rb +21 -21
  30. data/lib/sawmill/record_processor/filter_by_attributes.rb +22 -22
  31. data/lib/sawmill/record_processor/filter_by_block.rb +29 -29
  32. data/lib/sawmill/record_processor/filter_by_record_id.rb +22 -22
  33. data/lib/sawmill/record_processor/format.rb +24 -24
  34. data/lib/sawmill/record_processor/simple_queue.rb +44 -44
  35. data/lib/sawmill/rotater.rb +60 -60
  36. data/lib/sawmill/rotater/base.rb +28 -28
  37. data/lib/sawmill/rotater/date_based_log_file.rb +50 -50
  38. data/lib/sawmill/rotater/shifting_log_file.rb +34 -34
  39. data/lib/sawmill/stats_middleware.rb +21 -21
  40. data/lib/sawmill/stats_railtie.rb +33 -33
  41. data/lib/sawmill/util/heap.rb +41 -41
  42. data/lib/sawmill/util/processor_tools.rb +17 -17
  43. data/lib/sawmill/util/queue.rb +33 -33
  44. data/lib/sawmill/version.rb +9 -9
  45. data/test/tc_entry_processors.rb +27 -27
  46. data/test/tc_formatter_parser.rb +40 -40
  47. data/test/tc_levels.rb +27 -27
  48. data/test/tc_logger.rb +49 -49
  49. data/test/tc_multi_parser.rb +18 -18
  50. data/test/tc_record_processors.rb +21 -21
  51. data/test/tc_records.rb +39 -39
  52. data/test/tc_reports.rb +19 -19
  53. metadata +10 -5
@@ -1,15 +1,15 @@
1
1
  # -----------------------------------------------------------------------------
2
- #
2
+ #
3
3
  # Sawmill entry processor that interprets a stats log
4
- #
4
+ #
5
5
  # -----------------------------------------------------------------------------
6
6
  # Copyright 2009 Daniel Azuma
7
- #
7
+ #
8
8
  # All rights reserved.
9
- #
9
+ #
10
10
  # Redistribution and use in source and binary forms, with or without
11
11
  # modification, are permitted provided that the following conditions are met:
12
- #
12
+ #
13
13
  # * Redistributions of source code must retain the above copyright notice,
14
14
  # this list of conditions and the following disclaimer.
15
15
  # * Redistributions in binary form must reproduce the above copyright notice,
@@ -18,7 +18,7 @@
18
18
  # * Neither the name of the copyright holder, nor the names of any other
19
19
  # contributors to this software, may be used to endorse or promote products
20
20
  # derived from this software without specific prior written permission.
21
- #
21
+ #
22
22
  # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23
23
  # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24
24
  # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
@@ -35,22 +35,22 @@
35
35
 
36
36
 
37
37
  module Sawmill
38
-
38
+
39
39
  module EntryProcessor
40
-
41
-
40
+
41
+
42
42
  # This processor interprets a stats log.
43
-
43
+
44
44
  class InterpretStats < Base
45
-
46
-
45
+
46
+
47
47
  # Create a stats log interpreter.
48
-
48
+
49
49
  def initialize(opts_={}, &block_)
50
50
  @handler = opts_[:handler] || block_ || method(:handle_data)
51
51
  end
52
-
53
-
52
+
53
+
54
54
  def message(entry_)
55
55
  data_ = ::JSON.parse(entry_.message) rescue nil
56
56
  if data_.is_a?(::Hash)
@@ -59,21 +59,21 @@ module Sawmill
59
59
  nil
60
60
  end
61
61
  end
62
-
63
-
62
+
63
+
64
64
  def handle_data(data_)
65
65
  true
66
66
  end
67
-
68
-
67
+
68
+
69
69
  def finish
70
70
  @handler.call(nil)
71
71
  end
72
-
73
-
72
+
73
+
74
74
  end
75
-
76
-
75
+
76
+
77
77
  end
78
-
78
+
79
79
  end
@@ -1,15 +1,15 @@
1
1
  # -----------------------------------------------------------------------------
2
- #
2
+ #
3
3
  # Sawmill entry processor that queues entries
4
- #
4
+ #
5
5
  # -----------------------------------------------------------------------------
6
6
  # Copyright 2009 Daniel Azuma
7
- #
7
+ #
8
8
  # All rights reserved.
9
- #
9
+ #
10
10
  # Redistribution and use in source and binary forms, with or without
11
11
  # modification, are permitted provided that the following conditions are met:
12
- #
12
+ #
13
13
  # * Redistributions of source code must retain the above copyright notice,
14
14
  # this list of conditions and the following disclaimer.
15
15
  # * Redistributions in binary form must reproduce the above copyright notice,
@@ -18,7 +18,7 @@
18
18
  # * Neither the name of the copyright holder, nor the names of any other
19
19
  # contributors to this software, may be used to endorse or promote products
20
20
  # derived from this software without specific prior written permission.
21
- #
21
+ #
22
22
  # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23
23
  # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24
24
  # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
@@ -35,20 +35,20 @@
35
35
 
36
36
 
37
37
  module Sawmill
38
-
39
-
38
+
39
+
40
40
  module EntryProcessor
41
-
42
-
41
+
42
+
43
43
  # This processor simply queues up log entries for later use.
44
-
44
+
45
45
  class SimpleQueue < Base
46
-
47
-
46
+
47
+
48
48
  # Create a queue.
49
- #
49
+ #
50
50
  # Recognized options include:
51
- #
51
+ #
52
52
  # [<tt>:limit</tt>]
53
53
  # Size limit for the queue. If not specified, the queue can grow
54
54
  # arbitrarily large.
@@ -56,69 +56,69 @@ module Sawmill
56
56
  # If set to true, then when an item is added to a full queue, the
57
57
  # oldest item is dropped. If set to false or not specified, then
58
58
  # the new item is not added.
59
-
59
+
60
60
  def initialize(opts_={})
61
61
  @queue = Util::Queue.new(opts_)
62
62
  @closed = false
63
63
  end
64
-
65
-
64
+
65
+
66
66
  # Return the oldest entry in the queue, or nil if the queue is empty.
67
-
67
+
68
68
  def dequeue
69
69
  @queue.dequeue
70
70
  end
71
-
72
-
71
+
72
+
73
73
  # Return an array of the contents of the queue, in order.
74
-
74
+
75
75
  def dequeue_all
76
76
  @queue.dequeue_all
77
77
  end
78
-
79
-
78
+
79
+
80
80
  # Return the size of the queue, which is 0 if the queue is empty.
81
-
81
+
82
82
  def size
83
83
  @queue.size
84
84
  end
85
-
86
-
85
+
86
+
87
87
  def begin_record(entry_)
88
88
  @queue.enqueue(entry_) unless @closed
89
89
  !@closed
90
90
  end
91
-
91
+
92
92
  def end_record(entry_)
93
93
  @queue.enqueue(entry_) unless @closed
94
94
  !@closed
95
95
  end
96
-
96
+
97
97
  def message(entry_)
98
98
  @queue.enqueue(entry_) unless @closed
99
99
  !@closed
100
100
  end
101
-
101
+
102
102
  def attribute(entry_)
103
103
  @queue.enqueue(entry_) unless @closed
104
104
  !@closed
105
105
  end
106
-
106
+
107
107
  def unknown_data(entry_)
108
108
  @queue.enqueue(entry_) unless @closed
109
109
  !@closed
110
110
  end
111
-
111
+
112
112
  def finish
113
113
  @closed = true
114
114
  nil
115
115
  end
116
-
117
-
116
+
117
+
118
118
  end
119
-
120
-
119
+
120
+
121
121
  end
122
-
123
-
122
+
123
+
124
124
  end
@@ -1,15 +1,15 @@
1
1
  # -----------------------------------------------------------------------------
2
- #
2
+ #
3
3
  # Sawmill error classes
4
- #
4
+ #
5
5
  # -----------------------------------------------------------------------------
6
6
  # Copyright 2009 Daniel Azuma
7
- #
7
+ #
8
8
  # All rights reserved.
9
- #
9
+ #
10
10
  # Redistribution and use in source and binary forms, with or without
11
11
  # modification, are permitted provided that the following conditions are met:
12
- #
12
+ #
13
13
  # * Redistributions of source code must retain the above copyright notice,
14
14
  # this list of conditions and the following disclaimer.
15
15
  # * Redistributions in binary form must reproduce the above copyright notice,
@@ -18,7 +18,7 @@
18
18
  # * Neither the name of the copyright holder, nor the names of any other
19
19
  # contributors to this software, may be used to endorse or promote products
20
20
  # derived from this software without specific prior written permission.
21
- #
21
+ #
22
22
  # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23
23
  # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24
24
  # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
@@ -35,45 +35,45 @@
35
35
 
36
36
 
37
37
  module Sawmill
38
-
39
-
38
+
39
+
40
40
  # This is a namespace for errors that can be thrown by Sawmill.
41
-
41
+
42
42
  module Errors
43
-
44
-
43
+
44
+
45
45
  # Base class for all Sawmill exceptions
46
-
46
+
47
47
  class SawmillError < ::RuntimeError
48
48
  end
49
-
50
-
49
+
50
+
51
51
  # Tried to create an illegal record
52
-
52
+
53
53
  class IllegalRecordError < SawmillError
54
54
  end
55
-
56
-
55
+
56
+
57
57
  # Tried to log an entry with an unknown level code
58
-
58
+
59
59
  class UnknownLevelError < SawmillError
60
60
  end
61
-
62
-
61
+
62
+
63
63
  # Tried to register a method with a processor building DSL
64
64
  # where the method name was already taken.
65
-
65
+
66
66
  class DSLMethodError < SawmillError
67
67
  end
68
-
69
-
68
+
69
+
70
70
  # Could not open a log file because a uniquifier failed.
71
-
71
+
72
72
  class NoUniqueLogFileError < SawmillError
73
73
  end
74
-
75
-
74
+
75
+
76
76
  end
77
-
78
-
77
+
78
+
79
79
  end
@@ -1,15 +1,15 @@
1
1
  # -----------------------------------------------------------------------------
2
- #
2
+ #
3
3
  # Sawmill convenience interface
4
- #
4
+ #
5
5
  # -----------------------------------------------------------------------------
6
6
  # Copyright 2009 Daniel Azuma
7
- #
7
+ #
8
8
  # All rights reserved.
9
- #
9
+ #
10
10
  # Redistribution and use in source and binary forms, with or without
11
11
  # modification, are permitted provided that the following conditions are met:
12
- #
12
+ #
13
13
  # * Redistributions of source code must retain the above copyright notice,
14
14
  # this list of conditions and the following disclaimer.
15
15
  # * Redistributions in binary form must reproduce the above copyright notice,
@@ -18,7 +18,7 @@
18
18
  # * Neither the name of the copyright holder, nor the names of any other
19
19
  # contributors to this software, may be used to endorse or promote products
20
20
  # derived from this software without specific prior written permission.
21
- #
21
+ #
22
22
  # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23
23
  # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24
24
  # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
@@ -35,21 +35,21 @@
35
35
 
36
36
 
37
37
  # This module is a namespace for Sawmill.
38
- #
38
+ #
39
39
  # It also contains some convenience class methods.
40
40
 
41
41
  module Sawmill
42
-
42
+
43
43
  class << self
44
-
45
-
44
+
45
+
46
46
  # Creates a new logger that writes to a single logfile.
47
47
  # You may provide either the path to the logfile, or an IO object to
48
48
  # write to, such as STDOUT.
49
- #
49
+ #
50
50
  # You may pass the same options taken by Sawmill::Logger#new and
51
51
  # Sawmill::EntryProcessor::Format#new, which are:
52
- #
52
+ #
53
53
  # [<tt>:levels</tt>]
54
54
  # Use a custom Sawmill::LevelGroup. Normally, you should leave this
55
55
  # set to the default, which is Sawmill::STANDARD_LEVELS.
@@ -85,7 +85,7 @@ module Sawmill
85
85
  # [<tt>:iso_8601_time</tt>]
86
86
  # If true, outputs time in strict ISO 8601 format.
87
87
  # If false (the default), outputs a slightly more readable format.
88
-
88
+
89
89
  def simple_logger(filepath_=::STDOUT, opts_={})
90
90
  if filepath_.kind_of?(::String)
91
91
  io_ = ::File.open(filepath_)
@@ -97,29 +97,29 @@ module Sawmill
97
97
  processor_ = EntryProcessor::Format.new(io_, opts_.dup)
98
98
  Logger.new(opts_.merge(:processor => processor_))
99
99
  end
100
-
101
-
100
+
101
+
102
102
  # Creates a new logger that writes to a logfile that rotates
103
103
  # automatically by "shifting". This is a standard rotation strategy
104
104
  # used by many unix tools.
105
- #
105
+ #
106
106
  # You must provide the logfile path, a shifting period, and a maximum
107
107
  # file size.
108
- #
108
+ #
109
109
  # The period specifies how often to rotate the logfile. Possible values
110
110
  # include <tt>:yearly</tt>, <tt>:monthly</tt>, <tt>:daily</tt>, and
111
111
  # <tt>:hourly</tt>. You may also specify an integer value, which is
112
112
  # interpreted as a number of seconds. Finally, you may pass nil to
113
113
  # disable checking of the file's age. If you do pass nil, you should
114
114
  # provide a maximum size.
115
- #
115
+ #
116
116
  # The maximum size is the maximum file size in bytes. You may provide
117
117
  # a number, or nil to disable checking of the file size.
118
- #
118
+ #
119
119
  # You may pass the same options taken by Sawmill::Logger#new,
120
120
  # Sawmill::EntryProcessor::Format#new, Sawmill::Rotater#new, and
121
121
  # Sawmill::Rotater::ShiftingLogFile#new, which are:
122
- #
122
+ #
123
123
  # [<tt>:levels</tt>]
124
124
  # Use a custom Sawmill::LevelGroup. Normally, you should leave this
125
125
  # set to the default, which is Sawmill::STANDARD_LEVELS.
@@ -167,27 +167,27 @@ module Sawmill
167
167
  # [<tt>:encoding</tt>]
168
168
  # Specify an encoding name for file data. (Ruby 1.9 only)
169
169
  # If not specified, uses the default external encoding.
170
-
170
+
171
171
  def shifting_logfile(filepath_, period_, max_size_, opts_={})
172
172
  rotater_ = Rotater.new(Rotater::ShiftingLogFile, opts_.merge(:file_path => filepath_,
173
173
  :max_file_size => max_size_, :shift_period => period_))
174
174
  processor_ = EntryProcessor::Format.new(rotater_, opts_.dup)
175
175
  Logger.new(opts_.merge(:processor => processor_))
176
176
  end
177
-
178
-
177
+
178
+
179
179
  # Creates a new logger that writes to a logfile that rotates
180
180
  # automatically by tagging filenames with a datestamp.
181
- #
181
+ #
182
182
  # You must provide the file path prefix, and a turnover frequency.
183
183
  # Possible values for the turnover frequency are <tt>:yearly</tt>,
184
184
  # <tt>:monthly</tt>, <tt>:daily</tt>, <tt>:hourly</tt>, and
185
185
  # <tt>:never</tt>.
186
- #
186
+ #
187
187
  # You may pass the same options taken by Sawmill::Logger#new,
188
188
  # Sawmill::EntryProcessor::Format#new, Sawmill::Rotater#new, and
189
189
  # Sawmill::Rotater::DateBasedLogFile#new, which are:
190
- #
190
+ #
191
191
  # [<tt>:levels</tt>]
192
192
  # Use a custom Sawmill::LevelGroup. Normally, you should leave this
193
193
  # set to the default, which is Sawmill::STANDARD_LEVELS.
@@ -238,31 +238,31 @@ module Sawmill
238
238
  # [<tt>:encoding</tt>]
239
239
  # Specify an encoding name for file data. (Ruby 1.9 only)
240
240
  # If not specified, uses the default external encoding.
241
-
241
+
242
242
  def date_based_logfile(filepath_, frequency_, opts_={})
243
243
  rotater_ = Rotater.new(Rotater::DateBasedLogFile, opts_.merge(:path_prefix => filepath_,
244
244
  :turnover_frequency => frequency_))
245
245
  processor_ = EntryProcessor::Format.new(rotater_, opts_.dup)
246
246
  Logger.new(opts_.merge(:processor => processor_))
247
247
  end
248
-
249
-
248
+
249
+
250
250
  # Open one or more log files and run them through an entry processor.
251
251
  # The processor is built on the fly using the EntryProcessor DSL.
252
252
  # See EntryProcessor#build for more details.
253
- #
253
+ #
254
254
  # You may pass the same options taken by Sawmill::MultiParser#new,
255
255
  # which are:
256
- #
256
+ #
257
257
  # [<tt>:levels</tt>]
258
258
  # Sawmill::LevelGroup to use to parse log levels.
259
259
  # If not specified, Sawmill::STANDARD_LEVELS is used by default.
260
260
  # [<tt>:emit_incomplete_records_at_eof</tt>]
261
261
  # If set to true, causes any incomplete log records to be emitted
262
262
  # in their incomplete state when EOF is reached on all streams.
263
- #
263
+ #
264
264
  # Additionally, these options are recognized:
265
- #
265
+ #
266
266
  # [<tt>:encoding</tt>]
267
267
  # Specify an encoding for file data. (Ruby 1.9 only.)
268
268
  # You may specify an encoding name or an encoding object.
@@ -273,29 +273,29 @@ module Sawmill
273
273
  # Specify an encoding to transcode to. (Ruby 1.9 only.)
274
274
  # You may specify an encoding name or an encoding object.
275
275
  # If not specified, uses the encoding as read from the file.
276
-
276
+
277
277
  def open_entries(globs_, opts_={}, &block_)
278
278
  processor_ = EntryProcessor.build(&block_)
279
279
  open_files(globs_, processor_, opts_.merge(:finish => true))
280
280
  end
281
-
282
-
281
+
282
+
283
283
  # Open one or more log files and run them through a record processor.
284
284
  # The processor is built on the fly using the RecordProcessor DSL.
285
285
  # See RecordProcessor#build for more details.
286
- #
286
+ #
287
287
  # You may pass the same options taken by Sawmill::MultiParser#new,
288
288
  # which are:
289
- #
289
+ #
290
290
  # [<tt>:levels</tt>]
291
291
  # Sawmill::LevelGroup to use to parse log levels.
292
292
  # If not specified, Sawmill::STANDARD_LEVELS is used by default.
293
293
  # [<tt>:emit_incomplete_records_at_eof</tt>]
294
294
  # If set to true, causes any incomplete log records to be emitted
295
295
  # in their incomplete state when EOF is reached on all streams.
296
- #
296
+ #
297
297
  # Additionally, these options are recognized:
298
- #
298
+ #
299
299
  # [<tt>:encoding</tt>]
300
300
  # Specify an encoding for file data. (Ruby 1.9 only.)
301
301
  # You may specify an encoding name or an encoding object.
@@ -306,28 +306,28 @@ module Sawmill
306
306
  # Specify an encoding to transcode to. (Ruby 1.9 only.)
307
307
  # You may specify an encoding name or an encoding object.
308
308
  # If not specified, uses the encoding as read from the file.
309
-
309
+
310
310
  def open_records(globs_, opts_={}, &block_)
311
311
  processor_ = RecordProcessor.build(&block_)
312
312
  open_files(globs_, processor_, opts_.merge(:finish => true))
313
313
  end
314
-
315
-
314
+
315
+
316
316
  # Open one or more log files and run them through the given
317
317
  # EntryProcessor or RecordProcessor.
318
- #
318
+ #
319
319
  # You may pass the same options taken by Sawmill::MultiParser#new,
320
320
  # which are:
321
- #
321
+ #
322
322
  # [<tt>:levels</tt>]
323
323
  # Sawmill::LevelGroup to use to parse log levels.
324
324
  # If not specified, Sawmill::STANDARD_LEVELS is used by default.
325
325
  # [<tt>:emit_incomplete_records_at_eof</tt>]
326
326
  # If set to true, causes any incomplete log records to be emitted
327
327
  # in their incomplete state when EOF is reached on all streams.
328
- #
328
+ #
329
329
  # Additionally, these options are recognized:
330
- #
330
+ #
331
331
  # [<tt>:finish</tt>]
332
332
  # If set to true, the "finish" method is called on the processor
333
333
  # after all files have been parsed, and the return value is returned.
@@ -342,7 +342,7 @@ module Sawmill
342
342
  # Specify an encoding to transcode to. (Ruby 1.9 only.)
343
343
  # You may specify an encoding name or an encoding object.
344
344
  # If not specified, uses the encoding as read from the file.
345
-
345
+
346
346
  def open_files(globs_, processor_, opts_={})
347
347
  finish_opt_ = opts_.delete(:finish)
348
348
  encoding_ = opts_.delete(:encoding)
@@ -393,8 +393,8 @@ module Sawmill
393
393
  end
394
394
  finish_opt_ ? processor_.finish : nil
395
395
  end
396
-
397
-
396
+
397
+
398
398
  end
399
-
399
+
400
400
  end