sawmill 0.0.1

data/lib/sawmill/record.rb

@@ -0,0 +1,255 @@
+# -----------------------------------------------------------------------------
+# 
+# Sawmill log record class
+# 
+# -----------------------------------------------------------------------------
+# Copyright 2009 Daniel Azuma
+# 
+# All rights reserved.
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# * Neither the name of the copyright holder, nor the names of any other
+#   contributors to this software, may be used to endorse or promote products
+#   derived from this software without specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+# -----------------------------------------------------------------------------
+;
+
+
+module Sawmill
+  
+  
+  # A log record.
+  # 
+  # Log records are sequences of related log entries with a common record
+  # ID, beginning with a begin_record and ending with an end_record. Log
+  # records therefore can be analyzed as a group, for example to measure the
+  # time taken by an aggregate action such as a website request.
+  # 
+  # Log records must follow a particular protocol:
+  #
+  # * The first entry must be begin_record.
+  # * The last entry must be end_record.
+  # * No other begin_record or end_record entries may be present.
+  # * All entries must have the same non-nil record_id.
+  # * Entries must be in nondecreasing order of timestamp.
+  # 
+  # The only exception to these rules are incomplete log records, in which
+  # the final end_record is not present.
+  
+  class Record
+    
+    
+    # Create a log record.
+    # 
+    # You may optionally pass in an array of log entries to populate the
+    # record either partially or completely. If you do, note that
+    # Errors::IllegalRecordError may be raised if the log entries do not
+    # follow the log record protocol-- e.g. if the first entry is not a
+    # begin_record, etc.
+    
+    def initialize(entries_=nil)
+      @started = false
+      @complete = false
+      @message_count = 0
+      @entries = []
+      @attributes = {}
+      if entries_ && entries_.size > 0
+        entries_.each do |entry_|
+          add_entry(entry_)
+        end
+      end
+    end
+    
+    
+    def to_s  # :nodoc:
+      "#{type}: #{@line}"
+    end
+    
+    def inspect  # :nodoc:
+      "#<#{self.class}:0x#{object_id.to_s(16)} record_id=#{record_id.inspect}>"
+    end
+    
+    
+    # Append a log entry to this record.
+    # 
+    # Entries must be added in order. Raises Errors::IllegalRecordError if
+    # the log record protocol is violated
+    
+    def add_entry(entry_)
+      empty_ = @entries.size == 0
+      if entry_.type == :unknown_data
+        raise Errors::IllegalRecordError, "You cannot add an unknown_data entry to a record"
+      end
+      if empty_ && entry_.type != :begin_record
+        raise Errors::IllegalRecordError, "First entry in a record must be a begin_record"
+      elsif !empty_ && entry_.type == :begin_record
+        raise Errors::IllegalRecordError, "Extra begin_record found"
+      end
+      if @complete
+        raise Errors::IllegalRecordError, "Cannot have entries after end_record"
+      end
+      if empty_
+        if entry_.record_id.nil?
+          raise Errors::IllegalRecordError, "Entry has no record_id"
+        end
+      else
+        last_ = @entries.last
+        if last_.record_id != entry_.record_id
+          raise Errors::IllegalRecordError, "Entry has a mismatching record_id"
+        end
+        if last_.timestamp > entry_.timestamp
+          raise Errors::IllegalRecordError, "Entry's timestamp is earlier than the previous entry"
+        end
+      end
+      case entry_.type
+      when :begin_record
+        @started = true
+      when :end_record
+        @complete = true
+      when :attribute
+        case entry_.operation
+        when :set
+          @attributes[entry_.key] = entry_.value
+        when :append
+          val_ = @attributes[entry_.key]
+          case val_
+          when Array
+            val_ << entry_.value
+          when String
+            @attributes[entry_.key] = [val_, entry_.value]
+          when nil
+            @attributes[entry_.key] = [entry_.value]
+          end
+        end
+      when :message
+        @message_count += 1
+      end
+      @entries << entry_
+    end
+    
+    
+    # Returns true if the initial begin_record has been added.
+    
+    def started?
+      @started
+    end
+    
+    
+    # Returns true if the final end_record has been added.
+    
+    def complete?
+      @complete
+    end
+    
+    
+    # Returns the record ID as a string.
+    
+    def record_id
+      @entries.size > 0 ? @entries.first.record_id : nil
+    end
+    
+    
+    # Returns the beginning timestamp as a Time object, if the log record
+    # has been started, or nil otherwise.
+    
+    def begin_timestamp
+      @started ? @entries.first.timestamp : nil
+    end
+    
+    
+    # Returns the ending timestamp as a Time object, if the log record
+    # has been completed, or nil otherwise.
+    
+    def end_timestamp
+      @complete ? @entries.last.timestamp : nil
+    end
+    
+    
+    # Returns the number of log entries currently in this record.
+    
+    def entry_count
+      @entries.size
+    end
+    
+    alias_method :size, :entry_count
+    
+    
+    # Returns the number of message entries currently in this record.
+    
+    def message_count
+      @message_count
+    end
+    
+    
+    # Iterate over all log entries, passing each to the given block.
+    
+    def each_entry(&block_)
+      @entries.each(&block_)
+    end
+    
+    
+    # Iterate over all log message entries, passing each to the given block.
+    
+    def each_message
+      @entries.each do |entry_|
+        if entry_.type == :message
+          yield entry_
+        end
+      end
+    end
+    
+    
+    # Returns an array of all log entries.
+    
+    def all_entries
+      @entries.dup
+    end
+    
+    
+    # Returns an array of all log message entries.
+    
+    def all_messages
+      @entries.find_all{ |entry_| entry_.type == :message }
+    end
+    
+    
+    # Get the value of the given attribute.
+    # Returns a string if the attribute has a single value.
+    # Returns an array of strings if the attribute has multiple values.
+    # Returns nil if the attribute is not set.
+    
+    def attribute(key_)
+      @attributes[key_.to_s]
+    end
+    
+    
+    # Get an array of attribute keys present in this log record.
+    
+    def attribute_keys
+      @attributes.keys
+    end
+    
+    
+  end
+  
+  
+end

data/lib/sawmill/record_processor/conditionals.rb

@@ -0,0 +1,330 @@
+# -----------------------------------------------------------------------------
+# 
+# Sawmill basic entry processors that implement conditionals
+# 
+# -----------------------------------------------------------------------------
+# Copyright 2009 Daniel Azuma
+# 
+# All rights reserved.
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# * Neither the name of the copyright holder, nor the names of any other
+#   contributors to this software, may be used to endorse or promote products
+#   derived from this software without specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+# -----------------------------------------------------------------------------
+;
+
+
+module Sawmill
+  
+  module RecordProcessor
+    
+    
+    # An "if" conditional.
+    # 
+    # Takes a boolean condition processor and executes a processor on true
+    # (and optionally another one on false).
+    # 
+    # For example, this builds a processor that sends formatted log records
+    # to STDOUT only if they have a "user" attribute of "daniel":
+    # 
+    #  processor = Sawmill::RecordProcessor.build do
+    #    If(FilterByAttributes('user' => 'daniel'), Format(STDOUT))
+    #  end
+    
+    class If < Base
+      
+      
+      # Create an "if" conditional.
+      # 
+      # The first parameter must be a processor whose methods return a
+      # boolean value indicating whether the record should be accepted.
+      # The second parameter is a processor to run on accepted records.
+      # The optional third parameter is an "else" processor to run on
+      # rejected records.
+      
+      def initialize(condition_, on_true_, on_false_=nil)
+        @condition = condition_
+        @on_true = on_true_
+        @on_false = on_false_
+      end
+      
+      def record(record_)
+        if @condition.record(record_)
+          @on_true.record(record_)
+        elsif @on_false
+          @on_false.record(record_)
+        end
+      end
+      
+      def extra_entry(entry_)
+        if @condition.extra_entry(entry_)
+          @on_true.extra_entry(entry_)
+        elsif @on_false
+          @on_false.extra_entry(entry_)
+        end
+      end
+      
+      def close
+        @on_true.close
+        @on_false.close if @on_false
+      end
+      
+      
+    end
+    
+    
+    # A boolean processor that returns the boolean negation of a given
+    # processor.
+    # 
+    # For example, this builds a processor that sends formatted log records
+    # to STDOUT only if they do NOT have a "user" attribute of "daniel":
+    # 
+    #  processor = Sawmill::RecordProcessor.build do
+    #    If(Not(FilterByAttributes('user' => 'daniel')), Format(STDOUT))
+    #  end
+    
+    class Not < Base
+      
+      
+      # Create a "not" boolean.
+      # The parameter is a boolean processor to run. This processor returns
+      # the boolean negation of its output.
+      
+      def initialize(child_)
+        @child = _interpret_processor(child_)
+      end
+      
+      def record(record_)
+        !@child.record(record_)
+      end
+      
+      def extra_entry(entry_)
+        !@child.extra_entry(record_)
+      end
+      
+      def close
+        @child.close
+      end
+      
+      
+    end
+    
+    
+    # A boolean processor that returns true if and only if all its child
+    # processors return true. This version short-circuits the processing,
+    # so once one child returns false, subsequent children are not called
+    # at all.
+    # 
+    # For example, this builds a processor that sends formatted log records
+    # to STDOUT only if they have a "user" attribute of "daniel" AND their
+    # record ID is '12345678':
+    # 
+    #  processor = Sawmill::RecordProcessor.build do
+    #    If(And(FilterByAttributes('user' => 'daniel'),
+    #           FilterByRecordID('12345678')),
+    #       Format(STDOUT))
+    #  end
+    
+    class And < Base
+      
+      
+      # Create an "and" boolean.
+      # The parameters are child processors whose return values should be
+      # combined with an AND operation.
+      
+      def initialize(*children_)
+        @children = _interpret_processor_array(children_)
+      end
+      
+      def record(record_)
+        @children.each do |child_|
+          return false unless child_.record(record_)
+        end
+        true
+      end
+      
+      def extra_entry(entry_)
+        @children.each do |child_|
+          return false unless child_.extra_entry(entry_)
+        end
+        true
+      end
+      
+      def close
+        @children.each{ |child_| child_.close }
+      end
+      
+      
+    end
+    
+    
+    # A boolean processor that returns true if and only if any of its child
+    # processors returns true. This version short-circuits the processing,
+    # so once one child returns true, subsequent children are not called
+    # at all.
+    # 
+    # For example, this builds a processor that sends formatted log records
+    # to STDOUT only if they have a "user" attribute of "daniel" OR their
+    # record ID is '12345678':
+    # 
+    #  processor = Sawmill::RecordProcessor.build do
+    #    If(Or(FilterByAttributes('user' => 'daniel'),
+    #          FilterByRecordID('12345678')),
+    #       Format(STDOUT))
+    #  end
+    
+    class Or < Base
+      
+      
+      # Create an "or" boolean.
+      # The parameters are child processors whose return values should be
+      # combined with an OR operation.
+      
+      def initialize(*children_)
+        @children = _interpret_processor_array(children_)
+      end
+      
+      def record(record_)
+        @children.each do |child_|
+          return true if child_.record(record_)
+        end
+        false
+      end
+      
+      def extra_entry(entry_)
+        @children.each do |child_|
+          return true if child_.extra_entry(entry_)
+        end
+        false
+      end
+      
+      def close
+        @children.each{ |child_| child_.close }
+      end
+      
+      
+    end
+    
+    
+    # A boolean processor that returns true if and only if all its child
+    # processors return true. This version does not short-circuit the
+    # processing, so all children are always called even if an early one
+    # returns false. Thus, this processor is also a good one to use as a
+    # multiplexor to simply run a bunch of processors.
+    # 
+    # For example, this builds a processor that sends formatted log records
+    # to STDOUT only if they have a "user" attribute of "daniel" AND their
+    # record ID is '12345678':
+    # 
+    #  processor = Sawmill::RecordProcessor.build do
+    #    If(All(FilterByAttributes('user' => 'daniel'),
+    #           FilterByRecordID('12345678')),
+    #       Format(STDOUT))
+    #  end
+    # 
+    # This processor just formats both to STDOUT and STDERR:
+    # 
+    #  processor = Sawmill::RecordProcessor.build do
+    #    All(Format(STDOUT), Format(STDERR))
+    #  end
+    
+    class All < Base
+      
+      
+      # Create an "all" boolean.
+      # The parameters are child processors whose return values should be
+      # combined with an AND operation.
+      
+      def initialize(*children_)
+        @children = _interpret_processor_array(children_)
+      end
+      
+      def record(record_)
+        @children.inject(true) do |result_, child_|
+          child_.record(record_) && result_
+        end
+      end
+      
+      def extra_entry(entry_)
+        @children.inject(true) do |result_, child_|
+          child_.extra_entry(entry_) && result_
+        end
+      end
+      
+      def close
+        @children.each{ |child_| child_.close }
+      end
+      
+      
+    end
+    
+    
+    # A boolean processor that returns true if and only if any of its child
+    # processors returns true. This version does not short-circuit the
+    # processing, so all children are always called even if an early one
+    # returns true.
+    # 
+    # For example, this builds a processor that sends formatted log records
+    # to STDOUT only if they have a "user" attribute of "daniel" OR their
+    # record ID is '12345678':
+    # 
+    #  processor = Sawmill::RecordProcessor.build do
+    #    If(Any(FilterByAttributes('user' => 'daniel'),
+    #           FilterByRecordID('12345678')),
+    #       Format(STDOUT))
+    #  end
+    
+    class Any < Base
+      
+      
+      # Create an "any" boolean.
+      # The parameters are child processors whose return values should be
+      # combined with an OR operation.
+      
+      def initialize(*children_)
+        @children = _interpret_processor_array(children_)
+      end
+      
+      def record(record_)
+        @children.inject(false) do |result_, child_|
+          child_.record(record_) || result_
+        end
+      end
+      
+      def extra_entry(entry_)
+        @children.inject(false) do |result_, child_|
+          child_.extra_entry(entry_) || result_
+        end
+      end
+      
+      def close
+        @children.each{ |child_| child_.close }
+      end
+      
+      
+    end
+    
+    
+  end
+  
+end

data/lib/sawmill/record_processor/decompose.rb

@@ -0,0 +1,75 @@
+# -----------------------------------------------------------------------------
+# 
+# Sawmill record processor that decomposes a record into entries
+# 
+# -----------------------------------------------------------------------------
+# Copyright 2009 Daniel Azuma
+# 
+# All rights reserved.
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# * Neither the name of the copyright holder, nor the names of any other
+#   contributors to this software, may be used to endorse or promote products
+#   derived from this software without specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+# -----------------------------------------------------------------------------
+;
+
+
+module Sawmill
+  
+  module RecordProcessor
+    
+    
+    # A processor that decomposes records into constituent entries and
+    # passes those entries to an entry processor.
+    
+    class Decompose < Base
+      
+      
+      # Create a new decomposer that emits to the given entry processor.
+      
+      def initialize(processor_, opts_={})
+        @processor = processor_
+        @classifier = ::Sawmill::EntryClassifier.new(processor_)
+      end
+      
+      
+      def record(record_)
+        record_.each_entry{ |entry_| @classifier.entry(entry_) }
+        true
+      end
+      
+      def extra_entry(entry_)
+        @classifier.entry(entry_)
+        true
+      end
+      
+      def close
+        @processor.close
+      end
+      
+    end
+    
+    
+  end
+  
+end