winsome_wolverine 0.0.1

data/README.textile

@@ -0,0 +1,84 @@
+h1. Wolverine
+
+An appetite for logs.
+
+
+h2. Summary
+
+Wolverine is a library for processing event streams.  Processing starts with a source of events, such as a log file.  Filters wrap a source and themselves look like a source.  The DSL supplies shortcut methods to easily build a pipeline of filters in this way.  Filters are processed incrementally as events are required; one can quickly get the first few events from a very long stream.
+
+Events may be stored in a database for easy indexing.  See the @ActiveRecordSource@, which provides an optimized @where@ filter.
+
+For a list of convenience methods for building filter pipelines, see @FilterDSL@.
+
+When processing a Rails log with multiple lines per requests and multiple web processes writing to the log concurrently, Wolverine can filter requests matching a particular criteria in a particular context.  For example, it can tell you the number of exceptions thrown on POST requests, even though those bits of information are contained in different log messages.
+
+
+h2. Examples
+
+<pre>
+<code>
+  include Wolverine
+
+  # Read events from a file, one per line
+  requests = FileSource.new("log/production.log").
+
+    # In this log, subsequent lines of a message may appear indented,
+    # so coalesce them into one event.
+    append_indented.
+
+    # Parse text out into fields using regex captures.  Imagine a log
+    # line that looks like this:
+    #   Nov 19 19:08:27.931605 dhcp-172-25-211-77 45471: Hello World
+    field(/\A(\w+ \d+ \d+:\d+:\d+\.\d{6}) (\w+) (\w+):/m,
+          :timestamp, :host, :pid).
+
+    # Group messages from a single request together.
+    # Since the logs are from multiple web server processes, messages
+    # might be interleaved with those from other processes.  Use the
+    # :pid and :host fields to separate them.  Whenever a processes
+    # logs "Processing," that's a new request.
+    group(:following => [:pid, :host], :from => /: Processing/).
+
+    # Pull some per-request fields out of the requests.
+    fields(/Processing (\w+)#(\w+)/, :controller, :action).
+    fields(/Session ID: (\w+)/, :session_id)
+
+  # Filter by a field.  Note this returns a new filter immediately
+  # but does not consume any input yet.
+  accounts_requests = requests.where(:controller => "AccountsController")
+
+  # Consume the entire stream and count the number of events
+  accounts_requests.count
+
+  # View the first five matches with "less"
+  accounts_requests.head(5).less
+
+  # Group requests into session traces
+  sessions = requests.group(:following => :session_id)
+
+  # Turn the timestamp into a real ruby object
+  ts_requests = requests.parse_time(:timestamp)
+
+  # Shuffle two streams into one, with events in order of timestamp
+  ts_requests.interleave(other_host_requests)
+</code>
+</pre>
+
+Wolverine works well interactively:
+
+bc. $ irb -r wolverine
+
+You may also wish to store some helper functions specific to your log format and/or task at hand to pre-load into your session:
+
+bc. $ irb -r my-app
+
+
+h2. Road Map
+
+* Sources to read files over SSH
+* Sources that can tail a file
+* Histograms and tables
+* Faster field extraction (regex matching is the bottleneck)
+* Processing multiple pipelines in one pass over the underlying data
+* Sliding window filters that emit when a 5-minute average exceeds a threshold

data/lib/winsome_wolverine.rb

@@ -0,0 +1 @@
+require 'wolverine'

data/lib/wolverine.rb

@@ -0,0 +1,23 @@
+module Wolverine
+end
+
+require 'wolverine/filter_dsl'
+require 'wolverine/enumerable'
+require 'wolverine/source'
+require 'wolverine/file_source'
+require 'wolverine/active_record_source'
+require 'wolverine/array_source'
+require 'wolverine/filter'
+require 'wolverine/append_indented_filter'
+require 'wolverine/field_filter'
+require 'wolverine/parse_time_filter'
+require 'wolverine/head_filter'
+require 'wolverine/where_filter'
+require 'wolverine/group_filter'
+require 'wolverine/interleave_filter'
+require 'wolverine/sink'
+require 'wolverine/count_sink'
+require 'wolverine/less_sink'
+require 'wolverine/file_sink'
+require 'wolverine/active_record_sink'
+require 'wolverine/event'

data/lib/wolverine/Filter.rb

@@ -0,0 +1,11 @@
+module Wolverine
+  class Filter < Wolverine::Enumerable
+    def initialize(source)
+      @source = source
+    end
+    # The base filter implements the identity transformation.
+    def each(&block)
+      @source.each(&block)
+    end
+  end
+end

data/lib/wolverine/active_record_sink.rb

@@ -0,0 +1,32 @@
+module Wolverine
+  class ActiveRecordSink < Sink
+    attr_reader :columns, :klass
+    def initialize(source, klass)
+      super(source)
+      @klass = klass
+      @columns = klass.columns.map(&:name) - %w{id created_at updated_at}
+    end
+    def self.create(source, table_name, columns, connection_params)
+      require 'active_record'
+      klass = Class.new(ActiveRecord::Base)
+      klass.class_eval { self.table_name = table_name }
+      klass.establish_connection(connection_params)
+      klass.connection.create_table table_name do |t|
+        t.column :text, :text
+        columns.each do |col|
+          t.column col, :string
+        end
+        #t.timestamps
+      end
+      self.new(source, klass)
+    end
+    def run
+      @source.each do |evt|
+        hsh = {}
+        columns.each {|col| hsh[col] = evt.send(col) }
+        rec = @klass.new(hsh)
+        rec.save!
+      end
+    end
+  end
+end

data/lib/wolverine/active_record_source.rb

@@ -0,0 +1,71 @@
+module Wolverine
+  class ActiveRecordSource < Source
+    attr_reader :klass
+    def initialize(active_record_klass, conditions={}, search=nil)
+      @klass = active_record_klass
+      @conds = conditions
+      @search = search
+      require 'logger'
+      ActiveRecord::Base.logger = Logger.new(STDOUT)
+    end
+    def self.open(table_name, connection_params)
+      require 'active_record'
+      klass = Class.new(ActiveRecord::Base)
+      klass.class_eval do
+        self.table_name = table_name
+        def to_s
+          text
+        end
+      end
+      klass.establish_connection(connection_params)
+      self.new(klass)
+    end
+    def each
+      if can_search_with_conditions
+        evts = @klass.where({})
+        evts = evts.where(@conds) if !@conds.empty?
+        if @search
+          evts = evts.where(["text like ?", "%#{@search}%"])
+          # Add a fulltext search if we're on MySQL.  Boolean fulltext search
+          # is slightly less strict than "like", so it will only be a prefilter.
+          # Fixme: table could be Innodb...
+          if !@search.include? '"' and
+              evts.connection.class.name.include? "Mysql"
+            evts = evts.where(["MATCH (text) AGAINST (? IN BOOLEAN MODE)",
+                               "\"#{@search}\""])
+          end
+        end
+      else
+        if @search
+          evts = @klass.find(:all, :conditions =>
+                                      ["text like ?", "%#{@search}%"])
+        else
+          evts = @klass.find(:all, :conditions => @conds)
+        end
+      end
+      evts.each {|rec| yield rec }
+    end
+    def where(conditions=nil)
+      if @search && !can_search_with_conditions
+        raise Exception, "Can't do both where and search"
+      end
+
+      if conditions
+        conds = @conds.merge(conditions)
+        return self.class.new(@klass, conds, @search)
+      else
+        # Get ready for where.not(...), which should happen in-memory
+        super(conditions)
+      end
+    end
+    def search(word)
+      if @conds && !can_search_with_conditions
+        raise Exception, "Can't do both search and where"
+      end
+      self.class.new(@klass, @conds, word)
+    end
+    def can_search_with_conditions
+      @klass.respond_to? :where
+    end
+  end
+end

data/lib/wolverine/append_indented_filter.rb

@@ -0,0 +1,31 @@
+module Wolverine
+  class AppendIndentedFilter < Filter
+    def each
+      merged_evt = nil
+      prefix = self.prefix
+      @source.each do |evt|
+        if evt.to_s.start_with? prefix
+          if merged_evt
+            merged_evt = merge_events(merged_evt, evt)
+            # else drop the partial event
+          end
+        else
+          if merged_evt
+            yield merged_evt
+          end
+          merged_evt = evt
+        end
+      end
+      if merged_evt
+        yield merged_evt
+      end
+    end
+    def merge_events(ev1, ev2)
+      str = ev2.to_s
+      return ev1.to_s + str.slice(prefix.length..str.length)
+    end
+    def prefix
+      "  "
+    end
+  end
+end

data/lib/wolverine/array_source.rb

@@ -0,0 +1,10 @@
+module Wolverine
+  class ArraySource < Source
+    def initialize(array)
+      @array = array
+    end
+    def each
+      @array.each {|str| yield Event.new(str) }
+    end
+  end
+end

data/lib/wolverine/core_ext.rb

@@ -0,0 +1,9 @@
+module Wolverine::CoreExt::Symbol
+  def to_proc
+    proc { |obj, *args| obj.send(self, *args) }
+  end
+end
+
+unless :foo.respond_to? :to_proc
+  Symbol.send(:include, Wolverine::CoreExt::Symbol)
+end

data/lib/wolverine/count_sink.rb

@@ -0,0 +1,12 @@
+module Wolverine
+  class CountSink < Sink
+    def initialize(source)
+      @source = source
+    end
+    def run
+      cnt = 0
+      @source.each {|evt| cnt += 1 }
+      cnt
+    end
+  end
+end

data/lib/wolverine/enumerable.rb

@@ -0,0 +1,27 @@
+module Wolverine
+  class Enumerable
+    include ::Enumerable
+    include FilterDSL
+    def to_strings
+      map {|el| el.to_s}
+    end
+    def to_a
+      filterize super
+    end
+    def map(*args)
+      filterize super
+    end
+    alias collect map
+
+    protected
+
+    def filterize(arr)
+      arr.extend(FilterDSL)
+      def arr.inspect
+        super if size < 1
+        "[#{first.inspect}, (size: #{size})...]"
+      end
+      arr
+    end
+  end
+end

data/lib/wolverine/event.rb

@@ -0,0 +1,12 @@
+class Wolverine::Event
+  attr_reader :text
+  def initialize(text)
+    @text = text
+  end
+  def <=>(other)
+    to_s <=> other.to_s
+  end
+  def to_s
+    @text
+  end
+end

data/lib/wolverine/field_filter.rb

@@ -0,0 +1,34 @@
+module Wolverine
+  class FieldFilter < Filter
+    def initialize(source, regex, *fields)
+      super(source)
+      @regex = regex
+      @fields = fields
+      @class = Class.new(Event)
+      @class.class_eval <<-EOF, __FILE__, __LINE__
+        def initialize(text, evt, #{@fields.join(", ")})
+          super(text)
+          @evt = evt
+          #{@fields.map {|f| "@#{f}" }.join(", ")} = #{@fields.join(", ")}
+        end
+        def method_missing(name, *args)
+          # Danger: bypassing method access modifier (private/protected)
+          @evt.send(name) if args.empty?
+        end
+      EOF
+      @class.send(:attr_reader, *@fields)
+      # Explicitly enumerating the variables passed is faster than splat
+      self.instance_eval <<-EOF, __FILE__, __LINE__
+        def self.each
+          @source.each do |evt|
+            md = @regex.match(evt.to_s) || []
+            #md = ["pepper.bp", "12345"]
+            #{@fields.join(", ")} =
+              #{@fields.length > 1 ? "md[1..#{@fields.length}]" : "md[1]"}
+            yield @class.new(evt.to_s, evt, #{@fields.join(", ")})
+          end
+        end
+      EOF
+    end
+  end
+end

data/lib/wolverine/file_sink.rb

@@ -0,0 +1,13 @@
+module Wolverine
+  class FileSink < Sink
+    def initialize(source, filename)
+      super(source)
+      @filename = filename
+    end
+    def run
+      File.open(@filename, "w") do |file|
+        @source.each {|evt| file.write(evt) }
+      end
+    end
+  end
+end

data/lib/wolverine/file_source.rb

@@ -0,0 +1,38 @@
+require 'progressbar'
+module Wolverine
+  class FileSource < Source
+    attr_accessor :progress, :gzip
+    def initialize(filename, opts={})
+      @filename = filename
+      self.progress = opts[:progress]
+      self.gzip = opts.key?(:gzip) ? opts[:gzip] : filename.match(/\.gz$/i)
+      require 'zlib' if self.gzip
+    end
+    def each
+      progress = self.progress
+      if progress
+        size = File.size(@filename)
+        bar = ProgressBar.new(@filename, size)
+        bar.file_transfer_mode
+      end
+      File.open(@filename, "r") do |file|
+        realfile = file
+        file = Zlib::GzipReader.new(file) if self.gzip
+        bar_update = 10_000
+        cnt = 0
+        file.each do |line|
+          if progress
+            if cnt < bar_update
+              cnt += 1
+            else
+              bar.set(realfile.tell)
+              cnt = 0
+            end
+          end
+          yield line
+        end
+      end
+      bar.finish if progress
+    end
+  end
+end

data/lib/wolverine/filter_dsl.rb

@@ -0,0 +1,37 @@
+module Wolverine
+  module FilterDSL
+    def append_indented
+      AppendIndentedFilter.new(self)
+    end
+    def field(regex, *fields)
+      FieldFilter.new(self, regex, *fields)
+    end
+    def parse_time(source_field, dest_field=source_field)
+      ParseTimeFilter.new(self, source_field, dest_field)
+    end
+    def head(count, opts={})
+      HeadFilter.new(self, count, opts)
+    end
+    def where(conditions=nil)
+      WhereFilter.where(self, conditions)
+    end
+    def count
+      CountSink.new(self).run
+    end
+    def less
+      LessSink.new(self).run
+    end
+    def save_file(filename)
+      FileSink.new(self, filename).run
+    end
+    def save_records(active_record_klass)
+      ActiveRecordSink.new(self, active_record_klass).run
+    end
+    def group(opts)
+      GroupFilter.new(self, opts)
+    end
+    def interleave(other_source)
+      InterleaveFilter.new(self, other_source)
+    end
+  end
+end

data/lib/wolverine/group_filter.rb

@@ -0,0 +1,45 @@
+module Wolverine
+  class GroupFilter < Filter
+    def initialize(source, opts)
+      super(source)
+      @following = arg_to_a(opts[:following])
+      @from = opts[:from]
+    end
+    def each
+      groups = Hash.new { |hash,k| hash[k] = [] }
+      @source.each do |evt|
+        key = @following.map {|meth| evt.send(meth) }
+        group = groups[key] 
+        if @from
+          if @from.match(evt.to_s)
+            if group.any?
+              yield merge_events(group)
+              group.clear
+            end
+            group.push(evt)
+          else
+            group.push(evt) unless group.empty?
+          end
+        else
+          group.push(evt)
+        end
+      end
+      # ordering?
+      groups.each do |k,group|
+        yield merge_events(group)
+      end
+    end
+    private
+    def arg_to_a(arg)
+      if Array === arg
+        arg
+      else
+        [arg]
+      end
+    end
+    def merge_events(group)
+      Event.new(group.map {|evt| evt.to_s }.join(""))
+    end
+  end
+end
+

data/lib/wolverine/gzip_file_source.rb

@@ -0,0 +1,34 @@
+require 'progressbar'
+module Wolverine
+  class GzipFileSource < Source
+    attr_accessor :progress
+    def initialize(filename, opts={})
+      @filename = filename
+      self.progress = opts[:progress]
+    end
+    def each
+      progress = self.progress
+      if progress
+        size = File.size(@filename)
+        bar = ProgressBar.new(@filename, size)
+        bar.file_transfer_mode
+      end
+      File.open(@filename, "r") do |file|
+        bar_update = 10_000
+        cnt = 0
+        file.each do |line|
+          if progress
+            if cnt < bar_update
+              cnt += 1
+            else
+              bar.set(file.tell)
+              cnt = 0
+            end
+          end
+          yield Event.new(line)
+        end
+      end
+      bar.finish if progress
+    end
+  end
+end

data/lib/wolverine/head_filter.rb

@@ -0,0 +1,29 @@
+module Wolverine
+  class HeadFilter < Filter
+    attr_accessor :progress
+    def initialize(source, limit, opts={})
+      super(source)
+      @limit = limit
+      self.progress = opts[:progress]
+    end
+    def each
+      progress = self.progress
+      bar = ProgressBar.new("head(#{number_with_delimiter @limit})",
+                            @limit) if progress
+      cnt = 0
+      @source.each do |evt|
+        bar.set(cnt) if progress
+        if cnt >= @limit
+          bar.finish if progress
+          return
+        end
+        yield evt
+        cnt += 1
+      end
+    end
+    # From Rails.
+    def number_with_delimiter(number, delimiter=",")
+        number.to_s.gsub(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1#{delimiter}")
+    end
+  end
+end

data/lib/wolverine/interleave_filter.rb

@@ -0,0 +1,20 @@
+require 'generator'
+
+module Wolverine
+  class InterleaveFilter < Filter
+    def initialize(source, other_source)
+      super(source)
+      @other_source = other_source
+    end
+    def each(&block)
+      # The Generator docs warn these are slow b/c they use continuations
+      generators = [Generator.new(@source), Generator.new(@other_source)]
+      generators.reject!(&:end?)
+      while generators.any?
+        generators = generators.sort_by(&:current) 
+        yield generators.first.next
+        generators.shift if generators.first.end?
+      end
+    end
+  end
+end

data/lib/wolverine/less_sink.rb

@@ -0,0 +1,14 @@
+module Wolverine
+  class LessSink < Sink
+    def run
+      IO.popen(pager_command, "a") do |pipe|
+        @source.each {|evt| pipe.puts evt }
+        nil
+      end
+    end
+    def pager_command
+      "less"
+    end
+  end
+end
+

data/lib/wolverine/parse_time_filter.rb

@@ -0,0 +1,41 @@
+module Wolverine
+  class ParseTimeFilter < FieldFilter
+    def initialize(source, source_field, dest_field=source_field)
+      @source_field = source_field
+      @dest_field = dest_field
+      super(source, nil, dest_field)
+      # each is defined by eval in superclass
+      #self.send(:alias_method, :each, :_each)
+      def self.each(*args, &block)
+        _each(*args, &block)
+      end
+      @class.class_eval <<-EOF, __FILE__, __LINE__
+        def <=>(other)
+          mine = self.#@dest_field
+          yours = other.#@dest_field
+
+          # Handle nils; sort them first
+          if mine.nil?
+            if yours.nil?
+              return 0
+            end
+            return -1
+          end
+          if yours.nil?
+            return 1
+          end
+
+          mine <=> yours
+        end
+      EOF
+    end
+
+    def _each
+      @source.each do |evt|
+        ts_text = evt.send(@source_field)
+        ts = Time.parse(ts_text) if ts_text
+        yield @class.new(evt.to_s, evt, ts)
+      end
+    end
+  end
+end

data/lib/wolverine/sink.rb

@@ -0,0 +1,11 @@
+module Wolverine
+  class Sink
+    def initialize(source)
+      @source = source
+    end
+    def run
+      @source.each { }
+      nil
+    end
+  end
+end

data/lib/wolverine/source.rb

@@ -0,0 +1,8 @@
+module Wolverine
+  class Source < Wolverine::Enumerable
+    def each(&block)
+      raise Exception, "Override me"
+      yield event
+    end
+  end
+end

data/lib/wolverine/where_filter.rb

@@ -0,0 +1,50 @@
+module Wolverine
+  class WhereFilter < Filter
+    attr_reader :conditions
+    def initialize(source, conditions)
+      super(source)
+      @conditions = Conditions.new(self, conditions)
+    end
+    def each
+      @source.each do |evt|
+        next unless @conditions.match?(evt)
+        yield evt
+      end
+    end
+    def self.where(source, conditions=nil)
+      where = WhereFilter.new(source, conditions)
+      if conditions
+        where
+      else
+        where.conditions
+      end
+    end
+  end
+
+  class Conditions
+    def initialize(where, conds=nil)
+      @where = where
+      set_conds(conds)
+    end
+    def match?(evt)
+      @conditions_hash.each do |k,v|
+        return @negate unless v === evt.send(k)
+      end
+      return !@negate
+    end
+    def not(conds)
+      set_conds(conds)
+      @negate = true
+      @where
+    end
+    private
+    def set_conds(conds)
+      return unless conds
+      @conditions_hash = case conds
+        when Regexp then {:to_s => conds}
+        when Hash then conds
+        else raise ArgumentError, "Unknown parameter type: #{conds.class}"
+        end
+    end
+  end
+end

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification 
+name: winsome_wolverine
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Marcel M. Cary
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-11-28 00:00:00 -08:00
+default_executable: 
+dependencies: []
+
+description: Library and DSL to process log files with a pipe-and-filter architecture
+email: marcel@oak.homeunix.org
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- README.textile
+- lib/winsome_wolverine.rb
+- lib/wolverine/active_record_sink.rb
+- lib/wolverine/active_record_source.rb
+- lib/wolverine/append_indented_filter.rb
+- lib/wolverine/array_source.rb
+- lib/wolverine/core_ext.rb
+- lib/wolverine/count_sink.rb
+- lib/wolverine/enumerable.rb
+- lib/wolverine/event.rb
+- lib/wolverine/field_filter.rb
+- lib/wolverine/file_sink.rb
+- lib/wolverine/file_source.rb
+- lib/wolverine/Filter.rb
+- lib/wolverine/filter_dsl.rb
+- lib/wolverine/group_filter.rb
+- lib/wolverine/gzip_file_source.rb
+- lib/wolverine/head_filter.rb
+- lib/wolverine/interleave_filter.rb
+- lib/wolverine/less_sink.rb
+- lib/wolverine/parse_time_filter.rb
+- lib/wolverine/sink.rb
+- lib/wolverine/source.rb
+- lib/wolverine/where_filter.rb
+- lib/wolverine.rb
+has_rdoc: true
+homepage: http://github.com/mcary/wolverine
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.4.2
+signing_key: 
+specification_version: 3
+summary: Log file processor
+test_files: []
+