xail 0.0.1

data/.gitignore

@@ -0,0 +1,7 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+.idea
+*.iml
+*.swp

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in xail.gemspec
+gemspec

data/README.md

@@ -0,0 +1,71 @@
+# xail - tail for winners
+
+xail is a super lightweight Ruby DSL for building simple stream/log analysis
+tools.
+
+A simple log viewer: `logview.xail.rb`:
+
+
+    #!/usr/bin/env xail
+    group('fatal') {
+        contains 'fatal'
+        bell
+    }
+
+    group('error') {
+        contains 'error', 'exception'
+        red
+        bold
+    }
+
+    group('warning') {
+        contains 'warn'
+        yellow
+    }
+
+You can then run it directly:
+
+    $ ./logview.xail.rb
+
+And it will accept input on stdin or a filename on the command line.
+
+You can directly specify:
+
+## Filters
+
+There are filters and compound filters. A filter takes a line from a stream
+and performs some action, potentially altering the stream, or terminating the flow.
+Compound filters take a stream and also a set of subfilters. These generally implement
+flow control.
+
+Stdout is the ultimate consumer of the strings, unless they've been filtered.
+
+### Compound Filters
+* `cascade` -- stream the result of first subfilter that streams
+* `composition` -- applies each subfilter on the stream of the preceding subfilter, streaming the final result
+
+* `and` -- streams the original if all subfilters stream
+* `or` -- streams the original if any subfilter streams
+* `not` -- streams the original if no subfilters stream
+
+### Matching Filters
+* `contains` -- streams the original if any of the parameters are included
+* `replace` -- mutates the stream
+
+### Alerting Filters
+* `execute` executes a command, replacing `{}` with the line content
+* `bell` rings a terminal bell (if possible)
+
+### Styling Filters
+* `black`, `red`, `green`, `yellow`, `blue`, `magenta`, `cyan`, `white` -- adjust foreground color
+* `onblack`, `onred`, `ongreen`, `onyellow`, `onblue`, `onmagenta`, `oncyan`, `onwhite` -- adjust background color
+* `bold`, `blink`, `underscore`, `negative`, `dark` -- apply effects to the text
+
+### Special Filters
+* `sample` -- samples the stream, only printing at the rate requested
+* `stop` -- stops processing of this stream and continues with the next
+* `count` -- [todo] computes the rate of the stream for display (need UI aspect)
+
+### Custom Filters
+
+You can easily develop your own filters. Either as an anonymous block:

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/xail

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'xail'
+

data/examples/logview.xail.rb

@@ -0,0 +1,34 @@
+#!/usr/bin/env xail
+#
+# a simple log error and warning highlighter
+#
+
+class ExecuteFilter < AbstractFilter
+  def initialize(command)
+    @command = command
+  end
+
+  def streamLine(line)
+    puts "EXECUTING #{@command} #{line}"
+    return line
+  end
+end
+
+
+group('fatal') {
+  contains 'fatal'
+  red
+  onblue
+  bell
+}
+
+group('error') {
+  contains 'error', 'exception'
+  red
+  bold
+}
+
+group('warning') {
+  contains 'warn'
+  yellow
+}

data/examples/logviewtest.txt

@@ -0,0 +1,11 @@
+[trace] hi
+[trace] there
+[warn]  vagrant
+[trace] how
+[error] are you doing?
+[fatal] all that?
+[error] stuff?
+[warn ] test
+[trace] finit.
+
+

data/lib/xail/config.rb

@@ -0,0 +1,62 @@
+
+require 'xail/filter'
+
+module Xail
+  module DSL
+    def get_binding
+      binding
+    end
+
+    def filter_scope(compound)
+      filter_in_scope << compound
+      @filter_stack << compound
+
+      yield
+
+      @filter_stack.pop
+    end
+
+    def stream(name, source = null)
+      source ||= name
+    end
+
+    def group(name, &filters)
+      # TODO intergrate with UX
+      filter_scope(FilterComposition.new) {
+        filters.yield
+      }
+    end
+
+    def has_final
+      @has_final
+    end
+
+    def rest(&filters)
+      if @has_final
+        raise "rest may only be specified once"
+      end
+
+      @has_final = true
+      filter_scope(FilterComposition.new) {
+        filters.yield
+      }
+    end
+
+    def filter_in_scope
+      @filter_stack ||= [FilterCascade.new]
+      @filter_stack.last
+    end
+
+    def method_missing(name, *args, &block)
+      filterClass = FilterRegistry::get_filter(name.downcase)
+      filter = filterClass.new(*args)
+      filter_in_scope << filter
+
+    rescue UnknownFilter => error
+      abort error.to_s
+
+    rescue => error
+      abort "#{filter_in_scope} will not accept #{name} as subfilter: #{error}"
+    end
+  end
+end

data/lib/xail/filter.rb

@@ -0,0 +1,273 @@
+
+require 'term/ansicolor'
+
+module Xail
+
+  class UnknownFilter < Exception
+    def initialize(filter)
+      super("Unknown filter ``#{filter}''. Known: #{FilterRegistry.filters.keys.sort}")
+    end
+  end
+
+  class FilterRegistry
+    def self.filters
+      @@filters ||= FilterRegistry.find_filters
+    end
+
+
+    # naively assume all filters are defined in this file, or at least loaded
+    # before this executes
+    def self.find_filters
+      filters = Hash.new
+      ObjectSpace.each_object(Class).select { |classObject|
+        classObject < AbstractFilter and
+        not classObject.name.downcase.include? "abstract"
+      }.map { |filterclass|
+        filtername = filterclass.name.split('::').last.gsub(/filter/i, '')
+        filters[filtername.downcase] = filterclass
+      }
+
+      filters
+    end
+
+    def self.get_filter(key)
+      name = key.to_s
+
+      @@filters ||= FilterRegistry.filters
+
+      if @@filters.has_key? name
+        return @@filters[name]
+      end
+
+      # if we couldn't find the filter, rebuild the list and try
+      # again
+      @@filters = FilterRegistry.find_filters
+
+      if @@filters.has_key? name
+        return @@filters[name]
+      else
+        raise UnknownFilter.new(name)
+      end
+    end
+  end
+
+
+
+
+  class AbstractFilter
+    def filterName
+      self.class.name.split('::').last.gsub(/filter/i, '')
+    end
+
+    def streamLine(line)
+    end
+  end
+
+
+
+  #
+  # Filter Composer
+  #
+
+  class AbstractCompoundFilter < AbstractFilter
+    def initialize
+      @filters = []
+    end
+
+    def <<(filter)
+      @filters << filter
+    end
+  end
+
+
+  # a cascade streams the next filter on rejection
+  class FilterCascade < AbstractCompoundFilter
+    def streamLine(input)
+      @filters.each do |filter|
+        line = filter.streamLine(input)
+        if line != nil
+          return line
+        end
+      end
+
+      nil
+    end
+  end
+
+
+  # a composition streams the next filter on success
+  class FilterComposition < AbstractCompoundFilter
+    def streamLine(input)
+      @filters.inject(input) do |line,filter|
+        if line != nil
+          filter.streamLine(line)
+        else
+          nil
+        end
+      end
+    end
+  end
+
+
+  # the and filter streams the original if all component filters stream
+  class AndFilter < AbstractCompoundFilter
+    def streamLine(line)
+      @filters.each do |filter|
+        if !filter.streamLine(line)
+          return nil
+        end
+      end
+
+      line
+    end
+  end
+
+
+  # the or filter streams the original if any component filter streams
+  class OrFilter < AbstractCompoundFilter
+    def streamLine(line)
+      @filters.each do |filter|
+        if filter.streamLine line
+          return line
+        end
+      end
+
+      nil
+    end
+  end
+
+
+  # the not filter streams the original if none of the component filters stream
+  class NotFilter < AndFilter
+    def streamLine(line)
+      result = super.streamLine(line)
+
+      if result != nil
+        nil
+      else
+        line
+      end
+    end
+  end
+
+  class ContainsFilter < AbstractFilter
+    def initialize(*keys)
+      @keys = keys
+    end
+
+    def streamLine(line)
+      @keys.each do |key|
+        if line.include?(key)
+          return line
+        end
+      end
+
+      nil
+    end
+  end
+
+  class ReplaceFilter < AbstractFilter
+    def initialize(regexp)
+      @regexp = regexp
+    end
+
+    def streamLine(line)
+      # why fold when you can iterate
+      final = line
+      @regexp.each_pair do |patt,val|
+        final.gsub!(patt,val)
+      end
+      final
+    end
+  end
+
+
+  # the stop filter never streams
+  class StopFilter < AbstractFilter
+    def streamLine(line)
+      nil
+    end
+  end
+
+  # the pass through filter always streams
+  class PassThroughFilter < AbstractFilter
+    def streamLine(line)
+      line
+    end
+  end
+
+
+
+
+  #
+  # Stream Mutators
+  #
+
+  class SampleFilter < AbstractFilter
+    def initialize(params)
+      @rate = params.to_i
+      @count = 0
+    end
+
+    def streamLine(line)
+      res = if @count % @rate == 0
+        line
+      else
+        ""
+      end
+
+      @count += 1
+      res
+    end
+  end
+
+
+
+  #
+  # Stream Decorators
+  #
+
+  class BellFilter < AbstractFilter
+    def streamLine(line)
+      return "\a"+line
+    end
+  end
+
+  class AbstractColorFilter < AbstractFilter
+    def initialize
+      @colors = Term::ANSIColor
+    end
+
+    def streamLine(line)
+      return @colors.send(filterName.gsub(/On/,"on_").downcase, line)
+    end
+  end
+
+
+  # foregrounds
+  class Black      < AbstractColorFilter; end
+  class Red        < AbstractColorFilter; end
+  class Green      < AbstractColorFilter; end
+  class Yellow     < AbstractColorFilter; end
+  class Blue       < AbstractColorFilter; end
+  class Magenta    < AbstractColorFilter; end
+  class Cyan       < AbstractColorFilter; end
+  class White      < AbstractColorFilter; end
+
+  # backgrounds
+  class OnBlack    < AbstractColorFilter; end
+  class OnRed      < AbstractColorFilter; end
+  class OnGreen    < AbstractColorFilter; end
+  class OnYellow   < AbstractColorFilter; end
+  class OnBlue     < AbstractColorFilter; end
+  class OnMagenta  < AbstractColorFilter; end
+  class OnCyan     < AbstractColorFilter; end
+  class OnWhite    < AbstractColorFilter; end
+
+  # effects
+  class Bold       < AbstractColorFilter; end
+  class Blink      < AbstractColorFilter; end
+  class Underscore < AbstractColorFilter; end
+  class Negative   < AbstractColorFilter; end
+  class Dark       < AbstractColorFilter; end
+end

data/lib/xail/ui.rb

@@ -0,0 +1,12 @@
+module Xail
+  class UI
+    def write_line()
+
+    end
+
+    def write_status_line()
+
+    end
+  end
+end
+

data/lib/xail/version.rb

@@ -0,0 +1,3 @@
+module Xail
+  VERSION = "0.0.1"
+end

data/lib/xail.rb

@@ -0,0 +1,43 @@
+require 'yaml'
+require 'trollop'
+require 'pp'
+
+require 'xail/version'
+require 'xail/filter'
+require 'xail/config'
+
+module Xail
+  @opts = Trollop::options do
+    version "xail - #{VERSION} (c) 2012 w.k.macura - Released under BSD license"
+    banner <<-EOS
+A Ruby utility for performing basic stream processing, directly focused on increasing error visibility in logs
+    EOS
+  end
+
+
+  def Xail.run(configuration)
+
+    begin
+      extend Xail::DSL
+
+      eval(configuration)
+      filter = filter_in_scope
+
+      if !has_final
+        filter << PassThroughFilter.new
+      end
+    end
+
+
+    stream = $stdin
+    stream.each() do |line|
+      streamed = filter.streamLine(line)
+      if streamed and streamed.size > 0
+        printf streamed
+      end
+    end
+  end
+
+  config = IO.read(ARGV[0])
+  Xail.run(config)
+end

data/spec/filter_spec.rb

@@ -0,0 +1,83 @@
+
+require 'xail/filter'
+
+include Xail
+
+describe BellFilter do
+  it "prepends a bell to the stream" do
+    f = BellFilter.new
+    f.streamLine("hi").should eq("\ahi")
+  end
+end
+
+describe Yellow do
+  it "should apply escape codes to the filter" do
+    f = Yellow.new
+    f.streamLine("hi").should eq("\e[33mhi\e[0m")
+  end
+end
+
+describe PassThroughFilter do
+  it "should passthrough streams" do
+    f = PassThroughFilter.new
+    f.streamLine("hi").should eq("hi")
+  end
+end
+
+describe StopFilter do
+  it "should stop all streams" do
+    f = StopFilter.new
+    f.streamLine("hi").should eq(nil)
+  end
+end
+
+describe ContainsFilter do
+  it "should stream if it matches" do
+    f = ContainsFilter.new("hi")
+    f.streamLine("bye").should eq(nil)
+    f.streamLine("hi there").should eq("hi there")
+  end
+end
+
+describe ReplaceFilter do
+  it "should replace on streams" do
+    f = ReplaceFilter.new(/hi/ => 'bye', /there/ => 'potato')
+    f.streamLine("hi there").should eq("bye potato")
+  end
+end
+
+describe SampleFilter do
+  it "should sample at the given rate" do
+    f = SampleFilter.new(5)
+    f.streamLine("a").should eq "a"
+    f.streamLine("b").should eq ""
+    f.streamLine("c").should eq ""
+    f.streamLine("d").should eq ""
+    f.streamLine("e").should eq ""
+    f.streamLine("f").should eq "f"
+  end
+end
+
+describe FilterCascade do
+  it "should cascade through" do
+    f = FilterCascade.new
+    f << StopFilter.new
+    f << StopFilter.new
+    f << ContainsFilter.new("hi")
+    f << StopFilter.new
+
+    f.streamLine("bye").should eq(nil)
+    f.streamLine("hi").should eq("hi")
+  end
+end
+
+describe FilterComposition do
+  it "should compose through" do
+    f = FilterComposition.new
+    f << ReplaceFilter.new('a'=>'b')
+    f << ContainsFilter.new('hi')
+
+    f.streamLine('bye').should eq(nil)
+    f.streamLine('hi there alice').should eq('hi there blice')
+  end
+end

data/xail.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "xail/version"
+
+Gem::Specification.new do |s|
+  s.name        = "xail"
+  s.version     = Xail::VERSION
+  s.authors     = ["Wiktor Macura"]
+  s.email       = ["wiktor@tumblr.com"]
+  s.homepage    = ""
+  s.summary     = %q{tail for winners}
+  s.description = %q{A lightweight Ruby DSL for building text-stream filters}
+
+  s.rubyforge_project = "xail"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- spec/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # specify any dependencies here; for example:
+  s.add_development_dependency "rspec"
+
+  s.add_runtime_dependency "trollop"
+  s.add_runtime_dependency "term-ansicolor"
+end

metadata

@@ -0,0 +1,94 @@
+--- !ruby/object:Gem::Specification
+name: xail
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Wiktor Macura
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-02-24 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70198967869060 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70198967869060
+- !ruby/object:Gem::Dependency
+  name: trollop
+  requirement: &70198967868420 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70198967868420
+- !ruby/object:Gem::Dependency
+  name: term-ansicolor
+  requirement: &70198967867800 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70198967867800
+description: A lightweight Ruby DSL for building text-stream filters
+email:
+- wiktor@tumblr.com
+executables:
+- xail
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- README.md
+- Rakefile
+- bin/xail
+- examples/logview.xail.rb
+- examples/logviewtest.txt
+- lib/xail.rb
+- lib/xail/config.rb
+- lib/xail/filter.rb
+- lib/xail/ui.rb
+- lib/xail/version.rb
+- spec/filter_spec.rb
+- xail.gemspec
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: xail
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: tail for winners
+test_files:
+- spec/filter_spec.rb