jse 0.1.0

data/History.txt

@@ -0,0 +1,3 @@
+=== 0.1.0 / 2010-06-30
+
+* Initial release

data/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2010 Ryan Burrows
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/Manifest.txt

@@ -0,0 +1,14 @@
+LICENSE
+README.md
+History.txt
+Manifest.txt
+bin/jse
+lib/jse.rb
+lib/jse/cli.rb
+lib/jse/field_value_transform.rb
+lib/jse/filter.rb
+lib/jse/regexp_filter.rb
+lib/jse/stream.rb
+lib/jse/subset_transform.rb
+lib/jse/to_json_transform.rb
+lib/jse/version.rb

data/README.md

@@ -0,0 +1,24 @@
+# jse: Json Stream Editor
+
+jse allows for filtering and transforming of json streams. It is
+mainly designed for mining information out of json log files.
+
+## Usage
+
+Use the `jse` command
+
+### Filtering based on field values
+
+`jse field1:value field2:'another value' file.txt`
+
+### Regexp filtering on fields
+
+`jse field:/^pattern.+$/ file.txt`
+
+### Printing selected fields
+
+`jse message:/Error/ -f message log.txt`
+
+## Installation
+
+`gem install jse`

data/bin/jse

@@ -0,0 +1,7 @@
+#!/usr/bin/ruby
+
+require 'rubygems'
+require File.expand_path(File.dirname(__FILE__) + '/../lib/jse')
+require 'jse/cli'
+
+JSE::CLI.execute(STDOUT, ARGV)

data/lib/jse.rb

@@ -0,0 +1,12 @@
+$:.unshift(File.dirname(__FILE__)) unless
+  $:.include?(File.dirname(__FILE__))
+
+require 'jse/stream'
+require 'jse/filter'
+require 'jse/regexp_filter'
+require 'jse/subset_transform'
+require 'jse/to_json_transform'
+require 'jse/field_value_transform'
+
+module JSE
+end

data/lib/jse/cli.rb

@@ -0,0 +1,49 @@
+require 'optparse'
+
+module JSE
+  module CLI
+    def self.execute(stdout, arguments = [])
+      print = []
+      parser = OptionParser.new do |opts|
+        opts.banner = <<-BANNER.gsub(/^[ \t]*/, '')
+        Json Stream Editor.
+
+        Usage: #{File.basename($0)} [options]
+
+        Options are:
+      BANNER
+        opts.separator ""
+        opts.on("-f", "--fields a,b,c", Array,
+                "List of fields to return") do |fields|
+          print = fields
+        end
+        opts.on("-h", "--help",
+                "Show this help message.") { stdout.puts opts; exit }
+        opts.parse!(arguments)
+      end
+
+      if File.exist?(arguments.last)
+        stream = JSE::Stream.new(File.open(arguments.pop, 'r'))
+      else
+        stream = JSE::Stream.new(STDIN)
+      end
+
+      arguments.each do |arg|
+        field, text = arg.split(':')
+        stream.filter!(field, text)
+      end
+
+      unless print.empty?
+        stream.print!(*print)
+      end
+
+      begin
+        stream.each_line do |line|
+          stdout.puts line
+        end
+      rescue Errno::EPIPE
+        # Catch broken pipes so we can use head etc.
+      end
+    end
+  end
+end

data/lib/jse/field_value_transform.rb

@@ -0,0 +1,11 @@
+module JSE
+  class FieldValueTransform
+    def initialize(field)
+      @field = field
+    end
+
+    def apply(hash)
+      hash[@field]
+    end
+  end
+end

data/lib/jse/filter.rb

@@ -0,0 +1,11 @@
+module JSE
+  class Filter
+    def initialize(field, text)
+      @field, @text = field, text
+    end
+
+    def match?(json)
+      json[@field] == @text
+    end
+  end
+end

data/lib/jse/regexp_filter.rb

@@ -0,0 +1,13 @@
+module JSE
+  class RegexpFilter
+    def initialize(field, regexp)
+      @field = field
+      # regexp should have leading and trailing '/'
+      @regexp = Regexp.new(regexp[1,regexp.size-2])
+    end
+
+    def match?(json)
+      json[@field] =~ @regexp
+    end
+  end
+end

data/lib/jse/stream.rb

@@ -0,0 +1,91 @@
+require 'json'
+
+module JSE
+  class Stream
+    attr_accessor :filters, :transforms
+    attr_reader :inner
+
+    def initialize(inner)
+      @inner = inner
+      @transforms = []
+      @filters = []
+    end
+
+    def each_line
+      inner.each_line do |line|
+        unless filter_chain.eliminates?(line)
+          yield transform_chain.apply(line)
+        end
+      end
+    end
+
+    def filter!(field, text)
+      if looks_like_regexp?(text)
+        filters << RegexpFilter.new(field, text)
+      else
+        filters << Filter.new(field, text)
+      end
+    end
+
+    def print!(*field_list)
+      if field_list.size == 1
+        transforms << FieldValueTransform.new(field_list[0])
+      else
+        transforms << SubsetTransform.new(*field_list)
+        transforms << ToJsonTransform.new
+      end
+    end
+
+    private
+
+    def looks_like_regexp?(string)
+      string[0,1] == '/' && string[string.size-1,1] == '/'
+    end
+
+    def transform_chain
+      TransformChain.new(transforms)
+    end
+
+    class TransformChain
+      def initialize(transforms)
+        @transforms = transforms
+      end
+
+      def apply(line)
+        begin
+          if @transforms.empty?
+            line
+          else
+            json = JSON.parse(line)
+            @transforms.inject(json){ |j, transform| transform.apply(j) }
+          end
+        rescue JSON::ParserError
+          $stderr.puts "Error parsing line:"
+          $stderr.puts line
+          true
+        end
+      end
+    end
+
+    def filter_chain
+      FilterChain.new(filters)
+    end
+
+    class FilterChain
+      def initialize(filters)
+        @filters = filters
+      end
+
+      def eliminates?(line)
+        begin
+          json = JSON.parse(line)
+          !@filters.all?{ |f| f.match?(json) }
+        rescue JSON::ParserError
+          $stderr.puts "Error parsing line:"
+          $stderr.puts line
+          true
+        end
+      end
+    end
+  end
+end

data/lib/jse/subset_transform.rb

@@ -0,0 +1,14 @@
+module JSE
+  class SubsetTransform
+    def initialize(*fields)
+      @fields = fields
+    end
+
+    def apply(json)
+      @fields.inject({}) do |result, field|
+        result[field] = json[field]
+        result
+      end
+    end
+  end
+end

data/lib/jse/to_json_transform.rb

@@ -0,0 +1,7 @@
+module JSE
+  class ToJsonTransform
+    def apply(obj)
+      obj.to_json
+    end
+  end
+end

data/lib/jse/version.rb

@@ -0,0 +1,3 @@
+module JSE
+  VERSION = '0.1.0'
+end

metadata

@@ -0,0 +1,124 @@
+--- !ruby/object:Gem::Specification 
+name: jse
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Ryan Burrows
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-06-30 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: json
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: cucumber
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: fakefs
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id004
+description: "    JsonStreamEditor allows manipulation of streams of json\n"
+email: rhburrows@gmail.com
+executables: 
+- jse
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- LICENSE
+- README.md
+- History.txt
+- Manifest.txt
+- bin/jse
+- lib/jse.rb
+- lib/jse/cli.rb
+- lib/jse/field_value_transform.rb
+- lib/jse/filter.rb
+- lib/jse/regexp_filter.rb
+- lib/jse/stream.rb
+- lib/jse/subset_transform.rb
+- lib/jse/to_json_transform.rb
+- lib/jse/version.rb
+has_rdoc: true
+homepage: http://github.com/rhburrows/jse
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 1
+      - 3
+      - 6
+      version: 1.3.6
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: Stream editor for JSON streams
+test_files: []
+