parsely 0.1.1

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in parsely.gemspec
+gemspec

data/Rakefile

@@ -0,0 +1,19 @@
+require 'bundler/gem_tasks'
+
+ENV['PATH']+=':'+File.join(File.dirname(__FILE__),'bin/')
+ENV['RUBYLIB']||=''
+ENV['RUBYLIB']+=':'+File.join(File.dirname(__FILE__),'lib/')
+task :default => [:test]
+
+desc 'run tests'
+task :test do
+  ruby "-Ilib test/cli-runner.rb"
+end
+
+desc 'create new test dir'
+task :newtest, :name do |t, args|
+  old = Dir.entries('test/basic/').last
+  new = old[/^(\d+)/, 1].succ
+  new = new + '-' + args[:name] if args[:name]
+  cp_r 'test/basic/'+old, 'test/basic/'+new
+end

data/TODO

@@ -0,0 +1,20 @@
+- do by using regex to split on last by <field> and assume none if missing and just run as map over groups
+- _2 in dump by _1 in other
+- _2 in dump by _1 in other
++ _1 if /www/
+- _1 if _2 =~ x
+- .parselyrc
+- split by: if a X or not, or if _1 in other_file (subsides comm)
++ in "foo 1\nbar 2" "sum(_2) if _2 < 3" to see how much I can delete
+- _1._1 for submatch (e.g. user agent)
+- index of <foo> (e.g. when something is processing ines and you want to check where it is)
+- skip line e.g. 123\na\nb\nc -> parsely _2 unless idx = 0
+- normalize e.g. uniq on 
+/?email=digest%40digg.com&name=&callback=jsonp1310305312089
+/?email=digest%40digg.com&name=&callback=jsonp1310305312088
+/?email=digest%40digg.com&name=&callback=jsonp1310305312085
+/?email=digest%40digg.com&name=&callback=jsonp1310305312086
+skip jsonp bit
+summaryse is a cool extension from which to steal stuff. Maybe provide basic functionalities through that
+- given [/m/09_pbpl      /type/object/key        /soft/isbn/9780491035453,..] and [/m/077601h      /book/isbn/book_editions        /m/09_pbpl] get [/m/09_pbpl /soft/isbn/9780491035453]
+- use merge sorted and merge unsorted probably

data/bin/parsely

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require 'parsely'
+Parsely.new.main

data/lib/parsely.rb

@@ -0,0 +1,264 @@
+require 'set'
+require 'English'
+$OUTPUT_FIELD_SEPARATOR = ' '
+
+  def p args
+    STDERR.puts(args.inspect) #if $DEBUG
+  end
+
+class PseudoBinding
+  class PerlVar < String
+    def <=> other
+      if other.is_a? Numeric
+        to_f <=> other
+      else
+        super
+      end
+    end
+    def inspect
+      "PerlVar(#{super})"
+    end
+  end
+  PerlNil = PerlVar.new ''
+  attr :line
+  def initialize lineno, vals
+    @line, @vals = lineno, vals.map {|x| PerlVar.new(x)}
+  end
+  def method_missing name, *args
+    if args.empty?
+      if name =~ /_(\d+)/
+        @vals[$1.to_i] || PerlNil
+      else
+        name.to_s
+      end
+    else
+      super
+    end
+  end
+end
+class String
+  def value
+    to_s
+  end
+end
+class Proc
+  attr_accessor :single
+  def value
+    call
+  end
+end
+class Parsely
+  VERSION = "0.1.1"
+  def self.cmd(&block)
+    klass = Struct.new :value, &block
+    klass.class_eval do 
+      def process(items)
+        value.assign(items)
+        _process(value)
+      end
+    end
+    klass
+  end
+  RGX= /"(.*?)"|\[(.*?)\]|([^\s]+)/
+  Value = Struct.new :index, :value do
+    def assign(items)
+      self.value = items[index]
+    end
+    def process(items)
+      items[index]
+    end
+    def to_i
+      value.to_i
+    end
+    def to_f
+      value.to_f
+    end
+    def to_s
+      value.to_s
+    end
+  end
+  Ops = {
+    :min => cmd do
+      def initialize value
+        super
+        @running_value = Float::MAX #-Inf would be better
+        @result = proc { @running_value }
+        @result.single = true
+      end
+      def _process(value)
+        if value.to_f < @running_value  
+          @running_value = value.to_f
+        end
+        @result
+      end
+    end,
+      :max => cmd do
+      def initialize index
+        super
+        @running_value = Float::MIN #-Inf would be better
+        @result = proc { @running_value }
+        @result.single = true
+      end
+      def _process(value)
+        if value.to_f > @running_value
+          @running_value = value.to_f
+        end
+        @result
+      end
+      end,
+        :sum => cmd do
+        def initialize index
+          super
+          @running_value = 0
+          @result = proc { @running_value }
+          @result.single = true
+        end
+        def _process(value)
+          @running_value += value.to_i
+          @result
+        end
+    end,
+    :avg => cmd do
+      def initialize index
+        super
+        @running_value = 0
+        @running_count = 0
+        @result = proc { @running_value/@running_count.to_f }
+        @result.single = true
+      end
+      def _process(value)
+        @running_value += value.to_i
+        @running_count += 1
+        @result
+      end
+    end,
+    :freq => cmd do
+      def initialize index
+        super
+      	@running_freqs = Hash.new(0)
+        @running_count = 0
+	as_ary=nil
+        @result = proc do
+          if as_ary.nil?
+            as_ary=@running_freqs.sort_by do |k,v| [-v,k] end.each
+          end
+          k,v = as_ary.next 
+          [v, k]
+        end
+      end
+      def _process(value)
+        @running_freqs[value.to_s]+=1
+        @running_count += 1
+        @result
+      end
+    end,
+    :stats => cmd do
+      def initialize index
+        require 'rubygems'
+        require 'ministat'
+        super
+        @running_values = []
+        cached = nil
+        @result = proc do  
+          unless cached
+            data = MiniStat::Data.new(@running_values)
+            cached = { 
+              "Mean"=>data.mean,
+              "Geometric Mean"=>data.geometric_mean,
+              "Harmonic Mean"=>data.harmonic_mean,
+              "Median"=>data.median ,
+              "Min"=>data.data.min ,
+              "Q1"=>data.q1,
+              "Q3"=>data.q3,
+              "Max"=>data.data.max,
+              "IQR"=>data.iqr,
+              "Outliers"=>data.outliers.inspect,
+              "Variance"=>data.variance ,
+              "Std Dev"=>data.std_dev,
+            }.sort.each
+          end
+          cached.next
+        end
+      end
+      def _process(value)
+        @running_values << value.to_i
+        @result
+      end
+    end,
+  }
+
+  def parse(expr)
+    val, cond = expr.split(/ if /)
+                              # p [ val, cond]
+    elems=val.split
+    r=elems.map do |e|
+      case e
+      when /(\w+)\(\_(\d+)\)/
+        klass=Ops[$1.to_sym]
+        if klass.nil?
+          abort "unknown op '#$1'"
+        end
+        klass.new(Value.new($2.to_i))
+      when /\_(\d+)/
+        Value.new($1.to_i)
+      end
+    end
+    [r, parse_cond(cond)]
+  end
+
+  def parse_cond str
+    case str
+    when %r[/.*/]
+      proc { |bnd| bnd.instance_eval(str) === bnd._0 }
+    when nil, ''
+      proc { |bnd| true }
+    else
+    proc { |bnd| bnd.instance_eval(str) }
+    end
+  end
+
+  def main
+    if ARGV.empty?
+      abort("usage #$0 <expr> <file file file| stdin >")
+    end
+    expr = ARGV.shift
+    main_loop(expr,ARGF)
+  end
+
+  def main_loop(expr,lines)
+    ast, cond =parse(expr)
+    result = []
+    result = lines.map.with_index do |line, lineno|
+      line.chomp!
+      items = [line]+line.scan(RGX).map do |a| 
+        # XXX horrible
+        a.find do |e| 
+          !e.nil? 
+        end 
+      end
+      #XXX ugly
+      next unless items
+      b = PseudoBinding.new(lineno, items)
+      ast.map do |a| 
+        a.process(items) if cond[b]
+      end 
+    end
+    last = []
+    result.each do |cols|
+      result_line = cols.map do |col|
+        next if col.nil?
+        col.value 
+      end.join.strip 
+      same_results = cols.zip(last).map do |a,b| 
+        a.respond_to?(:single) && a.single && a.object_id == b.object_id && !a.is_a?(Numeric) 
+      end.all?
+      break if same_results
+      next if result_line.empty?
+      puts result_line
+      last = cols
+    end
+  rescue StopIteration
+    # end
+  end
+
+end

data/parsely.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "parsely"
+
+Gem::Specification.new do |s|
+  s.name        = "parsely"
+  s.version     = Parsely::VERSION
+  s.authors     = ["Gabriele Renzi"]
+  s.email       = ["rff.rff+parsely@gmail.com"]
+  s.homepage    = "http://github.com/riffraff/parsely"
+  s.summary     = %q{a simple tool for text file wrangling}
+  s.description = %q{parsely is a simple tool for managing text files.
+                     Mostly to replace a lot of awk/sed/ruby/perl one-off scripts.
+                     This is an internal release, guaranteed to break and ruin your life.}
+
+  #s.rubyforge_project = "parsely"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  s.add_development_dependency "rake"
+  
+end

data/test/basic/001/command

@@ -0,0 +1 @@
+parsely '_1' input.log

data/test/basic/001/input.log

@@ -0,0 +1,5 @@
+1 miao
+2 bau
+3
+4 er er er
+5

data/test/basic/001/output

@@ -0,0 +1,5 @@
+1
+2
+3
+4
+5

data/test/basic/002/command

@@ -0,0 +1 @@
+parsely '_2, _3' input.log

data/test/basic/002/input.log

@@ -0,0 +1,3 @@
+ciao 1 2 bau
+miao 2 3 lol
+roar 9 10 argh

data/test/basic/002/output

@@ -0,0 +1,3 @@
+1 2
+2 3
+9 10

data/test/basic/003/command

@@ -0,0 +1 @@
+parsely 'sum(_1)' input.log

data/test/basic/003/input.log

@@ -0,0 +1,5 @@
+1 miao
+2 bau
+3
+4 er er er
+5

data/test/basic/003/output

@@ -0,0 +1 @@
+15

data/test/basic/004/command

@@ -0,0 +1 @@
+parsely 'avg(_1)' input.log

data/test/basic/004/input.log

@@ -0,0 +1,5 @@
+1 miao
+2 bau
+3
+4 er er er
+5

data/test/basic/004/output

@@ -0,0 +1 @@
+3.0

data/test/basic/005/command

@@ -0,0 +1 @@
+parsely '_1 _2' input.log

data/test/basic/005/input.log

@@ -0,0 +1,5 @@
+1 miao
+2 bau
+3
+4 er er er
+5

data/test/basic/005/output

@@ -0,0 +1,5 @@
+1 miao
+2 bau
+3
+4 er
+5

data/test/basic/006/command

@@ -0,0 +1 @@
+parsely 'avg(_1) sum(_1) _2' input.log

data/test/basic/006/input.log

@@ -0,0 +1,5 @@
+1 miao
+2 bau
+3
+4 er er er
+5

data/test/basic/006/output

@@ -0,0 +1,5 @@
+3.0 15 miao
+3.0 15 bau
+3.0 15
+3.0 15 er
+3.0 15

data/test/basic/007/command

@@ -0,0 +1 @@
+parsely 'freq(_1)' input.log

data/test/basic/007/input.log

@@ -0,0 +1,8 @@
+1 miao
+1 again
+1 more
+2 bau
+2 again
+3
+4 er er er
+5

data/test/basic/007/output

@@ -0,0 +1,5 @@
+3 1
+2 2
+1 3
+1 4
+1 5

data/test/basic/008/command

@@ -0,0 +1 @@
+parsely 'min(_1) max(_2)' input.log

data/test/basic/008/input.log

@@ -0,0 +1,3 @@
+1.5 10
+2.9 20
+3 30

data/test/basic/008/output

@@ -0,0 +1 @@
+1.5 30.0

data/test/basic/009-if-global/command

@@ -0,0 +1 @@
+parsely '_0 if /www/' input.log

data/test/basic/009-if-global/input.log

@@ -0,0 +1,4 @@
+ignore
+www bla bla
+ignore again
+something www

data/test/basic/009-if-global/output

@@ -0,0 +1,2 @@
+www bla bla
+something www

data/test/basic/010-if-var/command

@@ -0,0 +1 @@
+parsely '_0 if _1 == www' input.log

data/test/basic/010-if-var/input.log

@@ -0,0 +1,4 @@
+ignore
+www bla bla
+ignore again
+something www

data/test/basic/010-if-var/output

@@ -0,0 +1 @@
+www bla bla

data/test/basic/011-sum-if/command

@@ -0,0 +1 @@
+parsely 'sum(_2) if _2 < 3' counts.txt

data/test/basic/011-sum-if/counts.txt

@@ -0,0 +1,7 @@
+keep 10
+kep 5
+foo 1
+bar 1
+baz 2
+quux 3
+

data/test/basic/011-sum-if/output

@@ -0,0 +1 @@
+4

data/test/basic/012-sum-if-2/command

@@ -0,0 +1 @@
+parsely 'sum(_2) if _2 < 3 && _1 != skip' counts.txt

data/test/basic/012-sum-if-2/counts.txt

@@ -0,0 +1,7 @@
+keep 10
+kep 5
+foo 1
+bar 1
+baz 2
+quux 3
+

data/test/basic/012-sum-if-2/output

@@ -0,0 +1 @@
+4

data/test/basic/013-if-non-math/command

@@ -0,0 +1 @@
+parsely 'sum(_2) if _1[0] == ?k' counts.txt

data/test/basic/013-if-non-math/counts.txt

@@ -0,0 +1,7 @@
+keep 10
+kep 5
+foo 1
+bar 1
+baz 2
+quux 3
+

data/test/basic/013-if-non-math/output

@@ -0,0 +1 @@
+15

data/test/basic/014-line-index/command

@@ -0,0 +1 @@
+parsely 'sum(_2) if line == 0' counts.txt

data/test/basic/014-line-index/counts.txt

@@ -0,0 +1,7 @@
+keep 10
+no 5
+foo 1
+bar 1
+baz 2
+quux 3
+

data/test/basic/014-line-index/output

@@ -0,0 +1 @@
+10

data/test/cli-runner.rb

@@ -0,0 +1,20 @@
+require 'test/unit'
+
+p ENV['PATH']
+def make_runner(dir)
+  klass= Class.new Test::Unit::TestCase do
+    test_dir = File.expand_path("test/#{dir}/0*")
+    Dir[test_dir].each do |d|
+      define_method 'test_'+dir+'_'+d.split("/").last do
+        Dir.chdir(d)
+        command = File.read('command')
+        output= `#{command} 2>&1`
+        expected = File.read('output')
+        assert_equal expected.chomp.strip, output.chomp.strip
+      end
+    end
+  end
+  Object.const_set 'Test_'+dir, klass
+end
+make_runner 'basic'
+

metadata

@@ -0,0 +1,152 @@
+--- !ruby/object:Gem::Specification
+name: parsely
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+  prerelease: 
+platform: ruby
+authors:
+- Gabriele Renzi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-07-14 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &2153010780 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *2153010780
+description: ! "parsely is a simple tool for managing text files.\n                     Mostly
+  to replace a lot of awk/sed/ruby/perl one-off scripts.\n                     This
+  is an internal release, guaranteed to break and ruin your life."
+email:
+- rff.rff+parsely@gmail.com
+executables:
+- parsely
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Rakefile
+- TODO
+- bin/parsely
+- lib/parsely.rb
+- parsely.gemspec
+- test/basic/001/command
+- test/basic/001/input.log
+- test/basic/001/output
+- test/basic/002/command
+- test/basic/002/input.log
+- test/basic/002/output
+- test/basic/003/command
+- test/basic/003/input.log
+- test/basic/003/output
+- test/basic/004/command
+- test/basic/004/input.log
+- test/basic/004/output
+- test/basic/005/command
+- test/basic/005/input.log
+- test/basic/005/output
+- test/basic/006/command
+- test/basic/006/input.log
+- test/basic/006/output
+- test/basic/007/command
+- test/basic/007/input.log
+- test/basic/007/output
+- test/basic/008/command
+- test/basic/008/input.log
+- test/basic/008/output
+- test/basic/009-if-global/command
+- test/basic/009-if-global/input.log
+- test/basic/009-if-global/output
+- test/basic/010-if-var/command
+- test/basic/010-if-var/input.log
+- test/basic/010-if-var/output
+- test/basic/011-sum-if/command
+- test/basic/011-sum-if/counts.txt
+- test/basic/011-sum-if/output
+- test/basic/012-sum-if-2/command
+- test/basic/012-sum-if-2/counts.txt
+- test/basic/012-sum-if-2/output
+- test/basic/013-if-non-math/command
+- test/basic/013-if-non-math/counts.txt
+- test/basic/013-if-non-math/output
+- test/basic/014-line-index/command
+- test/basic/014-line-index/counts.txt
+- test/basic/014-line-index/output
+- test/cli-runner.rb
+homepage: http://github.com/riffraff/parsely
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.5
+signing_key: 
+specification_version: 3
+summary: a simple tool for text file wrangling
+test_files:
+- test/basic/001/command
+- test/basic/001/input.log
+- test/basic/001/output
+- test/basic/002/command
+- test/basic/002/input.log
+- test/basic/002/output
+- test/basic/003/command
+- test/basic/003/input.log
+- test/basic/003/output
+- test/basic/004/command
+- test/basic/004/input.log
+- test/basic/004/output
+- test/basic/005/command
+- test/basic/005/input.log
+- test/basic/005/output
+- test/basic/006/command
+- test/basic/006/input.log
+- test/basic/006/output
+- test/basic/007/command
+- test/basic/007/input.log
+- test/basic/007/output
+- test/basic/008/command
+- test/basic/008/input.log
+- test/basic/008/output
+- test/basic/009-if-global/command
+- test/basic/009-if-global/input.log
+- test/basic/009-if-global/output
+- test/basic/010-if-var/command
+- test/basic/010-if-var/input.log
+- test/basic/010-if-var/output
+- test/basic/011-sum-if/command
+- test/basic/011-sum-if/counts.txt
+- test/basic/011-sum-if/output
+- test/basic/012-sum-if-2/command
+- test/basic/012-sum-if-2/counts.txt
+- test/basic/012-sum-if-2/output
+- test/basic/013-if-non-math/command
+- test/basic/013-if-non-math/counts.txt
+- test/basic/013-if-non-math/output
+- test/basic/014-line-index/command
+- test/basic/014-line-index/counts.txt
+- test/basic/014-line-index/output
+- test/cli-runner.rb