tb 0.3 → 0.4

data/lib/tb/cmd_join.rb

@@ -71,8 +71,6 @@ def (Tb::Cmd).main_join(argv)
     $stderr.puts "shared keys: #{(result.list_fields & tbl.list_fields).inspect}" if 1 <= Tb::Cmd.opt_debug
     result = result.natjoin2_outer(tbl, Tb::Cmd.opt_join_outer_missing, retain_left, retain_right)
   }
-  with_output {|out|
-    result.write_to_csv(out, !Tb::Cmd.opt_N)
-  }
+  output_tbenum(result)
 end
 

data/lib/tb/cmd_ls.rb

@@ -63,9 +63,7 @@ def (Tb::Cmd).main_ls(argv)
       ls.ls_run(Pathname(ls.real_pathname_string(arg)))
     }
   }
-  with_output {|out|
-    er.write_to_csv(out, !Tb::Cmd.opt_N)
-  }
+  output_tbenum(er)
   if ls.fail
     exit false
   end
@@ -106,7 +104,7 @@ class Tb::Cmd::Ls
       return
     end
     entries.map! {|filename| real_pathname_string(filename) }
-    entries = entries.sort_by {|filename| smart_cmp_value(filename) }
+    entries = entries.sort_by {|filename| Tb::Func.smart_cmp_value(filename) }
     if @opts[:a] || @opts[:A]
       entries1, entries2 = entries.partition {|filename| /\A\./ =~ filename }
       entries0, entries1 = entries1.partition {|filename| filename == '.' || filename == '..' }
@@ -155,7 +153,7 @@ class Tb::Cmd::Ls
       end
       @y.yield ls_long_info(path, st)
     else
-      @y.yield Tb::Pairs.new([['filename', path.to_s]])
+      @y.yield({'filename' => path.to_s})
     end
   end
 
@@ -179,9 +177,9 @@ class Tb::Cmd::Ls
   end
 
   def ls_long_info(path, st)
-    Tb::Pairs.new(ls_long_header.map {|info_type|
+    Hash[ls_long_header.map {|info_type|
       [info_type, self.send("ls_info_#{info_type}", path, st)]
-    })
+    }]
   end
 
   def ls_info_dev(path, st) sprintf("0x%x", st.dev) end
@@ -234,7 +232,7 @@ class Tb::Cmd::Ls
     if pw
       pw.name
     else
-      uid
+      uid.to_s
     end
   end
 
@@ -247,7 +245,7 @@ class Tb::Cmd::Ls
     if gr
       gr.name
     else
-      gid
+      gid.to_s
     end
   end
 
@@ -291,11 +289,6 @@ class Tb::Cmd::Ls
   end
 
   def real_pathname_string(str)
-    if str.respond_to? :force_encoding
-      # pathname is a sequence of bytes on Unix.
-      str.dup.force_encoding("ASCII-8BIT")
-    else
-      str
-    end
+    str.dup.force_encoding("ASCII-8BIT")
   end
 end

data/lib/tb/cmd_mheader.rb

@@ -68,7 +68,7 @@ def (Tb::Cmd).main_mheader(argv)
         }
         h2 = header_end_p.call
         if h2
-          pairs2 = Tb::Pairs.new(h2.map.with_index {|v, i| ["#{i+1}", v] })
+          pairs2 = Hash[h2.map.with_index {|v, i| ["#{i+1}", v] }]
           y.yield pairs2
           header = nil
         end
@@ -77,9 +77,8 @@ def (Tb::Cmd).main_mheader(argv)
       end
     }
   }
-  with_output {|out|
-    er.write_to_csv(out, false)
-  }
+  Tb::Cmd.opt_N = true
+  output_tbenum(er)
   if header
     warn "unique header fields not recognized."
   end

data/lib/tb/cmd_nest.rb

@@ -58,15 +58,12 @@ def (Tb::Cmd).main_nest(argv)
       cv = pairs.reject {|f, v|
         oldfields_hash[f]
       }.map {|f, v|
-        [smart_cmp_value(f), smart_cmp_value(v)]
+        [Tb::Func.smart_cmp_value(f), Tb::Func.smart_cmp_value(v)]
       }.sort
       [cv, pairs]
     }
 
     nested = nil
-    boundary_p = lambda {|(cv1, _), (cv2, _)|
-      cv1 != cv2
-    }
     before_group = lambda {|(_, _)|
       nested = []
     }
@@ -82,13 +79,11 @@ def (Tb::Cmd).main_nest(argv)
       }
       assoc = last_pairs.reject {|f, v| oldfields_hash[f] }.to_a
       assoc << [newfield, nested_csv]
-      pairs = Tb::Pairs.new(assoc)
+      pairs = Hash[assoc]
       y.yield pairs
     }
-    sorted.each_group_element(boundary_p, before_group, body, after_group)
-  }
-  with_output {|out|
-    er.write_to_csv(out, !Tb::Cmd.opt_N)
+    sorted.detect_group_by(before_group, after_group) {|cv,| cv }.each(&body)
   }
+  output_tbenum(er)
 end
 

data/lib/tb/cmd_newfield.rb

@@ -47,9 +47,7 @@ def (Tb::Cmd).main_newfield(argv)
   argv = ['-'] if argv.empty?
   creader = Tb::CatReader.open(argv, Tb::Cmd.opt_N)
   er = creader.newfield(field) {|pairs| pr.call(pairs) }
-  with_output {|out|
-    er.write_to_csv(out, !Tb::Cmd.opt_N)
-  }
+  output_tbenum(er)
 end
 
 

data/lib/tb/cmd_rename.rb

@@ -56,11 +56,9 @@ def (Tb::Cmd).main_rename(argv)
       }
       y.set_header header.map {|f| h.fetch(f, f) }
     }.each {|pairs|
-      y.yield Tb::Pairs.new(pairs.map {|f, v| [h.fetch(f, f), v] })
+      y.yield Hash[pairs.map {|f, v| [h.fetch(f, f), v] }]
     }
   }
-  with_output {|out|
-    er.write_to_csv(out, !Tb::Cmd.opt_N)
-  }
+  output_tbenum(er)
 end
 

data/lib/tb/cmd_shape.rb

@@ -67,8 +67,7 @@ def (Tb::Cmd).main_shape(argv)
                      'filename'=>filename})
     }
   }
-  with_output {|out|
-    result.write_to_csv(out, true)
-  }
+  Tb::Cmd.opt_N = false
+  output_tbenum(result)
 end
 

data/lib/tb/cmd_sort.rb

@@ -53,9 +53,9 @@ def (Tb::Cmd).main_sort(argv)
   creader = Tb::CatReader.open(argv, Tb::Cmd.opt_N)
   header = []
   if fs
-    blk = lambda {|pairs| fs.map {|f| smart_cmp_value(pairs[f]) } }
+    blk = lambda {|pairs| fs.map {|f| Tb::Func.smart_cmp_value(pairs[f]) } }
   else
-    blk = lambda {|pairs| header.map {|f| smart_cmp_value(pairs[f]) } }
+    blk = lambda {|pairs| header.map {|f| Tb::Func.smart_cmp_value(pairs[f]) } }
   end
   if Tb::Cmd.opt_sort_r
     blk1 = blk
@@ -71,9 +71,7 @@ def (Tb::Cmd).main_sort(argv)
       y.yield pairs
     }
   }.extsort_by(&blk)
-  with_output {|out|
-    er.write_to_csv(out, !Tb::Cmd.opt_N)
-  }
+  output_tbenum(er)
 end
 
 

data/lib/tb/cmd_svn_log.rb

@@ -101,11 +101,11 @@ class Tb::Cmd::SVNLOGListener
         @log['paths'].each {|h|
           assoc = @log.to_a.reject {|f, v| !%w[rev author date msg].include?(f) }
           assoc += h.to_a.reject {|f, v| !%w[kind action path].include?(f) }
-          @y.yield Tb::Pairs.new(assoc)
+          @y.yield Hash[assoc]
         }
       else
         assoc = @log.to_a.reject {|f, v| !%w[rev author date msg].include?(f) }
-        @y.yield Tb::Pairs.new(assoc)
+        @y.yield Hash[assoc]
       end
       @log = nil
     end
@@ -170,8 +170,6 @@ def (Tb::Cmd).main_svn_log(argv)
       REXML::Parsers::StreamParser.new(f, listener).parse
     }
   }
-  with_output {|out|
-    er.write_to_csv(out, !Tb::Cmd.opt_N)
-  }
+  output_tbenum(er)
 end
 

data/lib/tb/cmd_tar_tvf.rb

@@ -436,12 +436,10 @@ def (Tb::Cmd).main_tar_tvf(argv)
           formatted["tar_typeflag"] = h[:typeflag]
           formatted["tar_magic"] = h[:magic]
           formatted["tar_version"] = h[:version]
-          y.yield Tb::Pairs.new(header.map {|f2| [f2, formatted[f2]] })
+          y.yield Hash[header.map {|f2| [f2, formatted[f2]] }]
         }
       }
     }
   }
-  with_output {|out|
-    er.write_to_csv(out, !Tb::Cmd.opt_N)
-  }
+  output_tbenum(er)
 end

data/lib/tb/cmd_to_csv.rb

@@ -51,7 +51,7 @@ def (Tb::Cmd).main_to_csv(argv)
   er = Tb::Enumerator.new {|y|
     y.set_header header
     ter.each {|pairs|
-      y.yield Tb::Pairs.new(header.map {|f| [f, pairs[f]] })
+      y.yield Hash[header.map {|f| [f, pairs[f]] }]
     }
   }
   with_output {|out|

data/lib/tb/cmd_unnest.rb

@@ -111,8 +111,6 @@ def (Tb::Cmd).main_unnest(argv)
       end
     }
   }
-  with_output {|out|
-    er.write_to_csv(out, !Tb::Cmd.opt_N)
-  }
+  output_tbenum(er)
 end
 

data/lib/tb/cmdutil.rb

@@ -96,150 +96,36 @@ def err(msg)
   raise SystemExit.new(1, msg)
 end
 
-def smart_cmp_value(v)
-  case v
-  when nil
-    []
-  when Numeric
-    [0, v]
-  when String
-    if v.respond_to? :force_encoding
-      v = v.dup.force_encoding("ASCII-8BIT")
-    end
-    case v
-    when /\A\s*-?\d+\s*\z/
-      [0, v.to_i(10)]
-    when /\A\s*-?(\d+(\.\d*)?|\.\d+)([eE][-+]?\d+)?\s*\z/
-      [0, Float(v)]
-    else
-      a = []
-      v.scan(/(\d+)|\D+/) {
-        if $1
-          a << 0 << $1.to_i
-        else
-          a << 1 << $&
-        end
-      }
-      a
-    end
-  else
-    raise ArgumentError, "unexpected: #{v.inspect}"
-  end
-end
-
-def conv_to_numeric(v)
-  v = v.strip
-  if /\A-?\d+\z/ =~ v
-    v = v.to_i
-  elsif /\A-?(\d+(\.\d*)?|\.\d+)([eE][-+]?\d+)?\z/ =~ v
-    v = v.to_f
-  else
-    raise ArgumentError, "number string expected: #{v.inspect}"
-  end
-  v
-end
-
-class CountAggregator
-  def initialize() @result = 0 end
-  def update(v) @result += 1 end
-  def finish() @result end
-end
-
-class SumAggregator
-  def initialize() @result = 0 end 
-  def update(v) @result += conv_to_numeric(v) if !(v.nil? || v == '') end
-  def finish() @result end
-end
-
-class AvgAggregator
-  def initialize() @sum = 0; @count = 0 end 
-  def update(v) @count += 1; @sum += conv_to_numeric(v) if !(v.nil? || v == '') end
-  def finish() @sum / @count.to_f end
-end
-
-class MaxAggregator
-  def initialize() @v = nil; @cmp = nil end 
-  def update(v)
-    cmp = smart_cmp_value(v)
-    if @cmp == nil
-      @v, @cmp = v, cmp
-    else
-      @v, @cmp = v, cmp if (@cmp <=> cmp) < 0
-    end
-  end
-  def finish() @v end
-end
-
-class MinAggregator
-  def initialize() @v = @cmp = nil end 
-  def update(v)
-    cmp = smart_cmp_value(v)
-    if @cmp == nil
-      @v, @cmp = v, cmp
-    else
-      @v, @cmp = v, cmp if (@cmp <=> cmp) > 0
-    end
-  end
-  def finish() @v end
-end
-
-class ValuesAggregator
-  def initialize() @result = [] end 
-  def update(v) @result << v if v end
-  def finish() @result.join(",") end
-end
-
-class UniqueValuesAggregator
-  def initialize() @result = [] end 
-  def update(v) @result << v if v end
-  def finish() @result.uniq.join(",") end
-end
-
-class Selector
-  def initialize(i, aggregator) @i = i; @agg = aggregator end
-  def update(ary) @agg.update(ary[@i]) end
-  def finish() @agg.finish end
-end
-
-def make_aggregator(spec, fs)
+def parse_aggregator_spec(spec)
   case spec
   when 'count'
-    CountAggregator.new
+    ['count', nil]
   when /\Asum\((.*)\)\z/
-    field = $1
-    i = fs.index(field)
-    raise ArgumentError, "field not found: #{field.inspect}" if !i
-    Selector.new(i, SumAggregator.new)
+    ['sum', $1]
   when /\Aavg\((.*)\)\z/
-    field = $1
-    i = fs.index(field)
-    raise ArgumentError, "field not found: #{field.inspect}" if !i
-    Selector.new(i, AvgAggregator.new)
+    ['avg', $1]
   when /\Amax\((.*)\)\z/
-    field = $1
-    i = fs.index(field)
-    raise ArgumentError, "field not found: #{field.inspect}" if !i
-    Selector.new(i, MaxAggregator.new)
+    ['max', $1]
   when /\Amin\((.*)\)\z/
-    field = $1
-    i = fs.index(field)
-    raise ArgumentError, "field not found: #{field.inspect}" if !i
-    Selector.new(i, MinAggregator.new)
+    ['min', $1]
   when /\Avalues\((.*)\)\z/
-    field = $1
-    i = fs.index(field)
-    raise ArgumentError, "field not found: #{field.inspect}" if !i
-    Selector.new(i, ValuesAggregator.new)
+    ['values', $1]
   when /\Auniquevalues\((.*)\)\z/
-    field = $1
-    i = fs.index(field)
-    raise ArgumentError, "field not found: #{field.inspect}" if !i
-    Selector.new(i, UniqueValuesAggregator.new)
+    ['uniquevalues', $1]
   else
     raise ArgumentError, "unexpected aggregation spec: #{spec.inspect}"
   end
 end
 
+def parse_aggregator_spec2(spec)
+  name, field = parse_aggregator_spec(spec)
+  func = Tb::Func::AggregationFunctions[name]
+  if !func
+    raise ArgumentError, "unexpected aggregation spec: #{spec.inspect}"
+  end
+  [func, field]
+end
+
 def split_field_list_argument(arg)
   split_csv_argument(arg).map {|f| f || '' }
 end
@@ -266,14 +152,18 @@ def tbl_generate_tsv(tbl, out)
   end
 end
 
-def with_output
-  if Tb::Cmd.opt_output
-    tmp = Tb::Cmd.opt_output + ".part"
+def with_output(filename=Tb::Cmd.opt_output)
+  if filename && filename != '-'
+    tmp = filename + ".part"
     begin
       File.open(tmp, 'w') {|f|
         yield f
       }
-      File.rename tmp, Tb::Cmd.opt_output
+      if File.exist?(filename) && FileUtils.compare_file(filename, tmp)
+        File.unlink tmp
+      else
+        File.rename tmp, filename
+      end
     ensure
       File.unlink tmp if File.exist? tmp
     end
@@ -285,3 +175,35 @@ def with_output
     yield $stdout
   end
 end
+
+def output_tbenum(te)
+  filename = Tb::Cmd.opt_output
+  if /\A([a-z0-9]{2,}):/ =~ filename
+    fmt = $1
+    filename = $'
+  else
+    fmt = nil
+  end
+  if !fmt
+    case filename
+    when /\.csv\z/
+      fmt = 'csv'
+    when /\.json\z/
+      fmt = 'json'
+    end
+  end
+  if fmt
+    case fmt
+    when 'csv'
+      write_proc = lambda {|out| te.write_to_csv(out, !Tb::Cmd.opt_N) }
+    when 'json'
+      write_proc = lambda {|out| te.write_to_json(out) }
+    else
+      err("unexpected format: #{fmt.inspect}")
+    end
+  end
+  write_proc ||= lambda {|out| te.write_to_csv(out, !Tb::Cmd.opt_N) }
+  with_output(filename) {|out|
+    write_proc.call(out)
+  }
+end