tb 0.3 → 0.4

data/README

@@ -1,8 +1,9 @@
-= tb - manipulation tool for table: CSV, TSV, etc.
+= tb - manipulation tool for table: CSV, TSV, JSON, etc.
 
 tb provides a command and a library for manipulating tables:
 Unix filter like operations (grep, sort, cat, cut, ls, etc.),
 SQL like operations (join, group, etc.),
+information extractions (git-log, svn-log, tar-tvf),
 and more.
 
 == Example

data/lib/tb.rb

@@ -34,9 +34,10 @@ class Tb
 end
 
 require 'pp'
-require 'tb/enum'
+require 'tb/enumerable'
 require 'tb/enumerator'
-require 'tb/pairs'
+require 'tb/func'
+require 'tb/zipper'
 require 'tb/basic'
 require 'tb/record'
 require 'tb/csv'
@@ -48,6 +49,9 @@ require 'tb/ropen'
 require 'tb/catreader'
 require 'tb/fieldset'
 require 'tb/search'
-require 'tb/enumerable'
+require 'tb/ex_enumerable'
+require 'tb/ex_enumerator'
 require 'tb/fileenumerator'
 require 'tb/revcmp'
+require 'tb/customcmp'
+require 'tb/customeq'

data/lib/tb/basic.rb

@@ -55,7 +55,7 @@
 # It is an error to access a record by recordid which is not allocated.
 #
 class Tb
-  include Tb::Enum
+  include Tb::Enumerable
 
   # :call-seq:
   #   Tb.new

data/lib/tb/cmd_cat.rb

@@ -63,7 +63,5 @@ def (Tb::Cmd).main_cat(argv)
   exit_if_help('cat')
   argv = ['-'] if argv.empty?
   creader = Tb::CatReader.open(argv, Tb::Cmd.opt_N, Tb::Cmd.opt_cat_with_filename)
-  with_output {|out|
-    creader.write_to_csv(out, !Tb::Cmd.opt_N)
-  }
+  output_tbenum(creader)
 end

data/lib/tb/cmd_consecutive.rb

@@ -68,22 +68,20 @@ def (Tb::Cmd).main_consecutive(argv)
     }.each {|pairs, header|
       buf << pairs
       if buf.length == Tb::Cmd.opt_consecutive_n
-        pairs2 = []
+        pairs2 = {}
         header.each {|f|
           Tb::Cmd.opt_consecutive_n.times {|i|
             ps = buf[i]
             next if !ps.has_key?(f)
             v = ps[f]
-            pairs2 << ["#{f}_#{i+1}", v]
+            pairs2["#{f}_#{i+1}"] = v
           }
         }
         empty = false
-        y.yield Tb::Pairs.new(pairs2)
+        y.yield pairs2
         buf.shift
       end
     }
   }
-  with_output {|out|
-    er.write_to_csv(out, !Tb::Cmd.opt_N)
-  }
+  output_tbenum(er)
 end

data/lib/tb/cmd_crop.rb

@@ -84,9 +84,8 @@ def (Tb::Cmd).main_crop(argv)
         rownum += 1
       }
     }
-    with_output {|out|
-      er.write_to_csv(out, false)
-    }
+    Tb::Cmd.opt_N = true
+    output_tbenum(er)
   else
     creader = Tb::CatReader.open(argv, true)
     last_nonempty_row = nil
@@ -127,13 +126,12 @@ def (Tb::Cmd).main_crop(argv)
           break
         end
         ary.slice!(0, lmargin_min)
-        pairs = Tb::Pairs.new(ary.map.with_index {|v, i| ["#{i+1}", v]})
+        pairs = Hash[ary.map.with_index {|v, i| ["#{i+1}", v]}]
         y.yield pairs
       }
     }
-    with_output {|out|
-      er.write_to_csv(out, false)
-    }
+    Tb::Cmd.opt_N = true
+    output_tbenum(er)
   end
 end
 

data/lib/tb/cmd_cross.rb

@@ -49,36 +49,51 @@ def (Tb::Cmd).main_cross(argv)
   vkfs = split_field_list_argument(argv.shift)
   err('no hkey-fields given.') if argv.empty?
   hkfs = split_field_list_argument(argv.shift)
+  vhkfs = vkfs + hkfs
   if Tb::Cmd.opt_cross_fields.empty?
-    opt_cross_fields = [['count', 'count']]
+    num_aggregate_fields = 1
+    opt_cross_fields = (vkfs + hkfs).map {|f| [f, Tb::Func::First, f] } +
+      [['count', Tb::Func::Count, nil]]
   else
-    opt_cross_fields = Tb::Cmd.opt_cross_fields.map {|arg|
+    num_aggregate_fields = Tb::Cmd.opt_cross_fields.length
+    opt_cross_fields = (vkfs + hkfs).map {|f| [f, Tb::Func::First, f] } +
+      Tb::Cmd.opt_cross_fields.map {|arg|
       agg_spec, new_field = split_field_list_argument(arg)
       new_field ||= agg_spec
-      [agg_spec, new_field]
+      begin
+        func_srcf = parse_aggregator_spec2(agg_spec)
+      rescue ArgumentError
+        err($!.message)
+      end
+      [new_field, *func_srcf]
     }
   end
   argv = ['-'] if argv.empty?
   creader = Tb::CatReader.open(argv, Tb::Cmd.opt_N)
   er = Tb::Enumerator.new {|y|
-    header = nil
     hvs_hash = {}
     hvs_list = nil
-    sorted = creader.extsort_by {|pairs|
-      hvs = hkfs.map {|f| pairs[f] }
-      hvs_hash[hvs] = true
-      vcv = vkfs.map {|f| smart_cmp_value(pairs[f]) }
-      vcv
-    }
-    sorted2 = sorted.with_header {|header0|
-      header = header0
-      (vkfs + hkfs).each {|f|
+    aggs_hash = nil
+    op = Tb::Zipper.new(opt_cross_fields.map {|dstf, func, srcf| func })
+    er = creader.with_header {|header0|
+      vhkfs.each {|f|
         if !header0.include?(f)
           err("field not found: #{f}")
         end
       }
-      hvs_list = hvs_hash.keys.sort_by {|hvs| hvs.map {|hv| smart_cmp_value(hv) } }
-      n = vkfs.length + hvs_list.length * opt_cross_fields.length
+    }.extsort_reduce(op) {|pairs|
+      vvs = vkfs.map {|f| pairs[f] }
+      hvs = hkfs.map {|f| pairs[f] }
+      vvsc = vvs.map {|v| Tb::Func.smart_cmp_value(v) }
+      hvsc = hvs.map {|v| Tb::Func.smart_cmp_value(v) }
+      hvs_hash[hvs] = hvsc
+      aggs = opt_cross_fields.map {|dstf, func, srcf| func.start(srcf ? pairs[srcf] : true) }
+      [[vvsc, hvsc], aggs]
+    }
+    all_representative = lambda {|_| 1 }
+    all_before = lambda {|_|
+      hvs_list = hvs_hash.keys.sort_by {|hvs| hvs_hash[hvs] }
+      n = vkfs.length + hvs_list.length * num_aggregate_fields
       header1 = (1..n).map {|i| i.to_s }
       y.set_header header1
       hkfs.each_with_index {|hkf, i|
@@ -87,7 +102,7 @@ def (Tb::Cmd).main_cross(argv)
         j = vkfs.length
         h1[j.to_s] = hkf
         hvs_list.each {|hkvs|
-          opt_cross_fields.length.times {
+          num_aggregate_fields.times {
             j += 1
             h1[j.to_s] = hkvs[i]
           }
@@ -101,49 +116,35 @@ def (Tb::Cmd).main_cross(argv)
         h2[j.to_s] = vkf
       }
       hvs_list.each {|hkvs|
-        opt_cross_fields.each {|agg_spec, new_field|
+        opt_cross_fields.last(num_aggregate_fields).each {|dstf, func, srcf|
           j += 1
           if Tb::Cmd.opt_cross_compact
             h2[j.to_s] = hkvs[-1]
           else
-            h2[j.to_s] = new_field
+            h2[j.to_s] = dstf
           end
         }
       }
       y.yield h2
     }
-    boudary_p = lambda {|pairs1, pairs2|
-      vcv1 = vkfs.map {|f| smart_cmp_value(pairs1[f]) }
-      vcv2 = vkfs.map {|f| smart_cmp_value(pairs2[f]) }
-      vcv1 != vcv2
+    v_representative = lambda {|((vvsc, _), _)|
+      vvsc
     }
-    aggs = nil
-    before = lambda {|_|
-      aggs = {}
+    v_before = lambda {|_|
+      aggs_hash = {}
     }
-    body = lambda {|pairs|
-      hvs = hkfs.map {|f| pairs[f] }
-      if !aggs.has_key?(hvs)
-        aggs[hvs] = opt_cross_fields.map {|agg_spec, nf|
-          begin
-            make_aggregator(agg_spec, header)
-          rescue ArgumentError
-            err($!.message)
-          end
-        }
-      end
-      ary = header.map {|f| pairs[f] }
-      aggs[hvs].each {|agg|
-        agg.update(ary)
-      }
+    body = lambda {|(_, aggs)|
+      hvs = aggs[vkfs.length, hkfs.length]
+      aggs_hash[hvs] = op.aggregate(aggs)
     }
-    after = lambda {|last_pairs|
-      ary = vkfs.map {|f| last_pairs[f] }
+    v_after = lambda {|(_, aggs)|
+      vvs = aggs[0, vkfs.length]
+      ary = vvs
       hvs_list.each {|hvs|
-        if aggs.has_key? hvs
-          ary.concat(aggs[hvs].map {|agg| agg.finish })
+        if aggs_hash.has_key? hvs
+          ary.concat(aggs_hash[hvs].last(num_aggregate_fields))
         else
-          ary.concat([nil] * opt_cross_fields.length)
+          ary.concat([nil] * num_aggregate_fields)
         end
       }
       pairs = {}
@@ -152,10 +153,11 @@ def (Tb::Cmd).main_cross(argv)
       }
       y.yield pairs
     }
-    sorted2.each_group_element(boudary_p, before, body, after)
-  }
-  with_output {|out|
-    er.write_to_csv(out, false)
+    er.detect_nested_group_by(
+      [[all_representative, all_before],
+       [v_representative, v_before, v_after]]).each(&body)
   }
+  Tb::Cmd.opt_N = true
+  output_tbenum(er)
 end
 

data/lib/tb/cmd_cut.rb

@@ -56,9 +56,7 @@ def (Tb::Cmd).main_cut(argv)
           y.yield pairs.reject {|k, v| fs.include? k }
         }
       }
-      with_output {|out|
-        er.write_to_csv(out, !Tb::Cmd.opt_N)
-      }
+      output_tbenum(er)
     else
       er = Tb::Enumerator.new {|y|
         tblreader.with_header {|header0|
@@ -73,9 +71,7 @@ def (Tb::Cmd).main_cut(argv)
           y.yield pairs.reject {|k, v| !fs.include?(k) }
         }
       }
-      with_output {|out|
-        er.write_to_csv(out, !Tb::Cmd.opt_N)
-      }
+      output_tbenum(er)
     end
   }
 end

data/lib/tb/cmd_git_log.rb

@@ -29,15 +29,17 @@
 Tb::Cmd.subcommands << 'git-log'
 
 Tb::Cmd.default_option[:opt_git_log_git_command] = nil
-Tb::Cmd.default_option[:opt_git_log_debug_git_log_file] = nil
+Tb::Cmd.default_option[:opt_git_log_debug_input] = nil
+Tb::Cmd.default_option[:opt_git_log_debug_output] = nil
 
 def (Tb::Cmd).op_git_log
   op = OptionParser.new
   op.banner = "Usage: tb git-log [OPTS] [GIT-DIR ...]\n" +
     "Show the GIT log as a table."
-  define_common_option(op, "hNo", "--no-pager")
+  define_common_option(op, "hNod", "--no-pager", '--debug')
   op.def_option('--git-command COMMAND', 'specify the git command (default: git)') {|command| Tb::Cmd.opt_git_log_git_command = command }
-  op.def_option('--debug-git-log-file FILE', 'specify the result git log (for debug)') {|filename| Tb::Cmd.opt_git_log_debug_git_log_file = filename }
+  op.def_option('--debug-git-log-output FILE', 'store the raw output of git-log (for debug)') {|filename| Tb::Cmd.opt_git_log_debug_output = filename }
+  op.def_option('--debug-git-log-input FILE', 'use the file as output of git-log (for debug)') {|filename| Tb::Cmd.opt_git_log_debug_input = filename }
   op
 end
 
@@ -68,8 +70,8 @@ Tb::Cmd::GIT_LOG_PRETTY_FORMAT = 'format:%x01commit-separator%x01%n' +
 Tb::Cmd::GIT_LOG_HEADER = Tb::Cmd::GIT_LOG_FORMAT_SPEC.map {|k, v| k } + ['files']
 
 def (Tb::Cmd).git_log_with_git_log(dir)
-  if Tb::Cmd.opt_git_log_debug_git_log_file
-    File.open(Tb::Cmd.opt_git_log_debug_git_log_file) {|f|
+  if Tb::Cmd.opt_git_log_debug_input
+    File.open(Tb::Cmd.opt_git_log_debug_input) {|f|
       yield f
     }
   else
@@ -85,9 +87,18 @@ def (Tb::Cmd).git_log_with_git_log(dir)
         '.',
         {:chdir=>dir}
     ]
-    IO.popen(command) {|f|
-      yield f
-    }
+    $stderr.puts "git command line: #{command.inspect}" if 1 <= Tb::Cmd.opt_debug
+    if Tb::Cmd.opt_git_log_debug_output
+      command.last[:out] = Tb::Cmd.opt_git_log_debug_output
+      system(*command)
+      File.open(Tb::Cmd.opt_git_log_debug_output) {|f|
+        yield f
+      }
+    else
+      IO.popen(command) {|f|
+        yield f
+      }
+    end
   end
 end
 
@@ -186,8 +197,6 @@ def (Tb::Cmd).main_git_log(argv)
       }
     }
   }
-  with_output {|out|
-    er.write_to_csv(out, !Tb::Cmd.opt_N)
-  }
+  output_tbenum(er)
 end
 

data/lib/tb/cmd_grep.rb

@@ -80,8 +80,6 @@ def (Tb::Cmd).main_grep(argv)
       end
     }
   }
-  with_output {|out|
-    er.write_to_csv(out, !Tb::Cmd.opt_N)
-  }
+  output_tbenum(er)
 end
 

data/lib/tb/cmd_group.rb

@@ -46,61 +46,35 @@ def (Tb::Cmd).main_group(argv)
   exit_if_help('group')
   err("no key fields given.") if argv.empty?
   kfs = split_field_list_argument(argv.shift)
-  opt_group_fields = Tb::Cmd.opt_group_fields.map {|arg|
+  opt_group_fields = kfs.map {|f| [f, Tb::Func::First, f] } +
+    Tb::Cmd.opt_group_fields.map {|arg|
     aggregation_spec, new_field = split_field_list_argument(arg)
     new_field ||= aggregation_spec
     [new_field,
-      lambda {|fields|
-        begin
-          make_aggregator(aggregation_spec, fields)
-        rescue ArgumentError
-          err($!.message)
-        end
-      }
+      *begin
+        parse_aggregator_spec2(aggregation_spec)
+      rescue ArgumentError
+        err($!.message)
+      end
     ]
   }
   argv = ['-'] if argv.empty?
   creader = Tb::CatReader.open(argv, Tb::Cmd.opt_N)
   result = Tb::Enumerator.new {|y|
-    er = creader.extsort_by {|pairs|
-      kfs.map {|f| smart_cmp_value(pairs[f]) }
-    }
-    header = nil
-    row = nil
-    agg = nil
-    er2 = er.with_header {|header0|
-      header = header0
-      y.set_header(kfs + opt_group_fields.map {|f, maker| f })
-    }
-    boudary_p = lambda {|pairs1, pairs2|
-      kfs.any? {|f| pairs1[f] != pairs2[f] }
+    op = Tb::Zipper.new(opt_group_fields.map {|dstf, func, srcf| func })
+    er = creader.extsort_reduce(op) {|pairs|
+      [kfs.map {|f| Tb::Func.smart_cmp_value(pairs[f]) },
+       opt_group_fields.map {|dstf, func, srcf| func.start(srcf ? pairs[srcf] : true) } ]
     }
-    before = lambda {|first_pairs|
-      row = {}
-      kfs.each {|f|
-        row[f] = first_pairs[f]
-      }
-      agg = {}
-      opt_group_fields.each {|f, maker|
-        agg[f] = maker.call(header)
+    fields = opt_group_fields.map {|dstf, func, srcf| dstf }
+    y.set_header(fields)
+    er.each {|_, vals|
+      pairs = opt_group_fields.zip(vals).map {|(dstf, func, _), val|
+        [dstf, func.aggregate(val)]
       }
+      y.yield Hash[pairs]
     }
-    body = lambda {|pairs|
-      ary = header.map {|f| pairs[f] }
-      opt_group_fields.each {|f, maker|
-        agg[f].update(ary)
-      }
-    }
-    after = lambda {|last_pairs|
-      opt_group_fields.each {|f, maker|
-        row[f] = agg[f].finish
-      }
-      y.yield row
-    }
-    er2.each_group_element(boudary_p, before, body, after)
-  }
-  with_output {|out|
-    result.write_to_csv(out, !Tb::Cmd.opt_N)
   }
+  output_tbenum(result)
 end
 

data/lib/tb/cmd_gsub.rb

@@ -77,11 +77,9 @@ def (Tb::Cmd).main_gsub(argv)
           [f, v.gsub(re, repl)]
         }
       end
-      y.yield Tb::Pairs.new(pairs2)
+      y.yield Hash[pairs2]
     }
   }
-  with_output {|out|
-    er.write_to_csv(out, !Tb::Cmd.opt_N)
-  }
+  output_tbenum(er)
 end