tb 0.9 → 1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1b5601329ad0755149e198c2139d3da0707fb0af
-  data.tar.gz: 03997adcba0d54d8745235e6534d899ba23100ec
+  metadata.gz: 9aa78baf63b93601d7d70cba5e9a69ecda6aceff
+  data.tar.gz: df5877535b6a51efc81c27b3903f99e9d762e997
 SHA512:
-  metadata.gz: 59b8dea2fba883302eb50b2253c74c0e963ad408cf9031a908ee6957b91f0562f1c6ee93249e68ca13ed49d4144ddce863cfd381cba2e9c693801616791593e7
-  data.tar.gz: ef49a9b3f97c8490303bf50812b590b38c56fa6c8b045bf136f032debd5a8070cbf10e28e092699578a3aac442c50b7128d8ed37f4d8e24ecee5a7b74cc72a93
+  metadata.gz: d460033685e07e30d9b3e2de12eeacfcb5c0c451655e9ad75cce094576ac34188a5badd93d56316f663f2096ffa41826d1f2e5f5a376be06047093b794ffbd2f
+  data.tar.gz: 727ec54c18343cba5a725952789202e220370478e430cfe4c366df152e9908bbfdd475126feef7047fca1d939fddc1b31c968eb11728c0953ae7f037f9d3ebd8

data/README

@@ -1,12 +1,14 @@
-= tb - manipulation tool for table: CSV, TSV, JSON, LTSV, etc.
+= tb - manipulation tool for tables
 
 tb provides a command and a library for manipulating tables:
-Unix filter like operations (grep, sort, cat, cut, ls, etc.),
+Unix filter like operations (sort, cat, cut, ls, etc.),
 SQL like operations (join, group, etc.),
-other table operations (gsub, rename, cross, melt, unmelt, etc.),
+other table operations (search, gsub, rename, cross, melt, unmelt, etc.),
 information extractions (git, svn, tar),
 and more.
 
+tb supports various table formats: CSV, TSV, JSON, NDJSON, LTSV, etc.
+
 == Example
 
 There is a CSV file for programming languages and their birth year in
@@ -56,18 +58,18 @@ Also, the comparison method used in tb is smart to sort numbers correctly.
   BASIC,1964
   PL/I,1964
 
-"grep" subcommand search CSV file.
+"search" subcommand search CSV file.
 
-  % tb grep R sample/langs.csv
+  % tb search R sample/langs.csv
   language,year
   FORTRAN,1955
   Ruby,1993
 
-"grep" subcommand takes -f with field name and -v to show non-matching rows.
+"search" subcommand takes -f with field name and -v to show non-matching rows.
 You don't need to care field separators (comma) to match.
 Following example searches languages which name contains a non-alphabet character.
 
-  % tb grep -vf language '\A[A-Za-z]*\z' sample/langs.csv |cat
+  % tb search -vf language '\A[A-Za-z]*\z' sample/langs.csv
   language,year
   ALGOL 58,1958
   PL/I,1964
@@ -78,10 +80,10 @@ Following example searches languages which name contains a non-alphabet characte
   C#,2001
   F#,2002
 
-"grep" subcommand can take Ruby expression, instead of a regexp.
+"search" subcommand can take Ruby expression, instead of a regexp.
 The variable, "_", contains a hash which represents a record.
 
-  % tb grep --ruby '(1990..1999).include?(_["year"].to_i)' sample/langs.csv
+  % tb search --ruby '(1990..1999).include?(_["year"].to_i)' sample/langs.csv
   language,year
   Haskell,1990
   Python,1991
@@ -138,7 +140,7 @@ There are more subcommands.
     tb to-json [OPTS] [TABLE]
     tb to-yaml [OPTS] [TABLE]
     tb to-pp [OPTS] [TABLE]
-    tb grep [OPTS] REGEXP [TABLE ...]
+    tb search [OPTS] REGEXP [TABLE ...]
     tb gsub [OPTS] REGEXP STRING [TABLE ...]
     tb sort [OPTS] [TABLE]
     tb cut [OPTS] FIELD,... [TABLE]
@@ -172,7 +174,7 @@ tb help -s shows one line summary of the subcommands.
   to-json     : Convert a table to JSON (JavaScript Object Notation).
   to-yaml     : Convert a table to YAML (YAML Ain't a Markup Language).
   to-pp       : Convert a table to pretty printed format.
-  grep        : Search rows using regexp or ruby expression.
+  search      : Search rows using regexp or ruby expression.
   gsub        : Substitute cells.
   sort        : Sort rows.
   cut         : Select columns.

data/lib/tb.rb

@@ -1,6 +1,6 @@
 # lib/tb.rb - entry file for table library
 #
-# Copyright (C) 2010-2013 Tanaka Akira  <akr@fsij.org>
+# Copyright (C) 2010-2014 Tanaka Akira  <akr@fsij.org>
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -30,7 +30,7 @@
 
 require 'tempfile'
 
-class Tb
+module Tb
 end
 
 require 'pp'
@@ -38,17 +38,25 @@ require 'tb/enumerable'
 require 'tb/enumerator'
 require 'tb/func'
 require 'tb/zipper'
-require 'tb/basic'
-require 'tb/record'
+
+require 'tb/headerreader'
+require 'tb/headerwriter'
+
+require 'tb/numericreader'
+require 'tb/numericwriter'
+
+require 'tb/hashreader'
+require 'tb/hashwriter'
+
 require 'tb/csv'
 require 'tb/tsv'
 require 'tb/ltsv'
 require 'tb/pnm'
 require 'tb/json'
-require 'tb/reader'
+require 'tb/ndjson'
+
 require 'tb/ropen'
 require 'tb/catreader'
-require 'tb/fieldset'
 require 'tb/search'
 require 'tb/ex_enumerable'
 require 'tb/ex_enumerator'

data/lib/tb/catreader.rb

@@ -1,6 +1,6 @@
 # lib/tb/catreader.rb - Tb::CatReader class
 #
-# Copyright (C) 2011-2012 Tanaka Akira  <akr@fsij.org>
+# Copyright (C) 2011-2014 Tanaka Akira  <akr@fsij.org>
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -32,7 +32,7 @@ module Tb::CatReader
   def self.open(filenames, numeric=false, with_filename=false)
     readers = []
     filenames.each {|f|
-      r = Tb.open_reader(f, numeric ? {:numeric=>true} : {})
+      r = Tb.open_reader(f, numeric)
       if with_filename
         r = r.newfield("filename") { f }
       end

data/lib/tb/cmd_consecutive.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2011-2012 Tanaka Akira  <akr@fsij.org>
+# Copyright (C) 2011-2014 Tanaka Akira  <akr@fsij.org>
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -74,7 +74,11 @@ def (Tb::Cmd).main_consecutive(argv)
             ps = buf[i]
             next if !ps.has_key?(f)
             v = ps[f]
-            pairs2["#{f}_#{i+1}"] = v
+            if Tb::Cmd.opt_N
+              pairs2[((f.to_i-1) * Tb::Cmd.opt_consecutive_n + i + 1).to_s] = v
+            else
+              pairs2["#{f}_#{i+1}"] = v
+            end
           }
         }
         empty = false

data/lib/tb/cmd_crop.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2011-2012 Tanaka Akira  <akr@fsij.org>
+# Copyright (C) 2011-2014 Tanaka Akira  <akr@fsij.org>
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -35,10 +35,26 @@ def (Tb::Cmd).op_crop
   op.banner = "Usage: tb crop [OPTS] [TABLE ...]\n" +
     "Extract rectangle in a table."
   define_common_option(op, "ho", "--no-pager")
-  op.def_option('-r RANGE', 'range.  i.e. "2,1-4,3", "B1:D3"') {|arg| Tb::Cmd.opt_crop_range = arg }
+  op.def_option('-r RANGE', 'range.  i.e. "R2C1:R4C3", "B1:D3"') {|arg| Tb::Cmd.opt_crop_range = arg }
   op
 end
 
+Tb::Cmd.def_vhelp('crop', <<'End')
+Example:
+
+  % cat tst.csv
+  0,1,2,4
+  5,6,7,8
+  9,a,b,c
+  d,e,f,g
+  h,i,j,k
+  % tb crop -r R2C2:R4C3 tst.csv
+  6,7
+  a,b
+  e,f
+End
+
+
 def (Tb::Cmd).decode_a1_addressing_col(str)
   (26**str.length-1)/25+str.tr("A-Z", "0-9A-P").to_i(26)
 end
@@ -79,7 +95,10 @@ def (Tb::Cmd).main_crop(argv)
             f = f.to_i
             f < range_col1 || range_col2 < f
           }
-          y.yield pairs2
+          pairs2 = pairs2.map {|f, v|
+            [(f.to_i - range_col1 + 1).to_s, v]
+          }
+          y.yield Hash[pairs2]
         end
         rownum += 1
       }

data/lib/tb/cmd_cross.rb

@@ -42,6 +42,30 @@ def (Tb::Cmd).op_cross
   op
 end
 
+Tb::Cmd.def_vhelp('cross', <<'End')
+Example:
+
+  % cat tst.csv
+  a,b,c
+  A,X,2
+  A,Y,3
+  B,Y,4
+  % tb cross a b tst.csv
+  b,X,Y
+  a,count,count
+  A,1,1
+  B,,1
+  % tb cross -c a b tst.csv
+  a,X,Y
+  A,1,1
+  B,,1
+  % tb cross a b -a 'avg(c)' tst.csv
+  b,X,Y
+  a,avg(c),avg(c)
+  A,2.0,3.0
+  B,,4.0
+End
+
 def (Tb::Cmd).main_cross(argv)
   op_cross.parse!(argv)
   exit_if_help('cross')

data/lib/tb/cmd_cut.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2011-2012 Tanaka Akira  <akr@fsij.org>
+# Copyright (C) 2011-2014 Tanaka Akira  <akr@fsij.org>
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -39,6 +39,23 @@ def (Tb::Cmd).op_cut
   op
 end
 
+Tb::Cmd.def_vhelp('cut', <<'End')
+Example:
+
+  % cat tst.csv
+  a,b,c
+  0,1,2
+  4,5,6
+  % tb cut a,c tst.csv
+  a,c
+  0,2
+  4,6
+  % tb cut -v a tst.csv
+  b,c
+  1,2
+  5,6
+End
+
 def (Tb::Cmd).main_cut(argv)
   op_cut.parse!(argv)
   exit_if_help('cut')
@@ -59,15 +76,8 @@ def (Tb::Cmd).main_cut(argv)
       output_tbenum(er)
     else
       er = Tb::Enumerator.new {|y|
-        tblreader.with_header {|header0|
-          if header0
-            fieldset = Tb::FieldSet.new(*header0)
-            fs.each {|f|
-              fieldset.index_from_field_ex(f)
-            }
-          end
-          y.set_header fs
-        }.each {|pairs|
+        y.set_header fs
+        tblreader.each {|pairs|
           y.yield pairs.reject {|k, v| !fs.include?(k) }
         }
       }

data/lib/tb/cmd_git.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2011-2012 Tanaka Akira  <akr@fsij.org>
+# Copyright (C) 2011-2014 Tanaka Akira  <akr@fsij.org>
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -43,6 +43,20 @@ def (Tb::Cmd).op_git
   op
 end
 
+Tb::Cmd.def_vhelp('git', <<'End')
+Example:
+
+  % tb git | tb cut commit
+  commit
+  dedc77387459ea90b5f508be03cce9ecf0d990b5
+  bebf22d1d97921fb86f816ea48235948fa4c73b3
+  38d61f96a8b33a6c5d527e564dc78b4a47b3eff0
+  48c10d3b1b0609d389ce1d9203e57f63c1408703
+  d002c43e0a85ec7051f667140261d6b0e74e9039
+  9292cade7c71d0b8805898621a53ba7a4713d029
+  ...
+End
+
 Tb::Cmd::GIT_LOG_FORMAT_SPEC = [
   %w[commit %H],
   %w[tree %T],
@@ -150,12 +164,11 @@ def (Tb::Cmd).git_parse_commit(commit_info, files)
       warn "unexpected git output (raw/numstat): #{file_line.inspect}"
     end
   }
-  Tb.csv_stream_output(files_csv="") {|gen|
-    gen << %w[mode1 mode2 hash1 hash2 add del status filename]
-    files_raw.each {|filename, (mode1, mode2, hash1, hash2, status)|
-      add, del = files_numstat[filename]
-      gen << [mode1, mode2, hash1, hash2, add, del, status, filename]
-    }
+  files_csv = ""
+  files_csv << %w[mode1 mode2 hash1 hash2 add del status filename].to_csv
+  files_raw.each {|filename, (mode1, mode2, hash1, hash2, status)|
+    add, del = files_numstat[filename]
+    files_csv << [mode1, mode2, hash1, hash2, add, del, status, filename].to_csv
   }
   h = {}
   commit_info.each {|s|

data/lib/tb/cmd_group.rb

@@ -41,6 +41,38 @@ def (Tb::Cmd).op_group
   op
 end
 
+Tb::Cmd.def_vhelp('group', <<'End')
+Example:
+
+  % cat tst.csv
+  a,b,c
+  A,X,2
+  A,Y,3
+  B,Y,4
+  % tb group a tst.csv
+  a
+  A
+  B
+  % tb group a -a count tst.csv
+  a,count
+  A,2
+  B,1
+  % tb group a -a 'avg(c)' tst.csv
+  a,avg(c)
+  A,2.5
+  B,4.0
+  % tb group a,b tst.csv
+  a,b
+  A,X
+  A,Y
+  B,Y
+  % tb group a,b -a count tst.csv
+  a,b,count
+  A,X,1
+  A,Y,1
+  B,Y,1
+End
+
 def (Tb::Cmd).main_group(argv)
   op_group.parse!(argv)
   exit_if_help('group')

data/lib/tb/cmd_gsub.rb

@@ -41,6 +41,27 @@ def (Tb::Cmd).op_gsub
   op
 end
 
+Tb::Cmd.def_vhelp('gsub', <<'End')
+Example:
+
+  % cat tst.csv
+  foo,bar
+  baz,qux
+  hoge,moga
+  % tb gsub o X tst.csv
+  foo,bar
+  baz,qux
+  hXge,mXga
+  % tb gsub -f foo o X tst.csv
+  foo,bar
+  baz,qux
+  hXge,moga
+  % tb gsub '[aeiou]' '{\&}' tst.csv
+  foo,bar
+  b{a}z,q{u}x
+  h{o}g{e},m{o}g{a}
+End
+
 def (Tb::Cmd).main_gsub(argv)
   op_gsub.parse!(argv)
   exit_if_help('gsub')

data/lib/tb/cmd_join.rb

@@ -59,6 +59,34 @@ def (Tb::Cmd).op_join
   op
 end
 
+Tb::Cmd.def_vhelp('join', <<'End')
+Example:
+
+  % cat tst1.csv
+  name,length
+  A,20
+  B,30
+  C,25
+  % cat tst2.csv
+  name,weight
+  A,5
+  B,8
+  % tb join tst1.csv tst2.csv
+  name,length,weight
+  A,20,5
+  B,30,8
+  % tb join --left tst1.csv tst2.csv
+  name,length,weight
+  A,20,5
+  B,30,8
+  C,25,
+  % tb join --left --outer-missing=zzz tst1.csv tst2.csv
+  name,length,weight
+  A,20,5
+  B,30,8
+  C,25,zzz
+End
+
 def (Tb::Cmd).main_join(argv)
   op_join.parse!(argv)
   exit_if_help('join')

data/lib/tb/cmd_ls.rb

@@ -45,6 +45,15 @@ def (Tb::Cmd).op_ls
   op
 end
 
+Tb::Cmd.def_vhelp('ls', <<'End')
+Example:
+
+  % tb ls -l lib
+  filemode,nlink,user,group,size,mtime,filename,symlink
+  drwxr-xr-x,2,akr,akr,4096,2014-10-26T09:15:35+09:00,lib/tb,
+  -rw-r--r--,1,akr,akr,2113,2014-10-25T18:40:05+09:00,lib/tb.rb,
+End
+
 def (Tb::Cmd).main_ls(argv)
   op_ls.parse!(argv)
   exit_if_help('ls')