tb 0.2 → 0.3

data/lib/tb/cmd_to_csv.rb

@@ -0,0 +1,60 @@
+# Copyright (C) 2011-2012 Tanaka Akira  <akr@fsij.org>
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above
+#     copyright notice, this list of conditions and the following
+#     disclaimer in the documentation and/or other materials provided
+#     with the distribution.
+#  3. The name of the author may not be used to endorse or promote
+#     products derived from this software without specific prior
+#     written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Tb::Cmd.subcommands << 'to-csv'
+
+def (Tb::Cmd).op_to_csv
+  op = OptionParser.new
+  op.banner = "Usage: tb to-csv [OPTS] [TABLE ...]\n" +
+    "Convert a table to CSV (Comma Separated Value)."
+  define_common_option(op, "hNo", "--no-pager")
+  op
+end
+
+def (Tb::Cmd).main_to_csv(argv)
+  op_to_csv.parse!(argv)
+  exit_if_help('to-csv')
+  argv = ['-'] if argv.empty?
+  creader = Tb::CatReader.open(argv, Tb::Cmd.opt_N)
+  header = []
+  ter = Tb::Enumerator.new {|y|
+    creader.with_cumulative_header.each {|pairs, header1|
+      header = header1
+      y.yield pairs
+    }
+  }.to_fileenumerator
+  er = Tb::Enumerator.new {|y|
+    y.set_header header
+    ter.each {|pairs|
+      y.yield Tb::Pairs.new(header.map {|f| [f, pairs[f]] })
+    }
+  }
+  with_output {|out|
+    er.write_to_csv(out, !Tb::Cmd.opt_N)
+  }
+end

data/lib/tb/cmd_to_json.rb

@@ -0,0 +1,60 @@
+# Copyright (C) 2011-2012 Tanaka Akira  <akr@fsij.org>
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above
+#     copyright notice, this list of conditions and the following
+#     disclaimer in the documentation and/or other materials provided
+#     with the distribution.
+#  3. The name of the author may not be used to endorse or promote
+#     products derived from this software without specific prior
+#     written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Tb::Cmd.subcommands << 'to-json'
+
+def (Tb::Cmd).op_to_json
+  op = OptionParser.new
+  op.banner = "Usage: tb to-json [OPTS] [TABLE]\n" +
+    "Convert a table to JSON (JavaScript Object Notation)."
+  define_common_option(op, "hNo", "--no-pager")
+  op
+end
+
+def (Tb::Cmd).main_to_json(argv)
+  require 'json'
+  op_to_json.parse!(argv)
+  exit_if_help('to-json')
+  argv = ['-'] if argv.empty?
+  with_output {|out|
+    out.print "["
+    sep = nil
+    argv.each {|filename|
+      sep = ",\n\n" if sep
+      tablereader_open(filename) {|tblreader|
+        tblreader.each {|pairs|
+          out.print sep if sep
+          out.print JSON.pretty_generate(Hash[pairs.to_a])
+          sep = ",\n"
+        }
+      }
+    }
+    out.puts "]"
+  }
+end
+

data/lib/tb/cmd_to_pnm.rb

@@ -0,0 +1,48 @@
+# Copyright (C) 2011-2012 Tanaka Akira  <akr@fsij.org>
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above
+#     copyright notice, this list of conditions and the following
+#     disclaimer in the documentation and/or other materials provided
+#     with the distribution.
+#  3. The name of the author may not be used to endorse or promote
+#     products derived from this software without specific prior
+#     written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Tb::Cmd.subcommands << 'to-pnm'
+
+def (Tb::Cmd).op_to_pnm
+  op = OptionParser.new
+  op.banner = "Usage: tb to-pnm [OPTS] [TABLE]\n" +
+    "Convert a table to PNM (Portable Anymap: PPM, PGM, PBM)."
+  define_common_option(op, "hNo", "--no-pager")
+  op
+end
+
+def (Tb::Cmd).main_to_pnm(argv)
+  op_to_pnm.parse!(argv)
+  exit_if_help('to-pnm')
+  argv = ['-'] if argv.empty?
+  tbl = Tb::CatReader.open(argv, Tb::Cmd.opt_N).to_tb
+  with_output {|out|
+    tbl.generate_pnm(out)
+  }
+end
+

data/lib/tb/cmd_to_pp.rb

@@ -0,0 +1,71 @@
+# Copyright (C) 2011-2012 Tanaka Akira  <akr@fsij.org>
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above
+#     copyright notice, this list of conditions and the following
+#     disclaimer in the documentation and/or other materials provided
+#     with the distribution.
+#  3. The name of the author may not be used to endorse or promote
+#     products derived from this software without specific prior
+#     written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Tb::Cmd.subcommands << 'to-pp'
+
+def (Tb::Cmd).op_to_pp
+  op = OptionParser.new
+  op.banner = "Usage: tb to-pp [OPTS] [TABLE]\n" +
+    "Convert a table to pretty printed format."
+  define_common_option(op, "hNo", "--no-pager")
+  op
+end
+
+def (Tb::Cmd).main_to_pp(argv)
+  op_to_pp.parse!(argv)
+  exit_if_help('to-pp')
+  argv.unshift '-' if argv.empty?
+  with_output {|out|
+    argv.each {|filename|
+      tablereader_open(filename) {|tblreader|
+        tblreader.each {|pairs|
+          a = pairs.reject {|f, v| v.nil? }
+          q = PP.new(out, 79)
+          q.guard_inspect_key {
+            q.group(1, '{', '}') {
+              q.seplist(a, nil, :each) {|kv|
+                k, v = kv
+                q.group {
+                  q.pp k
+                  q.text '=>'
+                  q.group(1) {
+                    q.breakable ''
+                    q.pp v
+                  }
+                }
+              }
+            }
+          }
+          q.flush
+          out << "\n"
+        }
+      }
+    }
+  }
+end
+

data/lib/tb/cmd_to_tsv.rb

@@ -0,0 +1,48 @@
+# Copyright (C) 2011-2012 Tanaka Akira  <akr@fsij.org>
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above
+#     copyright notice, this list of conditions and the following
+#     disclaimer in the documentation and/or other materials provided
+#     with the distribution.
+#  3. The name of the author may not be used to endorse or promote
+#     products derived from this software without specific prior
+#     written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Tb::Cmd.subcommands << 'to-tsv'
+
+def (Tb::Cmd).op_to_tsv
+  op = OptionParser.new
+  op.banner = "Usage: tb to-tsv [OPTS] [TABLE]\n" +
+    "Convert a table to TSV (Tab Separated Value)."
+  define_common_option(op, "hNo", "--no-pager")
+  op
+end
+
+def (Tb::Cmd).main_to_tsv(argv)
+  op_to_tsv.parse!(argv)
+  exit_if_help('to-tsv')
+  argv = ['-'] if argv.empty?
+  tbl = Tb::CatReader.open(argv, Tb::Cmd.opt_N).to_tb
+  with_output {|out|
+    tbl_generate_tsv(tbl, out)
+  }
+end
+

data/lib/tb/cmd_to_yaml.rb

@@ -0,0 +1,52 @@
+# Copyright (C) 2011-2012 Tanaka Akira  <akr@fsij.org>
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above
+#     copyright notice, this list of conditions and the following
+#     disclaimer in the documentation and/or other materials provided
+#     with the distribution.
+#  3. The name of the author may not be used to endorse or promote
+#     products derived from this software without specific prior
+#     written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Tb::Cmd.subcommands << 'to-yaml'
+
+def (Tb::Cmd).op_to_yaml
+  op = OptionParser.new
+  op.banner = "Usage: tb to-yaml [OPTS] [TABLE]\n" +
+    "Convert a table to YAML (YAML Ain't a Markup Language)."
+  define_common_option(op, "hNo", "--no-pager")
+  op
+end
+
+def (Tb::Cmd).main_to_yaml(argv)
+  require 'yaml'
+  op_to_yaml.parse!(argv)
+  exit_if_help('to-yaml')
+  argv = ['-'] if argv.empty?
+  tbl = Tb::CatReader.open(argv, Tb::Cmd.opt_N).to_tb
+  ary = tbl.map {|rec| rec.to_h }
+  with_output {|out|
+    YAML.dump(ary, out)
+    out.puts
+  }
+end
+
+

data/lib/tb/cmd_unnest.rb

@@ -0,0 +1,118 @@
+# Copyright (C) 2011-2012 Tanaka Akira  <akr@fsij.org>
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above
+#     copyright notice, this list of conditions and the following
+#     disclaimer in the documentation and/or other materials provided
+#     with the distribution.
+#  3. The name of the author may not be used to endorse or promote
+#     products derived from this software without specific prior
+#     written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Tb::Cmd.subcommands << 'unnest'
+
+Tb::Cmd.default_option[:opt_unnest_prefix] = ''
+Tb::Cmd.default_option[:opt_unnest_outer] = nil
+
+def (Tb::Cmd).op_unnest
+  op = OptionParser.new
+  op.banner = "Usage: tb unnest [OPTS] FIELD [TABLE ...]\n" +
+    "Unnest a field."
+  define_common_option(op, "hNo", "--no-pager")
+  op.def_option('--prefix PREFIX', 'field prefix') {|prefix| Tb::Cmd.opt_unnest_prefix = prefix }
+  op.def_option('--outer', 'retain rows for empty nested table') { Tb::Cmd.opt_unnest_outer = true }
+  op
+end
+
+def (Tb::Cmd).main_unnest(argv)
+  op_unnest.parse!(argv)
+  exit_if_help('unnest')
+  err('no field given.') if argv.empty?
+  target_field = argv.shift
+  argv = ['-'] if argv.empty?
+  creader = Tb::CatReader.open(argv, Tb::Cmd.opt_N)
+  er = Tb::Enumerator.new {|y|
+    nested_fields = {}
+    ter = Tb::Enumerator.new {|y2|
+      creader.with_header {|header0|
+        unless header0.include? target_field
+          err("field not found: #{target_field.inspect}")
+        end
+        y2.set_header header0
+      }.each {|pairs|
+        pairs2 = {}
+        pairs.each {|f, v|
+          if f != target_field
+            pairs2[f] = v
+          elsif v.nil?
+            pairs2[f] = v
+          else
+            nested_tbl = Tb.parse_csv(v)
+            nested_tbl.list_fields.each {|f2|
+              unless nested_fields.has_key? f2
+                nested_fields[f2] = nested_fields.size
+              end
+            }
+            pairs2[f] = nested_tbl
+          end
+        }
+        y2.yield pairs2
+      }
+    }.to_fileenumerator
+    nested_fields_ary = nested_fields.keys
+    if Tb::Cmd.opt_unnest_prefix
+      nested_fields_ary.map! {|f| Tb::Cmd.opt_unnest_prefix + f }
+    end
+    ter.with_header {|header0|
+      header2 = []
+      header0.each {|f|
+        if f != target_field
+          header2 << f
+        else
+          header2.concat nested_fields_ary
+        end
+      }
+      y.set_header header2
+    }.each {|pairs|
+      pairs2 = pairs.reject {|f, v| f == target_field }
+      ntbl = pairs[target_field]
+      if ntbl.nil? || ntbl.empty?
+        if Tb::Cmd.opt_unnest_outer
+          y.yield pairs2
+        end
+      else
+        ntbl.each {|npairs|
+          pairs3 = pairs2.dup
+          npairs.each {|nf, nv|
+            if Tb::Cmd.opt_unnest_prefix
+              nf = Tb::Cmd.opt_unnest_prefix + nf
+            end
+            pairs3[nf] = nv
+          }
+          y.yield pairs3
+        }
+      end
+    }
+  }
+  with_output {|out|
+    er.write_to_csv(out, !Tb::Cmd.opt_N)
+  }
+end
+