tb 0.5 → 0.6

data/README

@@ -149,6 +149,7 @@ There are more subcommands.
     tb group [OPTS] KEY-FIELD1,... [TABLE ...]
     tb cross [OPTS] VKEY-FIELD1,... HKEY-FIELD1,... [TABLE ...]
     tb melt KEY-FIELDS-LIST [OPTS] [TABLE ...]
+    tb unmelt [OPTS] [TABLE ...]
     tb nest [OPTS] NEWFIELD,OLDFIELD1,OLDFIELD2,... [TABLE ...]
     tb unnest [OPTS] FIELD [TABLE ...]
     tb shape [OPTS] [TABLE ...]
@@ -181,6 +182,7 @@ tb help -s shows one line summary of the subcommands.
   group       : Group and aggregate rows.
   cross       : Create a cross table. (a.k.a contingency table, pivot table)
   melt        : split value fields into records.
+  unmelt      : merge melted records into a record.
   nest        : Nest fields.
   unnest      : Unnest a field.
   shape       : Show table size.

data/lib/tb/cmd_tar_tvf.rb

@@ -30,6 +30,7 @@ Tb::Cmd.subcommands << 'tar-tvf'
 
 Tb::Cmd.default_option[:opt_tar_tvf_l] = 0
 Tb::Cmd.default_option[:opt_tar_tvf_ustar] = nil
+Tb::Cmd.default_option[:opt_tar_tvf_hash] = []
 
 def (Tb::Cmd).op_tar_tvf
   op = OptionParser.new
@@ -38,6 +39,7 @@ def (Tb::Cmd).op_tar_tvf
   define_common_option(op, "hNo", "--no-pager")
   op.def_option('-l', 'show more attributes.') {|fs| Tb::Cmd.opt_tar_tvf_l += 1 }
   op.def_option('--ustar', 'ustar format (POSIX.1-1988).  No GNU and POSIX.1-2001 extension.') {|fs| Tb::Cmd.opt_tar_tvf_ustar = true }
+  op.def_option('--hash=ALGORITHMS', 'hash algorithms such as md5,sha256,sha384,sha512 (default: none)') {|hs| Tb::Cmd.opt_tar_tvf_hash.concat split_field_list_argument(hs) }
   op
 end
 
@@ -74,6 +76,14 @@ Tb::Cmd::TAR_TYPEFLAG = {
   '7' => :contiguous,           # [POSIX] Reserved for high-performance file.  (It is come from "contiguous file" (S_IFCTG) of Masscomp?)
 }
 
+Tb::Cmd::TAR_HASH_ALGORITHMS = {
+  'md5' => 'MD5',
+  'sha1' => 'SHA1',
+  'sha256' => 'SHA256',
+  'sha384' => 'SHA384',
+  'sha512' => 'SHA512',
+}
+
 def (Tb::Cmd).tar_tvf_parse_seconds_from_epoch(val)
   if /\./ =~ val
     num = ($` + $').to_i
@@ -196,6 +206,41 @@ class Tb::Cmd::TarReader
     end
     @offset += size
   end
+
+  def calculate_hash(blocklength, filesize, alg_list) # :yield: alg, result
+    digests = {}
+    alg_list.each {|alg|
+      c = Digest.const_get(Tb::Cmd::TAR_HASH_ALGORITHMS.fetch(alg))
+      digests[alg] = c.new
+    }
+    rest = blocklength
+    while 0 < rest
+      if rest < 4096
+        s = rest
+      else
+        s = 4096
+      end
+      ret = @input.read(s)
+      if !ret || ret.length != s
+        warn "premature end of tar archive content (#{kind})"
+        raise Tb::Cmd::TarFormatError
+      end
+      if filesize < s
+        ret = ret[0, filesize]
+      end
+      digests.each {|alg, d|
+        d.update ret
+      }
+      filesize -= s
+      rest -= s
+    end
+    @offset += blocklength
+    result = {}
+    digests.each {|alg, d|
+      result[alg] = d.hexdigest
+    }
+    result
+  end
 end
 
 def (Tb::Cmd).tar_tvf_read_end_of_archive_indicator(reader)
@@ -291,7 +336,13 @@ def (Tb::Cmd).tar_tvf_each(f)
     when :link, :symlink, :directory, :character_special, :block_special, :fifo
       # xxx: hardlink may have contents for posix archive.
     else
-      reader.skip(content_blocklength, 'file content')
+      if Tb::Cmd.opt_tar_tvf_hash.empty?
+        reader.skip(content_blocklength, 'file content')
+      else
+        reader.calculate_hash(content_blocklength, h[:size], Tb::Cmd.opt_tar_tvf_hash).each {|alg, result|
+          h[alg] = result
+        }
+      end
     end
     h[:size_in_tar] = reader.offset - offset
     yield h
@@ -405,6 +456,10 @@ end
 def (Tb::Cmd).main_tar_tvf(argv)
   op_tar_tvf.parse!(argv)
   exit_if_help('tar-tvf')
+  if Tb::Cmd.opt_tar_tvf_hash.any? {|alg| !Tb::Cmd::TAR_HASH_ALGORITHMS[alg] }
+    STDERR.puts "Unexpected hash algorithm: #{Tb::Cmd.opt_tar_tvf_hash.reject {|alg| Tb::Cmd::TAR_HASH_ALGORITHMS[alg] }.join(",")}"
+    exit false
+  end
   argv = ['-'] if argv.empty?
   er = Tb::Enumerator.new {|y|
     if Tb::Cmd.opt_tar_tvf_l == 0
@@ -412,6 +467,7 @@ def (Tb::Cmd).main_tar_tvf(argv)
     else
       header = Tb::Cmd::TAR_CSV_LONG_HEADER
     end
+    header += Tb::Cmd.opt_tar_tvf_hash
     y.set_header header
     argv.each {|filename|
       tar_tvf_open_with(filename) {|f|
@@ -436,6 +492,7 @@ def (Tb::Cmd).main_tar_tvf(argv)
           formatted["tar_typeflag"] = h[:typeflag]
           formatted["tar_magic"] = h[:magic]
           formatted["tar_version"] = h[:version]
+          Tb::Cmd.opt_tar_tvf_hash.each {|alg| formatted[alg] = h[alg] }
           y.yield Hash[header.map {|f2| [f2, formatted[f2]] }]
         }
       }

data/lib/tb/cmd_unmelt.rb

@@ -32,6 +32,7 @@ Tb::Cmd.default_option[:opt_unmelt_recnum] = nil
 Tb::Cmd.default_option[:opt_unmelt_keys] = []
 Tb::Cmd.default_option[:opt_unmelt_variable_field] = 'variable'
 Tb::Cmd.default_option[:opt_unmelt_value_field] = 'value'
+Tb::Cmd.default_option[:opt_unmelt_missing_value] = nil
 
 def (Tb::Cmd).op_unmelt
   op = OptionParser.new
@@ -51,6 +52,9 @@ def (Tb::Cmd).op_unmelt
   op.def_option('--value-field FIELD', 'value field. (default: value)') {|field|
     Tb::Cmd.opt_unmelt_value_field = field
   }
+  op.def_option('--missing-value FIELD', 'used for missing values. (default: not specified)') {|value|
+    Tb::Cmd.opt_unmelt_missing_value = value
+  }
   op
 end
 
@@ -69,6 +73,7 @@ def (Tb::Cmd).main_unmelt(argv)
     end
     key_fields += Tb::Cmd.opt_unmelt_keys
   end
+  melt_fields_hash = {}
   er = Tb::Enumerator.new {|y|
     creader.chunk {|pairs|
       keys = {}
@@ -92,6 +97,7 @@ def (Tb::Cmd).main_unmelt(argv)
       pairs_ary.each {|pairs|
         var = pairs[Tb::Cmd.opt_unmelt_variable_field]
         val = pairs[Tb::Cmd.opt_unmelt_value_field]
+        melt_fields_hash[var] = true
         if rec.has_key? var
           y.yield rec
           rec = keys.dup
@@ -101,6 +107,20 @@ def (Tb::Cmd).main_unmelt(argv)
       y.yield rec
     }
   }
-  output_tbenum(er)
+  if !Tb::Cmd.opt_unmelt_missing_value
+    er2 = er
+  else
+    er2 = Tb::Enumerator.new {|y|
+      er.to_fileenumerator.with_header {|header|
+        y.set_header header
+      }.each {|pairs|
+        melt_fields_hash.each_key {|f|
+          pairs[f] ||= Tb::Cmd.opt_unmelt_missing_value
+        }
+        y.yield pairs
+      }
+    }
+  end
+  output_tbenum(er2)
 end
 

data/lib/tb/cmdtop.rb

@@ -32,6 +32,7 @@ require 'pathname'
 require 'etc'
 require 'time'
 require 'enumerator'
+require 'digest'
 require 'tb/pager'
 require 'tb/cmdutil'
 require 'tb/cmd_help'

data/test/test_cmd_tar_tvf.rb

@@ -278,4 +278,30 @@ class TestTbCmdTarTvf < Test::Unit::TestCase
     }
   end
 
+  def test_hash_short
+    str = 'bar'
+    open('foo', 'w') {|f| f.print str }
+    assert(system('tar cf foo.tar foo'))
+    %w[MD5 SHA1 SHA256 SHA384 SHA512].each {|cname|
+      alg = cname.downcase
+      Tb::Cmd.main_tar_tvf(['-o', o='o.csv', "--hash=#{alg}", 'foo.tar'])
+      result = File.read(o)
+      assert_equal(2, result.count("\n"), "hash algorithm: #{alg}")
+      assert_match(/,#{Regexp.escape Digest.const_get(cname).hexdigest(str)}$/, result, "hash algorithm: #{alg}")
+    }
+  end
+
+  def test_hash_long
+    str = 'bx' * 8000
+    open('foo', 'w') {|f| f.print str }
+    assert(system('tar cf foo.tar foo'))
+    %w[MD5 SHA1 SHA256 SHA384 SHA512].each {|cname|
+      alg = cname.downcase
+      Tb::Cmd.main_tar_tvf(['-o', o='o.csv', "--hash=#{alg}", 'foo.tar'])
+      result = File.read(o)
+      assert_equal(2, result.count("\n"), "hash algorithm: #{alg}")
+      assert_match(/,#{Regexp.escape Digest.const_get(cname).hexdigest(str)}$/, result, "hash algorithm: #{alg}")
+    }
+  end
+
 end

data/test/test_cmd_unmelt.rb

@@ -154,4 +154,18 @@ class TestTbCmdUnmelt < Test::Unit::TestCase
     End
   end
 
+  def test_missing_value
+    File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
+      a,variable,value
+      0,x,2
+      1,y,3
+    End
+    Tb::Cmd.main_unmelt(['-o', o="o.csv", '--missing-value=9', i])
+    assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
+      a,x,y
+      0,2,9
+      1,9,3
+    End
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tb
 version: !ruby/object:Gem::Version
-  version: '0.5'
+  version: '0.6'
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-29 00:00:00.000000000 Z
+date: 2012-04-29 00:00:00.000000000 Z
 dependencies: []
 description: ! 'tb is a manipulation tool for table: CSV, TSV, JSON, etc.