tb 0.4 → 0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/README CHANGED
@@ -3,6 +3,7 @@
3
3
  tb provides a command and a library for manipulating tables:
4
4
  Unix filter like operations (grep, sort, cat, cut, ls, etc.),
5
5
  SQL like operations (join, group, etc.),
6
+ other table operations (gsub, rename, cross, melt, unmelt, etc.),
6
7
  information extractions (git-log, svn-log, tar-tvf),
7
8
  and more.
8
9
 
@@ -78,6 +79,7 @@ Following example searches languages which name contains a non-alphabet characte
78
79
  F#,2002
79
80
 
80
81
  "grep" subcommand can take Ruby expression, instead of a regexp.
82
+ The variable, "_", contains a hash which represents a record.
81
83
 
82
84
  % tb grep --ruby '(1990..1999).include?(_["year"].to_i)' sample/langs.csv
83
85
  language,year
@@ -145,17 +147,21 @@ There are more subcommands.
145
147
  tb join [OPTS] [TABLE1 TABLE2 ...]
146
148
  tb consecutive [OPTS] [TABLE ...]
147
149
  tb group [OPTS] KEY-FIELD1,... [TABLE ...]
148
- tb cross [OPTS] HKEY-FIELD1,... VKEY-FIELD1,... [TABLE ...]
150
+ tb cross [OPTS] VKEY-FIELD1,... HKEY-FIELD1,... [TABLE ...]
151
+ tb melt KEY-FIELDS-LIST [OPTS] [TABLE ...]
152
+ tb nest [OPTS] NEWFIELD,OLDFIELD1,OLDFIELD2,... [TABLE ...]
153
+ tb unnest [OPTS] FIELD [TABLE ...]
149
154
  tb shape [OPTS] [TABLE ...]
150
155
  tb mheader [OPTS] [TABLE]
151
156
  tb crop [OPTS] [TABLE ...]
152
157
  tb ls [OPTS] [FILE ...]
158
+ tb tar-tvf [OPTS] [TAR-FILE ...]
153
159
  tb svn-log [OPTS] -- [SVN-LOG-ARGS]
160
+ tb git-log [OPTS] [GIT-DIR ...]
154
161
 
155
- == Command Line Tool
156
-
157
- tb command has many subcommands.
162
+ tb help -s shows one line summary of the subcommands.
158
163
 
164
+ % tb help -s
159
165
  help : Show help message of tb command.
160
166
  to-csv : Convert a table to CSV (Comma Separated Value).
161
167
  to-tsv : Convert a table to TSV (Tab Separated Value).
@@ -173,12 +179,17 @@ tb command has many subcommands.
173
179
  join : Concatenate tables horizontally as left/right/full natural join.
174
180
  consecutive : Concatenate consecutive rows.
175
181
  group : Group and aggregate rows.
176
- cross : Create a contingency table.
182
+ cross : Create a cross table. (a.k.a contingency table, pivot table)
183
+ melt : split value fields into records.
184
+ nest : Nest fields.
185
+ unnest : Unnest a field.
177
186
  shape : Show table size.
178
187
  mheader : Collapse multi rows header.
179
188
  crop : Extract rectangle in a table.
180
189
  ls : List directory entries as a table.
190
+ tar-tvf : Show the file listing of tar file.
181
191
  svn-log : Show the SVN log as a table.
192
+ git-log : Show the GIT log as a table.
182
193
 
183
194
  == Install
184
195
 
@@ -82,14 +82,16 @@ def (Tb::Cmd).git_log_with_git_log(dir)
82
82
  'log',
83
83
  "--pretty=#{Tb::Cmd::GIT_LOG_PRETTY_FORMAT}",
84
84
  '--decorate=full',
85
- '--raw',
86
- '--abbrev=40',
87
- '.',
88
- {:chdir=>dir}
85
+ '--raw',
86
+ '--numstat',
87
+ '--abbrev=40',
88
+ '.',
89
+ {:chdir=>dir}
89
90
  ]
90
91
  $stderr.puts "git command line: #{command.inspect}" if 1 <= Tb::Cmd.opt_debug
91
92
  if Tb::Cmd.opt_git_log_debug_output
92
- command.last[:out] = Tb::Cmd.opt_git_log_debug_output
93
+ # File.realdirpath is required before Ruby 2.0.
94
+ command.last[:out] = File.realdirpath(Tb::Cmd.opt_git_log_debug_output)
93
95
  system(*command)
94
96
  File.open(Tb::Cmd.opt_git_log_debug_output) {|f|
95
97
  yield f
@@ -131,22 +133,34 @@ end
131
133
 
132
134
  def (Tb::Cmd).git_log_parse_commit(commit_info, files)
133
135
  commit_info = commit_info.split(/\n(?=[a-z])/)
134
- Tb.csv_stream_output(files_csv="") {|gen|
135
- gen << %w[mode1 mode2 hash1 hash2 status filename]
136
- files.split(/\n/).each {|file_line|
137
- if /\A:(\d+) (\d+) ([0-9a-f]+) ([0-9a-f]+) (\S+)\t(.+)\z/ !~ file_line
138
- warn "unexpected git-log output: #{file_line.inspect}"
139
- next
140
- end
136
+ files_raw = {}
137
+ files_numstat = {}
138
+ files.split(/\n/).each {|file_line|
139
+ if /\A:(\d+) (\d+) ([0-9a-f]+) ([0-9a-f]+) (\S+)\t(.+)\z/ =~ file_line
141
140
  mode1, mode2, hash1, hash2, status, filename = $1, $2, $3, $4, $5, $6
142
141
  filename = git_log_unescape_filename(filename)
143
- gen << [mode1, mode2, hash1, hash2, status, filename]
142
+ files_raw[filename] = [mode1, mode2, hash1, hash2, status]
143
+ elsif /\A(\d+|-)\t(\d+|-)\t(.+)\z/ =~ file_line
144
+ add, del, filename = $1, $2, $3
145
+ add = add == '-' ? nil : add.to_i
146
+ del = del == '-' ? nil : del.to_i
147
+ filename = git_log_unescape_filename(filename)
148
+ files_numstat[filename] = [add, del]
149
+ else
150
+ warn "unexpected git-log output (raw/numstat): #{file_line.inspect}"
151
+ end
152
+ }
153
+ Tb.csv_stream_output(files_csv="") {|gen|
154
+ gen << %w[mode1 mode2 hash1 hash2 add del status filename]
155
+ files_raw.each {|filename, (mode1, mode2, hash1, hash2, status)|
156
+ add, del = files_numstat[filename]
157
+ gen << [mode1, mode2, hash1, hash2, add, del, status, filename]
144
158
  }
145
159
  }
146
160
  h = {}
147
161
  commit_info.each {|s|
148
162
  if /:/ !~ s
149
- warn "unexpected git-log output: #{s.inspect}"
163
+ warn "unexpected git-log output (header:value): #{s.inspect}"
150
164
  next
151
165
  end
152
166
  k = $`
@@ -171,7 +185,7 @@ def (Tb::Cmd).git_log_each_commit(f)
171
185
  chunk.chomp!("\x01commit-separator\x01\n")
172
186
  next if chunk.empty? # beginning of the output
173
187
  if /\nend-commit\n/ !~ chunk
174
- warn "unexpected git-log output: #{chunk.inspect}"
188
+ warn "unexpected git-log output (end-commit): #{chunk.inspect}"
175
189
  next
176
190
  end
177
191
  commit_info, files = $`, $'
@@ -0,0 +1,105 @@
1
+ # Copyright (C) 2012 Tanaka Akira <akr@fsij.org>
2
+ #
3
+ # Redistribution and use in source and binary forms, with or without
4
+ # modification, are permitted provided that the following conditions
5
+ # are met:
6
+ #
7
+ # 1. Redistributions of source code must retain the above copyright
8
+ # notice, this list of conditions and the following disclaimer.
9
+ # 2. Redistributions in binary form must reproduce the above
10
+ # copyright notice, this list of conditions and the following
11
+ # disclaimer in the documentation and/or other materials provided
12
+ # with the distribution.
13
+ # 3. The name of the author may not be used to endorse or promote
14
+ # products derived from this software without specific prior
15
+ # written permission.
16
+ #
17
+ # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
18
+ # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
19
+ # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20
+ # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
21
+ # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22
+ # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
23
+ # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24
+ # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
25
+ # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
26
+ # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
27
+ # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
+
29
+ Tb::Cmd.subcommands << 'melt'
30
+
31
+ Tb::Cmd.default_option[:opt_melt_recnum] = nil
32
+ Tb::Cmd.default_option[:opt_melt_regexps] = []
33
+ Tb::Cmd.default_option[:opt_melt_list] = []
34
+ Tb::Cmd.default_option[:opt_melt_variable_field] = 'variable'
35
+ Tb::Cmd.default_option[:opt_melt_value_field] = 'value'
36
+
37
+ def (Tb::Cmd).op_melt
38
+ op = OptionParser.new
39
+ op.banner = "Usage: tb melt KEY-FIELDS-LIST [OPTS] [TABLE ...]\n" +
40
+ "split value fields into records."
41
+ define_common_option(op, "hNo", "--no-pager")
42
+ op.def_option('--recnum[=FIELD]',
43
+ 'add recnum field (default don\'t add)') {|field|
44
+ Tb::Cmd.opt_melt_recnum = field || 'recnum'
45
+ }
46
+ op.def_option('-R REGEXP',
47
+ '--melt-regexp REGEXP',
48
+ 'regexp for melt fields') {|regexp|
49
+ Tb::Cmd.opt_melt_regexps << Regexp.compile(regexp)
50
+ }
51
+ op.def_option('--melt-fields FIELD,...',
52
+ 'list of melt fields') {|fields|
53
+ Tb::Cmd.opt_melt_list.concat split_field_list_argument(fields)
54
+ }
55
+ op.def_option('--variable-field FIELD', 'variable field. (default: variable)') {|field|
56
+ Tb::Cmd.opt_melt_variable_field = field
57
+ }
58
+ op.def_option('--value-field FIELD', 'value field. (default: value)') {|field|
59
+ Tb::Cmd.opt_melt_value_field = field
60
+ }
61
+ op
62
+ end
63
+
64
+ def (Tb::Cmd).main_melt(argv)
65
+ op_melt.parse!(argv)
66
+ exit_if_help('melt')
67
+ err('no key-fields given.') if argv.empty?
68
+ key_fields = split_field_list_argument(argv.shift)
69
+ key_fields_hash = Hash[key_fields.map {|f| [f, true] }]
70
+ if Tb::Cmd.opt_melt_regexps.empty? && Tb::Cmd.opt_melt_list.empty?
71
+ melt_fields_pattern = //
72
+ else
73
+ list = Tb::Cmd.opt_melt_list + Tb::Cmd.opt_melt_regexps
74
+ melt_fields_pattern = /\A#{Regexp.union(list)}\z/
75
+ end
76
+ argv = ['-'] if argv.empty?
77
+ creader = Tb::CatReader.open(argv, Tb::Cmd.opt_N)
78
+ er = Tb::Enumerator.new {|y|
79
+ header = []
80
+ header << Tb::Cmd.opt_melt_recnum if Tb::Cmd.opt_melt_recnum
81
+ header.concat key_fields
82
+ header << Tb::Cmd.opt_melt_variable_field
83
+ header << Tb::Cmd.opt_melt_value_field
84
+ y.set_header header
85
+ creader.each_with_index {|pairs, i|
86
+ recnum = i + 1
87
+ h0 = {}
88
+ h0[Tb::Cmd.opt_melt_recnum] = nil if Tb::Cmd.opt_melt_recnum
89
+ key_fields.each {|kf|
90
+ h0[kf] = pairs[kf]
91
+ }
92
+ pairs.each {|f, v|
93
+ next if key_fields_hash[f]
94
+ next if melt_fields_pattern !~ f
95
+ h = h0.dup
96
+ h[Tb::Cmd.opt_melt_recnum] = recnum if Tb::Cmd.opt_melt_recnum
97
+ h[Tb::Cmd.opt_melt_variable_field] = f
98
+ h[Tb::Cmd.opt_melt_value_field] = v
99
+ y.yield h
100
+ }
101
+ }
102
+ }
103
+ output_tbenum(er)
104
+ end
105
+
@@ -97,14 +97,23 @@ class Tb::Cmd::SVNLOGListener
97
97
  end
98
98
  @y.set_header @header
99
99
  end
100
+ assoc = @log.to_a.reject {|f, v| !%w[rev author date msg].include?(f) }
101
+ if !assoc.assoc('author')
102
+ assoc << ['author', '(no author)']
103
+ end
104
+ if !assoc.assoc('date')
105
+ assoc << ['date', '(no date)']
106
+ end
107
+ if !assoc.assoc('msg')
108
+ assoc << ['msg', '']
109
+ end
100
110
  if @log['paths']
101
111
  @log['paths'].each {|h|
102
- assoc = @log.to_a.reject {|f, v| !%w[rev author date msg].include?(f) }
103
- assoc += h.to_a.reject {|f, v| !%w[kind action path].include?(f) }
104
- @y.yield Hash[assoc]
112
+ assoc2 = assoc.dup
113
+ assoc2.concat(h.to_a.reject {|f, v| !%w[kind action path].include?(f) })
114
+ @y.yield Hash[assoc2]
105
115
  }
106
116
  else
107
- assoc = @log.to_a.reject {|f, v| !%w[rev author date msg].include?(f) }
108
117
  @y.yield Hash[assoc]
109
118
  end
110
119
  @log = nil
@@ -0,0 +1,106 @@
1
+ # Copyright (C) 2012 Tanaka Akira <akr@fsij.org>
2
+ #
3
+ # Redistribution and use in source and binary forms, with or without
4
+ # modification, are permitted provided that the following conditions
5
+ # are met:
6
+ #
7
+ # 1. Redistributions of source code must retain the above copyright
8
+ # notice, this list of conditions and the following disclaimer.
9
+ # 2. Redistributions in binary form must reproduce the above
10
+ # copyright notice, this list of conditions and the following
11
+ # disclaimer in the documentation and/or other materials provided
12
+ # with the distribution.
13
+ # 3. The name of the author may not be used to endorse or promote
14
+ # products derived from this software without specific prior
15
+ # written permission.
16
+ #
17
+ # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
18
+ # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
19
+ # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20
+ # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
21
+ # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22
+ # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
23
+ # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24
+ # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
25
+ # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
26
+ # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
27
+ # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
+
29
+ Tb::Cmd.subcommands << 'unmelt'
30
+
31
+ Tb::Cmd.default_option[:opt_unmelt_recnum] = nil
32
+ Tb::Cmd.default_option[:opt_unmelt_keys] = []
33
+ Tb::Cmd.default_option[:opt_unmelt_variable_field] = 'variable'
34
+ Tb::Cmd.default_option[:opt_unmelt_value_field] = 'value'
35
+
36
+ def (Tb::Cmd).op_unmelt
37
+ op = OptionParser.new
38
+ op.banner = "Usage: tb unmelt [OPTS] [TABLE ...]\n" +
39
+ "merge melted records into a record."
40
+ define_common_option(op, "hNo", "--no-pager")
41
+ op.def_option('--recnum[=FIELD]',
42
+ 'use FIELD as an additional key and remove it from the result. (default: not specified)') {|field|
43
+ Tb::Cmd.opt_unmelt_recnum = field || 'recnum'
44
+ }
45
+ op.def_option('--keys FIELD,...', 'key fields. (default: all fields except variable and value)') {|fields|
46
+ Tb::Cmd.opt_unmelt_keys.concat split_field_list_argument(fields)
47
+ }
48
+ op.def_option('--variable-field FIELD', 'variable field. (default: variable)') {|field|
49
+ Tb::Cmd.opt_unmelt_variable_field = field
50
+ }
51
+ op.def_option('--value-field FIELD', 'value field. (default: value)') {|field|
52
+ Tb::Cmd.opt_unmelt_value_field = field
53
+ }
54
+ op
55
+ end
56
+
57
+ def (Tb::Cmd).main_unmelt(argv)
58
+ op_unmelt.parse!(argv)
59
+ exit_if_help('unmelt')
60
+ argv = ['-'] if argv.empty?
61
+ creader = Tb::CatReader.open(argv, Tb::Cmd.opt_N)
62
+ if Tb::Cmd.opt_unmelt_keys.empty?
63
+ key_fields = nil
64
+ else
65
+ if Tb::Cmd.opt_unmelt_recnum
66
+ key_fields = [Tb::Cmd.opt_unmelt_recnum]
67
+ else
68
+ key_fields = []
69
+ end
70
+ key_fields += Tb::Cmd.opt_unmelt_keys
71
+ end
72
+ er = Tb::Enumerator.new {|y|
73
+ creader.chunk {|pairs|
74
+ keys = {}
75
+ if key_fields
76
+ key_fields.each {|k|
77
+ keys[k] = pairs[k]
78
+ }
79
+ else
80
+ pairs.each_key {|k|
81
+ next if k == Tb::Cmd.opt_unmelt_variable_field ||
82
+ k == Tb::Cmd.opt_unmelt_value_field
83
+ keys[k] = pairs[k]
84
+ }
85
+ end
86
+ keys
87
+ }.each {|keys, pairs_ary|
88
+ if Tb::Cmd.opt_unmelt_recnum
89
+ keys.delete Tb::Cmd.opt_unmelt_recnum
90
+ end
91
+ rec = keys.dup
92
+ pairs_ary.each {|pairs|
93
+ var = pairs[Tb::Cmd.opt_unmelt_variable_field]
94
+ val = pairs[Tb::Cmd.opt_unmelt_value_field]
95
+ if rec.has_key? var
96
+ y.yield rec
97
+ rec = keys.dup
98
+ end
99
+ rec[var] = val
100
+ }
101
+ y.yield rec
102
+ }
103
+ }
104
+ output_tbenum(er)
105
+ end
106
+
@@ -52,6 +52,8 @@ require 'tb/cmd_join'
52
52
  require 'tb/cmd_consecutive'
53
53
  require 'tb/cmd_group'
54
54
  require 'tb/cmd_cross'
55
+ require 'tb/cmd_melt'
56
+ require 'tb/cmd_unmelt'
55
57
  require 'tb/cmd_nest'
56
58
  require 'tb/cmd_unnest'
57
59
  require 'tb/cmd_shape'
@@ -55,6 +55,7 @@ class Tb::Enumerator < Enumerator
55
55
  def self.new(&enumerator_proc)
56
56
  super() {|y|
57
57
  header_proc = Thread.current[:tb_enumerator_header_proc]
58
+ Thread.current[:tb_enumerator_header_proc] = nil
58
59
  ty = Tb::Yielder.new(header_proc, y)
59
60
  enumerator_proc.call(ty)
60
61
  if !ty.header_proc_called
@@ -77,24 +77,24 @@ module Tb::Func
77
77
  def Count.aggregate(count) count end
78
78
 
79
79
  module Sum; end
80
- def Sum.start(value) Tb::Func.smart_numerize(value) end
80
+ def Sum.start(value) value.nil? ? 0 : Tb::Func.smart_numerize(value) end
81
81
  def Sum.call(v1, v2) v1 + v2 end
82
82
  def Sum.aggregate(sum) sum end
83
83
 
84
84
  module Min; end
85
- def Min.start(value) [value, Tb::Func.smart_cmp_value(value)] end
86
- def Min.call(vc1, vc2) (vc1.last <=> vc2.last) <= 0 ? vc1 : vc2 end
87
- def Min.aggregate(vc) vc.first end
85
+ def Min.start(value) value.nil? ? nil : [value, Tb::Func.smart_cmp_value(value)] end
86
+ def Min.call(vc1, vc2) vc1.nil? ? vc2 : vc2.nil? ? vc1 : (vc1.last <=> vc2.last) <= 0 ? vc1 : vc2 end
87
+ def Min.aggregate(vc) vc.nil? ? nil : vc.first end
88
88
 
89
89
  module Max; end
90
- def Max.start(value) [value, Tb::Func.smart_cmp_value(value)] end
91
- def Max.call(vc1, vc2) (vc1.last <=> vc2.last) >= 0 ? vc1 : vc2 end
92
- def Max.aggregate(vc) vc.first end
90
+ def Max.start(value) value.nil? ? nil : [value, Tb::Func.smart_cmp_value(value)] end
91
+ def Max.call(vc1, vc2) vc1.nil? ? vc2 : vc2.nil? ? vc1 : (vc1.last <=> vc2.last) >= 0 ? vc1 : vc2 end
92
+ def Max.aggregate(vc) vc.nil? ? nil : vc.first end
93
93
 
94
94
  module Avg; end
95
- def Avg.start(value) [Tb::Func.smart_numerize(value), 1] end
95
+ def Avg.start(value) value.nil? ? [0, 0] : [Tb::Func.smart_numerize(value), 1] end
96
96
  def Avg.call(v1, v2) [v1[0] + v2[0], v1[1] + v2[1]] end
97
- def Avg.aggregate(sum_count) sum_count[0] / sum_count[1].to_f end
97
+ def Avg.aggregate(sum_count) sum_count[1] == 0 ? nil : sum_count[0] / sum_count[1].to_f end
98
98
 
99
99
  module First; end
100
100
  def First.start(value) value end
@@ -107,12 +107,12 @@ module Tb::Func
107
107
  def Last.aggregate(value) value end
108
108
 
109
109
  module Values; end
110
- def Values.start(value) [value] end
110
+ def Values.start(value) value.nil? ? [] : [value] end
111
111
  def Values.call(a1, a2) a1.concat a2 end
112
112
  def Values.aggregate(ary) ary.join(',') end
113
113
 
114
114
  module UniqueValues; end
115
- def UniqueValues.start(value) {value => true} end
115
+ def UniqueValues.start(value) value.nil? ? {} : {value => true} end
116
116
  def UniqueValues.call(h1, h2) h1.update h2 end
117
117
  def UniqueValues.aggregate(hash) hash.keys.join(',') end
118
118
 
@@ -159,4 +159,32 @@ class TestTbCmdGitLog < Test::Unit::TestCase
159
159
  assert(!log.empty?)
160
160
  end
161
161
 
162
+ def test_binary
163
+ system("git init -q")
164
+ File.open("foo", "w") {|f| f.print "\0\xff" }
165
+ system("git add foo")
166
+ system("git commit -q -m msg foo")
167
+ Tb::Cmd.main_git_log(['-o', o="o.csv"])
168
+ result = File.read(o)
169
+ tb = Tb.parse_csv(result)
170
+ assert_equal(1, tb.size)
171
+ assert_match(/,,,A,foo\n/, tb.get_record(0)["files"])
172
+ end
173
+
174
+ def test_subdir
175
+ system("git init -q")
176
+ File.open("foo", "w") {|f| f.print "foo" }
177
+ system("git add foo")
178
+ system("git commit -q -m msg foo")
179
+ Dir.mkdir("bar")
180
+ File.open("bar/baz", "w") {|f| f.print "baz" }
181
+ system("git add bar")
182
+ system("git commit -q -m msg bar")
183
+ Tb::Cmd.main_git_log(['-o', o="o.csv", "bar"])
184
+ result = File.read(o)
185
+ tb = Tb.parse_csv(result)
186
+ assert_equal(1, tb.size)
187
+ assert_not_match(/foo\n/, tb.get_record(0)["files"])
188
+ end
189
+
162
190
  end
@@ -178,4 +178,121 @@ class TestTbCmdGroup < Test::Unit::TestCase
178
178
  assert(!exc.success?)
179
179
  end
180
180
 
181
+ def test_sum_nil
182
+ File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
183
+ k,v
184
+ a,2
185
+ b,3
186
+ a,
187
+ b,4
188
+ End
189
+ Tb::Cmd.main_group(['-o', o="o.csv", 'k', '-a', 'sum(v)', i])
190
+ assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
191
+ k,sum(v)
192
+ a,2
193
+ b,7
194
+ End
195
+ end
196
+
197
+ def test_avg_nil
198
+ File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
199
+ k,v
200
+ a,2
201
+ b,3
202
+ c,
203
+ a,
204
+ b,4
205
+ c,
206
+ End
207
+ Tb::Cmd.main_group(['-o', o="o.csv", 'k', '-a', 'avg(v)', i])
208
+ assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
209
+ k,avg(v)
210
+ a,2.0
211
+ b,3.5
212
+ c,
213
+ End
214
+ end
215
+
216
+ def test_min_nil
217
+ File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
218
+ k,v
219
+ a,9
220
+ b,9
221
+ c,
222
+ a,2
223
+ b,
224
+ c,
225
+ a,
226
+ b,4
227
+ c,
228
+ End
229
+ Tb::Cmd.main_group(['-o', o="o.csv", 'k', '-a', 'min(v)', i])
230
+ assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
231
+ k,min(v)
232
+ a,2
233
+ b,4
234
+ c,
235
+ End
236
+ end
237
+
238
+ def test_max_nil
239
+ File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
240
+ k,v
241
+ a,1
242
+ b,1
243
+ c,
244
+ a,2
245
+ b,
246
+ c,
247
+ a,
248
+ b,4
249
+ c,
250
+ End
251
+ Tb::Cmd.main_group(['-o', o="o.csv", 'k', '-a', 'max(v)', i])
252
+ assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
253
+ k,max(v)
254
+ a,2
255
+ b,4
256
+ c,
257
+ End
258
+ end
259
+
260
+ def test_values_nil
261
+ File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
262
+ k,v
263
+ a,2
264
+ b,
265
+ c,
266
+ a,
267
+ b,4
268
+ c,
269
+ End
270
+ Tb::Cmd.main_group(['-o', o="o.csv", 'k', '-a', 'values(v)', i])
271
+ assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
272
+ k,values(v)
273
+ a,2
274
+ b,4
275
+ c,""
276
+ End
277
+ end
278
+
279
+ def test_uniquevalues_nil
280
+ File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
281
+ k,v
282
+ a,2
283
+ b,
284
+ c,
285
+ a,
286
+ b,4
287
+ c,
288
+ End
289
+ Tb::Cmd.main_group(['-o', o="o.csv", 'k', '-a', 'uniquevalues(v)', i])
290
+ assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
291
+ k,uniquevalues(v)
292
+ a,2
293
+ b,4
294
+ c,""
295
+ End
296
+ end
297
+
181
298
  end
@@ -0,0 +1,172 @@
1
+ require 'test/unit'
2
+ require 'tb/cmdtop'
3
+ require 'tmpdir'
4
+
5
+ class TestTbCmdMelt < Test::Unit::TestCase
6
+ def setup
7
+ Tb::Cmd.reset_option
8
+ @curdir = Dir.pwd
9
+ @tmpdir = Dir.mktmpdir
10
+ Dir.chdir @tmpdir
11
+ end
12
+ def teardown
13
+ Tb::Cmd.reset_option
14
+ Dir.chdir @curdir
15
+ FileUtils.rmtree @tmpdir
16
+ end
17
+
18
+ def test_basic
19
+ File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
20
+ a,b,c,d
21
+ 0,1,2,3
22
+ 4,5,6,7
23
+ 8,9,a,b
24
+ c,d,e,f
25
+ End
26
+ Tb::Cmd.main_melt(['-o', o="o.csv", 'a,b', i])
27
+ assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
28
+ a,b,variable,value
29
+ 0,1,c,2
30
+ 0,1,d,3
31
+ 4,5,c,6
32
+ 4,5,d,7
33
+ 8,9,c,a
34
+ 8,9,d,b
35
+ c,d,c,e
36
+ c,d,d,f
37
+ End
38
+ end
39
+
40
+ def test_recnum
41
+ File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
42
+ a,b,c,d
43
+ 0,1,2,3
44
+ 4,5,6,7
45
+ 8,9,a,b
46
+ c,d,e,f
47
+ End
48
+ Tb::Cmd.main_melt(['-o', o="o.csv", 'a,b', '--recnum', i])
49
+ assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
50
+ recnum,a,b,variable,value
51
+ 1,0,1,c,2
52
+ 1,0,1,d,3
53
+ 2,4,5,c,6
54
+ 2,4,5,d,7
55
+ 3,8,9,c,a
56
+ 3,8,9,d,b
57
+ 4,c,d,c,e
58
+ 4,c,d,d,f
59
+ End
60
+ end
61
+
62
+ def test_recnum_value
63
+ File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
64
+ a,b,c,d
65
+ 0,1,2,3
66
+ 4,5,6,7
67
+ 8,9,a,b
68
+ c,d,e,f
69
+ End
70
+ Tb::Cmd.main_melt(['-o', o="o.csv", 'a,b', '--recnum=rec', i])
71
+ assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
72
+ rec,a,b,variable,value
73
+ 1,0,1,c,2
74
+ 1,0,1,d,3
75
+ 2,4,5,c,6
76
+ 2,4,5,d,7
77
+ 3,8,9,c,a
78
+ 3,8,9,d,b
79
+ 4,c,d,c,e
80
+ 4,c,d,d,f
81
+ End
82
+ end
83
+
84
+ def test_variable_field
85
+ File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
86
+ a,b,c,d
87
+ 0,1,2,3
88
+ 4,5,6,7
89
+ 8,9,a,b
90
+ c,d,e,f
91
+ End
92
+ Tb::Cmd.main_melt(['-o', o="o.csv", 'a,b', '--variable-field=foo', i])
93
+ assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
94
+ a,b,foo,value
95
+ 0,1,c,2
96
+ 0,1,d,3
97
+ 4,5,c,6
98
+ 4,5,d,7
99
+ 8,9,c,a
100
+ 8,9,d,b
101
+ c,d,c,e
102
+ c,d,d,f
103
+ End
104
+ end
105
+
106
+ def test_value_field
107
+ File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
108
+ a,b,c,d
109
+ 0,1,2,3
110
+ 4,5,6,7
111
+ 8,9,a,b
112
+ c,d,e,f
113
+ End
114
+ Tb::Cmd.main_melt(['-o', o="o.csv", 'a,b', '--value-field=bar', i])
115
+ assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
116
+ a,b,variable,bar
117
+ 0,1,c,2
118
+ 0,1,d,3
119
+ 4,5,c,6
120
+ 4,5,d,7
121
+ 8,9,c,a
122
+ 8,9,d,b
123
+ c,d,c,e
124
+ c,d,d,f
125
+ End
126
+ end
127
+
128
+ def test_melt_regexp
129
+ File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
130
+ a,b,c,d
131
+ 0,1,2,3
132
+ 4,5,6,7
133
+ 8,9,a,b
134
+ c,d,e,f
135
+ End
136
+ Tb::Cmd.main_melt(['-o', o="o.csv", 'a', '--melt-regexp=[bd]', i])
137
+ assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
138
+ a,variable,value
139
+ 0,b,1
140
+ 0,d,3
141
+ 4,b,5
142
+ 4,d,7
143
+ 8,b,9
144
+ 8,d,b
145
+ c,b,d
146
+ c,d,f
147
+ End
148
+ end
149
+
150
+ def test_melt_list
151
+ File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
152
+ a,b,c,d
153
+ 0,1,2,3
154
+ 4,5,6,7
155
+ 8,9,a,b
156
+ c,d,e,f
157
+ End
158
+ Tb::Cmd.main_melt(['-o', o="o.csv", 'a', '--melt-fields=b,d', i])
159
+ assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
160
+ a,variable,value
161
+ 0,b,1
162
+ 0,d,3
163
+ 4,b,5
164
+ 4,d,7
165
+ 8,b,9
166
+ 8,d,b
167
+ c,b,d
168
+ c,d,f
169
+ End
170
+ end
171
+
172
+ end
@@ -84,4 +84,25 @@ class TestTbCmdSvnLog < Test::Unit::TestCase
84
84
  assert_equal(1, tb.size)
85
85
  assert_match(/baz/, tb.get_record(0)["msg"])
86
86
  end
87
+
88
+ def test_no_props
89
+ system("svnadmin create repo")
90
+ File.open("repo/hooks/pre-revprop-change", "w", 0755) {|f| f.print "#!/bin/sh\nexit 0\0" }
91
+ system("svn co -q file://#{@tmpdir}/repo .")
92
+ File.open("foo", "w") {|f| f.puts "bar" }
93
+ system("svn add -q foo")
94
+ system("svn commit -q -m baz foo")
95
+ system("svn update -q") # update the revision of the directory.
96
+ system("svn propdel -q svn:author --revprop -r 1 .")
97
+ system("svn propdel -q svn:date --revprop -r 1 .")
98
+ system("svn propdel -q svn:log --revprop -r 1 .")
99
+ ###
100
+ Tb::Cmd.main_svn_log(['-o', o="o.csv"])
101
+ result = File.read(o)
102
+ tb = Tb.parse_csv(result)
103
+ assert_equal(1, tb.size)
104
+ assert_equal('(no author)', tb.get_record(0)["author"])
105
+ assert_equal('(no date)', tb.get_record(0)["date"])
106
+ assert_equal('', tb.get_record(0)["msg"])
107
+ end
87
108
  end
@@ -0,0 +1,157 @@
1
+ require 'test/unit'
2
+ require 'tb/cmdtop'
3
+ require 'tmpdir'
4
+
5
+ class TestTbCmdUnmelt < Test::Unit::TestCase
6
+ def setup
7
+ Tb::Cmd.reset_option
8
+ @curdir = Dir.pwd
9
+ @tmpdir = Dir.mktmpdir
10
+ Dir.chdir @tmpdir
11
+ end
12
+ def teardown
13
+ Tb::Cmd.reset_option
14
+ Dir.chdir @curdir
15
+ FileUtils.rmtree @tmpdir
16
+ end
17
+
18
+ def test_basic
19
+ File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
20
+ a,b,variable,value
21
+ 0,1,x,3
22
+ 0,1,y,7
23
+ 4,5,x,b
24
+ 4,5,y,f
25
+ End
26
+ Tb::Cmd.main_unmelt(['-o', o="o.csv", i])
27
+ assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
28
+ a,b,x,y
29
+ 0,1,3,7
30
+ 4,5,b,f
31
+ End
32
+ end
33
+
34
+ def test_json
35
+ File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
36
+ a,b,variable,value
37
+ 0,1,x,3
38
+ 0,1,y,7
39
+ 4,5,x,b
40
+ 4,5,y,f
41
+ End
42
+ Tb::Cmd.main_unmelt(['-o', o="o.json", i])
43
+ assert_equal(<<-"End".gsub(/\s+/, ''), File.read(o).gsub(/\s+/, ''))
44
+ [
45
+ {"a":"0", "b":"1", "x":"3", "y":"7"},
46
+ {"a":"4", "b":"5", "x":"b", "y":"f"}
47
+ ]
48
+ End
49
+ end
50
+
51
+ def test_duplicated_variable
52
+ File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
53
+ a,b,variable,value
54
+ 0,1,x,3
55
+ 0,1,x,4
56
+ End
57
+ Tb::Cmd.main_unmelt(['-o', o="o.csv", i])
58
+ assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
59
+ a,b,x
60
+ 0,1,3
61
+ 0,1,4
62
+ End
63
+ end
64
+
65
+ def test_keys
66
+ File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
67
+ a,b,variable,value
68
+ 0,1,x,3
69
+ 0,1,y,7
70
+ 4,5,x,b
71
+ 4,5,y,f
72
+ End
73
+ Tb::Cmd.main_unmelt(['-o', o="o.csv", '--keys=a', i])
74
+ assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
75
+ a,x,y
76
+ 0,3,7
77
+ 4,b,f
78
+ End
79
+ end
80
+
81
+ def test_recnum_noneffective
82
+ File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
83
+ recnum,a,b,variable,value
84
+ 1,0,1,x,3
85
+ 1,0,1,y,7
86
+ 2,4,5,x,b
87
+ 2,4,5,y,f
88
+ End
89
+ Tb::Cmd.main_unmelt(['-o', o="o.csv", '--keys=a,b', '--recnum', i])
90
+ assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
91
+ a,b,x,y
92
+ 0,1,3,7
93
+ 4,5,b,f
94
+ End
95
+ end
96
+
97
+ def test_recnum_effective
98
+ File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
99
+ recnum,a,b,variable,value
100
+ 1,0,1,x,3
101
+ 2,0,1,y,f
102
+ End
103
+ Tb::Cmd.main_unmelt(['-o', o="o.csv", '--keys=a,b', '--recnum', i])
104
+ assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
105
+ a,b,x,y
106
+ 0,1,3
107
+ 0,1,,f
108
+ End
109
+ end
110
+
111
+ def test_recnum_value
112
+ File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
113
+ r,a,b,variable,value
114
+ 1,0,1,x,3
115
+ 2,0,1,y,f
116
+ End
117
+ Tb::Cmd.main_unmelt(['-o', o="o.csv", '--keys=a,b', '--recnum=r', i])
118
+ assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
119
+ a,b,x,y
120
+ 0,1,3
121
+ 0,1,,f
122
+ End
123
+ end
124
+
125
+ def test_variable_field
126
+ File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
127
+ a,b,foo,value
128
+ 0,1,x,3
129
+ 0,1,y,7
130
+ 4,5,x,b
131
+ 4,5,y,f
132
+ End
133
+ Tb::Cmd.main_unmelt(['-o', o="o.csv", '--variable-field=foo', i])
134
+ assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
135
+ a,b,x,y
136
+ 0,1,3,7
137
+ 4,5,b,f
138
+ End
139
+ end
140
+
141
+ def test_value_field
142
+ File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
143
+ a,b,variable,bar
144
+ 0,1,x,3
145
+ 0,1,y,7
146
+ 4,5,x,b
147
+ 4,5,y,f
148
+ End
149
+ Tb::Cmd.main_unmelt(['-o', o="o.csv", '--value-field=bar', i])
150
+ assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
151
+ a,b,x,y
152
+ 0,1,3,7
153
+ 4,5,b,f
154
+ End
155
+ end
156
+
157
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: tb
3
3
  version: !ruby/object:Gem::Version
4
- version: '0.4'
4
+ version: '0.5'
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-02-29 00:00:00.000000000 Z
12
+ date: 2012-03-29 00:00:00.000000000 Z
13
13
  dependencies: []
14
14
  description: ! 'tb is a manipulation tool for table: CSV, TSV, JSON, etc.
15
15
 
@@ -20,6 +20,8 @@ description: ! 'tb is a manipulation tool for table: CSV, TSV, JSON, etc.
20
20
 
21
21
  SQL like operations (join, group, etc.),
22
22
 
23
+ other table operations (gsub, rename, cross, melt, unmelt, etc.),
24
+
23
25
  information extractions (git-log, svn-log, tar-tvf),
24
26
 
25
27
  and more.
@@ -48,6 +50,7 @@ files:
48
50
  - lib/tb/cmd_help.rb
49
51
  - lib/tb/cmd_join.rb
50
52
  - lib/tb/cmd_ls.rb
53
+ - lib/tb/cmd_melt.rb
51
54
  - lib/tb/cmd_mheader.rb
52
55
  - lib/tb/cmd_nest.rb
53
56
  - lib/tb/cmd_newfield.rb
@@ -62,6 +65,7 @@ files:
62
65
  - lib/tb/cmd_to_pp.rb
63
66
  - lib/tb/cmd_to_tsv.rb
64
67
  - lib/tb/cmd_to_yaml.rb
68
+ - lib/tb/cmd_unmelt.rb
65
69
  - lib/tb/cmd_unnest.rb
66
70
  - lib/tb/cmdmain.rb
67
71
  - lib/tb/cmdtop.rb
@@ -109,6 +113,7 @@ files:
109
113
  - test/test_cmd_help.rb
110
114
  - test/test_cmd_join.rb
111
115
  - test/test_cmd_ls.rb
116
+ - test/test_cmd_melt.rb
112
117
  - test/test_cmd_mheader.rb
113
118
  - test/test_cmd_nest.rb
114
119
  - test/test_cmd_newfield.rb
@@ -123,6 +128,7 @@ files:
123
128
  - test/test_cmd_to_pp.rb
124
129
  - test/test_cmd_to_tsv.rb
125
130
  - test/test_cmd_to_yaml.rb
131
+ - test/test_cmd_unmelt.rb
126
132
  - test/test_cmd_unnest.rb
127
133
  - test/test_cmdtty.rb
128
134
  - test/test_cmdutil.rb
@@ -181,6 +187,7 @@ test_files:
181
187
  - test/test_cmd_help.rb
182
188
  - test/test_cmd_join.rb
183
189
  - test/test_cmd_ls.rb
190
+ - test/test_cmd_melt.rb
184
191
  - test/test_cmd_mheader.rb
185
192
  - test/test_cmd_nest.rb
186
193
  - test/test_cmd_newfield.rb
@@ -195,6 +202,7 @@ test_files:
195
202
  - test/test_cmd_to_pp.rb
196
203
  - test/test_cmd_to_tsv.rb
197
204
  - test/test_cmd_to_yaml.rb
205
+ - test/test_cmd_unmelt.rb
198
206
  - test/test_cmd_unnest.rb
199
207
  - test/test_cmdtty.rb
200
208
  - test/test_cmdutil.rb