scout-gear 7.3.0 → 8.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.vimproject +20 -9
- data/VERSION +1 -1
- data/bin/scout +6 -3
- data/lib/rbbt-scout.rb +1 -0
- data/lib/scout/cmd.rb +1 -1
- data/lib/scout/concurrent_stream.rb +26 -23
- data/lib/scout/config.rb +1 -1
- data/lib/scout/log/color.rb +4 -1
- data/lib/scout/log/progress/report.rb +1 -1
- data/lib/scout/log/progress/util.rb +58 -54
- data/lib/scout/log/progress.rb +1 -1
- data/lib/scout/log/trap.rb +107 -0
- data/lib/scout/log.rb +56 -21
- data/lib/scout/meta_extension.rb +13 -6
- data/lib/scout/misc/digest.rb +1 -1
- data/lib/scout/misc/format.rb +12 -0
- data/lib/scout/misc/insist.rb +1 -1
- data/lib/scout/misc/monitor.rb +11 -0
- data/lib/scout/misc/system.rb +10 -0
- data/lib/scout/named_array.rb +65 -3
- data/lib/scout/open/lock/lockfile.rb +587 -0
- data/lib/scout/open/lock.rb +28 -2
- data/lib/scout/open/remote.rb +4 -0
- data/lib/scout/open/stream.rb +90 -15
- data/lib/scout/open/util.rb +13 -3
- data/lib/scout/path/find.rb +9 -1
- data/lib/scout/path/util.rb +35 -0
- data/lib/scout/persist/serialize.rb +18 -5
- data/lib/scout/persist.rb +28 -12
- data/lib/scout/resource/path.rb +53 -0
- data/lib/scout/resource/produce.rb +0 -8
- data/lib/scout/resource/util.rb +2 -1
- data/lib/scout/tmpfile.rb +7 -8
- data/lib/scout/tsv/attach.rb +177 -0
- data/lib/scout/tsv/change_id.rb +40 -0
- data/lib/scout/tsv/dumper.rb +72 -46
- data/lib/scout/tsv/index.rb +69 -13
- data/lib/scout/tsv/open.rb +138 -84
- data/lib/scout/tsv/parser.rb +135 -80
- data/lib/scout/tsv/path.rb +1 -2
- data/lib/scout/tsv/persist/adapter.rb +15 -45
- data/lib/scout/tsv/persist/fix_width_table.rb +3 -0
- data/lib/scout/tsv/persist/tokyocabinet.rb +4 -1
- data/lib/scout/tsv/persist.rb +4 -0
- data/lib/scout/tsv/transformer.rb +141 -0
- data/lib/scout/tsv/traverse.rb +96 -92
- data/lib/scout/tsv/util/filter.rb +9 -0
- data/lib/scout/tsv/util/reorder.rb +81 -0
- data/lib/scout/tsv/util/select.rb +78 -33
- data/lib/scout/tsv/util/unzip.rb +86 -0
- data/lib/scout/tsv/util.rb +60 -11
- data/lib/scout/tsv.rb +26 -3
- data/lib/scout/work_queue/socket.rb +6 -1
- data/lib/scout/work_queue/worker.rb +5 -2
- data/lib/scout/work_queue.rb +15 -8
- data/lib/scout/workflow/definition.rb +21 -2
- data/lib/scout/workflow/step/dependencies.rb +24 -4
- data/lib/scout/workflow/step/info.rb +36 -5
- data/lib/scout/workflow/step/provenance.rb +8 -7
- data/lib/scout/workflow/step/status.rb +45 -0
- data/lib/scout/workflow/step.rb +100 -34
- data/lib/scout/workflow/task/inputs.rb +14 -20
- data/lib/scout/workflow/task.rb +81 -46
- data/lib/scout/workflow/usage.rb +8 -6
- data/scout-gear.gemspec +24 -20
- data/scout_commands/workflow/task +34 -7
- data/test/scout/open/test_stream.rb +60 -58
- data/test/scout/path/test_find.rb +10 -1
- data/test/scout/resource/test_produce.rb +15 -0
- data/test/scout/test_meta_extension.rb +25 -0
- data/test/scout/test_named_array.rb +18 -0
- data/test/scout/test_persist.rb +6 -0
- data/test/scout/test_tsv.rb +212 -2
- data/test/scout/test_work_queue.rb +21 -19
- data/test/scout/tsv/persist/test_adapter.rb +1 -1
- data/test/scout/tsv/persist/test_tokyocabinet.rb +29 -1
- data/test/scout/tsv/test_attach.rb +227 -0
- data/test/scout/tsv/test_change_id.rb +98 -0
- data/test/scout/tsv/test_dumper.rb +1 -1
- data/test/scout/tsv/test_index.rb +35 -3
- data/test/scout/tsv/test_open.rb +160 -2
- data/test/scout/tsv/test_parser.rb +19 -2
- data/test/scout/tsv/test_persist.rb +2 -0
- data/test/scout/tsv/test_transformer.rb +108 -0
- data/test/scout/tsv/test_traverse.rb +88 -3
- data/test/scout/tsv/test_util.rb +1 -0
- data/test/scout/tsv/util/test_reorder.rb +94 -0
- data/test/scout/tsv/util/test_select.rb +25 -11
- data/test/scout/tsv/util/test_unzip.rb +112 -0
- data/test/scout/work_queue/test_socket.rb +0 -1
- data/test/scout/workflow/step/test_status.rb +31 -0
- data/test/scout/workflow/task/test_inputs.rb +14 -14
- data/test/scout/workflow/test_step.rb +3 -3
- data/test/scout/workflow/test_task.rb +168 -32
- data/test/scout/workflow/test_usage.rb +33 -6
- metadata +20 -6
@@ -1,8 +1,9 @@
|
|
1
1
|
require_relative '../../open/lock'
|
2
|
+
require_relative '../../meta_extension'
|
2
3
|
require_relative 'serialize'
|
3
4
|
|
4
5
|
module TSVAdapter
|
5
|
-
attr_accessor :persistence_path, :persistence_class, :
|
6
|
+
attr_accessor :persistence_path, :persistence_class, :closed, :writable, :serializer
|
6
7
|
|
7
8
|
class << self
|
8
9
|
attr_accessor :lock_dir
|
@@ -19,23 +20,15 @@ module TSVAdapter
|
|
19
20
|
end
|
20
21
|
|
21
22
|
def load_extension_attr_hash
|
22
|
-
EXTENSION_ATTR_HASH_SERIALIZER.load(StringIO.new(self
|
23
|
+
EXTENSION_ATTR_HASH_SERIALIZER.load(StringIO.new(self.orig_get(EXTENSION_ATTR_HASH_KEY)))
|
23
24
|
end
|
24
25
|
|
25
26
|
def save_extension_attr_hash
|
26
|
-
self
|
27
|
+
self.orig_set(EXTENSION_ATTR_HASH_KEY, EXTENSION_ATTR_HASH_SERIALIZER.dump(self.extension_attr_hash))
|
27
28
|
end
|
28
29
|
|
29
30
|
def self.extended(base)
|
30
|
-
if
|
31
|
-
if (! TSVAdapter === base) && base.include?(EXTENSION_ATTR_HASH_KEY)
|
32
|
-
TSV.setup(base, base.load_extension_attr_hash)
|
33
|
-
elsif TSV === base
|
34
|
-
base[EXTENSION_ATTR_HASH_KEY] = EXTENSION_ATTR_HASH_SERIALIZER.dump(base.extension_attr_hash)
|
35
|
-
end
|
36
|
-
end
|
37
|
-
|
38
|
-
base.serializer = SERIALIZER_ALIAS[base.type]
|
31
|
+
return if base.respond_to?(:orig_set)
|
39
32
|
|
40
33
|
class << base
|
41
34
|
alias orig_set []=
|
@@ -54,45 +47,22 @@ module TSVAdapter
|
|
54
47
|
end
|
55
48
|
|
56
49
|
def load_value(str)
|
50
|
+
return nil if str.nil?
|
57
51
|
serializer.load(str)
|
58
52
|
end
|
59
53
|
|
60
54
|
def save_value(value)
|
61
55
|
serializer.dump(value)
|
62
56
|
end
|
63
|
-
|
64
|
-
|
65
57
|
end
|
66
58
|
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
# value
|
75
|
-
# end
|
76
|
-
# end
|
77
|
-
#when :list, :flat
|
78
|
-
# class << base
|
79
|
-
# def load_value(value)
|
80
|
-
# value.nil? ? nil : value.split("\t")
|
81
|
-
# end
|
82
|
-
# def save_value(value)
|
83
|
-
# value * "\t"
|
84
|
-
# end
|
85
|
-
# end
|
86
|
-
#when :double
|
87
|
-
# class << base
|
88
|
-
# def load_value(value)
|
89
|
-
# value.nil? ? nil : value.split("\t").collect{|v| v.split("|") }
|
90
|
-
# end
|
91
|
-
# def save_value(value)
|
92
|
-
# value.collect{|v| v * "|" } * "\t"
|
93
|
-
# end
|
94
|
-
# end
|
95
|
-
#end
|
59
|
+
if base.include?(EXTENSION_ATTR_HASH_KEY)
|
60
|
+
TSV.setup(base, base.load_extension_attr_hash)
|
61
|
+
else
|
62
|
+
base.instance_variable_get(:@extension_attrs).push :serializer
|
63
|
+
base.serializer = SERIALIZER_ALIAS[base.type] if base.serializer.nil?
|
64
|
+
base.save_extension_attr_hash
|
65
|
+
end
|
96
66
|
end
|
97
67
|
|
98
68
|
def keys(*args)
|
@@ -101,9 +71,9 @@ module TSVAdapter
|
|
101
71
|
end
|
102
72
|
|
103
73
|
if k[0] == EXTENSION_ATTR_HASH_KEY
|
104
|
-
k.slice(1,k.length
|
74
|
+
k.slice(1,k.length)
|
105
75
|
elsif k[-1] == EXTENSION_ATTR_HASH_KEY
|
106
|
-
k.slice(0,k.length-
|
76
|
+
k.slice(0,k.length-1)
|
107
77
|
else
|
108
78
|
k - [EXTENSION_ATTR_HASH_KEY]
|
109
79
|
end
|
@@ -2,6 +2,7 @@ class FixWidthTable
|
|
2
2
|
|
3
3
|
attr_accessor :filename, :file, :value_size, :record_size, :range, :size, :mask, :write
|
4
4
|
def initialize(filename, value_size = nil, range = nil, update = false, in_memory = true)
|
5
|
+
filename = filename.find if Path === filename
|
5
6
|
@filename = filename
|
6
7
|
|
7
8
|
if update || %w(memory stringio).include?(filename.to_s.downcase) || ! File.exist?(filename)
|
@@ -312,6 +313,8 @@ class FixWidthTable
|
|
312
313
|
acc.concat(new)
|
313
314
|
end
|
314
315
|
end
|
316
|
+
|
317
|
+
alias length size
|
315
318
|
end
|
316
319
|
|
317
320
|
Persist.save_drivers[:fwt] = proc do |file, content|
|
@@ -16,10 +16,13 @@ module ScoutCabinet
|
|
16
16
|
dir = File.dirname(File.expand_path(path))
|
17
17
|
Open.mkdir(dir) unless File.exist?(dir)
|
18
18
|
|
19
|
+
tokyocabinet_class = tokyocabinet_class.to_s if Symbol === tokyocabinet_class
|
19
20
|
tokyocabinet_class = TokyoCabinet::HDB if tokyocabinet_class == "HDB" or tokyocabinet_class.nil?
|
20
21
|
tokyocabinet_class = TokyoCabinet::BDB if tokyocabinet_class == "BDB"
|
21
22
|
|
22
|
-
|
23
|
+
# Hack - Ignore warning: undefining the allocator of T_DATA class
|
24
|
+
# TokyoCabinet::HDB_data
|
25
|
+
database = Log.ignore_stderr do Persist::CONNECTIONS[path] ||= tokyocabinet_class.new end
|
23
26
|
|
24
27
|
if big and not Open.exists?(path)
|
25
28
|
database.tune(nil,nil,nil,tokyocabinet_class::TLARGE | tokyocabinet_class::TDEFLATE)
|
data/lib/scout/tsv/persist.rb
CHANGED
@@ -0,0 +1,141 @@
|
|
1
|
+
module TSV
|
2
|
+
class Transformer
|
3
|
+
attr_accessor :unnamed, :parser, :dumper
|
4
|
+
|
5
|
+
def initialize(parser, dumper = nil, unnamed: false)
|
6
|
+
if TSV::Parser === parser
|
7
|
+
@parser = parser
|
8
|
+
elsif TSV === parser
|
9
|
+
@parser = parser
|
10
|
+
else
|
11
|
+
@parser = TSV::Parser.new parser
|
12
|
+
end
|
13
|
+
@unnamed = unnamed
|
14
|
+
if dumper.nil?
|
15
|
+
@dumper = TSV::Dumper.new(@parser)
|
16
|
+
@dumper.sep = "\t"
|
17
|
+
else
|
18
|
+
@dumper = dumper
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def key_field=(key_field)
|
23
|
+
@dumper.key_field = key_field
|
24
|
+
end
|
25
|
+
|
26
|
+
def fields=(fields)
|
27
|
+
@dumper.fields = fields
|
28
|
+
end
|
29
|
+
|
30
|
+
def type=(type)
|
31
|
+
@dumper.type = type
|
32
|
+
end
|
33
|
+
|
34
|
+
def type
|
35
|
+
@dumper.type
|
36
|
+
end
|
37
|
+
|
38
|
+
def sep=(sep)
|
39
|
+
@dumper.sep = sep
|
40
|
+
end
|
41
|
+
|
42
|
+
def include?(*args)
|
43
|
+
false
|
44
|
+
end
|
45
|
+
|
46
|
+
def key_field
|
47
|
+
@dumper.key_field
|
48
|
+
end
|
49
|
+
|
50
|
+
def fields
|
51
|
+
@dumper.fields
|
52
|
+
end
|
53
|
+
|
54
|
+
def all_fields
|
55
|
+
return nil if fields.nil?
|
56
|
+
[key_field] + fields
|
57
|
+
end
|
58
|
+
|
59
|
+
def options
|
60
|
+
@dumper.options
|
61
|
+
end
|
62
|
+
|
63
|
+
def identify_field(name)
|
64
|
+
TSV.identify_field key_field, fields, name
|
65
|
+
end
|
66
|
+
|
67
|
+
def traverse(*args, **kwargs, &block)
|
68
|
+
kwargs[:into] = @dumper
|
69
|
+
kwargs[:bar] = "Transform #{Log.fingerprint @parser} into #{Log.fingerprint @target}" if TrueClass === kwargs[:bar]
|
70
|
+
@dumper.init if @dumper.respond_to?(:init) && ! @dumper.initialized
|
71
|
+
Log.debug "Transform #{Log.fingerprint @parser} into #{Log.fingerprint @dumper}"
|
72
|
+
Open.traverse(@parser, *args, **kwargs) do |k,v|
|
73
|
+
NamedArray.setup(v, @parser.fields, k) unless @unnamed
|
74
|
+
block.call k, v
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
def each(*args, **kwargs, &block)
|
79
|
+
kwargs[:into] = @dumper
|
80
|
+
kwargs[:bar] = "Transform #{Log.fingerprint @parser} into #{Log.fingerprint @target}" if TrueClass === kwargs[:bar]
|
81
|
+
@dumper.init if @dumper.respond_to?(:init) && ! @dumper.initialized
|
82
|
+
Open.traverse(@parser, *args, **kwargs) do |k,v|
|
83
|
+
NamedArray.setup(v, @parser.fields, k) unless @unnamed
|
84
|
+
block.call k, v
|
85
|
+
[k, v]
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
def with_unnamed
|
90
|
+
begin
|
91
|
+
old_unnamed = @unnamed
|
92
|
+
@unnamed = true
|
93
|
+
yield
|
94
|
+
ensure
|
95
|
+
@unnamed = old_unnamed
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
def []=(key, value)
|
100
|
+
@dumper.init if @dumper.respond_to?(:init) && ! @dumper.initialized
|
101
|
+
@dumper.add key, value
|
102
|
+
end
|
103
|
+
|
104
|
+
def stream
|
105
|
+
@dumper.stream
|
106
|
+
end
|
107
|
+
|
108
|
+
def tsv(*args)
|
109
|
+
TSV === @dumper ? @dumper : TSV.open(stream, *args)
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
def to_list
|
114
|
+
res = self.annotate({})
|
115
|
+
transformer = Transformer.new self, res
|
116
|
+
transformer.type = :list
|
117
|
+
transformer.traverse do |k,v|
|
118
|
+
case self.type
|
119
|
+
when :single
|
120
|
+
[k, [v]]
|
121
|
+
when :double
|
122
|
+
[k, v.collect{|v| v.first }]
|
123
|
+
when :flat
|
124
|
+
[k, v.slice(0,1)]
|
125
|
+
end
|
126
|
+
end
|
127
|
+
res
|
128
|
+
end
|
129
|
+
|
130
|
+
def to_single
|
131
|
+
res = self.annotate({})
|
132
|
+
transformer = Transformer.new self, res
|
133
|
+
transformer.type = :single
|
134
|
+
transformer.traverse do |k,v|
|
135
|
+
v = v.first while Array === v
|
136
|
+
[k, v]
|
137
|
+
end
|
138
|
+
res
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
data/lib/scout/tsv/traverse.rb
CHANGED
@@ -1,22 +1,14 @@
|
|
1
1
|
require_relative 'parser'
|
2
2
|
module TSV
|
3
|
-
def
|
4
|
-
return :key if name == :key || key_field.start_with?(name.to_s)
|
5
|
-
name.collect!{|n| key_field == n ? :key : n } if Array === name
|
6
|
-
NamedArray.identify_name(fields, name)
|
7
|
-
end
|
8
|
-
|
9
|
-
def identify_field(name)
|
10
|
-
TSV.identify_field(@key_field, @fields, name)
|
11
|
-
end
|
12
|
-
|
13
|
-
def traverse(key_field_pos = :key, fields_pos = nil, type: nil, one2one: false, unnamed: false, key_field: nil, fields: nil, &block)
|
3
|
+
def traverse(key_field_pos = :key, fields_pos = nil, type: nil, one2one: false, unnamed: false, key_field: nil, fields: nil, bar: false, cast: nil, select: nil, &block)
|
14
4
|
key_field = key_field_pos if key_field.nil?
|
15
|
-
fields = fields_pos if fields.nil?
|
5
|
+
fields = fields_pos.dup if fields.nil?
|
16
6
|
type = @type if type.nil?
|
17
7
|
key_pos = self.identify_field(key_field)
|
8
|
+
fields = self.all_fields if fields == :all
|
18
9
|
fields = [fields] unless fields.nil? || Array === fields
|
19
|
-
positions = fields.nil? ? nil : self.identify_field(fields)
|
10
|
+
positions = fields.nil? || fields == :all ? nil : self.identify_field(fields)
|
11
|
+
|
20
12
|
|
21
13
|
if key_pos == :key
|
22
14
|
key_name = @key_field
|
@@ -33,7 +25,7 @@ module TSV
|
|
33
25
|
field_names = @fields
|
34
26
|
elsif positions.nil? && key_pos != :key
|
35
27
|
field_names = @fields.dup
|
36
|
-
field_names.delete_at key_pos
|
28
|
+
field_names.delete_at key_pos unless fields == :all
|
37
29
|
elsif positions.include?(:key)
|
38
30
|
field_names = positions.collect{|p| p == :key ? @key_field : @fields[p] }
|
39
31
|
else
|
@@ -43,98 +35,110 @@ module TSV
|
|
43
35
|
key_index = positions.index :key if positions
|
44
36
|
positions.delete :key if positions
|
45
37
|
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
if key_pos != :key
|
50
|
-
values = values.dup
|
51
|
-
key = values.delete_at(key_pos)
|
52
|
-
end
|
53
|
-
else
|
54
|
-
orig_key = key
|
55
|
-
key = values[key_pos] if key_pos != :key
|
38
|
+
log_message = "Traverse #{Log.fingerprint self}"
|
39
|
+
Log.debug log_message
|
40
|
+
bar = log_message if TrueClass === bar
|
56
41
|
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
42
|
+
Log::ProgressBar.with_obj_bar(self, bar) do |bar|
|
43
|
+
with_unnamed unnamed do
|
44
|
+
each do |key,values|
|
45
|
+
bar.tick if bar
|
46
|
+
values = [values] if @type == :single
|
47
|
+
if positions.nil?
|
48
|
+
if key_pos != :key
|
49
|
+
values = values.dup
|
50
|
+
key = values.delete_at(key_pos)
|
51
|
+
end
|
52
|
+
else
|
53
|
+
orig_key = key
|
54
|
+
key = values[key_pos] if key_pos != :key
|
55
|
+
|
56
|
+
values = values.values_at(*positions)
|
57
|
+
if key_index
|
58
|
+
if @type == :double
|
59
|
+
values.insert key_index, [orig_key]
|
60
|
+
else
|
61
|
+
values.insert key_index, orig_key
|
62
|
+
end
|
63
|
+
end
|
63
64
|
end
|
64
|
-
end
|
65
|
-
end
|
66
65
|
|
67
|
-
|
68
|
-
|
69
|
-
if
|
70
|
-
|
71
|
-
if
|
72
|
-
|
66
|
+
values = TSV.cast_value(values, cast) if cast
|
67
|
+
|
68
|
+
if Array === key
|
69
|
+
if @type == :double && one2one
|
70
|
+
if one2one == :strict
|
71
|
+
key.each_with_index do |key_i,i|
|
72
|
+
if type == :double
|
73
|
+
v_i = values.collect{|v| [v[i]] }
|
74
|
+
else
|
75
|
+
v_i = values.collect{|v| v[i] }
|
76
|
+
end
|
77
|
+
yield key_i, v_i
|
78
|
+
end
|
73
79
|
else
|
74
|
-
|
80
|
+
key.each_with_index do |key_i,i|
|
81
|
+
if type == :double
|
82
|
+
v_i = values.collect{|v| [v[i] || v.first] }
|
83
|
+
else
|
84
|
+
v_i = values.collect{|v| v[i] || v.first }
|
85
|
+
end
|
86
|
+
yield key_i, v_i, @fields
|
87
|
+
end
|
88
|
+
end
|
89
|
+
else
|
90
|
+
key.each_with_index do |key_i, i|
|
91
|
+
if type == :double
|
92
|
+
yield key_i, values
|
93
|
+
elsif type == :list
|
94
|
+
yield key_i, values.collect{|v| v[i] }
|
95
|
+
elsif type == :flat
|
96
|
+
yield key_i, values.flatten
|
97
|
+
elsif type == :single
|
98
|
+
yield key_i, values.first
|
99
|
+
end
|
75
100
|
end
|
76
|
-
yield key_i, v_i
|
77
101
|
end
|
78
102
|
else
|
79
|
-
|
80
|
-
if type == :
|
81
|
-
|
103
|
+
if type == @type
|
104
|
+
if type == :single
|
105
|
+
yield key, values.first
|
82
106
|
else
|
83
|
-
|
107
|
+
yield key, values
|
108
|
+
end
|
109
|
+
else
|
110
|
+
case [type, @type]
|
111
|
+
when [:double, :list]
|
112
|
+
yield key, values.collect{|v| [v] }
|
113
|
+
when [:double, :flat]
|
114
|
+
yield key, [values]
|
115
|
+
when [:double, :single]
|
116
|
+
yield key, [values]
|
117
|
+
when [:list, :double]
|
118
|
+
yield key, values.collect{|v| v.first }
|
119
|
+
when [:list, :flat]
|
120
|
+
yield key, [values.first]
|
121
|
+
when [:list, :single]
|
122
|
+
yield key, values
|
123
|
+
when [:flat, :double]
|
124
|
+
yield key, values.flatten
|
125
|
+
when [:flat, :list]
|
126
|
+
yield key, values.flatten
|
127
|
+
when [:flat, :single]
|
128
|
+
yield key, values
|
129
|
+
when [:single, :double]
|
130
|
+
yield key, values.flatten.first
|
131
|
+
when [:single, :list]
|
132
|
+
yield key, values.first
|
133
|
+
when [:single, :flat]
|
134
|
+
yield key, values.first
|
84
135
|
end
|
85
|
-
yield key_i, v_i
|
86
|
-
end
|
87
|
-
end
|
88
|
-
else
|
89
|
-
key.each_with_index do |key_i, i|
|
90
|
-
if type == :double
|
91
|
-
yield key_i, values
|
92
|
-
elsif type == :list
|
93
|
-
yield key_i, values.collect{|v| v[i] }
|
94
|
-
elsif type == :flat
|
95
|
-
yield key_i, values.flatten
|
96
|
-
elsif type == :single
|
97
|
-
yield key_i, values.first
|
98
136
|
end
|
99
137
|
end
|
100
138
|
end
|
101
|
-
else
|
102
|
-
if type == @type
|
103
|
-
if type == :single
|
104
|
-
yield key, values.first
|
105
|
-
else
|
106
|
-
yield key, values
|
107
|
-
end
|
108
|
-
else
|
109
|
-
case [type, @type]
|
110
|
-
when [:double, :list]
|
111
|
-
yield key, values.collect{|v| [v] }
|
112
|
-
when [:double, :flat]
|
113
|
-
yield key, [values]
|
114
|
-
when [:double, :single]
|
115
|
-
yield key, [values]
|
116
|
-
when [:list, :double]
|
117
|
-
yield key, values.collect{|v| v.first }
|
118
|
-
when [:list, :flat]
|
119
|
-
yield key, [values.first]
|
120
|
-
when [:list, :single]
|
121
|
-
yield key, values
|
122
|
-
when [:flat, :double]
|
123
|
-
yield key, values.flatten
|
124
|
-
when [:flat, :list]
|
125
|
-
yield key, values.flatten
|
126
|
-
when [:flat, :single]
|
127
|
-
yield key, values
|
128
|
-
when [:single, :double]
|
129
|
-
yield key, values.flatten.first
|
130
|
-
when [:single, :list]
|
131
|
-
yield key, values.first
|
132
|
-
when [:single, :flat]
|
133
|
-
yield key, values.first
|
134
|
-
end
|
135
|
-
end
|
136
139
|
end
|
137
140
|
end
|
141
|
+
|
138
142
|
|
139
143
|
[key_name, field_names]
|
140
144
|
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
require 'matrix'
|
2
|
+
|
3
|
+
module TSV
|
4
|
+
def reorder(key_field = nil, fields = nil, merge: true, one2one: true, **kwargs)
|
5
|
+
res = self.annotate({})
|
6
|
+
res.type = kwargs[:type] if kwargs.include?(:type)
|
7
|
+
kwargs[:one2one] = one2one
|
8
|
+
key_field_name, field_names = traverse key_field, fields, **kwargs do |k,v|
|
9
|
+
if @type == :double && merge && res.include?(k)
|
10
|
+
current = res[k]
|
11
|
+
if merge == :concat
|
12
|
+
v.each_with_index do |new,i|
|
13
|
+
next if new.empty?
|
14
|
+
current[i].concat(new)
|
15
|
+
end
|
16
|
+
else
|
17
|
+
merged = []
|
18
|
+
v.each_with_index do |new,i|
|
19
|
+
next if new.empty?
|
20
|
+
merged[i] = current[i] + new
|
21
|
+
end
|
22
|
+
res[k] = merged
|
23
|
+
end
|
24
|
+
else
|
25
|
+
res[k] = v
|
26
|
+
end
|
27
|
+
end
|
28
|
+
res.key_field = key_field_name
|
29
|
+
res.fields = field_names
|
30
|
+
res
|
31
|
+
end
|
32
|
+
|
33
|
+
def slice(fields, **kwargs)
|
34
|
+
reorder :key, fields, **kwargs
|
35
|
+
end
|
36
|
+
|
37
|
+
def column(field, **kwargs)
|
38
|
+
new_type = case type
|
39
|
+
when :double, :flat
|
40
|
+
:flat
|
41
|
+
else
|
42
|
+
:single
|
43
|
+
end
|
44
|
+
|
45
|
+
kwargs[:type] = new_type
|
46
|
+
slice(field, **kwargs)
|
47
|
+
end
|
48
|
+
|
49
|
+
def transpose_list(key_field="Unkown ID")
|
50
|
+
new_fields = keys.dup
|
51
|
+
new = self.annotate({})
|
52
|
+
TSV.setup(new, :key_field => key_field, :fields => new_fields, :type => type, :filename => filename, :identifiers => identifiers)
|
53
|
+
|
54
|
+
m = Matrix.rows values
|
55
|
+
new_rows = m.transpose.to_a
|
56
|
+
|
57
|
+
fields.zip(new_rows) do |key,row|
|
58
|
+
new[key] = row
|
59
|
+
end
|
60
|
+
|
61
|
+
new
|
62
|
+
end
|
63
|
+
|
64
|
+
def transpose_double(key_field = "Unkown ID")
|
65
|
+
sep = "-!SEP--#{rand 10000}!-"
|
66
|
+
tmp = self.to_list{|v| v * sep}
|
67
|
+
new = tmp.transpose_list(key_field)
|
68
|
+
new.to_double{|v| v.split(sep)}
|
69
|
+
end
|
70
|
+
|
71
|
+
def transpose(key_field = "Unkown ID")
|
72
|
+
case type
|
73
|
+
when :single, :flat
|
74
|
+
self.to_list.transpose_list key_field
|
75
|
+
when :list
|
76
|
+
transpose_list key_field
|
77
|
+
when :double
|
78
|
+
transpose_double key_field
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|