scout-gear 10.7.1 → 10.7.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (62) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +6 -0
  3. data/VERSION +1 -1
  4. data/lib/scout/association/index.rb +1 -1
  5. data/lib/scout/association.rb +21 -5
  6. data/lib/scout/entity/format.rb +9 -4
  7. data/lib/scout/entity/identifiers.rb +2 -2
  8. data/lib/scout/entity/named_array.rb +13 -0
  9. data/lib/scout/entity/property.rb +2 -1
  10. data/lib/scout/entity.rb +7 -4
  11. data/lib/scout/persist/tsv/adapter/base.rb +13 -1
  12. data/lib/scout/persist/tsv.rb +2 -1
  13. data/lib/scout/tsv/attach.rb +10 -2
  14. data/lib/scout/tsv/change_id.rb +3 -0
  15. data/lib/scout/tsv/dumper.rb +34 -30
  16. data/lib/scout/tsv/open.rb +1 -0
  17. data/lib/scout/tsv/parser.rb +21 -10
  18. data/lib/scout/tsv/path.rb +8 -0
  19. data/lib/scout/tsv/stream.rb +15 -8
  20. data/lib/scout/tsv/traverse.rb +12 -2
  21. data/lib/scout/tsv/util/process.rb +4 -1
  22. data/lib/scout/tsv/util/select.rb +8 -2
  23. data/lib/scout/tsv/util/sort.rb +23 -15
  24. data/lib/scout/tsv/util.rb +11 -2
  25. data/lib/scout/tsv.rb +23 -11
  26. data/lib/scout/workflow/definition.rb +3 -3
  27. data/lib/scout/workflow/deployment/orchestrator.rb +8 -5
  28. data/lib/scout/workflow/step/dependencies.rb +35 -11
  29. data/lib/scout/workflow/step/file.rb +2 -1
  30. data/lib/scout/workflow/step/info.rb +14 -2
  31. data/lib/scout/workflow/step/load.rb +5 -3
  32. data/lib/scout/workflow/step/progress.rb +6 -0
  33. data/lib/scout/workflow/step/provenance.rb +1 -1
  34. data/lib/scout/workflow/step/status.rb +10 -4
  35. data/lib/scout/workflow/step.rb +32 -12
  36. data/lib/scout/workflow/task/dependencies.rb +33 -24
  37. data/lib/scout/workflow/task/inputs.rb +40 -12
  38. data/lib/scout/workflow/task.rb +22 -10
  39. data/lib/scout/workflow/usage.rb +2 -2
  40. data/lib/scout/workflow.rb +1 -1
  41. data/scout-gear.gemspec +10 -4
  42. data/scout_commands/db/query +83 -0
  43. data/scout_commands/db/register +43 -0
  44. data/scout_commands/db/show +47 -0
  45. data/test/scout/entity/test_named_array.rb +21 -0
  46. data/test/scout/persist/test_tsv.rb +20 -0
  47. data/test/scout/persist/tsv/adapter/test_base.rb +20 -0
  48. data/test/scout/test_tsv.rb +40 -0
  49. data/test/scout/tsv/test_dumper.rb +24 -0
  50. data/test/scout/tsv/test_path.rb +24 -0
  51. data/test/scout/tsv/test_stream.rb +93 -0
  52. data/test/scout/tsv/test_traverse.rb +99 -0
  53. data/test/scout/tsv/test_util.rb +2 -0
  54. data/test/scout/tsv/util/test_select.rb +22 -0
  55. data/test/scout/tsv/util/test_sort.rb +24 -0
  56. data/test/scout/workflow/step/test_dependencies.rb +26 -0
  57. data/test/scout/workflow/step/test_info.rb +35 -0
  58. data/test/scout/workflow/task/test_dependencies.rb +67 -1
  59. data/test/scout/workflow/task/test_inputs.rb +24 -7
  60. data/test/scout/workflow/test_task.rb +36 -0
  61. data/test/scout/workflow/test_usage.rb +0 -1
  62. metadata +9 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 3e72b6d2bef9af0067bf2601e059aa992ffdab98fba772dba1e297f958edb02d
4
- data.tar.gz: 2c2cc3591c1645ccf34dbb250f1b2acda1b69040c0b051806c02c97eb07cc253
3
+ metadata.gz: 1182999631b54e94f842c90017e1f97dde904ddafe47129cd80b51097714a1c4
4
+ data.tar.gz: ec736a5762fc2e58450a7955dffffe61eb73584f0bdc53bf3cd368e194ae7cbf
5
5
  SHA512:
6
- metadata.gz: d1be99dd4127f54c5cf16833197e527a3cfa41014c7db4306ec291e8f01b78f8340cad7ee6e342dee2ed25221f2b02a6f6b665f46e81365ad5a12eea087b300d
7
- data.tar.gz: cc3a9b28e47520f6756cba18d6037d7e83b7cd4d0cded1597a7cafc2cc167234090c1014d2b38adcc0a6a69c8d3ba1a72255538bd924e14f55b95769f3eca977
6
+ metadata.gz: 66a5c6ab90c2aa047cfbd000731eaec3913994880adddc9ac2daac9802ee2ebef14a054c1e292200624a6411ac2244d5f915de8d3ed9113d1237edc3795ab027
7
+ data.tar.gz: 27a0e87ac1360420bffc87b34b01eb6f21631fdf3f6ef95983fb46152f6e0e390200518dd2bebcc95c880e74d6f6f8a98a76577be2fbad0aea7d1fbe9c0720b1
data/.vimproject CHANGED
@@ -107,6 +107,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
107
107
  object.rb
108
108
  format.rb
109
109
  identifiers.rb
110
+ named_array.rb
110
111
  }
111
112
  association.rb
112
113
  association=association{
@@ -137,6 +138,11 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
137
138
  update
138
139
  template
139
140
  offsite
141
+ db=db{
142
+ register
143
+ query
144
+ show
145
+ }
140
146
  workflow=workflow{
141
147
  task
142
148
  list
data/VERSION CHANGED
@@ -1 +1 @@
1
- 10.7.1
1
+ 10.7.2
@@ -157,7 +157,7 @@ module Association
157
157
  new.read
158
158
  end
159
159
 
160
- new.unnamed = true
160
+ new.unnamed = self.unnamed
161
161
 
162
162
  new.undirected = undirected
163
163
 
@@ -6,17 +6,32 @@ require_relative 'association/item'
6
6
 
7
7
  module Association
8
8
  def self.open(obj, source: nil, target: nil, fields: nil, source_format: nil, target_format: nil, format: nil, **kwargs)
9
+
10
+
11
+ if Path.is_filename?(obj)
12
+ options = TSV.parse_options(obj).merge(kwargs)
13
+ else
14
+ options = kwargs.dup
15
+ end
16
+
17
+ if String === obj && options[:namespace] && obj.include?("NAMESPACE")
18
+ new_obj = obj.gsub(/\[?NAMESPACE\]?/, options[:namespace])
19
+ obj.annotate(new_obj)
20
+ obj = new_obj
21
+ end
22
+
9
23
  all_fields = TSV.all_fields(obj)
10
- source_pos, field_pos, source_header, field_headers, source_format, target_format = headers(all_fields, fields, kwargs.merge(source: source, target: target, source_format: source_format, target_format: target_format, format: format))
24
+ source_pos, field_pos, source_header, field_headers, source_format, target_format = headers(all_fields, fields, options.merge(source: source, target: target, source_format: source_format, target_format: target_format, format: format))
11
25
 
12
26
  original_source_header = all_fields[source_pos]
13
27
  original_field_headers = all_fields.values_at(*field_pos)
14
28
  original_target_header = all_fields[field_pos.first]
15
29
 
16
- type, identifiers = IndiferentHash.process_options kwargs, :type, :identifiers
30
+ type, identifiers = IndiferentHash.process_options options, :type, :identifiers
17
31
 
18
32
  if source_format
19
33
  translation_files = [TSV.identifier_files(obj), Entity.identifier_files(source_format), identifiers].flatten.compact
34
+ translation_files.collect!{|f| Path.is_filename?(f, false) ? Path.setup(f.gsub(/\[?NAMESPACE\]?/, options[:namespace])) : f }
20
35
  source_index = begin
21
36
  TSV.translation_index(translation_files, source_header, source_format)
22
37
  rescue
@@ -26,6 +41,7 @@ module Association
26
41
 
27
42
  if target_format
28
43
  translation_files = [TSV.identifier_files(obj), Entity.identifier_files(target_format), identifiers].flatten.compact
44
+ translation_files.collect!{|f| Path.is_filename?(f, false) ? Path.setup(f.gsub(/\[?NAMESPACE\]?/, options[:namespace])) : f }
29
45
  target_index = begin
30
46
  TSV.translation_index(translation_files, field_headers.first, target_format)
31
47
  rescue
@@ -67,10 +83,10 @@ module Association
67
83
  if source_index.nil? && target_index.nil?
68
84
  if TSV === obj
69
85
  IndiferentHash.pull_keys kwargs, :persist
70
- type = kwargs[:type] || obj.type
71
- res = obj.reorder original_source_header, all_fields.values_at(*field_pos), **kwargs.merge(type: type, merge: true)
86
+ type = options[:type] || obj.type
87
+ res = obj.reorder original_source_header, all_fields.values_at(*field_pos), **options.merge(type: type, merge: true)
72
88
  else
73
- res = TSV.open(obj, key_field: original_source_header, fields: all_fields.values_at(*field_pos), **kwargs.merge(type: type))
89
+ res = TSV.open(obj, key_field: original_source_header, fields: all_fields.values_at(*field_pos), **options.merge(type: type))
74
90
  end
75
91
  res.key_field = final_key_field
76
92
  res.fields = final_fields
@@ -17,16 +17,20 @@ module Entity
17
17
  def find(value)
18
18
  @find_cache ||= {}
19
19
 
20
- @find_cache[value] ||= begin
20
+ if @find_cache.include?(value)
21
+ @find_cache[value]
22
+ else
23
+ @find_cache[value] = begin
21
24
  if orig_include? value
22
- @find_cache[value] = value
25
+ value
23
26
  else
27
+ value = value.to_s
24
28
  found = nil
25
29
  each do |k,v|
26
- if value.to_s == k.to_s
30
+ if value == k.to_s
27
31
  found = k
28
32
  break
29
- elsif value.to_s =~ /\(#{Regexp.quote k}\)/
33
+ elsif value =~ /\(#{Regexp.quote k.to_s}\)/
30
34
  found = k
31
35
  break
32
36
  end
@@ -34,6 +38,7 @@ module Entity
34
38
  found
35
39
  end
36
40
  end
41
+ end
37
42
  end
38
43
 
39
44
  def [](value)
@@ -91,6 +91,8 @@ module Entity
91
91
  end
92
92
  end
93
93
 
94
+ name = default if name.nil?
95
+
94
96
  self.send(:include, Entity::Identified) unless Entity::Identified === self
95
97
 
96
98
  self.format = all_fields
@@ -106,6 +108,4 @@ module Entity
106
108
  @identifier_files << file
107
109
  @identifier_files.uniq!
108
110
  end
109
-
110
-
111
111
  end
@@ -0,0 +1,13 @@
1
+ require 'scout/named_array'
2
+ require 'scout/entity'
3
+
4
+ module NamedArray
5
+
6
+ def [](key)
7
+ pos = NamedArray.identify_name(@fields, key)
8
+ return nil if pos.nil?
9
+ v = super(pos)
10
+ field = @fields && Integer === key && ! @fields.include?(key) ? @fields[key] : key
11
+ Entity.prepare_entity(v, field)
12
+ end
13
+ end
@@ -20,7 +20,8 @@ module Entity
20
20
  Persist.annotation_repo_persist(repo, [name, obj.id] * ":", &block)
21
21
  else
22
22
 
23
- Persist.persist([name, obj.id] * ":", type, options.dup, &block)
23
+ _id = obj.nil? ? 'nil' : obj.id
24
+ Persist.persist([name, _id] * ":", type, options.dup, &block)
24
25
  end
25
26
  end
26
27
 
data/lib/scout/entity.rb CHANGED
@@ -3,6 +3,7 @@ require_relative 'entity/format'
3
3
  require_relative 'entity/property'
4
4
  require_relative 'entity/object'
5
5
  require_relative 'entity/identifiers'
6
+ require_relative 'entity/named_array'
6
7
 
7
8
  module Entity
8
9
  def self.extended(base)
@@ -12,17 +13,19 @@ module Entity
12
13
  base.instance_variable_set(:@persisted_methods, {})
13
14
  base.include Entity::Object
14
15
  base.include AnnotatedArray
16
+ base.format = base.to_s
15
17
  base
16
18
  end
17
19
 
18
20
  def self.prepare_entity(entity, field, options = {})
19
21
  return entity unless defined? Entity
20
- return entity unless String === entity or Array === entity
21
- options ||= {}
22
-
23
- dup_array = options.delete :dup_array
22
+ return entity unless String === entity or Array === entity or Numeric === entity
24
23
 
25
24
  if Entity === field or (Entity.respond_to?(:formats) and (_format = Entity.formats.find(field)))
25
+ options ||= {}
26
+
27
+ dup_array = options.delete :dup_array
28
+
26
29
  params = options.dup
27
30
 
28
31
  params[:format] ||= params.delete "format"
@@ -23,7 +23,9 @@ module TSVAdapter
23
23
  end
24
24
 
25
25
  def save_annotation_hash
26
- self.orig_set(ANNOTATION_ATTR_HASH_KEY, ANNOTATION_ATTR_HASH_SERIALIZER.dump(self.annotation_hash))
26
+ self.with_write do
27
+ self.orig_set(ANNOTATION_ATTR_HASH_KEY, ANNOTATION_ATTR_HASH_SERIALIZER.dump(self.annotation_hash))
28
+ end
27
29
  end
28
30
 
29
31
  def self.extended(base)
@@ -163,6 +165,16 @@ module TSVAdapter
163
165
  end
164
166
  end
165
167
 
168
+ def with_write(*args, &block)
169
+ if @write
170
+ yield
171
+ elsif @closed
172
+ write_and_close &block
173
+ else
174
+ write_and_read &block
175
+ end
176
+ end
177
+
166
178
  def close(*args)
167
179
  begin
168
180
  super(*args)
@@ -48,7 +48,8 @@ module Persist
48
48
 
49
49
  def self.tsv(id, options = {}, engine: nil, persist_options: {})
50
50
  engine ||= persist_options[:engine] || :HDB
51
- Persist.persist(id, engine, persist_options.merge(:other_options => options)) do |filename|
51
+ persist_options[:other_options] = options unless persist_options.include?(:other_options)
52
+ Persist.persist(id, engine, persist_options) do |filename|
52
53
  if filename.nil?
53
54
  yield(persist_options[:data] || {})
54
55
  else
@@ -92,6 +92,9 @@ module TSV
92
92
  end
93
93
 
94
94
  other_field_positions = other.identify_field(fields.dup)
95
+ fields.zip(other_field_positions) do |o,n|
96
+ raise "Field #{o} not found. Options: #{Log.fingerprint other.fields}" if n.nil?
97
+ end
95
98
 
96
99
  log_message = "Attach #{Log.fingerprint fields - source.fields} to #{Log.fingerprint source} (#{[match_key, other_key] * "=~"})"
97
100
  Log.debug log_message
@@ -252,8 +255,13 @@ module TSV
252
255
  def self.identifier_files(obj)
253
256
  if TSV === obj
254
257
  obj.identifier_files
255
- elsif Path === obj
256
- obj.dirname.identifiers
258
+ elsif Path.is_filename?(obj)
259
+ path = Path === obj ? obj : Path.setup(obj)
260
+ if obj.dirname.identifiers.exists?
261
+ obj.dirname.identifiers
262
+ else
263
+ [TSV.parse_options(obj)[:identifiers]]
264
+ end
257
265
  else
258
266
  nil
259
267
  end
@@ -6,6 +6,9 @@ module TSV
6
6
  identifiers = source.identifiers if identifiers.nil? and source.respond_to?(:identifiers)
7
7
  if identifiers && source.identify_field(new_key_field, strict: true).nil?
8
8
  identifiers = identifiers.nil? ? source.identifiers : identifiers
9
+ if Array === identifiers
10
+ identifiers = identifiers.select{|f| f.identify_field(new_key_field) }.last
11
+ end
9
12
  new = source.attach(identifiers, fields: [new_key_field], insitu: false, one2one: true, persist_input: persist_identifiers)
10
13
  new = new.change_key(new_key_field, keep: keep, stream: stream, one2one: one2one, merge: merge)
11
14
  return new
@@ -26,12 +26,13 @@ module TSV
26
26
  end
27
27
 
28
28
 
29
- attr_accessor :options, :initialized, :type, :sep, :filename, :namespace
29
+ attr_accessor :options, :initialized, :type, :sep, :compact, :filename, :namespace
30
30
  def initialize(options = {})
31
31
  options = options.options.merge(sep: nil) if TSV::Parser === options || TSV === options
32
32
  @sep, @type = IndiferentHash.process_options options,
33
33
  :sep, :type,
34
34
  :sep => "\t", :type => :double
35
+ @compact = options[:compact]
35
36
  @options = options
36
37
  @options[:type] = @type
37
38
  @sout, @sin = Open.pipe
@@ -39,8 +40,9 @@ module TSV
39
40
  @initialized = false
40
41
  @filename = options[:filename]
41
42
  @mutex = Mutex.new
43
+ @namespace = options[:namespace]
42
44
  ConcurrentStream.setup(@sin, pair: @sout)
43
- ConcurrentStream.setup(@sout, pair: @sin)
45
+ ConcurrentStream.setup(@sout, pair: @sin, filename: filename)
44
46
  end
45
47
 
46
48
  def set_stream(stream)
@@ -96,11 +98,11 @@ module TSV
96
98
  when :list, :flat
97
99
  @sin << key + @sep + value * @sep << "\n"
98
100
  when :double
99
- @sin << key + @sep + value.collect{|v| Array === v ? v * "|" : v } * @sep << "\n"
101
+ @sin << key + @sep + value.collect{|v| Array === v ? (@compact ? v.compact : v) * "|" : v } * @sep << "\n"
100
102
  else
101
103
  if Array === value
102
104
  if Array === value.first
103
- @sin << key + @sep + value.collect{|v| Array === v ? v * "|" : v } * @sep << "\n"
105
+ @sin << key + @sep + value.collect{|v| Array === v ? (@compact ? v.compact : v) * "|" : v } * @sep << "\n"
104
106
  else
105
107
  @sin << key + @sep + value * @sep << "\n"
106
108
  end
@@ -173,13 +175,35 @@ module TSV
173
175
  end
174
176
  end
175
177
 
176
- if stream.nil?
177
- t = Thread.new do
178
+ self.with_unnamed do
179
+ if stream.nil?
180
+ t = Thread.new do
181
+ begin
182
+ Thread.current.report_on_exception = true
183
+ Thread.current["name"] = "Dumper thread"
184
+ dumper.init(preamble: preamble)
185
+
186
+ if keys
187
+ keys.each do |k|
188
+ dump_entry.call k, self[k]
189
+ end
190
+ else
191
+ self.each &dump_entry
192
+ end
193
+
194
+ dumper.close
195
+ rescue
196
+ dumper.abort($!)
197
+ end
198
+ end
199
+ Thread.pass until t["name"]
200
+ stream = dumper.stream
201
+ ConcurrentStream.setup(stream, :threads => [t])
202
+ stream
203
+ else
204
+ dumper.set_stream stream
178
205
  begin
179
- Thread.current.report_on_exception = true
180
- Thread.current["name"] = "Dumper thread"
181
206
  dumper.init(preamble: preamble)
182
-
183
207
  if keys
184
208
  keys.each do |k|
185
209
  dump_entry.call k, self[k]
@@ -192,28 +216,8 @@ module TSV
192
216
  rescue
193
217
  dumper.abort($!)
194
218
  end
219
+ stream
195
220
  end
196
- Thread.pass until t["name"]
197
- stream = dumper.stream
198
- ConcurrentStream.setup(stream, :threads => [t])
199
- stream
200
- else
201
- dumper.set_stream stream
202
- begin
203
- dumper.init(preamble: preamble)
204
- if keys
205
- keys.each do |k|
206
- dump_entry.call k, self[k]
207
- end
208
- else
209
- self.each &dump_entry
210
- end
211
-
212
- dumper.close
213
- rescue
214
- dumper.abort($!)
215
- end
216
- stream
217
221
  end
218
222
  end
219
223
 
@@ -45,6 +45,7 @@ module Open
45
45
  elsif Path === into
46
46
  Open.write(into) do |io|
47
47
  self.traverse(obj, into: io, cpus: cpus, bar: bar, callback: callback, unnamed: unnamed, **options, &block)
48
+ io.join if io.respond_to?(:join)
48
49
  end
49
50
  return into
50
51
  end
@@ -7,7 +7,11 @@ module TSV
7
7
  if Proc === cast
8
8
  cast.call value
9
9
  else
10
- value.send(cast)
10
+ if value.nil? || value.empty?
11
+ nil
12
+ else
13
+ value.send(cast)
14
+ end
11
15
  end
12
16
  end
13
17
  end
@@ -94,6 +98,8 @@ module TSV
94
98
  line.chomp!
95
99
  if Proc === fix
96
100
  line = fix.call line
101
+ break if (FalseClass === line) || :break == line
102
+ next if line.nil?
97
103
  elsif fix
98
104
  line = Misc.fixutf8(line)
99
105
  end
@@ -175,14 +181,14 @@ module TSV
175
181
  current = data[key]
176
182
  if merge == :concat
177
183
  these_items.each_with_index do |new,i|
178
- new = [nil] if new.empty?
184
+ new = one2one ? [nil] : [] if new.empty?
179
185
  current[i].concat(new)
180
186
  end
181
187
  else
182
188
  merged = []
183
189
  these_items.each_with_index do |new,i|
184
- new = [nil] if new.empty?
185
- merged[i] = current[i] + new
190
+ new = one2one ? [nil] : [] if new.empty?
191
+ merged[i] = (current[i] || []) + new
186
192
  end
187
193
  data[key] = merged
188
194
  end
@@ -290,7 +296,8 @@ module TSV
290
296
  opts[:cast] = opts[:cast].to_sym if opts[:cast]
291
297
 
292
298
  all_fields = [key_field] + fields if key_field && fields
293
- NamedArray.setup([opts, key_field, fields, first_line, preamble, all_fields], %w(options key_field fields first_line preamble all_fields))
299
+ namespace = opts[:namespace]
300
+ NamedArray.setup([opts, key_field, fields, first_line, preamble, all_fields, namespace], %w(options key_field fields first_line preamble all_fields namespace))
294
301
  rescue Exception
295
302
  raise stream.stream_exception if stream.respond_to?(:stream_exception) && stream.stream_exception
296
303
  stream.abort($!) if stream.respond_to?(:abort)
@@ -298,6 +305,10 @@ module TSV
298
305
  end
299
306
  end
300
307
 
308
+ def self.parse_options(...)
309
+ parse_header(...)[:options]
310
+ end
311
+
301
312
  KEY_PARAMETERS = begin
302
313
  params = []
303
314
  (method(:parse_line).parameters + method(:parse_stream).parameters).each do |type, name|
@@ -408,7 +419,7 @@ module TSV
408
419
  end
409
420
 
410
421
  if data
411
- TSV.setup(data, :key_field => key_field_name, :fields => field_names, :type => @type)
422
+ TSV.setup(data, @source_options.merge(:key_field => key_field_name, :fields => field_names, :type => @type))
412
423
  else
413
424
  [key_field || self.key_field, fields || self.fields]
414
425
  end
@@ -435,7 +446,7 @@ module TSV
435
446
  end
436
447
  end
437
448
 
438
- def self.parse(stream, fix: true, header_hash: "#", sep: "\t", filename: nil, namespace: nil, unnamed: false, serializer: nil, **kwargs, &block)
449
+ def self.parse(stream, fix: true, header_hash: "#", sep: "\t", filename: nil, namespace: nil, unnamed: nil, serializer: nil, **kwargs, &block)
439
450
  parser = TSV::Parser === stream ? stream : TSV::Parser.new(stream, fix: fix, header_hash: header_hash, sep: sep)
440
451
 
441
452
  cast = kwargs[:cast]
@@ -472,9 +483,9 @@ module TSV
472
483
  data = parser.traverse **kwargs, &block
473
484
  data.type = type
474
485
  data.cast = cast
475
- data.filename = filename || parser.options[:filename]
476
- data.namespace = namespace || parser.options[:namespace]
477
- data.identifiers = identifiers
486
+ data.filename = filename || parser.options[:filename] if data.filename.nil?
487
+ data.namespace = namespace || parser.options[:namespace] if data.namespace.nil?
488
+ data.identifiers = identifiers || parser.options[:identifiers] if data.identifiers.nil?
478
489
  data.unnamed = unnamed
479
490
  data.save_annotation_hash if data.respond_to?(:save_annotation_hash)
480
491
  data
@@ -14,4 +14,12 @@ module Path
14
14
  found = produce_and_find('tsv')
15
15
  TSV.index(found, *args, **kwargs, &block)
16
16
  end
17
+
18
+ def identifier_file_path
19
+ if self.dirname.identifiers.exists?
20
+ self.dirname.identifiers
21
+ else
22
+ nil
23
+ end
24
+ end
17
25
  end
@@ -1,5 +1,6 @@
1
1
  module TSV
2
- def self.paste_streams(streams, type: nil, sort: nil, sort_memory: nil, sep: nil, preamble: nil, header: nil, same_fields: nil, fix_flat: nil, all_match: nil, field_prefix: nil)
2
+ def self.paste_streams(streams, type: nil, sort: nil, sort_memory: nil, sep: nil, preamble: nil, header: nil, same_fields: nil, fix_flat: nil, all_match: nil, one2one: true, field_prefix: nil)
3
+ sep = "\t" if sep.nil?
3
4
 
4
5
  streams = streams.collect do |stream|
5
6
  case stream
@@ -37,7 +38,8 @@ module TSV
37
38
 
38
39
  streams = streams.collect do |stream|
39
40
 
40
- parser = TSV::Parser.new stream, type: type, sep: sep
41
+ parser = TSV::Parser.new stream, sep: sep
42
+ #parser.type = type
41
43
 
42
44
  sfields = parser.fields
43
45
 
@@ -102,8 +104,9 @@ module TSV
102
104
  keys[i]= nil
103
105
  parts[i]= nil
104
106
  else
105
- vs = line.chomp.split(sep, -1)
107
+ vs = line.split(sep, -1)
106
108
  key, *p = vs
109
+ p = [p] if parser_types[i] == :flat
107
110
  keys[i]= key
108
111
  parts[i]= p
109
112
  end
@@ -112,7 +115,7 @@ module TSV
112
115
  done_streams =[]
113
116
 
114
117
  fields = nil if fields && fields.empty?
115
- dumper = TSV::Dumper.new key_field: key_field, fields: fields, type: type
118
+ dumper = TSV::Dumper.new key_field: key_field, fields: fields, type: type, compact: !one2one
116
119
  dumper.init(preamble: preamble_txt || !!key_field)
117
120
 
118
121
  t = Thread.new do
@@ -130,7 +133,7 @@ module TSV
130
133
  keys.each_with_index do |key,i|
131
134
  case key
132
135
  when min
133
- new_values << parts[i]
136
+ new_parts = parts[i]
134
137
 
135
138
  begin
136
139
  line = lines[i]= begin
@@ -145,18 +148,20 @@ module TSV
145
148
  else
146
149
  k, *p = line.chomp.split(sep, -1)
147
150
  p = p.collect{|e| e.nil? ? "" : e }
151
+ p = [p] if parser_types[i] == :flat
148
152
 
149
153
  if k == keys[i]
150
- new_values = NamedArray.zip_fields(new_values).zip(p).collect{|p| [p.flatten * "|"] }
154
+ new_parts = NamedArray.zip_fields([new_parts]).zip(p).collect{|p| [p.flatten * "|"] }
151
155
  raise TryAgain
152
156
  end
153
157
  keys[i]= k
154
158
  parts[i]= p
155
159
  end
160
+
161
+ new_values << new_parts
156
162
  rescue TryAgain
157
163
  keys[i]= nil
158
164
  parts[i]= nil
159
- Log.debug "Skipping repeated key in stream #{i}: #{key} - #{min}"
160
165
  retry
161
166
  end
162
167
  else
@@ -180,6 +185,8 @@ module TSV
180
185
  new_values = new_values.inject([]){|acc,l| acc.concat l }
181
186
  end
182
187
 
188
+ new_values = new_values.collect{|l| Array === l ? l.compact : l } unless one2one
189
+
183
190
  dumper.add min, new_values
184
191
  end
185
192
 
@@ -187,7 +194,7 @@ module TSV
187
194
 
188
195
  streams.each do |stream|
189
196
  stream.close if stream.respond_to?(:close) && ! stream.closed?
190
- stream.join if stream.respond_to? :join
197
+ stream.join if stream.respond_to?(:join)
191
198
  end
192
199
  end
193
200
  rescue Aborted
@@ -1,6 +1,6 @@
1
1
  require_relative 'parser'
2
2
  module TSV
3
- def traverse(key_field_pos = :key, fields_pos = nil, type: nil, one2one: false, unnamed: false, key_field: nil, fields: nil, bar: false, cast: nil, select: nil, uniq: false, &block)
3
+ def traverse(key_field_pos = :key, fields_pos = nil, type: nil, one2one: false, unnamed: nil, key_field: nil, fields: nil, bar: false, cast: nil, select: nil, uniq: false, &block)
4
4
  key_field = key_field_pos if key_field.nil?
5
5
  fields = fields_pos.dup if fields.nil?
6
6
  type = @type if type.nil?
@@ -9,6 +9,8 @@ module TSV
9
9
  fields = [fields] unless fields.nil? || Array === fields
10
10
  positions = (fields.nil? || fields == :all) ? nil : self.identify_field(fields)
11
11
  positions = nil if fields == self.fields
12
+ unnamed = @unnamed if unnamed.nil?
13
+ unnamed = false if unnamed.nil?
12
14
 
13
15
  if key_pos == :key
14
16
  key_name = @key_field
@@ -61,7 +63,6 @@ module TSV
61
63
  key = @type == :flat ? values : values[key_pos] if key_pos != :key
62
64
 
63
65
  values = values.values_at(*positions)
64
- NamedArray.setup(values, fields)
65
66
  if key_index
66
67
  if @type == :double
67
68
  values.insert key_index, [orig_key]
@@ -71,6 +72,15 @@ module TSV
71
72
  end
72
73
  end
73
74
 
75
+ if ! unnamed && fields
76
+ case @type
77
+ when :flat, :single
78
+ values = Entity.prepare_entity(values, fields.first)
79
+ else
80
+ values = NamedArray.setup(values, fields, entity_options)
81
+ end
82
+ end
83
+
74
84
  values = TSV.cast_value(values, cast) if cast
75
85
 
76
86
  if Array === key
@@ -44,11 +44,14 @@ module TSV
44
44
  end
45
45
 
46
46
  def add_field(name = nil)
47
- through do |key, values|
47
+ keys.each do |key|
48
+ values = self[key]
48
49
  new_values = yield(key, values)
49
50
  new_values = [new_values].compact if type == :double and not Array === new_values
50
51
 
51
52
  case
53
+ when type == :single
54
+ values = new_values
52
55
  when (values.nil? and (fields.nil? or fields.empty?))
53
56
  values = [new_values]
54
57
  when values.nil?
@@ -156,8 +156,14 @@ module TSV
156
156
  case
157
157
  when ((Array === method) and (key == :key or key_field == key))
158
158
  with_unnamed do
159
- keys.each do |key|
160
- new[key] = self[key] if invert ^ (method.include? key)
159
+ if invert
160
+ keys.each do |key|
161
+ new[key] = self[key] unless method.include?(key)
162
+ end
163
+ else
164
+ method.each do |key|
165
+ new[key] = self[key] if self.include?(key)
166
+ end
161
167
  end
162
168
  end
163
169
  when Array === method