scout-gear 10.7.1 → 10.7.3

data/lib/scout/persist/tsv/adapter/base.rb

@@ -23,7 +23,9 @@ module TSVAdapter
   end
 
   def save_annotation_hash
-    self.orig_set(ANNOTATION_ATTR_HASH_KEY, ANNOTATION_ATTR_HASH_SERIALIZER.dump(self.annotation_hash))
+    self.with_write do
+      self.orig_set(ANNOTATION_ATTR_HASH_KEY, ANNOTATION_ATTR_HASH_SERIALIZER.dump(self.annotation_hash))
+    end
   end
 
   def self.extended(base)
@@ -163,6 +165,16 @@ module TSVAdapter
     end
   end
 
+  def with_write(*args, &block)
+    if @write
+      yield
+    elsif @closed
+      write_and_close &block
+    else
+      write_and_read &block
+    end
+  end
+
   def close(*args)
     begin
       super(*args)

data/lib/scout/persist/tsv.rb

@@ -48,7 +48,8 @@ module Persist
 
   def self.tsv(id, options = {}, engine: nil, persist_options: {})
     engine ||= persist_options[:engine] || :HDB
-    Persist.persist(id, engine, persist_options.merge(:other_options => options)) do |filename|
+    persist_options[:other_options] = options unless persist_options.include?(:other_options)
+    Persist.persist(id, engine, persist_options) do |filename|
       if filename.nil?
         yield(persist_options[:data] || {})
       else

data/lib/scout/tsv/annotation.rb

@@ -48,12 +48,12 @@ module Annotation
 
     fields = fields.flatten.compact.uniq
 
-    annotations = if Annotation.is_annotated?(objs) 
-                    objs.annotations 
+    annotations = if Annotation.is_annotated?(objs)
+                    objs.annotation_hash.keys
                   elsif (Array === objs && objs.any?)
                     first = objs.compact.first
                     if Annotation.is_annotated?(first)
-                      objs.compact.first.annotations
+                      objs.compact.first.annotation_hash.keys
                     else
                       raise "Objects didn't have annotations"
                     end
@@ -158,7 +158,7 @@ module Annotation
         Annotation.load_tsv_values(id, values, tsv.fields)
       end
 
-      case tsv.key_field 
+      case tsv.key_field
       when "List"
         annotated_objects.first
       else

data/lib/scout/tsv/attach.rb

@@ -92,6 +92,9 @@ module TSV
         end
 
         other_field_positions = other.identify_field(fields.dup) 
+        fields.zip(other_field_positions) do |o,n|
+          raise "Field #{o} not found. Options: #{Log.fingerprint other.fields}" if n.nil? 
+        end
 
         log_message = "Attach #{Log.fingerprint fields - source.fields} to #{Log.fingerprint source} (#{[match_key, other_key] * "=~"})"
         Log.debug log_message
@@ -252,8 +255,13 @@ module TSV
   def self.identifier_files(obj)
     if TSV === obj
       obj.identifier_files
-    elsif Path === obj
-      obj.dirname.identifiers
+    elsif Path.is_filename?(obj)
+      path = Path === obj ? obj : Path.setup(obj)
+      if obj.dirname.identifiers.exists?
+        obj.dirname.identifiers
+      else
+        [TSV.parse_options(obj)[:identifiers]]
+      end
     else
       nil
     end

data/lib/scout/tsv/change_id.rb

@@ -6,6 +6,9 @@ module TSV
     identifiers = source.identifiers if identifiers.nil? and source.respond_to?(:identifiers)
     if identifiers && source.identify_field(new_key_field, strict: true).nil?
       identifiers = identifiers.nil? ? source.identifiers : identifiers
+      if Array === identifiers
+        identifiers = identifiers.select{|f| f.identify_field(new_key_field) }.last
+      end
       new = source.attach(identifiers, fields: [new_key_field], insitu: false, one2one: true, persist_input: persist_identifiers)
       new = new.change_key(new_key_field, keep: keep, stream: stream, one2one: one2one, merge: merge)
       return new

data/lib/scout/tsv/dumper.rb

@@ -26,12 +26,13 @@ module TSV
     end
 
 
-    attr_accessor :options, :initialized, :type, :sep, :filename, :namespace
+    attr_accessor :options, :initialized, :type, :sep, :compact, :filename, :namespace
     def initialize(options = {})
       options = options.options.merge(sep: nil) if TSV::Parser === options || TSV === options
       @sep, @type = IndiferentHash.process_options options, 
         :sep, :type, 
         :sep => "\t", :type => :double
+      @compact = options[:compact]
       @options = options
       @options[:type] = @type
       @sout, @sin = Open.pipe
@@ -39,8 +40,9 @@ module TSV
       @initialized = false
       @filename = options[:filename]
       @mutex = Mutex.new
+      @namespace = options[:namespace]
       ConcurrentStream.setup(@sin, pair: @sout)
-      ConcurrentStream.setup(@sout, pair: @sin)
+      ConcurrentStream.setup(@sout, pair: @sin, filename: filename)
     end
 
     def set_stream(stream)
@@ -96,11 +98,11 @@ module TSV
           when :list, :flat
             @sin << key + @sep + value * @sep << "\n"
           when :double
-            @sin << key + @sep + value.collect{|v| Array === v ? v * "|" : v } * @sep << "\n"
+            @sin << key + @sep + value.collect{|v| Array === v ? (@compact ? v.compact : v) * "|" : v } * @sep << "\n"
           else
             if Array === value 
               if Array === value.first
-                @sin << key + @sep + value.collect{|v| Array === v ? v * "|" : v } * @sep << "\n"
+                @sin << key + @sep + value.collect{|v| Array === v ? (@compact ? v.compact : v) * "|" : v } * @sep << "\n"
               else
                 @sin << key + @sep + value * @sep << "\n"
               end
@@ -173,13 +175,35 @@ module TSV
       end
     end
 
-    if stream.nil?
-      t = Thread.new do 
+    self.with_unnamed do
+      if stream.nil?
+        t = Thread.new do 
+          begin
+            Thread.current.report_on_exception = true
+            Thread.current["name"] = "Dumper thread"
+            dumper.init(preamble: preamble)
+
+            if keys
+              keys.each do |k|
+                dump_entry.call k, self[k]
+              end
+            else
+              self.each &dump_entry
+            end
+
+            dumper.close
+          rescue
+            dumper.abort($!)
+          end
+        end
+        Thread.pass until t["name"]
+        stream = dumper.stream
+        ConcurrentStream.setup(stream, :threads => [t])
+        stream
+      else
+        dumper.set_stream stream
         begin
-          Thread.current.report_on_exception = true
-          Thread.current["name"] = "Dumper thread"
           dumper.init(preamble: preamble)
-
           if keys
             keys.each do |k|
               dump_entry.call k, self[k]
@@ -192,28 +216,8 @@ module TSV
         rescue
           dumper.abort($!)
         end
+        stream
       end
-      Thread.pass until t["name"]
-      stream = dumper.stream
-      ConcurrentStream.setup(stream, :threads => [t])
-      stream
-    else
-      dumper.set_stream stream
-      begin
-        dumper.init(preamble: preamble)
-        if keys
-          keys.each do |k|
-            dump_entry.call k, self[k]
-          end
-        else
-          self.each &dump_entry
-        end
-
-        dumper.close
-      rescue
-        dumper.abort($!)
-      end
-      stream
     end
   end
 

data/lib/scout/tsv/index.rb

@@ -66,8 +66,6 @@ module TSV
         index = TSV.setup({}, :type => :single)
       end
 
-      tsv_file = TSV.open(tsv_file, **data_options) if ! TSV === tsv_file
-
       log_msg = "Index #{Log.fingerprint tsv_file} target #{Log.fingerprint target}"
       Log.low log_msg
       bar = log_msg if TrueClass === bar

data/lib/scout/tsv/open.rb

@@ -45,6 +45,7 @@ module Open
     elsif Path === into
       Open.write(into) do |io|
         self.traverse(obj, into: io, cpus: cpus, bar: bar, callback: callback, unnamed: unnamed, **options, &block)
+        io.join if io.respond_to?(:join)
       end
       return into
     end

data/lib/scout/tsv/parser.rb

@@ -7,7 +7,11 @@ module TSV
       if Proc === cast
         cast.call value
       else
-        value.send(cast)
+        if value.nil? || value == ""
+          nil
+        else
+          value.send(cast)
+        end
       end
     end
   end
@@ -94,6 +98,8 @@ module TSV
           line.chomp!
           if Proc === fix
             line = fix.call line
+            break if (FalseClass === line) || :break == line
+            next if line.nil?
           elsif fix
             line = Misc.fixutf8(line)
           end
@@ -175,14 +181,14 @@ module TSV
               current = data[key]
               if merge == :concat
                 these_items.each_with_index do |new,i|
-                  new = [nil] if new.empty?
+                  new = one2one ? [nil] : [] if new.empty?
                   current[i].concat(new)
                 end
               else
                 merged = []
                 these_items.each_with_index do |new,i|
-                  new = [nil] if new.empty?
-                  merged[i] = current[i] + new
+                  new = one2one ? [nil] : [] if new.empty?
+                  merged[i] = (current[i] || []) + new
                 end
                 data[key] = merged
               end
@@ -290,7 +296,8 @@ module TSV
       opts[:cast] = opts[:cast].to_sym if opts[:cast]
 
       all_fields = [key_field] + fields if key_field && fields
-      NamedArray.setup([opts, key_field, fields, first_line, preamble, all_fields], %w(options key_field fields first_line preamble all_fields))
+      namespace = opts[:namespace]
+      NamedArray.setup([opts, key_field, fields, first_line, preamble, all_fields, namespace], %w(options key_field fields first_line preamble all_fields namespace))
     rescue Exception
       raise stream.stream_exception if stream.respond_to?(:stream_exception) && stream.stream_exception
       stream.abort($!) if stream.respond_to?(:abort)
@@ -298,6 +305,10 @@ module TSV
     end
   end
 
+  def self.parse_options(...)
+    parse_header(...)[:options]
+  end
+
   KEY_PARAMETERS = begin
                      params = []
                      (method(:parse_line).parameters + method(:parse_stream).parameters).each do |type, name|
@@ -408,7 +419,7 @@ module TSV
       end
 
       if data
-        TSV.setup(data, :key_field => key_field_name, :fields => field_names, :type => @type)
+        TSV.setup(data, @source_options.merge(:key_field => key_field_name, :fields => field_names, :type => @type))
       else
         [key_field || self.key_field, fields || self.fields]
       end
@@ -435,7 +446,7 @@ module TSV
     end
   end
 
-  def self.parse(stream, fix: true, header_hash: "#", sep: "\t", filename: nil, namespace: nil, unnamed: false, serializer: nil, **kwargs, &block)
+  def self.parse(stream, fix: true, header_hash: "#", sep: "\t", filename: nil, namespace: nil, unnamed: nil, serializer: nil, **kwargs, &block)
     parser = TSV::Parser === stream ? stream : TSV::Parser.new(stream, fix: fix, header_hash: header_hash, sep: sep)
 
     cast = kwargs[:cast]
@@ -472,9 +483,9 @@ module TSV
     data = parser.traverse **kwargs, &block
     data.type = type
     data.cast = cast
-    data.filename = filename || parser.options[:filename]
-    data.namespace = namespace || parser.options[:namespace]
-    data.identifiers = identifiers
+    data.filename = filename || parser.options[:filename] if data.filename.nil?
+    data.namespace = namespace || parser.options[:namespace] if data.namespace.nil?
+    data.identifiers = identifiers || parser.options[:identifiers] if data.identifiers.nil?
     data.unnamed = unnamed
     data.save_annotation_hash if data.respond_to?(:save_annotation_hash)
     data

data/lib/scout/tsv/path.rb

@@ -14,4 +14,12 @@ module Path
     found = produce_and_find('tsv')
     TSV.index(found, *args, **kwargs, &block)
   end
+
+  def identifier_file_path
+    if self.dirname.identifiers.exists?
+      self.dirname.identifiers
+    else
+      nil
+    end
+  end
 end

data/lib/scout/tsv/stream.rb

@@ -1,5 +1,6 @@
 module TSV
-  def self.paste_streams(streams, type: nil, sort: nil, sort_memory: nil, sep: nil, preamble: nil, header: nil, same_fields: nil, fix_flat: nil, all_match: nil, field_prefix: nil)
+  def self.paste_streams(streams, type: nil, sort: nil, sort_cmd_args: nil, sort_memory: nil, sep: nil, preamble: nil, header: nil, same_fields: nil, fix_flat: nil, all_match: nil, one2one: true, field_prefix: nil)
+    sep = "\t" if sep.nil?
 
     streams = streams.collect do |stream|
       case stream
@@ -19,7 +20,7 @@ module TSV
     num_streams = streams.length
 
     streams = streams.collect do |stream|
-      Open.sort_stream(stream, memory: sort_memory)
+      Open.sort_stream(stream, memory: sort_memory, cmd_args: sort_cmd_args)
     end if sort
 
     begin
@@ -37,7 +38,8 @@ module TSV
 
       streams = streams.collect do |stream|
 
-        parser = TSV::Parser.new stream, type: type, sep: sep
+        parser = TSV::Parser.new stream, sep: sep
+        #parser.type = type
 
         sfields = parser.fields
 
@@ -102,8 +104,9 @@ module TSV
           keys[i]= nil
           parts[i]= nil
         else
-          vs = line.chomp.split(sep, -1)
+          vs = line.split(sep, -1)
           key, *p = vs
+          p = [p] if parser_types[i] == :flat
           keys[i]= key
           parts[i]= p
         end
@@ -112,7 +115,7 @@ module TSV
       done_streams =[]
 
       fields = nil if fields && fields.empty?
-      dumper = TSV::Dumper.new key_field: key_field, fields: fields, type: type
+      dumper = TSV::Dumper.new key_field: key_field, fields: fields, type: type, compact: !one2one
       dumper.init(preamble: preamble_txt || !!key_field)
 
       t = Thread.new do
@@ -130,7 +133,7 @@ module TSV
           keys.each_with_index do |key,i|
             case key
             when min
-              new_values << parts[i]
+              new_parts = parts[i]
 
               begin
                 line = lines[i]= begin
@@ -145,18 +148,20 @@ module TSV
               else
                 k, *p = line.chomp.split(sep, -1)
                 p = p.collect{|e| e.nil? ? "" : e }
+                p = [p] if parser_types[i] == :flat
 
                 if k == keys[i]
-                  new_values = NamedArray.zip_fields(new_values).zip(p).collect{|p| [p.flatten * "|"] }
+                  new_parts = NamedArray.zip_fields([new_parts]).zip(p).collect{|p| [p.flatten * "|"] }
                   raise TryAgain 
                 end
                 keys[i]= k
                 parts[i]= p
               end
+
+              new_values << new_parts
             rescue TryAgain
               keys[i]= nil
               parts[i]= nil
-              Log.debug "Skipping repeated key in stream #{i}: #{key} - #{min}"
               retry
             end
           else
@@ -180,14 +185,16 @@ module TSV
           new_values = new_values.inject([]){|acc,l| acc.concat l }
         end
 
+        new_values = new_values.collect{|l| Array === l ? l.compact : l } unless one2one
+
         dumper.add min, new_values
       end
 
       dumper.close
 
       streams.each do |stream|
-        stream.close if stream.respond_to?(:close) && ! stream.closed?
-        stream.join if stream.respond_to? :join
+        stream.close if stream.respond_to?(:close)
+        stream.join if stream.respond_to?(:join)
       end
       end
     rescue Aborted

data/lib/scout/tsv/traverse.rb

@@ -1,6 +1,6 @@
 require_relative 'parser'
 module TSV
-  def traverse(key_field_pos = :key, fields_pos = nil, type: nil, one2one: false, unnamed: false, key_field: nil, fields: nil, bar: false, cast: nil, select: nil, uniq: false, &block)
+  def traverse(key_field_pos = :key, fields_pos = nil, type: nil, one2one: false, unnamed: nil, key_field: nil, fields: nil, bar: false, cast: nil, select: nil, uniq: false, &block)
     key_field = key_field_pos if key_field.nil?
     fields = fields_pos.dup if fields.nil?
     type = @type if type.nil?
@@ -9,6 +9,8 @@ module TSV
     fields = [fields] unless fields.nil? || Array === fields
     positions = (fields.nil? || fields == :all) ? nil : self.identify_field(fields)
     positions = nil if fields == self.fields
+    unnamed = @unnamed if unnamed.nil?
+    unnamed = false if unnamed.nil?
 
     if key_pos == :key
       key_name = @key_field
@@ -61,7 +63,6 @@ module TSV
             key = @type == :flat ? values : values[key_pos] if key_pos != :key 
 
             values = values.values_at(*positions)
-            NamedArray.setup(values, fields)
             if key_index
               if @type == :double
                 values.insert key_index, [orig_key]
@@ -71,6 +72,15 @@ module TSV
             end
           end
 
+          if ! unnamed && fields
+            case @type
+            when :flat, :single
+              values = Entity.prepare_entity(values, fields.first)
+            else
+              values = NamedArray.setup(values, fields, entity_options)
+            end
+          end
+
           values = TSV.cast_value(values, cast) if cast
 
           if Array === key 

data/lib/scout/tsv/util/process.rb

@@ -44,11 +44,14 @@ module TSV
   end
 
   def add_field(name = nil)
-    through do |key, values|
+    keys.each do |key|
+      values = self[key]
       new_values = yield(key, values)
       new_values = [new_values].compact if type == :double and not Array === new_values
 
       case
+      when type == :single
+        values = new_values
       when (values.nil? and (fields.nil? or fields.empty?))
         values = [new_values]
       when values.nil?  

data/lib/scout/tsv/util/select.rb

@@ -156,8 +156,14 @@ module TSV
       case
       when ((Array === method) and (key == :key or key_field == key))
         with_unnamed do
-          keys.each do |key|
-            new[key] = self[key] if invert ^ (method.include? key)
+          if invert
+            keys.each do |key|
+              new[key] = self[key] unless method.include?(key)
+            end
+          else
+            method.each do |key|
+              new[key] = self[key] if self.include?(key)
+            end
           end
         end
       when Array === method

data/lib/scout/tsv/util/sort.rb

@@ -1,4 +1,10 @@
+require 'scout/entity'
+
 module TSV
+  def prepare_entity(...)
+    Entity.prepare_entity(...)
+  end
+
   def sort_by(field = nil, just_keys = false, &block)
     field = :all if field.nil?
 
@@ -8,16 +14,18 @@ module TSV
       elems = []
       case type
       when :single
-        through :key, field do |key, field|
-          elems << [key, field]
+        through :key, field do |key, value|
+          elems << [key, value]
         end
       when :list, :flat
-        through :key, field do |key, fields|
-          elems << [key, fields.first]
+        through :key, field do |key, value|
+          v = field == :key ? key : value[0]
+          elems << [key, v]
         end
       when :double
-        through :key, field do |key, fields|
-          elems << [key, fields.first]
+        through :key, field do |key, value|
+          v = field == :key ? key : value[0]
+          elems << [key, v]
         end
       end
     end
@@ -81,16 +89,16 @@ module TSV
       elems = []
       case type
       when :single
-        through :key, field do |key, field|
-          elems << [key, field]
+        through :key, field do |key, value|
+          elems << [key, value]
         end
       when :list, :flat
-        through :key, field do |key, fields|
-          elems << [key, fields.first]
+        through :key, field do |key, values|
+          elems << [key, values[0]]
         end
       when :double
-        through :key, field do |key, fields|
-          elems << [key, fields.first]
+        through :key, field do |key, values|
+          elems << [key, values[0]]
         end
       end
     end
@@ -99,7 +107,7 @@ module TSV
       if fields == :all
         if just_keys
           keys = elems.sort_by{|key, value| key }.collect{|key, values| key}
-          keys = prepare_entity(keys, key_field, entity_options.merge(:dup_array => true))
+          keys = prepare_entity(keys, key_field, (entity_options || {}).merge(:dup_array => true))
         else
           elems.sort_by{|key, value| key }
         end
@@ -128,7 +136,7 @@ module TSV
         end
         if just_keys
           keys = sorted.collect{|key, value| key}
-          keys = prepare_entity(keys, key_field, entity_options.merge(:dup_array => true)) unless @unnamed
+          keys = prepare_entity(keys, key_field, (entity_options || {}).merge(:dup_array => true)) unless @unnamed
           keys
         else
           sorted.collect{|key, value| [key, self[key]]}
@@ -137,7 +145,7 @@ module TSV
     else
       if just_keys
         keys = elems.sort(&block).collect{|key, value| key}
-        keys = prepare_entity(keys, key_field, entity_options.merge(:dup_array => true)) unless @unnamed
+        keys = prepare_entity(keys, key_field, (entity_options || {}).merge(:dup_array => true)) unless @unnamed
         keys
       else
         elems.sort(&block).collect{|key, value| [key, self[key]]}

data/lib/scout/tsv/util.rb

@@ -32,6 +32,7 @@ module TSV
       text
     end
 
+    path = path.find if Path === path
     TmpFile.with_file(values.uniq * "\n", false) do |value_file|
       cmd = "cat '#{ path }' | sed 's/\\t/\\tHEADERNOMATCH/' | grep -w -F -f '#{ value_file }' | sed 's/HEADERNOMATCH//' |sort -u|cut -f 2  |sort|uniq -c|sed 's/^ *//;s/ /\t/'"
       begin
@@ -84,8 +85,9 @@ module TSV
 
   def each(*args, &block)
     if block_given?
+      actual_unnamed = @unnamed.nil? ? true : @unnamed
       super(*args) do |k,v|
-        NamedArray.setup(v, @fields) unless @unnamed || @type == :flat || ! (Array === v)
+        NamedArray.setup(v, @fields) unless actual_unnamed || @type == :flat || ! (Array === v)
         block.call(k, v)
       end
     else
@@ -105,7 +107,8 @@ module TSV
     end
   end
 
-  def with_unnamed(unnamed = true)
+  def with_unnamed(unnamed = nil)
+    unnamed = true if unnamed.nil?
     begin
       old_unnamed = @unnamed
       @unnamed = unnamed
@@ -177,4 +180,10 @@ Example:
     self.annotate(super(other))
   end
 
+  def merge_zip(other)
+    other.each do |k,v|
+      self.zip_new k, v
+    end
+  end
+
 end