scout-gear 10.3.0 → 10.6.1

data/lib/scout/tsv/annotation.rb

@@ -0,0 +1,169 @@
+require_relative 'annotation/repo'
+module Annotation
+
+  def self.obj_tsv_values(obj, fields)
+
+    annotation_info = obj.annotation_info
+    annotation_info[:annotated_array] = true if AnnotatedArray === obj
+
+    fields.collect do |field|
+      field = field.to_s if Symbol === field
+      case field
+      when Proc
+        field.call(obj)
+
+      when "JSON"
+        annotation_info.to_json
+
+      when "annotation_types"
+        annotation_info[:annotation_types].collect{|t| t.to_s} * "|"
+
+      when "annotated_array"
+        AnnotatedArray === obj
+
+      when "literal"
+        (Array === obj ? "Array:" << obj * "|" : obj).gsub(/\n|\t/, ' ')
+
+      else
+        if annotation_info.include?(field.to_sym)
+          res = annotation_info[field.to_sym]
+          Array === res ? "Array:" << res * "|" : res
+        elsif self.respond_to?(field)
+          res = self.send(field)
+          Array === res ? "Array:"<< res * "|" : res
+        else
+          raise
+        end
+      end
+    end
+  end
+
+  def self.list_tsv_values(objs, fields)
+    obj_tsv_values(objs, fields)
+  end
+  
+
+  def self.tsv(objs, *fields)
+    return nil if objs.nil?
+
+    fields = fields.flatten.compact.uniq
+
+    annotations = if Annotation.is_annotated?(objs) 
+                    objs.annotations 
+                  elsif (Array === objs && objs.any?)
+                    first = objs.compact.first
+                    if Annotation.is_annotated?(first)
+                      objs.compact.first.annotations
+                    else
+                      raise "Objects didn't have annotations"
+                    end
+                  else
+                    []
+                  end
+
+    if fields.empty?
+      fields = annotations + [:annotation_types]
+    elsif fields == ["all"] || fields == [:all]
+      fields = annotations + [:annotation_types, :literal]
+    end
+
+    fields = fields.collect{|f| Symbol === f ? f.to_s : f }
+
+    tsv = TSV.setup({}, :key_field => nil, :fields => fields, :type => :list, :unnamed => true)
+
+    case
+    when Annotation.is_annotated?(objs)
+      tsv.key_field = "List"
+
+      tsv[objs.annotation_id] = self.list_tsv_values(objs, fields).dup
+    when Array === objs 
+      tsv.key_field = "ID"
+
+      if Annotation.is_annotated?(objs.compact.first)
+        objs.compact.each_with_index do |obj,i|
+          tsv[obj.annotation_id + "#" << i.to_s] = self.obj_tsv_values(obj, fields).dup
+        end
+      elsif (objs.any? && Annotation.is_annotated?(objs.compact.first.compact.first))
+        objs.flatten.compact.each_with_index do |obj,i|
+          tsv[obj.annotation_id + "#" << i.to_s] = self.obj_tsv_values(obj, fields).dup
+        end
+      end
+    else
+      raise "Annotations need to be an Array to create TSV"
+    end
+
+    tsv
+  end
+
+  # Load TSV
+
+  def self.resolve_tsv_array(entry)
+    if String === entry && entry =~ /^Array:/
+      entry["Array:".length..-1].split("|")
+    else
+      entry
+    end
+  end
+
+  def self.load_info(fields, values)
+    info = {}
+    fields.each_with_index do |field,i|
+      next if field == "literal"
+
+      case field
+      when "JSON"
+        JSON.parse(values[i]).each do |key, value|
+          info[key.to_sym] = value
+        end
+      when nil
+        next
+      else
+        info[field.to_sym] = resolve_tsv_array(values[i])
+      end
+    end
+    info
+  end
+
+  def self.load_tsv_values(id, values, *fields)
+    fields = fields.flatten
+    literal_pos = fields.index "literal"
+
+    object = case
+             when literal_pos
+               values[literal_pos].tap{|o| o.force_encoding(Encoding.default_external)}
+             else
+               id.dup
+             end
+
+    object = resolve_tsv_array(object)
+
+    if Array === values.first
+      NamedArray.zip_fields(values).collect do |v|
+        info = load_info(fields, v)
+      end
+    else
+      info = load_info(fields, values)
+    end
+
+    self.setup(object, info[:annotation_types], info)
+
+    object.extend AnnotatedArray if Array === object
+
+    object
+  end
+
+  def self.load_tsv(tsv)
+    tsv.with_unnamed do
+      annotated_objects = tsv.collect do |id, values|
+        Annotation.load_tsv_values(id, values, tsv.fields)
+      end
+
+      case tsv.key_field 
+      when "List"
+        annotated_objects.first
+      else
+        annotated_objects
+      end
+    end
+  end
+end

data/lib/scout/tsv/attach.rb

@@ -1,7 +1,11 @@
 module TSV
 
   def self.match_keys(source, other, match_key: nil, other_key: nil)
-    match_key = (source.all_fields & other.all_fields).first if match_key.nil?
+    #match_key = (source.all_fields & other.all_fields).first if match_key.nil?
+    if match_key.nil?
+      match_key_pos = NamedArray.identify_name(source.all_fields, other.all_fields).first
+      match_key = source.all_fields[match_key_pos] if match_key_pos
+    end
 
     if match_key.nil?
       source.all_fields.collect do |f|
@@ -32,20 +36,26 @@ module TSV
 
     other_key = other.key_field if other_key.nil?
 
-    match_key = :key if match_key == source.key_field
-    other_key = :key if other_key == other.key_field
+    match_key = :key if NamedArray.field_match(match_key, source.key_field)
+    other_key = :key if NamedArray.field_match(other_key, other.key_field)
 
     [match_key, other_key]
   end
 
-  def self.attach(source, other, target: nil, fields: nil, match_key: nil, other_key: nil, one2one: true, complete: false, insitu: nil, persist_input: false, bar: nil)
+  def self.attach(source, other, target: nil, fields: nil, index: nil, identifiers: nil, match_key: nil, other_key: nil, one2one: true, complete: false, insitu: nil, persist_input: false, bar: nil)
     source = TSV::Transformer.new source unless TSV === source || TSV::Parser === source
-    other = TSV.open other, persist: persist_input unless TSV === other 
+    other = TSV::Parser.new other unless TSV === other || TSV::Parser === other
 
     fields = [fields] if String === fields
 
     match_key, other_key = TSV.match_keys(source, other, match_key: match_key, other_key: other_key)
 
+    if ! (TSV === other)
+      other_key_name = other_key == :key ? other.key_field : other.fields[other_key]
+      other = TSV.open other, key_field: other_key_name, fields: fields, one2one: true, persist: persist_input
+      other_key = :key if other.key_field == source.key_field
+    end
+
     if TSV::Transformer === source
       source.dumper = case target
                       when :stream
@@ -64,11 +74,24 @@ module TSV
         other_key_name = other.fields[other_key_name] if Integer === other_key
         fields = other.all_fields - [other_key_name, source.key_field] if fields.nil?
 
+        match_key_name = match_key == :key ? source.key_field : match_key_name
+
+        if index.nil? && ! source.identify_field(other_key_name)
+          identifier_files = []
+          identifier_files << identifiers if identifiers
+          identifier_files << source
+          identifier_files << TSV.identifier_files(source)
+          identifier_files << TSV.identifier_files(other)
+          identifier_files << other
+
+          index = TSV.translation_index(identifier_files.flatten, match_key_name, other_key_name)
+        end
+
         if other_key != :key 
-          other = other.reorder other_key, fields, one2one: one2one
+          other = other.reorder other_key, fields, one2one: one2one, merge: true, type: :double
         end
 
-        other_field_positions = other.identify_field(fields) 
+        other_field_positions = other.identify_field(fields.dup) 
 
         log_message = "Attach #{Log.fingerprint fields - source.fields} to #{Log.fingerprint source} (#{[match_key, other_key] * "=~"})"
         Log.debug log_message
@@ -79,8 +102,11 @@ module TSV
         source.fields = (source.fields + fields).uniq
 
         overlaps = source.identify_field(fields)
+        orig_type = source.type
+
+        type = source.type == :single ? :list : source.type
 
-        empty_other_values = case source.type
+        empty_other_values = case type
                              when :list
                                [nil] * other.fields.length
                              when :flat
@@ -89,20 +115,28 @@ module TSV
                                [[]] * other.fields.length
                              end
 
+        empty_other_values = nil if other.type == :single
+
         insitu = TSV === source ? true : false if insitu.nil?
+        insitu = false if source.type == :single
 
         match_key_pos = source.identify_field(match_key)
         source.traverse bar: bar, unnamed: true do |orig_key,current_values|
+          current_values = [current_values] if source.type == :single
+
           keys = (match_key == :key || match_key_pos == :key) ? [orig_key] : current_values[match_key_pos]
-          keys = [keys] unless Array === keys
+          keys = [keys].compact unless Array === keys
+
+          keys = index.chunked_values_at(keys).flatten if index
 
           current_values = current_values.dup unless insitu
+          keys = [nil] if keys.empty?
           keys.each do |current_key|
-            other_values = other[current_key]
+            other_values = current_key.nil? ? empty_other_values : other[current_key]
 
             if other_values.nil?
               other_values = empty_other_values
-            elsif other.type == :flat
+            elsif other.type == :flat 
               other_values = [other_values]
             elsif other.type == :list && source.type == :double
               other_values = other_values.collect{|v| [v] }
@@ -110,13 +144,25 @@ module TSV
               other_values = other_values.collect{|v| v.first }
             end
 
-            other_values = other_values.values_at *other_field_positions
+            other_values = other_field_positions.collect do |pos|
+              if pos == :key
+                current_key
+              else
+                other.type == :single ? other_values : other_values[pos]
+              end
+            end
 
             other_values.zip(overlaps).each do |v,overlap|
-              if source.type == :list
+              if type == :list
                 current_values[overlap] = v if current_values[overlap].nil? || String === current_values[overlap] && current_values[overlap].empty?
+              elsif type == :flat
+                next if v.nil?
+                v = [v] unless Array === v
+                current_values.concat v
               else
                 current_values[overlap] ||= []
+                next if v.nil?
+                v = [v] unless Array === v
                 current_values[overlap].concat (v - current_values[overlap])
               end
             end
@@ -126,7 +172,7 @@ module TSV
         end
 
         if complete && match_key == :key
-          empty_self_values = case source.type
+          empty_self_values = case type
                               when :list
                                 [nil] * source.fields.length
                               when :flat
@@ -136,15 +182,17 @@ module TSV
                               end
           other.each do |other_key,other_values|
             next if source.include?(other_key)
-            if other.type == :flat
+            if other.type == :flat 
               other_values = [other_values]
-            elsif other.type == :list && source.type == :double
+            elsif other.type == :single 
+              other_values = [other_values]
+            elsif other.type == :list && type == :double
               other_values = other_values.collect{|v| [v] }
-            elsif other.type == :double && source.type == :list
+            elsif other.type == :double && type == :list
               other_values = other_values.collect{|v| v.first }
             end
 
-            new_values = case source.type
+            new_values = case type
                          when :list
                            [nil] * source.fields.length
                          when :flat
@@ -154,17 +202,20 @@ module TSV
                          end
 
             other_values.zip(overlaps).each do |v,overlap|
-              if false && overlap == :key
+              next if v.nil?
+              if overlap == :key
                 other_key = Array === v ? v : v.first
-              elsif source.type == :list
+              elsif type == :list
                 new_values[overlap] = v if v[overlap].nil? || String === v[overlap] && v[overlap].empty?
               else
+                v = [v] unless Array === v
                 new_values[overlap].concat v
               end
             end
             source[other_key] = new_values
           end
         end
+        source.type = type
       end
     end
 
@@ -174,4 +225,37 @@ module TSV
   def attach(*args, **kwargs)
     TSV.attach(self, *args, **kwargs)
   end
+
+  def identifier_files
+    case
+    when (identifiers and TSV === identifiers)
+      [identifiers]
+    when (identifiers and Array === identifiers)
+      case
+      when (TSV === identifiers.first or identifiers.empty?)
+        identifiers
+      else
+        identifiers.collect{|f| Path === f ? f : Path.setup(f)}
+      end
+    when identifiers
+      [ Path === identifiers ? identifiers : Path.setup(identifiers) ]
+    when Path === filename
+      path_files = filename.dirname.identifiers
+      [path_files].flatten.compact.select{|f| f.exists?}
+    when filename
+      [Path.setup(filename.dup).dirname.identifiers]
+    else
+      []
+    end
+  end
+
+  def self.identifier_files(obj)
+    if TSV === obj
+      obj.identifier_files
+    elsif Path === obj
+      obj.dirname.identifiers
+    else
+      nil
+    end
+  end
 end

data/lib/scout/tsv/change_id/translate.rb

@@ -0,0 +1,148 @@
+module TSV
+
+  def self.identify_field_in_obj(obj, field)
+    case obj
+    when TSV
+      obj.identify_field(field)
+    when TSV::Parser, TSV::Dumper
+      TSV.identify_field(obj.key_field, obj.fields, field)
+    when Path, String
+      all_fields = TSV.parse_header(obj)["all_fields"]
+      identify_field_in_obj(all_fields, field)
+    when Array
+      key_field, *fields = obj
+      TSV.identify_field(key_field, fields, field)
+    end
+  end
+
+  def self.translation_path(file_fields, source, target)
+    target_files = file_fields.select{|f,fields| identify_field_in_obj(fields, target) }.collect{|file,f| file }
+    if source.nil?
+      source_files = file_fields.keys
+    else
+      source_files = file_fields.select{|f,fields| identify_field_in_obj(fields, source) }.collect{|file,f| file }
+    end
+
+    if source && (one_step = target_files & source_files).any?
+      [one_step.first]
+    else
+      source_fields = file_fields.values_at(*source_files).flatten
+      target_fields = file_fields.values_at(*target_files).flatten
+      if (common_fields = source_fields & target_fields).any?
+        source_file = source_files.select{|file| fields = file_fields[file]; (fields & common_fields).any? }.collect{|file,f| file }.first
+        target_file = target_files.select{|file| fields = file_fields[file]; (fields & common_fields).any? }.collect{|file,f| file }.first
+        [source_file, target_file]
+      else
+        file_fields.select{|f,fields| (fields & source_fields).any? && (fields & target_fields).any? }
+        middle_file, middle_fields = file_fields.select{|f,fields| (fields & source_fields).any? && (fields & target_fields).any? }.first
+        if middle_file
+          source_file = source_files.select{|file| fields = file_fields[file]; (fields & middle_fields).any? }.collect{|file,f| file }.first
+          target_file = target_files.select{|file| fields = file_fields[file]; (fields & middle_fields).any? }.collect{|file,f| file }.first
+          [source_file, middle_file, target_file]
+        else
+          raise "Could not traverse identifier path from #{Log.fingerprint source} to #{Log.fingerprint target} in #{Log.fingerprint file_fields}"
+        end
+      end
+    end
+  end
+
+  def self.translation_index(files, source, target, persist_options = {})
+    return nil if source == target
+    persist_options = IndiferentHash.add_defaults persist_options.dup, :persist => true, :prefix => "Translation index"
+
+    file_fields = {}
+
+    files = [files] unless Array === files
+
+    files.each do |file|
+      next if Path === file && ! Open.exist?(file)
+      file = file.find if Path === file
+      file_fields[file] = all_fields(file)
+    end
+
+    begin
+      path = translation_path(file_fields, source, target)
+    rescue
+      exception = $!
+      begin
+        path = translation_path(file_fields, source, target)
+      rescue
+        raise exception
+      end
+    end
+
+    name = [source || "all", target] * "->" + " (#{files.length} files - #{Misc.digest(files)})"
+    second_target = if path.length == 1
+                      target
+                    else
+                      file1, file2 = path.values_at 0, 1
+                      pos = NamedArray.identify_name(TSV.all_fields(file1), TSV.all_fields(file2))
+                      TSV.all_fields(file1)[pos.compact.first]
+                    end
+    Persist.persist(name, "HDB", persist_options) do 
+      index = path.inject(nil) do |acc,file|
+        if acc.nil?
+          if source.nil?
+            if TSV === file
+              acc = file.index target: second_target
+            else
+              acc = TSV.index(file, target: second_target)
+            end
+          else
+            if TSV === file
+              acc = (file.key_field == source || source.nil?) ? file.annotate(file.dup) : file.reorder(source)
+            else
+              acc = TSV.open(file, key_field: source)
+            end
+          end
+        else
+          acc = acc.attach file, insitu: false
+        end
+
+        acc
+      end
+      index.slice([target]).to_single
+    end
+  end
+
+  def self.translate(tsv, field, format, identifiers: nil, one2one: false, merge: true, stream: false, keep: false, persist_index: true)
+
+    identifiers ||= tsv.identifier_files
+    index = translation_index([tsv, identifiers].flatten.compact, field, format, persist: persist_index)
+
+    key_field, *fields = TSV.all_fields(tsv)
+    if field == key_field
+      new_key_field = format
+      new_fields = fields
+    else
+      new_key_field = key_field
+      new_fields = fields.collect{|f| f == field ? format : f }
+    end
+
+    field_pos = new_key_field == key_field ? new_fields.index(format) : :key
+
+    transformer = TSV::Transformer.new tsv
+    transformer.key_field = new_key_field
+    transformer.fields = new_fields
+    transformer.traverse one2one: one2one, unnamed: true do |k,v|
+      if field_pos == :key
+        [index[k], v]
+      else
+        v = v.dup
+        if Array === v[field_pos]
+          v[field_pos] = index.values_at(*v[field_pos]).compact
+        else
+          v[field_pos] = index[v[field_pos]]
+        end
+        [k, v]
+      end
+    end
+
+    stream ? transformer : transformer.tsv(merge: merge, one2one: one2one)
+  end
+
+  def translate(*args, **kwargs)
+    TSV.translate(self, *args, **kwargs)
+  end
+
+end

data/lib/scout/tsv/change_id.rb

@@ -1,10 +1,13 @@
+require_relative 'change_id/translate'
+
 module TSV
-  def self.change_key(source, new_key_field, identifiers: nil, one2one: false, stream: false, keep: false, persist_identifiers: nil)
+  def self.change_key(source, new_key_field, identifiers: nil, one2one: false, merge: true, stream: false, keep: false, persist_identifiers: nil)
     source = TSV::Parser.new source if String === source
+    identifiers = source.identifiers if identifiers.nil? and source.respond_to?(:identifiers)
     if identifiers && source.identify_field(new_key_field, strict: true).nil?
       identifiers = identifiers.nil? ? source.identifiers : identifiers
       new = source.attach(identifiers, fields: [new_key_field], insitu: false, one2one: true, persist_input: persist_identifiers)
-      new = new.change_key(new_key_field, keep: keep, stream: stream, one2one: one2one)
+      new = new.change_key(new_key_field, keep: keep, stream: stream, one2one: one2one, merge: merge)
       return new
     end
 
@@ -17,7 +20,7 @@ module TSV
       [k, v]
     end
 
-    stream ? transformer : transformer.tsv
+    stream ? transformer : transformer.tsv(merge: merge, one2one: one2one)
   end
 
   def change_key(*args, **kwargs)

data/lib/scout/tsv/csv.rb

@@ -0,0 +1,85 @@
+require 'csv'
+
+module TSV
+  def self.csv(obj, options = {}) 
+    options = IndiferentHash.add_defaults options, :headers => true, :type => :list
+    headers = options[:headers]
+
+    noheaders = ! headers
+
+    type = options.delete :type
+    cast = options.delete :cast
+    merge = options.delete :merge
+    key_field = options.delete :key_field
+    fields = options.delete :fields
+    
+    if key_field || fields
+      orig_type = type
+      type = :double
+      merge = true
+    end
+
+    options[:headers] = false
+
+    csv = case obj
+          when Path
+            CSV.read obj.find.open, **options
+          when String
+            if Open.remote?(obj)
+              CSV.read Open.open(obj), **options
+            elsif Path.is_filename?(obj)
+              CSV.read obj, **options
+            else
+              CSV.new obj, **options
+            end
+          else
+            CSV.new obj, **options
+          end
+
+    tsv = if noheaders
+            TSV.setup({}, :key_field => nil, :fields => nil, :type => type)
+          else
+            key, *csv_fields = csv.shift
+            TSV.setup({}, :key_field => key, :fields => csv_fields, :type => type)
+          end
+
+    csv.each_with_index do |row,i|
+      if noheaders
+        key, values = ["row-#{i}", row]
+      else
+        key, *values = row
+      end
+      
+      if cast
+        values = values.collect{|v| v.send cast }
+      end
+
+      case type
+      when :double, :flat
+        tsv.zip_new(key, values)
+      when :single
+        tsv[key] = values.first
+      when :list
+        tsv[key] = values
+      end
+    end
+
+    if key_field || fields
+      tsv = tsv.reorder(key_field, fields, :one2one => true, :merge => true)
+      if tsv.type != orig_type
+        tsv = case orig_type
+              when :list
+                tsv.to_list
+              when :single
+                tsv.to_single
+              when :list
+                tsv.to_list
+              when :flat
+                tsv.to_flat
+              end
+      end
+    end
+
+    tsv
+  end
+end