rbbt-util 5.14.33 → 5.14.34

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e156f608c54a1a71f1d7892428698d857075c4e7
-  data.tar.gz: 535f2b8b01c561226a389050cbb90b31b82ac9e2
+  metadata.gz: ec2d34290a61ca4f3f32cd875a07a15c9c7c06fd
+  data.tar.gz: 08ce21aa7a885530dcba3859231d94874f99bba2
 SHA512:
-  metadata.gz: 8bad7add5780e3764172ffb11c87b8b3cebc5267fe2f11dbe0efcdd46cebd92019b669346d31ff08a0ab32321356c2fc21ef5f776d1ce188d8527f2b1a4044c9
-  data.tar.gz: 543e58c6022ccacbbd164aa5a97df5ab7ab9c168582ec212b5b5cfce7714b02ca1f5b74b2c59a50c27297aa9113ccbf548ce62e3aafa3d530371361acb180bc3
+  metadata.gz: d532ef379f2238e8dbaf311f485a4e18ac774177624e327d7b1d432a6cd0304f17f945448523407bb0cbb8f14a1067b50d36585b6d554d646dd5a01124c5b4f2
+  data.tar.gz: b15a2660fd8be0a782fb0a139bc44b8670adf3629446be6c0b8dc3e6d27f017a2c71db0cb6d76602b306917c9909f03013226566d963c0247506a4723968ead8

data/bin/rbbt

@@ -145,6 +145,8 @@ def rbbt_usage(prev = nil)
   true
 end
 
+alias usage rbbt_usage 
+
 def print_error(error, backtrace = nil)
   puts Log.color :magenta, "## ERROR"
   puts

data/lib/rbbt/association/database.rb

@@ -0,0 +1,153 @@
+require 'rbbt/association/util'
+require 'rbbt/tsv/change_id'
+
+module Association
+
+  def self.add_reciprocal(tsv)
+    new = TSV.open(tsv.dumper_stream)
+    tsv.with_unnamed do
+      case tsv.type
+      when :double
+        tsv.through do |source, values|
+          Misc.zip_fields(values).each do |info|
+            target, *rest = info
+            new.zip_new target, [source] + rest
+          end
+        end
+      else
+      end
+    end
+
+    tsv.annotate(new)
+
+    new
+  end
+
+  def self.translate(tsv, source_final_format, target_final_format, options = {})
+    source_field = tsv.key_field
+    target_field = tsv.fields.first
+    namespace = tsv.namespace
+
+    if source_final_format and source_field != source_final_format 
+      Log.debug("Changing source format from #{tsv.key_field} to #{source_final_format}")
+
+      identifier_files = tsv.identifier_files.dup
+      identifier_files.concat Entity.identifier_files(source_final_format) if defined? Entity
+      identifier_files.uniq!
+      identifier_files.collect!{|f| f.annotate(f.gsub(/\bNAMESPACE\b/, namespace))} if namespace
+      identifier_files.reject!{|f| f.match(/\bNAMESPACE\b/)}
+
+      tsv = TSV.translate(tsv, source_field, source_final_format, options.merge(:identifier_files => identifier_files))
+    end
+
+    # Translate target 
+    if target_final_format and target_field != target_final_format
+      Log.debug("Changing target format from #{target_field} to #{target_final_format}")
+      old_key_field = tsv.key_field 
+      tsv.key_field = "MASK"
+
+      identifier_files = tsv.identifier_files.dup 
+      identifier_files.concat Entity.identifier_files(target_final_format) if defined? Entity
+      identifier_files.uniq!
+      identifier_files.collect!{|f| f.annotate(f.gsub(/\bNAMESPACE\b/, namespace))} if namespace
+      identifier_files.reject!{|f| f.match(/\bNAMESPACE\b/)}
+
+      tsv = TSV.translate(tsv, target_field, target_final_format, options.merge(:identifier_files => identifier_files))
+      tsv.key_field = old_key_field
+    end
+
+    tsv
+  end
+
+  def self.reorder_tsv(tsv, options = {})
+    fields, undirected, persist = Misc.process_options options, :fields, :undirected, :persist 
+    all_fields = tsv.all_fields
+
+    source_pos, field_pos, source_header, field_headers, source_format, target_format = headers(all_fields, fields, options)
+
+    source_field = source_pos == :key ? :key : all_fields[source_pos]
+    info_fields = field_pos.collect{|f| f == :key ? :key : all_fields[f]}
+    options = options.merge({:key_field => source_field, :fields =>  info_fields})
+
+    tsv = tsv.reorder source_field, fields, :zipped => true
+
+    tsv.key_field = source_header
+    tsv.fields = field_headers
+
+    tsv = translate tsv, source_format, target_format, :persist => persist if source_format or target_format
+
+    tsv = add_reciprocal tsv if undirected
+
+    tsv
+  end
+
+  def self.open_stream(stream, options = {})
+    fields, undirected, persist = Misc.process_options options, :fields, :undirected, :persist
+
+    parser = TSV::Parser.new stream, options.merge(:fields => nil, :key_field => nil)
+
+    key_field, *_fields = all_fields = parser.all_fields
+
+    source_pos, field_pos, source_header, field_headers, source_format, target_format = headers parser.all_fields, fields, options
+
+    parser.key_field = source_pos
+    parser.fields = field_pos
+
+    case parser.type
+    when :single
+      class << parser
+        def get_values(parts)
+          [parts[@key_field], parts.values_at(*@fields).first]
+        end
+      end
+    when :list
+      class << parser
+        def get_values(parts)
+          [parts[@key_field], parts.values_at(*@fields)]
+        end
+      end
+    when :double, :list, :single
+      class << parser
+        def get_values(parts)
+          [parts[@key_field].split(@sep2,-1), parts.values_at(*@fields).collect{|v| v.nil? ? [] : v.split(@sep2,-1) }]
+        end
+      end
+    when :flat
+      class << parser
+        def get_values(parts)
+          fields = (0..parts.length-1).to_a - [@key_field]
+          values = parts.values_at(*fields).compact.collect{|v| v.split(@sep2,-1) }.flatten
+          [parts[@key_field].split(@sep2,-1), values]
+        end
+      end
+    end
+
+    open_options = options.merge(parser.options).merge(:parser => parser)
+
+    tsv = TSV.parse parser.stream, {}, open_options
+    tsv.key_field = source_header
+    tsv.fields = field_headers
+
+    tsv = tsv.to_double unless tsv.type == :double
+
+    tsv = translate tsv, source_format, target_format, :persist => persist if source_format or target_format
+
+    tsv = add_reciprocal tsv if undirected
+
+    tsv
+  end
+
+  def self.database(file,  options = {})
+    case file
+    when TSV
+      file = file.to_double unless file.type == :double
+      reorder_tsv(file, options.dup)
+    when IO
+      open_stream(file, options.dup)
+    else
+      stream = TSV.get_stream(file)
+      open_stream(stream, options.dup)
+    end
+  end
+  
+end

data/lib/rbbt/association/index.rb

@@ -1,5 +1,85 @@
 require 'rbbt/tsv'
+require 'rbbt/association/open'
+
 module Association
+  def self.index(file, options = nil, persist_options = nil)
+    options = options.nil? ? {} : options.dup
+    persist_options = persist_options.nil? ?  Misc.pull_keys(options, :persist)  : persist_options.dup 
+
+    persist_options = Misc.add_defaults persist_options.dup, :persist => true, :engine => "BDB"
+    persist = persist_options[:persist]
+
+    file = version_file(file, options[:namespace]) if options[:namespace] and String === file
+
+    undirected = options[:undirected]
+    Persist.persist_tsv(file, "Association Index", options, persist_options.dup) do |data|
+      recycle = options[:recycle]
+
+      persist_options[:file] = persist_options[:file] + '.database' if persist_options[:file]
+      database = open(file, options, persist_options.dup)
+
+      fields = database.fields
+      source_field = database.key_field
+      target_field = fields.first.split(":").last
+      key_field = [source_field, target_field, undirected ? "undirected" : nil].compact * "~"
+
+      TSV.setup(data, :key_field => key_field, :fields => fields[1..-1], :type => :list, :serializer => :list)
+
+      data.key_field = key_field
+      data.fields = fields[1..-1]
+      data.type = :list
+      data.serializer = :list 
+
+      database.with_unnamed do
+        database.through do |source, values|
+          case database.type
+          when :single
+            values = [[values]]
+          when :list
+            values = values.collect{|v| [v] }
+          when :flat
+            values = [values]
+          end
+          next if values.empty?
+          next if source.nil? or source.empty?
+          next if values.empty?
+
+          targets, *rest = values
+
+          size = targets ? targets.length : 0
+
+          rest.each_with_index do |list,i|
+            list.replace [list.first] * size if list.length == 1
+          end if recycle and size > 1
+
+          rest = Misc.zip_fields rest
+
+          annotations = rest.length > 1 ?
+            targets.zip(rest) :
+            targets.zip(rest * targets.length) 
+
+          annotations.each do |target, info|
+            next if target.nil? or target.empty?
+            key = [source, target] * "~"
+            if data[key].nil? or info.nil?
+              data[key] = info
+            else
+              old_info = data[key]
+              info = old_info.zip(info).collect{|p| p * ";;" }
+              data[key] = info
+            end
+          end
+        end
+      end
+
+      data.close
+      data
+    end.tap do |data|
+      data.read if not Hash === data and data.respond_to? :read
+      Association::Index.setup data
+      data
+    end
+  end
   module Index
 
     attr_accessor :source_field, :target_field, :undirected
@@ -16,9 +96,15 @@ module Association
 
     def reverse
       @reverse ||= begin
-                     persistence_path = self.persistence_path
-                     persistence_path = persistence_path.find if Path === persistence_path
-                     reverse_filename = persistence_path + '.reverse'
+                     if self.respond_to? :persistence_path
+                       persistence_path = self.persistence_path
+                       persistence_path = persistence_path.find if Path === persistence_path
+                       reverse_filename = persistence_path + '.reverse'
+                     else
+                       raise "Can only reverse a TokyoCabinet::BDB dataset at the time"
+                     end
+
+                     self.read if self.respond_to? :read
 
                      if File.exists?(reverse_filename)
                        new = Persist.open_tokyocabinet(reverse_filename, false, serializer, TokyoCabinet::BDB)
@@ -66,16 +152,6 @@ module Association
 
     #{{{ Subset
 
-    def select_entities(entities)
-      source_type = Entity.formats[source_field] 
-      target_type = Entity.formats[target_field]
-
-      source_entities = entities[:source] || entities[source_field] || entities[Entity.formats[source_field].to_s] 
-      target_entities = entities[:target] || entities[target_field] || entities[Entity.formats[target_field].to_s]
-
-      [source_entities, target_entities]
-    end
-
     def subset(source, target)
       return [] if source.nil? or target.nil? or source.empty? or target.empty?
 
@@ -110,12 +186,5 @@ module Association
       target_matches.values_at(*target.uniq).flatten.compact
     end
 
-    def subset_entities(entities)
-      source, target = select_entities(entities)
-      return [] if source.nil? or target.nil?
-      return [] if Array === target and target.empty?
-      return [] if Array === source and source.empty?
-      subset source, target
-    end
   end
 end

data/lib/rbbt/association/open.rb

@@ -0,0 +1,37 @@
+require 'rbbt/association/database'
+
+module Association
+  def self.version_file(file, namespace)
+    old_file, file = file, file.sub('NAMESPACE', namespace) if namespace and String === file
+    old_file.annotate file if Path === old_file
+    file
+  end
+  
+  def self.open(file, options = nil, persist_options = nil)
+    options = options.nil? ? {} : options.dup
+    persist_options = persist_options.nil? ?  Misc.pull_keys(options, :persist)  : persist_options.dup 
+
+    options = Misc.add_defaults options, :zipped => true
+    persist_options = Misc.add_defaults persist_options, :persist => true, :dir => Rbbt.var.associations
+    persist = persist_options[:persist]
+
+    file = version_file(file, options[:namespace]) if options[:namespace] and String === file
+    file = file.call if Proc === file
+
+    data = Persist.persist_tsv(file, "Association Database", options, persist_options) do |data|
+      tsv = Association.database(file, options.merge(:persist => persist))
+      tsv = tsv.to_double unless tsv.type == :double
+      tsv.annotate data
+
+      data.serializer = :double if data.respond_to? :serializer
+      tsv.each do |k,v|
+        data[k] = v
+      end
+
+
+      data
+    end
+      data
+  end
+  
+end

data/lib/rbbt/association/util.rb

@@ -0,0 +1,133 @@
+require 'rbbt/entity'
+
+module Association
+  def self.identify_entity_format(format, fields)
+    entity_type = Entity.formats[format]
+    raise "Field #{ format } could not be resolved: #{fields}" if entity_type.nil?
+    main_field = fields.select{|f| Entity.formats[f] == entity_type}.first
+    raise "Field #{ format } not present, options: #{Misc.fingerprint fields}" if main_field.nil?
+    [main_field, nil, format]
+  end
+
+  def self.parse_field_specification(spec)
+    return [2,nil,nil] if Fixnum === spec
+    spec = spec.split "=>" unless Array === spec
+    field_part, final_format = spec
+
+    field, format = field_part.split "=~", -1
+
+    field = nil if field.nil? or field.empty?
+
+    [field, format, final_format]
+  end
+
+  def self.normalize_specs(spec, all_fields = nil)
+    return nil if spec.nil?
+    field, header, format = parse_field_specification spec 
+
+    specs = if all_fields.nil? or all_fields.include? field
+               [field, header, format]
+             else
+               if all_fields.nil?
+                 begin
+                   identify_entity_format field, all_fields 
+                 rescue
+                   [field, header, format]
+                 end
+               else
+                 [field, header, format]
+               end
+             end
+    specs
+  end
+
+  def self.extract_specs(all_fields=nil, options = {})
+    source, source_format, target, target_format = Misc.process_options options, :source, :source_format, :target, :target_format
+
+    key_field, *fields = all_fields.nil? ? [nil] : all_fields
+
+    source_specs = normalize_specs  source, all_fields
+    target_specs = normalize_specs  target, all_fields
+
+    source_specs = [nil, nil, nil] if source_specs.nil?
+    target_specs = [nil, nil, nil] if target_specs.nil?
+
+    source_specs[2] = source_format if source_format
+    target_specs[2] = target_format if target_format
+
+    if source_specs[0].nil? and target_specs[0].nil?
+      source_specs[0] = key_field 
+      target_specs[0] = fields[0]
+    elsif source_specs[0].nil? 
+      if target_specs[0] == :key or target_specs[0] == key_field
+        source_specs[0] = fields[0]
+      else
+        source_specs[0] = key_field
+      end
+    elsif target_specs[0].nil? 
+      if source_specs[0] == fields.first 
+        target_specs[0] = key_field
+      else
+        target_specs[0] = fields.first 
+      end
+    end
+
+    {:source => source_specs, :target => target_specs}
+  end
+
+  def self.process_formats(field, default_format = {})
+    return nil if default_format.nil? or default_format.empty?
+    default_format.each do |type, format|
+      entity_type = Entity.formats[field] || format
+      return format if entity_type.to_s === type 
+    end
+    return nil
+  end
+
+  def self.headers(all_fields, info_fields = nil, options = {})
+    specs = extract_specs all_fields, options
+
+    source_field = specs[:source][0]
+    target_field = specs[:target][0]
+
+    source_pos = all_fields.index source_field
+    target_pos = all_fields.index target_field
+
+    source_header = specs[:source][1] || specs[:source][0]
+    target_header = specs[:target][1] || specs[:target][0]
+
+    info_fields = all_fields.dup if info_fields.nil?
+    info_fields.delete source_field
+    info_fields.delete target_field
+    info_fields.unshift target_field
+
+    field_headers = [target_header] 
+    info_fields[1..-1].each do |field|
+      header = case field
+               when String 
+                 field
+               when Fixnum
+                 all_fields[field] 
+               when :key
+                 all_fields.first
+               end
+
+
+      field_headers << header
+    end
+
+    field_pos = info_fields.collect{|f| raise "Field #{f} not found. Options: #{info_fields* ", "}" unless all_fields.include?(f); f == :key ? 0 : all_fields.index(f);  }
+
+    source_format = specs[:source][2]
+    target_format = specs[:target][2]
+
+
+    if format = options[:format]
+      source_format = process_formats(specs[:source][1] || specs[:source][0], format) || source_format
+      target_format = process_formats(specs[:target][1] || specs[:target][0], format) || target_format
+    end
+
+    Log.low "Headers -- #{[source_pos, field_pos, source_header, field_headers, source_format, target_format]}"
+    [source_pos, field_pos, source_header, field_headers, source_format, target_format]
+  end
+end