scout-gear 10.4.0 → 10.7.0

data/Rakefile

@@ -39,6 +39,7 @@ Rake::TestTask.new(:test) do |test|
   test.libs << 'lib' << 'test'
   test.pattern = 'test/**/test_*.rb'
   test.verbose = true
+  test.warning = false
 end
 
 desc "Code coverage detail"

data/VERSION

@@ -1 +1 @@
-10.4.0
+10.7.0

data/bin/scout

@@ -6,8 +6,6 @@ ENV["SCOUT_NOCOLOR"] = "true" if ARGV.include? "--nocolor"
 
 ENV["SCOUT_NO_PROGRESS"] = "true" if ARGV.include? "--nobar"
 
-require 'scout-gear'
-
 class CmdStop < Exception
   attr_accessor :exit_status
   def initialize(exit_status = 0)
@@ -54,7 +52,7 @@ if dev_dir
   end
 end
 
-require 'scout'
+require 'scout-gear'
 
 require 'scout/simple_opt'
 

data/lib/scout/association/fields.rb

@@ -0,0 +1,170 @@
+require 'scout/entity'
+
+module Association
+
+  def self.identify_entity_format(format, fields)
+    entity_type = Entity.formats[format]
+    raise "Field #{ format } could not be resolved: #{fields}" if entity_type.nil?
+    main_field = fields.select{|f| Entity.formats[f] == entity_type}.first
+    raise "Field #{ format } not present, options: #{Log.fingerprint fields}" if main_field.nil?
+    [main_field, nil, format]
+  end
+
+  def self.parse_field_specification(spec)
+    return [spec,nil,nil] if Numeric === spec
+    spec = spec.split "=>" unless Array === spec
+    field_part, final_format = spec
+
+    field, format = field_part.split "=~", -1
+
+    field = nil if field.nil? or field.empty?
+
+    [field, format, final_format]
+  end
+
+  def self.normalize_specs(spec, all_fields = nil)
+    return nil if spec.nil?
+    field, header, format = parse_field_specification spec 
+
+    specs = if all_fields.nil? or all_fields.include? field
+               [field, header, format]
+             else
+               if all_fields.nil?
+                 begin
+                   identify_entity_format field, all_fields 
+                 rescue
+                   [field, header, format]
+                 end
+               else
+                 [field, header, format]
+               end
+             end
+    specs
+  end
+
+  def self.extract_specs(all_fields=nil, options = {})
+    source, source_format, target, target_format, format = IndiferentHash.process_options options, :source, :source_format, :target, :target_format, :format
+
+    key_field, *fields = all_fields.nil? ? [nil] : all_fields
+
+    source_specs = normalize_specs  source, all_fields
+    target_specs = normalize_specs  target, all_fields
+
+    source_specs = [nil, nil, nil] if source_specs.nil?
+    target_specs = [nil, nil, nil] if target_specs.nil?
+
+    source_specs[2] = source_format if source_format
+    target_specs[2] = target_format if target_format
+
+    if source_specs.first and not all_fields.include? source_specs.first and defined? Entity and (_format = Entity.formats[source_specs.first.to_s])
+      _source = all_fields.select{|f| Entity.formats[f.to_s] == _format }.first
+      raise "Source not found #{source_specs}. Options: #{Log.fingerprint all_fields}" if _source.nil?
+      source_specs[0] = _source
+    end
+
+    if target_specs.first and  not all_fields.include? target_specs.first and defined? Entity and (_format = Entity.formats[target_specs.first.to_s])
+      _target = all_fields.select{|f| Entity.formats[f.to_s].to_s == _format.to_s }.first
+      raise "Target not found #{target_specs}. Options: #{Log.fingerprint all_fields}" if _target.nil?
+      target_specs[0] = _target
+    end
+
+    if source_specs[0].nil? and target_specs[0].nil?
+      source_specs[0] = key_field 
+      target_specs[0] = fields[0]
+    elsif source_specs[0].nil? 
+      if target_specs[0] == :key or target_specs[0] == key_field
+        source_specs[0] = fields[0]
+      else
+        source_specs[0] = key_field
+      end
+    elsif target_specs[0].nil? 
+      if source_specs[0] == fields.first 
+        target_specs[0] = key_field
+      else
+        target_specs[0] = fields.first 
+      end
+    end
+
+    # If format is specified, then perhaps we need to change the
+    if target_specs[2].nil? 
+      target_type = Entity.formats[target_specs[1] || target_specs[0]]
+      target_specs[2] = format[target_type.to_s] if format
+      target_specs[2] = nil if target_specs[2] == target_specs[0] or target_specs[2] == target_specs[1]
+    end
+
+    if source_specs[2].nil? 
+      source_type = Entity.formats[source_specs[1] || source_specs[0]]
+      source_specs[2] = format[source_type.to_s] if format
+      source_specs[2] = nil if source_specs[2] == source_specs[0] or source_specs[2] == source_specs[1]
+    end
+
+    {:source => source_specs, :target => target_specs}
+  end
+
+  def self.process_formats(field, default_format = {})
+    return nil if default_format.nil? or default_format.empty?
+    default_format.each do |type, format|
+      entity_type = Entity.formats[field] || format
+      return format if entity_type.to_s === type 
+    end
+    return nil
+  end
+
+  def self.headers(all_fields, info_fields = nil, options = {})
+    specs = extract_specs all_fields, options
+
+    source_field = specs[:source][0]
+    target_field = specs[:target][0]
+
+    #source_pos = all_fields.index source_field
+    #target_pos = all_fields.index target_field
+   
+    source_pos = TSV.identify_field all_fields.first, all_fields[1..-1], source_field
+    target_pos = TSV.identify_field all_fields.first, all_fields[1..-1], target_field
+
+    source_pos = source_pos == :key ? 0 : source_pos + 1
+    target_pos = target_pos == :key ? 0 : target_pos + 1
+
+    source_header = specs[:source][1] || specs[:source][0]
+    target_header = specs[:target][1] || specs[:target][0]
+
+    info_fields = all_fields.dup if info_fields.nil?
+    info_fields.delete_at NamedArray.identify_name(info_fields, source_field) if NamedArray.identify_name(info_fields, source_field)
+    info_fields.delete_at NamedArray.identify_name(info_fields, target_field) if NamedArray.identify_name(info_fields, target_field)
+    info_fields.unshift target_field
+
+    field_headers = [target_header] 
+    info_fields[1..-1].each do |field|
+      header = case field
+               when String 
+                 field
+               when Numeric
+                 all_fields[field] 
+               when :key
+                 all_fields.first
+               end
+
+      field_headers << header
+    end
+
+    field_pos = info_fields.collect do |f| 
+      p = TSV.identify_field all_fields.first, all_fields[1..-1], f
+      p == :key ? 0 : p + 1
+    end
+
+    field_pos.delete source_pos
+
+    source_format = specs[:source][2]
+    target_format = specs[:target][2]
+
+
+    if format = options[:format]
+      source_format = process_formats(specs[:source][1] || specs[:source][0], format) || source_format unless source_format
+      target_format = process_formats(specs[:target][1] || specs[:target][0], format) || target_format unless target_format
+    end
+
+    res = [source_pos, field_pos, source_header, field_headers, source_format, target_format]
+    Log.low "Headers -- #{res}"
+    res
+  end
+end

data/lib/scout/association/index.rb

@@ -0,0 +1,229 @@
+require 'scout/annotation'
+module Association
+
+  def self.index(file, source: nil, target: nil, source_format: nil, target_format: nil, format: nil, **kwargs)
+    persist_options = IndiferentHash.pull_keys kwargs, :persist
+    index_persist_options = IndiferentHash.add_defaults persist_options.dup, persist: true, 
+      prefix: "Association::Index", 
+      other_options: kwargs.merge(source: source, target: target, source_format: source_format, target_format: target_format, format: format)
+
+    index = Persist.tsv(file, kwargs, engine: "BDB", persist_options: index_persist_options) do |data|
+      recycle, undirected = IndiferentHash.process_options kwargs, :recycle, :undirected
+
+      database = Association.open(file, source: source, target: target, source_format: source_format, target_format: target_format, **kwargs.merge(persist_prefix: "Association::Database"))
+
+      source_field = database.key_field
+      target_field, *fields = database.fields
+
+      undirected = true if undirected.nil? and source_field == target_field
+
+      key_field = [source_field, target_field, undirected ? "undirected" : nil].compact * "~"
+
+      dumper = TSV::Dumper.new database.options.merge(key_field: key_field, fields: fields, type: :list)
+      transformer = TSV::Transformer.new database, dumper
+
+      if database.type == :double
+        transformer.traverse do |source,value_list|
+          res = []
+          NamedArray.zip_fields(value_list).collect do |values|
+            target, *info = values
+            key = [source, target] * "~"
+            res << [key, info]
+            if undirected
+              key = [target, source] * "~"
+              res << [key, info]
+            end
+          end
+          res.extend MultipleResult
+        end
+      elsif database.type == :flat
+        transformer.traverse do |source,targets|
+          res = []
+          res.extend MultipleResult
+          targets.each do |target|
+            key = [source, target] * "~"
+            res << [key, []]
+            if undirected
+              key = [target, source] * "~"
+              res << [key, []]
+            end
+          end
+          res
+        end
+      else
+        transformer.traverse do |source,values|
+          res = []
+          res.extend MultipleResult
+          target, *info = values
+          key = [source, target] * "~"
+          res << [key, info]
+          if undirected
+            key = [target, source] * "~"
+            res << [key, info]
+          end
+          res
+        end
+      end
+
+      tsv = transformer.tsv **kwargs.merge(data: data, fields: fields)
+    end
+    index.extend Index
+    index.parse_key_field
+    index
+  end
+
+  module Index
+    extend Annotation
+
+    annotation :source_field, :target_field, :undirected
+
+    def parse_key_field
+      @source_field, @target_field, @undirected = key_field.split("~")
+    end
+
+    def match(entity)
+      return entity.inject([]){|acc,e| acc.concat match(e); acc } if Array === entity
+      return [] if entity.nil?
+      prefix(entity + "~")
+    end
+
+    def subset(source, target)
+      return [] if source.nil? or target.nil? or source.empty? or target.empty?
+
+      if source == :all or source == "all"
+        if target == :all or target == "all"
+          return keys
+        else
+          matches = reverse.subset(target, source)
+          return matches.collect{|m| r = m.partition "~"; r.reverse*"" }
+        end
+      end
+
+      matches = source.uniq.inject([]){|acc,e| 
+        if block_given?
+          acc.concat(match(e))
+        else
+          acc.concat(match(e))
+        end
+      }
+
+      return matches if target == :all or target == "all"
+
+      target_matches = {}
+
+      matches.each{|code| 
+        s,sep,t = code.partition "~"
+        next if undirected and t > s and source.include? t
+        target_matches[t] ||= []
+        target_matches[t] << code
+      }
+
+      target_matches.values_at(*target.uniq).flatten.compact
+    end
+
+    def reverse
+      @reverse ||= begin
+                     if self.respond_to? :persistence_path
+                       persistence_path = self.persistence_path
+                       persistence_path = persistence_path.find if Path === persistence_path
+                       reverse_filename = persistence_path + '.reverse'
+                     else
+                       raise "Can only reverse a TokyoCabinet::BDB dataset at the time"
+                     end
+
+                     if Open.exist?(reverse_filename)
+                       new = Persist.open_tokyocabinet(reverse_filename, false, serializer, TokyoCabinet::BDB)
+                       raise "Index has no info: #{reverse_filename}" if new.key_field.nil?
+                       new.extend Index
+                       new
+                     else
+                       FileUtils.mkdir_p File.dirname(reverse_filename) unless File.exist?(File.dirname(reverse_filename))
+
+                       new = Persist.open_tokyocabinet(reverse_filename, true, serializer, TokyoCabinet::BDB)
+
+                       self.with_unnamed do
+                         self.traverse do |key, value|
+                           new_key = key.split("~").reverse.join("~")
+                           new[new_key] = value
+                         end
+                       end
+                       annotate(new)
+                       new.key_field = key_field.split("~").values_at(1,0,2).compact * "~"
+                       new.save_annotation_hash
+                       new.read_and_close do
+                         Association::Index.setup new
+                       end
+                       new.parse_key_field
+                       new.read
+                     end
+
+                     new.unnamed = true
+
+                     new.undirected = undirected
+
+                     new
+                   rescue Exception
+                     Log.error "Deleting after error reversing database: #{ reverse_filename }"
+                     FileUtils.rm reverse_filename if File.exist? reverse_filename
+                     raise $!
+                   end
+    end
+
+    def filter(value_field = nil, target_value = nil, &block)
+      if block_given?
+        matches = []
+        if value_field
+          through :key, value_field do |key,values|
+            pass = block.call values
+            matches << key if pass
+          end
+        else
+          through do |key,values|
+            pass = block.call [key, values]
+            matches << key if pass
+          end
+        end
+        matches
+
+      else
+        matches = []
+        if target_value
+          target_value = [target_value] unless Array === target_value
+          through :key, value_field do |key,values|
+            pass = (values & target_value).any?
+            matches << key if pass
+          end
+        else
+          through :key, value_field do |key,values|
+            pass = false
+            values.each do |value|
+              pass = true unless value.nil? or value.empty? or value.downcase == 'false'
+            end
+            matches << key if pass
+          end
+        end
+        matches
+      end
+    end
+
+    def to_matrix(value_field = nil, &block)
+      value_field = fields.first if value_field.nil? and fields.length == 1
+      value_pos = identify_field value_field if value_field and String === value_field
+      key_field = source_field
+
+      tsv = if value_pos
+              AssociationItem.incidence self.keys, key_field do |key|
+                if block_given? 
+                  yield self[key][value_pos]
+                else
+                  self[key][value_pos]
+                end
+              end
+            elsif block_given?
+              AssociationItem.incidence self.keys, key_field, &block
+            else
+              AssociationItem.incidence self.keys, key_field 
+            end
+    end
+  end
+end

data/lib/scout/association/item.rb

@@ -0,0 +1,227 @@
+require_relative '../entity'
+
+module AssociationItem
+  extend Entity
+
+  annotation :knowledge_base
+  annotation :database
+  annotation :reverse
+
+  property :name => :single do
+    [source_entity, target_entity].collect{|e| e.respond_to?(:name)? e.name || e : e } * "~"
+  end 
+
+  property :full_name => :single do
+    database ? [database, name] * ":" : name
+  end 
+
+  property :invert => :both do
+    if Array === self
+      inverted = self.collect do |item|
+        s,_sep,t= item.partition "~"
+        new = [t,s] * _sep
+      end
+      self.annotate inverted
+      inverted.reverse = ! reverse
+      inverted
+    else
+      s,_sep,t= self.partition "~"
+      inverted = self.annotate([t,s] * _sep)
+      inverted.reverse = ! reverse
+      inverted
+    end
+  end
+
+  property :namespace => :both do
+    knowledge_base.namespace
+  end
+
+  property :part => :array2single do
+    self.purge.collect{|p| p.partition("~") }
+  end
+
+  property :target => :array2single do
+    self.part.collect{|p| p[2]}
+  end
+
+  property :source => :array2single do
+    self.purge.collect{|p| p[/[^~]+/] }
+  end
+
+  property :target_entity_type => :both do
+    Entity.formats[target_type].to_s
+  end
+
+  property :source_entity_type => :both do
+    Entity.formats[source_type].to_s
+  end
+
+  property :target_type => :both do
+    if reverse
+      knowledge_base.source(database)
+    else
+      knowledge_base.target(database)
+    end
+  end
+
+  property :source_type => :both do
+    if reverse
+      knowledge_base.target(database)
+    else
+      knowledge_base.source(database)
+    end
+  end
+
+  property :undirected => :both do
+    knowledge_base.undirected(database)
+  end
+
+  property :target_entity => :array2single do
+    type = target_type
+    knowledge_base.annotate self.target, type, database #if self.target.any?
+  end
+
+  property :source_entity => :array2single do
+    type = source_type
+    knowledge_base.annotate self.source, type, database #if self.source.any?
+  end
+
+  property :index => :both do |database|
+    @index ||= knowledge_base.get_index(database)
+  end
+  property :value => :array2single do
+    index = index(database)
+    value = self.reverse ? index.chunked_values_at(self.invert) : index.chunked_values_at(self)
+    value.collect{|v| NamedArray.setup(v, index.fields)}
+  end
+
+  property :info_fields => :both do
+    knowledge_base.index_fields(database)
+  end
+
+  property :info => :array2single do
+    fields = self.info_fields
+
+    return [{}] * self.length if fields.nil? or fields.empty?
+
+    value = self.value
+    value.collect{|v|
+      raise "No info for pair; not registered in index" if v.nil?
+      Hash[*fields.zip(v).flatten]
+    }
+  end
+
+  property :tsv => :array do
+    info_fields = self.info_fields
+    fields = [self.source_type, self.target_type].concat info_fields
+    type = [self.source_type, self.target_type] * "~"
+    tsv = TSV.setup({}, :key_field => type, :fields => fields, :type => :list, :namespace => self.namespace)
+    index = index(database)
+    index.with_unnamed do
+      index.chunked_values_at(self).each_with_index do |v,i|
+        p = self[i]
+        source, _sep, target = p.partition("~")
+        if info_fields.empty?
+          tsv[p] = [source, target]
+        else
+          tsv[p] = [source, target].concat v 
+        end
+      end
+    end
+    tsv.entity_options = {:namespace => namespace}
+    knowledge_base.entity_options.each do |type,options|
+      tsv.entity_options.merge! options
+    end
+    tsv
+  end
+
+  property :filter => :array do |*args,&block|
+    block = args.pop if Proc === args.last
+    keys = tsv.with_unnamed do tsv.select(*args, &block).keys end
+    keys = self.annotate keys
+    keys
+  end
+
+  def self.incidence(pairs, key_field = nil, &block)
+    matrix = {}
+    targets = []
+    sources = []
+    matches = {}
+
+    pairs.inject([]){|acc,m| acc << m; acc << m.invert if m.respond_to?(:undirected) and m.undirected; acc  }.each do |p|
+      s, sep, t = p.partition "~"
+
+      sources << s
+      targets << t
+      if block_given?
+        matches[s] ||= Hash.new{nil}
+        value = block.call p
+        matches[s][t] = value unless value.nil? or (mv = matches[s][t] and value > mv)
+      else
+        matches[s] ||= Hash.new{false}
+        matches[s][t] ||= true 
+      end
+    end
+
+    sources.uniq!
+    targets = targets.uniq.sort
+
+    matches.each do |s,hash|
+      matrix[s] = hash.values_at(*targets)
+    end
+
+    defined?(TSV)? TSV.setup(matrix, :key_field => (key_field || "Source") , :fields => targets, :type => :list) : matrix
+  end
+
+  def self.adjacency(pairs, key_field = nil, &block)
+    incidence = incidence(pairs, key_field, &block)
+
+    targets = incidence.fields
+    adjacency = TSV.setup({}, :key_field => incidence.key_field, :fields => ["Target"], :type => :double)
+    TSV.traverse incidence, :into => adjacency, :unnamed => true do |k,values|
+      target_values = targets.zip(values).reject{|t,v| v.nil? }.collect{|t,v| [t,v]}
+      next if target_values.empty?
+      [k, Misc.zip_fields(target_values)]
+    end
+  end
+
+  def self._select_match(orig, elem)
+    if Array === orig and Array === elem
+      (orig & elem).any?
+    elsif Array === orig
+      orig.include? elem
+    elsif Array === elem
+      elem.include? orig
+    else
+      elem === orif
+    end
+    false
+  end
+
+  def self.select(list, method = nil, &block)
+    if method and method.any?
+      list.select do |item|
+        method.collect do |key,value|
+          case key
+          when :target
+            _select_match item.target, value
+          when :source
+            _select_match item.source, value
+          else
+            orig = item.info[key]
+            orig = orig.split(";;") if String orig
+            _select_match orig, value 
+          end
+        end
+      end
+    else
+      list.select(&block)
+    end
+  end
+end
+
+module TSV
+  def self.incidence(tsv, **kwargs)
+    AssociationItem.incidence Association.index(tsv, **kwargs).keys
+  end
+end

data/lib/scout/association/util.rb

@@ -0,0 +1,7 @@
+module Association
+  def self.version_file(file, namespace)
+    old_file, file = file, file.sub(Entity::Identified::NAMESPACE_TAG, namespace) if namespace and String === file
+    old_file.annotate file if Path === old_file
+    file
+  end
+end