scout-gear 10.4.0 → 10.7.0

data/lib/scout/association.rb

@@ -0,0 +1,100 @@
+require_relative 'tsv'
+require_relative 'association/fields'
+require_relative 'association/util'
+require_relative 'association/index'
+require_relative 'association/item'
+
+module Association
+  def self.open(obj, source: nil, target: nil, fields: nil, source_format: nil, target_format: nil, format: nil, **kwargs)
+    all_fields = TSV.all_fields(obj)
+    source_pos, field_pos, source_header, field_headers, source_format, target_format = headers(all_fields, fields, kwargs.merge(source: source, target: target, source_format: source_format, target_format: target_format, format: format))
+
+    original_source_header = all_fields[source_pos]
+    original_field_headers = all_fields.values_at(*field_pos)
+    original_target_header = all_fields[field_pos.first]
+
+    type, identifiers = IndiferentHash.process_options kwargs, :type, :identifiers
+
+    if source_format
+      translation_files = [TSV.identifier_files(obj), Entity.identifier_files(source_format), identifiers].flatten.compact
+      source_index = begin
+                       TSV.translation_index(translation_files, source_header, source_format)
+                     rescue
+                       TSV.translation_index(translation_files, original_source_header, source_format)
+                     end
+    end
+
+    if target_format
+      translation_files = [TSV.identifier_files(obj), Entity.identifier_files(target_format), identifiers].flatten.compact
+      target_index = begin
+                       TSV.translation_index(translation_files, field_headers.first, target_format)
+                     rescue
+                       TSV.translation_index(translation_files, original_target_header, target_format)
+                     end
+    end
+
+    final_key_field = if source_format
+                        if m = original_source_header.match(/(.*) \(.*\)/)
+                          m[1] + " (#{source_format})"
+                        elsif m = source_header.match(/(.*) \(.*\)/)
+                          m[1] + " (#{source_format})"
+                        else
+                          source_format
+                        end
+                      else
+                        if source_header
+                          original_source_header.include?(source_header) ? original_source_header : source_header
+                        else
+                          original_source_header
+                        end
+                      end
+
+    fields = original_field_headers
+    final_target_field = if target_format
+                     if m = original_target_header.match(/(.*) \(.*\)/)
+                       m[1] + " (#{target_format})"
+                     elsif m = field_headers.first.match(/(.*) \(.*\)/)
+                       m[1] + " (#{target_format})"
+                     else
+                       target_format
+                     end
+                   else
+                     target_header = field_headers.first
+                     original_target_header.include?(target_header) ? original_target_header : target_header
+                   end
+    final_fields = [final_target_field] + original_field_headers[1..-1]
+
+    if source_index.nil? && target_index.nil?
+      if TSV === obj
+        IndiferentHash.pull_keys kwargs, :persist
+        type = kwargs[:type] || obj.type
+        res = obj.reorder original_source_header, all_fields.values_at(*field_pos), **kwargs.merge(type: type, merge: true)
+      else
+        res = TSV.open(obj, key_field: original_source_header, fields: all_fields.values_at(*field_pos), **kwargs.merge(type: type))
+      end
+      res.key_field = final_key_field
+      res.fields = final_fields
+
+      return res
+    end
+
+    transformer = TSV::Transformer.new obj
+    transformer.key_field = final_key_field
+    transformer.fields = final_fields
+    transformer.type = type if type
+
+    transformer.traverse key_field: original_source_header, fields: all_fields.values_at(*field_pos) do |k,v|
+      v = v.dup if TSV === obj
+      k = source_index[k] if source_index
+      v[0] = Array === v[0] ? target_index.values_at(*v[0]) : target_index[v[0]] if target_index
+      [k, v]
+    end
+
+    transformer
+  end
+
+  def self.database(*args, **kwargs)
+    tsv = open(*args, **kwargs)
+    TSV::Transformer === tsv ? tsv.tsv(merge: true) : tsv
+  end
+end

data/lib/scout/entity/format.rb

@@ -0,0 +1,62 @@
+module Entity
+  def format=(formats)
+    formats = [formats] unless Array === formats
+    formats.each do |format|
+      Entity.formats[format] ||= self
+    end
+  end
+
+  class FormatIndex < Hash
+
+    alias orig_include? include?
+
+    def initialize
+      @find_cache = {}
+    end
+
+    def find(value)
+      @find_cache ||= {}
+
+      @find_cache[value] ||= begin
+                               if orig_include? value
+                                 @find_cache[value] = value
+                               else
+                                 found = nil
+                                 each do |k,v|
+                                   if value.to_s == k.to_s
+                                     found = k
+                                     break
+                                   elsif value.to_s =~ /\(#{Regexp.quote k}\)/
+                                     found = k
+                                     break
+                                   end
+                                 end
+                                 found
+                               end
+                             end
+    end
+
+    def [](value)
+      res = super
+      return res if res
+      key = find(value)
+      key ? super(key) : nil
+    end
+
+    def []=(key,value)
+      @find_cache = {}
+      super(key, value)
+    end
+
+    def include?(value)
+      find(value) != nil
+    end
+  end
+
+  FORMATS ||= FormatIndex.new
+
+  def self.formats
+    FORMATS
+  end
+
+end

data/lib/scout/entity/identifiers.rb

@@ -0,0 +1,111 @@
+module Entity
+  def self.identifier_files(field)
+    entity_type = Entity.formats[field]
+    return [] unless entity_type and entity_type.include? Entity::Identified 
+    entity_type.identifier_files
+  end
+
+  module Identified
+    NAMESPACE_TAG = 'NAMESPACE'
+
+    def self.included(base)
+      base.annotation :format
+      base.annotation :namespace
+
+      class << base
+        attr_accessor :identifier_files, :formats, :default_format, :name_format, :description_format
+      end
+
+      base.property :to => :both do |target_format|
+
+        target_format = case target_format
+                        when :name
+                          identity_type.name_format 
+                        when :default
+                          identity_type.default_format 
+                        else
+                          target_format
+                        end
+
+        if target_format == format
+          self
+        elsif Array === self
+          self.annotate(identifier_index(target_format, self.format).values_at(*self))
+        else
+          self.annotate(identifier_index(target_format, self.format)[self])
+        end.tap{|o| o.format = target_format unless o.nil? }
+      end
+
+      base.property :name => :both do
+        to(:name)
+      end
+
+      base.property :default => :both do
+        to(:default)
+      end
+    end
+
+    def identifier_files 
+      files = identity_type.identifier_files.dup
+      return [] if files.nil?
+      files.collect!{|f| f.annotate f.gsub(/\b#{NAMESPACE_TAG}\b/, namespace.to_s) } if annotations.include? :namespace and self.namespace
+      if files.select{|f| f =~ /\b#{NAMESPACE_TAG}\b/ }.any?
+        Log.warn "Rejecting some identifier files for lack of 'namespace': " << files.select{|f| f =~ /\b#{NAMESPACE_TAG}\b/ } * ", "
+      end
+      files.reject!{|f| f =~ /\b#{NAMESPACE_TAG}\b/ } 
+      files
+    end
+
+    def identity_type
+      self.annotation_types.select{|m| m.include? Entity::Identified }.last
+    end
+
+    def identifier_index(format = nil, source = nil)
+      Persist.memory("Entity index #{identity_type}: #{format} (from #{source || "All"})", :persist => true, :format => format, :source => source) do
+        source ||= self.respond_to?(:format)? self.format : nil
+
+        begin
+          index = TSV.translation_index(identifier_files, source, format, :persist => true)
+          raise "No index from #{ Misc.fingerprint source } to #{ Misc.fingerprint format }: #{Misc.fingerprint identifier_files}" if index.nil?
+          index.unnamed = true
+          index
+        rescue
+          raise $! if source.nil?
+          source = nil
+          retry
+        end
+      end
+    end
+  end
+
+  def add_identifiers(file, default = nil, name = nil, description = nil)
+    if TSV === file
+      all_fields = file.all_fields
+    else
+      if file =~ /#{Identified::NAMESPACE_TAG}/
+        all_fields = file.sub(/#{Identified::NAMESPACE_TAG}/,'**').glob.collect do |f|
+          TSV.parse_header(f)["all_fields"]
+        end.flatten.compact.uniq
+      else
+        all_fields = TSV.parse_header(file)["all_fields"]
+      end
+    end
+
+    self.send(:include, Entity::Identified) unless Entity::Identified === self
+
+    self.format = all_fields
+    @formats ||= []
+    @formats.concat all_fields
+    @formats.uniq!
+
+    @default_format = default if default
+    @name_format = name if name
+    @description_format = description if description
+
+    @identifier_files ||= []
+    @identifier_files << file
+    @identifier_files.uniq!
+  end
+
+
+end

data/lib/scout/entity/object.rb

@@ -0,0 +1,20 @@
+module Entity
+  module Object
+
+    def entity_classes
+      annotation_types.select{|t| Entity === t}
+    end
+
+    def base_entity
+      entity_classes.last
+    end
+
+    def _ary_property_cache
+      @_ary_property_cache ||= {}
+    end
+
+    def all_properties
+      entity_classes.inject([]){|acc,e| acc.concat(e.properties) }
+    end
+  end
+end

data/lib/scout/entity/property.rb

@@ -0,0 +1,165 @@
+module Entity
+  class << self
+    attr_accessor :entity_property_cache
+
+    def entity_property_cache
+      @entity_property_cache ||= Path.setup('var/entity_property')
+    end
+  end
+
+  module Property
+
+    class MultipleEntityProperty < DontPersist; end
+
+    attr_accessor :persisted_methods, :properties
+
+    def self.persist(name, obj, type, options, &block)
+      return yield unless options[:persist]
+      if (type == :annotation || type == :annotations) && options[:annotation_repo]
+        repo = options[:annotation_repo]
+        Persist.annotation_repo_persist(repo, [name, obj.id] * ":", &block)
+      else
+
+        Persist.persist([name, obj.id] * ":", type, options.dup, &block)
+      end
+    end
+
+    def self.single_method(name)
+      ("_single_" + name.to_s).to_sym
+    end
+     
+    def self.array_method(name)
+      ("_ary_" + name.to_s).to_sym
+    end
+
+    def self.multi_method(name)
+      ("_multi_" + name.to_s).to_sym
+    end
+
+    DEFAULT_PROPERTY_TYPE = :both
+    def property(name_and_type, &block)
+      name, type = case name_and_type
+                   when Symbol, String
+                     [name_and_type.to_sym, DEFAULT_PROPERTY_TYPE]
+                   else
+                     name_and_type.collect.first
+                   end
+
+      real_method = case type
+                    when :single, :single2array
+                      Entity::Property.single_method(name)
+                    when :array, :array2single
+                      Entity::Property.array_method(name)
+                    when :multiple
+                      Entity::Property.multi_method(name)
+                    when :both
+                      name
+                    else
+                      raise "Type of property unknown #{type}"
+                    end
+
+      properties.push name
+
+
+      entity_class = self
+      if type == :multiple
+        self.define_method(real_method) do |*args,**kwargs|
+          if entity_class.persisted_methods && entity_class.persisted_methods[name]
+            type, options = entity_class.persisted_methods[name]
+          else
+            type, options = nil, {persist: false}
+          end
+
+          missing = []
+          responses = {}
+          self.each do |item|
+            begin
+              responses[item] = Entity::Property.persist(name, item, type, options) do
+                raise MultipleEntityProperty 
+              end
+            rescue MultipleEntityProperty
+              missing << item
+            end
+          end
+
+          if missing.any?
+            self.annotate(missing)
+
+            new_responses = missing.instance_exec(*args, **kwargs, &block)
+
+            missing.each do |item,i|
+              responses[item] = Entity::Property.persist(name, item, type, options) do
+                Array === new_responses ? new_responses[item.container_index] : new_responses[item]
+              end
+            end
+          end
+
+          responses.values_at(*self)
+        end
+      else
+        self.define_method(real_method) do |*args,**kwargs|
+          if entity_class.persisted_methods && entity_class.persisted_methods[name]
+            type, options = entity_class.persisted_methods[name]
+          else
+            type, options = nil, {persist: false}
+          end
+
+          Entity::Property.persist(name, self, type, options) do
+            self.instance_exec(*args, **kwargs, &block)
+          end
+        end
+      end
+
+      return if type == :both
+
+      self.define_method(name) do |*args,**kwargs|
+
+        method_type = %w(single_method array_method multi_method).select do |method_type|
+          self.methods.include?(Entity::Property.send(method_type, name))
+        end.first
+
+        real_method = Entity::Property.send(method_type, name)
+
+        if Array === self
+          case method_type
+          when 'single_method'
+            self.collect{|item| item.send(real_method, *args, **kwargs) }
+          when 'array_method', 'multi_method'
+            self.send(real_method, *args, **kwargs)
+          end
+        else
+          case method_type
+          when 'single_method'
+            self.send(real_method, *args, **kwargs)
+          when 'array_method', 'multi_method'
+            if AnnotatedArray.is_contained?(self)
+              cache_code = Misc.digest({:name => name, :args => args})
+              res = (self.container._ary_property_cache[cache_code] ||= self.container.send(real_method, *args, **kwargs))
+              Array === res ? res[self.container_index] : res[self]
+            else
+              res = self.make_array.send(real_method)
+              Array === res ? res[0] : res[self]
+            end
+          end
+        end
+      end
+    end
+
+    def persist(name, type = :marshal, options = {})
+      options = IndiferentHash.add_defaults options, persist: true,
+        dir: File.join(Entity.entity_property_cache, self.to_s, name.to_s)
+      @persisted_methods ||= {}
+      @persisted_methods[name] = [type, options]
+    end
+
+    def persisted?(name)
+      @persisted_methods ||= {}
+      @persisted_methods.include?(name)
+    end
+
+    def unpersist(name)
+      @persisted_methods ||= {}
+      @persisted_methods.delete(name)
+    end
+  end
+end

data/lib/scout/entity.rb

@@ -0,0 +1,41 @@
+require 'scout/annotation'
+require_relative 'entity/format'
+require_relative 'entity/property'
+require_relative 'entity/object'
+require_relative 'entity/identifiers'
+
+module Entity
+  def self.extended(base)
+    base.extend Annotation
+    base.extend Entity::Property
+    base.instance_variable_set(:@properties, [])
+    base.instance_variable_set(:@persisted_methods, {})
+    base.include Entity::Object
+    base.include AnnotatedArray
+    base
+  end
+
+  def self.prepare_entity(entity, field, options = {})
+    return entity unless defined? Entity
+    return entity unless String === entity or Array === entity
+    options ||= {}
+
+    dup_array = options.delete :dup_array
+
+    if Entity === field or (Entity.respond_to?(:formats) and (_format = Entity.formats.find(field)))
+      params = options.dup
+
+      params[:format] ||= params.delete "format"
+      params.merge!(:format => _format) unless _format.nil? or (params.include?(:format) and not ((f = params[:format]).nil? or (String === f and f.empty?)))
+
+      mod = Entity === field ? field : Entity.formats[field]
+
+      entity = entity.dup
+      entity = (entity.frozen? and not entity.nil?) ? entity.dup : ((Array === entity and dup_array) ? entity.collect{|e| e.nil? ? e : e.dup} : entity) 
+
+      entity = mod.setup(entity, params)
+    end
+
+    entity
+  end
+end

data/lib/scout/offsite/step.rb

@@ -4,8 +4,8 @@ require_relative 'sync'
 
 module OffsiteStep
 
-  extend MetaExtension
-  extension_attr :server, :workflow_name, :clean_id, :slurm
+  extend Annotation
+  annotation :server, :workflow_name, :clean_id, :slurm
 
   def inputs_directory
     @inputs_directory ||= begin

data/lib/scout/{tsv/persist → persist/engine}/fix_width_table.rb

@@ -1,24 +1,24 @@
 class FixWidthTable
 
-  attr_accessor :filename, :file, :value_size, :record_size, :range, :size, :mask, :write
-  def initialize(filename, value_size = nil, range = nil, update = false, in_memory = true)
-    filename = filename.find if Path === filename
-    @filename = filename
+  attr_accessor :persistence_path, :file, :value_size, :record_size, :range, :size, :mask, :write
+  def initialize(persistence_path, value_size = nil, range = nil, update = false, in_memory = true)
+    persistence_path = persistence_path.find if Path === persistence_path
+    @persistence_path = persistence_path
 
-    if update || %w(memory stringio).include?(filename.to_s.downcase) || ! File.exist?(filename)
-      Log.debug "FixWidthTable create: #{ filename }"
+    if update || %w(memory stringio).include?(persistence_path.to_s.downcase) || ! File.exist?(persistence_path)
+      Log.debug "FixWidthTable create: #{ persistence_path }"
       @value_size  = value_size
       @range       = range
       @record_size = @value_size + (@range ? 16 : 8)
       @write = true
 
-      if %w(memory stringio).include?(filename.to_s.downcase)
-        @filename = :memory
+      if %w(memory stringio).include?(persistence_path.to_s.downcase)
+        @persistence_path = :memory
         @file     = StringIO.new
       else
-        FileUtils.rm @filename if File.exist? @filename
-        FileUtils.mkdir_p File.dirname(@filename) unless File.exist? @filename
-        @file = File.open(@filename, 'w:ASCII-8BIT')
+        FileUtils.rm @persistence_path if File.exist? @persistence_path
+        FileUtils.mkdir_p File.dirname(@persistence_path) unless File.exist? @persistence_path
+        @file = File.open(@persistence_path, 'w:ASCII-8BIT')
       end
 
       @file.write [value_size].pack("L")
@@ -26,18 +26,18 @@ class FixWidthTable
 
       @size = 0
     else
-      Log.debug "FixWidthTable up-to-date: #{ filename } - (in_memory:#{in_memory})"
+      Log.debug "FixWidthTable up-to-date: #{ persistence_path } - (in_memory:#{in_memory})"
       if in_memory
-        @file        = Open.open(@filename, :mode => 'r:ASCII-8BIT'){|f| StringIO.new f.read}
+        @file        = Open.open(@persistence_path, :mode => 'r:ASCII-8BIT'){|f| StringIO.new f.read}
       else
-        @file        = File.open(@filename, 'r:ASCII-8BIT')
+        @file        = File.open(@persistence_path, 'r:ASCII-8BIT')
       end
       @value_size  = @file.read(4).unpack("L").first
       @range       = @file.read(1).unpack("C").first == 1
       @record_size = @value_size + (@range ? 16 : 8)
       @write = false
 
-      @size        = (File.size(@filename) - 5) / (@record_size)
+      @size        = (File.size(@persistence_path) - 5) / (@record_size)
     end
 
     @mask = "a#{@value_size}"
@@ -47,22 +47,22 @@ class FixWidthTable
     @write
   end
 
-  def persistence_path
-    @filename
+  def filename
+    @persistence_path
   end
 
   def persistence_path=(value)
-    @filename=value
+    @persistence_path=value
   end
 
-  def self.get(filename, value_size = nil, range = nil, update = false)
-    return self.new(filename, value_size, range, update) if filename == :memory
+  def self.get(persistence_path, value_size = nil, range = nil, update = false)
+    return self.new(persistence_path, value_size, range, update) if persistence_path == :memory
     case
-    when (!File.exist?(filename) or update or not Persist::CONNECTIONS.include?(filename))
-      Persist::CONNECTIONS[filename] = self.new(filename, value_size, range, update)
+    when (!File.exist?(persistence_path) or update or not Persist::CONNECTIONS.include?(persistence_path))
+      Persist::CONNECTIONS[persistence_path] = self.new(persistence_path, value_size, range, update)
     end
 
-    Persist::CONNECTIONS[filename] 
+    Persist::CONNECTIONS[persistence_path] 
   end
 
   def format(pos, value)
@@ -113,10 +113,10 @@ class FixWidthTable
   end
 
   def read(force = false)
-    return if @filename == :memory
+    return if @persistence_path == :memory
     @write = false
     @file.close unless @file.closed?
-    @file = File.open(filename, 'r:ASCII-8BIT')
+    @file = File.open(persistence_path, 'r:ASCII-8BIT')
   end
 
   def close
@@ -317,11 +317,3 @@ class FixWidthTable
   alias length size
 end
 
-Persist.save_drivers[:fwt] = proc do |file, content|
-  content.file.seek 0
-  Misc.sensiblewrite(file, content.file.read)
-end
-
-Persist.load_drivers[:fwt] = proc do |file| 
-  FixWidthTable.new file
-end