scout-gear 7.3.0 → 8.1.0

data/lib/scout/persist/serialize.rb

@@ -1,5 +1,6 @@
 require_relative '../open'
 require_relative 'open'
+require 'set'
 
 module Persist
   TRUE_STRINGS = Set.new ["true", "True", "TRUE", "t", "T", "1", "yes", "Yes", "YES", "y", "Y", "ON", "on"] unless defined? TRUE_STRINGS
@@ -19,7 +20,7 @@ module Persist
     type = type.to_sym if String === type
     type = SERIALIZER if type == :serializer
     case type
-    when nil, :string, :integer, :float, :boolean, :file, :path
+    when nil, :string, :text, :integer, :float, :boolean, :file, :path, :select, :folder, :binary
       if IO === content || StringIO === content
         content.read
       else
@@ -46,8 +47,9 @@ module Persist
   def self.deserialize(serialized, type)
     type = type.to_sym if String === type
     type = SERIALIZER if type == :serializer
+
     case type
-    when nil, :string, :file, :stream
+    when nil, :string, :text, :file, :stream, :select, :folder
       serialized
     when :path
       Path.setup(serialized)
@@ -56,7 +58,7 @@ module Persist
     when :float
       serialized.to_f
     when :boolean
-      TRUE_STRINGS.include? serialized
+      TRUE_STRINGS.include? serialized.strip
     when :array
       serialized.split("\n")
     when :yaml
@@ -99,9 +101,18 @@ module Persist
         return save_drivers[type].call(file, content)
       end
     end
-    serialized = serialize(content, type)
-    Open.sensible_write(file, serialized, :force => true)
-    return nil
+
+    if type == :binary
+      content.force_encoding("ASCII-8BIT") if content.respond_to? :force_encoding
+      Open.open(path, :mode => 'wb') do |f|
+        f.puts content
+      end
+      content
+   else
+      serialized = serialize(content, type)
+      Open.sensible_write(file, serialized, :force => true)
+      return nil
+    end
   end
 
   def self.load(file, type = :serializer)
@@ -118,6 +129,8 @@ module Persist
     end
 
     case type
+    when :binary
+      Open.read(file, :mode => 'rb')
     when :yaml
       Open.yaml(file)
     when :json

data/lib/scout/persist.rb

@@ -21,7 +21,9 @@ module Persist
   def self.persistence_path(name, options = {})
     options = IndiferentHash.add_defaults options, :dir => Persist.cache_dir
     other_options = IndiferentHash.pull_keys options, :other
-    TmpFile.tmp_for_file(name, options, other_options)
+    name = name.filename if name.respond_to?(:filename) && name.filename
+    persist_options = {}
+    TmpFile.tmp_for_file(name, options.merge(persist_options), other_options)
   end
 
   MEMORY_CACHE = {}
@@ -31,25 +33,31 @@ module Persist
     return yield if FalseClass === persist_options[:persist]
     file = persist_options[:path] || options[:path] || persistence_path(name, options)
 
-    lockfile = persist_options[:lockfile] || options[:lockfile] || Persist.persistence_path(file + '.persist', {:dir => Persist.lock_dir})
-
-    update = options[:update] || persist_options[:update]
-    update = Open.mtime(update) if Path === update
-    update = Open.mtime(file) >= update ? false : true if Time === update
-
     if type == :memory
       repo = options[:memory] || options[:repo] || MEMORY_CACHE
       repo[file] ||= yield
       return repo[file]
     end
 
+    update = options[:update] || persist_options[:update]
+    update = Open.mtime(update) if Path === update
+    update = Open.mtime(file) >= update ? false : true if Time === update
+
+    lockfile = persist_options[:lockfile] || options[:lockfile] || Persist.persistence_path(file + '.persist', {:dir => Persist.lock_dir})
+
     Open.lock lockfile do |lock|
       if Open.exist?(file) && ! update
         Persist.load(file, type)
       else
-        return yield(file) if block.arity == 1
-        res = yield
         begin
+          file = file.find if Path === file
+          return yield(file) if block.arity == 1
+          res = yield
+
+          if res.nil?
+            return Persist.load(file, type)
+          end
+
           Open.rm(file)
 
           if IO === res || StringIO === res
@@ -72,17 +80,25 @@ module Persist
             pres = Persist.save(res, file, type)
             res = pres unless pres.nil?
           end
-        rescue
+        rescue Exception
+          Thread.handle_interrupt(Exception => :never) do
+            if Open.exist?(file)
+              Log.debug "Failed persistence #{file} - erasing"
+              Open.rm file
+            else
+              Log.debug "Failed persistence #{file}"
+            end
+          end
           raise $! unless options[:canfail]
-          Log.debug "Could not persist #{type} on #{file}"
         end
         res
       end
     end
   end
 
-  def self.memory(name, *args, &block)
-    self.persist(name, :memory, *args, &block)
+  def self.memory(name, options = {}, &block)
+    options[:persist_path] ||= options[:path] ||= [name, options[:key]].compact * ":"
+    self.persist(name, :memory, options, &block)
   end
 
 end

data/lib/scout/resource/path.rb

@@ -1,9 +1,56 @@
 module Path
+  def produce(force = false)
+    return self if ! force && (Open.exist?(self) || @produced)
+    begin
+      if Resource === self.pkgdir
+        self.pkgdir.produce self
+      else
+        false
+      end
+    rescue ResourceNotFound
+      false
+    rescue
+      message = $!.message
+      message = "No exception message" if message.nil? || message.empty?
+      Log.warn "Error producing #{self}: #{message}"
+      false
+    ensure
+      @produced = true
+    end
+  end
+
+  def produce_with_extension(extension, *args)
+    begin
+      self.produce(*args)
+    rescue Exception
+      exception = $!
+      begin
+        self.set_extension(extension).produce(*args)
+      rescue Exception
+        raise exception
+      end
+    end
+  end
+
+  def produce_and_find(extension = nil, *args)
+    if extension
+      found = find_with_extension(extension, *args)
+      found.exists? ? found : produce_with_extension(extension, *args)
+    else
+      found = find
+      found.exists? ? found : produce(*args)
+    end
+  end
+
   def relocate
     return self if Open.exists?(self)
     Resource.relocate(self)
   end
 
+  def identify
+    Resource.identify(self)
+  end
+
   def open(*args, &block)
     produce
     Open.open(self, *args, &block)
@@ -17,4 +64,14 @@ module Path
   def write(*args, &block)
     Open.write(self.find, *args, &block)
   end
+
+  def list
+    found = produce_and_find('list')
+    Open.list(found)
+  end
+
+  def exists?
+    return true if Open.exists?(self.find)
+    self.produce
+  end
 end

data/lib/scout/resource/produce.rb

@@ -149,11 +149,3 @@ module Resource
   end
 
 end
-
-module Path
-  def produce(force = false)
-    return self if ! force && Open.exist?(self)
-    self.pkgdir.produce self if Resource === self.pkgdir
-    return self
-  end
-end

data/lib/scout/resource/util.rb

@@ -1,13 +1,13 @@
 module Resource
   def identify(path)
     return path unless path.start_with?("/")
-    path_maps = path.path_maps || self.path_maps || Path.path_maps
+    path_maps = path.path_maps if Path === path
+    path_maps ||= self.path_maps || Path.path_maps
     path = File.expand_path(path)
     path += "/" if File.directory?(path)
 
     map_order ||= (path_maps.keys & Path.basic_map_order) + (path_maps.keys - Path.basic_map_order)
     map_order -= [:current, "current"]
-    map_order << :current
 
     choices = []
     map_order.uniq.each do |name|
@@ -16,6 +16,7 @@ module Resource
       next if pattern.nil?
 
       pattern = pattern.sub('{PWD}', Dir.pwd)
+      pattern = pattern.sub('{HOME}', ENV["HOME"])
       if String ===  pattern and pattern.include?('{')
         regexp = "^" + pattern
           .gsub(/{(TOPLEVEL)}/,'(?<\1>[^/]+)')
@@ -34,14 +35,20 @@ module Resource
       end
     end
 
-    Path.setup(choices.sort_by{|s| s.length }.first, self, nil, path_maps)
+    identified = choices.sort_by{|s| s.length }.first
+
+    Path.setup(identified || path, self, nil, path_maps)
   end
 
-  def self.relocate(path)
-    return path if Open.exists?(path)
+  def self.identify(path)
     resource = path.pkgdir if Path === path
     resource = Scout unless Resource === resource
     unlocated = resource.identify path
+  end
+
+  def self.relocate(path)
+    return path if Open.exists?(path)
+    unlocated = identify(path)
     unlocated.find
   end
 end

data/lib/scout/tmpfile.rb

@@ -93,19 +93,18 @@ module TmpFile
     end
   end
 
+  SLASH_REPLACE = '·'
   def self.tmp_for_file(file, tmp_options = {}, other_options = {})
-    tmp_for_file = IndiferentHash.process_options tmp_options, :file
+    tmp_for_file, prefix, key, persistence_dir = IndiferentHash.process_options tmp_options, :file, :prefix, :key, :dir
     return tmp_for_file unless tmp_for_file.nil?
 
-    prefix = IndiferentHash.process_options tmp_options, :prefix
-
     if prefix.nil?
-      perfile = file.to_s.gsub(/\//, '>') 
+      perfile = file.to_s.sub(/\.b?gz$/,'')
     else
-      perfile = prefix.to_s + ":" + file.to_s.gsub(/\//, '>') 
+      perfile = prefix.to_s + ":" + file.to_s.sub(/\.b?gz$/,'')
     end
 
-    perfile.sub!(/\.b?gz$/,'')
+    perfile += "[#{ key }]" if key
 
     if other_options.include? :filters
       other_options[:filters].each do |match,value|
@@ -113,10 +112,10 @@ module TmpFile
       end
     end
 
-    persistence_dir = IndiferentHash.process_options(tmp_options, :dir) || TmpFile.tmpdir 
+    persistence_dir = TmpFile.tmpdir if persistence_dir.nil?
     Path.setup(persistence_dir) unless Path === persistence_dir
 
-    filename = perfile.gsub(/\s/,'_').gsub(/\//,'>')
+    filename = perfile.gsub(/\s/,'_').gsub('/', SLASH_REPLACE)
     clean_options = other_options.dup
     clean_options.delete :unnamed
     clean_options.delete "unnamed"

data/lib/scout/tsv/attach.rb

@@ -0,0 +1,177 @@
+module TSV
+
+  def self.match_keys(source, other, match_key: nil, other_key: nil)
+    match_key = (source.all_fields & other.all_fields).first if match_key.nil?
+
+    if match_key.nil?
+      source.all_fields.collect do |f|
+        other_key = other.identify_field(f)
+        if other_key
+          other_key = other.key_field if other_key == :key
+          match_key = f
+          break
+        end
+      end
+    end
+
+    if match_key.nil?
+      other.all_fields.collect do |f|
+        match_key = source.identify_field(f)
+        if match_key
+          other_key = f
+          break
+        end
+      end
+    end
+
+    match_key = source.key_field if match_key.nil? 
+
+    if other_key.nil?
+      other_key = other.identify_field(match_key)
+    end
+
+    other_key = other.key_field if other_key.nil?
+
+    match_key = :key if match_key == source.key_field
+    other_key = :key if other_key == other.key_field
+
+    [match_key, other_key]
+  end
+
+  def self.attach(source, other, target: nil, fields: nil, match_key: nil, other_key: nil, one2one: true, complete: false, insitu: nil, persist_input: false, bar: nil)
+    source = TSV::Transformer.new source unless TSV === source || TSV::Parser === source
+    other = TSV.open other, persist: persist_input unless TSV === other 
+
+    fields = [fields] if String === fields
+
+    match_key, other_key = TSV.match_keys(source, other, match_key: match_key, other_key: other_key)
+
+    if TSV::Transformer === source
+      source.dumper = case target
+                      when :stream
+                        TSV::Dumper.new(source.options.merge(sep: "\t"))
+                      when nil
+                        TSV.setup({}, **source.options.dup)
+                      else
+                        target
+                      end
+    end
+
+    other.with_unnamed do
+      source.with_unnamed do
+
+        other_key_name = other_key == :key ? other.key_field : other_key
+        other_key_name = other.fields[other_key_name] if Integer === other_key
+        fields = other.all_fields - [other_key_name, source.key_field] if fields.nil?
+
+        if other_key != :key 
+          other = other.reorder other_key, fields, one2one: one2one
+        end
+
+        other_field_positions = other.identify_field(fields) 
+
+        log_message = "Attach #{Log.fingerprint fields - source.fields} to #{Log.fingerprint source} (#{[match_key, other_key] * "=~"})"
+        Log.debug log_message
+        bar = log_message if TrueClass === bar
+
+        new = fields - source.fields
+
+        source.fields = (source.fields + fields).uniq
+
+        overlaps = source.identify_field(fields)
+
+        empty_other_values = case source.type
+                             when :list
+                               [nil] * other.fields.length
+                             when :flat
+                               []
+                             when :double
+                               [[]] * other.fields.length
+                             end
+
+        insitu = TSV === source ? true : false if insitu.nil?
+
+        match_key_pos = source.identify_field(match_key)
+        source.traverse bar: bar, unnamed: true do |orig_key,current_values|
+          keys = (match_key == :key || match_key_pos == :key) ? [orig_key] : current_values[match_key_pos]
+          keys = [keys] unless Array === keys
+
+          current_values = current_values.dup unless insitu
+          keys.each do |current_key|
+            other_values = other[current_key]
+
+            if other_values.nil?
+              other_values = empty_other_values
+            elsif other.type == :flat
+              other_values = [other_values]
+            elsif other.type == :list && source.type == :double
+              other_values = other_values.collect{|v| [v] }
+            elsif other.type == :double && source.type == :list
+              other_values = other_values.collect{|v| v.first }
+            end
+
+            other_values = other_values.values_at *other_field_positions
+
+            other_values.zip(overlaps).each do |v,overlap|
+              if source.type == :list
+                current_values[overlap] = v if current_values[overlap].nil? || String === current_values[overlap] && current_values[overlap].empty?
+              else
+                current_values[overlap] ||= []
+                current_values[overlap].concat (v - current_values[overlap])
+              end
+            end
+          end
+          source[orig_key] = current_values unless insitu
+          nil
+        end
+
+        if complete && match_key == :key
+          empty_self_values = case source.type
+                              when :list
+                                [nil] * source.fields.length
+                              when :flat
+                                []
+                              when :double
+                                [[]] * source.fields.length
+                              end
+          other.each do |other_key,other_values|
+            next if source.include?(other_key)
+            if other.type == :flat
+              other_values = [other_values]
+            elsif other.type == :list && source.type == :double
+              other_values = other_values.collect{|v| [v] }
+            elsif other.type == :double && source.type == :list
+              other_values = other_values.collect{|v| v.first }
+            end
+
+            new_values = case source.type
+                         when :list
+                           [nil] * source.fields.length
+                         when :flat
+                           []
+                         when :double
+                           source.fields.length.times.collect{ [] }
+                         end
+
+            other_values.zip(overlaps).each do |v,overlap|
+              if false && overlap == :key
+                other_key = Array === v ? v : v.first
+              elsif source.type == :list
+                new_values[overlap] = v if v[overlap].nil? || String === v[overlap] && v[overlap].empty?
+              else
+                new_values[overlap].concat v
+              end
+            end
+            source[other_key] = new_values
+          end
+        end
+      end
+    end
+
+    source
+  end
+
+  def attach(*args, **kwargs)
+    TSV.attach(self, *args, **kwargs)
+  end
+end

data/lib/scout/tsv/change_id.rb

@@ -0,0 +1,40 @@
+module TSV
+  def self.change_key(source, new_key_field, identifiers: nil, one2one: false, stream: false, keep: false, persist_identifiers: nil)
+    source = TSV::Parser.new source if String === source
+    if identifiers && source.identify_field(new_key_field, strict: true).nil?
+      identifiers = identifiers.nil? ? source.identifiers : identifiers
+      new = source.attach(identifiers, fields: [new_key_field], insitu: false, one2one: true, persist_input: persist_identifiers)
+      new = new.change_key(new_key_field, keep: keep, stream: stream, one2one: one2one)
+      return new
+    end
+
+    fields = source.fields.dup - [new_key_field]
+    fields.unshift source.key_field if keep
+    transformer = TSV::Transformer.new source
+    transformer.key_field = new_key_field
+    transformer.fields = fields
+    transformer.traverse key_field: new_key_field, fields: fields, one2one: one2one, unnamed: true do |k,v|
+      [k, v]
+    end
+
+    stream ? transformer : transformer.tsv
+  end
+
+  def change_key(*args, **kwargs)
+    TSV.change_key(self, *args, **kwargs)
+  end
+
+  def self.change_id(source, source_id, new_id, identifiers: nil, one2one: false, insitu: false)
+    source = TSV::Parser.new source if String === source
+
+    identifiers = identifiers.nil? ? source.identifiers : identifiers
+
+    new_fields = source.fields.dup
+    new_fields[new_fields.index(source_id)] = new_id
+    return source.attach(identifiers, fields: [new_id], insitu: insitu).slice(new_fields)
+  end
+
+  def change_id(*args, **kwargs)
+    TSV.change_id(self, *args, **kwargs)
+  end
+end