scout-gear 7.2.0 → 8.0.0

data/lib/scout/tsv/transformer.rb

@@ -0,0 +1,141 @@
+module TSV
+  class Transformer
+    attr_accessor :unnamed, :parser, :dumper
+
+    def initialize(parser, dumper = nil, unnamed: false)
+      if TSV::Parser === parser
+        @parser = parser
+      elsif TSV === parser
+        @parser = parser
+      else
+        @parser = TSV::Parser.new parser
+      end
+      @unnamed = unnamed
+      if dumper.nil?
+        @dumper = TSV::Dumper.new(@parser)
+        @dumper.sep = "\t"
+      else
+        @dumper = dumper
+      end
+    end
+
+    def key_field=(key_field)
+      @dumper.key_field = key_field
+    end
+
+    def fields=(fields)
+      @dumper.fields = fields
+    end
+
+    def type=(type)
+      @dumper.type = type
+    end
+
+    def type
+      @dumper.type
+    end
+
+    def sep=(sep)
+      @dumper.sep = sep
+    end
+
+    def include?(*args)
+      false
+    end
+
+    def key_field
+      @dumper.key_field
+    end
+
+    def fields
+      @dumper.fields
+    end
+
+    def all_fields
+      return nil if fields.nil?
+      [key_field] + fields
+    end
+
+    def options
+      @dumper.options
+    end
+
+    def identify_field(name)
+      TSV.identify_field key_field, fields, name
+    end
+
+    def traverse(*args, **kwargs, &block)
+      kwargs[:into] = @dumper
+      kwargs[:bar] = "Transform #{Log.fingerprint @parser} into #{Log.fingerprint @target}" if TrueClass === kwargs[:bar]
+      @dumper.init if @dumper.respond_to?(:init) && ! @dumper.initialized
+      Log.debug "Transform #{Log.fingerprint @parser} into #{Log.fingerprint @dumper}"
+      Open.traverse(@parser, *args, **kwargs) do |k,v|
+        NamedArray.setup(v, @parser.fields, k) unless @unnamed
+        block.call k, v
+      end
+    end
+
+    def each(*args, **kwargs, &block)
+      kwargs[:into] = @dumper
+      kwargs[:bar] = "Transform #{Log.fingerprint @parser} into #{Log.fingerprint @target}" if TrueClass === kwargs[:bar]
+      @dumper.init if @dumper.respond_to?(:init) && ! @dumper.initialized
+      Open.traverse(@parser, *args, **kwargs) do |k,v|
+        NamedArray.setup(v, @parser.fields, k) unless @unnamed
+        block.call k, v
+        [k, v]
+      end
+    end
+
+    def with_unnamed
+      begin
+        old_unnamed = @unnamed
+        @unnamed = true
+        yield
+      ensure
+        @unnamed = old_unnamed
+      end
+    end
+
+    def []=(key, value)
+      @dumper.init if @dumper.respond_to?(:init) && ! @dumper.initialized
+      @dumper.add key, value
+    end
+
+    def stream
+      @dumper.stream
+    end
+
+    def tsv(*args)
+      TSV === @dumper ? @dumper : TSV.open(stream, *args)
+    end
+  end
+
+  def to_list
+    res = self.annotate({})
+    transformer = Transformer.new self, res
+    transformer.type = :list
+    transformer.traverse do |k,v|
+      case self.type
+      when :single
+        [k, [v]]
+      when :double
+        [k, v.collect{|v| v.first }]
+      when :flat
+        [k, v.slice(0,1)]
+      end
+    end
+    res
+  end
+
+  def to_single
+    res = self.annotate({})
+    transformer = Transformer.new self, res
+    transformer.type = :single
+    transformer.traverse do |k,v|
+      v = v.first while Array === v
+      [k, v]
+    end
+    res
+  end
+end
+

data/lib/scout/tsv/traverse.rb

@@ -1,48 +1,147 @@
 require_relative 'parser'
 module TSV
-  def self.traverse_add(into, res)
-    case into
-    when TSV::Dumper
-      into.add *res
-    when TSV, Hash
-      key, value = res
-      into[key] = value
+  def traverse(key_field_pos = :key, fields_pos = nil, type: nil, one2one: false, unnamed: false, key_field: nil, fields: nil, bar: false, cast: nil, select: nil, &block)
+    key_field = key_field_pos if key_field.nil?
+    fields = fields_pos.dup if fields.nil?
+    type = @type if type.nil?
+    key_pos = self.identify_field(key_field)
+    fields = self.all_fields if fields == :all
+    fields = [fields] unless fields.nil? || Array === fields
+    positions = fields.nil? || fields == :all ? nil : self.identify_field(fields)
+
+
+    if key_pos == :key
+      key_name = @key_field
+    else
+      key_name = @fields[key_pos]
+      if positions.nil?
+        positions = (0..@fields.length-1).to_a
+        positions.delete_at key_pos
+        positions.unshift :key
+      end
     end
-  end
 
-  def self.traverse(obj, into: nil, cpus: nil, bar: nil, **options, &block)
-    case obj
-    when TSV
-      self.traverse(obj.stream, into: into, cpus: cpus, bar: bar, **options, &block)
-    when String
-      f = Open.open(obj)
-      self.traverse(f, into: into, cpus: cpus, bar: bar, **options, &block)
-    when Step
-      self.traverse(obj.get_stream, into: into, cpus: cpus, bar: bar, **options, &block)
-    when IO
-      if into
-        into_thread = Thread.new do 
-          Thread.current.report_on_exception = false
-          Thread.current["name"] = "Traverse into"
-          TSV.parse obj, **options do |k,v|
-            begin
-              res = block.call k, v
-              traverse_add into, res
-            rescue
-              into.abort $!
+    if positions.nil? && key_pos == :key
+      field_names = @fields
+    elsif positions.nil? && key_pos != :key
+      field_names = @fields.dup
+      field_names.delete_at key_pos unless fields == :all
+    elsif positions.include?(:key)
+      field_names = positions.collect{|p| p == :key ? @key_field : @fields[p] }
+    else
+      field_names = @fields.values_at *positions
+    end
+
+    key_index = positions.index :key if positions
+    positions.delete :key if positions
+
+    log_message = "Traverse #{Log.fingerprint self}"
+    Log.debug log_message
+    bar = log_message if TrueClass === bar
+
+    Log::ProgressBar.with_obj_bar(self, bar) do |bar|
+      with_unnamed unnamed do
+        each do |key,values|
+          bar.tick if bar
+          values = [values] if @type == :single
+          if positions.nil?
+            if key_pos != :key
+              values = values.dup
+              key = values.delete_at(key_pos)
+            end
+          else 
+            orig_key = key
+            key = values[key_pos] if key_pos != :key 
+
+            values = values.values_at(*positions)
+            if key_index
+              if @type == :double
+                values.insert key_index, [orig_key]
+              else
+                values.insert key_index, orig_key
+              end
+            end
+          end
+
+          values = TSV.cast_value(values, cast) if cast
+
+          if Array === key 
+            if @type == :double && one2one
+              if one2one == :strict
+                key.each_with_index do |key_i,i|
+                  if type == :double
+                    v_i = values.collect{|v| [v[i]] }
+                  else
+                    v_i = values.collect{|v| v[i] }
+                  end
+                  yield key_i, v_i
+                end
+              else
+                key.each_with_index do |key_i,i|
+                  if type == :double
+                    v_i = values.collect{|v| [v[i] || v.first] }
+                  else
+                    v_i = values.collect{|v| v[i] || v.first }
+                  end
+                  yield key_i, v_i, @fields
+                end
+              end
+            else
+              key.each_with_index do |key_i, i|
+                if type == :double
+                  yield key_i, values
+                elsif type == :list
+                  yield key_i, values.collect{|v| v[i] }
+                elsif type == :flat
+                  yield key_i, values.flatten
+                elsif type == :single
+                  yield key_i, values.first
+                end
+              end
+            end
+          else
+            if type == @type
+              if type == :single
+                yield key, values.first
+              else
+                yield key, values
+              end
+            else
+              case [type, @type]
+              when [:double, :list]
+                yield key, values.collect{|v| [v] }
+              when [:double, :flat]
+                yield key, [values]
+              when [:double, :single]
+                yield key, [values]
+              when [:list, :double]
+                yield key, values.collect{|v| v.first }
+              when [:list, :flat]
+                yield key, [values.first]
+              when [:list, :single]
+                yield key, values
+              when [:flat, :double]
+                yield key, values.flatten
+              when [:flat, :list]
+                yield key, values.flatten
+              when [:flat, :single]
+                yield key, values
+              when [:single, :double]
+                yield key, values.flatten.first
+              when [:single, :list]
+                yield key, values.first
+              when [:single, :flat]
+                yield key, values.first
+              end
             end
-            nil
           end
-          into.close if into.respond_to?(:close)
-        end
-        Thread.pass until into_thread
-        into
-      else
-        TSV.parse obj, **options do |k,v|
-          block.call k, v
-          nil
         end
       end
     end
+    
+
+    [key_name, field_names]
   end
+
+  alias through traverse
 end

data/lib/scout/tsv/util/filter.rb

@@ -0,0 +1,312 @@
+require 'set'
+module Filtered
+
+  class FilterArray
+    attr_accessor :filters
+
+    def ids
+      ids = filters.inject(nil){|list,filter| list.nil? ? filter.ids.dup : Misc.merge_sorted_arrays(list, filter.ids.dup)}
+    end
+
+    def method_missing(name, *args)
+      filters.each do |filter|
+        filter.send(name, *args)
+      end
+    end
+  end
+
+  #{{{ FILTER
+
+  class Filter
+    attr_accessor :data, :match, :fieldnum, :value, :list, :unsaved
+    attr_accessor :persistence
+
+    def initialize(data, match, value, persistence = nil)
+      @data = data
+      @match = match
+      @value = value
+      @unsaved = []
+
+      case 
+      when Hash === persistence
+        @persistence = persistence
+      when String === persistence
+        @persistence = begin
+                         file = ScoutCabinet.open(persistence, true, "HDB")
+                         TSV.setup file, :type => :list
+                         file.extend TSVAdapter
+                         file
+                       end
+        @persistence.read
+      end
+
+      @list = nil
+      case
+      when @match == :key
+        @value = Set.new(@value)
+        class << self
+          self
+        end.class_eval <<-EOC
+          def match_entry(key, entry)
+            key == @value or (Set === @value and @value.include? key)
+          end
+        EOC
+      when @match.match(/field:(.*)/)
+        @fieldnum = data.identify_field $1
+        class << self
+          self
+        end.class_eval <<-EOC
+          def match_entry(key, entry)
+            value = entry[@fieldnum] 
+            value == @value or (Array === value and value.include? @value)
+          end
+        EOC
+      else
+        raise "Unknown match: #{ @match }"
+      end
+    end
+
+    def key
+      case 
+      when String === value
+        value
+      else
+        Marshal.dump(value)
+      end
+    end
+
+    def save(ids)
+      if persistence
+        persistence.write_and_close do
+          persistence[self.key] = ids
+        end
+      else
+        if @list.nil?
+          @list = ids
+        else
+          @list.replace ids
+        end
+      end
+    end
+
+    def update
+      ids = []
+
+      data.with_unnamed do
+        data.unfiltered_each do |key, entry|
+          ids << key if match_entry(key, entry)
+        end
+      end
+
+      save(ids.sort)
+    end
+
+    def saved
+      if persistence.nil?
+        return nil if list.nil?
+        list
+      else
+        return nil if not persistence.include?(self.key)
+        persistence.write_and_close do
+          persistence[self.key]
+        end
+      end
+    end
+
+    def add_unsaved
+      save(Misc.merge_sorted_arrays(unsaved.sort, saved || [])) if unsaved.any?
+      unsaved.clear
+    end
+
+    def ids
+      add_unsaved
+
+      list = saved
+      if list.nil?
+        update
+        list = saved
+      end
+      list
+    end
+
+    def add(id)
+      unsaved.push id
+    end
+
+    def clean
+      add_unsaved
+      if persistence and persistence.include? self.key
+        persistence.write_and_close do
+          persistence.delete self.key
+        end
+      else
+        @list = nil
+      end
+    end
+
+    def reset
+      add_unsaved
+      if persistence
+        persistence.write_and_close do
+          persistence.clear
+        end
+      else
+        @list = nil
+      end
+    end
+  end
+
+  #}}} FILTER
+
+  def self.extended(base)
+    if not base.respond_to? :unfiltered_set
+      class << base
+        attr_accessor :filter_dir, :filters
+        
+        alias unfiltered_set []=
+        alias []= filtered_set
+
+        alias unfiltered_filename filename
+        alias filename filtered_filename
+
+        alias unfiltered_keys keys
+        alias keys filtered_keys
+
+        alias unfiltered_values values
+        alias values filtered_values
+
+        alias unfiltered_each each
+        alias each filtered_each
+
+        alias unfiltered_collect collect
+        alias collect filtered_collect
+
+        alias unfiltered_delete delete
+        alias delete filtered_delete
+      end
+    end
+    base.filters = []
+  end
+
+  def filtered_filename
+    if filters.empty?
+      unfiltered_filename
+    else
+      unfiltered_filename + ":Filtered[#{filters.collect{|f| [f.match, Array === f.value ? Misc.digest(:values => f.value) : f.value] * "="} * ", "}]"
+    end
+  end
+
+  def filtered_set(key, value, clean = false)
+    if filters.empty?
+      self.send(:unfiltered_set, key, value, clean)
+    else
+      filters.each do |filter| 
+        filter.add key if filter.match_entry key, value
+      end
+      self.send(:unfiltered_set, key, value, clean)
+    end
+  end
+
+  def filtered_keys
+    if filters.empty?
+      self.send(:unfiltered_keys)
+    else
+      filters.inject(nil){|list,filter| list.nil? ? filter.ids.dup : Misc.intersect_sorted_arrays(list, filter.ids.dup)}
+    end
+  end
+
+  def filtered_values 
+    if filters.empty?  
+      self.send(:unfiltered_values) 
+    else 
+      ids = filters.inject(nil){|list,filter| list.nil? ? filter.ids.dup : Misc.intersect_sorted_arrays(list, filter.ids.dup)}
+      self.send :values_at, *ids
+    end
+  end
+
+  def filtered_each(&block)
+    if filters.empty?
+      self.send(:unfiltered_each, &block)
+    else
+      ids = filters.inject(nil){|list,filter| list.nil? ? filter.ids.dup : Misc.intersect_sorted_arrays(list, filter.ids.dup)}
+
+      ids.each do |id|
+        value = self[id]
+        yield id, value if block_given?
+        [id, value]
+      end
+    end
+  end
+
+  def filtered_collect(&block)
+    if filters.empty?
+      self.send(:unfiltered_collect, &block)
+    else
+      ids = filters.inject(nil){|list,filter| list = (list.nil? ? filter.ids.dup : Misc.intersect_sorted_arrays(list, filter.ids.dup))}
+
+      new = self.annotate({})
+
+      ids.zip(self.send(:values_at, *ids)).each do |id, values|
+        new[id] = values
+      end
+      new.send :collect, &block
+    end
+  end
+
+  def filtered_delete(key)
+    if filters.empty?
+      self.send(:unfiltered_delete, key)
+    else
+      reset_filters
+      self.send :unfiltered_delete, key
+    end
+  end
+
+  def add_filter(match, value, persistence = nil)
+    if persistence.nil? and filter_dir
+      persistence = File.join(filter_dir, match.to_s + '.filter')
+    end
+
+    filter = Filter.new self, match, value, persistence
+    filters.push filter
+  end
+
+  def pop_filter
+    filters.pop.add_unsaved if filters.any?
+  end
+
+  def size
+    filters.empty? ? super : filters.collect{|f| f.ids.length }.min
+  end
+
+end
+
+module TSV
+  def filter(filter_dir = nil)
+    self.extend Filtered
+    self.filter_dir = filter_dir
+    self.filters = []
+    self
+  end
+
+  def reset_filters
+    if @filter_dir.nil? or @filter_dir.empty?
+      @filters.each do |filter| filter.reset end if Array === @filters
+      return
+    end
+
+    Dir.glob(File.join(@filter_dir, '*.filter')).each do |f|
+      FileUtils.rm f
+    end
+  end
+
+  def with_filters(filters, &block)
+    filter
+    begin
+      filters.each{|field,value| add_filter field, value }
+    ensure
+      reset_filters
+    end
+  end
+end
+

data/lib/scout/tsv/util/process.rb

@@ -0,0 +1,73 @@
+module TSV
+  def process(field, &block)
+    field_pos = identify_field field
+
+    through do |key, values|
+      case
+      when type == :single
+        field_values = values
+      when type == :flat
+        field_values = values
+      else
+        next if values.nil?
+        field_values = values[field_pos]
+      end
+
+      new_values = case 
+                   when block.arity == 1
+                     yield(field_values)
+                   when block.arity == 2
+                     yield(field_values, key)
+                   when block.arity == 3
+                     yield(field_values, key, values)
+                   else
+                     raise "Unexpected arity in block, must be 1, 2 or 3: #{block.arity}"
+                   end
+
+      case
+      when type == :single
+        self[key] = new_values
+      when type == :flat
+        self[key] = new_values
+      else
+        if ! values[field_pos].frozen? && ((String === values[field_pos] && String === new_values) ||
+          (Array === values[field_pos] && Array === new_values))
+           values[field_pos].replace new_values
+        else
+          values[field_pos] = new_values
+        end
+        self[key] = values
+      end
+    end
+
+    self
+  end
+
+  def add_field(name = nil)
+    through do |key, values|
+      new_values = yield(key, values)
+      new_values = [new_values] if type == :double and not Array === new_values
+
+      case
+      when (values.nil? and (fields.nil? or fields.empty?))
+        values = [new_values]
+      when values.nil?  
+        values = [nil] * fields.length + [new_values]
+      when Array === values
+        values += [new_values]
+      else
+        values << new_values
+      end
+
+      self[key] = values
+    end
+
+    if not fields.nil? and not name.nil?
+      new_fields = self.fields + [name]
+      self.fields = new_fields
+    end
+
+    self
+  end
+
+end