scout-gear 7.1.0 → 7.3.0

data/lib/scout/tsv/util/filter.rb

@@ -0,0 +1,303 @@
+require 'set'
+module Filtered
+
+  class FilterArray
+    attr_accessor :filters
+
+    def ids
+      ids = filters.inject(nil){|list,filter| list.nil? ? filter.ids.dup : Misc.merge_sorted_arrays(list, filter.ids.dup)}
+    end
+
+    def method_missing(name, *args)
+      filters.each do |filter|
+        filter.send(name, *args)
+      end
+    end
+  end
+
+  #{{{ FILTER
+
+  class Filter
+    attr_accessor :data, :match, :fieldnum, :value, :list, :unsaved
+    attr_accessor :persistence
+
+    def initialize(data, match, value, persistence = nil)
+      @data = data
+      @match = match
+      @value = value
+      @unsaved = []
+
+      case 
+      when Hash === persistence
+        @persistence = persistence
+      when String === persistence
+        @persistence = begin
+                         file = ScoutCabinet.open(persistence, true, "HDB")
+                         TSV.setup file, :type => :list
+                         file.extend TSVAdapter
+                         file
+                       end
+        @persistence.read
+      end
+
+      @list = nil
+      case
+      when @match == :key
+        @value = Set.new(@value)
+        class << self
+          self
+        end.class_eval <<-EOC
+          def match_entry(key, entry)
+            key == @value or (Set === @value and @value.include? key)
+          end
+        EOC
+      when @match.match(/field:(.*)/)
+        @fieldnum = data.identify_field $1
+        class << self
+          self
+        end.class_eval <<-EOC
+          def match_entry(key, entry)
+            value = entry[@fieldnum] 
+            value == @value or (Array === value and value.include? @value)
+          end
+        EOC
+      else
+        raise "Unknown match: #{ @match }"
+      end
+    end
+
+    def key
+      case 
+      when String === value
+        value
+      else
+        Marshal.dump(value)
+      end
+    end
+
+    def save(ids)
+      if persistence
+        persistence.write_and_close do
+          persistence[self.key] = ids
+        end
+      else
+        if @list.nil?
+          @list = ids
+        else
+          @list.replace ids
+        end
+      end
+    end
+
+    def update
+      ids = []
+
+      data.with_unnamed do
+        data.unfiltered_each do |key, entry|
+          ids << key if match_entry(key, entry)
+        end
+      end
+
+      save(ids.sort)
+    end
+
+    def saved
+      if persistence.nil?
+        return nil if list.nil?
+        list
+      else
+        return nil if not persistence.include?(self.key)
+        persistence.write_and_close do
+          persistence[self.key]
+        end
+      end
+    end
+
+    def add_unsaved
+      save(Misc.merge_sorted_arrays(unsaved.sort, saved || [])) if unsaved.any?
+      unsaved.clear
+    end
+
+    def ids
+      add_unsaved
+
+      list = saved
+      if list.nil?
+        update
+        list = saved
+      end
+      list
+    end
+
+    def add(id)
+      unsaved.push id
+    end
+
+    def clean
+      add_unsaved
+      if persistence and persistence.include? self.key
+        persistence.write_and_close do
+          persistence.delete self.key
+        end
+      else
+        @list = nil
+      end
+    end
+
+    def reset
+      add_unsaved
+      if persistence
+        persistence.write_and_close do
+          persistence.clear
+        end
+      else
+        @list = nil
+      end
+    end
+  end
+
+  #}}} FILTER
+
+  def self.extended(base)
+    if not base.respond_to? :unfiltered_set
+      class << base
+        attr_accessor :filter_dir, :filters
+        
+        alias unfiltered_set []=
+        alias []= filtered_set
+
+        alias unfiltered_filename filename
+        alias filename filtered_filename
+
+        alias unfiltered_keys keys
+        alias keys filtered_keys
+
+        alias unfiltered_values values
+        alias values filtered_values
+
+        alias unfiltered_each each
+        alias each filtered_each
+
+        alias unfiltered_collect collect
+        alias collect filtered_collect
+
+        alias unfiltered_delete delete
+        alias delete filtered_delete
+      end
+    end
+    base.filters = []
+  end
+
+  def filtered_filename
+    if filters.empty?
+      unfiltered_filename
+    else
+      unfiltered_filename + ":Filtered[#{filters.collect{|f| [f.match, Array === f.value ? Misc.digest(:values => f.value) : f.value] * "="} * ", "}]"
+    end
+  end
+
+  def filtered_set(key, value, clean = false)
+    if filters.empty?
+      self.send(:unfiltered_set, key, value, clean)
+    else
+      filters.each do |filter| 
+        filter.add key if filter.match_entry key, value
+      end
+      self.send(:unfiltered_set, key, value, clean)
+    end
+  end
+
+  def filtered_keys
+    if filters.empty?
+      self.send(:unfiltered_keys)
+    else
+      filters.inject(nil){|list,filter| list.nil? ? filter.ids.dup : Misc.intersect_sorted_arrays(list, filter.ids.dup)}
+    end
+  end
+
+  def filtered_values 
+    if filters.empty?  
+      self.send(:unfiltered_values) 
+    else 
+      ids = filters.inject(nil){|list,filter| list.nil? ? filter.ids.dup : Misc.intersect_sorted_arrays(list, filter.ids.dup)}
+      self.send :values_at, *ids
+    end
+  end
+
+  def filtered_each(&block)
+    if filters.empty?
+      self.send(:unfiltered_each, &block)
+    else
+      ids = filters.inject(nil){|list,filter| list.nil? ? filter.ids.dup : Misc.intersect_sorted_arrays(list, filter.ids.dup)}
+
+      ids.each do |id|
+        value = self[id]
+        yield id, value if block_given?
+        [id, value]
+      end
+    end
+  end
+
+  def filtered_collect(&block)
+    if filters.empty?
+      self.send(:unfiltered_collect, &block)
+    else
+      ids = filters.inject(nil){|list,filter| list = (list.nil? ? filter.ids.dup : Misc.intersect_sorted_arrays(list, filter.ids.dup))}
+
+      new = self.annotate({})
+
+      ids.zip(self.send(:values_at, *ids)).each do |id, values|
+        new[id] = values
+      end
+      new.send :collect, &block
+    end
+  end
+
+  def filtered_delete(key)
+    if filters.empty?
+      self.send(:unfiltered_delete, key)
+    else
+      reset_filters
+      self.send :unfiltered_delete, key
+    end
+  end
+
+  def add_filter(match, value, persistence = nil)
+    if persistence.nil? and filter_dir
+      persistence = File.join(filter_dir, match.to_s + '.filter')
+    end
+
+    filter = Filter.new self, match, value, persistence
+    filters.push filter
+  end
+
+  def pop_filter
+    filters.pop.add_unsaved if filters.any?
+  end
+
+  def size
+    filters.empty? ? super : filters.collect{|f| f.ids.length }.min
+  end
+
+end
+
+module TSV
+  def filter(filter_dir = nil)
+    self.extend Filtered
+    self.filter_dir = filter_dir
+    self.filters = []
+    self
+  end
+
+  def reset_filters
+    if @filter_dir.nil? or @filter_dir.empty?
+      @filters.each do |filter| filter.reset end if Array === @filters
+      return
+    end
+
+    Dir.glob(File.join(@filter_dir, '*.filter')).each do |f|
+      FileUtils.rm f
+    end
+  end
+end
+

data/lib/scout/tsv/util/process.rb

@@ -0,0 +1,73 @@
+module TSV
+  def process(field, &block)
+    field_pos = identify_field field
+
+    through do |key, values|
+      case
+      when type == :single
+        field_values = values
+      when type == :flat
+        field_values = values
+      else
+        next if values.nil?
+        field_values = values[field_pos]
+      end
+
+      new_values = case 
+                   when block.arity == 1
+                     yield(field_values)
+                   when block.arity == 2
+                     yield(field_values, key)
+                   when block.arity == 3
+                     yield(field_values, key, values)
+                   else
+                     raise "Unexpected arity in block, must be 1, 2 or 3: #{block.arity}"
+                   end
+
+      case
+      when type == :single
+        self[key] = new_values
+      when type == :flat
+        self[key] = new_values
+      else
+        if ! values[field_pos].frozen? && ((String === values[field_pos] && String === new_values) ||
+          (Array === values[field_pos] && Array === new_values))
+           values[field_pos].replace new_values
+        else
+          values[field_pos] = new_values
+        end
+        self[key] = values
+      end
+    end
+
+    self
+  end
+
+  def add_field(name = nil)
+    through do |key, values|
+      new_values = yield(key, values)
+      new_values = [new_values] if type == :double and not Array === new_values
+
+      case
+      when (values.nil? and (fields.nil? or fields.empty?))
+        values = [new_values]
+      when values.nil?  
+        values = [nil] * fields.length + [new_values]
+      when Array === values
+        values += [new_values]
+      else
+        values << new_values
+      end
+
+      self[key] = values
+    end
+
+    if not fields.nil? and not name.nil?
+      new_fields = self.fields + [name]
+      self.fields = new_fields
+    end
+
+    self
+  end
+
+end

data/lib/scout/tsv/util/select.rb

@@ -0,0 +1,220 @@
+module TSV
+  def select(method = nil, invert = false, &block)
+    new = TSV.setup({}, :key_field => key_field, :fields => fields, :type => type, :filename => filename, :identifiers => identifiers)
+
+    self.annotate(new)
+    
+    case
+    when (method.nil? and block_given?)
+      through do |key, values|
+        new[key] = values if invert ^ (yield key, values)
+      end
+    when Array === method
+      method = Set.new method
+      with_unnamed do
+        case type
+        when :single
+          through do |key, value|
+            new[key] = value if invert ^ (method.include? key or method.include? value)
+          end
+        when :list, :flat
+          through do |key, values|
+            new[key] = values if invert ^ (method.include? key or (method & values).length > 0)
+          end
+        else
+          through do |key, values|
+            new[key] = values if invert ^ (method.include? key or (method & values.flatten).length > 0)
+          end
+        end
+      end
+    when Regexp === method
+      with_unnamed do
+        through do |key, values|
+          new[key] = values if invert ^ ([key,values].flatten.select{|v| v =~ method}.any?)
+        end
+      end
+    when (String === method || Symbol === method)
+      if block_given?
+        case 
+        when block.arity == 1
+          with_unnamed do
+            case
+            when (method == key_field or method == :key)
+              through do |key, values|
+                new[key] = values if invert ^ (yield(key))
+              end
+            when (type == :single or type == :flat)
+              through do |key, value|
+                new[key] = value if invert ^ (yield(value))
+              end
+            else
+              pos = identify_field method
+              raise "Field #{ method } not identified. Available: #{ fields * ", " }" if pos.nil?
+
+              through do |key, values|
+                new[key] = values if invert ^ (yield(values[pos]))
+              end
+            end
+          end
+        when block.arity == 2
+          with_unnamed do
+            case
+            when (method == key_field or method == :key)
+              through do |key, values|
+                new[key] = values if invert ^ (yield(key, key))
+              end
+            when (type == :single or type == :flat)
+              through do |key, value|
+                new[key] = value if invert ^ (yield(key, value))
+              end
+            else
+              pos = identify_field method
+              through do |key, values|
+                new[key] = values if invert ^ (yield(key, values[pos]))
+              end
+            end
+
+          end
+        end
+
+      else
+        with_unnamed do
+          through do |key, values|
+            new[key] = values if invert ^ ([key,values].flatten.select{|v| v == method}.any?)
+          end
+        end
+      end
+    when Hash === method
+      key  = method.keys.first
+      method = method.values.first
+      case
+      when (Array === method and (key == :key or key_field == key))
+        with_unnamed do
+          Annotated.purge(method).each{|key| 
+            new[key] = self[key] if invert ^ (self.include? key)
+          }
+        end
+      when Array === method
+        with_unnamed do
+          method = Set.new method unless Set === method
+          case type
+          when :single
+            through :key, key do |key, value|
+              new[key] = self[key] if invert ^ (method.include? value)
+            end
+          when :list
+            through :key, key do |key, values|
+              new[key] = self[key] if invert ^ (method.include? values.first)
+            end
+          when :flat #untested
+            through :key, key do |key, values|
+              new[key] = self[key] if invert ^ ((method & values.flatten).any?)
+            end
+          else
+            through :key, key do |key, values|
+              new[key] = self[key] if invert ^ ((method & values.flatten).any?)
+            end
+          end
+        end
+
+      when Regexp === method
+        with_unnamed do
+          through :key, key do |key, values|
+            values = [values] if type == :single
+            new[key] = self[key] if invert ^ (values.flatten.select{|v| v =~ method}.any?)
+          end
+        end
+
+      when (String === method and method =~ /name:(.*)/)
+        name = $1
+        old_unnamed = self.unnamed
+        self.unnamed = false
+        if name.strip =~ /^\/(.*)\/$/
+          regexp = Regexp.new $1
+          through :key, key do |key, values|
+            case type
+            when :single
+              values = values.annotate([values])
+            when :double
+              values = values[0]
+            end
+            new[key] = self[key] if invert ^ (values.select{|v| v.name =~ regexp}.any?)
+          end
+        else
+          through :key, key do |key, values|
+            case type
+            when :single
+              values = values.annotate([values])
+            when :double
+              values = values[0]
+            end
+            new[key] = self[key] if invert ^ (values.select{|v| v.name == name}.any?)
+          end
+        end
+        self.unnamed = old_unnamed
+
+      when String === method
+        if method =~ /^([<>]=?)(.*)/
+          with_unnamed do
+            through :key, key do |key, values|
+              value = Array === values ? values.flatten.first : values
+              new[key] = self[key] if value.to_f.send($1, $2.to_f)
+            end
+          end
+        else
+          with_unnamed do
+            through :key, key do |key, values|
+              values = [values] if type == :single
+              new[key] = self[key] if invert ^ (values.flatten.select{|v| v == method}.length > 0)
+            end
+          end
+        end
+      when Numeric === method
+        with_unnamed do
+          through :key, key do |key, values|
+            new[key] = self[key] if invert ^ (values.flatten.length >= method)
+          end
+        end
+      when Proc === method
+        with_unnamed do
+          through :key, key do |key, values|
+            values = [values] if type == :single
+            new[key] = self[key] if invert ^ (values.flatten.select{|v| method.call(v)}.length > 0)
+          end
+        end
+      end
+    end
+    new
+  end
+
+  def reorder(key_field = nil, fields = nil, merge: true, one2one: :fill) 
+    res = self.annotate({})
+    key_field_name, field_names = through key_field, fields, one2one: one2one do |k,v|
+      if @type == :double && merge && res.include?(k)
+        current = res[k]
+        if merge == :concat
+          v.each_with_index do |new,i|
+            next if new.empty?
+            current[i].concat(new)
+          end
+        else
+          merged = []
+          v.each_with_index do |new,i|
+            next if new.empty?
+            merged[i] = current[i] + new
+          end
+          res[k] = merged
+        end
+      else
+        res[k] = v
+      end
+    end
+    res.key_field = key_field_name
+    res.fields = field_names
+    res
+  end
+
+  def slice(fields)
+    reorder :key, fields
+  end
+end

data/lib/scout/tsv/util.rb

@@ -0,0 +1,82 @@
+#require_relative '../../../modules/rbbt-util/lib/rbbt/tsv/manipulate'
+#Log.warn "USING OLD RBBT CODE: #{__FILE__}"
+require_relative 'traverse'
+require_relative 'util/process'
+require_relative 'util/select'
+module TSV
+  def [](*args)
+    v = super(*args)
+    NamedArray.setup(v, @fields) unless @unnamed || ! (Array === v)
+    v
+  end
+
+  def each(*args, &block)
+    if block_given?
+      super(*args) do |k,v|
+      NamedArray.setup(v, @fields) unless @unnamed || ! (Array === v)
+      block.call(k, v)
+    end
+    else
+      super(*args)
+    end
+  end
+
+  def collect(*args, &block)
+    if block_given?
+      res = []
+      each do |k,v|
+        res << yield(k, v)
+      end
+      res
+    else
+      super(*args)
+    end
+  end
+
+  def with_unnamed
+    begin
+      old_unnamed = unnamed
+      unnamed = true
+      yield
+    ensure
+      unnamed = old_unnamed
+    end
+  end
+
+  def summary
+    key = nil
+    values = nil
+    self.each do |k, v|
+      key = k
+      values = v
+      break
+    end
+
+    filename = @filename
+    filename = "No filename" if filename.nil? || filename.empty?
+    filename.find if Path === filename 
+    filename = File.basename(filename) + " [" + File.basename(persistence_path) + "]" if respond_to?(:persistence_path) and persistence_path
+
+    with_unnamed do
+      <<-EOF
+Filename = #{filename}
+Key field = #{key_field || "*No key field*"}
+Fields = #{fields ? Log.fingerprint(fields) : "*No field info*"}
+Type = #{type}
+Size = #{size}
+namespace = #{Log.fingerprint namespace}
+identifiers = #{Log.fingerprint identifiers}
+Example:
+  - #{key} -- #{Log.fingerprint values }
+      EOF
+    end
+  end
+
+  def all_fields
+    [@key_field] + @fields
+  end
+
+  def fingerprint
+    "TSV:{"<< Log.fingerprint(self.all_fields|| []) << ";" << Log.fingerprint(self.keys) << "}"
+  end
+end

data/lib/scout/tsv.rb

@@ -1,13 +1,26 @@
 require_relative 'meta_extension'
+require_relative 'tsv/util'
 require_relative 'tsv/parser'
+require_relative 'tsv/dumper'
+require_relative 'tsv/persist'
+require_relative 'tsv/index'
+require_relative 'tsv/path'
+require_relative 'tsv/traverse'
+require_relative 'tsv/open'
 
 module TSV
   extend MetaExtension
-  extension_attr :key_field, :fields
+  extension_attr :key_field, :fields, :type, :filename, :namespace, :unnamed, :identifiers
 
   def self.open(file, options = {})
-    Open.open(file) do |f|
-      TSV.parse(f,**options)
+    persist, type = IndiferentHash.process_options options, :persist, :persist_type, :persist => false, :persist_type => "HDB"
+    Persist.persist(file, type, options.merge(:persist => persist)) do |filename|
+      data = filename ? ScoutCabinet.open(filename, true, type) : nil
+      options[:data] = data if data
+      options[:filename] = file
+      Open.open(file) do |f|
+        TSV.parse(f, **options)
+      end
     end
   end
 end