scout-gear 7.1.0 → 7.3.0

data/lib/scout/tsv/parser.rb

@@ -1,9 +1,14 @@
+require_relative '../named_array'
 module TSV
   def self.cast_value(value, cast)
     if Array === value
       value.collect{|e| cast_value(e, cast) }
     else
-      value.send(cast)
+      if Proc === cast
+        cast.call value
+      else
+        value.send(cast)
+      end
     end
   end
 
@@ -13,9 +18,11 @@ module TSV
     if positions.nil? && key == 0
       key = items.shift
     elsif positions.nil? 
-      key = items.delete(key)
+      key = items.delete_at(key)
+      key = key.split(sep2) if type == :double
     else 
       key, items = items[key], items.values_at(*positions)
+      key = key.split(sep2) if type == :double
     end
 
     items = case type
@@ -29,7 +36,6 @@ module TSV
               items.collect{|i| i.split(sep2, -1) }
             end
 
-    key = key.partition(sep2).first if type == :double
 
     if cast
       items = cast_value(items, cast)
@@ -38,10 +44,12 @@ module TSV
     [key, items]
   end
 
-  def self.parse_stream(stream, data: nil, merge: true, type: :list, fix: true, bar: false, first_line: nil, **kargs, &block)
+  def self.parse_stream(stream, data: nil, source_type: nil, type: :list, merge: true, one2one: false, fix: true, bar: false, first_line: nil, **kargs, &block)
     begin
       bar = Log::ProgressBar.new_bar(bar) if bar
 
+      source_type = type if source_type.nil?
+
       data = {} if data.nil?
       merge = false if type != :double
       line = first_line || stream.gets
@@ -50,30 +58,86 @@ module TSV
           line.strip!
           line = Misc.fixutf8(line) if fix
           bar.tick if bar
-          key, items = parse_line(line, type: type, **kargs)
+          key, items = parse_line(line, type: source_type, **kargs)
 
-          if block_given?
-            res = block.call(key, items)
-            data[key] = res unless res.nil?
-            next
+          if Array === key
+            keys = key
+            if one2one
+              key_items = keys.length.times.collect{|i| items.collect{|list| [list[i] || list[0]] } }
+            else
+              key_items = false
+            end
+          else
+            keys = [key]
+            key_items = false
           end
 
-          if ! merge || ! data.include?(key)
-            data[key] = items
-          else
-            current = data[key]
-            if merge == :concat
-              items.each_with_index do |new,i|
-                next if new.empty?
-                current[i].concat(new)
+          keys.each_with_index do |key,i|
+            if key_items
+              these_items = key_items[i]
+            else
+              these_items = items
+            end
+
+            these_items = 
+              case [source_type, type]
+              when [:single, :single]
+                these_items
+              when [:list, :single]
+                these_items.first
+              when [:flat, :single]
+                these_items.first
+              when [:double, :single]
+                these_items.first.first
+              when [:single, :list]
+                [these_items]
+              when [:list, :list]
+                these_items
+              when [:flat, :list]
+                these_items
+              when [:double, :list]
+                these_items.collect{|l| l.first }
+              when [:single, :flat]
+                [these_items]
+              when [:list, :flat]
+                these_items
+              when [:flat, :flat]
+                these_items
+              when [:double, :flat]
+                these_items.flatten
+              when [:single, :double]
+                [[these_items]]
+              when [:list, :double]
+                these_items.collect{|l| [l] }
+              when [:flat, :double]
+                [these_items]
+              when [:double, :double]
+                these_items
               end
+
+            if block_given?
+              res = block.call(key, these_items)
+              data[key] = res unless res.nil? || FalseClass === data
+              next
+            end
+
+            if ! merge || ! data.include?(key)
+              data[key] = these_items
             else
-              merged = []
-              items.each_with_index do |new,i|
-                next if new.empty?
-                merged[i] = current[i] + new
+              current = data[key]
+              if merge == :concat
+                these_items.each_with_index do |new,i|
+                  next if new.empty?
+                  current[i].concat(new)
+                end
+              else
+                merged = []
+                these_items.each_with_index do |new,i|
+                  next if new.empty?
+                  merged[i] = current[i] + new
+                end
+                data[key] = merged
               end
-              data[key] = merged
             end
           end
         ensure
@@ -86,7 +150,7 @@ module TSV
     end
   end
 
-  def self.parse_header(stream, fix: true, header_hash: '#', sep: "\n")
+  def self.parse_header(stream, fix: true, header_hash: '#', sep: "\t")
     raise "Closed stream" if IO === stream && stream.closed?
 
     options = {}
@@ -131,14 +195,150 @@ module TSV
     [options, key_field, fields, first_line, preamble]
   end
 
-  def self.parse(stream, **kwargs)
-    options, key_field, fields, first_line, preamble = parse_header(stream)
+  KEY_PARAMETERS = begin
+                     params = []
+                     (method(:parse_line).parameters + method(:parse_stream).parameters).each do |type, name|
+                       params << name if type == :key
+                     end
+                     params
+                   end
 
-    options.each do |option,value|
-      option = option.to_sym
-      kwargs[option] = value unless kwargs.include?(option)
+  class Parser
+    attr_accessor :stream, :options, :key_field, :fields, :first_line, :preamble
+    def initialize(file, fix: true, header_hash: "#", sep: "\t")
+      if IO === file
+        @stream = file
+      else
+        @stream = Open.open(file)
+      end
+      @options, @key_field, @fields, @first_line, @preamble = TSV.parse_header(@stream, fix:fix, header_hash:header_hash, sep:sep)
+      @options[:sep] = sep if @options[:sep].nil?
+    end
+
+    def all_fields
+      [@key_field] + @fields
     end
-    data = parse_stream(stream, first_line: first_line, **kwargs)
-    TSV.setup data, :key_field => key_field, :fields => fields
+
+    def traverse(key_field: nil, fields: nil, filename: nil, namespace: nil,  **kwargs, &block)
+      if fields
+        all_field_names ||= [@key_field] + @fields
+        positions = NamedArray.identify_name(all_field_names, fields)
+        kwargs[:positions] = positions
+        field_names = all_field_names.values_at *positions
+      else
+        field_names = @fields
+      end
+
+      if key_field
+        all_field_names ||= [@key_field] + @fields
+        key = NamedArray.identify_name(all_field_names, key_field)
+        kwargs[:key] = key == :key ? 0 : key
+        key_field_name = key === :key ? @key_field : all_field_names[key]
+        if fields.nil?
+          field_names = all_field_names - [@key_field]
+        end
+      else
+        key_field_name = @key_field
+      end
+
+      @options.each do |option,value|
+        option = option.to_sym
+        next unless KEY_PARAMETERS.include? option
+        kwargs[option] = value unless kwargs.include?(option)
+      end
+
+      kwargs[:source_type] = @options[:type]
+      kwargs[:data] = false if kwargs[:data].nil?
+
+      data = TSV.parse_stream(@stream, first_line: @first_line, **kwargs, &block)
+
+      if data
+        TSV.setup(data, :key_field => key_field_name, :fields => field_names, :type => @type)
+      else
+        self
+      end
+    end
+
   end
+
+  def self.parse(stream, fix: true, header_hash: "#", sep: "\t", filename: nil, namespace: nil, unnamed: false, serializer: nil, **kwargs, &block)
+    parser = TSV::Parser.new stream, fix: fix, header_hash: header_hash, sep: sep
+
+    cast = parser.options[:cast] || kwargs[:cast]
+    type = kwargs[:type] ||=  parser.options[:type] ||= :double
+    if (data = kwargs[:data]) && data.respond_to?(:persistence_class)
+      TSV.setup(data, type: type)
+      data.extend TSVAdapter
+      if serializer
+        data.serializer = serializer
+      elsif cast
+        data.serializer = 
+          case [cast, type]
+          when [:to_i, :single]
+            :integer
+          when [:to_i, :list], [:to_i, :flat]
+            :integer_array
+          when [:to_f, :single]
+            :float
+          when [:to_f, :list], [:to_f, :flat]
+            :float_array
+          else
+            type
+          end
+      else
+        data.serializer = type
+      end
+    end
+
+    kwargs[:data] = {} if kwargs[:data].nil?
+
+    data = parser.traverse **kwargs, &block
+    data.type = type
+    data.filename = filename
+    data.namespace = namespace
+    data.unnamed = unnamed
+    data
+  end
+
+  #def self.parse_alt(stream, key_field: nil, fields: nil, filename: nil, namespace: nil,  **kwargs, &block)
+  #  options, key_field_name, field_names, first_line, preamble = parse_header(stream)
+
+  #  if fields
+  #    all_field_names ||= [key_field_name] + field_names
+  #    positions = NamedArray.identify_name(all_field_names, fields)
+  #    kwargs[:positions] = positions
+  #    field_names = all_field_names.values_at *positions
+  #  end
+
+  #  if key_field
+  #    all_field_names ||= [key_field_name] + field_names
+  #    key = NamedArray.identify_name(all_field_names, key_field)
+  #    kwargs[:key] = key
+  #    key_field_name = all_field_names[key]
+  #    if fields.nil?
+  #      field_names = all_field_names - [key_field_name]
+  #    end
+  #  end
+
+  #  options.each do |option,value|
+  #    option = option.to_sym
+  #    next unless KEY_PARAMETERS.include? option
+  #    kwargs[option] = value unless kwargs.include?(option)
+  #  end
+
+  #  kwargs[:source_type] = options[:type]
+
+  #  type = kwargs[:type] ||= :double
+  #  if (data = kwargs[:data]) && data.respond_to?(:persistence_class)
+  #    TSV.setup(data, type: type, key_field: key_field_name, fields: field_names)
+  #    data.extend TSVAdapter
+  #  end
+
+  #  data = parse_stream(stream, first_line: first_line, **kwargs, &block)
+
+  #  TSV.setup(data, :key_field => key_field_name, :fields => field_names, :type => type, filename: filename, namespace: namespace)
+
+  #  data
+  #end
+
 end

data/lib/scout/tsv/path.rb

@@ -0,0 +1,13 @@
+module Path
+  def tsv(*args, **kwargs, &block)
+    found = self.find
+    found = self.set_extension('tsv').find unless found.exists?
+    TSV.open(found, *args, **kwargs, &block)
+  end
+
+  def index(*args, **kwargs, &block)
+    found = self.find
+    found = self.set_extension('tsv').find unless found.exists?
+    TSV.index(found, *args, **kwargs, &block)
+  end
+end

data/lib/scout/tsv/persist/adapter.rb

@@ -0,0 +1,367 @@
+require_relative '../../open/lock'
+require_relative 'serialize'
+
+module TSVAdapter
+  attr_accessor :persistence_path, :persistence_class, :serializer, :closed, :writable
+
+  class << self
+    attr_accessor :lock_dir
+    def lock_dir
+      @lock_dir ||= Path.setup('tmp/tsv_locks')
+    end
+  end
+
+  EXTENSION_ATTR_HASH_KEY = "__extension_attr_hash__"
+  EXTENSION_ATTR_HASH_SERIALIZER = Marshal
+
+  def serializer=(serializer)
+    @serializer = Symbol === serializer ? SERIALIZER_ALIAS[serializer] : serializer
+  end
+
+  def load_extension_attr_hash
+    EXTENSION_ATTR_HASH_SERIALIZER.load(StringIO.new(self[EXTENSION_ATTR_HASH_KEY]))
+  end
+
+  def save_extension_attr_hash
+    self[EXTENSION_ATTR_HASH_KEY]= EXTENSION_ATTR_HASH_SERIALIZER.dump(self.extension_attr_hash)
+  end
+
+  def self.extended(base)
+    if ! TSVAdapter === base
+      if (! TSVAdapter === base) && base.include?(EXTENSION_ATTR_HASH_KEY)
+        TSV.setup(base, base.load_extension_attr_hash)
+      elsif TSV === base
+        base[EXTENSION_ATTR_HASH_KEY] = EXTENSION_ATTR_HASH_SERIALIZER.dump(base.extension_attr_hash)
+      end
+    end
+
+    base.serializer = SERIALIZER_ALIAS[base.type]
+
+    class << base
+      alias orig_set []=
+        alias orig_get []
+
+      def [](key)
+        self.read_lock do
+          load_value(super(key))
+        end
+      end
+
+      def []=(key, value)
+        self.write_lock do
+          super(key, save_value(value))
+        end
+      end
+
+      def load_value(str)
+        serializer.load(str)
+      end
+
+      def save_value(value)
+        serializer.dump(value)
+      end
+
+
+    end
+
+    #case base.type
+    #when :single
+    #  class << base
+    #    def load_value(value)
+    #      value
+    #    end
+    #    def save_value(value)
+    #      value
+    #    end
+    #  end
+    #when :list, :flat
+    #  class << base
+    #    def load_value(value)
+    #      value.nil? ? nil : value.split("\t")
+    #    end
+    #    def save_value(value)
+    #      value * "\t"
+    #    end
+    #  end
+    #when :double
+    #  class << base
+    #    def load_value(value)
+    #      value.nil? ? nil : value.split("\t").collect{|v| v.split("|") }
+    #    end
+    #    def save_value(value)
+    #      value.collect{|v| v * "|" } * "\t"
+    #    end
+    #  end
+    #end
+  end
+
+  def keys(*args)
+    k = self.read_lock do
+      super(*args)
+    end
+
+    if k[0] == EXTENSION_ATTR_HASH_KEY
+      k.slice(1,k.length-1)
+    elsif k[-1] == EXTENSION_ATTR_HASH_KEY
+      k.slice(0,k.length-2)
+    else
+      k - [EXTENSION_ATTR_HASH_KEY]
+    end
+  end
+
+  def each(&block)
+    self.read_lock do
+      super do |k,v|
+        next if k == EXTENSION_ATTR_HASH_KEY
+        yield(k, load_value(v))
+      end
+    end
+  end
+
+  def collect(&block)
+    res = []
+    if block_given?
+      each do |k,v|
+        res << yield(k, v)
+      end
+    else
+      each do |k,v|
+        res << [k, v]
+      end
+    end
+    res
+  end
+
+  def values
+    collect{|k,v| v }
+  end
+
+  alias map collect
+
+  def closed?
+    @closed
+  end
+
+  def write?
+    @writable
+  end
+
+  def read?
+    ! (write? || closed?)
+  end
+
+  def write(*args)
+    begin
+      super(*args)
+      @writable = true
+    rescue NoMethodError
+    end
+  end
+
+  def close(*args)
+    begin
+      super(*args)
+      @closed = true
+    rescue NoMethodError
+    end
+    self
+  end
+
+  def read(*args)
+    begin
+      super(*args)
+    rescue NoMethodError
+    end
+  end
+
+  def delete(key)
+    self.write_lock do
+      out(key)
+    end
+  end
+
+  def lock
+    return yield if @locked
+    lock_filename = Persist.persistence_path(persistence_path, {:dir => TSVAdapter.lock_dir})
+    Open.lock(lock_filename) do
+      begin
+        @locked = true
+        yield
+      ensure
+        @locked = false
+      end
+    end
+  end
+
+  def lock_and_close
+    lock do
+      begin
+        yield
+      ensure
+        close
+      end
+    end
+  end
+
+  def write_and_read
+    if write?
+      begin
+        return yield
+      ensure
+        read
+      end
+    end
+
+    lock do
+      write(true) if closed? || !write?
+      begin
+        yield
+      ensure
+        read
+      end
+    end
+  end
+
+  def write_and_close
+    if write?
+      begin
+        return yield
+      ensure
+        close unless @locked
+      end
+    end
+
+    lock do
+      write(true) if closed? || ! write?
+      res = begin
+              yield
+            ensure
+              close
+            end
+      res
+    end
+  end
+
+  def with_read(&block)
+    if read? || write?
+      return yield
+    else
+      read_and_close &block
+    end
+  end
+
+  def with_write(&block)
+    if write?
+      return yield
+    else
+      if self.read?
+        self.write_and_read do
+          return yield
+        end
+      else
+        self.write_and_close do
+          return yield
+        end
+      end
+    end
+  end
+
+
+  def read_and_close
+    if read? || write?
+      begin
+        return yield
+      ensure
+        close unless @locked
+      end
+    end
+
+    lock do
+      read true if closed? || ! read?
+      begin
+        yield
+      ensure
+        close
+      end
+    end
+  end
+
+  def read_lock
+    read if closed?
+    if read? || write?
+      return yield
+    end
+
+    lock do
+      close
+      read true
+      begin
+        yield
+      end
+    end
+  end
+
+  def write_lock
+    write if closed?
+    if write?
+      return yield
+    end
+
+    lock do
+      close
+      write true
+      begin
+        yield
+      end
+    end
+  end
+
+  def merge!(hash)
+    hash.each do |key,values|
+      self[key] = values
+    end
+  end
+
+  def range(*args)
+    begin
+      self.read_lock do
+        super(*args)
+      end
+    rescue
+      []
+    end
+  end
+
+  def include?(*args)
+    self.read_lock do
+      super(*args) #- TSV::ENTRY_KEYS.to_a
+    end
+  end
+
+  MAX_CHAR = 255.chr
+
+  def prefix(key)
+    self.read_lock do
+      range(key, 1, key + MAX_CHAR, 1)
+    end
+  end
+
+  def get_prefix(key)
+    keys = prefix(key)
+    select(:key => keys)
+  end
+
+  def size(*args)
+    self.read_lock do
+      super(*args)
+    end
+  end
+
+  def values_at(*keys)
+    self.read_lock do
+      keys.collect do |k|
+        self[k]
+      end
+    end
+  end
+end