scout-gear 10.4.0 → 10.7.0

data/lib/scout/persist/tsv/adapter/fix_width_table.rb

@@ -0,0 +1,106 @@
+require_relative 'base'
+require_relative '../../engine/fix_width_table'
+
+module FWTAdapter
+  include TSVAdapter
+
+  def self.extended(base)
+    base.extend TSVAdapter
+    base
+  end
+
+  attr_accessor :pos_function
+
+  def persistence_path=(value)
+    @persistence_path = value
+    @filename = value
+  end
+
+  def metadata_file
+    @metadata_file ||= self.persistence_path + '.metadata'
+  end
+
+  def load_annotation_hash
+    ANNOTATION_ATTR_HASH_SERIALIZER.load(Open.read(metadata_file, mode: 'b'))
+  end
+
+  def save_annotation_hash
+    Open.write(metadata_file, ANNOTATION_ATTR_HASH_SERIALIZER.dump(self.annotation_hash), mode: 'wb')
+  end
+
+  def add(key, value)
+    key = pos_function.call(key) if pos_function and not (@range and Array === key)
+    value = save_value(value)
+    super(key, value)
+  end
+
+  def add_range_point(key, value)
+    key = pos_function.call(key) if pos_function
+    value = save_value(value)
+    super(key, value)
+  end
+
+  def [](key, clean = false)
+    key = pos_function.call(key) if pos_function and not clean
+    res = super(key)
+    return nil if res.nil?
+    res = res.collect{|r| load_value(r) }
+    res.extend MultipleResult
+    res
+  end
+
+  def []=(key, value)
+    self.add(key, value)
+  end
+
+  def include?(i)
+    return true if Numeric === i and i < pos(@size)
+    @annotations
+    false
+  end
+
+  def size
+    @size
+  end
+
+  def each
+    read
+    @size.times do |i|
+      v = idx_value(i)
+      yield i, v
+    end
+  end
+
+  def keys
+    []
+  end
+end
+
+module Persist
+
+  def self.open_fwt(path, value_size, range = false, serializer = nil, update = false, in_memory = false, &pos_function)
+    FileUtils.mkdir_p File.dirname(path) unless File.exist?(File.dirname(path))
+
+    database = FixWidthTable.new(path, value_size, range, update, in_memory, &pos_function)
+
+    database.extend FWTAdapter
+
+    database.pos_function = pos_function
+
+    unless serializer == :clean
+      TSV.setup database
+      database.serializer ||= TSVAdapter.serializer_module(serializer)
+    end
+
+    database
+  end
+end
+
+Persist.save_drivers[:fwt] = proc do |file, content|
+  content.file.seek 0
+  Misc.sensiblewrite(file, content.file.read)
+end
+
+Persist.load_drivers[:fwt] = proc do |file| 
+  FixWidthTable.new file
+end

data/lib/scout/persist/tsv/adapter/packed_index.rb

@@ -0,0 +1,95 @@
+require_relative 'base'
+require_relative '../../engine/packed_index'
+
+module PKIAdapter
+  include TSVAdapter
+
+  attr_accessor :pos_function
+
+  def self.extended(base)
+    base.extend TSVAdapter
+    base
+  end
+
+  def persistence_path=(value)
+    @persistence_path = value
+    @file = value
+  end
+
+  def metadata_file
+    @metadata_file ||= self.persistence_path + '.metadata'
+  end
+
+  def load_annotation_hash
+    ANNOTATION_ATTR_HASH_SERIALIZER.load(Open.read(metadata_file, mode: 'rb'))
+  end
+
+  def save_annotation_hash
+    Open.write(metadata_file, ANNOTATION_ATTR_HASH_SERIALIZER.dump(self.annotation_hash), mode: 'wb')
+  end
+
+  def [](key, clean = false)
+    key = pos_function.call(key) if pos_function and not clean
+    res = super(key)
+    res.extend MultipleResult unless res.nil?
+    res
+  end
+
+  def value(pos)
+    self.send(:[], pos, true)
+  end
+
+  def []=(key, value)
+    add key, value
+  end
+
+  def add(key, value)
+    key = pos_function.call(key) if pos_function 
+    if Numeric === key
+      @_last ||= -1
+      skipped = key - @_last - 1
+      skipped.times do
+        self.send(:<<, nil)
+      end
+      @_last = key
+    end
+    self.send(:<<, value)
+  end
+
+  def add_range_point(key, value)
+    key = pos_function.call(key) if pos_function
+    super(key, value)
+  end
+
+  def include?(i)
+    return true if Numeric === i and i < size
+    false
+  end
+
+  def each
+    size.times do |i|
+      yield i, value(i)
+    end
+  end
+
+  def keys
+    []
+  end
+
+  def size
+    orig_size
+  end
+end
+
+module Persist
+  def self.open_pki(path, write, pattern, &pos_function)
+    FileUtils.mkdir_p File.dirname(path) unless File.exist?(File.dirname(path))
+
+    database = PackedIndex.new(path, write, pattern, &pos_function)
+    database.extend PKIAdapter
+
+    database.pos_function = pos_function
+
+    database
+  end
+end

data/lib/scout/persist/tsv/adapter/sharder.rb

@@ -0,0 +1,54 @@
+require_relative 'base'
+require_relative '../../engine/sharder'
+ 
+module ShardAdapter
+  include TSVAdapter
+
+  def self.extended(obj)
+    obj.extend TSVAdapter
+    class << obj
+      alias keys orig_keys
+      alias each orig_each
+    end
+  
+    obj
+  end
+
+  def metadata_file
+    @metadata_file ||= File.join(self.persistence_path, 'metadata')
+  end
+
+  def load_annotation_hash
+    ANNOTATION_ATTR_HASH_SERIALIZER.load(Open.read(metadata_file, mode: 'rb'))
+  end
+
+  def save_annotation_hash
+    Open.write(metadata_file, ANNOTATION_ATTR_HASH_SERIALIZER.dump(self.annotation_hash), mode: 'wb')
+  end
+
+  def size
+    databases.values.inject(0){|acc,d| acc += d.size }
+  end
+
+  def keys
+    databases.values.inject([]){|acc,d| acc.concat(d.keys); acc }
+  end
+
+  def prefix(...)
+    databases.values.inject([]){|acc,d| acc.concat(d.prefix(...)); acc }
+  end
+
+  def include?(key)
+    self[key] != nil
+  end
+end
+
+module Persist
+  def self.open_sharder(persistence_path, write=false, db_type=nil, persist_options={}, &block)
+    database = Sharder.new(persistence_path, write, db_type, persist_options, &block)
+    database.extend ShardAdapter
+    database.serializer = TSVAdapter.serializer_module(persist_options[:serializer]) if persist_options[:serializer]
+    database.save_annotation_hash
+    database
+  end
+end

data/lib/scout/persist/tsv/adapter/tkrzw.rb

@@ -0,0 +1,18 @@
+require_relative 'base'
+require_relative '../../engine/tkrzw'
+
+Persist.save_drivers[:tkh] = proc do |file, content|
+  data = ScoutTKRZW.open(file, true, "tkh")
+  content.annotate(data)
+  data.extend TSVAdapter
+  data.merge!(content)
+  data.close
+  data.read
+  data
+end
+
+Persist.load_drivers[:tkh] = proc do |file| 
+  data = ScoutTKRZW.open(file, false, "tkh")
+  data.extend TSVAdapter unless TSVAdapter === data
+  data
+end

data/lib/scout/persist/tsv/adapter/tokyocabinet.rb

@@ -0,0 +1,65 @@
+require_relative 'base'
+require_relative '../../engine/tokyocabinet'
+
+module TKAdapter 
+  include TSVAdapter
+  def self.extended(obj)
+    obj.extend TSVAdapter
+    obj
+  end
+end
+
+module Persist
+  def self.open_tokyocabinet(path, write, serializer = nil, tokyocabinet_class = TokyoCabinet::HDB)
+    write = true unless File.exist? path
+
+    FileUtils.mkdir_p File.dirname(path) unless File.exist?(File.dirname(path))
+
+    database = ScoutCabinet.open(path, write, tokyocabinet_class)
+
+    database.extend TKAdapter
+    database.serializer ||= TSVAdapter.serializer_module(serializer)
+
+    database
+  end
+end
+
+Persist.save_drivers[:HDB] = proc do |file, content|
+  if ScoutCabinet === content
+    Open.mv(content.persistence_path, file)
+    content.persistence_path = file
+    content
+  else
+    data = ScoutCabinet.open(file, true, "HDB")
+    content.annotate(data)
+    data.extend TKAdapter
+    data.merge!(content)
+    data
+  end
+end
+
+Persist.load_drivers[:HDB] = proc do |file| 
+  data = ScoutCabinet.open(file, false, "HDB")
+  data.extend TKAdapter unless TKAdapter === data
+  data
+end
+
+Persist.save_drivers[:BDB] = proc do |file, content|
+  if ScoutCabinet === content
+    Open.mv(content.persistence_path, file)
+    content.persistence_path = file
+    content
+  else
+    data = ScoutCabinet.open(file, true, "BDB")
+    content.annotate(data)
+    data.extend TKAdapter
+    data.merge!(content)
+    data
+  end
+end
+
+Persist.load_drivers[:BDB] = proc do |file| 
+  data = ScoutCabinet.open(file, false, "BDB")
+  data.extend TKAdapter unless TKAdapter === data
+  data
+end

data/lib/scout/persist/tsv/adapter.rb

@@ -0,0 +1,6 @@
+require_relative 'serialize'
+require_relative 'adapter/base'
+require_relative 'adapter/fix_width_table'
+require_relative 'adapter/packed_index'
+require_relative 'adapter/tokyocabinet'
+require_relative 'adapter/sharder'

data/lib/scout/{tsv/persist → persist/tsv}/serialize.rb

@@ -108,10 +108,15 @@ module TSVAdapter
     :strict_integer_array => StrictIntegerArraySerializer,
     :strict_float_array => StrictFloatArraySerializer,
     :marshal => Marshal,
+    :json => JSON,
     :string => StringSerializer,
     :binary => BinarySerializer,
     :tsv => TSVSerializer,
     :marshal_tsv => TSVMarshalSerializer
   }
 
+  def self.serializer_module(serializer)
+    Symbol === serializer ? SERIALIZER_ALIAS[serializer] : serializer
+  end
+
 end

data/lib/scout/persist/tsv.rb

@@ -0,0 +1,107 @@
+require 'scout/persist'
+require_relative 'engine'
+require_relative 'tsv/adapter'
+
+Persist.save_drivers[:tsv] = proc do |file,content| 
+  case content
+  when IO
+    Open.sensible_write(file, content)
+  when TSV
+    Open.sensible_write(file, content)
+  else
+    stream = if content.respond_to?(:stream)
+               content.stream
+             elsif content.respond_to?(:dumper_stream)
+               content.dumper_stream
+             else
+               content
+             end
+    Open.sensible_write(file, stream)
+  end
+end
+
+Persist.load_drivers[:tsv] = proc do |file| TSV.open file end
+
+module Persist
+
+  def self.open_database(path, write, serializer = nil, type = "HDB", options = {})
+    db = case type
+         when 'fwt'
+           value_size, range, update, in_memory, pos_function = IndiferentHash.process_options options.dup, :value_size, :range, :update, :in_memory, :pos_function
+           if pos_function
+             Persist.open_fwt(path, value_size, range, serializer, update, in_memory, &pos_function)
+           else
+             Persist.open_fwt(path, value_size, range, serializer, update, in_memory)
+           end
+         when 'pki'
+           pattern, pos_function = IndiferentHash.process_options options.dup, :pattern, :pos_function
+           if pos_function
+             Persist.open_pki(path, write, pattern, &pos_function)
+           else
+             Persist.open_pki(path, write, pattern)
+           end
+         else
+           Persist.open_tokyocabinet(path, write, serializer, type)
+         end
+    db
+  end
+
+  def self.tsv(id, options = {}, engine: nil, persist_options: {})
+    engine ||= persist_options[:engine] || :HDB
+    Persist.persist(id, engine, persist_options.merge(:other_options => options)) do |filename|
+      if filename.nil?
+        yield(persist_options[:data] || {})
+      else
+        if persist_options.include?(:shard_function)
+          data = persist_options[:data] ||= Persist.open_sharder(filename, true, engine, options.merge(persist_options), &persist_options[:shard_function])
+        else
+          data = persist_options[:data] ||= Persist.open_database(filename, true, persist_options[:serializer], engine, options)
+        end
+
+        data.serializer = TSVAdapter.serializer_module(persist_options[:serializer]) if persist_options[:serializer]
+
+        yield(data)
+        data.save_annotation_hash if Annotation.is_annotated?(data)
+        data
+      end
+    end
+  end
+
+  def self.persist_tsv(file, filename = nil, options = {}, persist_options = {}, &block)
+    persist_options = IndiferentHash.add_defaults persist_options,
+      IndiferentHash.pull_keys(options, :persist)
+
+    persist_options[:data] ||= options.delete(:data)
+    engine = IndiferentHash.process_options persist_options, :engine, engine: "HDB"
+    other_options = IndiferentHash.pull_keys persist_options, :other
+    other_options[:original_options] = options
+
+    Persist.tsv(file, engine: engine, persist_options: persist_options.merge(other: other_options), &block)
+  end
+
+  #def self.persist_tsv(file, filename = nil, options = {}, persist_options = {})
+  #  persist_options = IndiferentHash.add_defaults persist_options,
+  #    IndiferentHash.pull_keys(options, :persist)
+
+  #  persist_options[:data] ||= options.delete(:data)
+  #  engine = IndiferentHash.process_options persist_options, :engine, engine: "HDB"
+  #  other_options = IndiferentHash.pull_keys persist_options, :other
+  #  other_options[:original_options] = options
+  #  Persist.persist(file, engine, persist_options.merge(:other => other_options)) do |filename|
+  #    if filename
+  #      if persist_options.include?(:shard_function)
+  #        data = persist_options[:data] ||= Persist.open_sharder(filename, true, engine, options.merge(persist_options), &persist_options[:shard_function])
+  #      else
+  #        data = persist_options[:data] ||= Persist.open_database(filename, true, persist_options[:serializer], engine, options)
+  #      end
+
+
+  #      yield(data)
+  #      data.save_annotation_hash if Annotation.is_annotated?(data)
+  #      data
+  #    else
+  #      yield(persist_options[:data] || {})
+  #    end
+  #  end
+  #end
+end

data/lib/scout/tsv/annotation/repo.rb

@@ -0,0 +1,87 @@
+module Persist
+  REPO_CACHE = {}
+  def self.annotation_repo_persist(repo, name, &block)
+
+    if String === repo
+      repo = REPO_CACHE[repo] ||= begin
+                                    repo = repo.find if Path === repo
+                                    repo = Persist.open_tokyocabinet(repo, false, :list, :BDB)
+                                    repo_fields = ["literal", "annotation_types", "JSON"]
+                                    TSV.setup(repo, :fields => repo_fields, :key_field => "Annotation ID")
+                                    repo.save_annotation_hash
+                                    repo
+                                  end
+      repo.close
+    end
+
+    repo_fields = repo.fields
+
+    subkey = name + ":"
+
+    keys = repo.read_and_close do
+      repo.range subkey + 0.chr, true, subkey + 254.chr, true
+    end
+
+    case
+    when (keys.length == 1 and keys.first == subkey + 'NIL')
+      nil
+    when (keys.length == 1 and keys.first == subkey + 'EMPTY')
+      []
+    when (keys.length == 1 && keys.first =~ /:SINGLE$/)
+      key = keys.first
+      values = repo.read_and_close do
+        repo[key]
+      end
+      Annotation.load_tsv_values(key, values, *repo_fields)
+    when (keys.any? and not keys.first =~ /ANNOTATED_DOUBLE_ARRAY/)
+      repo.read_and_close do
+        keys.sort_by{|k| k.split(":").last.to_i}.collect{|key|
+          v = repo[key]
+          Annotation.load_tsv_values(key, v, *repo_fields)
+        }
+      end
+    when (keys.any? and keys.first =~ /ANNOTATED_DOUBLE_ARRAY/)
+      repo.read_and_close do
+
+        res = keys.sort_by{|k| k.split(":").last.to_i}.collect{|key|
+          v = repo[key]
+          Annotation.load_tsv_values(key, v, *repo_fields)
+        }
+
+        res.first.annotate res
+        res.extend AnnotatedArray
+
+        res
+      end
+    else
+      annotations = yield
+
+      repo.write_and_close do 
+        case
+        when annotations.nil?
+          repo[subkey + "NIL"] = nil
+        when annotations.empty?
+          repo[subkey + "EMPTY"] = nil
+        when (not Array === annotations or (AnnotatedArray === annotations and not Array === annotations.first))
+          tsv_values = Annotation.obj_tsv_values(annotations, repo_fields) 
+          repo[subkey + annotations.id << ":" << "SINGLE"] = tsv_values
+        when (not Array === annotations or (AnnotatedArray === annotations and AnnotatedArray === annotations.first))
+          annotations.each_with_index do |e,i|
+            next if e.nil?
+            tsv_values = Annotation.obj_tsv_values(e, repo_fields) 
+            repo[subkey + "ANNOTATED_DOUBLE_ARRAY:" << i.to_s] = tsv_values
+          end
+        else
+          annotations.each_with_index do |e,i|
+            next if e.nil?
+            tsv_values = Annotation.obj_tsv_values(e, repo_fields) 
+            repo[subkey + i.to_s] = tsv_values
+          end
+        end
+      end
+
+      annotations
+    end
+
+  end
+end