scout-gear 6.0.0 → 7.2.0

data/lib/scout/tsv/persist/adapter.rb

@@ -0,0 +1,348 @@
+require_relative '../../open/lock'
+
+module TSVAdapter
+  attr_accessor :persistence_path, :persistence_class, :closed, :writable
+
+  class << self
+    attr_accessor :lock_dir
+    def lock_dir
+      @lock_dir ||= Path.setup('tmp/tsv_locks')
+    end
+  end
+
+  EXTENSION_ATTR_HASH_KEY = "__extension_attr_hash__"
+  EXTENSION_ATTR_HASH_SERIALIZER = Marshal
+
+  def load_extension_attr_hash
+    EXTENSION_ATTR_HASH_SERIALIZER.load(self[EXTENSION_ATTR_HASH_KEY])
+  end
+
+  def save_extension_attr_hash
+    self[EXTENSION_ATTR_HASH_KEY]= EXTENSION_ATTR_HASH_SERIALIZER.dump(self.extension_attr_hash)
+  end
+
+  def self.extended(base)
+    if base.include?(EXTENSION_ATTR_HASH_KEY)
+      TSV.setup(base, base.load_extension_attr_hash)
+    elsif TSV === base
+      base[EXTENSION_ATTR_HASH_KEY] = EXTENSION_ATTR_HASH_SERIALIZER.dump(base.extension_attr_hash)
+    end
+
+    class << base
+      alias orig_set []=
+        alias orig_get []
+
+      def [](key)
+        self.read_lock do
+          load_value(super(key))
+        end
+      end
+
+      def []=(key, value)
+        self.write_lock do
+          super(key, save_value(value))
+        end
+      end
+    end
+
+    case base.type
+    when :single
+      class << base
+        def load_value(value)
+          value
+        end
+        def save_value(value)
+          value
+        end
+      end
+    when :list, :flat
+      class << base
+        def load_value(value)
+          value.nil? ? nil : value.split("\t")
+        end
+        def save_value(value)
+          value * "\t"
+        end
+      end
+    when :double
+      class << base
+        def load_value(value)
+          value.nil? ? nil : value.split("\t").collect{|v| v.split("|") }
+        end
+        def save_value(value)
+          value.collect{|v| v * "|" } * "\t"
+        end
+      end
+    end
+  end
+
+  def keys(*args)
+    k = self.read_lock do
+      super(*args)
+    end
+
+    if k[0] == EXTENSION_ATTR_HASH_KEY
+      k.slice(1,k.length-1)
+    elsif k[-1] == EXTENSION_ATTR_HASH_KEY
+      k.slice(0,k.length-2)
+    else
+      k - [EXTENSION_ATTR_HASH_KEY]
+    end
+  end
+
+  def each(&block)
+    self.read_lock do
+      super do |k,v|
+        next if k == EXTENSION_ATTR_HASH_KEY
+        yield(k, load_value(v))
+      end
+    end
+  end
+
+  def collect(&block)
+    res = []
+    if block_given?
+      each do |k,v|
+        res << yield(k, v)
+      end
+    else
+      each do |k,v|
+        res << [k, v]
+      end
+    end
+    res
+  end
+
+  def values
+    collect{|k,v| v }
+  end
+
+  alias map collect
+
+  def closed?
+    @closed
+  end
+
+  def write?
+    @writable
+  end
+
+  def read?
+    ! (write? || closed?)
+  end
+
+  def write(*args)
+    begin
+      super(*args)
+      @writable = true
+    rescue NoMethodError
+    end
+  end
+
+  def close(*args)
+    begin
+      super(*args)
+      @closed = true
+    rescue NoMethodError
+    end
+    self
+  end
+
+  def read(*args)
+    begin
+      super(*args)
+    rescue NoMethodError
+    end
+  end
+
+  def delete(key)
+    self.write_lock do
+      out(key)
+    end
+  end
+
+  def lock
+    return yield if @locked
+    lock_filename = Persist.persistence_path(persistence_path, {:dir => TSVAdapter.lock_dir})
+    Open.lock(lock_filename) do
+      begin
+        @locked = true
+        yield
+      ensure
+        @locked = false
+      end
+    end
+  end
+
+  def lock_and_close
+    lock do
+      begin
+        yield
+      ensure
+        close
+      end
+    end
+  end
+
+  def write_and_read
+    if write?
+      begin
+        return yield
+      ensure
+        read
+      end
+    end
+
+    lock do
+      write(true) if closed? || !write?
+      begin
+        yield
+      ensure
+        read
+      end
+    end
+  end
+
+  def write_and_close
+    if write?
+      begin
+        return yield
+      ensure
+        close unless @locked
+      end
+    end
+
+    lock do
+      write(true) if closed? || ! write?
+      res = begin
+              yield
+            ensure
+              close
+            end
+      res
+    end
+  end
+
+  def with_read(&block)
+    if read? || write?
+      return yield
+    else
+      read_and_close &block
+    end
+  end
+
+  def with_write(&block)
+    if write?
+      return yield
+    else
+      if self.read?
+        self.write_and_read do
+          return yield
+        end
+      else
+        self.write_and_close do
+          return yield
+        end
+      end
+    end
+  end
+
+
+  def read_and_close
+    if read? || write?
+      begin
+        return yield
+      ensure
+        close unless @locked
+      end
+    end
+
+    lock do
+      read true if closed? || ! read?
+      begin
+        yield
+      ensure
+        close
+      end
+    end
+  end
+
+  def read_lock
+    read if closed?
+    if read? || write?
+      return yield
+    end
+
+    lock do
+      close
+      read true
+      begin
+        yield
+      end
+    end
+  end
+
+  def write_lock
+    write if closed?
+    if write?
+      return yield
+    end
+
+    lock do
+      close
+      write true
+      begin
+        yield
+      end
+    end
+  end
+
+  def merge!(hash)
+    hash.each do |key,values|
+      self[key] = values
+    end
+  end
+
+  def range(*args)
+    begin
+      self.read_lock do
+        super(*args)
+      end
+    rescue
+      []
+    end
+  end
+
+  def include?(*args)
+    self.read_lock do
+      super(*args) #- TSV::ENTRY_KEYS.to_a
+    end
+  end
+
+  MAX_CHAR = 255.chr
+
+  def prefix(key)
+    self.read_lock do
+      range(key, 1, key + MAX_CHAR, 1)
+    end
+  end
+
+  def get_prefix(key)
+    keys = prefix(key)
+    select(:key => keys)
+  end
+
+  def size(*args)
+    self.read_lock do
+      super(*args)
+    end
+  end
+
+  def values_at(*keys)
+    self.read_lock do
+      keys.collect do |k|
+        self[k]
+      end
+    end
+  end
+end

data/lib/scout/tsv/persist/tokyocabinet.rb

@@ -0,0 +1,113 @@
+require 'tokyocabinet'
+require_relative 'adapter'
+
+module ScoutCabinet
+  attr_accessor :persistence_path, :persistence_class
+
+  CONNECTIONS = {}
+  def self.open(path, write, tokyocabinet_class = TokyoCabinet::HDB)
+    if String === tokyocabinet_class && tokyocabinet_class.include?(":big")
+      big = true
+      tokyocabinet_class = tokyocabinet_class.split(":").first
+    else
+      big = false
+    end
+
+    dir = File.dirname(File.expand_path(path))
+    Open.mkdir(dir) unless File.exist?(dir)
+
+    tokyocabinet_class = TokyoCabinet::HDB if tokyocabinet_class == "HDB" or tokyocabinet_class.nil?
+    tokyocabinet_class = TokyoCabinet::BDB if tokyocabinet_class == "BDB"
+
+    database = CONNECTIONS[path] ||= tokyocabinet_class.new
+
+    if big and not Open.exists?(path)
+      database.tune(nil,nil,nil,tokyocabinet_class::TLARGE | tokyocabinet_class::TDEFLATE) 
+    end
+
+    flags = (write ? tokyocabinet_class::OWRITER | tokyocabinet_class::OCREAT : tokyocabinet_class::OREADER)
+    database.close 
+
+    if !database.open(path, flags)
+      ecode = database.ecode
+      raise "Open error: #{database.errmsg(ecode)}. Trying to open file #{path}"
+    end
+
+    database.extend ScoutCabinet
+    database.persistence_path ||= path
+    database.persistence_class = tokyocabinet_class
+
+    database.open(path, tokyocabinet_class::OREADER)
+
+    CONNECTIONS[path] = database
+
+    database
+  end
+
+  def close
+    @closed = true
+    @writable = false
+    super
+  end
+
+  def read(force = false)
+    return if ! write? && ! closed && ! force
+    self.close
+    if !self.open(@persistence_path, persistence_class::OREADER)
+      ecode = self.ecode
+      raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
+    end
+
+    @writable = false
+    @closed = false
+
+    self
+  end
+
+  def write(force = true)
+    return if write? && ! closed && ! force
+    self.close
+
+    if !self.open(@persistence_path, persistence_class::OWRITER)
+      ecode = self.ecode
+      raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
+    end
+
+    @writable = true
+    @closed = false
+
+    self
+  end
+
+  #def self.open_tokyocabinet(path, write, serializer = nil, tokyocabinet_class = TokyoCabinet::HDB)
+  #  raise
+  #  write = true unless File.exist? path
+
+  #  FileUtils.mkdir_p File.dirname(path) unless File.exist?(File.dirname(path))
+
+  #  database = Persist::TCAdapter.open(path, write, tokyocabinet_class)
+
+  #  unless serializer == :clean
+  #    TSV.setup database
+  #    database.write_and_read do
+  #      database.serializer = serializer
+  #    end if serializer && database.serializer != serializer
+  #  end
+
+  #  database
+  #end
+end
+
+Persist.save_drivers[:HDB] = proc do |file, content|
+  data = ScoutCabinet.open(file, true, "HDB")
+  content.annotate(data)
+  data.extend TSVAdapter
+  data.merge!(content)
+  data
+end
+
+Persist.load_drivers[:HDB] = proc do |file| 
+  data = ScoutCabinet.open(file, false, "HDB")
+  data.extend TSVAdapter unless TSVAdapter === data
+  data
+end

data/lib/scout/tsv/persist.rb

@@ -0,0 +1,15 @@
+require_relative 'persist/adapter'
+require_relative 'persist/tokyocabinet'
+
+Persist.save_drivers[:tsv] = proc do |file,content| 
+  stream = if IO === content
+             content
+           elsif content.respond_to?(:get_stream)
+             content.get_stream
+           elsif content.respond_to?(:stream)
+             content.stream
+           end
+  Open.sensible_write(file, stream)
+end
+
+Persist.load_drivers[:tsv] = proc do |file| TSV.open file end

data/lib/scout/tsv/traverse.rb

@@ -0,0 +1,48 @@
+require_relative 'parser'
+module TSV
+  def self.traverse_add(into, res)
+    case into
+    when TSV::Dumper
+      into.add *res
+    when TSV, Hash
+      key, value = res
+      into[key] = value
+    end
+  end
+
+  def self.traverse(obj, into: nil, cpus: nil, bar: nil, **options, &block)
+    case obj
+    when TSV
+      self.traverse(obj.stream, into: into, cpus: cpus, bar: bar, **options, &block)
+    when String
+      f = Open.open(obj)
+      self.traverse(f, into: into, cpus: cpus, bar: bar, **options, &block)
+    when Step
+      self.traverse(obj.get_stream, into: into, cpus: cpus, bar: bar, **options, &block)
+    when IO
+      if into
+        into_thread = Thread.new do 
+          Thread.current.report_on_exception = false
+          Thread.current["name"] = "Traverse into"
+          TSV.parse obj, **options do |k,v|
+            begin
+              res = block.call k, v
+              traverse_add into, res
+            rescue
+              into.abort $!
+            end
+            nil
+          end
+          into.close if into.respond_to?(:close)
+        end
+        Thread.pass until into_thread
+        into
+      else
+        TSV.parse obj, **options do |k,v|
+          block.call k, v
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/scout/tsv/util.rb

@@ -0,0 +1,24 @@
+#require_relative '../../../modules/rbbt-util/lib/rbbt/tsv/manipulate'
+#Log.warn "USING OLD RBBT CODE: #{__FILE__}"
+module TSV
+  #[:each, :collect, :map].each do |method|
+  #  define_method(method) do |*args,&block|
+  #    super(*args) do |k,v|
+  #      NamedArray.setup(v, @fields) unless @unnamed
+  #      block.call k, v
+  #    end
+  #  end
+  #end
+
+  #[:select, :reject].each do |method|
+  #  define_method(method) do |*args,&block|
+  #    res = super(*args) do |k,v|
+  #      NamedArray.setup(v, @fields) unless @unnamed
+  #      block.call k, v
+  #    end
+  #    self.annotate(res)
+  #    res
+  #  end
+  #end
+
+end

data/lib/scout/tsv.rb

@@ -0,0 +1,27 @@
+require_relative 'meta_extension'
+require_relative 'tsv/util'
+require_relative 'tsv/parser'
+require_relative 'tsv/dumper'
+require_relative 'tsv/persist'
+require_relative 'tsv/index'
+require_relative 'tsv/path'
+require_relative 'tsv/traverse'
+
+module TSV
+  extend MetaExtension
+  extension_attr :key_field, :fields, :type, :filename, :namespace, :unnamed
+
+  def self.open(file, options = {})
+    persist, type = IndiferentHash.process_options options, :persist, :persist_type, :persist => false, :persist_type => "HDB"
+    Persist.persist(file, type, options.merge(:persist => persist)) do |filename|
+      data = filename ? ScoutCabinet.open(filename, true, type) : nil
+      options[:data] = data if data
+      options[:filename] = file
+      Open.open(file) do |f|
+        TSV.parse(f, **options)
+      end
+    end
+  end
+
+end
+

data/lib/scout/work_queue/worker.rb

@@ -1,16 +1,18 @@
 class WorkQueue
   class Worker
     attr_accessor :pid, :ignore_ouput
-    def initialize
+    def initialize(ignore_ouput = false)
+      @ignore_output = ignore_ouput
     end
 
     def run
       @pid = Process.fork do
+        Log.debug "Worker start with #{Process.pid}"
         yield
       end
     end
 
-    def process(input, output, &block)
+    def process(input, output = nil, &block)
       run do
         begin
           while obj = input.read
@@ -22,21 +24,25 @@ class WorkQueue
             output.write res unless ignore_ouput || res == :ignore 
           end
         rescue DoneProcessing
-          Log.log "Worker #{Process.pid} done"
+        rescue Interrupt
         rescue Exception
-          Log.exception $!
+          output.write WorkerException.new($!, Process.pid)
           exit -1
         end
       end
     end
 
-    def join
-      Log.log "Joining worker #{@pid}"
-      Process.waitpid @pid
+    def abort
+      begin
+        Log.debug "Aborting worker #{@pid}"
+        Process.kill "INT", @pid 
+      rescue Errno::ECHILD
+      end
     end
 
-    def exit(status)
-      Log.log "Worker #{@pid} exited with status #{Log.color(:green, status)}"
+    def join
+      Log.debug "Joining worker #{@pid}"
+      Process.waitpid @pid
     end
 
     def self.join(workers)
@@ -44,8 +50,7 @@ class WorkQueue
       begin
         while pid = Process.wait 
           status = $?
-          worker = workers.select{|w| w.pid == pid }.first
-          worker.exit status.exitstatus if worker
+            worker = workers.select{|w| w.pid == pid }.first
         end
       rescue Errno::ECHILD
       end