scout-gear 7.1.0 → 7.3.0

data/lib/scout/semaphore.rb

@@ -37,7 +37,13 @@ if continue
     int ret;
     sem_t* sem;
     sem = sem_open(name, 0);
+    if (sem == SEM_FAILED){
+      return(errno);
+    }
     ret = sem_wait(sem);
+    if (ret == -1){
+      return(errno);
+    }
     sem_close(sem);
     return(ret);
   }
@@ -51,6 +57,7 @@ if continue
     sem_close(sem);
   }
       EOF
+
     end
 
     SEM_MUTEX = Mutex.new
@@ -66,7 +73,7 @@ if continue
 
     def self.with_semaphore(size, file = nil)
       if file.nil?
-        file = "/" << Misc.digest(rand(1000000000000).to_s) if file.nil?
+        file = "/scout-" << Misc.digest(rand(100000000000).to_s)[0..10] if file.nil?
       else
         file = file.gsub('/', '_') if file
       end

data/lib/scout/tsv/dumper.rb

@@ -0,0 +1,112 @@
+module TSV
+  class Dumper
+    def self.header_lines(key_field, fields, entry_hash = nil)
+      if Hash === entry_hash 
+        sep = entry_hash[:sep] ? entry_hash[:sep] : "\t"
+        preamble = entry_hash[:preamble]
+        header_hash = entry_hash[:header_hash]
+      end
+
+      header_hash = "#" if header_hash.nil?
+
+      preamble = "#: " << Misc.hash2string(entry_hash.merge(:key_field => nil, :fields => nil)) << "\n" if preamble.nil? and entry_hash and entry_hash.values.compact.any?
+
+      str = "" 
+      str << preamble.strip << "\n" if preamble and not preamble.empty?
+      if fields
+        if fields.empty?
+          str << header_hash << (key_field || "ID").to_s << "\n" 
+        else
+          str << header_hash << (key_field || "ID").to_s << sep << (fields * sep) << "\n" 
+        end
+      end
+
+      str
+    end
+
+    def self.header(options={})
+      key_field, fields, sep, header_hash, preamble = IndiferentHash.process_options options, 
+        :key_field, :fields, :sep, :header_hash, :preamble,
+        :sep => "\t", :header_hash => "#", :preamble => true
+
+      if fields.nil? || key_field.nil?
+        fields_str = nil
+      else
+        fields_str = "#{header_hash}#{key_field}#{sep}#{fields*sep}"
+      end
+
+      if preamble && options.values.compact.any?
+        preamble_str = "#: " << IndiferentHash.hash2string(options)
+      else
+        preamble_str = nil
+      end
+
+      [preamble_str, fields_str].compact * "\n"
+    end
+
+
+    attr_accessor :options
+    def initialize(options = {})
+      @sep, @type = IndiferentHash.process_options options, 
+        :sep, :type, 
+        :sep => "\t", :type => :double
+      @options = options
+      @sout, @sin = Open.pipe
+      ConcurrentStream.setup(@sin, pair: @sout)
+      ConcurrentStream.setup(@sout, pair: @sin)
+    end
+
+    def init
+      header = Dumper.header(@options.merge(:type => @type, :sep => @sep))
+      @sin.puts header if header and ! header.empty?
+    end
+
+    def add(key, value)
+
+      case @type
+      when :single
+        @sin.puts key + @sep + value
+      when :list, :flat
+        @sin.puts key + @sep + value * @sep
+      when :double
+        @sin.puts key + @sep + value.collect{|v| v * "|" } * @sep
+      end
+    end
+
+    def close
+      @sin.close
+      @sin.join
+    end
+
+    def stream
+      @sout
+    end
+
+    def abort(exception=nil)
+      @sin.abort(exception)
+    end
+  end
+
+  def stream
+    dumper = TSV::Dumper.new self.extension_attr_hash
+    dumper.init
+    t = Thread.new do 
+      begin
+        Thread.current.report_on_exception = true
+        Thread.current["name"] = "Dumper thread"
+        self.each do |k,v|
+          dumper.add k, v
+        end
+        dumper.close
+      rescue
+        dumper.abort($!)
+      end
+    end
+    Thread.pass until t["name"]
+    dumper.stream
+  end
+
+  def to_s
+    stream.read
+  end
+end

data/lib/scout/tsv/index.rb

@@ -0,0 +1,161 @@
+require_relative 'parser'
+require_relative 'persist/fix_width_table'
+module TSV
+  def self.index(tsv_file, target: 0, fields: nil, order: true, **kwargs)
+    persist, type = IndiferentHash.process_options kwargs,
+      :persist, :persist_type,
+      :persist => false, :persist_type => "HDB"
+    kwargs.delete :type
+
+    Persist.persist(tsv_file, type, kwargs.merge(:persist => persist, :persist_prefix => "Index")) do |filename|
+      if filename
+        index = ScoutCabinet.open(filename, true, type)
+        TSV.setup(index, :type => :single)
+        index.extend TSVAdapter 
+      else
+        index = TSV.setup({}, :type => :single)
+      end
+
+      dummy_data = TSV.setup({}, :key_field => "Key", :fields => ["Target"])
+      if order
+        tmp_index = {}
+        key_field, field_names = TSV.traverse tsv_file, key_field: target, fields: fields, type: :double, into: dummy_data, unnamed: true, **kwargs do |k,values|
+          values.each_with_index do |list,i|
+            list.each do |e|
+              tmp_index[e] ||= []
+              tmp_index[e][i] ||= []
+              tmp_index[e][i] << k
+            end
+          end
+        end
+        tmp_index.each do |e,list|
+          index[e] = list.flatten.compact.uniq.first
+        end
+      else
+        key_field, field_names = TSV.traverse tsv_file, key_field: target, fields: fields, type: :flat, into: dummy_data, unnamed: true, **kwargs do |k,values|
+          values.each do |e|
+            index[e] = k unless index.include?(e)
+          end
+        end
+      end
+
+      index.key_field = dummy_data.fields * ", "
+      index.fields = [dummy_data.key_field]
+      index
+    end
+  end
+
+  def index(*args, **kwargs, &block)
+    TSV.index(self, *args, **kwargs, &block)
+  end
+
+  def self.range_index(tsv_file, start_field = nil, end_field = nil, key_field: :key, **kwargs)
+    persist, type = IndiferentHash.process_options kwargs,
+      :persist, :persist_type,
+      :persist => false, :persist_type => :fwt
+    kwargs.delete :type
+
+    Persist.persist(tsv_file, type, kwargs.merge(:persist => persist, :persist_prefix => "Index")) do |filename|
+
+      max_key_size = 0
+      index_data = []
+      TSV.traverse tsv_file, key_field: key_field, fields: [start_field, end_field] do |key, values|
+        key_size = key.length
+        max_key_size = key_size if key_size > max_key_size
+
+        start_pos, end_pos = values
+        if Array === start_pos
+          start_pos.zip(end_pos).each do |s,e|
+            index_data << [key, [s.to_i, e.to_i]]
+          end
+        else
+          index_data << [key, [start_pos.to_i, end_pos.to_i]]
+        end
+      end
+
+      filename = :memory if filename.nil?
+      index = FixWidthTable.get(filename, max_key_size, true)
+      index.add_range index_data
+      index.read
+      index
+    end
+  end
+
+  def range_index(*args, **kwargs, &block)
+    TSV.range_index(self, *args, **kwargs, &block)
+  end
+
+
+  #def range_index(start_field = nil, end_field = nil, options = {})
+  #  start_field ||= "Start"
+  #  end_field ||= "End"
+
+  #  options = Misc.add_defaults options,
+  #    :persist => false, :persist_file => nil, :persist_update => false 
+
+  #  persist_options = Misc.pull_keys options, :persist
+  #  persist_options[:prefix] ||= "RangeIndex[#{start_field}-#{end_field}]"
+
+  #  Persist.persist(filename || self.object_id.to_s, :fwt, persist_options) do 
+  #    max_key_size = 0
+  #    index_data = []
+  #    with_unnamed do
+  #      with_monitor :desc => "Creating Index Data", :step => 10000 do
+  #        through :key, [start_field, end_field] do |key, values|
+  #          key_size = key.length
+  #          max_key_size = key_size if key_size > max_key_size
+
+  #          start_pos, end_pos = values
+  #          if Array === start_pos
+  #            start_pos.zip(end_pos).each do |s,e|
+  #              index_data << [key, [s.to_i, e.to_i]]
+  #            end
+  #          else
+  #            index_data << [key, [start_pos.to_i, end_pos.to_i]]
+  #          end
+  #        end
+  #      end
+  #    end
+
+  #    index = FixWidthTable.get(:memory, max_key_size, true)
+  #    index.add_range index_data
+  #    index.read
+  #    index
+  #  end
+  #end
+
+  #def self.range_index(file, start_field = nil, end_field = nil, options = {})
+  #  start_field ||= "Start"
+  #  end_field ||= "End"
+
+  #  data_options = Misc.pull_keys options, :data
+  #  filename = case
+  #             when (String === file or Path === file)
+  #               file
+  #             when file.respond_to?(:filename)
+  #               file.filename
+  #             else
+  #               file.object_id.to_s
+  #             end
+  #  persist_options = Misc.pull_keys options, :persist
+  #  persist_options[:prefix] ||= "StaticRangeIndex[#{start_field}-#{end_field}]"
+
+  #  filters = Misc.process_options options, :filters
+
+  #  if filters
+  #    filename += ":Filtered[#{filters.collect{|f| f * "="} * ", "}]"
+  #  end
+
+  #  Persist.persist(filename, :fwt, persist_options) do
+  #    tsv = TSV.open(file, data_options)
+  #    if filters
+  #      tsv.filter
+  #      filters.each do |match, value|
+  #        tsv.add_filter match, value
+  #      end
+  #    end
+ 
+  #    tsv.range_index(start_field, end_field, options)
+  #  end
+  #end
+end

data/lib/scout/tsv/open.rb

@@ -0,0 +1,128 @@
+require_relative '../open'
+module Open
+  def self.traverse_add(into, res)
+    case into
+    when TSV::Dumper
+      into.add *res
+    when TSV, Hash
+      key, value = res
+      into[key] = value
+    end
+  end
+
+  #def self.traverse(obj, into: nil, cpus: nil, bar: nil, **options, &block)
+  #  case obj
+  #  when TSV
+  #    obj.traverse options[:key_field], options[:fields], **options do |k,v|
+  #      res = yield k, v
+  #    end
+  #  when String
+  #    f = Open.open(obj)
+  #    self.traverse(f, into: into, cpus: cpus, bar: bar, **options, &block)
+  #  when Step
+  #    self.traverse(obj.stream, into: into, cpus: cpus, bar: bar, **options, &block)
+  #  when IO
+  #    if into && (IO === into || into.respond_to?(:stream) )
+  #      into_thread = Thread.new do 
+  #        Thread.current.report_on_exception = false
+  #        Thread.current["name"] = "Traverse into"
+  #        TSV.parse obj, **options do |k,v|
+  #          begin
+  #            res = block.call k, v
+  #            traverse_add into, res
+  #          rescue
+  #            into.abort $!
+  #          end
+  #          nil
+  #        end
+  #        into.close if into.respond_to?(:close)
+  #      end
+  #      Thread.pass until into_thread
+  #      into
+  #    else
+  #      TSV.parse obj, **options do |k,v|
+  #        block.call k, v
+  #        nil
+  #      end
+  #    end
+  #  end
+  #end
+
+  def self.traverse(obj, into: nil, cpus: nil, bar: nil, callback: nil, unnamed: true, **options, &block)
+
+    if into || bar
+      orig_callback = callback if callback
+      bar = Log::ProgressBar.get_obj_bar(bar, obj)
+      callback = proc do |res|
+        bar.tick if bar
+        traverse_add into, res if into
+        orig_callback.call res if orig_callback
+      end
+
+      if into.respond_to?(:close)
+        into_thread = Thread.new do 
+          Thread.current.report_on_exception = false
+          Thread.current["name"] = "Traverse into"
+          error = false
+          begin
+            self.traverse(obj, callback: callback, **options, &block)
+            into.close if into.respond_to?(:close)
+            bar.remove if bar
+          rescue Exception
+            into.abort($!) if into.respond_to?(:abort)
+            bar.remove($!) if bar
+          end
+        end
+        Thread.pass until into_thread
+        return into
+      end
+    end
+
+    begin
+      case obj
+      when TSV
+        obj.traverse options[:key_field], options[:fields], unnamed: unnamed, **options do |k,v|
+          res = block.call(k, v)
+          callback.call res if callback
+          nil
+        end
+      when Array
+        obj.each do |line|
+          res = block.call(line)
+          callback.call res if callback
+          nil
+        end
+      when String
+        f = Open.open(obj)
+        self.traverse(f, cpus: cpus, callback: callback, **options, &block)
+      when Step
+        raise obj.exception if obj.error?
+        self.traverse(obj.stream, cpus: cpus, callback: callback, **options, &block)
+      when IO
+        TSV.parse obj, **options do |k,v|
+          res = block.call k, v
+          callback.call res if callback
+          nil
+        end
+      else
+        TSV.parse obj, **options do |k,v|
+          res = block.call k, v
+          callback.call res if callback
+          nil
+        end
+      end
+      bar.remove if bar
+    rescue
+      bar.abort($!) if bar
+      raise $!
+    end
+
+    into
+  end
+end
+
+module TSV
+  def self.traverse(*args, **kwargs, &block)
+    Open.traverse(*args, **kwargs, &block)
+  end
+end