scout-gear 10.11.6 → 10.11.8

data/lib/scout/knowledge_base/traverse.rb

@@ -45,7 +45,7 @@ class KnowledgeBase
 
       [source_entities, target_entities]
     end
-    
+
     def reassign(matches, source, target)
       #assignments[source] = (matches.any? ? matches.collect{|m| m.source_entity }.uniq : nil) if is_wildcard? source
       #assignments[target] = (matches.any? ? matches.collect{|m| m.target_entity }.uniq : nil) if is_wildcard? target
@@ -64,7 +64,7 @@ class KnowledgeBase
           assigned = assignments[source] || []
           matches = matches.select{|m| assigned.include? m.partition("~").first }
         end
-        
+
         if is_wildcard? target
           assigned = assignments[target] || []
           matches = matches.select{|m| assigned.include? m.partition("~").last }
@@ -110,7 +110,7 @@ class KnowledgeBase
 
       return false if paths.empty?
 
-      paths 
+      paths
     end
 
     def _ep(paths)
@@ -172,8 +172,8 @@ class KnowledgeBase
         _name, _sep, _kb = db.partition("@")
         case
         when _name[0] == '?'
-          dbs = all_dbs.select{|_db| 
-            n,_s,d=_db.partition("@"); 
+          dbs = all_dbs.select{|_db|
+            n,_s,d=_db.partition("@");
             d.nil? or d.empty? or (d == _kb and assignments[_name].include?(n))
           }
         when _kb[0] == '?'
@@ -247,7 +247,7 @@ class KnowledgeBase
           else
             dbs = id_dbs(db)
             names = names.collect{|name| assignments.include?(name) ? assignments[name] : name}.flatten
-            ids = names.collect{|name| 
+            ids = names.collect{|name|
               id = nil
               dbs.each do |db|
                 sid, tid = identify db, name, name
@@ -286,44 +286,11 @@ class KnowledgeBase
       [assignments, paths]
     end
 
-    #def traverse
-    #  all_matches = []
-
-    #  rules.each do |rule|
-    #    rule = rule.strip
-    #    next if rule.empty?
-    #    source, db, target, conditions = rule.match(/([^\s]+)\s+([^\s]+)\s+([^\s]+)(?:\s+-\s+([^\s]+))?/).captures
-
-    #    source_entities, target_entities = identify db, source, target
-
-    #    matches = kb.subset(db, :source => source_entities, :target => target_entities)
-
-    #    if conditions
-    #      conditions.split(/\s+/).each do |condition|
-    #        if condition.index "="
-    #          key, value = conditions.split("=")
-    #          matches = matches.select{|m| m.info[key.strip].to_s =~ /\b#{value.strip}\b/}
-    #        else
-    #          matches = matches.select{|m| m.info[condition.strip].to_s =~ /\btrue\b/}
-    #        end
-    #      end
-    #    end
-
-    #    reassign matches, source, target
-
-    #    all_matches << matches
-    #  end
-
-    #  paths = find_paths rules, all_matches, assignments
-
-    #  [assignments, paths]
-    #end
-
   end
 
   def traverse(rules, nopaths=false)
     traverser = KnowledgeBase::Traverser.new self, rules
     traverser.traverse nopaths
   end
-    
+
 end

data/lib/scout/persist/engine/fix_width_table.rb

@@ -62,7 +62,7 @@ class FixWidthTable
       Persist::CONNECTIONS[persistence_path] = self.new(persistence_path, value_size, range, update)
     end
 
-    Persist::CONNECTIONS[persistence_path] 
+    Persist::CONNECTIONS[persistence_path]
   end
 
   def format(pos, value)
@@ -209,10 +209,10 @@ class FixWidthTable
     values = []
     l_start = idx_pos(idx)
     l_end   = idx_pos_end(idx)
-    
+
     if return_idx
       while l_start <= r_end
-        values << idx if l_end >= r_start 
+        values << idx if l_end >= r_start
         idx += 1
         break if idx >= size
         l_start = idx_pos(idx)
@@ -220,7 +220,7 @@ class FixWidthTable
       end
     else
       while l_start <= r_end
-        values << idx_value(idx) if l_end >= r_start 
+        values << idx_value(idx) if l_end >= r_start
         idx += 1
         break if idx >= size
         l_start = idx_pos(idx)
@@ -254,7 +254,7 @@ class FixWidthTable
     values = []
     l_start = idx_pos(idx)
     l_end   = idx_pos_end(idx)
-    if return_idx 
+    if return_idx
       while l_start <= r_end
         values << idx
         idx += 1
@@ -284,7 +284,7 @@ class FixWidthTable
       get_point(pos)
     end
   end
-  
+
   def overlaps(pos, value = false)
     return [] if size == 0
     idxs = if @range

data/lib/scout/persist/engine/packed_index.rb

@@ -56,7 +56,7 @@ class PackedIndex
   def file
     @persistence_path
   end
-  
+
   def close
     @stream.close
   end
@@ -82,7 +82,7 @@ class PackedIndex
   def get_position(position)
     @stream.seek(position * item_size + offset)
     encoded = @stream.read(item_size)
-    return nil if encoded.nil? or encoded == nil_string 
+    return nil if encoded.nil? or encoded == nil_string
     encoded.unpack mask
   end
 

data/lib/scout/persist/engine/sharder.rb

@@ -10,7 +10,7 @@ class Sharder
     @db_type = db_type
 
     if write
-      @databases = {} 
+      @databases = {}
     end
   end
 
@@ -210,9 +210,9 @@ class Sharder
   end
 
   def size
-    databases.inject(0){|acc,i| 
-      shard, db = i; 
-      acc += db.size 
+    databases.inject(0){|acc,i|
+      shard, db = i;
+      acc += db.size
     }
   end
 end

data/lib/scout/persist/engine/tkrzw.rb

@@ -5,7 +5,7 @@ module ScoutTKRZW
 
   def self.open(path, write = true, persistence_class = 'tkh', options = {})
     open_options = IndiferentHash.add_defaults options, truncate: true, num_buckets: 100, dbm: "HashDBM", sync_hard: true, encoding: "UTF-8"
-  
+
     path = path.find if Path === path
 
     dir = File.dirname(File.expand_path(path))

data/lib/scout/persist/engine/tokyocabinet.rb

@@ -31,11 +31,11 @@ if continue
       database = Log.ignore_stderr do Persist::CONNECTIONS[path] ||= tokyocabinet_class.new end
 
       if big and not Open.exists?(path)
-        database.tune(nil, nil, nil, tokyocabinet_class::TLARGE | tokyocabinet_class::TDEFLATE) 
+        database.tune(nil, nil, nil, tokyocabinet_class::TLARGE | tokyocabinet_class::TDEFLATE)
       end
 
       flags = (write ? tokyocabinet_class::OWRITER | tokyocabinet_class::OCREAT : tokyocabinet_class::OREADER)
-      database.close 
+      database.close
 
       if !database.open(path, flags)
         ecode = database.ecode

data/lib/scout/persist/tsv/adapter/fix_width_table.rb

@@ -101,6 +101,6 @@ Persist.save_drivers[:fwt] = proc do |file, content|
   Misc.sensiblewrite(file, content.file.read)
 end
 
-Persist.load_drivers[:fwt] = proc do |file| 
+Persist.load_drivers[:fwt] = proc do |file|
   FixWidthTable.new file
 end

data/lib/scout/persist/tsv/adapter/packed_index.rb

@@ -44,7 +44,7 @@ module PKIAdapter
   end
 
   def add(key, value)
-    key = pos_function.call(key) if pos_function 
+    key = pos_function.call(key) if pos_function
     if Numeric === key
       @_last ||= -1
       skipped = key - @_last - 1

data/lib/scout/persist/tsv/adapter/tkrzw.rb

@@ -11,7 +11,7 @@ Persist.save_drivers[:tkh] = proc do |file, content|
   data
 end
 
-Persist.load_drivers[:tkh] = proc do |file| 
+Persist.load_drivers[:tkh] = proc do |file|
   data = ScoutTKRZW.open(file, false, "tkh")
   data.extend TSVAdapter unless TSVAdapter === data
   data

data/lib/scout/persist/tsv/adapter/tokyocabinet.rb

@@ -1,7 +1,7 @@
 require_relative 'base'
 require_relative '../../engine/tokyocabinet'
 
-module TKAdapter 
+module TKAdapter
   include TSVAdapter
   def self.extended(obj)
     obj.extend TSVAdapter
@@ -38,7 +38,7 @@ Persist.save_drivers[:HDB] = proc do |file, content|
   end
 end
 
-Persist.load_drivers[:HDB] = proc do |file| 
+Persist.load_drivers[:HDB] = proc do |file|
   data = ScoutCabinet.open(file, false, "HDB")
   data.extend TKAdapter unless TKAdapter === data
   data
@@ -58,7 +58,7 @@ Persist.save_drivers[:BDB] = proc do |file, content|
   end
 end
 
-Persist.load_drivers[:BDB] = proc do |file| 
+Persist.load_drivers[:BDB] = proc do |file|
   data = ScoutCabinet.open(file, false, "BDB")
   data.extend TKAdapter unless TKAdapter === data
   data

data/lib/scout/persist/tsv/serialize.rb

@@ -10,7 +10,7 @@ module TSVAdapter
     def self.dump(o); [o].pack('m'); end
     def self.load(str); str.unpack('m').first; end
   end
-  
+
   class IntegerSerializer
     def self.dump(i); [i].pack("l"); end
     def self.load(str); str.unpack("l").first; end
@@ -101,8 +101,8 @@ module TSVAdapter
     :flat => StringArraySerializer,
     :double => StringDoubleArraySerializer,
     :clean => CleanSerializer,
-    :integer => IntegerSerializer, 
-    :float => FloatSerializer, 
+    :integer => IntegerSerializer,
+    :float => FloatSerializer,
     :integer_array => IntegerArraySerializer,
     :float_array => FloatArraySerializer,
     :strict_integer_array => StrictIntegerArraySerializer,

data/lib/scout/persist/tsv.rb

@@ -2,7 +2,7 @@ require 'scout/persist'
 require_relative 'engine'
 require_relative 'tsv/adapter'
 
-Persist.save_drivers[:tsv] = proc do |file,content| 
+Persist.save_drivers[:tsv] = proc do |file,content|
   case content
   when IO
     Open.sensible_write(file, content)

data/lib/scout/semaphore.rb

@@ -1,5 +1,6 @@
 begin
   require 'inline'
+  require 'fileutils'
   continue = true
 rescue Exception
   Log.warn "The RubyInline gem could not be loaded: semaphore synchronization will not work"
@@ -159,9 +160,55 @@ if continue
       end
     end
 
+    # Try to create the semaphore while holding a per-semaphore lock to avoid races
+    def self.ensure_or_create(name, size = 1)
+      # Normalize and make a safe lock path under Scout.tmp.semaphore_locks
+      lock_dir = if defined?(Scout) && Scout.respond_to?(:tmp) && Scout.tmp.respond_to?(:semaphore_locks)
+                   Scout.tmp.semaphore_locks
+                 else
+                   File.join('/tmp', 'scout', 'semaphore_locks')
+                 end
+
+      FileUtils.mkdir_p(lock_dir) unless File.exist?(lock_dir)
+
+      lock_base = File.join(lock_dir, name.gsub(%r{^/+}, '').gsub('/', '_'))
+
+      begin
+        Open.lock(lock_base) do |_lf|
+          # If someone else created it while waiting for the lock, we're done
+          return true if self.exists?(name)
+
+          Log.info "Semaphore #{name} missing; creating under lock #{lock_base}"
+          begin
+            # call the lower-level C create and let create_semaphore perform checks/retries
+            ret = ScoutSemaphore.create_semaphore_c(name, size)
+            if ret < 0
+              Log.warn "create_semaphore_c failed for #{name}: errno=#{-ret}"
+              return false
+            end
+
+            # best-effort: ensure the file shows up
+            unless self.exists?(name)
+              Log.warn "Semaphore #{name} created but /dev/shm entry not visible"
+            end
+
+            Log.info "Semaphore #{name} created"
+            return true
+          rescue Exception => e
+            Log.warn "Exception while creating semaphore #{name}: #{e.message}"
+            return false
+          end
+        end
+      rescue Exception => e
+        Log.warn "Failed to acquire creation lock for #{name}: #{e.message}"
+        return false
+      end
+    end
+
     # Safe wrappers that raise SystemCallError on final failure
     def self.create_semaphore(name, value, **opts)
       ret = with_retry(**opts) { ScoutSemaphore.create_semaphore_c(name, value) }
+      # After creation attempt, make sure the /dev/shm entry exists (cluster may remove entries)
       raise SystemCallError.new("Semaphore missing (#{name})") unless self.exists?(name)
       if ret < 0
         raise SystemCallError.new("create_semaphore(#{name}) failed", -ret)
@@ -172,14 +219,28 @@ if continue
     def self.delete_semaphore(name, **opts)
       ret = with_retry(**opts) { ScoutSemaphore.delete_semaphore_c(name) }
       if ret < 0
-        raise SystemCallError.new("delete_semaphore(#{name}) failed", -ret)
+        Log.warn("delete_semaphore(#{name}) failed")
       end
       ret
     end
 
     def self.wait_semaphore(name, **opts)
-      raise SystemCallError.new("Semaphore missing (#{name})") unless self.exists?(name)
+      # Try a normal wait first
       ret = with_retry(**opts) { ScoutSemaphore.wait_semaphore_c(name) }
+
+      if ret < 0
+        err = -ret
+        # If semaphore missing or removed, try to recreate it under a lock and retry once
+        if err == Errno::ENOENT.new.errno || err == Errno::EIDRM.new.errno
+          Log.warn "wait_semaphore: semaphore #{name} appears missing (errno=#{err}); attempting recreate"
+          created = ensure_or_create(name, opts.fetch(:create_size, 1))
+          if created
+            # retry the wait after creating
+            ret = with_retry(**opts) { ScoutSemaphore.wait_semaphore_c(name) }
+          end
+        end
+      end
+
       if ret < 0
         err = -ret
         if err == Errno::EINTR.new.errno
@@ -188,12 +249,26 @@ if continue
           raise SystemCallError.new("wait_semaphore(#{name}) failed", err)
         end
       end
+
       ret
     end
 
     def self.post_semaphore(name, **opts)
-      raise SystemCallError.new("Semaphore missing (#{name})") unless self.exists?(name)
+      # Try normal post first
       ret = with_retry(**opts) { ScoutSemaphore.post_semaphore_c(name) }
+
+      if ret < 0
+        err = -ret
+        # If semaphore missing or removed, try to recreate it under a lock and then post
+        if err == Errno::ENOENT.new.errno || err == Errno::EIDRM.new.errno
+          Log.warn "post_semaphore: semaphore #{name} appears missing (errno=#{err}); attempting recreate"
+          created = ensure_or_create(name, opts.fetch(:create_size, 1))
+          if created
+            ret = with_retry(**opts) { ScoutSemaphore.post_semaphore_c(name) }
+          end
+        end
+      end
+
       if ret < 0
         raise SystemCallError.new("post_semaphore(#{name}) failed", -ret)
       end

data/lib/scout/tsv/annotation/repo.rb

@@ -56,25 +56,25 @@ module Persist
     else
       annotations = yield
 
-      repo.write_and_close do 
+      repo.write_and_close do
         case
         when annotations.nil?
           repo[subkey + "NIL"] = nil
         when annotations.empty?
           repo[subkey + "EMPTY"] = nil
         when (not Array === annotations or (AnnotatedArray === annotations and not Array === annotations.first))
-          tsv_values = Annotation.obj_tsv_values(annotations, repo_fields) 
+          tsv_values = Annotation.obj_tsv_values(annotations, repo_fields)
           repo[subkey + annotations.id << ":" << "SINGLE"] = tsv_values
         when (not Array === annotations or (AnnotatedArray === annotations and AnnotatedArray === annotations.first))
           annotations.each_with_index do |e,i|
             next if e.nil?
-            tsv_values = Annotation.obj_tsv_values(e, repo_fields) 
+            tsv_values = Annotation.obj_tsv_values(e, repo_fields)
             repo[subkey + "ANNOTATED_DOUBLE_ARRAY:" << i.to_s] = tsv_values
           end
         else
           annotations.each_with_index do |e,i|
             next if e.nil?
-            tsv_values = Annotation.obj_tsv_values(e, repo_fields) 
+            tsv_values = Annotation.obj_tsv_values(e, repo_fields)
             repo[subkey + i.to_s] = tsv_values
           end
         end

data/lib/scout/tsv/annotation.rb

@@ -41,7 +41,7 @@ module Annotation
   def self.list_tsv_values(objs, fields)
     obj_tsv_values(objs, fields)
   end
-  
+
 
   def self.tsv(objs, *fields)
     return nil if objs.nil?
@@ -76,7 +76,7 @@ module Annotation
       tsv.key_field = "List"
 
       tsv[objs.id] = self.list_tsv_values(objs, fields).dup
-    when Array === objs 
+    when Array === objs
       tsv.key_field = "ID"
 
       if Annotation.is_annotated?(objs.compact.first)

data/lib/scout/tsv/attach.rb

@@ -28,7 +28,7 @@ module TSV
       end
     end
 
-    match_key = source.key_field if match_key.nil? 
+    match_key = source.key_field if match_key.nil?
 
     if other_key.nil?
       other_key = other.identify_field(match_key)
@@ -87,13 +87,13 @@ module TSV
           index = TSV.translation_index(identifier_files.flatten, match_key_name, other_key_name)
         end
 
-        if other_key != :key 
+        if other_key != :key
           other = other.reorder other_key, fields, one2one: one2one, merge: true, type: :double
         end
 
-        other_field_positions = other.identify_field(fields.dup) 
+        other_field_positions = other.identify_field(fields.dup)
         fields.zip(other_field_positions) do |o,n|
-          raise "Field #{o} not found. Options: #{Log.fingerprint other.fields}" if n.nil? 
+          raise "Field #{o} not found. Options: #{Log.fingerprint other.fields}" if n.nil?
         end
 
         log_message = "Attach #{Log.fingerprint fields - source.fields} to #{Log.fingerprint source} (#{[match_key, other_key] * "=~"})"
@@ -139,7 +139,7 @@ module TSV
 
             if other_values.nil?
               other_values = empty_other_values
-            elsif other.type == :flat 
+            elsif other.type == :flat
               other_values = [other_values]
             elsif other.type == :list && source.type == :double
               other_values = other_values.collect{|v| [v] }
@@ -185,9 +185,9 @@ module TSV
                               end
           other.each do |other_key,other_values|
             next if source.include?(other_key)
-            if other.type == :flat 
+            if other.type == :flat
               other_values = [other_values]
-            elsif other.type == :single 
+            elsif other.type == :single
               other_values = [other_values]
             elsif other.type == :list && type == :double
               other_values = other_values.collect{|v| [v] }

data/lib/scout/tsv/change_id/translate.rb

@@ -87,7 +87,7 @@ module TSV
                       pos = NamedArray.identify_name(TSV.all_fields(file1), TSV.all_fields(file2))
                       TSV.all_fields(file1)[pos.compact.first]
                     end
-    Persist.persist(name, "HDB", persist_options) do 
+    Persist.persist(name, "HDB", persist_options) do
       index = path.inject(nil) do |acc,file|
         if acc.nil?
           if source.nil?

data/lib/scout/tsv/csv.rb

@@ -1,7 +1,7 @@
 require 'csv'
 
 module TSV
-  def self.csv(obj, options = {}) 
+  def self.csv(obj, options = {})
     options = IndiferentHash.add_defaults options, :headers => true, :type => :list
     headers = options[:headers]
 
@@ -12,7 +12,7 @@ module TSV
     merge = options.delete :merge
     key_field = options.delete :key_field
     fields = options.delete :fields
-    
+
     if key_field || fields
       orig_type = type
       type = :double
@@ -49,7 +49,7 @@ module TSV
       else
         key, *values = row
       end
-      
+
       if cast
         values = values.collect{|v| v.send cast }
       end

data/lib/scout/tsv/dumper.rb

@@ -1,7 +1,7 @@
 module TSV
   class Dumper
     def self.header(options={})
-      key_field, fields, sep, header_hash, preamble, unnamed = IndiferentHash.process_options options, 
+      key_field, fields, sep, header_hash, preamble, unnamed = IndiferentHash.process_options options,
         :key_field, :fields, :sep, :header_hash, :preamble, :unnamed,
         :sep => "\t", :header_hash => "#", :preamble => true
 
@@ -29,8 +29,8 @@ module TSV
     attr_accessor :options, :initialized, :type, :sep, :compact, :filename, :namespace
     def initialize(options = {})
       options = options.options.merge(sep: nil) if TSV::Parser === options || TSV === options
-      @sep, @type = IndiferentHash.process_options options, 
-        :sep, :type, 
+      @sep, @type = IndiferentHash.process_options options,
+        :sep, :type,
         :sep => "\t", :type => :double
       @compact = options[:compact]
       @options = options
@@ -54,7 +54,7 @@ module TSV
     def key_field
       @options[:key_field]
     end
-    
+
     def fields
       @options[:fields]
     end
@@ -62,7 +62,7 @@ module TSV
     def key_field=(key_field)
       @options[:key_field] = key_field
     end
-    
+
     def fields=(fields)
       @options[:fields] = fields
     end
@@ -100,7 +100,7 @@ module TSV
           when :double
             @sin << key + @sep + value.collect{|v| Array === v ? (@compact ? v.compact : v) * "|" : v } * @sep << "\n"
           else
-            if Array === value 
+            if Array === value
               if Array === value.first
                 @sin << key + @sep + value.collect{|v| Array === v ? (@compact ? v.compact : v) * "|" : v } * @sep << "\n"
               else
@@ -147,7 +147,7 @@ module TSV
   end
 
   def dumper_stream(options = {})
-    preamble, unmerge, keys, stream = IndiferentHash.process_options options, 
+    preamble, unmerge, keys, stream = IndiferentHash.process_options options,
       :preamble, :unmerge, :keys, :stream,
       :preamble => true, :unmerge => false
     unmerge = false unless @type === :double
@@ -177,7 +177,7 @@ module TSV
 
     self.with_unnamed do
       if stream.nil?
-        t = Thread.new do 
+        t = Thread.new do
           begin
             Thread.current.report_on_exception = true
             Thread.current["name"] = "Dumper thread"

data/lib/scout/tsv/index.rb

@@ -61,7 +61,7 @@ module TSV
       if filename
         index = ScoutCabinet.open(filename, true, engine)
         TSV.setup(index, :type => :single)
-        index.extend TSVAdapter 
+        index.extend TSVAdapter
       else
         index = TSV.setup({}, :type => :single)
       end

data/lib/scout/tsv/open.rb

@@ -62,7 +62,7 @@ module Open
 
       if into.respond_to?(:close)
         obj = obj.find if Path === obj
-        into_thread = Thread.new do 
+        into_thread = Thread.new do
           Thread.current.report_on_exception = false
           Thread.current["name"] = "Traverse into"
           error = false
@@ -96,7 +96,7 @@ module Open
       queue.process do |res|
         callback.call res if callback
       end
-      
+
       begin
         self.traverse(obj, **options) do |*args|
           queue.write args
@@ -210,7 +210,7 @@ module TSV
 
   def self.process_stream(stream, header_hash: "#", &block)
     sout = Open.open_pipe do |sin|
-      while line = stream.gets 
+      while line = stream.gets
         break unless line.start_with?(header_hash)
         sin.puts line
       end

data/lib/scout/tsv/stream.rb

@@ -77,7 +77,7 @@ module TSV
         fields = fields.compact.flatten
       end
 
-      options = input_options.first 
+      options = input_options.first
       type ||= options[:type]
       type ||= :list if type == :single
       type ||= :double if type == :flat
@@ -152,7 +152,7 @@ module TSV
 
                 if k == keys[i]
                   new_parts = NamedArray.zip_fields([new_parts]).zip(p).collect{|p| [p.flatten * "|"] }
-                  raise TryAgain 
+                  raise TryAgain
                 end
                 keys[i]= k
                 parts[i]= p