scout-gear 10.11.4 → 10.11.7

data/lib/scout/knowledge_base/traverse.rb

@@ -45,7 +45,7 @@ class KnowledgeBase
 
       [source_entities, target_entities]
     end
-    
+
     def reassign(matches, source, target)
       #assignments[source] = (matches.any? ? matches.collect{|m| m.source_entity }.uniq : nil) if is_wildcard? source
       #assignments[target] = (matches.any? ? matches.collect{|m| m.target_entity }.uniq : nil) if is_wildcard? target
@@ -64,7 +64,7 @@ class KnowledgeBase
           assigned = assignments[source] || []
           matches = matches.select{|m| assigned.include? m.partition("~").first }
         end
-        
+
         if is_wildcard? target
           assigned = assignments[target] || []
           matches = matches.select{|m| assigned.include? m.partition("~").last }
@@ -110,7 +110,7 @@ class KnowledgeBase
 
       return false if paths.empty?
 
-      paths 
+      paths
     end
 
     def _ep(paths)
@@ -172,8 +172,8 @@ class KnowledgeBase
         _name, _sep, _kb = db.partition("@")
         case
         when _name[0] == '?'
-          dbs = all_dbs.select{|_db| 
-            n,_s,d=_db.partition("@"); 
+          dbs = all_dbs.select{|_db|
+            n,_s,d=_db.partition("@");
             d.nil? or d.empty? or (d == _kb and assignments[_name].include?(n))
           }
         when _kb[0] == '?'
@@ -247,7 +247,7 @@ class KnowledgeBase
           else
             dbs = id_dbs(db)
             names = names.collect{|name| assignments.include?(name) ? assignments[name] : name}.flatten
-            ids = names.collect{|name| 
+            ids = names.collect{|name|
               id = nil
               dbs.each do |db|
                 sid, tid = identify db, name, name
@@ -286,44 +286,11 @@ class KnowledgeBase
       [assignments, paths]
     end
 
-    #def traverse
-    #  all_matches = []
-
-    #  rules.each do |rule|
-    #    rule = rule.strip
-    #    next if rule.empty?
-    #    source, db, target, conditions = rule.match(/([^\s]+)\s+([^\s]+)\s+([^\s]+)(?:\s+-\s+([^\s]+))?/).captures
-
-    #    source_entities, target_entities = identify db, source, target
-
-    #    matches = kb.subset(db, :source => source_entities, :target => target_entities)
-
-    #    if conditions
-    #      conditions.split(/\s+/).each do |condition|
-    #        if condition.index "="
-    #          key, value = conditions.split("=")
-    #          matches = matches.select{|m| m.info[key.strip].to_s =~ /\b#{value.strip}\b/}
-    #        else
-    #          matches = matches.select{|m| m.info[condition.strip].to_s =~ /\btrue\b/}
-    #        end
-    #      end
-    #    end
-
-    #    reassign matches, source, target
-
-    #    all_matches << matches
-    #  end
-
-    #  paths = find_paths rules, all_matches, assignments
-
-    #  [assignments, paths]
-    #end
-
   end
 
   def traverse(rules, nopaths=false)
     traverser = KnowledgeBase::Traverser.new self, rules
     traverser.traverse nopaths
   end
-    
+
 end

data/lib/scout/persist/engine/fix_width_table.rb

@@ -62,7 +62,7 @@ class FixWidthTable
       Persist::CONNECTIONS[persistence_path] = self.new(persistence_path, value_size, range, update)
     end
 
-    Persist::CONNECTIONS[persistence_path] 
+    Persist::CONNECTIONS[persistence_path]
   end
 
   def format(pos, value)
@@ -209,10 +209,10 @@ class FixWidthTable
     values = []
     l_start = idx_pos(idx)
     l_end   = idx_pos_end(idx)
-    
+
     if return_idx
       while l_start <= r_end
-        values << idx if l_end >= r_start 
+        values << idx if l_end >= r_start
         idx += 1
         break if idx >= size
         l_start = idx_pos(idx)
@@ -220,7 +220,7 @@ class FixWidthTable
       end
     else
       while l_start <= r_end
-        values << idx_value(idx) if l_end >= r_start 
+        values << idx_value(idx) if l_end >= r_start
         idx += 1
         break if idx >= size
         l_start = idx_pos(idx)
@@ -254,7 +254,7 @@ class FixWidthTable
     values = []
     l_start = idx_pos(idx)
     l_end   = idx_pos_end(idx)
-    if return_idx 
+    if return_idx
       while l_start <= r_end
         values << idx
         idx += 1
@@ -284,7 +284,7 @@ class FixWidthTable
       get_point(pos)
     end
   end
-  
+
   def overlaps(pos, value = false)
     return [] if size == 0
     idxs = if @range

data/lib/scout/persist/engine/packed_index.rb

@@ -56,7 +56,7 @@ class PackedIndex
   def file
     @persistence_path
   end
-  
+
   def close
     @stream.close
   end
@@ -82,7 +82,7 @@ class PackedIndex
   def get_position(position)
     @stream.seek(position * item_size + offset)
     encoded = @stream.read(item_size)
-    return nil if encoded.nil? or encoded == nil_string 
+    return nil if encoded.nil? or encoded == nil_string
     encoded.unpack mask
   end
 

data/lib/scout/persist/engine/sharder.rb

@@ -10,7 +10,7 @@ class Sharder
     @db_type = db_type
 
     if write
-      @databases = {} 
+      @databases = {}
     end
   end
 
@@ -210,9 +210,9 @@ class Sharder
   end
 
   def size
-    databases.inject(0){|acc,i| 
-      shard, db = i; 
-      acc += db.size 
+    databases.inject(0){|acc,i|
+      shard, db = i;
+      acc += db.size
     }
   end
 end

data/lib/scout/persist/engine/tkrzw.rb

@@ -5,7 +5,7 @@ module ScoutTKRZW
 
   def self.open(path, write = true, persistence_class = 'tkh', options = {})
     open_options = IndiferentHash.add_defaults options, truncate: true, num_buckets: 100, dbm: "HashDBM", sync_hard: true, encoding: "UTF-8"
-  
+
     path = path.find if Path === path
 
     dir = File.dirname(File.expand_path(path))

data/lib/scout/persist/engine/tokyocabinet.rb

@@ -31,11 +31,11 @@ if continue
       database = Log.ignore_stderr do Persist::CONNECTIONS[path] ||= tokyocabinet_class.new end
 
       if big and not Open.exists?(path)
-        database.tune(nil, nil, nil, tokyocabinet_class::TLARGE | tokyocabinet_class::TDEFLATE) 
+        database.tune(nil, nil, nil, tokyocabinet_class::TLARGE | tokyocabinet_class::TDEFLATE)
       end
 
       flags = (write ? tokyocabinet_class::OWRITER | tokyocabinet_class::OCREAT : tokyocabinet_class::OREADER)
-      database.close 
+      database.close
 
       if !database.open(path, flags)
         ecode = database.ecode

data/lib/scout/persist/tsv/adapter/fix_width_table.rb

@@ -101,6 +101,6 @@ Persist.save_drivers[:fwt] = proc do |file, content|
   Misc.sensiblewrite(file, content.file.read)
 end
 
-Persist.load_drivers[:fwt] = proc do |file| 
+Persist.load_drivers[:fwt] = proc do |file|
   FixWidthTable.new file
 end

data/lib/scout/persist/tsv/adapter/packed_index.rb

@@ -44,7 +44,7 @@ module PKIAdapter
   end
 
   def add(key, value)
-    key = pos_function.call(key) if pos_function 
+    key = pos_function.call(key) if pos_function
     if Numeric === key
       @_last ||= -1
       skipped = key - @_last - 1

data/lib/scout/persist/tsv/adapter/tkrzw.rb

@@ -11,7 +11,7 @@ Persist.save_drivers[:tkh] = proc do |file, content|
   data
 end
 
-Persist.load_drivers[:tkh] = proc do |file| 
+Persist.load_drivers[:tkh] = proc do |file|
   data = ScoutTKRZW.open(file, false, "tkh")
   data.extend TSVAdapter unless TSVAdapter === data
   data

data/lib/scout/persist/tsv/adapter/tokyocabinet.rb

@@ -1,7 +1,7 @@
 require_relative 'base'
 require_relative '../../engine/tokyocabinet'
 
-module TKAdapter 
+module TKAdapter
   include TSVAdapter
   def self.extended(obj)
     obj.extend TSVAdapter
@@ -38,7 +38,7 @@ Persist.save_drivers[:HDB] = proc do |file, content|
   end
 end
 
-Persist.load_drivers[:HDB] = proc do |file| 
+Persist.load_drivers[:HDB] = proc do |file|
   data = ScoutCabinet.open(file, false, "HDB")
   data.extend TKAdapter unless TKAdapter === data
   data
@@ -58,7 +58,7 @@ Persist.save_drivers[:BDB] = proc do |file, content|
   end
 end
 
-Persist.load_drivers[:BDB] = proc do |file| 
+Persist.load_drivers[:BDB] = proc do |file|
   data = ScoutCabinet.open(file, false, "BDB")
   data.extend TKAdapter unless TKAdapter === data
   data

data/lib/scout/persist/tsv/serialize.rb

@@ -10,7 +10,7 @@ module TSVAdapter
     def self.dump(o); [o].pack('m'); end
     def self.load(str); str.unpack('m').first; end
   end
-  
+
   class IntegerSerializer
     def self.dump(i); [i].pack("l"); end
     def self.load(str); str.unpack("l").first; end
@@ -101,8 +101,8 @@ module TSVAdapter
     :flat => StringArraySerializer,
     :double => StringDoubleArraySerializer,
     :clean => CleanSerializer,
-    :integer => IntegerSerializer, 
-    :float => FloatSerializer, 
+    :integer => IntegerSerializer,
+    :float => FloatSerializer,
     :integer_array => IntegerArraySerializer,
     :float_array => FloatArraySerializer,
     :strict_integer_array => StrictIntegerArraySerializer,

data/lib/scout/persist/tsv.rb

@@ -2,7 +2,7 @@ require 'scout/persist'
 require_relative 'engine'
 require_relative 'tsv/adapter'
 
-Persist.save_drivers[:tsv] = proc do |file,content| 
+Persist.save_drivers[:tsv] = proc do |file,content|
   case content
   when IO
     Open.sensible_write(file, content)

data/lib/scout/semaphore.rb

@@ -1,5 +1,6 @@
 begin
   require 'inline'
+  require 'fileutils'
   continue = true
 rescue Exception
   Log.warn "The RubyInline gem could not be loaded: semaphore synchronization will not work"
@@ -25,7 +26,7 @@ if continue
 
       # Create a named semaphore. Return 0 on success, -errno on error.
       builder.c_singleton <<-EOF
-  int create_semaphore(char* name, int value){
+  int create_semaphore_c(char* name, int value){
     sem_t* sem;
     sem = sem_open(name, O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO, value);
     if (sem == SEM_FAILED){
@@ -39,7 +40,7 @@ if continue
 
       # Unlink (remove) a named semaphore. Return 0 on success, -errno on error.
       builder.c_singleton <<-EOF
-  int delete_semaphore(char* name){
+  int delete_semaphore_c(char* name){
     int ret = sem_unlink(name);
     if (ret == -1) {
       return -errno;
@@ -50,7 +51,7 @@ if continue
 
       # Wait (sem_wait) on a named semaphore. Return 0 on success, -errno on error.
       builder.c_singleton <<-EOF
-  int wait_semaphore(char* name){
+  int wait_semaphore_c(char* name){
     sem_t* sem;
     sem = sem_open(name, 0);
     if (sem == SEM_FAILED){
@@ -76,7 +77,7 @@ if continue
 
       # Post (sem_post) on a named semaphore. Return 0 on success, -errno on error.
       builder.c_singleton <<-EOF
-  int post_semaphore(char* name){
+  int post_semaphore_c(char* name){
     sem_t* sem;
     sem = sem_open(name, 0);
     if (sem == SEM_FAILED){
@@ -113,6 +114,11 @@ if continue
       s
     end
 
+    def self.exists?(name)
+      file = File.join('/dev/shm', 'sem.' + name[1..-1])
+      Open.exists? file
+    end
+
     # Errno numeric lists
     RETRIABLE_ERRNOS = [
       Errno::ENOENT,
@@ -154,25 +160,87 @@ if continue
       end
     end
 
+    # Try to create the semaphore while holding a per-semaphore lock to avoid races
+    def self.ensure_or_create(name, size = 1)
+      # Normalize and make a safe lock path under Scout.tmp.semaphore_locks
+      lock_dir = if defined?(Scout) && Scout.respond_to?(:tmp) && Scout.tmp.respond_to?(:semaphore_locks)
+                   Scout.tmp.semaphore_locks
+                 else
+                   File.join('/tmp', 'scout', 'semaphore_locks')
+                 end
+
+      FileUtils.mkdir_p(lock_dir) unless File.exist?(lock_dir)
+
+      lock_base = File.join(lock_dir, name.gsub(%r{^/+}, '').gsub('/', '_'))
+
+      begin
+        Open.lock(lock_base) do |_lf|
+          # If someone else created it while waiting for the lock, we're done
+          return true if self.exists?(name)
+
+          Log.info "Semaphore #{name} missing; creating under lock #{lock_base}"
+          begin
+            # call the lower-level C create and let create_semaphore perform checks/retries
+            ret = ScoutSemaphore.create_semaphore_c(name, size)
+            if ret < 0
+              Log.warn "create_semaphore_c failed for #{name}: errno=#{-ret}"
+              return false
+            end
+
+            # best-effort: ensure the file shows up
+            unless self.exists?(name)
+              Log.warn "Semaphore #{name} created but /dev/shm entry not visible"
+            end
+
+            Log.info "Semaphore #{name} created"
+            return true
+          rescue Exception => e
+            Log.warn "Exception while creating semaphore #{name}: #{e.message}"
+            return false
+          end
+        end
+      rescue Exception => e
+        Log.warn "Failed to acquire creation lock for #{name}: #{e.message}"
+        return false
+      end
+    end
+
     # Safe wrappers that raise SystemCallError on final failure
-    def self.safe_create_semaphore(name, value, **opts)
-      ret = with_retry(**opts) { ScoutSemaphore.create_semaphore(name, value) }
+    def self.create_semaphore(name, value, **opts)
+      ret = with_retry(**opts) { ScoutSemaphore.create_semaphore_c(name, value) }
+      # After creation attempt, make sure the /dev/shm entry exists (cluster may remove entries)
+      raise SystemCallError.new("Semaphore missing (#{name})") unless self.exists?(name)
       if ret < 0
         raise SystemCallError.new("create_semaphore(#{name}) failed", -ret)
       end
       ret
     end
 
-    def self.safe_delete_semaphore(name, **opts)
-      ret = with_retry(**opts) { ScoutSemaphore.delete_semaphore(name) }
+    def self.delete_semaphore(name, **opts)
+      ret = with_retry(**opts) { ScoutSemaphore.delete_semaphore_c(name) }
       if ret < 0
-        raise SystemCallError.new("delete_semaphore(#{name}) failed", -ret)
+        Log.warn("delete_semaphore(#{name}) failed")
       end
       ret
     end
 
-    def self.safe_wait_semaphore(name, **opts)
-      ret = with_retry(**opts) { ScoutSemaphore.wait_semaphore(name) }
+    def self.wait_semaphore(name, **opts)
+      # Try a normal wait first
+      ret = with_retry(**opts) { ScoutSemaphore.wait_semaphore_c(name) }
+
+      if ret < 0
+        err = -ret
+        # If semaphore missing or removed, try to recreate it under a lock and retry once
+        if err == Errno::ENOENT.new.errno || err == Errno::EIDRM.new.errno
+          Log.warn "wait_semaphore: semaphore #{name} appears missing (errno=#{err}); attempting recreate"
+          created = ensure_or_create(name, opts.fetch(:create_size, 1))
+          if created
+            # retry the wait after creating
+            ret = with_retry(**opts) { ScoutSemaphore.wait_semaphore_c(name) }
+          end
+        end
+      end
+
       if ret < 0
         err = -ret
         if err == Errno::EINTR.new.errno
@@ -181,11 +249,26 @@ if continue
           raise SystemCallError.new("wait_semaphore(#{name}) failed", err)
         end
       end
+
       ret
     end
 
-    def self.safe_post_semaphore(name, **opts)
-      ret = with_retry(**opts) { ScoutSemaphore.post_semaphore(name) }
+    def self.post_semaphore(name, **opts)
+      # Try normal post first
+      ret = with_retry(**opts) { ScoutSemaphore.post_semaphore_c(name) }
+
+      if ret < 0
+        err = -ret
+        # If semaphore missing or removed, try to recreate it under a lock and then post
+        if err == Errno::ENOENT.new.errno || err == Errno::EIDRM.new.errno
+          Log.warn "post_semaphore: semaphore #{name} appears missing (errno=#{err}); attempting recreate"
+          created = ensure_or_create(name, opts.fetch(:create_size, 1))
+          if created
+            ret = with_retry(**opts) { ScoutSemaphore.post_semaphore_c(name) }
+          end
+        end
+      end
+
       if ret < 0
         raise SystemCallError.new("post_semaphore(#{name}) failed", -ret)
       end
@@ -198,7 +281,7 @@ if continue
 
       # wait_semaphore returns 0 on success or -errno on error
       begin
-        ScoutSemaphore.safe_wait_semaphore(sem)
+        ScoutSemaphore.wait_semaphore(sem)
       rescue SemaphoreInterrupted
         raise
       rescue SystemCallError => e
@@ -210,7 +293,7 @@ if continue
         yield
       ensure
         begin
-          ScoutSemaphore.safe_post_semaphore(sem)
+          ScoutSemaphore.post_semaphore(sem)
         rescue SystemCallError => e
           # Log but don't raise from ensure
           # Log.warn "post_semaphore(#{sem}) failed in ensure: #{e.message}"
@@ -232,7 +315,7 @@ if continue
       begin
         Log.debug "Creating semaphore (#{ size }): #{file}"
         begin
-          ScoutSemaphore.safe_create_semaphore(file, size)
+          ScoutSemaphore.create_semaphore(file, size)
         rescue SystemCallError => e
           Log.error "Failed to create semaphore #{file}: #{e.message}"
           raise
@@ -242,7 +325,7 @@ if continue
       ensure
         Log.debug "Removing semaphore #{ file }"
         begin
-          ScoutSemaphore.safe_delete_semaphore(file)
+          ScoutSemaphore.delete_semaphore(file)
         rescue SystemCallError => e
           Log.warn "delete_semaphore(#{file}) failed: #{e.message}"
         end

data/lib/scout/tsv/annotation/repo.rb

@@ -56,25 +56,25 @@ module Persist
     else
       annotations = yield
 
-      repo.write_and_close do 
+      repo.write_and_close do
         case
         when annotations.nil?
           repo[subkey + "NIL"] = nil
         when annotations.empty?
           repo[subkey + "EMPTY"] = nil
         when (not Array === annotations or (AnnotatedArray === annotations and not Array === annotations.first))
-          tsv_values = Annotation.obj_tsv_values(annotations, repo_fields) 
+          tsv_values = Annotation.obj_tsv_values(annotations, repo_fields)
           repo[subkey + annotations.id << ":" << "SINGLE"] = tsv_values
         when (not Array === annotations or (AnnotatedArray === annotations and AnnotatedArray === annotations.first))
           annotations.each_with_index do |e,i|
             next if e.nil?
-            tsv_values = Annotation.obj_tsv_values(e, repo_fields) 
+            tsv_values = Annotation.obj_tsv_values(e, repo_fields)
             repo[subkey + "ANNOTATED_DOUBLE_ARRAY:" << i.to_s] = tsv_values
           end
         else
           annotations.each_with_index do |e,i|
             next if e.nil?
-            tsv_values = Annotation.obj_tsv_values(e, repo_fields) 
+            tsv_values = Annotation.obj_tsv_values(e, repo_fields)
             repo[subkey + i.to_s] = tsv_values
           end
         end

data/lib/scout/tsv/annotation.rb

@@ -41,7 +41,7 @@ module Annotation
   def self.list_tsv_values(objs, fields)
     obj_tsv_values(objs, fields)
   end
-  
+
 
   def self.tsv(objs, *fields)
     return nil if objs.nil?
@@ -76,7 +76,7 @@ module Annotation
       tsv.key_field = "List"
 
       tsv[objs.id] = self.list_tsv_values(objs, fields).dup
-    when Array === objs 
+    when Array === objs
       tsv.key_field = "ID"
 
       if Annotation.is_annotated?(objs.compact.first)

data/lib/scout/tsv/attach.rb

@@ -28,7 +28,7 @@ module TSV
       end
     end
 
-    match_key = source.key_field if match_key.nil? 
+    match_key = source.key_field if match_key.nil?
 
     if other_key.nil?
       other_key = other.identify_field(match_key)
@@ -87,13 +87,13 @@ module TSV
           index = TSV.translation_index(identifier_files.flatten, match_key_name, other_key_name)
         end
 
-        if other_key != :key 
+        if other_key != :key
           other = other.reorder other_key, fields, one2one: one2one, merge: true, type: :double
         end
 
-        other_field_positions = other.identify_field(fields.dup) 
+        other_field_positions = other.identify_field(fields.dup)
         fields.zip(other_field_positions) do |o,n|
-          raise "Field #{o} not found. Options: #{Log.fingerprint other.fields}" if n.nil? 
+          raise "Field #{o} not found. Options: #{Log.fingerprint other.fields}" if n.nil?
         end
 
         log_message = "Attach #{Log.fingerprint fields - source.fields} to #{Log.fingerprint source} (#{[match_key, other_key] * "=~"})"
@@ -139,7 +139,7 @@ module TSV
 
             if other_values.nil?
               other_values = empty_other_values
-            elsif other.type == :flat 
+            elsif other.type == :flat
               other_values = [other_values]
             elsif other.type == :list && source.type == :double
               other_values = other_values.collect{|v| [v] }
@@ -185,9 +185,9 @@ module TSV
                               end
           other.each do |other_key,other_values|
             next if source.include?(other_key)
-            if other.type == :flat 
+            if other.type == :flat
               other_values = [other_values]
-            elsif other.type == :single 
+            elsif other.type == :single
               other_values = [other_values]
             elsif other.type == :list && type == :double
               other_values = other_values.collect{|v| [v] }

data/lib/scout/tsv/change_id/translate.rb

@@ -87,7 +87,7 @@ module TSV
                       pos = NamedArray.identify_name(TSV.all_fields(file1), TSV.all_fields(file2))
                       TSV.all_fields(file1)[pos.compact.first]
                     end
-    Persist.persist(name, "HDB", persist_options) do 
+    Persist.persist(name, "HDB", persist_options) do
       index = path.inject(nil) do |acc,file|
         if acc.nil?
           if source.nil?

data/lib/scout/tsv/csv.rb

@@ -1,7 +1,7 @@
 require 'csv'
 
 module TSV
-  def self.csv(obj, options = {}) 
+  def self.csv(obj, options = {})
     options = IndiferentHash.add_defaults options, :headers => true, :type => :list
     headers = options[:headers]
 
@@ -12,7 +12,7 @@ module TSV
     merge = options.delete :merge
     key_field = options.delete :key_field
     fields = options.delete :fields
-    
+
     if key_field || fields
       orig_type = type
       type = :double
@@ -49,7 +49,7 @@ module TSV
       else
         key, *values = row
       end
-      
+
       if cast
         values = values.collect{|v| v.send cast }
       end