rbbt-util 5.44.0 → 6.0.3

data/lib/rbbt/knowledge_base/registry.rb

@@ -9,7 +9,7 @@ class KnowledgeBase
       Log.debug("Registering #{ name } from code block")
       @registry[name] = [block, options]
     else
-      Log.debug("Registering #{ name }: #{ Misc.fingerprint file } #{Misc.fingerprint options}")
+      Log.debug("Registering #{ name }: #{ Log.fingerprint file } #{Log.fingerprint options}")
       @registry[name] = [file, options]
     end
   end
@@ -40,7 +40,7 @@ class KnowledgeBase
 
   def get_index(name, options = {})
     name = name.to_s
-    options[:organism] ||= options[:namespace] ||= self.namespace unless self.namespace.nil?
+    options[:namespace] ||= self.namespace unless self.namespace.nil?
     @indices[[name, options]] ||= 
       begin 
         if options.empty?
@@ -49,7 +49,7 @@ class KnowledgeBase
           key = options[:key]
           key = name if key == :name
         else
-          fp = Misc.hash2md5(options)
+          fp = Misc.digest(options)
           key = name.to_s + "_" + fp
         end
 
@@ -57,30 +57,33 @@ class KnowledgeBase
           options = options.dup
 
           persist_dir = dir
-          persist_file = persist_dir[key].find
+          persist_path = persist_dir[key].find
           file, registered_options = registry[name]
 
-          options = Misc.add_defaults options, registered_options if registered_options and registered_options.any?
-          options = Misc.add_defaults options, :persist_file => persist_file, :persist_dir => persist_dir, :format => format, :persist => true
+          options = IndiferentHash.add_defaults options, registered_options if registered_options and registered_options.any?
+          options = IndiferentHash.add_defaults options, :persist_path => persist_path, :persist_dir => persist_dir, :persist => true
 
           if entity_options
             options[:entity_options] ||= {}
             entity_options.each do |type, info|
               options[:entity_options][type] ||= {}
-              options[:entity_options][type] = Misc.add_defaults options[:entity_options][type], info
+              options[:entity_options][type] = IndiferentHash.add_defaults options[:entity_options][type], info
             end
           end
 
-          persist_options = Misc.pull_keys options, :persist
+          persist_options = IndiferentHash.pull_keys options, :persist
+          persist_options = IndiferentHash.add_defaults persist_options, :path => persist_path
+          
 
-          index = if persist_file.exists? and persist_options[:persist] and not persist_options[:update]
-                    Log.low "Re-opening index #{ name } from #{ Misc.fingerprint persist_file }. #{options}"
-                    Association.index(file, options, persist_options.dup)
+          index = if persist_path.exists? and persist_options[:persist] and not persist_options[:update]
+                    Log.low "Re-opening index #{ name } from #{ Log.fingerprint persist_path }. #{options}"
+                    Association.index(file, **options.merge(persist_options: persist_options.dup))
                   else
-                    options = Misc.add_defaults options, registered_options if registered_options
+                    options = IndiferentHash.add_defaults options, registered_options if registered_options
                     raise "Repo #{ name } not found and not registered" if file.nil?
-                    Log.medium "Opening index #{ name } from #{ Misc.fingerprint file }. #{options}"
-                    Association.index(file, options, persist_options.dup)
+                    Log.medium "Opening index #{ name } from #{ Log.fingerprint file }. #{options}"
+                    file = file.call if Proc === file
+                    Association.index(file, **options.merge(persist_options: persist_options.dup))
                   end
 
           index.namespace = self.namespace unless self.namespace
@@ -97,51 +100,49 @@ class KnowledgeBase
     if self.namespace == options[:namespace]
       options.delete(:namespace) 
     end
-    if self.namespace == options[:organism]
-      options.delete(:organism) 
-    end
     @databases[[name, options]] ||= 
       begin 
-        fp = Misc.fingerprint([name,options])
+        fp = Log.fingerprint([name,options])
 
         if options.empty?
           key = name.to_s
         else
-          fp = Misc.hash2md5(options)
+          fp = Misc.digest(options)
           key = name.to_s + "_" + fp
         end
 
-        options[:organism] ||= options[:namespace] ||= self.namespace unless self.namespace.nil?
+        options[:namespace] ||= self.namespace unless self.namespace.nil?
 
         key += '.database'
         Persist.memory("Database:" << [key, dir] * "@") do
           options = options.dup
 
           persist_dir = dir
-          persist_file = persist_dir[key].find
+          persist_path = persist_dir[key].find
           file, registered_options = registry[name]
 
-          options = Misc.add_defaults options, registered_options if registered_options and registered_options.any?
-          options = Misc.add_defaults options, :persist_file => persist_file, :format => format, :persist => true
+          options = IndiferentHash.add_defaults options, registered_options if registered_options and registered_options.any?
+          options = IndiferentHash.add_defaults options, :persist_path => persist_path, :persist => true
 
           if entity_options
             options[:entity_options] ||= {}
             entity_options.each do |type, info|
               options[:entity_options][type] ||= {}
-              options[:entity_options][type] = Misc.add_defaults options[:entity_options][type], info
+              options[:entity_options][type] = IndiferentHash.add_defaults options[:entity_options][type], info
             end
           end
 
-          persist_options = Misc.pull_keys options, :persist
+          persist_options = IndiferentHash.pull_keys options, :persist
 
-          database = if persist_file.exists? and persist_options[:persist] and not persist_options[:update]
-                       Log.low "Re-opening database #{ name } from #{ Misc.fingerprint persist_file }. #{options}"
-                       Association.open(file, options, persist_options)
+          database = if persist_path.exists? and persist_options[:persist] and not persist_options[:update]
+                       Log.low "Re-opening database #{ name } from #{ Log.fingerprint persist_path }. #{options}"
+                       Association.database(file, options, persist_options)
                      else
-                       options = Misc.add_defaults options, registered_options if registered_options
+                       options = IndiferentHash.add_defaults options, registered_options if registered_options
                        raise "Repo #{ name } not found and not registered" if file.nil?
-                       Log.medium "Opening database #{ name } from #{ Misc.fingerprint file }. #{options}"
-                       Association.open(file, options, persist_options)
+                       Log.medium "Opening database #{ name } from #{ Log.fingerprint file }. #{options}"
+                       file = file.call if Proc === file
+                       Association.database(file, **options)
                      end
 
           database.namespace = self.namespace if self.namespace

data/lib/rbbt/knowledge_base/traverse.rb

@@ -80,7 +80,7 @@ class KnowledgeBase
 
       paths = {}
       matches = clean_matches[rule]
-      Annotated.purge(matches).each do |match|
+      Annotation.purge(matches).each do |match|
         new_assignments = nil
         match_source, _sep, match_target = match.partition "~"
 

data/lib/rbbt/knowledge_base.rb

@@ -1,4 +1,4 @@
-
+require 'rbbt/annotations'
 require 'rbbt/association'
 require 'rbbt/association/item'
 require 'rbbt/knowledge_base/entity'

data/lib/rbbt/monitor.rb

@@ -8,7 +8,7 @@ module Rbbt
   LOCK_DIRS = Rbbt.tmp.tsv_open_locks.find_all + Rbbt.tmp.persist_locks.find_all + Rbbt.tmp.sensiblewrite_locks.find_all +
     Rbbt.tmp.produce_locks.find_all + Rbbt.tmp.step_info_locks.find_all
 
-  SENSIBLE_WRITE_DIRS = Misc.sensiblewrite_dir.find_all
+  SENSIBLE_WRITE_DIRS = Open.sensible_write_dir.find_all
 
   PERSIST_DIRS    = Rbbt.share.find_all  + Rbbt.var.cache.persistence.find_all
 
@@ -142,23 +142,28 @@ module Rbbt
       tasks_dirs = if dir == '.'
                     ["."]
                    else
-                     workflowdirs = if (dir_sub_path = Open.find_repo_dir(workflowdir))
-                                      repo_dir, sub_path = dir_sub_path
-                                      Open.list_repo_files(*dir_sub_path).collect{|f| f.split("/").first}.uniq.collect{|f| File.join(repo_dir, f)}.uniq
-                                    else
-                                      dir.glob("*")
-                                    end
+                     #workflowdirs = if (dir_sub_path = Open.find_repo_dir(workflowdir))
+                     #                 repo_dir, sub_path = dir_sub_path
+                     #                 Open.list_repo_files(*dir_sub_path).collect{|f| f.split("/").first}.uniq.collect{|f| File.join(repo_dir, f)}.uniq
+                     #               else
+                     #                 dir.glob("*")
+                     #               end
+
+                     workflowdirs = dir.glob("*")
 
                      workflowdirs.collect do |workflowdir|
                        workflow = File.basename(workflowdir)
                        next if workflows and not workflows.include? workflow
 
-                       task_dirs = if (dir_sub_path = Open.find_repo_dir(workflowdir))
-                                     repo_dir, sub_path = dir_sub_path
-                                     Open.list_repo_files(*dir_sub_path).collect{|f| f.split("/").first}.uniq.collect{|f| File.join(repo_dir, f)}.uniq
-                                   else
-                                     workflowdir.glob("*")
-                                   end
+                       #task_dirs = if (dir_sub_path = Open.find_repo_dir(workflowdir))
+                       #              repo_dir, sub_path = dir_sub_path
+                       #              Open.list_repo_files(*dir_sub_path).collect{|f| f.split("/").first}.uniq.collect{|f| File.join(repo_dir, f)}.uniq
+                       #            else
+                       #              workflowdir.glob("*")
+                       #            end
+
+                       task_dirs = workflowdir.glob("*")
+
                        task_dirs.each do |tasks_dir|
                          task_dir_workflows[tasks_dir] = workflow
                        end
@@ -170,18 +175,24 @@ module Rbbt
         next if tasks and not tasks.include? task
 
 
-        files = if (dir_sub_path = Open.find_repo_dir(taskdir))
-                  repo_dir, sub_path = dir_sub_path
-                  Open.list_repo_files(*dir_sub_path).reject do |f|
-                    f.include?("/.info/") ||
-                      f.include?(".files/") ||
-                      f.include?(".pid/") ||
-                      File.directory?(f)
-                  end.collect do |f|
-                    File.join(repo_dir, f)
-                  end
-                else
-                  #cmd = "find -L '#{ taskdir }/'  -not \\( -path \"#{taskdir}/*.files/*\" -prune \\) -not -name '*.pid' -not -name '*.notify' -not -name '\\.*' 2>/dev/null"
+        #files = if (dir_sub_path = Open.find_repo_dir(taskdir))
+        #          repo_dir, sub_path = dir_sub_path
+        #          Open.list_repo_files(*dir_sub_path).reject do |f|
+        #            f.include?("/.info/") ||
+        #              f.include?(".files/") ||
+        #              f.include?(".pid/") ||
+        #              File.directory?(f)
+        #          end.collect do |f|
+        #            File.join(repo_dir, f)
+        #          end
+        #        else
+        #          #cmd = "find -L '#{ taskdir }/'  -not \\( -path \"#{taskdir}/*.files/*\" -prune \\) -not -name '*.pid' -not -name '*.notify' -not -name '\\.*' 2>/dev/null"
+        #          cmd = "find -L '#{ taskdir }/' -not \\( -path \"#{taskdir}/.info/*\" -prune \\) -not \\( -path \"#{taskdir}/*.files/*\" -prune \\) -not -name '*.pid' -not -name '*.md5' -not -name '*.notify' -not -name '\\.*' \\( -not -type d -o -name '*.files' \\)  2>/dev/null"
+
+        #          CMD.cmd(cmd, :pipe => true).read.split("\n")
+        #        end
+
+        files = begin
                   cmd = "find -L '#{ taskdir }/' -not \\( -path \"#{taskdir}/.info/*\" -prune \\) -not \\( -path \"#{taskdir}/*.files/*\" -prune \\) -not -name '*.pid' -not -name '*.md5' -not -name '*.notify' -not -name '\\.*' \\( -not -type d -o -name '*.files' \\)  2>/dev/null"
 
                   CMD.cmd(cmd, :pipe => true).read.split("\n")

data/lib/rbbt/persist/refactor.rb

@@ -0,0 +1,166 @@
+module Persist
+  TSVAdapter = Object::TSVAdapter
+  def self.is_persisted?(path, persist_options = {})
+    return true if Open.remote?(path)
+    return true if Open.ssh?(path)
+    return false if not Open.exists? path
+    return false if TrueClass === persist_options[:update]
+
+    expiration = persist_options[:expiration]
+    if expiration
+      seconds = Misc.timespan(expiration)
+      patht = Open.mtime(path)
+      return false if Time.now > patht + seconds
+    end
+
+    check = persist_options[:check]
+    return true if check.nil?
+
+    missing = check.reject{|file| Open.exists?(file) }
+    return false if missing.any?
+
+    return true unless ENV["RBBT_UPDATE"]
+
+    if Array === check
+      newer = check.select{|file| newer? path, file}
+      return true if newer.empty?
+      Log.medium "Persistence check for #{path} failed in: #{ Misc.fingerprint(newer)}"
+      return false 
+    else
+      ! newer?(path, check)
+    end
+  end
+
+  # Is 'file' newer than 'path'? return non-true if path is newer than file
+  def self.newer?(path, file, by_link = false)
+    return true if not Open.exists?(file)
+    path = path.find if Path === path
+    file = file.find if Path === file
+    if by_link
+      patht = File.exist?(path) ? File.lstat(path).mtime : nil
+      filet = File.exist?(file) ? File.lstat(file).mtime : nil
+    else
+      patht = Open.mtime(path)
+      filet = Open.mtime(file)
+    end
+    return true if patht.nil? || filet.nil?
+    diff = patht - filet
+    return diff if diff < 0
+    return false
+  end
+
+  #def self.persist_tsv(source, filename = nil, options = {}, persist_options = {}, &block)
+  #  engine = IndiferentHash.process_options persist_options, :engine, engine: "HDB"
+  #  Persist.persist(name, engine, persist_options, &block)
+  #end
+
+#
+#  def self.persist_tsv(source, filename = nil, options = {}, persist_options = {}, &block)
+#    persist_options[:prefix] ||= "TSV"
+#
+#    if data = persist_options[:data]
+#      Log.debug "TSV persistence creating with data: #{ Misc.fingerprint(data) }"
+#      yield data
+#      return data 
+#    end
+#
+#    filename ||= get_filename(source)
+#
+#    if not persist_options[:persist]
+#      data = {}
+#
+#      yield(data) 
+#
+#      return data 
+#    end
+#
+#    path = persistence_path(filename, options)
+#
+#    if ENV["RBBT_UPDATE_TSV_PERSIST"] == 'true' and filename
+#      check_options = {:check => [filename]}
+#    else
+#      check_options = {}
+#    end
+#
+#    if is_persisted?(path, check_options) and not persist_options[:update]
+#      path = path.find if Path === path
+#      Log.debug "TSV persistence up-to-date: #{ path }"
+#      if persist_options[:shard_function]
+#        return open_sharder(path, false, nil, persist_options[:engine], persist_options, &persist_options[:shard_function]) 
+#      else
+#        return open_database(path, false, nil, persist_options[:engine] || TokyoCabinet::HDB, persist_options) 
+#      end
+#    end
+#
+#    lock_filename = Persist.persistence_path(path, {:dir => TSV.lock_dir})
+#    Misc.lock lock_filename do
+#      begin
+#        if is_persisted?(path, check_options) and not persist_options[:update]
+#          path = path.find if Path === path
+#          Log.debug "TSV persistence (suddenly) up-to-date: #{ path }"
+#
+#          if persist_options[:shard_function]
+#            return open_sharder(path, false, nil, persist_options[:engine], persist_options, &persist_options[:shard_function]) 
+#          else
+#            return open_database(path, false, nil, persist_options[:engine] || TokyoCabinet::HDB, persist_options) 
+#          end
+#        end
+#        path = path.find if Path === path
+#
+#        FileUtils.rm_rf path if File.exist? path
+#
+#        Log.medium "TSV persistence creating: #{ path }"
+#
+#        tmp_path = path + '.persist'
+#
+#        data = if persist_options[:shard_function]
+#                 open_sharder(tmp_path, true, persist_options[:serializer], persist_options[:engine], persist_options, &persist_options[:shard_function]) 
+#               else
+#                 open_database(tmp_path, true, persist_options[:serializer], persist_options[:engine] || TokyoCabinet::HDB, persist_options) 
+#               end
+#
+#        if TSV === data and data.serializer.nil?
+#          data.serializer = :type 
+#        end
+#
+#        if persist_options[:persist] == :preload
+#          tmp_tsv = yield({})
+#          tmp_tsv.annotate data
+#          data.serializer = tmp_tsv.type
+#          data.write_and_read do
+#            tmp_tsv.each do |k,v|
+#              data[k] = v
+#            end
+#          end
+#        else
+#          data.write_and_read do
+#            yield data
+#          end
+#        end
+#
+#        data.write_and_read do
+#          FileUtils.mv data.persistence_path, path if File.exist? data.persistence_path and not File.exist? path
+#          tsv = CONNECTIONS[path] = CONNECTIONS.delete tmp_path
+#          tsv.persistence_path = path
+#
+#          tsv.fix_io if tsv.respond_to? :fix_io
+#        end
+#
+#        data
+#      rescue Exception
+#        Log.error "Captured error during persist_tsv. Erasing: #{path}"
+#        FileUtils.rm_rf tmp_path if tmp_path and File.exist? tmp_path
+#        FileUtils.rm_rf path if path and File.exist? path
+#        raise $!
+#      end
+#    end
+#  end
+end
+
+#Persist.save_drivers[:annotations] = proc do |file,content|
+#  Persist.save(content, file, :)
+#end
+#
+#Persist.load_drivers[:annotations] = proc do |file|
+#  Persist.load(file, :meta_extension)
+#end

data/lib/rbbt/persist/tsv/sharder.rb

@@ -214,7 +214,7 @@ module Persist
     def [](key, clean=false)
       database = database(key)
       return nil if database.nil?
-      v = database.send(:[], key)
+      database.send(:[], key)
     end
 
     def <<(p)

data/lib/rbbt/persist/tsv/tokyocabinet.rb

@@ -1,105 +1,105 @@
-require 'tokyocabinet'
-
-module Persist
-
-  module TCAdapter
-    include Persist::TSVAdapter
-
-    attr_accessor :tokyocabinet_class
-
-    def self.open(path, write, tokyocabinet_class = TokyoCabinet::HDB)
-      if String === tokyocabinet_class && tokyocabinet_class.include?(":big")
-        big = true
-        tokyocabinet_class = tokyocabinet_class.split(":").first
-      else
-        big = false
-      end
-
-      dir = File.dirname(File.expand_path(path))
-      File.mkdir(dir) unless File.exist?(dir)
-
-      tokyocabinet_class = TokyoCabinet::HDB if tokyocabinet_class == "HDB" or tokyocabinet_class.nil?
-      tokyocabinet_class = TokyoCabinet::BDB if tokyocabinet_class == "BDB"
-
-      database = CONNECTIONS[path] ||= Log.ignore_stderr do tokyocabinet_class.new end
-
-      if big and not Open.exists?(path)
-        database.tune(nil,nil,nil,tokyocabinet_class::TLARGE | tokyocabinet_class::TDEFLATE) 
-      end
-
-      flags = (write ? tokyocabinet_class::OWRITER | tokyocabinet_class::OCREAT : tokyocabinet_class::OREADER)
-      database.close 
-
-      if !database.open(path, flags)
-        ecode = database.ecode
-        raise "Open error: #{database.errmsg(ecode)}. Trying to open file #{path}"
-      end
-
-      database.extend Persist::TCAdapter unless Persist::TCAdapter === database
-      database.persistence_path ||= path
-      database.tokyocabinet_class = tokyocabinet_class
-
-      database.mutex = Mutex.new
-      database.read
-
-      CONNECTIONS[path] = database
-
-      database
-    end
-
-    def close
-      @closed = true
-      @writable = false
-      super
-    end
-
-    def read(force = false)
-      return if ! write? && ! closed && ! force
-      self.close
-      if !self.open(@persistence_path, tokyocabinet_class::OREADER)
-        ecode = self.ecode
-        raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
-      end
-
-      @writable = false
-      @closed = false
-
-      self
-    end
-
-    def write(force = true)
-      return if write? && ! closed && ! force
-      self.close
-
-      if !self.open(@persistence_path, tokyocabinet_class::OWRITER)
-        ecode = self.ecode
-        raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
-      end
-
-      @writable = true
-      @closed = false
-
-      self
-    end
-
-  end
-
-  def self.open_tokyocabinet(path, write, serializer = nil, tokyocabinet_class = TokyoCabinet::HDB)
-    write = true unless File.exist? path
-
-    FileUtils.mkdir_p File.dirname(path) unless File.exist?(File.dirname(path))
-
-    database = Persist::TCAdapter.open(path, write, tokyocabinet_class)
-
-    unless serializer == :clean
-      TSV.setup database
-      database.write_and_read do
-        database.serializer = serializer
-      end if serializer && database.serializer != serializer
-    end
-
-    database
-  end
-end
-
-require 'rbbt/persist/tsv/tokyocabinet/marshal'
+#require 'tokyocabinet'
+#
+#module Persist
+#
+#  module TCAdapter
+#    include Persist::TSVAdapter
+#
+#    attr_accessor :tokyocabinet_class
+#
+#    def self.open(path, write, tokyocabinet_class = TokyoCabinet::HDB)
+#      if String === tokyocabinet_class && tokyocabinet_class.include?(":big")
+#        big = true
+#        tokyocabinet_class = tokyocabinet_class.split(":").first
+#      else
+#        big = false
+#      end
+#
+#      dir = File.dirname(File.expand_path(path))
+#      File.mkdir(dir) unless File.exist?(dir)
+#
+#      tokyocabinet_class = TokyoCabinet::HDB if tokyocabinet_class == "HDB" or tokyocabinet_class.nil?
+#      tokyocabinet_class = TokyoCabinet::BDB if tokyocabinet_class == "BDB"
+#
+#      database = CONNECTIONS[path] ||= Log.ignore_stderr do tokyocabinet_class.new end
+#
+#      if big and not Open.exists?(path)
+#        database.tune(nil,nil,nil,tokyocabinet_class::TLARGE | tokyocabinet_class::TDEFLATE) 
+#      end
+#
+#      flags = (write ? tokyocabinet_class::OWRITER | tokyocabinet_class::OCREAT : tokyocabinet_class::OREADER)
+#      database.close 
+#
+#      if !database.open(path, flags)
+#        ecode = database.ecode
+#        raise "Open error: #{database.errmsg(ecode)}. Trying to open file #{path}"
+#      end
+#
+#      database.extend Persist::TCAdapter unless Persist::TCAdapter === database
+#      database.persistence_path ||= path
+#      database.tokyocabinet_class = tokyocabinet_class
+#
+#      database.mutex = Mutex.new
+#      database.read
+#
+#      CONNECTIONS[path] = database
+#
+#      database
+#    end
+#
+#    def close
+#      @closed = true
+#      @writable = false
+#      super
+#    end
+#
+#    def read(force = false)
+#      return if ! write? && ! closed && ! force
+#      self.close
+#      if !self.open(@persistence_path, tokyocabinet_class::OREADER)
+#        ecode = self.ecode
+#        raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
+#      end
+#
+#      @writable = false
+#      @closed = false
+#
+#      self
+#    end
+#
+#    def write(force = true)
+#      return if write? && ! closed && ! force
+#      self.close
+#
+#      if !self.open(@persistence_path, tokyocabinet_class::OWRITER)
+#        ecode = self.ecode
+#        raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
+#      end
+#
+#      @writable = true
+#      @closed = false
+#
+#      self
+#    end
+#
+#  end
+#
+#  def self.open_tokyocabinet(path, write, serializer = nil, tokyocabinet_class = TokyoCabinet::HDB)
+#    write = true unless File.exist? path
+#
+#    FileUtils.mkdir_p File.dirname(path) unless File.exist?(File.dirname(path))
+#
+#    database = Persist::TCAdapter.open(path, write, tokyocabinet_class)
+#
+#    unless serializer == :clean
+#      TSV.setup database
+#      database.write_and_read do
+#        database.serializer = serializer
+#      end if serializer && database.serializer != serializer
+#    end
+#
+#    database
+#  end
+#end
+#
+#require 'rbbt/persist/tsv/tokyocabinet/marshal'