scout-gear 10.7.4 → 10.7.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 92262edf62404f5935bf749d3ab71d2ce39ffc7292a1b3d860fd5dc712501fcb
-  data.tar.gz: 47f69a80c0a135d3a9b3b42558e7b91e766b7781ff2360cd8344ef9c0f7ac8b9
+  metadata.gz: f346d283c182fbb5a090329100ad4bd32a50e7e84c058c156014ca1fd2e64772
+  data.tar.gz: d4457afe22a81e426232ee94ddc798921fc69dfcf35a628c8ecafa39fb3a271b
 SHA512:
-  metadata.gz: 3d532c6a4ac2bd3d0b5584761531b0364f381c317a9864b85b22c7a23bd61dbf2b31d1655f5fc7085c893c843f1b3f36a1b23fffae4c4b9dc2253352f3f5febf
-  data.tar.gz: 50e36c694239f0d74528b9156435b8769d16ae172dd87029ee741c0002b8ef3606f5088fc2c8f881d413b78f1f00f66134f06fc14129da3015648546130809c6
+  metadata.gz: 27ac7e8599c5caabad5d63282c2f316404bce25d82dc0cb302acbb249c1ddb76eff171044b0f9c766e05ee582f399f4ba27076c6514bb3abab16857f11974846
+  data.tar.gz: 475df8c7b163a30dd09d608119633443c8bfc9eec1f6bcc657e3f2116bf23a678e18f3224f8da399c6923ac222f5cc1b953dd4035f44514bff90d1ee2b1ffcb7

data/.gitmodules

@@ -1,4 +0,0 @@
-[submodule "modules/rbbt-util"]
-	path = modules/rbbt-util
-	url = https://github.com/mikisvaz/rbbt-util.git
-	branch = scout

data/.vimproject

@@ -8,7 +8,9 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
    workflow.rb
    workflow=workflow{
     exceptions.rb
+    export.rb
     definition.rb
+    persist.rb
     documentation.rb
     usage.rb
     util.rb
@@ -35,9 +37,11 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     }
     deployment.rb
     deployment=deployment{
+     queue.rb
      trace.rb
      orchestrator.rb
     }
+    entity.rb
    }
    work_queue.rb
    work_queue=work_queue{
@@ -121,12 +125,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     traverse.rb
     enrichment.rb
     list.rb
-   }
-   offsite.rb
-   offsite=offsite{
-    ssh.rb
-    sync.rb
-    step.rb
+    description.rb
    }
    semaphore.rb
   }
@@ -144,7 +143,6 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
   doc
   update
   template
-  offsite
   kb=kb{
    config
    entities
@@ -162,6 +160,9 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
    install
    trace
    prov
+   queue
+   process
+   cmd
   }
   batch=batch{
    list
@@ -177,6 +178,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
   test_scout.rb
   data=data filter="*"{
    person=person{
+    README.md
     brothers
     identifiers
     marriages
@@ -186,7 +188,6 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
   scout=scout{
    test_association.rb
    test_entity.rb
-   test_offsite.rb
    test_semaphore.rb
    test_tsv.rb
    test_workflow.rb
@@ -248,12 +249,6 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
      test_tokyocabinet.rb
     }
    }
-   offsite=offsite{
-    test_ssh.rb
-    test_step.rb
-    test_sync.rb
-    test_task.rb
-   }
   }
  }
  share=share{

data/Rakefile

@@ -12,7 +12,7 @@ Juwelier::Tasks.new do |gem|
   gem.homepage = "http://github.com/mikisvaz/scout-gear"
   gem.license = "MIT"
   gem.summary = %Q{basic gear for scouts}
-  gem.description = %Q{Temporary files, logs, path, resources, persistence, workflows, TSV, etc.}
+  gem.description = %Q{Scout gear: workflow, TSVs, persistence, entities, associations, and knowledge_bases.}
   gem.email = "mikisvaz@gmail.com"
   gem.authors = ["Miguel Vazquez"]
 
@@ -26,10 +26,7 @@ Juwelier::Tasks.new do |gem|
   gem.add_runtime_dependency 'RubyInline'
   #gem.add_runtime_dependency 'tokyocabinet'
 
-  gem.add_development_dependency "rdoc", "~> 3.12"
-  gem.add_development_dependency "bundler", "~> 1.0"
   gem.add_development_dependency "juwelier", "~> 2.1.0"
-  gem.add_development_dependency "simplecov", ">= 0"
 end
 Juwelier::RubygemsDotOrgTasks.new
 

data/VERSION

@@ -1 +1 @@
-10.7.4
+10.7.6

data/lib/scout/association/index.rb

@@ -1,7 +1,7 @@
 require 'scout/annotation'
 module Association
 
-  def self.index(file, source: nil, target: nil, source_format: nil, target_format: nil, format: nil, **kwargs)
+  def self.index(file, source: nil, target: nil, source_format: nil, target_format: nil, format: nil, database: nil, **kwargs)
     IndiferentHash.setup(kwargs)
     source = kwargs.delete :source if kwargs.include?(:source)
     target = kwargs.delete :target if kwargs.include?(:target)
@@ -14,7 +14,7 @@ module Association
     index = Persist.tsv(file, kwargs, engine: "BDB", persist_options: index_persist_options) do |data|
       recycle, undirected = IndiferentHash.process_options kwargs, :recycle, :undirected
 
-      database = Association.open(file, source: source, target: target, source_format: source_format, target_format: target_format, **kwargs.merge(persist_prefix: "Association::Database"))
+      database ||= Association.open(file, source: source, target: target, source_format: source_format, target_format: target_format, **kwargs.merge(persist_prefix: "Association::Database"))
 
       source_field = database.key_field
       target_field, *fields = database.fields
@@ -141,7 +141,7 @@ module Association
                        new.extend Index
                        new
                      else
-                       FileUtils.mkdir_p File.dirname(reverse_filename) unless File.exist?(File.dirname(reverse_filename))
+                       Open.mkdir File.dirname(reverse_filename) unless Open.exist?(File.dirname(reverse_filename))
 
                        new = Persist.open_tokyocabinet(reverse_filename, true, serializer, TokyoCabinet::BDB)
 

data/lib/scout/entity/identifiers.rb

@@ -64,13 +64,16 @@ module Entity
       Persist.memory("Entity index #{identity_type}: #{format} (from #{source || "All"})", :persist => true, :format => format, :source => source) do
         source ||= self.respond_to?(:format)? self.format : nil
 
+        saved_exception = nil
         begin
           index = TSV.translation_index(identifier_files, source, format, :persist => true)
           raise "No index from #{ Misc.fingerprint source } to #{ Misc.fingerprint format }: #{Misc.fingerprint identifier_files}" if index.nil?
           index.unnamed = true
           index
         rescue
-          raise $! if source.nil?
+          raise saved_exception || $! if source.nil?
+          Log.debug "Retrying identifier index without specifying source"
+          saved_exception = $!
           source = nil
           retry
         end
@@ -93,7 +96,7 @@ module Entity
 
     name = default if name.nil?
 
-    self.send(:include, Entity::Identified) unless Entity::Identified === self
+    self.send(:include, Entity::Identified) unless self.include?(Entity::Identified)
 
     self.format = all_fields
     @formats ||= []

data/lib/scout/entity/property.rb

@@ -148,7 +148,7 @@ module Entity
 
     def persist(name, type = :marshal, options = {})
       options = IndiferentHash.add_defaults options, persist: true,
-        dir: File.join(Entity.entity_property_cache, self.to_s, name.to_s)
+        dir: Entity.entity_property_cache[self.to_s][name.to_s]
       @persisted_methods ||= {}
       @persisted_methods[name] = [type, options]
     end

data/lib/scout/knowledge_base/description.rb

@@ -0,0 +1,108 @@
+class KnowledgeBase
+  def self.doc_parse_up_to(str, pattern, keep = false)
+    pre, _pat, _post = str.partition pattern
+    if _pat
+      [pre, (keep ? _pat << _post : _post)]
+    else
+      _post
+    end
+  end
+
+  def self.doc_parse_chunks(str, pattern)
+    parts = str.split(pattern)
+    return {} if parts.length < 2
+    databases = Hash[*parts[1..-1].collect{|v| v.strip }]
+    databases.delete_if{|t,d| d.empty?}
+    databases.transform_keys!(&:downcase)
+    databases
+  end
+
+  def self.parse_knowledge_base_doc(doc)
+    description, db_description = doc_parse_up_to doc, /^#/, true
+    databases = doc_parse_chunks db_description, /^# (.*)/ 
+    IndiferentHash.setup({:description => description.strip, :databases => databases})
+  end
+
+  def documentation_markdown
+    return "" if @libdir.nil?
+    file = @libdir['README.md'].find unless file.exists?
+    if file.exists?
+      file.read
+    else
+      ""
+    end
+  end
+  
+  def database_description_file(name)
+    dir[name.to_s + '.md']
+  end
+  
+  def knowledge_base_description_file(name)
+    file = dir['README.md']
+    return file if file.exists?
+
+    file, options = registry[name]
+    file = Path.setup(file.dup) unless file.nil? or Path === file
+    source_readme = file.dirname['README.md'] if file
+    return source_readme if source_readme  && source_readme.exists?
+  end
+  
+  def description(name)
+    return registered_options(name)[:description] if registered_options(name)[:description]
+    
+    return database_description_file(name).read if database_description_file(name).exist? 
+
+    if knowledge_base_description_file(name)
+      KnowledgeBase.parse_knowledge_base_doc(knowledge_base_description_file(name).read)[:databases][name.to_s.downcase]
+    end
+  end
+
+  def markdown(name)
+    description = description(name)
+    source_type = source_type(name)
+    target_type = target_type(name)
+
+    full_description = []
+    empty_line = ''
+    full_description << ("# " << Misc.humanize(name))
+    full_description << empty_line
+
+    source_formats = begin
+                       source_index(name).key_field.split(',')
+                     rescue
+                       []
+                     end
+
+    target_formats = begin
+                       target_index(name).key_field.split(',')
+                     rescue
+                       []
+                     end
+
+    if source_type 
+      full_description << "Source: #{source_type} - #{source(name)}"
+    else
+      full_description << "Source: #{source(name)}"
+    end
+    #full_description.last << ". Accepted formats: #{source_formats*", "}" if source_formats.any?
+
+    if target_type 
+      full_description << "Target: #{target_type} - #{target(name)}"
+    else
+      full_description << "Target: #{target(name)}"
+    end
+    #full_description.last << ". Accepted formats: #{target_formats*", "}" if target_formats.any?
+    
+    if undirected?(name) 
+      full_description << "Undirected database, source and target can be reversed."
+    end
+
+    if description
+      full_description << empty_line
+      full_description << description
+      full_description << empty_line
+    end
+
+    full_description * "\n"
+  end
+end

data/lib/scout/knowledge_base/entity.rb

@@ -80,6 +80,7 @@ class KnowledgeBase
       end
       identifier_files.concat Entity.identifier_files(source(name)) if defined? Entity
       identifier_files.uniq!
+      identifier_files.collect!{|f| (Path === f) ? f : Path.setup(f.dup) }
       identifier_files.collect!{|f| f.annotate(f.gsub(/\bNAMESPACE\b/, namespace))} if namespace
       identifier_files.collect!{|f| f.annotate(f.gsub(/\bNAMESPACE\b/, db_namespace(name)))} if not namespace and db_namespace(name)
       identifier_files.reject!{|f| f.match(/\bNAMESPACE\b/)}
@@ -89,7 +90,11 @@ class KnowledgeBase
   
   def target_index(name)
     Persist.memory("Target index #{name}: KB directory #{dir}") do
-      identifier_files = identifier_files(name)
+      if @identifier_files && @identifier_files.any?
+        identifier_files = @identifier_files
+      else
+        identifier_files = database_identifier_files(name)
+      end
       identifier_files.concat Entity.identifier_files(target(name)) if defined? Entity
       identifier_files.uniq!
       identifier_files.collect!{|f| f.annotate(f.gsub(/\bNAMESPACE\b/, namespace))} if self.namespace

data/lib/scout/knowledge_base/registry.rb

@@ -15,9 +15,29 @@ class KnowledgeBase
     end
   end
 
+  def present_databases
+    dir.glob("*.database").collect{|file| File.basename(file, '.database')}
+  end
+
   def all_databases
     return [] unless @registry
-    @registry.keys 
+    (@registry.keys + present_databases).uniq
+  end
+  
+  def database_file(name)
+    if @registry[name].nil?
+      nil
+    else
+      @registry[name].first
+    end
+  end
+
+  def registered_options(name)
+    if @registry[name].nil?
+      IndiferentHash.setup({})
+    else
+      IndiferentHash.setup(@registry[name].last)
+    end
   end
 
   def include?(name)
@@ -29,21 +49,21 @@ class KnowledgeBase
     @fields[name] ||= get_index(name).fields
   end
 
-  def description(name)
-    @descriptions ||= {}
-    @descriptions[name] ||= get_index(name).key_field.split("~")
+  def pair(name)
+    @pairs ||= {}
+    @pairs[name] ||= get_index(name).key_field.split("~")
   end
 
   def source(name)
-    description(name)[0]
+    pair(name)[0]
   end
 
   def target(name)
-    description(name)[1]
+    pair(name)[1]
   end
 
   def undirected(name)
-    description(name).length == 3
+    pair(name).length == 3
   end
 
   alias undirected? undirected
@@ -69,7 +89,12 @@ class KnowledgeBase
 
           persist_dir = dir
           persist_path = persist_dir[key].find
-          file, registered_options = registry[name]
+
+          file = database_file(name)
+          registered_options = registered_options(name)
+          registered_options =  IndiferentHash.setup(registered_options).except(:description)
+
+          registered_options = IndiferentHash.add_defaults registered_options, identifiers: self.identifier_files if registered_options
 
           options = IndiferentHash.add_defaults options, registered_options if registered_options and registered_options.any?
           options = IndiferentHash.add_defaults options, :persist_path => persist_path, :persist_dir => persist_dir, :persist => true
@@ -89,11 +114,11 @@ class KnowledgeBase
                     Log.low "Re-opening index #{ name } from #{ Log.fingerprint persist_path }. #{options}"
                     Association.index(file, **options, persist_options: persist_options.dup)
                   else
-                    options = IndiferentHash.add_defaults options, registered_options if registered_options
-                    raise "Repo #{ name } not found and not registered" if file.nil?
-                    Log.medium "Opening index #{ name } from #{ Log.fingerprint file }. #{options}"
+                    database = get_database name if file.nil?
                     file = file.call if Proc === file
-                    Association.index(file, **options, persist_options: persist_options.dup)
+                    Log.medium "Opening index #{ name } from #{ Log.fingerprint database }. #{options}"
+                    options = IndiferentHash.add_defaults options, registered_options if registered_options
+                    Association.index(file, **options, persist_options: persist_options.dup, database: database)
                   end
 
           index.namespace = self.namespace unless self.namespace
@@ -128,7 +153,12 @@ class KnowledgeBase
 
           persist_dir = dir
           persist_path = persist_dir[key].find
-          file, registered_options = registry[name]
+
+          file = database_file(name)
+          registered_options = registered_options(name)
+          registered_options =  IndiferentHash.setup(registered_options).except(:description)
+
+          registered_options = IndiferentHash.add_defaults registered_options, identifiers: self.identifier_files if registered_options
 
           options = IndiferentHash.add_defaults options, registered_options if registered_options and registered_options.any?
           options = IndiferentHash.add_defaults options, :persist_path => persist_path, :persist => true
@@ -145,7 +175,6 @@ class KnowledgeBase
 
           database = if persist_path.exists? and persist_options[:persist] and not persist_options[:update]
                        Log.low "Re-opening database #{ name } from #{ Log.fingerprint persist_path }. #{options}"
-                       #Association.database(file, **options, persist_options: persist_options)
                        Association.database(file, **options.merge(persist_options: persist_options))
                      else
                        options = IndiferentHash.add_defaults options, registered_options if registered_options
@@ -153,7 +182,6 @@ class KnowledgeBase
                        raise "Repo #{ name } not found and not registered" if file.nil?
                        Log.medium "Opening database #{ name } from #{ Log.fingerprint file }. #{options}"
                        file = file.call if Proc === file
-                       #Association.database(file, **options, persist_options: persist_options)
                        Association.database(file, **options.merge(persist_options: persist_options))
                      end
 

data/lib/scout/knowledge_base.rb

@@ -5,6 +5,7 @@ require_relative 'knowledge_base/entity'
 require_relative 'knowledge_base/query'
 require_relative 'knowledge_base/traverse'
 require_relative 'knowledge_base/list'
+require_relative 'knowledge_base/description'
 #require 'scout/knowledge_base/query'
 #require 'scout/knowledge_base/syndicate'
 
@@ -23,12 +24,12 @@ class KnowledgeBase
     @entity_options ||= IndiferentHash.setup({})
 
     @format         ||= IndiferentHash.setup({})
-    @descriptions   ||= IndiferentHash.setup({})
+    pairs          ||= IndiferentHash.setup({})
     @indices        ||= IndiferentHash.setup({})
   end
 
   def config_file(name)
-    @dir.config[name.to_s]
+    @dir['config'][name.to_s]
   end
 
   def save_variable(name)

data/lib/scout/tsv/change_id/translate.rb

@@ -40,7 +40,7 @@ module TSV
           target_file = target_files.select{|file| fields = file_fields[file]; (fields & middle_fields).any? }.collect{|file,f| file }.first
           [source_file, middle_file, target_file]
         else
-          raise "Could not traverse identifier path from #{Log.fingerprint source} to #{Log.fingerprint target} in #{Log.fingerprint file_fields}"
+          raise "Could not traverse identifier path from #{Log.fingerprint source} to #{Log.fingerprint target}. #{file_fields.empty? ? "No identifier files" : Log.fingerprint(file_fields)}"
         end
       end
     end
@@ -55,7 +55,14 @@ module TSV
     files = [files] unless Array === files
 
     files.each do |file|
-      next if Path === file && ! Open.exist?(file)
+      #next if Path === file && ! Open.exist?(file)
+      begin
+        file = file.produce if Path === file
+        raise "Could no produce file" if FalseClass === file
+      rescue
+        Log.warn $!.message
+        next
+      end
       file = file.find if Path === file
       file_fields[file] = all_fields(file)
     end

data/lib/scout/tsv/open.rb

@@ -151,6 +151,16 @@ module Open
                   res = block.call(line)
                   callback.call res if callback
                 end
+                obj.close
+                obj.join if obj.respond_to? :join
+              elsif options[:type] == :matrix
+                Log.low "Traverse stream by lines #{Log.fingerprint obj}"
+                parser = options[:sep].nil? ? TSV::Parser.new(obj) : TSV::Parser.new(obj, sep: options[:sep])
+                parser.traverse **options do |parts|
+                  res = block.call parts
+                  callback.call res if callback
+                  nil
+                end
               else
                 Log.low "Traverse stream with parser #{Log.fingerprint obj}"
                 parser = options[:sep].nil? ? TSV::Parser.new(obj) : TSV::Parser.new(obj, sep: options[:sep])

data/lib/scout/tsv/parser.rb

@@ -31,6 +31,12 @@ module TSV
 
     return nil if select && ! TSV.select(items[0], items[1..-1], select, fields: field_names, type: type, sep: sep2)
 
+    if String === key
+      raise "Key by name, but no field names" if field_names.nil?
+      key = field_names.index key
+      raise "Key #{key} not found in field names #{Log.fingerprint field_names}" if key.nil?
+    end
+
     if positions.nil? && key == 0
       key = items.shift
     elsif positions.nil?
@@ -65,7 +71,7 @@ module TSV
     [key, items]
   end
 
-  def self.parse_stream(stream, data: nil, source_type: nil, type: :list, merge: true, one2one: false, fix: true, bar: false, first_line: nil, field_names: nil, head: nil, **kwargs, &block)
+  def self.parse_stream(stream, data: nil, source_type: nil, sep: "\t", type: :list, merge: true, one2one: false, fix: true, bar: false, first_line: nil, field_names: nil, head: nil, **kwargs, &block)
     begin
       bar = "Parsing #{Log.fingerprint stream}" if TrueClass === bar
       bar = Log::ProgressBar.get_obj_bar(stream, bar) if bar
@@ -81,7 +87,7 @@ module TSV
           data.serializer.to_s.include?("String") &&
           same_type && 
           ! (head || kwargs[:cast] || kwargs[:positions] || (kwargs[:key] && kwargs[:key] != 0) || Proc === fix ) &&
-          (kwargs[:sep].nil? || kwargs[:sep] == "\t")
+          (sep.nil? || sep == "\t")
 
 
         Log.debug "Loading #{Log.fingerprint stream} directly into #{Log.fingerprint data}"
@@ -114,12 +120,17 @@ module TSV
             line = Misc.fixutf8(line)
           end
           bar.tick if bar
+
           if type == :array || type == :line
             block.call line
             next
+          elsif type == :matrix
+            parts = line.split(sep)
+            block.call parts
+            next
           end
 
-          key, items = parse_line(line, type: source_type, field_names: field_names, **kwargs)
+          key, items = parse_line(line, type: source_type, field_names: field_names, sep: sep, **kwargs)
 
           next if key.nil?
 

data/lib/scout/workflow/deployment/orchestrator.rb

@@ -1,6 +1,8 @@
 module Workflow
   class Orchestrator
 
+    class NoWork < Exception; end
+
     def self.job_workload(job)
       workload = {job => []}
       return workload if job.done? && job.updated?
@@ -202,7 +204,7 @@ module Workflow
 
           candidates = resources_used.keys + Orchestrator.candidates(workload, rules)
           candidates.uniq!
-          raise "No candidates and no running jobs" if candidates.empty?
+          raise NoWork, "No candidates and no running jobs" if candidates.empty?
 
           candidates.each do |job|
             case 
@@ -272,4 +274,10 @@ module Workflow
     orchestrator.process({}, produce_list)
     produce_list
   end
+
+  def self.produce(jobs, produce_cpus: Etc.nprocessors, produce_timer: 5)
+    jobs = [jobs] unless Array === jobs
+    orchestrator = Orchestrator.new produce_timer.to_i, cpus: produce_cpus.to_i
+    orchestrator.process({}, jobs)
+  end
 end

data/lib/scout/workflow/deployment/queue.rb

@@ -0,0 +1,26 @@
+module Workflow
+  def self.name2clean_name(name)
+    name.reverse.partition("_").last.reverse
+  end
+
+  def self.queue_job(file)
+    workflow, task, name = file.split("/").values_at(-3, -2, -1) if file
+    workflow = Workflow.require_workflow workflow
+
+    if Open.directory?(file)
+      clean_name = name2clean_name name
+      inputs = workflow.tasks[task].load_inputs(file)
+      workflow.job(task, clean_name, inputs)
+    else
+      workflow.job(task, name)
+    end
+  end
+
+  def self.unqueue(file, &block)
+    Open.lock file do
+      job = queue_job(file)
+      puts job.run
+      Open.rm_rf file
+    end
+  end
+end