scout-essentials 1.8.5 → 1.8.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bf37bb130a8c4d14dd4560df43a3e98f6ea43ea7ad38c9b44525cf113e84b7b4
-  data.tar.gz: 504f47bbf850cae9fa91b898b460ec632fe6a64df77a8ab89f9d7b6c1abc67b7
+  metadata.gz: 3193beb1b7024505d5a238bb6d9caa280ddab034b23431652138697ebc9ac7f0
+  data.tar.gz: 29ef13d1a7862228eabc8f153f602ba728eda98cf4907b23770a8fb65cb36b98
 SHA512:
-  metadata.gz: b977260d1670a0a391583c2f20cd5a0780fba2b6d5a8a6685370664fb8bf66e91a7cac5f9f77e01de3dffe89cf218581bf02df95a5ae4350789881adfbd11273
-  data.tar.gz: 124cd1a4e62ca11e516144e8e37c49808440d0e67904e7756eabc8c3f71da053bd3856b8f21e7ed2a8e5fb5372bd43e6510d17ae5a70b538d23cf0e9e0cab683
+  metadata.gz: c074200ead16664e4ecdcd262315f389500cd59391b533bcdd1f4a84bdf9ab3fcf96152698f643393d198d9cc1d552707144fd4fbc9c962badbb33d57c79d364
+  data.tar.gz: 57663384b0258537a626eb867e8dd209c9e84bc477dbc462c6096af539f54b176936b1cd4ace7cd47525b5e791a6b13ac41a000fdf2709a9add3a830eb1a33ab

data/.vimproject

@@ -1,144 +1,11 @@
 scout-essentials=/$PWD filter="*.rb *.txt *.md *.conf *.yaml" {
  LICENSE.txt
  chats=chats filter="*"{
+  improve_doc
+  intro
+
   review
   document.rb
-  docs=docs{
-   Analyze Annotation tests
-   Analyze CMD and related tests
-   Analyze CMD tests
-   Analyze log tests
-   Analyze Open tests
-   Analyze Path tests
-   Analyze Persist tests
-   Analyze Resource tests
-   Analyze selected tests 
-   simple_opt, log, open, persist, cmd, tmpfile
-   
-   Analyze SimpleOpt tests
-   Analyze simple_opt tests
-   Analyze tests: simple_opt, log, open, persist, cmd, tmpfile
-   Analyze tmpfile tests
-   Analyze Utilities tests
-   Assemble Annotation docs
-   Assemble CMD docs
-   Assemble CMD docs 
-   update
-   
-   Assemble Open docs
-   Assemble Open docs 
-   update
-   
-   Assemble Path docs
-   Assemble Persist docs
-   Assemble Persist docs 
-   update
-   
-   Assemble Resource docs
-   Assemble Resource docs 
-   update
-   
-   Assemble Top-level README
-   Assemble Top-level README 
-   update
-   
-   Assemble Top-level README draft
-   Assemble Utilities docs
-   Assemble Utilities docs 
-   update
-   
-   Confirm·Explore Utilities dirs
-   Explore Annotation dirs
-   Explore CMD dirs
-   Explore Log dir
-   Explore Log lib dir
-   Explore Log lib dir 
-   refresh
-   
-   Explore Log test dir
-   Explore Log test dir 
-   refresh
-   
-   Explore Open dirs
-   Explore Path dirs
-   Explore Persist dirs
-   Explore Resource dirs
-   Explore Utilities dirs
-   Finalize README step 
-   refresh
-   
-   Finalize README.md into memory key 
-   .md
-    
-   conditional
-   
-   NEXT_STEPS
-   QA Annotation docs
-   QA CMD docs
-   QA Integrated README
-   QA Integrated README 
-   post-update
-   
-   QA Integrated README 
-   post-update
-    
-   refresh
-   
-   QA Open docs
-   QA Path docs
-   QA Persist docs
-   QA Resource docs
-   QA Utilities docs
-   Read Annotation source files
-   Read CMD source files
-   Read CMD source files 
-   refresh
-   
-   Read Log source files
-   Read Log source files 
-   iterate
-   
-   Read Log source files 
-   refresh
-   
-   Read Log source files 
-   use explorer output
-   
-   Read Open source files
-   Read Open stream and lockfile
-   Read Path source files
-   Read Persist serialize and open
-   Read Persist source files
-   Read Resource scout.rb
-   Read Resource source files
-   Read SimpleOpt source files
-   Read TmpFile and IndiferentHash source files
-   Read Utilities source files
-   Run QA Integrator on assembled docs
-   Update CMD docs 
-   assemble
-   
-   Update Open docs 
-   assemble
-   
-   Update Persist docs 
-   assemble
-   
-   Update Resource docs 
-   assemble
-   
-   Update Utilities docs 
-   assemble
-   
-   final=final{
-    annotation.md
-    cmd.md
-    open.md
-    path.md
-    persist.md
-    resource.md
-   }
-  }
  }
  doc=doc{
   Annotation.md

data/VERSION

@@ -1 +1 @@
-1.8.5
+1.8.7

data/doc/CMD.md

@@ -32,12 +32,72 @@ puts stream.read   # streaming consumption
 stream.join        # wait for producers and check exit status
 ```
 
+## Common gotchas
+
+These are the most common sources of confusion when using `CMD.cmd`:
+
+- **String vs Symbol as the first argument**:
+  - `CMD.cmd('mytool ...')` runs exactly that shell command.
+  - `CMD.cmd(:mytool, ...)` goes through `CMD.get_tool(:mytool)` first (tool registry / bootstrap). This is useful when you want
+tool discovery/installation behavior.
+  - If you do *not* rely on the tool registry, prefer passing the command as a String.
+
+- **Pipe mode needs a `join` (for correct error detection)**:
+  - With `:pipe => true`, you typically do `io = CMD.cmd(..., pipe: true)` and then `io.read`.
+  - To reliably detect failures (non-zero exit), call `io.join` (unless you deliberately set `no_fail: true`).
+
+- **Non-pipe mode returns a `StringIO`**:
+  - With `:pipe => false` (default), CMD collects stdout and returns a `StringIO` *after* the process completes.
+  - This is convenient, but can be memory-heavy for large outputs.
+
+- **`save_stderr` vs logging**:
+  - `log: true` (default in many contexts) logs stderr lines as they arrive.
+  - `save_stderr: true` additionally accumulates stderr into `io.std_err` (useful for raising helpful exceptions).
+
+- **`no_fail` suppresses exceptions**:
+  - If you pass `no_fail: true`, CMD will not raise `ProcessFailed` / `ConcurrentStreamProcessFailed` on non-zero exits.
+  - This is useful for "try it" probes, but make sure you explicitly check `io.exit_status` (or parse output) when you need correctness.
+
+- **`{opt}` placeholder**:
+  - If the command string contains the exact substring `'{opt}'`, CMD replaces it with the processed options string.
+  - Otherwise, options are appended to the end of the command.
+
+- **Second argument ambiguity**:
+  - `CMD.cmd(tool, {...})` means the second argument is treated as options and `cmd_fragment_or_options` becomes nil.
+  - If you intended to pass a command fragment, pass it as a String, e.g. `CMD.cmd('cut', "-f 2", in: "a b")`.
+
 ---
 
 ## Important options
 
 All options are passed as an options Hash (converted with IndiferentHash), and many are special keys:
 
+### Understanding `CMD.cmd` arguments
+
+`CMD.cmd` has a flexible signature:
+
+- `CMD.cmd(tool_or_cmd, cmd_fragment_or_options = nil, options = {})`
+
+Common calling styles:
+
+```ruby
+# 1) Single string command
+io = CMD.cmd("echo hello")
+
+# 2) Tool + command fragment
+io = CMD.cmd("cut", "-f 2 -d ' '", in: "a b")
+
+# 3) Tool + options hash (options are converted to CLI flags)
+io = CMD.cmd("cut", {"-f" => 2, "-d" => " "}, in: "a b")
+
+# 4) Tool registry symbol (uses CMD.get_tool first)
+io = CMD.cmd(:python, "--version")
+```
+
+Notes:
+- If the *second* argument is a Hash, it is treated as option flags and `cmd_fragment_or_options` becomes nil.
+- Options are shell-quoted; values containing single quotes are escaped.
+
 - :pipe (boolean) — if true, return a stream you can read from; otherwise CMD returns a StringIO after the process completes.
 - :in — input to feed to the command:
   - String will be wrapped by StringIO and streamed to process stdin.
@@ -177,6 +237,39 @@ CMD.cmd('grep . NONEXISTINGFILE', :pipe => true).join
 
 ## Recommendations & patterns
 
+### Robust error-handling pattern
+
+A common robust pattern is:
+
+```ruby
+io = CMD.cmd("SomeTool", "--flag value", log: true, save_stderr: true, no_fail: true)
+
+# Decide how to handle failure
+if io.exit_status != 0
+  raise ScoutException, io.read + "
+" + io.std_err.to_s
+end
+```
+
+- `no_fail: true` prevents immediate exceptions (useful so you can include stderr in your own error message).
+- If you want CMD to raise automatically, omit `no_fail` and rely on `ProcessFailed` / `ConcurrentStreamProcessFailed`.
+
+### Large outputs
+
+- Prefer `:pipe => true` when you want to stream and transform output without buffering everything in memory.
+- If you need to keep the full output, write it to a file as you consume it, and return/keep only a small summary in memory.
+
+### Pipelines
+
+When composing multiple commands, use pipe mode and pass the upstream stream as `:in`:
+
+```ruby
+io1 = CMD.cmd("tool1", "--emit", pipe: true)
+io2 = CMD.cmd("tool2", "--filter", in: io1, pipe: true)
+out = io2.read
+io2.join
+```
+
 - Prefer `:pipe => true` + ConcurrentStream when you want streaming processing without waiting for full output in memory.
 - Provide `:in` as an IO to stream large inputs into a subprocess.
 - Use `:autojoin => true` to automatically join producers on EOF/close (useful for simple consumers).
@@ -200,4 +293,71 @@ CMD.cmd('grep . NONEXISTINGFILE', :pipe => true).join
 
 ---
 
-CMD centralizes robust process execution patterns needed throughout the framework: streaming, joining, logging, error detection and tool bootstrap. Use its options to control behavior for production-grade command invocation.
+CMD centralizes robust process execution patterns needed throughout the framework: streaming, joining, logging, error detection and tool bootstrap. Use its options to control behavior for production-grade command invocation.
+
+---
+
+## Designing command wrappers for workflows and agents
+
+When you wrap external CLI tools inside workflow tasks (or expose them to agents), use a pattern that maximizes reproducibility and keeps outputs small for downstream consumers:
+
+- Backend task (tool runner)
+  - Run the binary, write all full outputs to `step.files_dir` (use `file('name')` helpers).
+  - Return a compact JSON summary with:
+    - `files`: list of important output file paths (full paths inside `.files/`).
+    - `params`: what parameters were used (seeds, time, sample_count, fixed clamps, etc.).
+    - optional small parsed snippets (e.g. final probabilities) if tiny.
+  - Use `CMD.cmd('Binary', ...)` (string) unless the tool is registered; include `save_stderr: true` so errors are captured.
+
+- Analysis task (summary)
+  - `dep` on the backend task and parse only the necessary outputs into a compact summary suitable for the interactive/LLM context (JSON, small tables, phenotype probs).
+  - Echo the backend `params` in the returned result so cached runs are auditable.
+
+Benefits:
+- Clear cache boundaries: the backend is the expensive, cacheable step; the analysis is cheap and reproducible given the backend outputs.
+- Agents never have to load entire trace files; they get a compact summary.
+
+
+## Quick checklist when using CMD in workflows
+
+- Prefer `CMD.cmd('Binary', ...)` string invocation unless you intentionally want the tool registry/install behavior via symbol form.
+- For long-running streaming tasks use `:pipe => true` and **always** `join` the returned stream (or rely on `autojoin`) to detect failures.
+- If you must ignore process failures for probing, use `no_fail: true` but explicitly check exit code or outputs later.
+- Use `save_stderr: true` when you will raise on error: include `io.std_err` in exception messages.
+- Avoid returning raw `StringIO` of huge outputs from tasks — write to `step.files_dir` instead and return a small summary.
+
+
+## Minimal patterns/examples
+
+Read-only capture (small output):
+```ruby
+out = CMD.cmd("echo hello").read   # synchronous, small stdout
+```
+
+Streaming safe consumption:
+```ruby
+stream = CMD.cmd("tail -f /some/log", pipe: true)
+begin
+  data = stream.read
+ensure
+  stream.join   # ensure you detect non-zero exit and collect stderr
+end
+```
+
+Pass an IO to stdin safely:
+```ruby
+f = Open.open('input.txt')
+io = CMD.cmd('someprog', :in => f, pipe: true)
+puts io.read
+io.join
+```
+
+Tool registry vs direct call:
+```ruby
+# use tool registry (only if the tool is registered in CMD.tool)
+CMD.cmd(:my_registered_tool, '--version')
+# prefer direct call when portability is desired
+CMD.cmd('my_tool --version')
+```
+
+For more advanced patterns and examples see the `CMD` implementation and tests in the codebase.

data/lib/scout/annotation/annotated_object.rb

@@ -64,6 +64,11 @@ module Annotation
         new.remove_instance_variable(:@container_index)
       end
 
+      new.instance_variables.each do |var|
+        value = new.instance_variable_get(var)
+        new.instance_variable_set(var, Annotation.purge(value))
+      end
+
       new
     end
 

data/lib/scout/annotation.rb

@@ -26,7 +26,7 @@ module Annotation
   end
 
   def self.is_annotated?(obj)
-    obj.instance_variables.include?(:@annotation_types)
+    obj.instance_variables.include?(:@annotation_types) && obj.respond_to?(:purge)
   end
 
   def self.purge(obj)

data/lib/scout/cmd.rb

@@ -212,8 +212,10 @@ module CMD
             in_content.close unless in_content.closed?
             in_content.join if in_content.respond_to? :join
           end
-        rescue
+        rescue Exception
           Log.error "Error in CMD  [#{pid}] #{cmd}: #{$!.message}" unless no_fail
+          sin.close  unless sin.closed?
+          sin.join if sin.respond_to? :join
           raise $!
         end
       end

data/lib/scout/concurrent_stream.rb

@@ -222,7 +222,7 @@ module ConcurrentStream
     if autojoin
       begin
         super(*args)
-      rescue
+      rescue Exception
         self.abort
         self.join
         stream_raise_exception $!

data/lib/scout/log/fingerprint.rb

@@ -16,6 +16,7 @@ module Log
     when Symbol
       ":" + obj.to_s
     when String
+      obj = obj.gsub("\n", '\n')
       if obj.length > FP_MAX_STRING
         digest = Digest::MD5.hexdigest(obj)
         middle = "<...#{obj.length} - #{digest[0..4]}...>"
@@ -67,6 +68,8 @@ module Log
       else
         obj.inspect
       end
+    when Set
+      fingerprint(obj.to_a)
     else
       obj.to_s
     end

data/lib/scout/log/progress/report.rb

@@ -89,7 +89,7 @@ module Log
       percent = self.percent
       time = Time.now
 
-      indicator = ""
+      indicator = "".dup
       10.times{|i|
         if i < percent / 10 then
           indicator << Log.color(:yellow, ".")

data/lib/scout/log/progress/util.rb

@@ -90,7 +90,7 @@ module Log
         yield bar
       rescue KeepBar
         keep = true
-      rescue
+      rescue Exception
         error = true
         raise $!
       ensure

data/lib/scout/misc/digest.rb

@@ -80,6 +80,9 @@ module Misc
   def self.digest_file(file)
     file = file.find if Path === file
     file = File.expand_path(file)
+    if Open.exist? file + '.md5'
+      return Open.read(file + '.md5').strip
+    end
     if File.size(file) > 10_000_000
       fast_file_md5(file)
     else

data/lib/scout/misc/format.rb

@@ -36,7 +36,7 @@ module Misc
     end
   end
 
-  MAX_TTY_LINE_WIDTH = 160
+  MAX_TTY_LINE_WIDTH = 120
   def self.format_paragraph(text, size = nil, indent = nil, offset = nil)
     size ||= Log.tty_size || MAX_TTY_LINE_WIDTH
     size = MAX_TTY_LINE_WIDTH if size > MAX_TTY_LINE_WIDTH
@@ -45,8 +45,8 @@ module Misc
 
     i = 0
     #size = size + offset + indent
-    re = /((?:\n\s*\n\s*)|(?:\n\s*(?=\*)))/
-      text.split(re).collect do |paragraph|
+    re = /((?:\n\s*\n\s*)|(?:\n\s*(?=\*))|(?:\n```)|(?:\n\s*[-\*])|(?:\n\s\s+))/
+    text.split(re).collect do |paragraph|
       i += 1
       str = if i % 2 == 1
               words = paragraph.gsub(/\s+/, "\s").split(" ")
@@ -75,11 +75,12 @@ module Misc
   def self.format_definition_list_item(dt, dd,  indent = nil, size = nil, color: :yellow)
     if size.nil?
       base_size = Log.tty_size || MAX_TTY_LINE_WIDTH
+      base_size = MAX_TTY_LINE_WIDTH if base_size > MAX_TTY_LINE_WIDTH
       base_indent = indent || (base_size / 3)
       size = base_size - base_indent
     end
 
-    indent ||= base_indent || size / 3
+    indent ||= base_indent || (size / 3)
 
     dd = "" if dd.nil?
     dt = Log.color color, dt if color
@@ -87,12 +88,12 @@ module Misc
     len = Log.uncolor(dt).length
 
     if indent < 0
-      text = format_paragraph(dd, size, indent.abs-1, 0)
+      text = format_paragraph(dd, base_size, indent.abs-1, 0)
       text = dt << "\n" << text
     else
       offset = len - indent
       offset = 0 if offset < 0
-      text = format_paragraph(dd, size, indent.abs+1, offset)
+      text = format_paragraph(dd, base_size, indent.abs+1, offset)
       text[0..len-1] = dt
     end
     text

data/lib/scout/misc/hook.rb

@@ -41,4 +41,10 @@ module Hook
       end
     end
   end
+
+  def self.hook_method(base_class, source_class, method_name)
+    base_class.define_singleton_method(method_name) do |*args, &block|
+      source_class.send(method_name, *args, &block)
+    end
+  end
 end

data/lib/scout/misc/system.rb

@@ -48,17 +48,7 @@ module Misc
     end
   end
 
-  def self.with_env(var, value, &block)
-    old_value = ENV[var]
-    begin
-      ENV[var] = value
-      yield
-    ensure
-      ENV[var] = old_value
-    end
-  end
-
-  def self.with_envs(hash, &block)
+  def self.with_env_hash(hash, &block)
     old_value = {}
     begin
       hash.each do |var,value|
@@ -73,6 +63,24 @@ module Misc
     end
   end
 
+  def self.with_env(var, value = nil, &block)
+    if Hash === value
+      with_env_hash(var, &block)
+    else
+      old_value = ENV[var]
+      begin
+        ENV[var] = value
+        yield
+      ensure
+        ENV[var] = old_value
+      end
+    end
+  end
+
+  class << self
+    alias with_envs with_env_hash
+  end
+
 
   def self.update_git(gem_name = 'scout-essentials')
     gem_name = 'scout-essentials' if gem_name.nil?

data/lib/scout/open/final.rb

@@ -144,8 +144,11 @@ module Open
     file = file.find if Path === file
     begin
       if File.symlink?(file) || File.stat(file).nlink > 1
-        if File.exist?(file + '.info') && defined?(Step)
-          done = Persist.load(file + '.info', Step::SERIALIZER)[:done]
+        info_file = file + '.info'
+        if File.exist?(info_file) && defined?(Step)
+          done = Persist.memory([info_file, File.mtime(info_file)], persist: true ) do
+            Persist.load(info_file, Step::SERIALIZER)[:done]
+          end
           return done if done
         end
 

data/lib/scout/open/util.rb

@@ -35,21 +35,25 @@ module Open
     Bgzf.setup stream
   end
    
-  def self.gunzip(stream)
-    CMD.cmd('zcat', :in => stream, :pipe => true, :no_fail => true, :no_wait => true)
+  def self.gunzip(stream, options = {})
+    options = IndiferentHash.add_defaults options, :pipe => true, :no_fail => false, :no_wait => true, in: stream
+    CMD.cmd('zcat', options)
   end
 
-  def self.gzip(stream)
-    CMD.cmd('gzip', :in => stream, :pipe => true, :no_fail => true, :no_wait => true)
+  def self.gzip(stream, options = {})
+    options = IndiferentHash.add_defaults options, :pipe => true, :no_fail => false, :no_wait => true, in: stream
+    CMD.cmd('gzip', options)
   end
 
-  def self.bgzip(stream)
-    CMD.cmd('bgzip', :in => stream, :pipe => true, :no_fail => true, :no_wait => true)
+  def self.bgzip(stream, options = {})
+    options = IndiferentHash.add_defaults options, :pipe => true, :no_fail => false, :no_wait => true, in: stream
+    CMD.cmd('bgzip', options)
   end
 
-  def self.unzip(stream)
+  def self.unzip(stream, options = {})
+    options = IndiferentHash.add_defaults options, :pipe => true, :no_fail => false, :no_wait => true, in: stream
     TmpFile.with_file(stream.read) do |filename|
-      StringIO.new(CMD.cmd("unzip '{opt}' #{filename}", "-p" => true, :pipe => true).read)
+      StringIO.new(CMD.cmd("unzip '{opt}' #{filename}", options))
     end
   end
 

data/lib/scout/open.rb

@@ -52,9 +52,9 @@ module Open
 
     io = file_open(file, grep, mode, invert_grep, fixed_grep, options)
 
-    io = unzip(io)   if ((String === file and zip?(file))   and not options[:noz]) or options[:zip]
-    io = gunzip(io)  if ((String === file and gzip?(file))  and not options[:noz]) or options[:gzip]
-    io = bgunzip(io) if ((String === file and bgzip?(file)) and not options[:noz]) or options[:bgzip]
+    io = unzip(io, options)   if ((String === file and zip?(file))   and not options[:noz]) or options[:zip]
+    io = gunzip(io, options)  if ((String === file and gzip?(file))  and not options[:noz]) or options[:gzip]
+    io = bgunzip(io, options) if ((String === file and bgzip?(file)) and not options[:noz]) or options[:bgzip]
 
     io.extend NamedStream
     io.filename = file

data/lib/scout/path/find.rb

@@ -203,7 +203,7 @@ module Path
   def annotate_found_where(found, where)
     self.annotate(found).tap{|p| 
       p.instance_variable_set("@where", where) 
-      p.instance_variable_set("@original", self.dup) 
+      p.instance_variable_set("@original", self.annotate(self.dup))
     }
   end
 
@@ -288,15 +288,15 @@ module Path
 
   def find_with_extension(extension, *args, produce: true)
     found = self.find(*args)
-    return found if found.exists?(produce: produce) && ! found.directory?
+    return found if found.exists? && ! found.directory?
     if Array === extension
       extension.each do |ext|
         found_with_extension = self.set_extension(ext).find
-        return found_with_extension if found_with_extension.exists?(produce: produce)
+        return found_with_extension if found_with_extension.exists?
       end
     else
       found_with_extension = self.set_extension(extension).find
-      return found_with_extension if found_with_extension.exists?(produce: produce)
+      return found_with_extension if found_with_extension.exists?
     end
     return found
   end

data/lib/scout/path/util.rb

@@ -23,6 +23,15 @@ module Path
     return true
   end
 
+  def self.can_write?(string)
+    return false if Open.remote?(string)
+    return false unless Path.is_filename?(string)
+    string = string.find if Path === string
+    return false if File.directory?(string)
+    return File.writable?(string) if File.exist?(string)
+    return File.writable?(File.dirname(string))
+  end
+
   def self.sanitize_filename(filename, length = 254)
     if filename.length > length
       if filename =~ /(\..{2,9})$/

data/lib/scout/path.rb

@@ -5,7 +5,7 @@ require_relative 'path/digest'
 
 module Path
   extend Annotation
-  annotation :pkgdir, :libdir, :path_maps, :map_order
+  annotation :pkgdir, :libdir, :path_maps, :map_order, :where, :original
 
   def self.default_pkgdir
     @@default_pkgdir ||= 'scout'

data/lib/scout/persist/serialize.rb

@@ -129,7 +129,7 @@ module Persist
     type = MEMORY if type == :memory
     return unless Hash === type || Open.exist?(file) 
 
-    Log.debug "Load #{Log.fingerprint type} on #{file}"
+    Log.debug "Load #{Log.fingerprint type} from #{file}"
     if load_drivers[type]
       return load_drivers[type].call(file)
     end

data/lib/scout/persist.rb

@@ -42,16 +42,18 @@ module Persist
     update = Open.mtime(update) if Path === update
 
     update = true if file.outdated?(check) if Path === file && check
-    file_mtime = Open.mtime(file)
-    update = file_mtime >= update ? false : true if file_mtime && Time === update
-    update = file_mtime >= (Time.now - update) ? false : true if file_mtime && Numeric === update
+    if update
+      file_mtime = Open.mtime(file)
+      update = file_mtime >= update ? false : true if file_mtime && Time === update
+      update = file_mtime >= (Time.now - update) ? false : true if file_mtime && Numeric === update
+    end
 
     if type == :memory
       repo = options[:memory] || options[:repo] || MEMORY_CACHE
       if update
         repo[file] = yield
       else
-        repo[file] ||= yield
+        repo[file] = yield unless repo.include? file
       end
       return repo[file]
     end

data/lib/scout/resource/path.rb

@@ -76,6 +76,26 @@ module Path
 
   def exists?(produce: true)
     return true if Open.exists?(self.find)
-    self.produce if produce
+    if produce
+      self.produce
+      Open.exists?(self.find)
+    else
+      false
+    end
+  end
+
+  def find_with_extension(extension, *args, produce: true)
+    found = self.find(*args)
+    return found if found.exists?(produce: produce) && ! found.directory?
+    if Array === extension
+      extension.each do |ext|
+        found_with_extension = self.set_extension(ext).find
+        return found_with_extension if found_with_extension.exists?(produce: produce)
+      end
+    else
+      found_with_extension = self.set_extension(extension).find
+      return found_with_extension if found_with_extension.exists?(produce: produce)
+    end
+    return found
   end
 end

data/lib/scout/resource/produce/rake.rb

@@ -56,12 +56,13 @@ module ScoutRake
           Rake::FileTask.clear_files
         end
       rescue Exception
+        Log.exception $!
         Log.error "Error in rake: #{$!.message}"
         Kernel.exit! -1
       end
       Kernel.exit! 0
     }
-    Process.waitpid(pid)
+    Misc.wait_child(pid)
     raise "Rake failed" unless $?.success?
 
   end

data/scout-essentials.gemspec

@@ -2,11 +2,11 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-essentials 1.8.5 ruby lib
+# stub: scout-essentials 1.8.7 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-essentials".freeze
-  s.version = "1.8.5".freeze
+  s.version = "1.8.7".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]

data/test/scout/open/test_remote.rb

@@ -21,5 +21,15 @@ class TestOpenRemote < Test::Unit::TestCase
       end
     end if false
   end
+
+  def test_error
+    teardown
+    sss 0
+    assert_raises ConcurrentStreamProcessFailed do
+      stream = Open.open("ftp://ftp.ncbi.nlm.nih.gov/pub/geo/DATA/SOFT/GDS/GDS3148.soft.gz", nocache: true)
+      stream.read
+    end
+  end
+
 end
 

data/test/scout/path/test_util.rb

@@ -37,5 +37,35 @@ class TestPathUtil < Test::Unit::TestCase
       refute Path.newer? dir.f2.find, dir.f1.find
     end
   end
+
+  def test_can_read
+    TmpFile.with_path do |dir|
+      Open.write dir.f1, 'test1'
+      sleep 0.1
+      Open.write dir.f2, 'test2'
+      Open.write dir.f4, 'test4'
+
+      CMD.cmd("chmod -w #{dir.f4.find}")
+
+      assert Path.can_read? dir.f1
+      assert Path.can_read? dir.f3
+      assert Path.can_write? dir.f3
+
+      refute Path.can_write? dir.f4
+
+      CMD.cmd("chmod -w #{dir.find}")
+      refute Path.can_write? dir.f3
+      CMD.cmd("chmod +w #{dir.find}")
+    end
+  end
+
+  def test_find_exists
+    TmpFile.with_path do |dir|
+      Open.write dir.f1.set_extension("rb"), 'test1'
+
+      assert dir.f1.find_with_extension("rb").exists?
+      refute dir.f1.find_with_extension("rp").exists?
+    end
+  end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scout-essentials
 version: !ruby/object:Gem::Version
-  version: 1.8.5
+  version: 1.8.7
 platform: ruby
 authors:
 - Miguel Vazquez