scout-essentials 1.8.6 → 1.8.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 860143272938aa7cf5cea99f3c6b58206ed6670cf0c8638adcfe62774fcab925
-  data.tar.gz: 6c92e398a80f3d39c3cae9db84d947c9386d8a08cb0bfffab2239c0810d1ba6e
+  metadata.gz: 3193beb1b7024505d5a238bb6d9caa280ddab034b23431652138697ebc9ac7f0
+  data.tar.gz: 29ef13d1a7862228eabc8f153f602ba728eda98cf4907b23770a8fb65cb36b98
 SHA512:
-  metadata.gz: aef77cc47e9de21fa0da8ddc9e0246afd91849aad07b3e7d8b33813098cde01a9971c6dcd58d8b07d2b9c372a580ce74aaa484258b5cf914c48e0eae28dc165e
-  data.tar.gz: 4638f2423e319274ff0ad0b5885b9669b156c519278e775e6ab33733faf0b1c2abeb561f7256c8ebada13c4faaaf2325897a39e0d85b0c49d19413dd0fc6c125
+  metadata.gz: c074200ead16664e4ecdcd262315f389500cd59391b533bcdd1f4a84bdf9ab3fcf96152698f643393d198d9cc1d552707144fd4fbc9c962badbb33d57c79d364
+  data.tar.gz: 57663384b0258537a626eb867e8dd209c9e84bc477dbc462c6096af539f54b176936b1cd4ace7cd47525b5e791a6b13ac41a000fdf2709a9add3a830eb1a33ab

data/.vimproject

@@ -1,146 +1,11 @@
 scout-essentials=/$PWD filter="*.rb *.txt *.md *.conf *.yaml" {
  LICENSE.txt
  chats=chats filter="*"{
+  improve_doc
   intro
 
   review
   document.rb
-  docs=docs{
-   Analyze Annotation tests
-   Analyze CMD and related tests
-   Analyze CMD tests
-   Analyze log tests
-   Analyze Open tests
-   Analyze Path tests
-   Analyze Persist tests
-   Analyze Resource tests
-   Analyze selected tests 
-   simple_opt, log, open, persist, cmd, tmpfile
-   
-   Analyze SimpleOpt tests
-   Analyze simple_opt tests
-   Analyze tests: simple_opt, log, open, persist, cmd, tmpfile
-   Analyze tmpfile tests
-   Analyze Utilities tests
-   Assemble Annotation docs
-   Assemble CMD docs
-   Assemble CMD docs 
-   update
-   
-   Assemble Open docs
-   Assemble Open docs 
-   update
-   
-   Assemble Path docs
-   Assemble Persist docs
-   Assemble Persist docs 
-   update
-   
-   Assemble Resource docs
-   Assemble Resource docs 
-   update
-   
-   Assemble Top-level README
-   Assemble Top-level README 
-   update
-   
-   Assemble Top-level README draft
-   Assemble Utilities docs
-   Assemble Utilities docs 
-   update
-   
-   Confirm·Explore Utilities dirs
-   Explore Annotation dirs
-   Explore CMD dirs
-   Explore Log dir
-   Explore Log lib dir
-   Explore Log lib dir 
-   refresh
-   
-   Explore Log test dir
-   Explore Log test dir 
-   refresh
-   
-   Explore Open dirs
-   Explore Path dirs
-   Explore Persist dirs
-   Explore Resource dirs
-   Explore Utilities dirs
-   Finalize README step 
-   refresh
-   
-   Finalize README.md into memory key 
-   .md
-    
-   conditional
-   
-   NEXT_STEPS
-   QA Annotation docs
-   QA CMD docs
-   QA Integrated README
-   QA Integrated README 
-   post-update
-   
-   QA Integrated README 
-   post-update
-    
-   refresh
-   
-   QA Open docs
-   QA Path docs
-   QA Persist docs
-   QA Resource docs
-   QA Utilities docs
-   Read Annotation source files
-   Read CMD source files
-   Read CMD source files 
-   refresh
-   
-   Read Log source files
-   Read Log source files 
-   iterate
-   
-   Read Log source files 
-   refresh
-   
-   Read Log source files 
-   use explorer output
-   
-   Read Open source files
-   Read Open stream and lockfile
-   Read Path source files
-   Read Persist serialize and open
-   Read Persist source files
-   Read Resource scout.rb
-   Read Resource source files
-   Read SimpleOpt source files
-   Read TmpFile and IndiferentHash source files
-   Read Utilities source files
-   Run QA Integrator on assembled docs
-   Update CMD docs 
-   assemble
-   
-   Update Open docs 
-   assemble
-   
-   Update Persist docs 
-   assemble
-   
-   Update Resource docs 
-   assemble
-   
-   Update Utilities docs 
-   assemble
-   
-   final=final{
-    annotation.md
-    cmd.md
-    open.md
-    path.md
-    persist.md
-    resource.md
-   }
-  }
  }
  doc=doc{
   Annotation.md

data/VERSION

@@ -1 +1 @@
-1.8.6
+1.8.7

data/doc/CMD.md

@@ -32,12 +32,72 @@ puts stream.read   # streaming consumption
 stream.join        # wait for producers and check exit status
 ```
 
+## Common gotchas
+
+These are the most common sources of confusion when using `CMD.cmd`:
+
+- **String vs Symbol as the first argument**:
+  - `CMD.cmd('mytool ...')` runs exactly that shell command.
+  - `CMD.cmd(:mytool, ...)` goes through `CMD.get_tool(:mytool)` first (tool registry / bootstrap). This is useful when you want
+tool discovery/installation behavior.
+  - If you do *not* rely on the tool registry, prefer passing the command as a String.
+
+- **Pipe mode needs a `join` (for correct error detection)**:
+  - With `:pipe => true`, you typically do `io = CMD.cmd(..., pipe: true)` and then `io.read`.
+  - To reliably detect failures (non-zero exit), call `io.join` (unless you deliberately set `no_fail: true`).
+
+- **Non-pipe mode returns a `StringIO`**:
+  - With `:pipe => false` (default), CMD collects stdout and returns a `StringIO` *after* the process completes.
+  - This is convenient, but can be memory-heavy for large outputs.
+
+- **`save_stderr` vs logging**:
+  - `log: true` (default in many contexts) logs stderr lines as they arrive.
+  - `save_stderr: true` additionally accumulates stderr into `io.std_err` (useful for raising helpful exceptions).
+
+- **`no_fail` suppresses exceptions**:
+  - If you pass `no_fail: true`, CMD will not raise `ProcessFailed` / `ConcurrentStreamProcessFailed` on non-zero exits.
+  - This is useful for "try it" probes, but make sure you explicitly check `io.exit_status` (or parse output) when you need correctness.
+
+- **`{opt}` placeholder**:
+  - If the command string contains the exact substring `'{opt}'`, CMD replaces it with the processed options string.
+  - Otherwise, options are appended to the end of the command.
+
+- **Second argument ambiguity**:
+  - `CMD.cmd(tool, {...})` means the second argument is treated as options and `cmd_fragment_or_options` becomes nil.
+  - If you intended to pass a command fragment, pass it as a String, e.g. `CMD.cmd('cut', "-f 2", in: "a b")`.
+
 ---
 
 ## Important options
 
 All options are passed as an options Hash (converted with IndiferentHash), and many are special keys:
 
+### Understanding `CMD.cmd` arguments
+
+`CMD.cmd` has a flexible signature:
+
+- `CMD.cmd(tool_or_cmd, cmd_fragment_or_options = nil, options = {})`
+
+Common calling styles:
+
+```ruby
+# 1) Single string command
+io = CMD.cmd("echo hello")
+
+# 2) Tool + command fragment
+io = CMD.cmd("cut", "-f 2 -d ' '", in: "a b")
+
+# 3) Tool + options hash (options are converted to CLI flags)
+io = CMD.cmd("cut", {"-f" => 2, "-d" => " "}, in: "a b")
+
+# 4) Tool registry symbol (uses CMD.get_tool first)
+io = CMD.cmd(:python, "--version")
+```
+
+Notes:
+- If the *second* argument is a Hash, it is treated as option flags and `cmd_fragment_or_options` becomes nil.
+- Options are shell-quoted; values containing single quotes are escaped.
+
 - :pipe (boolean) — if true, return a stream you can read from; otherwise CMD returns a StringIO after the process completes.
 - :in — input to feed to the command:
   - String will be wrapped by StringIO and streamed to process stdin.
@@ -177,6 +237,39 @@ CMD.cmd('grep . NONEXISTINGFILE', :pipe => true).join
 
 ## Recommendations & patterns
 
+### Robust error-handling pattern
+
+A common robust pattern is:
+
+```ruby
+io = CMD.cmd("SomeTool", "--flag value", log: true, save_stderr: true, no_fail: true)
+
+# Decide how to handle failure
+if io.exit_status != 0
+  raise ScoutException, io.read + "
+" + io.std_err.to_s
+end
+```
+
+- `no_fail: true` prevents immediate exceptions (useful so you can include stderr in your own error message).
+- If you want CMD to raise automatically, omit `no_fail` and rely on `ProcessFailed` / `ConcurrentStreamProcessFailed`.
+
+### Large outputs
+
+- Prefer `:pipe => true` when you want to stream and transform output without buffering everything in memory.
+- If you need to keep the full output, write it to a file as you consume it, and return/keep only a small summary in memory.
+
+### Pipelines
+
+When composing multiple commands, use pipe mode and pass the upstream stream as `:in`:
+
+```ruby
+io1 = CMD.cmd("tool1", "--emit", pipe: true)
+io2 = CMD.cmd("tool2", "--filter", in: io1, pipe: true)
+out = io2.read
+io2.join
+```
+
 - Prefer `:pipe => true` + ConcurrentStream when you want streaming processing without waiting for full output in memory.
 - Provide `:in` as an IO to stream large inputs into a subprocess.
 - Use `:autojoin => true` to automatically join producers on EOF/close (useful for simple consumers).
@@ -200,4 +293,71 @@ CMD.cmd('grep . NONEXISTINGFILE', :pipe => true).join
 
 ---
 
-CMD centralizes robust process execution patterns needed throughout the framework: streaming, joining, logging, error detection and tool bootstrap. Use its options to control behavior for production-grade command invocation.
+CMD centralizes robust process execution patterns needed throughout the framework: streaming, joining, logging, error detection and tool bootstrap. Use its options to control behavior for production-grade command invocation.
+
+---
+
+## Designing command wrappers for workflows and agents
+
+When you wrap external CLI tools inside workflow tasks (or expose them to agents), use a pattern that maximizes reproducibility and keeps outputs small for downstream consumers:
+
+- Backend task (tool runner)
+  - Run the binary, write all full outputs to `step.files_dir` (use `file('name')` helpers).
+  - Return a compact JSON summary with:
+    - `files`: list of important output file paths (full paths inside `.files/`).
+    - `params`: what parameters were used (seeds, time, sample_count, fixed clamps, etc.).
+    - optional small parsed snippets (e.g. final probabilities) if tiny.
+  - Use `CMD.cmd('Binary', ...)` (string) unless the tool is registered; include `save_stderr: true` so errors are captured.
+
+- Analysis task (summary)
+  - `dep` on the backend task and parse only the necessary outputs into a compact summary suitable for the interactive/LLM context (JSON, small tables, phenotype probs).
+  - Echo the backend `params` in the returned result so cached runs are auditable.
+
+Benefits:
+- Clear cache boundaries: the backend is the expensive, cacheable step; the analysis is cheap and reproducible given the backend outputs.
+- Agents never have to load entire trace files; they get a compact summary.
+
+
+## Quick checklist when using CMD in workflows
+
+- Prefer `CMD.cmd('Binary', ...)` string invocation unless you intentionally want the tool registry/install behavior via symbol form.
+- For long-running streaming tasks use `:pipe => true` and **always** `join` the returned stream (or rely on `autojoin`) to detect failures.
+- If you must ignore process failures for probing, use `no_fail: true` but explicitly check exit code or outputs later.
+- Use `save_stderr: true` when you will raise on error: include `io.std_err` in exception messages.
+- Avoid returning raw `StringIO` of huge outputs from tasks — write to `step.files_dir` instead and return a small summary.
+
+
+## Minimal patterns/examples
+
+Read-only capture (small output):
+```ruby
+out = CMD.cmd("echo hello").read   # synchronous, small stdout
+```
+
+Streaming safe consumption:
+```ruby
+stream = CMD.cmd("tail -f /some/log", pipe: true)
+begin
+  data = stream.read
+ensure
+  stream.join   # ensure you detect non-zero exit and collect stderr
+end
+```
+
+Pass an IO to stdin safely:
+```ruby
+f = Open.open('input.txt')
+io = CMD.cmd('someprog', :in => f, pipe: true)
+puts io.read
+io.join
+```
+
+Tool registry vs direct call:
+```ruby
+# use tool registry (only if the tool is registered in CMD.tool)
+CMD.cmd(:my_registered_tool, '--version')
+# prefer direct call when portability is desired
+CMD.cmd('my_tool --version')
+```
+
+For more advanced patterns and examples see the `CMD` implementation and tests in the codebase.

data/lib/scout/cmd.rb

@@ -212,8 +212,10 @@ module CMD
             in_content.close unless in_content.closed?
             in_content.join if in_content.respond_to? :join
           end
-        rescue
+        rescue Exception
           Log.error "Error in CMD  [#{pid}] #{cmd}: #{$!.message}" unless no_fail
+          sin.close  unless sin.closed?
+          sin.join if sin.respond_to? :join
           raise $!
         end
       end

data/lib/scout/concurrent_stream.rb

@@ -222,7 +222,7 @@ module ConcurrentStream
     if autojoin
       begin
         super(*args)
-      rescue
+      rescue Exception
         self.abort
         self.join
         stream_raise_exception $!

data/lib/scout/log/fingerprint.rb

@@ -16,6 +16,7 @@ module Log
     when Symbol
       ":" + obj.to_s
     when String
+      obj = obj.gsub("\n", '\n')
       if obj.length > FP_MAX_STRING
         digest = Digest::MD5.hexdigest(obj)
         middle = "<...#{obj.length} - #{digest[0..4]}...>"

data/lib/scout/log/progress/report.rb

@@ -89,7 +89,7 @@ module Log
       percent = self.percent
       time = Time.now
 
-      indicator = ""
+      indicator = "".dup
       10.times{|i|
         if i < percent / 10 then
           indicator << Log.color(:yellow, ".")

data/lib/scout/log/progress/util.rb

@@ -90,7 +90,7 @@ module Log
         yield bar
       rescue KeepBar
         keep = true
-      rescue
+      rescue Exception
         error = true
         raise $!
       ensure

data/lib/scout/misc/format.rb

@@ -45,7 +45,7 @@ module Misc
 
     i = 0
     #size = size + offset + indent
-    re = /((?:\n\s*\n\s*)|(?:\n\s*(?=\*))|(?:\n\s\s+)|(?:\n\s*[-\*]))/
+    re = /((?:\n\s*\n\s*)|(?:\n\s*(?=\*))|(?:\n```)|(?:\n\s*[-\*])|(?:\n\s\s+))/
     text.split(re).collect do |paragraph|
       i += 1
       str = if i % 2 == 1

data/lib/scout/misc/hook.rb

@@ -41,4 +41,10 @@ module Hook
       end
     end
   end
+
+  def self.hook_method(base_class, source_class, method_name)
+    base_class.define_singleton_method(method_name) do |*args, &block|
+      source_class.send(method_name, *args, &block)
+    end
+  end
 end

data/lib/scout/open/util.rb

@@ -35,21 +35,25 @@ module Open
     Bgzf.setup stream
   end
    
-  def self.gunzip(stream)
-    CMD.cmd('zcat', :in => stream, :pipe => true, :no_fail => true, :no_wait => true)
+  def self.gunzip(stream, options = {})
+    options = IndiferentHash.add_defaults options, :pipe => true, :no_fail => false, :no_wait => true, in: stream
+    CMD.cmd('zcat', options)
   end
 
-  def self.gzip(stream)
-    CMD.cmd('gzip', :in => stream, :pipe => true, :no_fail => true, :no_wait => true)
+  def self.gzip(stream, options = {})
+    options = IndiferentHash.add_defaults options, :pipe => true, :no_fail => false, :no_wait => true, in: stream
+    CMD.cmd('gzip', options)
   end
 
-  def self.bgzip(stream)
-    CMD.cmd('bgzip', :in => stream, :pipe => true, :no_fail => true, :no_wait => true)
+  def self.bgzip(stream, options = {})
+    options = IndiferentHash.add_defaults options, :pipe => true, :no_fail => false, :no_wait => true, in: stream
+    CMD.cmd('bgzip', options)
   end
 
-  def self.unzip(stream)
+  def self.unzip(stream, options = {})
+    options = IndiferentHash.add_defaults options, :pipe => true, :no_fail => false, :no_wait => true, in: stream
     TmpFile.with_file(stream.read) do |filename|
-      StringIO.new(CMD.cmd("unzip '{opt}' #{filename}", "-p" => true, :pipe => true).read)
+      StringIO.new(CMD.cmd("unzip '{opt}' #{filename}", options))
     end
   end
 

data/lib/scout/open.rb

@@ -52,9 +52,9 @@ module Open
 
     io = file_open(file, grep, mode, invert_grep, fixed_grep, options)
 
-    io = unzip(io)   if ((String === file and zip?(file))   and not options[:noz]) or options[:zip]
-    io = gunzip(io)  if ((String === file and gzip?(file))  and not options[:noz]) or options[:gzip]
-    io = bgunzip(io) if ((String === file and bgzip?(file)) and not options[:noz]) or options[:bgzip]
+    io = unzip(io, options)   if ((String === file and zip?(file))   and not options[:noz]) or options[:zip]
+    io = gunzip(io, options)  if ((String === file and gzip?(file))  and not options[:noz]) or options[:gzip]
+    io = bgunzip(io, options) if ((String === file and bgzip?(file)) and not options[:noz]) or options[:bgzip]
 
     io.extend NamedStream
     io.filename = file

data/lib/scout/path/find.rb

@@ -288,15 +288,15 @@ module Path
 
   def find_with_extension(extension, *args, produce: true)
     found = self.find(*args)
-    return found if found.exists?(produce: produce) && ! found.directory?
+    return found if found.exists? && ! found.directory?
     if Array === extension
       extension.each do |ext|
         found_with_extension = self.set_extension(ext).find
-        return found_with_extension if found_with_extension.exists?(produce: produce)
+        return found_with_extension if found_with_extension.exists?
       end
     else
       found_with_extension = self.set_extension(extension).find
-      return found_with_extension if found_with_extension.exists?(produce: produce)
+      return found_with_extension if found_with_extension.exists?
     end
     return found
   end

data/lib/scout/resource/path.rb

@@ -76,6 +76,26 @@ module Path
 
   def exists?(produce: true)
     return true if Open.exists?(self.find)
-    self.produce if produce
+    if produce
+      self.produce
+      Open.exists?(self.find)
+    else
+      false
+    end
+  end
+
+  def find_with_extension(extension, *args, produce: true)
+    found = self.find(*args)
+    return found if found.exists?(produce: produce) && ! found.directory?
+    if Array === extension
+      extension.each do |ext|
+        found_with_extension = self.set_extension(ext).find
+        return found_with_extension if found_with_extension.exists?(produce: produce)
+      end
+    else
+      found_with_extension = self.set_extension(extension).find
+      return found_with_extension if found_with_extension.exists?(produce: produce)
+    end
+    return found
   end
 end

data/lib/scout/resource/produce/rake.rb

@@ -56,12 +56,13 @@ module ScoutRake
           Rake::FileTask.clear_files
         end
       rescue Exception
+        Log.exception $!
         Log.error "Error in rake: #{$!.message}"
         Kernel.exit! -1
       end
       Kernel.exit! 0
     }
-    Process.waitpid(pid)
+    Misc.wait_child(pid)
     raise "Rake failed" unless $?.success?
 
   end

data/scout-essentials.gemspec

@@ -2,11 +2,11 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-essentials 1.8.6 ruby lib
+# stub: scout-essentials 1.8.7 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-essentials".freeze
-  s.version = "1.8.6".freeze
+  s.version = "1.8.7".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
@@ -159,7 +159,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://github.com/mikisvaz/scout-essentials".freeze
   s.licenses = ["MIT".freeze]
-  s.rubygems_version = "3.7.2".freeze
+  s.rubygems_version = "3.7.0.dev".freeze
   s.summary = "Scout essential tools".freeze
 
   s.specification_version = 4

data/test/scout/open/test_remote.rb

@@ -21,5 +21,15 @@ class TestOpenRemote < Test::Unit::TestCase
       end
     end if false
   end
+
+  def test_error
+    teardown
+    sss 0
+    assert_raises ConcurrentStreamProcessFailed do
+      stream = Open.open("ftp://ftp.ncbi.nlm.nih.gov/pub/geo/DATA/SOFT/GDS/GDS3148.soft.gz", nocache: true)
+      stream.read
+    end
+  end
+
 end
 

data/test/scout/path/test_util.rb

@@ -58,5 +58,14 @@ class TestPathUtil < Test::Unit::TestCase
       CMD.cmd("chmod +w #{dir.find}")
     end
   end
+
+  def test_find_exists
+    TmpFile.with_path do |dir|
+      Open.write dir.f1.set_extension("rb"), 'test1'
+
+      assert dir.f1.find_with_extension("rb").exists?
+      refute dir.f1.find_with_extension("rp").exists?
+    end
+  end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scout-essentials
 version: !ruby/object:Gem::Version
-  version: 1.8.6
+  version: 1.8.7
 platform: ruby
 authors:
 - Miguel Vazquez
@@ -298,7 +298,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.7.2
+rubygems_version: 3.7.0.dev
 specification_version: 4
 summary: Scout essential tools
 test_files: []