scout-essentials 1.9.1 → 1.9.3

data/lib/scout/testing/availability.rb

@@ -0,0 +1,264 @@
+# Opt-in generic execution/availability primitives for test suites.
+#
+# Layering contract (see research/testing-availability-*.md in this repo):
+# this module deliberately contains ONLY stdlib-generic primitives — PATH
+# lookup, bounded child execution, and a bounded TCP connect. No interpreter,
+# package, model, or network-service knowledge lives here: those probes
+# belong to the layers that own them.
+#
+# This file is deliberately not required by scout-essentials.
+require 'socket'
+
+module Scout
+  module Testing
+    module Availability
+      DEFAULT_TIMEOUT = 30
+      TCP_TIMEOUT = 5
+      OUTPUT_LIMIT = 65_536
+      READ_CHUNK = 16 * 1024
+      KILL_GRACE = 0.1
+      CHUNK_POLL = 0.05
+      TRUNCATION_NOTE = "\n[output truncated]"
+
+      # Result of a run_command call: everything the caller needs to decide,
+      # including the failure modes that are reported rather than raised.
+      CommandResult = Struct.new(
+        :argv, :success, :exit_status, :stdout, :stderr,
+        :timed_out, :launch_error, keyword_init: true
+      ) do
+        def success?
+          !!success
+        end
+
+        # Reason for a non-successful run, preferring the concrete failure
+        # mode over raw output. Order: launch failure, timeout, exit status,
+        # stderr, stdout.
+        def reason
+          return nil if success?
+          return launch_error if launch_error
+          return 'command timed out' if timed_out
+          return "exit status #{exit_status}" if exit_status
+
+          detail = stderr.to_s.strip
+          return detail unless detail.empty?
+
+          detail = stdout.to_s.strip
+          return detail unless detail.empty?
+
+          'could not start executable'
+        end
+      end
+
+      class << self
+        # True when name resolves to an executable on PATH, or contains a
+        # path separator and names an executable file directly. No shell.
+        def executable?(name)
+          return false if name.to_s.empty?
+
+          candidate = name.to_s
+          if candidate.include?(File::SEPARATOR) || candidate.include?('\\')
+            return File.file?(candidate) && File.executable?(candidate)
+          end
+
+          ENV['PATH'].to_s.split(File::PATH_SEPARATOR).any? do |directory|
+            next false if directory.to_s.empty?
+
+            path = File.join(directory, candidate)
+            File.file?(path) && File.executable?(path)
+          end
+        end
+
+        # Run argv (never a shell string) with a hard deadline. Both output
+        # streams are drained concurrently so a chatty child cannot deadlock
+        # a pipe, output is bounded per stream, and a timeout escalates
+        # TERM -> KILL for the child process group.
+        #
+        # Launch failure (e.g. Errno::ENOENT) and timeout are reported on the
+        # returned CommandResult, never raised. ArgumentError is raised only
+        # for a non-positive/non-finite timeout or an empty argv, which are
+        # programming errors. Not memoized: the answer can change per call.
+        def run_command(argv, timeout: DEFAULT_TIMEOUT)
+          argv = Array(argv).map(&:to_s)
+          raise ArgumentError, 'argv must not be empty' if argv.empty?
+
+          seconds = checked_timeout(timeout)
+
+          out_read, out_write = IO.pipe
+          err_read, err_write = IO.pipe
+          pid = nil
+          reaped = false
+          killed = false
+          readers = [out_read, err_read]
+          stdout = +''
+          stderr = +''
+          status = nil
+
+          begin
+            pid = Process.spawn(*argv, out: out_write, err: err_write,
+                                    pgroup: true, close_others: true)
+          rescue StandardError => error
+            close_all(readers + [out_write, err_write])
+            return CommandResult.new(
+              argv: argv, success: false, exit_status: nil,
+              stdout: '', stderr: '', timed_out: false,
+              launch_error: "#{error.class}: #{error.message}"
+            )
+          ensure
+            # Closed in the parent regardless of spawn outcome, so EOF on the
+            # read ends is meaningful.
+            [out_write, err_write].each { |io| io.close unless io.closed? }
+          end
+
+          timed_out = false
+          deadline = monotonic + seconds
+
+          # Drain both pipes while waiting, so neither fills up and blocks
+          # the child while this side is blocked on the other stream.
+          until status && readers.empty?
+            remaining = deadline - monotonic
+            if remaining <= 0
+              timed_out = true
+              break
+            end
+
+            ready, = IO.select(readers, nil, nil, [remaining, CHUNK_POLL].min)
+            Array(ready).each do |io|
+              begin
+                chunk = io.read_nonblock(READ_CHUNK)
+                append_limited(io.equal?(out_read) ? stdout : stderr, chunk)
+              rescue EOFError
+                readers.delete(io)
+                io.close
+              end
+            end
+
+            next if status
+
+            waited, candidate = Process.waitpid2(pid, Process::WNOHANG)
+            if waited
+              status = candidate
+              reaped = true
+            end
+          end
+
+          if timed_out
+            killed = true
+            status ||= terminate_group(pid)
+            reaped = true
+            stderr = note(stderr, "command timed out after #{seconds}s")
+          end
+
+          close_all(readers)
+
+          CommandResult.new(
+            argv: argv,
+            success: !timed_out && status&.exitstatus == 0,
+            exit_status: status&.exitstatus,
+            stdout: stdout,
+            stderr: stderr,
+            timed_out: timed_out,
+            launch_error: nil
+          )
+        ensure
+          close_all(readers) if readers
+          unless pid.nil? || reaped
+            terminate_group(pid) unless killed
+            reap(pid)
+          end
+        end
+
+        # True means only that a TCP connection to host:port was established
+        # within the timeout. A malformed port (non-numeric string, out of
+        # range) reports false rather than raising; only an invalid timeout
+        # raises, since that is a programming error. No memoization:
+        # reachability can change.
+        def tcp_reachable?(host, port, timeout: TCP_TIMEOUT)
+          seconds = checked_timeout(timeout)
+          port = Integer(port) rescue nil
+          return false unless (1..65_535).cover?(port) && !host.to_s.empty?
+
+          begin
+            Socket.tcp(host.to_s, port, connect_timeout: seconds) do |socket|
+              socket.close
+            end
+            true
+          rescue SocketError, SystemCallError, IOError, ArgumentError
+            false
+          end
+        end
+
+        private
+
+        def monotonic
+          Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        end
+
+        def checked_timeout(value)
+          number = Float(value)
+          unless number.positive? && number.finite?
+            raise ArgumentError, 'timeout must be positive and finite'
+          end
+
+          number
+        end
+
+        def close_all(ios)
+          ios.each { |io| io.close unless io.closed? }
+        rescue IOError
+          nil
+        end
+
+        # TERM the process group, wait a short grace period, then KILL.
+        # Returns the exit status when this call reaped the child, nil when
+        # the child was already reaped elsewhere or had to be killed.
+        def terminate_group(pid)
+          signal_group(pid, 'TERM')
+          grace = monotonic + KILL_GRACE
+          loop do
+            waited, status = waitpid_nohang(pid)
+            return status if waited
+            break if monotonic >= grace
+
+            sleep(CHUNK_POLL / 2)
+          end
+          signal_group(pid, 'KILL')
+          nil
+        end
+
+        def waitpid_nohang(pid)
+          Process.waitpid2(pid, Process::WNOHANG)
+        rescue Errno::ECHILD
+          [false, nil]
+        end
+
+        def signal_group(pid, signal)
+          Process.kill(signal, -pid)
+        rescue SystemCallError
+          begin
+            Process.kill(signal, pid)
+          rescue SystemCallError
+            nil
+          end
+        end
+
+        def reap(pid)
+          waited, status = Process.waitpid2(pid)
+          waited ? status : nil
+        rescue Errno::ECHILD
+          nil
+        end
+
+        def append_limited(buffer, chunk)
+          remaining = OUTPUT_LIMIT - buffer.bytesize
+          buffer << chunk.byteslice(0, remaining) if remaining.positive?
+          marker = chunk.bytesize > remaining && !buffer.end_with?(TRUNCATION_NOTE)
+          buffer << TRUNCATION_NOTE if marker
+        end
+
+        def note(text, addition)
+          text.empty? ? addition : "#{text}\n#{addition}"
+        end
+      end
+    end
+  end
+end

data/lib/scout/testing/scout_state.rb

@@ -0,0 +1,193 @@
+# Opt-in Scout global-state isolation for tests.
+require_relative '../testing'
+require_relative '../tmpfile'
+require_relative '../persist'
+require_relative '../open'
+
+module Scout
+  module Testing
+    class State
+      class RestoreError < StandardError
+        attr_reader :errors, :original_error
+
+        def initialize(errors, original_error: nil)
+          @errors = errors
+          @original_error = original_error
+          details = errors.map { |error| "#{error.class}: #{error.message}" }.join('; ')
+          message = "Failed to restore #{errors.length} state binding(s): #{details}"
+          message += " (while handling #{original_error.class}: #{original_error.message})" if original_error
+          super(message)
+        end
+      end
+
+      def initialize(root, bindings: [])
+        @root = File.expand_path(root)
+        @bindings = normalize_bindings(bindings)
+      end
+
+      def activate
+        raise RuntimeError, 'State is already active' if @saved
+
+        @saved = {}
+        @activated_bindings = []
+        begin
+          [[TmpFile, :@tmpdir], [Persist, :@cache_dir], [Persist, :@lock_dir],
+           [Open, :@remote_cache_dir], [Open, :@sensible_write_dir],
+           [Open, :@sensible_write_lock_dir]].each do |owner, ivar|
+            @saved[[owner, ivar]] = [owner.instance_variable_defined?(ivar), owner.instance_variable_get(ivar)]
+          end
+          @binding_snapshots = @bindings.to_h do |binding|
+            [binding, [binding[:owner].instance_variable_defined?(binding[:ivar]),
+                       binding[:owner].instance_variable_get(binding[:ivar])]]
+          end
+          @memory_cache = Persist::MEMORY_CACHE.dup
+
+          FileUtils.mkdir_p(@root)
+          @root = File.realpath(@root)
+          TmpFile.tmpdir = File.join(@root, 'tmpfiles')
+          Persist.cache_dir = File.join(@root, 'persistence-cache')
+          Persist.lock_dir = File.join(@root, 'persistence-locks')
+          Open.remote_cache_dir = File.join(@root, 'remote-cache')
+          Open.sensible_write_dir = File.join(@root, 'sensible-write')
+          Open.sensible_write_lock_dir = File.join(@root, 'sensible-write-locks')
+          @bindings.each do |binding|
+            binding[:target] = File.expand_path(binding[:relative_path], @root)
+            create_binding_directory(binding[:target])
+            @activated_bindings << binding
+            binding[:owner].public_send(binding[:writer], binding[:target])
+          end
+          Persist::MEMORY_CACHE.clear
+          self
+        rescue Exception => activation_error
+          begin
+            restore
+          rescue State::RestoreError => restore_error
+            raise State::RestoreError.new(restore_error.errors, original_error: activation_error)
+          end
+          raise
+        end
+      end
+
+      def restore
+        return self unless @saved
+
+        errors = []
+        @activated_bindings.reverse_each do |binding|
+          was_defined, value = @binding_snapshots.fetch(binding)
+          begin
+            # Accessor-backed bindings are intended for simple settings whose
+            # backing instance variable is the conventional @<accessor>.
+            binding[:owner].public_send(binding[:writer], value) if was_defined
+          rescue Exception => error
+            errors << error
+          ensure
+            begin
+              if was_defined
+                binding[:owner].instance_variable_set(binding[:ivar], value)
+              elsif binding[:owner].instance_variable_defined?(binding[:ivar])
+                binding[:owner].remove_instance_variable(binding[:ivar])
+              end
+            rescue Exception => error
+              errors << error
+            end
+          end
+        end
+
+        @saved.each do |(owner, ivar), (was_defined, value)|
+          begin
+            if was_defined
+              owner.instance_variable_set(ivar, value)
+            elsif owner.instance_variable_defined?(ivar)
+              owner.remove_instance_variable(ivar)
+            end
+          rescue Exception => error
+            errors << error
+          end
+        end
+        begin
+          Persist::MEMORY_CACHE.replace(@memory_cache) if @memory_cache
+        rescue Exception => error
+          errors << error
+        ensure
+          @saved = nil
+          @binding_snapshots = nil
+          @activated_bindings = nil
+          @memory_cache = nil
+        end
+
+        raise RestoreError, errors unless errors.empty?
+        self
+      end
+
+      private
+
+      def create_binding_directory(target)
+        prefix = "#{@root}#{File::SEPARATOR}"
+        raise ArgumentError, "binding path must stay under state root: #{target}" unless target.start_with?(prefix)
+
+        relative = target.delete_prefix(prefix)
+        current = @root
+        relative.split(File::SEPARATOR).each do |part|
+          current = File.join(current, part)
+          raise ArgumentError, "binding path traverses a symlink: #{current}" if File.symlink?(current)
+
+          begin
+            Dir.mkdir(current)
+          rescue Errno::EEXIST
+            raise unless File.directory?(current) && !File.symlink?(current)
+          end
+          raise ArgumentError, "binding path is not a directory: #{current}" unless File.directory?(current) && !File.symlink?(current)
+        end
+      end
+
+      def normalize_bindings(bindings)
+        raise ArgumentError, 'bindings must be an Array' unless Array === bindings
+
+        seen = {}
+        bindings.map do |binding|
+          unless Array === binding && binding.length == 3
+            raise ArgumentError, 'each binding must be [owner, accessor, relative_path]'
+          end
+          owner, accessor, relative_path = binding
+          unless owner.respond_to?(:instance_variable_defined?) && Symbol === accessor &&
+              accessor.to_s.match?(/\A[a-zA-Z_]\w*\z/)
+            raise ArgumentError, "invalid state binding owner or accessor: #{binding.inspect}"
+          end
+          writer = "#{accessor}=".to_sym
+          raise ArgumentError, "#{owner} does not respond to #{writer}" unless owner.respond_to?(writer)
+          unless String === relative_path && !relative_path.empty? && !relative_path.start_with?(File::SEPARATOR)
+            raise ArgumentError, "binding path must be a non-empty relative path: #{relative_path.inspect}"
+          end
+          target = File.expand_path(relative_path, @root)
+          unless target.start_with?(@root.end_with?(File::SEPARATOR) ? @root : "#{@root}#{File::SEPARATOR}")
+            raise ArgumentError, "binding path must stay under state root: #{relative_path.inspect}"
+          end
+          key = [owner, accessor]
+          raise ArgumentError, "duplicate state binding: #{owner}.#{accessor}" if seen[key]
+          seen[key] = true
+          { owner: owner, accessor: accessor, writer: writer,
+            ivar: "@#{accessor}".to_sym, target: target, relative_path: relative_path }
+        end
+      end
+    end
+
+    module_function
+
+    def with_isolated_state(root, bindings: [])
+      state = State.new(root, bindings: bindings)
+      begin
+        state.activate
+        yield state
+      rescue Exception => original_error
+        begin
+          state.restore
+        rescue State::RestoreError => restore_error
+          raise State::RestoreError.new(restore_error.errors, original_error: original_error)
+        end
+        raise
+      else
+        state.restore
+      end
+    end
+  end
+end

data/lib/scout/testing.rb

@@ -0,0 +1,82 @@
+# Opt-in support for generic test helpers; deliberately not required by scout-essentials.
+require 'tmpdir'
+require 'fileutils'
+
+module Scout
+  module Testing
+    module Assertions
+      def assert_equal_path(expected, actual, message = nil)
+        assert_equal File.expand_path(expected), File.expand_path(actual), message
+      end
+
+      def assert_path_exists(path, message = nil)
+        assert File.exist?(path), message || "Expected path to exist: #{path}"
+      end
+    end
+
+    module_function
+
+    def create_tmpdir(parent: Dir.tmpdir, prefix: 'scout-test-')
+      FileUtils.mkdir_p(parent)
+      Dir.mktmpdir(prefix, parent)
+    end
+
+    def remove_tmpdir(path)
+      FileUtils.remove_entry(path) if path && File.exist?(path)
+    end
+
+    def with_tmpdir(parent: Dir.tmpdir, prefix: 'scout-test-')
+      path = create_tmpdir(parent: parent, prefix: prefix)
+      begin
+        yield path
+      ensure
+        remove_tmpdir(path)
+      end
+    end
+
+    # Resolve against the consumer's repository, not this helper's source tree.
+    def fixture_root(from:, directory: 'test/fixtures')
+      File.expand_path(directory, repository_root(from))
+    end
+
+    def fixture_path(*parts, from:, directory: 'test/fixtures')
+      File.join(fixture_root(from: from, directory: directory), *parts)
+    end
+
+    def repository_root(from)
+      start = File.expand_path(from)
+      start = File.dirname(start) unless File.directory?(start)
+      ancestors = []
+      path = start
+      loop do
+        ancestors << path
+        parent = File.dirname(path)
+        break if parent == path
+        path = parent
+      end
+
+      # A Git worktree is authoritative when present, including for paths
+      # nested below a consumer's test or lib directory.
+      git_root = ancestors.find do |candidate|
+        File.directory?(File.join(candidate, '.git')) || File.file?(File.join(candidate, '.git'))
+      end
+      return git_root if git_root
+
+      # For non-Git consumers, infer the root from the conventional test/ or
+      # lib/ layout surrounding the supplied file/directory. This is a
+      # heuristic, not a universal repository detector: arbitrary paths with
+      # no such marker retain the conservative file-local fallback below.
+      layout_directory = ancestors.find { |candidate| %w[test lib].include?(File.basename(candidate)) }
+      return File.dirname(layout_directory) if layout_directory
+
+      # Also support a consumer root passed directly (or a directory beneath
+      # it) when that root contains a conventional top-level test/ or lib/.
+      root_with_layout = ancestors.find do |candidate|
+        File.directory?(File.join(candidate, 'test')) || File.directory?(File.join(candidate, 'lib'))
+      end
+      return root_with_layout if root_with_layout
+
+      File.dirname(File.expand_path(from))
+    end
+  end
+end

data/research/doc-audit-findings.md

@@ -3,11 +3,13 @@
 Audit target: `/bulk/mvazque2/git/scout-essentials` (gem v1.8.8, HEAD `a0324a4`).
 Outcome: `doc/` fully rewritten (17 pages) + 5 new pages + `README.md` rewritten;
 Critic final gate: **ACCEPT_WITH_NOTES** (notes applied).
-This file is the retained summary. The full evidence trail (coverage matrix with
-261 rows, 22 per-claim ledgers, 3 implementation inventories, 66 behavior probes,
-ecosystem attribution, phase reports and critic gates) lives in
-`research/doc_audit/`, `research/` and `tmp/critic/` — see
-`research/doc_audit/resumption-reference.md` §2 for the index.
+This file is the retained summary. Of the full evidence trail, the
+durable parts still in the repository are
+`research/behavior-probes.md` (66 behavior probes, executed
+transcripts) and the three `research/implementation-inventory-*.md`
+file:line inventories; the transient working directories
+(`research/doc_audit/`, `tmp/critic/`) were removed after the audit
+closed.
 
 Disposition counts (coverage-matrix sections A–F, 238 rows): OK 3 · FIX 117 ·
 EXPAND 67 · NEW_PAGE 27 · INTERNAL 6 · ELSEWHERE 13 · OUT_OF_SCOPE 6.

data/research/doc-structure.md

@@ -0,0 +1,89 @@
+# Documentation review — how `doc/` and `research/` are organized
+
+**Status:** Non-normative process artifact.
+
+This note records how the scout-essentials documentation set is
+structured after the 2026-09 review, and the standard each layer holds
+itself to. It is for future maintainers deciding where new material
+belongs.
+
+## The two-layer contract
+
+- **`doc/`** — normative, well-proven, core concepts only. A claim
+  belongs here once it is stable, verified against the current `lib/`,
+  and something a user or developer needs to operate the library. Pages
+  cite `file:line` anchors rather than probes or transcripts: the source
+  itself is the citation.
+- **`research/`** — non-normative investigations, process records, and
+  claims that are interesting but either not core, not fully proven, or
+  machine/lesson specific. Every file declares
+  `**Status:** Non-normative investigation artifact. May be outdated.`
+
+## Layout
+
+```
+doc/
+  StartHere.md          — the entry point + reading paths
+  Improvements.md       — running log of known defects / improvement ideas
+  user/                 — how to *use* each subsystem, task-oriented
+    WorkingWithFiles, RemoteData, RunningCommands, HandlingStreams,
+    CachingResults, ProducingResources, AnnotatingData,
+    LoggingAndProgress, CommandLineOptions, Cookbook
+    (Path itself is covered by developer/PathResolution.md and used
+    throughout the user pages)
+  developer/            — how each subsystem *works*, mechanism-oriented
+    Architecture, DesignPrinciples, AnnotationSystem, PathResolution,
+    PersistenceAndResources, StreamingModel, Configuration,
+    ErrorHandling, LockingAndConcurrency, CoreUtilities
+research/
+  *-analysis.md         — per-chunk behavioural analyses (annotations,
+                          commands/streaming, io/paths, design philosophy)
+  implementation-inventory-*.md — file:line inventories per chunk
+  doc-audit-findings.md — forensic audit report (hallucinations found,
+                          corrected claims, decisions taken)
+  ecosystem-attribution.md — which repo owns which constant
+  behavior-probes.md    — historical probe transcripts (the probes
+                          themselves were promoted to tests / deleted)
+  lock-namespace-blindness.md — promoted-from-cortex behavioural hazard
+```
+
+## Where new material goes
+
+| kind of finding | destination |
+|---|---|
+| core, verified behaviour a user needs | `doc/user/<Topic>.md` |
+| core, verified mechanism a maintainer needs | `doc/developer/<Topic>.md` |
+| known defect / improvement candidate | `doc/Improvements.md` (numbered `I#`) |
+| hazard, heuristic, environment-specific behaviour | `research/<topic>.md` |
+| one-off verification transcript | nowhere — write a regression test instead |
+| cross-repo attribution of a constant | `research/ecosystem-attribution.md` |
+
+## The promotion path (2026-09 episode)
+
+The 2026-09 documentation review followed this ladder:
+
+```
+probes (cortex probe/*.rb artifacts)
+  → claims (cortex *.md artifacts)
+  → doc/ pages (claims rewritten pedagogically, probe citations removed)
+  → tests (selected probes promoted to test/scout/**/test_*.rb)
+  → cortex probes + execution records deleted
+```
+
+Rules applied:
+
+1. **No probe references in `doc/`.** Documentation cites `file:line`
+   anchors and regression tests; transcripts are process, not product.
+2. **Probes whose claims were promoted became tests, not corpses.**
+   Eight regression tests were extracted (NamedArray resolution,
+   CaseInsensitiveHash staleness, `string2hash` false-asymmetry,
+   `Misc.timespan` tokens, Open compression detection, broken
+   `:gzip => true`, annotation dup/clone, TmpFile cleanup).
+3. **Probes with no durable claim were deleted**, together with their
+   `properties/Observation/probe` execution records. The four promoted
+   claims artifacts stay in cortex with a "promoted out of cortex"
+   banner naming their new home.
+4. **Cortex claims that document a hazard stay in cortex** only while
+   unproven; `persist_lock_theft_across_agents` was promoted to
+   `research/lock-namespace-blindness.md` with the operational advice
+   folded into `doc/developer/LockingAndConcurrency.md`.