woods 1.6.1 → 2.0.0.beta2

data/lib/woods/evaluation/ablation_runner.rb

@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+
+require 'shellwords'
+
+require_relative 'ablation_worktree'
+require_relative 'ablation_summary'
+require_relative 'ablation_provenance'
+require_relative 'ablation_agent_payload'
+require_relative 'ablation_executor'
+require_relative 'ablation_timed_executor'
+
+module Woods
+  module Evaluation
+    # Runs every task in an {AblationTaskSet} twice, index on and index off,
+    # each trial in its own disposable {AblationWorktree} checked out from a
+    # fixed baseline commit, and reports resolution rate, tokens, cost,
+    # turns, and errors per condition (#280).
+    #
+    # This is a harness for collecting paired on/off runs, not a source of
+    # causal evidence: sample size, task selection, and agent nondeterminism
+    # all bear on what a result means. See docs/EVALUATION.md.
+    #
+    # The agent is any command that prints one JSON object on stdout in the
+    # `claude -p --output-format json` shape. The executor is injectable so
+    # the runner never invokes a real agent in specs.
+    #
+    # Every command a trial runs (worktree add/remove, the optional reset,
+    # the agent invocation, and the check) shares one {AblationTimedExecutor}
+    # and is therefore bound by the same `timeout`, applied per call rather
+    # than once for the whole trial: a slow reset does not eat into the
+    # agent's budget, and vice versa. A timed-out call is terminated (TERM,
+    # then KILL if still alive after a short grace period) when the
+    # underlying executor exposes a pid.
+    #
+    # @example
+    #   set = AblationTaskSet.load('config/eval_ablation.json')
+    #   report = AblationRunner.new(task_set: set, workdir: Rails.root.to_s).run
+    #   report.summary[:delta][:mean_tokens] # => negative when the index saves tokens
+    #
+    class AblationRunner
+      Result = Struct.new(:task_id, :condition, :resolved, :total_tokens, :cost_usd, :turns, :duration_ms,
+                          :error, :provenance, keyword_init: true)
+      Report = Struct.new(:results, :summary, keyword_init: true)
+
+      DEFAULT_TIMEOUT = 600
+
+      # Verifies Woods availability before a trial runs, distinguishing "MCP
+      # enabled" (the agent command is wired to reach the Woods MCP server)
+      # from "index present" (an index actually exists in the checkout): the
+      # `:on` condition needs both. The `:off` condition needs both signals
+      # too, in the negative: `--strict-mcp-config` declared, and no
+      # `--mcp-config` reference at all. A command carrying both flags is
+      # still wired to Woods and fails preflight.
+      DEFAULT_WOODS_PROBE = lambda do |condition, chdir, command|
+        mcp_wired = command.include?('--mcp-config')
+        strict = command.include?('--strict-mcp-config')
+        index_present = File.exist?(File.join(chdir, 'tmp', 'woods', 'generation.json'))
+        if condition == :on
+          mcp_wired && index_present
+        else
+          strict && !mcp_wired
+        end
+      end
+
+      # @param task_set [AblationTaskSet::Definition]
+      # @param workdir [String] the host application root
+      # @param executor [#call] `call(command, chdir:)` returning `[stdout, stderr, success]`.
+      #   Wrapped in an {AblationTimedExecutor} exactly once, so every command a trial runs
+      #   through it (worktree add/remove, reset, agent, check) shares the same per-call
+      #   timeout; see `timeout` below.
+      # @param conditions [Array<Symbol>] subset of `%i[on off]`
+      # @param trial_options [Hash]
+      # @option trial_options [Numeric] :timeout seconds allowed per call the trial makes
+      #   (worktree add/remove, the optional reset, the agent invocation, and the check),
+      #   applied independently to each one rather than shared across the whole trial
+      # @option trial_options [#call] :woods_probe `call(condition, chdir, command)` returning a boolean
+      def initialize(task_set:, workdir:, executor: nil, conditions: %i[on off], **trial_options)
+        @task_set = task_set
+        @workdir = workdir
+        @conditions = conditions
+        @timeout = trial_options.fetch(:timeout, DEFAULT_TIMEOUT)
+        @woods_probe = trial_options.fetch(:woods_probe, DEFAULT_WOODS_PROBE)
+        @executor = AblationTimedExecutor.new(executor || AblationExecutor.new, timeout: @timeout)
+      end
+
+      # @return [Report]
+      def run
+        baseline_sha = resolve_baseline_sha
+        results = @task_set.tasks.flat_map do |task|
+          @conditions.map { |condition| run_trial(task, condition, baseline_sha) }
+        end
+        Report.new(results: results, summary: AblationSummary.build(results, @conditions))
+      end
+
+      private
+
+      def resolve_baseline_sha
+        out, _err, ok = @executor.call('git rev-parse HEAD', chdir: @workdir)
+        ok ? out.strip : nil
+      end
+
+      def run_trial(task, condition, baseline_sha)
+        worktree = AblationWorktree.new(repo_root: @workdir, baseline_sha: baseline_sha, executor: @executor)
+        result = nil
+        setup_error = worktree.trial(@task_set.reset) { |path| result = execute(task, condition, path, baseline_sha) }
+        result || failed_result(task, condition, blank_provenance(condition, baseline_sha), setup_error)
+      end
+
+      def execute(task, condition, path, baseline_sha)
+        chdir = File.expand_path(task.workdir, path)
+        command = agent_command(condition).sub('{prompt}', Shellwords.escape(task.prompt))
+        provenance = AblationProvenance.build(agent_command: command, chdir: chdir, baseline_sha: baseline_sha)
+
+        probe_error = preflight_error(condition, chdir, command)
+        return failed_result(task, condition, provenance, probe_error) if probe_error
+
+        run_agent_and_check(task, condition, chdir, command, provenance)
+      end
+
+      def run_agent_and_check(task, condition, chdir, command, provenance)
+        stdout, stderr, success = @executor.call(command, chdir: chdir)
+        payload = AblationAgentPayload.parse(stdout)
+        provenance.model = payload && payload['model']
+        _check_stdout, check_stderr, resolved = @executor.call(task.check, chdir: chdir)
+
+        Result.new(task_id: task.id, condition: condition, resolved: resolved == true,
+                   total_tokens: payload && AblationAgentPayload.token_total(payload['usage']),
+                   cost_usd: payload && payload['total_cost_usd'],
+                   turns: payload && payload['num_turns'],
+                   duration_ms: payload && payload['duration_ms'],
+                   error: agent_error(success, payload, stderr) || check_timeout_error(check_stderr),
+                   provenance: provenance)
+      end
+
+      def agent_error(success, payload, stderr)
+        return nil if success && payload
+
+        [stderr.to_s.strip, payload ? nil : 'agent printed no JSON'].compact.join('; ')
+      end
+
+      # The check's own exit status already drives `resolved`; a normal test
+      # failure is not a harness error. A timeout is: {AblationTimedExecutor}
+      # marks it with a recognizable stderr message, so it still surfaces as
+      # an error (and is counted in the summary) even though `resolved` is
+      # also false.
+      def check_timeout_error(check_stderr)
+        return nil unless check_stderr.to_s.include?('timed out after')
+
+        "check #{check_stderr.strip}"
+      end
+
+      def preflight_error(condition, chdir, command)
+        return nil if @woods_probe.call(condition, chdir, command)
+
+        "Woods availability check failed for condition #{condition}"
+      end
+
+      def failed_result(task, condition, provenance, error)
+        Result.new(task_id: task.id, condition: condition, resolved: false, total_tokens: nil, cost_usd: nil,
+                   turns: nil, duration_ms: nil, error: error, provenance: provenance)
+      end
+
+      def blank_provenance(condition, baseline_sha)
+        AblationProvenance.new(agent_command: agent_command(condition), model: nil, config: nil,
+                               woods_generation: nil, baseline_sha: baseline_sha)
+      end
+
+      def agent_command(condition)
+        condition == :on ? @task_set.agent_on : @task_set.agent_off
+      end
+    end
+  end
+end

data/lib/woods/evaluation/ablation_summary.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Woods
+  module Evaluation
+    # Aggregates {AblationRunner::Result} values into a per-condition summary
+    # (resolution rate, mean tokens/cost/turns, task and error counts) plus
+    # the on-minus-off delta (#280).
+    class AblationSummary
+      DELTA_METRICS = %i[resolution_rate mean_tokens mean_cost_usd mean_turns].freeze
+
+      # @param results [Array<AblationRunner::Result>]
+      # @param conditions [Array<Symbol>] subset of `%i[on off]`
+      # @return [Hash{Symbol=>Hash}]
+      def self.build(results, conditions)
+        new(results, conditions).build
+      end
+
+      def initialize(results, conditions)
+        @results = results
+        @conditions = conditions
+      end
+
+      def build
+        summary = {}
+        @conditions.each { |condition| summary[condition] = condition_summary(condition) }
+        summary[:delta] = delta(summary[:on], summary[:off]) if summary.key?(:on) && summary.key?(:off)
+        summary
+      end
+
+      private
+
+      def condition_summary(condition)
+        results = @results.select { |result| result.condition == condition }
+        {
+          resolution_rate: resolution_rate(results),
+          mean_tokens: mean(results.map(&:total_tokens)),
+          mean_cost_usd: mean(results.map(&:cost_usd)),
+          mean_turns: mean(results.map(&:turns)),
+          tasks: results.size,
+          errors: results.count(&:error)
+        }
+      end
+
+      def resolution_rate(results)
+        return 0.0 if results.empty?
+
+        (results.count(&:resolved).to_f / results.size).round(4)
+      end
+
+      def delta(on, off)
+        DELTA_METRICS.to_h do |key|
+          value = on[key].nil? || off[key].nil? ? nil : (on[key] - off[key]).round(4)
+          [key, value]
+        end
+      end
+
+      def mean(values)
+        present = values.compact
+        return nil if present.empty?
+
+        (present.sum.to_f / present.size).round(4)
+      end
+    end
+  end
+end

data/lib/woods/evaluation/ablation_task.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require_relative 'ablation_task_set'
+require_relative 'ablation_runner'
+require_relative 'ablation_report_writer'
+
+module Woods
+  module Evaluation
+    # Entry point for `woods:evaluate:ablation`: loads a task set, applies
+    # environment overrides, runs the harness, and writes the report (#280).
+    # Kept out of the rake file so the task definition stays a thin
+    # delegation and the logic is testable without Rake.
+    #
+    # File map for `lib/woods/evaluation/ablation_*.rb`: {AblationTaskSet}
+    # loads and validates the task set file; {AblationRunner} orchestrates a
+    # trial per task per condition; {AblationWorktree} gives each trial its
+    # own disposable git checkout; {AblationExecutor} is the default
+    # subprocess executor; {AblationTimedExecutor} wraps any executor with
+    # the per-call timeout and kill-on-timeout behavior; {AblationProvenance}
+    # and {AblationAgentPayload} build a trial's provenance and parse the
+    # agent's JSON payload; {AblationSummary} aggregates results into the
+    # per-condition and delta summary; {AblationReportWriter} serializes the
+    # report and renders the terminal summary; this class wires them
+    # together for the rake task.
+    class AblationTask
+      DEFAULT_TASKS = 'config/eval_ablation.json'
+      DEFAULT_OUTPUT = 'tmp/eval_ablation.json'
+
+      # @param task_set_path [String, nil]
+      # @param workdir [String]
+      # @return [AblationRunner::Report]
+      def self.run(task_set_path: nil, workdir: Dir.pwd)
+        new(task_set_path: task_set_path, workdir: workdir).run
+      end
+
+      def initialize(task_set_path: nil, workdir: Dir.pwd)
+        @task_set_path = task_set_path || ENV.fetch('EVAL_ABLATION_TASKS', DEFAULT_TASKS)
+        @output = ENV.fetch('EVAL_ABLATION_OUTPUT', DEFAULT_OUTPUT)
+        @workdir = workdir
+      end
+
+      # @return [AblationRunner::Report]
+      def run
+        task_set = load_task_set
+        puts "Running #{task_set.tasks.size} task(s) with the index on and off, " \
+             'one disposable worktree per trial...'
+        report = AblationRunner.new(task_set: task_set, workdir: @workdir).run
+
+        writer = AblationReportWriter.new(report)
+        writer.write(@output)
+        puts writer.summary_text
+        puts "Report saved to: #{@output}"
+        report
+      end
+
+      private
+
+      def load_task_set
+        task_set = AblationTaskSet.load(@task_set_path)
+        task_set.agent_on = ENV['EVAL_AGENT_ON'] if ENV['EVAL_AGENT_ON']
+        task_set.agent_off = ENV['EVAL_AGENT_OFF'] if ENV['EVAL_AGENT_OFF']
+        task_set
+      end
+    end
+  end
+end

data/lib/woods/evaluation/ablation_task_set.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Woods
+  module Evaluation
+    # A task set for the agent-level ablation (#280): the same coding tasks
+    # run with the Woods index on and off.
+    #
+    # File shape (schema_version 1):
+    #
+    #   {
+    #     "schema_version": 1,
+    #     "agent_on":  "claude -p {prompt} --output-format json --mcp-config .mcp.json",
+    #     "agent_off": "claude -p {prompt} --output-format json --strict-mcp-config",
+    #     "reset":     "git checkout -- . && git clean -fdq",   // optional
+    #     "tasks": [
+    #       { "id": "...", "prompt": "...", "check": "bin/rspec spec/...", "workdir": "." }
+    #     ]
+    #   }
+    #
+    # `{prompt}` is replaced with the shell-escaped prompt. `check` is any
+    # command whose exit status says whether the task was resolved.
+    class AblationTaskSet
+      SUPPORTED_SCHEMA_VERSIONS = [1].freeze
+
+      Task = Struct.new(:id, :prompt, :check, :workdir, keyword_init: true)
+
+      # Named `Definition` rather than `Data`: Ruby 3.2 introduces a
+      # top-level `::Data` class, and shadowing it inside this class body is
+      # legal on the 3.0 floor but confusing to read.
+      Definition = Struct.new(:schema_version, :agent_on, :agent_off, :reset, :tasks, keyword_init: true)
+
+      class << self
+        # @param path [String]
+        # @return [Definition]
+        # @raise [Woods::Error] on an unreadable file, invalid JSON, or an invalid shape
+        def load(path)
+          raw = JSON.parse(File.read(path.to_s, encoding: 'UTF-8'))
+          version = raw['schema_version']
+          unless SUPPORTED_SCHEMA_VERSIONS.include?(version)
+            raise Woods::Error, "Unsupported ablation schema_version #{version.inspect} in #{path}"
+          end
+
+          Definition.new(
+            schema_version: version,
+            agent_on: command!(raw, 'agent_on', path),
+            agent_off: command!(raw, 'agent_off', path),
+            reset: raw['reset'],
+            tasks: Array(raw['tasks']).map { |task| build_task(task, path) }
+          )
+        rescue JSON::ParserError => e
+          raise Woods::Error, "Invalid JSON in ablation task set #{path}: #{e.message}"
+        rescue Errno::ENOENT => e
+          raise Woods::Error, "Ablation task set not found: #{e.message}"
+        end
+
+        private
+
+        def command!(raw, key, path)
+          value = raw[key].to_s
+          raise Woods::Error, "#{key} in #{path} must contain {prompt}" unless value.include?('{prompt}')
+
+          value
+        end
+
+        def build_task(task, path)
+          %w[id prompt check].each do |key|
+            raise Woods::Error, "Task #{task['id'].inspect} in #{path} is missing #{key}" if task[key].to_s.empty?
+          end
+
+          Task.new(id: task['id'], prompt: task['prompt'], check: task['check'], workdir: task['workdir'] || '.')
+        end
+      end
+    end
+  end
+end

data/lib/woods/evaluation/ablation_timed_executor.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require 'timeout'
+
+module Woods
+  module Evaluation
+    # Bounds every call an ablation trial makes to the same per-call timeout
+    # (#280 review): a per-command budget applied independently to each
+    # command a trial runs (worktree add/remove, the optional reset, the
+    # agent invocation, the check), not one budget shared across the whole
+    # trial. `AblationRunner` wraps whatever executor it is given with this
+    # class exactly once, so `AblationWorktree` and the runner itself both
+    # get the bound for free by calling `@executor.call` as normal.
+    #
+    # On timeout, `Timeout.timeout` only interrupts the calling thread; a
+    # subprocess started by the wrapped executor keeps running unless
+    # something kills it. When the wrapped executor exposes its most recent
+    # pid (see {AblationExecutor}), this sends TERM, waits briefly, then KILL
+    # if the process is still alive, before reporting the call as a timed-out
+    # failure. An executor that does not expose a pid (for example a fake
+    # executor in a spec) still gets the timeout, just without a process to
+    # terminate.
+    class AblationTimedExecutor
+      TERM_GRACE_SECONDS = 2
+      REAP_GRACE_SECONDS = 2
+      POLL_INTERVAL = 0.05
+
+      # @param executor [#call] `call(command, chdir:)` returning `[stdout, stderr, success]`
+      # @param timeout [Numeric] seconds allowed per call
+      def initialize(executor, timeout:)
+        @executor = executor
+        @timeout = timeout
+      end
+
+      # @return [Array(String, String, Boolean)] stdout, stderr, success
+      def call(command, chdir:)
+        Timeout.timeout(@timeout) { @executor.call(command, chdir: chdir) }
+      rescue Timeout::Error
+        ['', "timed out after #{@timeout}s#{terminate_process}", false]
+      end
+
+      private
+
+      # @return [String] empty, or a note that the pid outlived the reap
+      #   grace period, appended to the timeout message
+      def terminate_process
+        pid = executor_pid
+        return '' unless pid
+
+        Process.kill('TERM', pid)
+        Process.kill('KILL', pid) unless process_exited?(pid, within: TERM_GRACE_SECONDS)
+        reap(pid, within: REAP_GRACE_SECONDS) ? '' : " (pid #{pid} did not reap within #{REAP_GRACE_SECONDS}s)"
+      rescue Errno::ESRCH, Errno::ECHILD
+        ''
+      end
+
+      def executor_pid
+        @executor.pid if @executor.respond_to?(:pid)
+      end
+
+      def process_exited?(pid, within:)
+        deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + within
+        until Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
+          begin
+            Process.kill(0, pid)
+          rescue Errno::ESRCH
+            return true
+          end
+          sleep POLL_INTERVAL
+        end
+        false
+      end
+
+      # Polls a non-blocking wait instead of a plain Process.wait (#280
+      # review, minor): a pid some other bug left un-reapable must not hang
+      # the whole trial forever just because we tried to clean up after it.
+      def reap(pid, within:)
+        deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + within
+        until Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
+          reaped_pid, = Process.wait2(pid, Process::WNOHANG)
+          return true if reaped_pid
+
+          sleep POLL_INTERVAL
+        end
+        false
+      rescue Errno::ECHILD
+        true
+      end
+    end
+  end
+end

data/lib/woods/evaluation/ablation_worktree.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require 'tmpdir'
+require 'fileutils'
+require 'shellwords'
+
+module Woods
+  module Evaluation
+    # Sets up and tears down one disposable git worktree per ablation trial
+    # (#280), checked out from a fixed baseline SHA so an on/off pair for
+    # the same task starts from the identical commit. The trial never runs
+    # in the caller's own working directory, and a reset failure aborts the
+    # trial rather than being ignored.
+    #
+    # @example
+    #   worktree = AblationWorktree.new(repo_root: '/app', baseline_sha: sha, executor: executor)
+    #   error = worktree.trial('git checkout -- . && git clean -fdq') { |path| run_agent(path) }
+    #   error # => nil on success, an error string otherwise (the block never ran)
+    class AblationWorktree
+      # @param repo_root [String] the repository the trial is checked out from
+      # @param baseline_sha [String, nil] the commit every trial checks out
+      # @param executor [#call] `call(command, chdir:)` returning `[stdout, stderr, success]`
+      def initialize(repo_root:, baseline_sha:, executor:)
+        @repo_root = repo_root
+        @baseline_sha = baseline_sha
+        @executor = executor
+      end
+
+      # @param reset_command [String, nil]
+      # @yieldparam path [String] the checkout root, only when setup succeeded
+      # @return [String, nil] an error message when the checkout or reset
+      #   failed (the block never ran), else nil
+      def trial(reset_command)
+        parent = nil
+        path = nil
+        added = false
+
+        parent = Dir.mktmpdir('woods-ablation-')
+        path = File.join(parent, 'checkout')
+        error = add_worktree(path)
+        added = error.nil?
+        error ||= reset(path, reset_command) if added && reset_command
+        yield path if error.nil?
+        error
+      ensure
+        teardown(path) if added
+        FileUtils.remove_entry(parent, true) if parent
+      end
+
+      private
+
+      def add_worktree(path)
+        _out, err, ok = run_in_repo("git worktree add --detach #{Shellwords.escape(path)} #{@baseline_sha}")
+        ok ? nil : "git worktree add failed: #{err.to_s.strip}"
+      end
+
+      def reset(path, reset_command)
+        _out, err, ok = @executor.call(reset_command, chdir: path)
+        ok ? nil : "reset failed: #{err.to_s.strip}"
+      end
+
+      def teardown(path)
+        run_in_repo("git worktree remove --force #{Shellwords.escape(path)}")
+      end
+
+      def run_in_repo(command)
+        @executor.call(command, chdir: @repo_root)
+      end
+    end
+  end
+end

data/lib/woods/evaluation/baseline.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Woods
+  module Evaluation
+    # Loads a versioned baseline/thresholds file for `woods:evaluate`.
+    #
+    # File shape (schema_version 1):
+    #
+    #   {
+    #     "schema_version": 1,
+    #     "query_set": "config/eval_queries.json",
+    #     "captured_at": "2026-08-20T00:00:00Z",   // or null if never captured
+    #     "thresholds": {
+    #       "mean_precision_at5": 0.0,
+    #       "mean_recall": 0.0,
+    #       "mean_mrr": 0.0
+    #     },
+    #     "notes": "free text"
+    #   }
+    #
+    # `thresholds` keys must match {Evaluator::EvaluationReport}'s +aggregates+
+    # keys (e.g. "mean_recall", not "recall") — the evaluator compares them
+    # directly against computed aggregates.
+    #
+    # The checked-in example under spec/fixtures/evaluation_baseline.example.json
+    # is a format fixture only: its thresholds are 0.0 placeholders, not a
+    # captured v2 corpus baseline. Capturing the real baseline — running the
+    # harness against the pinned v2 corpus and recording the resulting scores
+    # as thresholds — is future work.
+    class Baseline
+      SUPPORTED_SCHEMA_VERSIONS = [1].freeze
+
+      Data = Struct.new(:schema_version, :query_set, :captured_at, :thresholds, :notes, keyword_init: true)
+
+      class << self
+        # @param path [String] path to a baseline JSON file
+        # @return [Data]
+        # @raise [Woods::Error] on an unreadable file, invalid JSON, or an
+        #   unsupported schema_version
+        def load(path)
+          raw = JSON.parse(File.read(path.to_s, encoding: 'UTF-8'))
+          version = raw['schema_version']
+          unless SUPPORTED_SCHEMA_VERSIONS.include?(version)
+            raise Woods::Error, "Unsupported baseline schema_version #{version.inspect} in #{path}"
+          end
+
+          Data.new(
+            schema_version: version,
+            query_set: raw['query_set'],
+            captured_at: raw['captured_at'],
+            thresholds: (raw['thresholds'] || {}).transform_keys(&:to_sym),
+            notes: raw['notes']
+          )
+        end
+      end
+    end
+  end
+end

data/lib/woods/evaluation/baseline_runner.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative '../storage_identity'
+
 module Woods
   module Evaluation
     # Runs simple baseline strategies for comparison against the full
@@ -51,6 +53,12 @@ module Woods
 
       private
 
+      # Evaluation ground truth uses public names, not vector/metadata storage keys.
+      # Coexisting types count once because the evaluation contract is name-based.
+      def public_identifiers
+        @metadata_store.all_identifiers.map { |id| StorageIdentity.identifier(id) }.uniq
+      end
+
       # Grep strategy: substring match on unit identifiers.
       #
       # Extracts words from the query and matches identifiers that contain
@@ -60,7 +68,7 @@ module Woods
       # @param limit [Integer] Max results
       # @return [Array<String>]
       def run_grep(query, limit)
-        all_ids = @metadata_store.all_identifiers
+        all_ids = public_identifiers
         keywords = extract_keywords(query)
 
         return all_ids.first(limit) if keywords.empty?
@@ -85,7 +93,7 @@ module Woods
       # @return [Array<String>]
       def run_random(_query, limit)
         @random_mutex.synchronize do
-          @metadata_store.all_identifiers.sample(limit, random: @random)
+          public_identifiers.sample(limit, random: @random)
         end
       end
 
@@ -96,7 +104,7 @@ module Woods
       # @param limit [Integer] Max results
       # @return [Array<String>]
       def run_file_level(query, limit)
-        all_ids = @metadata_store.all_identifiers
+        all_ids = public_identifiers
         keywords = extract_keywords(query)
 
         return all_ids.first(limit) if keywords.empty?