clinkx 0.3.0 → 1.1.0

package/dist/context-builder/discovery.js

@@ -5,10 +5,12 @@ import { isSamePathOrDescendant } from "../allowed-roots.js";
 import { CancellationError } from "../errors.js";
 import { CONTEXT_BUILDER_INTERNAL_CAP_DEFAULTS, } from "./config.js";
 import { CONTEXT_BUILDER_REQUESTED_EVIDENCE_OMISSION_REASON, requestedEvidenceOmission, } from "./diagnostics.js";
+import { CONTEXT_BUILDER_SCAN_PROGRESS_EVERY_FILES } from "./discovery-progress.js";
 import { compileContextBuilderGlobs, CONTEXT_BUILDER_DEFAULT_EXCLUDE_GLOBS, } from "./globs.js";
 import { normalizeContextBuilderRootRelativePath, rootRelativePathForAbsolutePath, } from "./paths.js";
 import { directoryDistance, genericPathKind, isTestPath, normalizedPathStem, } from "./path-signals.js";
-import { isBlockingSecretFinding, redactHighConfidenceAssignmentSecrets, scanHighConfidenceSecretContent, scanSecretPath, } from "./secrets.js";
+import { buildContextReviewObligations, planRegressionHeadSliceCandidates, } from "./review-coverage.js";
+import { sanitizeHighConfidenceSecretContent, scanSecretPath, } from "./secrets.js";
 import { CONTEXT_BUILDER_PUBLIC_BUDGET_HARD_LIMITS, } from "./schema.js";
 const SEARCH_WINDOW_LINES_BEFORE = 19;
 const SEARCH_WINDOW_LINES_AFTER = 20;
@@ -16,13 +18,32 @@ const SEARCH_BRIDGE_GAP_LINES = 20;
 const SEARCH_MAX_MERGED_SPAN_LINES = 160;
 const DEFAULT_MAX_FULL_FOCUS_SLICE_BYTES = 48_000;
 const DEFAULT_MAX_FULL_IMPLEMENTATION_FOCUS_SLICE_BYTES = 96_000;
+// Non-silent, material signal for an explicitly-focused file that exceeded the
+// bounded full-file focus size and had no per-file search evidence: instead of
+// dropping it, we include a bounded head slice and record this reason so the
+// receipt reflects a partial inclusion (not a silent omission).
+const CONTEXT_BUILDER_FOCUS_SLICED_REASON = "focus_sliced";
+// Non-silent, material signal for an explicitly-focused file whose FIRST logical
+// line alone exceeds the bounded focus-slice cap. The line-range model cannot
+// express a partial line, so emitting a head slice would blow the byte bound.
+// We omit the file with this explicit material reason instead — honoring both
+// the byte bound and "never silently drop an explicitly-focused file".
+const CONTEXT_BUILDER_FOCUS_OVER_CAP_REASON = "focus_over_cap";
+// Secret sanitization can run multi-second synchronous stretches. The walk
+// yields a macrotask at least this often (and after every sanitize) so timers,
+// cancellation notifications, and lock heartbeats keep running mid-scan.
+const DISCOVERY_YIELD_EVERY_FILES = 32;
 const IMPORT_EXPANSION_PASSES = 3;
+const MAX_EXPANDED_SEARCH_TERMS = 128;
+const MAX_EXPANDED_SEARCH_TERM_BYTES = 16_384;
+const MAX_DERIVED_SEARCH_TERM_LENGTH = 256;
 const PROJECT_SUPPORT_FILENAMES = Object.freeze([
     "package.json",
     "vitest.config.ts",
     "vite.config.ts",
 ]);
 export async function discoverContextBuilderCandidates(options) {
+    const allowSecrets = options.allowSecrets ?? true;
     const internalCaps = options.internal_caps ?? CONTEXT_BUILDER_INTERNAL_CAP_DEFAULTS;
     const matcher = compileContextBuilderGlobs({
         include: [...(options.include_globs ?? [])],
@@ -37,6 +58,8 @@ export async function discoverContextBuilderCandidates(options) {
         omissions: [],
         requestedEvidenceOmissionKeys: new Set(),
         processedFileKeys: new Set(),
+        traversedDirectoryKeys: new Set(),
+        focusScanExemptFiles: 0,
         metrics: {
             files_seen: 0,
             files_considered: 0,
@@ -48,6 +71,9 @@ export async function discoverContextBuilderCandidates(options) {
             bytes_read: 0,
         },
         stopped: false,
+        onProgress: options.onProgress,
+        filesSinceYield: 0,
+        filesSinceProgress: 0,
     };
     for (const root of options.roots) {
         if (state.stopped) {
@@ -57,6 +83,7 @@ export async function discoverContextBuilderCandidates(options) {
         await processExactFocusFiles({
             root,
             focusPaths: options.focus_paths ?? [],
+            allowSecrets,
             matcher,
             budget: options.budget,
             internalCaps,
@@ -70,12 +97,31 @@ export async function discoverContextBuilderCandidates(options) {
             },
             state,
         });
+        await processFocusDirectories({
+            root,
+            focusPaths: options.focus_paths ?? [],
+            allowSecrets,
+            matcher,
+            budget: options.budget,
+            internalCaps,
+            excludedRealRoots: options.excluded_real_roots ?? [],
+            ...(options.signal == null ? {} : { signal: options.signal }),
+            fs: {
+                readdir: options.fs?.readdir ?? fsReaddir,
+                lstat: options.fs?.lstat ?? fsLstat,
+                open: options.fs?.open ?? fsOpen,
+                realpath: options.fs?.realpath ?? fsRealpath,
+                stat: options.fs?.stat ?? fsStat,
+            },
+            state,
+        });
         await traverseDirectory({
             root,
             absoluteDir: root.path,
             relativeDir: "",
             matcher,
             focusPaths: options.focus_paths ?? [],
+            allowSecrets,
             budget: options.budget,
             internalCaps,
             excludedRealRoots: options.excluded_real_roots ?? [],
@@ -93,6 +139,7 @@ export async function discoverContextBuilderCandidates(options) {
             root,
             matcher,
             focusPaths: options.focus_paths ?? [],
+            allowSecrets,
             budget: options.budget,
             internalCaps,
             ...(options.signal == null ? {} : { signal: options.signal }),
@@ -108,11 +155,12 @@ export async function discoverContextBuilderCandidates(options) {
     }
     state.tree.sort(compareTreeFiles);
     state.contentCandidates.sort(compareContentCandidates);
+    const expandedSearchTerms = expandSearchTerms([...(options.search_terms ?? [])]);
     const search = buildCandidateSearchRecords({
         candidates: state.contentCandidates,
-        ...(options.task == null ? {} : { task: options.task }),
         focusPaths: [...(options.focus_paths ?? [])],
-        searchTerms: expandSearchTerms([...(options.search_terms ?? [])]),
+        searchTerms: expandedSearchTerms.terms,
+        callerTermCount: expandedSearchTerms.callerTermCount,
         maxSearchResults: options.budget.max_search_results,
         maxPreviewBytes: internalCaps.max_search_preview_bytes,
         maxCandidateBytes: internalCaps.max_candidate_bytes,
@@ -135,6 +183,50 @@ export async function discoverContextBuilderCandidates(options) {
     if (slices.capHit) {
         state.metrics.candidate_cap_hit = true;
     }
+    const reviewFocusPaths = (options.focus_paths ?? []).map((focus) => ({
+        root_index: focus.root_index,
+        path: focus.root_relative_path,
+        kind: focus.kind,
+    }));
+    let reviewObligations = buildContextReviewObligations({
+        candidates: state.contentCandidates,
+        focusPaths: reviewFocusPaths,
+        searchRecords: search.records,
+        slices: slices.records,
+        policy: options.coverage_policy,
+    });
+    if (reviewObligations != null) {
+        const headSlices = buildRegressionHeadSliceRecords({
+            candidates: state.contentCandidates,
+            planned: planRegressionHeadSliceCandidates({
+                candidates: state.contentCandidates,
+                searchRecords: search.records,
+                slices: slices.records,
+                obligations: reviewObligations,
+            }),
+            existingRecords: slices.records,
+            maxCandidateSlices: internalCaps.max_candidate_slices,
+            maxCandidateBytes: Math.max(0, internalCaps.max_candidate_bytes - search.candidateBytes),
+            maxBytesPerFile: options.budget.max_bytes_per_file,
+        });
+        if (headSlices.length > 0) {
+            slices.records.push(...headSlices);
+            reviewObligations = buildContextReviewObligations({
+                candidates: state.contentCandidates,
+                focusPaths: reviewFocusPaths,
+                searchRecords: search.records,
+                slices: slices.records,
+                policy: options.coverage_policy,
+            });
+        }
+    }
+    classifyContextEvidenceCandidates({
+        candidates: state.contentCandidates,
+        focusPaths: options.focus_paths ?? [],
+        searchRecords: search.records,
+        policy: options.coverage_policy,
+        obligations: reviewObligations,
+    });
     auditScanCapFocusedPaths({
         focusPaths: [...(options.focus_paths ?? [])],
         state,
@@ -146,30 +238,86 @@ export async function discoverContextBuilderCandidates(options) {
         slices: slices.records,
         omissions: state.omissions,
         metrics: state.metrics,
+        search_coverage: search.coverage,
+        ...(reviewObligations == null ? {} : { review_obligations: reviewObligations }),
     };
 }
-function expandSearchTerms(searchTerms) {
+export function expandSearchTerms(searchTerms) {
     const expanded = [];
     const seen = new Set();
+    let aggregateBytes = 0;
+    // Caller terms are admitted first and are never displaced by heuristic
+    // expansion. Public post-parse validation owns their individual/aggregate
+    // bounds; this function only bounds derived fan-out before stream creation.
+    for (const term of searchTerms) {
+        const added = addExpandedSearchTerm(expanded, seen, term);
+        aggregateBytes += added;
+    }
+    const callerTermCount = expanded.length;
     for (const term of searchTerms) {
-        addExpandedSearchTerm(expanded, seen, term);
         for (const derived of deriveGenericSearchTerms(term)) {
-            addExpandedSearchTerm(expanded, seen, derived);
+            const normalized = derived.trim();
+            if (normalized.length > MAX_DERIVED_SEARCH_TERM_LENGTH ||
+                expanded.length >= MAX_EXPANDED_SEARCH_TERMS) {
+                continue;
+            }
+            const bytes = Buffer.byteLength(normalized, "utf8");
+            if (aggregateBytes + bytes > MAX_EXPANDED_SEARCH_TERM_BYTES) {
+                continue;
+            }
+            aggregateBytes += addExpandedSearchTerm(expanded, seen, normalized);
         }
     }
-    return expanded;
+    return { terms: expanded, callerTermCount };
+}
+function classifyContextEvidenceCandidates(input) {
+    if (input.policy?.id == null) {
+        return;
+    }
+    const directSearchKeys = new Set(input.searchRecords.map(candidateKey));
+    const focusedFileKeys = new Set(input.focusPaths
+        .filter((focus) => focus.kind === "file")
+        .map((focus) => candidateKey({
+        root_index: focus.root_index,
+        path: focus.root_relative_path,
+    })));
+    const targetCandidateIds = new Set(input.obligations?.targets.map((target) => target.candidate_id) ?? []);
+    const regressionCandidateIds = new Set(input.obligations?.targets.flatMap((target) => target.regression_links.map((link) => link.candidate_id)) ?? []);
+    for (const candidate of input.candidates) {
+        const classes = [];
+        const key = candidateKey(candidate);
+        const packetCandidateId = `c:${String(candidate.root_index)}:${candidate.path}`;
+        const isReviewTarget = targetCandidateIds.has(packetCandidateId);
+        if (isReviewTarget) {
+            classes.push("review_target");
+        }
+        if (isTestPath(candidate.path) && regressionCandidateIds.has(packetCandidateId)) {
+            classes.push("regression_test");
+        }
+        const hasDiscoveryEvidence = directSearchKeys.has(key) ||
+            focusedFileKeys.has(key) ||
+            candidate.selection_hint != null;
+        if (genericPathKind(candidate.path) === "config" && hasDiscoveryEvidence) {
+            classes.push("configuration");
+        }
+        if (candidate.selection_hint === "local_dependency") {
+            classes.push("dependency");
+        }
+        candidate.evidence_classes = classes;
+    }
 }
 function addExpandedSearchTerm(expanded, seen, term) {
     const normalized = term.trim();
     if (normalized.length === 0) {
-        return;
+        return 0;
     }
     const key = normalized.toLowerCase();
     if (seen.has(key)) {
-        return;
+        return 0;
     }
     expanded.push(normalized);
     seen.add(key);
+    return Buffer.byteLength(normalized, "utf8");
 }
 function deriveGenericSearchTerms(term) {
     const tokens = tokenizeGenericSearchTerm(term);
@@ -238,6 +386,7 @@ async function processImportAndSupportFiles(input) {
                 relativePath,
                 matcher: input.matcher,
                 focusPaths: input.focusPaths,
+                allowSecrets: input.allowSecrets,
                 budget: input.budget,
                 internalCaps: input.internalCaps,
                 ignoreIncludeGlob: true,
@@ -264,6 +413,7 @@ async function processImportAndSupportFiles(input) {
             relativePath,
             matcher: input.matcher,
             focusPaths: input.focusPaths,
+            allowSecrets: input.allowSecrets,
             budget: input.budget,
             internalCaps: input.internalCaps,
             ignoreIncludeGlob: true,
@@ -477,9 +627,51 @@ async function processExactFocusFiles(input) {
             relativePath: focus.root_relative_path,
             matcher: input.matcher,
             focusPaths: input.focusPaths,
+            allowSecrets: input.allowSecrets,
             budget: input.budget,
             internalCaps: input.internalCaps,
-            ignoreScanCap: true,
+            focusScanExempt: true,
+            ...(input.signal == null ? {} : { signal: input.signal }),
+            fs: input.fs,
+            state: input.state,
+        });
+    }
+}
+// Pre-walk explicitly-focused directories before global traversal so a focus
+// directory late in sort order cannot be starved by max_scan_files exhaustion
+// from unrelated trees. Files processed here are focus-scan-exempt (they never
+// erode traversal headroom) but remain bounded by the discovery byte and
+// candidate caps; glob/secret/binary semantics are unchanged because the walk
+// reuses traverseDirectory/processFile verbatim.
+async function processFocusDirectories(input) {
+    const focusDirectories = [...input.focusPaths]
+        .filter((focus) => focus.root_index === input.root.root_index)
+        .filter((focus) => focus.kind === "directory")
+        .sort(compareFocusPathsForDiscovery);
+    for (const focus of focusDirectories) {
+        assertNotAborted(input.signal);
+        // A root-level focus gains nothing from a pre-walk (global traversal
+        // starts there immediately), and pre-walking it would exempt the entire
+        // root from max_scan_files.
+        if (focus.root_relative_path === "") {
+            continue;
+        }
+        // Leave include-glob-pruned focus directories to global traversal, which
+        // emits the existing requested-evidence omission for them.
+        if (!input.matcher.canContainIncludedPath(focus.root_relative_path)) {
+            continue;
+        }
+        await traverseDirectory({
+            root: input.root,
+            absoluteDir: focus.absolute_path,
+            relativeDir: focus.root_relative_path,
+            matcher: input.matcher,
+            focusPaths: input.focusPaths,
+            allowSecrets: input.allowSecrets,
+            budget: input.budget,
+            internalCaps: input.internalCaps,
+            excludedRealRoots: input.excludedRealRoots,
+            focusScanExempt: true,
             ...(input.signal == null ? {} : { signal: input.signal }),
             fs: input.fs,
             state: input.state,
@@ -495,14 +687,41 @@ export function buildCandidateSearchRecords(input) {
     let previewBytes = 0;
     let candidateBytes = 0;
     let capHit = false;
-    if (input.searchTerms.length === 0) {
-        return { records, capHit, candidateBytes };
-    }
+    const callerTermCount = Math.max(0, Math.min(input.callerTermCount ?? 0, input.searchTerms.length));
     const searchTerms = input.searchTerms
         .map((term, index) => ({ term, index }))
         .filter(({ term }) => term.trim().length > 0);
+    const callerTerms = searchTerms.filter(({ index }) => index < callerTermCount);
+    const derivedTerms = searchTerms.filter(({ index }) => index >= callerTermCount);
+    // Candidates whose content was examined for at least one term before a hard
+    // cap ended the walk. Key set is bounded by the candidate pool; only counts
+    // are surfaced.
+    const searchedCandidateKeys = new Set();
+    const callerTermCoverage = callerTerms.map(({ term, index }) => ({
+        term,
+        search_term_index: index,
+        hit_count: 0,
+        scan_complete: false,
+    }));
+    let callerRecordCount = 0;
+    const finalize = () => ({
+        records,
+        capHit,
+        candidateBytes,
+        coverage: {
+            eligible_candidate_count: input.candidates.length,
+            searched_candidate_count: searchedCandidateKeys.size,
+            unsearched_candidate_count: input.candidates.length - searchedCandidateKeys.size,
+            caller_term_count: callerTerms.length,
+            derived_term_count: derivedTerms.length,
+            caller_record_count: callerRecordCount,
+            derived_record_count: records.length - callerRecordCount,
+            caller_coverage_complete: callerTermCoverage.every((coverage) => coverage.scan_complete),
+            caller_terms: callerTermCoverage,
+        },
+    });
     if (searchTerms.length === 0) {
-        return { records, capHit, candidateBytes };
+        return finalize();
     }
     const focusRank = new Map();
     for (const focus of input.focusPaths ?? []) {
@@ -521,70 +740,154 @@ export function buildCandidateSearchRecords(input) {
         const leftRank = focusRank.get(candidateKey(left)) ?? Number.MAX_SAFE_INTEGER;
         const rightRank = focusRank.get(candidateKey(right)) ?? Number.MAX_SAFE_INTEGER;
         return (leftRank - rightRank ||
-            discoveryTaskClosureRank(left.path, input.task ?? "") -
-                discoveryTaskClosureRank(right.path, input.task ?? "") ||
             compareContentCandidates(left, right));
     });
-    const streams = [];
+    const candidateLines = [];
     for (const candidate of candidates) {
         assertNotAborted(input.signal);
-        const lines = splitTextLines(candidate.content);
-        for (const searchTerm of searchTerms) {
-            streams.push({
+        candidateLines.push(splitTextLines(candidate.content));
+    }
+    // Emits one found hit unless a hard cap rejects it. Cap semantics match the
+    // pre-phasing scheduler: the record/byte caps are checked after the hit is
+    // found, and tripping either one ends the whole search with capHit set.
+    const emitSearchHit = (stream, hit) => {
+        if (records.length >= input.maxSearchResults) {
+            capHit = true;
+            return false;
+        }
+        const preview = boundedPreview(hit.line, hit.columnIndex, stream.searchTerm.term.length);
+        const previewSize = Buffer.byteLength(preview, "utf8");
+        const bounded = previewBytes + previewSize <= maxPreviewBytes ? preview : "";
+        const record = {
+            root_index: stream.candidate.root_index,
+            path: stream.candidate.path,
+            line: hit.lineIndex + 1,
+            column: hit.columnIndex + 1,
+            search_term: stream.searchTerm.term,
+            search_term_index: stream.searchTerm.index,
+            preview: bounded,
+        };
+        const recordBytes = Buffer.byteLength(`${JSON.stringify(record)}\n`, "utf8");
+        if (candidateBytes + recordBytes > maxCandidateBytes) {
+            capHit = true;
+            return false;
+        }
+        previewBytes += Buffer.byteLength(bounded, "utf8");
+        candidateBytes += recordBytes;
+        records.push(record);
+        return true;
+    };
+    if (callerTerms.length > 0) {
+        // Phase 1 — exact caller terms get first claim on the record budget. One
+        // cursor per caller term walks the complete candidate list (focus rank
+        // first, one hit per candidate per pass), and the scheduler round-robins
+        // across the term cursors, so neither a common sibling term nor an early
+        // alphabetical candidate prefix can starve a rare late-path caller hit.
+        const cursors = callerTermCoverage.map((coverage) => ({
+            coverage,
+            streams: candidates.map((candidate, candidateIndex) => ({
                 candidate,
-                searchTerm,
-                lines,
+                searchTerm: { term: coverage.term, index: coverage.search_term_index },
+                lines: candidateLines[candidateIndex] ?? [],
                 lineIndex: 0,
                 fromIndex: 0,
                 done: false,
-            });
+            })),
+            cursor: 0,
+            active: candidates.length,
+            exhausted: false,
+        }));
+        let liveCursors = cursors.length;
+        while (liveCursors > 0) {
+            for (const cursor of cursors) {
+                assertNotAborted(input.signal);
+                if (cursor.exhausted) {
+                    continue;
+                }
+                const next = nextCallerTermHit(cursor, searchedCandidateKeys);
+                if (next == null) {
+                    // Natural exhaustion: this term's cursor examined every candidate.
+                    cursor.exhausted = true;
+                    cursor.coverage.scan_complete = true;
+                    liveCursors--;
+                    continue;
+                }
+                if (!emitSearchHit(next.stream, next.hit)) {
+                    return finalize();
+                }
+                cursor.coverage.hit_count++;
+                callerRecordCount++;
+            }
         }
     }
-    let activeStreams = streams.length;
-    while (activeStreams > 0) {
-        let foundInRound = false;
-        for (const stream of streams) {
+    if (derivedTerms.length > 0) {
+        // Phase 2 — derived/heuristic terms spend whatever budget remains, using
+        // the pre-existing candidate-major one-hit-per-stream round-robin.
+        const streams = [];
+        for (const [candidateIndex, candidate] of candidates.entries()) {
             assertNotAborted(input.signal);
-            if (stream.done) {
-                continue;
-            }
-            const hit = nextSearchHit(stream);
-            if (hit == null) {
-                stream.done = true;
-                activeStreams--;
-                continue;
+            const lines = candidateLines[candidateIndex] ?? [];
+            for (const searchTerm of derivedTerms) {
+                streams.push({
+                    candidate,
+                    searchTerm,
+                    lines,
+                    lineIndex: 0,
+                    fromIndex: 0,
+                    done: false,
+                });
             }
-            foundInRound = true;
-            if (records.length >= input.maxSearchResults) {
-                capHit = true;
-                return { records, capHit, candidateBytes };
+        }
+        let activeStreams = streams.length;
+        while (activeStreams > 0) {
+            let foundInRound = false;
+            for (const stream of streams) {
+                assertNotAborted(input.signal);
+                if (stream.done) {
+                    continue;
+                }
+                searchedCandidateKeys.add(candidateKey(stream.candidate));
+                const hit = nextSearchHit(stream);
+                if (hit == null) {
+                    stream.done = true;
+                    activeStreams--;
+                    continue;
+                }
+                foundInRound = true;
+                if (!emitSearchHit(stream, hit)) {
+                    return finalize();
+                }
             }
-            const preview = boundedPreview(hit.line, hit.columnIndex, stream.searchTerm.term.length);
-            const previewSize = Buffer.byteLength(preview, "utf8");
-            const bounded = previewBytes + previewSize <= maxPreviewBytes ? preview : "";
-            const record = {
-                root_index: stream.candidate.root_index,
-                path: stream.candidate.path,
-                line: hit.lineIndex + 1,
-                column: hit.columnIndex + 1,
-                search_term: stream.searchTerm.term,
-                search_term_index: stream.searchTerm.index,
-                preview: bounded,
-            };
-            const recordBytes = Buffer.byteLength(`${JSON.stringify(record)}\n`, "utf8");
-            if (candidateBytes + recordBytes > maxCandidateBytes) {
-                capHit = true;
-                return { records, capHit, candidateBytes };
+            if (!foundInRound) {
+                break;
             }
-            previewBytes += Buffer.byteLength(bounded, "utf8");
-            candidateBytes += recordBytes;
-            records.push(record);
         }
-        if (!foundInRound) {
-            break;
+    }
+    return finalize();
+}
+// Advances one caller term's cursor to its next hit, cycling the candidate
+// list one hit per candidate per pass. Scanning is free — only emitted records
+// consume the budget — so a rare term reaches late-sorting candidates even
+// when earlier candidates are saturated with other terms' hits. Returns
+// undefined exactly when every candidate stream is exhausted.
+function nextCallerTermHit(cursor, searchedCandidateKeys) {
+    const total = cursor.streams.length;
+    for (let visited = 0; visited < total && cursor.active > 0; visited++) {
+        const stream = cursor.streams[cursor.cursor];
+        cursor.cursor = (cursor.cursor + 1) % total;
+        if (stream == null || stream.done) {
+            continue;
+        }
+        searchedCandidateKeys.add(candidateKey(stream.candidate));
+        const hit = nextSearchHit(stream);
+        if (hit == null) {
+            stream.done = true;
+            cursor.active--;
+            continue;
         }
+        return { stream, hit };
     }
-    return { records, capHit, candidateBytes };
+    return undefined;
 }
 export function buildCandidateSliceRecords(input) {
     const maxCandidateSlices = input.maxCandidateSlices ??
@@ -603,7 +906,7 @@ export function buildCandidateSliceRecords(input) {
     const expectedFocusedSearchRanges = new Map();
     const survivedFocusedSearchRanges = new Map();
     const focusSliceSurvivors = new Set();
-    const omittedLargeFocusedCandidates = new Set();
+    const slicedLargeFocusedCandidates = new Set();
     const omittedRequestedCandidates = new Set();
     const pushRequestedCandidateOmission = (candidate, kind, message) => {
         const key = candidateKey(candidate);
@@ -701,12 +1004,8 @@ export function buildCandidateSliceRecords(input) {
                     (focus.kind === "directory" &&
                         isRootRelativeDescendant(candidate.path, focus.root_relative_path)))) {
                 const key = candidateKey(candidate);
-                if (candidate.byte_length > fullFocusSliceByteLimit(candidate.path, maxFullFocusSliceBytes)) {
-                    if (!searchRanges.has(key) && !omittedLargeFocusedCandidates.has(key)) {
-                        pushRequestedCandidateOmission(candidate, "too_large", "focused file exceeded bounded full-file focus size and had no per-file search evidence");
-                        omittedLargeFocusedCandidates.add(key);
-                    }
-                    else if (searchRanges.has(key)) {
+                if (candidate.byte_length > maxFullFocusSliceBytes) {
+                    if (searchRanges.has(key)) {
                         focusedLargeSearchCandidates.set(key, candidate);
                         expectedFocusedSearchRanges.set(key, mergeLineRanges(searchRanges.get(key) ?? [], {
                             bridgeGapLines: SEARCH_BRIDGE_GAP_LINES,
@@ -714,6 +1013,27 @@ export function buildCandidateSliceRecords(input) {
                         }));
                         survivedFocusedSearchRanges.set(key, []);
                     }
+                    else if (!slicedLargeFocusedCandidates.has(key)) {
+                        // Never silently drop an explicitly-focused file. With no per-file
+                        // search evidence, include a bounded head slice up to the focus
+                        // byte cap and record a material, non-silent partial-focus signal
+                        // instead of a too_large omission.
+                        if (firstLineExceedsCap(candidate.content, maxFullFocusSliceBytes)) {
+                            // The first logical line alone exceeds the cap (e.g. a minified
+                            // single-line file). The line-range model cannot express a partial
+                            // line, so a head slice would blow the byte bound. Omit the file
+                            // with an explicit material reason rather than exceeding the bound.
+                            omissions.push(focusOverCapNotice(candidate, maxFullFocusSliceBytes));
+                            slicedLargeFocusedCandidates.add(key);
+                            continue;
+                        }
+                        const headEndLine = headSliceEndLine(candidate.content, maxFullFocusSliceBytes);
+                        const ranges = focusRanges.get(key) ?? [];
+                        ranges.push({ start_line: 1, end_line: headEndLine });
+                        focusRanges.set(key, ranges);
+                        omissions.push(focusSlicedNotice(candidate, headEndLine));
+                        slicedLargeFocusedCandidates.add(key);
+                    }
                     continue;
                 }
                 const ranges = focusRanges.get(candidateKey(candidate)) ?? [];
@@ -735,6 +1055,30 @@ export function buildCandidateSliceRecords(input) {
             { start_line: 1, end_line: Math.max(1, candidate.line_count) },
         ]);
     }
+    // Focus slices are appended FIRST so max_candidate_slices/max_candidate_bytes
+    // pressure prunes search and dependency material before explicitly-focused
+    // ranges — never the other way around.
+    for (const [key, ranges] of [...focusRanges.entries()].sort(([a], [b]) => a.localeCompare(b))) {
+        assertNotAborted(input.signal);
+        const candidate = candidateByIdentity.get(key);
+        if (candidate == null) {
+            continue;
+        }
+        for (const range of mergeLineRanges(ranges)) {
+            if (!append({
+                root_index: candidate.root_index,
+                path: candidate.path,
+                start_line: range.start_line,
+                end_line: range.end_line,
+                source: "focus",
+                content: sliceContent(candidate.content, range.start_line, range.end_line),
+            })) {
+                pushMissingFocusedRangeOmissions(focusRanges.keys());
+                pushMissingFocusedSearchOmissions(focusedLargeSearchCandidates.keys());
+                return { records, capHit, omissions };
+            }
+        }
+    }
     for (const [key, ranges] of [...searchRanges.entries()].sort(([a], [b]) => a.localeCompare(b))) {
         assertNotAborted(input.signal);
         const candidate = candidateByIdentity.get(key);
@@ -787,27 +1131,52 @@ export function buildCandidateSliceRecords(input) {
             }
         }
     }
-    for (const [key, ranges] of [...focusRanges.entries()].sort(([a], [b]) => a.localeCompare(b))) {
-        assertNotAborted(input.signal);
-        const candidate = candidateByIdentity.get(key);
-        if (candidate == null) {
+    return { records, capHit, omissions };
+}
+// Head-slice line bound for synthesized regression evidence: enough to show a
+// test's imports and first cases without competing with real search windows.
+const REGRESSION_HEAD_SLICE_MAX_LINES = 60;
+/**
+ * Materializes the tests chosen by `planRegressionHeadSliceCandidates` as real
+ * slice records so selection, reconstruction, and rendering can carry them.
+ * Source is `search` WITHOUT `search_term_index`: every slice consumer already
+ * defaults a missing index to the lowest rank, and `focus` would wrongly mark
+ * the test as caller-requested evidence. Content comes from the already
+ * sanitized content candidate, so `allow_secrets:false` redaction applies
+ * exactly as for other slices. Candidate slice/byte caps still bound the
+ * total; a slice that does not fit is simply not synthesized.
+ */
+function buildRegressionHeadSliceRecords(input) {
+    const candidateByIdentity = new Map(input.candidates.map((candidate) => [candidateKey(candidate), candidate]));
+    let count = input.existingRecords.length;
+    let bytes = input.existingRecords.reduce((total, record) => total + Buffer.byteLength(`${JSON.stringify(record)}\n`, "utf8"), 0);
+    const records = [];
+    for (const planned of input.planned) {
+        const candidate = candidateByIdentity.get(candidateKey(planned));
+        if (candidate == null ||
+            candidate.line_count === 0 ||
+            firstLineExceedsCap(candidate.content, input.maxBytesPerFile)) {
             continue;
         }
-        for (const range of mergeLineRanges(ranges)) {
-            if (!append({
-                root_index: candidate.root_index,
-                path: candidate.path,
-                start_line: range.start_line,
-                end_line: range.end_line,
-                source: "focus",
-                content: sliceContent(candidate.content, range.start_line, range.end_line),
-            })) {
-                pushMissingFocusedRangeOmissions(focusRanges.keys());
-                return { records, capHit, omissions };
-            }
+        const endLine = Math.min(candidate.line_count, REGRESSION_HEAD_SLICE_MAX_LINES, headSliceEndLine(candidate.content, input.maxBytesPerFile));
+        const record = {
+            root_index: candidate.root_index,
+            path: candidate.path,
+            start_line: 1,
+            end_line: endLine,
+            source: "search",
+            content: sliceContent(candidate.content, 1, endLine),
+        };
+        const recordBytes = Buffer.byteLength(`${JSON.stringify(record)}\n`, "utf8");
+        if (count >= input.maxCandidateSlices ||
+            bytes + recordBytes > input.maxCandidateBytes) {
+            continue;
         }
+        records.push(record);
+        count++;
+        bytes += recordBytes;
     }
-    return { records, capHit, omissions };
+    return records;
 }
 export function splitTextLines(text) {
     if (text.length === 0) {
@@ -859,11 +1228,55 @@ export function mergeLineRanges(ranges, options = {}) {
     }
     return merged;
 }
-function fullFocusSliceByteLimit(path, maxFullFocusSliceBytes) {
-    if (isTestPath(path)) {
-        return Math.min(maxFullFocusSliceBytes, DEFAULT_MAX_FULL_FOCUS_SLICE_BYTES);
+// Largest 1-based line count whose cumulative UTF-8 bytes (lines joined by
+// "\n", matching sliceContent) stay within capBytes. Always returns >= 1 so an
+// explicitly-focused file is never reduced to nothing.
+// CALLER COUPLING: line 1 is admitted unconditionally (the `count > 0 &&` guard
+// skips the cap check for the first line), so callers MUST pre-screen with
+// firstLineExceedsCap — otherwise an over-cap first line silently breaks the
+// byte bound.
+function headSliceEndLine(content, capBytes) {
+    const lines = splitTextLines(content);
+    let bytes = 0;
+    let count = 0;
+    for (const line of lines) {
+        const next = bytes + (count === 0 ? 0 : 1) + Buffer.byteLength(line, "utf8");
+        if (count > 0 && next > capBytes) {
+            break;
+        }
+        bytes = next;
+        count++;
+    }
+    return Math.max(1, count);
+}
+// True when even the first logical line's own UTF-8 byte length exceeds capBytes,
+// i.e. no head slice (which always covers at least line 1) can fit within the cap.
+function firstLineExceedsCap(content, capBytes) {
+    const lines = splitTextLines(content);
+    const first = lines[0];
+    if (first == null) {
+        return false;
     }
-    return maxFullFocusSliceBytes;
+    return Buffer.byteLength(first, "utf8") > capBytes;
+}
+function focusOverCapNotice(candidate, capBytes) {
+    return {
+        root_index: candidate.root_index,
+        reason: CONTEXT_BUILDER_FOCUS_OVER_CAP_REASON,
+        message: "focused file's first line alone exceeded the bounded focus-slice cap " +
+            `(${capBytes} bytes); omitted rather than exceeding the byte bound`,
+        paths: [candidate.path],
+    };
+}
+function focusSlicedNotice(candidate, headEndLine) {
+    return {
+        root_index: candidate.root_index,
+        reason: CONTEXT_BUILDER_FOCUS_SLICED_REASON,
+        message: "focused file exceeded bounded full-file focus size and had no per-file " +
+            `search evidence; included a bounded head slice (lines 1-${headEndLine}) ` +
+            "instead of dropping it",
+        paths: [candidate.path],
+    };
 }
 function focusedSearchRangesCovered(expected, survived) {
     return expected.every((range) => rangeCoveredByRanges(range, survived));
@@ -889,6 +1302,14 @@ async function traverseDirectory(input) {
     if (input.state.stopped) {
         return;
     }
+    const directoryKey = candidateKey({
+        root_index: input.root.root_index,
+        path: input.relativeDir,
+    });
+    if (input.state.traversedDirectoryKeys.has(directoryKey)) {
+        return;
+    }
+    input.state.traversedDirectoryKeys.add(directoryKey);
     if (input.relativeDir !== "" &&
         (input.matcher.excludes(input.relativeDir) ||
             input.matcher.excludes(`${input.relativeDir}/__dir__`))) {
@@ -1025,6 +1446,35 @@ function pushDirectoryOmission(input, omission) {
     });
 }
 async function processFile(input) {
+    await processFileEntry(input);
+    await settleDiscoveryFileBoundary(input.state, input.signal);
+}
+async function settleDiscoveryFileBoundary(state, signal) {
+    state.filesSinceProgress++;
+    if (state.onProgress != null &&
+        state.filesSinceProgress >= CONTEXT_BUILDER_SCAN_PROGRESS_EVERY_FILES) {
+        state.filesSinceProgress = 0;
+        try {
+            state.onProgress(Object.freeze({ ...state.metrics }));
+        }
+        catch {
+            // Progress is observability only; a failing sink must never stop the walk.
+        }
+    }
+    assertNotAborted(signal);
+    state.filesSinceYield++;
+    if (state.filesSinceYield >= DISCOVERY_YIELD_EVERY_FILES) {
+        await yieldDiscoveryToEventLoop(state, signal);
+    }
+}
+async function yieldDiscoveryToEventLoop(state, signal) {
+    state.filesSinceYield = 0;
+    await new Promise((resolve) => {
+        setImmediate(resolve);
+    });
+    assertNotAborted(signal);
+}
+async function processFileEntry(input) {
     const fileKey = candidateKey({
         root_index: input.root.root_index,
         path: input.relativePath,
@@ -1038,7 +1488,7 @@ async function processFile(input) {
     if (!alreadyProcessed) {
         input.state.processedFileKeys.add(fileKey);
     }
-    const omitted = (reason, message) => {
+    const omitted = (reason, message, secretFindings) => {
         input.state.omissions.push({
             root_index: input.root.root_index,
             reason,
@@ -1051,7 +1501,7 @@ async function processFile(input) {
             focusPaths: input.focusPaths,
             state: input.state,
             kind: requestedOmissionKindForDiscoveryReason(reason),
-            message: sanitizedFocusedOmissionMessage(reason),
+            message: sanitizedFocusedOmissionMessage(reason, secretFindings),
         });
     };
     const requestedOmitted = (kind, message) => {
@@ -1103,8 +1553,13 @@ async function processFile(input) {
         });
         return;
     }
+    // files_seen stays accurate for observability, but focus-processed files are
+    // subtracted from the cap check so explicit focus never consumes the
+    // max_scan_files headroom reserved for global traversal.
     if (input.ignoreScanCap !== true &&
-        input.state.metrics.files_seen >= input.budget.max_scan_files) {
+        input.focusScanExempt !== true &&
+        input.state.metrics.files_seen - input.state.focusScanExemptFiles >=
+            input.budget.max_scan_files) {
         input.state.metrics.scan_cap_hit = true;
         pushRequestedFocusOmission({
             root_index: input.root.root_index,
@@ -1118,9 +1573,12 @@ async function processFile(input) {
         return;
     }
     input.state.metrics.files_seen++;
-    const pathSecrets = scanSecretPath(input.relativePath);
+    if (input.focusScanExempt === true) {
+        input.state.focusScanExemptFiles++;
+    }
+    const pathSecrets = input.allowSecrets ? [] : scanSecretPath(input.relativePath);
     if (pathSecrets.length > 0) {
-        omitted("secret_path", `Secret-looking path omitted (${pathSecrets[0]?.provenance})`);
+        omitted("secret_path", `Secret-looking path omitted (${pathSecrets[0]?.provenance})`, pathSecrets);
         pushTree({
             eligible: false,
             content_eligible: false,
@@ -1235,10 +1693,15 @@ async function processFile(input) {
     }
     const rawContent = contentBuffer.toString("utf8");
     input.state.metrics.bytes_read += contentBuffer.byteLength;
-    const contentSecrets = scanHighConfidenceSecretContent(rawContent);
-    const blockingSecrets = contentSecrets.filter(isBlockingSecretFinding);
-    if (blockingSecrets.length > 0) {
-        omitted("content_secret", `Content omitted due to high-confidence secret (${summarizeSecretFindings(blockingSecrets)})`);
+    const sanitized = input.allowSecrets
+        ? { status: "unchanged", content: rawContent, findings: [] }
+        : sanitizeHighConfidenceSecretContent(rawContent);
+    if (!input.allowSecrets) {
+        assertNotAborted(input.signal);
+        await yieldDiscoveryToEventLoop(input.state, input.signal);
+    }
+    if (sanitized.status === "blocked") {
+        omitted("content_secret", `Content omitted due to high-confidence secret (${summarizeSecretFindings(sanitized.findings)})`, sanitized.findings);
         pushTree({
             size_bytes: read.sizeBytes,
             eligible: true,
@@ -1247,9 +1710,7 @@ async function processFile(input) {
         });
         return;
     }
-    const content = contentSecrets.length > 0
-        ? redactHighConfidenceAssignmentSecrets(rawContent)
-        : rawContent;
+    const content = sanitized.content;
     const contentBytes = Buffer.byteLength(content, "utf8");
     if (input.state.contentCandidates.length >= input.internalCaps.max_discovery_candidates) {
         input.state.metrics.discovery_candidate_cap_hit = true;
@@ -1403,74 +1864,6 @@ function compareTreeFiles(left, right) {
 function compareContentCandidates(left, right) {
     return left.root_index - right.root_index || left.path.localeCompare(right.path);
 }
-function discoveryTaskClosureRank(path, task) {
-    const normalizedTask = task.toLowerCase();
-    if (normalizedTask.length === 0) {
-        return 0;
-    }
-    const taskTokens = new Set(tokenizeGenericSearchTerm(normalizedTask));
-    const normalizedPath = path.toLowerCase();
-    if (/\b(discover|discovery)\b.*\b(provider-specific|provider\s+semantics|providers?|claude|agent|symmetry)\b|\b(provider-specific|provider\s+semantics|providers?|claude|agent|symmetry)\b.*\b(discover|discovery)\b/.test(normalizedTask) &&
-        hasAnyToken(taskTokens, ["provider", "runtime", "claude", "agent", "symmetry"]) &&
-        [/\/claude_agent\.py$/, /\/runtime\/base\.py$/].some((pattern) => pattern.test(normalizedPath))) {
-        return -30;
-    }
-    if (/\b(discover|discovery)\b/.test(normalizedTask) &&
-        hasAnyToken(taskTokens, ["queue", "migration", "runtime", "schema", "setting", "selector"]) &&
-        [
-            /\/tasks\/chat_processor\.py$/,
-            /\/hooks\/usemessageactions\.ts$/,
-            /\/hooks\/usechatstreaming\.ts$/,
-            /\/hooks\/usechatruntimesettings\.ts$/,
-            /\/components\/chat\/chat-window\/chat\.tsx$/,
-            /\/components\/chat\/message-input\/input\.tsx$/,
-            /\/components\/chat\/message-input\/inputsettingspopover\.tsx$/,
-            /\/pages\/chatpage\.tsx$/,
-            /\/landingpage\.tsx$/,
-            /\/models\/db_models\/chat\.py$/,
-            /\/[^/]*(permission|thinking).*selector[^/]*\.(ts|tsx)$/,
-            /\/tests\/test_chat\.py$/,
-            /\/tests?\/.*runtime.*\.(py|ts|tsx)$/,
-        ].some((pattern) => pattern.test(normalizedPath))) {
-        return -24;
-    }
-    if (/\breview\b/.test(normalizedTask) &&
-        hasAnyToken(taskTokens, ["codex", "approval", "permission", "session", "runtime"]) &&
-        [
-            /\/api\/endpoints\/permissions\.py$/,
-            /\/services\/runtime\/codex_approval\.py$/,
-            /\/services\/permission_manager\.py$/,
-            /\/services\/chat\.py$/,
-            /\/api\/endpoints\/chat\.py$/,
-            /\/models\/db_models\/ai_model\.py$/,
-            /\/services\/chatservice\.ts$/,
-            /\/types\/chat\.types\.ts$/,
-            /\/core\/config\.py$/,
-            /\/constants\.py$/,
-            /\/hooks\/usepermissionrequest\.ts$/,
-            /\/tests\/test_permissions\.py$/,
-            /(^|\/)docs?\/.*(codex|approval|permission|session|runtime).*\.md$/,
-        ].some((pattern) => pattern.test(normalizedPath))) {
-        return -24;
-    }
-    if (/\b(debug|debugging|diagnose|diagnosis|diagnostic)\b/.test(normalizedTask) &&
-        hasAnyToken(taskTokens, ["devtool", "trace", "archive", "message"]) &&
-        [
-            /\/api\/endpoints\/devtools\.py$/,
-            /\/services\/session_trace_archive\.py$/,
-            /\/models\/db_models\/session_trace_archive\.py$/,
-            /\/.*trace.*archive.*\.test\.(ts|tsx|py)$/,
-            /\/tests?\/.*trace.*archive.*\.(py|ts|tsx)$/,
-            /\/messagerenderer\.tsx$/,
-            /\/devtools\.types\.ts$/,
-        ].some((pattern) => pattern.test(normalizedPath))) {
-        return -24;
-    }
-    return 0;
-}
-function hasAnyToken(tokens, candidates) {
-    return candidates.some((candidate) => tokens.has(candidate));
-}
 function compareSearchRecords(left, right) {
     return (left.root_index - right.root_index ||
         left.path.localeCompare(right.path) ||
@@ -1569,14 +1962,14 @@ function requestedEvidenceOmissionKey(rootIndex, relativePath) {
 function requestedOmissionKindForDiscoveryReason(reason) {
     return reason === "file_too_large" ? "too_large" : "unavailable";
 }
-function sanitizedFocusedOmissionMessage(reason) {
+function sanitizedFocusedOmissionMessage(reason, secretFindings) {
     switch (reason) {
         case "caller_include_glob":
             return "focused path was outside include globs";
         case "exclude_glob":
             return "focused path was omitted by exclude globs";
         case "secret_path":
-            return "focused path was omitted because it matched a secret-path rule";
+            return withSecretFindingProvenance("focused path was omitted because it matched a secret-path rule", secretFindings);
         case "symlink_omitted":
             return "focused path was omitted because symlinks are not followed";
         case "path_escape":
@@ -1591,13 +1984,32 @@ function sanitizedFocusedOmissionMessage(reason) {
         case "binary_file":
             return "focused binary file cannot be represented as text";
         case "content_secret":
-            return "focused file content was omitted because secret-like content was detected";
+            return withSecretFindingProvenance("focused file content was omitted because secret-like content was detected", secretFindings);
         case "file_too_large":
             return "focused file exceeded max_bytes_per_file";
         default:
             return "focused path could not be represented as local evidence";
     }
 }
+const MAX_SECRET_FINDING_PROVENANCE_DETAILS = 3;
+// Appends detector provenance labels and line numbers (never matched secret
+// text or values) to a sanitized focused-omission message, e.g.
+// "... (aws_access_key_id at line 12)".
+function withSecretFindingProvenance(base, secretFindings) {
+    const details = [];
+    for (const finding of secretFindings ?? []) {
+        const detail = finding.line == null
+            ? finding.provenance
+            : `${finding.provenance} at line ${finding.line}`;
+        if (!details.includes(detail)) {
+            details.push(detail);
+        }
+        if (details.length >= MAX_SECRET_FINDING_PROVENANCE_DETAILS) {
+            break;
+        }
+    }
+    return details.length === 0 ? base : `${base} (${details.join(", ")})`;
+}
 function candidateKey(candidate) {
     return `${candidate.root_index}\u0000${candidate.path}`;
 }