@iowarp/clio-coder 0.4.7 → 0.4.9

package/src/tools/spawn-hygiene.ts

@@ -1,6 +1,9 @@
+import { AsyncLocalStorage } from "node:async_hooks";
 import { spawn } from "node:child_process";
 import { createInterface } from "node:readline";
+import { StringDecoder } from "node:string_decoder";
 import { buildSafeToolEnv } from "../core/safe-exec.js";
+import { truncateUtf8 } from "./truncate-utf8.js";
 
 /**
  * Hygienic line-stream spawner for the search binaries (rg, fd). Matches the
@@ -13,7 +16,55 @@ import { buildSafeToolEnv } from "../core/safe-exec.js";
 
 export const SEARCH_SPAWN_TIMEOUT_MS = 30_000;
 const KILL_GRACE_MS = 3_000;
-const STDERR_CAP_BYTES = 16 * 1024;
+export const SEARCH_STDERR_CAP_BYTES = 16 * 1024;
+const timeoutContext = new AsyncLocalStorage<number>();
+
+/** Leave an incomplete trailing code point undecoded, and bound replacement text for invalid input. */
+function decodeBoundedUtf8(buffer: Buffer): string {
+	const decoded = new StringDecoder("utf8").write(buffer);
+	return truncateUtf8(decoded, SEARCH_STDERR_CAP_BYTES, "");
+}
+
+/** Scope a shorter process deadline without changing tool arguments or concurrent calls. */
+export function withSearchTimeout<T>(timeoutMs: number, run: () => T): T {
+	if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) throw new Error("Invalid search timeout");
+	return timeoutContext.run(timeoutMs, run);
+}
+
+/** Decode UTF-8 lines incrementally with one fixed-size byte buffer, including without newlines. */
+export class BoundedDiagnosticDecoder {
+	private readonly buffer = Buffer.alloc(SEARCH_STDERR_CAP_BYTES);
+	private used = 0;
+	private truncated = false;
+	oversizedLines = 0;
+	constructor(private readonly onLine: (line: string, truncated: boolean) => void) {}
+	get bufferedBytes(): number {
+		return this.used;
+	}
+	write(chunk: Buffer): void {
+		let start = 0;
+		while (start < chunk.length) {
+			const newline = chunk.indexOf(10, start);
+			const end = newline < 0 ? chunk.length : newline;
+			const take = Math.min(end - start, this.buffer.length - this.used);
+			chunk.copy(this.buffer, this.used, start, start + take);
+			this.used += take;
+			if (take < end - start) this.truncated = true;
+			if (newline < 0) break;
+			this.flush();
+			start = newline + 1;
+		}
+	}
+	end(): void {
+		if (this.used || this.truncated) this.flush();
+	}
+	private flush(): void {
+		if (this.truncated) this.oversizedLines++;
+		this.onLine(decodeBoundedUtf8(this.buffer.subarray(0, this.used)).replace(/\r$/, ""), this.truncated);
+		this.used = 0;
+		this.truncated = false;
+	}
+}
 
 /**
  * Per-argument byte ceiling for the search binaries. Linux caps a single argv
@@ -40,6 +91,96 @@ export function validateSearchPatternSize(pattern: string): { ok: true } | { ok:
 	};
 }
 
+export interface SearchCompleteness {
+	complete: boolean;
+	reason?: "timeout" | "errors" | "limit" | "cancelled";
+	skipped: { count: number; samples: string[]; unknown?: boolean };
+}
+
+export function newSearchCompleteness(): SearchCompleteness {
+	return { complete: true, skipped: { count: 0, samples: [] } };
+}
+
+export function skipSearchPath(
+	search: SearchCompleteness,
+	path: string,
+	allows: (path: string) => boolean = () => true,
+): void {
+	search.complete = false;
+	search.reason ??= "errors";
+	search.skipped.count += 1;
+	if (allows(path) && search.skipped.samples.length < 5) search.skipped.samples.push(path);
+}
+
+/** Parse the path prefix, independently of the language used for OS error text. */
+function searchDiagnosticPath(line: string): string | null {
+	const cleaned = line.replace(/^(?:rg: |\[fd error\]:? )/, "").replace(/^IO error for operation on /, "");
+	if (/^(?:regex parse error|error parsing (?:glob|regex)|invalid (?:glob|regex)|error:|Usage:)/i.test(cleaned))
+		return null;
+	// Skip a Windows drive colon. Diagnostics use a colon followed by whitespace
+	// or end of line; this also handles a path header followed by continuation lines.
+	const match = /^((?:[A-Za-z]:[\\/])?[^\r\n]+?):(?:\s|$)/.exec(cleaned);
+	return match?.[1] ?? null;
+}
+
+export function createSearchDiagnostics(search: SearchCompleteness, allows: (path: string) => boolean) {
+	let continuation = false;
+	let realError = false;
+	return {
+		get realError() {
+			return realError;
+		},
+		onLine(line: string, truncated = false): void {
+			if (truncated) {
+				search.skipped.unknown = true;
+				search.complete = false;
+				search.reason ??= "errors";
+			}
+			if (!line.trim()) return;
+			if (continuation && /^\s/.test(line)) return;
+			const path = searchDiagnosticPath(line);
+			if (path) {
+				skipSearchPath(search, path, allows);
+				continuation = true;
+				return;
+			}
+			if (continuation && /\(os error \d+\)/.test(line)) return;
+			continuation = false;
+			if (
+				/^(?:rg: |\[fd error\]:? )?(?:regex parse error|error parsing (?:glob|regex)|invalid (?:glob|regex)|error:|Usage:)/i.test(
+					line,
+				)
+			)
+				realError = true;
+			else search.skipped.unknown = true;
+			search.complete = false;
+			search.reason ??= "errors";
+		},
+	};
+}
+
+export function finishSearch(search: SearchCompleteness, result: LineStreamResult): void {
+	const reason = result.aborted
+		? "cancelled"
+		: result.timedOut
+			? "timeout"
+			: result.stoppedEarly
+				? "limit"
+				: result.exitCode !== 0 && result.exitCode !== 1
+					? "errors"
+					: search.reason;
+	if (reason) {
+		search.complete = false;
+		search.reason = reason;
+	}
+}
+
+export function searchNotice(search: SearchCompleteness, shown: number, unit: string): string {
+	return search.complete
+		? ""
+		: `\n[Search incomplete: ${search.reason}; ${shown} ${unit} shown; ${search.skipped.count}${search.skipped.unknown ? "+ (diagnostic coverage unknown)" : ""} paths skipped.]`;
+}
+
 export interface LineStreamOptions {
 	/** Spawn cwd; defaults to the workspace root (process.cwd()). */
 	cwd?: string;
@@ -47,11 +188,14 @@ export interface LineStreamOptions {
 	timeoutMs?: number;
 	/** Called per stdout line. Invoke `stop()` to kill the child early. */
 	onLine(line: string, stop: () => void): void;
+	/** Receives every stderr line, independently of the retained diagnostic cap. */
+	onStderrLine?(line: string, truncated: boolean): void;
 }
 
 export interface LineStreamResult {
 	exitCode: number | null;
 	stderr: string;
+	oversizedDiagnostics: number;
 	timedOut: boolean;
 	aborted: boolean;
 	/** True when the caller's `stop()` killed the child (not an error). */
@@ -66,13 +210,14 @@ export function spawnLineStream(
 	options: LineStreamOptions,
 ): Promise<LineStreamResult> {
 	return new Promise((resolve) => {
-		const timeoutMs = options.timeoutMs ?? SEARCH_SPAWN_TIMEOUT_MS;
+		const timeoutMs = options.timeoutMs ?? timeoutContext.getStore() ?? SEARCH_SPAWN_TIMEOUT_MS;
 		let timedOut = false;
 		let aborted = false;
 		let stoppedEarly = false;
 		let killSent = false;
 		let settled = false;
-		let stderr = "";
+		const retainedStderr = Buffer.alloc(SEARCH_STDERR_CAP_BYTES);
+		let stderrBytes = 0;
 		let killGraceTimer: ReturnType<typeof setTimeout> | null = null;
 
 		// spawn can throw synchronously (e.g. an argv entry over MAX_ARG_STRLEN
@@ -100,6 +245,7 @@ export function spawnLineStream(
 			resolve({
 				exitCode: null,
 				stderr: "",
+				oversizedDiagnostics: 0,
 				timedOut: false,
 				aborted: false,
 				stoppedEarly: false,
@@ -109,6 +255,7 @@ export function spawnLineStream(
 		}
 		const child = spawned.child;
 		const rl = createInterface({ input: child.stdout });
+		const stderrLines = new BoundedDiagnosticDecoder((line, truncated) => options.onStderrLine?.(line, truncated));
 
 		const sendSignal = (signalName: NodeJS.Signals): void => {
 			const pid = child.pid;
@@ -157,9 +304,11 @@ export function spawnLineStream(
 			if (killGraceTimer) clearTimeout(killGraceTimer);
 			options.signal?.removeEventListener("abort", onAbort);
 			rl.close();
+			stderrLines.end();
 			resolve({
 				exitCode: child.exitCode,
-				stderr,
+				stderr: decodeBoundedUtf8(retainedStderr.subarray(0, stderrBytes)),
+				oversizedDiagnostics: stderrLines.oversizedLines,
 				timedOut,
 				aborted,
 				stoppedEarly,
@@ -174,7 +323,10 @@ export function spawnLineStream(
 			options.onLine(line, stop);
 		});
 		child.stderr?.on("data", (chunk: Buffer) => {
-			if (stderr.length < STDERR_CAP_BYTES) stderr += chunk.toString("utf8");
+			const take = Math.min(chunk.length, retainedStderr.length - stderrBytes);
+			chunk.copy(retainedStderr, stderrBytes, 0, take);
+			stderrBytes += take;
+			stderrLines.write(chunk);
 		});
 		child.on("error", (error) => finish(error.message));
 		child.on("close", () => finish(null));

package/src/tools/surface.ts

@@ -0,0 +1,163 @@
+/**
+ * Tool placement: which registered tools carry an attached schema on every
+ * turn (direct) and which are reached through the gateway's find/describe/call
+ * results (gateway). One table for the builtins, one prefix rule for each
+ * dynamic namespace, and the helpers every consumer of a placement decision
+ * shares, so moving a tool between the two surfaces is a one-row edit here.
+ *
+ * Placement changes what the model sees, never what a call is allowed to do:
+ * a capability behind the gateway runs through the same admission as a direct
+ * call, under its own name and action class, and the ledger consumers unwrap
+ * a gateway call back to that capability through {@link effectiveToolCall}.
+ *
+ * Pure module: no I/O, no registry construction. Imported by the prompt
+ * compiler and by ledger consumers, so it depends on core/tool-names only.
+ */
+
+import {
+	type BuiltinToolName,
+	isBuiltinToolName,
+	isHarnessExtensionToolName,
+	isMcpToolName,
+	type ToolName,
+	ToolNames,
+} from "../core/tool-names.js";
+
+export type ToolPlacement = "direct" | "gateway";
+
+/**
+ * Builtin placement. Direct: the fundamental coding operations, context and
+ * repository understanding, executable verification, the script runner, the
+ * gateway itself, and the ORCHESTRATE and INTERACT planes (operator decision,
+ * v0.4.9 handoff §5). Gateway: everything secondary, discovered on demand.
+ */
+export const TOOL_PLACEMENT: Readonly<Record<BuiltinToolName, ToolPlacement>> = {
+	[ToolNames.Read]: "direct",
+	[ToolNames.Evidence]: "gateway",
+	[ToolNames.Grep]: "direct",
+	[ToolNames.Find]: "direct",
+	[ToolNames.Ls]: "direct",
+	[ToolNames.CodeNav]: "direct",
+	[ToolNames.Context]: "direct",
+	[ToolNames.CredentialPresent]: "gateway",
+	[ToolNames.ClioDocs]: "gateway",
+	[ToolNames.ClioLibrary]: "gateway",
+	[ToolNames.Data]: "gateway",
+	[ToolNames.Write]: "direct",
+	[ToolNames.Edit]: "direct",
+	[ToolNames.Bash]: "direct",
+	[ToolNames.Git]: "gateway",
+	[ToolNames.Verify]: "direct",
+	[ToolNames.RunScript]: "direct",
+	[ToolNames.Dispatch]: "direct",
+	[ToolNames.Monitor]: "direct",
+	[ToolNames.Steer]: "direct",
+	[ToolNames.Tasks]: "direct",
+	[ToolNames.Ledger]: "direct",
+	[ToolNames.Panes]: "direct",
+	[ToolNames.Limitation]: "direct",
+	[ToolNames.Decide]: "direct",
+	[ToolNames.WebRead]: "gateway",
+	[ToolNames.WebFetch]: "gateway",
+	[ToolNames.AskUser]: "direct",
+	[ToolNames.Artifact]: "gateway",
+	[ToolNames.Gateway]: "direct",
+};
+
+/** Placement of a tool by name: the builtin table, then the dynamic namespaces. Unknown names are direct. */
+export function toolPlacement(name: string): ToolPlacement {
+	if (isBuiltinToolName(name as ToolName)) return TOOL_PLACEMENT[name as BuiltinToolName];
+	if (isHarnessExtensionToolName(name) || isMcpToolName(name)) return "gateway";
+	return "direct";
+}
+
+/** Placement of a registered spec: its explicit declaration wins, then the name rule. */
+export function toolSpecPlacement(spec: { name: string; placement?: ToolPlacement }): ToolPlacement {
+	return spec.placement ?? toolPlacement(spec.name);
+}
+
+function isGatewayPlacedToolName(name: string): boolean {
+	return toolPlacement(name) === "gateway";
+}
+
+export type GatewayCapabilityKind = "builtin" | "extension" | "mcp";
+
+/** Which namespace a gateway capability belongs to, for find listings and describe notes. */
+export function gatewayCapabilityKind(name: string): GatewayCapabilityKind {
+	if (isMcpToolName(name)) return "mcp";
+	if (isHarnessExtensionToolName(name)) return "extension";
+	return "builtin";
+}
+
+/**
+ * Add `gateway` to a capability list that names any gateway-placed capability.
+ * The dispatch admission filter and the worker's `effectiveToolNames` both
+ * apply this rule to the same admitted list, which is what keeps the
+ * orchestrator's approved tool signature equal to the one the worker attests.
+ * A list that reaches nothing behind the gateway gains nothing.
+ */
+export function withGatewayForCapabilities(names: ReadonlyArray<ToolName>): ToolName[] {
+	const out = [...new Set(names)];
+	if (out.includes(ToolNames.Gateway)) return out;
+	if (!out.some((name) => isGatewayPlacedToolName(name))) return out;
+	out.push(ToolNames.Gateway);
+	return out;
+}
+
+/**
+ * The direct projection of a capability surface: the names that carry an
+ * attached schema. Gateway-placed capabilities drop out and, when any was
+ * present, `gateway` stands in for them. Used wherever a prompt or a display
+ * lists "direct tools" from an admitted capability list.
+ */
+export function directSurfaceNames(names: ReadonlyArray<string>): string[] {
+	const seen = new Set<string>();
+	const out: string[] = [];
+	let reachesGateway = false;
+	for (const name of names) {
+		if (seen.has(name)) continue;
+		seen.add(name);
+		if (isGatewayPlacedToolName(name)) {
+			reachesGateway = true;
+			continue;
+		}
+		out.push(name);
+	}
+	if (reachesGateway && !out.includes(ToolNames.Gateway)) out.push(ToolNames.Gateway);
+	return out;
+}
+
+export interface EffectiveToolCall {
+	/** The capability that actually ran: the inner tool for a gateway call, the tool itself otherwise. */
+	toolName: string;
+	/** That capability's own arguments, when the record carried any. */
+	args: Record<string, unknown> | undefined;
+	/** True when the record was a gateway call unwrapped to its capability. */
+	viaGateway: boolean;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+/**
+ * Unwrap a ledger tool record to the capability it stands for. Every consumer
+ * keyed on tool names (artifact folding, the read ledger, the path index,
+ * mutation observers, evidence, transcript rendering) resolves the record
+ * through this one helper so a `gateway` row never becomes an opaque "gateway
+ * succeeded" entry that hides the artifact, git, or fetch it performed.
+ *
+ * The capability comes from the result details when they are present (the
+ * gateway stamps `details.capability` on every call result) and from the
+ * call arguments (`op: "call"`, `capability`) otherwise. Find and describe
+ * records stay `gateway`.
+ */
+export function effectiveToolCall(toolName: string, args: unknown, details?: unknown): EffectiveToolCall {
+	const record = isRecord(args) ? args : undefined;
+	if (toolName !== ToolNames.Gateway) return { toolName, args: record, viaGateway: false };
+	const fromDetails = isRecord(details) && typeof details.capability === "string" ? details.capability : null;
+	const fromArgs = record?.op === "call" && typeof record.capability === "string" ? record.capability : null;
+	const capability = (fromDetails ?? fromArgs)?.trim() ?? "";
+	if (capability.length === 0 || capability === ToolNames.Gateway) return { toolName, args: record, viaGateway: false };
+	return { toolName: capability, args: isRecord(record?.args) ? record.args : undefined, viaGateway: true };
+}

package/src/tools/verify/authoring.ts

@@ -716,7 +716,7 @@ function validationContractProposals(workspaceRoot: string, diagnostics: string[
  * command is deliberately absent, so the proposal cannot become executable
  * until the operator confirms an exact argv through `verifiers add`.
  */
-function numericContractProposals(workspaceRoot: string): NumericProposal[] {
+function numericContractProposals(workspaceRoot: string, diagnostics: string[]): NumericProposal[] {
 	const loaded = loadValidationContract(workspaceRoot);
 	if (!loaded.ok || loaded.contract === null) return [];
 	const proposals: NumericProposal[] = [];
@@ -724,7 +724,12 @@ function numericContractProposals(workspaceRoot: string): NumericProposal[] {
 	for (const artifact of loaded.contract.artifacts ?? []) {
 		if (artifact.numerical_tolerances === undefined) continue;
 		const tolerance = normalizeNumericTolerance(artifact.numerical_tolerances);
-		if (tolerance instanceof Error) continue;
+		if (tolerance instanceof Error) {
+			diagnostics.push(
+				`${loaded.path}: artifact ${JSON.stringify(artifact.path)} numeric proposal skipped: ${tolerance.message}`,
+			);
+			continue;
+		}
 		const base = slug(path.basename(artifact.path).replace(/\.[^.]+$/u, ""), "artifact");
 		const preferred = boundedId(`numeric-${base}`);
 		let id = preferred;
@@ -842,7 +847,7 @@ export function discoverVerifierAuthoring(workspaceRoot = process.cwd()): Verifi
 		activeChecks,
 		existingChecks,
 		proposals,
-		numericProposals: numericContractProposals(workspaceRoot),
+		numericProposals: numericContractProposals(workspaceRoot, diagnostics),
 		diagnostics,
 		manualEntry,
 		...(typeof catalogText === "string" ? { catalogText } : {}),

package/src/tools/verify/catalog.ts

@@ -39,6 +39,12 @@ export const DECLARED_CHECK_KINDS: ReadonlyArray<DeclaredCheckKind> = ["command"
 export interface DeclaredNumericCompare {
 	/** Repository-relative JSON file of the same shape as the command's stdout. */
 	reference: string;
+	/**
+	 * At least one of `relative`, `absolute`, `ulp`, plus the optional rule
+	 * fields `combine` (`all`, the default, or `any`) and `nonFinite` (`fail`,
+	 * the default, or `match`). A catalog that omits the rule fields judges
+	 * exactly as before they existed.
+	 */
 	tolerance: NumericTolerance;
 }
 
@@ -80,7 +86,14 @@ export type ProjectCatalogLoadResult = { ok: true; source: DeclaredCheckSource |
 
 const ROOT_FIELDS = new Set(["version", "checks"]);
 const CHECK_FIELDS = new Set(["id", "description", "command", "cwd", "timeoutMs", "tags"]);
-/** Version 2 adds the kind and its parameters; every one is optional and kind-gated. */
+/**
+ * Version 2 adds the kind and its parameters; every one is optional and
+ * kind-gated. The numeric tolerance object's own fields (`relative`,
+ * `absolute`, `ulp`, `combine`, `nonFinite`) are validated by
+ * `normalizeNumericTolerance`, so this version admits a `combine` or
+ * `nonFinite` rule without a catalog version bump: a file that omits them
+ * keeps the original strict-conjunction, non-finite-fails judgement.
+ */
 const CHECK_FIELDS_V2 = new Set([...CHECK_FIELDS, "kind", "reference", "tolerance", "budget", "baseline"]);
 const TAG_PATTERN = /^[a-z0-9][a-z0-9._-]*$/;
 const SHELL_EXECUTABLES = new Set([