@danypops/vehicle-conformance 0.2.1 → 0.4.0

package/README.md

@@ -10,8 +10,66 @@ bun add -d @danypops/vehicle-conformance
 ```
 
 ```ts
-import { registerConformanceOperations, runVehicleClientConformance } from "@danypops/vehicle-conformance";
+import {
+  registerConformanceOperations,
+  runVehicleClientConformance,
+  runToolShellDualChannelConformance,
+} from "@danypops/vehicle-conformance";
 ```
 
+`runToolShellDualChannelConformance(fixture)` is the host-neutral Tool Shell
+matrix. A fixture adapts one provider's real projection/rendering boundary via
+`execute`, `render`, `replay`, `renderCall`, and `invalidProjection`; the shared
+suite checks independent model/presentation sentinels and named bounds,
+JSON-safe secret-free details, malformed/unknown replay fallback, collapsed vs.
+expanded immutability, schema-sensitive call rendering, 40/80/120-column
+physical-line safety, partial output, and projector exception policy. Pi-specific
+component construction stays in the adapter fixture rather than this package.
+
+## The five boundaries
+
+Every conformant Tool Shell provider keeps five things independent:
+
+1. **Application DTO** -- the domain's own real output shape, transport-neutral,
+   untouched by any presentation concern.
+2. **Model content** -- what the LLM reads: independently bounded, ANSI-free,
+   semantic. Never derived from or coupled to what a human sees.
+3. **Persisted presentation details** -- a *projected*, versioned, discriminated-union
+   DTO, independently bounded, with explicit `{total, returned, omitted}`
+   completeness metadata. Projected once, before persistence -- never inferred
+   from raw output at render time.
+4. **Interactive component** -- the rendered view of #3. Expanded mode may only
+   reveal rows already inside the bounded DTO, never bypass the bound by
+   reaching back into raw application output.
+5. **CLI presenters** -- a separate, JSON/human-text presentation path outside
+   the interactive TUI entirely; out of scope for this suite.
+
+The fail-closed rule that matters most: a parser for #3 must reject a
+malformed/unknown-version/oversized/cyclic details object and fall back to
+**content** (#2) -- never render raw, unbounded application output as a human
+view.
+
+## Declared-value coverage
+
+A fixture's `ToolShellDualChannelSubject` can optionally supply
+`declaredValueCases` (one `{ value, rawPayload }` per value of a discriminator
+field the provider's own schema declares -- a `format`/`kind`/`action`/...) plus
+a matching `renderDeclaredValue(value, rawPayload, options)`. When present, the
+suite renders one result per declared value and fails if fewer than
+`min(2, cases.length)` of them escape being textually indistinguishable from a
+raw `JSON.stringify(rawPayload, null, 2)` dump.
+
+This is the generic version of a `never`-typed exhaustiveness guard on a
+discriminated switch -- it catches the same bug class (most declared values
+silently falling through to an undifferentiated raw-JSON view) in bespoke
+non-switch code too (an `if`-chain, a plain-string `action` switch with no
+compile-time exhaustiveness), which a TypeScript-only lint rule would miss
+entirely. `evaluateDeclaredValueCoverage(cases, renderDeclaredValue, options)`
+is exported separately for direct unit testing of the classifier against a
+known-bad fixture shape, independent of the wrapping `bun:test` assertion.
+
+Omit `declaredValueCases` entirely for a subject with no such discriminator --
+the check then no-ops.
+
 See the [workspace README](https://github.com/DanyPops/vehicle#readme) for
 the full Vehicle package layout.

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@danypops/vehicle-conformance",
-	"version": "0.2.1",
+	"version": "0.4.0",
 	"description": "Host-neutral conformance suite for any VehicleClient implementation: one shared bun:test assertion set that LocalVehicleClient, RemoteVehicleClient, and any future transport must satisfy identically. A Bun-only devDependency for testing, not a runtime library -- ships raw TypeScript, never precompiled.",
 	"license": "MIT",
 	"type": "module",
@@ -14,13 +14,13 @@
 		"typecheck": "tsc --noEmit"
 	},
 	"dependencies": {
-		"@danypops/vehicle-core": "^0.10.0",
-		"@danypops/vehicle-server": "^0.13.0"
+		"@danypops/vehicle-core": "^0.17.1",
+		"@danypops/vehicle-server": "^0.25.2"
 	},
 	"devDependencies": {
-		"@danypops/vehicle-client": "^0.5.1",
+		"@danypops/vehicle-client": "^0.10.3",
 		"@types/node": "^22.0.0",
-		"typescript": "^5.9.2"
+		"typescript": "^5.9.3"
 	},
 	"repository": {
 		"type": "git",

package/src/vehicle-conformance.ts

@@ -418,3 +418,210 @@ export function runVehicleClientConformance(fixture: VehicleConformanceFixture):
 		});
 	});
 }
+
+export interface ToolShellConformanceSnapshot {
+	readonly content: string;
+	readonly details: unknown;
+}
+
+export interface ToolShellRenderOptions {
+	readonly width: 40 | 80 | 120;
+	readonly expanded: boolean;
+	readonly partial?: boolean;
+}
+
+/**
+ * Host adapter for the Tool Shell's two independent persisted channels. The
+ * conformance package stays Pi-free: a Pi adapter supplies component output,
+ * while a CLI or another host can supply its own renderer through this same API.
+ */
+export interface ToolShellDualChannelSubject {
+	readonly bounds: { readonly modelContentBytes: number; readonly presentationDetailsBytes: number };
+	execute(): Promise<ToolShellConformanceSnapshot>;
+	render(snapshot: ToolShellConformanceSnapshot, options: ToolShellRenderOptions): readonly string[];
+	replay(details: unknown, fallbackContent: string, options: ToolShellRenderOptions): readonly string[];
+	renderCall(args: unknown, width: 40 | 80 | 120): readonly string[];
+	invalidProjection(): Promise<unknown>;
+	/**
+	 * Optional -- the discriminator values (a `format`/`kind`/`action`/... field) this provider's
+	 * own presentation-details schema declares, each paired with a representative raw application
+	 * payload for that value. Supplying this (together with renderDeclaredValue) enables the
+	 * declared-value coverage check below, which generically catches the pi-web-spider bug class
+	 * (see doc 4e9e08c1, Finding 1/4): most declared values falling through to an undifferentiated
+	 * JSON.stringify dump of their own payload instead of a real projected view. Omit entirely for
+	 * a subject with no such discriminator -- the check then no-ops.
+	 */
+	readonly declaredValueCases?: readonly ToolShellDeclaredValueCase[];
+	/** Required alongside declaredValueCases: renders the expanded view for one declared value's
+	 * own raw payload, through exactly the same projection+render pipeline the real handler uses. */
+	renderDeclaredValue?(value: string, rawPayload: unknown, options: ToolShellRenderOptions): readonly string[];
+}
+
+export interface ToolShellDeclaredValueCase {
+	/** e.g. a WebFormat value ('search'/'lean'/...), a PackageToolDetails['kind'], a tickets action name. */
+	readonly value: string;
+	/** The real, untransformed application output this declared value would carry. */
+	readonly rawPayload: unknown;
+}
+
+export interface DeclaredValueCoverageResult {
+	/** Declared values whose rendered output is NOT indistinguishable from a raw JSON.stringify dump of their own payload. */
+	readonly nonRawValues: readonly string[];
+	/** Declared values whose rendered output IS indistinguishable from a raw JSON.stringify dump of their own payload. */
+	readonly rawValues: readonly string[];
+}
+
+function normalizeForComparison(text: string): string {
+	return text.replace(/\s+/g, "");
+}
+
+/**
+ * True when `renderedLines` is textually indistinguishable (ignoring ANSI styling and whitespace)
+ * from `JSON.stringify(rawPayload, null, 2)` -- the exact shape pi-web-spider's `primaryLines()`
+ * fell back to for every non-"markdown" format. Whitespace-insensitive so a renderer that reflows
+ * the same JSON text to a narrower width still counts as "raw", matching the real bug (a Text
+ * component wrapping the identical JSON.stringify output).
+ */
+function looksLikeRawJsonDump(renderedLines: readonly string[], rawPayload: unknown): boolean {
+	let rawJson: string;
+	try {
+		rawJson = JSON.stringify(rawPayload, null, 2) ?? "";
+	} catch {
+		return false;
+	}
+	if (rawJson.length === 0) return false;
+	const renderedText = renderedLines.join("\n").replace(ANSI_CSI_PATTERN, "");
+	return normalizeForComparison(renderedText) === normalizeForComparison(rawJson);
+}
+
+/**
+ * Pure, independently unit-testable core of the declared-value coverage check -- separated from
+ * the bun:test `it()` wiring below so a fixture reproducing a known-bad shape (e.g.
+ * pi-web-spider's own pre-fix behavior) can be asserted against directly, proving the classifier
+ * itself actually detects that bug class rather than trusting the wrapping `it()` alone.
+ */
+export function evaluateDeclaredValueCoverage(
+	cases: readonly ToolShellDeclaredValueCase[],
+	renderDeclaredValue: (value: string, rawPayload: unknown, options: ToolShellRenderOptions) => readonly string[],
+	options: ToolShellRenderOptions = { width: 80, expanded: true },
+): DeclaredValueCoverageResult {
+	const nonRawValues: string[] = [];
+	const rawValues: string[] = [];
+	for (const { value, rawPayload } of cases) {
+		const lines = renderDeclaredValue(value, rawPayload, options);
+		(looksLikeRawJsonDump(lines, rawPayload) ? rawValues : nonRawValues).push(value);
+	}
+	return { nonRawValues, rawValues };
+}
+
+export interface ToolShellDualChannelFixture {
+	readonly label: string;
+	create(): Promise<{ readonly subject: ToolShellDualChannelSubject; readonly cleanup: () => Promise<void> }>;
+}
+
+function utf8Length(value: string): number {
+	return new TextEncoder().encode(value).byteLength;
+}
+
+// biome-ignore lint/complexity/useRegexLiterals: a constructor avoids control-character lint on the equivalent literal.
+const ANSI_CSI_PATTERN = new RegExp("\\u001B\\[[0-?]*[ -/]*[@-~]", "g");
+
+function assertPhysicalLines(lines: readonly string[], width: number): void {
+	expect(lines.length).toBeGreaterThan(0);
+	for (const line of lines) {
+		expect(line).not.toContain("\n");
+		// ANSI is forbidden in model content, but permitted in host rendering.
+		const visible = line.replace(ANSI_CSI_PATTERN, "");
+		expect([...visible].length).toBeLessThanOrEqual(width);
+	}
+}
+
+/** Reusable provider-facing dual-channel contract matrix. */
+export function runToolShellDualChannelConformance(fixture: ToolShellDualChannelFixture): void {
+	describe(`Vehicle Tool Shell dual-channel conformance: ${fixture.label}`, () => {
+		it("keeps model and persisted-presentation sentinels isolated under independent named bounds", async () => {
+			const { subject, cleanup } = await fixture.create();
+			try {
+				const snapshot = await subject.execute();
+				expect(snapshot.content).toContain("MODEL_ONLY");
+				expect(snapshot.content).not.toContain("PRESENTATION_ONLY");
+				const details = JSON.stringify(snapshot.details);
+				expect(details).toContain("PRESENTATION_ONLY");
+				expect(details).not.toContain("MODEL_ONLY");
+				expect(details).not.toContain("RAW_SECRET");
+				expect(utf8Length(snapshot.content)).toBeLessThanOrEqual(subject.bounds.modelContentBytes);
+				expect(utf8Length(details)).toBeLessThanOrEqual(subject.bounds.presentationDetailsBytes);
+			} finally {
+				await cleanup();
+			}
+		});
+
+		it("keeps model content semantic, ANSI-free, and useful when replay details reject", async () => {
+			const { subject, cleanup } = await fixture.create();
+			try {
+				const snapshot = await subject.execute();
+				expect(snapshot.content).not.toContain("\u001b[");
+				for (const details of [{ schema: "unknown/v99" }, { malformed: true }, { output: { legacy: true } }, undefined]) {
+					const lines = subject.replay(details, snapshot.content, { width: 80, expanded: false });
+					expect(lines.join("\n")).toContain("MODEL_ONLY");
+				}
+			} finally {
+				await cleanup();
+			}
+		});
+
+		it("changes only human rendering across collapsed/expanded and 40/80/120 layouts", async () => {
+			const { subject, cleanup } = await fixture.create();
+			try {
+				const snapshot = await subject.execute();
+				const before = JSON.stringify(snapshot);
+				for (const width of [40, 80, 120] as const) {
+					assertPhysicalLines(subject.render(snapshot, { width, expanded: false }), width);
+					assertPhysicalLines(subject.render(snapshot, { width, expanded: true }), width);
+					assertPhysicalLines(subject.render(snapshot, { width, expanded: false, partial: true }), width);
+				}
+				expect(JSON.stringify(snapshot)).toBe(before);
+			} finally {
+				await cleanup();
+			}
+		});
+
+		it("never echoes schema-sensitive call input and follows the documented projector exception policy", async () => {
+			const { subject, cleanup } = await fixture.create();
+			try {
+				for (const width of [40, 80, 120] as const) {
+					const call = subject.renderCall({ name: "safe-task", token: "RAW_SECRET" }, width).join("\n");
+					expect(call).toContain("safe-task");
+					expect(call).not.toContain("RAW_SECRET");
+				}
+				await expect(subject.invalidProjection()).rejects.toBeTruthy();
+			} finally {
+				await cleanup();
+			}
+		});
+
+		it("renders most of its own declared discriminator values as more than a raw JSON dump of their own payload", async () => {
+			const { subject, cleanup } = await fixture.create();
+			try {
+				const cases = subject.declaredValueCases;
+				if (!cases || cases.length === 0) return; // opt-in: no discriminator declared, nothing to check
+				if (!subject.renderDeclaredValue) {
+					throw new Error("declaredValueCases supplied without a matching renderDeclaredValue implementation");
+				}
+				const renderDeclaredValue = subject.renderDeclaredValue.bind(subject);
+				const options: ToolShellRenderOptions = { width: 80, expanded: true };
+				for (const { value, rawPayload } of cases) {
+					assertPhysicalLines(renderDeclaredValue(value, rawPayload, options), options.width);
+				}
+				const { nonRawValues, rawValues } = evaluateDeclaredValueCoverage(cases, renderDeclaredValue, options);
+				expect(
+					nonRawValues.length,
+					`declared values [${cases.map((c) => c.value).join(", ")}] mostly render as an undifferentiated JSON.stringify dump of ` +
+						`their own payload -- only [${nonRawValues.join(", ") || "none"}] escape it, [${rawValues.join(", ")}] don't`,
+				).toBeGreaterThanOrEqual(Math.min(2, cases.length));
+			} finally {
+				await cleanup();
+			}
+		});
+	});
+}