@termwright/conformance 0.2.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 gorce-ai
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,189 @@
+# @termwright/conformance
+
+The suites that decide whether an implementation of termwright is actually one:
+runnable fixtures, the driver conformance matrix from the origin spec (§20), and
+an **adapter contract suite you can run against any adapter, in any language**.
+
+Nothing depends on this package; it is allowed to depend on everything.
+
+## Install
+
+```sh
+pnpm add -D @termwright/conformance
+```
+
+`vitest` is an optional peer dependency, needed only to run the exported suite.
+
+## Certifying an adapter
+
+The only thing an adapter author writes is the binding between their app and the
+suite. It drives the adapter as a subprocess and looks at bytes and frames, so a
+Python, Go or Rust adapter certifies exactly like the TypeScript one.
+
+```ts
+// my-adapter.conformance.test.ts
+import { runAdapterConformance } from '@termwright/conformance';
+
+await runAdapterConformance({
+  name: 'termwright-py',
+  spawn: () => ({ command: ['python', 'examples/demo_app.py'] }),
+  // Optional: the same UI with the adapter compiled out. When given, the
+  // dormant run is compared against it byte for byte.
+  baseline: () => ({ command: ['python', 'examples/demo_app.py'], env: { PLAIN: '1' } }),
+  ready: 'Ready',
+  interaction: { input: '\t', expect: '[Save]' },
+  quit: { input: '', exitCode: 0 },
+  columns: 80,
+  rows: 24,
+  expectAbsoluteBounds: true,
+});
+```
+
+The byte-for-byte comparison covers the **startup** stream, with nothing
+written to the child: a pseudo-terminal echoes the suite's own keystrokes, so a
+stream containing our input compares the tty's timing rather than the adapter's
+output (measured on the Ink fixture: 3 mismatches in 30 pairs with input —
+always a stray `0x09`, the tab the suite itself sent — and 0 in 40 without).
+
+Two requirements the registration has to respect, because the suite exercises
+the app rather than mocking it: `interaction.input` is sent **more than once**,
+so pick something whose repetition is harmless; and `quit.input` must work from
+**any** state that repetition can reach — a key that quits only while one widget
+has focus is not a quit input.
+
+Pass `requires` to declare the toolchain the adapter needs. When the probe
+fails, the whole registration skips and the reason appears in the block's name,
+exactly as a missing pseudo-terminal does:
+
+```ts
+requires: {
+  probe: ['python3', '-c', 'import termwright, textual'],
+  label: 'python3 with termwright and textual installed',
+},
+```
+
+An adapter that announces the `logs` capability declares how to exercise it,
+and the obligations above are then asserted rather than skipped:
+
+```ts
+logs: { input: 'l', expect: 'conformance log record' },
+```
+
+`input` is optional: omit it for an app that logs on its own, as the tview
+example does at startup, and the obligation waits for the record instead of
+provoking one.
+
+Conventions the fixture has to opt into — an annotated test id, an empty
+textbox, a container with no label — are declared under `conventions`; the rest
+run for every adapter:
+
+```ts
+conventions: {
+  emptyTextboxTestId: 'reason',
+  readmePath: 'clients/go/README.md',
+},
+```
+
+Deviations are **not** declared here. They are read from the adapter's own
+`## Deviations` section, which is where rule 6 puts them and where a user reads
+them; repeating them in a registration would give two copies of one fact that
+eventually disagree. That gives three outcomes rather than two: compliant, a
+failure the README declares (a documented limitation, not an error), and a
+failure it does not (an error).
+
+Every run writes a per-adapter roll-up of those declarations, which
+`pnpm conformance` prints under the matrix. It is generated rather than
+maintained: a hand-written table of per-adapter gaps went stale within a round
+of being written, and a stale overview in a document people trust is worse than
+none.
+
+Rules 1, 2 and 4 cannot be judged in full from outside a subprocess, so the
+README check is advisory: a missing `## Deviations` heading writes a warning to
+stderr rather than failing the run.
+
+It checks the five obligations an adapter has:
+
+| Obligation | What is asserted |
+|---|---|
+| Dormant rule | Without `TERMWRIGHT_ENDPOINT` it opens no channel and writes no marker; with `baseline`, byte-for-byte identical startup output |
+| Tree before input | Once the handshake completes and *before any input*, the tree is non-empty and has at least one node a locator could address. Opt out with `treeBeforeInput: {required: false, reason}` |
+| Handshake | `hello` first and once, correct protocol id, non-empty adapter identity, capabilities from the closed set |
+| Snapshot validity | Every snapshot passes `validateSnapshot`, carries this session's id, has resolvable parents and monotonic revisions |
+| Revision ordering | For each revision: snapshot → `revision-commit` → a marker that verifies against the session token, markers strictly increasing |
+| Channel loss | Cutting the socket leaves the application rendering and alive, and the adapter does not reconnect |
+| Logs | An adapter that did not announce `logs` sends none. One that declares them in the registration must deliver a record whose `seq` is unique and increasing and whose message never appears on the terminal |
+| Deltas (when announced) | With `subscribe: 'diffs'`, the deltas an adapter emits compose — through the protocol's own `applyTreeDelta` — to the same tree it reports when asked with `get-tree` |
+| Conventions | The machine-checkable half of the protocol README's "Adapter semantics conventions": containers are not named from their content (rule 2), an annotated test id reaches the wire (rule 3), an empty textbox publishes `value: ''` and no value is derived outside `{textbox, progressbar}` or from a boolean (rule 5) |
+
+`await` it at the top level: `vitest` is imported dynamically so the package can
+also be used from a plain script.
+
+## Fixtures
+
+`CONFORMANCE_FIXTURES` returns absolute paths to programs you can launch with
+`node <path>`:
+
+| Fixture | Purpose | Dependencies |
+|---|---|---|
+| `generic()` | Uninstrumented app: menu, colours, mouse/paste/focus modes, Unicode, alternate screen, scrollback (§20.1) | none |
+| `prompt()` | Shell-shaped app emitting OSC 133 marks; `--marks=off` suppresses them, `--work=<ms>` sets the command duration | none |
+| `adversarialPeer()` | Raw wire peer; takes a scenario name as `argv[2]` (§20.3) | none |
+| `inkProbe()` | Ordinary `ink.render` app launched through the zero-config Ink probe | `ink`, `react`, `@termwright/probe-ink` |
+
+The first three import nothing at all — the adversarial peer re-derives the
+framing and the marker MAC from the specification rather than importing
+`@termwright/protocol`, so a drift between spec and implementation shows up
+instead of cancelling out.
+
+```ts
+import { CONFORMANCE_FIXTURES } from '@termwright/conformance';
+import { launchTerminal } from '@termwright/driver';
+
+const terminal = await launchTerminal({ command: ['node', CONFORMANCE_FIXTURES.generic()] });
+```
+
+## Running the matrix
+
+The MCP suite drives `@termwright/mcp` over real HTTP with several concurrent
+sessions, and checks close ownership against real pids rather than against the
+registry's bookkeeping — a registry can forget a session while its terminals
+keep running, and only a pid probed afterwards tells the two apart.
+
+```sh
+pnpm --filter @termwright/conformance conformance     # every suite, one matrix
+pnpm --filter @termwright/conformance test            # plain vitest
+pnpm --filter @termwright/conformance test:hostile    # adversarial suite, 128 MB heap cap
+```
+
+```
+area                        spec     result         tests    time
+generic fallback            §20.1    pass           10/10    2.0s
+hostile peer                §20.3    pass           25/25    14.5s
+interaction                 §20.4    pass           12/12    2.6s
+readiness + env             §5.3     pass           10/10    2.2s
+adapter contract (py/go)    §7       pass, 8 skip   6/14     0.6s
+hostile peer @ 128 MB heap  §10      pass           25/25    14.4s
+```
+
+Certifying the py/go rows needs their toolchains on the runner
+(`pip install -e clients/python[dev]` and a Go toolchain); without them those
+rows skip honestly, with the probe's failure on stderr.
+
+A partly-skipped area is reported as such rather than as a clean pass: the
+language adapters skip their whole registration when the toolchain is absent,
+and a matrix that hid it would claim coverage the machine never produced.
+
+Sessions launch with the driver's secret-safe `envMode: 'replace'` default, so
+a fixture only sees the documented allowlist plus what a suite declares.
+
+Every suite needs a pseudo-terminal and skips itself where none can be opened;
+`TERMWRIGHT_SKIP_PTY=1` skips them explicitly. A run where everything skipped
+says so rather than reporting success.
+
+## Why a second, smaller driver
+
+`AdapterProbe` speaks the protocol itself — endpoint, handshake, framing, marker
+verification — instead of using `@termwright/driver`. The driver deliberately
+hides frame ordering behind a settled tree, which is the right API for testing
+applications and the wrong one for testing adapters. The probe is exported for
+checks the suite does not cover; it never renders, locates or acts.

package/dist/index.d.ts

@@ -0,0 +1,351 @@
+import { AdapterToDriverMessage, LogRecord, TreeDelta, SemanticSnapshot } from '@termwright/protocol';
+import { LaunchOptions, TerminalHarness } from '@termwright/driver';
+
+/** How a fixture is started. Everything else about it is opaque to the probe. */
+interface AdapterCommand {
+    readonly command: readonly string[];
+    readonly env?: Readonly<Record<string, string>>;
+    readonly cwd?: string;
+}
+/** One message the adapter sent, stamped with the stdout position at arrival. */
+interface RecordedMessage {
+    readonly message: AdapterToDriverMessage;
+    /** Bytes of stdout the probe had received when this frame was parsed. */
+    readonly stdoutBytes: number;
+    readonly atMs: number;
+}
+/** One verified render marker found in stdout. */
+interface RecordedMarker {
+    readonly revision: number;
+    /** Offset of the marker's first byte in the stdout stream. */
+    readonly offset: number;
+    readonly atMs: number;
+}
+/** A frame the probe refused; a conforming adapter produces none. */
+interface RecordedFault {
+    readonly code: string;
+    readonly detail: string;
+}
+interface ProbeOptions {
+    readonly columns?: number;
+    readonly rows?: number;
+    /** Set `false` to withhold the instrumentation env — the dormant-run case. */
+    readonly instrument?: boolean;
+    /**
+     * What the probe asks the adapter to push. `'diffs'` is a preference, not a
+     * prohibition: an adapter may still send a full tree when a delta would not
+     * pay for itself, and the first publication always is one.
+     */
+    readonly subscribe?: 'snapshots' | 'diffs';
+}
+/** Everything the probe observed, readable while the child is still running. */
+interface ProbeObservation {
+    readonly messages: readonly RecordedMessage[];
+    readonly markers: readonly RecordedMarker[];
+    readonly faults: readonly RecordedFault[];
+    readonly connections: number;
+    readonly stdout: Uint8Array;
+    /** Raw bytes decoded as UTF-8: what was written, in order, escapes included. */
+    readonly text: string;
+    /** The visible grid, one row per line — what a user would actually see. */
+    readonly screen: string;
+    /** Application log records the adapter sent, in arrival order. */
+    readonly logs: readonly LogRecord[];
+    /** Deltas the adapter sent, in arrival order. */
+    readonly deltas: readonly TreeDelta[];
+    /**
+     * The tree obtained by composing every snapshot and delta received, in order,
+     * with the protocol's own `applyTreeDelta`. This is the oracle an adapter's
+     * deltas are checked against: a producer that also composed would only prove
+     * it agrees with itself.
+     */
+    readonly composed: SemanticSnapshot | null;
+    /** Why composition stopped, when it did. A conforming adapter produces none. */
+    readonly compositionError: string | null;
+}
+/**
+ * Runs one fixture under a pseudo-terminal with a protocol endpoint attached.
+ *
+ * @example
+ * ```ts
+ * const probe = await AdapterProbe.start({ command: ['node', 'app.mjs'] }, {});
+ * await probe.waitForText('Ready');
+ * await probe.write('\t');
+ * const { messages, markers } = probe.observe();
+ * await probe.stop();
+ * ```
+ */
+declare class AdapterProbe {
+    #private;
+    readonly sessionId: string;
+    readonly token: string;
+    private constructor();
+    /** Creates the endpoint (unless dormant), then spawns the fixture. */
+    static start(command: AdapterCommand, options?: ProbeOptions): Promise<AdapterProbe>;
+    /** Everything observed so far. Safe to call at any point. */
+    observe(): ProbeObservation;
+    /** The visible grid as text, trailing whitespace trimmed per row. */
+    screenText(): string;
+    /** The child's exit status, or `null` while it is still running. */
+    get exitStatus(): {
+        code: number | null;
+        signal: string | null;
+    } | null;
+    /** Writes raw bytes to the child, exactly as a terminal would. */
+    write(input: string): Promise<void>;
+    /**
+     * Resolves once `needle` appears on the rendered grid.
+     *
+     * Matching the byte stream instead would only work for adapters that happen
+     * to write their text contiguously: a framework that positions each run of
+     * cells never emits `focus: reject` as those twelve bytes in a row.
+     */
+    waitForText(needle: string | RegExp, timeoutMs?: number): Promise<void>;
+    /**
+     * Resolves once `predicate` holds over the current observation.
+     *
+     * `what` names the thing being waited for, and the failure carries what the
+     * probe could see when it gave up. "Condition never became true" is not a
+     * result anybody can act on: an adapter that never connected, one that
+     * connected and published nothing, and one whose binary died all produce it,
+     * and only the observation tells them apart.
+     */
+    waitFor(predicate: (observation: ProbeObservation) => boolean, timeoutMs?: number, what?: string): Promise<void>;
+    /** What the probe has seen so far, for a failure that has to explain itself. */
+    describe(): string;
+    /** Waits for the child to exit and returns its status. */
+    waitForExit(timeoutMs?: number): Promise<{
+        code: number | null;
+        signal: string | null;
+    }>;
+    /** Cuts the semantic channel without touching the child: the disconnect case. */
+    cutChannel(): void;
+    /** Stops the child and releases the endpoint. Idempotent. */
+    stop(): Promise<void>;
+    /**
+     * Asks the adapter for a full tree and resolves with it.
+     *
+     * This is what turns composition into a check rather than a belief: the
+     * locally composed tree is compared against one the adapter built itself.
+     */
+    requestTree(timeoutMs?: number): Promise<SemanticSnapshot | null>;
+}
+/** The marker prefix, re-exported so suites can assert on dormant output. */
+declare const MARKER_TEXT_PREFIX = "\u001B]8487;twm;";
+
+/** How to start, drive and stop the adapter under test. */
+interface AdapterConformanceOptions {
+    /** Name of the adapter, used in the test titles. */
+    readonly name: string;
+    /** Command that starts the instrumented application. */
+    spawn(): AdapterCommand;
+    /**
+     * Optional command rendering the same UI with the adapter compiled out. When
+     * given, the dormant run is compared against it byte for byte — the strongest
+     * form of the dormant rule. Without it, a dormant run is only checked for
+     * silence on the wire.
+     */
+    baseline?(): AdapterCommand;
+    /** Text that proves the first frame reached the terminal. */
+    readonly ready: string | RegExp;
+    /**
+     * An input that changes the screen, and the text that proves it landed.
+     *
+     * The suite sends it **more than once** — a dormant run, a run that has to
+     * produce a second revision, and a run after the channel was cut all need a
+     * render. Pick something whose repetition is harmless.
+     */
+    readonly interaction: {
+        readonly input: string;
+        readonly expect: string | RegExp;
+    };
+    /**
+     * An input that makes the application exit, and the status it exits with.
+     *
+     * It must work from **any** state repeated `interaction` can reach. A key
+     * that quits only while a particular widget has focus is not a quit input:
+     * the tview example's documented `q` types into its text field once focus has
+     * cycled that far, so its registration uses Ctrl+C instead.
+     */
+    readonly quit: {
+        readonly input: string;
+        readonly exitCode?: number;
+    };
+    readonly columns?: number;
+    readonly rows?: number;
+    /** Assert that published bounds are viewport-absolute (an `absolute-bounds` claim). */
+    readonly expectAbsoluteBounds?: boolean;
+    /**
+     * Opt out of the "publishes a tree before any input" obligation.
+     *
+     * By default an adapter must publish a non-empty tree once the handshake
+     * completes, with no input sent — an app that is addressable only after the
+     * first keystroke is not addressable at all to a driver that has just
+     * launched it. Some apps legitimately render nothing until an event arrives;
+     * pass a reason, which is printed in the test title so the exemption stays
+     * visible rather than becoming folklore.
+     */
+    readonly treeBeforeInput?: {
+        readonly required: false;
+        readonly reason: string;
+    };
+    /**
+     * How to make the application log, for an adapter that announces the `logs`
+     * capability. Without it the log obligations are skipped; with it they are
+     * asserted, and an adapter that announces `logs` but never delivers one
+     * fails here rather than in a user's test.
+     */
+    /**
+     * How to check the normative adapter conventions (protocol README, "Adapter
+     * semantics conventions"). Rules 1, 2 and 4 are largely judgement calls from
+     * outside; what is listed here is what a subprocess can actually observe.
+     *
+     * A rule an adapter cannot follow is a *declared deviation*, not a failure:
+     * name it in `deviations` and the matching check is skipped with the reason
+     * in the test title, so the exemption stays visible instead of becoming
+     * folklore.
+     */
+    readonly conventions?: {
+        /** A test id the fixture sets by author annotation (rule 3). */
+        readonly annotatedTestId?: string;
+        /** A textbox whose field is empty, to prove `value: ''` (rule 5). */
+        readonly emptyTextboxTestId?: string;
+        /** A container with no name of its own, wrapping text (rule 2). */
+        readonly unnamedContainerTestId?: string;
+        /**
+         * Test ids whose `value` is author-annotated. The role gate in rule 5
+         * bounds *derived* values; an annotation may put one on any role, and only
+         * the registration knows which is which.
+         */
+        readonly annotatedValues?: readonly string[];
+        /**
+         * The adapter's README. Its `## Deviations` section is the single source of
+         * truth for what this adapter cannot do (rule 6), so a declared limitation
+         * is read from there rather than repeated in the registration — two copies
+         * of the same fact disagree eventually, and the README is the one a user
+         * reads.
+         */
+        readonly readmePath?: string;
+    };
+    readonly logs?: {
+        /**
+         * Input that makes the application write a record. Omit it for an app that
+         * logs on its own (at startup, say) — the obligation then waits for a
+         * record rather than provoking one.
+         */
+        readonly input?: string;
+        /** A substring of the logged message, used to prove it stayed off-screen. */
+        readonly expect: string;
+    };
+    /** How long the handshake may take. Default 10 s. */
+    readonly timeoutMs?: number;
+    /**
+     * A command that must succeed before this adapter can be certified here —
+     * its interpreter, or a build step that produces the binary `spawn` runs.
+     * When it fails the whole suite skips and the reason is in the block's name,
+     * exactly as a missing pseudo-terminal does.
+     */
+    readonly requires?: {
+        readonly probe: readonly string[];
+        readonly label: string;
+        readonly cwd?: string;
+        readonly timeoutMs?: number;
+    };
+}
+/**
+ * Rule numbers declared in an adapter's `## Deviations` section.
+ *
+ * Three shapes are in use and all are accepted, because the suite should not
+ * dictate anyone's prose: `**Rule 2 — …**` (a heading per entry, Ink),
+ * `- **…** (rule 3).` (the number at the end of a bullet, the language
+ * clients), and a markdown table whose first column is `2 — …` (OpenTUI).
+ *
+ * Entries that name no rule are kept under `other`: they are still declared
+ * limitations, and dropping them would make the roll-up quietly incomplete.
+ */
+declare function parseDeclaredDeviations(readme: string): Map<string, string[]>;
+/**
+ * Registers the adapter contract suite for one adapter.
+ *
+ * Call it at the top level of a test file; it declares its own `describe`.
+ *
+ * `vitest` is imported dynamically, so the package stays importable from a
+ * plain script that only wants the fixture paths or the probe. That is why the
+ * function is async: `await` it at the top level of the test file, which is
+ * where vitest collects the suite from.
+ *
+ * @example
+ * ```ts
+ * await runAdapterConformance({
+ *   name: 'my-framework-probe',
+ *   spawn: () => ({ command: ['node', 'app.mjs'] }),
+ *   ready: 'Ready',
+ *   interaction: { input: '\t', expect: '[Save]' },
+ *   quit: { input: 'q', exitCode: 0 },
+ * });
+ * ```
+ */
+declare function runAdapterConformance(options: AdapterConformanceOptions): Promise<void>;
+
+/**
+ * Absolute path of a fixture that ships with this package.
+ *
+ * The fixtures are shipped as sources rather than bundled, because they are
+ * meant to be launched as `node <path>` by suites in other packages and in
+ * other languages' CI. They are therefore located from the package root, which
+ * is the one anchor that is the same whether this module was loaded from `src/`
+ * during development or from the bundled `dist/`.
+ */
+declare function fixturePath(name: string): string;
+/** The fixtures published for adapter authors and other packages' suites. */
+declare const CONFORMANCE_FIXTURES: Readonly<{
+    /** Uninstrumented app: proves the generic fallback (§20.1). */
+    generic: () => string;
+    /** Shell-shaped app emitting OSC 133 marks; `--marks=off` suppresses them. */
+    prompt: () => string;
+    /** Normal-render Ink app used to exercise launch-time probe attachment. */
+    inkProbe: () => string;
+    /** Hostile wire peer; takes a scenario name as its first argument (§20.3). */
+    adversarialPeer: () => string;
+}>;
+/**
+ * Whether this machine can open a pseudo-terminal at all.
+ *
+ * @returns `false` when `TERMWRIGHT_SKIP_PTY=1` or spawning a trivial child
+ * through the PTY backend throws. Probed once and cached.
+ */
+declare function ptyAvailable(): boolean;
+/** `process.env` with the `undefined` values dropped, as PTY spawning requires. */
+declare function environment(extra?: Readonly<Record<string, string>>): Record<string, string>;
+/** Options every conformance session shares; suites override what they need. */
+interface FixtureLaunchOptions extends Partial<Omit<LaunchOptions, 'command'>> {
+    /** Extra arguments appended to `node <fixture>`. */
+    readonly args?: readonly string[];
+    /** Attach the zero-config framework probe to the otherwise normal command. */
+    readonly probe?: 'ink';
+    /**
+     * Text that proves the fixture started drawing. Waiting for it here rather
+     * than in each suite is what lets the failure be diagnosed: a fixture that
+     * printed nothing at all failed to start, which is a very different problem
+     * from one that started and drew the wrong thing.
+     */
+    readonly ready?: string | RegExp;
+}
+/**
+ * A set of sessions closed together, so one wedged fixture cannot leak a child
+ * process into the next test.
+ *
+ * @example
+ * ```ts
+ * const sessions = createSessionPool();
+ * afterEach(sessions.closeAll);
+ * const terminal = await sessions.launch(CONFORMANCE_FIXTURES.generic());
+ * ```
+ */
+interface SessionPool {
+    launch(fixture: string, options?: FixtureLaunchOptions): Promise<TerminalHarness>;
+    closeAll(): Promise<void>;
+}
+declare function createSessionPool(): SessionPool;
+
+export { type AdapterCommand, type AdapterConformanceOptions, AdapterProbe, CONFORMANCE_FIXTURES, type FixtureLaunchOptions, MARKER_TEXT_PREFIX, type ProbeObservation, type ProbeOptions, type RecordedFault, type RecordedMarker, type RecordedMessage, type SessionPool, createSessionPool, environment, fixturePath, parseDeclaredDeviations, ptyAvailable, runAdapterConformance };