@openagentsinc/conformance-kit 0.2.0-rc.2

package/src/reducer-laws.test.ts

@@ -0,0 +1,17 @@
+import {
+  applyUiChunk,
+  initialUiMessage,
+  reduceUiMessageStream,
+  UiMessageReducerError,
+} from "@openagentsinc/agent-harness-contract";
+
+import { runReducerLaws } from "./reducer-laws.ts";
+
+// The kit run against the in-repo reference reducer proves the kit itself works.
+runReducerLaws({
+  label: "ui-message-reducer",
+  initial: () => initialUiMessage(),
+  applyChunk: applyUiChunk,
+  isReducerError: (error) => error instanceof UiMessageReducerError,
+  reduceStream: (stream) => reduceUiMessageStream(stream),
+});

package/src/reducer-laws.ts

@@ -0,0 +1,277 @@
+import { Effect, Stream } from "effect";
+import {
+  toolIdentity,
+  type UiMessage,
+  type UiMessageChunk,
+  type UiMessageStreamHandle,
+} from "@openagentsinc/agent-harness-contract";
+import { describe, expect, test } from "vite-plus/test";
+
+const BASH = toolIdentity("Bash");
+
+/**
+ * Configuration for {@link runReducerLaws}. The implementation under test is a
+ * progressive reducer: a pure `applyChunk` fold step plus its `initial`
+ * message. The kit feeds known chunk scripts and asserts the progressive
+ * snapshot laws hold. `isReducerError` recognizes the implementation's tagged
+ * error so a malformed sequence is proven to fail loudly, never to silently
+ * corrupt state.
+ */
+export interface ReducerLawsConfig {
+  /** A short label naming the implementation under test, used in test titles. */
+  readonly label: string;
+  /** The empty starting message. */
+  readonly initial: () => UiMessage;
+  /** The pure fold step. Must throw the tagged reducer error on a malformed sequence. */
+  readonly applyChunk: (message: UiMessage, chunk: UiMessageChunk) => UiMessage;
+  /** Recognizes the implementation's tagged reducer error. */
+  readonly isReducerError: (error: unknown) => boolean;
+  /**
+   * Optional stream reducer. When present, the kit also proves the streamed
+   * fold publishes the same progressive snapshots the pure fold produces and
+   * `done` equals the pure fold's final state.
+   */
+  readonly reduceStream?: (
+    stream: Stream.Stream<UiMessageChunk>,
+  ) => Effect.Effect<UiMessageStreamHandle<never>>;
+}
+
+/**
+ * The progressive-reducer snapshot laws, parameterized over any reducer.
+ *
+ * Promoted, published form of
+ * `agent-harness-contract/src/ui-message-reducer.test.ts`:
+ *
+ * - **Progressive fold.** Text grows across intermediate snapshots and closes
+ *   on finish; status advances `streaming -> complete`. Every intermediate
+ *   state is observable, not just the final one.
+ * - **Tool state machine.** `input-streaming -> input-available ->
+ *   output-available` transitions land in order with accumulated input text.
+ * - **Transient bypass.** A transient chunk never mutates the persisted message.
+ * - **Fail loud, never corrupt.** A malformed sequence (ghost output, state
+ *   regression, text after end) throws the tagged reducer error.
+ */
+export const runReducerLaws = (config: ReducerLawsConfig): void => {
+  const { label, initial, applyChunk, isReducerError } = config;
+
+  /** Fold chunks into every intermediate snapshot, starting from the initial. */
+  const snapshotsOf = (
+    chunks: ReadonlyArray<UiMessageChunk>,
+    start: UiMessage = initial(),
+  ): ReadonlyArray<UiMessage> => {
+    const snapshots: Array<UiMessage> = [start];
+    let current = start;
+    for (const chunk of chunks) {
+      current = applyChunk(current, chunk);
+      snapshots.push(current);
+    }
+    return snapshots;
+  };
+
+  const textOf = (message: UiMessage): string =>
+    message.parts
+      .filter((part) => part.type === "text")
+      .map((part) => (part.type === "text" ? part.text : ""))
+      .join("");
+
+  const toolStateOf = (message: UiMessage): string | undefined => {
+    const part = message.parts.find((candidate) => candidate.type === "tool");
+    return part?.type === "tool" ? part.state : undefined;
+  };
+
+  const toolInputTextOf = (message: UiMessage): string | undefined => {
+    const part = message.parts.find((candidate) => candidate.type === "tool");
+    return part?.type === "tool" ? part.inputText : undefined;
+  };
+
+  describe(`[${label}] reducer — progressive snapshots`, () => {
+    test("text grows across intermediate snapshots and closes on finish", () => {
+      const chunks: ReadonlyArray<UiMessageChunk> = [
+        { type: "message-start", messageId: "t1" },
+        { type: "text-delta", id: "msg.t1", delta: "Hello " },
+        { type: "text-delta", id: "msg.t1", delta: "world" },
+        { type: "text-end", id: "msg.t1" },
+        { type: "message-finish", finishReason: "stop" },
+      ];
+      const snapshots = snapshotsOf(chunks);
+
+      expect(snapshots.map(textOf)).toEqual([
+        "",
+        "",
+        "Hello ",
+        "Hello world",
+        "Hello world",
+        "Hello world",
+      ]);
+      expect(snapshots.map((snapshot) => snapshot.status)).toEqual([
+        "streaming",
+        "streaming",
+        "streaming",
+        "streaming",
+        "streaming",
+        "complete",
+      ]);
+      expect(snapshots.at(-1)?.finishReason).toBe("stop");
+    });
+
+    test("the tool-call state machine transitions land in order with accumulated input", () => {
+      const chunks: ReadonlyArray<UiMessageChunk> = [
+        {
+          type: "tool-input-streaming",
+          toolCallId: "call.1",
+          tool: BASH,
+          inputTextDelta: '{"command":',
+        },
+        { type: "tool-input-streaming", toolCallId: "call.1", tool: BASH, inputTextDelta: '"ls"}' },
+        {
+          type: "tool-input-available",
+          toolCallId: "call.1",
+          tool: BASH,
+          inputRef: "input.call.1",
+        },
+        {
+          type: "tool-output-available",
+          toolCallId: "call.1",
+          tool: BASH,
+          resultRef: "result.call.1",
+        },
+      ];
+      const snapshots = snapshotsOf(chunks).slice(1);
+
+      expect(snapshots.map(toolStateOf)).toEqual([
+        "input-streaming",
+        "input-streaming",
+        "input-available",
+        "output-available",
+      ]);
+      expect(snapshots.map(toolInputTextOf)).toEqual([
+        '{"command":',
+        '{"command":"ls"}',
+        '{"command":"ls"}',
+        '{"command":"ls"}',
+      ]);
+    });
+
+    test("tool-output-error lands the error state with the safe text", () => {
+      const chunks: ReadonlyArray<UiMessageChunk> = [
+        { type: "tool-input-available", toolCallId: "call.1", tool: BASH },
+        {
+          type: "tool-output-error",
+          toolCallId: "call.1",
+          tool: BASH,
+          errorText: "command failed",
+          errorRef: "error.call.1",
+        },
+      ];
+      const final = snapshotsOf(chunks).at(-1)!;
+      const part = final.parts.find((candidate) => candidate.type === "tool");
+      expect(part?.type === "tool" ? part.state : undefined).toBe("output-error");
+      const errorText =
+        part?.type === "tool" && part.state === "output-error" ? part.errorText : undefined;
+      expect(errorText).toBe("command failed");
+    });
+
+    test("transient chunks bypass the persisted message", () => {
+      const message = applyChunk(initial(), {
+        type: "text-delta",
+        id: "m",
+        delta: "ephemeral",
+        transient: true,
+      });
+      expect(message).toEqual(initial());
+    });
+  });
+
+  describe(`[${label}] reducer — fail loud, never corrupt`, () => {
+    test("output for a tool call that never streamed input throws the tagged error", () => {
+      let thrown: unknown;
+      try {
+        applyChunk(initial(), {
+          type: "tool-output-available",
+          toolCallId: "call.ghost",
+          tool: BASH,
+          resultRef: "result.ghost",
+        });
+      } catch (error) {
+        thrown = error;
+      }
+      expect(isReducerError(thrown)).toBe(true);
+    });
+
+    test("a state regression (input after output) throws the tagged error", () => {
+      const settled = snapshotsOf([
+        { type: "tool-input-available", toolCallId: "call.1", tool: BASH },
+        { type: "tool-output-available", toolCallId: "call.1", tool: BASH, resultRef: "r.1" },
+      ]).at(-1)!;
+      let thrown: unknown;
+      try {
+        applyChunk(settled, {
+          type: "tool-input-streaming",
+          toolCallId: "call.1",
+          tool: BASH,
+          inputTextDelta: "more",
+        });
+      } catch (error) {
+        thrown = error;
+      }
+      expect(isReducerError(thrown)).toBe(true);
+    });
+
+    test("text delta after text end throws the tagged error", () => {
+      const closed = snapshotsOf([
+        { type: "text-delta", id: "m", delta: "a" },
+        { type: "text-end", id: "m" },
+      ]).at(-1)!;
+      let thrown: unknown;
+      try {
+        applyChunk(closed, { type: "text-delta", id: "m", delta: "b" });
+      } catch (error) {
+        thrown = error;
+      }
+      expect(isReducerError(thrown)).toBe(true);
+    });
+  });
+
+  if (config.reduceStream !== undefined) {
+    const reduceStream = config.reduceStream;
+    describe(`[${label}] reducer — streamed fold equals pure fold`, () => {
+      test("done equals the pure fold's final state", async () => {
+        const chunks: ReadonlyArray<UiMessageChunk> = [
+          { type: "message-start", messageId: "t1" },
+          { type: "text-delta", id: "msg.t1", delta: "Hello " },
+          { type: "text-delta", id: "msg.t1", delta: "world" },
+          { type: "text-end", id: "msg.t1" },
+          { type: "message-finish", finishReason: "stop" },
+        ];
+        const pureFinal = snapshotsOf(chunks).at(-1)!;
+
+        const streamedFinal = await Effect.runPromise(
+          Effect.gen(function* () {
+            const handle = yield* reduceStream(Stream.fromIterable(chunks));
+            return yield* handle.done;
+          }),
+        );
+        expect(streamedFinal).toEqual(pureFinal);
+      });
+
+      test("a malformed sequence fails done with the tagged error", async () => {
+        const outcome = await Effect.runPromise(
+          Effect.gen(function* () {
+            const handle = yield* reduceStream(
+              Stream.fromIterable<UiMessageChunk>([
+                {
+                  type: "tool-output-available",
+                  toolCallId: "call.ghost",
+                  tool: BASH,
+                  resultRef: "result.ghost",
+                },
+              ]),
+            );
+            return yield* Effect.flip(handle.done);
+          }),
+        );
+        expect(isReducerError(outcome)).toBe(true);
+      });
+    });
+  }
+};

package/src/rlm-cap-laws.test.ts

@@ -0,0 +1,11 @@
+import { makeRlm, rlmInlineCorpusSourceLayer } from "@openagentsinc/rlm";
+
+import { runRlmCapLaws } from "./rlm-cap-laws.ts";
+
+// The kit run against the in-repo reference RLM engine proves the kit itself
+// works.
+runRlmCapLaws({
+  label: "rlm-engine",
+  makeEngine: makeRlm,
+  corpusSourceLayer: rlmInlineCorpusSourceLayer,
+});

package/src/rlm-cap-laws.ts

@@ -0,0 +1,207 @@
+import { Effect, type Layer } from "effect";
+import {
+  buildInlineCorpusInput,
+  type MakeRlmOptions,
+  RLM_REQUEST_SCHEMA_ID,
+  RlmCorpusSource,
+  type RlmDeterministicRequest,
+  type RlmShape,
+  type RlmTerminalResult,
+} from "@openagentsinc/rlm";
+import { describe, expect, test } from "vite-plus/test";
+
+/** A small inline corpus: four public entries that all contain the word "fact". */
+const corpus = buildInlineCorpusInput({
+  corpusRef: "corpus.conformance",
+  scopeRef: "scope.conformance",
+  entries: [
+    {
+      scopeRef: "scope.conformance",
+      sourceKind: "fixture",
+      sourceAddress: { addressSchemaId: "test.addr.v1", encodedAddress: "a0" },
+      text: "alpha fact one",
+      visibility: "public",
+      redactionClass: "none",
+    },
+    {
+      scopeRef: "scope.conformance",
+      sourceKind: "fixture",
+      sourceAddress: { addressSchemaId: "test.addr.v1", encodedAddress: "a1" },
+      text: "beta fact two",
+      visibility: "public",
+      redactionClass: "none",
+    },
+    {
+      scopeRef: "scope.conformance",
+      sourceKind: "fixture",
+      sourceAddress: { addressSchemaId: "test.addr.v1", encodedAddress: "a2" },
+      text: "gamma fact three",
+      visibility: "public",
+      redactionClass: "none",
+    },
+    {
+      scopeRef: "scope.conformance",
+      sourceKind: "fixture",
+      sourceAddress: { addressSchemaId: "test.addr.v1", encodedAddress: "a3" },
+      text: "delta fact four",
+      visibility: "public",
+      redactionClass: "none",
+    },
+  ],
+});
+
+const GENEROUS = {
+  maxEntriesScanned: 10_000,
+  maxSpans: 64,
+  maxCharsPerSpan: 2_048,
+  maxObservationChars: 8_192,
+} as const;
+
+const deterministicRequest = (params: {
+  readonly runRef: string;
+  readonly limits: RlmDeterministicRequest["limits"];
+}): RlmDeterministicRequest => ({
+  _tag: "Deterministic",
+  schemaId: RLM_REQUEST_SCHEMA_ID,
+  runRef: params.runRef,
+  corpus,
+  operation: { _tag: "Grep", pattern: "fact" },
+  limits: params.limits,
+});
+
+/**
+ * Configuration for {@link runRlmCapLaws}. The implementation under test is the
+ * recursive-language-model engine, supplied as its factory plus the corpus
+ * source layer it runs against. The kit drives deterministic runs whose caps
+ * bite, and asserts every cap surfaces as an honest `Partial` — never a
+ * `Completed` result that quietly dropped work.
+ */
+export interface RlmCapLawsConfig {
+  /** A short label naming the implementation under test, used in test titles. */
+  readonly label: string;
+  /** Build the engine shape (reference: `makeRlm`). */
+  readonly makeEngine: (options: MakeRlmOptions) => Effect.Effect<RlmShape, never, RlmCorpusSource>;
+  /** The corpus source layer the engine resolves against (reference: `rlmInlineCorpusSourceLayer`). */
+  readonly corpusSourceLayer: Layer.Layer<RlmCorpusSource>;
+}
+
+/**
+ * The RLM cap laws, parameterized over any engine + corpus source.
+ *
+ * Promoted from `rlm/src/conformance/paper-fidelity.test.ts` and the honesty
+ * contract in `rlm/src/schemas/request-result.ts`. The single invariant these
+ * laws defend is **no laundering**: every budget/cap that bites must surface as
+ * an honest `Partial` whose `honesty.capsHit` names the cap — the engine must
+ * never present a truncated run as a `Completed` result.
+ *
+ * - **Every deterministic cap → honest Partial.** A tight `maxSpans` or
+ *   `maxEntriesScanned` yields `Partial(cap_truncated)` with the cap in
+ *   `capsHit` and a `bestOutput` (the partial findings are surfaced, not
+ *   silently dropped).
+ * - **Generous caps → Completed with empty `capsHit`.**
+ * - **No laundering.** Across a matrix of limits, a non-empty `capsHit` is
+ *   never a `Completed` result, and a `Completed` result always has empty
+ *   `capsHit`.
+ * - **Deterministic never touches a model.** A deterministic run reports
+ *   `usage.modelCalls === 0` and never invokes the supplied model plan.
+ */
+export const runRlmCapLaws = (config: RlmCapLawsConfig): void => {
+  const { label, makeEngine, corpusSourceLayer } = config;
+
+  const run = (
+    request: RlmDeterministicRequest,
+    options: MakeRlmOptions = {},
+  ): Promise<RlmTerminalResult> =>
+    Effect.runPromise(
+      Effect.gen(function* () {
+        const shape = yield* makeEngine(options);
+        return yield* shape.run(request);
+      }).pipe(Effect.provide(corpusSourceLayer)),
+    );
+
+  describe(`[${label}] rlm — every cap yields an honest Partial`, () => {
+    test("maxSpans truncation is Partial(cap_truncated) naming the cap with a bestOutput", async () => {
+      const result = await run(
+        deterministicRequest({ runRef: "run.cap.spans", limits: { ...GENEROUS, maxSpans: 2 } }),
+      );
+      expect(result._tag).toBe("Partial");
+      if (result._tag === "Partial") {
+        expect(result.reason).toBe("cap_truncated");
+        expect(result.honesty.capsHit).toContain("maxSpans");
+        expect(result.bestOutput).toBeDefined();
+      }
+    });
+
+    test("maxEntriesScanned truncation is Partial(cap_truncated) naming the cap", async () => {
+      const result = await run(
+        deterministicRequest({
+          runRef: "run.cap.scanned",
+          limits: { ...GENEROUS, maxEntriesScanned: 2 },
+        }),
+      );
+      expect(result._tag).toBe("Partial");
+      if (result._tag === "Partial") {
+        expect(result.reason).toBe("cap_truncated");
+        expect(result.honesty.capsHit).toContain("maxEntriesScanned");
+      }
+    });
+
+    test("generous caps complete with an empty capsHit", async () => {
+      const result = await run(
+        deterministicRequest({ runRef: "run.complete", limits: { ...GENEROUS } }),
+      );
+      expect(result._tag).toBe("Completed");
+      if (result._tag === "Completed") {
+        expect(result.honesty.capsHit).toEqual([]);
+      }
+    });
+  });
+
+  describe(`[${label}] rlm — no laundering`, () => {
+    test("across a matrix of limits, capsHit is non-empty iff the result is Partial", async () => {
+      const cases: ReadonlyArray<{
+        readonly runRef: string;
+        readonly limits: RlmDeterministicRequest["limits"];
+      }> = [
+        { runRef: "m.0", limits: { ...GENEROUS } },
+        { runRef: "m.1", limits: { ...GENEROUS, maxSpans: 1 } },
+        { runRef: "m.2", limits: { ...GENEROUS, maxSpans: 3 } },
+        { runRef: "m.3", limits: { ...GENEROUS, maxEntriesScanned: 1 } },
+        { runRef: "m.4", limits: { ...GENEROUS, maxEntriesScanned: 3 } },
+      ];
+      for (const c of cases) {
+        const result = await run(deterministicRequest(c));
+        const capped = result.honesty.capsHit.length > 0;
+        if (capped) {
+          // A truncated run is never presented as Completed.
+          expect(result._tag).toBe("Partial");
+        } else {
+          expect(result._tag).toBe("Completed");
+        }
+      }
+    });
+  });
+
+  describe(`[${label}] rlm — deterministic never touches a model`, () => {
+    test("a deterministic run reports zero model calls and never invokes the model plan", async () => {
+      let modelTouched = false;
+      const result = await run(
+        deterministicRequest({ runRef: "run.no-model", limits: { ...GENEROUS } }),
+        {
+          admitSemantic: true,
+          model: {
+            completeRoot: () =>
+              Effect.sync(() => {
+                modelTouched = true;
+                return { text: "{}" };
+              }),
+          },
+        },
+      );
+      expect(modelTouched).toBe(false);
+      if (result._tag === "Completed" || result._tag === "Partial") {
+        expect(result.usage.modelCalls).toBe(0);
+      }
+    });
+  });
+};