@frockbot/plugin-models 0.3.14 → 0.3.17

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@frockbot/plugin-models",
-  "version": "0.3.14",
+  "version": "0.3.17",
   "private": false,
   "type": "module",
   "exports": {
@@ -17,7 +17,7 @@
     "typecheck": "tsc --noEmit -p tsconfig.json"
   },
   "dependencies": {
-    "@frockbot/kernel-contracts": "0.3.14",
+    "@frockbot/kernel-contracts": "0.3.17",
     "cordis": "4.0.0-rc.8"
   },
   "devDependencies": {

package/src/llm.ts

@@ -6,6 +6,10 @@ import {
   type LlmStreamEvent,
   type ModelInvocation,
   type NormalizedModelRequest,
+  type JsonSchemaResponseFormatV1,
+  parseStructuredOutputJsonV1,
+  type StructuredModelResultV1,
+  validateStructuredOutputV1,
 } from "@frockbot/kernel-contracts";
 
 export class LlmRegistry extends Service implements ModelInvocation {
@@ -44,9 +48,72 @@ export class LlmRegistry extends Service implements ModelInvocation {
       throw new LlmEffectNotStartedError(
         `LLM provider "${request.provider}" is unavailable`,
       );
-    return this.ctx.waterfall("llm/stream", request, signal, () =>
+    const events = this.ctx.waterfall("llm/stream", request, signal, () =>
       provider.stream(request, signal),
     );
+    return this.validatedStream(request, events);
+  }
+
+  private async *validatedStream(
+    request: NormalizedModelRequest,
+    events: AsyncIterable<LlmStreamEvent>,
+  ): AsyncIterable<LlmStreamEvent> {
+    let text = "";
+    let sawStructuredFailure = false;
+    let validatedResponseFormat = false;
+    for await (const event of events) {
+      if (event.type === "text-delta") text += event.text;
+      if (event.type === "structured-output-failure") {
+        sawStructuredFailure = true;
+      }
+      if (
+        event.type === "finish" &&
+        request.responseFormat &&
+        !sawStructuredFailure
+      ) {
+        validatedResponseFormat = true;
+        const failure = responseFormatFailureV1(request, text);
+        if (failure) yield { type: "structured-output-failure", failure };
+      }
+      yield event;
+    }
+    if (
+      request.responseFormat &&
+      !sawStructuredFailure &&
+      !validatedResponseFormat
+    ) {
+      const failure = responseFormatFailureV1(request, text);
+      if (failure) yield { type: "structured-output-failure", failure };
+    }
+  }
+
+  async structured<T>(
+    request: NormalizedModelRequest,
+    format: Omit<JsonSchemaResponseFormatV1, "type">,
+    signal: AbortSignal,
+  ): Promise<StructuredModelResultV1<T>> {
+    const structuredRequest: NormalizedModelRequest = {
+      ...request,
+      responseFormat: { type: "json_schema", ...format },
+    };
+    let raw = "";
+    let failure:
+      | Extract<
+          Awaited<ReturnType<typeof validateStructuredOutputV1>>,
+          { status: "failed" }
+        >["failure"]
+      | undefined;
+    for await (const event of this.stream(structuredRequest, signal)) {
+      if (event.type === "text-delta") raw += event.text;
+      if (event.type === "structured-output-failure") {
+        failure = event.failure;
+      }
+    }
+    if (failure) return { status: "failed", failure, raw };
+    const validated = validateStructuredOutputV1(raw, format.schema);
+    if (validated.status === "failed") return validated;
+    // SAFETY: callers choose T alongside the schema that was just validated.
+    return { status: "completed", value: validated.value as T, raw };
   }
 
   async reconcile(
@@ -69,10 +136,31 @@ export class LlmRegistry extends Service implements ModelInvocation {
       };
     }
     try {
-      return await provider.reconciliation.retrieve(
+      const outcome = await provider.reconciliation.retrieve(
         { providerEffectId: request.requestId, request },
         signal,
       );
+      if (outcome.status !== "recovered" || !request.responseFormat) {
+        return outcome;
+      }
+      const raw = outcome.events
+        .filter(
+          (event): event is Extract<LlmStreamEvent, { type: "text-delta" }> =>
+            event.type === "text-delta",
+        )
+        .map((event) => event.text)
+        .join("");
+      const failure = responseFormatFailureV1(request, raw);
+      if (!failure) return outcome;
+      const finish = outcome.events.findIndex(
+        (event) => event.type === "finish",
+      );
+      const events = [...outcome.events];
+      events.splice(finish < 0 ? events.length : finish, 0, {
+        type: "structured-output-failure",
+        failure,
+      });
+      return { status: "recovered", events };
     } catch (error) {
       signal.throwIfAborted();
       return {
@@ -85,3 +173,18 @@ export class LlmRegistry extends Service implements ModelInvocation {
     }
   }
 }
+
+function responseFormatFailureV1(
+  request: NormalizedModelRequest,
+  text: string,
+):
+  | Extract<StructuredModelResultV1<unknown>, { status: "failed" }>["failure"]
+  | undefined {
+  const format = request.responseFormat;
+  if (!format) return undefined;
+  const result =
+    format.type === "json_schema"
+      ? validateStructuredOutputV1(text, format.schema)
+      : parseStructuredOutputJsonV1(text);
+  return result.status === "failed" ? result.failure : undefined;
+}

package/src/structured-output.integration.test.ts

@@ -0,0 +1,134 @@
+import { describe, expect, test } from "bun:test";
+import {
+  type LlmProvider,
+  type NormalizedModelRequest,
+} from "@frockbot/kernel-contracts";
+import { Context } from "cordis";
+import { LlmRegistry } from "./llm.js";
+
+const schema = {
+  type: "object" as const,
+  properties: { answer: { type: "string" as const } },
+  required: ["answer"],
+  additionalProperties: false,
+};
+
+function request(): NormalizedModelRequest {
+  return {
+    requestId: "request-1",
+    provider: "fake-structured",
+    model: "fake-model",
+    system: "Answer briefly.",
+    messages: [{ role: "user", content: "Is this typed?" }],
+    tools: [],
+  };
+}
+
+function fakeProvider(content: string): LlmProvider {
+  return {
+    id: "fake-structured",
+    supports: { structuredOutput: "json_schema" },
+    async *stream(modelRequest) {
+      expect(modelRequest.responseFormat).toEqual({
+        type: "json_schema",
+        name: "answer",
+        schema,
+      });
+      yield { type: "text-delta", text: content };
+      yield { type: "finish", reason: "completed" };
+    },
+  };
+}
+
+async function mounted(content: string): Promise<Context> {
+  const root = new Context();
+  await root.plugin(LlmRegistry);
+  root.llm.register(fakeProvider(content));
+  return root;
+}
+
+describe("structured output through a fake provider", () => {
+  test("round-trips a validated typed value", async () => {
+    const root = await mounted('{"answer":"yes"}');
+    const result = await root.llm.structured<{ answer: string }>(
+      request(),
+      { name: "answer", schema },
+      new AbortController().signal,
+    );
+    expect(result).toEqual({
+      status: "completed",
+      value: { answer: "yes" },
+      raw: '{"answer":"yes"}',
+    });
+    await root.fiber.dispose();
+  });
+
+  test("emits and returns a typed validation failure", async () => {
+    const root = await mounted('{"answer":4}');
+    const events = [];
+    for await (const event of root.llm.stream(
+      {
+        ...request(),
+        responseFormat: { type: "json_schema", name: "answer", schema },
+      },
+      new AbortController().signal,
+    )) {
+      events.push(event);
+    }
+    expect(events.at(-2)).toEqual({
+      type: "structured-output-failure",
+      failure: {
+        code: "schema-mismatch",
+        message: "The model response did not match the requested schema",
+        issues: [
+          {
+            path: "$.answer",
+            code: "type",
+            message: "$.answer must be a string",
+          },
+        ],
+      },
+    });
+
+    const result = await root.llm.structured<{ answer: string }>(
+      { ...request(), requestId: "request-2" },
+      { name: "answer", schema },
+      new AbortController().signal,
+    );
+    expect(result.status).toBe("failed");
+    if (result.status === "failed") {
+      expect(result.failure.code).toBe("schema-mismatch");
+    }
+    await root.fiber.dispose();
+  });
+
+  test("emits a typed invalid-JSON failure for schema-free JSON mode", async () => {
+    const root = new Context();
+    await root.plugin(LlmRegistry);
+    root.llm.register({
+      id: "fake-json",
+      supports: { structuredOutput: "json" },
+      async *stream(modelRequest) {
+        expect(modelRequest.responseFormat).toEqual({ type: "json" });
+        yield { type: "text-delta", text: "not json" };
+        yield { type: "finish", reason: "completed" };
+      },
+    });
+    const events = [];
+    for await (const event of root.llm.stream(
+      {
+        ...request(),
+        provider: "fake-json",
+        responseFormat: { type: "json" },
+      },
+      new AbortController().signal,
+    )) {
+      events.push(event);
+    }
+    expect(events.at(-2)).toMatchObject({
+      type: "structured-output-failure",
+      failure: { code: "invalid-json" },
+    });
+    await root.fiber.dispose();
+  });
+});