@frockbot/plugin-provider-frock-ai 0.3.16 → 0.3.18

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@frockbot/plugin-provider-frock-ai",
-  "version": "0.3.16",
+  "version": "0.3.18",
   "private": false,
   "type": "module",
   "exports": {
@@ -20,12 +20,12 @@
     "typecheck": "tsc --noEmit -p tsconfig.json"
   },
   "dependencies": {
-    "@frockbot/configuration-core": "0.3.16",
-    "@frockbot/connection-core": "0.3.16",
-    "@frockbot/kernel-agent-loop": "0.3.16",
-    "@frockbot/kernel-contracts": "0.3.16",
-    "@frockbot/plugin-models": "0.3.16",
-    "@frockbot/provider-openai-compatible": "0.3.16",
+    "@frockbot/configuration-core": "0.3.18",
+    "@frockbot/connection-core": "0.3.18",
+    "@frockbot/kernel-agent-loop": "0.3.18",
+    "@frockbot/kernel-contracts": "0.3.18",
+    "@frockbot/plugin-models": "0.3.18",
+    "@frockbot/provider-openai-compatible": "0.3.18",
     "cordis": "4.0.0-rc.8"
   },
   "devDependencies": {

package/src/catalog.test.ts

@@ -5,6 +5,8 @@ import {
   frockAiStaticCatalogV1,
   frockModelIdForCloudflareIdV1,
   gatewayModelForFrockIdV1,
+  gatewayModelForFrockRequestV1,
+  FROCK_AI_STRUCTURED_MODEL,
   normalizeFrockModelIdV1,
 } from "./catalog.js";
 
@@ -43,6 +45,15 @@ describe("Frock AI catalog", () => {
     ).toBe("dynamic/production-auto");
   });
 
+  test("pins Auto schema work to a Workers AI model that supports it", () => {
+    expect(gatewayModelForFrockRequestV1(FROCK_AI_DEFAULT_MODEL, true)).toBe(
+      FROCK_AI_STRUCTURED_MODEL,
+    );
+    expect(gatewayModelForFrockRequestV1(FROCK_AI_DEFAULT_MODEL, false)).toBe(
+      "dynamic/flock-auto",
+    );
+  });
+
   test("rejects ids outside the Frock AI namespace", () => {
     expect(() => gatewayModelForFrockIdV1("@cf/not/frock")).toThrow(
       'must start with "@frock/"',

package/src/catalog.ts

@@ -17,6 +17,9 @@ export const FROCK_AI_DEFAULT_MODEL = "@frock/auto";
  * named `flock-auto`; the value is the resource's name, not ours.
  */
 export const FROCK_AI_DEFAULT_AUTO_ROUTE = "flock-auto";
+/** Workers AI model selected when Auto must honor a JSON Schema request. */
+export const FROCK_AI_STRUCTURED_MODEL =
+  "workers-ai/@cf/meta/llama-3.3-70b-instruct-fp8-fast";
 
 /** The pre-rename model-id prefix. Bots bound before the rename still carry it. */
 export const FROCK_AI_LEGACY_MODEL_PREFIX = "@flock/";
@@ -86,6 +89,18 @@ export function gatewayModelForFrockIdV1(
   return `workers-ai/${cloudflareModelIdForFrockIdV1(id)}`;
 }
 
+/** Auto routes ordinary chat dynamically and pins schema work to a capable model. */
+export function gatewayModelForFrockRequestV1(
+  input: string,
+  structured: boolean,
+  autoRoute = FROCK_AI_DEFAULT_AUTO_ROUTE,
+): string {
+  const id = normalizeFrockModelIdV1(input);
+  return structured && id === FROCK_AI_DEFAULT_MODEL
+    ? FROCK_AI_STRUCTURED_MODEL
+    : gatewayModelForFrockIdV1(id, autoRoute);
+}
+
 const STATIC_CATALOG: ConnectionModelCatalogV1 = {
   schemaVersion: 1,
   generation: "flock-ai-static-v1",

package/src/runtime.test.ts

@@ -294,6 +294,7 @@ describe("Frock AI runtime Contribution", () => {
             sse(
               'data: {"choices":[{"delta":{"content":"Working"}}]}\n\n' +
                 'data: {"choices":[{"delta":{"tool_calls":[{"index":0,"id":"call-1","function":{"name":"weather","arguments":"{\\"city\\":\\"Sydney\\"}"}}]},"finish_reason":"tool_calls"}]}\n\n' +
+                'data: {"choices":[],"usage":{"input_tokens":18,"output_tokens":6,"input_tokens_details":{"cached_tokens":4},"output_tokens_details":{"reasoning_tokens":2}}}\n\n' +
                 "data: [DONE]\n\n",
             ),
           );
@@ -314,6 +315,7 @@ describe("Frock AI runtime Contribution", () => {
         gatewayModel: "dynamic/configured-auto",
         body: {
           stream: true,
+          stream_options: { include_usage: true },
           messages: [
             { role: "system", content: "Be concise." },
             { role: "user", content: "hello" },
@@ -333,6 +335,15 @@ describe("Frock AI runtime Contribution", () => {
     ]);
     expect(events).toEqual([
       { type: "text-delta", text: "Working" },
+      {
+        type: "usage",
+        usage: {
+          inputTokens: 18,
+          outputTokens: 6,
+          cachedInputTokens: 4,
+          reasoningTokens: 2,
+        },
+      },
       {
         type: "tool-call",
         call: {
@@ -346,6 +357,67 @@ describe("Frock AI runtime Contribution", () => {
     await root.fiber.dispose();
   });
 
+  test("sends Auto schemas in Workers AI's direct non-stream shape", async () => {
+    let call:
+      { gatewayModel: string; body: Record<string, unknown> } | undefined;
+    const root = new Context();
+    await root.plugin(LlmRegistry);
+    await root.plugin(
+      createFrockAiRuntimePlugin(
+        runtimeConfig((gatewayModel, body) => {
+          call = { gatewayModel, body };
+          return Promise.resolve(
+            new Response(
+              JSON.stringify({
+                choices: [
+                  {
+                    message: { content: '{"answer":"yes"}' },
+                    finish_reason: "stop",
+                  },
+                ],
+              }),
+            ).body!,
+          );
+        }),
+      ),
+    );
+    const events = [];
+    for await (const event of root.llm.stream(
+      {
+        ...request,
+        responseFormat: {
+          type: "json_schema",
+          name: "answer",
+          schema: {
+            type: "object",
+            properties: { answer: { type: "string" } },
+            required: ["answer"],
+            additionalProperties: false,
+          },
+        },
+      },
+      new AbortController().signal,
+    )) {
+      events.push(event);
+    }
+    expect(call?.gatewayModel).toContain("llama-3.3-70b-instruct-fp8-fast");
+    expect(call?.body.stream).toBe(false);
+    expect(call?.body.response_format).toEqual({
+      type: "json_schema",
+      json_schema: {
+        type: "object",
+        properties: { answer: { type: "string" } },
+        required: ["answer"],
+        additionalProperties: false,
+      },
+    });
+    expect(events).toEqual([
+      { type: "text-delta", text: '{"answer":"yes"}' },
+      { type: "finish", reason: "completed" },
+    ]);
+    await root.fiber.dispose();
+  });
+
   test("refuses a request outside its pinned Connection generation", async () => {
     let calls = 0;
     const root = new Context();

package/src/runtime.ts

@@ -2,6 +2,7 @@ import {
   boundedModelProviderReasonV1,
   type LlmProvider,
   type LlmReconciliationCapability,
+  type LlmStreamEvent,
   ModelProviderFailureError,
   type ModelProviderFailureClassV1,
   ModelRequestDeadlineError,
@@ -10,7 +11,7 @@ import {
 import {
   classifyOpenAICompatibleFailureV1,
   type ModelRequestDeadlineOptionsV1,
-  requestToWire,
+  planOpenAICompatibleRequestV1,
   streamWithModelRequestDeadlinesV1,
 } from "@frockbot/provider-openai-compatible";
 import type { Agent } from "@frockbot/kernel-agent-loop/agent";
@@ -18,7 +19,8 @@ import type { Plugin } from "cordis";
 import {
   FROCK_AI_DEFAULT_MODEL,
   FROCK_AI_PROVIDER_TYPE,
-  gatewayModelForFrockIdV1,
+  gatewayModelForFrockRequestV1,
+  normalizeFrockModelIdV1,
 } from "./catalog.js";
 
 export type OpenAICompatibleChatCompletionBodyV1 = Record<string, unknown>;
@@ -125,6 +127,7 @@ export interface FrockAiRuntimeConfig {
 
 class FrockAiProvider implements LlmProvider {
   readonly id = FROCK_AI_PROVIDER_TYPE;
+  readonly supports = { structuredOutput: "json_schema" } as const;
   readonly autoFallbackFailures = new WeakSet<ModelProviderFailureError>();
 
   /**
@@ -143,7 +146,10 @@ class FrockAiProvider implements LlmProvider {
 
   constructor(private readonly config: FrockAiRuntimeConfig) {}
 
-  async *stream(request: NormalizedModelRequest, signal: AbortSignal) {
+  async *stream(
+    request: NormalizedModelRequest,
+    signal: AbortSignal,
+  ): AsyncIterable<LlmStreamEvent> {
     const binding = request.modelBinding;
     if (
       binding?.connectionId !== this.config.connectionId ||
@@ -155,10 +161,17 @@ class FrockAiProvider implements LlmProvider {
       });
     }
     signal.throwIfAborted();
-    const wire = requestToWire(request);
-    const { model: _model, ...body } = wire;
-    const gatewayModel = gatewayModelForFrockIdV1(
+    const auto =
+      normalizeFrockModelIdV1(request.model) === FROCK_AI_DEFAULT_MODEL;
+    const plan = planOpenAICompatibleRequestV1(request, {
+      structuredOutput: auto ? "json_schema" : "none",
+      responseFormatDialect: "workers-ai",
+    });
+    if (plan.note) yield { type: "response-format-note", note: plan.note };
+    const { model: _model, ...body } = plan.body;
+    const gatewayModel = gatewayModelForFrockRequestV1(
       request.model,
+      request.responseFormat !== undefined,
       this.config.autoRoute,
     );
     // A rejection here happened before a stream existed, so no provider effect