@workglow/test-contract 0.6.1

package/dist/ai-provider.js

@@ -0,0 +1,876 @@
+// src/ai-provider/fixtures.ts
+var DEFAULT_CONFORMANCE_FIXTURE = {
+  textPrompt: "Say hello in one short sentence.",
+  weatherTool: {
+    name: "get_weather",
+    description: "Get the current weather for a given city.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        location: { type: "string", description: "City name, e.g. San Francisco" }
+      },
+      required: ["location"]
+    }
+  },
+  weatherToolPrompt: "What is the weather in San Francisco?",
+  multiTurnTranscript: [
+    { role: "user", text: "What is the weather in Tokyo?" },
+    { role: "assistant", text: "Let me check." },
+    { role: "tool", text: '{"temperature":22,"conditions":"sunny"}' }
+  ],
+  structuredSchema: {
+    type: "object",
+    properties: {
+      name: { type: "string" },
+      age: { type: "number" }
+    },
+    required: ["name", "age"],
+    additionalProperties: false
+  },
+  structuredPrompt: "Generate a JSON object with a person's name and age. Use name 'Alice' and age 30.",
+  maxTokens: 100,
+  abortGraceMs: 50
+};
+function resolveFixture(override) {
+  if (!override)
+    return DEFAULT_CONFORMANCE_FIXTURE;
+  return { ...DEFAULT_CONFORMANCE_FIXTURE, ...override };
+}
+// src/ai-provider/runAiProviderConformance.ts
+import { afterAll, beforeAll as beforeAll2, describe as describe10 } from "vitest";
+
+// src/ai-provider/assertions/capabilityHonesty.ts
+import { getAiProviderRegistry, getGlobalModelRepository, textEmbedding } from "@workglow/ai";
+import { describe, expect } from "vitest";
+
+// src/creditExhaustedSkip.ts
+import { it as vitestIt } from "vitest";
+var BILLING_CODES = new Set([
+  "insufficient_quota",
+  "billing_hard_limit_reached",
+  "insufficient_credits",
+  "insufficient_balance"
+]);
+var CREDIT_MESSAGE = /insufficient credits|not enough credits|no credits remaining|out of credits|insufficient[\s_-]balance|balance is (?:too low|insufficient)|exceeded[\s\S]{0,80}credits|monthly included credits|payment required|exceeded your current quota|check your plan and billing|billing_hard_limit/i;
+var HTTP_402_MESSAGE = /\bHTTP\/?\d?(?:\.\d)?\s*402\b|\bstatus(?:\s*code)?\s*[:=]\s*402\b|\berror code:?\s*402\b|(?:^|:\s+)402\s+[a-z]/im;
+function isCreditExhaustedError(err) {
+  const seen = new Set;
+  let current = err;
+  while (current !== undefined && current !== null && !seen.has(current)) {
+    seen.add(current);
+    if (matchesCreditExhaustion(current))
+      return true;
+    current = causeOf(current);
+  }
+  return false;
+}
+function matchesCreditExhaustion(err) {
+  if (typeof err === "string")
+    return messageLooksLikeCreditExhaustion(err);
+  if (!err || typeof err !== "object")
+    return false;
+  const rec = err;
+  const status = numericStatus(rec);
+  if (status === 402)
+    return true;
+  const code = stringish(rec.code) ?? stringish(nested(rec.error, "code"));
+  const type = stringish(rec.type) ?? stringish(nested(rec.error, "type"));
+  if (code && BILLING_CODES.has(code))
+    return true;
+  if (type && BILLING_CODES.has(type))
+    return true;
+  const message = [
+    stringish(rec.message),
+    stringish(nested(rec.error, "message")),
+    stringifyUnknown(err)
+  ].filter((part) => typeof part === "string").join(`
+`);
+  return messageLooksLikeCreditExhaustion(message);
+}
+function messageLooksLikeCreditExhaustion(message) {
+  return CREDIT_MESSAGE.test(message) || HTTP_402_MESSAGE.test(message);
+}
+function numericStatus(rec) {
+  for (const key of ["status", "statusCode"]) {
+    const value = rec[key];
+    if (typeof value === "number" && Number.isFinite(value))
+      return value;
+  }
+  const nestedStatus = nested(rec.error, "status");
+  if (typeof nestedStatus === "number" && Number.isFinite(nestedStatus))
+    return nestedStatus;
+  return;
+}
+function nested(value, key) {
+  if (!value || typeof value !== "object")
+    return;
+  return value[key];
+}
+function stringish(value) {
+  return typeof value === "string" && value.length > 0 ? value : undefined;
+}
+function stringifyUnknown(err) {
+  if (err instanceof Error)
+    return err.message;
+  return String(err);
+}
+function causeOf(err) {
+  if (!err || typeof err !== "object")
+    return;
+  return err.cause;
+}
+function creditSkipNote(err) {
+  const message = err instanceof Error ? err.message : String(err);
+  return `skipped: provider out of credits (${message})`;
+}
+var SKIP_OVERRIDE_ENV = "WORKGLOW_CREDIT_EXHAUSTED_SKIP";
+function isTruthyFlag(value) {
+  return !/^(?:0|false|off|no)$/i.test(value.trim());
+}
+function creditExhaustionSkipsTest(env = process.env) {
+  const override = env[SKIP_OVERRIDE_ENV];
+  if (override !== undefined && override.trim() !== "")
+    return isTruthyFlag(override);
+  for (const key of ["CI", "GITHUB_ACTIONS"]) {
+    const value = env[key];
+    if (value !== undefined && value.trim() !== "" && isTruthyFlag(value))
+      return true;
+  }
+  return false;
+}
+function reportLocalCreditFailure(err) {
+  const message = err instanceof Error ? err.message : String(err);
+  console.error(`provider out of credits (${message})
+` + `  Failing because this is not CI. Top the account up, or set ${SKIP_OVERRIDE_ENV}=1 to skip instead.`);
+}
+async function runWithCreditSkip(ctx, fn) {
+  try {
+    return await fn();
+  } catch (err) {
+    if (isCreditExhaustedError(err)) {
+      if (!creditExhaustionSkipsTest()) {
+        reportLocalCreditFailure(err);
+        throw err;
+      }
+      ctx.skip(creditSkipNote(err));
+      return;
+    }
+    throw err;
+  }
+}
+function wrapTestBodyForCreditSkip(fn) {
+  return async function wrapped(...args) {
+    try {
+      return await fn.apply(this, args);
+    } catch (err) {
+      if (!isCreditExhaustedError(err))
+        throw err;
+      if (!creditExhaustionSkipsTest()) {
+        reportLocalCreditFailure(err);
+        throw err;
+      }
+      const ctx = args[0];
+      if (ctx && typeof ctx.skip === "function") {
+        ctx.skip(creditSkipNote(err));
+        return;
+      }
+      console.warn(creditSkipNote(err));
+      return;
+    }
+  };
+}
+function wrapCallable(fn) {
+  return new Proxy(fn, {
+    apply(target, thisArg, argArray) {
+      const wrapped = argArray.map((arg) => typeof arg === "function" ? wrapTestBodyForCreditSkip(arg) : arg);
+      const result = Reflect.apply(target, thisArg, wrapped);
+      return typeof result === "function" ? wrapCallable(result) : result;
+    },
+    get(target, prop, receiver) {
+      const value = Reflect.get(target, prop, receiver);
+      if (typeof value === "function") {
+        return wrapCallable(value.bind(target));
+      }
+      return value;
+    }
+  });
+}
+var it = wrapCallable(vitestIt);
+
+// src/ai-provider/assertions/capabilityHonesty.ts
+var TEXT_GENERATION = ["text.generation"];
+var TOOL_USE = ["text.generation", "tool-use"];
+var STRUCTURED = ["text.generation", "json-mode"];
+var TEXT_EMBEDDING = ["text.embedding"];
+function capabilityHonestyBlock(opts) {
+  describe("Capability honesty", () => {
+    it.skipIf(opts.capabilities.streaming || !opts.models.textGeneration)("declares streaming=false → registry has no stream function", async () => {
+      const registry = getAiProviderRegistry();
+      const model = await getGlobalModelRepository().findByName(opts.models.textGeneration);
+      expect(model).toBeDefined();
+      const fn = registry.getRunFnFor(model.provider, TEXT_GENERATION);
+      expect(fn).toBeUndefined();
+    }, opts.timeout);
+    it.skipIf(opts.capabilities.tools || !opts.models.textGeneration)("declares tools=false → registry rejects ToolCallingTask lookup", async () => {
+      const registry = getAiProviderRegistry();
+      const model = await getGlobalModelRepository().findByName(opts.models.textGeneration);
+      expect(model).toBeDefined();
+      const fn = registry.getRunFnFor(model.provider, TOOL_USE);
+      expect(fn).toBeUndefined();
+    }, opts.timeout);
+    it.skipIf(opts.capabilities.embeddings || !opts.models.textGeneration)("declares embeddings=false → registry rejects TextEmbeddingTask lookup", async () => {
+      const registry = getAiProviderRegistry();
+      const model = await getGlobalModelRepository().findByName(opts.models.textGeneration);
+      expect(model).toBeDefined();
+      const fn = registry.getRunFnFor(model.provider, TEXT_EMBEDDING);
+      expect(fn).toBeUndefined();
+    }, opts.timeout);
+    it.skipIf(opts.capabilities.structured || !opts.models.textGeneration)("declares structured=false → registry rejects StructuredGenerationTask lookup", async () => {
+      const registry = getAiProviderRegistry();
+      const model = await getGlobalModelRepository().findByName(opts.models.textGeneration);
+      expect(model).toBeDefined();
+      const fn = registry.getRunFnFor(model.provider, STRUCTURED);
+      expect(fn).toBeUndefined();
+    }, opts.timeout);
+    it.skipIf(!opts.capabilities.streaming || !opts.models.textGeneration)("declares streaming=true → registry has a stream function", async () => {
+      const registry = getAiProviderRegistry();
+      const model = await getGlobalModelRepository().findByName(opts.models.textGeneration);
+      expect(model).toBeDefined();
+      const fn = registry.getRunFnFor(model.provider, TEXT_GENERATION);
+      expect(fn).toBeDefined();
+    }, opts.timeout);
+    it.skipIf(!opts.capabilities.tools || !opts.models.toolCalling)("declares tools=true → registry has a ToolCallingTask run function", async () => {
+      const registry = getAiProviderRegistry();
+      const model = await getGlobalModelRepository().findByName(opts.models.toolCalling);
+      expect(model).toBeDefined();
+      const fn = registry.getRunFnFor(model.provider, TOOL_USE);
+      expect(fn).toBeDefined();
+    }, opts.timeout);
+    it.skipIf(!opts.capabilities.structured || !opts.models.structured)("declares structured=true → registry has a StructuredGenerationTask run function", async () => {
+      const registry = getAiProviderRegistry();
+      const model = await getGlobalModelRepository().findByName(opts.models.structured);
+      expect(model).toBeDefined();
+      const fn = registry.getRunFnFor(model.provider, STRUCTURED);
+      expect(fn).toBeDefined();
+    }, opts.timeout);
+    it.skipIf(!opts.capabilities.embeddings || !opts.models.embeddings)("declares embeddings=true → registry has a run function and produces a finite vector", async () => {
+      const registry = getAiProviderRegistry();
+      const model = await getGlobalModelRepository().findByName(opts.models.embeddings);
+      expect(model).toBeDefined();
+      const fn = registry.getRunFnFor(model.provider, TEXT_EMBEDDING);
+      expect(fn).toBeDefined();
+      const result = await textEmbedding({
+        model: opts.models.embeddings,
+        text: "hello"
+      });
+      const vector = result.vector;
+      const isTypedArray = ArrayBuffer.isView(vector) && !(vector instanceof DataView);
+      expect(isTypedArray, "vector must be a numeric typed array").toBe(true);
+      const values = Array.from(vector);
+      expect(values.length).toBeGreaterThan(0);
+      for (const v of values) {
+        expect(Number.isFinite(v)).toBe(true);
+      }
+    }, opts.timeout);
+  });
+}
+
+// src/ai-provider/assertions/dispose.ts
+import { getLogger } from "@workglow/util";
+import { describe as describe2, expect as expect2 } from "vitest";
+function disposeBlock(opts, getHandle) {
+  describe2("Dispose", () => {
+    it("release resources observable via inspect()", async () => {
+      const handle = getHandle();
+      const before = handle.inspect();
+      if (!before.sessionMap && (!before.disposables || before.disposables.length === 0)) {
+        getLogger().warn(`[conformance] ${opts.name} exposes no inspect() handles; dispose assertion skipped`);
+        return;
+      }
+      await handle.dispose();
+      const after = handle.inspect();
+      if (after.sessionMap) {
+        expect2(after.sessionMap.size).toBe(0);
+      }
+      if (after.disposables) {
+        for (const d of after.disposables) {
+          expect2(d.alive).toBe(false);
+        }
+      }
+    }, opts.timeout);
+  });
+}
+
+// src/ai-provider/assertions/registryCoverage.ts
+import { getAiProviderRegistry as getAiProviderRegistry2, getGlobalModelRepository as getGlobalModelRepository2 } from "@workglow/ai";
+import { describe as describe3, expect as expect3 } from "vitest";
+function registryCoverageBlock(opts) {
+  describe3("Registry coverage", () => {
+    it("resolves a run function for every capability advertised by the provider's models", async () => {
+      const candidateModelIds = [
+        opts.models.textGeneration,
+        opts.models.toolCalling,
+        opts.models.structured,
+        opts.models.embeddings
+      ].filter((id) => typeof id === "string");
+      expect3(candidateModelIds.length, "registryCoverageBlock requires at least one model id in opts.models").toBeGreaterThan(0);
+      const registry = getAiProviderRegistry2();
+      const repo = getGlobalModelRepository2();
+      const seenProviders = new Set;
+      const seenCapabilities = new Set;
+      for (const modelId of candidateModelIds) {
+        const model = await repo.findByName(modelId);
+        expect3(model, `model "${modelId}" not found in repository`).toBeDefined();
+        const providerKey = model.provider;
+        const provider = registry.getProvider(providerKey);
+        expect3(provider, `provider "${providerKey}" not registered`).toBeDefined();
+        const capabilities = model.capabilities ?? [];
+        expect3(capabilities.length, `model "${modelId}" advertises zero capabilities`).toBeGreaterThan(0);
+        seenProviders.add(providerKey);
+        for (const capability of capabilities) {
+          seenCapabilities.add(`${providerKey}::${capability}`);
+          const runFn = registry.getRunFnFor(providerKey, [capability]);
+          expect3(runFn, `provider "${providerKey}" advertises capability "${capability}" via model "${modelId}" but the registry has no matching run function`).toBeDefined();
+        }
+      }
+      expect3(seenProviders.size, "no providers reached during coverage walk").toBeGreaterThan(0);
+      expect3(seenCapabilities.size, "no capabilities reached during coverage walk").toBeGreaterThan(0);
+    }, opts.timeout);
+  });
+}
+
+// src/ai-provider/assertions/sessionReuse.ts
+import { accumulatingEmit, getAiProviderRegistry as getAiProviderRegistry3, getGlobalModelRepository as getGlobalModelRepository3 } from "@workglow/ai";
+import { getLogger as getLogger2 } from "@workglow/util";
+import { beforeAll, describe as describe4, expect as expect4 } from "vitest";
+
+// src/itExpectFail.ts
+var itExpectFail = (name, fn, timeout) => {
+  const native = it.fails;
+  if (typeof native === "function") {
+    native(name, fn, timeout);
+    return;
+  }
+  it(`${name} [expected-fail]`, async () => {
+    let passed = false;
+    try {
+      await fn();
+      passed = true;
+    } catch (err) {
+      if (isCreditExhaustedError(err))
+        throw err;
+    }
+    if (passed) {
+      throw new Error(`Test "${name}" was marked as expected-fail but passed. Remove its name from opts.expectedFailures.`);
+    }
+  }, timeout);
+};
+
+// src/ai-provider/assertions/sessionReuse.ts
+var TEXT_GENERATION2 = ["text.generation"];
+function sessionReuseBlock(opts, fixture, getHandle) {
+  const enabled = opts.capabilities.sessions && !!opts.models.textGeneration;
+  const expectFails = new Set(opts.expectedFailures ?? []);
+  const itImpl = expectFails.has("session.reuse") ? itExpectFail : it;
+  describe4.skipIf(!enabled)("Session reuse", () => {
+    beforeAll(async () => {
+      await getHandle().releaseTransients?.();
+    });
+    itImpl("two invocations with the same sessionId yield exactly one session-map entry", async () => {
+      const handle = getHandle();
+      const map = handle.inspect().sessionMap;
+      if (!map) {
+        getLogger2().warn(`[conformance] ${opts.name} declares sessions=true but inspect().sessionMap is undefined; skipping`);
+        return;
+      }
+      const sizeBefore = map.size;
+      const registry = getAiProviderRegistry3();
+      const model = await getGlobalModelRepository3().findByName(opts.models.textGeneration);
+      expect4(model).toBeDefined();
+      const runFn = registry.getRunFnFor(model.provider, TEXT_GENERATION2);
+      expect4(runFn, `provider "${model.provider}" has no run-fn for ["text.generation"]`).toBeDefined();
+      const sessionId = `conformance-${Date.now()}`;
+      {
+        const { emit } = accumulatingEmit();
+        await runFn({ prompt: fixture.textPrompt, maxTokens: fixture.maxTokens }, model, new AbortController().signal, emit, undefined, { sessionId });
+      }
+      {
+        const { emit } = accumulatingEmit();
+        await runFn({ prompt: fixture.textPrompt, maxTokens: fixture.maxTokens }, model, new AbortController().signal, emit, undefined, { sessionId });
+      }
+      const newEntries = map.size - sizeBefore;
+      expect4(newEntries).toBe(1);
+    }, opts.timeout);
+  });
+}
+
+// src/ai-provider/assertions/signalHonoring.ts
+import { accumulatingEmit as accumulatingEmit2, getAiProviderRegistry as getAiProviderRegistry4, getGlobalModelRepository as getGlobalModelRepository4 } from "@workglow/ai";
+import { describe as describe5, expect as expect5 } from "vitest";
+var TEXT_GENERATION3 = ["text.generation"];
+function isAbortError(err) {
+  if (!err || typeof err !== "object")
+    return false;
+  const e = err;
+  return e.name === "AbortError" || /abort/i.test(e.message ?? "");
+}
+function signalHonoringBlock(opts, fixture) {
+  const expectFails = new Set(opts.expectedFailures ?? []);
+  const itNonStreaming = expectFails.has("signal.nonStreaming") ? itExpectFail : it;
+  const skipMid = !opts.capabilities.abortMidStream;
+  const itMidStream = expectFails.has("signal.midStream") ? itExpectFail : it;
+  describe5.skipIf(!opts.models.textGeneration)("Signal honoring", () => {
+    itNonStreaming("run-fn rejects with AbortError when aborted before invocation", async () => {
+      const registry = getAiProviderRegistry4();
+      const repo = getGlobalModelRepository4();
+      const model = await repo.findByName(opts.models.textGeneration);
+      expect5(model).toBeDefined();
+      const runFn = registry.getRunFnFor(model.provider, TEXT_GENERATION3);
+      expect5(runFn, `provider "${model.provider}" has no run-fn for ["text.generation"]`).toBeDefined();
+      const ac = new AbortController;
+      ac.abort();
+      let caught;
+      try {
+        const { emit } = accumulatingEmit2();
+        await runFn({ prompt: fixture.textPrompt, maxTokens: fixture.maxTokens }, model, ac.signal, emit, undefined, undefined);
+      } catch (err) {
+        caught = err;
+      }
+      expect5(caught).toBeDefined();
+      expect5(isAbortError(caught)).toBe(true);
+    }, opts.timeout);
+    if (!skipMid) {
+      itMidStream("streaming run-fn terminates within abortGraceMs * 4 when aborted mid-stream", async () => {
+        const registry = getAiProviderRegistry4();
+        const repo = getGlobalModelRepository4();
+        const model = await repo.findByName(opts.models.textGeneration);
+        expect5(model).toBeDefined();
+        const streamFn = registry.getRunFnFor(model.provider, TEXT_GENERATION3);
+        if (!streamFn)
+          return;
+        const ac = new AbortController;
+        const start = Date.now();
+        setTimeout(() => ac.abort(), fixture.abortGraceMs);
+        try {
+          const emit = (_e) => {};
+          await streamFn({ prompt: fixture.textPrompt, maxTokens: fixture.maxTokens }, model, ac.signal, emit, undefined, undefined);
+        } catch (err) {
+          if (!isAbortError(err))
+            throw err;
+        }
+        const elapsed = Date.now() - start;
+        expect5(elapsed).toBeLessThan(fixture.abortGraceMs * 4 + 2000);
+      }, opts.timeout);
+    }
+  });
+}
+
+// src/ai-provider/assertions/structuredGeneration.ts
+import { structuredGeneration } from "@workglow/ai";
+import { describe as describe6, expect as expect6 } from "vitest";
+function structuredGenerationBlock(opts, fixture) {
+  const enabled = opts.capabilities.structured && !!opts.models.structured;
+  describe6.skipIf(!enabled)("Structured generation", () => {
+    it("produces an object that contains the schema's required fields", async () => {
+      const result = await structuredGeneration({
+        model: opts.models.structured,
+        prompt: fixture.structuredPrompt,
+        outputSchema: fixture.structuredSchema,
+        maxTokens: fixture.maxTokens
+      });
+      expect6(result).toBeDefined();
+      expect6(result.object).toBeDefined();
+      expect6(typeof result.object).toBe("object");
+      expect6(result.object).toHaveProperty("name");
+      expect6(result.object).toHaveProperty("age");
+    }, opts.timeout);
+  });
+}
+
+// src/ai-provider/assertions/textGenerationSmoke.ts
+import { getAiProviderRegistry as getAiProviderRegistry5, getGlobalModelRepository as getGlobalModelRepository5, textGeneration } from "@workglow/ai";
+import { describe as describe7, expect as expect7 } from "vitest";
+var TEXT_GENERATION4 = ["text.generation"];
+function textGenerationSmokeBlock(opts, fixture) {
+  describe7.skipIf(!opts.models.textGeneration)("TextGeneration smoke", () => {
+    it("task-level textGeneration() returns non-empty text", async () => {
+      const result = await textGeneration({
+        model: opts.models.textGeneration,
+        prompt: fixture.textPrompt,
+        maxTokens: fixture.maxTokens
+      });
+      expect7(result).toBeDefined();
+      expect7(typeof result.text).toBe("string");
+      expect7(result.text.length).toBeGreaterThan(0);
+    }, opts.timeout);
+    it.skipIf(!opts.capabilities.streaming)("streaming yields ≥1 text-delta and exactly one finish, with no event after finish", async () => {
+      const registry = getAiProviderRegistry5();
+      const model = await getGlobalModelRepository5().findByName(opts.models.textGeneration);
+      expect7(model).toBeDefined();
+      const streamFn = registry.getRunFnFor(model.provider, TEXT_GENERATION4);
+      expect7(streamFn).toBeDefined();
+      let textDeltaCount = 0;
+      let finishCount = 0;
+      let sawEventAfterFinish = false;
+      const emit = (ev) => {
+        if (finishCount > 0) {
+          sawEventAfterFinish = true;
+        }
+        const e = ev;
+        if (e.type === "text-delta")
+          textDeltaCount += 1;
+        if (e.type === "finish")
+          finishCount += 1;
+      };
+      await streamFn({ prompt: fixture.textPrompt, maxTokens: fixture.maxTokens }, model, new AbortController().signal, emit, undefined, undefined);
+      expect7(textDeltaCount).toBeGreaterThanOrEqual(1);
+      expect7(finishCount).toBe(1);
+      expect7(sawEventAfterFinish).toBe(false);
+    }, opts.timeout);
+  });
+}
+
+// src/ai-provider/assertions/toolCallAccumulator.ts
+import { toolCalling } from "@workglow/ai";
+import { parsePartialJson } from "@workglow/util/schema";
+import { describe as describe8, expect as expect8 } from "vitest";
+function toolCallAccumulatorBlock(opts, fixture) {
+  const enabled = opts.capabilities.tools && !!opts.models.toolCalling;
+  describe8.skipIf(!enabled)("Tool-call accumulator", () => {
+    it("produces ≥1 call with non-empty id, expected name, and parsable final args", async () => {
+      const tool = {
+        name: fixture.weatherTool.name,
+        description: fixture.weatherTool.description,
+        inputSchema: fixture.weatherTool.inputSchema
+      };
+      const result = await toolCalling({
+        model: opts.models.toolCalling,
+        prompt: fixture.weatherToolPrompt,
+        tools: [tool],
+        toolChoice: "required",
+        maxTokens: fixture.maxTokens,
+        messages: undefined
+      });
+      const calls = result.toolCalls;
+      expect8(calls.length).toBeGreaterThan(0);
+      const call = calls[0];
+      expect8(call.name).toBe(fixture.weatherTool.name);
+      expect8(call.id).toBeTruthy();
+      expect8(call.input).toBeDefined();
+      const serialized = typeof call.input === "string" ? call.input : JSON.stringify(call.input);
+      const parsed = parsePartialJson(serialized);
+      expect8(parsed).toBeDefined();
+    }, opts.timeout);
+    it("produces no tool calls with toolChoice none", async () => {
+      const tool = {
+        name: fixture.weatherTool.name,
+        description: fixture.weatherTool.description,
+        inputSchema: fixture.weatherTool.inputSchema
+      };
+      const result = await toolCalling({
+        model: opts.models.toolCalling,
+        prompt: fixture.weatherToolPrompt,
+        tools: [tool],
+        toolChoice: "none",
+        maxTokens: fixture.maxTokens,
+        messages: undefined
+      });
+      expect8(result).toBeDefined();
+      expect8(typeof result.text).toBe("string");
+      expect8(result.text.length).toBeGreaterThan(0);
+      expect8(result.toolCalls).toHaveLength(0);
+    }, opts.timeout);
+  });
+}
+
+// src/ai-provider/assertions/toolCallMultiTurn.ts
+import { Workflow } from "@workglow/task-graph";
+import { describe as describe9, expect as expect9 } from "vitest";
+function toolCallMultiTurnBlock(opts, fixture) {
+  const enabled = opts.capabilities.tools && !!opts.models.toolCalling;
+  describe9.skipIf(!enabled)("Tool-call multi-turn", () => {
+    it("second turn returns non-empty text after a tool result is provided", async () => {
+      const tool = {
+        name: fixture.weatherTool.name,
+        description: fixture.weatherTool.description,
+        inputSchema: fixture.weatherTool.inputSchema
+      };
+      const wf1 = new Workflow;
+      wf1.toolCalling({
+        model: opts.models.toolCalling,
+        prompt: fixture.multiTurnTranscript[0].text,
+        tools: [tool],
+        toolChoice: "auto",
+        maxTokens: fixture.maxTokens
+      });
+      const r1 = await wf1.run();
+      if (!r1.toolCalls || r1.toolCalls.length === 0)
+        return;
+      const call = r1.toolCalls[0];
+      const wf2 = new Workflow;
+      wf2.toolCalling({
+        model: opts.models.toolCalling,
+        prompt: fixture.multiTurnTranscript[0].text,
+        tools: [tool],
+        toolChoice: "auto",
+        maxTokens: fixture.maxTokens,
+        messages: [
+          {
+            role: "user",
+            content: [{ type: "text", text: fixture.multiTurnTranscript[0].text }]
+          },
+          {
+            role: "assistant",
+            content: [
+              { type: "text", text: r1.text || fixture.multiTurnTranscript[1].text },
+              {
+                type: "tool_use",
+                id: call.id,
+                name: call.name,
+                input: call.input,
+                providerSignature: call.providerSignature
+              }
+            ]
+          },
+          {
+            role: "tool",
+            content: [
+              {
+                type: "tool_result",
+                tool_use_id: call.id,
+                content: [{ type: "text", text: fixture.multiTurnTranscript[2].text }],
+                is_error: undefined
+              }
+            ]
+          }
+        ]
+      });
+      const r2 = await wf2.run();
+      expect9(typeof r2.text).toBe("string");
+      expect9(r2.text.length).toBeGreaterThan(0);
+    }, opts.timeout);
+  });
+}
+
+// src/ai-provider/runAiProviderConformance.ts
+function runAiProviderConformance(opts) {
+  const requiredModels = [
+    [opts.capabilities.streaming, "streaming", "textGeneration"],
+    [opts.capabilities.tools, "tools", "toolCalling"],
+    [opts.capabilities.structured, "structured", "structured"],
+    [opts.capabilities.embeddings, "embeddings", "embeddings"]
+  ];
+  for (const [claimed, capabilityKey, modelKey] of requiredModels) {
+    if (claimed && !opts.models[modelKey]) {
+      throw new Error(`AiProvider conformance "${opts.name}": capabilities.${capabilityKey} is true but models.${modelKey} is not set. ` + `Either provide a model id or set capabilities.${capabilityKey} to false.`);
+    }
+  }
+  describe10.skipIf(opts.skip)(`AiProvider conformance: ${opts.name}`, () => {
+    let handle;
+    const getHandle = () => {
+      if (!handle)
+        throw new Error("conformance handle not initialized");
+      return handle;
+    };
+    beforeAll2(async () => {
+      handle = await opts.factory();
+      await handle.register();
+    }, opts.timeout);
+    afterAll(async () => {
+      if (handle)
+        await handle.dispose();
+    });
+    const fixture = resolveFixture(opts.fixture);
+    registryCoverageBlock(opts);
+    textGenerationSmokeBlock(opts, fixture);
+    signalHonoringBlock(opts, fixture);
+    toolCallAccumulatorBlock(opts, fixture);
+    toolCallMultiTurnBlock(opts, fixture);
+    structuredGenerationBlock(opts, fixture);
+    sessionReuseBlock(opts, fixture, getHandle);
+    capabilityHonestyBlock(opts);
+    disposeBlock(opts, getHandle);
+  });
+}
+// src/ai-provider/assertions/inferAdvertisesRegistered.ts
+import { expect as expect10 } from "vitest";
+function assertInferAdvertisesRegistered(name, registered, inferred) {
+  const registeredCaps = new Set(registered.flat());
+  const inferredCaps = new Set(inferred.flat());
+  const missing = [...registeredCaps].filter((cap) => !inferredCaps.has(cap)).sort();
+  expect10(missing, `${name} registers capabilities that inferCapabilities never returns: ${missing.join(", ")}`).toEqual([]);
+}
+// src/ai-provider/assertions/inferServesInferred.ts
+import { expect as expect11 } from "vitest";
+var MARKER_CAPABILITIES = new Set(["vision-input"]);
+var SESSION_CAPABILITIES = new Set(["cache.checkpoint", "session.dispose"]);
+function assertInferServesInferred(name, registered, inferred) {
+  const unservable = [];
+  for (const { id, capabilities } of inferred) {
+    const inferredSet = new Set(capabilities);
+    for (const capability of capabilities) {
+      if (MARKER_CAPABILITIES.has(capability))
+        continue;
+      const servable = registered.some((serves) => serves.includes(capability) && serves.every((cap) => inferredSet.has(cap)));
+      if (!servable) {
+        unservable.push(`${id}: ${capability} — no run-fn serves it within this model's set`);
+      } else if (SESSION_CAPABILITIES.has(capability) && !inferredSet.has("text.generation")) {
+        unservable.push(`${id}: ${capability} — no session without text.generation`);
+      }
+    }
+  }
+  expect11(unservable, `${name} infers capabilities it cannot serve for that model:
+  ${unservable.join(`
+  `)}`).toEqual([]);
+}
+// src/ai-provider/assertions/pricingMatchesModality.ts
+import { expect as expect12 } from "vitest";
+var NOT_TOKEN_BILLED = new Set([
+  "image.generation",
+  "audio.speech",
+  "audio.transcription"
+]);
+function quotesTokenRate(pricing) {
+  if (pricing === undefined)
+    return false;
+  return typeof pricing.input === "number" || typeof pricing.output === "number" || typeof pricing.cached === "number";
+}
+function assertPricingMatchesModality(name, models, namedByTable = []) {
+  const named = new Set(namedByTable);
+  const fabricated = [];
+  for (const { id, capabilities, pricing } of models) {
+    if (!capabilities.some((capability) => NOT_TOKEN_BILLED.has(capability)))
+      continue;
+    if (!quotesTokenRate(pricing))
+      continue;
+    if (named.has(id))
+      continue;
+    const free = (pricing?.input ?? 0) === 0 && (pricing?.output ?? 0) === 0;
+    if (free)
+      continue;
+    const billed = capabilities.filter((capability) => NOT_TOKEN_BILLED.has(capability));
+    fabricated.push(`${id}: ${billed.join(", ")} but priced ` + `input=${String(pricing?.input)} output=${String(pricing?.output)} per 1M tokens`);
+  }
+  expect12(fabricated, `${name} borrows a per-token rate for a model not billed per token`).toEqual([]);
+  const stale = namedByTable.filter((id) => !models.some(({ id: modelId, capabilities, pricing }) => modelId === id && capabilities.some((capability) => NOT_TOKEN_BILLED.has(capability)) && quotesTokenRate(pricing)));
+  expect12(stale, `${name}: recorded as deliberately priced but no longer is: ${stale.join(", ")}`).toEqual([]);
+}
+function assertPricedGapDoesNotGrow(name, models, knownUnpriced) {
+  const unpriced = models.filter(({ capabilities, pricing }) => capabilities.length > 0 && !capabilities.some((capability) => NOT_TOKEN_BILLED.has(capability)) && pricing === undefined).map(({ id }) => id).sort();
+  const known = new Set(knownUnpriced);
+  expect12(unpriced.filter((id) => !known.has(id)), `${name} has a token-billed model with no rate card that was not recorded`).toEqual([]);
+  const closed = knownUnpriced.filter((id) => !unpriced.includes(id));
+  expect12(closed, `${name}: now priced — drop from the recorded set: ${closed.join(", ")}`).toEqual([]);
+}
+// src/ai-provider/assertions/usageNormalization.ts
+import { describe as describe11, expect as expect13 } from "vitest";
+var USAGE_COUNTER_FIELDS = [
+  "input",
+  "output",
+  "cached",
+  "cacheWrite",
+  "reasoning",
+  "total"
+];
+function assertUsageShape(usage) {
+  for (const field of USAGE_COUNTER_FIELDS) {
+    const value = usage[field];
+    if (value === undefined)
+      continue;
+    expect13(typeof value, `${field} must be a number when reported`).toBe("number");
+    expect13(Number.isFinite(value), `${field} must be finite`).toBe(true);
+  }
+  if (usage.extra !== undefined) {
+    for (const [key, value] of Object.entries(usage.extra)) {
+      expect13(["number", "string"], `extra.${key} must be a number or string`).toContain(typeof value);
+    }
+  }
+}
+function assertUsageDisjointness(usage) {
+  if (usage.reasoning !== undefined && usage.output !== undefined) {
+    expect13(usage.reasoning <= usage.output, `reasoning (${usage.reasoning}) is a subset of output (${usage.output})`).toBe(true);
+  }
+  if (usage.total === undefined)
+    return;
+  const parts = [usage.input, usage.cached, usage.cacheWrite, usage.output];
+  if (parts.every((part) => part === undefined))
+    return;
+  const sum = parts.reduce((acc, part) => acc + (part ?? 0), 0);
+  expect13(sum, `input + cached + cacheWrite + output must equal the provider's stated total`).toBe(usage.total);
+}
+function usageNormalizationBlock(opts) {
+  describe11(`${opts.provider} usage normalization contract`, () => {
+    it("reports nothing when the provider sent no usage payload", () => {
+      for (const empty of [undefined, null, {}, "usage", 42]) {
+        expect13(opts.mapUsage(empty), `mapped ${JSON.stringify(empty) ?? "undefined"}`).toBe(undefined);
+      }
+    });
+    it("leaves counters the provider did not report as undefined, never 0", () => {
+      const usage = opts.mapUsage(opts.sparseFrame);
+      expect13(usage, "a frame with real counters must map to a Usage").toBeDefined();
+      const reported = new Set(opts.sparseReports);
+      for (const field of USAGE_COUNTER_FIELDS) {
+        if (reported.has(field)) {
+          expect13(usage[field], `${field} was reported and must survive`).toBeTypeOf("number");
+        } else {
+          expect13(usage[field], `${field} was not reported and must be undefined`).toBe(undefined);
+        }
+      }
+    });
+    it("preserves a genuinely reported zero rather than dropping it", () => {
+      const usage = opts.mapUsage(opts.zeroFrame);
+      expect13(usage, "a frame reporting 0 still reported something").toBeDefined();
+      for (const field of opts.sparseReports) {
+        expect13(usage[field], `${field} reported 0 and must stay 0`).toBe(0);
+      }
+    });
+    it("maps only numbers and undefined into the normalized shape", () => {
+      for (const testCase of opts.cases) {
+        const usage = opts.mapUsage(testCase.frame);
+        if (usage)
+          assertUsageShape(usage);
+      }
+      const sparse = opts.mapUsage(opts.sparseFrame);
+      if (sparse)
+        assertUsageShape(sparse);
+    });
+    it("keeps input/cached/cacheWrite disjoint and reasoning inside output", () => {
+      for (const testCase of opts.cases) {
+        const usage = opts.mapUsage(testCase.frame);
+        if (usage)
+          assertUsageDisjointness(usage);
+      }
+      const sparse = opts.mapUsage(opts.sparseFrame);
+      if (sparse)
+        assertUsageDisjointness(sparse);
+    });
+    for (const testCase of opts.cases) {
+      it(`maps ${testCase.name}`, () => {
+        expect13(opts.mapUsage(testCase.frame)).toEqual(testCase.expected);
+      });
+    }
+  });
+}
+export {
+  DEFAULT_CONFORMANCE_FIXTURE,
+  assertInferAdvertisesRegistered,
+  assertInferServesInferred,
+  assertPricedGapDoesNotGrow,
+  assertPricingMatchesModality,
+  assertUsageDisjointness,
+  assertUsageShape,
+  capabilityHonestyBlock,
+  disposeBlock,
+  registryCoverageBlock,
+  resolveFixture,
+  runAiProviderConformance,
+  sessionReuseBlock,
+  signalHonoringBlock,
+  structuredGenerationBlock,
+  textGenerationSmokeBlock,
+  toolCallAccumulatorBlock,
+  toolCallMultiTurnBlock,
+  usageNormalizationBlock
+};
+
+//# debugId=A13FF2AAA5D2709B64756E2164756E21