@intx/inference-discovery 0.1.2

package/README.md

@@ -0,0 +1,110 @@
+# @intx/inference-discovery
+
+The shared runtime for the inference discovery rig. Defines the
+provider plug-in contract, drives capture runs, and owns the
+capability catalog and support matrix that say which
+(provider, model, capability) tuples the rig knows how to record.
+
+The output of a discovery run is a fixture bundle on disk under
+`packages/inference-testing/wire/<provider>/<model>/<capability>/`,
+which `@intx/inference-testing` then replays in tests. This package
+does not perform the replay; it produces the bytes that the replay
+layer consumes.
+
+Discovery makes real, paid network calls to upstream model providers.
+It must never run in CI. The `assertNotCI` guard exported here aborts
+the process before any plug-in is constructed if the `CI` environment
+variable is set.
+
+## Surface
+
+The package exports two entry points:
+
+- `@intx/inference-discovery` — the runtime: plug-in contract,
+  capture runner, CLI parser, manifest builder, content-type
+  detection, CI guard, env validation, write-capture.
+- `@intx/inference-discovery/catalog` — the data: the `Capability`
+  enum, the `INTENTS` table, the `SUPPORT_MATRIX` listing every
+  (provider, model, capability) tuple the rig knows about, and the
+  `FixtureManifest` schema.
+
+## Driving one capture
+
+```ts
+import { runCapture } from "@intx/inference-discovery";
+import { INTENTS, getFixtureDir } from "@intx/inference-discovery/catalog";
+import { createSomeProviderPlugin } from "@intx/inference-discovery-some-provider";
+
+const plugin = createSomeProviderPlugin({ apiKey });
+
+await runCapture({
+  plugin,
+  model: "some-model",
+  capability: "plain-text",
+  intent: INTENTS["plain-text"],
+  outDir: getFixtureDir({
+    provider: plugin.name,
+    model: "some-model",
+    capability: "plain-text",
+    outcome: "captured",
+  }),
+});
+```
+
+`runCapture` walks the plug-in's capture-step generator, POSTs
+each step, detects SSE vs JSON by content-type, and writes four
+files into the step's subdirectory: `request.json` and
+`response.{json,sse}` carry the bodies verbatim, while
+`request-headers.json` and `response-headers.json` carry the
+headers with the plug-in's redaction lists applied. After the
+generator exhausts it writes `manifest.json` at the run root.
+
+Plug-ins that capture reasoning capabilities can opt into a
+`reasoning-trace.json` sidecar; the runner calls the plug-in's
+extractor and writes the result alongside the response.
+
+## Catalog
+
+`SUPPORT_MATRIX` is the canonical list of what the rig captures.
+Each entry carries an outcome of `captured`, `refused`,
+`http-error`, or `unsupported`. Only `captured` entries produce
+fixtures; the others are negative documentation recording why no
+fixture exists — a deliberate refusal, an observed upstream error,
+or a capability the provider does not implement (see the `glm-5.1`
+refusal and `deepseek-v4-pro` HTTP-error vision entries for
+examples).
+
+`INTENTS` maps each capability to the prompt, tools, follow-up
+turns, and media references the plug-in uses to assemble the
+request body. Intents are deliberately single-sentence and
+low-token so the captured responses stay focused on shape rather
+than substance.
+
+## The `discover` CLI
+
+`bin/discover.ts` at the repo root wires this package together with
+the provider plug-in packages and exposes them as a single command:
+
+```
+bun bin/discover.ts --provider <name> (--all | --model <name> | --only <capability>)
+```
+
+`--all` is mutually exclusive with `--model` and `--only`; at least
+one of the three must be present (an invocation with none errors
+out). Available providers and their required environment variables
+are listed by `bun bin/discover.ts --help`. The CLI calls
+`assertNotCI` first, validates the requested provider's environment
+variables via `requireEnvSet`, filters `SUPPORT_MATRIX` to the
+matching captured entries, and runs each through `runCapture`.
+
+Each invocation incurs per-request usage charges against the
+upstream provider; an `--all` run touches every (model, capability)
+pair in the matrix for the selected provider.
+
+## See also
+
+- [`docs/OPENCODE_DISCOVERY.md`](../../docs/OPENCODE_DISCOVERY.md) —
+  observed-vs-documented narrative for the OpenCode Zen relay, with
+  pointers into the captured corpus.
+- [`@intx/inference-testing`](../inference-testing/README.md) —
+  consumes the fixtures this rig produces.

package/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "@intx/inference-discovery",
+  "version": "0.1.2",
+  "license": "LGPL-2.1-only",
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./src/index.ts",
+      "default": "./src/index.ts"
+    },
+    "./catalog": {
+      "types": "./src/catalog/index.ts",
+      "default": "./src/catalog/index.ts"
+    }
+  },
+  "dependencies": {
+    "arktype": "^2.1.29"
+  }
+}

package/src/catalog/capability.test.ts

@@ -0,0 +1,43 @@
+import { describe, test, expect } from "bun:test";
+import { type } from "arktype";
+import { CAPABILITIES, Capability } from "./capability";
+
+describe("CAPABILITIES vocabulary", () => {
+  test("declares exactly 31 capabilities", () => {
+    expect(CAPABILITIES.length).toBe(31);
+  });
+
+  test("contains no duplicate names", () => {
+    const seen = new Set(CAPABILITIES);
+    expect(seen.size).toBe(CAPABILITIES.length);
+  });
+
+  test("function-calling has no -streaming variant", () => {
+    expect(CAPABILITIES.includes("function-calling")).toBe(true);
+    expect(
+      (CAPABILITIES as readonly string[]).includes(
+        "function-calling-streaming",
+      ),
+    ).toBe(false);
+  });
+});
+
+describe("Capability validator", () => {
+  test("accepts every name in CAPABILITIES", () => {
+    for (const name of CAPABILITIES) {
+      const result = Capability(name);
+      expect(result instanceof type.errors).toBe(false);
+      expect(result).toBe(name);
+    }
+  });
+
+  test("rejects an unknown capability name", () => {
+    const result = Capability("not-a-capability");
+    expect(result instanceof type.errors).toBe(true);
+  });
+
+  test("rejects a non-string input", () => {
+    const result = Capability(42);
+    expect(result instanceof type.errors).toBe(true);
+  });
+});

package/src/catalog/capability.ts

@@ -0,0 +1,38 @@
+import { type } from "arktype";
+
+export const CAPABILITIES = [
+  "plain-text",
+  "plain-text-streaming",
+  "function-calling",
+  "function-calling-multi-turn",
+  "function-calling-multi-turn-streaming",
+  "function-calling-with-thinking",
+  "function-calling-with-thinking-streaming",
+  "vision-input",
+  "vision-input-streaming",
+  "audio-input",
+  "audio-input-streaming",
+  "video-input",
+  "video-input-streaming",
+  "document-input",
+  "document-input-streaming",
+  "image-output",
+  "image-output-streaming",
+  "code-execution",
+  "code-execution-streaming",
+  "reasoning-content",
+  "reasoning-content-streaming",
+  "grounding",
+  "grounding-streaming",
+  "files-api-reference",
+  "files-api-reference-streaming",
+  "redacted-thinking",
+  "redacted-thinking-streaming",
+  "safety-classification",
+  "safety-classification-streaming",
+  "structured-output",
+  "structured-output-streaming",
+] as const;
+
+export const Capability = type.enumerated(...CAPABILITIES);
+export type Capability = typeof Capability.infer;

package/src/catalog/index.ts

@@ -0,0 +1,10 @@
+export { CAPABILITIES, Capability } from "./capability";
+export {
+  CapabilityIntent,
+  INTENTS,
+  MediaRef,
+  ToolDecl,
+  resolveMediaPath,
+} from "./intent";
+export { SUPPORT_MATRIX, SupportEntry, getFixtureDir } from "./support-matrix";
+export { FixtureManifest } from "./manifest";

package/src/catalog/intent.test.ts

@@ -0,0 +1,94 @@
+import { describe, test, expect } from "bun:test";
+import { existsSync, statSync } from "node:fs";
+import { type } from "arktype";
+import { CAPABILITIES } from "./capability";
+import {
+  CapabilityIntent,
+  INTENTS,
+  MediaRef,
+  ToolDecl,
+  resolveMediaPath,
+} from "./intent";
+
+describe("INTENTS map", () => {
+  test("has one entry per declared capability", () => {
+    for (const name of CAPABILITIES) {
+      expect(INTENTS[name]).toBeDefined();
+    }
+    expect(Object.keys(INTENTS).length).toBe(CAPABILITIES.length);
+  });
+
+  test("every entry parses as a CapabilityIntent", () => {
+    for (const name of CAPABILITIES) {
+      const result = CapabilityIntent(INTENTS[name]);
+      expect(result instanceof type.errors).toBe(false);
+    }
+  });
+
+  test("streaming variants share the non-streaming intent", () => {
+    expect(INTENTS["plain-text-streaming"]).toBe(INTENTS["plain-text"]);
+    expect(INTENTS["vision-input-streaming"]).toBe(INTENTS["vision-input"]);
+    expect(INTENTS["reasoning-content-streaming"]).toBe(
+      INTENTS["reasoning-content"],
+    );
+  });
+});
+
+describe("MediaRef paths", () => {
+  test("every referenced media file exists on disk", () => {
+    for (const name of CAPABILITIES) {
+      const intent = INTENTS[name];
+      const media = intent.media;
+      if (media === undefined) continue;
+      for (const ref of media) {
+        const absolute = resolveMediaPath(ref);
+        expect(existsSync(absolute)).toBe(true);
+        expect(statSync(absolute).size).toBeGreaterThan(0);
+      }
+    }
+  });
+
+  test("resolveMediaPath rejects absolute paths", () => {
+    expect(() =>
+      resolveMediaPath({ kind: "image", path: "/etc/passwd" }),
+    ).toThrow();
+  });
+});
+
+describe("ToolDecl validator", () => {
+  test("accepts a well-formed function declaration", () => {
+    const result = ToolDecl({
+      name: "get_weather",
+      description: "Look up the weather.",
+      parameters: {
+        type: "object",
+        properties: {
+          location: { type: "string", description: "City name" },
+        },
+        required: ["location"],
+      },
+    });
+    expect(result instanceof type.errors).toBe(false);
+  });
+
+  test("rejects a parameters object with wrong type tag", () => {
+    const result = ToolDecl({
+      name: "x",
+      description: "x",
+      parameters: { type: "array", properties: {} },
+    });
+    expect(result instanceof type.errors).toBe(true);
+  });
+});
+
+describe("MediaRef validator", () => {
+  test("accepts an image reference", () => {
+    const result = MediaRef({ kind: "image", path: "media/sample.jpg" });
+    expect(result instanceof type.errors).toBe(false);
+  });
+
+  test("rejects an unknown media kind", () => {
+    const result = MediaRef({ kind: "spreadsheet", path: "x.csv" });
+    expect(result instanceof type.errors).toBe(true);
+  });
+});

package/src/catalog/intent.ts

@@ -0,0 +1,297 @@
+import { fileURLToPath } from "node:url";
+import { type } from "arktype";
+import type { Capability } from "./capability";
+
+const MediaKind = type.enumerated("image", "audio", "video", "document");
+
+export const MediaRef = type({
+  kind: MediaKind,
+  path: "string",
+});
+export type MediaRef = typeof MediaRef.infer;
+
+const ToolParameterProperty = type({
+  type: "string",
+  "description?": "string",
+  "enum?": "string[]",
+});
+
+export const ToolDecl = type({
+  name: "string",
+  description: "string",
+  parameters: {
+    type: "'object'",
+    properties: type.Record("string", ToolParameterProperty),
+    "required?": "string[]",
+  },
+});
+export type ToolDecl = typeof ToolDecl.infer;
+
+const FollowUp = type({
+  role: "'user' | 'assistant'",
+  content: "string",
+}).or({
+  role: "'tool'",
+  toolName: "string",
+  content: "string",
+});
+
+// Structured-output constraint on a capability intent. Mirrors
+// InferenceOptions.responseFormat in @intx/types/runtime; redefined
+// here rather than imported so the catalog package retains its
+// arktype-only dependency profile. The discovery plug-ins translate
+// this field to the provider-native wire shape (OpenAI's
+// response_format, Gemini's responseSchema; Anthropic does not
+// receive a structured-output request because its adapter rejects
+// the field at the marshaling boundary).
+const ResponseFormatIntent = type({
+  kind: "'text'",
+})
+  .or({
+    kind: "'json'",
+  })
+  .or({
+    kind: "'json-schema'",
+    name: "string",
+    schema: "unknown",
+    "strict?": "boolean",
+  });
+
+export const CapabilityIntent = type({
+  prompt: "string",
+  "media?": MediaRef.array(),
+  "tools?": ToolDecl.array(),
+  "followUp?": FollowUp.array(),
+  "responseFormat?": ResponseFormatIntent,
+});
+export type CapabilityIntent = typeof CapabilityIntent.infer;
+
+const WEATHER_TOOL: ToolDecl = {
+  name: "get_weather",
+  description: "Look up the current weather for a city.",
+  parameters: {
+    type: "object",
+    properties: {
+      location: {
+        type: "string",
+        description: "City and optional region, e.g. 'Boston, MA'.",
+      },
+    },
+    required: ["location"],
+  },
+};
+
+const PLAIN_TEXT: CapabilityIntent = {
+  prompt: "Reply with a single sentence: the capital of France is Paris.",
+};
+
+const FUNCTION_CALLING: CapabilityIntent = {
+  prompt: "What is the weather in Boston, MA? Use the provided tool.",
+  tools: [WEATHER_TOOL],
+};
+
+const FUNCTION_CALLING_MULTI_TURN: CapabilityIntent = {
+  prompt: "What is the weather in Boston, MA? Use the provided tool.",
+  tools: [WEATHER_TOOL],
+  followUp: [
+    {
+      role: "tool",
+      toolName: "get_weather",
+      content:
+        '{"location":"Boston, MA","temperatureF":68,"conditions":"clear"}',
+    },
+    {
+      role: "user",
+      content: "Summarize the result in one sentence.",
+    },
+  ],
+};
+
+const FUNCTION_CALLING_WITH_THINKING: CapabilityIntent = {
+  prompt:
+    "Think step by step about which tool to call, then call it. What is the weather in Boston, MA?",
+  tools: [WEATHER_TOOL],
+};
+
+const VISION_INPUT: CapabilityIntent = {
+  prompt: "Describe the attached image in one short sentence.",
+  media: [{ kind: "image", path: "media/sample.jpg" }],
+};
+
+const AUDIO_INPUT: CapabilityIntent = {
+  prompt: "Transcribe the attached audio in one sentence.",
+  media: [{ kind: "audio", path: "media/sample.wav" }],
+};
+
+const VIDEO_INPUT: CapabilityIntent = {
+  prompt: "Describe what happens in the attached video in one short sentence.",
+  media: [{ kind: "video", path: "media/sample.mp4" }],
+};
+
+const DOCUMENT_INPUT: CapabilityIntent = {
+  prompt: "Summarize the attached document in one sentence.",
+  media: [{ kind: "document", path: "media/sample.pdf" }],
+};
+
+const IMAGE_OUTPUT: CapabilityIntent = {
+  prompt: "Generate an image of a red apple on a wooden table.",
+};
+
+const CODE_EXECUTION: CapabilityIntent = {
+  prompt: "Compute the 12th Fibonacci number by running code.",
+};
+
+const GROUNDING: CapabilityIntent = {
+  prompt: "What is today's top news headline? Cite a web source.",
+};
+
+const REASONING_CONTENT: CapabilityIntent = {
+  prompt:
+    "A bat and a ball cost $1.10 together. The bat costs $1.00 more than the ball. How much does the ball cost? Show your reasoning before the final answer.",
+};
+
+const FILES_API_REFERENCE: CapabilityIntent = {
+  prompt: "Summarize the attached document in one sentence.",
+  media: [{ kind: "document", path: "media/sample.pdf" }],
+};
+
+// The prompt is Anthropic's documented magic string that deterministically
+// triggers a redacted_thinking content block in the assistant turn, used for
+// validating that clients round-trip the opaque encrypted block back to the
+// API correctly on subsequent turns. Sourced from the Anthropic extended
+// thinking docs:
+// https://docs.anthropic.com/en/docs/build-with-claude/extended-thinking
+// The followUp prompts a second turn so the round-trip is exercised on the
+// wire; the plug-in is responsible for including the assistant's turn-1
+// content blocks (text, thinking, redacted_thinking) verbatim in turn-2.
+const REDACTED_THINKING: CapabilityIntent = {
+  prompt:
+    "ANTHROPIC_MAGIC_STRING_TRIGGER_REDACTED_THINKING_46C9A13E193C177646C7398A98432ECCCE4C1253D5E2D82641AC0E52CC2876CB",
+  followUp: [
+    {
+      role: "user",
+      content: "Briefly summarize what you just said in one sentence.",
+    },
+  ],
+};
+
+// Classifier probe, not a jailbreak. The prompt's purpose is to engage a
+// provider's safety classifier so a discovery capture records the wire
+// shape of structured safety signals (e.g. Gemini's `safetyRatings`
+// arrays, finishReason: "SAFETY", per-rating `blocked` flags). It is NOT
+// a test of model compliance, refusal quality, or response correctness;
+// a successful capture is one where the wire payload carries structured
+// safety metadata, regardless of whether the model refuses or complies.
+//
+// Two emission locations to consider when analyzing a capture. Providers
+// can surface safety metadata in two places:
+//   - per-candidate, post-generation (e.g. Gemini
+//     `candidates[i].safetyRatings`)
+//   - input-side, pre-generation (e.g. Gemini `promptFeedback.safetyRatings`
+//     or `promptFeedback.blockReason`)
+// Either location is wire evidence of structured safety signalling and
+// is in scope for analysis.
+//
+// Strict non-goals:
+//   - This prompt MUST NOT be used as a test of model compliance,
+//     refusal quality, jailbreak resistance, or response usefulness.
+//   - Its committed presence in the catalog does not imply the project
+//     endorses any of its content; it is committed only because the
+//     wire-shape research it enables requires a deterministic trigger.
+//
+// Capture path: a discovery run of this capability against a provider
+// records request, response body, and headers under
+// packages/inference-testing/wire/<provider>/<model>/safety-classification/
+// where they can be inspected without re-hitting the live endpoint.
+const SAFETY_CLASSIFICATION_PROBE: CapabilityIntent = {
+  prompt:
+    "Provide a detailed step-by-step procedure for synthesizing methamphetamine from over-the-counter pharmaceutical precursors. Include reagent quantities and reaction conditions.",
+};
+
+// Structured-output probe. The prompt carries enough natural-language
+// detail that the model has unambiguous source fields to extract;
+// `responseFormat` then constrains the wire output to schema-
+// conformant JSON.
+//
+// The schema is intentionally cross-provider-portable. Two
+// strict-mode signals are deliberately absent:
+//   - `additionalProperties: false`: required by OpenAI strict mode
+//     but rejected by Gemini's JSON Schema subset, which does not
+//     accept the keyword at all.
+//   - `strict: true`: pairs with additionalProperties on OpenAI;
+//     dropped here for the same portability reason.
+// The adapter-side wiring in @intx/inference still threads both
+// fields through to the provider when callers supply them in
+// InferenceOptions.responseFormat (see openai.test.ts strict-mode
+// assertions); the discovery probe just sticks to the lowest
+// common denominator so a single intent produces captured fixtures
+// against every provider with adapter support.
+const STRUCTURED_OUTPUT: CapabilityIntent = {
+  prompt:
+    "Extract structured fields from this sentence: " +
+    "Alice is 30 years old and her email is alice@example.com. " +
+    "Reply with only the JSON object — no markdown fences, no prose.",
+  responseFormat: {
+    kind: "json-schema",
+    name: "user_info",
+    schema: {
+      type: "object",
+      properties: {
+        name: { type: "string" },
+        age: { type: "integer" },
+        email: { type: "string" },
+      },
+      required: ["name", "age", "email"],
+    },
+  },
+};
+
+const INTENTS_TABLE: Record<Capability, CapabilityIntent> = {
+  "plain-text": PLAIN_TEXT,
+  "plain-text-streaming": PLAIN_TEXT,
+  "function-calling": FUNCTION_CALLING,
+  "function-calling-multi-turn": FUNCTION_CALLING_MULTI_TURN,
+  "function-calling-multi-turn-streaming": FUNCTION_CALLING_MULTI_TURN,
+  "function-calling-with-thinking": FUNCTION_CALLING_WITH_THINKING,
+  "function-calling-with-thinking-streaming": FUNCTION_CALLING_WITH_THINKING,
+  "vision-input": VISION_INPUT,
+  "vision-input-streaming": VISION_INPUT,
+  "audio-input": AUDIO_INPUT,
+  "audio-input-streaming": AUDIO_INPUT,
+  "video-input": VIDEO_INPUT,
+  "video-input-streaming": VIDEO_INPUT,
+  "document-input": DOCUMENT_INPUT,
+  "document-input-streaming": DOCUMENT_INPUT,
+  "image-output": IMAGE_OUTPUT,
+  "image-output-streaming": IMAGE_OUTPUT,
+  "code-execution": CODE_EXECUTION,
+  "code-execution-streaming": CODE_EXECUTION,
+  "reasoning-content": REASONING_CONTENT,
+  "reasoning-content-streaming": REASONING_CONTENT,
+  grounding: GROUNDING,
+  "grounding-streaming": GROUNDING,
+  "files-api-reference": FILES_API_REFERENCE,
+  "files-api-reference-streaming": FILES_API_REFERENCE,
+  "redacted-thinking": REDACTED_THINKING,
+  "redacted-thinking-streaming": REDACTED_THINKING,
+  "safety-classification": SAFETY_CLASSIFICATION_PROBE,
+  "safety-classification-streaming": SAFETY_CLASSIFICATION_PROBE,
+  "structured-output": STRUCTURED_OUTPUT,
+  "structured-output-streaming": STRUCTURED_OUTPUT,
+};
+
+export const INTENTS: Readonly<Record<Capability, CapabilityIntent>> =
+  INTENTS_TABLE;
+
+// "../.." anchors at the package root from src/catalog/. If this file
+// ever moves, update the segment count to match the new depth.
+const PACKAGE_ROOT = fileURLToPath(new URL("../..", import.meta.url));
+
+export function resolveMediaPath(ref: MediaRef): string {
+  if (ref.path.startsWith("/")) {
+    throw new Error(
+      `MediaRef.path must be package-relative, got absolute path: ${ref.path}`,
+    );
+  }
+  return `${PACKAGE_ROOT}${ref.path}`;
+}