@intx/inference-discovery 0.1.2

package/src/cli.ts

@@ -0,0 +1,133 @@
+export interface ParsedCLIRun {
+  kind: "run";
+  provider: string;
+  models: readonly string[];
+  capabilities: readonly string[];
+  all: boolean;
+}
+
+export interface ParsedCLIHelp {
+  kind: "help";
+  message: string;
+}
+
+export interface ParsedCLIError {
+  kind: "error";
+  message: string;
+}
+
+export type ParsedCLI = ParsedCLIRun | ParsedCLIHelp | ParsedCLIError;
+
+export const HELP_TEXT = `Usage: discover --provider <name> [options]
+
+Options:
+  --provider <name>     Required. Selects the provider plug-in to invoke.
+  --model <name>        Restrict to this model. Repeatable.
+  --only <capability>   Restrict to this capability. Repeatable.
+  --all                 Run every supported model x capability combination.
+                        Mutually exclusive with --model and --only.
+  --help, -h            Show this message.
+
+When --all is omitted and no --model/--only flags are given, the runner
+will fail; pass --all explicitly or narrow the scope with --model/--only.
+`;
+
+function takeValue(
+  argv: readonly string[],
+  index: number,
+  flag: string,
+): { value: string; nextIndex: number } | { error: string } {
+  const next = argv[index + 1];
+  if (next === undefined || next.startsWith("-")) {
+    return { error: `Flag ${flag} requires a value` };
+  }
+  return { value: next, nextIndex: index + 2 };
+}
+
+export function parseCLI(argv: readonly string[]): ParsedCLI {
+  let provider: string | undefined;
+  const models: string[] = [];
+  const capabilities: string[] = [];
+  let all = false;
+
+  let i = 0;
+  while (i < argv.length) {
+    const arg = argv[i];
+    if (arg === undefined) {
+      i++;
+      continue;
+    }
+    if (arg === "--help" || arg === "-h") {
+      return { kind: "help", message: HELP_TEXT };
+    }
+    if (arg === "--all") {
+      all = true;
+      i++;
+      continue;
+    }
+    if (arg === "--provider") {
+      const result = takeValue(argv, i, "--provider");
+      if ("error" in result) {
+        return { kind: "error", message: result.error };
+      }
+      if (provider !== undefined) {
+        return {
+          kind: "error",
+          message: "--provider may only be specified once",
+        };
+      }
+      provider = result.value;
+      i = result.nextIndex;
+      continue;
+    }
+    if (arg === "--model") {
+      const result = takeValue(argv, i, "--model");
+      if ("error" in result) {
+        return { kind: "error", message: result.error };
+      }
+      models.push(result.value);
+      i = result.nextIndex;
+      continue;
+    }
+    if (arg === "--only") {
+      const result = takeValue(argv, i, "--only");
+      if ("error" in result) {
+        return { kind: "error", message: result.error };
+      }
+      capabilities.push(result.value);
+      i = result.nextIndex;
+      continue;
+    }
+    return { kind: "error", message: `Unknown argument: ${arg}` };
+  }
+
+  if (provider === undefined) {
+    return {
+      kind: "error",
+      message: "--provider <name> is required",
+    };
+  }
+
+  if (all && (models.length > 0 || capabilities.length > 0)) {
+    return {
+      kind: "error",
+      message: "--all is mutually exclusive with --model and --only",
+    };
+  }
+
+  if (!all && models.length === 0 && capabilities.length === 0) {
+    return {
+      kind: "error",
+      message:
+        "Specify --all or narrow the scope with --model/--only (at least one of either)",
+    };
+  }
+
+  return {
+    kind: "run",
+    provider,
+    models,
+    capabilities,
+    all,
+  };
+}

package/src/content-type.test.ts

@@ -0,0 +1,45 @@
+import { describe, test, expect } from "bun:test";
+import { detectResponseKind } from "./content-type";
+
+describe("detectResponseKind", () => {
+  test("returns 'json' for application/json", () => {
+    const headers = new Headers({ "content-type": "application/json" });
+    expect(detectResponseKind(headers)).toBe("json");
+  });
+
+  test("returns 'json' for application/json with charset", () => {
+    const headers = new Headers({
+      "content-type": "application/json; charset=utf-8",
+    });
+    expect(detectResponseKind(headers)).toBe("json");
+  });
+
+  test("returns 'sse' for text/event-stream", () => {
+    const headers = new Headers({ "content-type": "text/event-stream" });
+    expect(detectResponseKind(headers)).toBe("sse");
+  });
+
+  test("returns 'sse' for text/event-stream with charset", () => {
+    const headers = new Headers({
+      "content-type": "text/event-stream; charset=utf-8",
+    });
+    expect(detectResponseKind(headers)).toBe("sse");
+  });
+
+  test("matches Content-Type case-insensitively", () => {
+    const headers = new Headers({ "content-type": "Application/JSON" });
+    expect(detectResponseKind(headers)).toBe("json");
+  });
+
+  test("throws when content-type is missing", () => {
+    const headers = new Headers();
+    expect(() => detectResponseKind(headers)).toThrow(
+      /response has no Content-Type/,
+    );
+  });
+
+  test("throws on unknown content-type", () => {
+    const headers = new Headers({ "content-type": "text/plain" });
+    expect(() => detectResponseKind(headers)).toThrow(/text\/plain/);
+  });
+});

package/src/content-type.ts

@@ -0,0 +1,20 @@
+export type ResponseKind = "sse" | "json";
+
+export function detectResponseKind(headers: Headers): ResponseKind {
+  const raw = headers.get("content-type");
+  if (raw === null) {
+    throw new Error(
+      "Cannot detect response kind: response has no Content-Type header",
+    );
+  }
+  const normalized = raw.trim().toLowerCase();
+  if (normalized.startsWith("text/event-stream")) {
+    return "sse";
+  }
+  if (normalized.startsWith("application/json")) {
+    return "json";
+  }
+  throw new Error(
+    `Unsupported response Content-Type: ${raw}. Expected text/event-stream or application/json.`,
+  );
+}

package/src/env.test.ts

@@ -0,0 +1,31 @@
+import { describe, test, expect } from "bun:test";
+import { requireEnv, requireEnvSet } from "./env";
+
+describe("requireEnv", () => {
+  test("returns the value when set", () => {
+    expect(requireEnv("FOO", { FOO: "bar" })).toBe("bar");
+  });
+
+  test("throws when missing", () => {
+    expect(() => requireEnv("FOO", {})).toThrow(/FOO/);
+  });
+
+  test("throws when empty", () => {
+    expect(() => requireEnv("FOO", { FOO: "" })).toThrow(/FOO/);
+  });
+});
+
+describe("requireEnvSet", () => {
+  test("returns all values when all set", () => {
+    const result = requireEnvSet(["A", "B"], { A: "1", B: "2", C: "3" });
+    expect(result).toEqual({ A: "1", B: "2" });
+  });
+
+  test("throws listing all missing names", () => {
+    expect(() => requireEnvSet(["A", "B", "C"], { B: "2" })).toThrow(/A, C/);
+  });
+
+  test("treats empty values as missing", () => {
+    expect(() => requireEnvSet(["A"], { A: "" })).toThrow(/A/);
+  });
+});

package/src/env.ts

@@ -0,0 +1,36 @@
+export function requireEnv(
+  name: string,
+  env: NodeJS.ProcessEnv = process.env,
+): string {
+  const value = env[name];
+  if (value === undefined || value === "") {
+    throw new Error(
+      `Required environment variable ${name} is not set. ` +
+        `Export it before running discovery.`,
+    );
+  }
+  return value;
+}
+
+export function requireEnvSet(
+  names: readonly string[],
+  env: NodeJS.ProcessEnv = process.env,
+): Record<string, string> {
+  const result: Record<string, string> = {};
+  const missing: string[] = [];
+  for (const name of names) {
+    const value = env[name];
+    if (value === undefined || value === "") {
+      missing.push(name);
+      continue;
+    }
+    result[name] = value;
+  }
+  if (missing.length > 0) {
+    throw new Error(
+      `Required environment variables are not set: ${missing.join(", ")}. ` +
+        `Export them before running discovery.`,
+    );
+  }
+  return result;
+}

package/src/index.ts

@@ -0,0 +1,24 @@
+export type {
+  ProviderPlugin,
+  CaptureStep,
+  CapturedResponse,
+  IterateCaptureStepsOpts,
+} from "./plugin";
+export { runCapture, type FetchLike, type RunCaptureOpts } from "./runner";
+export {
+  writeCapture,
+  type ResponseBody,
+  type WriteCaptureInput,
+} from "./write-capture";
+export { detectResponseKind, type ResponseKind } from "./content-type";
+export { assertNotCI } from "./ci-guard";
+export { requireEnv, requireEnvSet } from "./env";
+export {
+  parseCLI,
+  HELP_TEXT,
+  type ParsedCLI,
+  type ParsedCLIRun,
+  type ParsedCLIHelp,
+  type ParsedCLIError,
+} from "./cli";
+export { buildManifest, type BuildManifestOpts } from "./manifest";

package/src/manifest.test.ts

@@ -0,0 +1,56 @@
+import { describe, test, expect } from "bun:test";
+import { type } from "arktype";
+import { FixtureManifest } from "./catalog";
+import { buildManifest } from "./manifest";
+
+describe("buildManifest", () => {
+  test("produces a record accepted by FixtureManifest", () => {
+    const manifest = buildManifest({
+      provider: "google-genai",
+      model: "gemini-2.5-flash",
+      capability: "plain-text",
+      now: () => new Date("2026-05-22T00:00:00Z"),
+    });
+    const validated = FixtureManifest(manifest);
+    expect(validated instanceof type.errors).toBe(false);
+    expect(manifest.provider).toBe("google-genai");
+    expect(manifest.model).toBe("gemini-2.5-flash");
+    expect(manifest.capability).toBe("plain-text");
+    expect(manifest.capturedAt).toBe("2026-05-22T00:00:00.000Z");
+    expect(manifest.schemaVersion).toBe("1");
+    expect(manifest.observedModelVersion).toBeUndefined();
+  });
+
+  test("omits observedModelVersion when not provided", () => {
+    const manifest = buildManifest({
+      provider: "p",
+      model: "m",
+      capability: "plain-text",
+      now: () => new Date("2026-05-22T00:00:00Z"),
+    });
+    expect("observedModelVersion" in manifest).toBe(false);
+  });
+
+  test("includes observedModelVersion when provided", () => {
+    const manifest = buildManifest({
+      provider: "p",
+      model: "m",
+      capability: "plain-text",
+      now: () => new Date("2026-05-22T00:00:00Z"),
+      observedModelVersion: "abc123",
+    });
+    expect(manifest.observedModelVersion).toBe("abc123");
+  });
+
+  test("uses real Date by default", () => {
+    const before = new Date().toISOString();
+    const manifest = buildManifest({
+      provider: "p",
+      model: "m",
+      capability: "plain-text",
+    });
+    const after = new Date().toISOString();
+    expect(manifest.capturedAt >= before).toBe(true);
+    expect(manifest.capturedAt <= after).toBe(true);
+  });
+});

package/src/manifest.ts

@@ -0,0 +1,29 @@
+import { type } from "arktype";
+import { type Capability, FixtureManifest } from "./catalog";
+
+export interface BuildManifestOpts {
+  provider: string;
+  model: string;
+  capability: Capability;
+  now?: () => Date;
+  observedModelVersion?: string;
+}
+
+export function buildManifest(opts: BuildManifestOpts): FixtureManifest {
+  const now = opts.now ?? (() => new Date());
+  const base: Record<string, unknown> = {
+    provider: opts.provider,
+    model: opts.model,
+    capability: opts.capability,
+    capturedAt: now().toISOString(),
+    schemaVersion: "1",
+  };
+  if (opts.observedModelVersion !== undefined) {
+    base.observedModelVersion = opts.observedModelVersion;
+  }
+  const validated = FixtureManifest(base);
+  if (validated instanceof type.errors) {
+    throw new Error(`Invalid manifest: ${validated.summary}`);
+  }
+  return validated;
+}

package/src/plugin.ts

@@ -0,0 +1,70 @@
+import type { Capability, CapabilityIntent } from "./catalog";
+
+export interface CapturedResponse {
+  status: number;
+  headers: Record<string, string>;
+  // Populated for application/json responses; null for SSE.
+  parsed: unknown | null;
+  // Populated for text/event-stream responses; null otherwise. Iterators
+  // that consume streaming turn-1 responses to build a turn-2 body parse
+  // these bytes themselves — the runner does not interpret SSE.
+  bytes: Uint8Array | null;
+}
+
+export interface IterateCaptureStepsOpts {
+  model: string;
+  capability: Capability;
+  intent: CapabilityIntent;
+}
+
+interface CaptureStepBase {
+  // Subdirectory under the capture root for this step's artifacts.
+  // null places the step's files directly under the capture root; a string
+  // segregates them (e.g. "turn-1", "turn-2", "upload", "generate").
+  subdir: string | null;
+  url: string;
+  // Defaults to "POST" when omitted.
+  method?: "POST" | "PUT" | "PATCH";
+  // Headers the step contributes on top of the runner's content-type default.
+  // Per-step headers may override the default content-type (for example,
+  // a multipart upload). They MUST NOT collide with the plug-in's auth
+  // headers — the runner detects that collision and throws, on the
+  // principle that auth is a plug-in-wide invariant and capability-
+  // specific overrides belong on the step.
+  headers?: Record<string, string>;
+}
+
+// A step whose body is a JSON-serializable value. The runner writes it to
+// `request.json` after JSON.stringify and sends it with the default
+// `Content-Type: application/json` unless overridden via `headers`.
+export interface JsonCaptureStep extends CaptureStepBase {
+  kind: "json";
+  body: unknown;
+}
+
+// A step whose body is raw bytes (e.g. a multipart upload envelope, a
+// single-part octet-stream). The runner writes the bytes to `request.bin`
+// and sends them verbatim with the supplied `contentType`. The plug-in
+// owns content-type because there is no sensible default for non-JSON
+// bodies.
+export interface RawCaptureStep extends CaptureStepBase {
+  kind: "raw";
+  contentType: string;
+  body: Uint8Array;
+}
+
+export type CaptureStep = JsonCaptureStep | RawCaptureStep;
+
+export interface ProviderPlugin {
+  name: string;
+  models: readonly string[];
+  redactRequestHeaders: readonly string[];
+  redactResponseHeaders: readonly string[];
+  // Plug-in-wide credentials only. Capability-specific headers (beta
+  // flags, upload-protocol markers) belong on the step's `headers` map.
+  buildAuthHeaders(): Record<string, string>;
+  extractReasoningTrace?(parsed: unknown): unknown | null;
+  iterateCaptureSteps(
+    opts: IterateCaptureStepsOpts,
+  ): Generator<CaptureStep, void, CapturedResponse>;
+}