@yanlinglabs/winter-provider-conformance 0.0.2

package/dist/live/cases.d.ts

@@ -0,0 +1,87 @@
+import type { ProviderAdapter, ProviderContext } from "@yanlinglabs/winter-provider-runtime";
+import type { WinterModelDescriptor } from "@yanlinglabs/winter-provider-catalog";
+/**
+ * A case's OWN assertion failure — the one error whose `.message` the runner prints (review round 1,
+ * I1).
+ *
+ * The distinction is not stylistic. Every message constructed with this class is Winter-authored
+ * text in this file, built from measurements: a stop reason, a byte count, a normalized error code.
+ * Every OTHER error reaching the runner came from an adapter, and an adapter failure is a
+ * `ProviderRequestError` whose message embeds a 200-character snippet of the provider's response
+ * body (`provider-runtime/src/errors.ts`). That snippet is scrubbed of credential-shaped strings, but
+ * it is still response CONTENT, and the constraint on this gate's output is verbatim: identifiers and
+ * byte counts only.
+ *
+ * So the type IS the permission to print. A message that is safe to render is one this file wrote.
+ */
+export declare class LiveCaseAssertionError extends Error {
+    constructor(message: string);
+}
+export type LiveCaseId = "discovery" | "text-turn" | "tool-round" | "thinking-summary" | "count-tokens" | "honest-identity-inference";
+export interface LiveCaseSpec {
+    id: LiveCaseId;
+    question: string;
+}
+/** The legs the gate runs, in order: cheapest and least stateful first. */
+export declare const LIVE_CASES: readonly LiveCaseSpec[];
+/**
+ * The auth-shaped keys a refused inference request's body may be reported by NAME (WS-13b §4).
+ *
+ * AN ALLOWLIST, and it is the whole safety argument for reporting anything at all. A provider's error
+ * body is content and this gate never prints it (`live/index.ts`'s rendering rule) — but when a
+ * subscription bearer is refused, the ONE thing an operator needs is which auth dimension the vendor
+ * says was missing, and that is a handful of named scalar fields. Anything not named here, including
+ * every human-readable message, is dropped.
+ *
+ * `x_xai_token_auth` and `auth_kind` are the two xAI's own proxy reports (Lane O's capture); `scope`
+ * and `token_auth` are the neighbouring spellings the same family uses.
+ */
+export declare const AUTH_DIMENSION_FIELDS: readonly string[];
+/**
+ * Pulls ONLY the allowlisted auth dimensions out of an error message, as `field=value`.
+ *
+ * Exported for its own unit test, because "a marker elsewhere in the same body never survives" is the
+ * property that makes this safe and it must be falsifiable without a vendor. Values are bounded to a
+ * scalar shape (`[\w.:/-]+` — the `/` admits a scoped value like `grok-cli:access` written as a path
+ * and a bare URL host), so a field whose value is a sentence contributes only its first token.
+ */
+export declare function authDimensionsOf(message: string): string[];
+export interface LiveCaseContext {
+    /** The catalog PROVIDER id. Named in a report's remediation, so it is threaded rather than assumed. */
+    providerId: string;
+    adapter: ProviderAdapter;
+    ctx: ProviderContext;
+    /** The provider-local id that goes on the wire (never the catalog key). */
+    model: string;
+    /** Absent only for an `allowUnlisted` pass-through, in which case every capability-gated case skips. */
+    descriptor?: WinterModelDescriptor;
+    /**
+     * The PROVIDER row's pricing basis (WS-13b §1) — a provider fact, which is why it does not come off
+     * the model descriptor. The inference-path reversion case is about an entitlement, and
+     * `subscription` is what "an entitlement rather than a key" means in catalog terms.
+     */
+    pricingBasis?: "token" | "subscription" | "free";
+    /**
+     * WHICH DOCUMENTED THIRD-PARTY PATH this run's credential came down (WS-13b §1), supplied by the
+     * gate that planned the target.
+     *
+     * It is the discriminator the reversion case needs and the one `pricingBasis` cannot be:
+     * `subscription` catches four rows and two of them (`clinepass`, `kimi-coding`) are ordinary
+     * API-KEY products with a seat price. Reporting a bad key on one of those as "the vendor rejected
+     * Winter's identity" would be a false alarm about the one thing this phase is careful about.
+     *
+     * `ProviderContext.authRef` cannot answer it: its `kind` is a LOCATOR (`keychain`/`env`/…), never
+     * the material's — a host may legitimately keep an api key in the Keychain. Absent, the case
+     * declines rather than guessing.
+     */
+    targetKind?: "api-key" | "oauth" | "keyless";
+    signal?: AbortSignal;
+}
+export type LiveCaseResult = {
+    status: "ok";
+    detail: string;
+} | {
+    status: "skipped";
+    detail: string;
+};
+export declare const LIVE_CASE_IMPLS: Record<LiveCaseId, (ctx: LiveCaseContext) => Promise<LiveCaseResult>>;

package/dist/live/index.d.ts

@@ -0,0 +1,101 @@
+import type { ProviderAdapter, ProviderContext } from "@yanlinglabs/winter-provider-runtime";
+import type { WinterModelDescriptor } from "@yanlinglabs/winter-provider-catalog";
+import { LIVE_CASES, LiveCaseAssertionError, type LiveCaseContext, type LiveCaseId, type LiveCaseSpec } from "./cases.js";
+export { LIVE_CASES, LiveCaseAssertionError };
+export type { LiveCaseContext, LiveCaseId, LiveCaseSpec };
+export interface LiveCaseOutcome {
+    id: LiveCaseId;
+    status: "ok" | "skipped" | "failed";
+    detail: string;
+    /** Wall-clock, because a live gate's most common real failure is "it answered, eventually". */
+    ms: number;
+}
+export interface LiveReport {
+    providerId: string;
+    adapterId: string;
+    adapterVersion: string;
+    /** The catalog key, so a reader can tell which row this run is evidence for. */
+    modelKey: string;
+    outcomes: LiveCaseOutcome[];
+    /** True when nothing FAILED. A skip is a recorded capability fact, not a failure. */
+    ok: boolean;
+}
+export interface RunLiveCasesOptions {
+    providerId: string;
+    modelKey: string;
+    adapter: ProviderAdapter;
+    ctx: ProviderContext;
+    /** The provider-local id that goes on the wire. */
+    model: string;
+    descriptor?: WinterModelDescriptor;
+    /** The PROVIDER row's pricing basis (WS-13b §1). Gates the inference-path reversion case. */
+    pricingBasis?: "token" | "subscription" | "free";
+    /** Which documented third-party path this run's credential came down. `runLiveTarget` supplies it from the gate's own plan. */
+    targetKind?: LiveTargetKindLabel;
+    signal?: AbortSignal;
+    /** Progress, one line per case as it finishes. Identifiers and counts only — the case details already obey that rule. */
+    onProgress?: (outcome: LiveCaseOutcome) => void;
+}
+export declare function runLiveCases(opts: RunLiveCasesOptions): Promise<LiveReport>;
+/** How the target's credential was named — WS-13b §1's three documented third-party paths. Mirrors the gate's `LiveTargetKind`. */
+export type LiveTargetKindLabel = "api-key" | "oauth" | "keyless";
+export interface LiveRowSummary {
+    providerId: string;
+    /** The CATALOG KEY (`<providerId>/<model>`), never the provider-local id and never a display name. */
+    model: string;
+    kind: LiveTargetKindLabel;
+    /** True when no case FAILED. A skipped case is a recorded capability fact, not a failure. */
+    ok: boolean;
+    /** The summed wall-clock of every case. A live gate's most common real failure is "it answered, eventually". */
+    latencyMs: number;
+    /**
+     * True only when the tool round ran and PASSED.
+     *
+     * A skip therefore reads `false` — deliberately, because the question this column answers is "can
+     * this row be driven agentically?", and "the descriptor says it cannot" is a no. The per-case line
+     * above always says WHICH of the two it was, so the row is a summary and never the whole story.
+     */
+    toolCallOk: boolean;
+    /**
+     * The identity string THIS BUILD sends (`winterUserAgent()`), recorded beside the result so a live
+     * run's output carries the identity claim WS-13b §1 makes.
+     *
+     * It is not a wire observation — this runner never sees an outgoing header. That Winter's own
+     * user-agent is genuinely on every family's requests is pinned by the corpus (`corpus/*.test.ts`,
+     * "every request carries Winter's OWN user-agent"), against the live request a fake received.
+     */
+    identityHeader: string;
+    /**
+     * The row's ADMISSION EVIDENCE TIER (fix-wave R-FW-3, condition (b)).
+     *
+     * Printed because promotion is TWO-KEY: a live pass alone does not promote a row — it takes a live
+     * pass AND a fetched vendor document, with the citation upgraded in the same reviewed commit. A
+     * green row from a `pinned-upstream` provider says "the path works", not "the evidence is
+     * complete", and an operator reading a pasted row is exactly who would otherwise promote it by
+     * habit.
+     *
+     * A catalog fact, read off the resolved provider row — never re-derived here.
+     */
+    admissionTier: string;
+}
+export interface LiveRowSummaryOptions {
+    kind: LiveTargetKindLabel;
+    identityHeader: string;
+    admissionTier: string;
+}
+/** Folds a finished `LiveReport` into its one-line row. Pure: no adapter, no endpoint, no clock. */
+export declare function liveRowSummary(report: LiveReport, opts: LiveRowSummaryOptions): LiveRowSummary;
+/** `key=value` pairs, in a fixed order, so a run's rows grep and diff. */
+export declare function formatLiveRow(row: LiveRowSummary): string;
+/**
+ * The per-target case: run every live leg against one target, and fold the result into its row.
+ *
+ * One call so the gate cannot run the cases and then forget the row, or report a row built from
+ * something other than the run it names.
+ */
+export declare function runLiveTarget(opts: RunLiveCasesOptions & LiveRowSummaryOptions): Promise<{
+    report: LiveReport;
+    row: LiveRowSummary;
+}>;
+/** One line per case. Identifiers, counts and durations only — never a byte of what a provider returned. */
+export declare function formatLiveReport(report: LiveReport): string;

package/package.json

@@ -0,0 +1,50 @@
+{
+  "name": "@yanlinglabs/winter-provider-conformance",
+  "version": "0.0.2",
+  "license": "MIT",
+  "type": "module",
+  "engines": {
+    "bun": ">=1.2"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/yanlingLabs/winter-agent-sdk.git",
+    "directory": "packages/provider-conformance"
+  },
+  "homepage": "https://github.com/yanlingLabs/winter-agent-sdk",
+  "bugs": {
+    "url": "https://github.com/yanlingLabs/winter-agent-sdk/issues"
+  },
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./fakes": {
+      "types": "./dist/fakes/index.d.ts",
+      "default": "./dist/fakes/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "NOTICE",
+    "README.md",
+    "LICENSE"
+  ],
+  "publishConfig": {
+    "access": "restricted"
+  },
+  "winter": {
+    "publish": {
+      "npm": true,
+      "harness": true
+    }
+  },
+  "dependencies": {
+    "@yanlinglabs/winter-provider-catalog": "0.0.2",
+    "@yanlinglabs/winter-provider-runtime": "0.0.2"
+  },
+  "scripts": {}
+}