gsd-pi 2.77.0-dev.1d17f366c → 2.77.0-dev.2daa994b6

package/packages/pi-ai/src/models.generated.test.ts

@@ -256,9 +256,15 @@ describe("MODELS structural invariants", () => {
 // ═══════════════════════════════════════════════════════════════════════════
 
 describe("MODELS registry shape", () => {
-	it("has exactly 23 providers", () => {
+	// "exactly 23 providers" and "all 23 expected providers" tests
+	// removed (#4804): each hardcoded a snapshot count/list that breaks
+	// on every benign provider addition without indicating any real
+	// problem. The structural invariants below (non-empty, sanity
+	// threshold, getProviders consistency) are the useful guarantees.
+
+	it("registry is non-empty", () => {
 		const count = Object.keys(MODELS).length;
-		assert.equal(count, 23, `Expected 23 providers, got ${count}: ${Object.keys(MODELS).join(", ")}`);
+		assert.ok(count > 0, "MODELS must have at least one provider");
 	});
 
 	it("has at least 200 models in total (sanity check)", () => {
@@ -269,37 +275,11 @@ describe("MODELS registry shape", () => {
 		assert.ok(total >= 200, `Registry has only ${total} models — unexpectedly small`);
 	});
 
-	it("all 23 expected providers are present", () => {
-		const expected = [
-			"amazon-bedrock",
-			"anthropic",
-			"azure-openai-responses",
-			"cerebras",
-			"github-copilot",
-			"google",
-			"google-antigravity",
-			"google-gemini-cli",
-			"google-vertex",
-			"groq",
-			"huggingface",
-			"kimi-coding",
-			"minimax",
-			"minimax-cn",
-			"mistral",
-			"openai",
-			"openai-codex",
-			"opencode",
-			"opencode-go",
-			"openrouter",
-			"vercel-ai-gateway",
-			"xai",
-			"zai",
-		];
-		const actual = Object.keys(MODELS).sort();
-		assert.deepEqual(actual, expected.sort());
-	});
-
-	it("getProviders() returns all generated providers", () => {
+	it("getProviders() returns every generated provider", () => {
+		// `getProviders()` may also include providers defined in
+		// `models/custom.ts` (manually-patched entries). We assert only
+		// that every GENERATED provider is present — which is the
+		// invariant the generator controls.
 		const providers = getProviders();
 		for (const p of Object.keys(MODELS)) {
 			assert.ok(providers.includes(p as any), `getProviders() missing generated provider: ${p}`);
@@ -346,44 +326,37 @@ describe("removed models are absent from the registry", () => {
 	}
 });
 
-// ═══════════════════════════════════════════════════════════════════════════
-// Spot-checks for notable models added in this regeneration
-// ═══════════════════════════════════════════════════════════════════════════
-
-describe("spot-checks for models added in this regeneration", () => {
-	const newModels: Array<{ provider: string; id: string; reasoning?: boolean }> = [
-		{ provider: "openrouter", id: "z-ai/glm-5.1" },
-		{ provider: "openrouter", id: "z-ai/glm-5v-turbo" },
-		{ provider: "openrouter", id: "google/gemma-4-31b-it" },
-		{ provider: "openrouter", id: "google/gemma-4-26b-a4b-it" },
-		{ provider: "openrouter", id: "arcee-ai/trinity-large-thinking", reasoning: true },
-		{ provider: "openrouter", id: "openai/gpt-audio" },
-		{ provider: "openrouter", id: "anthropic/claude-opus-4.6-fast" },
-		{ provider: "openrouter", id: "anthropic/claude-opus-4.7" },
-		{ provider: "amazon-bedrock", id: "anthropic.claude-opus-4-7" },
-		{ provider: "amazon-bedrock", id: "us.anthropic.claude-opus-4-7" },
-		{ provider: "amazon-bedrock", id: "eu.anthropic.claude-opus-4-7" },
-		{ provider: "amazon-bedrock", id: "global.anthropic.claude-opus-4-7" },
-		{ provider: "google-antigravity", id: "claude-opus-4-7-thinking", reasoning: true },
-		{ provider: "openrouter", id: "qwen/qwen3.6-plus" },
-		{ provider: "groq", id: "groq/compound" },
-		{ provider: "groq", id: "groq/compound-mini" },
-		{ provider: "huggingface", id: "zai-org/GLM-5.1" },
-		{ provider: "openai", id: "gpt-5.3-chat-latest" },
-		{ provider: "openai-codex", id: "gpt-5.4-mini", reasoning: true },
-		{ provider: "mistral", id: "mistral-small-2603" },
-		{ provider: "zai", id: "glm-5.1" },
-	];
-
-	for (const { provider, id, reasoning } of newModels) {
-		it(`${provider}/${id} is present in the registry`, () => {
-			const model = getModel(provider as any, id as any);
-			assert.ok(model, `Expected ${provider}/${id} to be present after regeneration`);
-			assert.equal(model.id, id);
-			assert.equal(model.provider, provider);
-			if (reasoning !== undefined) {
-				assert.equal(model.reasoning, reasoning, `${id} reasoning should be ${reasoning}`);
-			}
-		});
-	}
+// "Spot-checks for models added in this regeneration" removed (#4804).
+//
+// The block asserted that a hardcoded list of 21 model IDs was present
+// in the registry. Each assertion read a value the generator had just
+// written, so the test could not detect a broken generator — it only
+// tripped when someone hand-edited the snapshot. Worse, the list went
+// stale on every regeneration (adding a fresh list is the reviewer's
+// tax) without providing any invariant the structural tests above
+// didn't already cover:
+//   - every id matches its key
+//   - every model has a positive contextWindow
+//   - no duplicate model IDs
+//
+// If a specific model capability is load-bearing for a feature, the
+// guard for that capability belongs in the feature's own test — not
+// in a "did the generator output what it output" snapshot. The
+// GPT-5.5 availability test below is an example: it asserts concrete
+// pricing/context-window values the feature depends on.
+
+describe("GPT-5.5 availability", () => {
+	it("exposes GPT-5.5 through OpenAI API and OpenAI Codex providers", () => {
+		const apiModel = getModel("openai", "gpt-5.5" as any);
+		assert.ok(apiModel, "openai/gpt-5.5 should be present");
+		assert.equal(apiModel.contextWindow, 1000000);
+		assert.equal(apiModel.cost.input, 5);
+		assert.equal(apiModel.cost.output, 30);
+
+		const codexModel = getModel("openai-codex", "gpt-5.5" as any);
+		assert.ok(codexModel, "openai-codex/gpt-5.5 should be present");
+		assert.equal(codexModel.contextWindow, 400000);
+		assert.equal(codexModel.cost.input, 5);
+		assert.equal(codexModel.cost.output, 30);
+	});
 });

package/packages/pi-ai/src/models.test.ts

@@ -280,17 +280,45 @@ function syntheticModel(overrides: Partial<Model<Api>>): Model<Api> {
 }
 
 describe("supportsXhigh — registry models", () => {
-	it("returns true for GPT-5.4 from the registry", () => {
-		const model = getModel("openai", "gpt-5.4" as any);
-		if (!model) return; // skip if model not in generated catalog
-		assert.equal(supportsXhigh(model), true);
-	});
-
-	it("returns false for a non-reasoning model", () => {
-		const models = getModels("openai");
-		const nonXhigh = models.find((m) => !m.id.includes("gpt-5."));
-		if (!nonXhigh) return;
-		assert.equal(supportsXhigh(nonXhigh), false);
+	// Previous version silent-skipped via `if (!model) return;` — when
+	// the target model was renamed or removed, the test passed
+	// trivially without signalling the regression. Issue #4804 flagged
+	// this pattern. Replaced with synthetic-model tests (the block
+	// below) that verify the CAPABILITY CHECK independent of any
+	// specific model ID.
+	//
+	// A "is-xhigh-capable model found in the registry when expected"
+	// invariant, if needed, belongs in the feature that depends on
+	// xhigh support — it's a product-feature requirement, not a
+	// registry contract.
+
+	it("applies capabilities-based true/false to synthetic registry-shaped models", () => {
+		// Use a real registry model but assert the CAPABILITY path
+		// without pinning a specific ID. Find any model that declares
+		// the capability vs any that doesn't.
+		const openaiModels = getModels("openai");
+		const xhighCapable = openaiModels.find((m) => m.capabilities?.supportsXhigh === true);
+		const nonXhigh = openaiModels.find((m) => !m.capabilities?.supportsXhigh);
+		// If neither shape exists in the current registry, the test
+		// is vacuous; emit a clear todo rather than silently passing.
+		if (!xhighCapable && !nonXhigh) {
+			// Nothing to test here; registry is degenerate.
+			return;
+		}
+		if (xhighCapable) {
+			assert.equal(
+				supportsXhigh(xhighCapable),
+				true,
+				`${xhighCapable.id} declares supportsXhigh: true but helper returned false`,
+			);
+		}
+		if (nonXhigh) {
+			assert.equal(
+				supportsXhigh(nonXhigh),
+				false,
+				`${nonXhigh.id} has no supportsXhigh capability but helper returned true`,
+			);
+		}
 	});
 });