@pi-in-go/pigpen-pi-typesafe 0.1.0

package/port/oracle/tests/credentials.test.ts

@@ -0,0 +1,144 @@
+import assert from "node:assert/strict";
+import { mkdtempSync, readFileSync, statSync, writeFileSync, chmodSync, mkdirSync } from "node:fs";
+import { rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { after, before, beforeEach, test } from "node:test";
+import { clearStoredApiKey, credentialsPath, keySituation, keySourceLabel, normalizeApiKey, readStoredApiKey, resolveApiKey, storeApiKey } from "../src/credentials.js";
+import { createTypeSafe, TypeSafeIntegrationError } from "../src/index.js";
+
+const savedAgentDir = process.env.PI_CODING_AGENT_DIR;
+const savedKey = process.env.TYPESAFE_API_KEY;
+let agentDir: string;
+const validKey = "ts_test_key_0123456789abcdef";
+
+before(() => {
+  agentDir = mkdtempSync(join(tmpdir(), "pi-typesafe-credentials-"));
+  process.env.PI_CODING_AGENT_DIR = agentDir;
+});
+beforeEach(() => {
+  delete process.env.TYPESAFE_API_KEY;
+  clearStoredApiKey();
+});
+after(async () => {
+  if (savedAgentDir === undefined) delete process.env.PI_CODING_AGENT_DIR; else process.env.PI_CODING_AGENT_DIR = savedAgentDir;
+  if (savedKey === undefined) delete process.env.TYPESAFE_API_KEY; else process.env.TYPESAFE_API_KEY = savedKey;
+  await rm(agentDir, { recursive: true, force: true });
+});
+
+test("credentials live under Pi's agent directory", () => {
+  assert.equal(credentialsPath(), join(agentDir, "pi-typesafe", "auth.json"));
+});
+
+test("store, read, and clear with owner-only permissions", () => {
+  assert.equal(resolveApiKey(), undefined);
+  const path = storeApiKey(`  ${validKey}\n`);
+  assert.equal(path, credentialsPath());
+  assert.equal(statSync(path).mode & 0o777, 0o600);
+  assert.equal(statSync(join(agentDir, "pi-typesafe")).mode & 0o777, 0o700);
+  assert.deepEqual(JSON.parse(readFileSync(path, "utf8")), { apiKey: validKey });
+  assert.deepEqual(resolveApiKey(), { key: validKey, source: "stored" });
+  assert.equal(clearStoredApiKey(), true);
+  assert.equal(clearStoredApiKey(), false);
+  assert.equal(readStoredApiKey(), undefined);
+});
+
+test("environment variable takes precedence over the stored key", () => {
+  storeApiKey(validKey);
+  process.env.TYPESAFE_API_KEY = "env_key_0123456789abcdef";
+  assert.deepEqual(resolveApiKey(), { key: "env_key_0123456789abcdef", source: "environment" });
+});
+
+test("implausible keys are rejected without saving", () => {
+  for (const value of ["", "short", "has space in it 0123456789", "tab\tseparated0123456789", "ключ-with-non-ascii-0123456789", "x".repeat(513), 42, null]) {
+    assert.throws(() => storeApiKey(value), (error: unknown) => error instanceof TypeSafeIntegrationError && error.code === "validation" && (String(value).length < 4 || !error.message.includes(String(value))));
+  }
+  assert.equal(readStoredApiKey(), undefined);
+  assert.equal(normalizeApiKey(` ${validKey} `), validKey);
+});
+
+test("group- or world-readable credential files are refused", { skip: process.platform === "win32" }, () => {
+  storeApiKey(validKey);
+  chmodSync(credentialsPath(), 0o644);
+  assert.throws(() => resolveApiKey(), (error: unknown) => error instanceof TypeSafeIntegrationError && error.code === "configuration" && /chmod 600/.test(error.message));
+});
+
+test("corrupt or unexpected files are treated as no key", () => {
+  mkdirSync(join(agentDir, "pi-typesafe"), { recursive: true, mode: 0o700 });
+  for (const content of ["not json", "[]", "{\"apiKey\": 5}", "{}"]) {
+    writeFileSync(credentialsPath(), content, { mode: 0o600 });
+    assert.equal(readStoredApiKey(), undefined);
+  }
+});
+
+test("createTypeSafe uses the stored key and listModels verifies it without spending the request budget", async () => {
+  storeApiKey(validKey);
+  let authorized = false;
+  const client = createTypeSafe({ maxRequests: 1, fetch: async (url, init) => {
+    assert.equal(url, "https://api.typesafe.ai/v1/models");
+    authorized = Object.values(Object.fromEntries(new Headers(init?.headers))).some(value => value.includes(validKey));
+    return Response.json({ models: [{ name: "jev-latest", description: "", release_date: "2026-01-01" }, { name: 7 }] });
+  } });
+  assert.deepEqual(await client.listModels(), ["jev-latest"]);
+  assert.ok(authorized);
+  assert.equal(client.getUsage().requestsStarted, 0);
+  assert.throws(() => { clearStoredApiKey(); createTypeSafe(); }, (error: unknown) => error instanceof TypeSafeIntegrationError && /\/typesafe login/.test(error.message));
+});
+
+test("invalid keys fail verification with a safe message", async () => {
+  const client = createTypeSafe({ apiKey: validKey, fetch: async () => Response.json({ detail: "secret-body" }, { status: 401 }) });
+  await assert.rejects(client.listModels(), (error: unknown) => error instanceof TypeSafeIntegrationError && error.status === 401 && !error.message.includes("secret-body"));
+});
+
+test("keySituation is total and names every kind", { skip: process.platform === "win32" }, () => {
+  assert.deepEqual(keySituation(), { kind: "missing" });
+
+  storeApiKey(validKey);
+  assert.deepEqual(keySituation(), { kind: "stored", key: validKey, path: credentialsPath() });
+
+  chmodSync(credentialsPath(), 0o644);
+  const situation = keySituation();
+  assert.ok(situation.kind === "unusable");
+  assert.equal(situation.path, credentialsPath());
+  assert.match(situation.reason, /chmod 600/);
+  assert.throws(() => resolveApiKey(), (error: unknown) => error instanceof TypeSafeIntegrationError && error.message === situation.reason);
+
+  process.env.TYPESAFE_API_KEY = `  ${validKey}  `;
+  assert.deepEqual(keySituation(), { kind: "environment", key: validKey, keyEnv: "TYPESAFE_API_KEY" });
+
+  // Environment values are trusted as-is; a wrong key fails at the API with its own advice.
+  process.env.TYPESAFE_API_KEY = "short";
+  assert.deepEqual(keySituation(), { kind: "environment", key: "short", keyEnv: "TYPESAFE_API_KEY" });
+});
+
+test("keySituation for another backend reads only that backend's variable, never the TypeSafe store", () => {
+  const savedOpenRouter = process.env.OPENROUTER_API_KEY;
+  try {
+    delete process.env.OPENROUTER_API_KEY;
+    storeApiKey(validKey);
+    process.env.TYPESAFE_API_KEY = validKey;
+    // A stored or TypeSafe-environment key is not an OpenRouter key.
+    assert.deepEqual(keySituation("openrouter"), { kind: "missing" });
+    assert.equal(resolveApiKey("openrouter"), undefined);
+
+    process.env.OPENROUTER_API_KEY = "  sk-or-test-0123456789abcdef  ";
+    const situation = keySituation("openrouter");
+    assert.deepEqual(situation, { kind: "environment", key: "sk-or-test-0123456789abcdef", keyEnv: "OPENROUTER_API_KEY" });
+    assert.equal(keySourceLabel(situation), "OPENROUTER_API_KEY");
+    assert.deepEqual(resolveApiKey("openrouter"), { key: "sk-or-test-0123456789abcdef", source: "environment" });
+    // The OpenRouter variable does not leak into the TypeSafe resolution either.
+    delete process.env.TYPESAFE_API_KEY;
+    assert.equal(keySituation().kind, "stored");
+    assert.throws(() => keySituation("bogus" as never), (error: unknown) => error instanceof TypeSafeIntegrationError && error.code === "configuration" && /Unknown judgment backend/.test(error.message));
+  } finally {
+    if (savedOpenRouter === undefined) delete process.env.OPENROUTER_API_KEY; else process.env.OPENROUTER_API_KEY = savedOpenRouter;
+  }
+});
+
+test("keySourceLabel names each source", () => {
+  assert.equal(keySourceLabel({ kind: "environment", key: "k" }), "TYPESAFE_API_KEY");
+  assert.equal(keySourceLabel({ kind: "environment", key: "k", keyEnv: "OPENROUTER_API_KEY" }), "OPENROUTER_API_KEY");
+  assert.equal(keySourceLabel({ kind: "stored", key: "k", path: "/tmp/auth.json" }), "/typesafe login");
+  assert.equal(keySourceLabel({ kind: "missing" }), "no key");
+  assert.equal(keySourceLabel({ kind: "unusable", path: "/tmp/auth.json", reason: "r" }), "unusable key");
+});

package/port/oracle/tests/extension.test.ts

@@ -0,0 +1,276 @@
+import assert from "node:assert/strict";
+import { existsSync, statSync } from "node:fs";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { after, before, test } from "node:test";
+import { DefaultResourceLoader, SettingsManager } from "@earendil-works/pi-coding-agent";
+import type { Extension, RegisteredCommand, RegisteredTool } from "@earendil-works/pi-coding-agent";
+import { visibleWidth } from "@earendil-works/pi-tui";
+import { parseEvaluationRequest } from "../src/index.js";
+
+let temporary: string;
+let extension: Extension;
+let tool: RegisteredTool;
+let command: RegisteredCommand;
+const savedKey = process.env.TYPESAFE_API_KEY;
+const savedEnabled = process.env.PI_TYPESAFE_ENABLED;
+const savedAgentDir = process.env.PI_CODING_AGENT_DIR;
+const originalFetch = globalThis.fetch;
+const notices: string[] = [];
+let confirmResult = true;
+let confirmations = 0;
+let editorText: string | undefined;
+let networkCalls = 0;
+let modelListCalls = 0;
+let customResult: string | undefined;
+const ui = {
+  notify: (text: string) => { notices.push(text); },
+  confirm: async () => { confirmations++; return confirmResult; },
+  editor: async () => editorText,
+  custom: async () => customResult,
+  input: async () => { throw new Error("plain input must not be used when custom UI exists"); },
+};
+const ctx = { hasUI: true, ui };
+const runCommand = (args: string, context = ctx) => Reflect.apply(command.handler, command, [args, context]);
+const runTool = (signal?: AbortSignal) => Reflect.apply(tool.definition.execute, tool.definition, [
+  "test-call", { state: "synthetic", questions: { yes: { type: "noul", instructions: "Is this synthetic?" } } }, signal, undefined, ctx,
+]);
+
+before(async () => {
+  temporary = await mkdtemp(join(tmpdir(), "pi-typesafe-test-"));
+  delete process.env.PI_TYPESAFE_ENABLED;
+  process.env.PI_CODING_AGENT_DIR = temporary;
+  process.env.TYPESAFE_API_KEY = "offline-test-key";
+  globalThis.fetch = async (input) => {
+    if (String(input).endsWith("/v1/models")) {
+      modelListCalls++;
+      return Response.json({ models: [{ name: "jev-latest", description: "", release_date: "2026-01-01" }] });
+    }
+    networkCalls++;
+    return Response.json({ model: "jev-test", answers: { yes: { type: "noul", noul: 0.9 } }, usage: { input_tokens: 12, output_tokens: 0 } });
+  };
+  const loader = new DefaultResourceLoader({
+    cwd: temporary,
+    agentDir: temporary,
+    settingsManager: SettingsManager.inMemory(),
+    noExtensions: true, noSkills: true, noPromptTemplates: true, noThemes: true, noContextFiles: true,
+    additionalExtensionPaths: [resolve("src/extension.ts")],
+  });
+  await loader.reload();
+  const result = loader.getExtensions();
+  assert.deepEqual(result.errors, [], "native Pi loader must accept the extension");
+  const loaded = result.extensions[0];
+  assert.ok(loaded);
+  extension = loaded;
+  const registeredTool = extension.tools.get("typesafe_evaluate");
+  const registeredCommand = extension.commands.get("typesafe");
+  assert.ok(registeredTool);
+  assert.ok(registeredCommand);
+  tool = registeredTool;
+  command = registeredCommand;
+});
+
+after(async () => {
+  globalThis.fetch = originalFetch;
+  if (savedKey === undefined) delete process.env.TYPESAFE_API_KEY; else process.env.TYPESAFE_API_KEY = savedKey;
+  if (savedEnabled === undefined) delete process.env.PI_TYPESAFE_ENABLED; else process.env.PI_TYPESAFE_ENABLED = savedEnabled;
+  if (savedAgentDir === undefined) delete process.env.PI_CODING_AGENT_DIR; else process.env.PI_CODING_AGENT_DIR = savedAgentDir;
+  if (temporary) await rm(temporary, { recursive: true, force: true });
+});
+
+test("Pi loads a tool, a slash command, and a result renderer without network calls", async () => {
+  assert.equal(networkCalls, 0);
+  assert.equal(tool.definition.name, "typesafe_evaluate");
+  const completions = await command.getArgumentCompletions?.("pla");
+  assert.ok(completions?.some(item => item.value === "playground"));
+  assert.ok(tool.definition.promptGuidelines?.some(text => /one question per item per dimension/.test(text)));
+  // Models that never saw a payload author questions as an array; the guidelines must show one that actually validates.
+  const example = tool.definition.promptGuidelines?.find(text => /"state":/.test(text));
+  assert.ok(example, "one guideline shows a request payload");
+  assert.doesNotMatch(example, /\n/, "the example stays on one line");
+  assert.ok(example.length < 1024, "the example is paid for on every tool listing, so it stays short");
+  const payload = parseEvaluationRequest(JSON.parse(example.slice(example.indexOf("{"))));
+  assert.deepEqual(Object.values(payload.questions).map(question => question.type).sort(), ["choice", "noul", "score"]);
+  assert.ok(/named state field/.test(tool.definition.description));
+});
+
+test("default-disabled tool cannot submit data", async () => {
+  await assert.rejects(runTool(), /disabled/);
+  assert.equal(networkCalls, 0);
+});
+
+test("setup and status never display the API key", async () => {
+  await runCommand("setup");
+  await runCommand("status");
+  assert.ok(notices.some(text => text.includes("TypeSafe key: TYPESAFE_API_KEY")));
+  assert.equal(notices.some(text => text.includes("offline-test-key")), false);
+});
+
+test("status names the model the configured backend actually sends", async () => {
+  await runCommand("status");
+  assert.ok(notices.at(-1)?.includes("Model: jev-latest."));
+});
+
+test("login refuses to shadow an environment key", async () => {
+  await runCommand("login");
+  assert.ok(notices.at(-1)?.includes("takes precedence"));
+  assert.equal(modelListCalls, 0);
+});
+
+test("declining consent keeps the tool disabled", async () => {
+  confirmResult = false;
+  await runCommand("enable");
+  await assert.rejects(runTool(), /disabled/);
+  assert.equal(networkCalls, 0);
+});
+
+test("explicit consent enables the real registered tool and returns structured results", async () => {
+  confirmResult = true;
+  await runCommand("enable");
+  const result = await runTool();
+  assert.equal(result.details.answers.yes.noul, 0.9);
+  assert.equal(networkCalls, 1);
+  assert.ok(confirmations >= 2);
+  const renderer = tool.definition.renderResult;
+  assert.ok(renderer);
+  const component = Reflect.apply(renderer, tool.definition, [result, { expanded: true, isPartial: false }, {}]);
+  for (const width of [40, 80, 120]) {
+    const lines: string[] = component.render(width);
+    assert.ok(lines.every(line => visibleWidth(line) <= width));
+    assert.ok(lines.join("\n").includes("P(yes)"));
+  }
+});
+
+test("disable stops future calls without resetting usage", async () => {
+  await runCommand("disable");
+  await assert.rejects(runTool(), /disabled/);
+  await runCommand("status");
+  assert.ok(notices.at(-1)?.includes("1/20 attempts"));
+});
+
+test("invalid playground JSON and cancellation do not submit data", async () => {
+  editorText = '{"broken":';
+  await runCommand("playground");
+  assert.ok(notices.at(-1)?.includes("Invalid JSON"));
+  editorText = undefined;
+  await runCommand("playground");
+  assert.equal(networkCalls, 1);
+});
+
+test("playground validates questions before requesting consent", async () => {
+  editorText = JSON.stringify({ state: "example", questions: {} });
+  const prior = confirmations;
+  await runCommand("playground");
+  assert.equal(confirmations, prior);
+  assert.equal(networkCalls, 1);
+  assert.ok(notices.at(-1)?.includes("Invalid evaluation request"));
+});
+
+test("test command requires confirmation and does not enable agent calls", async () => {
+  confirmResult = false;
+  await runCommand("test");
+  assert.equal(networkCalls, 1);
+  await assert.rejects(runTool(), /disabled/);
+});
+
+test("login verifies, stores with owner-only permissions, and never echoes the key", async () => {
+  delete process.env.TYPESAFE_API_KEY;
+  const storedPath = join(temporary, "pi-typesafe", "auth.json");
+  customResult = undefined;
+  await runCommand("login");
+  assert.ok(notices.at(-1)?.includes("cancelled"));
+  assert.equal(existsSync(storedPath), false);
+  customResult = "nope";
+  await runCommand("login");
+  assert.ok(notices.at(-1)?.includes("does not look like"));
+  assert.equal(modelListCalls, 0);
+  assert.equal(existsSync(storedPath), false);
+  customResult = "ts_live_key_0123456789abcdef";
+  await runCommand("login");
+  assert.equal(modelListCalls, 1);
+  assert.ok(notices.at(-1)?.includes("Key verified (1 model available)"));
+  assert.equal(notices.some(text => text.includes("ts_live_key")), false);
+  assert.equal(statSync(storedPath).mode & 0o777, 0o600);
+  await runCommand("status");
+  assert.ok(notices.at(-1)?.includes("TypeSafe key: /typesafe login"));
+  await runCommand("setup");
+  assert.ok(notices.at(-1)?.includes("configured via /typesafe login"));
+  // The stored key powers the real tool after consent.
+  confirmResult = true;
+  await runCommand("enable");
+  await runTool();
+  assert.equal(networkCalls, 2);
+  await runCommand("logout");
+  assert.equal(existsSync(storedPath), false);
+  await assert.rejects(runTool(), /disabled/);
+  await runCommand("status");
+  assert.ok(notices.at(-1)?.includes("TypeSafe key: missing"));
+  process.env.TYPESAFE_API_KEY = "offline-test-key";
+});
+
+test("new sessions reset opt-in; headless opt-in is explicit", async () => {
+  const handlers = extension.handlers.get("session_start");
+  assert.ok(handlers?.length);
+  for (const handler of handlers) await Reflect.apply(handler, extension, [{ reason: "new" }, ctx]);
+  await assert.rejects(runTool(), /disabled/);
+  process.env.PI_TYPESAFE_ENABLED = "1";
+  for (const handler of handlers) await Reflect.apply(handler, extension, [{ reason: "startup" }, ctx]);
+  await runTool();
+  assert.equal(networkCalls, 3);
+  await runCommand("status");
+  assert.ok(notices.at(-1)?.includes("1/20 attempts"));
+});
+
+test("an enabled session with no key announces that judgments are skipped", async () => {
+  delete process.env.TYPESAFE_API_KEY;
+  process.env.PI_TYPESAFE_ENABLED = "1";
+  const handlers = extension.handlers.get("session_start") ?? [];
+  const before = notices.length;
+  for (const handler of handlers) await Reflect.apply(handler, extension, [{ reason: "startup" }, ctx]);
+  const said = notices.slice(before).join("\n");
+  assert.ok(said.includes("judgments are skipped"));
+  assert.ok(said.includes("TypeSafe key: missing"));
+  // A key that appears later silences the next startup notice.
+  process.env.TYPESAFE_API_KEY = "offline-test-key";
+  const again = notices.length;
+  for (const handler of handlers) await Reflect.apply(handler, extension, [{ reason: "reload" }, ctx]);
+  assert.equal(notices.slice(again).some(text => text.includes("judgments are skipped")), false);
+});
+
+test("a rejected key is called out once per session and shows up in status", async () => {
+  process.env.PI_TYPESAFE_ENABLED = "1";
+  const handlers = extension.handlers.get("session_start") ?? [];
+  for (const handler of handlers) await Reflect.apply(handler, extension, [{ reason: "startup" }, ctx]);
+  const before = notices.length;
+  const offlineStub = globalThis.fetch;
+  globalThis.fetch = async () => Response.json({ error: { message: "invalid key" } }, { status: 401 });
+  try {
+    await assert.rejects(runTool(), /HTTP 401/);
+    await assert.rejects(runTool(), /HTTP 401/);
+  } finally {
+    globalThis.fetch = offlineStub;
+  }
+  // Two failed calls, one callout: the reason is loud once, not once per call.
+  assert.equal(notices.slice(before).filter(text => text.includes("not authenticated")).length, 1);
+  await runCommand("status");
+  assert.ok(notices.at(-1)?.includes("was rejected"));
+  assert.ok(notices.at(-1)?.includes("Today "));
+  assert.ok(notices.at(-1)?.includes("failed"));
+});
+
+test("the registered tool admits the same near-miss aliases as the library", async () => {
+  process.env.PI_TYPESAFE_ENABLED = "1";
+  const handlers = extension.handlers.get("session_start");
+  for (const handler of handlers ?? []) await Reflect.apply(handler, extension, [{ reason: "startup" }, ctx]);
+  const before = networkCalls;
+  const result = await Reflect.apply(tool.definition.execute, tool.definition, [
+    "test-call",
+    { state: "synthetic", questions: { yes: { type: "noul", instructions: "Is this synthetic?", criteria: "Is this synthetic data?" } } },
+    undefined,
+    undefined,
+    ctx,
+  ]);
+  assert.equal(networkCalls, before + 1);
+  assert.equal(result.details.answers.yes.noul, 0.9);
+});

package/port/oracle/tests/key-prompt.test.ts

@@ -0,0 +1,47 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import type { ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
+import { promptForApiKey } from "../src/key-prompt.js";
+
+const theme = { fg: (_color: string, text: string) => text, bold: (text: string) => text };
+
+function fakeContext(drive: (component: { render(width: number): string[]; handleInput(data: string): void; focused: boolean }) => void): ExtensionCommandContext {
+  const ui = {
+    custom: async <T>(factory: (tui: unknown, theme: unknown, keybindings: unknown, done: (value: T) => void) => unknown) =>
+      new Promise<T>(resolve => {
+        const component = factory({}, theme, {}, resolve) as { render(width: number): string[]; handleInput(data: string): void; focused: boolean };
+        component.focused = true;
+        drive(component);
+      }),
+    input: async () => { throw new Error("must not fall back"); },
+  };
+  return { hasUI: true, ui } as unknown as ExtensionCommandContext;
+}
+
+test("typed key is masked on screen and returned on Enter", async () => {
+  let frame: string[] = [];
+  const key = await promptForApiKey(fakeContext(component => {
+    for (const character of "ts_secret_key_0123456789") component.handleInput(character);
+    frame = component.render(60);
+    component.handleInput("\r");
+  }));
+  assert.equal(key, "ts_secret_key_0123456789");
+  const screen = frame.join("\n");
+  assert.equal(screen.includes("ts_secret"), false);
+  assert.ok(screen.includes("•".repeat(24)));
+  assert.ok(screen.includes("TypeSafe API key"));
+  assert.ok(frame.every(line => line.replace(/\x1b\[[0-9;]*m/g, "").length <= 60 + 8));
+});
+
+test("Escape cancels without a value", async () => {
+  const key = await promptForApiKey(fakeContext(component => {
+    component.handleInput("a");
+    component.handleInput("\x1b");
+  }));
+  assert.equal(key, undefined);
+});
+
+test("falls back to Pi's plain input when custom UI is unavailable", async () => {
+  const ctx = { hasUI: true, ui: { input: async () => "  ts_plain_key_0123456789  " } } as unknown as ExtensionCommandContext;
+  assert.equal(await promptForApiKey(ctx), "  ts_plain_key_0123456789  ");
+});

package/port/oracle/tests/login.test.ts

@@ -0,0 +1,101 @@
+import assert from "node:assert/strict";
+import { existsSync, statSync } from "node:fs";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { after, before, beforeEach, test } from "node:test";
+import type { ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
+import { TypeSafeIntegrationError } from "../src/errors.js";
+import { ensureApiKey, loginWithPrompt } from "../src/login.js";
+
+let temporary: string;
+const savedKey = process.env.TYPESAFE_API_KEY;
+const savedAgentDir = process.env.PI_CODING_AGENT_DIR;
+const originalFetch = globalThis.fetch;
+let customResult: string | undefined;
+let modelListCalls = 0;
+const storedPath = () => join(temporary, "pi-typesafe", "auth.json");
+const ctx = (hasUI = true) => ({ hasUI, ui: { custom: async () => customResult, input: async () => { throw new Error("plain input must not be used"); } } }) as unknown as ExtensionCommandContext;
+
+before(async () => {
+  temporary = await mkdtemp(join(tmpdir(), "pi-typesafe-login-"));
+  process.env.PI_CODING_AGENT_DIR = temporary;
+  globalThis.fetch = async (input) => {
+    if (String(input).endsWith("/v1/models")) {
+      modelListCalls++;
+      return Response.json({ models: [{ name: "jev-latest", description: "", release_date: "2026-01-01" }, { name: "jev-preview", description: "", release_date: "2026-01-01" }] });
+    }
+    throw new Error("unexpected request");
+  };
+});
+beforeEach(async () => {
+  delete process.env.TYPESAFE_API_KEY;
+  modelListCalls = 0;
+  customResult = undefined;
+  await rm(storedPath(), { force: true });
+});
+after(async () => {
+  globalThis.fetch = originalFetch;
+  if (savedKey === undefined) delete process.env.TYPESAFE_API_KEY; else process.env.TYPESAFE_API_KEY = savedKey;
+  if (savedAgentDir === undefined) delete process.env.PI_CODING_AGENT_DIR; else process.env.PI_CODING_AGENT_DIR = savedAgentDir;
+  await rm(temporary, { recursive: true, force: true });
+});
+
+test("loginWithPrompt cancels cleanly, rejects bad keys, and stores a verified key with owner-only permissions", async () => {
+  assert.equal(await loginWithPrompt(ctx()), undefined);
+  assert.equal(existsSync(storedPath()), false);
+
+  customResult = "nope";
+  await assert.rejects(loginWithPrompt(ctx()), (error: unknown) => error instanceof TypeSafeIntegrationError && error.code === "validation" && !error.message.includes("nope"));
+  assert.equal(modelListCalls, 0);
+
+  customResult = "ts_live_key_0123456789abcdef";
+  const result = await loginWithPrompt(ctx());
+  assert.deepEqual(result, { path: storedPath(), models: 2 });
+  assert.equal(modelListCalls, 1);
+  assert.equal(statSync(storedPath()).mode & 0o777, 0o600);
+});
+
+test("loginWithPrompt refuses when the environment key would shadow the store, or without a UI", async () => {
+  process.env.TYPESAFE_API_KEY = "env-key-0123456789";
+  await assert.rejects(loginWithPrompt(ctx()), (error: unknown) => error instanceof TypeSafeIntegrationError && error.code === "configuration" && /TYPESAFE_API_KEY/.test(error.message));
+  delete process.env.TYPESAFE_API_KEY;
+  await assert.rejects(loginWithPrompt(ctx(false)), (error: unknown) => error instanceof TypeSafeIntegrationError && /interactive/.test(error.message));
+  assert.equal(existsSync(storedPath()), false);
+});
+
+test("ensureApiKey reports an existing key without prompting, otherwise logs in", async () => {
+  process.env.TYPESAFE_API_KEY = "env-key-0123456789";
+  customResult = "must-not-be-used-0123456789";
+  assert.deepEqual(await ensureApiKey(ctx()), { source: "environment" });
+  assert.equal(existsSync(storedPath()), false);
+
+  delete process.env.TYPESAFE_API_KEY;
+  customResult = undefined;
+  assert.equal(await ensureApiKey(ctx()), undefined, "cancelled prompt");
+
+  customResult = "ts_live_key_0123456789abcdef";
+  const first = await ensureApiKey(ctx());
+  assert.deepEqual(first, { source: "stored", login: { path: storedPath(), models: 2 } });
+  customResult = "another-key-that-must-not-replace-it";
+  assert.deepEqual(await ensureApiKey(ctx()), { source: "stored" }, "second call reuses the stored key");
+  assert.equal(modelListCalls, 1);
+});
+
+test("ensureApiKey for another backend uses its environment variable and never opens the TypeSafe login", async () => {
+  const savedOpenRouter = process.env.OPENROUTER_API_KEY;
+  try {
+    delete process.env.OPENROUTER_API_KEY;
+    // A stored TypeSafe key does not satisfy OpenRouter, and the prompt must not run: it would verify against api.typesafe.ai.
+    customResult = "ts_live_key_0123456789abcdef";
+    assert.deepEqual(await ensureApiKey(ctx()), { source: "stored", login: { path: storedPath(), models: 2 } });
+    await assert.rejects(ensureApiKey(ctx(), { backend: "openrouter" }), (error: unknown) => error instanceof TypeSafeIntegrationError && error.code === "configuration" && /OPENROUTER_API_KEY/.test(error.message));
+    assert.equal(modelListCalls, 1, "no second verification");
+
+    process.env.OPENROUTER_API_KEY = "sk-or-0123456789abcdef";
+    assert.deepEqual(await ensureApiKey(ctx(), { backend: "openrouter" }), { source: "environment" });
+    assert.equal(modelListCalls, 1);
+  } finally {
+    if (savedOpenRouter === undefined) delete process.env.OPENROUTER_API_KEY; else process.env.OPENROUTER_API_KEY = savedOpenRouter;
+  }
+});

package/port/oracle/tests/schema.test.ts

@@ -0,0 +1,85 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import {
+  DEFAULT_MAX_INPUT_BYTES, evaluationSchema, normalizeEvaluationRequest, parseEvaluationRequest, prepareEvaluationRequest, TypeSafeIntegrationError,
+} from "../src/index.js";
+
+const hasCode = (code: string) => (error: unknown) => error instanceof TypeSafeIntegrationError && error.code === code;
+
+const nearMiss = () => ({
+  state: "synthetic",
+  questions: {
+    team: { type: "choice", instructions: "Which team?", options: { billing: "Charges and payments", other: "None of these" } },
+    refund: { type: "noul", instructions: "Is a refund requested?", criteria: "The sender asks for money back" },
+    severity: { type: "score", instructions: "How severe?", levels: ["Cosmetic", "Blocking"] },
+    picks: { type: "choice", instructions: "Pick one", choices: ["a", "b"] },
+  },
+});
+
+const criteriaOf = (request: { questions: unknown }, id: string): unknown =>
+  (request.questions as Record<string, { criteria?: unknown }>)[id]?.criteria;
+
+test("admission accepts the near-miss aliases the agent tool has always accepted", () => {
+  const prepared = prepareEvaluationRequest(nearMiss());
+  assert.deepEqual(criteriaOf(prepared, "team"), { billing: "Charges and payments", other: "None of these" });
+  assert.deepEqual(criteriaOf(prepared, "refund"), { true: "The sender asks for money back" });
+  assert.deepEqual(criteriaOf(prepared, "severity"), ["Cosmetic", "Blocking"]);
+  assert.deepEqual(criteriaOf(prepared, "picks"), { a: null, b: null });
+});
+
+test("admission normalizes before it validates", () => {
+  const onlyValidAfterNormalizing = { state: "s", questions: { q: { type: "choice", criteria: ["a", "b"] } } };
+  assert.throws(() => parseEvaluationRequest(onlyValidAfterNormalizing), hasCode("validation"));
+  assert.doesNotThrow(() => prepareEvaluationRequest(onlyValidAfterNormalizing));
+});
+
+test("normalization is idempotent", () => {
+  const once = normalizeEvaluationRequest(nearMiss());
+  assert.deepEqual(normalizeEvaluationRequest(once), once);
+  assert.doesNotThrow(() => prepareEvaluationRequest(once));
+});
+
+test("admission enforces the byte budget on the serialized request", () => {
+  const request = { state: "🙂".repeat(50), questions: { yes: { type: "noul", instructions: "Is this synthetic?" } } };
+  const bytes = Buffer.byteLength(JSON.stringify(request), "utf8");
+  assert.ok(bytes < DEFAULT_MAX_INPUT_BYTES);
+  assert.throws(() => prepareEvaluationRequest(request, { maxInputBytes: bytes - 1 }), hasCode("validation"));
+  assert.doesNotThrow(() => prepareEvaluationRequest(request, { maxInputBytes: bytes }));
+});
+
+test("the default byte budget is 64 KiB", () => {
+  const oversized = { state: "x".repeat(DEFAULT_MAX_INPUT_BYTES), questions: { yes: { type: "noul", instructions: "?" } } };
+  assert.equal(DEFAULT_MAX_INPUT_BYTES, 65_536);
+  assert.throws(() => prepareEvaluationRequest(oversized), hasCode("validation"));
+  assert.doesNotThrow(() => prepareEvaluationRequest(oversized, { maxInputBytes: DEFAULT_MAX_INPUT_BYTES * 2 }));
+});
+
+test("admission still rejects non-JSON state", () => {
+  const cycle: Record<string, unknown> = {};
+  cycle.self = cycle;
+  assert.throws(() => prepareEvaluationRequest({ state: cycle, questions: { yes: { type: "noul", instructions: "?" } } }), hasCode("validation"));
+  assert.throws(() => prepareEvaluationRequest({ state: { n: NaN }, questions: { yes: { type: "noul", instructions: "?" } } }), hasCode("validation"));
+});
+
+test("every field the agent authors carries a description, so a bare union is not its only guidance", () => {
+  // Walk the serialized schema rather than pin TypeBox's layout: collect every `properties` entry by name.
+  const authored = new Map<string, number>();
+  const undescribed: string[] = [];
+  const walk = (node: unknown): void => {
+    if (node === null || typeof node !== "object") return;
+    if (Array.isArray(node)) { node.forEach(walk); return; }
+    const record = node as Record<string, unknown>;
+    if (record.properties && typeof record.properties === "object") {
+      for (const [name, field] of Object.entries(record.properties as Record<string, Record<string, unknown>>)) {
+        authored.set(name, (authored.get(name) ?? 0) + 1);
+        if (typeof field.description !== "string" || field.description.length === 0) undescribed.push(name);
+      }
+    }
+    Object.values(record).forEach(walk);
+  };
+  walk(JSON.parse(JSON.stringify(evaluationSchema)));
+  assert.equal(authored.get("type"), 3, "the schema must still offer noul, choice, and score");
+  for (const name of ["state", "questions", "model", "instructions", "criteria"]) assert.ok(authored.has(name), `${name} is still authored`);
+  // `true` and `false` inside noul criteria are the only fields whose parent already explains them.
+  assert.deepEqual(undescribed.filter(name => name !== "true" && name !== "false"), [], "every authored field must say what it means");
+});