@evoclock/pi-agentic-driver 0.9.1 → 0.9.3

package/janus/tests/schema.test.ts

@@ -0,0 +1,356 @@
+// SPDX-FileCopyrightText: 2026 Julen Gamboa <j.a.r.gamboa@gmail.com>
+// SPDX-License-Identifier: AGPL-3.0-only
+
+// Unit tests: question schema contract v1 enforcement (§6), atomic
+// fail-closed semantics, and the boolean threshold contract.
+
+import test from "node:test";
+import assert from "node:assert/strict";
+import { enforceRequest } from "../janus/schema.js";
+import { JanusError } from "../janus/errors.js";
+import { enforceAnswers } from "../janus/verdict.js";
+
+function expectValidationError(fn: () => unknown, fragment?: string): void {
+  try {
+    fn();
+    assert.fail("expected validation_error");
+  } catch (error) {
+    assert.ok(error instanceof JanusError);
+    assert.equal(error.code, "validation_error");
+    if (fragment) assert.match(error.message, new RegExp(fragment));
+  }
+}
+
+test("boolean: default threshold is 0.9 with >= comparison", () => {
+  const request = enforceRequest({
+    state: "hello",
+    questions: [{ type: "boolean", name: "refund", question: "Was a refund issued?" }],
+  });
+  assert.equal(request.questions.length, 1);
+  const q = request.questions[0]!;
+  assert.equal(q.kind, "boolean");
+  assert.equal((q as { threshold: number }).threshold, 0.9);
+});
+
+test("boolean: custom threshold accepted within [0,1]", () => {
+  const request = enforceRequest({
+    state: "hello",
+    questions: [{ type: "boolean", name: "q", question: "q", threshold: 0.5 }],
+  });
+  assert.equal((request.questions[0] as { threshold: number }).threshold, 0.5);
+});
+
+test("boolean: threshold out of range is validation_error", () => {
+  expectValidationError(() =>
+    enforceRequest({
+      state: "s",
+      questions: [{ type: "boolean", name: "q", question: "q", threshold: 1.5 }],
+    }),
+  );
+  expectValidationError(() =>
+    enforceRequest({
+      state: "s",
+      questions: [{ type: "boolean", name: "q", question: "q", threshold: "high" }],
+    }),
+  );
+});
+
+test("choice: choices array is REQUIRED (§6)", () => {
+  expectValidationError(() =>
+    enforceRequest({ state: "s", questions: [{ type: "choice", name: "route", question: "route" }] }),
+  );
+  expectValidationError(() =>
+    enforceRequest({ state: "s", questions: [{ type: "choice", name: "route", question: "route", choices: [] }] }),
+  );
+  expectValidationError(() =>
+    enforceRequest({
+      state: "s",
+      questions: [{ type: "choice", name: "route", question: "route", choices: ["a", "a"] }],
+    }),
+  );
+  expectValidationError(() =>
+    enforceRequest({
+      state: "s",
+      questions: [{ type: "choice", name: "route", question: "route", choices: ["a", 3] }],
+    }),
+  );
+});
+
+test("choice: valid choices accepted", () => {
+  const request = enforceRequest({
+    state: "s",
+    questions: [{ type: "choice", name: "route", question: "route", choices: ["billing", "tech", "other"] }],
+  });
+  assert.deepEqual((request.questions[0] as { choices: string[] }).choices, [
+    "billing",
+    "tech",
+    "other",
+  ]);
+});
+
+test("score: fixed 0..5 vocabulary; ordinal vocabularies must be choice (§6)", () => {
+  const request = enforceRequest({
+    state: "s",
+    questions: [{ type: "score", name: "rate", question: "rate" }],
+  });
+  assert.equal((request.questions[0] as { levels: number }).levels, 6);
+  // Closed validation: per-question overrides like choices/levels are rejected.
+  expectValidationError(() =>
+    enforceRequest({
+      state: "s",
+      questions: [{ type: "score", name: "rate", question: "rate", choices: ["poor", "good"] }],
+    }),
+  );
+  expectValidationError(() =>
+    enforceRequest({
+      state: "s",
+      questions: [{ type: "score", name: "rate", question: "rate", levels: 10 }],
+    }),
+  );
+});
+
+test("unknown question type is validation_error", () => {
+  expectValidationError(() =>
+    enforceRequest({ state: "s", questions: [{ type: "ranking", name: "q", question: "q" }] }),
+  );
+});
+
+test("atomic fail-closed: one malformed question fails the whole request", () => {
+  expectValidationError(() =>
+    enforceRequest({
+      state: "s",
+      questions: [
+        { type: "boolean", name: "ok", question: "ok?" },
+        { type: "choice", name: "route", question: "route" }, // missing choices
+      ],
+    }),
+  );
+});
+
+test("missing state / missing questions / bad body shape are validation_error", () => {
+  expectValidationError(() => enforceRequest({ questions: [{ type: "boolean", name: "q", question: "q" }] }));
+  expectValidationError(() => enforceRequest({ state: "s", questions: [] }));
+  expectValidationError(() => enforceRequest({ state: "s" }));
+  expectValidationError(() => enforceRequest(null));
+  expectValidationError(() => enforceRequest("state only"));
+});
+
+test("duplicate question name is validation_error", () => {
+  expectValidationError(() =>
+    enforceRequest({
+      state: "s",
+      questions: [
+        { type: "boolean", name: "same", question: "same" },
+        { type: "boolean", name: "same", question: "same" },
+      ],
+    }),
+  );
+});
+
+test("state accepts string, object, and array", () => {
+  for (const state of ["text", { a: 1 }, [{ role: "user" }]]) {
+    const request = enforceRequest({
+      state,
+      questions: [{ type: "boolean", name: "q", question: "q" }],
+    });
+    assert.deepEqual(request.state, state);
+  }
+  expectValidationError(() =>
+    enforceRequest({ state: 42, questions: [{ type: "boolean", name: "q", question: "q" }] }),
+  );
+});
+
+test("verdict enforcement: model returned false is a normal 200 verdict with passed=false", () => {
+  const request = enforceRequest({
+    state: "s",
+    questions: [{ type: "boolean", name: "ok", question: "ok?" }],
+  });
+  const verdict = enforceAnswers(
+    request.questions,
+    { ok: { type: "boolean", probability: 0.2 } },
+    { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
+  );
+  assert.equal(verdict.answers.ok!.type, "boolean");
+  assert.equal((verdict.answers.ok as { passed: boolean }).passed, false);
+});
+
+test("verdict enforcement: probability >= threshold passes", () => {
+  const request = enforceRequest({
+    state: "s",
+    questions: [{ type: "boolean", name: "ok", question: "ok?", threshold: 0.9 }],
+  });
+  const verdict = enforceAnswers(
+    request.questions,
+    { ok: { type: "boolean", probability: 0.9 } },
+    { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
+  );
+  assert.equal((verdict.answers.ok as { passed: boolean }).passed, true);
+});
+
+test("verdict enforcement: choice answer must be a member of the choices array", () => {
+  const request = enforceRequest({
+    state: "s",
+    questions: [{ type: "choice", name: "route", question: "route", choices: ["a", "b"] }],
+  });
+  assert.throws(
+    () =>
+      enforceAnswers(
+        request.questions,
+        { route: { type: "choice", choice: "c" } },
+        { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
+      ),
+    (error: unknown) => error instanceof JanusError && error.code === "validation_error",
+  );
+});
+
+test("verdict enforcement: missing answer fails the whole request with validation_error (atomic, review fix #8)", () => {
+  const request = enforceRequest({
+    state: "s",
+    questions: [
+      { type: "boolean", name: "a", question: "a" },
+      { type: "boolean", name: "b", question: "b" },
+    ],
+  });
+  assert.throws(
+    () =>
+      enforceAnswers(
+        request.questions,
+        { a: { type: "boolean", probability: 0.99 } },
+        { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
+      ),
+    (error: unknown) => error instanceof JanusError && error.code === "validation_error",
+  );
+});
+
+test("verdict enforcement: score is rounded to one decimal and range-checked", () => {
+  const request = enforceRequest({
+    state: "s",
+    questions: [{ type: "score", name: "rate", question: "rate" }],
+  });
+  const verdict = enforceAnswers(
+    request.questions,
+    { rate: { type: "score", score: 2.96 } },
+    { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
+  );
+  assert.equal((verdict.answers.rate as { score: number }).score, 3.0);
+  assert.throws(
+    () =>
+      enforceAnswers(
+        request.questions,
+        { rate: { type: "score", score: 5.5 } },
+        { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
+      ),
+    (error: unknown) => error instanceof JanusError && error.code === "validation_error",
+  );
+});
+
+test("question name is REQUIRED; missing or blank is validation_error (review fix #1)", () => {
+  expectValidationError(() =>
+    enforceRequest({ state: "s", questions: [{ type: "boolean", question: "ok?" }] }),
+  );
+  expectValidationError(() =>
+    enforceRequest({ state: "s", questions: [{ type: "boolean", name: "  ", question: "ok?" }] }),
+  );
+});
+
+test("question text is REQUIRED; missing or blank is validation_error, never synthetic question[i] (review fix #1)", () => {
+  expectValidationError(() =>
+    enforceRequest({ state: "s", questions: [{ type: "boolean", name: "q" }] }),
+  );
+  expectValidationError(() =>
+    enforceRequest({ state: "s", questions: [{ type: "boolean", name: "q", question: "" }] }),
+  );
+  expectValidationError(() =>
+    enforceRequest({ state: "s", questions: [{ type: "boolean", name: "q", question: "   " }] }),
+  );
+});
+
+test("enforced question carries both name and question text verbatim", () => {
+  const request = enforceRequest({
+    state: "s",
+    questions: [{ type: "boolean", name: "refund-gate", question: "Was a refund issued?" }],
+  });
+  const q = request.questions[0]! as { name: string; question: string };
+  assert.equal(q.name, "refund-gate");
+  assert.equal(q.question, "Was a refund issued?");
+});
+
+test("verdict enforcement: wrong answer type fails with validation_error (review fix #8)", () => {
+  const request = enforceRequest({
+    state: "s",
+    questions: [{ type: "boolean", name: "ok", question: "ok?" }],
+  });
+  assert.throws(
+    () =>
+      enforceAnswers(
+        request.questions,
+        { ok: { type: "score", score: 3 } },
+        { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
+      ),
+    (error: unknown) => error instanceof JanusError && error.code === "validation_error",
+  );
+});
+
+test("verdict enforcement: invalid probability and non-finite score fail with validation_error (review fix #8)", () => {
+  const booleanRequest = enforceRequest({
+    state: "s",
+    questions: [{ type: "boolean", name: "ok", question: "ok?" }],
+  });
+  assert.throws(
+    () =>
+      enforceAnswers(
+        booleanRequest.questions,
+        { ok: { type: "boolean", probability: 1.5 } },
+        { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
+      ),
+    (error: unknown) => error instanceof JanusError && error.code === "validation_error",
+  );
+  assert.throws(
+    () =>
+      enforceAnswers(
+        booleanRequest.questions,
+        { ok: { type: "boolean", probability: "high" } },
+        { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
+      ),
+    (error: unknown) => error instanceof JanusError && error.code === "validation_error",
+  );
+  const scoreRequest = enforceRequest({
+    state: "s",
+    questions: [{ type: "score", name: "rate", question: "rate" }],
+  });
+  assert.throws(
+    () =>
+      enforceAnswers(
+        scoreRequest.questions,
+        { rate: { type: "score", score: Number.NaN } },
+        { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
+      ),
+    (error: unknown) => error instanceof JanusError && error.code === "validation_error",
+  );
+});
+
+test("verdict enforcement: malformed probabilities map fails with validation_error (review fix #8)", () => {
+  const request = enforceRequest({
+    state: "s",
+    questions: [{ type: "choice", name: "route", question: "route", choices: ["a", "b"] }],
+  });
+  assert.throws(
+    () =>
+      enforceAnswers(
+        request.questions,
+        { route: { type: "choice", choice: "a", probabilities: { a: "high" } } },
+        { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
+      ),
+    (error: unknown) => error instanceof JanusError && error.code === "validation_error",
+  );
+  // Probabilities keys outside the vocabulary are rejected.
+  assert.throws(
+    () =>
+      enforceAnswers(
+        request.questions,
+        { route: { type: "choice", choice: "a", probabilities: { zzz: 0.5 } } },
+        { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
+      ),
+    (error: unknown) => error instanceof JanusError && error.code === "validation_error",
+  );
+});

package/janus/tests/server.test.ts

@@ -0,0 +1,288 @@
+// SPDX-FileCopyrightText: 2026 Julen Gamboa <j.a.r.gamboa@gmail.com>
+// SPDX-License-Identifier: AGPL-3.0-only
+
+// HTTP surface tests: loopback bind, endpoint contract, JSON error shape,
+// and the GET routes. The SDK is mocked; no live gateway calls.
+
+import test from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { createJanusService, type JanusService } from "../janus/server.js";
+import { Evaluator } from "../janus/evaluate.js";
+import { SingleFlight } from "../janus/single_flight.js";
+import { DailyBudget } from "../janus/budget_breaker.js";
+import { AuditLog } from "../janus/audit.js";
+import { testConfig, mockChain } from "./helpers.js";
+
+async function listenOnRandomPort(): Promise<{ service: JanusService; base: string; cleanup: () => Promise<void> }> {
+  const dir = mkdtempSync(join(tmpdir(), "janus-server-test-"));
+  const config = testConfig({
+    port: 0,
+    audit: { path: join(dir, "audit"), maxBytes: 1024 * 1024, retentionDays: 90 },
+  });
+  const audit = new AuditLog(config.audit.path, config.audit.maxBytes, config.audit.retentionDays);
+  audit.start();
+  const evaluator = new Evaluator({
+    config,
+    singleFlight: new SingleFlight(4),
+    budget: new DailyBudget(0, 0, 0.5),
+    audit,
+    chain: mockChain({ answers: { q: { type: "boolean", probability: 0.95 } } }) as never,
+  });
+  const service = createJanusService(config, evaluator, async () => ({ ready: true }));
+  await new Promise<void>((resolvePromise) => service.server.on("listening", resolvePromise));
+  const address = service.server.address();
+  assert.ok(address && typeof address === "object");
+  const base = `http://127.0.0.1:${address.port}`;
+  return {
+    service,
+    base,
+    cleanup: async () => {
+      await service.stop();
+      audit.stop();
+      rmSync(dir, { recursive: true, force: true });
+    },
+  };
+}
+
+test("binds loopback only", async () => {
+  const { service, cleanup } = await listenOnRandomPort();
+  try {
+    const address = service.server.address();
+    assert.ok(address && typeof address === "object");
+    assert.equal(address.address, "127.0.0.1");
+  } finally {
+    await cleanup();
+  }
+});
+
+test("GET /healthz returns ok", async () => {
+  const { base, cleanup } = await listenOnRandomPort();
+  try {
+    const res = await fetch(`${base}/healthz`);
+    assert.equal(res.status, 200);
+    const body = (await res.json()) as { status: string };
+    assert.equal(body.status, "ok");
+  } finally {
+    await cleanup();
+  }
+});
+
+test("GET /version returns janus, AI SDK, schema versions, and allowlist", async () => {
+  const { base, cleanup } = await listenOnRandomPort();
+  try {
+    const res = await fetch(`${base}/version`);
+    assert.equal(res.status, 200);
+    const body = (await res.json()) as Record<string, unknown>;
+    assert.equal(body.janus, "0.1.0");
+    assert.equal(body.typesafeSdk, "0.6.0");
+    assert.equal(body.questionSchema, "janus.question-schema.v2");
+    assert.deepEqual(body.providers, ["typesafe-direct"]);
+  } finally {
+    await cleanup();
+  }
+});
+
+test("POST /evaluate happy path returns verdict with request_id", async () => {
+  const { base, cleanup } = await listenOnRandomPort();
+  try {
+    const res = await fetch(`${base}/evaluate`, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ state: "s", questions: [{ type: "boolean", name: "q", question: "q" }] }),
+    });
+    assert.equal(res.status, 200);
+    const body = (await res.json()) as { request_id: string; verdict: Record<string, unknown> };
+    assert.ok(body.request_id.length > 0);
+    assert.deepEqual(body.verdict, { q: { type: "boolean", probability: 0.95, passed: true } });
+  } finally {
+    await cleanup();
+  }
+});
+
+test("POST /evaluate invalid JSON is validation_error", async () => {
+  const { base, cleanup } = await listenOnRandomPort();
+  try {
+    const res = await fetch(`${base}/evaluate`, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: "{not json",
+    });
+    assert.equal(res.status, 400);
+    const body = (await res.json()) as { error: string };
+    assert.equal(body.error, "validation_error");
+  } finally {
+    await cleanup();
+  }
+});
+
+test("unknown route is 404 with the taxonomy shape", async () => {
+  const { base, cleanup } = await listenOnRandomPort();
+  try {
+    const res = await fetch(`${base}/nope`);
+    assert.equal(res.status, 404);
+    const body = (await res.json()) as { error: string };
+    assert.equal(body.error, "validation_error");
+  } finally {
+    await cleanup();
+  }
+});
+
+test("POST /evaluate without content-type is validation_error (review fix #10)", async () => {
+  const { base, cleanup } = await listenOnRandomPort();
+  try {
+    const res = await fetch(`${base}/evaluate`, {
+      method: "POST",
+      body: JSON.stringify({ state: "s", questions: [] }),
+    });
+    assert.equal(res.status, 400);
+    const body = (await res.json()) as { error: string; message: string };
+    assert.equal(body.error, "validation_error");
+    assert.match(body.message, /content-type/);
+  } finally {
+    await cleanup();
+  }
+});
+
+test("POST /evaluate with malformed content-type is validation_error (review fix #10)", async () => {
+  const { base, cleanup } = await listenOnRandomPort();
+  try {
+    const res = await fetch(`${base}/evaluate`, {
+      method: "POST",
+      headers: { "content-type": "not a valid type;;;" },
+      body: "{}",
+    });
+    assert.equal(res.status, 400);
+    const body = (await res.json()) as { error: string };
+    assert.equal(body.error, "validation_error");
+  } finally {
+    await cleanup();
+  }
+});
+
+test("POST /evaluate with non-JSON content-type is validation_error (review fix #10)", async () => {
+  const { base, cleanup } = await listenOnRandomPort();
+  try {
+    const res = await fetch(`${base}/evaluate`, {
+      method: "POST",
+      headers: { "content-type": "text/plain" },
+      body: "{}",
+    });
+    assert.equal(res.status, 400);
+    const body = (await res.json()) as { error: string };
+    assert.equal(body.error, "validation_error");
+  } finally {
+    await cleanup();
+  }
+});
+
+test("POST /evaluate accepts application/json with parameters (charset) (review fix #10)", async () => {
+  const { base, cleanup } = await listenOnRandomPort();
+  try {
+    const res = await fetch(`${base}/evaluate`, {
+      method: "POST",
+      headers: { "content-type": "application/json; charset=utf-8" },
+      body: JSON.stringify({ state: "s", questions: [{ type: "boolean", name: "q", question: "q" }] }),
+    });
+    assert.equal(res.status, 200);
+  } finally {
+    await cleanup();
+  }
+});
+
+test("oversized body returns structured JSON 413 (review fix #10)", async () => {
+  const { base, cleanup } = await listenOnRandomPort();
+  try {
+    // 9 MB body against the 8 MB cap, sent with a JSON content-type.
+    const big = "x".repeat(9 * 1024 * 1024);
+    const res = await fetch(`${base}/evaluate`, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ state: big, questions: [{ type: "boolean", name: "q", question: "q" }] }),
+    });
+    assert.ok([400, 413].includes(res.status));
+    const body = (await res.json()) as { error: string };
+    assert.equal(body.error, "validation_error");
+  } finally {
+    await cleanup();
+  }
+});
+
+test("GET /readyz reports readiness from the provider chain", async () => {
+  const dir = mkdtempSync(join(tmpdir(), "janus-server-test-"));
+  try {
+    const config = testConfig({
+      port: 0,
+      audit: { path: join(dir, "audit"), maxBytes: 1024 * 1024, retentionDays: 90 },
+    });
+    const audit = new AuditLog(config.audit.path, config.audit.maxBytes, config.audit.retentionDays);
+    audit.start();
+    const evaluator = new Evaluator({
+      config,
+      singleFlight: new SingleFlight(4),
+      budget: new DailyBudget(0, 0, 0.5),
+      audit,
+      chain: mockChain({ answers: { q: { type: "boolean", probability: 0.95 } } }) as never,
+    });
+    const service = createJanusService(config, evaluator, async () => ({
+      ready: false,
+      reason: "no provider credential resolvable (checked: typesafe-direct)",
+    }));
+    await new Promise<void>((resolvePromise) => service.server.on("listening", resolvePromise));
+    const address = service.server.address();
+    assert.ok(address && typeof address === "object");
+    const res = await fetch(`http://127.0.0.1:${address.port}/readyz`);
+    assert.equal(res.status, 503);
+    const body = (await res.json()) as { reason: string };
+    assert.match(body.reason, /no provider credential resolvable/);
+    assert.ok(!/sk-|test-key/i.test(body.reason), "no key fragments in the public reason");
+    await service.stop();
+    audit.stop();
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+test("internal evaluator crash returns a fixed public degraded message (review fix #5)", async () => {
+  const dir = mkdtempSync(join(tmpdir(), "janus-server-test-"));
+  try {
+    const config = testConfig({
+      port: 0,
+      audit: { path: join(dir, "audit"), maxBytes: 1024 * 1024, retentionDays: 90 },
+    });
+    const audit = new AuditLog(config.audit.path, config.audit.maxBytes, config.audit.retentionDays);
+    audit.start();
+    const evaluator = new Evaluator({
+      config,
+      singleFlight: new SingleFlight(4),
+      budget: new DailyBudget(0, 0, 0.5),
+      audit,
+      chain: {
+        evaluate: async () => {
+          throw new Error("SECRET sk-live-abc123 internal stack trace detail");
+        },
+        readiness: async () => ({ ready: true, providers: [] }),
+      } as never,
+    });
+    const service = createJanusService(config, evaluator, async () => ({ ready: true }));
+    await new Promise<void>((resolvePromise) => service.server.on("listening", resolvePromise));
+    const address = service.server.address();
+    assert.ok(address && typeof address === "object");
+    const res = await fetch(`http://127.0.0.1:${address.port}/evaluate`, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ state: "s", questions: [{ type: "boolean", name: "q", question: "q" }] }),
+    });
+    assert.equal(res.status, 503);
+    const body = (await res.json()) as { error: string; message: string };
+    assert.equal(body.error, "gateway_unavailable");
+    assert.equal(body.message, "provider evaluation failed");
+    assert.ok(!body.message.includes("SECRET"), "internal error text must never reach the caller");
+    await service.stop();
+    audit.stop();
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+});