jev-layer 0.1.0

package/test/receipts.test.mjs

@@ -0,0 +1,71 @@
+import assert from "node:assert/strict";
+import { mkdtemp, readFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { test } from "node:test";
+import { appendExecutionReceipt, appendRoutingCase, readRoutingCases } from "../src/receipts.mjs";
+import { routeRequest } from "../src/route.mjs";
+
+const request = {
+  harness: "receipt-test",
+  intent: "inspect the repository",
+  context: { workspace: "test" },
+  actor_permissions: ["read"],
+  capabilities: [{
+    id: "read_repo",
+    kind: "tool",
+    name: "Read repository",
+    description: "Read repository metadata without modifying files",
+    permissions: ["read"],
+    risk: "low",
+  }],
+};
+
+test("execution receipt carries one correlation id through host completion", async () => {
+  const provider = {
+    name: "receipt-test-provider",
+    async decide() {
+      return {
+        answers: {
+          tool: {
+            type: "choice",
+            choice: "read_repo",
+            probabilities: { read_repo: 1 },
+            confidence: 1,
+          },
+        },
+        usage: { cost: 0.000123 },
+      };
+    },
+  };
+  const decision = await routeRequest(request, { provider });
+  const path = join(await mkdtemp(join(tmpdir(), "jev-receipts-")), "cases.jsonl");
+  await appendRoutingCase({ path, request, decision });
+  const completed = await appendExecutionReceipt({
+    path,
+    request,
+    decision,
+    host: {
+      harness: "receipt-test",
+      capability_id: "read_repo",
+      status: "completed",
+      result: { bytes: 42 },
+      exit_status: 0,
+      duration_ms: 12.5,
+    },
+  });
+
+  assert.equal(decision.correlation_id, decision.receipt.correlation_id);
+  assert.equal(decision.receipt.cost_usd, 0.000123);
+  assert.equal(completed.record.correlation_id, decision.correlation_id);
+  assert.equal(completed.record.jev.cost_usd, 0.000123);
+  assert.equal(completed.record.host.exit_status, 0);
+  assert.equal(completed.record.host.duration_ms, 12.5);
+  assert.deepEqual(completed.record.host.result, { bytes: 42 });
+
+  const routingCases = await readRoutingCases(path);
+  assert.equal(routingCases.length, 1);
+  assert.equal(routingCases[0].case_id, decision.correlation_id);
+  const firstRecord = JSON.parse((await readFile(path, "utf8")).split(/\r?\n/)[0]);
+  assert.equal(firstRecord.record_type, "routing_case");
+});

package/test/route.test.mjs

@@ -0,0 +1,74 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { projectState } from "../src/context-filter.mjs";
+import { routeRequest } from "../src/route.mjs";
+
+const candidates = [
+  { id: "search", kind: "mcp", name: "Repository search", description: "Search repository files and symbols", permissions: ["read"], risk: "low" },
+  { id: "write", kind: "tool", name: "File writer", description: "Modify repository files", permissions: ["write"], risk: "high" },
+];
+
+function request(overrides = {}) {
+  return {
+    harness: "test",
+    intent: "Search repository files",
+    context: { workspace: "test", api_key: "hidden" },
+    actor_permissions: ["read"],
+    capabilities: candidates,
+    ...overrides,
+  };
+}
+
+test("routes to an eligible capability without executing it", async () => {
+  const decision = await routeRequest(request(), { provider: "demo" });
+  assert.equal(decision.status, "selected");
+  assert.equal(decision.selected, "search");
+  assert.deepEqual(decision.execution, { enabled: false, status: "not_started" });
+  assert.equal(decision.candidates.find((candidate) => candidate.id === "write").filtered, true);
+});
+
+test("returns no_decision when every candidate is unavailable or forbidden", async () => {
+  const decision = await routeRequest(request({ capabilities: [
+    { ...candidates[0], availability: { available: false, reason: "offline" } },
+    candidates[1],
+  ] }), { provider: "demo" });
+  assert.equal(decision.status, "no_decision");
+  assert.equal(decision.execution.enabled, false);
+});
+
+test("fails open on provider errors and malformed responses", async () => {
+  const providerError = await routeRequest(request(), { provider: { name: "test-error", async decide() { throw new Error("down"); } } });
+  assert.equal(providerError.status, "fallback");
+  assert.equal(providerError.fallback.type, "provider_error");
+
+  const malformed = await routeRequest(request(), { provider: { name: "test-malformed", async decide() { return { answers: { tool: { type: "score" } } }; } } });
+  assert.equal(malformed.status, "fallback");
+  assert.equal(malformed.fallback.type, "malformed_response");
+});
+
+test("falls back when confidence is below the code threshold", async () => {
+  const decision = await routeRequest(request({ intent: "Do something unrelated", actor_permissions: ["read", "write"] }), { provider: "demo", policy: { min_confidence: 0.99 } });
+  assert.equal(decision.status, "fallback");
+  assert.equal(decision.fallback.type, "low_confidence");
+});
+
+test("filters secrets and bounds provider state", () => {
+  const projection = projectState(request({ context: { api_key: "hidden", nested: { password: "hidden-too" }, note: "safe" } }), candidates, 1_000);
+  const serialized = JSON.stringify(projection.state);
+  assert.equal(serialized.includes("hidden"), false);
+  assert.equal(serialized.includes("safe"), true);
+  assert.ok(projection.context_bytes <= 1_000);
+});
+
+test("honors explicit disablement with fail-open fallback", async () => {
+  const previous = process.env.JEV_LAYER_ENABLED;
+  process.env.JEV_LAYER_ENABLED = "0";
+  try {
+    const decision = await routeRequest(request(), { provider: "demo" });
+    assert.equal(decision.status, "fallback");
+    assert.equal(decision.fallback.type, "disabled");
+  } finally {
+    if (previous === undefined) delete process.env.JEV_LAYER_ENABLED;
+    else process.env.JEV_LAYER_ENABLED = previous;
+  }
+});

package/test/supervision.test.mjs

@@ -0,0 +1,99 @@
+import assert from "node:assert/strict";
+import { mkdtemp, readFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { test } from "node:test";
+import {
+  deterministicSupervisionPolicy,
+  normalizeAssessment,
+  superviseWork,
+  supervisionQuestions,
+} from "../src/supervision.mjs";
+import { readSupervisionCases } from "../src/receipts.mjs";
+
+function assessment(overrides = {}) {
+  return {
+    requirements_addressed: 0.8,
+    verification_needed: 0.1,
+    meaningful_progress: 0.8,
+    worker_stuck: 0.1,
+    work_off_track: 0.1,
+    completion: 0.8,
+    ...overrides,
+  };
+}
+
+test("supervision contract exposes bounded dimensions and parses Noul answers", () => {
+  assert.deepEqual(Object.keys(supervisionQuestions()), [
+    "requirements_addressed",
+    "verification_needed",
+    "meaningful_progress",
+    "worker_stuck",
+    "work_off_track",
+    "completion",
+  ]);
+  const parsed = normalizeAssessment({
+    answers: {
+      requirements_addressed: { type: "noul", probability: 0.8 },
+      verification_needed: { type: "noul", p_true: 0.2 },
+      meaningful_progress: { type: "noul", score: 0.7 },
+      worker_stuck: true,
+      work_off_track: 0.1,
+      completion: { type: "noul", noul: 0.9 },
+    },
+  });
+  assert.deepEqual(parsed, {
+    requirements_addressed: 0.8,
+    verification_needed: 0.2,
+    meaningful_progress: 0.7,
+    worker_stuck: 1,
+    work_off_track: 0.1,
+    completion: 0.9,
+  });
+});
+
+test("deterministic host policy owns the supervision action", () => {
+  assert.equal(deterministicSupervisionPolicy({ assessment: assessment({ verification_needed: 0.9 }) }).action, "verify");
+  assert.equal(deterministicSupervisionPolicy({ assessment: assessment({ worker_stuck: 0.9 }), attempts: 0 }).action, "retry");
+  assert.equal(deterministicSupervisionPolicy({ assessment: assessment({ worker_stuck: 0.9 }), attempts: 2 }).action, "escalate");
+  assert.equal(deterministicSupervisionPolicy({ assessment: assessment(), evidence: { tests_passed: true } }).action, "finish");
+});
+
+test("supervision is opt-in, fail-open, and records replayable judgments", async () => {
+  const disabled = await superviseWork({ enabled: false, provider: { async evaluate() { throw new Error("must not call"); } } });
+  assert.equal(disabled.status, "fallback");
+  assert.equal(disabled.action, "continue");
+  assert.equal(disabled.fallback.type, "disabled");
+
+  const directory = await mkdtemp(join(tmpdir(), "jev-supervision-test-"));
+  const path = join(directory, "cases.jsonl");
+  let calls = 0;
+  const result = await superviseWork({
+    enabled: true,
+    harness: "supervision-test",
+    job: { requirements: ["run tests"] },
+    observation: { status: "ready" },
+    evidence: { tests_passed: true },
+    provider: {
+      name: "injected-supervision",
+      async evaluate({ questions }) {
+        calls += 1;
+        return {
+          answers: Object.fromEntries(Object.keys(questions).map((key) => [key, { type: "noul", probability: assessment()[key] }])),
+          usage: { cost: 0.0003 },
+        };
+      },
+    },
+    receiptPath: path,
+  });
+  assert.equal(calls, 1);
+  assert.equal(result.status, "judged");
+  assert.equal(result.action, "finish");
+  assert.equal(result.metrics.jev_calls, 1);
+  assert.equal(result.metrics.cost_usd, 0.0003);
+  const cases = await readSupervisionCases(path);
+  assert.equal(cases.length, 1);
+  assert.equal(cases[0].record_type, "supervision_case");
+  assert.equal(cases[0].supervision.action, "finish");
+  assert.equal(JSON.parse(await readFile(path, "utf8")).record_type, "supervision_case");
+});