jev-layer 0.1.0

package/scripts/replay-eval.mjs

@@ -0,0 +1,124 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { dirname } from "node:path";
+import { readRoutingCases, readSupervisionCases, replayCasePath } from "../src/receipts.mjs";
+import { deterministicSupervisionPolicy } from "../src/supervision.mjs";
+import { routeRequest } from "../src/route.mjs";
+
+const args = parseArgs(process.argv.slice(2));
+const inputPath = replayCasePath(args.input);
+const provider = args.provider ?? process.env.JEV_REPLAY_PROVIDER ?? "demo";
+const allCases = await readRoutingCases(inputPath);
+const cases = args.limit ? allCases.slice(0, args.limit) : allCases;
+const results = [];
+
+for (const routingCase of cases) {
+  const decision = await routeRequest(routingCase.request, { provider });
+  if (decision.execution?.enabled) throw new Error(`replay attempted host execution for ${routingCase.case_id}`);
+  results.push({
+    source_case_id: routingCase.case_id,
+    source_correlation_id: routingCase.correlation_id,
+    replay_correlation_id: decision.correlation_id,
+    surface: routingCase.request.context?.browser ? "browser" : "generic",
+    harness: routingCase.harness,
+    selected: decision.selected,
+    confidence: decision.confidence,
+    status: decision.status,
+    fallback: decision.fallback?.type ?? null,
+    latency_ms: decision.receipt?.latency_ms ?? null,
+    cost_usd: decision.receipt?.cost_usd ?? decision.raw_jev?.usage?.cost ?? null,
+    execution: decision.execution,
+  });
+}
+
+const supervisionCases = args.limit
+  ? (await readSupervisionCases(inputPath)).slice(0, args.limit)
+  : await readSupervisionCases(inputPath);
+const supervisionResults = supervisionCases.map((record) => ({
+  source_case_id: record.case_id,
+  source_correlation_id: record.correlation_id,
+  harness: record.harness,
+  assessment: record.supervision.assessment,
+  recorded_action: record.supervision.action,
+  replayed_policy: deterministicSupervisionPolicy({
+    assessment: record.supervision.assessment,
+    evidence: record.request.context?.evidence ?? {},
+  }),
+  host_execution: false,
+}));
+
+
+const summary = summarize(results);
+summary.supervision_case_count = supervisionResults.length;
+summary.supervision_policy_match_rate = rate(
+  supervisionResults.filter((result) => result.replayed_policy.action === result.recorded_action).length,
+  supervisionResults.length,
+);
+const report = {
+  ok: true,
+  source: inputPath,
+  provider,
+  replayed_cases: results.length,
+  replayed_supervision_cases: supervisionResults.length,
+  host_execution_calls: 0,
+  summary,
+  cases: results,
+  supervision_cases: supervisionResults,
+};
+
+if (args.output) {
+  await mkdir(dirname(args.output), { recursive: true });
+  await writeFile(args.output, `${JSON.stringify(report, null, 2)}\n`, "utf8");
+}
+console.log(JSON.stringify(report, null, 2));
+function summarize(results) {
+  const total = results.length;
+  const fallbackCount = results.filter((result) => result.status === "fallback").length;
+  const selectedCount = results.filter((result) => result.status === "selected" || result.status === "needs_confirmation").length;
+  const noDecisionCount = results.filter((result) => result.status === "no_decision").length;
+  const browserResults = results.filter((result) => result.surface === "browser");
+  const confidences = results.map((result) => result.confidence).filter((value) => typeof value === "number");
+  const latencies = results.map((result) => result.latency_ms).filter((value) => typeof value === "number");
+  const costs = results.map((result) => result.cost_usd).filter((value) => typeof value === "number");
+  return {
+    selected_rate: rate(selectedCount, total),
+    fallback_rate: rate(fallbackCount, total),
+    no_decision_rate: rate(noDecisionCount, total),
+    browser_case_count: browserResults.length,
+    browser_selected_rate: rate(browserResults.filter((result) => result.status === "selected" || result.status === "needs_confirmation").length, browserResults.length),
+    average_confidence: average(confidences),
+    average_latency_ms: average(latencies),
+    p50_latency_ms: percentile(latencies, 0.5),
+    p95_latency_ms: percentile(latencies, 0.95),
+    total_cost_usd: Number(costs.reduce((sum, value) => sum + value, 0).toFixed(9)),
+    status_counts: Object.fromEntries([...new Set(results.map((result) => result.status))].map((status) => [status, results.filter((result) => result.status === status).length])),
+  };
+}
+
+function parseArgs(argv) {
+  const result = {};
+  for (let index = 0; index < argv.length; index += 1) {
+    const value = argv[index];
+    if (value === "--input") result.input = argv[++index];
+    else if (value === "--provider") result.provider = argv[++index];
+    else if (value === "--limit") result.limit = Number(argv[++index]);
+    else if (value === "--output") result.output = argv[++index];
+  }
+  if (result.limit !== undefined && (!Number.isInteger(result.limit) || result.limit < 0)) throw new Error("--limit must be a non-negative integer");
+  return result;
+}
+
+function average(values) {
+  return values.length ? Number((values.reduce((sum, value) => sum + value, 0) / values.length).toFixed(3)) : null;
+}
+
+function percentile(values, fraction) {
+  if (!values.length) return null;
+  const sorted = [...values].sort((left, right) => left - right);
+  const rank = Math.max(0, Math.ceil(sorted.length * fraction) - 1);
+  return Number(sorted[Math.min(sorted.length - 1, rank)].toFixed(3));
+}
+
+
+function rate(part, total) {
+  return total ? Number((part / total).toFixed(4)) : 0;
+}

package/scripts/smoke.mjs

@@ -0,0 +1,34 @@
+import assert from "node:assert/strict";
+import { readFile } from "node:fs/promises";
+import { routeRequest } from "../src/route.mjs";
+
+const request = JSON.parse(await readFile(new URL("../examples/route-request.json", import.meta.url), "utf8"));
+const selected = await routeRequest(request, { provider: "demo" });
+assert.equal(selected.status, "selected");
+assert.equal(selected.selected, "repo_search");
+assert.equal(selected.execution.enabled, false);
+assert.equal(selected.execution.status, "not_started");
+assert.ok(selected.receipt.context_bytes > 0);
+assert.equal(JSON.stringify(selected.raw_jev).includes("must-not-reach-provider"), false);
+
+const unavailable = await routeRequest({
+  ...request,
+  intent: "Modify a file",
+  capabilities: request.capabilities.map((capability) => ({ ...capability, availability: capability.id === "repo_search" ? { available: false, reason: "offline" } : undefined })),
+}, { provider: "demo" });
+assert.equal(unavailable.status, "no_decision");
+
+const providerFailure = await routeRequest(request, {
+  provider: { name: "failing-test-provider", async decide() { throw new Error("synthetic provider failure"); } },
+});
+assert.equal(providerFailure.status, "fallback");
+assert.equal(providerFailure.fallback.type, "provider_error");
+assert.equal(providerFailure.execution.enabled, false);
+
+console.log(JSON.stringify({
+  ok: true,
+  selected: selected.selected,
+  unavailable_status: unavailable.status,
+  provider_failure: providerFailure.fallback.type,
+  execution_started: selected.execution.enabled,
+}));

package/scripts/supervision-e2e.mjs

@@ -0,0 +1,109 @@
+import assert from "node:assert/strict";
+import { mkdtemp, readFile } from "node:fs/promises";
+import { spawn } from "node:child_process";
+import { createInterface } from "node:readline";
+import { fileURLToPath } from "node:url";
+import { dirname, join, resolve } from "node:path";
+import { tmpdir } from "node:os";
+import { deterministicSupervisionPolicy } from "../src/supervision.mjs";
+
+const root = resolve(dirname(fileURLToPath(import.meta.url)), "..");
+const casesPath = join(await mkdtemp(join(tmpdir(), "jev-supervision-e2e-")), "cases.jsonl");
+const child = spawn(process.env.JEV_NODE ?? "node", [resolve(root, "src/mcp-server.mjs")], {
+  cwd: root,
+  env: { ...process.env, JEV_REPLAY_CASES: casesPath },
+  stdio: ["pipe", "pipe", "pipe"],
+});
+const lines = createInterface({ input: child.stdout });
+const pending = new Map();
+lines.on("line", (line) => {
+  if (!line.trim()) return;
+  const message = JSON.parse(line);
+  const waiter = pending.get(message.id);
+  if (waiter) {
+    pending.delete(message.id);
+    waiter(message);
+  }
+});
+
+try {
+  const initialized = await request(1, "initialize", {});
+  assert.equal(initialized.result.serverInfo.name, "jev-layer");
+  await notification("notifications/initialized", {});
+  const listed = await request(2, "tools/list", {});
+  assert.ok(listed.result.tools.some((tool) => tool.name === "jev_supervise"));
+
+  const disabled = await request(3, "tools/call", {
+    name: "jev_supervise",
+    arguments: { enabled: false, provider: "demo", job: {}, observation: {} },
+  });
+  assert.equal(disabled.result.structuredContent.fallback.type, "disabled");
+
+  const judged = await request(4, "tools/call", {
+    name: "jev_supervise",
+    arguments: {
+      enabled: true,
+      provider: process.env.JEV_SUPERVISION_PROVIDER ?? "demo",
+      harness: "supervision-e2e",
+      job: { requirements: ["run tests", "inspect receipts"] },
+      observation: { status: "ready", progress: "complete" },
+      evidence: {
+        tests_passed: true,
+        judgments: {
+          requirements_addressed: 0.9,
+          verification_needed: 0.1,
+          meaningful_progress: 0.9,
+          worker_stuck: 0.05,
+          work_off_track: 0.05,
+          completion: 0.9,
+        },
+      },
+    },
+  });
+  const result = judged.result.structuredContent;
+  assert.equal(result.status, "judged");
+  assert.equal(result.action, "finish");
+  assert.equal(result.metrics.jev_calls, 1);
+  assert.ok(result.receipt.correlation_id);
+
+  const records = (await readFile(casesPath, "utf8")).trim().split(/\r?\n/).map(JSON.parse);
+  assert.deepEqual(records.map((record) => record.record_type), ["supervision_case"]);
+  assert.equal(records[0].supervision.action, "finish");
+  const replayed = deterministicSupervisionPolicy({
+    assessment: records[0].supervision.assessment,
+    evidence: records[0].request.context.evidence,
+  });
+  assert.equal(replayed.action, records[0].supervision.action);
+  console.log(JSON.stringify({
+    ok: true,
+    cases_path: casesPath,
+    action: result.action,
+    status: result.status,
+    dimensions: Object.keys(result.assessment).length,
+    latency_ms: result.receipt.latency_ms,
+    cost_usd: result.receipt.cost_usd,
+    receipts: records.length,
+  }, null, 2));
+} finally {
+  child.kill();
+  await new Promise((resolveExit) => child.once("close", resolveExit));
+}
+
+function request(id, method, params) {
+  return new Promise((resolveResponse, reject) => {
+    const timer = setTimeout(() => {
+      pending.delete(id);
+      reject(new Error(`MCP request timed out: ${method}`));
+    }, 15_000);
+    pending.set(id, (message) => {
+      clearTimeout(timer);
+      resolveResponse(message);
+    });
+    child.stdin.write(`${JSON.stringify({ jsonrpc: "2.0", id, method, params })}\n`);
+  });
+}
+
+function notification(method, params) {
+  child.stdin.write(`${JSON.stringify({ jsonrpc: "2.0", method, params })}\n`);
+  return new Promise((resolveNotification) => setImmediate(resolveNotification));
+}