jev-layer 0.1.0 → 0.2.0

package/scripts/mcp-receipt-smoke.mjs

@@ -1,98 +1,113 @@
-import assert from "node:assert/strict";
-import { mkdtemp, readFile } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join, resolve } from "node:path";
-import { spawn } from "node:child_process";
-import { createInterface } from "node:readline";
-
-const root = resolve(new URL("..", import.meta.url).pathname);
-const casesPath = join(await mkdtemp(join(tmpdir(), "jev-mcp-receipt-")), "cases.jsonl");
-const child = spawn(process.env.JEV_NODE ?? "node", [resolve(root, "src/mcp-server.mjs")], {
-  cwd: root,
-  env: { ...process.env, JEV_REPLAY_CASES: casesPath },
-  stdio: ["pipe", "pipe", "pipe"],
-});
-const lines = createInterface({ input: child.stdout });
-const pending = new Map();
-lines.on("line", (line) => {
-  if (!line.trim()) return;
-  const message = JSON.parse(line);
-  const waiter = pending.get(message.id);
-  if (waiter) {
-    pending.delete(message.id);
-    waiter(message);
-  }
-});
-
-try {
-  const initialized = await request(1, "initialize", {});
-  assert.equal(initialized.result.serverInfo.name, "jev-layer");
-  await notification("notifications/initialized", {});
-  const listed = await request(2, "tools/list", {});
-  assert.ok(listed.result.tools.some((tool) => tool.name === "jev_route"));
-  assert.ok(listed.result.tools.some((tool) => tool.name === "jev_record_execution"));
-
-  const routed = await request(3, "tools/call", {
-    name: "jev_route",
-    arguments: {
-      provider: "demo",
-      harness: "mcp-receipt-smoke",
-      intent: "read repository metadata",
-      actor_permissions: ["read"],
-      capabilities: [{
-        id: "read_metadata",
-        kind: "tool",
-        name: "Read metadata",
-        description: "Read repository metadata without modifying files",
-        permissions: ["read"],
-        risk: "low",
-      }],
-    },
-  });
-  const decision = routed.result.structuredContent;
-  assert.equal(decision.status, "selected");
-  assert.equal(decision.receipt.replay_persisted, true);
-
-  const recorded = await request(4, "tools/call", {
-    name: "jev_record_execution",
-    arguments: {
-      correlation_id: decision.correlation_id,
-      harness: "mcp-receipt-smoke",
-      capability_id: decision.selected,
-      status: "completed",
-      result: { observed: "metadata" },
-      exit_status: 0,
-      duration_ms: 3.2,
-    },
-  });
-  const receipt = recorded.result.structuredContent;
-  assert.equal(receipt.correlation_id, decision.correlation_id);
-  assert.equal(receipt.host.exit_status, 0);
-  assert.equal(receipt.host.duration_ms, 3.2);
-
-  const records = (await readFile(casesPath, "utf8")).trim().split(/\r?\n/).map(JSON.parse);
-  assert.deepEqual(records.map((record) => record.record_type), ["routing_case", "execution_receipt"]);
-  console.log(JSON.stringify({ ok: true, cases_path: casesPath, receipt }, null, 2));
-} finally {
-  child.kill();
-  await new Promise((resolveExit) => child.once("close", resolveExit));
-}
-
-function request(id, method, params) {
-  return new Promise((resolveResponse, reject) => {
-    const timer = setTimeout(() => {
-      pending.delete(id);
-      reject(new Error(`MCP request timed out: ${method}`));
-    }, 15_000);
-    pending.set(id, (message) => {
-      clearTimeout(timer);
-      resolveResponse(message);
-    });
-    child.stdin.write(`${JSON.stringify({ jsonrpc: "2.0", id, method, params })}\n`);
-  });
-}
-
-function notification(method, params) {
-  child.stdin.write(`${JSON.stringify({ jsonrpc: "2.0", method, params })}\n`);
-  return new Promise((resolveNotification) => setImmediate(resolveNotification));
-}
+import assert from "node:assert/strict";
+import { mkdtemp, readFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { spawn } from "node:child_process";
+import { createInterface } from "node:readline";
+
+const root = resolve(new URL("..", import.meta.url).pathname);
+const casesPath = join(await mkdtemp(join(tmpdir(), "jev-mcp-receipt-")), "cases.jsonl");
+const child = spawn(process.env.JEV_NODE ?? "node", [resolve(root, "src/mcp-server.mjs")], {
+  cwd: root,
+  env: { ...process.env, JEV_REPLAY_CASES: casesPath },
+  stdio: ["pipe", "pipe", "pipe"],
+});
+const lines = createInterface({ input: child.stdout });
+const pending = new Map();
+lines.on("line", (line) => {
+  if (!line.trim()) return;
+  const message = JSON.parse(line);
+  const waiter = pending.get(message.id);
+  if (waiter) {
+    pending.delete(message.id);
+    waiter(message);
+  }
+});
+
+try {
+  const initialized = await request(1, "initialize", {});
+  assert.equal(initialized.result.serverInfo.name, "jev-layer");
+  await notification("notifications/initialized", {});
+  const listed = await request(2, "tools/list", {});
+  assert.ok(listed.result.tools.some((tool) => tool.name === "jev_route"));
+  assert.ok(listed.result.tools.some((tool) => tool.name === "jev_model_route"));
+  assert.ok(listed.result.tools.some((tool) => tool.name === "jev_shadow_compaction"));
+  assert.ok(listed.result.tools.some((tool) => tool.name === "jev_record_execution"));
+
+  const routed = await request(3, "tools/call", {
+    name: "jev_route",
+    arguments: {
+      provider: "demo",
+      harness: "mcp-receipt-smoke",
+      intent: "read repository metadata",
+      actor_permissions: ["read"],
+      capabilities: [{
+        id: "read_metadata",
+        kind: "tool",
+        name: "Read metadata",
+        description: "Read repository metadata without modifying files",
+        permissions: ["read"],
+        risk: "low",
+      }],
+    },
+  });
+  const decision = routed.result.structuredContent;
+  assert.equal(decision.status, "selected");
+  assert.equal(decision.receipt.replay_persisted, true);
+
+  const recorded = await request(4, "tools/call", {
+    name: "jev_record_execution",
+    arguments: {
+      correlation_id: decision.correlation_id,
+      harness: "mcp-receipt-smoke",
+      capability_id: decision.selected,
+      status: "completed",
+      result: { observed: "metadata" },
+      exit_status: 0,
+      duration_ms: 3.2,
+    },
+  });
+  const receipt = recorded.result.structuredContent;
+  assert.equal(receipt.correlation_id, decision.correlation_id);
+  assert.equal(receipt.host.exit_status, 0);
+  assert.equal(receipt.host.duration_ms, 3.2);
+
+  const shadow = await request(5, "tools/call", {
+    name: "jev_shadow_compaction",
+    arguments: {
+      provider: "demo",
+      intent: "inspect context before compaction",
+      context: { messages: ["stale note", "Requirement: preserve /workspace/config.json"] },
+    },
+  });
+  const shadowReport = shadow.result.structuredContent;
+  assert.equal(shadowReport.mode, "jev_shadow");
+  assert.equal(shadowReport.changed, false);
+  assert.equal(shadowReport.protected_count, 1);
+
+  const records = (await readFile(casesPath, "utf8")).trim().split(/\r?\n/).map(JSON.parse);
+  assert.deepEqual(records.map((record) => record.record_type), ["routing_case", "execution_receipt"]);
+  console.log(JSON.stringify({ ok: true, cases_path: casesPath, receipt }, null, 2));
+} finally {
+  child.kill();
+  await new Promise((resolveExit) => child.once("close", resolveExit));
+}
+
+function request(id, method, params) {
+  return new Promise((resolveResponse, reject) => {
+    const timer = setTimeout(() => {
+      pending.delete(id);
+      reject(new Error(`MCP request timed out: ${method}`));
+    }, 15_000);
+    pending.set(id, (message) => {
+      clearTimeout(timer);
+      resolveResponse(message);
+    });
+    child.stdin.write(`${JSON.stringify({ jsonrpc: "2.0", id, method, params })}\n`);
+  });
+}
+
+function notification(method, params) {
+  child.stdin.write(`${JSON.stringify({ jsonrpc: "2.0", method, params })}\n`);
+  return new Promise((resolveNotification) => setImmediate(resolveNotification));
+}

package/scripts/model-route-report.mjs

@@ -0,0 +1,15 @@
+#!/usr/bin/env node
+import { readFile } from "node:fs/promises";
+import { buildModelRouteReport } from "../src/model-route-metrics.mjs";
+
+const path = process.argv[2] ?? process.env.JEV_REPLAY_CASES ?? ".jev/replay/cases.jsonl";
+let text = "";
+try {
+  text = await readFile(path, "utf8");
+} catch (error) {
+  if (error?.code !== "ENOENT") throw error;
+}
+const records = text.split(/\r?\n/).filter(Boolean).flatMap((line) => {
+  try { return [JSON.parse(line)]; } catch { return []; }
+});
+console.log(JSON.stringify(buildModelRouteReport(records), null, 2));

package/scripts/openrouter-choice.mjs

@@ -1,59 +1,59 @@
-import { routeRequest } from "../src/route.mjs";
-
-const endpoint = process.env.OPENROUTER_DECISIONS_ENDPOINT ?? "https://openrouter.ai/api/alpha/decisions";
-const model = process.env.OPENROUTER_DECISIONS_MODEL ?? "typesafe/jev-1.13";
-const decision = await routeRequest({
-  schema_version: 1,
-  harness: "jev-layer-openrouter-smoke",
-  intent: "Find where the OpenRouter Decisions provider is implemented without modifying files",
-  context: {
-    purpose: "one real bounded Choice call",
-    secret_note: "must be redacted before provider transmission",
-  },
-  actor_permissions: ["read", "browser", "terminal:read"],
-  capabilities: [
-    {
-      id: "repo_search",
-      kind: "mcp",
-      name: "Repository search",
-      description: "Search repository files and symbols without changing files",
-      permissions: ["read"],
-      risk: "low",
-    },
-    {
-      id: "browser_inspect",
-      kind: "mcp",
-      name: "Browser inspection",
-      description: "Open a webpage and inspect visible browser state",
-      permissions: ["browser"],
-      risk: "medium",
-    },
-    {
-      id: "terminal_read",
-      kind: "cli",
-      name: "Read-only terminal",
-      description: "Run a read-only shell command and return its output",
-      permissions: ["terminal:read"],
-      risk: "low",
-    },
-  ],
-}, {
-  provider: "openrouter",
-  openrouter: { endpoint, model, timeoutMs: 15_000 },
-});
-
-const raw = decision.raw_jev ?? {};
-const usage = raw.usage && typeof raw.usage === "object" ? raw.usage : {};
-console.log(JSON.stringify({
-  request: { endpoint, model, method: "POST" },
-  response: { id: raw.id ?? null, model: raw.model ?? null, provider: raw.provider ?? null },
-  status: decision.status,
-  selected: decision.selected,
-  probabilities: decision.probabilities,
-  confidence: decision.confidence,
-  latency_ms: decision.receipt.latency_ms,
-  cost_usd: typeof usage.cost === "number" ? usage.cost : null,
-  usage,
-  fallback: decision.fallback,
-  execution: decision.execution,
-}, null, 2));
+import { routeRequest } from "../src/route.mjs";
+
+const endpoint = process.env.OPENROUTER_DECISIONS_ENDPOINT ?? "https://openrouter.ai/api/alpha/decisions";
+const model = process.env.OPENROUTER_DECISIONS_MODEL ?? "typesafe/jev-1.13";
+const decision = await routeRequest({
+  schema_version: 1,
+  harness: "jev-layer-openrouter-smoke",
+  intent: "Find where the OpenRouter Decisions provider is implemented without modifying files",
+  context: {
+    purpose: "one real bounded Choice call",
+    secret_note: "must be redacted before provider transmission",
+  },
+  actor_permissions: ["read", "browser", "terminal:read"],
+  capabilities: [
+    {
+      id: "repo_search",
+      kind: "mcp",
+      name: "Repository search",
+      description: "Search repository files and symbols without changing files",
+      permissions: ["read"],
+      risk: "low",
+    },
+    {
+      id: "browser_inspect",
+      kind: "mcp",
+      name: "Browser inspection",
+      description: "Open a webpage and inspect visible browser state",
+      permissions: ["browser"],
+      risk: "medium",
+    },
+    {
+      id: "terminal_read",
+      kind: "cli",
+      name: "Read-only terminal",
+      description: "Run a read-only shell command and return its output",
+      permissions: ["terminal:read"],
+      risk: "low",
+    },
+  ],
+}, {
+  provider: "openrouter",
+  openrouter: { endpoint, model, timeoutMs: 15_000 },
+});
+
+const raw = decision.raw_jev ?? {};
+const usage = raw.usage && typeof raw.usage === "object" ? raw.usage : {};
+console.log(JSON.stringify({
+  request: { endpoint, model, method: "POST" },
+  response: { id: raw.id ?? null, model: raw.model ?? null, provider: raw.provider ?? null },
+  status: decision.status,
+  selected: decision.selected,
+  probabilities: decision.probabilities,
+  confidence: decision.confidence,
+  latency_ms: decision.receipt.latency_ms,
+  cost_usd: typeof usage.cost === "number" ? usage.cost : null,
+  usage,
+  fallback: decision.fallback,
+  execution: decision.execution,
+}, null, 2));

package/scripts/replay-eval.mjs

@@ -1,124 +1,124 @@
-import { mkdir, writeFile } from "node:fs/promises";
-import { dirname } from "node:path";
-import { readRoutingCases, readSupervisionCases, replayCasePath } from "../src/receipts.mjs";
-import { deterministicSupervisionPolicy } from "../src/supervision.mjs";
-import { routeRequest } from "../src/route.mjs";
-
-const args = parseArgs(process.argv.slice(2));
-const inputPath = replayCasePath(args.input);
-const provider = args.provider ?? process.env.JEV_REPLAY_PROVIDER ?? "demo";
-const allCases = await readRoutingCases(inputPath);
-const cases = args.limit ? allCases.slice(0, args.limit) : allCases;
-const results = [];
-
-for (const routingCase of cases) {
-  const decision = await routeRequest(routingCase.request, { provider });
-  if (decision.execution?.enabled) throw new Error(`replay attempted host execution for ${routingCase.case_id}`);
-  results.push({
-    source_case_id: routingCase.case_id,
-    source_correlation_id: routingCase.correlation_id,
-    replay_correlation_id: decision.correlation_id,
-    surface: routingCase.request.context?.browser ? "browser" : "generic",
-    harness: routingCase.harness,
-    selected: decision.selected,
-    confidence: decision.confidence,
-    status: decision.status,
-    fallback: decision.fallback?.type ?? null,
-    latency_ms: decision.receipt?.latency_ms ?? null,
-    cost_usd: decision.receipt?.cost_usd ?? decision.raw_jev?.usage?.cost ?? null,
-    execution: decision.execution,
-  });
-}
-
-const supervisionCases = args.limit
-  ? (await readSupervisionCases(inputPath)).slice(0, args.limit)
-  : await readSupervisionCases(inputPath);
-const supervisionResults = supervisionCases.map((record) => ({
-  source_case_id: record.case_id,
-  source_correlation_id: record.correlation_id,
-  harness: record.harness,
-  assessment: record.supervision.assessment,
-  recorded_action: record.supervision.action,
-  replayed_policy: deterministicSupervisionPolicy({
-    assessment: record.supervision.assessment,
-    evidence: record.request.context?.evidence ?? {},
-  }),
-  host_execution: false,
-}));
-
-
-const summary = summarize(results);
-summary.supervision_case_count = supervisionResults.length;
-summary.supervision_policy_match_rate = rate(
-  supervisionResults.filter((result) => result.replayed_policy.action === result.recorded_action).length,
-  supervisionResults.length,
-);
-const report = {
-  ok: true,
-  source: inputPath,
-  provider,
-  replayed_cases: results.length,
-  replayed_supervision_cases: supervisionResults.length,
-  host_execution_calls: 0,
-  summary,
-  cases: results,
-  supervision_cases: supervisionResults,
-};
-
-if (args.output) {
-  await mkdir(dirname(args.output), { recursive: true });
-  await writeFile(args.output, `${JSON.stringify(report, null, 2)}\n`, "utf8");
-}
-console.log(JSON.stringify(report, null, 2));
-function summarize(results) {
-  const total = results.length;
-  const fallbackCount = results.filter((result) => result.status === "fallback").length;
-  const selectedCount = results.filter((result) => result.status === "selected" || result.status === "needs_confirmation").length;
-  const noDecisionCount = results.filter((result) => result.status === "no_decision").length;
-  const browserResults = results.filter((result) => result.surface === "browser");
-  const confidences = results.map((result) => result.confidence).filter((value) => typeof value === "number");
-  const latencies = results.map((result) => result.latency_ms).filter((value) => typeof value === "number");
-  const costs = results.map((result) => result.cost_usd).filter((value) => typeof value === "number");
-  return {
-    selected_rate: rate(selectedCount, total),
-    fallback_rate: rate(fallbackCount, total),
-    no_decision_rate: rate(noDecisionCount, total),
-    browser_case_count: browserResults.length,
-    browser_selected_rate: rate(browserResults.filter((result) => result.status === "selected" || result.status === "needs_confirmation").length, browserResults.length),
-    average_confidence: average(confidences),
-    average_latency_ms: average(latencies),
-    p50_latency_ms: percentile(latencies, 0.5),
-    p95_latency_ms: percentile(latencies, 0.95),
-    total_cost_usd: Number(costs.reduce((sum, value) => sum + value, 0).toFixed(9)),
-    status_counts: Object.fromEntries([...new Set(results.map((result) => result.status))].map((status) => [status, results.filter((result) => result.status === status).length])),
-  };
-}
-
-function parseArgs(argv) {
-  const result = {};
-  for (let index = 0; index < argv.length; index += 1) {
-    const value = argv[index];
-    if (value === "--input") result.input = argv[++index];
-    else if (value === "--provider") result.provider = argv[++index];
-    else if (value === "--limit") result.limit = Number(argv[++index]);
-    else if (value === "--output") result.output = argv[++index];
-  }
-  if (result.limit !== undefined && (!Number.isInteger(result.limit) || result.limit < 0)) throw new Error("--limit must be a non-negative integer");
-  return result;
-}
-
-function average(values) {
-  return values.length ? Number((values.reduce((sum, value) => sum + value, 0) / values.length).toFixed(3)) : null;
-}
-
-function percentile(values, fraction) {
-  if (!values.length) return null;
-  const sorted = [...values].sort((left, right) => left - right);
-  const rank = Math.max(0, Math.ceil(sorted.length * fraction) - 1);
-  return Number(sorted[Math.min(sorted.length - 1, rank)].toFixed(3));
-}
-
-
-function rate(part, total) {
-  return total ? Number((part / total).toFixed(4)) : 0;
-}
+import { mkdir, writeFile } from "node:fs/promises";
+import { dirname } from "node:path";
+import { readRoutingCases, readSupervisionCases, replayCasePath } from "../src/receipts.mjs";
+import { deterministicSupervisionPolicy } from "../src/supervision.mjs";
+import { routeRequest } from "../src/route.mjs";
+
+const args = parseArgs(process.argv.slice(2));
+const inputPath = replayCasePath(args.input);
+const provider = args.provider ?? process.env.JEV_REPLAY_PROVIDER ?? "demo";
+const allCases = await readRoutingCases(inputPath);
+const cases = args.limit ? allCases.slice(0, args.limit) : allCases;
+const results = [];
+
+for (const routingCase of cases) {
+  const decision = await routeRequest(routingCase.request, { provider });
+  if (decision.execution?.enabled) throw new Error(`replay attempted host execution for ${routingCase.case_id}`);
+  results.push({
+    source_case_id: routingCase.case_id,
+    source_correlation_id: routingCase.correlation_id,
+    replay_correlation_id: decision.correlation_id,
+    surface: routingCase.request.context?.browser ? "browser" : "generic",
+    harness: routingCase.harness,
+    selected: decision.selected,
+    confidence: decision.confidence,
+    status: decision.status,
+    fallback: decision.fallback?.type ?? null,
+    latency_ms: decision.receipt?.latency_ms ?? null,
+    cost_usd: decision.receipt?.cost_usd ?? decision.raw_jev?.usage?.cost ?? null,
+    execution: decision.execution,
+  });
+}
+
+const supervisionCases = args.limit
+  ? (await readSupervisionCases(inputPath)).slice(0, args.limit)
+  : await readSupervisionCases(inputPath);
+const supervisionResults = supervisionCases.map((record) => ({
+  source_case_id: record.case_id,
+  source_correlation_id: record.correlation_id,
+  harness: record.harness,
+  assessment: record.supervision.assessment,
+  recorded_action: record.supervision.action,
+  replayed_policy: deterministicSupervisionPolicy({
+    assessment: record.supervision.assessment,
+    evidence: record.request.context?.evidence ?? {},
+  }),
+  host_execution: false,
+}));
+
+
+const summary = summarize(results);
+summary.supervision_case_count = supervisionResults.length;
+summary.supervision_policy_match_rate = rate(
+  supervisionResults.filter((result) => result.replayed_policy.action === result.recorded_action).length,
+  supervisionResults.length,
+);
+const report = {
+  ok: true,
+  source: inputPath,
+  provider,
+  replayed_cases: results.length,
+  replayed_supervision_cases: supervisionResults.length,
+  host_execution_calls: 0,
+  summary,
+  cases: results,
+  supervision_cases: supervisionResults,
+};
+
+if (args.output) {
+  await mkdir(dirname(args.output), { recursive: true });
+  await writeFile(args.output, `${JSON.stringify(report, null, 2)}\n`, "utf8");
+}
+console.log(JSON.stringify(report, null, 2));
+function summarize(results) {
+  const total = results.length;
+  const fallbackCount = results.filter((result) => result.status === "fallback").length;
+  const selectedCount = results.filter((result) => result.status === "selected" || result.status === "needs_confirmation").length;
+  const noDecisionCount = results.filter((result) => result.status === "no_decision").length;
+  const browserResults = results.filter((result) => result.surface === "browser");
+  const confidences = results.map((result) => result.confidence).filter((value) => typeof value === "number");
+  const latencies = results.map((result) => result.latency_ms).filter((value) => typeof value === "number");
+  const costs = results.map((result) => result.cost_usd).filter((value) => typeof value === "number");
+  return {
+    selected_rate: rate(selectedCount, total),
+    fallback_rate: rate(fallbackCount, total),
+    no_decision_rate: rate(noDecisionCount, total),
+    browser_case_count: browserResults.length,
+    browser_selected_rate: rate(browserResults.filter((result) => result.status === "selected" || result.status === "needs_confirmation").length, browserResults.length),
+    average_confidence: average(confidences),
+    average_latency_ms: average(latencies),
+    p50_latency_ms: percentile(latencies, 0.5),
+    p95_latency_ms: percentile(latencies, 0.95),
+    total_cost_usd: Number(costs.reduce((sum, value) => sum + value, 0).toFixed(9)),
+    status_counts: Object.fromEntries([...new Set(results.map((result) => result.status))].map((status) => [status, results.filter((result) => result.status === status).length])),
+  };
+}
+
+function parseArgs(argv) {
+  const result = {};
+  for (let index = 0; index < argv.length; index += 1) {
+    const value = argv[index];
+    if (value === "--input") result.input = argv[++index];
+    else if (value === "--provider") result.provider = argv[++index];
+    else if (value === "--limit") result.limit = Number(argv[++index]);
+    else if (value === "--output") result.output = argv[++index];
+  }
+  if (result.limit !== undefined && (!Number.isInteger(result.limit) || result.limit < 0)) throw new Error("--limit must be a non-negative integer");
+  return result;
+}
+
+function average(values) {
+  return values.length ? Number((values.reduce((sum, value) => sum + value, 0) / values.length).toFixed(3)) : null;
+}
+
+function percentile(values, fraction) {
+  if (!values.length) return null;
+  const sorted = [...values].sort((left, right) => left - right);
+  const rank = Math.max(0, Math.ceil(sorted.length * fraction) - 1);
+  return Number(sorted[Math.min(sorted.length - 1, rank)].toFixed(3));
+}
+
+
+function rate(part, total) {
+  return total ? Number((part / total).toFixed(4)) : 0;
+}