jev-layer 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/CONTRIBUTING.md +146 -0
- package/LICENSE +21 -0
- package/README.md +126 -0
- package/README.ru.md +117 -0
- package/README.zh-CN.md +117 -0
- package/RELEASE.md +53 -0
- package/SECURITY.md +56 -0
- package/bin/jev.mjs +214 -0
- package/config/codex.mcp.toml +8 -0
- package/config/generic-mcp.json +13 -0
- package/config/hermes.mcp.yaml +9 -0
- package/config/jev.example.json +22 -0
- package/config/omp.mcp.json +14 -0
- package/config/providers.env.example +13 -0
- package/docs/COMPATIBILITY.md +15 -0
- package/docs/SCHEMA-VERSIONING.md +94 -0
- package/examples/capabilities.json +39 -0
- package/examples/route-request.json +28 -0
- package/integrations/codex/.codex-plugin/plugin.json +19 -0
- package/integrations/codex/.mcp.json +11 -0
- package/integrations/codex/AGENTS.md +1 -0
- package/integrations/codex/run-mcp.mjs +9 -0
- package/integrations/codex/skills/jev-route/SKILL.md +17 -0
- package/integrations/hermes/__init__.py +51 -0
- package/integrations/hermes/plugin.yaml +5 -0
- package/integrations/hermes/schemas.py +12 -0
- package/integrations/omp/extension.js +105 -0
- package/integrations/template/README.md +10 -0
- package/integrations/template/adapter.mjs +87 -0
- package/package.json +59 -0
- package/scripts/benchmark.mjs +35 -0
- package/scripts/browser-benchmark.mjs +92 -0
- package/scripts/browser-e2e.mjs +117 -0
- package/scripts/capability-e2e.mjs +31 -0
- package/scripts/clean-install-smoke.mjs +162 -0
- package/scripts/codex-mcp-smoke.mjs +119 -0
- package/scripts/context-filter-e2e.mjs +45 -0
- package/scripts/fail-open-smoke.mjs +120 -0
- package/scripts/feature-flags-smoke.mjs +46 -0
- package/scripts/mcp-receipt-smoke.mjs +98 -0
- package/scripts/openrouter-choice.mjs +59 -0
- package/scripts/replay-eval.mjs +124 -0
- package/scripts/smoke.mjs +34 -0
- package/scripts/supervision-e2e.mjs +109 -0
- package/src/browser.mjs +569 -0
- package/src/cli.mjs +33 -0
- package/src/config.mjs +75 -0
- package/src/context-filter.mjs +56 -0
- package/src/contract.mjs +72 -0
- package/src/discovery.mjs +65 -0
- package/src/mcp-server.mjs +210 -0
- package/src/providers/demo.mjs +52 -0
- package/src/providers/typesafe.mjs +126 -0
- package/src/receipts.mjs +226 -0
- package/src/registry.mjs +109 -0
- package/src/relevance-filter.mjs +99 -0
- package/src/route.mjs +221 -0
- package/src/supervision.mjs +244 -0
- package/test/browser.test.mjs +199 -0
- package/test/capability.test.mjs +54 -0
- package/test/context-filter.test.mjs +65 -0
- package/test/openrouter-provider.test.mjs +55 -0
- package/test/receipts.test.mjs +71 -0
- package/test/route.test.mjs +74 -0
- package/test/supervision.test.mjs +99 -0
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { cp, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
|
3
|
+
import { execFile, spawn } from "node:child_process";
|
|
4
|
+
import { fileURLToPath } from "node:url";
|
|
5
|
+
import { delimiter, dirname, join, resolve } from "node:path";
|
|
6
|
+
import { promisify } from "node:util";
|
|
7
|
+
import { tmpdir } from "node:os";
|
|
8
|
+
|
|
9
|
+
const execFileAsync = promisify(execFile);
|
|
10
|
+
const sourceRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
|
11
|
+
const stage = await mkdtemp(join(tmpdir(), "jev-clean-install-"));
|
|
12
|
+
const packageRoot = join(stage, "package");
|
|
13
|
+
const workspace = join(stage, "workspace");
|
|
14
|
+
|
|
15
|
+
try {
|
|
16
|
+
await copyDistribution(packageRoot);
|
|
17
|
+
await execFileAsync("npm", ["install", "--ignore-scripts", "--no-audit", "--no-fund"], { cwd: packageRoot, env: process.env });
|
|
18
|
+
await mkdir(workspace, { recursive: true });
|
|
19
|
+
await writeFile(join(workspace, "package.json"), JSON.stringify({ name: "clean-install-workspace", private: true }, null, 2) + "\n", "utf8");
|
|
20
|
+
await execFileAsync("npm", ["install", "--ignore-scripts", "--no-audit", "--no-fund", "--no-save", packageRoot], { cwd: workspace, env: process.env });
|
|
21
|
+
await runNode(packageRoot, ["bin/jev.mjs", "install", "--project", workspace]);
|
|
22
|
+
await runNode(packageRoot, ["bin/jev.mjs", "add", "generic", "--project", workspace]);
|
|
23
|
+
const doctor = await runNode(packageRoot, ["bin/jev.mjs", "doctor", "--project", workspace]);
|
|
24
|
+
const doctorReport = JSON.parse(doctor.stdout);
|
|
25
|
+
assert.equal(doctorReport.ok, true);
|
|
26
|
+
|
|
27
|
+
const cli = await runJev(workspace, ["cli", "--provider", "demo", "--input", join(packageRoot, "examples", "route-request.json")]);
|
|
28
|
+
const cliDecision = JSON.parse(cli.stdout.trim().split(/\r?\n/).at(-1));
|
|
29
|
+
assert.equal(cliDecision.execution.enabled, false);
|
|
30
|
+
assert.ok(cliDecision.correlation_id);
|
|
31
|
+
|
|
32
|
+
const genericConfig = JSON.parse(await readFile(join(workspace, ".jev", "adapters", "generic.json"), "utf8"));
|
|
33
|
+
const genericServer = genericConfig.mcpServers.jev;
|
|
34
|
+
assert.deepEqual(genericServer.args, ["mcp"]);
|
|
35
|
+
const mcpEnv = localEnv(workspace);
|
|
36
|
+
const mcp = spawn(genericServer.command, genericServer.args, {
|
|
37
|
+
cwd: workspace,
|
|
38
|
+
env: { ...mcpEnv, JEV_LAYER_PROVIDER: "demo", JEV_REPLAY_CASES: join(workspace, ".jev", "replay", "cases.jsonl") },
|
|
39
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
40
|
+
});
|
|
41
|
+
const pending = new Map();
|
|
42
|
+
let nextId = 1;
|
|
43
|
+
const stderr = [];
|
|
44
|
+
let buffer = "";
|
|
45
|
+
mcp.stderr.on("data", (chunk) => stderr.push(String(chunk)));
|
|
46
|
+
mcp.stdout.on("data", (chunk) => {
|
|
47
|
+
buffer += String(chunk);
|
|
48
|
+
for (const line of buffer.split(/\r?\n/).slice(0, -1)) {
|
|
49
|
+
if (!line.trim()) continue;
|
|
50
|
+
const message = JSON.parse(line);
|
|
51
|
+
pending.get(message.id)?.(message);
|
|
52
|
+
pending.delete(message.id);
|
|
53
|
+
}
|
|
54
|
+
buffer = buffer.split(/\r?\n/).at(-1) ?? "";
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
const request = (method, params) => new Promise((resolveRequest, reject) => {
|
|
58
|
+
const id = nextId++;
|
|
59
|
+
pending.set(id, resolveRequest);
|
|
60
|
+
mcp.stdin.write(`${JSON.stringify({ jsonrpc: "2.0", id, method, params })}\n`);
|
|
61
|
+
setTimeout(() => {
|
|
62
|
+
if (pending.delete(id)) reject(new Error(`timeout waiting for ${method}`));
|
|
63
|
+
}, 10_000).unref();
|
|
64
|
+
});
|
|
65
|
+
const notify = (method, params) => mcp.stdin.write(`${JSON.stringify({ jsonrpc: "2.0", method, params })}\n`);
|
|
66
|
+
|
|
67
|
+
const initialized = await request("initialize", {});
|
|
68
|
+
assert.equal(initialized.result.serverInfo.name, "jev-layer");
|
|
69
|
+
notify("notifications/initialized", {});
|
|
70
|
+
const listed = await request("tools/list", {});
|
|
71
|
+
assert.ok(listed.result.tools.some((tool) => tool.name === "jev_route"));
|
|
72
|
+
assert.ok(listed.result.tools.some((tool) => tool.name === "jev_record_execution"));
|
|
73
|
+
const routed = await request("tools/call", {
|
|
74
|
+
name: "jev_route",
|
|
75
|
+
arguments: {
|
|
76
|
+
harness: "clean-install",
|
|
77
|
+
intent: "read the installed package manifest",
|
|
78
|
+
actor_permissions: ["read"],
|
|
79
|
+
capabilities: [{
|
|
80
|
+
id: "read_manifest",
|
|
81
|
+
kind: "tool",
|
|
82
|
+
name: "Read package manifest",
|
|
83
|
+
description: "Read package.json without changing files",
|
|
84
|
+
permissions: ["read"],
|
|
85
|
+
risk: "low",
|
|
86
|
+
}],
|
|
87
|
+
},
|
|
88
|
+
});
|
|
89
|
+
const decision = routed.result.structuredContent;
|
|
90
|
+
assert.equal(decision.status, "selected");
|
|
91
|
+
const hostResult = await readFile(join(packageRoot, "package.json"), "utf8");
|
|
92
|
+
const recorded = await request("tools/call", {
|
|
93
|
+
name: "jev_record_execution",
|
|
94
|
+
arguments: {
|
|
95
|
+
correlation_id: decision.correlation_id,
|
|
96
|
+
harness: "clean-install",
|
|
97
|
+
capability_id: decision.selected,
|
|
98
|
+
status: "completed",
|
|
99
|
+
result: { bytes: Buffer.byteLength(hostResult) },
|
|
100
|
+
exit_status: 0,
|
|
101
|
+
duration_ms: 1,
|
|
102
|
+
},
|
|
103
|
+
});
|
|
104
|
+
const receipt = recorded.result.structuredContent;
|
|
105
|
+
assert.equal(receipt.correlation_id, decision.correlation_id);
|
|
106
|
+
assert.equal(receipt.host.exit_status, 0);
|
|
107
|
+
assert.equal(receipt.host.duration_ms, 1);
|
|
108
|
+
mcp.stdin.end();
|
|
109
|
+
await onceExit(mcp);
|
|
110
|
+
assert.equal(stderr.join(""), "");
|
|
111
|
+
|
|
112
|
+
const records = (await readFile(join(workspace, ".jev", "replay", "cases.jsonl"), "utf8"))
|
|
113
|
+
.trim().split(/\r?\n/).map((line) => JSON.parse(line));
|
|
114
|
+
assert.equal(records.length, 2);
|
|
115
|
+
assert.deepEqual(records.map((record) => record.record_type), ["routing_case", "execution_receipt"]);
|
|
116
|
+
assert.equal(records[0].correlation_id, records[1].correlation_id);
|
|
117
|
+
const replay = await runNode(packageRoot, ["scripts/replay-eval.mjs", "--provider", "demo", "--input", join(workspace, ".jev", "replay", "cases.jsonl"), "--limit", "1"]);
|
|
118
|
+
const replayReport = JSON.parse(replay.stdout);
|
|
119
|
+
assert.equal(replayReport.replayed_cases, 1);
|
|
120
|
+
assert.equal(replayReport.host_execution_calls, 0);
|
|
121
|
+
console.log(JSON.stringify({
|
|
122
|
+
ok: true,
|
|
123
|
+
stage,
|
|
124
|
+
package_root: packageRoot,
|
|
125
|
+
workspace,
|
|
126
|
+
install: "npm install",
|
|
127
|
+
cli: { status: cliDecision.status, correlation_id: cliDecision.correlation_id },
|
|
128
|
+
mcp: { tools: listed.result.tools.map((tool) => tool.name), selected: decision.selected, host_exit_status: receipt.host.exit_status },
|
|
129
|
+
receipts: records.length,
|
|
130
|
+
replay_cases: records.filter((record) => record.record_type === "routing_case").length,
|
|
131
|
+
replay: { replayed_cases: replayReport.replayed_cases, host_execution_calls: replayReport.host_execution_calls },
|
|
132
|
+
}, null, 2));
|
|
133
|
+
} finally {
|
|
134
|
+
await rm(stage, { recursive: true, force: true });
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
async function copyDistribution(destination) {
|
|
138
|
+
const entries = ["bin", "config", "examples", "integrations", "scripts", "src", "test", "README.md", "package.json", "package-lock.json"];
|
|
139
|
+
for (const entry of entries) await cp(join(sourceRoot, entry), join(destination, entry), { recursive: true });
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
function runNode(cwd, args) {
|
|
143
|
+
return execFileAsync(process.execPath, args, { cwd, env: process.env, maxBuffer: 2 * 1024 * 1024 });
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function runJev(cwd, args) {
|
|
147
|
+
return execFileAsync("jev", args, { cwd, env: localEnv(cwd), maxBuffer: 2 * 1024 * 1024 });
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function localEnv(workspaceRoot) {
|
|
151
|
+
return {
|
|
152
|
+
...process.env,
|
|
153
|
+
PATH: `${join(workspaceRoot, "node_modules", ".bin")}${delimiter}${process.env.PATH ?? ""}`,
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function onceExit(child) {
|
|
158
|
+
return new Promise((resolveExit, reject) => {
|
|
159
|
+
child.once("error", reject);
|
|
160
|
+
child.once("close", (code) => code === 0 ? resolveExit() : reject(new Error(`MCP exited with ${code}`)));
|
|
161
|
+
});
|
|
162
|
+
}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { readFile } from "node:fs/promises";
|
|
3
|
+
import { spawn } from "node:child_process";
|
|
4
|
+
import { createInterface } from "node:readline";
|
|
5
|
+
import { fileURLToPath } from "node:url";
|
|
6
|
+
import { dirname, resolve } from "node:path";
|
|
7
|
+
|
|
8
|
+
const root = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
|
9
|
+
const runner = resolve(root, "integrations/codex/run-mcp.mjs");
|
|
10
|
+
const child = spawn(process.env.JEV_NODE ?? "node", [runner], {
|
|
11
|
+
cwd: root,
|
|
12
|
+
env: {
|
|
13
|
+
...process.env,
|
|
14
|
+
JEV_LAYER_ROOT: root,
|
|
15
|
+
JEV_LAYER_PROVIDER: "openrouter",
|
|
16
|
+
},
|
|
17
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
18
|
+
});
|
|
19
|
+
const lines = createInterface({ input: child.stdout });
|
|
20
|
+
const pending = new Map();
|
|
21
|
+
const stderr = [];
|
|
22
|
+
child.stderr.on("data", (chunk) => stderr.push(String(chunk)));
|
|
23
|
+
lines.on("line", (line) => {
|
|
24
|
+
if (!line.trim()) return;
|
|
25
|
+
const message = JSON.parse(line);
|
|
26
|
+
const waiter = pending.get(message.id);
|
|
27
|
+
if (waiter) {
|
|
28
|
+
pending.delete(message.id);
|
|
29
|
+
waiter(message);
|
|
30
|
+
}
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
try {
|
|
34
|
+
const initialized = await request(1, "initialize", {});
|
|
35
|
+
assert.equal(initialized.result.serverInfo.name, "jev-layer");
|
|
36
|
+
await notification("notifications/initialized", {});
|
|
37
|
+
const listed = await request(2, "tools/list", {});
|
|
38
|
+
assert.ok(listed.result.tools.some((tool) => tool.name === "jev_route"));
|
|
39
|
+
|
|
40
|
+
const response = await request(3, "tools/call", {
|
|
41
|
+
name: "jev_route",
|
|
42
|
+
arguments: {
|
|
43
|
+
intent: "inspect the routing implementation",
|
|
44
|
+
harness: "codex",
|
|
45
|
+
context: { workspace: "jev-layer", operation: "read-only" },
|
|
46
|
+
actor_permissions: ["read"],
|
|
47
|
+
capabilities: [
|
|
48
|
+
{
|
|
49
|
+
id: "read_route",
|
|
50
|
+
name: "read",
|
|
51
|
+
kind: "tool",
|
|
52
|
+
description: "Read src/route.mjs without modifying it.",
|
|
53
|
+
execution: { mode: "host", target: "src/route.mjs" },
|
|
54
|
+
permissions: ["read"],
|
|
55
|
+
risk: "low",
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
id: "read_manifest",
|
|
59
|
+
name: "read",
|
|
60
|
+
kind: "tool",
|
|
61
|
+
description: "Read package.json without modifying it.",
|
|
62
|
+
execution: { mode: "host", target: "package.json" },
|
|
63
|
+
permissions: ["read"],
|
|
64
|
+
risk: "low",
|
|
65
|
+
},
|
|
66
|
+
],
|
|
67
|
+
},
|
|
68
|
+
});
|
|
69
|
+
const decision = response.result.structuredContent;
|
|
70
|
+
assert.equal(decision.status, "selected");
|
|
71
|
+
assert.ok(decision.selected);
|
|
72
|
+
|
|
73
|
+
const target = decision.selected === "read_route" ? "src/route.mjs" : "package.json";
|
|
74
|
+
const content = await readFile(resolve(root, target), "utf8");
|
|
75
|
+
console.log(JSON.stringify({
|
|
76
|
+
ok: true,
|
|
77
|
+
transport: "codex-mcp-runner",
|
|
78
|
+
tool: "jev_route",
|
|
79
|
+
decision: {
|
|
80
|
+
selected: decision.selected,
|
|
81
|
+
probabilities: decision.probabilities,
|
|
82
|
+
confidence: decision.confidence,
|
|
83
|
+
status: decision.status,
|
|
84
|
+
latency_ms: decision.receipt?.latency_ms ?? null,
|
|
85
|
+
cost_usd: decision.raw_jev?.usage?.cost ?? null,
|
|
86
|
+
fallback: decision.fallback,
|
|
87
|
+
provider: decision.receipt?.provider ?? null,
|
|
88
|
+
},
|
|
89
|
+
host_execution: {
|
|
90
|
+
mode: "adapter-transport-fixture",
|
|
91
|
+
target,
|
|
92
|
+
bytes: Buffer.byteLength(content),
|
|
93
|
+
note: "Codex CLI model execution was not exercised by this fixture.",
|
|
94
|
+
},
|
|
95
|
+
}, null, 2));
|
|
96
|
+
} finally {
|
|
97
|
+
child.kill();
|
|
98
|
+
await new Promise((resolveExit) => child.once("close", resolveExit));
|
|
99
|
+
if (stderr.length > 0) process.stderr.write(stderr.join(""));
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function request(id, method, params) {
|
|
103
|
+
return new Promise((resolveResponse, reject) => {
|
|
104
|
+
const timer = setTimeout(() => {
|
|
105
|
+
pending.delete(id);
|
|
106
|
+
reject(new Error(`MCP request timed out: ${method}`));
|
|
107
|
+
}, 15_000);
|
|
108
|
+
pending.set(id, (message) => {
|
|
109
|
+
clearTimeout(timer);
|
|
110
|
+
resolveResponse(message);
|
|
111
|
+
});
|
|
112
|
+
child.stdin.write(`${JSON.stringify({ jsonrpc: "2.0", id, method, params })}\n`);
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function notification(method, params) {
|
|
117
|
+
child.stdin.write(`${JSON.stringify({ jsonrpc: "2.0", method, params })}\n`);
|
|
118
|
+
return new Promise((resolveNotification) => setImmediate(resolveNotification));
|
|
119
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { routeRequest } from "../src/route.mjs";
|
|
3
|
+
|
|
4
|
+
const messages = [
|
|
5
|
+
...Array.from({ length: 12 }, (_, index) => `stale unrelated worker note ${index}`),
|
|
6
|
+
"Requirement: preserve /workspace/project/config.json exactly",
|
|
7
|
+
"Error: command `npm test` failed with exit status 1",
|
|
8
|
+
"recent worker state",
|
|
9
|
+
];
|
|
10
|
+
const context = { messages };
|
|
11
|
+
const before = JSON.stringify(context);
|
|
12
|
+
let projected;
|
|
13
|
+
const decision = await routeRequest({
|
|
14
|
+
harness: "context-filter-e2e",
|
|
15
|
+
intent: "inspect project state",
|
|
16
|
+
context,
|
|
17
|
+
policy: { context_filter_mode: "conservative" },
|
|
18
|
+
capabilities: [{ id: "inspect", kind: "tool", name: "Inspect", description: "Inspect context", risk: "low" }],
|
|
19
|
+
}, {
|
|
20
|
+
provider: {
|
|
21
|
+
name: "context-filter-e2e-provider",
|
|
22
|
+
async decide({ state }) {
|
|
23
|
+
projected = state;
|
|
24
|
+
return { answers: { tool: { type: "choice", choice: "inspect", probabilities: { inspect: 1 }, confidence: 1 } } };
|
|
25
|
+
},
|
|
26
|
+
},
|
|
27
|
+
});
|
|
28
|
+
assert.equal(decision.status, "selected");
|
|
29
|
+
assert.equal(projected.context_filter.mode, "conservative");
|
|
30
|
+
assert.ok(projected.context_filter.dropped > 0);
|
|
31
|
+
assert.ok(projected.context.messages.includes("Requirement: preserve /workspace/project/config.json exactly"));
|
|
32
|
+
assert.ok(projected.context.messages.includes("Error: command `npm test` failed with exit status 1"));
|
|
33
|
+
assert.equal(JSON.stringify(context), before);
|
|
34
|
+
console.log(JSON.stringify({
|
|
35
|
+
ok: true,
|
|
36
|
+
status: decision.status,
|
|
37
|
+
mode: projected.context_filter.mode,
|
|
38
|
+
kept: projected.context_filter.kept,
|
|
39
|
+
dropped: projected.context_filter.dropped,
|
|
40
|
+
pinned: projected.context_filter.pinned,
|
|
41
|
+
preserved: [
|
|
42
|
+
projected.context.messages.includes("Requirement: preserve /workspace/project/config.json exactly"),
|
|
43
|
+
projected.context.messages.includes("Error: command `npm test` failed with exit status 1"),
|
|
44
|
+
],
|
|
45
|
+
}, null, 2));
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { routeRequest } from "../src/route.mjs";
|
|
3
|
+
|
|
4
|
+
const candidates = [
|
|
5
|
+
{
|
|
6
|
+
id: "read_manifest",
|
|
7
|
+
name: "read",
|
|
8
|
+
kind: "tool",
|
|
9
|
+
description: "Read the repository package manifest without modifying it.",
|
|
10
|
+
execution: { mode: "native", target: "read" },
|
|
11
|
+
permissions: ["read"],
|
|
12
|
+
risk: "low",
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
id: "read_route",
|
|
16
|
+
name: "read",
|
|
17
|
+
kind: "tool",
|
|
18
|
+
description: "Read the routing implementation without modifying it.",
|
|
19
|
+
execution: { mode: "native", target: "read" },
|
|
20
|
+
permissions: ["read"],
|
|
21
|
+
risk: "low",
|
|
22
|
+
},
|
|
23
|
+
];
|
|
24
|
+
|
|
25
|
+
const request = {
|
|
26
|
+
intent: "inspect repository metadata",
|
|
27
|
+
harness: "fail-open-smoke",
|
|
28
|
+
context: { workspace: "jev-layer", operation: "read-only" },
|
|
29
|
+
actor_permissions: ["read"],
|
|
30
|
+
capabilities: candidates,
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
const originalEnabled = process.env.JEV_LAYER_ENABLED;
|
|
34
|
+
const disabled = await withEnv("JEV_LAYER_ENABLED", "0", () => routeRequest(request, { provider: "openrouter" }));
|
|
35
|
+
restoreEnv("JEV_LAYER_ENABLED", originalEnabled);
|
|
36
|
+
assert.equal(disabled.status, "fallback");
|
|
37
|
+
assert.equal(disabled.fallback.type, "disabled");
|
|
38
|
+
assert.equal(disabled.execution.enabled, false);
|
|
39
|
+
|
|
40
|
+
const unavailable = await routeRequest(request, {
|
|
41
|
+
provider: "openrouter",
|
|
42
|
+
openrouter: {
|
|
43
|
+
apiKey: "smoke-token",
|
|
44
|
+
endpoint: "http://127.0.0.1:1/unavailable",
|
|
45
|
+
timeoutMs: 100,
|
|
46
|
+
},
|
|
47
|
+
});
|
|
48
|
+
assert.equal(unavailable.status, "fallback");
|
|
49
|
+
assert.equal(unavailable.fallback.type, "provider_error");
|
|
50
|
+
assert.equal(unavailable.execution.enabled, false);
|
|
51
|
+
|
|
52
|
+
const noDecision = await routeRequest({
|
|
53
|
+
...request,
|
|
54
|
+
capabilities: candidates.map((candidate) => ({
|
|
55
|
+
...candidate,
|
|
56
|
+
availability: { available: false, reason: "host capability intentionally unavailable" },
|
|
57
|
+
})),
|
|
58
|
+
}, { provider: "demo" });
|
|
59
|
+
assert.equal(noDecision.status, "no_decision");
|
|
60
|
+
assert.equal(noDecision.execution.enabled, false);
|
|
61
|
+
|
|
62
|
+
const lowConfidence = await routeRequest(request, {
|
|
63
|
+
provider: {
|
|
64
|
+
name: "low-confidence-smoke-provider",
|
|
65
|
+
async decide() {
|
|
66
|
+
return {
|
|
67
|
+
answers: {
|
|
68
|
+
tool: {
|
|
69
|
+
type: "choice",
|
|
70
|
+
choice: "read_manifest",
|
|
71
|
+
probabilities: { read_manifest: 0.51, read_route: 0.49 },
|
|
72
|
+
confidence: 0.51,
|
|
73
|
+
},
|
|
74
|
+
},
|
|
75
|
+
};
|
|
76
|
+
},
|
|
77
|
+
},
|
|
78
|
+
policy: { min_confidence: 0.8 },
|
|
79
|
+
});
|
|
80
|
+
assert.equal(lowConfidence.status, "fallback");
|
|
81
|
+
assert.equal(lowConfidence.fallback.type, "low_confidence");
|
|
82
|
+
assert.equal(lowConfidence.selected, "read_manifest");
|
|
83
|
+
assert.equal(lowConfidence.execution.enabled, false);
|
|
84
|
+
|
|
85
|
+
console.log(JSON.stringify({
|
|
86
|
+
ok: true,
|
|
87
|
+
scenarios: [
|
|
88
|
+
summarize("disabled", disabled),
|
|
89
|
+
summarize("provider_unavailable", unavailable),
|
|
90
|
+
summarize("no_decision", noDecision),
|
|
91
|
+
summarize("low_confidence", lowConfidence),
|
|
92
|
+
],
|
|
93
|
+
}, null, 2));
|
|
94
|
+
|
|
95
|
+
async function withEnv(name, value, operation) {
|
|
96
|
+
const previous = process.env[name];
|
|
97
|
+
process.env[name] = value;
|
|
98
|
+
try {
|
|
99
|
+
return await operation();
|
|
100
|
+
} finally {
|
|
101
|
+
restoreEnv(name, previous);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function restoreEnv(name, value) {
|
|
106
|
+
if (value === undefined) delete process.env[name];
|
|
107
|
+
else process.env[name] = value;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function summarize(name, result) {
|
|
111
|
+
return {
|
|
112
|
+
name,
|
|
113
|
+
status: result.status,
|
|
114
|
+
selected: result.selected,
|
|
115
|
+
fallback: result.fallback?.type ?? null,
|
|
116
|
+
confidence: result.confidence ?? null,
|
|
117
|
+
latency_ms: result.latency_ms,
|
|
118
|
+
execution: result.execution,
|
|
119
|
+
};
|
|
120
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { loadConfig } from "../src/config.mjs";
|
|
3
|
+
import { decideBrowserStep } from "../src/browser.mjs";
|
|
4
|
+
import { routeRequest } from "../src/route.mjs";
|
|
5
|
+
import { superviseWork } from "../src/supervision.mjs";
|
|
6
|
+
|
|
7
|
+
const { config } = await loadConfig({ cwd: "/tmp/jev-no-config" });
|
|
8
|
+
assert.equal(config.features.browser_fast_path, false);
|
|
9
|
+
assert.equal(config.features.supervision, false);
|
|
10
|
+
assert.equal(config.features.context_filter, null);
|
|
11
|
+
|
|
12
|
+
const browser = await decideBrowserStep({ goal: "scroll down", observation: { scroll: { down: true } } }, {
|
|
13
|
+
provider: { name: "must-not-run", async decide() { throw new Error("disabled browser must not call Jev"); } },
|
|
14
|
+
});
|
|
15
|
+
assert.equal(browser.decision.fallback.type, "browser_fast_path_disabled");
|
|
16
|
+
|
|
17
|
+
const previousSupervision = process.env.JEV_SUPERVISION;
|
|
18
|
+
delete process.env.JEV_SUPERVISION;
|
|
19
|
+
try {
|
|
20
|
+
const supervision = await superviseWork({ provider: { async evaluate() { throw new Error("disabled supervision must not call Jev"); } } });
|
|
21
|
+
assert.equal(supervision.fallback.type, "disabled");
|
|
22
|
+
} finally {
|
|
23
|
+
if (previousSupervision === undefined) delete process.env.JEV_SUPERVISION;
|
|
24
|
+
else process.env.JEV_SUPERVISION = previousSupervision;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
let calls = 0;
|
|
28
|
+
const route = await routeRequest({
|
|
29
|
+
intent: "inspect",
|
|
30
|
+
context: { messages: ["plain context"] },
|
|
31
|
+
capabilities: [{ id: "inspect", kind: "tool", name: "Inspect", description: "Inspect context", risk: "low" }],
|
|
32
|
+
}, {
|
|
33
|
+
provider: { name: "default-route-provider", async decide() {
|
|
34
|
+
calls += 1;
|
|
35
|
+
return { answers: { tool: { type: "choice", choice: "inspect", probabilities: { inspect: 1 }, confidence: 1 } } };
|
|
36
|
+
} },
|
|
37
|
+
});
|
|
38
|
+
assert.equal(route.status, "selected");
|
|
39
|
+
assert.equal(calls, 1);
|
|
40
|
+
console.log(JSON.stringify({
|
|
41
|
+
ok: true,
|
|
42
|
+
browser: browser.decision.fallback.type,
|
|
43
|
+
supervision: "disabled",
|
|
44
|
+
context_filter: "disabled",
|
|
45
|
+
default_route_provider_calls: calls,
|
|
46
|
+
}, null, 2));
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { mkdtemp, readFile } from "node:fs/promises";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { join, resolve } from "node:path";
|
|
5
|
+
import { spawn } from "node:child_process";
|
|
6
|
+
import { createInterface } from "node:readline";
|
|
7
|
+
|
|
8
|
+
const root = resolve(new URL("..", import.meta.url).pathname);
|
|
9
|
+
const casesPath = join(await mkdtemp(join(tmpdir(), "jev-mcp-receipt-")), "cases.jsonl");
|
|
10
|
+
const child = spawn(process.env.JEV_NODE ?? "node", [resolve(root, "src/mcp-server.mjs")], {
|
|
11
|
+
cwd: root,
|
|
12
|
+
env: { ...process.env, JEV_REPLAY_CASES: casesPath },
|
|
13
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
14
|
+
});
|
|
15
|
+
const lines = createInterface({ input: child.stdout });
|
|
16
|
+
const pending = new Map();
|
|
17
|
+
lines.on("line", (line) => {
|
|
18
|
+
if (!line.trim()) return;
|
|
19
|
+
const message = JSON.parse(line);
|
|
20
|
+
const waiter = pending.get(message.id);
|
|
21
|
+
if (waiter) {
|
|
22
|
+
pending.delete(message.id);
|
|
23
|
+
waiter(message);
|
|
24
|
+
}
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
try {
|
|
28
|
+
const initialized = await request(1, "initialize", {});
|
|
29
|
+
assert.equal(initialized.result.serverInfo.name, "jev-layer");
|
|
30
|
+
await notification("notifications/initialized", {});
|
|
31
|
+
const listed = await request(2, "tools/list", {});
|
|
32
|
+
assert.ok(listed.result.tools.some((tool) => tool.name === "jev_route"));
|
|
33
|
+
assert.ok(listed.result.tools.some((tool) => tool.name === "jev_record_execution"));
|
|
34
|
+
|
|
35
|
+
const routed = await request(3, "tools/call", {
|
|
36
|
+
name: "jev_route",
|
|
37
|
+
arguments: {
|
|
38
|
+
provider: "demo",
|
|
39
|
+
harness: "mcp-receipt-smoke",
|
|
40
|
+
intent: "read repository metadata",
|
|
41
|
+
actor_permissions: ["read"],
|
|
42
|
+
capabilities: [{
|
|
43
|
+
id: "read_metadata",
|
|
44
|
+
kind: "tool",
|
|
45
|
+
name: "Read metadata",
|
|
46
|
+
description: "Read repository metadata without modifying files",
|
|
47
|
+
permissions: ["read"],
|
|
48
|
+
risk: "low",
|
|
49
|
+
}],
|
|
50
|
+
},
|
|
51
|
+
});
|
|
52
|
+
const decision = routed.result.structuredContent;
|
|
53
|
+
assert.equal(decision.status, "selected");
|
|
54
|
+
assert.equal(decision.receipt.replay_persisted, true);
|
|
55
|
+
|
|
56
|
+
const recorded = await request(4, "tools/call", {
|
|
57
|
+
name: "jev_record_execution",
|
|
58
|
+
arguments: {
|
|
59
|
+
correlation_id: decision.correlation_id,
|
|
60
|
+
harness: "mcp-receipt-smoke",
|
|
61
|
+
capability_id: decision.selected,
|
|
62
|
+
status: "completed",
|
|
63
|
+
result: { observed: "metadata" },
|
|
64
|
+
exit_status: 0,
|
|
65
|
+
duration_ms: 3.2,
|
|
66
|
+
},
|
|
67
|
+
});
|
|
68
|
+
const receipt = recorded.result.structuredContent;
|
|
69
|
+
assert.equal(receipt.correlation_id, decision.correlation_id);
|
|
70
|
+
assert.equal(receipt.host.exit_status, 0);
|
|
71
|
+
assert.equal(receipt.host.duration_ms, 3.2);
|
|
72
|
+
|
|
73
|
+
const records = (await readFile(casesPath, "utf8")).trim().split(/\r?\n/).map(JSON.parse);
|
|
74
|
+
assert.deepEqual(records.map((record) => record.record_type), ["routing_case", "execution_receipt"]);
|
|
75
|
+
console.log(JSON.stringify({ ok: true, cases_path: casesPath, receipt }, null, 2));
|
|
76
|
+
} finally {
|
|
77
|
+
child.kill();
|
|
78
|
+
await new Promise((resolveExit) => child.once("close", resolveExit));
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function request(id, method, params) {
|
|
82
|
+
return new Promise((resolveResponse, reject) => {
|
|
83
|
+
const timer = setTimeout(() => {
|
|
84
|
+
pending.delete(id);
|
|
85
|
+
reject(new Error(`MCP request timed out: ${method}`));
|
|
86
|
+
}, 15_000);
|
|
87
|
+
pending.set(id, (message) => {
|
|
88
|
+
clearTimeout(timer);
|
|
89
|
+
resolveResponse(message);
|
|
90
|
+
});
|
|
91
|
+
child.stdin.write(`${JSON.stringify({ jsonrpc: "2.0", id, method, params })}\n`);
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function notification(method, params) {
|
|
96
|
+
child.stdin.write(`${JSON.stringify({ jsonrpc: "2.0", method, params })}\n`);
|
|
97
|
+
return new Promise((resolveNotification) => setImmediate(resolveNotification));
|
|
98
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { routeRequest } from "../src/route.mjs";
|
|
2
|
+
|
|
3
|
+
const endpoint = process.env.OPENROUTER_DECISIONS_ENDPOINT ?? "https://openrouter.ai/api/alpha/decisions";
|
|
4
|
+
const model = process.env.OPENROUTER_DECISIONS_MODEL ?? "typesafe/jev-1.13";
|
|
5
|
+
const decision = await routeRequest({
|
|
6
|
+
schema_version: 1,
|
|
7
|
+
harness: "jev-layer-openrouter-smoke",
|
|
8
|
+
intent: "Find where the OpenRouter Decisions provider is implemented without modifying files",
|
|
9
|
+
context: {
|
|
10
|
+
purpose: "one real bounded Choice call",
|
|
11
|
+
secret_note: "must be redacted before provider transmission",
|
|
12
|
+
},
|
|
13
|
+
actor_permissions: ["read", "browser", "terminal:read"],
|
|
14
|
+
capabilities: [
|
|
15
|
+
{
|
|
16
|
+
id: "repo_search",
|
|
17
|
+
kind: "mcp",
|
|
18
|
+
name: "Repository search",
|
|
19
|
+
description: "Search repository files and symbols without changing files",
|
|
20
|
+
permissions: ["read"],
|
|
21
|
+
risk: "low",
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
id: "browser_inspect",
|
|
25
|
+
kind: "mcp",
|
|
26
|
+
name: "Browser inspection",
|
|
27
|
+
description: "Open a webpage and inspect visible browser state",
|
|
28
|
+
permissions: ["browser"],
|
|
29
|
+
risk: "medium",
|
|
30
|
+
},
|
|
31
|
+
{
|
|
32
|
+
id: "terminal_read",
|
|
33
|
+
kind: "cli",
|
|
34
|
+
name: "Read-only terminal",
|
|
35
|
+
description: "Run a read-only shell command and return its output",
|
|
36
|
+
permissions: ["terminal:read"],
|
|
37
|
+
risk: "low",
|
|
38
|
+
},
|
|
39
|
+
],
|
|
40
|
+
}, {
|
|
41
|
+
provider: "openrouter",
|
|
42
|
+
openrouter: { endpoint, model, timeoutMs: 15_000 },
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
const raw = decision.raw_jev ?? {};
|
|
46
|
+
const usage = raw.usage && typeof raw.usage === "object" ? raw.usage : {};
|
|
47
|
+
console.log(JSON.stringify({
|
|
48
|
+
request: { endpoint, model, method: "POST" },
|
|
49
|
+
response: { id: raw.id ?? null, model: raw.model ?? null, provider: raw.provider ?? null },
|
|
50
|
+
status: decision.status,
|
|
51
|
+
selected: decision.selected,
|
|
52
|
+
probabilities: decision.probabilities,
|
|
53
|
+
confidence: decision.confidence,
|
|
54
|
+
latency_ms: decision.receipt.latency_ms,
|
|
55
|
+
cost_usd: typeof usage.cost === "number" ? usage.cost : null,
|
|
56
|
+
usage,
|
|
57
|
+
fallback: decision.fallback,
|
|
58
|
+
execution: decision.execution,
|
|
59
|
+
}, null, 2));
|