jev-layer 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +45 -34
- package/CONTRIBUTING.md +146 -146
- package/LICENSE +21 -21
- package/README.md +123 -126
- package/README.ru.md +138 -117
- package/README.zh-CN.md +137 -117
- package/RELEASE.md +53 -53
- package/SECURITY.md +56 -56
- package/bin/jev.mjs +213 -213
- package/config/codex.mcp.toml +8 -8
- package/config/generic-mcp.json +13 -13
- package/config/hermes.mcp.yaml +9 -9
- package/config/jev.example.json +22 -22
- package/config/omp.mcp.json +14 -14
- package/config/providers.env.example +13 -13
- package/docs/AGENT-IMPLEMENTATION.md +250 -0
- package/docs/COMPATIBILITY.md +15 -15
- package/docs/HERMES-LOCAL-AGENT-HANDOFF.ru.md +177 -0
- package/docs/PROVIDERS.md +67 -0
- package/docs/SCHEMA-VERSIONING.md +96 -94
- package/docs/architecture.svg +77 -0
- package/docs/jev-layer-banner.svg +38 -0
- package/examples/capabilities.json +39 -39
- package/examples/route-request.json +28 -28
- package/integrations/codex/.codex-plugin/plugin.json +19 -19
- package/integrations/codex/.mcp.json +11 -11
- package/integrations/codex/AGENTS.md +1 -1
- package/integrations/codex/run-mcp.mjs +9 -9
- package/integrations/codex/skills/jev-route/SKILL.md +17 -17
- package/integrations/hermes/__init__.py +150 -51
- package/integrations/hermes/plugin.yaml +10 -5
- package/integrations/hermes/schemas.py +89 -12
- package/integrations/omp/extension.js +105 -105
- package/integrations/template/README.md +10 -10
- package/integrations/template/adapter.mjs +87 -87
- package/package.json +60 -59
- package/scripts/benchmark.mjs +35 -35
- package/scripts/browser-benchmark.mjs +92 -92
- package/scripts/browser-e2e.mjs +117 -117
- package/scripts/capability-e2e.mjs +31 -31
- package/scripts/clean-install-smoke.mjs +162 -162
- package/scripts/codex-mcp-smoke.mjs +119 -119
- package/scripts/context-filter-e2e.mjs +45 -45
- package/scripts/fail-open-smoke.mjs +120 -120
- package/scripts/feature-flags-smoke.mjs +46 -46
- package/scripts/mcp-receipt-smoke.mjs +113 -98
- package/scripts/model-route-report.mjs +15 -0
- package/scripts/openrouter-choice.mjs +59 -59
- package/scripts/replay-eval.mjs +124 -124
- package/scripts/smoke.mjs +34 -34
- package/scripts/supervision-e2e.mjs +138 -109
- package/src/browser.mjs +571 -569
- package/src/cli.mjs +33 -33
- package/src/config.mjs +85 -75
- package/src/context-filter.mjs +56 -56
- package/src/contract.mjs +72 -72
- package/src/discovery.mjs +65 -65
- package/src/hermes-adapter.mjs +123 -0
- package/src/mcp-server.mjs +267 -210
- package/src/model-route-metrics.mjs +89 -0
- package/src/model-routing.mjs +87 -0
- package/src/providers/demo.mjs +52 -52
- package/src/providers/index.mjs +12 -0
- package/src/providers/typesafe.mjs +126 -126
- package/src/receipts.mjs +227 -226
- package/src/registry.mjs +109 -109
- package/src/relevance-filter.mjs +99 -99
- package/src/route.mjs +219 -221
- package/src/shadow-compaction.mjs +170 -0
- package/src/supervision.mjs +268 -244
- package/test/browser.test.mjs +199 -199
- package/test/capability.test.mjs +54 -54
- package/test/context-filter.test.mjs +65 -65
- package/test/hermes-adapter.test.mjs +62 -0
- package/test/model-route-metrics.test.mjs +61 -0
- package/test/model-routing.test.mjs +59 -0
- package/test/openrouter-provider.test.mjs +55 -55
- package/test/provider-transports.test.mjs +205 -0
- package/test/providers.test.mjs +126 -0
- package/test/receipts.test.mjs +71 -71
- package/test/route.test.mjs +74 -74
- package/test/shadow-compaction.test.mjs +67 -0
- package/test/supervision.test.mjs +117 -99
package/test/browser.test.mjs
CHANGED
|
@@ -1,199 +1,199 @@
|
|
|
1
|
-
import assert from "node:assert/strict";
|
|
2
|
-
import { mkdtemp, readFile } from "node:fs/promises";
|
|
3
|
-
import { tmpdir } from "node:os";
|
|
4
|
-
import { join } from "node:path";
|
|
5
|
-
import { test } from "node:test";
|
|
6
|
-
import { buildBrowserRequest, browserActions, decideBrowserStep, runBrowserFastPath } from "../src/browser.mjs";
|
|
7
|
-
import { readRoutingCases } from "../src/receipts.mjs";
|
|
8
|
-
import { routeRequest } from "../src/route.mjs";
|
|
9
|
-
function providerFor(choices) {
|
|
10
|
-
let call = 0;
|
|
11
|
-
return {
|
|
12
|
-
name: "browser-test-provider",
|
|
13
|
-
async decide({ candidates }) {
|
|
14
|
-
const requested = choices[call++] ?? "browser:handoff";
|
|
15
|
-
const choice = candidates.some((candidate) => candidate.id === requested)
|
|
16
|
-
? requested
|
|
17
|
-
: candidates.find((candidate) => candidate.id === "browser:handoff")?.id ?? candidates[0]?.id;
|
|
18
|
-
return {
|
|
19
|
-
answers: { tool: { type: "choice", choice, probabilities: Object.fromEntries(candidates.map((candidate) => [candidate.id, candidate.id === choice ? 1 : 0])), confidence: 1 } },
|
|
20
|
-
usage: { cost: 0.0001 },
|
|
21
|
-
};
|
|
22
|
-
},
|
|
23
|
-
};
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
test("browser request exposes only bounded structured actions", () => {
|
|
27
|
-
const request = buildBrowserRequest({
|
|
28
|
-
goal: "inspect the visible page",
|
|
29
|
-
observation: {
|
|
30
|
-
url: "https://example.test",
|
|
31
|
-
visible_text: "Visible content",
|
|
32
|
-
scroll: { up: false, down: true },
|
|
33
|
-
targets: [{ id: "submit", name: "Submit", role: "button", clickable: true }, { id: "country", name: "Country", options: [{ id: "de", label: "Germany" }] }],
|
|
34
|
-
tabs: [{ id: "tab-1", title: "Current", active: true }, { id: "tab-2", title: "Other", active: false }],
|
|
35
|
-
},
|
|
36
|
-
});
|
|
37
|
-
assert.ok(request.capabilities.some((candidate) => candidate.id === "browser:scroll:down"));
|
|
38
|
-
assert.ok(request.capabilities.some((candidate) => candidate.id === "browser:click:submit"));
|
|
39
|
-
assert.ok(request.capabilities.some((candidate) => candidate.id === "browser:select:country:de"));
|
|
40
|
-
assert.ok(request.capabilities.some((candidate) => candidate.id === "browser:switch_tab:tab-2"));
|
|
41
|
-
assert.ok(request.capabilities.every((candidate) => candidate.permissions.includes("browser")));
|
|
42
|
-
assert.equal(request.capabilities.some((candidate) => candidate.id.includes("type")), false);
|
|
43
|
-
});
|
|
44
|
-
|
|
45
|
-
test("browser progress removes completed targets and exposes link navigation", () => {
|
|
46
|
-
const request = buildBrowserRequest({
|
|
47
|
-
goal: "click the action then navigate",
|
|
48
|
-
observation: {
|
|
49
|
-
url: "https://example.test/start",
|
|
50
|
-
targets: [
|
|
51
|
-
{ id: "action", role: "button", name: "Action", clickable: true },
|
|
52
|
-
{ id: "next", role: "a", name: "Next", href: "/next", clickable: true },
|
|
53
|
-
],
|
|
54
|
-
},
|
|
55
|
-
progress: {
|
|
56
|
-
executed_actions: [{ id: "browser:click:action", operation: "click", target_id: "action", status: "completed" }],
|
|
57
|
-
used_targets: ["action"],
|
|
58
|
-
current_url: "https://example.test/start",
|
|
59
|
-
last_action: { id: "browser:click:action", operation: "click", target_id: "action", status: "completed" },
|
|
60
|
-
last_result: { status: "completed", state_changed: true },
|
|
61
|
-
},
|
|
62
|
-
});
|
|
63
|
-
assert.equal(request.capabilities.some((candidate) => candidate.id === "browser:click:action"), false);
|
|
64
|
-
assert.ok(request.capabilities.some((candidate) => candidate.id === "browser:navigate:https%3A%2F%2Fexample.test%2Fnext:next"));
|
|
65
|
-
assert.equal(request.context.browser.progress.goal, "click the action then navigate");
|
|
66
|
-
assert.equal(request.context.browser.progress.current_url, "https://example.test/start");
|
|
67
|
-
assert.equal(request.context.browser.progress.last_action.target_id, "action");
|
|
68
|
-
});
|
|
69
|
-
|
|
70
|
-
test("browser progress infers actionable roles and deterministic goal order", () => {
|
|
71
|
-
const request = buildBrowserRequest({
|
|
72
|
-
goal: "Click Increment, then select Beta, then navigate to the next page",
|
|
73
|
-
observation: {
|
|
74
|
-
url: "https://example.test/start",
|
|
75
|
-
targets: [
|
|
76
|
-
{ id: "increment", role: "button", name: "Increment" },
|
|
77
|
-
{ id: "choice", role: "select", name: "Choice", options: [{ id: "beta", value: "beta", label: "Beta" }] },
|
|
78
|
-
{ id: "next", role: "a", name: "Next", href: "/next" },
|
|
79
|
-
],
|
|
80
|
-
},
|
|
81
|
-
});
|
|
82
|
-
assert.deepEqual(request.context.browser.progress.remaining_expected_progress, [
|
|
83
|
-
"browser:click:increment",
|
|
84
|
-
"browser:select:choice:beta",
|
|
85
|
-
"browser:navigate:https%3A%2F%2Fexample.test%2Fnext:next",
|
|
86
|
-
]);
|
|
87
|
-
assert.deepEqual(request.capabilities.map((candidate) => candidate.id), ["browser:click:increment", "browser:handoff"]);
|
|
88
|
-
});
|
|
89
|
-
|
|
90
|
-
test("browser progress only permits repeated scroll after measurable progress", () => {
|
|
91
|
-
const base = {
|
|
92
|
-
goal: "scroll down",
|
|
93
|
-
observation: { url: "https://example.test", scroll: { down: true, position: 100, max: 1000 } },
|
|
94
|
-
};
|
|
95
|
-
const blocked = browserActions({
|
|
96
|
-
...base,
|
|
97
|
-
progress: {
|
|
98
|
-
last_action: { id: "browser:scroll:down", operation: "scroll", direction: "down", status: "completed" },
|
|
99
|
-
last_result: { status: "completed", state_changed: false, scroll_progress: false },
|
|
100
|
-
},
|
|
101
|
-
});
|
|
102
|
-
assert.equal(blocked.some((candidate) => candidate.id === "browser:scroll:down"), false);
|
|
103
|
-
const allowed = browserActions({
|
|
104
|
-
...base,
|
|
105
|
-
progress: {
|
|
106
|
-
last_action: { id: "browser:scroll:down", operation: "scroll", direction: "down", status: "completed" },
|
|
107
|
-
last_result: { status: "completed", state_changed: true, scroll_progress: true },
|
|
108
|
-
},
|
|
109
|
-
});
|
|
110
|
-
assert.ok(allowed.some((candidate) => candidate.id === "browser:scroll:down"));
|
|
111
|
-
});
|
|
112
|
-
|
|
113
|
-
test("browser fast-path is opt-in and fails open without an executor", async () => {
|
|
114
|
-
const disabled = await decideBrowserStep({ goal: "inspect", observation: {} }, { enabled: false, provider: "demo" });
|
|
115
|
-
assert.equal(disabled.decision.status, "fallback");
|
|
116
|
-
assert.equal(disabled.decision.fallback.type, "browser_fast_path_disabled");
|
|
117
|
-
const unavailable = await runBrowserFastPath({ goal: "inspect", enabled: true, provider: "demo" });
|
|
118
|
-
assert.equal(unavailable.status, "handoff");
|
|
119
|
-
assert.equal(unavailable.reason, "browser_executor_unavailable");
|
|
120
|
-
});
|
|
121
|
-
|
|
122
|
-
test("browser fast-path executes safe actions then hands control back", async () => {
|
|
123
|
-
const path = join(await mkdtemp(join(tmpdir(), "jev-browser-")), "cases.jsonl");
|
|
124
|
-
const executed = [];
|
|
125
|
-
const observations = [
|
|
126
|
-
{ url: "https://example.test", scroll: { up: false, down: true }, tabs: [{ id: "tab-1", active: true }, { id: "tab-2", title: "Other", active: false }] },
|
|
127
|
-
{ url: "https://example.test", scroll: { up: false, down: false }, tabs: [{ id: "tab-1", active: true }, { id: "tab-2", title: "Other", active: false }] },
|
|
128
|
-
];
|
|
129
|
-
const executor = {
|
|
130
|
-
async observe() { return observations.shift() ?? observations.at(-1) ?? {}; },
|
|
131
|
-
async execute(action) {
|
|
132
|
-
executed.push(action);
|
|
133
|
-
return { result: { operation: action.operation }, observation: observations.shift() ?? {}, handoff: executed.length === 2 };
|
|
134
|
-
},
|
|
135
|
-
};
|
|
136
|
-
const result = await runBrowserFastPath({
|
|
137
|
-
goal: "scroll and switch to the other tab",
|
|
138
|
-
harness: "browser-test",
|
|
139
|
-
enabled: true,
|
|
140
|
-
provider: providerFor(["browser:scroll:down", "browser:switch_tab:tab-2"]),
|
|
141
|
-
executor,
|
|
142
|
-
receiptPath: path,
|
|
143
|
-
});
|
|
144
|
-
assert.equal(result.status, "handoff");
|
|
145
|
-
assert.equal(result.reason, "host_handoff");
|
|
146
|
-
assert.deepEqual(executed.map((action) => action.operation), ["scroll", "switch_tab"]);
|
|
147
|
-
assert.equal(result.metrics.jev_calls, 2);
|
|
148
|
-
assert.equal(result.metrics.browser_actions, 2);
|
|
149
|
-
assert.equal(result.metrics.failures, 0);
|
|
150
|
-
assert.equal(result.metrics.cost_usd, 0.0002);
|
|
151
|
-
const records = (await readFile(path, "utf8")).trim().split(/\r?\n/).map(JSON.parse);
|
|
152
|
-
assert.equal(records.length, 4);
|
|
153
|
-
assert.equal(records.filter((record) => record.record_type === "routing_case").length, 2);
|
|
154
|
-
assert.equal(records.filter((record) => record.record_type === "execution_receipt").length, 2);
|
|
155
|
-
assert.equal(records[1].host.browser.operation, "scroll");
|
|
156
|
-
assert.equal(records[3].host.browser.operation, "switch_tab");
|
|
157
|
-
const replayCases = await readRoutingCases(path);
|
|
158
|
-
const replayed = await Promise.all(replayCases.map((routingCase) => routeRequest(routingCase.request, { provider: "demo" })));
|
|
159
|
-
assert.equal(replayed.length, 2);
|
|
160
|
-
assert.ok(replayed.every((decision) => decision.execution.enabled === false));
|
|
161
|
-
});
|
|
162
|
-
|
|
163
|
-
test("consequential browser action hands off for host approval", async () => {
|
|
164
|
-
let executeCalls = 0;
|
|
165
|
-
const result = await runBrowserFastPath({
|
|
166
|
-
goal: "click the visible action",
|
|
167
|
-
enabled: true,
|
|
168
|
-
provider: providerFor(["browser:click:action"]),
|
|
169
|
-
executor: {
|
|
170
|
-
async observe() { return { targets: [{ id: "action", name: "Action", clickable: true }] }; },
|
|
171
|
-
async approve() { return false; },
|
|
172
|
-
async execute() { executeCalls += 1; return {}; },
|
|
173
|
-
},
|
|
174
|
-
});
|
|
175
|
-
assert.equal(result.status, "handoff");
|
|
176
|
-
assert.equal(result.reason, "host_confirmation_required");
|
|
177
|
-
assert.equal(executeCalls, 0);
|
|
178
|
-
});
|
|
179
|
-
|
|
180
|
-
test("native select uses the optional host select executor", async () => {
|
|
181
|
-
let selectCalls = 0;
|
|
182
|
-
let executeCalls = 0;
|
|
183
|
-
const result = await runBrowserFastPath({
|
|
184
|
-
goal: "select beta",
|
|
185
|
-
enabled: true,
|
|
186
|
-
maxSteps: 1,
|
|
187
|
-
provider: providerFor(["browser:select:choice:beta"]),
|
|
188
|
-
executor: {
|
|
189
|
-
async observe() { return { targets: [{ id: "choice", role: "select", options: [{ id: "beta", label: "Beta" }] }] }; },
|
|
190
|
-
async approve() { return true; },
|
|
191
|
-
async select(action) { selectCalls += 1; return { result: { value: action.option_id }, observation: { targets: [] } }; },
|
|
192
|
-
async execute() { executeCalls += 1; },
|
|
193
|
-
},
|
|
194
|
-
});
|
|
195
|
-
assert.equal(result.reason, "browser_step_budget_exhausted");
|
|
196
|
-
assert.equal(result.metrics.browser_actions, 1);
|
|
197
|
-
assert.equal(selectCalls, 1);
|
|
198
|
-
assert.equal(executeCalls, 0);
|
|
199
|
-
});
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { mkdtemp, readFile } from "node:fs/promises";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import { test } from "node:test";
|
|
6
|
+
import { buildBrowserRequest, browserActions, decideBrowserStep, runBrowserFastPath } from "../src/browser.mjs";
|
|
7
|
+
import { readRoutingCases } from "../src/receipts.mjs";
|
|
8
|
+
import { routeRequest } from "../src/route.mjs";
|
|
9
|
+
function providerFor(choices) {
|
|
10
|
+
let call = 0;
|
|
11
|
+
return {
|
|
12
|
+
name: "browser-test-provider",
|
|
13
|
+
async decide({ candidates }) {
|
|
14
|
+
const requested = choices[call++] ?? "browser:handoff";
|
|
15
|
+
const choice = candidates.some((candidate) => candidate.id === requested)
|
|
16
|
+
? requested
|
|
17
|
+
: candidates.find((candidate) => candidate.id === "browser:handoff")?.id ?? candidates[0]?.id;
|
|
18
|
+
return {
|
|
19
|
+
answers: { tool: { type: "choice", choice, probabilities: Object.fromEntries(candidates.map((candidate) => [candidate.id, candidate.id === choice ? 1 : 0])), confidence: 1 } },
|
|
20
|
+
usage: { cost: 0.0001 },
|
|
21
|
+
};
|
|
22
|
+
},
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
test("browser request exposes only bounded structured actions", () => {
|
|
27
|
+
const request = buildBrowserRequest({
|
|
28
|
+
goal: "inspect the visible page",
|
|
29
|
+
observation: {
|
|
30
|
+
url: "https://example.test",
|
|
31
|
+
visible_text: "Visible content",
|
|
32
|
+
scroll: { up: false, down: true },
|
|
33
|
+
targets: [{ id: "submit", name: "Submit", role: "button", clickable: true }, { id: "country", name: "Country", options: [{ id: "de", label: "Germany" }] }],
|
|
34
|
+
tabs: [{ id: "tab-1", title: "Current", active: true }, { id: "tab-2", title: "Other", active: false }],
|
|
35
|
+
},
|
|
36
|
+
});
|
|
37
|
+
assert.ok(request.capabilities.some((candidate) => candidate.id === "browser:scroll:down"));
|
|
38
|
+
assert.ok(request.capabilities.some((candidate) => candidate.id === "browser:click:submit"));
|
|
39
|
+
assert.ok(request.capabilities.some((candidate) => candidate.id === "browser:select:country:de"));
|
|
40
|
+
assert.ok(request.capabilities.some((candidate) => candidate.id === "browser:switch_tab:tab-2"));
|
|
41
|
+
assert.ok(request.capabilities.every((candidate) => candidate.permissions.includes("browser")));
|
|
42
|
+
assert.equal(request.capabilities.some((candidate) => candidate.id.includes("type")), false);
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
test("browser progress removes completed targets and exposes link navigation", () => {
|
|
46
|
+
const request = buildBrowserRequest({
|
|
47
|
+
goal: "click the action then navigate",
|
|
48
|
+
observation: {
|
|
49
|
+
url: "https://example.test/start",
|
|
50
|
+
targets: [
|
|
51
|
+
{ id: "action", role: "button", name: "Action", clickable: true },
|
|
52
|
+
{ id: "next", role: "a", name: "Next", href: "/next", clickable: true },
|
|
53
|
+
],
|
|
54
|
+
},
|
|
55
|
+
progress: {
|
|
56
|
+
executed_actions: [{ id: "browser:click:action", operation: "click", target_id: "action", status: "completed" }],
|
|
57
|
+
used_targets: ["action"],
|
|
58
|
+
current_url: "https://example.test/start",
|
|
59
|
+
last_action: { id: "browser:click:action", operation: "click", target_id: "action", status: "completed" },
|
|
60
|
+
last_result: { status: "completed", state_changed: true },
|
|
61
|
+
},
|
|
62
|
+
});
|
|
63
|
+
assert.equal(request.capabilities.some((candidate) => candidate.id === "browser:click:action"), false);
|
|
64
|
+
assert.ok(request.capabilities.some((candidate) => candidate.id === "browser:navigate:https%3A%2F%2Fexample.test%2Fnext:next"));
|
|
65
|
+
assert.equal(request.context.browser.progress.goal, "click the action then navigate");
|
|
66
|
+
assert.equal(request.context.browser.progress.current_url, "https://example.test/start");
|
|
67
|
+
assert.equal(request.context.browser.progress.last_action.target_id, "action");
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
test("browser progress infers actionable roles and deterministic goal order", () => {
|
|
71
|
+
const request = buildBrowserRequest({
|
|
72
|
+
goal: "Click Increment, then select Beta, then navigate to the next page",
|
|
73
|
+
observation: {
|
|
74
|
+
url: "https://example.test/start",
|
|
75
|
+
targets: [
|
|
76
|
+
{ id: "increment", role: "button", name: "Increment" },
|
|
77
|
+
{ id: "choice", role: "select", name: "Choice", options: [{ id: "beta", value: "beta", label: "Beta" }] },
|
|
78
|
+
{ id: "next", role: "a", name: "Next", href: "/next" },
|
|
79
|
+
],
|
|
80
|
+
},
|
|
81
|
+
});
|
|
82
|
+
assert.deepEqual(request.context.browser.progress.remaining_expected_progress, [
|
|
83
|
+
"browser:click:increment",
|
|
84
|
+
"browser:select:choice:beta",
|
|
85
|
+
"browser:navigate:https%3A%2F%2Fexample.test%2Fnext:next",
|
|
86
|
+
]);
|
|
87
|
+
assert.deepEqual(request.capabilities.map((candidate) => candidate.id), ["browser:click:increment", "browser:handoff"]);
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
test("browser progress only permits repeated scroll after measurable progress", () => {
|
|
91
|
+
const base = {
|
|
92
|
+
goal: "scroll down",
|
|
93
|
+
observation: { url: "https://example.test", scroll: { down: true, position: 100, max: 1000 } },
|
|
94
|
+
};
|
|
95
|
+
const blocked = browserActions({
|
|
96
|
+
...base,
|
|
97
|
+
progress: {
|
|
98
|
+
last_action: { id: "browser:scroll:down", operation: "scroll", direction: "down", status: "completed" },
|
|
99
|
+
last_result: { status: "completed", state_changed: false, scroll_progress: false },
|
|
100
|
+
},
|
|
101
|
+
});
|
|
102
|
+
assert.equal(blocked.some((candidate) => candidate.id === "browser:scroll:down"), false);
|
|
103
|
+
const allowed = browserActions({
|
|
104
|
+
...base,
|
|
105
|
+
progress: {
|
|
106
|
+
last_action: { id: "browser:scroll:down", operation: "scroll", direction: "down", status: "completed" },
|
|
107
|
+
last_result: { status: "completed", state_changed: true, scroll_progress: true },
|
|
108
|
+
},
|
|
109
|
+
});
|
|
110
|
+
assert.ok(allowed.some((candidate) => candidate.id === "browser:scroll:down"));
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
test("browser fast-path is opt-in and fails open without an executor", async () => {
|
|
114
|
+
const disabled = await decideBrowserStep({ goal: "inspect", observation: {} }, { enabled: false, provider: "demo" });
|
|
115
|
+
assert.equal(disabled.decision.status, "fallback");
|
|
116
|
+
assert.equal(disabled.decision.fallback.type, "browser_fast_path_disabled");
|
|
117
|
+
const unavailable = await runBrowserFastPath({ goal: "inspect", enabled: true, provider: "demo" });
|
|
118
|
+
assert.equal(unavailable.status, "handoff");
|
|
119
|
+
assert.equal(unavailable.reason, "browser_executor_unavailable");
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
test("browser fast-path executes safe actions then hands control back", async () => {
|
|
123
|
+
const path = join(await mkdtemp(join(tmpdir(), "jev-browser-")), "cases.jsonl");
|
|
124
|
+
const executed = [];
|
|
125
|
+
const observations = [
|
|
126
|
+
{ url: "https://example.test", scroll: { up: false, down: true }, tabs: [{ id: "tab-1", active: true }, { id: "tab-2", title: "Other", active: false }] },
|
|
127
|
+
{ url: "https://example.test", scroll: { up: false, down: false }, tabs: [{ id: "tab-1", active: true }, { id: "tab-2", title: "Other", active: false }] },
|
|
128
|
+
];
|
|
129
|
+
const executor = {
|
|
130
|
+
async observe() { return observations.shift() ?? observations.at(-1) ?? {}; },
|
|
131
|
+
async execute(action) {
|
|
132
|
+
executed.push(action);
|
|
133
|
+
return { result: { operation: action.operation }, observation: observations.shift() ?? {}, handoff: executed.length === 2 };
|
|
134
|
+
},
|
|
135
|
+
};
|
|
136
|
+
const result = await runBrowserFastPath({
|
|
137
|
+
goal: "scroll and switch to the other tab",
|
|
138
|
+
harness: "browser-test",
|
|
139
|
+
enabled: true,
|
|
140
|
+
provider: providerFor(["browser:scroll:down", "browser:switch_tab:tab-2"]),
|
|
141
|
+
executor,
|
|
142
|
+
receiptPath: path,
|
|
143
|
+
});
|
|
144
|
+
assert.equal(result.status, "handoff");
|
|
145
|
+
assert.equal(result.reason, "host_handoff");
|
|
146
|
+
assert.deepEqual(executed.map((action) => action.operation), ["scroll", "switch_tab"]);
|
|
147
|
+
assert.equal(result.metrics.jev_calls, 2);
|
|
148
|
+
assert.equal(result.metrics.browser_actions, 2);
|
|
149
|
+
assert.equal(result.metrics.failures, 0);
|
|
150
|
+
assert.equal(result.metrics.cost_usd, 0.0002);
|
|
151
|
+
const records = (await readFile(path, "utf8")).trim().split(/\r?\n/).map(JSON.parse);
|
|
152
|
+
assert.equal(records.length, 4);
|
|
153
|
+
assert.equal(records.filter((record) => record.record_type === "routing_case").length, 2);
|
|
154
|
+
assert.equal(records.filter((record) => record.record_type === "execution_receipt").length, 2);
|
|
155
|
+
assert.equal(records[1].host.browser.operation, "scroll");
|
|
156
|
+
assert.equal(records[3].host.browser.operation, "switch_tab");
|
|
157
|
+
const replayCases = await readRoutingCases(path);
|
|
158
|
+
const replayed = await Promise.all(replayCases.map((routingCase) => routeRequest(routingCase.request, { provider: "demo" })));
|
|
159
|
+
assert.equal(replayed.length, 2);
|
|
160
|
+
assert.ok(replayed.every((decision) => decision.execution.enabled === false));
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
test("consequential browser action hands off for host approval", async () => {
|
|
164
|
+
let executeCalls = 0;
|
|
165
|
+
const result = await runBrowserFastPath({
|
|
166
|
+
goal: "click the visible action",
|
|
167
|
+
enabled: true,
|
|
168
|
+
provider: providerFor(["browser:click:action"]),
|
|
169
|
+
executor: {
|
|
170
|
+
async observe() { return { targets: [{ id: "action", name: "Action", clickable: true }] }; },
|
|
171
|
+
async approve() { return false; },
|
|
172
|
+
async execute() { executeCalls += 1; return {}; },
|
|
173
|
+
},
|
|
174
|
+
});
|
|
175
|
+
assert.equal(result.status, "handoff");
|
|
176
|
+
assert.equal(result.reason, "host_confirmation_required");
|
|
177
|
+
assert.equal(executeCalls, 0);
|
|
178
|
+
});
|
|
179
|
+
|
|
180
|
+
test("native select uses the optional host select executor", async () => {
|
|
181
|
+
let selectCalls = 0;
|
|
182
|
+
let executeCalls = 0;
|
|
183
|
+
const result = await runBrowserFastPath({
|
|
184
|
+
goal: "select beta",
|
|
185
|
+
enabled: true,
|
|
186
|
+
maxSteps: 1,
|
|
187
|
+
provider: providerFor(["browser:select:choice:beta"]),
|
|
188
|
+
executor: {
|
|
189
|
+
async observe() { return { targets: [{ id: "choice", role: "select", options: [{ id: "beta", label: "Beta" }] }] }; },
|
|
190
|
+
async approve() { return true; },
|
|
191
|
+
async select(action) { selectCalls += 1; return { result: { value: action.option_id }, observation: { targets: [] } }; },
|
|
192
|
+
async execute() { executeCalls += 1; },
|
|
193
|
+
},
|
|
194
|
+
});
|
|
195
|
+
assert.equal(result.reason, "browser_step_budget_exhausted");
|
|
196
|
+
assert.equal(result.metrics.browser_actions, 1);
|
|
197
|
+
assert.equal(selectCalls, 1);
|
|
198
|
+
assert.equal(executeCalls, 0);
|
|
199
|
+
});
|
package/test/capability.test.mjs
CHANGED
|
@@ -1,54 +1,54 @@
|
|
|
1
|
-
import assert from "node:assert/strict";
|
|
2
|
-
import { test } from "node:test";
|
|
3
|
-
import { discoverCapabilities } from "../src/discovery.mjs";
|
|
4
|
-
import { routeRequest } from "../src/route.mjs";
|
|
5
|
-
|
|
6
|
-
function discovered() {
|
|
7
|
-
return discoverCapabilities({
|
|
8
|
-
skills: [{ id: "skill:inspect", name: "Inspect skill", description: "Inspect work", verified: true }],
|
|
9
|
-
mcp: [{ id: "mcp:search", name: "Search MCP", description: "Search files" }],
|
|
10
|
-
cli: [{ id: "cli:git", name: "Git CLI", command: "git", description: "Run git" }],
|
|
11
|
-
dsh: [{ id: "dsh:worker", name: "Worker DSH", description: "Delegate work" }],
|
|
12
|
-
subagents: [{ id: "subagent:review", name: "Review subagent", description: "Review code" }],
|
|
13
|
-
models: [{ id: "model:jev", name: "Jev model", description: "Make bounded decisions" }],
|
|
14
|
-
tools: [{ id: "tool:read", name: "Read tool", description: "Read files" }],
|
|
15
|
-
});
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
test("discovery normalizes skills, MCP, CLI, DSH, subagent, model, and tool candidates", () => {
|
|
19
|
-
const capabilities = discovered();
|
|
20
|
-
assert.deepEqual(new Set(capabilities.map((capability) => capability.kind)), new Set(["skill", "mcp", "cli", "dsh", "subagent", "model", "tool"]));
|
|
21
|
-
assert.equal(capabilities.find((capability) => capability.id === "cli:git").metadata.cli, "git");
|
|
22
|
-
assert.equal(capabilities.find((capability) => capability.id === "skill:inspect").verified, true);
|
|
23
|
-
});
|
|
24
|
-
|
|
25
|
-
test("explicit deterministic policy selects one discovered capability before Jev", async () => {
|
|
26
|
-
let providerCalls = 0;
|
|
27
|
-
const decision = await routeRequest({
|
|
28
|
-
intent: "make a bounded model decision",
|
|
29
|
-
context: { request: "model" },
|
|
30
|
-
policy: { deterministic: true, prefer_kind: "model", prefer_verified: true },
|
|
31
|
-
}, {
|
|
32
|
-
discovery: { models: [{ id: "model:jev", name: "Jev model", description: "Make bounded decisions", verified: true, risk: "low" }] },
|
|
33
|
-
provider: { name: "must-not-run", async decide() { providerCalls += 1; throw new Error("deterministic route should not call Jev"); } },
|
|
34
|
-
});
|
|
35
|
-
assert.equal(providerCalls, 0);
|
|
36
|
-
assert.equal(decision.status, "selected");
|
|
37
|
-
assert.equal(decision.selected, "model:jev");
|
|
38
|
-
assert.equal(decision.receipt.provider, "must-not-run:deterministic");
|
|
39
|
-
assert.equal(decision.candidates[0].verified, true);
|
|
40
|
-
});
|
|
41
|
-
|
|
42
|
-
test("availability policy excludes an unavailable discovered capability", async () => {
|
|
43
|
-
const decision = await routeRequest({
|
|
44
|
-
intent: "inspect",
|
|
45
|
-
context: {},
|
|
46
|
-
policy: { deterministic: true, deterministic_capability_id: "tool:offline" },
|
|
47
|
-
}, {
|
|
48
|
-
discovery: { tools: [{ id: "tool:offline", name: "Offline", description: "Unavailable", available: false }] },
|
|
49
|
-
provider: "demo",
|
|
50
|
-
});
|
|
51
|
-
assert.equal(decision.status, "no_decision");
|
|
52
|
-
assert.equal(decision.candidates[0].filtered, true);
|
|
53
|
-
assert.equal(decision.candidates[0].filter_reason, "capability unavailable");
|
|
54
|
-
});
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { test } from "node:test";
|
|
3
|
+
import { discoverCapabilities } from "../src/discovery.mjs";
|
|
4
|
+
import { routeRequest } from "../src/route.mjs";
|
|
5
|
+
|
|
6
|
+
function discovered() {
|
|
7
|
+
return discoverCapabilities({
|
|
8
|
+
skills: [{ id: "skill:inspect", name: "Inspect skill", description: "Inspect work", verified: true }],
|
|
9
|
+
mcp: [{ id: "mcp:search", name: "Search MCP", description: "Search files" }],
|
|
10
|
+
cli: [{ id: "cli:git", name: "Git CLI", command: "git", description: "Run git" }],
|
|
11
|
+
dsh: [{ id: "dsh:worker", name: "Worker DSH", description: "Delegate work" }],
|
|
12
|
+
subagents: [{ id: "subagent:review", name: "Review subagent", description: "Review code" }],
|
|
13
|
+
models: [{ id: "model:jev", name: "Jev model", description: "Make bounded decisions" }],
|
|
14
|
+
tools: [{ id: "tool:read", name: "Read tool", description: "Read files" }],
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
test("discovery normalizes skills, MCP, CLI, DSH, subagent, model, and tool candidates", () => {
|
|
19
|
+
const capabilities = discovered();
|
|
20
|
+
assert.deepEqual(new Set(capabilities.map((capability) => capability.kind)), new Set(["skill", "mcp", "cli", "dsh", "subagent", "model", "tool"]));
|
|
21
|
+
assert.equal(capabilities.find((capability) => capability.id === "cli:git").metadata.cli, "git");
|
|
22
|
+
assert.equal(capabilities.find((capability) => capability.id === "skill:inspect").verified, true);
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
test("explicit deterministic policy selects one discovered capability before Jev", async () => {
|
|
26
|
+
let providerCalls = 0;
|
|
27
|
+
const decision = await routeRequest({
|
|
28
|
+
intent: "make a bounded model decision",
|
|
29
|
+
context: { request: "model" },
|
|
30
|
+
policy: { deterministic: true, prefer_kind: "model", prefer_verified: true },
|
|
31
|
+
}, {
|
|
32
|
+
discovery: { models: [{ id: "model:jev", name: "Jev model", description: "Make bounded decisions", verified: true, risk: "low" }] },
|
|
33
|
+
provider: { name: "must-not-run", async decide() { providerCalls += 1; throw new Error("deterministic route should not call Jev"); } },
|
|
34
|
+
});
|
|
35
|
+
assert.equal(providerCalls, 0);
|
|
36
|
+
assert.equal(decision.status, "selected");
|
|
37
|
+
assert.equal(decision.selected, "model:jev");
|
|
38
|
+
assert.equal(decision.receipt.provider, "must-not-run:deterministic");
|
|
39
|
+
assert.equal(decision.candidates[0].verified, true);
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
test("availability policy excludes an unavailable discovered capability", async () => {
|
|
43
|
+
const decision = await routeRequest({
|
|
44
|
+
intent: "inspect",
|
|
45
|
+
context: {},
|
|
46
|
+
policy: { deterministic: true, deterministic_capability_id: "tool:offline" },
|
|
47
|
+
}, {
|
|
48
|
+
discovery: { tools: [{ id: "tool:offline", name: "Offline", description: "Unavailable", available: false }] },
|
|
49
|
+
provider: "demo",
|
|
50
|
+
});
|
|
51
|
+
assert.equal(decision.status, "no_decision");
|
|
52
|
+
assert.equal(decision.candidates[0].filtered, true);
|
|
53
|
+
assert.equal(decision.candidates[0].filter_reason, "capability unavailable");
|
|
54
|
+
});
|
|
@@ -1,65 +1,65 @@
|
|
|
1
|
-
import assert from "node:assert/strict";
|
|
2
|
-
import { test } from "node:test";
|
|
3
|
-
import { filterContext } from "../src/relevance-filter.mjs";
|
|
4
|
-
import { projectState } from "../src/context-filter.mjs";
|
|
5
|
-
import { routeRequest } from "../src/route.mjs";
|
|
6
|
-
|
|
7
|
-
const capabilities = [{ id: "inspect", kind: "tool", name: "Inspect", description: "Inspect the supplied context", risk: "low" }];
|
|
8
|
-
|
|
9
|
-
function messages() {
|
|
10
|
-
return [
|
|
11
|
-
...Array.from({ length: 10 }, (_, index) => `old unrelated note ${index}`),
|
|
12
|
-
"Requirement: preserve /workspace/project/config.json exactly",
|
|
13
|
-
"command: `npm test` failed with exit status 1; Error: fixture failed",
|
|
14
|
-
"recent worker update",
|
|
15
|
-
];
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
test("shadow mode never mutates context and reports evidence", () => {
|
|
19
|
-
const context = { messages: messages() };
|
|
20
|
-
const before = JSON.stringify(context);
|
|
21
|
-
const result = filterContext(context, { mode: "shadow", recent: 2 });
|
|
22
|
-
assert.strictEqual(result.context, context);
|
|
23
|
-
assert.equal(JSON.stringify(context), before);
|
|
24
|
-
assert.equal(result.report.mode, "shadow");
|
|
25
|
-
assert.equal(result.report.changed, false);
|
|
26
|
-
assert.ok(result.report.pinned >= 2);
|
|
27
|
-
});
|
|
28
|
-
|
|
29
|
-
test("conservative mode drops stale noise but preserves exact evidence", () => {
|
|
30
|
-
const result = filterContext({ messages: messages() }, { mode: "conservative", recent: 2 });
|
|
31
|
-
assert.equal(result.report.mode, "conservative");
|
|
32
|
-
assert.equal(result.report.changed, true);
|
|
33
|
-
assert.ok(result.report.dropped > 0);
|
|
34
|
-
assert.ok(result.context.messages.includes("Requirement: preserve /workspace/project/config.json exactly"));
|
|
35
|
-
assert.ok(result.context.messages.includes("command: `npm test` failed with exit status 1; Error: fixture failed"));
|
|
36
|
-
assert.ok(!result.context.messages.includes("old unrelated note 0"));
|
|
37
|
-
});
|
|
38
|
-
|
|
39
|
-
test("route projection applies explicit filtering without losing pinned evidence", async () => {
|
|
40
|
-
let captured;
|
|
41
|
-
const decision = await routeRequest({
|
|
42
|
-
intent: "inspect project state",
|
|
43
|
-
policy: { context_filter_mode: "conservative" },
|
|
44
|
-
context: { messages: messages() },
|
|
45
|
-
capabilities,
|
|
46
|
-
}, {
|
|
47
|
-
provider: {
|
|
48
|
-
name: "context-filter-test-provider",
|
|
49
|
-
async decide({ state }) {
|
|
50
|
-
captured = state;
|
|
51
|
-
return { answers: { tool: { type: "choice", choice: "inspect", probabilities: { inspect: 1 }, confidence: 1 } } };
|
|
52
|
-
},
|
|
53
|
-
},
|
|
54
|
-
});
|
|
55
|
-
assert.equal(decision.status, "selected");
|
|
56
|
-
assert.equal(captured.context_filter.mode, "conservative");
|
|
57
|
-
assert.ok(captured.context.messages.includes("Requirement: preserve /workspace/project/config.json exactly"));
|
|
58
|
-
assert.ok(captured.context.messages.includes("command: `npm test` failed with exit status 1; Error: fixture failed"));
|
|
59
|
-
});
|
|
60
|
-
|
|
61
|
-
test("no mode keeps the existing projection contract", () => {
|
|
62
|
-
const result = projectState({ intent: "inspect", context: { messages: ["plain"] } }, capabilities, 6_000);
|
|
63
|
-
assert.equal(result.state.context_filter, undefined);
|
|
64
|
-
assert.deepEqual(result.state.context, { messages: ["plain"] });
|
|
65
|
-
});
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { test } from "node:test";
|
|
3
|
+
import { filterContext } from "../src/relevance-filter.mjs";
|
|
4
|
+
import { projectState } from "../src/context-filter.mjs";
|
|
5
|
+
import { routeRequest } from "../src/route.mjs";
|
|
6
|
+
|
|
7
|
+
const capabilities = [{ id: "inspect", kind: "tool", name: "Inspect", description: "Inspect the supplied context", risk: "low" }];
|
|
8
|
+
|
|
9
|
+
function messages() {
|
|
10
|
+
return [
|
|
11
|
+
...Array.from({ length: 10 }, (_, index) => `old unrelated note ${index}`),
|
|
12
|
+
"Requirement: preserve /workspace/project/config.json exactly",
|
|
13
|
+
"command: `npm test` failed with exit status 1; Error: fixture failed",
|
|
14
|
+
"recent worker update",
|
|
15
|
+
];
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
test("shadow mode never mutates context and reports evidence", () => {
|
|
19
|
+
const context = { messages: messages() };
|
|
20
|
+
const before = JSON.stringify(context);
|
|
21
|
+
const result = filterContext(context, { mode: "shadow", recent: 2 });
|
|
22
|
+
assert.strictEqual(result.context, context);
|
|
23
|
+
assert.equal(JSON.stringify(context), before);
|
|
24
|
+
assert.equal(result.report.mode, "shadow");
|
|
25
|
+
assert.equal(result.report.changed, false);
|
|
26
|
+
assert.ok(result.report.pinned >= 2);
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
test("conservative mode drops stale noise but preserves exact evidence", () => {
|
|
30
|
+
const result = filterContext({ messages: messages() }, { mode: "conservative", recent: 2 });
|
|
31
|
+
assert.equal(result.report.mode, "conservative");
|
|
32
|
+
assert.equal(result.report.changed, true);
|
|
33
|
+
assert.ok(result.report.dropped > 0);
|
|
34
|
+
assert.ok(result.context.messages.includes("Requirement: preserve /workspace/project/config.json exactly"));
|
|
35
|
+
assert.ok(result.context.messages.includes("command: `npm test` failed with exit status 1; Error: fixture failed"));
|
|
36
|
+
assert.ok(!result.context.messages.includes("old unrelated note 0"));
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
test("route projection applies explicit filtering without losing pinned evidence", async () => {
|
|
40
|
+
let captured;
|
|
41
|
+
const decision = await routeRequest({
|
|
42
|
+
intent: "inspect project state",
|
|
43
|
+
policy: { context_filter_mode: "conservative" },
|
|
44
|
+
context: { messages: messages() },
|
|
45
|
+
capabilities,
|
|
46
|
+
}, {
|
|
47
|
+
provider: {
|
|
48
|
+
name: "context-filter-test-provider",
|
|
49
|
+
async decide({ state }) {
|
|
50
|
+
captured = state;
|
|
51
|
+
return { answers: { tool: { type: "choice", choice: "inspect", probabilities: { inspect: 1 }, confidence: 1 } } };
|
|
52
|
+
},
|
|
53
|
+
},
|
|
54
|
+
});
|
|
55
|
+
assert.equal(decision.status, "selected");
|
|
56
|
+
assert.equal(captured.context_filter.mode, "conservative");
|
|
57
|
+
assert.ok(captured.context.messages.includes("Requirement: preserve /workspace/project/config.json exactly"));
|
|
58
|
+
assert.ok(captured.context.messages.includes("command: `npm test` failed with exit status 1; Error: fixture failed"));
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
test("no mode keeps the existing projection contract", () => {
|
|
62
|
+
const result = projectState({ intent: "inspect", context: { messages: ["plain"] } }, capabilities, 6_000);
|
|
63
|
+
assert.equal(result.state.context_filter, undefined);
|
|
64
|
+
assert.deepEqual(result.state.context, { messages: ["plain"] });
|
|
65
|
+
});
|