jev-layer 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +73 -34
- package/CONTRIBUTING.md +146 -146
- package/LICENSE +21 -21
- package/README.md +125 -126
- package/README.ru.md +140 -117
- package/README.zh-CN.md +139 -117
- package/RELEASE.md +53 -53
- package/SECURITY.md +64 -56
- package/bin/jev.mjs +215 -213
- package/config/codex.mcp.toml +8 -8
- package/config/generic-mcp.json +13 -13
- package/config/hermes.mcp.yaml +9 -9
- package/config/jev.example.json +23 -22
- package/config/omp.mcp.json +14 -14
- package/config/providers.env.example +13 -13
- package/docs/AGENT-IMPLEMENTATION.md +306 -0
- package/docs/COMPATIBILITY.md +24 -15
- package/docs/HERMES-LOCAL-AGENT-HANDOFF.ru.md +187 -0
- package/docs/PROVIDERS.md +109 -0
- package/docs/SCHEMA-VERSIONING.md +101 -94
- package/docs/architecture.svg +77 -0
- package/docs/jev-layer-banner.svg +38 -0
- package/examples/capabilities.json +39 -39
- package/examples/route-request.json +28 -28
- package/integrations/codex/.codex-plugin/plugin.json +19 -19
- package/integrations/codex/.mcp.json +11 -11
- package/integrations/codex/AGENTS.md +1 -1
- package/integrations/codex/run-mcp.mjs +9 -9
- package/integrations/codex/skills/jev-route/SKILL.md +17 -17
- package/integrations/hermes/__init__.py +156 -51
- package/integrations/hermes/plugin.yaml +11 -5
- package/integrations/hermes/schemas.py +120 -12
- package/integrations/omp/extension.js +106 -105
- package/integrations/template/README.md +10 -10
- package/integrations/template/adapter.mjs +87 -87
- package/package.json +60 -59
- package/scripts/benchmark.mjs +35 -35
- package/scripts/browser-benchmark.mjs +92 -92
- package/scripts/browser-e2e.mjs +117 -117
- package/scripts/capability-e2e.mjs +31 -31
- package/scripts/clean-install-smoke.mjs +176 -162
- package/scripts/codex-mcp-smoke.mjs +119 -119
- package/scripts/context-filter-e2e.mjs +45 -45
- package/scripts/fail-open-smoke.mjs +120 -120
- package/scripts/feature-flags-smoke.mjs +46 -46
- package/scripts/mcp-receipt-smoke.mjs +115 -98
- package/scripts/model-route-report.mjs +15 -0
- package/scripts/openrouter-choice.mjs +59 -59
- package/scripts/replay-eval.mjs +124 -124
- package/scripts/smoke.mjs +34 -34
- package/scripts/supervision-e2e.mjs +138 -109
- package/src/browser.mjs +583 -569
- package/src/cli.mjs +33 -33
- package/src/computer.mjs +196 -0
- package/src/config.mjs +86 -75
- package/src/context-filter.mjs +56 -56
- package/src/contract.mjs +74 -72
- package/src/discovery.mjs +65 -65
- package/src/hermes-adapter.mjs +134 -0
- package/src/images.mjs +98 -0
- package/src/mcp-server.mjs +313 -210
- package/src/model-route-metrics.mjs +89 -0
- package/src/model-routing.mjs +87 -0
- package/src/providers/demo.mjs +52 -52
- package/src/providers/index.mjs +12 -0
- package/src/providers/typesafe.mjs +206 -126
- package/src/receipts.mjs +228 -226
- package/src/registry.mjs +109 -109
- package/src/relevance-filter.mjs +99 -99
- package/src/route.mjs +228 -221
- package/src/shadow-compaction.mjs +170 -0
- package/src/supervision.mjs +268 -244
- package/test/browser.test.mjs +222 -199
- package/test/capability.test.mjs +54 -54
- package/test/computer.test.mjs +195 -0
- package/test/context-filter.test.mjs +65 -65
- package/test/hermes-adapter.test.mjs +63 -0
- package/test/image-fixture.mjs +2 -0
- package/test/images.test.mjs +85 -0
- package/test/model-route-metrics.test.mjs +61 -0
- package/test/model-routing.test.mjs +59 -0
- package/test/openrouter-provider.test.mjs +55 -55
- package/test/provider-transports.test.mjs +205 -0
- package/test/providers.test.mjs +125 -0
- package/test/receipts.test.mjs +71 -71
- package/test/route.test.mjs +86 -74
- package/test/shadow-compaction.test.mjs +67 -0
- package/test/supervision.test.mjs +117 -99
|
@@ -1,87 +1,87 @@
|
|
|
1
|
-
/*
|
|
2
|
-
* Copy this file to integrations/<your-harness>/adapter.mjs.
|
|
3
|
-
* The three injected functions are the only Jev/host boundaries:
|
|
4
|
-
* jevRoute(request) -> decision from jev_route
|
|
5
|
-
* jevRecordExecution(args) -> execution_receipt from jev_record_execution
|
|
6
|
-
* executeNative(capabilityId, request) -> host-owned result
|
|
7
|
-
*/
|
|
8
|
-
|
|
9
|
-
export function createTemplateAdapter({ jevRoute, jevRecordExecution, executeNative, approve = async () => true }) {
|
|
10
|
-
for (const [name, value] of Object.entries({ jevRoute, jevRecordExecution, executeNative, approve })) {
|
|
11
|
-
if (typeof value !== "function") throw new TypeError(`${name} callback is required`);
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
return {
|
|
15
|
-
async run(request) {
|
|
16
|
-
let decision;
|
|
17
|
-
try {
|
|
18
|
-
decision = await jevRoute(request);
|
|
19
|
-
} catch (error) {
|
|
20
|
-
return nativeFallback("jev_unavailable", error);
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
if (decision?.status !== "selected" || typeof decision.selected !== "string" || decision.selected === "") {
|
|
24
|
-
return { status: "native_fallback", reason: decision?.reason ?? "no_selection", decision };
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
const approval = await approve(decision.selected, decision, request);
|
|
28
|
-
if (!approval) {
|
|
29
|
-
const receipt = await record(jevRecordExecution, {
|
|
30
|
-
correlation_id: decision.correlation_id,
|
|
31
|
-
harness: request.harness,
|
|
32
|
-
capability_id: decision.selected,
|
|
33
|
-
status: "not_started",
|
|
34
|
-
error: "host approval denied",
|
|
35
|
-
});
|
|
36
|
-
return { status: "native_fallback", reason: "host_confirmation_required", decision, receipt };
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
const started = Date.now();
|
|
40
|
-
let host;
|
|
41
|
-
try {
|
|
42
|
-
host = await executeNative(decision.selected, request, decision);
|
|
43
|
-
} catch (error) {
|
|
44
|
-
const receipt = await record(jevRecordExecution, {
|
|
45
|
-
correlation_id: decision.correlation_id,
|
|
46
|
-
harness: request.harness,
|
|
47
|
-
capability_id: decision.selected,
|
|
48
|
-
status: "failed",
|
|
49
|
-
error: boundedError(error),
|
|
50
|
-
duration_ms: Date.now() - started,
|
|
51
|
-
});
|
|
52
|
-
return { status: "native_fallback", reason: "host_execution_failed", decision, error: boundedError(error), receipt };
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
const receipt = await record(jevRecordExecution, {
|
|
56
|
-
correlation_id: decision.correlation_id,
|
|
57
|
-
harness: request.harness,
|
|
58
|
-
capability_id: decision.selected,
|
|
59
|
-
status: "completed",
|
|
60
|
-
result: host,
|
|
61
|
-
duration_ms: Date.now() - started,
|
|
62
|
-
});
|
|
63
|
-
return { status: "completed", decision, host, receipt };
|
|
64
|
-
},
|
|
65
|
-
};
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
async function record(jevRecordExecution, args) {
|
|
69
|
-
try {
|
|
70
|
-
return await jevRecordExecution(args);
|
|
71
|
-
} catch (error) {
|
|
72
|
-
// Execution ownership stays with the host even if receipt persistence fails.
|
|
73
|
-
return { persisted: false, error: boundedError(error) };
|
|
74
|
-
}
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
function nativeFallback(reason, error) {
|
|
78
|
-
return {
|
|
79
|
-
status: "native_fallback",
|
|
80
|
-
reason,
|
|
81
|
-
error: error ? boundedError(error) : null,
|
|
82
|
-
};
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
function boundedError(error) {
|
|
86
|
-
return String(error instanceof Error ? error.message : error).slice(0, 2_000);
|
|
87
|
-
}
|
|
1
|
+
/*
|
|
2
|
+
* Copy this file to integrations/<your-harness>/adapter.mjs.
|
|
3
|
+
* The three injected functions are the only Jev/host boundaries:
|
|
4
|
+
* jevRoute(request) -> decision from jev_route
|
|
5
|
+
* jevRecordExecution(args) -> execution_receipt from jev_record_execution
|
|
6
|
+
* executeNative(capabilityId, request) -> host-owned result
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
export function createTemplateAdapter({ jevRoute, jevRecordExecution, executeNative, approve = async () => true }) {
|
|
10
|
+
for (const [name, value] of Object.entries({ jevRoute, jevRecordExecution, executeNative, approve })) {
|
|
11
|
+
if (typeof value !== "function") throw new TypeError(`${name} callback is required`);
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
return {
|
|
15
|
+
async run(request) {
|
|
16
|
+
let decision;
|
|
17
|
+
try {
|
|
18
|
+
decision = await jevRoute(request);
|
|
19
|
+
} catch (error) {
|
|
20
|
+
return nativeFallback("jev_unavailable", error);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
if (decision?.status !== "selected" || typeof decision.selected !== "string" || decision.selected === "") {
|
|
24
|
+
return { status: "native_fallback", reason: decision?.reason ?? "no_selection", decision };
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const approval = await approve(decision.selected, decision, request);
|
|
28
|
+
if (!approval) {
|
|
29
|
+
const receipt = await record(jevRecordExecution, {
|
|
30
|
+
correlation_id: decision.correlation_id,
|
|
31
|
+
harness: request.harness,
|
|
32
|
+
capability_id: decision.selected,
|
|
33
|
+
status: "not_started",
|
|
34
|
+
error: "host approval denied",
|
|
35
|
+
});
|
|
36
|
+
return { status: "native_fallback", reason: "host_confirmation_required", decision, receipt };
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const started = Date.now();
|
|
40
|
+
let host;
|
|
41
|
+
try {
|
|
42
|
+
host = await executeNative(decision.selected, request, decision);
|
|
43
|
+
} catch (error) {
|
|
44
|
+
const receipt = await record(jevRecordExecution, {
|
|
45
|
+
correlation_id: decision.correlation_id,
|
|
46
|
+
harness: request.harness,
|
|
47
|
+
capability_id: decision.selected,
|
|
48
|
+
status: "failed",
|
|
49
|
+
error: boundedError(error),
|
|
50
|
+
duration_ms: Date.now() - started,
|
|
51
|
+
});
|
|
52
|
+
return { status: "native_fallback", reason: "host_execution_failed", decision, error: boundedError(error), receipt };
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const receipt = await record(jevRecordExecution, {
|
|
56
|
+
correlation_id: decision.correlation_id,
|
|
57
|
+
harness: request.harness,
|
|
58
|
+
capability_id: decision.selected,
|
|
59
|
+
status: "completed",
|
|
60
|
+
result: host,
|
|
61
|
+
duration_ms: Date.now() - started,
|
|
62
|
+
});
|
|
63
|
+
return { status: "completed", decision, host, receipt };
|
|
64
|
+
},
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
async function record(jevRecordExecution, args) {
|
|
69
|
+
try {
|
|
70
|
+
return await jevRecordExecution(args);
|
|
71
|
+
} catch (error) {
|
|
72
|
+
// Execution ownership stays with the host even if receipt persistence fails.
|
|
73
|
+
return { persisted: false, error: boundedError(error) };
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function nativeFallback(reason, error) {
|
|
78
|
+
return {
|
|
79
|
+
status: "native_fallback",
|
|
80
|
+
reason,
|
|
81
|
+
error: error ? boundedError(error) : null,
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function boundedError(error) {
|
|
86
|
+
return String(error instanceof Error ? error.message : error).slice(0, 2_000);
|
|
87
|
+
}
|
package/package.json
CHANGED
|
@@ -1,59 +1,60 @@
|
|
|
1
|
-
{
|
|
2
|
-
"name": "jev-layer",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"license": "MIT",
|
|
5
|
-
"repository": {
|
|
6
|
-
"type": "git",
|
|
7
|
-
"url": "git+https://github.com/typakon4/jev-layer.git"
|
|
8
|
-
},
|
|
9
|
-
"bugs": {
|
|
10
|
-
"url": "https://github.com/typakon4/jev-layer/issues"
|
|
11
|
-
},
|
|
12
|
-
"homepage": "https://github.com/typakon4/jev-layer#readme",
|
|
13
|
-
"type": "module",
|
|
14
|
-
"bin": {
|
|
15
|
-
"jev": "bin/jev.mjs"
|
|
16
|
-
},
|
|
17
|
-
"files": [
|
|
18
|
-
"bin/",
|
|
19
|
-
"config/",
|
|
20
|
-
"docs/",
|
|
21
|
-
"examples/",
|
|
22
|
-
"integrations/",
|
|
23
|
-
"scripts/",
|
|
24
|
-
"src/",
|
|
25
|
-
"test/",
|
|
26
|
-
"CHANGELOG.md",
|
|
27
|
-
"CONTRIBUTING.md",
|
|
28
|
-
"LICENSE",
|
|
29
|
-
"README.md",
|
|
30
|
-
"README.ru.md",
|
|
31
|
-
"README.zh-CN.md",
|
|
32
|
-
"RELEASE.md",
|
|
33
|
-
"SECURITY.md",
|
|
34
|
-
"package.json",
|
|
35
|
-
"package-lock.json"
|
|
36
|
-
],
|
|
37
|
-
"engines": {
|
|
38
|
-
"node": ">=20"
|
|
39
|
-
},
|
|
40
|
-
"scripts": {
|
|
41
|
-
"jev": "node bin/jev.mjs",
|
|
42
|
-
"test": "node --test test/*.test.mjs",
|
|
43
|
-
"smoke": "node scripts/smoke.mjs",
|
|
44
|
-
"benchmark": "node scripts/benchmark.mjs",
|
|
45
|
-
"openrouter:choice": "node scripts/openrouter-choice.mjs",
|
|
46
|
-
"mcp": "node src/mcp-server.mjs",
|
|
47
|
-
"fail-open-smoke": "node scripts/fail-open-smoke.mjs",
|
|
48
|
-
"codex:mcp-smoke": "node scripts/codex-mcp-smoke.mjs",
|
|
49
|
-
"replay:evaluate": "node scripts/replay-eval.mjs",
|
|
50
|
-
"receipt:mcp-smoke": "node scripts/mcp-receipt-smoke.mjs",
|
|
51
|
-
"
|
|
52
|
-
"browser:
|
|
53
|
-
"
|
|
54
|
-
"
|
|
55
|
-
"
|
|
56
|
-
"
|
|
57
|
-
"
|
|
58
|
-
|
|
59
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"name": "jev-layer",
|
|
3
|
+
"version": "0.3.0",
|
|
4
|
+
"license": "MIT",
|
|
5
|
+
"repository": {
|
|
6
|
+
"type": "git",
|
|
7
|
+
"url": "git+https://github.com/typakon4/jev-layer.git"
|
|
8
|
+
},
|
|
9
|
+
"bugs": {
|
|
10
|
+
"url": "https://github.com/typakon4/jev-layer/issues"
|
|
11
|
+
},
|
|
12
|
+
"homepage": "https://github.com/typakon4/jev-layer#readme",
|
|
13
|
+
"type": "module",
|
|
14
|
+
"bin": {
|
|
15
|
+
"jev": "bin/jev.mjs"
|
|
16
|
+
},
|
|
17
|
+
"files": [
|
|
18
|
+
"bin/",
|
|
19
|
+
"config/",
|
|
20
|
+
"docs/",
|
|
21
|
+
"examples/",
|
|
22
|
+
"integrations/",
|
|
23
|
+
"scripts/",
|
|
24
|
+
"src/",
|
|
25
|
+
"test/",
|
|
26
|
+
"CHANGELOG.md",
|
|
27
|
+
"CONTRIBUTING.md",
|
|
28
|
+
"LICENSE",
|
|
29
|
+
"README.md",
|
|
30
|
+
"README.ru.md",
|
|
31
|
+
"README.zh-CN.md",
|
|
32
|
+
"RELEASE.md",
|
|
33
|
+
"SECURITY.md",
|
|
34
|
+
"package.json",
|
|
35
|
+
"package-lock.json"
|
|
36
|
+
],
|
|
37
|
+
"engines": {
|
|
38
|
+
"node": ">=20"
|
|
39
|
+
},
|
|
40
|
+
"scripts": {
|
|
41
|
+
"jev": "node bin/jev.mjs",
|
|
42
|
+
"test": "node --test test/*.test.mjs",
|
|
43
|
+
"smoke": "node scripts/smoke.mjs",
|
|
44
|
+
"benchmark": "node scripts/benchmark.mjs",
|
|
45
|
+
"openrouter:choice": "node scripts/openrouter-choice.mjs",
|
|
46
|
+
"mcp": "node src/mcp-server.mjs",
|
|
47
|
+
"fail-open-smoke": "node scripts/fail-open-smoke.mjs",
|
|
48
|
+
"codex:mcp-smoke": "node scripts/codex-mcp-smoke.mjs",
|
|
49
|
+
"replay:evaluate": "node scripts/replay-eval.mjs",
|
|
50
|
+
"receipt:mcp-smoke": "node scripts/mcp-receipt-smoke.mjs",
|
|
51
|
+
"model-route:report": "node scripts/model-route-report.mjs",
|
|
52
|
+
"browser:e2e": "node scripts/browser-e2e.mjs",
|
|
53
|
+
"browser:benchmark": "node scripts/browser-benchmark.mjs",
|
|
54
|
+
"supervision:e2e": "node scripts/supervision-e2e.mjs",
|
|
55
|
+
"context-filter:e2e": "node scripts/context-filter-e2e.mjs",
|
|
56
|
+
"capability:e2e": "node scripts/capability-e2e.mjs",
|
|
57
|
+
"feature-flags:smoke": "node scripts/feature-flags-smoke.mjs",
|
|
58
|
+
"clean-install-smoke": "node scripts/clean-install-smoke.mjs"
|
|
59
|
+
}
|
|
60
|
+
}
|
package/scripts/benchmark.mjs
CHANGED
|
@@ -1,35 +1,35 @@
|
|
|
1
|
-
import { routeRequest } from "../src/route.mjs";
|
|
2
|
-
|
|
3
|
-
const capabilities = [
|
|
4
|
-
{ id: "repo_search", kind: "mcp", name: "Repository search", description: "Search repository files and symbols", permissions: ["read"], risk: "low" },
|
|
5
|
-
{ id: "browser_inspect", kind: "mcp", name: "Browser inspection", description: "Open and inspect webpages", permissions: ["browser"], risk: "medium" },
|
|
6
|
-
{ id: "terminal_read", kind: "cli", name: "Read-only terminal", description: "Run read-only terminal commands", permissions: ["terminal:read"], risk: "low" },
|
|
7
|
-
{ id: "file_write", kind: "tool", name: "File writer", description: "Modify repository files", permissions: ["write"], risk: "high" },
|
|
8
|
-
];
|
|
9
|
-
const request = {
|
|
10
|
-
harness: "benchmark",
|
|
11
|
-
intent: "Search repository files for the routing implementation",
|
|
12
|
-
context: { workspace: "jev-layer" },
|
|
13
|
-
actor_permissions: ["read"],
|
|
14
|
-
capabilities,
|
|
15
|
-
};
|
|
16
|
-
const samples = [];
|
|
17
|
-
const started = performance.now();
|
|
18
|
-
for (let index = 0; index < 1_000; index += 1) {
|
|
19
|
-
const before = performance.now();
|
|
20
|
-
await routeRequest(request, { provider: "demo" });
|
|
21
|
-
samples.push(performance.now() - before);
|
|
22
|
-
}
|
|
23
|
-
samples.sort((left, right) => left - right);
|
|
24
|
-
const percentile = (p) => Number(samples[Math.min(samples.length - 1, Math.floor(samples.length * p))].toFixed(3));
|
|
25
|
-
console.log(JSON.stringify({
|
|
26
|
-
provider: "jev-demo",
|
|
27
|
-
iterations: samples.length,
|
|
28
|
-
total_ms: Number((performance.now() - started).toFixed(3)),
|
|
29
|
-
p50_ms: percentile(0.5),
|
|
30
|
-
p95_ms: percentile(0.95),
|
|
31
|
-
p99_ms: percentile(0.99),
|
|
32
|
-
provider_calls_per_route: 1,
|
|
33
|
-
host_execution_calls: 0,
|
|
34
|
-
paid_provider_calls: 0,
|
|
35
|
-
}));
|
|
1
|
+
import { routeRequest } from "../src/route.mjs";
|
|
2
|
+
|
|
3
|
+
const capabilities = [
|
|
4
|
+
{ id: "repo_search", kind: "mcp", name: "Repository search", description: "Search repository files and symbols", permissions: ["read"], risk: "low" },
|
|
5
|
+
{ id: "browser_inspect", kind: "mcp", name: "Browser inspection", description: "Open and inspect webpages", permissions: ["browser"], risk: "medium" },
|
|
6
|
+
{ id: "terminal_read", kind: "cli", name: "Read-only terminal", description: "Run read-only terminal commands", permissions: ["terminal:read"], risk: "low" },
|
|
7
|
+
{ id: "file_write", kind: "tool", name: "File writer", description: "Modify repository files", permissions: ["write"], risk: "high" },
|
|
8
|
+
];
|
|
9
|
+
const request = {
|
|
10
|
+
harness: "benchmark",
|
|
11
|
+
intent: "Search repository files for the routing implementation",
|
|
12
|
+
context: { workspace: "jev-layer" },
|
|
13
|
+
actor_permissions: ["read"],
|
|
14
|
+
capabilities,
|
|
15
|
+
};
|
|
16
|
+
const samples = [];
|
|
17
|
+
const started = performance.now();
|
|
18
|
+
for (let index = 0; index < 1_000; index += 1) {
|
|
19
|
+
const before = performance.now();
|
|
20
|
+
await routeRequest(request, { provider: "demo" });
|
|
21
|
+
samples.push(performance.now() - before);
|
|
22
|
+
}
|
|
23
|
+
samples.sort((left, right) => left - right);
|
|
24
|
+
const percentile = (p) => Number(samples[Math.min(samples.length - 1, Math.floor(samples.length * p))].toFixed(3));
|
|
25
|
+
console.log(JSON.stringify({
|
|
26
|
+
provider: "jev-demo",
|
|
27
|
+
iterations: samples.length,
|
|
28
|
+
total_ms: Number((performance.now() - started).toFixed(3)),
|
|
29
|
+
p50_ms: percentile(0.5),
|
|
30
|
+
p95_ms: percentile(0.95),
|
|
31
|
+
p99_ms: percentile(0.99),
|
|
32
|
+
provider_calls_per_route: 1,
|
|
33
|
+
host_execution_calls: 0,
|
|
34
|
+
paid_provider_calls: 0,
|
|
35
|
+
}));
|
|
@@ -1,92 +1,92 @@
|
|
|
1
|
-
import { runBrowserFastPath } from "../src/browser.mjs";
|
|
2
|
-
|
|
3
|
-
const iterations = Number(process.env.JEV_BROWSER_BENCHMARK_ITERATIONS ?? 25);
|
|
4
|
-
const baseline = [];
|
|
5
|
-
const jev = [];
|
|
6
|
-
|
|
7
|
-
for (let iteration = 0; iteration < iterations; iteration += 1) {
|
|
8
|
-
baseline.push(await runBaseline());
|
|
9
|
-
jev.push(await runJev());
|
|
10
|
-
}
|
|
11
|
-
|
|
12
|
-
const report = {
|
|
13
|
-
ok: true,
|
|
14
|
-
iterations,
|
|
15
|
-
provider: "injected-jev-demo-choice",
|
|
16
|
-
baseline: summarize(baseline),
|
|
17
|
-
jev: summarize(jev),
|
|
18
|
-
delta: {
|
|
19
|
-
wall_time_reduction_percent: percent(mean(baseline, "wall_time_ms"), mean(jev, "wall_time_ms")),
|
|
20
|
-
main_model_turn_reduction: mean(baseline, "main_model_turns") - mean(jev, "main_model_turns"),
|
|
21
|
-
extra_jev_calls: mean(jev, "jev_calls") - mean(baseline, "jev_calls"),
|
|
22
|
-
browser_action_delta: mean(jev, "browser_actions") - mean(baseline, "browser_actions"),
|
|
23
|
-
failure_delta: mean(jev, "failures") - mean(baseline, "failures"),
|
|
24
|
-
},
|
|
25
|
-
};
|
|
26
|
-
console.log(JSON.stringify(report, null, 2));
|
|
27
|
-
|
|
28
|
-
async function runBaseline() {
|
|
29
|
-
const started = performance.now();
|
|
30
|
-
const actions = ["scroll", "switch_tab"];
|
|
31
|
-
for (const operation of actions) await Promise.resolve(operation);
|
|
32
|
-
return {
|
|
33
|
-
wall_time_ms: elapsed(started),
|
|
34
|
-
main_model_turns: 2,
|
|
35
|
-
jev_calls: 0,
|
|
36
|
-
browser_actions: actions.length,
|
|
37
|
-
failures: 0,
|
|
38
|
-
cost_usd: 0,
|
|
39
|
-
};
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
async function runJev() {
|
|
43
|
-
let index = 0;
|
|
44
|
-
const choices = ["browser:scroll:down", "browser:switch_tab:tab-2"];
|
|
45
|
-
const result = await runBrowserFastPath({
|
|
46
|
-
goal: "scroll down and switch to the other tab",
|
|
47
|
-
enabled: true,
|
|
48
|
-
provider: {
|
|
49
|
-
name: "browser-benchmark-provider",
|
|
50
|
-
async decide({ candidates }) {
|
|
51
|
-
const requested = choices[index++] ?? "browser:handoff";
|
|
52
|
-
const choice = candidates.some((candidate) => candidate.id === requested)
|
|
53
|
-
? requested
|
|
54
|
-
: candidates.find((candidate) => candidate.id === "browser:handoff")?.id ?? candidates[0]?.id;
|
|
55
|
-
return {
|
|
56
|
-
answers: { tool: { type: "choice", choice, probabilities: Object.fromEntries(candidates.map((candidate) => [candidate.id, candidate.id === choice ? 1 : 0])), confidence: 1 } },
|
|
57
|
-
usage: { cost: 0.0001 },
|
|
58
|
-
};
|
|
59
|
-
},
|
|
60
|
-
},
|
|
61
|
-
executor: {
|
|
62
|
-
state: 0,
|
|
63
|
-
async observe() {
|
|
64
|
-
return this.state === 0
|
|
65
|
-
? { scroll: { up: false, down: true }, tabs: [{ id: "tab-1", active: true }, { id: "tab-2", title: "Other", active: false }] }
|
|
66
|
-
: { scroll: { up: false, down: false }, tabs: [{ id: "tab-1", active: true }, { id: "tab-2", title: "Other", active: false }] };
|
|
67
|
-
},
|
|
68
|
-
async execute() {
|
|
69
|
-
this.state += 1;
|
|
70
|
-
return { observation: await this.observe(), handoff: this.state >= 2 };
|
|
71
|
-
},
|
|
72
|
-
},
|
|
73
|
-
mainModelTurns: 1,
|
|
74
|
-
});
|
|
75
|
-
return result.metrics;
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
function summarize(rows) {
|
|
79
|
-
return Object.fromEntries(["wall_time_ms", "main_model_turns", "jev_calls", "browser_actions", "failures", "cost_usd"].map((key) => [key, mean(rows, key)]));
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
function mean(rows, key) {
|
|
83
|
-
return Number((rows.reduce((sum, row) => sum + row[key], 0) / rows.length).toFixed(3));
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
function percent(before, after) {
|
|
87
|
-
return before ? Number((((before - after) / before) * 100).toFixed(2)) : 0;
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
function elapsed(started) {
|
|
91
|
-
return Number((performance.now() - started).toFixed(3));
|
|
92
|
-
}
|
|
1
|
+
import { runBrowserFastPath } from "../src/browser.mjs";
|
|
2
|
+
|
|
3
|
+
const iterations = Number(process.env.JEV_BROWSER_BENCHMARK_ITERATIONS ?? 25);
|
|
4
|
+
const baseline = [];
|
|
5
|
+
const jev = [];
|
|
6
|
+
|
|
7
|
+
for (let iteration = 0; iteration < iterations; iteration += 1) {
|
|
8
|
+
baseline.push(await runBaseline());
|
|
9
|
+
jev.push(await runJev());
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
const report = {
|
|
13
|
+
ok: true,
|
|
14
|
+
iterations,
|
|
15
|
+
provider: "injected-jev-demo-choice",
|
|
16
|
+
baseline: summarize(baseline),
|
|
17
|
+
jev: summarize(jev),
|
|
18
|
+
delta: {
|
|
19
|
+
wall_time_reduction_percent: percent(mean(baseline, "wall_time_ms"), mean(jev, "wall_time_ms")),
|
|
20
|
+
main_model_turn_reduction: mean(baseline, "main_model_turns") - mean(jev, "main_model_turns"),
|
|
21
|
+
extra_jev_calls: mean(jev, "jev_calls") - mean(baseline, "jev_calls"),
|
|
22
|
+
browser_action_delta: mean(jev, "browser_actions") - mean(baseline, "browser_actions"),
|
|
23
|
+
failure_delta: mean(jev, "failures") - mean(baseline, "failures"),
|
|
24
|
+
},
|
|
25
|
+
};
|
|
26
|
+
console.log(JSON.stringify(report, null, 2));
|
|
27
|
+
|
|
28
|
+
async function runBaseline() {
|
|
29
|
+
const started = performance.now();
|
|
30
|
+
const actions = ["scroll", "switch_tab"];
|
|
31
|
+
for (const operation of actions) await Promise.resolve(operation);
|
|
32
|
+
return {
|
|
33
|
+
wall_time_ms: elapsed(started),
|
|
34
|
+
main_model_turns: 2,
|
|
35
|
+
jev_calls: 0,
|
|
36
|
+
browser_actions: actions.length,
|
|
37
|
+
failures: 0,
|
|
38
|
+
cost_usd: 0,
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
async function runJev() {
|
|
43
|
+
let index = 0;
|
|
44
|
+
const choices = ["browser:scroll:down", "browser:switch_tab:tab-2"];
|
|
45
|
+
const result = await runBrowserFastPath({
|
|
46
|
+
goal: "scroll down and switch to the other tab",
|
|
47
|
+
enabled: true,
|
|
48
|
+
provider: {
|
|
49
|
+
name: "browser-benchmark-provider",
|
|
50
|
+
async decide({ candidates }) {
|
|
51
|
+
const requested = choices[index++] ?? "browser:handoff";
|
|
52
|
+
const choice = candidates.some((candidate) => candidate.id === requested)
|
|
53
|
+
? requested
|
|
54
|
+
: candidates.find((candidate) => candidate.id === "browser:handoff")?.id ?? candidates[0]?.id;
|
|
55
|
+
return {
|
|
56
|
+
answers: { tool: { type: "choice", choice, probabilities: Object.fromEntries(candidates.map((candidate) => [candidate.id, candidate.id === choice ? 1 : 0])), confidence: 1 } },
|
|
57
|
+
usage: { cost: 0.0001 },
|
|
58
|
+
};
|
|
59
|
+
},
|
|
60
|
+
},
|
|
61
|
+
executor: {
|
|
62
|
+
state: 0,
|
|
63
|
+
async observe() {
|
|
64
|
+
return this.state === 0
|
|
65
|
+
? { scroll: { up: false, down: true }, tabs: [{ id: "tab-1", active: true }, { id: "tab-2", title: "Other", active: false }] }
|
|
66
|
+
: { scroll: { up: false, down: false }, tabs: [{ id: "tab-1", active: true }, { id: "tab-2", title: "Other", active: false }] };
|
|
67
|
+
},
|
|
68
|
+
async execute() {
|
|
69
|
+
this.state += 1;
|
|
70
|
+
return { observation: await this.observe(), handoff: this.state >= 2 };
|
|
71
|
+
},
|
|
72
|
+
},
|
|
73
|
+
mainModelTurns: 1,
|
|
74
|
+
});
|
|
75
|
+
return result.metrics;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function summarize(rows) {
|
|
79
|
+
return Object.fromEntries(["wall_time_ms", "main_model_turns", "jev_calls", "browser_actions", "failures", "cost_usd"].map((key) => [key, mean(rows, key)]));
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function mean(rows, key) {
|
|
83
|
+
return Number((rows.reduce((sum, row) => sum + row[key], 0) / rows.length).toFixed(3));
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function percent(before, after) {
|
|
87
|
+
return before ? Number((((before - after) / before) * 100).toFixed(2)) : 0;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function elapsed(started) {
|
|
91
|
+
return Number((performance.now() - started).toFixed(3));
|
|
92
|
+
}
|