opencode-agent-skill 13.0.0-beta.2 → 14.2.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1556 -607
- package/bin/ocskill.mjs +172 -24
- package/docs/OPENCODE-COMPAT.md +34 -97
- package/docs/PI-COMPAT.md +188 -0
- package/docs/V14-CONTEXT-MEMORY-FABRIC.md +70 -0
- package/docs/V14.1-QUALITY-PERFORMANCE-FABRIC.md +114 -0
- package/docs/V14.2-TURBO-WEAK-MODEL-RUNTIME.md +448 -0
- package/evals/v14/tasks.json +46 -0
- package/global-config/agents/executor.md +7 -0
- package/global-config/agents/visual-verifier.md +22 -3
- package/global-config/plugins/ues-router/index.js +13 -8
- package/global-config/plugins/ues-router/policy-runtime.js +7 -0
- package/global-config/plugins/ues-router/router.js +12 -2
- package/global-config/skills/ecommerce-engineering/SKILL.md +1 -1
- package/global-config/skills/file-upload-engineering/SKILL.md +1 -1
- package/global-config/skills/git-safety/SKILL.md +1 -1
- package/global-config/skills/nestjs-engineering/SKILL.md +1 -1
- package/global-config/skills/performance-engineering/SKILL.md +1 -1
- package/global-config/skills/react-native-engineering/SKILL.md +1 -1
- package/global-config/skills/rest-api-design/SKILL.md +1 -1
- package/global-config/skills/ui-ux-engineering/SKILL.md +1 -1
- package/lib/adaptive-context-budget.mjs +97 -0
- package/lib/affected-tests.mjs +260 -0
- package/lib/benchmark-confidence.mjs +41 -2
- package/lib/browser-mcp-routing.mjs +166 -0
- package/lib/capability-fabric.mjs +336 -0
- package/lib/capability-registry.mjs +9 -0
- package/lib/context-engine-v11.mjs +65 -1
- package/lib/context-graph-rank.mjs +118 -0
- package/lib/context-manifest.mjs +97 -18
- package/lib/control-center.mjs +19 -1
- package/lib/dynamic-workflow.mjs +3 -1
- package/lib/evidence-store.mjs +82 -1
- package/lib/hierarchical-context.mjs +215 -0
- package/lib/memory-engine.mjs +465 -0
- package/lib/model-performance.mjs +33 -8
- package/lib/model-policy.mjs +3 -3
- package/lib/orchestrator-policy.mjs +5 -209
- package/lib/performance-fabric.mjs +229 -0
- package/lib/pi-rpc-pool.mjs +433 -0
- package/lib/process-hang-detector.mjs +83 -0
- package/lib/process-supervisor.mjs +193 -0
- package/lib/prompt-cache.mjs +2 -0
- package/lib/repo-graph.mjs +53 -2
- package/lib/runtime-config.mjs +31 -0
- package/lib/safety.mjs +132 -0
- package/lib/semantic-index.mjs +52 -3
- package/lib/skill-compiler.mjs +128 -0
- package/lib/skill-quality.mjs +48 -2
- package/lib/task-engine.mjs +66 -5
- package/lib/task-policy.mjs +235 -0
- package/lib/verification-broker.mjs +284 -0
- package/lib/verification-command.mjs +111 -0
- package/lib/windows-shim.mjs +35 -0
- package/lib/workspace-fingerprint.mjs +198 -0
- package/package.json +52 -42
- package/pi/extensions/ues-child-runtime.ts +238 -0
- package/pi/extensions/ues.ts +3200 -0
- package/pi/prompts/ues-audit.md +9 -0
- package/pi/prompts/ues-critique.md +9 -0
- package/pi/prompts/ues-debug.md +9 -0
- package/pi/prompts/ues-feature.md +9 -0
- package/pi/prompts/ues-fix.md +9 -0
- package/pi/prompts/ues-plan.md +9 -0
- package/pi/prompts/ues-research.md +9 -0
- package/pi/prompts/ues-resume.md +9 -0
- package/pi/prompts/ues-review.md +7 -0
- package/pi/prompts/ues-run.md +17 -0
- package/pi/prompts/ues-verify.md +9 -0
- package/scripts/check-release-consistency.mjs +119 -185
- package/scripts/check-runtime-exports.mjs +66 -0
- package/scripts/check-source-integrity.mjs +184 -0
- package/scripts/eval-pi.mjs +492 -0
- package/scripts/install.mjs +16 -0
- package/scripts/smoke-package-closure.mjs +110 -0
- package/scripts/smoke-packed-install.mjs +24 -11
- package/scripts/smoke-pi-extension.mjs +144 -0
- package/scripts/uninstall.mjs +44 -0
- package/CHANGELOG.md +0 -415
- package/docs/DETERMINISTIC-TOOLS.md +0 -105
- package/docs/ENGINEERING-DESIGN.md +0 -194
- package/docs/EVALS.md +0 -158
- package/docs/GITHUB-RULESET.md +0 -50
- package/docs/NPM-PUBLISH.md +0 -116
- package/docs/RESEARCH-SOURCES.md +0 -37
- package/docs/TRACE-SCHEMA.md +0 -122
- package/docs/V11-PERCEPTION-ADAPTIVE-EXECUTION.md +0 -75
- package/docs/V11-PERCEPTION-ADAPTIVE.md +0 -220
- package/docs/V12-WEAK-MODEL-INTELLIGENCE.md +0 -27
- package/docs/V13-PARALLEL-WEAK-MODEL-RUNTIME.md +0 -86
- package/docs/V7-INTELLIGENCE-RUNTIME.md +0 -166
- package/docs/V8-INTELLIGENCE-RELIABILITY.md +0 -206
- package/docs/V9-SPEED-INTELLIGENCE.md +0 -102
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: nestjs-engineering
|
|
3
|
-
description: Work on NestJS apps: modules, controllers, providers, DTO validation, guards, interceptors, persistence, and tests.
|
|
3
|
+
description: "Work on NestJS apps: modules, controllers, providers, DTO validation, guards, interceptors, persistence, and tests."
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# Nestjs Engineering
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: performance-engineering
|
|
3
|
-
description: Diagnose and improve performance using evidence: rendering, database, network, memory, caching, bundles, concurrency, and hot paths.
|
|
3
|
+
description: "Diagnose and improve performance using evidence: rendering, database, network, memory, caching, bundles, concurrency, and hot paths."
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# Performance Engineering
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: react-native-engineering
|
|
3
|
-
description: Work on React Native/Expo apps: components, hooks, navigation, styling, APIs, native modules, Android/iOS build failures, platform behavior, and performance with version-aware verification.
|
|
3
|
+
description: "Work on React Native/Expo apps: components, hooks, navigation, styling, APIs, native modules, Android/iOS build failures, platform behavior, and performance with version-aware verification."
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# React Native Engineering
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: rest-api-design
|
|
3
|
-
description: Design/review REST APIs: resources, methods, status codes, pagination, validation, errors, versioning, and idempotency.
|
|
3
|
+
description: "Design/review REST APIs: resources, methods, status codes, pagination, validation, errors, versioning, and idempotency."
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# Rest Api Design
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: ui-ux-engineering
|
|
3
|
-
description: Improve production UI/UX: hierarchy, responsive layout, components, forms, states, accessibility, realistic content, and design consistency.
|
|
3
|
+
description: "Improve production UI/UX: hierarchy, responsive layout, components, forms, states, accessibility, realistic content, and design consistency."
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# Ui Ux Engineering
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
const ROLE_TARGETS = Object.freeze({
|
|
2
|
+
fast: {
|
|
3
|
+
architect: 8_000,
|
|
4
|
+
"plan-checker": 6_000,
|
|
5
|
+
executor: 6_000,
|
|
6
|
+
debugger: 8_000,
|
|
7
|
+
verifier: 4_500,
|
|
8
|
+
"integration-verifier": 8_000,
|
|
9
|
+
"visual-verifier": 5_000,
|
|
10
|
+
reviewer: 5_000,
|
|
11
|
+
critic: 6_000,
|
|
12
|
+
"codebase-mapper": 6_000,
|
|
13
|
+
researcher: 6_000,
|
|
14
|
+
"merge-arbiter": 8_000,
|
|
15
|
+
},
|
|
16
|
+
standard: {
|
|
17
|
+
architect: 12_000,
|
|
18
|
+
"plan-checker": 10_000,
|
|
19
|
+
executor: 11_000,
|
|
20
|
+
debugger: 14_000,
|
|
21
|
+
verifier: 8_000,
|
|
22
|
+
"integration-verifier": 12_000,
|
|
23
|
+
"visual-verifier": 8_000,
|
|
24
|
+
reviewer: 8_000,
|
|
25
|
+
critic: 10_000,
|
|
26
|
+
"codebase-mapper": 10_000,
|
|
27
|
+
researcher: 10_000,
|
|
28
|
+
"merge-arbiter": 12_000,
|
|
29
|
+
},
|
|
30
|
+
deep: {
|
|
31
|
+
architect: 30_000,
|
|
32
|
+
"plan-checker": 24_000,
|
|
33
|
+
executor: 26_000,
|
|
34
|
+
debugger: 32_000,
|
|
35
|
+
verifier: 20_000,
|
|
36
|
+
"integration-verifier": 30_000,
|
|
37
|
+
"visual-verifier": 18_000,
|
|
38
|
+
reviewer: 20_000,
|
|
39
|
+
critic: 24_000,
|
|
40
|
+
"codebase-mapper": 28_000,
|
|
41
|
+
researcher: 24_000,
|
|
42
|
+
"merge-arbiter": 30_000,
|
|
43
|
+
},
|
|
44
|
+
})
|
|
45
|
+
|
|
46
|
+
function clamp(value, fallback, min, max) {
|
|
47
|
+
const parsed = Number(value)
|
|
48
|
+
if (!Number.isFinite(parsed)) return fallback
|
|
49
|
+
return Math.max(min, Math.min(max, Math.trunc(parsed)))
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export function adaptiveContextBudget(taskPolicy = {}, role = "executor", attempt = 1, options = {}) {
|
|
53
|
+
const base = clamp(
|
|
54
|
+
taskPolicy.contextBudget ?? taskPolicy.profile?.contextBudget,
|
|
55
|
+
20_000,
|
|
56
|
+
4_000,
|
|
57
|
+
48_000,
|
|
58
|
+
)
|
|
59
|
+
const profile = String(taskPolicy.executionProfile || taskPolicy.profile?.name || "standard")
|
|
60
|
+
const highRisk = taskPolicy.risk === "high" || taskPolicy.risk === "critical"
|
|
61
|
+
|
|
62
|
+
// High-risk work keeps the original evidence budget. Turbo mode must not trade
|
|
63
|
+
// away security/payment/schema evidence for latency.
|
|
64
|
+
if (highRisk || options.disabled === true) {
|
|
65
|
+
return {
|
|
66
|
+
schemaVersion: 1,
|
|
67
|
+
budget: base,
|
|
68
|
+
baseBudget: base,
|
|
69
|
+
adaptive: false,
|
|
70
|
+
reason: highRisk ? "high-risk-preserves-base-budget" : "disabled",
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const table = ROLE_TARGETS[profile] || ROLE_TARGETS.standard
|
|
75
|
+
const target = clamp(table?.[role], base, 4_000, base)
|
|
76
|
+
const normalizedAttempt = Math.max(1, Math.trunc(Number(attempt || 1)))
|
|
77
|
+
|
|
78
|
+
// Failed attempts expand deterministically back toward the policy ceiling.
|
|
79
|
+
let budget = target
|
|
80
|
+
if (normalizedAttempt === 2) budget = Math.min(base, Math.max(target, Math.round(target * 1.5)))
|
|
81
|
+
else if (normalizedAttempt >= 3) budget = base
|
|
82
|
+
|
|
83
|
+
if (options.contextInsufficient === true) {
|
|
84
|
+
budget = Math.min(base, Math.max(budget, Math.round(target * 1.75)))
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
return {
|
|
88
|
+
schemaVersion: 1,
|
|
89
|
+
budget,
|
|
90
|
+
baseBudget: base,
|
|
91
|
+
adaptive: budget < base,
|
|
92
|
+
profile,
|
|
93
|
+
role,
|
|
94
|
+
attempt: normalizedAttempt,
|
|
95
|
+
reason: budget < base ? "role-bounded-expand-on-failure" : "policy-ceiling",
|
|
96
|
+
}
|
|
97
|
+
}
|
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
import { existsSync } from "node:fs"
|
|
2
|
+
import { readFile, readdir, stat } from "node:fs/promises"
|
|
3
|
+
import path from "node:path"
|
|
4
|
+
import { spawnSync } from "node:child_process"
|
|
5
|
+
import { runtimeWorkspaceFingerprint } from "./workspace-fingerprint.mjs"
|
|
6
|
+
|
|
7
|
+
const TEST_RE = /(?:^|[/.\\])(?:__tests__|tests?|spec)(?:[/.\\]|$)|\.(?:test|spec|e2e-spec)\.[cm]?[jt]sx?$/i
|
|
8
|
+
const SOURCE_EXT = new Set([".js",".jsx",".mjs",".cjs",".ts",".tsx",".py",".java",".kt",".kts",".cs",".go",".rs",".dart",".swift"])
|
|
9
|
+
const SKIP = new Set([".git","node_modules",".next","dist","build","coverage",".venv","venv","target","Pods","DerivedData",".gradle",".ues-cache",".ues-work",".ues-sandboxes",".ues-traces",".ues-learning",".ues-dashboard",".ues-memory",".ues-evals"])
|
|
10
|
+
const AFFECTED_TEST_CACHE = new Map()
|
|
11
|
+
const AFFECTED_TEST_INFLIGHT = new Map()
|
|
12
|
+
|
|
13
|
+
function git(root, args) {
|
|
14
|
+
return spawnSync("git", args, { cwd: root, encoding: "utf8", maxBuffer: 8 * 1024 * 1024 })
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
function normalize(file) {
|
|
18
|
+
return String(file || "").replaceAll("\\", "/").replace(/^\.\//, "")
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function stem(file) {
|
|
22
|
+
return path.basename(file)
|
|
23
|
+
.replace(/\.(?:test|spec|e2e-spec)(?=\.)/i, "")
|
|
24
|
+
.replace(/\.[^.]+$/, "")
|
|
25
|
+
.toLowerCase()
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function tokens(file) {
|
|
29
|
+
return [...new Set(normalize(file).toLowerCase().split(/[^a-z0-9]+/).filter((x) => x.length >= 3))]
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
async function walk(root, options = {}) {
|
|
33
|
+
const maxFiles = Math.max(200, Number(options.maxFiles || 6000))
|
|
34
|
+
const rows = []
|
|
35
|
+
async function visit(dir, depth) {
|
|
36
|
+
if (rows.length >= maxFiles || depth > 14) return
|
|
37
|
+
const entries = await readdir(dir, { withFileTypes: true }).catch(() => [])
|
|
38
|
+
for (const entry of entries) {
|
|
39
|
+
if (rows.length >= maxFiles) break
|
|
40
|
+
if (entry.isDirectory() && SKIP.has(entry.name)) continue
|
|
41
|
+
const full = path.join(dir, entry.name)
|
|
42
|
+
if (entry.isDirectory()) await visit(full, depth + 1)
|
|
43
|
+
else if (entry.isFile()) rows.push(full)
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
await visit(root, 0)
|
|
47
|
+
return rows
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function gitChangedFiles(root = process.cwd()) {
|
|
51
|
+
root = path.resolve(root)
|
|
52
|
+
const inside = git(root, ["rev-parse", "--is-inside-work-tree"])
|
|
53
|
+
if (inside.status !== 0) return []
|
|
54
|
+
|
|
55
|
+
const names = new Set()
|
|
56
|
+
for (const args of [
|
|
57
|
+
["diff", "--name-only", "--"],
|
|
58
|
+
["diff", "--cached", "--name-only", "--"],
|
|
59
|
+
["ls-files", "--others", "--exclude-standard"],
|
|
60
|
+
]) {
|
|
61
|
+
const result = git(root, args)
|
|
62
|
+
if (result.status !== 0) continue
|
|
63
|
+
for (const row of String(result.stdout || "").split(/\r?\n/)) {
|
|
64
|
+
const file = normalize(row.trim())
|
|
65
|
+
if (file) names.add(file)
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
return [...names]
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function scoreTest(testFile, changedFile, content = "") {
|
|
72
|
+
const test = normalize(testFile).toLowerCase()
|
|
73
|
+
const changed = normalize(changedFile).toLowerCase()
|
|
74
|
+
let score = 0
|
|
75
|
+
const reasons = []
|
|
76
|
+
const changedStem = stem(changed)
|
|
77
|
+
const testStem = stem(test)
|
|
78
|
+
|
|
79
|
+
if (changedStem && testStem.includes(changedStem)) {
|
|
80
|
+
score += 35
|
|
81
|
+
reasons.push("same-stem")
|
|
82
|
+
}
|
|
83
|
+
const testDir = path.posix.dirname(test)
|
|
84
|
+
const changedDir = path.posix.dirname(changed)
|
|
85
|
+
if (testDir === changedDir) {
|
|
86
|
+
score += 20
|
|
87
|
+
reasons.push("same-directory")
|
|
88
|
+
} else if (testDir.startsWith(changedDir + "/") || changedDir.startsWith(testDir + "/")) {
|
|
89
|
+
score += 10
|
|
90
|
+
reasons.push("nearby-directory")
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const changedTokens = tokens(changed)
|
|
94
|
+
const overlap = changedTokens.filter((token) => test.includes(token)).length
|
|
95
|
+
if (overlap) {
|
|
96
|
+
score += Math.min(20, overlap * 4)
|
|
97
|
+
reasons.push("path-token-overlap")
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const base = path.basename(changed).replace(/\.[^.]+$/, "")
|
|
101
|
+
if (base.length >= 3 && content.toLowerCase().includes(base.toLowerCase())) {
|
|
102
|
+
score += 25
|
|
103
|
+
reasons.push("content-reference")
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
return { score, reasons }
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
async function nearestPackage(root, file) {
|
|
110
|
+
let dir = path.dirname(path.resolve(root, file))
|
|
111
|
+
const base = path.resolve(root)
|
|
112
|
+
while (dir === base || dir.startsWith(base + path.sep)) {
|
|
113
|
+
const packageFile = path.join(dir, "package.json")
|
|
114
|
+
if (existsSync(packageFile)) {
|
|
115
|
+
try {
|
|
116
|
+
return { dir, json: JSON.parse(await readFile(packageFile, "utf8")) }
|
|
117
|
+
} catch {}
|
|
118
|
+
}
|
|
119
|
+
if (dir === base) break
|
|
120
|
+
dir = path.dirname(dir)
|
|
121
|
+
}
|
|
122
|
+
return null
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function packageManager(root) {
|
|
126
|
+
if (existsSync(path.join(root, "pnpm-lock.yaml"))) return "pnpm"
|
|
127
|
+
if (existsSync(path.join(root, "yarn.lock"))) return "yarn"
|
|
128
|
+
if (existsSync(path.join(root, "bun.lock")) || existsSync(path.join(root, "bun.lockb"))) return "bun"
|
|
129
|
+
return "npm"
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function targetedCommand(root, pkg, testFile) {
|
|
133
|
+
const script = String(pkg?.json?.scripts?.test || "")
|
|
134
|
+
if (!script || !/(jest|vitest|node\s+--test|tsx\s+--test)/i.test(script)) return null
|
|
135
|
+
const manager = packageManager(root)
|
|
136
|
+
const relativeDir = normalize(path.relative(root, pkg.dir)) || "."
|
|
137
|
+
const relativeTest = normalize(path.relative(pkg.dir, path.resolve(root, testFile)))
|
|
138
|
+
|
|
139
|
+
if (manager === "pnpm") {
|
|
140
|
+
return { command: "pnpm", args: ["--dir", relativeDir, "run", "test", "--", relativeTest], confidence: "high" }
|
|
141
|
+
}
|
|
142
|
+
if (manager === "yarn") {
|
|
143
|
+
return { command: "yarn", args: ["--cwd", relativeDir, "test", relativeTest], confidence: "high" }
|
|
144
|
+
}
|
|
145
|
+
if (manager === "bun") {
|
|
146
|
+
return { command: "bun", args: ["--cwd", relativeDir, "run", "test", "--", relativeTest], confidence: "high" }
|
|
147
|
+
}
|
|
148
|
+
return { command: "npm", args: ["--prefix", relativeDir, "test", "--", relativeTest], confidence: "high" }
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
export async function resolveAffectedTests(root = process.cwd(), options = {}) {
|
|
152
|
+
root = path.resolve(root)
|
|
153
|
+
const changed = (options.changedFiles || gitChangedFiles(root))
|
|
154
|
+
.map(normalize)
|
|
155
|
+
.filter((file) => SOURCE_EXT.has(path.extname(file).toLowerCase()) && !TEST_RE.test(file))
|
|
156
|
+
|
|
157
|
+
if (!changed.length) {
|
|
158
|
+
return {
|
|
159
|
+
schemaVersion: 1,
|
|
160
|
+
root,
|
|
161
|
+
changedFiles: [],
|
|
162
|
+
tests: [],
|
|
163
|
+
suggestedCommands: [],
|
|
164
|
+
truncated: false,
|
|
165
|
+
cacheHit: false,
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
const limit = Math.max(1, Math.min(40, Number(options.limit || 12)))
|
|
170
|
+
let fingerprint = String(options.workspaceFingerprint || "")
|
|
171
|
+
if (!fingerprint || fingerprint === "unknown") {
|
|
172
|
+
try { fingerprint = runtimeWorkspaceFingerprint(root) } catch { fingerprint = "" }
|
|
173
|
+
}
|
|
174
|
+
if (fingerprint === "unknown") fingerprint = ""
|
|
175
|
+
|
|
176
|
+
const cacheKey = JSON.stringify([
|
|
177
|
+
root,
|
|
178
|
+
fingerprint || "no-fingerprint",
|
|
179
|
+
[...changed].sort(),
|
|
180
|
+
Number(options.maxFiles || 6000),
|
|
181
|
+
limit,
|
|
182
|
+
])
|
|
183
|
+
|
|
184
|
+
if (fingerprint && AFFECTED_TEST_CACHE.has(cacheKey)) {
|
|
185
|
+
return { ...AFFECTED_TEST_CACHE.get(cacheKey), cacheHit: true }
|
|
186
|
+
}
|
|
187
|
+
if (fingerprint && AFFECTED_TEST_INFLIGHT.has(cacheKey)) {
|
|
188
|
+
const value = await AFFECTED_TEST_INFLIGHT.get(cacheKey)
|
|
189
|
+
return { ...value, cacheHit: true, cacheCoalesced: true }
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
const compute = async () => {
|
|
193
|
+
const files = await walk(root, options)
|
|
194
|
+
const tests = files
|
|
195
|
+
.map((file) => normalize(path.relative(root, file)))
|
|
196
|
+
.filter((file) => TEST_RE.test(file))
|
|
197
|
+
|
|
198
|
+
const ranked = []
|
|
199
|
+
for (const testFile of tests) {
|
|
200
|
+
const info = await stat(path.join(root, testFile)).catch(() => null)
|
|
201
|
+
const content = info?.size && info.size <= 512 * 1024
|
|
202
|
+
? await readFile(path.join(root, testFile), "utf8").catch(() => "")
|
|
203
|
+
: ""
|
|
204
|
+
let best = { score: 0, reasons: [], changedFile: null }
|
|
205
|
+
for (const changedFile of changed) {
|
|
206
|
+
const current = scoreTest(testFile, changedFile, content)
|
|
207
|
+
if (current.score > best.score) best = { ...current, changedFile }
|
|
208
|
+
}
|
|
209
|
+
if (best.score > 0) ranked.push({ path: testFile, ...best })
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
ranked.sort((a, b) => b.score - a.score || a.path.localeCompare(b.path))
|
|
213
|
+
const selected = ranked.slice(0, limit)
|
|
214
|
+
const suggestedCommands = []
|
|
215
|
+
const seen = new Set()
|
|
216
|
+
|
|
217
|
+
for (const item of selected.slice(0, 8)) {
|
|
218
|
+
const pkg = await nearestPackage(root, item.path)
|
|
219
|
+
const command = pkg ? targetedCommand(root, pkg, item.path) : null
|
|
220
|
+
if (!command) continue
|
|
221
|
+
const key = JSON.stringify([command.command, command.args])
|
|
222
|
+
if (seen.has(key)) continue
|
|
223
|
+
seen.add(key)
|
|
224
|
+
suggestedCommands.push({ ...command, test: item.path })
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
return {
|
|
228
|
+
schemaVersion: 1,
|
|
229
|
+
root,
|
|
230
|
+
changedFiles: changed,
|
|
231
|
+
tests: selected,
|
|
232
|
+
suggestedCommands,
|
|
233
|
+
truncated: ranked.length > selected.length,
|
|
234
|
+
cacheHit: false,
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
const pending = compute()
|
|
239
|
+
if (fingerprint) AFFECTED_TEST_INFLIGHT.set(cacheKey, pending)
|
|
240
|
+
|
|
241
|
+
try {
|
|
242
|
+
const result = await pending
|
|
243
|
+
if (fingerprint) {
|
|
244
|
+
AFFECTED_TEST_CACHE.set(cacheKey, result)
|
|
245
|
+
while (AFFECTED_TEST_CACHE.size > 24) {
|
|
246
|
+
const oldest = AFFECTED_TEST_CACHE.keys().next().value
|
|
247
|
+
if (!oldest) break
|
|
248
|
+
AFFECTED_TEST_CACHE.delete(oldest)
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
return result
|
|
252
|
+
} finally {
|
|
253
|
+
if (fingerprint) AFFECTED_TEST_INFLIGHT.delete(cacheKey)
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
export function clearAffectedTestCache() {
|
|
258
|
+
AFFECTED_TEST_CACHE.clear()
|
|
259
|
+
AFFECTED_TEST_INFLIGHT.clear()
|
|
260
|
+
}
|
|
@@ -125,8 +125,36 @@ export function pairedBenchmarkConfidence(results, options = {}) {
|
|
|
125
125
|
initialInputAcceptable,
|
|
126
126
|
tokensAcceptable,
|
|
127
127
|
}
|
|
128
|
+
|
|
129
|
+
const baselineContaminated = pairs.filter((pair) => pair.baseline?.baselineIsolated === false).length
|
|
130
|
+
const baselineIsolationUnknown = pairs.filter((pair) => pair.baseline?.baselineIsolated !== true).length - baselineContaminated
|
|
131
|
+
const uesControllerUnproven = pairs.filter((pair) => pair.ues?.telemetry?.controllerUsed !== true).length
|
|
132
|
+
const uesFalsePasses = pairs.filter((pair) =>
|
|
133
|
+
pair.ues?.telemetry?.controllerPass === true &&
|
|
134
|
+
Number(pair.ues?.graderExit) !== 0
|
|
135
|
+
).length
|
|
136
|
+
const qualityRegressionTolerance = Math.max(0, Number(options.qualityRegressionTolerance || 0))
|
|
137
|
+
const qualityNonRegression = delta >= -qualityRegressionTolerance
|
|
138
|
+
const efficiencySignals = [
|
|
139
|
+
durationRatio == null ? null : durationRatio < 1,
|
|
140
|
+
initialInputRatio == null ? null : initialInputRatio < 1,
|
|
141
|
+
tokenRatio == null ? null : tokenRatio < 1,
|
|
142
|
+
].filter((value) => value !== null)
|
|
143
|
+
const efficiencyImproved = efficiencySignals.some(Boolean)
|
|
144
|
+
const turboChecks = {
|
|
145
|
+
pairedCoverage: total >= minPairs,
|
|
146
|
+
baselineIsolated: baselineContaminated === 0 && baselineIsolationUnknown === 0,
|
|
147
|
+
uesControllerUsed: uesControllerUnproven === 0,
|
|
148
|
+
qualityNonRegression,
|
|
149
|
+
noSuiteRegression,
|
|
150
|
+
noControllerFalsePass: uesFalsePasses === 0,
|
|
151
|
+
speedAcceptable,
|
|
152
|
+
initialInputAcceptable,
|
|
153
|
+
tokensAcceptable,
|
|
154
|
+
efficiencyImproved,
|
|
155
|
+
}
|
|
128
156
|
return {
|
|
129
|
-
schemaVersion:
|
|
157
|
+
schemaVersion: 3,
|
|
130
158
|
kind: "ues-paired-benchmark-confidence",
|
|
131
159
|
pairs: total,
|
|
132
160
|
bothPass,
|
|
@@ -169,5 +197,16 @@ export function pairedBenchmarkConfidence(results, options = {}) {
|
|
|
169
197
|
suites,
|
|
170
198
|
checks,
|
|
171
199
|
promotionEligible: Object.values(checks).every(Boolean),
|
|
200
|
+
turbo: {
|
|
201
|
+
policy: "quality-non-regression-with-efficiency-gain",
|
|
202
|
+
qualityRegressionTolerance,
|
|
203
|
+
baselineContaminated,
|
|
204
|
+
baselineIsolationUnknown,
|
|
205
|
+
uesControllerUnproven,
|
|
206
|
+
uesFalsePasses,
|
|
207
|
+
efficiencyImproved,
|
|
208
|
+
checks: turboChecks,
|
|
209
|
+
promotionEligible: Object.values(turboChecks).every(Boolean),
|
|
210
|
+
},
|
|
172
211
|
}
|
|
173
|
-
}
|
|
212
|
+
}
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
import { inferTaskCapabilities } from "./capability-registry.mjs"
|
|
2
|
+
|
|
3
|
+
const BROWSER_ROLES = new Set([
|
|
4
|
+
"executor",
|
|
5
|
+
"debugger",
|
|
6
|
+
"verifier",
|
|
7
|
+
"integration-verifier",
|
|
8
|
+
"visual-verifier",
|
|
9
|
+
])
|
|
10
|
+
|
|
11
|
+
const VISUAL_SIGNAL =
|
|
12
|
+
/(visual fidelity|visual regression|screenshot|figma|pixel|layout|responsive|viewport|accessibility snapshot|giao diện|\bui\b|\bcss\b|styling|kiểm tra hiển thị)/i
|
|
13
|
+
|
|
14
|
+
const PROVIDER_SIGNAL =
|
|
15
|
+
/(playwright|browser automation|browser mcp|chromium|webkit|firefox)/i
|
|
16
|
+
|
|
17
|
+
const BROWSER_TOOL_NAME =
|
|
18
|
+
/(^|[_:.])(browser|playwright)([_:.]|$)|^browser_|^playwright_|mcp.*(?:browser|playwright)/i
|
|
19
|
+
|
|
20
|
+
const ACTION_PRIORITY = [
|
|
21
|
+
["snapshot", 18],
|
|
22
|
+
["screenshot", 17],
|
|
23
|
+
["console", 16],
|
|
24
|
+
["network", 15],
|
|
25
|
+
["navigate", 14],
|
|
26
|
+
["viewport", 13],
|
|
27
|
+
["click", 12],
|
|
28
|
+
["fill", 11],
|
|
29
|
+
["type", 10],
|
|
30
|
+
["select", 9],
|
|
31
|
+
["press", 8],
|
|
32
|
+
["wait", 7],
|
|
33
|
+
["hover", 6],
|
|
34
|
+
["locator", 5],
|
|
35
|
+
["evaluate", 4],
|
|
36
|
+
["close", 1],
|
|
37
|
+
]
|
|
38
|
+
|
|
39
|
+
const BLOCKED_BUILTINS = new Set([
|
|
40
|
+
"read",
|
|
41
|
+
"grep",
|
|
42
|
+
"find",
|
|
43
|
+
"ls",
|
|
44
|
+
"bash",
|
|
45
|
+
"powershell",
|
|
46
|
+
"edit",
|
|
47
|
+
"write",
|
|
48
|
+
"ues_cli",
|
|
49
|
+
"ues_execute",
|
|
50
|
+
"ues_dispatch",
|
|
51
|
+
])
|
|
52
|
+
|
|
53
|
+
function textForTool(tool = {}) {
|
|
54
|
+
return [
|
|
55
|
+
tool.name,
|
|
56
|
+
tool.label,
|
|
57
|
+
tool.description,
|
|
58
|
+
].filter(Boolean).join(" ")
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function normalizeExplicitNames(values = []) {
|
|
62
|
+
if (typeof values === "string") values = values.split(",")
|
|
63
|
+
return [...new Set(
|
|
64
|
+
(Array.isArray(values) ? values : [])
|
|
65
|
+
.map((value) => String(value || "").trim())
|
|
66
|
+
.filter(Boolean),
|
|
67
|
+
)]
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export function visualEvidenceNeeded(text = "") {
|
|
71
|
+
const value = String(text || "")
|
|
72
|
+
const capabilities = inferTaskCapabilities(value, { coding: true })
|
|
73
|
+
return capabilities.required.vision === true || VISUAL_SIGNAL.test(value)
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export function browserEvidenceNeeded(text = "", role = "") {
|
|
77
|
+
const normalizedRole = String(role || "").replace(/^ues-/, "")
|
|
78
|
+
if (!BROWSER_ROLES.has(normalizedRole)) return false
|
|
79
|
+
if (normalizedRole === "visual-verifier") return true
|
|
80
|
+
|
|
81
|
+
const capabilities = inferTaskCapabilities(String(text || ""), { coding: true })
|
|
82
|
+
return (
|
|
83
|
+
capabilities.required.browser === true ||
|
|
84
|
+
capabilities.required.vision === true ||
|
|
85
|
+
VISUAL_SIGNAL.test(String(text || ""))
|
|
86
|
+
)
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export function selectBrowserMcpToolNames(tools = [], options = {}) {
|
|
90
|
+
const limit = Math.max(1, Math.min(Number(options.limit || 14), 32))
|
|
91
|
+
const explicit = normalizeExplicitNames(options.explicitNames)
|
|
92
|
+
const rows = []
|
|
93
|
+
const available = new Map()
|
|
94
|
+
|
|
95
|
+
for (const raw of Array.isArray(tools) ? tools : []) {
|
|
96
|
+
const name = String(raw?.name || "").trim()
|
|
97
|
+
if (!name || BLOCKED_BUILTINS.has(name) || name.startsWith("ues_")) continue
|
|
98
|
+
available.set(name, raw)
|
|
99
|
+
|
|
100
|
+
const descriptor = textForTool(raw)
|
|
101
|
+
const providerHit = PROVIDER_SIGNAL.test(descriptor)
|
|
102
|
+
const nameHit = BROWSER_TOOL_NAME.test(name)
|
|
103
|
+
if (!providerHit && !nameHit) continue
|
|
104
|
+
|
|
105
|
+
let score = providerHit ? 100 : 70
|
|
106
|
+
if (/playwright/i.test(descriptor)) score += 30
|
|
107
|
+
if (/^browser_|(?:^|[_:.])browser[_:.]/i.test(name)) score += 20
|
|
108
|
+
for (const [signal, weight] of ACTION_PRIORITY) {
|
|
109
|
+
if (name.toLowerCase().includes(signal)) {
|
|
110
|
+
score += weight
|
|
111
|
+
break
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
rows.push({ name, score })
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
const selected = []
|
|
118
|
+
for (const name of explicit) {
|
|
119
|
+
if (available.has(name) && !selected.includes(name)) selected.push(name)
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
for (const row of rows.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name))) {
|
|
123
|
+
if (selected.length >= limit) break
|
|
124
|
+
if (!selected.includes(row.name)) selected.push(row.name)
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
return selected.slice(0, limit)
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
export function selectBrowserToolsForTask(names = [], text = "", role = "") {
|
|
131
|
+
const value = String(text || "").toLowerCase()
|
|
132
|
+
const normalizedRole = String(role || "").replace(/^ues-/, "")
|
|
133
|
+
const visual = visualEvidenceNeeded(text) || normalizedRole === "visual-verifier"
|
|
134
|
+
const interactive = /(click|fill|type|select|press|submit|login|checkout|flow|interaction|form|navigate|đăng nhập|nhấn|điền|chọn)/i.test(value)
|
|
135
|
+
const diagnostics = /(console|network|request|response|error|failed|api|socket|log)/i.test(value)
|
|
136
|
+
|
|
137
|
+
const keep = []
|
|
138
|
+
const addIf = (name, predicate) => {
|
|
139
|
+
const lower = name.toLowerCase()
|
|
140
|
+
if (predicate(lower) && !keep.includes(name)) keep.push(name)
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
for (const name of names) {
|
|
144
|
+
addIf(name, (lower) => /snapshot|accessibility|navigate|close/.test(lower))
|
|
145
|
+
}
|
|
146
|
+
if (visual) {
|
|
147
|
+
for (const name of names) addIf(name, (lower) => /screenshot|viewport|snapshot|accessibility/.test(lower))
|
|
148
|
+
}
|
|
149
|
+
if (interactive) {
|
|
150
|
+
for (const name of names) addIf(name, (lower) => /click|fill|type|select|press|wait|hover|locator|navigate/.test(lower))
|
|
151
|
+
}
|
|
152
|
+
if (diagnostics || normalizedRole === "verifier" || normalizedRole === "integration-verifier") {
|
|
153
|
+
for (const name of names) addIf(name, (lower) => /console|network|request|response/.test(lower))
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
// Keep a small escape hatch for providers whose naming does not expose action semantics.
|
|
157
|
+
if (keep.length < 3) {
|
|
158
|
+
for (const name of names) {
|
|
159
|
+
if (!keep.includes(name)) keep.push(name)
|
|
160
|
+
if (keep.length >= Math.min(6, names.length)) break
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
const limit = visual || interactive ? 10 : 6
|
|
165
|
+
return keep.slice(0, limit)
|
|
166
|
+
}
|