opencode-agent-skill 7.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. package/CHANGELOG.md +163 -0
  2. package/LICENSE +9 -0
  3. package/README.md +581 -0
  4. package/bin/ocskill.mjs +975 -0
  5. package/docs/DETERMINISTIC-TOOLS.md +88 -0
  6. package/docs/ENGINEERING-DESIGN.md +176 -0
  7. package/docs/EVALS.md +136 -0
  8. package/docs/NPM-PUBLISH.md +102 -0
  9. package/docs/OPENCODE-COMPAT.md +109 -0
  10. package/docs/RESEARCH-SOURCES.md +37 -0
  11. package/docs/TRACE-SCHEMA.md +109 -0
  12. package/docs/V7-INTELLIGENCE-RUNTIME.md +166 -0
  13. package/evals/live/fixtures/engineering-bench/package.json +1 -0
  14. package/evals/live/fixtures/engineering-bench/src/api-errors.mjs +3 -0
  15. package/evals/live/fixtures/engineering-bench/src/authz.mjs +3 -0
  16. package/evals/live/fixtures/engineering-bench/src/cache-tags.mjs +3 -0
  17. package/evals/live/fixtures/engineering-bench/src/config.mjs +3 -0
  18. package/evals/live/fixtures/engineering-bench/src/contract-consumer.mjs +6 -0
  19. package/evals/live/fixtures/engineering-bench/src/contract-producer.mjs +3 -0
  20. package/evals/live/fixtures/engineering-bench/src/dedupe.mjs +3 -0
  21. package/evals/live/fixtures/engineering-bench/src/dependency.mjs +3 -0
  22. package/evals/live/fixtures/engineering-bench/src/discount.mjs +3 -0
  23. package/evals/live/fixtures/engineering-bench/src/inventory.mjs +3 -0
  24. package/evals/live/fixtures/engineering-bench/src/migration.mjs +3 -0
  25. package/evals/live/fixtures/engineering-bench/src/money.mjs +3 -0
  26. package/evals/live/fixtures/engineering-bench/src/pagination.mjs +5 -0
  27. package/evals/live/fixtures/engineering-bench/src/path-safe.mjs +5 -0
  28. package/evals/live/fixtures/engineering-bench/src/payment.mjs +5 -0
  29. package/evals/live/fixtures/engineering-bench/src/query-sort.mjs +3 -0
  30. package/evals/live/fixtures/engineering-bench/src/react-state.mjs +8 -0
  31. package/evals/live/fixtures/engineering-bench/src/retry.mjs +3 -0
  32. package/evals/live/fixtures/engineering-bench/src/rn-platform.mjs +3 -0
  33. package/evals/live/fixtures/engineering-bench/src/upload.mjs +3 -0
  34. package/evals/live/fixtures/engineering-bench/src/webhook.mjs +3 -0
  35. package/evals/live/graders/engineering-bench.mjs +239 -0
  36. package/evals/live/tasks.json +126 -0
  37. package/evals/long/fixtures/long-horizon/package.json +1 -0
  38. package/evals/long/fixtures/long-horizon/src/auth.mjs +5 -0
  39. package/evals/long/fixtures/long-horizon/src/checkout.mjs +9 -0
  40. package/evals/long/fixtures/long-horizon/src/inventory.mjs +5 -0
  41. package/evals/long/fixtures/long-horizon/src/money.mjs +3 -0
  42. package/evals/long/fixtures/long-horizon/src/payment.mjs +7 -0
  43. package/evals/long/fixtures/long-horizon/src/product-api.mjs +10 -0
  44. package/evals/long/fixtures/long-horizon/src/product-cache.mjs +3 -0
  45. package/evals/long/fixtures/long-horizon/src/product-service.mjs +6 -0
  46. package/evals/long/fixtures/long-horizon/src/product-state.mjs +8 -0
  47. package/evals/long/fixtures/long-horizon/src/project-api.mjs +9 -0
  48. package/evals/long/fixtures/long-horizon/src/project-service.mjs +7 -0
  49. package/evals/long/fixtures/long-horizon/src/user-consumer.mjs +3 -0
  50. package/evals/long/fixtures/long-horizon/src/user-migration.mjs +7 -0
  51. package/evals/long/fixtures/long-horizon/src/user-serializer.mjs +3 -0
  52. package/evals/long/fixtures/long-horizon/src/user-validation.mjs +3 -0
  53. package/evals/long/graders/long-horizon.mjs +209 -0
  54. package/evals/long/tasks.json +36 -0
  55. package/evals/router-triggers.json +1238 -0
  56. package/evals/routing.json +321 -0
  57. package/global-config/AGENTS.md +212 -0
  58. package/global-config/agents/architect.md +38 -0
  59. package/global-config/agents/codebase-mapper.md +41 -0
  60. package/global-config/agents/critic.md +37 -0
  61. package/global-config/agents/debugger.md +36 -0
  62. package/global-config/agents/executor.md +48 -0
  63. package/global-config/agents/integration-verifier.md +43 -0
  64. package/global-config/agents/plan-checker.md +43 -0
  65. package/global-config/agents/researcher.md +30 -0
  66. package/global-config/agents/reviewer.md +30 -0
  67. package/global-config/agents/verifier.md +33 -0
  68. package/global-config/commands/audit.md +8 -0
  69. package/global-config/commands/critique.md +8 -0
  70. package/global-config/commands/debug.md +8 -0
  71. package/global-config/commands/feature.md +8 -0
  72. package/global-config/commands/fix.md +8 -0
  73. package/global-config/commands/plan.md +8 -0
  74. package/global-config/commands/research.md +8 -0
  75. package/global-config/commands/resume.md +22 -0
  76. package/global-config/commands/review.md +8 -0
  77. package/global-config/commands/run.md +29 -0
  78. package/global-config/commands/verify.md +8 -0
  79. package/global-config/plugins/ues-router/capabilities.js +20 -0
  80. package/global-config/plugins/ues-router/index.js +277 -0
  81. package/global-config/plugins/ues-router/router.js +74 -0
  82. package/global-config/plugins/ues-router/safety.js +16 -0
  83. package/global-config/skills/accessibility/SKILL.md +10 -0
  84. package/global-config/skills/accessibility/references/workflow.md +17 -0
  85. package/global-config/skills/api-contract/SKILL.md +12 -0
  86. package/global-config/skills/api-contract/references/workflow.md +17 -0
  87. package/global-config/skills/auth-security/SKILL.md +12 -0
  88. package/global-config/skills/auth-security/references/workflow.md +15 -0
  89. package/global-config/skills/bug-diagnosis/SKILL.md +28 -0
  90. package/global-config/skills/change-impact-analysis/SKILL.md +23 -0
  91. package/global-config/skills/code-review/SKILL.md +19 -0
  92. package/global-config/skills/context-engineering/SKILL.md +18 -0
  93. package/global-config/skills/context-engineering/references/large-repo.md +16 -0
  94. package/global-config/skills/database-engineering/SKILL.md +12 -0
  95. package/global-config/skills/database-engineering/references/workflow.md +15 -0
  96. package/global-config/skills/dependency-management/SKILL.md +12 -0
  97. package/global-config/skills/dependency-management/references/workflow.md +14 -0
  98. package/global-config/skills/devops-engineering/SKILL.md +10 -0
  99. package/global-config/skills/devops-engineering/references/workflow.md +11 -0
  100. package/global-config/skills/django-engineering/SKILL.md +10 -0
  101. package/global-config/skills/django-engineering/references/workflow.md +11 -0
  102. package/global-config/skills/documentation-engineering/SKILL.md +10 -0
  103. package/global-config/skills/documentation-engineering/references/workflow.md +15 -0
  104. package/global-config/skills/dotnet-engineering/SKILL.md +10 -0
  105. package/global-config/skills/dotnet-engineering/references/workflow.md +11 -0
  106. package/global-config/skills/ecommerce-engineering/SKILL.md +10 -0
  107. package/global-config/skills/ecommerce-engineering/references/workflow.md +17 -0
  108. package/global-config/skills/engineering-orchestrator/SKILL.md +29 -0
  109. package/global-config/skills/engineering-orchestrator/references/delegation.md +22 -0
  110. package/global-config/skills/engineering-orchestrator/references/evaluator-loop.md +18 -0
  111. package/global-config/skills/engineering-orchestrator/references/long-horizon.md +57 -0
  112. package/global-config/skills/engineering-orchestrator/references/model-escalation.md +19 -0
  113. package/global-config/skills/engineering-orchestrator/references/retry-policy.md +12 -0
  114. package/global-config/skills/engineering-orchestrator/references/routing.md +39 -0
  115. package/global-config/skills/engineering-orchestrator/references/verification-matrix.md +18 -0
  116. package/global-config/skills/fastapi-engineering/SKILL.md +10 -0
  117. package/global-config/skills/fastapi-engineering/references/workflow.md +13 -0
  118. package/global-config/skills/file-upload-engineering/SKILL.md +10 -0
  119. package/global-config/skills/file-upload-engineering/references/workflow.md +17 -0
  120. package/global-config/skills/flutter-engineering/SKILL.md +10 -0
  121. package/global-config/skills/flutter-engineering/references/workflow.md +13 -0
  122. package/global-config/skills/git-safety/SKILL.md +10 -0
  123. package/global-config/skills/git-safety/references/workflow.md +15 -0
  124. package/global-config/skills/implementation-engineer/SKILL.md +10 -0
  125. package/global-config/skills/implementation-engineer/references/workflow.md +14 -0
  126. package/global-config/skills/java-spring-engineering/SKILL.md +10 -0
  127. package/global-config/skills/java-spring-engineering/references/workflow.md +13 -0
  128. package/global-config/skills/long-task-state/SKILL.md +28 -0
  129. package/global-config/skills/long-task-state/references/context-ledger.md +27 -0
  130. package/global-config/skills/long-task-state/templates/STATE.md +48 -0
  131. package/global-config/skills/nestjs-engineering/SKILL.md +10 -0
  132. package/global-config/skills/nestjs-engineering/references/workflow.md +11 -0
  133. package/global-config/skills/nextjs-engineering/SKILL.md +12 -0
  134. package/global-config/skills/nextjs-engineering/references/workflow.md +15 -0
  135. package/global-config/skills/nodejs-engineering/SKILL.md +12 -0
  136. package/global-config/skills/nodejs-engineering/references/workflow.md +11 -0
  137. package/global-config/skills/payment-engineering/SKILL.md +12 -0
  138. package/global-config/skills/payment-engineering/references/workflow.md +19 -0
  139. package/global-config/skills/performance-engineering/SKILL.md +10 -0
  140. package/global-config/skills/performance-engineering/references/workflow.md +17 -0
  141. package/global-config/skills/python-engineering/SKILL.md +10 -0
  142. package/global-config/skills/python-engineering/references/workflow.md +11 -0
  143. package/global-config/skills/react-engineering/SKILL.md +14 -0
  144. package/global-config/skills/react-engineering/references/workflow.md +16 -0
  145. package/global-config/skills/react-native-engineering/SKILL.md +14 -0
  146. package/global-config/skills/react-native-engineering/references/workflow.md +16 -0
  147. package/global-config/skills/repo-explorer/SKILL.md +17 -0
  148. package/global-config/skills/research-verification/SKILL.md +18 -0
  149. package/global-config/skills/research-verification/references/source-hierarchy.md +12 -0
  150. package/global-config/skills/rest-api-design/SKILL.md +10 -0
  151. package/global-config/skills/rest-api-design/references/workflow.md +18 -0
  152. package/global-config/skills/software-architect/SKILL.md +10 -0
  153. package/global-config/skills/software-architect/references/workflow.md +18 -0
  154. package/global-config/skills/task-planner/SKILL.md +21 -0
  155. package/global-config/skills/task-planner/references/plan-schema.md +56 -0
  156. package/global-config/skills/test-driven-development/SKILL.md +22 -0
  157. package/global-config/skills/test-driven-development/references/writing-good-tests.md +21 -0
  158. package/global-config/skills/test-verification/SKILL.md +20 -0
  159. package/global-config/skills/ui-ux-engineering/SKILL.md +10 -0
  160. package/global-config/skills/ui-ux-engineering/references/workflow.md +17 -0
  161. package/global-config/skills/web-security-review/SKILL.md +10 -0
  162. package/global-config/skills/web-security-review/references/workflow.md +22 -0
  163. package/lib/context-manifest.mjs +101 -0
  164. package/lib/control-center.mjs +148 -0
  165. package/lib/eval-auth.mjs +21 -0
  166. package/lib/eval-report.mjs +72 -0
  167. package/lib/eval-telemetry.mjs +155 -0
  168. package/lib/evidence-receipt.mjs +50 -0
  169. package/lib/hermes-bridge.mjs +28 -0
  170. package/lib/ids.mjs +21 -0
  171. package/lib/installer.mjs +636 -0
  172. package/lib/learning-engine.mjs +161 -0
  173. package/lib/model-config.mjs +88 -0
  174. package/lib/model-policy.mjs +71 -0
  175. package/lib/opencode-compat.mjs +86 -0
  176. package/lib/orchestrator-policy.mjs +35 -0
  177. package/lib/process-runner.mjs +117 -0
  178. package/lib/repo-graph.mjs +135 -0
  179. package/lib/repo-inspect.mjs +264 -0
  180. package/lib/review-scope.mjs +98 -0
  181. package/lib/router-config.mjs +40 -0
  182. package/lib/task-engine.mjs +777 -0
  183. package/lib/task-graph.mjs +262 -0
  184. package/lib/update-resolver.mjs +43 -0
  185. package/lib/verification-plan.mjs +52 -0
  186. package/lib/version.mjs +47 -0
  187. package/lib/workspace-snapshot.mjs +45 -0
  188. package/lib/worktree-sandbox.mjs +59 -0
  189. package/package.json +69 -0
  190. package/scripts/check-working-tree.mjs +2 -0
  191. package/scripts/collect-evidence.mjs +2 -0
  192. package/scripts/control-center.mjs +37 -0
  193. package/scripts/detect-stack.mjs +2 -0
  194. package/scripts/detect-test-commands.mjs +2 -0
  195. package/scripts/eval-live.mjs +437 -0
  196. package/scripts/eval-report.mjs +49 -0
  197. package/scripts/eval-router.mjs +45 -0
  198. package/scripts/eval-skills.mjs +71 -0
  199. package/scripts/impact-map.mjs +4 -0
  200. package/scripts/install.mjs +61 -0
  201. package/scripts/repo-map.mjs +2 -0
  202. package/scripts/smoke-packed-install.mjs +326 -0
  203. package/scripts/syntax-check.mjs +35 -0
  204. package/scripts/uninstall.mjs +21 -0
  205. package/scripts/validate-live-suite.mjs +65 -0
  206. package/scripts/validate.mjs +133 -0
@@ -0,0 +1,437 @@
1
+ import { existsSync, readFileSync } from "node:fs"
2
+ import { cp, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"
3
+ import os from "node:os"
4
+ import path from "node:path"
5
+ import { spawnSync } from "node:child_process"
6
+ import { fileURLToPath } from "node:url"
7
+ import { installResources } from "../lib/installer.mjs"
8
+ import { copyCurrentOpenCodeAuth } from "../lib/eval-auth.mjs"
9
+ import { parseOpenCodeTelemetry } from "../lib/eval-telemetry.mjs"
10
+ import { buildOpenCodeRunArgs, parseOpenCodeMajor } from "../lib/opencode-compat.mjs"
11
+ import { snapshotWorkspace, diffWorkspaceSnapshots } from "../lib/workspace-snapshot.mjs"
12
+ import { runProcess } from "../lib/process-runner.mjs"
13
+
14
+ const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..")
15
+ const args = process.argv.slice(2)
16
+ const abortController = new AbortController()
17
+ let interrupted = false
18
+ process.once("SIGINT", () => {
19
+ interrupted = true
20
+ console.warn("\n[eval] interrupt requested; stopping the active OpenCode process tree...")
21
+ abortController.abort()
22
+ })
23
+
24
+ function argValue(name, fallback = null) {
25
+ const index = args.indexOf(name)
26
+ return index >= 0 && index + 1 < args.length ? args[index + 1] : fallback
27
+ }
28
+
29
+ function hasArg(name) {
30
+ return args.includes(name)
31
+ }
32
+
33
+ function positiveInt(value, fallback) {
34
+ const parsed = Number.parseInt(value ?? "", 10)
35
+ return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback
36
+ }
37
+
38
+ function findWindowsCommand(name) {
39
+ if (path.isAbsolute(name) && existsSync(name)) return name
40
+ const result = spawnSync("where", [name], { encoding: "utf8" })
41
+ if (result.status !== 0 || !result.stdout) return null
42
+ const matches = result.stdout.split(/\r?\n/).map((line) => line.trim()).filter(Boolean)
43
+ return matches.find((item) => /\.(exe|cmd|bat)$/i.test(item)) || matches[0] || null
44
+ }
45
+
46
+ function quoteCmd(value) {
47
+ if (/^[A-Za-z0-9_@%+=:,./\\-]+$/.test(value)) return value
48
+ return '"' + value.replaceAll('"', '""') + '"'
49
+ }
50
+
51
+ function findNodeShimEntry(cmdPath) {
52
+ const dir = path.dirname(cmdPath)
53
+ let shim = ""
54
+ try {
55
+ shim = readFileSync(cmdPath, "utf8")
56
+ } catch {
57
+ return null
58
+ }
59
+ const match = shim.match(/node_modules[\\/][^\s"]+?\.(?:js|mjs)/gi)?.at(-1)
60
+ if (!match) return null
61
+ const entry = path.resolve(dir, match)
62
+ return existsSync(entry) ? entry : null
63
+ }
64
+
65
+ function runCommand(executable, commandArgs, options = {}) {
66
+ if (process.platform !== "win32") {
67
+ return spawnSync(executable, commandArgs, options)
68
+ }
69
+
70
+ const resolved = findWindowsCommand(executable)
71
+ if (!resolved) {
72
+ return { status: 127, stdout: "", stderr: "Command not found: " + executable }
73
+ }
74
+
75
+ if (/\.(cmd|bat)$/i.test(resolved)) {
76
+ const entry = findNodeShimEntry(resolved)
77
+ if (entry) return spawnSync(process.execPath, [entry, ...commandArgs], options)
78
+
79
+ const line = [resolved, ...commandArgs].map(quoteCmd).join(" ")
80
+ return spawnSync(
81
+ process.env.ComSpec || "cmd.exe",
82
+ ["/d", "/s", "/c", line],
83
+ options,
84
+ )
85
+ }
86
+
87
+ return spawnSync(resolved, commandArgs, options)
88
+ }
89
+
90
+
91
+ function runAsyncCommand(executable, commandArgs, options = {}) {
92
+ if (process.platform !== "win32") return runProcess(executable, commandArgs, options)
93
+
94
+ const resolved = findWindowsCommand(executable)
95
+ if (!resolved) {
96
+ return Promise.resolve({
97
+ status: 127,
98
+ signal: null,
99
+ stdout: "",
100
+ stderr: "Command not found: " + executable,
101
+ durationMs: 0,
102
+ timedOut: false,
103
+ idleTimedOut: false,
104
+ aborted: false,
105
+ })
106
+ }
107
+
108
+ if (/\.(cmd|bat)$/i.test(resolved)) {
109
+ const entry = findNodeShimEntry(resolved)
110
+ if (entry) return runProcess(process.execPath, [entry, ...commandArgs], options)
111
+ const line = [resolved, ...commandArgs].map(quoteCmd).join(" ")
112
+ return runProcess(process.env.ComSpec || "cmd.exe", ["/d", "/s", "/c", line], options)
113
+ }
114
+
115
+ return runProcess(resolved, commandArgs, options)
116
+ }
117
+
118
+ function excerpt(value, limit = 12000) {
119
+ if (!value) return ""
120
+ const text = String(value)
121
+ return text.length <= limit ? text : text.slice(0, limit) + "\n...[truncated]"
122
+ }
123
+
124
+ async function inspectLongOrchestration(workspace) {
125
+ const workRoot = path.join(workspace, ".ues-work")
126
+ const dirs = await readdir(workRoot, { withFileTypes: true }).catch(() => [])
127
+ const items = []
128
+
129
+ for (const entry of dirs) {
130
+ if (!entry.isDirectory()) continue
131
+ const dir = path.join(workRoot, entry.name)
132
+ try {
133
+ const [state, plan, evidence] = await Promise.all([
134
+ readFile(path.join(dir, "STATE.json"), "utf8").then(JSON.parse),
135
+ readFile(path.join(dir, "PLAN.json"), "utf8").then(JSON.parse),
136
+ readFile(path.join(dir, "EVIDENCE.json"), "utf8").then(JSON.parse),
137
+ ])
138
+ const tasks = Object.values(state.tasks || {})
139
+ const evidenceTasks = new Set((evidence.entries || []).map((item) => item.task))
140
+ const receiptBackedTasks = new Set(
141
+ (evidence.receipts || []).filter((item) => item.passed).map((item) => item.task),
142
+ )
143
+ const plannedIDs = (plan.tasks || []).map((item) => item.id)
144
+ const item = {
145
+ slug: entry.name,
146
+ taskCount: Array.isArray(plan.tasks) ? plan.tasks.length : 0,
147
+ planApproved: state.planApproval?.status === "passed",
148
+ attemptedTasks: tasks.filter((task) => Number(task.attempts || 0) > 0).length,
149
+ completedTasks: tasks.filter((task) => task.status === "completed").length,
150
+ receiptBackedTasks: plannedIDs.filter((id) => receiptBackedTasks.has(id)).length,
151
+ receiptCoverage: plannedIDs.length
152
+ ? plannedIDs.filter((id) => receiptBackedTasks.has(id)).length / plannedIDs.length
153
+ : 0,
154
+ integrationPassed: state.integrationVerification?.status === "PASS",
155
+ integrationEvidence: evidenceTasks.has("__integration_verification__"),
156
+ finalizedEvidence: evidenceTasks.has("__integration__"),
157
+ completed: state.status === "completed",
158
+ }
159
+ item.valid =
160
+ item.taskCount >= 2 &&
161
+ item.planApproved &&
162
+ item.attemptedTasks === item.taskCount &&
163
+ item.completedTasks === item.taskCount &&
164
+ item.receiptBackedTasks === item.taskCount &&
165
+ item.integrationPassed &&
166
+ item.integrationEvidence &&
167
+ item.finalizedEvidence &&
168
+ item.completed
169
+ items.push(item)
170
+ } catch (error) {
171
+ items.push({ slug: entry.name, valid: false, error: String(error?.message || error) })
172
+ }
173
+ }
174
+
175
+ return {
176
+ required: true,
177
+ valid: items.some((item) => item.valid),
178
+ items,
179
+ }
180
+ }
181
+
182
+ const model = argValue("--model", process.env.UES_EVAL_MODEL)
183
+ const variant = argValue("--variant", process.env.UES_EVAL_VARIANT)
184
+ const trials = Math.min(positiveInt(argValue("--trials"), 1), 20)
185
+ const taskFilter = argValue("--task")
186
+ const requestedMode = argValue("--mode", "both")
187
+ const keep = hasArg("--keep")
188
+ const authMode = argValue("--auth", "env-only")
189
+ const suiteName = argValue("--suite", "live")
190
+ const timeoutMs = positiveInt(argValue("--timeout-ms"), 15 * 60_000)
191
+ const idleTimeoutMs = positiveInt(argValue("--idle-timeout-ms"), 5 * 60_000)
192
+ const heartbeatMs = positiveInt(argValue("--heartbeat-ms"), 30_000)
193
+ const suiteRoot = path.join(root, "evals", suiteName)
194
+
195
+ if (!model) {
196
+ console.error("Usage: node scripts/eval-live.mjs --model provider/model [--suite live|long] [--variant high] [--trials N] [--task id] [--mode baseline|ues|both] [--auth env-only|current] [--output-dir path] [--keep] [--timeout-ms N] [--idle-timeout-ms N] [--heartbeat-ms N]")
197
+ console.error("You can also set UES_EVAL_MODEL and UES_EVAL_VARIANT.")
198
+ process.exit(2)
199
+ }
200
+
201
+ if (!["baseline", "ues", "both"].includes(requestedMode)) {
202
+ console.error("--mode must be baseline, ues, or both")
203
+ process.exit(2)
204
+ }
205
+
206
+ if (!["env-only", "current"].includes(authMode)) {
207
+ console.error("--auth must be env-only or current")
208
+ process.exit(2)
209
+ }
210
+
211
+ const probe = runCommand("opencode", ["--version"], { encoding: "utf8" })
212
+ if (probe.status !== 0) {
213
+ console.error("OpenCode CLI is required for live evals.")
214
+ console.error(excerpt(probe.stderr || probe.stdout))
215
+ process.exit(2)
216
+ }
217
+ const opencodeVersion = String(probe.stdout || probe.stderr || "").trim()
218
+ const parsedOpenCodeMajor = parseOpenCodeMajor(opencodeVersion)
219
+ const opencodeMajor = parsedOpenCodeMajor ?? 1
220
+ if (parsedOpenCodeMajor === null) {
221
+ console.warn("[eval] could not parse OpenCode version; using the conservative OpenCode 1.x invocation.")
222
+ }
223
+
224
+ if (!["live", "long"].includes(suiteName)) {
225
+ console.error("--suite must be live or long")
226
+ process.exit(2)
227
+ }
228
+
229
+ const suite = JSON.parse(await readFile(path.join(suiteRoot, "tasks.json"), "utf8"))
230
+ let tasks = suite.tasks || []
231
+ if (taskFilter) tasks = tasks.filter((task) => task.id === taskFilter)
232
+ if (tasks.length === 0) {
233
+ console.error(taskFilter ? "Unknown task: " + taskFilter : "No live eval tasks found.")
234
+ process.exit(2)
235
+ }
236
+
237
+ const modes = requestedMode === "both" ? ["baseline", "ues"] : [requestedMode]
238
+ const invocationDir = process.cwd()
239
+ const runRoot = await mkdtemp(path.join(os.tmpdir(), "ues-live-eval-"))
240
+ const resultDir = path.resolve(
241
+ argValue("--output-dir", path.join(invocationDir, ".ues-evals")),
242
+ )
243
+ await mkdir(resultDir, { recursive: true })
244
+
245
+ const results = []
246
+ const oldConfigDir = process.env.OPENCODE_CONFIG_DIR
247
+
248
+ try {
249
+ for (const task of tasks) {
250
+ if (interrupted) break
251
+ for (const mode of modes) {
252
+ if (interrupted) break
253
+ for (let trial = 1; trial <= trials; trial += 1) {
254
+ if (interrupted) break
255
+ const isolatedRoot = path.join(runRoot, task.id + "-" + mode + "-" + trial)
256
+ const workspace = path.join(isolatedRoot, "workspace")
257
+ const isolatedHome = path.join(isolatedRoot, "home")
258
+ const xdgRoot = path.join(isolatedHome, ".config")
259
+ const configDir = path.join(xdgRoot, "opencode")
260
+ const dataRoot = path.join(isolatedRoot, "data")
261
+ const cacheRoot = path.join(isolatedRoot, "cache")
262
+ const stateRoot = path.join(isolatedRoot, "state")
263
+
264
+ await mkdir(configDir, { recursive: true })
265
+ await mkdir(dataRoot, { recursive: true })
266
+ await mkdir(cacheRoot, { recursive: true })
267
+ await mkdir(stateRoot, { recursive: true })
268
+ await cp(path.join(suiteRoot, task.fixture), workspace, { recursive: true })
269
+ const beforeSnapshot = await snapshotWorkspace(workspace)
270
+
271
+ if (authMode === "current") {
272
+ const auth = await copyCurrentOpenCodeAuth(dataRoot)
273
+ if (!auth.copied) {
274
+ console.warn("[eval] current auth requested but auth.json was not found; continuing with environment credentials only.")
275
+ }
276
+ }
277
+
278
+ if (mode === "ues") {
279
+ process.env.OPENCODE_CONFIG_DIR = configDir
280
+ const installed = await installResources({ sourceRoot: root })
281
+ if (installed.stateError) {
282
+ throw new Error("UES install failed for live eval: " + installed.stateError)
283
+ }
284
+ }
285
+
286
+ const childEnv = {
287
+ ...process.env,
288
+ HOME: isolatedHome,
289
+ USERPROFILE: isolatedHome,
290
+ OPENCODE_CONFIG_DIR: configDir,
291
+ XDG_CONFIG_HOME: xdgRoot,
292
+ XDG_DATA_HOME: dataRoot,
293
+ XDG_CACHE_HOME: cacheRoot,
294
+ XDG_STATE_HOME: stateRoot,
295
+ OPENCODE_DISABLE_AUTOUPDATE: "true",
296
+ }
297
+
298
+ const opencodeArgs = buildOpenCodeRunArgs({
299
+ major: opencodeMajor,
300
+ model,
301
+ workspace,
302
+ variant,
303
+ prompt: task.prompt,
304
+ })
305
+
306
+ console.log("[" + mode + "] " + task.id + " trial " + trial + ": starting agent")
307
+ const agentRun = await runAsyncCommand("opencode", opencodeArgs, {
308
+ cwd: workspace,
309
+ env: childEnv,
310
+ maxBuffer: 4 * 1024 * 1024,
311
+ heartbeatMs,
312
+ timeoutMs,
313
+ idleTimeoutMs,
314
+ signal: abortController.signal,
315
+ onHeartbeat: ({ elapsedMs, idleMs }) => {
316
+ console.log(
317
+ "[" + mode + "] " + task.id + " trial " + trial +
318
+ ": still running (" + Math.round(elapsedMs / 1000) + "s elapsed, " +
319
+ Math.round(idleMs / 1000) + "s since output)",
320
+ )
321
+ },
322
+ })
323
+ const durationMs = agentRun.durationMs
324
+
325
+ const graderPath = path.join(suiteRoot, task.grader)
326
+ const graderRun = spawnSync(process.execPath, [graderPath], {
327
+ cwd: workspace,
328
+ env: { ...childEnv, UES_EVAL_WORKSPACE: workspace, UES_EVAL_TASK: task.id },
329
+ encoding: "utf8",
330
+ maxBuffer: 1024 * 1024,
331
+ })
332
+
333
+ const afterSnapshot = await snapshotWorkspace(workspace)
334
+ const telemetry = parseOpenCodeTelemetry(agentRun.stdout)
335
+ const changedFiles = diffWorkspaceSnapshots(beforeSnapshot, afterSnapshot)
336
+ const orchestration =
337
+ suiteName === "long" && mode === "ues"
338
+ ? await inspectLongOrchestration(workspace)
339
+ : { required: false, valid: true, items: [] }
340
+ const passed =
341
+ agentRun.status === 0 &&
342
+ graderRun.status === 0 &&
343
+ (!orchestration.required || orchestration.valid)
344
+ results.push({
345
+ task: task.id,
346
+ mode,
347
+ model,
348
+ variant: variant || null,
349
+ trial,
350
+ passed,
351
+ agentExit: agentRun.status,
352
+ graderExit: graderRun.status,
353
+ durationMs,
354
+ timedOut: agentRun.timedOut,
355
+ idleTimedOut: agentRun.idleTimedOut,
356
+ aborted: agentRun.aborted,
357
+ authMode,
358
+ opencodeVersion,
359
+ opencodeMajor,
360
+ telemetry,
361
+ changedFiles,
362
+ orchestration,
363
+ agentStdout: excerpt(agentRun.stdout),
364
+ agentStderr: excerpt(agentRun.stderr),
365
+ graderStdout: excerpt(graderRun.stdout),
366
+ graderStderr: excerpt(graderRun.stderr),
367
+ workspace: keep ? workspace : null,
368
+ timestamp: new Date().toISOString(),
369
+ })
370
+
371
+ if (agentRun.aborted) interrupted = true
372
+ console.log(
373
+ "[" + mode + "] " + task.id + " trial " + trial + ": " +
374
+ (passed ? "PASS" : "FAIL") +
375
+ " (agent=" + agentRun.status + ", grader=" + graderRun.status +
376
+ (orchestration.required ? ", orchestration=" + (orchestration.valid ? "PASS" : "FAIL") : "") +
377
+ (agentRun.timedOut ? ", timeout=hard" : agentRun.idleTimedOut ? ", timeout=idle" : "") +
378
+ ", " + durationMs + "ms)",
379
+ )
380
+ }
381
+ }
382
+ }
383
+ } finally {
384
+ if (oldConfigDir === undefined) delete process.env.OPENCODE_CONFIG_DIR
385
+ else process.env.OPENCODE_CONFIG_DIR = oldConfigDir
386
+
387
+ if (!keep) await rm(runRoot, { recursive: true, force: true })
388
+ }
389
+
390
+ const summary = {}
391
+ for (const mode of modes) {
392
+ const subset = results.filter((item) => item.mode === mode)
393
+ const passed = subset.filter((item) => item.passed).length
394
+ summary[mode] = {
395
+ passed,
396
+ total: subset.length,
397
+ passRate: subset.length ? passed / subset.length : 0,
398
+ }
399
+ }
400
+
401
+ const safeModel = model.replace(/[^a-zA-Z0-9._-]+/g, "-")
402
+ const stamp = new Date().toISOString().replace(/[:.]/g, "-")
403
+ const resultFile = path.join(resultDir, stamp + "-" + suiteName + "-" + safeModel + ".json")
404
+ await writeFile(
405
+ resultFile,
406
+ JSON.stringify(
407
+ {
408
+ schemaVersion: 1,
409
+ suite: suiteName,
410
+ suiteVersion: suite.version,
411
+ model,
412
+ variant: variant || null,
413
+ trials,
414
+ taskFilter: taskFilter || null,
415
+ authMode,
416
+ timeoutMs,
417
+ idleTimeoutMs,
418
+ heartbeatMs,
419
+ opencodeVersion,
420
+ opencodeMajor,
421
+ modes,
422
+ summary,
423
+ results,
424
+ },
425
+ null,
426
+ 2,
427
+ ) + "\n",
428
+ "utf8",
429
+ )
430
+
431
+ console.log("\nSummary")
432
+ for (const [mode, item] of Object.entries(summary)) {
433
+ console.log("- " + mode + ": " + item.passed + "/" + item.total + " (" + (item.passRate * 100).toFixed(1) + "%)")
434
+ }
435
+ console.log("Result: " + resultFile)
436
+ if (keep) console.log("Temporary workspaces kept under: " + runRoot)
437
+ if (interrupted) process.exitCode = 130
@@ -0,0 +1,49 @@
1
+ import { existsSync } from "node:fs"
2
+ import { readFile, readdir } from "node:fs/promises"
3
+ import path from "node:path"
4
+ import { summarizeEvalResults } from "../lib/eval-report.mjs"
5
+
6
+ const args = process.argv.slice(2)
7
+ let files = []
8
+
9
+ if (args.length) {
10
+ for (const value of args) {
11
+ const resolved = path.resolve(value)
12
+ if (!existsSync(resolved)) {
13
+ console.error("Eval result path not found: " + resolved)
14
+ process.exit(2)
15
+ }
16
+ if (resolved.endsWith(".json")) files.push(resolved)
17
+ else {
18
+ const entries = await readdir(resolved)
19
+ files.push(...entries.filter((name) => name.endsWith(".json")).map((name) => path.join(resolved, name)))
20
+ }
21
+ }
22
+ } else {
23
+ const dir = path.resolve(".ues-evals")
24
+ if (existsSync(dir)) {
25
+ const entries = await readdir(dir)
26
+ files = entries.filter((name) => name.endsWith(".json")).map((name) => path.join(dir, name))
27
+ }
28
+ }
29
+
30
+ if (files.length === 0) {
31
+ console.error("No eval result JSON files found. Pass files/directories or run live evals first.")
32
+ process.exit(2)
33
+ }
34
+
35
+ const runs = []
36
+ const results = []
37
+ for (const file of [...new Set(files)].sort()) {
38
+ const parsed = JSON.parse(await readFile(file, "utf8"))
39
+ runs.push({ file, model: parsed.model, variant: parsed.variant, trials: parsed.trials })
40
+ results.push(...(parsed.results || []))
41
+ }
42
+
43
+ const summary = summarizeEvalResults(results)
44
+ console.log(JSON.stringify({
45
+ schemaVersion: 1,
46
+ files: runs,
47
+ totalResults: results.length,
48
+ ...summary,
49
+ }, null, 2))
@@ -0,0 +1,45 @@
1
+ import { readFile } from "node:fs/promises"
2
+ import path from "node:path"
3
+ import { fileURLToPath } from "node:url"
4
+ import { routeSkills } from "../global-config/plugins/ues-router/router.js"
5
+
6
+ const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..")
7
+ const suite = JSON.parse(await readFile(path.join(root, "evals", "router-triggers.json"), "utf8"))
8
+ const failures = []
9
+ let positives = 0
10
+ let positiveHits = 0
11
+ let negatives = 0
12
+ let negativeHits = 0
13
+
14
+ for (const item of suite.cases || []) {
15
+ const routed = routeSkills(item.prompt, item.maxSkills || 6)
16
+
17
+ for (const id of item.include || []) {
18
+ positives += 1
19
+ if (routed.includes(id)) positiveHits += 1
20
+ else failures.push(item.id + ": missing required route " + id + " from [" + routed.join(", ") + "]")
21
+ }
22
+
23
+ for (const id of item.exclude || []) {
24
+ negatives += 1
25
+ if (!routed.includes(id)) negativeHits += 1
26
+ else failures.push(item.id + ": unexpected route " + id + " in [" + routed.join(", ") + "]")
27
+ }
28
+ }
29
+
30
+ const recall = positives ? positiveHits / positives : 1
31
+ const specificity = negatives ? negativeHits / negatives : 1
32
+
33
+ if (failures.length) {
34
+ console.error("Router trigger evaluation failed:")
35
+ for (const failure of failures) console.error("- " + failure)
36
+ console.error("Recall: " + (recall * 100).toFixed(1) + "%")
37
+ console.error("Negative guard accuracy: " + (specificity * 100).toFixed(1) + "%")
38
+ process.exit(1)
39
+ }
40
+
41
+ console.log(
42
+ "Validated " + suite.cases.length + " router cases: " +
43
+ positiveHits + "/" + positives + " required routes and " +
44
+ negativeHits + "/" + negatives + " negative guards passed.",
45
+ )
@@ -0,0 +1,71 @@
1
+ import { readFile, readdir } from "node:fs/promises"
2
+ import path from "node:path"
3
+ import { fileURLToPath } from "node:url"
4
+
5
+ const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..")
6
+ const skillsRoot = path.join(root, "global-config", "skills")
7
+ const evalFile = path.join(root, "evals", "routing.json")
8
+
9
+ const skillEntries = await readdir(skillsRoot, { withFileTypes: true })
10
+ const skillIDs = new Set(skillEntries.filter((entry) => entry.isDirectory()).map((entry) => entry.name))
11
+ const suite = JSON.parse(await readFile(evalFile, "utf8"))
12
+ const errors = []
13
+ const names = new Set()
14
+ const referenced = new Set()
15
+
16
+ if (!Array.isArray(suite.scenarios) || suite.scenarios.length < 30) {
17
+ errors.push("routing suite must contain at least 30 scenarios")
18
+ }
19
+
20
+ for (const scenario of suite.scenarios || []) {
21
+ if (!scenario.name || names.has(scenario.name)) {
22
+ errors.push(`invalid or duplicate scenario name: ${scenario.name || "<missing>"}`)
23
+ continue
24
+ }
25
+ names.add(scenario.name)
26
+
27
+ if (!scenario.prompt || typeof scenario.prompt !== "string") {
28
+ errors.push(`${scenario.name}: missing prompt`)
29
+ }
30
+
31
+ if (!Array.isArray(scenario.expect) || scenario.expect.length === 0) {
32
+ errors.push(`${scenario.name}: expect must contain at least one skill`)
33
+ continue
34
+ }
35
+
36
+ if (scenario.expect.length > 5) {
37
+ errors.push(`${scenario.name}: expects more than five skills; routing should stay focused`)
38
+ }
39
+
40
+ for (const id of scenario.expect) {
41
+ referenced.add(id)
42
+ if (!skillIDs.has(id)) errors.push(`${scenario.name}: unknown skill ${id}`)
43
+ }
44
+ }
45
+
46
+ if (!referenced.has("engineering-orchestrator")) {
47
+ errors.push("routing suite must exercise engineering-orchestrator")
48
+ }
49
+ if (!referenced.has("bug-diagnosis")) {
50
+ errors.push("routing suite must exercise bug-diagnosis")
51
+ }
52
+ if (!referenced.has("test-verification")) {
53
+ errors.push("routing suite must exercise test-verification")
54
+ }
55
+ if (!referenced.has("research-verification")) {
56
+ errors.push("routing suite must exercise research-verification")
57
+ }
58
+
59
+ for (const id of [...skillIDs].sort()) {
60
+ if (!referenced.has(id)) errors.push(`routing suite does not exercise skill ${id}`)
61
+ }
62
+
63
+ if (errors.length) {
64
+ console.error("Skill routing eval contract failed:")
65
+ for (const error of errors) console.error(`- ${error}`)
66
+ process.exit(1)
67
+ }
68
+
69
+ console.log(
70
+ `Validated ${suite.scenarios.length} routing scenarios across ${referenced.size} referenced skills.`,
71
+ )
@@ -0,0 +1,4 @@
1
+ import { impactMap } from "../lib/repo-inspect.mjs"
2
+ const query = process.argv[2]
3
+ if (!query) { console.error("Usage: node scripts/impact-map.mjs <query> [dir]"); process.exit(2) }
4
+ console.log(JSON.stringify(await impactMap(process.argv[3] || process.cwd(), query), null, 2))
@@ -0,0 +1,61 @@
1
+ import { spawnSync } from "node:child_process"
2
+ import path from "node:path"
3
+ import { fileURLToPath } from "node:url"
4
+ import { installResources } from "../lib/installer.mjs"
5
+
6
+ const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..")
7
+
8
+ function npmGlobalPrefix() {
9
+ if (process.platform === "win32") {
10
+ const result = spawnSync(
11
+ process.env.ComSpec || "cmd.exe",
12
+ ["/d", "/s", "/c", "npm prefix -g"],
13
+ { encoding: "utf8" },
14
+ )
15
+ return result.status === 0 ? result.stdout?.trim() : null
16
+ }
17
+
18
+ const result = spawnSync("npm", ["prefix", "-g"], { encoding: "utf8" })
19
+ return result.status === 0 ? result.stdout?.trim() : null
20
+ }
21
+
22
+ function isGlobalInstall() {
23
+ if (process.env.npm_config_global === "true" || process.env.npm_config_global === "1") {
24
+ return true
25
+ }
26
+
27
+ const prefix = npmGlobalPrefix()
28
+ if (!prefix) return false
29
+
30
+ return packageRoot.toLowerCase().startsWith(path.resolve(prefix).toLowerCase())
31
+ }
32
+
33
+ if (!isGlobalInstall()) {
34
+ console.log("[ocskill] Local npm install detected; skipping OpenCode global setup.")
35
+ console.log("[ocskill] Use 'node bin/ocskill.mjs install' to test installation manually.")
36
+ process.exit(0)
37
+ }
38
+
39
+ try {
40
+ const result = await installResources()
41
+ if (result.stateError) {
42
+ for (const warning of result.warnings) {
43
+ console.warn(`[ocskill] WARNING: ${warning}`)
44
+ }
45
+ console.error(`[ocskill] ERROR: ${result.stateError}`)
46
+ process.exit(1)
47
+ }
48
+ console.log(`[ocskill] Installed resources for v${result.version}`)
49
+ console.log(`[ocskill] OpenCode config: ${result.configDir}`)
50
+ console.log(`[ocskill] Skills: ${result.skills.length}`)
51
+ console.log(`[ocskill] Commands: ${result.commands.length}`)
52
+ console.log(`[ocskill] Subagents: ${result.agents.length}`)
53
+ for (const warning of result.warnings) {
54
+ console.warn(`[ocskill] WARNING: ${warning}`)
55
+ }
56
+ console.log("[ocskill] Start a new OpenCode session to pick up workflow changes.")
57
+ } catch (error) {
58
+ console.error("[ocskill] Installation failed.")
59
+ console.error(error instanceof Error ? error.stack : error)
60
+ process.exit(1)
61
+ }
@@ -0,0 +1,2 @@
1
+ import { repoMap } from "../lib/repo-inspect.mjs"
2
+ console.log(JSON.stringify(await repoMap(process.argv[2] || process.cwd()), null, 2))