opencode-agent-skill 7.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +163 -0
- package/LICENSE +9 -0
- package/README.md +581 -0
- package/bin/ocskill.mjs +975 -0
- package/docs/DETERMINISTIC-TOOLS.md +88 -0
- package/docs/ENGINEERING-DESIGN.md +176 -0
- package/docs/EVALS.md +136 -0
- package/docs/NPM-PUBLISH.md +102 -0
- package/docs/OPENCODE-COMPAT.md +109 -0
- package/docs/RESEARCH-SOURCES.md +37 -0
- package/docs/TRACE-SCHEMA.md +109 -0
- package/docs/V7-INTELLIGENCE-RUNTIME.md +166 -0
- package/evals/live/fixtures/engineering-bench/package.json +1 -0
- package/evals/live/fixtures/engineering-bench/src/api-errors.mjs +3 -0
- package/evals/live/fixtures/engineering-bench/src/authz.mjs +3 -0
- package/evals/live/fixtures/engineering-bench/src/cache-tags.mjs +3 -0
- package/evals/live/fixtures/engineering-bench/src/config.mjs +3 -0
- package/evals/live/fixtures/engineering-bench/src/contract-consumer.mjs +6 -0
- package/evals/live/fixtures/engineering-bench/src/contract-producer.mjs +3 -0
- package/evals/live/fixtures/engineering-bench/src/dedupe.mjs +3 -0
- package/evals/live/fixtures/engineering-bench/src/dependency.mjs +3 -0
- package/evals/live/fixtures/engineering-bench/src/discount.mjs +3 -0
- package/evals/live/fixtures/engineering-bench/src/inventory.mjs +3 -0
- package/evals/live/fixtures/engineering-bench/src/migration.mjs +3 -0
- package/evals/live/fixtures/engineering-bench/src/money.mjs +3 -0
- package/evals/live/fixtures/engineering-bench/src/pagination.mjs +5 -0
- package/evals/live/fixtures/engineering-bench/src/path-safe.mjs +5 -0
- package/evals/live/fixtures/engineering-bench/src/payment.mjs +5 -0
- package/evals/live/fixtures/engineering-bench/src/query-sort.mjs +3 -0
- package/evals/live/fixtures/engineering-bench/src/react-state.mjs +8 -0
- package/evals/live/fixtures/engineering-bench/src/retry.mjs +3 -0
- package/evals/live/fixtures/engineering-bench/src/rn-platform.mjs +3 -0
- package/evals/live/fixtures/engineering-bench/src/upload.mjs +3 -0
- package/evals/live/fixtures/engineering-bench/src/webhook.mjs +3 -0
- package/evals/live/graders/engineering-bench.mjs +239 -0
- package/evals/live/tasks.json +126 -0
- package/evals/long/fixtures/long-horizon/package.json +1 -0
- package/evals/long/fixtures/long-horizon/src/auth.mjs +5 -0
- package/evals/long/fixtures/long-horizon/src/checkout.mjs +9 -0
- package/evals/long/fixtures/long-horizon/src/inventory.mjs +5 -0
- package/evals/long/fixtures/long-horizon/src/money.mjs +3 -0
- package/evals/long/fixtures/long-horizon/src/payment.mjs +7 -0
- package/evals/long/fixtures/long-horizon/src/product-api.mjs +10 -0
- package/evals/long/fixtures/long-horizon/src/product-cache.mjs +3 -0
- package/evals/long/fixtures/long-horizon/src/product-service.mjs +6 -0
- package/evals/long/fixtures/long-horizon/src/product-state.mjs +8 -0
- package/evals/long/fixtures/long-horizon/src/project-api.mjs +9 -0
- package/evals/long/fixtures/long-horizon/src/project-service.mjs +7 -0
- package/evals/long/fixtures/long-horizon/src/user-consumer.mjs +3 -0
- package/evals/long/fixtures/long-horizon/src/user-migration.mjs +7 -0
- package/evals/long/fixtures/long-horizon/src/user-serializer.mjs +3 -0
- package/evals/long/fixtures/long-horizon/src/user-validation.mjs +3 -0
- package/evals/long/graders/long-horizon.mjs +209 -0
- package/evals/long/tasks.json +36 -0
- package/evals/router-triggers.json +1238 -0
- package/evals/routing.json +321 -0
- package/global-config/AGENTS.md +212 -0
- package/global-config/agents/architect.md +38 -0
- package/global-config/agents/codebase-mapper.md +41 -0
- package/global-config/agents/critic.md +37 -0
- package/global-config/agents/debugger.md +36 -0
- package/global-config/agents/executor.md +48 -0
- package/global-config/agents/integration-verifier.md +43 -0
- package/global-config/agents/plan-checker.md +43 -0
- package/global-config/agents/researcher.md +30 -0
- package/global-config/agents/reviewer.md +30 -0
- package/global-config/agents/verifier.md +33 -0
- package/global-config/commands/audit.md +8 -0
- package/global-config/commands/critique.md +8 -0
- package/global-config/commands/debug.md +8 -0
- package/global-config/commands/feature.md +8 -0
- package/global-config/commands/fix.md +8 -0
- package/global-config/commands/plan.md +8 -0
- package/global-config/commands/research.md +8 -0
- package/global-config/commands/resume.md +22 -0
- package/global-config/commands/review.md +8 -0
- package/global-config/commands/run.md +29 -0
- package/global-config/commands/verify.md +8 -0
- package/global-config/plugins/ues-router/capabilities.js +20 -0
- package/global-config/plugins/ues-router/index.js +277 -0
- package/global-config/plugins/ues-router/router.js +74 -0
- package/global-config/plugins/ues-router/safety.js +16 -0
- package/global-config/skills/accessibility/SKILL.md +10 -0
- package/global-config/skills/accessibility/references/workflow.md +17 -0
- package/global-config/skills/api-contract/SKILL.md +12 -0
- package/global-config/skills/api-contract/references/workflow.md +17 -0
- package/global-config/skills/auth-security/SKILL.md +12 -0
- package/global-config/skills/auth-security/references/workflow.md +15 -0
- package/global-config/skills/bug-diagnosis/SKILL.md +28 -0
- package/global-config/skills/change-impact-analysis/SKILL.md +23 -0
- package/global-config/skills/code-review/SKILL.md +19 -0
- package/global-config/skills/context-engineering/SKILL.md +18 -0
- package/global-config/skills/context-engineering/references/large-repo.md +16 -0
- package/global-config/skills/database-engineering/SKILL.md +12 -0
- package/global-config/skills/database-engineering/references/workflow.md +15 -0
- package/global-config/skills/dependency-management/SKILL.md +12 -0
- package/global-config/skills/dependency-management/references/workflow.md +14 -0
- package/global-config/skills/devops-engineering/SKILL.md +10 -0
- package/global-config/skills/devops-engineering/references/workflow.md +11 -0
- package/global-config/skills/django-engineering/SKILL.md +10 -0
- package/global-config/skills/django-engineering/references/workflow.md +11 -0
- package/global-config/skills/documentation-engineering/SKILL.md +10 -0
- package/global-config/skills/documentation-engineering/references/workflow.md +15 -0
- package/global-config/skills/dotnet-engineering/SKILL.md +10 -0
- package/global-config/skills/dotnet-engineering/references/workflow.md +11 -0
- package/global-config/skills/ecommerce-engineering/SKILL.md +10 -0
- package/global-config/skills/ecommerce-engineering/references/workflow.md +17 -0
- package/global-config/skills/engineering-orchestrator/SKILL.md +29 -0
- package/global-config/skills/engineering-orchestrator/references/delegation.md +22 -0
- package/global-config/skills/engineering-orchestrator/references/evaluator-loop.md +18 -0
- package/global-config/skills/engineering-orchestrator/references/long-horizon.md +57 -0
- package/global-config/skills/engineering-orchestrator/references/model-escalation.md +19 -0
- package/global-config/skills/engineering-orchestrator/references/retry-policy.md +12 -0
- package/global-config/skills/engineering-orchestrator/references/routing.md +39 -0
- package/global-config/skills/engineering-orchestrator/references/verification-matrix.md +18 -0
- package/global-config/skills/fastapi-engineering/SKILL.md +10 -0
- package/global-config/skills/fastapi-engineering/references/workflow.md +13 -0
- package/global-config/skills/file-upload-engineering/SKILL.md +10 -0
- package/global-config/skills/file-upload-engineering/references/workflow.md +17 -0
- package/global-config/skills/flutter-engineering/SKILL.md +10 -0
- package/global-config/skills/flutter-engineering/references/workflow.md +13 -0
- package/global-config/skills/git-safety/SKILL.md +10 -0
- package/global-config/skills/git-safety/references/workflow.md +15 -0
- package/global-config/skills/implementation-engineer/SKILL.md +10 -0
- package/global-config/skills/implementation-engineer/references/workflow.md +14 -0
- package/global-config/skills/java-spring-engineering/SKILL.md +10 -0
- package/global-config/skills/java-spring-engineering/references/workflow.md +13 -0
- package/global-config/skills/long-task-state/SKILL.md +28 -0
- package/global-config/skills/long-task-state/references/context-ledger.md +27 -0
- package/global-config/skills/long-task-state/templates/STATE.md +48 -0
- package/global-config/skills/nestjs-engineering/SKILL.md +10 -0
- package/global-config/skills/nestjs-engineering/references/workflow.md +11 -0
- package/global-config/skills/nextjs-engineering/SKILL.md +12 -0
- package/global-config/skills/nextjs-engineering/references/workflow.md +15 -0
- package/global-config/skills/nodejs-engineering/SKILL.md +12 -0
- package/global-config/skills/nodejs-engineering/references/workflow.md +11 -0
- package/global-config/skills/payment-engineering/SKILL.md +12 -0
- package/global-config/skills/payment-engineering/references/workflow.md +19 -0
- package/global-config/skills/performance-engineering/SKILL.md +10 -0
- package/global-config/skills/performance-engineering/references/workflow.md +17 -0
- package/global-config/skills/python-engineering/SKILL.md +10 -0
- package/global-config/skills/python-engineering/references/workflow.md +11 -0
- package/global-config/skills/react-engineering/SKILL.md +14 -0
- package/global-config/skills/react-engineering/references/workflow.md +16 -0
- package/global-config/skills/react-native-engineering/SKILL.md +14 -0
- package/global-config/skills/react-native-engineering/references/workflow.md +16 -0
- package/global-config/skills/repo-explorer/SKILL.md +17 -0
- package/global-config/skills/research-verification/SKILL.md +18 -0
- package/global-config/skills/research-verification/references/source-hierarchy.md +12 -0
- package/global-config/skills/rest-api-design/SKILL.md +10 -0
- package/global-config/skills/rest-api-design/references/workflow.md +18 -0
- package/global-config/skills/software-architect/SKILL.md +10 -0
- package/global-config/skills/software-architect/references/workflow.md +18 -0
- package/global-config/skills/task-planner/SKILL.md +21 -0
- package/global-config/skills/task-planner/references/plan-schema.md +56 -0
- package/global-config/skills/test-driven-development/SKILL.md +22 -0
- package/global-config/skills/test-driven-development/references/writing-good-tests.md +21 -0
- package/global-config/skills/test-verification/SKILL.md +20 -0
- package/global-config/skills/ui-ux-engineering/SKILL.md +10 -0
- package/global-config/skills/ui-ux-engineering/references/workflow.md +17 -0
- package/global-config/skills/web-security-review/SKILL.md +10 -0
- package/global-config/skills/web-security-review/references/workflow.md +22 -0
- package/lib/context-manifest.mjs +101 -0
- package/lib/control-center.mjs +148 -0
- package/lib/eval-auth.mjs +21 -0
- package/lib/eval-report.mjs +72 -0
- package/lib/eval-telemetry.mjs +155 -0
- package/lib/evidence-receipt.mjs +50 -0
- package/lib/hermes-bridge.mjs +28 -0
- package/lib/ids.mjs +21 -0
- package/lib/installer.mjs +636 -0
- package/lib/learning-engine.mjs +161 -0
- package/lib/model-config.mjs +88 -0
- package/lib/model-policy.mjs +71 -0
- package/lib/opencode-compat.mjs +86 -0
- package/lib/orchestrator-policy.mjs +35 -0
- package/lib/process-runner.mjs +117 -0
- package/lib/repo-graph.mjs +135 -0
- package/lib/repo-inspect.mjs +264 -0
- package/lib/review-scope.mjs +98 -0
- package/lib/router-config.mjs +40 -0
- package/lib/task-engine.mjs +777 -0
- package/lib/task-graph.mjs +262 -0
- package/lib/update-resolver.mjs +43 -0
- package/lib/verification-plan.mjs +52 -0
- package/lib/version.mjs +47 -0
- package/lib/workspace-snapshot.mjs +45 -0
- package/lib/worktree-sandbox.mjs +59 -0
- package/package.json +69 -0
- package/scripts/check-working-tree.mjs +2 -0
- package/scripts/collect-evidence.mjs +2 -0
- package/scripts/control-center.mjs +37 -0
- package/scripts/detect-stack.mjs +2 -0
- package/scripts/detect-test-commands.mjs +2 -0
- package/scripts/eval-live.mjs +437 -0
- package/scripts/eval-report.mjs +49 -0
- package/scripts/eval-router.mjs +45 -0
- package/scripts/eval-skills.mjs +71 -0
- package/scripts/impact-map.mjs +4 -0
- package/scripts/install.mjs +61 -0
- package/scripts/repo-map.mjs +2 -0
- package/scripts/smoke-packed-install.mjs +326 -0
- package/scripts/syntax-check.mjs +35 -0
- package/scripts/uninstall.mjs +21 -0
- package/scripts/validate-live-suite.mjs +65 -0
- package/scripts/validate.mjs +133 -0
|
@@ -0,0 +1,437 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from "node:fs"
|
|
2
|
+
import { cp, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"
|
|
3
|
+
import os from "node:os"
|
|
4
|
+
import path from "node:path"
|
|
5
|
+
import { spawnSync } from "node:child_process"
|
|
6
|
+
import { fileURLToPath } from "node:url"
|
|
7
|
+
import { installResources } from "../lib/installer.mjs"
|
|
8
|
+
import { copyCurrentOpenCodeAuth } from "../lib/eval-auth.mjs"
|
|
9
|
+
import { parseOpenCodeTelemetry } from "../lib/eval-telemetry.mjs"
|
|
10
|
+
import { buildOpenCodeRunArgs, parseOpenCodeMajor } from "../lib/opencode-compat.mjs"
|
|
11
|
+
import { snapshotWorkspace, diffWorkspaceSnapshots } from "../lib/workspace-snapshot.mjs"
|
|
12
|
+
import { runProcess } from "../lib/process-runner.mjs"
|
|
13
|
+
|
|
14
|
+
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..")
|
|
15
|
+
const args = process.argv.slice(2)
|
|
16
|
+
const abortController = new AbortController()
|
|
17
|
+
let interrupted = false
|
|
18
|
+
process.once("SIGINT", () => {
|
|
19
|
+
interrupted = true
|
|
20
|
+
console.warn("\n[eval] interrupt requested; stopping the active OpenCode process tree...")
|
|
21
|
+
abortController.abort()
|
|
22
|
+
})
|
|
23
|
+
|
|
24
|
+
function argValue(name, fallback = null) {
|
|
25
|
+
const index = args.indexOf(name)
|
|
26
|
+
return index >= 0 && index + 1 < args.length ? args[index + 1] : fallback
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function hasArg(name) {
|
|
30
|
+
return args.includes(name)
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function positiveInt(value, fallback) {
|
|
34
|
+
const parsed = Number.parseInt(value ?? "", 10)
|
|
35
|
+
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function findWindowsCommand(name) {
|
|
39
|
+
if (path.isAbsolute(name) && existsSync(name)) return name
|
|
40
|
+
const result = spawnSync("where", [name], { encoding: "utf8" })
|
|
41
|
+
if (result.status !== 0 || !result.stdout) return null
|
|
42
|
+
const matches = result.stdout.split(/\r?\n/).map((line) => line.trim()).filter(Boolean)
|
|
43
|
+
return matches.find((item) => /\.(exe|cmd|bat)$/i.test(item)) || matches[0] || null
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function quoteCmd(value) {
|
|
47
|
+
if (/^[A-Za-z0-9_@%+=:,./\\-]+$/.test(value)) return value
|
|
48
|
+
return '"' + value.replaceAll('"', '""') + '"'
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function findNodeShimEntry(cmdPath) {
|
|
52
|
+
const dir = path.dirname(cmdPath)
|
|
53
|
+
let shim = ""
|
|
54
|
+
try {
|
|
55
|
+
shim = readFileSync(cmdPath, "utf8")
|
|
56
|
+
} catch {
|
|
57
|
+
return null
|
|
58
|
+
}
|
|
59
|
+
const match = shim.match(/node_modules[\\/][^\s"]+?\.(?:js|mjs)/gi)?.at(-1)
|
|
60
|
+
if (!match) return null
|
|
61
|
+
const entry = path.resolve(dir, match)
|
|
62
|
+
return existsSync(entry) ? entry : null
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function runCommand(executable, commandArgs, options = {}) {
|
|
66
|
+
if (process.platform !== "win32") {
|
|
67
|
+
return spawnSync(executable, commandArgs, options)
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const resolved = findWindowsCommand(executable)
|
|
71
|
+
if (!resolved) {
|
|
72
|
+
return { status: 127, stdout: "", stderr: "Command not found: " + executable }
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
if (/\.(cmd|bat)$/i.test(resolved)) {
|
|
76
|
+
const entry = findNodeShimEntry(resolved)
|
|
77
|
+
if (entry) return spawnSync(process.execPath, [entry, ...commandArgs], options)
|
|
78
|
+
|
|
79
|
+
const line = [resolved, ...commandArgs].map(quoteCmd).join(" ")
|
|
80
|
+
return spawnSync(
|
|
81
|
+
process.env.ComSpec || "cmd.exe",
|
|
82
|
+
["/d", "/s", "/c", line],
|
|
83
|
+
options,
|
|
84
|
+
)
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
return spawnSync(resolved, commandArgs, options)
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
function runAsyncCommand(executable, commandArgs, options = {}) {
|
|
92
|
+
if (process.platform !== "win32") return runProcess(executable, commandArgs, options)
|
|
93
|
+
|
|
94
|
+
const resolved = findWindowsCommand(executable)
|
|
95
|
+
if (!resolved) {
|
|
96
|
+
return Promise.resolve({
|
|
97
|
+
status: 127,
|
|
98
|
+
signal: null,
|
|
99
|
+
stdout: "",
|
|
100
|
+
stderr: "Command not found: " + executable,
|
|
101
|
+
durationMs: 0,
|
|
102
|
+
timedOut: false,
|
|
103
|
+
idleTimedOut: false,
|
|
104
|
+
aborted: false,
|
|
105
|
+
})
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
if (/\.(cmd|bat)$/i.test(resolved)) {
|
|
109
|
+
const entry = findNodeShimEntry(resolved)
|
|
110
|
+
if (entry) return runProcess(process.execPath, [entry, ...commandArgs], options)
|
|
111
|
+
const line = [resolved, ...commandArgs].map(quoteCmd).join(" ")
|
|
112
|
+
return runProcess(process.env.ComSpec || "cmd.exe", ["/d", "/s", "/c", line], options)
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
return runProcess(resolved, commandArgs, options)
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function excerpt(value, limit = 12000) {
|
|
119
|
+
if (!value) return ""
|
|
120
|
+
const text = String(value)
|
|
121
|
+
return text.length <= limit ? text : text.slice(0, limit) + "\n...[truncated]"
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
async function inspectLongOrchestration(workspace) {
|
|
125
|
+
const workRoot = path.join(workspace, ".ues-work")
|
|
126
|
+
const dirs = await readdir(workRoot, { withFileTypes: true }).catch(() => [])
|
|
127
|
+
const items = []
|
|
128
|
+
|
|
129
|
+
for (const entry of dirs) {
|
|
130
|
+
if (!entry.isDirectory()) continue
|
|
131
|
+
const dir = path.join(workRoot, entry.name)
|
|
132
|
+
try {
|
|
133
|
+
const [state, plan, evidence] = await Promise.all([
|
|
134
|
+
readFile(path.join(dir, "STATE.json"), "utf8").then(JSON.parse),
|
|
135
|
+
readFile(path.join(dir, "PLAN.json"), "utf8").then(JSON.parse),
|
|
136
|
+
readFile(path.join(dir, "EVIDENCE.json"), "utf8").then(JSON.parse),
|
|
137
|
+
])
|
|
138
|
+
const tasks = Object.values(state.tasks || {})
|
|
139
|
+
const evidenceTasks = new Set((evidence.entries || []).map((item) => item.task))
|
|
140
|
+
const receiptBackedTasks = new Set(
|
|
141
|
+
(evidence.receipts || []).filter((item) => item.passed).map((item) => item.task),
|
|
142
|
+
)
|
|
143
|
+
const plannedIDs = (plan.tasks || []).map((item) => item.id)
|
|
144
|
+
const item = {
|
|
145
|
+
slug: entry.name,
|
|
146
|
+
taskCount: Array.isArray(plan.tasks) ? plan.tasks.length : 0,
|
|
147
|
+
planApproved: state.planApproval?.status === "passed",
|
|
148
|
+
attemptedTasks: tasks.filter((task) => Number(task.attempts || 0) > 0).length,
|
|
149
|
+
completedTasks: tasks.filter((task) => task.status === "completed").length,
|
|
150
|
+
receiptBackedTasks: plannedIDs.filter((id) => receiptBackedTasks.has(id)).length,
|
|
151
|
+
receiptCoverage: plannedIDs.length
|
|
152
|
+
? plannedIDs.filter((id) => receiptBackedTasks.has(id)).length / plannedIDs.length
|
|
153
|
+
: 0,
|
|
154
|
+
integrationPassed: state.integrationVerification?.status === "PASS",
|
|
155
|
+
integrationEvidence: evidenceTasks.has("__integration_verification__"),
|
|
156
|
+
finalizedEvidence: evidenceTasks.has("__integration__"),
|
|
157
|
+
completed: state.status === "completed",
|
|
158
|
+
}
|
|
159
|
+
item.valid =
|
|
160
|
+
item.taskCount >= 2 &&
|
|
161
|
+
item.planApproved &&
|
|
162
|
+
item.attemptedTasks === item.taskCount &&
|
|
163
|
+
item.completedTasks === item.taskCount &&
|
|
164
|
+
item.receiptBackedTasks === item.taskCount &&
|
|
165
|
+
item.integrationPassed &&
|
|
166
|
+
item.integrationEvidence &&
|
|
167
|
+
item.finalizedEvidence &&
|
|
168
|
+
item.completed
|
|
169
|
+
items.push(item)
|
|
170
|
+
} catch (error) {
|
|
171
|
+
items.push({ slug: entry.name, valid: false, error: String(error?.message || error) })
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
return {
|
|
176
|
+
required: true,
|
|
177
|
+
valid: items.some((item) => item.valid),
|
|
178
|
+
items,
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
const model = argValue("--model", process.env.UES_EVAL_MODEL)
|
|
183
|
+
const variant = argValue("--variant", process.env.UES_EVAL_VARIANT)
|
|
184
|
+
const trials = Math.min(positiveInt(argValue("--trials"), 1), 20)
|
|
185
|
+
const taskFilter = argValue("--task")
|
|
186
|
+
const requestedMode = argValue("--mode", "both")
|
|
187
|
+
const keep = hasArg("--keep")
|
|
188
|
+
const authMode = argValue("--auth", "env-only")
|
|
189
|
+
const suiteName = argValue("--suite", "live")
|
|
190
|
+
const timeoutMs = positiveInt(argValue("--timeout-ms"), 15 * 60_000)
|
|
191
|
+
const idleTimeoutMs = positiveInt(argValue("--idle-timeout-ms"), 5 * 60_000)
|
|
192
|
+
const heartbeatMs = positiveInt(argValue("--heartbeat-ms"), 30_000)
|
|
193
|
+
const suiteRoot = path.join(root, "evals", suiteName)
|
|
194
|
+
|
|
195
|
+
if (!model) {
|
|
196
|
+
console.error("Usage: node scripts/eval-live.mjs --model provider/model [--suite live|long] [--variant high] [--trials N] [--task id] [--mode baseline|ues|both] [--auth env-only|current] [--output-dir path] [--keep] [--timeout-ms N] [--idle-timeout-ms N] [--heartbeat-ms N]")
|
|
197
|
+
console.error("You can also set UES_EVAL_MODEL and UES_EVAL_VARIANT.")
|
|
198
|
+
process.exit(2)
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
if (!["baseline", "ues", "both"].includes(requestedMode)) {
|
|
202
|
+
console.error("--mode must be baseline, ues, or both")
|
|
203
|
+
process.exit(2)
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
if (!["env-only", "current"].includes(authMode)) {
|
|
207
|
+
console.error("--auth must be env-only or current")
|
|
208
|
+
process.exit(2)
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
const probe = runCommand("opencode", ["--version"], { encoding: "utf8" })
|
|
212
|
+
if (probe.status !== 0) {
|
|
213
|
+
console.error("OpenCode CLI is required for live evals.")
|
|
214
|
+
console.error(excerpt(probe.stderr || probe.stdout))
|
|
215
|
+
process.exit(2)
|
|
216
|
+
}
|
|
217
|
+
const opencodeVersion = String(probe.stdout || probe.stderr || "").trim()
|
|
218
|
+
const parsedOpenCodeMajor = parseOpenCodeMajor(opencodeVersion)
|
|
219
|
+
const opencodeMajor = parsedOpenCodeMajor ?? 1
|
|
220
|
+
if (parsedOpenCodeMajor === null) {
|
|
221
|
+
console.warn("[eval] could not parse OpenCode version; using the conservative OpenCode 1.x invocation.")
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
if (!["live", "long"].includes(suiteName)) {
|
|
225
|
+
console.error("--suite must be live or long")
|
|
226
|
+
process.exit(2)
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
const suite = JSON.parse(await readFile(path.join(suiteRoot, "tasks.json"), "utf8"))
|
|
230
|
+
let tasks = suite.tasks || []
|
|
231
|
+
if (taskFilter) tasks = tasks.filter((task) => task.id === taskFilter)
|
|
232
|
+
if (tasks.length === 0) {
|
|
233
|
+
console.error(taskFilter ? "Unknown task: " + taskFilter : "No live eval tasks found.")
|
|
234
|
+
process.exit(2)
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
const modes = requestedMode === "both" ? ["baseline", "ues"] : [requestedMode]
|
|
238
|
+
const invocationDir = process.cwd()
|
|
239
|
+
const runRoot = await mkdtemp(path.join(os.tmpdir(), "ues-live-eval-"))
|
|
240
|
+
const resultDir = path.resolve(
|
|
241
|
+
argValue("--output-dir", path.join(invocationDir, ".ues-evals")),
|
|
242
|
+
)
|
|
243
|
+
await mkdir(resultDir, { recursive: true })
|
|
244
|
+
|
|
245
|
+
const results = []
|
|
246
|
+
const oldConfigDir = process.env.OPENCODE_CONFIG_DIR
|
|
247
|
+
|
|
248
|
+
try {
|
|
249
|
+
for (const task of tasks) {
|
|
250
|
+
if (interrupted) break
|
|
251
|
+
for (const mode of modes) {
|
|
252
|
+
if (interrupted) break
|
|
253
|
+
for (let trial = 1; trial <= trials; trial += 1) {
|
|
254
|
+
if (interrupted) break
|
|
255
|
+
const isolatedRoot = path.join(runRoot, task.id + "-" + mode + "-" + trial)
|
|
256
|
+
const workspace = path.join(isolatedRoot, "workspace")
|
|
257
|
+
const isolatedHome = path.join(isolatedRoot, "home")
|
|
258
|
+
const xdgRoot = path.join(isolatedHome, ".config")
|
|
259
|
+
const configDir = path.join(xdgRoot, "opencode")
|
|
260
|
+
const dataRoot = path.join(isolatedRoot, "data")
|
|
261
|
+
const cacheRoot = path.join(isolatedRoot, "cache")
|
|
262
|
+
const stateRoot = path.join(isolatedRoot, "state")
|
|
263
|
+
|
|
264
|
+
await mkdir(configDir, { recursive: true })
|
|
265
|
+
await mkdir(dataRoot, { recursive: true })
|
|
266
|
+
await mkdir(cacheRoot, { recursive: true })
|
|
267
|
+
await mkdir(stateRoot, { recursive: true })
|
|
268
|
+
await cp(path.join(suiteRoot, task.fixture), workspace, { recursive: true })
|
|
269
|
+
const beforeSnapshot = await snapshotWorkspace(workspace)
|
|
270
|
+
|
|
271
|
+
if (authMode === "current") {
|
|
272
|
+
const auth = await copyCurrentOpenCodeAuth(dataRoot)
|
|
273
|
+
if (!auth.copied) {
|
|
274
|
+
console.warn("[eval] current auth requested but auth.json was not found; continuing with environment credentials only.")
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
if (mode === "ues") {
|
|
279
|
+
process.env.OPENCODE_CONFIG_DIR = configDir
|
|
280
|
+
const installed = await installResources({ sourceRoot: root })
|
|
281
|
+
if (installed.stateError) {
|
|
282
|
+
throw new Error("UES install failed for live eval: " + installed.stateError)
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
const childEnv = {
|
|
287
|
+
...process.env,
|
|
288
|
+
HOME: isolatedHome,
|
|
289
|
+
USERPROFILE: isolatedHome,
|
|
290
|
+
OPENCODE_CONFIG_DIR: configDir,
|
|
291
|
+
XDG_CONFIG_HOME: xdgRoot,
|
|
292
|
+
XDG_DATA_HOME: dataRoot,
|
|
293
|
+
XDG_CACHE_HOME: cacheRoot,
|
|
294
|
+
XDG_STATE_HOME: stateRoot,
|
|
295
|
+
OPENCODE_DISABLE_AUTOUPDATE: "true",
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
const opencodeArgs = buildOpenCodeRunArgs({
|
|
299
|
+
major: opencodeMajor,
|
|
300
|
+
model,
|
|
301
|
+
workspace,
|
|
302
|
+
variant,
|
|
303
|
+
prompt: task.prompt,
|
|
304
|
+
})
|
|
305
|
+
|
|
306
|
+
console.log("[" + mode + "] " + task.id + " trial " + trial + ": starting agent")
|
|
307
|
+
const agentRun = await runAsyncCommand("opencode", opencodeArgs, {
|
|
308
|
+
cwd: workspace,
|
|
309
|
+
env: childEnv,
|
|
310
|
+
maxBuffer: 4 * 1024 * 1024,
|
|
311
|
+
heartbeatMs,
|
|
312
|
+
timeoutMs,
|
|
313
|
+
idleTimeoutMs,
|
|
314
|
+
signal: abortController.signal,
|
|
315
|
+
onHeartbeat: ({ elapsedMs, idleMs }) => {
|
|
316
|
+
console.log(
|
|
317
|
+
"[" + mode + "] " + task.id + " trial " + trial +
|
|
318
|
+
": still running (" + Math.round(elapsedMs / 1000) + "s elapsed, " +
|
|
319
|
+
Math.round(idleMs / 1000) + "s since output)",
|
|
320
|
+
)
|
|
321
|
+
},
|
|
322
|
+
})
|
|
323
|
+
const durationMs = agentRun.durationMs
|
|
324
|
+
|
|
325
|
+
const graderPath = path.join(suiteRoot, task.grader)
|
|
326
|
+
const graderRun = spawnSync(process.execPath, [graderPath], {
|
|
327
|
+
cwd: workspace,
|
|
328
|
+
env: { ...childEnv, UES_EVAL_WORKSPACE: workspace, UES_EVAL_TASK: task.id },
|
|
329
|
+
encoding: "utf8",
|
|
330
|
+
maxBuffer: 1024 * 1024,
|
|
331
|
+
})
|
|
332
|
+
|
|
333
|
+
const afterSnapshot = await snapshotWorkspace(workspace)
|
|
334
|
+
const telemetry = parseOpenCodeTelemetry(agentRun.stdout)
|
|
335
|
+
const changedFiles = diffWorkspaceSnapshots(beforeSnapshot, afterSnapshot)
|
|
336
|
+
const orchestration =
|
|
337
|
+
suiteName === "long" && mode === "ues"
|
|
338
|
+
? await inspectLongOrchestration(workspace)
|
|
339
|
+
: { required: false, valid: true, items: [] }
|
|
340
|
+
const passed =
|
|
341
|
+
agentRun.status === 0 &&
|
|
342
|
+
graderRun.status === 0 &&
|
|
343
|
+
(!orchestration.required || orchestration.valid)
|
|
344
|
+
results.push({
|
|
345
|
+
task: task.id,
|
|
346
|
+
mode,
|
|
347
|
+
model,
|
|
348
|
+
variant: variant || null,
|
|
349
|
+
trial,
|
|
350
|
+
passed,
|
|
351
|
+
agentExit: agentRun.status,
|
|
352
|
+
graderExit: graderRun.status,
|
|
353
|
+
durationMs,
|
|
354
|
+
timedOut: agentRun.timedOut,
|
|
355
|
+
idleTimedOut: agentRun.idleTimedOut,
|
|
356
|
+
aborted: agentRun.aborted,
|
|
357
|
+
authMode,
|
|
358
|
+
opencodeVersion,
|
|
359
|
+
opencodeMajor,
|
|
360
|
+
telemetry,
|
|
361
|
+
changedFiles,
|
|
362
|
+
orchestration,
|
|
363
|
+
agentStdout: excerpt(agentRun.stdout),
|
|
364
|
+
agentStderr: excerpt(agentRun.stderr),
|
|
365
|
+
graderStdout: excerpt(graderRun.stdout),
|
|
366
|
+
graderStderr: excerpt(graderRun.stderr),
|
|
367
|
+
workspace: keep ? workspace : null,
|
|
368
|
+
timestamp: new Date().toISOString(),
|
|
369
|
+
})
|
|
370
|
+
|
|
371
|
+
if (agentRun.aborted) interrupted = true
|
|
372
|
+
console.log(
|
|
373
|
+
"[" + mode + "] " + task.id + " trial " + trial + ": " +
|
|
374
|
+
(passed ? "PASS" : "FAIL") +
|
|
375
|
+
" (agent=" + agentRun.status + ", grader=" + graderRun.status +
|
|
376
|
+
(orchestration.required ? ", orchestration=" + (orchestration.valid ? "PASS" : "FAIL") : "") +
|
|
377
|
+
(agentRun.timedOut ? ", timeout=hard" : agentRun.idleTimedOut ? ", timeout=idle" : "") +
|
|
378
|
+
", " + durationMs + "ms)",
|
|
379
|
+
)
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
} finally {
|
|
384
|
+
if (oldConfigDir === undefined) delete process.env.OPENCODE_CONFIG_DIR
|
|
385
|
+
else process.env.OPENCODE_CONFIG_DIR = oldConfigDir
|
|
386
|
+
|
|
387
|
+
if (!keep) await rm(runRoot, { recursive: true, force: true })
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
const summary = {}
|
|
391
|
+
for (const mode of modes) {
|
|
392
|
+
const subset = results.filter((item) => item.mode === mode)
|
|
393
|
+
const passed = subset.filter((item) => item.passed).length
|
|
394
|
+
summary[mode] = {
|
|
395
|
+
passed,
|
|
396
|
+
total: subset.length,
|
|
397
|
+
passRate: subset.length ? passed / subset.length : 0,
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
const safeModel = model.replace(/[^a-zA-Z0-9._-]+/g, "-")
|
|
402
|
+
const stamp = new Date().toISOString().replace(/[:.]/g, "-")
|
|
403
|
+
const resultFile = path.join(resultDir, stamp + "-" + suiteName + "-" + safeModel + ".json")
|
|
404
|
+
await writeFile(
|
|
405
|
+
resultFile,
|
|
406
|
+
JSON.stringify(
|
|
407
|
+
{
|
|
408
|
+
schemaVersion: 1,
|
|
409
|
+
suite: suiteName,
|
|
410
|
+
suiteVersion: suite.version,
|
|
411
|
+
model,
|
|
412
|
+
variant: variant || null,
|
|
413
|
+
trials,
|
|
414
|
+
taskFilter: taskFilter || null,
|
|
415
|
+
authMode,
|
|
416
|
+
timeoutMs,
|
|
417
|
+
idleTimeoutMs,
|
|
418
|
+
heartbeatMs,
|
|
419
|
+
opencodeVersion,
|
|
420
|
+
opencodeMajor,
|
|
421
|
+
modes,
|
|
422
|
+
summary,
|
|
423
|
+
results,
|
|
424
|
+
},
|
|
425
|
+
null,
|
|
426
|
+
2,
|
|
427
|
+
) + "\n",
|
|
428
|
+
"utf8",
|
|
429
|
+
)
|
|
430
|
+
|
|
431
|
+
console.log("\nSummary")
|
|
432
|
+
for (const [mode, item] of Object.entries(summary)) {
|
|
433
|
+
console.log("- " + mode + ": " + item.passed + "/" + item.total + " (" + (item.passRate * 100).toFixed(1) + "%)")
|
|
434
|
+
}
|
|
435
|
+
console.log("Result: " + resultFile)
|
|
436
|
+
if (keep) console.log("Temporary workspaces kept under: " + runRoot)
|
|
437
|
+
if (interrupted) process.exitCode = 130
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { existsSync } from "node:fs"
|
|
2
|
+
import { readFile, readdir } from "node:fs/promises"
|
|
3
|
+
import path from "node:path"
|
|
4
|
+
import { summarizeEvalResults } from "../lib/eval-report.mjs"
|
|
5
|
+
|
|
6
|
+
const args = process.argv.slice(2)
|
|
7
|
+
let files = []
|
|
8
|
+
|
|
9
|
+
if (args.length) {
|
|
10
|
+
for (const value of args) {
|
|
11
|
+
const resolved = path.resolve(value)
|
|
12
|
+
if (!existsSync(resolved)) {
|
|
13
|
+
console.error("Eval result path not found: " + resolved)
|
|
14
|
+
process.exit(2)
|
|
15
|
+
}
|
|
16
|
+
if (resolved.endsWith(".json")) files.push(resolved)
|
|
17
|
+
else {
|
|
18
|
+
const entries = await readdir(resolved)
|
|
19
|
+
files.push(...entries.filter((name) => name.endsWith(".json")).map((name) => path.join(resolved, name)))
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
} else {
|
|
23
|
+
const dir = path.resolve(".ues-evals")
|
|
24
|
+
if (existsSync(dir)) {
|
|
25
|
+
const entries = await readdir(dir)
|
|
26
|
+
files = entries.filter((name) => name.endsWith(".json")).map((name) => path.join(dir, name))
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
if (files.length === 0) {
|
|
31
|
+
console.error("No eval result JSON files found. Pass files/directories or run live evals first.")
|
|
32
|
+
process.exit(2)
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const runs = []
|
|
36
|
+
const results = []
|
|
37
|
+
for (const file of [...new Set(files)].sort()) {
|
|
38
|
+
const parsed = JSON.parse(await readFile(file, "utf8"))
|
|
39
|
+
runs.push({ file, model: parsed.model, variant: parsed.variant, trials: parsed.trials })
|
|
40
|
+
results.push(...(parsed.results || []))
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const summary = summarizeEvalResults(results)
|
|
44
|
+
console.log(JSON.stringify({
|
|
45
|
+
schemaVersion: 1,
|
|
46
|
+
files: runs,
|
|
47
|
+
totalResults: results.length,
|
|
48
|
+
...summary,
|
|
49
|
+
}, null, 2))
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { readFile } from "node:fs/promises"
|
|
2
|
+
import path from "node:path"
|
|
3
|
+
import { fileURLToPath } from "node:url"
|
|
4
|
+
import { routeSkills } from "../global-config/plugins/ues-router/router.js"
|
|
5
|
+
|
|
6
|
+
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..")
|
|
7
|
+
const suite = JSON.parse(await readFile(path.join(root, "evals", "router-triggers.json"), "utf8"))
|
|
8
|
+
const failures = []
|
|
9
|
+
let positives = 0
|
|
10
|
+
let positiveHits = 0
|
|
11
|
+
let negatives = 0
|
|
12
|
+
let negativeHits = 0
|
|
13
|
+
|
|
14
|
+
for (const item of suite.cases || []) {
|
|
15
|
+
const routed = routeSkills(item.prompt, item.maxSkills || 6)
|
|
16
|
+
|
|
17
|
+
for (const id of item.include || []) {
|
|
18
|
+
positives += 1
|
|
19
|
+
if (routed.includes(id)) positiveHits += 1
|
|
20
|
+
else failures.push(item.id + ": missing required route " + id + " from [" + routed.join(", ") + "]")
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
for (const id of item.exclude || []) {
|
|
24
|
+
negatives += 1
|
|
25
|
+
if (!routed.includes(id)) negativeHits += 1
|
|
26
|
+
else failures.push(item.id + ": unexpected route " + id + " in [" + routed.join(", ") + "]")
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const recall = positives ? positiveHits / positives : 1
|
|
31
|
+
const specificity = negatives ? negativeHits / negatives : 1
|
|
32
|
+
|
|
33
|
+
if (failures.length) {
|
|
34
|
+
console.error("Router trigger evaluation failed:")
|
|
35
|
+
for (const failure of failures) console.error("- " + failure)
|
|
36
|
+
console.error("Recall: " + (recall * 100).toFixed(1) + "%")
|
|
37
|
+
console.error("Negative guard accuracy: " + (specificity * 100).toFixed(1) + "%")
|
|
38
|
+
process.exit(1)
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
console.log(
|
|
42
|
+
"Validated " + suite.cases.length + " router cases: " +
|
|
43
|
+
positiveHits + "/" + positives + " required routes and " +
|
|
44
|
+
negativeHits + "/" + negatives + " negative guards passed.",
|
|
45
|
+
)
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { readFile, readdir } from "node:fs/promises"
|
|
2
|
+
import path from "node:path"
|
|
3
|
+
import { fileURLToPath } from "node:url"
|
|
4
|
+
|
|
5
|
+
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..")
|
|
6
|
+
const skillsRoot = path.join(root, "global-config", "skills")
|
|
7
|
+
const evalFile = path.join(root, "evals", "routing.json")
|
|
8
|
+
|
|
9
|
+
const skillEntries = await readdir(skillsRoot, { withFileTypes: true })
|
|
10
|
+
const skillIDs = new Set(skillEntries.filter((entry) => entry.isDirectory()).map((entry) => entry.name))
|
|
11
|
+
const suite = JSON.parse(await readFile(evalFile, "utf8"))
|
|
12
|
+
const errors = []
|
|
13
|
+
const names = new Set()
|
|
14
|
+
const referenced = new Set()
|
|
15
|
+
|
|
16
|
+
if (!Array.isArray(suite.scenarios) || suite.scenarios.length < 30) {
|
|
17
|
+
errors.push("routing suite must contain at least 30 scenarios")
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
for (const scenario of suite.scenarios || []) {
|
|
21
|
+
if (!scenario.name || names.has(scenario.name)) {
|
|
22
|
+
errors.push(`invalid or duplicate scenario name: ${scenario.name || "<missing>"}`)
|
|
23
|
+
continue
|
|
24
|
+
}
|
|
25
|
+
names.add(scenario.name)
|
|
26
|
+
|
|
27
|
+
if (!scenario.prompt || typeof scenario.prompt !== "string") {
|
|
28
|
+
errors.push(`${scenario.name}: missing prompt`)
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
if (!Array.isArray(scenario.expect) || scenario.expect.length === 0) {
|
|
32
|
+
errors.push(`${scenario.name}: expect must contain at least one skill`)
|
|
33
|
+
continue
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
if (scenario.expect.length > 5) {
|
|
37
|
+
errors.push(`${scenario.name}: expects more than five skills; routing should stay focused`)
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
for (const id of scenario.expect) {
|
|
41
|
+
referenced.add(id)
|
|
42
|
+
if (!skillIDs.has(id)) errors.push(`${scenario.name}: unknown skill ${id}`)
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
if (!referenced.has("engineering-orchestrator")) {
|
|
47
|
+
errors.push("routing suite must exercise engineering-orchestrator")
|
|
48
|
+
}
|
|
49
|
+
if (!referenced.has("bug-diagnosis")) {
|
|
50
|
+
errors.push("routing suite must exercise bug-diagnosis")
|
|
51
|
+
}
|
|
52
|
+
if (!referenced.has("test-verification")) {
|
|
53
|
+
errors.push("routing suite must exercise test-verification")
|
|
54
|
+
}
|
|
55
|
+
if (!referenced.has("research-verification")) {
|
|
56
|
+
errors.push("routing suite must exercise research-verification")
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
for (const id of [...skillIDs].sort()) {
|
|
60
|
+
if (!referenced.has(id)) errors.push(`routing suite does not exercise skill ${id}`)
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
if (errors.length) {
|
|
64
|
+
console.error("Skill routing eval contract failed:")
|
|
65
|
+
for (const error of errors) console.error(`- ${error}`)
|
|
66
|
+
process.exit(1)
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
console.log(
|
|
70
|
+
`Validated ${suite.scenarios.length} routing scenarios across ${referenced.size} referenced skills.`,
|
|
71
|
+
)
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { impactMap } from "../lib/repo-inspect.mjs"
|
|
2
|
+
const query = process.argv[2]
|
|
3
|
+
if (!query) { console.error("Usage: node scripts/impact-map.mjs <query> [dir]"); process.exit(2) }
|
|
4
|
+
console.log(JSON.stringify(await impactMap(process.argv[3] || process.cwd(), query), null, 2))
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { spawnSync } from "node:child_process"
|
|
2
|
+
import path from "node:path"
|
|
3
|
+
import { fileURLToPath } from "node:url"
|
|
4
|
+
import { installResources } from "../lib/installer.mjs"
|
|
5
|
+
|
|
6
|
+
const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..")
|
|
7
|
+
|
|
8
|
+
function npmGlobalPrefix() {
|
|
9
|
+
if (process.platform === "win32") {
|
|
10
|
+
const result = spawnSync(
|
|
11
|
+
process.env.ComSpec || "cmd.exe",
|
|
12
|
+
["/d", "/s", "/c", "npm prefix -g"],
|
|
13
|
+
{ encoding: "utf8" },
|
|
14
|
+
)
|
|
15
|
+
return result.status === 0 ? result.stdout?.trim() : null
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
const result = spawnSync("npm", ["prefix", "-g"], { encoding: "utf8" })
|
|
19
|
+
return result.status === 0 ? result.stdout?.trim() : null
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function isGlobalInstall() {
|
|
23
|
+
if (process.env.npm_config_global === "true" || process.env.npm_config_global === "1") {
|
|
24
|
+
return true
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const prefix = npmGlobalPrefix()
|
|
28
|
+
if (!prefix) return false
|
|
29
|
+
|
|
30
|
+
return packageRoot.toLowerCase().startsWith(path.resolve(prefix).toLowerCase())
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
if (!isGlobalInstall()) {
|
|
34
|
+
console.log("[ocskill] Local npm install detected; skipping OpenCode global setup.")
|
|
35
|
+
console.log("[ocskill] Use 'node bin/ocskill.mjs install' to test installation manually.")
|
|
36
|
+
process.exit(0)
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
try {
|
|
40
|
+
const result = await installResources()
|
|
41
|
+
if (result.stateError) {
|
|
42
|
+
for (const warning of result.warnings) {
|
|
43
|
+
console.warn(`[ocskill] WARNING: ${warning}`)
|
|
44
|
+
}
|
|
45
|
+
console.error(`[ocskill] ERROR: ${result.stateError}`)
|
|
46
|
+
process.exit(1)
|
|
47
|
+
}
|
|
48
|
+
console.log(`[ocskill] Installed resources for v${result.version}`)
|
|
49
|
+
console.log(`[ocskill] OpenCode config: ${result.configDir}`)
|
|
50
|
+
console.log(`[ocskill] Skills: ${result.skills.length}`)
|
|
51
|
+
console.log(`[ocskill] Commands: ${result.commands.length}`)
|
|
52
|
+
console.log(`[ocskill] Subagents: ${result.agents.length}`)
|
|
53
|
+
for (const warning of result.warnings) {
|
|
54
|
+
console.warn(`[ocskill] WARNING: ${warning}`)
|
|
55
|
+
}
|
|
56
|
+
console.log("[ocskill] Start a new OpenCode session to pick up workflow changes.")
|
|
57
|
+
} catch (error) {
|
|
58
|
+
console.error("[ocskill] Installation failed.")
|
|
59
|
+
console.error(error instanceof Error ? error.stack : error)
|
|
60
|
+
process.exit(1)
|
|
61
|
+
}
|