opencode-agent-skill 7.7.0 → 10.0.0

package/lib/benchmark-confidence.mjs

@@ -0,0 +1,173 @@
+function asBool(value) {
+  return value === true
+}
+
+function pairKey(item) {
+  return [item.suite || "", item.task || "", item.trial || 1].join("::")
+}
+
+function exactSignPValue(a, b) {
+  const n = a + b
+  if (n <= 0) return 1
+  const k = Math.min(a, b)
+  let probability = 2 ** (-n)
+  let cumulative = probability
+  for (let i = 1; i <= k; i += 1) {
+    probability *= (n - i + 1) / i
+    cumulative += probability
+  }
+  return Math.min(1, cumulative * 2)
+}
+
+function mean(values) {
+  const usable = values.filter((value) => Number.isFinite(value))
+  return usable.length ? usable.reduce((sum, value) => sum + value, 0) / usable.length : null
+}
+
+function positiveMean(values) {
+  return mean(values.map(Number).filter((value) => Number.isFinite(value) && value > 0))
+}
+
+function ratio(candidate, reference) {
+  return reference && candidate ? candidate / reference : null
+}
+
+export function pairedBenchmarkConfidence(results, options = {}) {
+  const byKey = new Map()
+  for (const item of results || []) {
+    if (!["baseline", "ues"].includes(item?.mode)) continue
+    const key = pairKey(item)
+    const pair = byKey.get(key) || {}
+    pair[item.mode] = item
+    byKey.set(key, pair)
+  }
+  const pairs = [...byKey.entries()]
+    .filter(([, pair]) => pair.baseline && pair.ues)
+    .map(([key, pair]) => ({ key, ...pair }))
+
+  let bothPass = 0
+  let bothFail = 0
+  let baselineOnly = 0
+  let uesOnly = 0
+  for (const pair of pairs) {
+    const b = asBool(pair.baseline.passed)
+    const u = asBool(pair.ues.passed)
+    if (b && u) bothPass += 1
+    else if (!b && !u) bothFail += 1
+    else if (b) baselineOnly += 1
+    else uesOnly += 1
+  }
+  const total = pairs.length
+  const baselinePassed = bothPass + baselineOnly
+  const uesPassed = bothPass + uesOnly
+  const baselinePassRate = total ? baselinePassed / total : 0
+  const uesPassRate = total ? uesPassed / total : 0
+  const delta = uesPassRate - baselinePassRate
+  const pValue = exactSignPValue(uesOnly, baselineOnly)
+
+  const suites = {}
+  for (const pair of pairs) {
+    const suite = pair.baseline.suite || pair.ues.suite || "unknown"
+    suites[suite] ??= { total: 0, baselinePassed: 0, uesPassed: 0 }
+    suites[suite].total += 1
+    if (pair.baseline.passed) suites[suite].baselinePassed += 1
+    if (pair.ues.passed) suites[suite].uesPassed += 1
+  }
+  for (const item of Object.values(suites)) {
+    item.baselinePassRate = item.total ? item.baselinePassed / item.total : 0
+    item.uesPassRate = item.total ? item.uesPassed / item.total : 0
+    item.delta = item.uesPassRate - item.baselinePassRate
+  }
+
+  const baselineDuration = mean(pairs.map((pair) => Number(pair.baseline.durationMs)))
+  const uesDuration = mean(pairs.map((pair) => Number(pair.ues.durationMs)))
+  const durationRatio = ratio(uesDuration, baselineDuration)
+
+  const baselineCost = positiveMean(pairs.map((pair) => pair.baseline.telemetry?.cost))
+  const uesCost = positiveMean(pairs.map((pair) => pair.ues.telemetry?.cost))
+  const costRatio = ratio(uesCost, baselineCost)
+
+  const baselineInitialInput = positiveMean(
+    pairs.map((pair) => pair.baseline.telemetry?.firstUsage?.input),
+  )
+  const uesInitialInput = positiveMean(
+    pairs.map((pair) => pair.ues.telemetry?.firstUsage?.input),
+  )
+  const initialInputRatio = ratio(uesInitialInput, baselineInitialInput)
+
+  const baselineTokens = positiveMean(
+    pairs.map((pair) => pair.baseline.telemetry?.tokens?.total),
+  )
+  const uesTokens = positiveMean(
+    pairs.map((pair) => pair.ues.telemetry?.tokens?.total),
+  )
+  const tokenRatio = ratio(uesTokens, baselineTokens)
+
+  const minPairs = Math.max(1, Number(options.minPairs || 20))
+  const alpha = Math.min(0.5, Math.max(0.0001, Number(options.alpha || 0.05)))
+  const minDelta = Math.max(0, Number(options.minDelta || 0))
+  const suiteRegressionTolerance = Math.max(0, Number(options.suiteRegressionTolerance || 0))
+  const maxDurationRatio = Math.max(1, Number(options.maxDurationRatio || 1.75))
+  const maxInitialInputRatio = Math.max(1, Number(options.maxInitialInputRatio || 1.5))
+  const maxTokenRatio = Math.max(1, Number(options.maxTokenRatio || 1.75))
+  const noSuiteRegression = Object.values(suites).every((item) => item.delta >= -suiteRegressionTolerance)
+  const speedAcceptable = durationRatio == null || durationRatio <= maxDurationRatio
+  const initialInputAcceptable = initialInputRatio == null || initialInputRatio <= maxInitialInputRatio
+  const tokensAcceptable = tokenRatio == null || tokenRatio <= maxTokenRatio
+
+  const checks = {
+    pairedCoverage: total >= minPairs,
+    positiveDelta: delta > minDelta,
+    moreWinsThanLosses: uesOnly > baselineOnly,
+    statisticallySupported: pValue <= alpha,
+    noSuiteRegression,
+    speedAcceptable,
+    initialInputAcceptable,
+    tokensAcceptable,
+  }
+  return {
+    schemaVersion: 2,
+    kind: "ues-paired-benchmark-confidence",
+    pairs: total,
+    bothPass,
+    bothFail,
+    baselineOnly,
+    uesOnly,
+    baselinePassed,
+    uesPassed,
+    baselinePassRate,
+    uesPassRate,
+    delta,
+    pValue,
+    alpha,
+    minPairs,
+    minDelta,
+    suiteRegressionTolerance,
+    duration: {
+      baselineMeanMs: baselineDuration,
+      uesMeanMs: uesDuration,
+      ratio: durationRatio,
+      maxRatio: maxDurationRatio,
+    },
+    initialInput: {
+      baselineMeanTokens: baselineInitialInput,
+      uesMeanTokens: uesInitialInput,
+      ratio: initialInputRatio,
+      maxRatio: maxInitialInputRatio,
+    },
+    tokens: {
+      baselineMean: baselineTokens,
+      uesMean: uesTokens,
+      ratio: tokenRatio,
+      maxRatio: maxTokenRatio,
+    },
+    cost: {
+      baselineMean: baselineCost,
+      uesMean: uesCost,
+      ratio: costRatio,
+    },
+    suites,
+    checks,
+    promotionEligible: Object.values(checks).every(Boolean),
+  }
+}

package/lib/cli-utils.mjs

@@ -0,0 +1,41 @@
+import { readFileSync } from "node:fs"
+import path from "node:path"
+
+export function clipOutput(value, { head = 8000, tail = 8000 } = {}) {
+  const text = String(value ?? "")
+  if (text.length <= head + tail) return text
+  return text.slice(0, head) + "\n...[truncated]\n" + text.slice(-tail)
+}
+
+export function positionalArg(args, index) {
+  const value = args[index]
+  if (value == null || value === "" || value === "--" || value.startsWith("--")) return null
+  return value
+}
+
+export function readJsonFile(file) {
+  return JSON.parse(readFileSync(path.resolve(file), "utf8"))
+}
+
+export function readTextFile(file) {
+  return readFileSync(path.resolve(file), "utf8")
+}
+
+export function optionInt(args, name, fallback) {
+  const index = args.indexOf(name)
+  const raw = index >= 0 ? args[index + 1] : null
+  return Number.parseInt(raw || String(fallback), 10)
+}
+
+export function optionIntOrUndefined(args, name) {
+  return optionInt(args, name, 0) || undefined
+}
+
+export function optionValue(args, name) {
+  const index = args.indexOf(name)
+  return index >= 0 ? args[index + 1] : null
+}
+
+export function errorMessage(error) {
+  return error instanceof Error ? error.message : error
+}

package/lib/container-sandbox.mjs

@@ -0,0 +1,102 @@
+import { spawnSync } from "node:child_process"
+import path from "node:path"
+
+function probe(engine) {
+  const result = spawnSync(engine, ["--version"], { encoding: "utf8", timeout: 5000 })
+  return {
+    engine,
+    available: result.status === 0,
+    version: result.status === 0 ? String(result.stdout || result.stderr || "").trim() : null,
+    error: result.status === 0 ? null : String(result.stderr || result.stdout || "").trim() || "unavailable",
+  }
+}
+
+export function containerSandboxCapability(preferred = null) {
+  const names = preferred ? [preferred] : ["docker", "podman"]
+  const probes = names.map(probe)
+  const selected = probes.find((item) => item.available) || null
+  return {
+    schemaVersion: 1,
+    available: Boolean(selected),
+    selected,
+    probes,
+    guarantees: selected ? [
+      "network disabled by default",
+      "Linux capabilities dropped",
+      "no-new-privileges",
+      "read-only container root",
+      "bounded pids/memory/cpu",
+      "only requested workspace bind-mounted read-write",
+      "no host environment secrets forwarded by default",
+    ] : [],
+    limitation: "This isolates deterministic verification commands, not the OpenCode model session itself.",
+  }
+}
+
+function validImage(image) {
+  return /^[A-Za-z0-9][A-Za-z0-9._/:@-]{0,240}$/.test(String(image || ""))
+}
+
+export function buildContainerSandboxArgs(root, image, command, commandArgs = [], options = {}) {
+  if (!validImage(image)) throw new Error("container image is required and must be a concrete image reference")
+  if (!command) throw new Error("sandbox command is required")
+  root = path.resolve(root)
+  const memory = String(options.memory || "2g")
+  const cpus = String(options.cpus || "2")
+  const pids = String(options.pids || "256")
+  const args = [
+    "run", "--rm", "--init",
+    "--network", options.network === "bridge" ? "bridge" : "none",
+    "--cap-drop", "ALL",
+    "--security-opt", "no-new-privileges",
+    "--read-only",
+    "--pids-limit", pids,
+    "--memory", memory,
+    "--cpus", cpus,
+    "--tmpfs", "/tmp:rw,noexec,nosuid,size=256m",
+    "--mount", "type=bind,source=" + root + ",target=/workspace",
+    "--workdir", "/workspace",
+    "--env", "CI=true",
+  ]
+  if (process.platform !== "win32" && typeof process.getuid === "function" && typeof process.getgid === "function") {
+    args.push("--user", process.getuid() + ":" + process.getgid())
+  }
+  args.push(image, command, ...commandArgs)
+  return args
+}
+
+export function runContainerSandbox(root, input = {}) {
+  const capability = containerSandboxCapability(input.engine || null)
+  if (!capability.available) {
+    const error = new Error("container sandbox requested but Docker/Podman is unavailable")
+    error.capability = capability
+    throw error
+  }
+  const args = buildContainerSandboxArgs(root, input.image, input.command, input.args || [], input)
+  const started = Date.now()
+  const result = spawnSync(capability.selected.engine, args, {
+    cwd: path.resolve(root),
+    encoding: "utf8",
+    maxBuffer: Math.max(1024 * 1024, Number(input.maxBuffer || 8 * 1024 * 1024)),
+    timeout: Math.max(1000, Number(input.timeoutMs || 10 * 60_000)),
+    env: {
+      SystemRoot: process.env.SystemRoot,
+      WINDIR: process.env.WINDIR,
+      PATH: process.env.PATH,
+      PATHEXT: process.env.PATHEXT,
+      HOME: process.env.HOME,
+      USERPROFILE: process.env.USERPROFILE,
+    },
+  })
+  return {
+    schemaVersion: 1,
+    engine: capability.selected,
+    args,
+    status: result.status ?? 1,
+    signal: result.signal || null,
+    stdout: String(result.stdout || ""),
+    stderr: String(result.stderr || ""),
+    durationMs: Date.now() - started,
+    timedOut: Boolean(result.error?.code === "ETIMEDOUT"),
+  }
+}