akm-cli 0.7.0-rc1 → 0.7.0

package/dist/tests/bench/run-curate-test.js

@@ -0,0 +1,32 @@
+/**
+ * OBSOLETE: superseded by `bun run tests/bench/cli.ts tests/bench/configs/curate-test.json`.
+ * Kept for backward compatibility; will be removed in the standalone-bench-repo extraction.
+ *
+ * Test akm curate as first command on configure-scaling.
+ * Usage: bun run tests/bench/run-curate-test.ts
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { loadTask } from "./corpus";
+import { loadOpencodeProviders } from "./opencode-config";
+import { runUtility } from "./runner";
+process.stderr.write("[obsolete] run-curate-test.ts → see tests/bench/configs/curate-test.json (`bun run tests/bench/cli.ts tests/bench/configs/curate-test.json`)\n");
+const tasks = [loadTask("inkwell/configure-scaling")];
+const LOCAL = path.resolve(__dirname, "..", "fixtures", "bench", "opencode-providers.local.json");
+const DEFAULT = path.resolve(__dirname, "..", "fixtures", "bench", "opencode-providers.json");
+const providers = loadOpencodeProviders(fs.existsSync(LOCAL) ? LOCAL : DEFAULT);
+process.stderr.write(`Running configure-scaling × 5 seeds (curate as first cmd)\nModel: ${providers.defaultModel}\n\n`);
+const report = await runUtility({
+    tasks,
+    arms: ["akm"],
+    model: providers.defaultModel,
+    seedsPerArm: 5,
+    budgetTokens: 25000,
+    budgetWallMs: 360000,
+    parallel: 3,
+    opencodeProviders: providers,
+});
+process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
+const t = report.tasks?.[0];
+const rate = t?.akm?.passRate ?? 0;
+process.stderr.write(`\nconfigure-scaling: ${(rate * 100).toFixed(0)}% (baseline 80%)\n`);

package/dist/tests/bench/run-failing-tasks.js

@@ -0,0 +1,56 @@
+/**
+ * OBSOLETE: superseded by `bun run tests/bench/cli.ts tests/bench/configs/failing-tasks.json`.
+ * Kept for backward compatibility; will be removed in the standalone-bench-repo extraction.
+ *
+ * Targeted retest of failing/partial tasks after stash improvements.
+ * Usage: bun run tests/bench/run-failing-tasks.ts
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { loadTask } from "./corpus";
+import { loadOpencodeProviders } from "./opencode-config";
+import { runUtility } from "./runner";
+process.stderr.write("[obsolete] run-failing-tasks.ts → see tests/bench/configs/failing-tasks.json (`bun run tests/bench/cli.ts tests/bench/configs/failing-tasks.json`)\n");
+const TASK_IDS = [
+    "drillbit/backup-policy",
+    "drillbit/canary-enable",
+    "inkwell/add-healthcheck",
+    "inkwell/configure-scaling",
+    "opencode/select-correct-skill",
+];
+const tasks = TASK_IDS.map((id) => loadTask(id));
+const LOCAL = path.resolve(__dirname, "..", "fixtures", "bench", "opencode-providers.local.json");
+const DEFAULT = path.resolve(__dirname, "..", "fixtures", "bench", "opencode-providers.json");
+const providers = loadOpencodeProviders(fs.existsSync(LOCAL) ? LOCAL : DEFAULT);
+process.stderr.write(`Running ${tasks.length} tasks × 5 seeds (akm only)\nModel: ${providers.defaultModel}\n\n`);
+const report = await runUtility({
+    tasks,
+    arms: ["akm"],
+    model: providers.defaultModel,
+    seedsPerArm: 5,
+    budgetTokens: 25000,
+    budgetWallMs: 360000,
+    parallel: 3,
+    opencodeProviders: providers,
+});
+process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
+const agg = report.aggregateAkm;
+process.stderr.write(`\n=== RESULTS vs BASELINE ===\n`);
+// Qwen 9B baseline for comparison
+const BASELINE = {
+    "drillbit/backup-policy": 1.0,
+    "drillbit/canary-enable": 1.0,
+    "inkwell/add-healthcheck": 0.8,
+    "inkwell/configure-scaling": 0.8,
+    "opencode/select-correct-skill": 1.0,
+};
+for (const t of report.tasks ?? []) {
+    const rate = t.akm?.passRate ?? 0;
+    const base = BASELINE[t.id] ?? 0;
+    const delta = rate - base;
+    const arrow = delta > 0 ? "↑" : delta < 0 ? "↓" : "=";
+    const bar = "█".repeat(Math.round(rate * 5)) + "░".repeat(5 - Math.round(rate * 5));
+    const deltaStr = delta !== 0 ? ` (${arrow}${Math.abs(delta * 100).toFixed(0)}pp)` : "";
+    process.stderr.write(`${t.id.padEnd(48)} ${(rate * 100).toFixed(0).padStart(3)}%  ${bar}${deltaStr}\n`);
+}
+process.stderr.write(`\nOverall: ${((agg?.passRate ?? 0) * 100).toFixed(1)}%\n`);

package/dist/tests/bench/run-full-bench.js

@@ -0,0 +1,51 @@
+/**
+ * OBSOLETE: superseded by `bun run tests/bench/cli.ts tests/bench/configs/full.json`.
+ * Kept for backward compatibility; will be removed in the standalone-bench-repo extraction.
+ *
+ * Full benchmark run — all tasks, 5 seeds, akm arm only.
+ * Usage: bun run tests/bench/run-full-bench.ts
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { listTasks } from "./corpus";
+import { loadOpencodeProviders } from "./opencode-config";
+import { runUtility } from "./runner";
+process.stderr.write("[obsolete] run-full-bench.ts → see tests/bench/configs/full.json (`bun run tests/bench/cli.ts tests/bench/configs/full.json`)\n");
+const tasks = listTasks();
+const LOCAL = path.resolve(__dirname, "..", "fixtures", "bench", "opencode-providers.local.json");
+const DEFAULT = path.resolve(__dirname, "..", "fixtures", "bench", "opencode-providers.json");
+const providers = loadOpencodeProviders(fs.existsSync(LOCAL) ? LOCAL : DEFAULT);
+process.stderr.write(`Running ${tasks.length} tasks × 5 seeds (akm only)\nModel: ${providers.defaultModel}\n\n`);
+const report = await runUtility({
+    tasks,
+    arms: ["akm"],
+    model: providers.defaultModel,
+    seedsPerArm: 5,
+    budgetTokens: 25000,
+    budgetWallMs: 360000,
+    parallel: 3,
+    opencodeProviders: providers,
+});
+process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
+const BASELINE = {
+    "drillbit/backup-policy": 1.0,
+    "drillbit/canary-enable": 1.0,
+    "inkwell/add-healthcheck": 0.8,
+    "inkwell/configure-scaling": 0.8,
+    "opencode/select-correct-skill": 1.0,
+};
+process.stderr.write(`\n=== RESULTS vs BASELINE ===\n`);
+for (const t of report.tasks ?? []) {
+    const rate = t.akm?.passRate ?? 0;
+    const base = BASELINE[t.id] ?? null;
+    const bar = "█".repeat(Math.round(rate * 5)) + "░".repeat(5 - Math.round(rate * 5));
+    const deltaStr = base !== null
+        ? (() => {
+            const d = rate - base;
+            const arrow = d > 0 ? "↑" : d < 0 ? "↓" : "=";
+            return d !== 0 ? ` (${arrow}${Math.abs(d * 100).toFixed(0)}pp)` : " (=)";
+        })()
+        : "";
+    process.stderr.write(`${t.id.padEnd(52)} ${(rate * 100).toFixed(0).padStart(3)}%  ${bar}${deltaStr}\n`);
+}
+process.stderr.write(`\nOverall: ${((report.aggregateAkm?.passRate ?? 0) * 100).toFixed(1)}%\n`);

package/dist/tests/bench/run-items36-targeted.js

@@ -0,0 +1,69 @@
+/**
+ * Items 3-6 targeted bench — tasks most directly affected by fixture stash
+ * content additions and skill frontmatter strip (commit 92196c7).
+ * Usage: bun run tests/bench/run-items36-targeted.ts
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { loadTask } from "./corpus";
+import { loadOpencodeProviders } from "./opencode-config";
+import { runUtility } from "./runner";
+const TARGET_TASKS = [
+    // item 4: env_file section moved to top of compose-conventions.md
+    "docker-homelab/env-from-file",
+    // item 3: az-storage-lifecycle knowledge added to az-cli stash
+    "workflow-compliance/repeated-fail-storage-lifecycle-a",
+    // item 5: memory assets (compound-tag-filter, null-value-trap)
+    "az-cli/query-by-tag",
+    // item 5: memory asset (healthcheck-test-cmd)
+    "inkwell/add-healthcheck-train",
+    // item 6: skill frontmatter strip — previously low-scoring tasks
+    "docker-homelab/restart-policy",
+    "docker-homelab/redis-healthcheck",
+    "docker-homelab/named-volume",
+    "az-cli/storage-account-create",
+    "inkwell/configure-scaling",
+];
+const tasks = TARGET_TASKS.map((id) => loadTask(id));
+const LOCAL = path.resolve(__dirname, "..", "fixtures", "bench", "opencode-providers.local.json");
+const DEFAULT = path.resolve(__dirname, "..", "fixtures", "bench", "opencode-providers.json");
+const providers = loadOpencodeProviders(fs.existsSync(LOCAL) ? LOCAL : DEFAULT);
+process.stderr.write(`Items 3-6 targeted bench: ${tasks.length} tasks × 3 seeds\nModel: ${providers.defaultModel}\n\n`);
+const report = await runUtility({
+    tasks,
+    arms: ["akm"],
+    model: providers.defaultModel,
+    seedsPerArm: 3,
+    budgetTokens: 25000,
+    budgetWallMs: 360000,
+    parallel: 3,
+    opencodeProviders: providers,
+});
+process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
+// Wave G baselines from 2026-05-03 targeted run
+const BASELINE = {
+    "docker-homelab/env-from-file": 0.0,
+    "workflow-compliance/repeated-fail-storage-lifecycle-a": 0.0,
+    "az-cli/query-by-tag": 0.4,
+    "inkwell/add-healthcheck-train": 0.67,
+    "docker-homelab/restart-policy": 0.33,
+    "docker-homelab/redis-healthcheck": 0.33,
+    "docker-homelab/named-volume": 0.33,
+    "az-cli/storage-account-create": 1.0,
+    "inkwell/configure-scaling": 0.6,
+};
+process.stderr.write(`\n=== RESULTS vs Wave G BASELINE ===\n`);
+for (const t of report.tasks ?? []) {
+    const rate = t.akm?.passRate ?? 0;
+    const base = BASELINE[t.id] ?? null;
+    const bar = "█".repeat(Math.round(rate * 5)) + "░".repeat(5 - Math.round(rate * 5));
+    const deltaStr = base !== null
+        ? (() => {
+            const d = rate - base;
+            const arrow = d > 0 ? "↑" : d < 0 ? "↓" : "=";
+            return d !== 0 ? ` (${arrow}${Math.abs(d * 100).toFixed(0)}pp)` : " (=)";
+        })()
+        : "";
+    process.stderr.write(`${t.id.padEnd(52)} ${(rate * 100).toFixed(0).padStart(3)}%  ${bar}${deltaStr}\n`);
+}
+process.stderr.write(`\nOverall: ${((report.aggregateAkm?.passRate ?? 0) * 100).toFixed(1)}%\n`);

package/dist/tests/bench/run-nano-quick.js

@@ -0,0 +1,42 @@
+/**
+ * OBSOLETE: superseded by `bun run tests/bench/cli.ts tests/bench/configs/nano-quick.json`.
+ * Kept for backward compatibility; will be removed in the standalone-bench-repo extraction.
+ *
+ * Quick 5-task × 2-seed run for Nemotron Nano evaluation.
+ * Usage: bun run tests/bench/run-nano-quick.ts
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { loadTask } from "./corpus";
+import { loadOpencodeProviders } from "./opencode-config";
+import { runUtility } from "./runner";
+process.stderr.write("[obsolete] run-nano-quick.ts → see tests/bench/configs/nano-quick.json (`bun run tests/bench/cli.ts tests/bench/configs/nano-quick.json`)\n");
+const TASK_IDS = [
+    "drillbit/backup-policy",
+    "drillbit/canary-enable",
+    "inkwell/add-healthcheck",
+    "inkwell/configure-scaling",
+    "opencode/select-correct-skill",
+];
+const tasks = TASK_IDS.map((id) => loadTask(id));
+const LOCAL = path.resolve(__dirname, "..", "fixtures", "bench", "opencode-providers.local.json");
+const DEFAULT = path.resolve(__dirname, "..", "fixtures", "bench", "opencode-providers.json");
+const providers = loadOpencodeProviders(fs.existsSync(LOCAL) ? LOCAL : DEFAULT);
+process.stderr.write(`Running ${tasks.length} tasks × 2 seeds\nModel: ${providers.defaultModel}\n\n`);
+const report = await runUtility({
+    tasks,
+    arms: ["akm"],
+    model: providers.defaultModel,
+    seedsPerArm: 2,
+    budgetTokens: 25000,
+    budgetWallMs: 360000,
+    parallel: 2,
+    opencodeProviders: providers,
+});
+process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
+for (const t of report.tasks ?? []) {
+    const rate = t.akm?.passRate ?? 0;
+    const bar = "█".repeat(Math.round(rate * 5)) + "░".repeat(5 - Math.round(rate * 5));
+    process.stderr.write(`${t.id.padEnd(48)} ${(rate * 100).toFixed(0).padStart(3)}%  ${bar}\n`);
+}
+process.stderr.write(`\nOverall: ${((report.aggregateAkm?.passRate ?? 0) * 100).toFixed(1)}%\n`);

package/dist/tests/bench/run-waveg-targeted.js

@@ -0,0 +1,62 @@
+/**
+ * Wave G targeted bench — 9 previously-failing tasks, 3 seeds.
+ * Usage: bun run tests/bench/run-waveg-targeted.ts
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { loadTask } from "./corpus";
+import { loadOpencodeProviders } from "./opencode-config";
+import { runUtility } from "./runner";
+const TARGET_TASKS = [
+    "inkwell/configure-scaling",
+    "inkwell/add-healthcheck-train",
+    "inkwell/full-config",
+    "az-cli/storage-account-create",
+    "docker-homelab/bridge-network",
+    "docker-homelab/compose-version-upgrade",
+    "docker-homelab/env-from-file",
+    "workflow-compliance/feedback-trap-az-tag-list",
+    "workflow-compliance/repeated-fail-storage-lifecycle-a",
+];
+const tasks = TARGET_TASKS.map((id) => loadTask(id));
+const LOCAL = path.resolve(__dirname, "..", "fixtures", "bench", "opencode-providers.local.json");
+const DEFAULT = path.resolve(__dirname, "..", "fixtures", "bench", "opencode-providers.json");
+const providers = loadOpencodeProviders(fs.existsSync(LOCAL) ? LOCAL : DEFAULT);
+process.stderr.write(`Wave G targeted bench: ${tasks.length} tasks × 3 seeds\nModel: ${providers.defaultModel}\n\n`);
+const report = await runUtility({
+    tasks,
+    arms: ["akm"],
+    model: providers.defaultModel,
+    seedsPerArm: 3,
+    budgetTokens: 25000,
+    budgetWallMs: 360000,
+    parallel: 3,
+    opencodeProviders: providers,
+});
+process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
+const BASELINE = {
+    "inkwell/configure-scaling": 0.6,
+    "inkwell/add-healthcheck-train": 0.4,
+    "inkwell/full-config": 0.0,
+    "az-cli/storage-account-create": 0.4,
+    "docker-homelab/bridge-network": 0.2,
+    "docker-homelab/compose-version-upgrade": 0.4,
+    "docker-homelab/env-from-file": 0.0,
+    "workflow-compliance/feedback-trap-az-tag-list": 0.2,
+    "workflow-compliance/repeated-fail-storage-lifecycle-a": 0.0,
+};
+process.stderr.write(`\n=== RESULTS vs 2026-05-03 BASELINE ===\n`);
+for (const t of report.tasks ?? []) {
+    const rate = t.akm?.passRate ?? 0;
+    const base = BASELINE[t.id] ?? null;
+    const bar = "█".repeat(Math.round(rate * 5)) + "░".repeat(5 - Math.round(rate * 5));
+    const deltaStr = base !== null
+        ? (() => {
+            const d = rate - base;
+            const arrow = d > 0 ? "↑" : d < 0 ? "↓" : "=";
+            return d !== 0 ? ` (${arrow}${Math.abs(d * 100).toFixed(0)}pp)` : " (=)";
+        })()
+        : "";
+    process.stderr.write(`${t.id.padEnd(52)} ${(rate * 100).toFixed(0).padStart(3)}%  ${bar}${deltaStr}\n`);
+}
+process.stderr.write(`\nOverall: ${((report.aggregateAkm?.passRate ?? 0) * 100).toFixed(1)}%\n`);