@ted-galago/wave-cli 0.1.4 → 0.1.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ted-galago/wave-cli",
-  "version": "0.1.4",
+  "version": "0.1.6",
   "type": "module",
   "bin": {
     "wave": "dist/index.js"
@@ -14,6 +14,7 @@
     "dev": "tsx src/index.ts",
     "test": "vitest run",
     "verify:dev": "node scripts/verify-dev-api.mjs",
+    "benchmark:cli": "node scripts/benchmark-cli.mjs",
     "postinstall": "node scripts/postinstall-local-bin.mjs"
   },
   "engines": {

package/scripts/benchmark-cli.mjs

@@ -0,0 +1,626 @@
+#!/usr/bin/env node
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { spawnSync } from "node:child_process";
+
+const DEFAULTS = {
+  runs: 10,
+  warmup: 2,
+  query: "notes",
+  timeoutMs: 15000,
+  waveEntry: "dist/index.js",
+  verbose: false,
+  skipWave: false,
+  skipObsidian: false,
+  obsidianBin: ""
+};
+
+function printUsage() {
+  console.log(`Wave vs Obsidian CLI benchmark\n\nUsage:\n  node scripts/benchmark-cli.mjs [options]\n\nOptions:\n  --runs <n>             Measured runs per scenario (default: ${DEFAULTS.runs})\n  --warmup <n>           Warmup runs per scenario (default: ${DEFAULTS.warmup})\n  --query <text>         Search query used in search scenarios (default: ${DEFAULTS.query})\n  --timeout-ms <n>       Timeout per command in ms (default: ${DEFAULTS.timeoutMs})\n  --obsidian-bin <path>  Override Obsidian binary path\n  --wave-entry <path>    Override Wave CLI entry script (default: ${DEFAULTS.waveEntry})\n  --json-out <path>      Write full benchmark output as JSON\n  --verbose              Print per-run details\n  --skip-wave            Skip Wave CLI benchmarks\n  --skip-obsidian        Skip Obsidian CLI benchmarks\n  --help                 Show this help\n\nExamples:\n  node scripts/benchmark-cli.mjs\n  node scripts/benchmark-cli.mjs --runs 20 --warmup 3 --query \"meeting notes\"\n  node scripts/benchmark-cli.mjs --obsidian-bin /Applications/Obsidian.app/Contents/MacOS/obsidian-cli\n`);
+}
+
+function parseArgs(argv) {
+  const options = { ...DEFAULTS, jsonOut: "" };
+
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+
+    if (arg === "--help") {
+      options.help = true;
+      continue;
+    }
+
+    if (arg === "--verbose") {
+      options.verbose = true;
+      continue;
+    }
+
+    if (arg === "--skip-wave") {
+      options.skipWave = true;
+      continue;
+    }
+
+    if (arg === "--skip-obsidian") {
+      options.skipObsidian = true;
+      continue;
+    }
+
+    if (arg.startsWith("--")) {
+      const key = arg;
+      const value = argv[i + 1];
+      if (value === undefined || value.startsWith("--")) {
+        throw new Error(`Missing value for ${key}`);
+      }
+
+      if (key === "--runs") {
+        options.runs = toPositiveInt(value, "--runs");
+      } else if (key === "--warmup") {
+        options.warmup = toNonNegativeInt(value, "--warmup");
+      } else if (key === "--query") {
+        options.query = value;
+      } else if (key === "--timeout-ms") {
+        options.timeoutMs = toPositiveInt(value, "--timeout-ms");
+      } else if (key === "--obsidian-bin") {
+        options.obsidianBin = value;
+      } else if (key === "--wave-entry") {
+        options.waveEntry = value;
+      } else if (key === "--json-out") {
+        options.jsonOut = value;
+      } else {
+        throw new Error(`Unknown option: ${key}`);
+      }
+
+      i += 1;
+      continue;
+    }
+
+    throw new Error(`Unexpected argument: ${arg}`);
+  }
+
+  return options;
+}
+
+function toPositiveInt(raw, name) {
+  const num = Number(raw);
+  if (!Number.isInteger(num) || num <= 0) {
+    throw new Error(`${name} must be a positive integer.`);
+  }
+  return num;
+}
+
+function toNonNegativeInt(raw, name) {
+  const num = Number(raw);
+  if (!Number.isInteger(num) || num < 0) {
+    throw new Error(`${name} must be a non-negative integer.`);
+  }
+  return num;
+}
+
+function loadDotEnv(filePath) {
+  if (!existsSync(filePath)) return {};
+  const raw = readFileSync(filePath, "utf8");
+  const env = {};
+  for (const line of raw.split(/\r?\n/)) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith("#")) continue;
+    const match = trimmed.match(/^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/);
+    if (!match) continue;
+    const [, key, value] = match;
+    env[key] = value;
+  }
+  return env;
+}
+
+function commandExists(command) {
+  const probe = spawnSync("/bin/zsh", ["-lc", `command -v ${escapeShellWord(command)}`], {
+    encoding: "utf8"
+  });
+  if (probe.status !== 0) return "";
+  return probe.stdout.trim();
+}
+
+function escapeShellWord(value) {
+  return `'${String(value).replace(/'/g, `'\\''`)}'`;
+}
+
+function resolveObsidianBinary(explicitPath) {
+  if (explicitPath) {
+    return { path: explicitPath, source: "--obsidian-bin" };
+  }
+
+  const inPath = commandExists("obsidian");
+  if (inPath) {
+    return { path: inPath, source: "PATH" };
+  }
+
+  const appBundlePath = "/Applications/Obsidian.app/Contents/MacOS/obsidian-cli";
+  if (existsSync(appBundlePath)) {
+    return { path: appBundlePath, source: "app-bundle" };
+  }
+
+  return { path: "", source: "not-found" };
+}
+
+function nowIso() {
+  return new Date().toISOString();
+}
+
+function runTimed(binary, args, env, timeoutMs) {
+  const startedAt = process.hrtime.bigint();
+  const child = spawnSync(binary, args, {
+    env,
+    cwd: process.cwd(),
+    encoding: "utf8",
+    timeout: timeoutMs,
+    maxBuffer: 10 * 1024 * 1024
+  });
+  const endedAt = process.hrtime.bigint();
+
+  const durationMs = Number(endedAt - startedAt) / 1_000_000;
+
+  return {
+    command: [binary, ...args].join(" "),
+    args,
+    status: child.status,
+    signal: child.signal,
+    durationMs,
+    stdout: child.stdout ?? "",
+    stderr: child.stderr ?? "",
+    timedOut: child.error?.code === "ETIMEDOUT"
+  };
+}
+
+function parseWaveEnvelope(stdout) {
+  const text = (stdout || "").trim();
+  if (!text) return null;
+  try {
+    return JSON.parse(text);
+  } catch {
+    return null;
+  }
+}
+
+function evaluateRun(toolId, expectation, mode, result) {
+  if (result.timedOut) {
+    return {
+      pass: false,
+      reason: "timeout",
+      waveJson: false,
+      waveOk: false
+    };
+  }
+
+  if (toolId === "wave") {
+    const envelope = parseWaveEnvelope(result.stdout);
+    const waveJson = Boolean(envelope);
+    const waveOk = envelope?.ok === true;
+
+    if (mode === "exit_only") {
+      if (expectation === "success") {
+        const pass = result.status === 0;
+        return {
+          pass,
+          reason: pass ? "ok" : `exit_${result.status ?? "null"}`,
+          waveJson,
+          waveOk
+        };
+      }
+
+      const pass = result.status !== 0;
+      return {
+        pass,
+        reason: pass ? "expected_failure" : "unexpected_success",
+        waveJson,
+        waveOk
+      };
+    }
+
+    if (expectation === "success") {
+      const pass = result.status === 0 && waveOk;
+      return {
+        pass,
+        reason: pass ? "ok" : envelope ? `wave_not_ok:${envelope?.error?.code ?? "unknown"}` : "non_json_output",
+        waveJson,
+        waveOk
+      };
+    }
+
+    const pass = result.status !== 0 && envelope?.ok === false;
+    return {
+      pass,
+      reason: pass ? "expected_failure" : "unexpected_success_or_format",
+      waveJson,
+      waveOk
+    };
+  }
+
+  if (expectation === "success") {
+    const pass = result.status === 0;
+    const combined = `${result.stdout || ""}\n${result.stderr || ""}`.toLowerCase();
+    const reason = pass
+      ? "ok"
+      : combined.includes("unable to find obsidian")
+        ? "obsidian_not_running"
+        : `exit_${result.status ?? "null"}`;
+    return {
+      pass,
+      reason,
+      waveJson: false,
+      waveOk: false
+    };
+  }
+
+  const pass = result.status !== 0;
+  return {
+    pass,
+    reason: pass ? "expected_failure" : "unexpected_success",
+    waveJson: false,
+    waveOk: false
+  };
+}
+
+function percentile(values, p) {
+  if (values.length === 0) return null;
+  const sorted = [...values].sort((a, b) => a - b);
+  const idx = Math.ceil(sorted.length * p) - 1;
+  return sorted[Math.max(0, Math.min(sorted.length - 1, idx))];
+}
+
+function mean(values) {
+  if (values.length === 0) return null;
+  return values.reduce((acc, v) => acc + v, 0) / values.length;
+}
+
+function formatMs(value) {
+  if (value === null || Number.isNaN(value)) return "n/a";
+  return `${value.toFixed(1)}ms`;
+}
+
+function pad(value, width) {
+  const text = String(value);
+  if (text.length >= width) return text;
+  return `${text}${" ".repeat(width - text.length)}`;
+}
+
+function printTable(rows) {
+  if (rows.length === 0) return;
+
+  const headers = [
+    "Scenario",
+    "Tool",
+    "Pass",
+    "Avg",
+    "P50",
+    "P95",
+    "Min",
+    "Max",
+    "Runs",
+    "JSON"
+  ];
+
+  const widths = headers.map((h) => h.length);
+  const normalizedRows = rows.map((row) => {
+    const cells = [
+      row.scenario,
+      row.tool,
+      `${row.passRate.toFixed(0)}%`,
+      formatMs(row.avgMs),
+      formatMs(row.p50Ms),
+      formatMs(row.p95Ms),
+      formatMs(row.minMs),
+      formatMs(row.maxMs),
+      String(row.runs),
+      row.jsonRate
+    ];
+    cells.forEach((cell, index) => {
+      widths[index] = Math.max(widths[index], String(cell).length);
+    });
+    return cells;
+  });
+
+  const headerLine = headers.map((h, i) => pad(h, widths[i])).join("  ");
+  const separator = widths.map((w) => "-".repeat(w)).join("  ");
+
+  console.log("\nBenchmark Summary");
+  console.log(headerLine);
+  console.log(separator);
+
+  normalizedRows.forEach((cells) => {
+    console.log(cells.map((cell, i) => pad(cell, widths[i])).join("  "));
+  });
+}
+
+function buildScenarios(query) {
+  return [
+    {
+      id: "cold_help",
+      label: "Cold help",
+      category: "mechanics",
+      expectation: "success",
+      runs: 1,
+      warmup: 0,
+      modes: {
+        wave: "exit_only",
+        obsidian: "exit_only"
+      },
+      commands: {
+        wave: ["--help"],
+        obsidian: ["help"]
+      }
+    },
+    {
+      id: "warm_help",
+      label: "Warm help",
+      category: "mechanics",
+      expectation: "success",
+      modes: {
+        wave: "exit_only",
+        obsidian: "exit_only"
+      },
+      commands: {
+        wave: ["--help"],
+        obsidian: ["help"]
+      }
+    },
+    {
+      id: "list_read",
+      label: "List/read",
+      category: "domain",
+      expectation: "success",
+      commands: {
+        wave: ["tasks", "list", "--page", "1", "--per", "10"],
+        obsidian: ["files", "total"]
+      }
+    },
+    {
+      id: "search",
+      label: "Search",
+      category: "domain",
+      expectation: "success",
+      commands: {
+        wave: ["find", query, "--limit", "10"],
+        obsidian: ["search", `query=${query}`]
+      }
+    },
+    {
+      id: "invalid_input",
+      label: "Invalid input",
+      category: "error",
+      expectation: "failure",
+      commands: {
+        wave: ["tasks", "show"],
+        obsidian: ["__wave_benchmark_invalid_command__"]
+      }
+    }
+  ];
+}
+
+function summarizeRuns(scenario, tool, runs) {
+  const latencies = runs.map((run) => run.durationMs);
+  const passCount = runs.filter((run) => run.pass).length;
+  const waveJsonCount = runs.filter((run) => run.waveJson).length;
+
+  return {
+    scenario: scenario.label,
+    scenarioId: scenario.id,
+    category: scenario.category,
+    tool,
+    runs: runs.length,
+    passRate: runs.length === 0 ? 0 : (passCount / runs.length) * 100,
+    avgMs: mean(latencies),
+    p50Ms: percentile(latencies, 0.5),
+    p95Ms: percentile(latencies, 0.95),
+    minMs: latencies.length ? Math.min(...latencies) : null,
+    maxMs: latencies.length ? Math.max(...latencies) : null,
+    jsonRate: tool === "wave" ? `${Math.round((waveJsonCount / Math.max(1, runs.length)) * 100)}%` : "n/a"
+  };
+}
+
+function printPreflight(preflight) {
+  console.log("Preflight");
+  console.log(`  Timestamp: ${preflight.timestamp}`);
+  console.log(`  Wave entry: ${preflight.waveEntry}`);
+  console.log(`  Wave ready: ${preflight.waveReady ? "yes" : "no"}`);
+  if (preflight.waveMissing.length > 0) {
+    console.log(`  Wave missing env: ${preflight.waveMissing.join(", ")}`);
+  }
+  console.log(`  Obsidian binary: ${preflight.obsidianPath || "not found"}`);
+  console.log(`  Obsidian source: ${preflight.obsidianSource}`);
+  console.log(`  Obsidian ready: ${preflight.obsidianReady ? "yes" : "no"}`);
+}
+
+function printOperationalNotes(preflight) {
+  console.log("\nOperational Notes");
+  console.log("  Wave CLI: requires API base URL, token, and organization ID; outputs strict JSON envelope.");
+  console.log("  Obsidian CLI: requires Obsidian desktop CLI registration and usually a running app instance.");
+  if (!preflight.obsidianReady) {
+    console.log("  Obsidian benchmarks were skipped because no binary was found.");
+  }
+  if (!preflight.waveReady) {
+    console.log("  Wave benchmarks were skipped because required env/build prerequisites were missing.");
+  }
+}
+
+function printFailureReasons(allRunResults) {
+  const failedRuns = allRunResults.filter((run) => !run.pass);
+  if (failedRuns.length === 0) return;
+
+  const grouped = new Map();
+  for (const run of failedRuns) {
+    const key = `${run.scenarioLabel} | ${run.tool}`;
+    if (!grouped.has(key)) grouped.set(key, new Map());
+    const reasonMap = grouped.get(key);
+    reasonMap.set(run.reason, (reasonMap.get(run.reason) || 0) + 1);
+  }
+
+  console.log("\nTop Failure Reasons");
+  for (const [group, reasonMap] of grouped.entries()) {
+    const sorted = [...reasonMap.entries()].sort((a, b) => b[1] - a[1]);
+    const [reason, count] = sorted[0];
+    console.log(`  ${group}: ${reason} (${count})`);
+  }
+}
+
+function main() {
+  const options = parseArgs(process.argv.slice(2));
+  if (options.help) {
+    printUsage();
+    return;
+  }
+
+  const cwd = process.cwd();
+  const dotEnvPath = resolve(cwd, ".env");
+  const dotEnv = loadDotEnv(dotEnvPath);
+  const baseEnv = {
+    ...process.env,
+    ...dotEnv,
+    WAVE_AGENT_RUN_ID: process.env.WAVE_AGENT_RUN_ID || `benchmark-${Date.now()}`,
+    WAVE_AGENT_NAME: process.env.WAVE_AGENT_NAME || dotEnv.WAVE_AGENT_NAME || "cli-benchmark"
+  };
+
+  const waveEntryAbs = resolve(cwd, options.waveEntry);
+  const waveMissing = ["WAVE_API_BASE_URL", "WAVE_API_TOKEN", "WAVE_ORGANIZATION_ID"].filter((key) => !baseEnv[key]);
+  const waveReady = existsSync(waveEntryAbs) && waveMissing.length === 0;
+
+  const obsidian = resolveObsidianBinary(options.obsidianBin);
+  const obsidianReady = Boolean(obsidian.path);
+
+  const preflight = {
+    timestamp: nowIso(),
+    waveEntry: waveEntryAbs,
+    waveReady,
+    waveMissing,
+    obsidianReady,
+    obsidianPath: obsidian.path,
+    obsidianSource: obsidian.source,
+    options
+  };
+
+  printPreflight(preflight);
+
+  const tools = [];
+  if (!options.skipWave && waveReady) {
+    tools.push({
+      id: "wave",
+      label: "Wave",
+      binary: process.execPath,
+      baseArgs: [waveEntryAbs],
+      env: baseEnv
+    });
+  }
+
+  if (!options.skipObsidian && obsidianReady) {
+    tools.push({
+      id: "obsidian",
+      label: "Obsidian",
+      binary: obsidian.path,
+      baseArgs: [],
+      env: process.env
+    });
+  }
+
+  if (tools.length === 0) {
+    console.log("\nNo tools available to benchmark. Check preflight status or remove skip flags.");
+    process.exitCode = 1;
+    return;
+  }
+
+  const scenarios = buildScenarios(options.query);
+  const allRunResults = [];
+  const summaryRows = [];
+
+  for (const scenario of scenarios) {
+    const measuredRuns = scenario.runs ?? options.runs;
+    const warmupRuns = scenario.warmup ?? options.warmup;
+
+    for (const tool of tools) {
+      const argsForTool = scenario.commands[tool.id];
+      if (!argsForTool) continue;
+
+      console.log(`\n[${scenario.id}] ${tool.label} (warmup=${warmupRuns}, runs=${measuredRuns})`);
+
+      for (let i = 0; i < warmupRuns; i += 1) {
+        const warmup = runTimed(tool.binary, [...tool.baseArgs, ...argsForTool], tool.env, options.timeoutMs);
+        if (options.verbose) {
+          console.log(`  warmup ${i + 1}/${warmupRuns} ${warmup.status === 0 ? "ok" : "fail"} ${warmup.durationMs.toFixed(1)}ms`);
+        }
+      }
+
+      const runRows = [];
+      for (let i = 0; i < measuredRuns; i += 1) {
+        const result = runTimed(tool.binary, [...tool.baseArgs, ...argsForTool], tool.env, options.timeoutMs);
+        const mode = scenario.modes?.[tool.id] ?? "strict_json";
+        const evalResult = evaluateRun(tool.id, scenario.expectation, mode, result);
+        const merged = {
+          ...result,
+          ...evalResult,
+          scenarioId: scenario.id,
+          scenarioLabel: scenario.label,
+          tool: tool.id,
+          run: i + 1,
+          measuredRuns,
+          warmupRuns
+        };
+
+        if (options.verbose) {
+          console.log(`  run ${i + 1}/${measuredRuns} ${merged.pass ? "ok" : "fail"} ${merged.durationMs.toFixed(1)}ms (${merged.reason})`);
+        }
+
+        runRows.push(merged);
+        allRunResults.push(merged);
+      }
+
+      const summary = summarizeRuns(scenario, tool.id, runRows);
+      summaryRows.push(summary);
+      console.log(`  => pass ${summary.passRate.toFixed(0)}% | avg ${formatMs(summary.avgMs)} | p95 ${formatMs(summary.p95Ms)}`);
+    }
+  }
+
+  printTable(summaryRows);
+  printFailureReasons(allRunResults);
+  printOperationalNotes(preflight);
+
+  const winners = [];
+  for (const scenario of scenarios) {
+    const wave = summaryRows.find((row) => row.scenarioId === scenario.id && row.tool === "wave");
+    const obsidianRow = summaryRows.find((row) => row.scenarioId === scenario.id && row.tool === "obsidian");
+    if (!wave || !obsidianRow) continue;
+    if (wave.avgMs === null || obsidianRow.avgMs === null) continue;
+
+    const winner = wave.avgMs < obsidianRow.avgMs ? "Wave" : "Obsidian";
+    const delta = Math.abs(wave.avgMs - obsidianRow.avgMs);
+    winners.push({ scenario: scenario.label, winner, deltaMs: delta });
+  }
+
+  if (winners.length > 0) {
+    console.log("\nSpeed Winner By Scenario");
+    for (const winner of winners) {
+      console.log(`  ${winner.scenario}: ${winner.winner} (${winner.deltaMs.toFixed(1)}ms faster on average)`);
+    }
+  }
+
+  if (options.jsonOut) {
+    const outPath = resolve(cwd, options.jsonOut);
+    const payload = {
+      generatedAt: nowIso(),
+      cwd,
+      options,
+      preflight,
+      summary: summaryRows,
+      runs: allRunResults
+    };
+    writeFileSync(outPath, JSON.stringify(payload, null, 2));
+    console.log(`\nWrote JSON report: ${outPath}`);
+  }
+
+  const failures = summaryRows.filter((row) => row.passRate < 100).length;
+  process.exitCode = failures > 0 ? 1 : 0;
+}
+
+try {
+  main();
+} catch (error) {
+  const message = error instanceof Error ? error.message : String(error);
+  console.error(`benchmark-cli failed: ${message}`);
+  process.exit(1);
+}