kushi-agents 5.7.4 → 5.7.5

package/package.json

@@ -1,60 +1,60 @@
-{
-  "name": "kushi-agents",
-  "version": "5.7.4",
-  "description": "Install Kushi — multi-source project evidence agent with Comprehensive Structured Capture (CSC) into weekly-only files across Email, Teams, OneNote, Loop, SharePoint, Meetings, CRM, ADO. Meetings retain a sibling verbatim/ audit folder. WorkIQ-only for M365 sources (Graph / m365_* FORBIDDEN as fallbacks; user-paste is first-class). Host-agnostic.",
-  "type": "module",
-  "bin": {
-    "kushi-agents": "./bin/cli.mjs"
-  },
-  "files": [
-    "bin/",
-    "src/",
-    "plugin/",
-    ".github/copilot-instructions.kushi.md"
-  ],
-  "engines": {
-    "node": ">=18.0.0"
-  },
-  "dependencies": {
-    "@azure/identity": "^4.5.0",
-    "@mozilla/readability": "^0.6.0",
-    "jsdom": "^29.1.1",
-    "jsonc-parser": "^3.3.1",
-    "yaml": "^2.6.0"
-  },
-  "keywords": [
-    "vscode",
-    "copilot",
-    "agents",
-    "kushi",
-    "project-evidence",
-    "workiq",
-    "m365",
-    "ai",
-    "cli"
-  ],
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/gim-home/kushi.git"
-  },
-  "homepage": "https://gim-home.github.io/kushi/",
-  "bugs": {
-    "url": "https://github.com/gim-home/kushi/issues"
-  },
-  "license": "MIT",
-  "scripts": {
-    "test": "node --test src/check-workiq.test.mjs src/seed-config.test.mjs src/sanitize-workiq-input.test.mjs src/detect-vertex-repo.test.mjs src/vertex-validate.test.mjs src/emit-vertex.e2e.test.mjs src/config-root-resolve.test.mjs src/forbidden-workiq-phrasings.test.mjs src/multi-host-install.test.mjs src/eval-aggregator.test.mjs src/eval-runner.test.mjs src/skill-creator.test.mjs src/skill-checker.test.mjs src/hooks-dispatcher.test.mjs src/parallel-refresh.test.mjs src/otel-emit.test.mjs src/teach.test.mjs src/schema-evolve.test.mjs src/global-wiki.test.mjs src/promote.test.mjs src/doctor.test.mjs src/setup-wizard.test.mjs src/cli-no-args.test.mjs src/cli-no-args-tty.test.mjs src/per-user-files.test.mjs src/layout-portable.test.mjs src/profile-coverage.test.mjs src/get-kushi-config.test.mjs src/seed-config-derived.test.mjs plugin/runners/test/unit/*.test.mjs",
-    "test:runners": "node --test plugin/runners/test/unit/*.test.mjs",
-    "test:runners:integration": "node --test plugin/runners/test/integration/*.test.mjs",
-    "test:integration:bootstrap": "node src/bootstrap-dryrun.integration.test.mjs",
-    "smoke": "node scripts/smoke.mjs",
-    "eval": "pwsh plugin/skills/eval/run-evals.ps1 -Skill",
-    "eval:all": "pwsh plugin/skills/eval/run-evals.ps1 -All",
-    "eval:canary": "pwsh plugin/skills/eval/run-evals.ps1 -Canary",
-    "eval:baseline": "pwsh plugin/skills/eval/run-evals.ps1 -All -UpdateBaseline",
-    "prepublishOnly": "npm test && npm run smoke"
-  },
-  "publishConfig": {
-    "access": "public"
-  }
-}
+{
+  "name": "kushi-agents",
+  "version": "5.7.5",
+  "description": "Install Kushi — multi-source project evidence agent with Comprehensive Structured Capture (CSC) into weekly-only files across Email, Teams, OneNote, Loop, SharePoint, Meetings, CRM, ADO. Meetings retain a sibling verbatim/ audit folder. WorkIQ-only for M365 sources (Graph / m365_* FORBIDDEN as fallbacks; user-paste is first-class). Host-agnostic.",
+  "type": "module",
+  "bin": {
+    "kushi-agents": "./bin/cli.mjs"
+  },
+  "files": [
+    "bin/",
+    "src/",
+    "plugin/",
+    ".github/copilot-instructions.kushi.md"
+  ],
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "dependencies": {
+    "@azure/identity": "^4.5.0",
+    "@mozilla/readability": "^0.6.0",
+    "jsdom": "^29.1.1",
+    "jsonc-parser": "^3.3.1",
+    "yaml": "^2.6.0"
+  },
+  "keywords": [
+    "vscode",
+    "copilot",
+    "agents",
+    "kushi",
+    "project-evidence",
+    "workiq",
+    "m365",
+    "ai",
+    "cli"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/gim-home/kushi.git"
+  },
+  "homepage": "https://gim-home.github.io/kushi/",
+  "bugs": {
+    "url": "https://github.com/gim-home/kushi/issues"
+  },
+  "license": "MIT",
+  "scripts": {
+    "test": "node --test src/check-workiq.test.mjs src/seed-config.test.mjs src/sanitize-workiq-input.test.mjs src/detect-vertex-repo.test.mjs src/vertex-validate.test.mjs src/emit-vertex.e2e.test.mjs src/config-root-resolve.test.mjs src/forbidden-workiq-phrasings.test.mjs src/multi-host-install.test.mjs src/eval-aggregator.test.mjs src/eval-runner.test.mjs src/skill-creator.test.mjs src/skill-checker.test.mjs src/hooks-dispatcher.test.mjs src/parallel-refresh.test.mjs src/otel-emit.test.mjs src/teach.test.mjs src/schema-evolve.test.mjs src/global-wiki.test.mjs src/promote.test.mjs src/doctor.test.mjs src/setup-wizard.test.mjs src/cli-no-args.test.mjs src/cli-no-args-tty.test.mjs src/per-user-files.test.mjs src/layout-portable.test.mjs src/profile-coverage.test.mjs src/get-kushi-config.test.mjs src/seed-config-derived.test.mjs plugin/runners/test/unit/*.test.mjs",
+    "test:runners": "node --test plugin/runners/test/unit/*.test.mjs",
+    "test:runners:integration": "node --test plugin/runners/test/integration/*.test.mjs",
+    "test:integration:bootstrap": "node src/bootstrap-dryrun.integration.test.mjs",
+    "smoke": "node scripts/smoke.mjs",
+    "eval": "pwsh plugin/skills/eval/run-evals.ps1 -Skill",
+    "eval:all": "pwsh plugin/skills/eval/run-evals.ps1 -All",
+    "eval:canary": "pwsh plugin/skills/eval/run-evals.ps1 -Canary",
+    "eval:baseline": "pwsh plugin/skills/eval/run-evals.ps1 -All -UpdateBaseline",
+    "prepublishOnly": "npm test && npm run smoke"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/plugin/learnings/cross-cutting.md

@@ -4,6 +4,37 @@ Newest on top. Format defined in [`README.md`](./README.md). Use this file when
 
 ---
 
+### 2026-05-29 — Per-step stderr alone is not enough; long single steps need heartbeat ticks
+
+**Symptom**: After v5.7.4 added per-source `[discover] → email ...` stderr lines, a user re-ran discover from VS Code Copilot Chat. Each WorkIQ call still took 30–90s, and the host **still** killed the runner because between sources there was no output for 30+ seconds. The single `[discover] → email ...` line at the start of each source wasn't enough — the host watchdog measures *idle output* (no bytes on either stream for N seconds), not just *no progress at all*.
+
+**Root cause**: A single line of stderr at the start of a long-running operation doesn't satisfy host idle-output watchdogs. They typically measure the gap between successive output bytes. With 30s+ WorkIQ calls and no output in between, the gap exceeds the watchdog threshold (typically 30–60s in Copilot Chat).
+
+**Fix shipped (v5.7.5, 2026-05-29)**:
+
+1. **`workiq.mjs runProcess()` accepts an `onHeartbeat` callback + `heartbeatMs` interval** (default 10s). While the child workiq process runs, the callback fires every `heartbeatMs` with `{ elapsedMs, stdoutBytes, stderrBytes }`.
+2. **`discover.mjs` passes a heartbeat that emits intelligent ticks**:
+   ```
+   [discover] [1/7] email: querying workiq (180000ms budget)...
+   [discover]   email: ...waiting for first byte (10s/180s, no output yet)
+   [discover]   email: ...still working (20s/180s, 1247 bytes received)
+   [discover]   email: ✓ 28412ms, 4892 bytes, 4 row(s), 4 block(s)
+   ```
+   Tells the user (and host watchdog): which source, how long it's been running, how much budget is left, whether anything has come back yet.
+3. **Distinguish empty result from timeout**: previously a fast `exit 0` with empty stdout *and* a 90s SIGKILL both produced `WORKIQ_TIMEOUT`. v5.7.5 reports the empty case as `workiq-empty-response` (with byte count in the stderr log) so users can immediately tell "WorkIQ found nothing" vs "WorkIQ is hanging".
+4. **Default timeout bumped 90s → 180s** since real kushi-style queries (deeper than `who am i`) can legitimately take 30–90s each.
+5. **New regression tests** (2): empty response is reported distinctly from timeout; heartbeat ticks fire on slow workiq calls.
+
+**Lesson — long-running steps need active liveness signals**:
+- A startup line ("starting X...") is necessary but not sufficient.
+- Anything that can take >10s should emit a tick at least every 10s with **useful information**: elapsed/budget, bytes received so far, what the runner is waiting on.
+- Heartbeats should be *parameterized* (env var or option) so tests can run them at sub-second intervals without slowing the suite.
+- "Intelligent" beats "noisy": `...waiting for first byte (10s/180s)` is more useful than `...still working (10s)` because it tells you whether progress has been made.
+
+**Files changed**: `plugin/runners/lib/workiq.mjs` (added heartbeat plumbing), `plugin/runners/discover.mjs` (heartbeat + telemetry + empty-vs-timeout split + default 180s timeout), `plugin/runners/test/unit/discover.test.mjs` (+2 tests).
+
+---
+
 ### 2026-05-29 — Silent runners get killed by host watchdogs (GitHub Copilot Chat)
 
 **Symptom**: User invoked `discover.mjs` from inside GitHub Copilot Chat in VS Code. The runner appeared to hang for 60+ seconds, then the host announced "discover did not return completion output (it stayed silent/hung)" and gave up. Direct PowerShell invocation outside the host showed the same long pause but eventually returned.

package/plugin/runners/discover.mjs

@@ -24,7 +24,7 @@ import { ask as workiqAsk, resolveWorkiqBin } from './lib/workiq.mjs';
 const ALL_SOURCES = ['email', 'teams', 'meetings', 'onenote', 'sharepoint', 'crm', 'ado'];
 
 function parseArgs(argv) {
-  const args = { force: false, dryRun: false, timeoutMs: 90_000, sources: null };
+  const args = { force: false, dryRun: false, timeoutMs: 180_000, sources: null };
   for (let i = 0; i < argv.length; i++) {
     const a = argv[i];
     if (a === '--project') args.project = argv[++i];
@@ -32,7 +32,7 @@ function parseArgs(argv) {
     else if (a === '--source') (args.sources ??= []).push(argv[++i]);
     else if (a === '--force') args.force = true;
     else if (a === '--dry-run') args.dryRun = true;
-    else if (a === '--timeout-ms') args.timeoutMs = Number(argv[++i]) || 90_000;
+    else if (a === '--timeout-ms') args.timeoutMs = Number(argv[++i]) || 180_000;
     else if (a === '--help' || a === '-h') args.help = true;
   }
   return args;
@@ -201,27 +201,54 @@ async function main() {
   let boundsDirty = false;
   let integDirty = false;
 
-  // Stream per-source progress to stderr so users see the runner is alive.
-  // stdout is reserved for the final JSON envelope.
+  // Stream per-source progress to stderr so users (and host watchdogs) see the runner is alive.
+  // stdout is reserved for the final JSON envelope. Heartbeat ticks every 10s during long
+  // workiq calls keep the runner visibly alive — Copilot Chat / Cursor watchdogs typically
+  // kill processes silent on output for 30–60s.
   const log = (msg) => process.stderr.write(`[discover] ${msg}\n`);
   log(`workiq: ${workiqBin}`);
   log(`sources: ${sourcesToRun.join(', ')} (timeout ${args.timeoutMs}ms each)`);
 
+  const total = sourcesToRun.length;
+  const heartbeatMs = Number(process.env.KUSHI_DISCOVER_HEARTBEAT_MS) || 10_000;
+  let idx = 0;
   for (const source of sourcesToRun) {
+    idx++;
     const prompt = buildPrompt(source, path.basename(root));
     let rows = [];
     let asked = true;
     let skipReason = null;
     const t0 = Date.now();
-    log(`→ ${source} ...`);
+    log(`[${idx}/${total}] ${source}: querying workiq (${args.timeoutMs}ms budget)...`);
+    const onHeartbeat = ({ elapsedMs, stdoutBytes }) => {
+      const sec = Math.round(elapsedMs / 1000);
+      const budget = Math.round(args.timeoutMs / 1000);
+      if (stdoutBytes === 0) {
+        log(`  ${source}: ...waiting for first byte (${sec}s/${budget}s, no output yet)`);
+      } else {
+        log(`  ${source}: ...still working (${sec}s/${budget}s, ${stdoutBytes} bytes received)`);
+      }
+    };
     try {
-      const { blocks } = await workiqAsk(prompt, { bin: workiqBin, timeoutMs: args.timeoutMs });
+      const { blocks, stdout: workiqStdout } = await workiqAsk(prompt, { bin: workiqBin, timeoutMs: args.timeoutMs, onHeartbeat, heartbeatMs });
       rows = rowsFromBlocks(blocks, source);
-      log(`  ${source} ok in ${Date.now() - t0}ms (${rows.length} row(s))`);
+      const elapsed = Date.now() - t0;
+      const bytes = Buffer.byteLength(workiqStdout || '', 'utf8');
+      if (rows.length === 0 && bytes < 8) {
+        // Distinguish "workiq returned empty" from "timeout" — both used to look the same.
+        skipReason = 'workiq-empty-response';
+        log(`  ${source}: empty response in ${elapsed}ms (${bytes} bytes) — no data found`);
+      } else {
+        log(`  ${source}: ✓ ${elapsed}ms, ${bytes} bytes, ${rows.length} row(s), ${blocks.length} block(s)`);
+      }
     } catch (e) {
       asked = true;
       skipReason = e.code || 'workiq-error';
-      log(`  ${source} ${skipReason} after ${Date.now() - t0}ms`);
+      const elapsed = Date.now() - t0;
+      const detail = e.code === 'WORKIQ_TIMEOUT'
+        ? `TIMEOUT after ${elapsed}ms (received ${e.stdoutBytes ?? 0} bytes before kill)`
+        : `${skipReason} after ${elapsed}ms: ${(e.message || '').split('\n')[0].slice(0, 200)}`;
+      log(`  ${source}: ✗ ${detail}`);
     }
     const { boundariesPatch, integrationsPatch, accepted } = applyRows(source, rows, bounds, integ);
     if (boundariesPatch) {
@@ -235,6 +262,8 @@ async function main() {
     sourceResults.push({ source, asked, found: rows.length, accepted, skipped_reason: skipReason });
   }
 
+  log(`done: ${sourceResults.filter(r => r.found > 0).length}/${total} sources returned data`);
+
   if (!args.dryRun) {
     if (boundsDirty) await writeAtomic(boundsPath, YAML.stringify(bounds), { skipIfUnchanged: true });
     if (integDirty) await writeAtomic(integPath, YAML.stringify(integ), { skipIfUnchanged: true });

package/plugin/runners/lib/workiq.mjs

@@ -47,16 +47,19 @@ function whichSync(name) {
  * Ask WorkIQ a question. Returns the raw stdout text and any parsed CSC blocks.
  *
  * @param {string} prompt
- * @param {object} opts { bin, timeoutMs, env }
+ * @param {object} opts { bin, timeoutMs, env, onHeartbeat }
+ *   onHeartbeat: optional ({ elapsedMs, stdoutBytes, stderrBytes }) => void
+ *                called every ~heartbeatMs while the child is running.
+ *   heartbeatMs: heartbeat interval in ms (default 10000).
  */
-export async function ask(prompt, { bin, timeoutMs = 120_000, env = process.env } = {}) {
+export async function ask(prompt, { bin, timeoutMs = 120_000, env = process.env, onHeartbeat = null, heartbeatMs = 10_000 } = {}) {
   const exe = bin || resolveWorkiqBin();
   if (!await pathExists(exe)) {
     const err = new Error(`workiq: binary not found at ${exe}. Set KUSHI_WORKIQ_BIN or install workiq.`);
     err.code = 'WORKIQ_NOT_FOUND';
     throw err;
   }
-  const { stdout, stderr, exitCode } = await runProcess(exe, ['ask', '-q', prompt], { timeoutMs, env });
+  const { stdout, stderr, exitCode } = await runProcess(exe, ['ask', '-q', prompt], { timeoutMs, env, onHeartbeat, heartbeatMs });
   if (exitCode !== 0) {
     const err = new Error(`workiq exited ${exitCode}: ${stderr.slice(0, 1000)}`);
     err.code = 'WORKIQ_EXIT_NONZERO';
@@ -114,7 +117,7 @@ function parseKvLines(s) {
 
 async function pathExists(p) { try { await fs.access(p); return true; } catch { return false; } }
 
-function runProcess(exe, args, { timeoutMs, env }) {
+function runProcess(exe, args, { timeoutMs, env, onHeartbeat = null, heartbeatMs = 10_000 }) {
   return new Promise((resolve, reject) => {
     // Node 20.12+ refuses to spawn .cmd/.bat on Windows without shell:true
     // (CVE-2024-27980). Detect and route through cmd.exe explicitly with
@@ -132,17 +135,36 @@ function runProcess(exe, args, { timeoutMs, env }) {
     const child = spawn(spawnExe, spawnArgs, spawnOpts);
     let stdout = '';
     let stderr = '';
+    let stdoutBytes = 0;
+    let stderrBytes = 0;
+    const t0 = Date.now();
     let timeoutTimer = null;
+    let heartbeatTimer = null;
+    const cleanup = () => {
+      if (timeoutTimer) { clearTimeout(timeoutTimer); timeoutTimer = null; }
+      if (heartbeatTimer) { clearInterval(heartbeatTimer); heartbeatTimer = null; }
+    };
     if (timeoutMs > 0) {
       timeoutTimer = setTimeout(() => {
+        cleanup();
         try { child.kill('SIGKILL'); } catch {}
-        reject(Object.assign(new Error(`workiq: timeout after ${timeoutMs}ms`), { code: 'WORKIQ_TIMEOUT' }));
+        reject(Object.assign(new Error(`workiq: timeout after ${timeoutMs}ms (received ${stdoutBytes} bytes)`), {
+          code: 'WORKIQ_TIMEOUT',
+          elapsedMs: Date.now() - t0,
+          stdoutBytes,
+          stderrBytes,
+        }));
       }, timeoutMs);
     }
-    child.stdout.on('data', d => { stdout += d.toString('utf8'); });
-    child.stderr.on('data', d => { stderr += d.toString('utf8'); });
-    child.on('error', e => { if (timeoutTimer) clearTimeout(timeoutTimer); reject(e); });
-    child.on('close', code => { if (timeoutTimer) clearTimeout(timeoutTimer); resolve({ stdout, stderr, exitCode: code ?? 0 }); });
+    if (typeof onHeartbeat === 'function' && heartbeatMs > 0) {
+      heartbeatTimer = setInterval(() => {
+        try { onHeartbeat({ elapsedMs: Date.now() - t0, stdoutBytes, stderrBytes }); } catch {}
+      }, heartbeatMs);
+    }
+    child.stdout.on('data', d => { const s = d.toString('utf8'); stdout += s; stdoutBytes += Buffer.byteLength(s, 'utf8'); });
+    child.stderr.on('data', d => { const s = d.toString('utf8'); stderr += s; stderrBytes += Buffer.byteLength(s, 'utf8'); });
+    child.on('error', e => { cleanup(); reject(e); });
+    child.on('close', code => { cleanup(); resolve({ stdout, stderr, exitCode: code ?? 0, elapsedMs: Date.now() - t0, stdoutBytes, stderrBytes }); });
   });
 }
 

package/plugin/runners/test/unit/discover.test.mjs

@@ -148,5 +148,53 @@ test('discover: streams [discover] progress to stderr (host-watchdog regression
   assert.equal(r.code, 0, r.stderr);
   assert.match(r.stderr, /\[discover\] workiq:/, `expected workiq header on stderr, got: ${r.stderr}`);
   assert.match(r.stderr, /\[discover\] sources: email/, `expected sources header on stderr, got: ${r.stderr}`);
-  assert.match(r.stderr, /\[discover\] → email/, `expected per-source progress on stderr, got: ${r.stderr}`);
+  assert.match(r.stderr, /\[discover\] \[1\/1\] email: querying workiq/, `expected per-source progress on stderr, got: ${r.stderr}`);
+});
+
+test('discover: empty workiq response is reported as workiq-empty-response, not WORKIQ_TIMEOUT (v5.7.5)', async () => {
+  // Regression: previously `found: 0` with a fast-empty workiq response and a hard timeout
+  // both produced WORKIQ_TIMEOUT in the run logs, making it impossible to distinguish
+  // "workiq found nothing" from "workiq is hanging". v5.7.5 adds workiq-empty-response.
+  const bs = runRunner(BOOTSTRAP, ['--project', 'empty-proj', '--alias', 'alice']);
+  assert.equal(bs.code, 0, bs.stderr);
+
+  const fakeBin = path.join(tmpRoot, process.platform === 'win32' ? 'fake-empty.cmd' : 'fake-empty');
+  const fakeBody = process.platform === 'win32' ? '@echo off\r\nexit /b 0\r\n' : '#!/bin/sh\nexit 0\n';
+  await fs.writeFile(fakeBin, fakeBody);
+  if (process.platform !== 'win32') await fs.chmod(fakeBin, 0o755);
+
+  const r = runRunner(RUNNER, ['--project', 'empty-proj', '--alias', 'alice', '--source', 'email'], {
+    KUSHI_WORKIQ_BIN: fakeBin,
+  });
+  assert.equal(r.code, 0, r.stderr);
+  assert.equal(r.json.status, 'ok');
+  assert.equal(r.json.sources.length, 1);
+  assert.equal(r.json.sources[0].skipped_reason, 'workiq-empty-response',
+    `expected workiq-empty-response, got: ${JSON.stringify(r.json.sources[0])}`);
+  assert.match(r.stderr, /empty response/, `expected 'empty response' in stderr, got: ${r.stderr}`);
+});
+
+test('discover: heartbeat ticks emit during slow workiq calls (v5.7.5 host-watchdog defense)', async () => {
+  // Hosts like GitHub Copilot Chat kill processes silent on output for ~30-60s.
+  // For slow workiq queries (some kushi-style queries take 60-120s), we MUST emit
+  // periodic heartbeats. v5.7.5 emits a heartbeat every ~10s; this test uses a
+  // sleeping fake-workiq and a 1s heartbeat to assert the mechanism works.
+  const bs = runRunner(BOOTSTRAP, ['--project', 'hb-proj', '--alias', 'alice']);
+  assert.equal(bs.code, 0, bs.stderr);
+
+  // Sleep 4s, then exit 0 with empty stdout — gives heartbeat time to fire.
+  const fakeBin = path.join(tmpRoot, process.platform === 'win32' ? 'fake-slow.cmd' : 'fake-slow');
+  const fakeBody = process.platform === 'win32'
+    ? '@echo off\r\nping -n 5 127.0.0.1 >nul\r\nexit /b 0\r\n'
+    : '#!/bin/sh\nsleep 4\nexit 0\n';
+  await fs.writeFile(fakeBin, fakeBody);
+  if (process.platform !== 'win32') await fs.chmod(fakeBin, 0o755);
+
+  const r = runRunner(RUNNER, ['--project', 'hb-proj', '--alias', 'alice', '--source', 'email'], {
+    KUSHI_WORKIQ_BIN: fakeBin,
+    KUSHI_DISCOVER_HEARTBEAT_MS: '1000', // 1s heartbeat for the test
+  });
+  assert.equal(r.code, 0, r.stderr);
+  assert.match(r.stderr, /still working|waiting for first byte/,
+    `expected heartbeat tick on stderr, got: ${r.stderr}`);
 });