@c4t4/heyamigo 0.8.0 → 0.8.1

package/dist/ai/claude.js

@@ -3,7 +3,7 @@ import { resolve } from 'path';
 import { config } from '../config.js';
 import { logger } from '../logger.js';
 import { logPrompt } from '../promptlog.js';
-import { runClaude, TIMEOUT_MS } from './spawn.js';
+import { parseStreamJson, runClaude, TIMEOUT_MS } from './spawn.js';
 let cachedSystemPrompt = null;
 function systemPrompt() {
     if (cachedSystemPrompt !== null)
@@ -25,10 +25,14 @@ export function reloadSystemPrompt() {
     cachedSystemPrompt = null;
 }
 function buildArgs(params) {
+    // stream-json gives per-event visibility into the agent loop (system init,
+    // assistant messages, tool_use, tool_result, final result). We parse the
+    // final 'result' event for the return shape, and log event types for
+    // diagnostic purposes.
     const args = [
         '-p',
         '--output-format',
-        config.claude.outputFormat,
+        'stream-json',
         '--model',
         config.claude.model,
         '--permission-mode',
@@ -57,35 +61,32 @@ function buildArgs(params) {
 export async function askClaude(params) {
     const args = buildArgs(params);
     logger.debug({ resume: !!params.sessionId, inputChars: params.input.length }, 'spawning claude');
-    const { stdout, durationMs } = await runClaude({
+    const { stdout, stderr, durationMs } = await runClaude({
         args,
         input: params.input,
         timeoutMs: TIMEOUT_MS.main,
         caller: 'worker',
     });
     const startedAt = Date.now() - durationMs;
-    let parsed;
-    try {
-        parsed = JSON.parse(stdout);
-    }
-    catch (err) {
-        throw new Error(`failed to parse claude output: ${err.message}\nstdout: ${stdout.slice(0, 500)}`);
+    const parsed = parseStreamJson(stdout);
+    if (!parsed) {
+        throw new Error(`claude stream-json produced no result event; stdout: ${stdout.slice(0, 500)}`);
     }
-    if (parsed.is_error || parsed.subtype !== 'success') {
-        throw new Error(`claude returned error (subtype=${parsed.subtype}): ${parsed.result ?? ''}`);
+    if (parsed.isError || parsed.subtype !== 'success') {
+        throw new Error(`claude returned error (subtype=${parsed.subtype}): ${parsed.result}`);
     }
-    if (!parsed.result || !parsed.session_id) {
+    if (!parsed.result || !parsed.sessionId) {
         throw new Error(`claude output missing result or session_id: ${stdout.slice(0, 200)}`);
     }
     const result = {
         reply: parsed.result,
-        sessionId: parsed.session_id,
+        sessionId: parsed.sessionId,
         usage: {
             inputTokens: parsed.usage?.input_tokens ?? 0,
             cacheReadTokens: parsed.usage?.cache_read_input_tokens ?? 0,
             cacheCreationTokens: parsed.usage?.cache_creation_input_tokens ?? 0,
             outputTokens: parsed.usage?.output_tokens ?? 0,
-            numTurns: parsed.num_turns ?? 0,
+            numTurns: parsed.numTurns ?? 0,
         },
     };
     void logPrompt({
@@ -97,6 +98,8 @@ export async function askClaude(params) {
         sessionId: result.sessionId,
         usage: result.usage,
         durationMs,
+        stderr,
+        eventTypes: parsed.eventTypes,
     });
     return result;
 }

package/dist/ai/spawn.js

@@ -21,6 +21,60 @@ export class ClaudeSpawnError extends Error {
         this.name = 'ClaudeSpawnError';
     }
 }
+// Parse Claude CLI's --output-format stream-json output. Each line is a JSON
+// event; the final event with type === 'result' carries the completion
+// summary (same shape as the old single-json output format). Returns null if
+// no result event is found — caller should treat that as an error.
+export function parseStreamJson(stdout) {
+    const events = [];
+    const eventTypes = [];
+    const lines = stdout.split(/\r?\n/);
+    for (const line of lines) {
+        const trimmed = line.trim();
+        if (!trimmed)
+            continue;
+        try {
+            const parsed = JSON.parse(trimmed);
+            events.push(parsed);
+            if (typeof parsed.type === 'string')
+                eventTypes.push(parsed.type);
+        }
+        catch {
+            // Ignore malformed lines — Claude CLI occasionally emits preamble or
+            // debug lines that aren't JSON; the structured events we need are
+            // always well-formed.
+        }
+    }
+    // Find the final result event. Walk from end to handle any stray events
+    // after 'result' (shouldn't happen but be defensive).
+    let resultEvent = null;
+    for (let i = events.length - 1; i >= 0; i--) {
+        if (events[i].type === 'result') {
+            resultEvent = events[i];
+            break;
+        }
+    }
+    if (!resultEvent)
+        return null;
+    return {
+        result: typeof resultEvent.result === 'string' ? resultEvent.result : '',
+        sessionId: typeof resultEvent.session_id === 'string'
+            ? resultEvent.session_id
+            : null,
+        usage: resultEvent.usage && typeof resultEvent.usage === 'object'
+            ? resultEvent.usage
+            : undefined,
+        isError: !!resultEvent.is_error,
+        subtype: typeof resultEvent.subtype === 'string'
+            ? resultEvent.subtype
+            : undefined,
+        numTurns: typeof resultEvent.num_turns === 'number'
+            ? resultEvent.num_turns
+            : undefined,
+        eventTypes,
+        events,
+    };
+}
 // Kill the process group of a detached child. Playwright MCP and any Chromium
 // children sit under the claude subprocess; without process-group kill they
 // linger after we SIGTERM the parent and accumulate on the host.
@@ -54,6 +108,15 @@ export async function runClaude(opts) {
             // detached:true puts the child in its own process group, so killGroup
             // can SIGTERM the whole tree (Playwright MCP, Chromium, etc.) at once.
             detached: true,
+            // ANTHROPIC_LOG=debug surfaces the SDK's HTTP layer to stderr:
+            // request URLs, status codes, retries, rate-limit notices. We
+            // capture stderr and put a truncated copy into the promptlog so
+            // we can diagnose API hangs/rate-limits post-mortem instead of
+            // staring at "Claude subprocess is idle, why?".
+            env: {
+                ...process.env,
+                ANTHROPIC_LOG: process.env.ANTHROPIC_LOG ?? 'debug',
+            },
         });
         let stdout = '';
         let stderr = '';
@@ -106,7 +169,7 @@ export async function runClaude(opts) {
                 logFail(`exit ${code}: ${stderr.slice(0, 500)}`);
                 return rejectPromise(new ClaudeSpawnError(caller, `claude exited with code ${code}: ${stderr.slice(0, 500)}`));
             }
-            resolvePromise({ stdout, durationMs });
+            resolvePromise({ stdout, stderr, durationMs });
         });
         child.stdin.write(input);
         child.stdin.end();

package/dist/memory/compressed.js

@@ -1,7 +1,7 @@
 import { existsSync, readFileSync, readdirSync, writeFileSync, } from 'fs';
 import { dirname, resolve } from 'path';
 import { mkdirSync } from 'fs';
-import { runClaude, TIMEOUT_MS } from '../ai/spawn.js';
+import { parseStreamJson, runClaude, TIMEOUT_MS } from '../ai/spawn.js';
 import { config } from '../config.js';
 import { logger } from '../logger.js';
 import { logPrompt } from '../promptlog.js';
@@ -224,28 +224,25 @@ async function spawnGenerator(prompt) {
     const args = [
         '-p',
         '--output-format',
-        'json',
+        'stream-json',
         '--model',
         config.claude.model,
         '--permission-mode',
         'acceptEdits',
     ];
-    const { stdout, durationMs } = await runClaude({
+    const { stdout, stderr, durationMs } = await runClaude({
         args,
         input: prompt,
         timeoutMs: TIMEOUT_MS.background,
         caller: 'compressed',
     });
     const startedAt = Date.now() - durationMs;
-    let parsed;
-    try {
-        parsed = JSON.parse(stdout);
-    }
-    catch (err) {
-        throw new Error(`compressed parse failed: ${err.message}`);
+    const parsed = parseStreamJson(stdout);
+    if (!parsed) {
+        throw new Error(`compressed stream-json produced no result event: ${stdout.slice(0, 200)}`);
     }
-    if (parsed.is_error || parsed.subtype !== 'success' || !parsed.result) {
-        throw new Error(`compressed bad output: ${parsed.result ?? stdout.slice(0, 200)}`);
+    if (parsed.isError || parsed.subtype !== 'success' || !parsed.result) {
+        throw new Error(`compressed bad output: ${parsed.result || stdout.slice(0, 200)}`);
     }
     const output = parsed.result.trim();
     void logPrompt({
@@ -255,6 +252,8 @@ async function spawnGenerator(prompt) {
         input: prompt,
         output,
         durationMs,
+        stderr,
+        eventTypes: parsed.eventTypes,
     });
     return output;
 }

package/dist/memory/digest.js

@@ -1,4 +1,4 @@
-import { runClaude, TIMEOUT_MS } from '../ai/spawn.js';
+import { parseStreamJson, runClaude, TIMEOUT_MS } from '../ai/spawn.js';
 import { config } from '../config.js';
 import { logger } from '../logger.js';
 import { logPrompt } from '../promptlog.js';
@@ -13,28 +13,25 @@ async function spawnDigester(prompt) {
     const args = [
         '-p',
         '--output-format',
-        'json',
+        'stream-json',
         '--model',
         config.claude.model,
         '--permission-mode',
         'acceptEdits',
     ];
-    const { stdout, durationMs } = await runClaude({
+    const { stdout, stderr, durationMs } = await runClaude({
         args,
         input: prompt,
         timeoutMs: TIMEOUT_MS.background,
         caller: 'digester',
     });
     const startedAt = Date.now() - durationMs;
-    let parsed;
-    try {
-        parsed = JSON.parse(stdout);
-    }
-    catch (err) {
-        throw new Error(`digester parse failed: ${err.message}`);
+    const parsed = parseStreamJson(stdout);
+    if (!parsed) {
+        throw new Error(`digester stream-json produced no result event: ${stdout.slice(0, 200)}`);
     }
-    if (parsed.is_error || parsed.subtype !== 'success' || !parsed.result) {
-        throw new Error(`digester bad output: ${parsed.result ?? stdout.slice(0, 200)}`);
+    if (parsed.isError || parsed.subtype !== 'success' || !parsed.result) {
+        throw new Error(`digester bad output: ${parsed.result || stdout.slice(0, 200)}`);
     }
     const output = parsed.result.trim();
     void logPrompt({
@@ -44,6 +41,8 @@ async function spawnDigester(prompt) {
         input: prompt,
         output,
         durationMs,
+        stderr,
+        eventTypes: parsed.eventTypes,
     });
     return output;
 }

package/dist/memory/journal-nudger.js

@@ -1,4 +1,4 @@
-import { runClaude, TIMEOUT_MS } from '../ai/spawn.js';
+import { parseStreamJson, runClaude, TIMEOUT_MS } from '../ai/spawn.js';
 import { config } from '../config.js';
 import { initiate } from '../gateway/outgoing.js';
 import { logger } from '../logger.js';
@@ -18,28 +18,25 @@ async function spawnComposer(prompt) {
     const args = [
         '-p',
         '--output-format',
-        'json',
+        'stream-json',
         '--model',
         config.claude.model,
         '--permission-mode',
         'acceptEdits',
     ];
-    const { stdout, durationMs } = await runClaude({
+    const { stdout, stderr, durationMs } = await runClaude({
         args,
         input: prompt,
         timeoutMs: TIMEOUT_MS.background,
         caller: 'journal-nudger',
     });
     const startedAt = Date.now() - durationMs;
-    let parsed;
-    try {
-        parsed = JSON.parse(stdout);
-    }
-    catch (err) {
-        throw new Error(`nudger parse failed: ${err.message}`);
+    const parsed = parseStreamJson(stdout);
+    if (!parsed) {
+        throw new Error(`nudger stream-json produced no result event: ${stdout.slice(0, 200)}`);
     }
-    if (parsed.is_error || parsed.subtype !== 'success' || !parsed.result) {
-        throw new Error(`nudger bad output: ${parsed.result ?? stdout.slice(0, 200)}`);
+    if (parsed.isError || parsed.subtype !== 'success' || !parsed.result) {
+        throw new Error(`nudger bad output: ${parsed.result || stdout.slice(0, 200)}`);
     }
     const output = parsed.result.trim();
     void logPrompt({
@@ -49,6 +46,8 @@ async function spawnComposer(prompt) {
         input: prompt,
         output,
         durationMs,
+        stderr,
+        eventTypes: parsed.eventTypes,
     });
     return output;
 }

package/dist/memory/journal-observer.js

@@ -1,4 +1,4 @@
-import { runClaude, TIMEOUT_MS } from '../ai/spawn.js';
+import { parseStreamJson, runClaude, TIMEOUT_MS } from '../ai/spawn.js';
 import { config } from '../config.js';
 import { logger } from '../logger.js';
 import { logPrompt } from '../promptlog.js';
@@ -14,28 +14,25 @@ async function spawnObserver(prompt) {
     const args = [
         '-p',
         '--output-format',
-        'json',
+        'stream-json',
         '--model',
         config.claude.model,
         '--permission-mode',
         'acceptEdits',
     ];
-    const { stdout, durationMs } = await runClaude({
+    const { stdout, stderr, durationMs } = await runClaude({
         args,
         input: prompt,
         timeoutMs: TIMEOUT_MS.background,
         caller: 'journal-observer',
     });
     const startedAt = Date.now() - durationMs;
-    let parsed;
-    try {
-        parsed = JSON.parse(stdout);
-    }
-    catch (err) {
-        throw new Error(`journal observer parse failed: ${err.message}`);
+    const parsed = parseStreamJson(stdout);
+    if (!parsed) {
+        throw new Error(`journal observer stream-json produced no result event: ${stdout.slice(0, 200)}`);
     }
-    if (parsed.is_error || parsed.subtype !== 'success' || !parsed.result) {
-        throw new Error(`journal observer bad output: ${parsed.result ?? stdout.slice(0, 200)}`);
+    if (parsed.isError || parsed.subtype !== 'success' || !parsed.result) {
+        throw new Error(`journal observer bad output: ${parsed.result || stdout.slice(0, 200)}`);
     }
     const output = parsed.result.trim();
     void logPrompt({
@@ -45,6 +42,8 @@ async function spawnObserver(prompt) {
         input: prompt,
         output,
         durationMs,
+        stderr,
+        eventTypes: parsed.eventTypes,
     });
     return output;
 }

package/dist/promptlog.js

@@ -1,6 +1,21 @@
 import { appendFile, mkdir, readdir, unlink } from 'fs/promises';
 import { resolve } from 'path';
 import { config } from './config.js';
+// Hard caps on fields that can grow unbounded. Prevents promptlog entries
+// from exploding when a run produces huge stdout/stderr. prunePrompts
+// still handles multi-day retention separately.
+const STDOUT_MAX_BYTES = 100_000;
+const STDERR_MAX_BYTES = 50_000;
+const INPUT_MAX_BYTES = 200_000;
+function truncateWithMarker(s, maxBytes) {
+    if (!s)
+        return s;
+    // Rough byte size via length — fine for mostly-ASCII prompt payloads.
+    if (s.length <= maxBytes)
+        return s;
+    const extra = s.length - maxBytes;
+    return s.slice(0, maxBytes) + `\n… [truncated ${extra} bytes]`;
+}
 let dirReady = false;
 function promptsDir() {
     return resolve(process.cwd(), 'storage/prompts');
@@ -18,7 +33,17 @@ function logFilePath() {
 export async function logPrompt(entry) {
     try {
         await ensureDir();
-        await appendFile(logFilePath(), JSON.stringify(entry) + '\n', 'utf-8');
+        const capped = {
+            ...entry,
+            input: truncateWithMarker(entry.input, INPUT_MAX_BYTES),
+        };
+        if (entry.output !== undefined) {
+            capped.output = truncateWithMarker(entry.output, STDOUT_MAX_BYTES);
+        }
+        if (entry.stderr !== undefined) {
+            capped.stderr = truncateWithMarker(entry.stderr, STDERR_MAX_BYTES);
+        }
+        await appendFile(logFilePath(), JSON.stringify(capped) + '\n', 'utf-8');
     }
     catch {
         // logging must never break the main flow

package/dist/queue/async-tasks.js

@@ -1,6 +1,6 @@
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
 import { dirname, resolve } from 'path';
-import { runClaude, TIMEOUT_MS } from '../ai/spawn.js';
+import { parseStreamJson, runClaude, TIMEOUT_MS } from '../ai/spawn.js';
 import { config } from '../config.js';
 import fastq from 'fastq';
 import { initiate } from '../gateway/outgoing.js';
@@ -113,7 +113,7 @@ function buildArgs(task) {
     const args = [
         '-p',
         '--output-format',
-        'json',
+        'stream-json',
         '--model',
         config.claude.model,
         '--permission-mode',
@@ -135,22 +135,19 @@ function buildArgs(task) {
 }
 async function spawnClaudeForTask(task, prompt) {
     const args = buildArgs(task);
-    const { stdout, durationMs } = await runClaude({
+    const { stdout, stderr, durationMs } = await runClaude({
         args,
         input: prompt,
         timeoutMs: TIMEOUT_MS.async,
         caller: 'async-task',
     });
     const startedAt = Date.now() - durationMs;
-    let parsed;
-    try {
-        parsed = JSON.parse(stdout);
-    }
-    catch (err) {
-        throw new Error(`async task parse failed: ${err.message}`);
+    const parsed = parseStreamJson(stdout);
+    if (!parsed) {
+        throw new Error(`async task stream-json produced no result event: ${stdout.slice(0, 200)}`);
     }
-    if (parsed.is_error || parsed.subtype !== 'success' || !parsed.result) {
-        throw new Error(`async task bad output: ${parsed.result ?? stdout.slice(0, 200)}`);
+    if (parsed.isError || parsed.subtype !== 'success' || !parsed.result) {
+        throw new Error(`async task bad output: ${parsed.result || stdout.slice(0, 200)}`);
     }
     const output = parsed.result.trim();
     void logPrompt({
@@ -160,6 +157,8 @@ async function spawnClaudeForTask(task, prompt) {
         input: prompt,
         output,
         durationMs,
+        stderr,
+        eventTypes: parsed.eventTypes,
     });
     return output;
 }
@@ -398,7 +397,7 @@ function buildBrowserArgs(task, sessionId) {
     const args = [
         '-p',
         '--output-format',
-        'json',
+        'stream-json',
         '--model',
         config.claude.model,
         '--permission-mode',
@@ -435,6 +434,7 @@ async function runBrowserTask(task) {
     const startedAtMs = Date.now();
     const elapsedLog = () => `${Math.round((Date.now() - task.startedAt * 1000) / 1000)}s`;
     let stdout;
+    let stderr;
     let durationMs;
     try {
         const result = await runClaude({
@@ -444,6 +444,7 @@ async function runBrowserTask(task) {
             caller: 'browser-task',
         });
         stdout = result.stdout;
+        stderr = result.stderr;
         durationMs = result.durationMs;
     }
     catch (err) {
@@ -454,20 +455,17 @@ async function runBrowserTask(task) {
         });
         return;
     }
-    let parsed;
-    try {
-        parsed = JSON.parse(stdout);
-    }
-    catch (err) {
-        logger.error({ err, id: task.id }, 'browser task: failed to parse claude output');
+    const parsed = parseStreamJson(stdout);
+    if (!parsed) {
+        logger.error({ id: task.id }, 'browser task stream-json produced no result event');
         await initiate({
             jid: task.jid,
             text: `Heads up: the browser task "${truncate(task.description, 80)}" returned an unparseable response.`,
         });
         return;
     }
-    if (parsed.is_error || parsed.subtype !== 'success' || !parsed.result) {
-        logger.error({ parsed, id: task.id }, 'browser task bad output');
+    if (parsed.isError || parsed.subtype !== 'success' || !parsed.result) {
+        logger.error({ id: task.id, subtype: parsed.subtype, isError: parsed.isError }, 'browser task bad output');
         await initiate({
             jid: task.jid,
             text: `Heads up: the browser task "${truncate(task.description, 80)}" returned an error.`,
@@ -476,7 +474,7 @@ async function runBrowserTask(task) {
     }
     // Persist the session id. On first call Claude returns the new sessionId;
     // on resume it may return the same or a rotated one.
-    const returnedSessionId = parsed.session_id ?? null;
+    const returnedSessionId = parsed.sessionId;
     if (returnedSessionId) {
         const now = Math.floor(Date.now() / 1000);
         saveBrowserSession({
@@ -494,6 +492,8 @@ async function runBrowserTask(task) {
         output: parsed.result,
         sessionId: returnedSessionId ?? undefined,
         durationMs,
+        stderr,
+        eventTypes: parsed.eventTypes,
     });
     // Route markers the same way the general async lane does.
     const { extractFlags } = await import('../memory/digest-flag.js');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@c4t4/heyamigo",
-  "version": "0.8.0",
+  "version": "0.8.1",
   "description": "WhatsApp AI bot powered by Claude with long-term memory, browser control, and role-based access",
   "type": "module",
   "main": "dist/index.js",