@c4t4/heyamigo 0.9.6 → 0.9.8

package/dist/boot.js

@@ -0,0 +1,67 @@
+// Shared bootstrap for both the library entry (src/index.ts) and the
+// CLI start command (src/cli/start.ts). Single source of truth for
+// startup order — there used to be two parallel main() functions that
+// drifted; this prevents that.
+import { setBaileysSocket } from './channels/index.js';
+import { closeDb, initDb } from './db/index.js';
+import { syncIdentitiesFromAccess } from './db/identity-sync.js';
+import { attachIncoming } from './gateway/incoming.js';
+import { logger } from './logger.js';
+import { startScheduler } from './memory/scheduler.js';
+import { startChatWorkers, stopChatWorkers } from './queue/chat-worker.js';
+import { requestShutdown, startOrchestrator, stopOrchestrator, } from './queue/orchestrator.js';
+import { startSenderWorker, stopSenderWorker } from './queue/sender-worker.js';
+import { startSocket } from './wa/socket.js';
+let booted = false;
+export async function bootBot() {
+    if (booted) {
+        logger.warn('bootBot called twice; ignoring');
+        return;
+    }
+    booted = true;
+    logger.info('heyamigo starting');
+    // Migrations + drift check first. Refuses to start on schema mismatch
+    // — protects production data from a half-applied schema upgrade.
+    initDb();
+    // Derived view: persons + identities from access.json (idempotent).
+    syncIdentitiesFromAccess();
+    // Orchestrator first so workers see a control surface from the moment
+    // they register. Drain hook stops everything in reverse order.
+    startOrchestrator({
+        onShutdownDrained: () => {
+            stopChatWorkers();
+            stopSenderWorker();
+            stopOrchestrator();
+            closeDb();
+        },
+    });
+    // Workers next. Inbound table is the source of truth; anything left
+    // from a previous crash gets claimed by the new pool automatically.
+    // No separate replay step needed.
+    startSenderWorker();
+    startChatWorkers();
+    startScheduler();
+    await startSocket((sock) => {
+        attachIncoming(sock);
+        // Point the Baileys adapter at the live socket. Called on each
+        // reconnect with a fresh sock; the adapter just keeps the latest.
+        setBaileysSocket(sock);
+    });
+}
+// Install once. Both signals trigger the same graceful drain:
+// orchestrator picks up the shutdown control row, waits for busy
+// workers, then runs onShutdownDrained and exits.
+let signalsInstalled = false;
+export function installShutdownSignals() {
+    if (signalsInstalled)
+        return;
+    signalsInstalled = true;
+    process.on('SIGINT', () => {
+        logger.info('SIGINT received, requesting graceful shutdown');
+        requestShutdown('SIGINT');
+    });
+    process.on('SIGTERM', () => {
+        logger.info('SIGTERM received, requesting graceful shutdown');
+        requestShutdown('SIGTERM');
+    });
+}

package/dist/cli/start.js

@@ -1,10 +1,6 @@
 import { execSync } from 'child_process';
-import { attachIncoming } from '../gateway/incoming.js';
-import { handleReply } from '../gateway/outgoing.js';
+import { bootBot, installShutdownSignals } from '../boot.js';
 import { logger } from '../logger.js';
-import { startScheduler } from '../memory/scheduler.js';
-import { replayPending } from '../queue/queue.js';
-import { startSocket } from '../wa/socket.js';
 export async function main() {
     try {
         execSync('which claude', { stdio: 'pipe' });
@@ -14,23 +10,9 @@ export async function main() {
             '  npm install -g @anthropic-ai/claude-code\n');
         process.exit(1);
     }
-    logger.info('heyamigo starting');
-    startScheduler();
-    await startSocket((sock) => {
-        attachIncoming(sock);
-    });
-    void replayPending(async (job, result) => {
-        await handleReply(job, result, {});
-    }).catch((err) => logger.error({ err }, 'replay failed'));
+    installShutdownSignals();
+    await bootBot();
 }
-process.on('SIGINT', () => {
-    logger.info('SIGINT received, shutting down');
-    process.exit(0);
-});
-process.on('SIGTERM', () => {
-    logger.info('SIGTERM received, shutting down');
-    process.exit(0);
-});
 main().catch((err) => {
     logger.error({ err }, 'fatal error during boot');
     process.exit(1);

package/dist/config.js

@@ -36,6 +36,14 @@ const ConfigSchema = z.object({
         outputFormat: z.enum(['json', 'text', 'stream-json']),
         contextWindow: z.number(),
     }),
+    chatPool: z
+        .object({
+        // How many chat workers run in parallel. Per-address
+        // serialization means N workers serve up to N different chats
+        // concurrently; per-chat ordering is preserved naturally.
+        size: z.number().int().positive().default(5),
+    })
+        .default({ size: 5 }),
     codex: z
         .object({
         // Optional model override. If unset, Codex uses its default. Passed

package/dist/db/schema.js

@@ -159,6 +159,12 @@ export const inbound = sqliteTable('inbound', {
     mediaBytes: integer('media_bytes'),
     pushName: text('push_name'), // sender's display name at send time
     triggerReason: text('trigger_reason'), // 'alias'|'mention'|'reply'|'owner'|...
+    // Producer-built worker payload (JSON). Chat worker deserializes
+    // at claim time to reconstruct the Job. Keeps the rebuild logic
+    // out of the worker for Phase 4; later phases may move portions
+    // (memory preamble assembly) into the worker for fresh-at-claim
+    // semantics. Nullable so non-chat inserters can leave it off.
+    payload: text('payload'),
     // 'pending'|'claimed'|'done'|'failed'|'dlq'
     status: text('status').notNull(),
     attempts: integer('attempts').notNull().default(0),

package/dist/gateway/commands.js

@@ -59,5 +59,11 @@ export async function tryCommand(ctx) {
         }).catch(() => undefined);
         return true;
     }
+    if (cmd === 'queues') {
+        const { takeQueuesSnapshot, formatQueuesSnapshot } = await import('../queue/observability.js');
+        const snap = takeQueuesSnapshot();
+        await sendText(ctx.sock, ctx.jid, formatQueuesSnapshot(snap), ctx.quoted);
+        return true;
+    }
     return false;
 }

package/dist/gateway/incoming.js

@@ -2,10 +2,12 @@ import { unlink } from 'fs/promises';
 import { getContentType, isJidGroup, jidDecode, jidNormalizedUser, } from 'baileys';
 import { getProvider } from '../ai/providers.js';
 import { getSession } from '../ai/sessions.js';
+import { formatAddress, jidToAddress } from '../db/address.js';
+import { personIdForAddress } from '../db/identity-sync.js';
 import { config } from '../config.js';
 import { logger } from '../logger.js';
 import { buildMemoryPreamble } from '../memory/preamble.js';
-import { enqueue } from '../queue/queue.js';
+import { enqueueInbound } from '../queue/inbound.js';
 import { detectMediaType, downloadAndSave, getMediaSize, mediaPromptTag, } from '../store/media.js';
 import { append } from '../store/messages.js';
 import { getDailyTokens } from '../store/usage.js';
@@ -13,7 +15,6 @@ import { sendText } from '../wa/sender.js';
 import { checkAccess, discoverGroupIfNew, getLimitsForUser, getRoleForContext, } from '../wa/whitelist.js';
 import { buildInitPayload, buildRecentContext } from './bootstrap.js';
 import { tryCommand } from './commands.js';
-import { handleReply } from './outgoing.js';
 import { checkTrigger } from './triggers.js';
 export function attachIncoming(sock) {
     const ownerJid = sock.user?.id
@@ -201,46 +202,29 @@ async function processMessages(messages, sock, ownerJid, isHistorySync = false)
                 fromMe: stored.fromMe,
                 allowedTools: role.tools,
             };
-            // Start typing indicator immediately; refresh every 10s (WA expires ~15s)
-            let typingHeartbeat = null;
-            if (config.reply.typingIndicator) {
-                void sock
-                    .sendPresenceUpdate('composing', stored.jid)
-                    .catch(() => undefined);
-                typingHeartbeat = setInterval(() => {
-                    void sock
-                        .sendPresenceUpdate('composing', stored.jid)
-                        .catch(() => undefined);
-                }, 10000);
-            }
-            const stopTyping = () => {
-                if (typingHeartbeat)
-                    clearInterval(typingHeartbeat);
-                typingHeartbeat = null;
-            };
-            // Defense-in-depth: if nothing else clears the heartbeat within 10 min
-            // (e.g. a code path forgot), force-stop. Prevents runaway "typing..."
-            // indicators when the pipeline silently fails.
-            const typingSafetyCap = setTimeout(() => {
-                if (typingHeartbeat) {
-                    logger.warn({ jid: job.jid }, 'typingHeartbeat safety-cap fired, forcing clear');
-                    stopTyping();
-                }
-            }, 10 * 60 * 1000);
-            typingSafetyCap.unref();
-            enqueue(job)
-                .then((result) => handleReply(job, result, msg))
-                .catch((err) => {
-                const isTimeout = err instanceof Error && err.name === 'ClaudeTimeoutError';
-                logger.error({ err, jid: job.jid, isTimeout }, 'pipeline failed');
-                const replyText = isTimeout
-                    ? 'That request timed out. The task was cancelled, queue is moving.'
-                    : config.reply.errorMessage;
-                return handleReply(job, { reply: replyText }, msg).catch((e) => logger.error({ err: e, jid: job.jid }, 'failed to send error reply'));
-            })
-                .finally(() => {
-                stopTyping();
-                clearTimeout(typingSafetyCap);
+            // Enqueue into the inbound table; chat worker pool drains and
+            // calls processJob + handleReply asynchronously. Typing indicator
+            // is temporarily dropped (was tied to the old synchronous flow);
+            // re-add via ChannelAdapter.sendTyping() in a follow-up commit.
+            const chatAddress = formatAddress(jidToAddress(stored.jid));
+            const senderAddress = stored.senderNumber
+                ? formatAddress(jidToAddress(`${stored.senderNumber}@s.whatsapp.net`))
+                : null;
+            const personId = personIdForAddress(chatAddress);
+            const actorPersonId = senderAddress
+                ? personIdForAddress(senderAddress)
+                : null;
+            enqueueInbound({
+                address: chatAddress,
+                actorAddress: senderAddress,
+                personId,
+                actorPersonId,
+                externalMsgId: msg.key.id ?? null,
+                text: stored.text,
+                pushName: stored.pushName ?? null,
+                triggerReason,
+                receivedAt: stored.timestamp,
+                payload: job,
             });
         }
         catch (err) {

package/dist/index.js

@@ -1,59 +1,7 @@
-import { setBaileysSocket } from './channels/index.js';
-import { closeDb, initDb } from './db/index.js';
-import { syncIdentitiesFromAccess } from './db/identity-sync.js';
-import { attachIncoming } from './gateway/incoming.js';
-import { handleReply } from './gateway/outgoing.js';
+import { bootBot, installShutdownSignals } from './boot.js';
 import { logger } from './logger.js';
-import { startScheduler } from './memory/scheduler.js';
-import { requestShutdown, startOrchestrator, stopOrchestrator, } from './queue/orchestrator.js';
-import { replayPending } from './queue/queue.js';
-import { startSenderWorker, stopSenderWorker } from './queue/sender-worker.js';
-import { startSocket } from './wa/socket.js';
-async function main() {
-    logger.info('heyamigo starting');
-    // Migrations + drift check first; refuses to start on schema mismatch.
-    initDb();
-    // Derived view: populate persons + identities from access.json.
-    syncIdentitiesFromAccess();
-    // Orchestrator handles cross-cutting bookkeeping: control table
-    // signals, stuck-claim reclaim, dead-worker detection, cron polling
-    // (Phase 2.2+). Starts before workers so it can see them register.
-    startOrchestrator({
-        onShutdownDrained: () => {
-            stopSenderWorker();
-            stopOrchestrator();
-            closeDb();
-        },
-    });
-    // Sender worker drains outbound queue → channel adapters. Started
-    // before the socket so it's ready when handleReply enqueues rows.
-    startSenderWorker();
-    startScheduler();
-    await startSocket((sock) => {
-        attachIncoming(sock);
-        // Point the Baileys adapter at the live socket. Called on each
-        // reconnect with a fresh sock; the adapter just keeps the latest.
-        setBaileysSocket(sock);
-    });
-    // Replay any jobs left from a previous crash (no original WAMessage
-    // available, so replies are sent as plain messages, not quoted).
-    void replayPending(async (job, result) => {
-        await handleReply(job, result, {});
-    }).catch((err) => logger.error({ err }, 'replay failed'));
-}
-// Graceful shutdown: signal handler writes a 'shutdown' row to the
-// control table; orchestrator picks it up, drains in-flight work,
-// then runs onShutdownDrained (stops workers, closes DB) and exits.
-// A 30s timer inside the orchestrator force-exits if drain hangs.
-process.on('SIGINT', () => {
-    logger.info('SIGINT received, requesting graceful shutdown');
-    requestShutdown('SIGINT');
-});
-process.on('SIGTERM', () => {
-    logger.info('SIGTERM received, requesting graceful shutdown');
-    requestShutdown('SIGTERM');
-});
-main().catch((err) => {
+installShutdownSignals();
+bootBot().catch((err) => {
     logger.error({ err }, 'fatal error during boot');
     process.exit(1);
 });

package/dist/queue/chat-worker.js

@@ -0,0 +1,183 @@
+// Chat worker pool. N workers drain the inbound queue; per-address
+// serialization ensures one in-flight job per chat while different
+// chats run in parallel (Phase 4's headline feature).
+//
+// Each worker:
+//   1. claimNextInbound (atomic, serialized per address)
+//   2. deserialize the producer-built Job payload
+//   3. call processJob (existing AI + marker handling, unchanged)
+//   4. call handleReply (enqueues outbound rows)
+//   5. markInboundDone — or markInboundRetryOrDlq on error
+//
+// Replaces the in-memory fastq queue (queue/queue.ts). The old
+// queue + persistence files stay in place for now in case they're
+// still referenced anywhere; clean removal lands after we confirm
+// nothing breaks.
+import { hostname } from 'os';
+import { eq } from 'drizzle-orm';
+import { config } from '../config.js';
+import { getDb } from '../db/index.js';
+import { workers } from '../db/schema.js';
+import { handleReply } from '../gateway/outgoing.js';
+import { logger } from '../logger.js';
+import { processJob } from './worker.js';
+import { claimNextInbound, markInboundDone, markInboundFailed, markInboundRetryOrDlq, } from './inbound.js';
+const HEARTBEAT_INTERVAL_MS = 5_000;
+const IDLE_POLL_INTERVAL_MS = 250;
+const BUSY_POLL_INTERVAL_MS = 0; // immediately try next after a successful claim
+const activeWorkers = [];
+let stopping = false;
+let heartbeatTimer = null;
+function newWorkerId(slot) {
+    return `${hostname()}-${process.pid}-chat-${slot}`;
+}
+function registerWorker(id) {
+    const db = getDb();
+    const now = Math.floor(Date.now() / 1000);
+    db.insert(workers)
+        .values({
+        id,
+        kind: 'chat',
+        status: 'idle',
+        currentJob: null,
+        lastSeen: now,
+        startedAt: now,
+    })
+        .onConflictDoUpdate({
+        target: workers.id,
+        set: { status: 'idle', currentJob: null, lastSeen: now, startedAt: now },
+    })
+        .run();
+}
+function setWorkerStatus(id, status, currentJob = null) {
+    const db = getDb();
+    db.update(workers)
+        .set({
+        status,
+        currentJob,
+        lastSeen: Math.floor(Date.now() / 1000),
+    })
+        .where(eq(workers.id, id))
+        .run();
+}
+function heartbeatAll() {
+    if (activeWorkers.length === 0)
+        return;
+    const db = getDb();
+    const now = Math.floor(Date.now() / 1000);
+    for (const id of activeWorkers) {
+        db.update(workers)
+            .set({ lastSeen: now })
+            .where(eq(workers.id, id))
+            .run();
+    }
+}
+// Reconstruct the Job from an inbound row's payload. The payload was
+// built by the producer (gateway/incoming.ts) at enqueue time.
+function jobFromRow(row) {
+    if (!row.payload) {
+        logger.warn({ id: row.id }, 'inbound row has no payload; cannot reconstruct Job');
+        return null;
+    }
+    try {
+        return JSON.parse(row.payload);
+    }
+    catch (err) {
+        logger.error({ err, id: row.id }, 'inbound row payload is not JSON');
+        return null;
+    }
+}
+async function processOne(workerId, row) {
+    setWorkerStatus(workerId, 'busy', `inbound:${row.id}`);
+    const job = jobFromRow(row);
+    if (!job) {
+        markInboundFailed(row.id, workerId, 'invalid payload');
+        setWorkerStatus(workerId, 'idle');
+        return;
+    }
+    try {
+        const result = await processJob(job);
+        // handleReply's third arg was the original WAMessage (used for
+        // group quoting). That regression was already deferred in Phase 1;
+        // pass an empty stub here too. The compiler type is loose enough.
+        await handleReply(job, result, {});
+        const ok = markInboundDone(row.id, workerId);
+        if (!ok) {
+            logger.warn({ id: row.id, workerId }, 'inbound markDone failed (claim lost?). reply was already enqueued.');
+        }
+        logger.info({
+            id: row.id,
+            address: row.address,
+            chars: result.reply.length,
+            dur: result.stats?.durationMs,
+        }, 'inbound processed');
+    }
+    catch (err) {
+        const isTimeout = err instanceof Error && err.name === 'ClaudeTimeoutError';
+        const msg = err instanceof Error ? err.message : String(err);
+        const result = markInboundRetryOrDlq(row.id, workerId, msg);
+        if (result.deadLettered) {
+            logger.error({ err, id: row.id, address: row.address, isTimeout }, 'inbound dead-lettered after max attempts');
+            // Send a user-facing failure ack so the chat isn't left hanging.
+            try {
+                const failText = isTimeout
+                    ? 'That request timed out. The task was cancelled, queue is moving.'
+                    : config.reply.errorMessage;
+                await handleReply(job, { reply: failText }, {});
+            }
+            catch (e) {
+                logger.error({ err: e, id: row.id }, 'failed to send DLQ-ack reply');
+            }
+        }
+        else if (result.retried) {
+            logger.warn({ err, id: row.id, address: row.address, isTimeout }, 'inbound transient fail, will retry');
+        }
+    }
+    finally {
+        setWorkerStatus(workerId, 'idle');
+    }
+}
+async function loop(workerId) {
+    while (!stopping) {
+        let processed = false;
+        try {
+            const row = claimNextInbound(workerId);
+            if (row) {
+                await processOne(workerId, row);
+                processed = true;
+            }
+        }
+        catch (err) {
+            logger.error({ err, workerId }, 'chat worker loop error');
+        }
+        const delay = processed ? BUSY_POLL_INTERVAL_MS : IDLE_POLL_INTERVAL_MS;
+        if (delay > 0) {
+            await new Promise((res) => setTimeout(res, delay));
+        }
+        else {
+            // Yield without sleeping so other microtasks (heartbeat etc) run.
+            await new Promise((res) => setImmediate(res));
+        }
+    }
+    setWorkerStatus(workerId, 'dead');
+}
+export function startChatWorkers() {
+    if (activeWorkers.length > 0) {
+        logger.warn('chat workers already started; ignoring');
+        return;
+    }
+    const pool = Math.max(1, config.chatPool?.size ?? 5);
+    for (let i = 0; i < pool; i++) {
+        const id = newWorkerId(i);
+        activeWorkers.push(id);
+        registerWorker(id);
+        void loop(id).catch((err) => logger.fatal({ err, workerId: id }, 'chat worker loop crashed'));
+    }
+    heartbeatTimer = setInterval(heartbeatAll, HEARTBEAT_INTERVAL_MS);
+    logger.info({ pool }, 'chat worker pool started');
+}
+export function stopChatWorkers() {
+    stopping = true;
+    if (heartbeatTimer)
+        clearInterval(heartbeatTimer);
+}

package/dist/queue/inbound.js

@@ -37,6 +37,7 @@ export function enqueueInbound(input) {
         mediaBytes: input.mediaBytes ?? null,
         pushName: input.pushName ?? null,
         triggerReason: input.triggerReason ?? null,
+        payload: input.payload === undefined ? null : JSON.stringify(input.payload),
         status: 'pending',
         attempts: 0,
         nextAttemptAt: null,