@c4t4/heyamigo 0.8.15 → 0.9.1

package/dist/db/schema.js

@@ -0,0 +1,97 @@
+// Schema source of truth. Every DDL change goes here first, then
+// `npx drizzle-kit generate` produces a SQL migration in migrations/.
+// Direct ALTER/CREATE/DROP outside this flow is forbidden — see the
+// "Cardinal rule" section in refactor.md.
+import { sql } from 'drizzle-orm';
+import { sqliteTable, text, integer, primaryKey, index, uniqueIndex } from 'drizzle-orm/sqlite-core';
+// ──────────────────────────────────────────────────────────────────
+// Identity
+// ──────────────────────────────────────────────────────────────────
+// Canonical humans the bot knows about. One row per real person; same
+// person on multiple channels = multiple rows in `identities`, one row
+// here.
+export const persons = sqliteTable('persons', {
+    id: text('id').primaryKey(),
+    displayName: text('display_name'),
+    timezone: text('timezone'),
+    createdAt: integer('created_at').notNull(),
+});
+// Channel-addressable identities resolving to a person. Address is
+// channel-prefixed: wa:dm:..., wa:group:..., tg:dm:..., etc.
+export const identities = sqliteTable('identities', {
+    personId: text('person_id').notNull().references(() => persons.id),
+    address: text('address').notNull().unique(),
+    addedAt: integer('added_at').notNull(),
+}, t => ({
+    pk: primaryKey({ columns: [t.personId, t.address] }),
+}));
+// ──────────────────────────────────────────────────────────────────
+// Runtime control
+// ──────────────────────────────────────────────────────────────────
+// Worker registry. Every worker (chat, async, browser, sender, memory,
+// orchestrator) inserts a row at startup and updates last_seen as a
+// heartbeat. Orchestrator uses this for liveness detection and
+// graceful shutdown drain (see refactor.md §Workers).
+export const workers = sqliteTable('workers', {
+    id: text('id').primaryKey(), // `${hostname}-${pid}-${slot}`
+    kind: text('kind').notNull(), // 'chat'|'async'|'browser'|'sender'|'memory'|'orchestrator'
+    status: text('status').notNull(), // 'idle'|'busy'|'draining'|'dead'
+    currentJob: text('current_job'), // 'queue:id' if busy
+    lastSeen: integer('last_seen').notNull(),
+    startedAt: integer('started_at').notNull(),
+}, t => ({
+    byKindStatus: index('workers_by_kind_status').on(t.kind, t.status),
+    byLastSeen: index('workers_by_last_seen').on(t.lastSeen),
+}));
+// Bot-wide control signals. Insert a row to request shutdown/pause/
+// reload; orchestrator picks it up on its next tick. Single-key, so
+// using PK on key gives us upsert semantics.
+export const control = sqliteTable('control', {
+    key: text('key').primaryKey(), // 'shutdown'|'pause'|'reload_config'
+    value: text('value'),
+    requestedBy: text('requested_by'),
+    requestedAt: integer('requested_at').notNull(),
+});
+// ──────────────────────────────────────────────────────────────────
+// Outbound queue (Phase 1)
+// ──────────────────────────────────────────────────────────────────
+// Replies waiting to be sent on any channel. Workers (chat / async /
+// browser) insert rows here; the sender worker drains by channel
+// adapter. AI code never touches the WA socket directly.
+//
+// Status lifecycle:
+//   pending → sending → done   (happy path)
+//   sending → pending          (transient fail, attempts++, backoff)
+//   sending → failed           (permanent fail, kept for inspection)
+//   sending → dlq              (attempts > N; dead-letter)
+//
+// idempotency_key prevents double-sends when a worker is reclaimed
+// after TTL and its delayed insert collides with the replacement
+// worker's insert. Format suggested: `from-inbound-<id>` or
+// `from-async-<task-id>`.
+export const outbound = sqliteTable('outbound', {
+    id: integer('id').primaryKey({ autoIncrement: true }),
+    address: text('address').notNull(), // 'wa:dm:...' | 'wa:group:...' | etc.
+    kind: text('kind').notNull(), // 'text'|'image'|'video'|'audio'|'document'
+    text: text('text'), // body or caption
+    mediaPath: text('media_path'), // relative to storage/ when set
+    mediaMime: text('media_mime'),
+    mediaBytes: integer('media_bytes'), // enforced cap
+    quoteMsgId: text('quote_msg_id'),
+    idempotencyKey: text('idempotency_key'),
+    status: text('status').notNull(), // 'pending'|'sending'|'done'|'failed'|'dlq'
+    attempts: integer('attempts').notNull().default(0),
+    nextAttemptAt: integer('next_attempt_at'), // unix seconds; null = ready immediately
+    lastError: text('last_error'),
+    claimedBy: text('claimed_by'),
+    claimedAt: integer('claimed_at'),
+    createdAt: integer('created_at').notNull(),
+    updatedAt: integer('updated_at').notNull(),
+}, t => ({
+    byStatusNext: index('outbound_by_status_next').on(t.status, t.nextAttemptAt),
+    byAddress: index('outbound_by_address').on(t.address),
+    // Sparse unique: enforced only when idempotencyKey is non-null.
+    uniqIdemp: uniqueIndex('outbound_idempotency_key_uq')
+        .on(t.idempotencyKey)
+        .where(sql `${t.idempotencyKey} IS NOT NULL`),
+}));

package/dist/gateway/outgoing.js

@@ -1,10 +1,11 @@
-import { existsSync, unlinkSync } from 'fs';
+import { existsSync, statSync } from 'fs';
+import { extname } from 'path';
 import { isJidGroup } from 'baileys';
 import { config } from '../config.js';
+import { formatAddress, jidToAddress } from '../db/address.js';
 import { logger } from '../logger.js';
-import { append } from '../store/messages.js';
-import { detectMediaType, sendFile, sendText } from '../wa/sender.js';
-import { getSocket } from '../wa/socket.js';
+import { enqueueOutbound } from '../queue/outbound.js';
+import { detectMediaType } from '../wa/sender.js';
 // Matches [FILE: path], [IMAGE: path], [VIDEO: path], [AUDIO: path], [DOCUMENT: path]
 const FILE_TAG_RE = /\[(?:FILE|IMAGE|VIDEO|AUDIO|DOCUMENT):\s*([^\]]+)\]/gi;
 function extractFiles(reply) {
@@ -21,104 +22,140 @@ function extractFiles(reply) {
     }).trim();
     return { text, files };
 }
-export async function handleReply(job, result, originalMsg) {
-    const sock = getSocket();
-    if (!sock) {
-        logger.warn({ jid: job.jid }, 'no socket available to send reply');
-        return;
+function kindForFile(filePath) {
+    return detectMediaType(filePath);
+}
+const MIME_MAP = {
+    '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg',
+    '.gif': 'image/gif', '.webp': 'image/webp',
+    '.mp4': 'video/mp4', '.avi': 'video/avi', '.mov': 'video/quicktime',
+    '.mkv': 'video/x-matroska',
+    '.mp3': 'audio/mpeg', '.ogg': 'audio/ogg', '.opus': 'audio/opus',
+    '.m4a': 'audio/mp4', '.wav': 'audio/wav',
+    '.pdf': 'application/pdf',
+    '.doc': 'application/msword',
+    '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+    '.xls': 'application/vnd.ms-excel',
+    '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+    '.csv': 'text/csv', '.txt': 'text/plain', '.zip': 'application/zip',
+};
+function mimeFor(filePath) {
+    return MIME_MAP[extname(filePath).toLowerCase()] ?? 'application/octet-stream';
+}
+function fileSize(filePath) {
+    try {
+        return statSync(filePath).size;
+    }
+    catch {
+        return undefined;
     }
+}
+// `originalMsg` is currently ignored when routing through the outbound
+// queue — Baileys quoting needs the full WAMessage embedded in
+// contextInfo, which we'd have to serialize through the DB row and
+// reconstruct. Known regression for Phase 1; see refactor-scrap.md.
+// Kept in the signature so existing callers don't change.
+export async function handleReply(job, result, _originalMsg) {
     const raw = result.reply?.replaceAll('—', ', ').replaceAll('–', '-');
     if (!raw)
         return;
     const { text, files } = extractFiles(raw);
     const isGroup = isJidGroup(job.jid) === true;
-    const quoted = isGroup && config.reply.quoteInGroups ? originalMsg : undefined;
+    void isGroup; // quoting deferred; see comment above
+    const address = formatAddress(jidToAddress(job.jid));
     const footer = result.stats && config.reply.showStats
         ? formatStatsFooter(result.stats)
         : '';
-    try {
-        // Send files first (images, videos, PDFs, audio, etc.)
-        for (const filePath of files) {
-            const isFirst = filePath === files[0];
-            const mediaType = detectMediaType(filePath);
-            // First file gets caption if text is short + single file + supports captions
-            const supportsCaption = mediaType !== 'audio';
-            const caption = isFirst && text && text.length <= 1000 && files.length === 1 && supportsCaption
-                ? text
-                : undefined;
-            // Append footer to caption at send time only (not to storage). Only
-            // when this media file is the final user-facing payload (no text
-            // coming after, single file with caption case).
-            const willHaveTextAfter = !!text &&
-                !(files.length === 1 && text.length <= 1000 && supportsCaption);
-            const captionForSend = caption && footer && !willHaveTextAfter
-                ? `${caption}\n\n${footer}`
-                : caption;
-            await sendFile(sock, job.jid, filePath, captionForSend, isFirst ? quoted : undefined);
-            await append({
-                id: `reply-file-${Date.now()}`,
-                jid: job.jid,
-                direction: 'out',
-                fromMe: true,
-                sender: sock.user?.id ?? '',
-                senderNumber: config.owner.number,
-                timestamp: Math.floor(Date.now() / 1000),
-                text: caption || `[${mediaType}: ${filePath}]`,
-                messageType: `${mediaType}Message`,
-                mediaPath: filePath,
-                mediaType,
-            });
-            logger.info({ jid: job.jid, path: filePath, mediaType }, 'file sent');
-            // Clean up temp file after sending
-            try {
-                unlinkSync(filePath);
-            }
-            catch { }
-            if (files.length > 1)
-                await sleep(config.reply.chunkDelayMs);
-        }
-        // Send text (skip if already used as caption on single file)
-        const textAlreadySent = files.length === 1 && text && text.length <= 1000 && detectMediaType(files[0]) !== 'audio';
-        if (text && !textAlreadySent) {
-            const chunks = chunkText(text, config.reply.chunkChars);
-            for (let i = 0; i < chunks.length; i++) {
-                const chunk = chunks[i];
-                const q = i === 0 && files.length === 0 ? quoted : undefined;
-                const isLast = i === chunks.length - 1;
-                const chunkForSend = isLast && footer ? `${chunk}\n\n${footer}` : chunk;
-                await sendText(sock, job.jid, chunkForSend, q);
-                await append({
-                    id: `reply-${Date.now()}-${i}`,
-                    jid: job.jid,
-                    direction: 'out',
-                    fromMe: true,
-                    sender: sock.user?.id ?? '',
-                    senderNumber: config.owner.number,
-                    timestamp: Math.floor(Date.now() / 1000),
-                    text: chunk,
-                    messageType: 'conversation',
-                });
-                if (i < chunks.length - 1)
-                    await sleep(config.reply.chunkDelayMs);
-            }
-        }
-        if (config.reply.typingIndicator) {
-            await sock
-                .sendPresenceUpdate('paused', job.jid)
-                .catch(() => undefined);
-        }
-        logger.info({
-            jid: job.jid,
-            files: files.length,
-            chars: text.length,
-        }, 'reply sent');
+    let pieceIdx = 0;
+    const baseKey = `reply-${job.jid}-${Date.now()}`;
+    const enqueuePiece = (input) => {
+        enqueueOutbound({ ...input, idempotencyKey: `${baseKey}-${pieceIdx++}` });
+    };
+    // Files first. Caption goes on the single-file-with-short-text case,
+    // matching pre-refactor behavior.
+    for (let i = 0; i < files.length; i++) {
+        const filePath = files[i];
+        const isFirst = i === 0;
+        const kind = kindForFile(filePath);
+        const supportsCaption = kind !== 'audio';
+        const caption = isFirst && text && text.length <= 1000 && files.length === 1 && supportsCaption
+            ? text
+            : undefined;
+        const willHaveTextAfter = !!text && !(files.length === 1 && text.length <= 1000 && supportsCaption);
+        const captionForSend = caption && footer && !willHaveTextAfter
+            ? `${caption}\n\n${footer}`
+            : caption;
+        enqueuePiece({
+            address,
+            kind,
+            text: captionForSend,
+            mediaPath: filePath,
+            mediaMime: mimeFor(filePath),
+            mediaBytes: fileSize(filePath),
+        });
     }
-    catch (err) {
-        logger.error({ err, jid: job.jid }, 'failed to send reply');
+    // Text — skip if already used as a caption on a single file.
+    const textAlreadySent = files.length === 1 &&
+        text &&
+        text.length <= 1000 &&
+        kindForFile(files[0]) !== 'audio';
+    if (text && !textAlreadySent) {
+        const chunks = chunkText(text, config.reply.chunkChars);
+        for (let i = 0; i < chunks.length; i++) {
+            const chunk = chunks[i];
+            const isLast = i === chunks.length - 1;
+            const chunkForSend = isLast && footer ? `${chunk}\n\n${footer}` : chunk;
+            enqueuePiece({ address, kind: 'text', text: chunkForSend });
+        }
     }
+    logger.info({ jid: job.jid, files: files.length, chars: text.length, pieces: pieceIdx }, 'reply enqueued for outbound');
 }
-function sleep(ms) {
-    return new Promise((r) => setTimeout(r, ms));
+// Proactive outbound: send a message to a chat without an incoming
+// trigger. Same parsing as handleReply; enqueues outbound rows.
+// `text` may contain [FILE:...] tags; they're extracted and enqueued
+// as media. Returns true if anything was enqueued.
+export async function initiate(params) {
+    const raw = params.text.replaceAll('—', ', ').replaceAll('–', '-');
+    if (!raw.trim())
+        return false;
+    const { text, files } = extractFiles(raw);
+    if (!text && files.length === 0)
+        return false;
+    const address = formatAddress(jidToAddress(params.jid));
+    let pieceIdx = 0;
+    const baseKey = `initiate-${params.jid}-${Date.now()}`;
+    const enqueuePiece = (input) => {
+        enqueueOutbound({ ...input, idempotencyKey: `${baseKey}-${pieceIdx++}` });
+    };
+    for (let i = 0; i < files.length; i++) {
+        const filePath = files[i];
+        const isFirst = i === 0;
+        const kind = kindForFile(filePath);
+        const supportsCaption = kind !== 'audio';
+        const caption = isFirst && text && text.length <= 1000 && files.length === 1 && supportsCaption
+            ? text
+            : undefined;
+        enqueuePiece({
+            address,
+            kind,
+            text: caption,
+            mediaPath: filePath,
+            mediaMime: mimeFor(filePath),
+            mediaBytes: fileSize(filePath),
+        });
+    }
+    const textAlreadySent = files.length === 1 &&
+        text &&
+        text.length <= 1000 &&
+        kindForFile(files[0]) !== 'audio';
+    if (text && !textAlreadySent) {
+        const chunks = chunkText(text, config.reply.chunkChars);
+        for (const chunk of chunks) {
+            enqueuePiece({ address, kind: 'text', text: chunk });
+        }
+    }
+    logger.info({ jid: params.jid, files: files.length, chars: text.length, pieces: pieceIdx }, 'proactive message enqueued for outbound');
+    return pieceIdx > 0;
 }
 // Append-only-at-send footer. Never stored, never in Claude's recent-context
 // feedback loop. Adaptive: shows only what's interesting for this reply.
@@ -142,16 +179,12 @@ export function formatStatsFooter(stats) {
         else if (pct >= 70)
             parts.push(`${pct}% ctx`);
     }
-    // Fresh session — resume is default, says nothing
     if (stats.fresh)
         parts.push('fresh');
-    // Journal flagged — show each slug (usually 0 or 1)
     for (const slug of stats.journalSlugs)
         parts.push(`+journal:${slug}`);
-    // Digest fired
     if (stats.hasDigest)
         parts.push('+digest');
-    // Async spawned
     if (stats.asyncCount > 0) {
         parts.push(stats.asyncCount === 1 ? '+async' : `+${stats.asyncCount} async`);
     }
@@ -164,89 +197,6 @@ function compactTokens(n) {
         return `${(n / 1000).toFixed(1)}k`;
     return `${Math.round(n / 1000)}k`;
 }
-// Proactive outbound: send a message to a chat without an incoming trigger.
-// Extracts [FILE:]/[IMAGE:]/etc tags the same way handleReply does — files
-// get sent as WhatsApp media, remaining text sent normally. Chunks, persists
-// to the message log, never throws. Callers are responsible for the
-// canSendProactive() gate — this function does not re-check it.
-export async function initiate(params) {
-    const sock = getSocket();
-    if (!sock) {
-        logger.warn({ jid: params.jid }, 'initiate: no socket available');
-        return false;
-    }
-    const raw = params.text.replaceAll('—', ', ').replaceAll('–', '-');
-    if (!raw.trim())
-        return false;
-    const { text, files } = extractFiles(raw);
-    try {
-        // Send any files first — images, video, PDFs, audio, etc.
-        for (const filePath of files) {
-            const isFirst = filePath === files[0];
-            const mediaType = detectMediaType(filePath);
-            const supportsCaption = mediaType !== 'audio';
-            const caption = isFirst &&
-                text &&
-                text.length <= 1000 &&
-                files.length === 1 &&
-                supportsCaption
-                ? text
-                : undefined;
-            await sendFile(sock, params.jid, filePath, caption);
-            await append({
-                id: `initiate-file-${Date.now()}`,
-                jid: params.jid,
-                direction: 'out',
-                fromMe: true,
-                sender: sock.user?.id ?? '',
-                senderNumber: config.owner.number,
-                timestamp: Math.floor(Date.now() / 1000),
-                text: caption || `[${mediaType}: ${filePath}]`,
-                messageType: `${mediaType}Message`,
-                mediaPath: filePath,
-                mediaType,
-            });
-            logger.info({ jid: params.jid, path: filePath, mediaType }, 'proactive file sent');
-            try {
-                unlinkSync(filePath);
-            }
-            catch { }
-            if (files.length > 1)
-                await sleep(config.reply.chunkDelayMs);
-        }
-        // Send text — skip only when it was used as the caption on a single file
-        const textAlreadySent = files.length === 1 &&
-            text &&
-            text.length <= 1000 &&
-            detectMediaType(files[0]) !== 'audio';
-        if (text && !textAlreadySent) {
-            const chunks = chunkText(text, config.reply.chunkChars);
-            for (let i = 0; i < chunks.length; i++) {
-                const chunk = chunks[i];
-                await sendText(sock, params.jid, chunk);
-                await append({
-                    id: `initiate-${Date.now()}-${i}`,
-                    jid: params.jid,
-                    direction: 'out',
-                    fromMe: true,
-                    sender: sock.user?.id ?? '',
-                    senderNumber: config.owner.number,
-                    timestamp: Math.floor(Date.now() / 1000),
-                    text: chunk,
-                    messageType: 'conversation',
-                });
-                if (i < chunks.length - 1)
-                    await sleep(config.reply.chunkDelayMs);
-            }
-        }
-        logger.info({ jid: params.jid, files: files.length, chars: text.length }, 'proactive message sent');
-        return true;
-    }
-    catch (err) {
-        logger.error({ err, jid: params.jid }, 'initiate failed');
-        return false;
-    }
-}
 export function chunkText(text, maxChars) {
     if (text.length <= maxChars)
         return [text];

package/dist/index.js

@@ -1,14 +1,28 @@
+import { setBaileysSocket } from './channels/index.js';
+import { closeDb, initDb } from './db/index.js';
+import { syncIdentitiesFromAccess } from './db/identity-sync.js';
 import { attachIncoming } from './gateway/incoming.js';
 import { handleReply } from './gateway/outgoing.js';
 import { logger } from './logger.js';
 import { startScheduler } from './memory/scheduler.js';
 import { replayPending } from './queue/queue.js';
+import { startSenderWorker, stopSenderWorker } from './queue/sender-worker.js';
 import { startSocket } from './wa/socket.js';
 async function main() {
     logger.info('heyamigo starting');
+    // Migrations + drift check first; refuses to start on schema mismatch.
+    initDb();
+    // Derived view: populate persons + identities from access.json.
+    syncIdentitiesFromAccess();
+    // Sender worker drains outbound queue → channel adapters. Started
+    // before the socket so it's ready when handleReply enqueues rows.
+    startSenderWorker();
     startScheduler();
     await startSocket((sock) => {
         attachIncoming(sock);
+        // Point the Baileys adapter at the live socket. Called on each
+        // reconnect with a fresh sock; the adapter just keeps the latest.
+        setBaileysSocket(sock);
     });
     // Replay any jobs left from a previous crash (no original WAMessage
     // available, so replies are sent as plain messages, not quoted).
@@ -18,10 +32,14 @@ async function main() {
 }
 process.on('SIGINT', () => {
     logger.info('SIGINT received, shutting down');
+    stopSenderWorker();
+    closeDb();
     process.exit(0);
 });
 process.on('SIGTERM', () => {
     logger.info('SIGTERM received, shutting down');
+    stopSenderWorker();
+    closeDb();
     process.exit(0);
 });
 main().catch((err) => {

package/dist/queue/outbound-postsend.js

@@ -0,0 +1,77 @@
+// Post-send bookkeeping run by the sender worker after a successful
+// channel send. Kept separate from the worker loop so it's easy to
+// extend (e.g. emit metrics, fire webhooks) without bloating the loop.
+//
+// Today's job:
+//  - Append a row to the message log so Claude sees what the bot
+//    said in future bootstrap context.
+//  - Delete the local file if the outbound row pointed at one and the
+//    path looks like ephemeral outbox content. We trust producers to
+//    only point at files they want unlinked after send.
+import { existsSync, unlinkSync } from 'fs';
+import { isAbsolute, resolve } from 'path';
+import { config } from '../config.js';
+import { addressToExternalId, parseAddress } from '../db/address.js';
+import { logger } from '../logger.js';
+import { append } from '../store/messages.js';
+export async function afterSend(row, sentMsgId) {
+    await persistToMessageLog(row, sentMsgId);
+    maybeUnlinkMedia(row);
+}
+async function persistToMessageLog(row, msgId) {
+    let address;
+    try {
+        address = parseAddress(row.address);
+    }
+    catch {
+        return;
+    }
+    // Message log is currently WA-specific (keyed by JID). Only persist
+    // WA-bound replies for now; multi-channel log unification comes in a
+    // later phase.
+    if (address.channel !== 'wa')
+        return;
+    const jid = addressToExternalId(address);
+    const messageType = row.kind === 'text' ? 'conversation' : `${row.kind}Message`;
+    try {
+        await append({
+            id: `outbound-${row.id}-${msgId}`,
+            jid,
+            direction: 'out',
+            fromMe: true,
+            sender: '',
+            senderNumber: config.owner.number,
+            timestamp: Math.floor(Date.now() / 1000),
+            text: row.text ?? (row.mediaPath ? `[${row.kind}: ${row.mediaPath}]` : ''),
+            messageType,
+            mediaPath: row.mediaPath ?? undefined,
+            mediaType: row.kind === 'text' ? undefined : row.kind,
+        });
+    }
+    catch (err) {
+        logger.warn({ err, outboundId: row.id }, 'failed to append to message log (send already succeeded)');
+    }
+}
+function maybeUnlinkMedia(row) {
+    if (!row.mediaPath)
+        return;
+    // Resolve relative paths the same way the sender worker did when
+    // calling the adapter, so we delete the actual file.
+    const path = isAbsolute(row.mediaPath)
+        ? row.mediaPath
+        : resolve(process.cwd(), row.mediaPath);
+    // Only auto-delete files inside known-ephemeral directories. Inbound
+    // media in storage/media/ has its own retention cron and should not
+    // be touched here. config.storage.mediaDir = inbound media.
+    const inboundMediaDir = resolve(process.cwd(), config.storage.mediaDir);
+    if (path.startsWith(inboundMediaDir)) {
+        return;
+    }
+    try {
+        if (existsSync(path))
+            unlinkSync(path);
+    }
+    catch (err) {
+        logger.warn({ err, path }, 'failed to unlink outbound media file');
+    }
+}