@c4t4/heyamigo 0.9.9 → 0.9.11

package/dist/boot.js

@@ -9,6 +9,7 @@ import { attachIncoming } from './gateway/incoming.js';
 import { logger } from './logger.js';
 import { startScheduler } from './memory/scheduler.js';
 import { startChatWorkers, stopChatWorkers } from './queue/chat-worker.js';
+import { startMemoryWorker, stopMemoryWorker, } from './queue/memory-worker.js';
 import { requestShutdown, startOrchestrator, stopOrchestrator, } from './queue/orchestrator.js';
 import { startSenderWorker, stopSenderWorker } from './queue/sender-worker.js';
 import { startSocket } from './wa/socket.js';
@@ -31,14 +32,16 @@ export async function bootBot() {
         onShutdownDrained: () => {
             stopChatWorkers();
             stopSenderWorker();
+            stopMemoryWorker();
             stopOrchestrator();
             closeDb();
         },
     });
-    // Workers next. Inbound table is the source of truth; anything left
+    // Workers next. Queue tables are the source of truth; anything left
     // from a previous crash gets claimed by the new pool automatically.
     // No separate replay step needed.
     startSenderWorker();
+    startMemoryWorker();
     startChatWorkers();
     startScheduler();
     await startSocket((sock) => {

package/dist/db/schema.js

@@ -185,3 +185,34 @@ export const inbound = sqliteTable('inbound', {
         .on(t.externalMsgId)
         .where(sql `${t.externalMsgId} IS NOT NULL`),
 }));
+// ──────────────────────────────────────────────────────────────────
+// memory_writes queue (Phase 5a)
+// ──────────────────────────────────────────────────────────────────
+// All memory mutations (journal append, journal create, digest
+// trigger, compressed-view invalidation, future observe op) flow
+// through here. One memory worker drains, serializing writes so
+// parallel chat / async workers can't race on file edits.
+//
+// op + payload (JSON): operation-specific. The worker switches on op
+// and calls the matching handler. Adding a new op = adding a switch
+// arm + ensuring the handler is idempotent on re-delivery (since the
+// queue retries on transient failure).
+export const memoryWrites = sqliteTable('memory_writes', {
+    id: integer('id').primaryKey({ autoIncrement: true }),
+    op: text('op').notNull(), // 'append_journal'|'create_journal'|'trigger_digest'|'mark_compressed_dirty'|...
+    payload: text('payload').notNull(), // JSON
+    idempotencyKey: text('idempotency_key'),
+    status: text('status').notNull(), // 'pending'|'claimed'|'done'|'failed'|'dlq'
+    attempts: integer('attempts').notNull().default(0),
+    nextAttemptAt: integer('next_attempt_at'),
+    lastError: text('last_error'),
+    claimedBy: text('claimed_by'),
+    claimedAt: integer('claimed_at'),
+    createdAt: integer('created_at').notNull(),
+    updatedAt: integer('updated_at').notNull(),
+}, t => ({
+    byStatusNext: index('memwr_by_status_next').on(t.status, t.nextAttemptAt),
+    uniqIdemp: uniqueIndex('memwr_idemp_uq')
+        .on(t.idempotencyKey)
+        .where(sql `${t.idempotencyKey} IS NOT NULL`),
+}));

package/dist/gateway/incoming.js

@@ -201,6 +201,7 @@ async function processMessages(messages, sock, ownerJid, isHistorySync = false)
                 senderNumber: stored.senderNumber,
                 fromMe: stored.fromMe,
                 allowedTools: role.tools,
+                allowedTags: role.tags,
             };
             // Enqueue into the inbound table; chat worker pool drains and
             // calls processJob + handleReply asynchronously. Typing indicator

package/dist/memory/digest-flag.js

@@ -123,6 +123,23 @@ export function extractFlags(reply) {
         sendTexts,
     };
 }
+// Strip flags that the sender's role isn't permitted to emit. The
+// agent's reply still goes out as text — only the side-effect markers
+// get suppressed. allowedTags='all' or undefined → no filtering.
+export function filterFlagsByRole(flags, allowedTags) {
+    if (allowedTags === 'all' || allowedTags === undefined)
+        return flags;
+    const allowed = new Set(allowedTags);
+    return {
+        clean: flags.clean,
+        digest: allowed.has('DIGEST') ? flags.digest : null,
+        journals: allowed.has('JOURNAL') ? flags.journals : [],
+        journalCreates: allowed.has('JOURNAL-NEW') ? flags.journalCreates : [],
+        asyncTasks: allowed.has('ASYNC') ? flags.asyncTasks : [],
+        asyncBrowserTasks: allowed.has('ASYNC-BROWSER') ? flags.asyncBrowserTasks : [],
+        sendTexts: allowed.has('SEND-TEXT') ? flags.sendTexts : [],
+    };
+}
 // Legacy helper kept so existing callers still compile.
 export function extractDigestFlag(reply) {
     const r = extractFlags(reply);

package/dist/queue/async-tasks.js

@@ -138,48 +138,47 @@ async function executeAsyncTask(task) {
             });
         }
     }
-    // Journal creates run first so an entry flagged in the same output against
-    // a new slug lands correctly.
-    const { appendEntry, createJournal, getJournal, isValidSlug } = await import('../memory/journals.js');
-    for (const op of journalCreates) {
+    // All memory mutations through memory_writes queue (Phase 5a).
+    const { enqueueMemoryWrite } = await import('./memory-writes.js');
+    const { isValidSlug } = await import('../memory/journals.js');
+    const memBase = `async-${task.id}`;
+    for (let i = 0; i < journalCreates.length; i++) {
+        const op = journalCreates[i];
         if (!isValidSlug(op.slug)) {
             logger.warn({ op, id: task.id }, 'async JOURNAL-NEW: invalid slug, dropped');
             continue;
         }
-        if (getJournal(op.slug))
-            continue;
-        try {
-            createJournal({
-                slug: op.slug,
-                name: titleCaseSlug(op.slug),
-                purpose: op.purpose,
-            });
-            logger.info({ slug: op.slug, id: task.id }, 'journal created via async marker');
-        }
-        catch (err) {
-            logger.error({ err, op, id: task.id }, 'async JOURNAL-NEW failed');
-        }
+        enqueueMemoryWrite({
+            op: 'create_journal',
+            payload: { slug: op.slug, name: titleCaseSlug(op.slug), purpose: op.purpose },
+            idempotencyKey: `${memBase}-create-${i}`,
+        });
     }
+    // Treat enqueued appends as "appended" for the reporting line below;
+    // the memory worker logs the actual append+slug-validity outcomes.
     let appendedCount = 0;
-    for (const j of journals) {
-        const ok = appendEntry(j.slug, {
-            source: 'async',
-            jid: task.jid,
-            senderNumber: task.senderNumber,
-            note: j.note,
+    for (let i = 0; i < journals.length; i++) {
+        const j = journals[i];
+        enqueueMemoryWrite({
+            op: 'append_journal',
+            payload: {
+                slug: j.slug,
+                entry: {
+                    source: 'async',
+                    jid: task.jid,
+                    senderNumber: task.senderNumber,
+                    note: j.note,
+                },
+            },
+            idempotencyKey: `${memBase}-append-${i}`,
         });
-        if (ok)
-            appendedCount++;
-        else {
-            logger.warn({ slug: j.slug, id: task.id }, 'async JOURNAL marker pointed at unknown slug, dropped');
-        }
+        appendedCount++;
     }
     if (digest) {
-        const { scheduleDigest } = await import('../memory/scheduler.js');
-        scheduleDigest({
-            jid: task.jid,
-            number: task.senderNumber,
-            reason: digest,
+        enqueueMemoryWrite({
+            op: 'trigger_digest',
+            payload: { jid: task.jid, number: task.senderNumber, reason: digest },
+            idempotencyKey: `${memBase}-digest`,
         });
     }
     // The clean (marker-stripped) text IS the chat reply. Always send it when
@@ -428,41 +427,42 @@ async function runBrowserTask(task) {
             });
         }
     }
-    const { appendEntry, createJournal, getJournal, isValidSlug } = await import('../memory/journals.js');
-    for (const op of journalCreates) {
+    const { enqueueMemoryWrite } = await import('./memory-writes.js');
+    const { isValidSlug } = await import('../memory/journals.js');
+    const memBase = `browser-${task.id}`;
+    for (let i = 0; i < journalCreates.length; i++) {
+        const op = journalCreates[i];
         if (!isValidSlug(op.slug))
             continue;
-        if (getJournal(op.slug))
-            continue;
-        try {
-            createJournal({
-                slug: op.slug,
-                name: titleCaseSlug(op.slug),
-                purpose: op.purpose,
-            });
-            logger.info({ slug: op.slug, id: task.id }, 'journal created via browser task marker');
-        }
-        catch (err) {
-            logger.error({ err, op, id: task.id }, 'browser JOURNAL-NEW failed');
-        }
+        enqueueMemoryWrite({
+            op: 'create_journal',
+            payload: { slug: op.slug, name: titleCaseSlug(op.slug), purpose: op.purpose },
+            idempotencyKey: `${memBase}-create-${i}`,
+        });
     }
     let appendedCount = 0;
-    for (const j of journals) {
-        const ok = appendEntry(j.slug, {
-            source: 'async',
-            jid: task.jid,
-            senderNumber: task.senderNumber,
-            note: j.note,
+    for (let i = 0; i < journals.length; i++) {
+        const j = journals[i];
+        enqueueMemoryWrite({
+            op: 'append_journal',
+            payload: {
+                slug: j.slug,
+                entry: {
+                    source: 'async',
+                    jid: task.jid,
+                    senderNumber: task.senderNumber,
+                    note: j.note,
+                },
+            },
+            idempotencyKey: `${memBase}-append-${i}`,
         });
-        if (ok)
-            appendedCount++;
+        appendedCount++;
     }
     if (digest) {
-        const { scheduleDigest } = await import('../memory/scheduler.js');
-        scheduleDigest({
-            jid: task.jid,
-            number: task.senderNumber,
-            reason: digest,
+        enqueueMemoryWrite({
+            op: 'trigger_digest',
+            payload: { jid: task.jid, number: task.senderNumber, reason: digest },
+            idempotencyKey: `${memBase}-digest`,
         });
     }
     const chatText = clean.trim();

package/dist/queue/memory-worker.js

@@ -0,0 +1,169 @@
+// Memory worker. Single concurrency by design: serializes ALL memory
+// mutations into one writer thread so parallel chat / async workers
+// can't race on file edits (full-file rewrites in particular).
+//
+// Dispatches by `op` to the existing handlers in src/memory/. The
+// handlers themselves stay synchronous file writes — only the
+// orchestration moved.
+import { hostname } from 'os';
+import { eq } from 'drizzle-orm';
+import { getDb } from '../db/index.js';
+import { workers } from '../db/schema.js';
+import { logger } from '../logger.js';
+import { claimNextMemoryWrite, markMemoryWriteDone, markMemoryWriteRetryOrDlq, } from './memory-writes.js';
+const HEARTBEAT_INTERVAL_MS = 5_000;
+const IDLE_POLL_INTERVAL_MS = 250;
+let workerId = null;
+let stopping = false;
+let heartbeatTimer = null;
+function newWorkerId() {
+    return `${hostname()}-${process.pid}-memory-0`;
+}
+function registerWorker(id) {
+    const db = getDb();
+    const now = Math.floor(Date.now() / 1000);
+    db.insert(workers)
+        .values({
+        id,
+        kind: 'memory',
+        status: 'idle',
+        currentJob: null,
+        lastSeen: now,
+        startedAt: now,
+    })
+        .onConflictDoUpdate({
+        target: workers.id,
+        set: { status: 'idle', currentJob: null, lastSeen: now, startedAt: now },
+    })
+        .run();
+}
+function setWorkerStatus(id, status, currentJob = null) {
+    const db = getDb();
+    db.update(workers)
+        .set({
+        status,
+        currentJob,
+        lastSeen: Math.floor(Date.now() / 1000),
+    })
+        .where(eq(workers.id, id))
+        .run();
+}
+function heartbeat(id) {
+    const db = getDb();
+    db.update(workers)
+        .set({ lastSeen: Math.floor(Date.now() / 1000) })
+        .where(eq(workers.id, id))
+        .run();
+}
+async function applyOp(row) {
+    const payload = JSON.parse(row.payload);
+    switch (row.op) {
+        case 'append_journal': {
+            const { appendEntry } = await import('../memory/journals.js');
+            const slug = payload.slug;
+            const entry = payload.entry;
+            const ok = appendEntry(slug, entry);
+            if (!ok) {
+                // Unknown slug — log + treat as done (no point retrying).
+                logger.warn({ slug }, 'memory_writes: append_journal slug not found, dropped');
+            }
+            return;
+        }
+        case 'create_journal': {
+            const { createJournal, getJournal, isValidSlug } = await import('../memory/journals.js');
+            const slug = payload.slug;
+            if (!isValidSlug(slug)) {
+                logger.warn({ slug }, 'memory_writes: create_journal invalid slug, dropped');
+                return;
+            }
+            if (getJournal(slug)) {
+                logger.info({ slug }, 'memory_writes: create_journal for existing slug, ignored');
+                return;
+            }
+            createJournal({
+                slug,
+                name: payload.name,
+                purpose: payload.purpose,
+            });
+            logger.info({ slug }, 'journal created via memory_writes');
+            return;
+        }
+        case 'trigger_digest': {
+            const { scheduleDigest } = await import('../memory/scheduler.js');
+            scheduleDigest({
+                jid: payload.jid,
+                number: payload.number,
+                reason: payload.reason,
+            });
+            return;
+        }
+        case 'mark_compressed_dirty': {
+            const { markCompressedDirty } = await import('../memory/compressed.js');
+            markCompressedDirty();
+            return;
+        }
+        default:
+            throw new Error(`unknown memory_writes op: ${row.op}`);
+    }
+}
+async function processOne(workerId, row) {
+    setWorkerStatus(workerId, 'busy', `memory_writes:${row.id}`);
+    try {
+        await applyOp(row);
+        const ok = markMemoryWriteDone(row.id, workerId);
+        if (!ok) {
+            logger.warn({ id: row.id, workerId }, 'memory_writes markDone failed (claim lost?). op already applied.');
+        }
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        const result = markMemoryWriteRetryOrDlq(row.id, workerId, msg);
+        if (result.deadLettered) {
+            logger.error({ err, id: row.id, op: row.op }, 'memory_writes dead-lettered');
+        }
+        else if (result.retried) {
+            logger.warn({ err, id: row.id, op: row.op }, 'memory_writes transient fail, will retry');
+        }
+    }
+    finally {
+        setWorkerStatus(workerId, 'idle');
+    }
+}
+async function loop(workerId) {
+    while (!stopping) {
+        let processed = false;
+        try {
+            const row = claimNextMemoryWrite(workerId);
+            if (row) {
+                await processOne(workerId, row);
+                processed = true;
+            }
+        }
+        catch (err) {
+            logger.error({ err, workerId }, 'memory worker loop error');
+        }
+        if (!processed) {
+            await new Promise((res) => setTimeout(res, IDLE_POLL_INTERVAL_MS));
+        }
+        else {
+            await new Promise((res) => setImmediate(res));
+        }
+    }
+    setWorkerStatus(workerId, 'dead');
+}
+export function startMemoryWorker() {
+    if (workerId) {
+        logger.warn('memory worker already started; ignoring');
+        return;
+    }
+    workerId = newWorkerId();
+    registerWorker(workerId);
+    heartbeatTimer = setInterval(() => workerId && heartbeat(workerId), HEARTBEAT_INTERVAL_MS);
+    void loop(workerId).catch((err) => logger.fatal({ err }, 'memory worker loop crashed'));
+    logger.info({ workerId }, 'memory worker started');
+}
+export function stopMemoryWorker() {
+    stopping = true;
+    if (heartbeatTimer)
+        clearInterval(heartbeatTimer);
+}

package/dist/queue/memory-writes.js

@@ -0,0 +1,136 @@
+// memory_writes queue helpers. Producers (chat workers, async
+// workers, journal observer, etc.) call enqueueMemoryWrite. The
+// single memory worker drains. Op is a discriminator; payload is
+// op-specific JSON.
+import { and, asc, eq, isNull, lte, or, sql } from 'drizzle-orm';
+import { getDb } from '../db/index.js';
+import { memoryWrites } from '../db/schema.js';
+export function enqueueMemoryWrite(input) {
+    const db = getDb();
+    const now = Math.floor(Date.now() / 1000);
+    if (input.idempotencyKey) {
+        const found = db
+            .select()
+            .from(memoryWrites)
+            .where(eq(memoryWrites.idempotencyKey, input.idempotencyKey))
+            .get();
+        if (found)
+            return { inserted: false, row: found };
+    }
+    const row = db
+        .insert(memoryWrites)
+        .values({
+        op: input.op,
+        payload: JSON.stringify(input.payload),
+        idempotencyKey: input.idempotencyKey ?? null,
+        status: 'pending',
+        attempts: 0,
+        nextAttemptAt: null,
+        lastError: null,
+        claimedBy: null,
+        claimedAt: null,
+        createdAt: now,
+        updatedAt: now,
+    })
+        .returning()
+        .get();
+    return { inserted: true, row };
+}
+export function claimNextMemoryWrite(workerId) {
+    const db = getDb();
+    const now = Math.floor(Date.now() / 1000);
+    return db.transaction((tx) => {
+        const target = tx
+            .select({ id: memoryWrites.id })
+            .from(memoryWrites)
+            .where(and(eq(memoryWrites.status, 'pending'), or(isNull(memoryWrites.nextAttemptAt), lte(memoryWrites.nextAttemptAt, now))))
+            .orderBy(asc(memoryWrites.id))
+            .limit(1)
+            .get();
+        if (!target)
+            return null;
+        const claimed = tx
+            .update(memoryWrites)
+            .set({
+            status: 'claimed',
+            claimedBy: workerId,
+            claimedAt: now,
+            updatedAt: now,
+        })
+            .where(and(eq(memoryWrites.id, target.id), eq(memoryWrites.status, 'pending')))
+            .returning()
+            .get();
+        return claimed ?? null;
+    });
+}
+export function markMemoryWriteDone(id, workerId) {
+    const db = getDb();
+    const now = Math.floor(Date.now() / 1000);
+    const result = db
+        .update(memoryWrites)
+        .set({ status: 'done', updatedAt: now })
+        .where(and(eq(memoryWrites.id, id), eq(memoryWrites.status, 'claimed'), eq(memoryWrites.claimedBy, workerId)))
+        .returning({ id: memoryWrites.id })
+        .all();
+    return result.length > 0;
+}
+// Memory mutations are cheap; if one fails we don't want to bury it
+// in long backoffs. Quick retries, fast DLQ.
+const BACKOFF_SECONDS = [1, 5, 30];
+const MAX_ATTEMPTS = BACKOFF_SECONDS.length;
+export function markMemoryWriteRetryOrDlq(id, workerId, errorMessage) {
+    const db = getDb();
+    return db.transaction((tx) => {
+        const row = tx.select().from(memoryWrites).where(eq(memoryWrites.id, id)).get();
+        if (!row || row.status !== 'claimed' || row.claimedBy !== workerId) {
+            return { retried: false, deadLettered: false };
+        }
+        const now = Math.floor(Date.now() / 1000);
+        const nextAttempts = row.attempts + 1;
+        if (nextAttempts > MAX_ATTEMPTS) {
+            tx.update(memoryWrites)
+                .set({
+                status: 'dlq',
+                attempts: nextAttempts,
+                lastError: errorMessage,
+                claimedBy: null,
+                claimedAt: null,
+                updatedAt: now,
+            })
+                .where(eq(memoryWrites.id, id))
+                .run();
+            return { retried: false, deadLettered: true };
+        }
+        const backoff = BACKOFF_SECONDS[Math.min(row.attempts, BACKOFF_SECONDS.length - 1)];
+        tx.update(memoryWrites)
+            .set({
+            status: 'pending',
+            attempts: nextAttempts,
+            nextAttemptAt: now + backoff,
+            lastError: errorMessage,
+            claimedBy: null,
+            claimedAt: null,
+            updatedAt: now,
+        })
+            .where(eq(memoryWrites.id, id))
+            .run();
+        return { retried: true, deadLettered: false };
+    });
+}
+const CLAIM_TTL_SECONDS = 60;
+export function reclaimStuckMemoryWrites() {
+    const db = getDb();
+    const cutoff = Math.floor(Date.now() / 1000) - CLAIM_TTL_SECONDS;
+    const result = db
+        .update(memoryWrites)
+        .set({
+        status: 'pending',
+        claimedBy: null,
+        claimedAt: null,
+        updatedAt: sql `${memoryWrites.updatedAt}`,
+    })
+        .where(and(eq(memoryWrites.status, 'claimed'), lte(memoryWrites.claimedAt, cutoff)))
+        .returning({ id: memoryWrites.id })
+        .all();
+    return result.length;
+}

package/dist/queue/orchestrator.js

@@ -17,6 +17,7 @@ import { getDb } from '../db/index.js';
 import { workers } from '../db/schema.js';
 import { logger } from '../logger.js';
 import { reclaimStuckInbound } from './inbound.js';
+import { reclaimStuckMemoryWrites } from './memory-writes.js';
 import { reclaimStuckOutbound } from './outbound.js';
 import { clearControl, readControl, requestControl } from './control.js';
 import { listDueCrons, markCronFired } from './crons.js';
@@ -107,6 +108,10 @@ async function tick(id) {
         if (reclaimedInbound > 0) {
             logger.info({ reclaimed: reclaimedInbound }, 'reclaimed stuck inbound rows');
         }
+        const reclaimedMemWr = reclaimStuckMemoryWrites();
+        if (reclaimedMemWr > 0) {
+            logger.info({ reclaimed: reclaimedMemWr }, 'reclaimed stuck memory_writes rows');
+        }
         // Fire any due crons. Order: dispatch each in turn; if dispatch
         // throws (it shouldn't — dispatch swallows), the cron is NOT
         // marked fired and we'll retry on the next tick.