@c4t4/heyamigo 0.9.15 → 0.9.16

package/dist/boot.js

@@ -8,6 +8,7 @@ import { syncIdentitiesFromAccess } from './db/identity-sync.js';
 import { attachIncoming } from './gateway/incoming.js';
 import { logger } from './logger.js';
 import { startScheduler } from './memory/scheduler.js';
+import { startBrowserWorkers, stopBrowserWorkers } from './queue/browser-worker.js';
 import { startChatWorkers, stopChatWorkers } from './queue/chat-worker.js';
 import { startMemoryWorker, stopMemoryWorker, } from './queue/memory-worker.js';
 import { requestShutdown, startOrchestrator, stopOrchestrator, } from './queue/orchestrator.js';
@@ -31,6 +32,7 @@ export async function bootBot() {
     startOrchestrator({
         onShutdownDrained: () => {
             stopChatWorkers();
+            stopBrowserWorkers();
             stopSenderWorker();
             stopMemoryWorker();
             stopOrchestrator();
@@ -42,6 +44,7 @@ export async function bootBot() {
     // No separate replay step needed.
     startSenderWorker();
     startMemoryWorker();
+    startBrowserWorkers();
     startChatWorkers();
     startScheduler();
     await startSocket((sock) => {

package/dist/db/schema.js

@@ -216,3 +216,33 @@ export const memoryWrites = sqliteTable('memory_writes', {
         .on(t.idempotencyKey)
         .where(sql `${t.idempotencyKey} IS NOT NULL`),
 }));
+// ──────────────────────────────────────────────────────────────────
+// Browser tasks (Phase 4, durable)
+// ──────────────────────────────────────────────────────────────────
+// Browser-driven background tasks ([ASYNC-BROWSER:] markers). Replaces
+// the in-memory fastq queue with SQLite-backed durable storage: tasks
+// survive process crashes and reclaim via TTL.
+//
+// Browser worker pool (config.browser.maxWorkers) drains; each task
+// runs as a fresh agent (no persistent session — Phase 4) and opens
+// its own tab on the shared Chrome.
+export const browserTasks = sqliteTable('browser_tasks', {
+    id: integer('id').primaryKey({ autoIncrement: true }),
+    address: text('address').notNull(),
+    actorPersonId: text('actor_person_id'),
+    description: text('description').notNull(),
+    originatingMessage: text('originating_message').notNull(),
+    senderNumber: text('sender_number').notNull(),
+    senderName: text('sender_name'),
+    allowedTools: text('allowed_tools'), // JSON: 'all' | string[]
+    status: text('status').notNull(),
+    attempts: integer('attempts').notNull().default(0),
+    nextAttemptAt: integer('next_attempt_at'),
+    lastError: text('last_error'),
+    claimedBy: text('claimed_by'),
+    claimedAt: integer('claimed_at'),
+    createdAt: integer('created_at').notNull(),
+    updatedAt: integer('updated_at').notNull(),
+}, t => ({
+    byStatusNext: index('btasks_by_status_next').on(t.status, t.nextAttemptAt),
+}));

package/dist/queue/async-tasks.js

@@ -1,8 +1,10 @@
 import { getProvider } from '../ai/providers.js';
+import { formatAddress, jidToAddress } from '../db/address.js';
 import { config } from '../config.js';
 import fastq from 'fastq';
 import { initiate } from '../gateway/outgoing.js';
 import { logger } from '../logger.js';
+import { enqueueBrowserJob } from './browser-queue.js';
 // Concurrency: how many async workers can run simultaneously.
 // Start conservative — each process is expensive (Playwright, multi-minute runs).
 // Tune via config.asyncTasks.concurrency once we have real usage data.
@@ -241,36 +243,32 @@ function truncate(s, n) {
 //   (the chat-track agent writes self-contained task descriptions).
 //   Per-task tab isolation is enforced by the prompt instructions
 //   below.
-// Browser pool: multiple agents share one Chrome (the logged-in
-// profile), each task opens its own tab. Persistent agent session is
-// dropped — every task is fresh, with self-contained instructions
-// from the chat-track agent. The trade-off: no cross-task agent
-// memory; the win: real parallelism.
-const BROWSER_CONCURRENCY = Math.max(1, config.browser?.maxWorkers ?? 3);
-const browserQueue = fastq.promise(async (task) => {
-    inProgress.set(task.id, task);
-    try {
-        await runBrowserTask(task);
-    }
-    catch (err) {
-        logger.error({ err, id: task.id, jid: task.jid }, 'browser task failed unexpectedly');
-    }
-    finally {
-        inProgress.delete(task.id);
-    }
-}, BROWSER_CONCURRENCY);
+// Browser tasks now go into the durable browser_tasks SQLite table.
+// The browser worker pool (src/queue/browser-worker.ts) drains it.
+// In-flight tasks survive process crashes; the orchestrator reclaims
+// stuck claims via the TTL on the table.
 export function enqueueBrowserTask(input) {
+    // Keep AsyncTask shape exported so existing callers (worker.ts)
+    // don't change. The returned id is informational only — the real
+    // row id is the DB auto-increment.
     const task = {
         ...input,
         id: `browser-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
         startedAt: Math.floor(Date.now() / 1000),
     };
+    enqueueBrowserJob({
+        address: formatAddress(jidToAddress(task.jid)),
+        description: task.description,
+        originatingMessage: task.originatingMessage,
+        senderNumber: task.senderNumber,
+        senderName: task.senderName ?? null,
+        allowedTools: task.allowedTools,
+    });
     logger.info({
         id: task.id,
         jid: task.jid,
         description: task.description.slice(0, 200),
     }, 'browser task enqueued');
-    browserQueue.push(task).catch((err) => logger.error({ err, id: task.id }, 'browser queue push failed'));
     return task;
 }
 function buildBrowserPrompt(task) {
@@ -324,7 +322,10 @@ function browserAddDirs() {
         config.storage.mediaDir,
     ];
 }
-async function runBrowserTask(task) {
+// Exported so the browser worker (src/queue/browser-worker.ts) can
+// invoke it for each claimed row. Body unchanged from the pre-queue
+// version — just rehomed for direct invocation by the pool.
+export async function runBrowserTask(task) {
     const provider = getProvider();
     // Each task is fresh (Phase 4 browser parallelism). No persistent
     // session — would force serialization on concurrent tasks.

package/dist/queue/browser-queue.js

@@ -0,0 +1,141 @@
+// Browser tasks queue helpers. Producers call enqueueBrowserJob;
+// the browser worker pool drains via claimNextBrowserTask. Same
+// primitives as inbound/outbound — claim is atomic, retry uses
+// per-task backoff, claimed_by safety check on completion.
+//
+// No per-address serialization: multiple browser tasks for the same
+// originating chat CAN run concurrently (each opens its own tab on
+// the shared Chrome). Reply order isn't a concern because each browser
+// task ends with an outbound row, and the sender worker serializes
+// per-address there.
+import { and, asc, eq, isNull, lte, or, sql } from 'drizzle-orm';
+import { getDb } from '../db/index.js';
+import { browserTasks } from '../db/schema.js';
+export function enqueueBrowserJob(input) {
+    const db = getDb();
+    const now = Math.floor(Date.now() / 1000);
+    return db
+        .insert(browserTasks)
+        .values({
+        address: input.address,
+        actorPersonId: input.actorPersonId ?? null,
+        description: input.description,
+        originatingMessage: input.originatingMessage,
+        senderNumber: input.senderNumber,
+        senderName: input.senderName ?? null,
+        allowedTools: input.allowedTools
+            ? JSON.stringify(input.allowedTools)
+            : null,
+        status: 'pending',
+        attempts: 0,
+        nextAttemptAt: null,
+        lastError: null,
+        claimedBy: null,
+        claimedAt: null,
+        createdAt: now,
+        updatedAt: now,
+    })
+        .returning()
+        .get();
+}
+export function claimNextBrowserTask(workerId) {
+    const db = getDb();
+    const now = Math.floor(Date.now() / 1000);
+    return db.transaction((tx) => {
+        const target = tx
+            .select({ id: browserTasks.id })
+            .from(browserTasks)
+            .where(and(eq(browserTasks.status, 'pending'), or(isNull(browserTasks.nextAttemptAt), lte(browserTasks.nextAttemptAt, now))))
+            .orderBy(asc(browserTasks.id))
+            .limit(1)
+            .get();
+        if (!target)
+            return null;
+        const claimed = tx
+            .update(browserTasks)
+            .set({
+            status: 'claimed',
+            claimedBy: workerId,
+            claimedAt: now,
+            updatedAt: now,
+        })
+            .where(and(eq(browserTasks.id, target.id), eq(browserTasks.status, 'pending')))
+            .returning()
+            .get();
+        return claimed ?? null;
+    });
+}
+export function markBrowserTaskDone(id, workerId) {
+    const db = getDb();
+    const now = Math.floor(Date.now() / 1000);
+    const result = db
+        .update(browserTasks)
+        .set({ status: 'done', updatedAt: now })
+        .where(and(eq(browserTasks.id, id), eq(browserTasks.status, 'claimed'), eq(browserTasks.claimedBy, workerId)))
+        .returning({ id: browserTasks.id })
+        .all();
+    return result.length > 0;
+}
+// Browser tasks are expensive (multi-minute Playwright sessions) so
+// retries are sparse: 30s, 5min, give up (DLQ after 2 attempts past
+// the first). Most browser failures are deterministic (login wall,
+// bot detection) and won't benefit from rapid retries.
+const BACKOFF_SECONDS = [30, 300];
+const MAX_ATTEMPTS = BACKOFF_SECONDS.length;
+export function markBrowserTaskRetryOrDlq(id, workerId, errorMessage) {
+    const db = getDb();
+    return db.transaction((tx) => {
+        const row = tx.select().from(browserTasks).where(eq(browserTasks.id, id)).get();
+        if (!row || row.status !== 'claimed' || row.claimedBy !== workerId) {
+            return { retried: false, deadLettered: false };
+        }
+        const now = Math.floor(Date.now() / 1000);
+        const nextAttempts = row.attempts + 1;
+        if (nextAttempts > MAX_ATTEMPTS) {
+            tx.update(browserTasks)
+                .set({
+                status: 'dlq',
+                attempts: nextAttempts,
+                lastError: errorMessage,
+                claimedBy: null,
+                claimedAt: null,
+                updatedAt: now,
+            })
+                .where(eq(browserTasks.id, id))
+                .run();
+            return { retried: false, deadLettered: true };
+        }
+        const backoff = BACKOFF_SECONDS[Math.min(row.attempts, BACKOFF_SECONDS.length - 1)];
+        tx.update(browserTasks)
+            .set({
+            status: 'pending',
+            attempts: nextAttempts,
+            nextAttemptAt: now + backoff,
+            lastError: errorMessage,
+            claimedBy: null,
+            claimedAt: null,
+            updatedAt: now,
+        })
+            .where(eq(browserTasks.id, id))
+            .run();
+        return { retried: true, deadLettered: false };
+    });
+}
+// Browser tasks take 1-15 min routinely. Generous reclaim TTL.
+const CLAIM_TTL_SECONDS = 20 * 60;
+export function reclaimStuckBrowserTasks() {
+    const db = getDb();
+    const cutoff = Math.floor(Date.now() / 1000) - CLAIM_TTL_SECONDS;
+    const result = db
+        .update(browserTasks)
+        .set({
+        status: 'pending',
+        claimedBy: null,
+        claimedAt: null,
+        updatedAt: sql `${browserTasks.updatedAt}`,
+    })
+        .where(and(eq(browserTasks.status, 'claimed'), lte(browserTasks.claimedAt, cutoff)))
+        .returning({ id: browserTasks.id })
+        .all();
+    return result.length;
+}

package/dist/queue/browser-worker.js

@@ -0,0 +1,170 @@
+// Browser worker pool. N workers (config.browser.maxWorkers, default
+// 3) drain the browser_tasks SQLite table. Each task runs as a fresh
+// agent with its own tab on the shared Chrome — same model as before
+// the durability change, just claimable from the DB now.
+//
+// Differences vs in-memory fastq:
+// - Tasks survive process crashes (durable rows).
+// - Orchestrator reclaims stuck claims via reclaimStuckBrowserTasks.
+// - Retry / DLQ semantics live in the queue helpers.
+import { hostname } from 'os';
+import { eq } from 'drizzle-orm';
+import { config } from '../config.js';
+import { getDb } from '../db/index.js';
+import { parseAddress } from '../db/address.js';
+import { workers } from '../db/schema.js';
+import { logger } from '../logger.js';
+import { claimNextBrowserTask, markBrowserTaskDone, markBrowserTaskRetryOrDlq, } from './browser-queue.js';
+import { initiate } from '../gateway/outgoing.js';
+import { runBrowserTask } from './async-tasks.js';
+const HEARTBEAT_INTERVAL_MS = 5_000;
+const IDLE_POLL_INTERVAL_MS = 500;
+const BUSY_POLL_INTERVAL_MS = 0;
+const activeWorkers = [];
+let stopping = false;
+let heartbeatTimer = null;
+function newWorkerId(slot) {
+    return `${hostname()}-${process.pid}-browser-${slot}`;
+}
+function registerWorker(id) {
+    const db = getDb();
+    const now = Math.floor(Date.now() / 1000);
+    db.insert(workers)
+        .values({
+        id,
+        kind: 'browser',
+        status: 'idle',
+        currentJob: null,
+        lastSeen: now,
+        startedAt: now,
+    })
+        .onConflictDoUpdate({
+        target: workers.id,
+        set: { status: 'idle', currentJob: null, lastSeen: now, startedAt: now },
+    })
+        .run();
+}
+function setWorkerStatus(id, status, currentJob = null) {
+    const db = getDb();
+    db.update(workers)
+        .set({
+        status,
+        currentJob,
+        lastSeen: Math.floor(Date.now() / 1000),
+    })
+        .where(eq(workers.id, id))
+        .run();
+}
+function heartbeatAll() {
+    if (activeWorkers.length === 0)
+        return;
+    const db = getDb();
+    const now = Math.floor(Date.now() / 1000);
+    for (const id of activeWorkers) {
+        db.update(workers)
+            .set({ lastSeen: now })
+            .where(eq(workers.id, id))
+            .run();
+    }
+}
+// Convert a row into the AsyncTask shape that runBrowserTask expects.
+// The id field is a synthetic string for log lines; the real row id
+// is used for queue bookkeeping.
+function rowToAsyncTask(row) {
+    let allowedTools = 'all';
+    if (row.allowedTools) {
+        try {
+            allowedTools = JSON.parse(row.allowedTools);
+        }
+        catch {
+            // bad JSON → fall back to 'all'
+        }
+    }
+    return {
+        id: `browser-${row.id}`,
+        jid: parseAddress(row.address).externalId,
+        senderNumber: row.senderNumber,
+        senderName: row.senderName ?? undefined,
+        description: row.description,
+        originatingMessage: row.originatingMessage,
+        allowedTools,
+        startedAt: row.claimedAt ?? row.createdAt,
+    };
+}
+async function processOne(workerId, row) {
+    setWorkerStatus(workerId, 'busy', `browser_tasks:${row.id}`);
+    const task = rowToAsyncTask(row);
+    try {
+        await runBrowserTask(task);
+        const ok = markBrowserTaskDone(row.id, workerId);
+        if (!ok) {
+            logger.warn({ id: row.id, workerId }, 'browser task markDone failed (claim lost?). work already done.');
+        }
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        const result = markBrowserTaskRetryOrDlq(row.id, workerId, msg);
+        if (result.deadLettered) {
+            logger.error({ err, id: row.id, address: row.address }, 'browser task dead-lettered after max attempts');
+            // User-facing failure ack so the chat isn't left hanging.
+            try {
+                await initiate({
+                    jid: parseAddress(row.address).externalId,
+                    text: `Heads up: the browser task "${row.description.slice(0, 80)}" failed. Ask me again and I'll retry.`,
+                });
+            }
+            catch (e) {
+                logger.error({ err: e, id: row.id }, 'failed to send DLQ-ack reply');
+            }
+        }
+        else if (result.retried) {
+            logger.warn({ err, id: row.id, address: row.address }, 'browser task transient fail, will retry');
+        }
+    }
+    finally {
+        setWorkerStatus(workerId, 'idle');
+    }
+}
+async function loop(workerId) {
+    while (!stopping) {
+        let processed = false;
+        try {
+            const row = claimNextBrowserTask(workerId);
+            if (row) {
+                await processOne(workerId, row);
+                processed = true;
+            }
+        }
+        catch (err) {
+            logger.error({ err, workerId }, 'browser worker loop error');
+        }
+        const delay = processed ? BUSY_POLL_INTERVAL_MS : IDLE_POLL_INTERVAL_MS;
+        if (delay > 0) {
+            await new Promise((res) => setTimeout(res, delay));
+        }
+        else {
+            await new Promise((res) => setImmediate(res));
+        }
+    }
+    setWorkerStatus(workerId, 'dead');
+}
+export function startBrowserWorkers() {
+    if (activeWorkers.length > 0) {
+        logger.warn('browser workers already started; ignoring');
+        return;
+    }
+    const pool = Math.max(1, config.browser?.maxWorkers ?? 3);
+    for (let i = 0; i < pool; i++) {
+        const id = newWorkerId(i);
+        activeWorkers.push(id);
+        registerWorker(id);
+        void loop(id).catch((err) => logger.fatal({ err, workerId: id }, 'browser worker loop crashed'));
+    }
+    heartbeatTimer = setInterval(heartbeatAll, HEARTBEAT_INTERVAL_MS);
+    logger.info({ pool }, 'browser worker pool started');
+}
+export function stopBrowserWorkers() {
+    stopping = true;
+    if (heartbeatTimer)
+        clearInterval(heartbeatTimer);
+}

package/dist/queue/orchestrator.js

@@ -16,6 +16,7 @@ import { and, eq, lt, ne } from 'drizzle-orm';
 import { getDb } from '../db/index.js';
 import { workers } from '../db/schema.js';
 import { logger } from '../logger.js';
+import { reclaimStuckBrowserTasks } from './browser-queue.js';
 import { reclaimStuckInbound } from './inbound.js';
 import { reclaimStuckMemoryWrites } from './memory-writes.js';
 import { reclaimStuckOutbound } from './outbound.js';
@@ -112,6 +113,10 @@ async function tick(id) {
         if (reclaimedMemWr > 0) {
             logger.info({ reclaimed: reclaimedMemWr }, 'reclaimed stuck memory_writes rows');
         }
+        const reclaimedBrowser = reclaimStuckBrowserTasks();
+        if (reclaimedBrowser > 0) {
+            logger.info({ reclaimed: reclaimedBrowser }, 'reclaimed stuck browser_tasks rows');
+        }
         // Fire any due crons. Order: dispatch each in turn; if dispatch
         // throws (it shouldn't — dispatch swallows), the cron is NOT
         // marked fired and we'll retry on the next tick.

package/migrations/0006_phase4_browser_tasks.sql

@@ -0,0 +1,20 @@
+CREATE TABLE `browser_tasks` (
+	`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+	`address` text NOT NULL,
+	`actor_person_id` text,
+	`description` text NOT NULL,
+	`originating_message` text NOT NULL,
+	`sender_number` text NOT NULL,
+	`sender_name` text,
+	`allowed_tools` text,
+	`status` text NOT NULL,
+	`attempts` integer DEFAULT 0 NOT NULL,
+	`next_attempt_at` integer,
+	`last_error` text,
+	`claimed_by` text,
+	`claimed_at` integer,
+	`created_at` integer NOT NULL,
+	`updated_at` integer NOT NULL
+);
+--> statement-breakpoint
+CREATE INDEX `btasks_by_status_next` ON `browser_tasks` (`status`,`next_attempt_at`);