@c4t4/heyamigo 0.8.15 → 0.9.1

package/dist/queue/outbound.js

@@ -0,0 +1,185 @@
+// Outbound queue helpers. Producers (chat workers, async workers,
+// crons, external triggers) call enqueueOutbound. The sender worker
+// drains via claimNextOutbound + markOutbound{Done,Failed,Retry}.
+//
+// All mutations preserve the claimed_by safety check on completion:
+// only the holder of a claim can mark it done/failed. A slow worker
+// that comes back after TTL-reclaim will harmlessly no-op.
+import { and, asc, eq, isNull, lte, or, sql } from 'drizzle-orm';
+import { getDb } from '../db/index.js';
+import { outbound } from '../db/schema.js';
+// Insert a row, or no-op when the same idempotency_key already exists.
+// Returns the row either way so callers can log/observe.
+export function enqueueOutbound(input) {
+    const db = getDb();
+    const now = Math.floor(Date.now() / 1000);
+    // Idempotency: look up first. SQLite has no INSERT ... ON CONFLICT
+    // returning the previous row, so we serve it in two queries inside
+    // a transaction.
+    if (input.idempotencyKey) {
+        const found = db
+            .select()
+            .from(outbound)
+            .where(eq(outbound.idempotencyKey, input.idempotencyKey))
+            .get();
+        if (found)
+            return { inserted: false, row: found };
+    }
+    const inserted = db
+        .insert(outbound)
+        .values({
+        address: input.address,
+        kind: input.kind,
+        text: input.text ?? null,
+        mediaPath: input.mediaPath ?? null,
+        mediaMime: input.mediaMime ?? null,
+        mediaBytes: input.mediaBytes ?? null,
+        quoteMsgId: input.quoteMsgId ?? null,
+        idempotencyKey: input.idempotencyKey ?? null,
+        status: 'pending',
+        attempts: 0,
+        nextAttemptAt: null,
+        lastError: null,
+        claimedBy: null,
+        claimedAt: null,
+        createdAt: now,
+        updatedAt: now,
+    })
+        .returning()
+        .get();
+    return { inserted: true, row: inserted };
+}
+// Atomic claim. Returns the row or null if nothing's ready.
+// Reserves rows whose nextAttemptAt is null (ready immediately) OR in
+// the past (backoff elapsed). Single-statement so two workers can't
+// claim the same row.
+export function claimNextOutbound(workerId) {
+    const db = getDb();
+    const now = Math.floor(Date.now() / 1000);
+    // SQLite supports UPDATE ... RETURNING since 3.35.
+    return db.transaction((tx) => {
+        const target = tx
+            .select({ id: outbound.id })
+            .from(outbound)
+            .where(and(eq(outbound.status, 'pending'), or(isNull(outbound.nextAttemptAt), lte(outbound.nextAttemptAt, now))))
+            .orderBy(asc(outbound.id))
+            .limit(1)
+            .get();
+        if (!target)
+            return null;
+        const claimed = tx
+            .update(outbound)
+            .set({
+            status: 'sending',
+            claimedBy: workerId,
+            claimedAt: now,
+            updatedAt: now,
+        })
+            .where(and(eq(outbound.id, target.id), eq(outbound.status, 'pending')))
+            .returning()
+            .get();
+        return claimed ?? null;
+    });
+}
+// Mark done — succeeds only when the row is still owned by the caller.
+// Returns whether the update actually applied.
+export function markOutboundDone(id, workerId) {
+    const db = getDb();
+    const now = Math.floor(Date.now() / 1000);
+    const result = db
+        .update(outbound)
+        .set({ status: 'done', updatedAt: now })
+        .where(and(eq(outbound.id, id), eq(outbound.status, 'sending'), eq(outbound.claimedBy, workerId)))
+        .returning({ id: outbound.id })
+        .all();
+    return result.length > 0;
+}
+// Backoff schedule: 1s, 5s, 30s, 2min, give up.
+const BACKOFF_SECONDS = [1, 5, 30, 120];
+const MAX_ATTEMPTS = BACKOFF_SECONDS.length;
+// Transient failure: return to pending with next_attempt_at set, or
+// move to DLQ if attempts exceeded. Caller-owned check applies.
+export function markOutboundRetryOrDlq(id, workerId, errorMessage) {
+    const db = getDb();
+    return db.transaction((tx) => {
+        const row = tx
+            .select()
+            .from(outbound)
+            .where(eq(outbound.id, id))
+            .get();
+        if (!row || row.status !== 'sending' || row.claimedBy !== workerId) {
+            return { retried: false, deadLettered: false };
+        }
+        const now = Math.floor(Date.now() / 1000);
+        const nextAttempts = row.attempts + 1;
+        if (nextAttempts > MAX_ATTEMPTS) {
+            tx.update(outbound)
+                .set({
+                status: 'dlq',
+                attempts: nextAttempts,
+                lastError: errorMessage,
+                claimedBy: null,
+                claimedAt: null,
+                updatedAt: now,
+            })
+                .where(eq(outbound.id, id))
+                .run();
+            return { retried: false, deadLettered: true };
+        }
+        const backoff = BACKOFF_SECONDS[Math.min(row.attempts, BACKOFF_SECONDS.length - 1)];
+        tx.update(outbound)
+            .set({
+            status: 'pending',
+            attempts: nextAttempts,
+            nextAttemptAt: now + backoff,
+            lastError: errorMessage,
+            claimedBy: null,
+            claimedAt: null,
+            updatedAt: now,
+        })
+            .where(eq(outbound.id, id))
+            .run();
+        return { retried: true, deadLettered: false };
+    });
+}
+// Permanent failure (no retry). Used when the error is unrecoverable
+// — e.g. media file missing, malformed address.
+export function markOutboundFailed(id, workerId, errorMessage) {
+    const db = getDb();
+    const now = Math.floor(Date.now() / 1000);
+    const result = db
+        .update(outbound)
+        .set({
+        status: 'failed',
+        lastError: errorMessage,
+        claimedBy: null,
+        claimedAt: null,
+        updatedAt: now,
+    })
+        .where(and(eq(outbound.id, id), eq(outbound.status, 'sending'), eq(outbound.claimedBy, workerId)))
+        .returning({ id: outbound.id })
+        .all();
+    return result.length > 0;
+}
+// Orchestrator helper: reclaim rows whose worker died mid-send.
+// "Stuck in sending past TTL" → return to pending so another worker
+// can pick them up. attempts NOT incremented (the worker may have
+// died before even talking to the channel).
+const CLAIM_TTL_SECONDS = 60;
+export function reclaimStuckOutbound() {
+    const db = getDb();
+    const cutoff = Math.floor(Date.now() / 1000) - CLAIM_TTL_SECONDS;
+    const result = db
+        .update(outbound)
+        .set({
+        status: 'pending',
+        claimedBy: null,
+        claimedAt: null,
+        // intentionally leaving updatedAt as-is so observability can spot reclaims
+        updatedAt: sql `${outbound.updatedAt}`,
+    })
+        .where(and(eq(outbound.status, 'sending'), lte(outbound.claimedAt, cutoff)))
+        .returning({ id: outbound.id })
+        .all();
+    return result.length;
+}

package/dist/queue/sender-worker.js

@@ -0,0 +1,199 @@
+// Sender worker. Drains the outbound queue and pushes each row to the
+// matching channel adapter. One per process (no concurrency) so
+// per-address ordering is preserved naturally and rate-limiting lives
+// in one place.
+import { hostname } from 'os';
+import { resolve } from 'path';
+import { eq } from 'drizzle-orm';
+import { config } from '../config.js';
+import { getDb } from '../db/index.js';
+import { parseAddress } from '../db/address.js';
+import { workers } from '../db/schema.js';
+import { getChannelAdapter, PermanentChannelError, TransientChannelError, } from '../channels/index.js';
+import { logger } from '../logger.js';
+import { claimNextOutbound, markOutboundDone, markOutboundFailed, markOutboundRetryOrDlq, } from './outbound.js';
+import { afterSend } from './outbound-postsend.js';
+const HEARTBEAT_INTERVAL_MS = 5_000;
+const IDLE_POLL_INTERVAL_MS = 500; // when queue empty
+const BUSY_POLL_INTERVAL_MS = 50; // immediately fetch next after a successful send
+let workerId = null;
+let stopping = false;
+let heartbeatTimer = null;
+function newWorkerId() {
+    return `${hostname()}-${process.pid}-sender-0`;
+}
+function registerWorker(id) {
+    const db = getDb();
+    const now = Math.floor(Date.now() / 1000);
+    db.insert(workers)
+        .values({
+        id,
+        kind: 'sender',
+        status: 'idle',
+        currentJob: null,
+        lastSeen: now,
+        startedAt: now,
+    })
+        .onConflictDoUpdate({
+        target: workers.id,
+        set: { status: 'idle', currentJob: null, lastSeen: now, startedAt: now },
+    })
+        .run();
+}
+function setWorkerStatus(id, status, currentJob = null) {
+    const db = getDb();
+    db.update(workers)
+        .set({
+        status,
+        currentJob,
+        lastSeen: Math.floor(Date.now() / 1000),
+    })
+        .where(eq(workers.id, id))
+        .run();
+}
+function heartbeat(id) {
+    const db = getDb();
+    db.update(workers)
+        .set({ lastSeen: Math.floor(Date.now() / 1000) })
+        .where(eq(workers.id, id))
+        .run();
+}
+// Translate an outbound row into the channel-agnostic message shape.
+// Media paths are stored relative to the project root in the row;
+// resolved to absolute here so the adapter can readFileSync directly.
+function rowToMessage(row) {
+    return {
+        kind: row.kind,
+        text: row.text ?? undefined,
+        mediaPath: row.mediaPath ? resolve(process.cwd(), row.mediaPath) : undefined,
+        mediaMime: row.mediaMime ?? undefined,
+        quoteMsgId: row.quoteMsgId ?? undefined,
+    };
+}
+// Enforce media-size cap. mediaBytes is stored on the row by the
+// producer; if missing, we trust the channel to enforce its own limit.
+function tooLarge(row) {
+    const cap = config.reply.maxOutboundMediaBytes ?? null;
+    if (cap === null)
+        return null;
+    if (row.mediaBytes !== null && row.mediaBytes > cap) {
+        return `media too large: ${row.mediaBytes} > ${cap} bytes`;
+    }
+    return null;
+}
+async function processOne(id, row) {
+    setWorkerStatus(id, 'busy', `outbound:${row.id}`);
+    // Size cap → permanent fail; no point retrying same payload.
+    const sizeError = tooLarge(row);
+    if (sizeError) {
+        markOutboundFailed(row.id, id, sizeError);
+        logger.warn({ id: row.id, address: row.address }, sizeError);
+        setWorkerStatus(id, 'idle');
+        return;
+    }
+    let address;
+    try {
+        address = parseAddress(row.address);
+    }
+    catch (err) {
+        markOutboundFailed(row.id, id, `bad address: ${row.address}`);
+        logger.warn({ id: row.id, err }, 'outbound row has unparseable address');
+        setWorkerStatus(id, 'idle');
+        return;
+    }
+    // system:* addresses are bot-internal; not real channels. Drop them
+    // with a friendly log so future cron-emitted system messages don't
+    // accidentally try to "send" anywhere.
+    if (address.channel === 'system') {
+        markOutboundFailed(row.id, id, 'system addresses are not sendable');
+        logger.warn({ id: row.id, address: row.address }, 'system address routed to sender');
+        setWorkerStatus(id, 'idle');
+        return;
+    }
+    let adapter;
+    try {
+        adapter = getChannelAdapter(address.channel);
+    }
+    catch (err) {
+        markOutboundFailed(row.id, id, err.message);
+        logger.error({ id: row.id, channel: address.channel }, 'no adapter for channel');
+        setWorkerStatus(id, 'idle');
+        return;
+    }
+    try {
+        const result = await adapter.send(address.externalId, rowToMessage(row));
+        const ok = markOutboundDone(row.id, id);
+        if (!ok) {
+            // Lost the claim — orchestrator reclaimed it as stuck, or
+            // status changed under us. The send already happened though
+            // (channel returned a msg_id), so we just log and move on. The
+            // reclaimed copy may re-send → that's why idempotency_key
+            // matters at the producer side.
+            logger.warn({ id: row.id, msgId: result.msgId }, 'outbound sent but markDone failed (claim lost?)');
+        }
+        else {
+            await afterSend(row, result.msgId).catch((err) => {
+                logger.error({ err, id: row.id }, 'afterSend hook failed');
+            });
+            logger.info({ id: row.id, address: row.address, kind: row.kind, msgId: result.msgId }, 'outbound sent');
+        }
+    }
+    catch (err) {
+        if (err instanceof PermanentChannelError) {
+            markOutboundFailed(row.id, id, err.message);
+            logger.error({ id: row.id, err: err.message }, 'outbound permanent failure');
+        }
+        else if (err instanceof TransientChannelError) {
+            const result = markOutboundRetryOrDlq(row.id, id, err.message);
+            if (result.deadLettered) {
+                logger.error({ id: row.id }, 'outbound dead-lettered after max attempts');
+            }
+            else if (result.retried) {
+                logger.warn({ id: row.id, err: err.message }, 'outbound transient fail, will retry');
+            }
+        }
+        else {
+            // Unexpected throw — treat as transient (safer to retry than to
+            // give up on something we don't understand).
+            const result = markOutboundRetryOrDlq(row.id, id, `unexpected error: ${err.message}`);
+            logger.error({ id: row.id, err, retried: result.retried, dlq: result.deadLettered }, 'outbound unexpected error');
+        }
+    }
+    finally {
+        setWorkerStatus(id, 'idle');
+    }
+}
+async function loop(id) {
+    while (!stopping) {
+        let processed = false;
+        try {
+            const row = claimNextOutbound(id);
+            if (row) {
+                await processOne(id, row);
+                processed = true;
+            }
+        }
+        catch (err) {
+            logger.error({ err }, 'sender worker loop error');
+        }
+        const delay = processed ? BUSY_POLL_INTERVAL_MS : IDLE_POLL_INTERVAL_MS;
+        await new Promise((res) => setTimeout(res, delay));
+    }
+    setWorkerStatus(id, 'dead');
+}
+export function startSenderWorker() {
+    if (workerId) {
+        logger.warn('sender worker already started; ignoring');
+        return;
+    }
+    workerId = newWorkerId();
+    registerWorker(workerId);
+    heartbeatTimer = setInterval(() => workerId && heartbeat(workerId), HEARTBEAT_INTERVAL_MS);
+    void loop(workerId).catch((err) => logger.fatal({ err }, 'sender worker loop crashed'));
+    logger.info({ workerId }, 'sender worker started');
+}
+export function stopSenderWorker() {
+    stopping = true;
+    if (heartbeatTimer)
+        clearInterval(heartbeatTimer);
+}

package/migrations/0000_phase0_identity_control.sql

@@ -0,0 +1,34 @@
+CREATE TABLE `control` (
+	`key` text PRIMARY KEY NOT NULL,
+	`value` text,
+	`requested_by` text,
+	`requested_at` integer NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE `identities` (
+	`person_id` text NOT NULL,
+	`address` text NOT NULL,
+	`added_at` integer NOT NULL,
+	PRIMARY KEY(`person_id`, `address`),
+	FOREIGN KEY (`person_id`) REFERENCES `persons`(`id`) ON UPDATE no action ON DELETE no action
+);
+--> statement-breakpoint
+CREATE UNIQUE INDEX `identities_address_unique` ON `identities` (`address`);--> statement-breakpoint
+CREATE TABLE `persons` (
+	`id` text PRIMARY KEY NOT NULL,
+	`display_name` text,
+	`timezone` text,
+	`created_at` integer NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE `workers` (
+	`id` text PRIMARY KEY NOT NULL,
+	`kind` text NOT NULL,
+	`status` text NOT NULL,
+	`current_job` text,
+	`last_seen` integer NOT NULL,
+	`started_at` integer NOT NULL
+);
+--> statement-breakpoint
+CREATE INDEX `workers_by_kind_status` ON `workers` (`kind`,`status`);--> statement-breakpoint
+CREATE INDEX `workers_by_last_seen` ON `workers` (`last_seen`);

package/migrations/0001_phase1_outbound.sql

@@ -0,0 +1,23 @@
+CREATE TABLE `outbound` (
+	`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+	`address` text NOT NULL,
+	`kind` text NOT NULL,
+	`text` text,
+	`media_path` text,
+	`media_mime` text,
+	`media_bytes` integer,
+	`quote_msg_id` text,
+	`idempotency_key` text,
+	`status` text NOT NULL,
+	`attempts` integer DEFAULT 0 NOT NULL,
+	`next_attempt_at` integer,
+	`last_error` text,
+	`claimed_by` text,
+	`claimed_at` integer,
+	`created_at` integer NOT NULL,
+	`updated_at` integer NOT NULL
+);
+--> statement-breakpoint
+CREATE INDEX `outbound_by_status_next` ON `outbound` (`status`,`next_attempt_at`);--> statement-breakpoint
+CREATE INDEX `outbound_by_address` ON `outbound` (`address`);--> statement-breakpoint
+CREATE UNIQUE INDEX `outbound_idempotency_key_uq` ON `outbound` (`idempotency_key`) WHERE "outbound"."idempotency_key" IS NOT NULL;

package/migrations/meta/0000_snapshot.json

@@ -0,0 +1,223 @@
+{
+  "version": "6",
+  "dialect": "sqlite",
+  "id": "16f29db1-6a21-41fb-b7e2-933ea7384dad",
+  "prevId": "00000000-0000-0000-0000-000000000000",
+  "tables": {
+    "control": {
+      "name": "control",
+      "columns": {
+        "key": {
+          "name": "key",
+          "type": "text",
+          "primaryKey": true,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "value": {
+          "name": "value",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        },
+        "requested_by": {
+          "name": "requested_by",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        },
+        "requested_at": {
+          "name": "requested_at",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "checkConstraints": {}
+    },
+    "identities": {
+      "name": "identities",
+      "columns": {
+        "person_id": {
+          "name": "person_id",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "address": {
+          "name": "address",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "added_at": {
+          "name": "added_at",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        }
+      },
+      "indexes": {
+        "identities_address_unique": {
+          "name": "identities_address_unique",
+          "columns": [
+            "address"
+          ],
+          "isUnique": true
+        }
+      },
+      "foreignKeys": {
+        "identities_person_id_persons_id_fk": {
+          "name": "identities_person_id_persons_id_fk",
+          "tableFrom": "identities",
+          "tableTo": "persons",
+          "columnsFrom": [
+            "person_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "no action",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {
+        "identities_person_id_address_pk": {
+          "columns": [
+            "person_id",
+            "address"
+          ],
+          "name": "identities_person_id_address_pk"
+        }
+      },
+      "uniqueConstraints": {},
+      "checkConstraints": {}
+    },
+    "persons": {
+      "name": "persons",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "text",
+          "primaryKey": true,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "display_name": {
+          "name": "display_name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        },
+        "timezone": {
+          "name": "timezone",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "checkConstraints": {}
+    },
+    "workers": {
+      "name": "workers",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "text",
+          "primaryKey": true,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "kind": {
+          "name": "kind",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "status": {
+          "name": "status",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "current_job": {
+          "name": "current_job",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        },
+        "last_seen": {
+          "name": "last_seen",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "started_at": {
+          "name": "started_at",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        }
+      },
+      "indexes": {
+        "workers_by_kind_status": {
+          "name": "workers_by_kind_status",
+          "columns": [
+            "kind",
+            "status"
+          ],
+          "isUnique": false
+        },
+        "workers_by_last_seen": {
+          "name": "workers_by_last_seen",
+          "columns": [
+            "last_seen"
+          ],
+          "isUnique": false
+        }
+      },
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "checkConstraints": {}
+    }
+  },
+  "views": {},
+  "enums": {},
+  "_meta": {
+    "schemas": {},
+    "tables": {},
+    "columns": {}
+  },
+  "internal": {
+    "indexes": {}
+  }
+}