ompchamber 1.2.1 → 2.0.0

package/src/server/lib/queue/delivery.server.test.ts

@@ -16,67 +16,13 @@
  * follow-up stranded, while a freshly started server delivered the identical
  * item correctly.
  *
- * The store is mocked so the contract under test is delivery's own: which RPCs
- * a claimed item produces, and that a lost timer is superseded rather than
- * allowed to wedge the session. Time is driven by fake timers, so nothing here
- * waits on the wall clock.
+ * This file covers the SCHEDULING contract — when a delivery runs, when it
+ * defers, and how it recovers. What a dispatch actually sends lives in
+ * `delivery-dispatch.server.test.ts`; both share `delivery-harness.ts`.
  */
 
-import { afterAll, afterEach, beforeEach, describe, expect, jest, mock, test } from 'bun:test';
-import type { QueuedMessage } from '@/shared/types/chat';
-import type { QueueDeliveryHost } from '@/server/lib/queue/delivery.server';
-
-/** In-memory stand-in for the SQLite-backed queue. */
-const queue = new Map<string, QueuedMessage[]>();
-
-mock.module('@/server/lib/queue/store.server', () => ({
-  claimHeadQueueItem: (sessionId: string): QueuedMessage | null => {
-    const items = queue.get(sessionId);
-    return items?.shift() ?? null;
-  },
-  requeueHeadQueueItem: async (sessionId: string, item: QueuedMessage): Promise<void> => {
-    const items = queue.get(sessionId) ?? [];
-    items.unshift(item);
-    queue.set(sessionId, items);
-  },
-}));
-
-const { scheduleQueueDelivery } = await import('@/server/lib/queue/delivery.server');
-
-const MODEL = { provider: 'kenari', modelId: 'deepseek-v4-pro', thinkingLevel: 'max', accessMode: 'yolo' as const };
-
-function queueItem(text: string, model: QueuedMessage['model'] = MODEL): QueuedMessage {
-  return { id: `item-${text}`, text, attachments: [], model };
-}
-
-/** Records the RPCs a delivery pushes at the session. */
-function makeHost(sessionId: string) {
-  const sent: Record<string, unknown>[] = [];
-  const host: QueueDeliveryHost = {
-    sessionId,
-    isAlive: () => true,
-    isBusy: () => false,
-    send: async (command) => {
-      sent.push(command);
-      return {};
-    },
-  };
-  return { host, sent };
-}
-
-/** Let the delivery's promise chain settle without advancing the clock. */
-async function flush(): Promise<void> {
-  for (let i = 0; i < 20; i++) await Promise.resolve();
-}
-
-let sessionId = '';
-let counter = 0;
-
-beforeEach(() => {
-  sessionId = `queue-test-${counter++}`;
-  queue.clear();
-  jest.useFakeTimers();
-});
+import { afterAll, afterEach, describe, expect, jest, mock, test } from 'bun:test';
+import { beginQueueTest, flush, makeHost, queue, queueItem, scheduleQueueDelivery, settleDelivery } from '@/server/lib/queue/delivery-harness';
 
 afterEach(() => {
   jest.useRealTimers();
@@ -86,147 +32,130 @@ afterAll(() => {
   mock.restore();
 });
 
-describe('queue delivery', () => {
-  test('delivers the head with the model snapshot it was queued with', async () => {
-    queue.set(sessionId, [queueItem('queued hello')]);
+describe('queue delivery scheduling', () => {
+  test('a busy session holds the item instead of sending it', async () => {
+    const sessionId = beginQueueTest();
+    queue.set(sessionId, [queueItem('held')]);
     const { host, sent } = makeHost(sessionId);
+    host.isBusy = () => true;
 
-    scheduleQueueDelivery(host);
-    jest.advanceTimersByTime(600);
-    await flush();
+    await settleDelivery(host);
 
-    expect(sent).toEqual([
-      { type: 'set_model', provider: 'kenari', modelId: 'deepseek-v4-pro' },
-      { type: 'set_thinking_level', level: 'max' },
-      { type: 'prompt', message: 'queued hello', accessMode: 'yolo' },
-    ]);
-    expect(queue.get(sessionId)).toEqual([]);
+    expect(sent).toEqual([]);
+    expect(queue.get(sessionId)?.length).toBe(1);
   });
 
-  test("an 'auto' thinking level is left to the session", async () => {
-    queue.set(sessionId, [queueItem('no level', { ...MODEL, thinkingLevel: 'auto' })]);
+  // The regression this file exists for: a delivery that finds the session busy
+  // at its tick used to return and stay returned. The run-end that scheduled it
+  // is the LAST trigger that run produces, so nothing re-armed it — the queue
+  // stalled with no error, and even a manual nudge could not recover it because
+  // `scheduleQueueDelivery` early-returns while a fresh entry sits in the map.
+  test('a delivery that finds the session busy retries until it goes idle', async () => {
+    const sessionId = beginQueueTest();
+    queue.set(sessionId, [queueItem('held then sent')]);
     const { host, sent } = makeHost(sessionId);
+    let busy = true;
+    host.isBusy = () => busy;
 
-    scheduleQueueDelivery(host);
-    jest.advanceTimersByTime(600);
-    await flush();
+    await settleDelivery(host);
 
-    expect(sent).toEqual([
-      { type: 'set_model', provider: 'kenari', modelId: 'deepseek-v4-pro' },
-      { type: 'prompt', message: 'no level', accessMode: 'yolo' },
-    ]);
-  });
+    // Held at the first tick — the item must survive untouched.
+    expect(sent).toEqual([]);
+    expect(queue.get(sessionId)?.length).toBe(1);
 
-  test('an item with no model snapshot is sent without touching the model', async () => {
-    queue.set(sessionId, [queueItem('session model', null)]);
-    const { host, sent } = makeHost(sessionId);
+    // Still busy across several retry intervals: the item must not be lost.
+    jest.advanceTimersByTime(6_000);
+    await flush();
+    expect(sent).toEqual([]);
+    expect(queue.get(sessionId)?.length).toBe(1);
 
-    scheduleQueueDelivery(host);
-    jest.advanceTimersByTime(600);
+    // The session frees up on its own (the subagent finished, the dialog was
+    // answered). No new agent_end arrives; the retry is the only thing that can
+    // deliver it.
+    busy = false;
+    jest.advanceTimersByTime(2_000);
     await flush();
 
-    expect(sent).toEqual([{ type: 'prompt', message: 'session model' }]);
+    expect(sent.filter((c) => c.type === 'prompt').length).toBe(1);
+    expect(queue.get(sessionId)).toEqual([]);
   });
 
-  test('a busy session holds the item instead of sending it', async () => {
-    queue.set(sessionId, [queueItem('held')]);
+  test('a retry stops arming once the queue has emptied elsewhere', async () => {
+    const sessionId = beginQueueTest();
+    queue.set(sessionId, [queueItem('taken by another tab')]);
     const { host, sent } = makeHost(sessionId);
     host.isBusy = () => true;
 
-    scheduleQueueDelivery(host);
-    jest.advanceTimersByTime(600);
+    await settleDelivery(host);
+
+    // Another tab claimed it while this session was busy.
+    queue.set(sessionId, []);
+    jest.advanceTimersByTime(2_000);
     await flush();
 
-    expect(sent).toEqual([]);
-    expect(queue.get(sessionId)?.length).toBe(1);
+    // No poll loop is left behind: the host is never asked to deliver again.
+    const probesAfterEmpty = sent.length;
+    jest.advanceTimersByTime(60_000);
+    await flush();
+    expect(sent.length).toBe(probesAfterEmpty);
   });
 
   test('a failed dispatch returns the item to the head', async () => {
+    const sessionId = beginQueueTest();
     queue.set(sessionId, [queueItem('retry me')]);
     const { host } = makeHost(sessionId);
     host.send = async () => {
       throw new Error('rpc down');
     };
 
-    scheduleQueueDelivery(host);
-    jest.advanceTimersByTime(600);
-    await flush();
+    await settleDelivery(host);
 
     expect(queue.get(sessionId)?.map((i) => i.text)).toEqual(['retry me']);
   });
 
-  test('repeated triggers deliver the head exactly once', async () => {
-    queue.set(sessionId, [queueItem('once')]);
+  // A failed dispatch re-arms, so a transient RPC failure drains on its own
+  // rather than waiting for a run end that may never come.
+  test('a failed dispatch retries and delivers once the send recovers', async () => {
+    const sessionId = beginQueueTest();
+    queue.set(sessionId, [queueItem('transient')]);
     const { host, sent } = makeHost(sessionId);
+    let failing = true;
+    host.send = async (command) => {
+      if (failing) throw new Error('rpc down');
+      sent.push(command);
+      return {};
+    };
 
-    scheduleQueueDelivery(host);
-    scheduleQueueDelivery(host);
-    scheduleQueueDelivery(host);
-    jest.advanceTimersByTime(600);
+    await settleDelivery(host);
+    expect(sent).toEqual([]);
+
+    failing = false;
+    jest.advanceTimersByTime(2_000);
     await flush();
 
     expect(sent.filter((c) => c.type === 'prompt').length).toBe(1);
     expect(queue.get(sessionId)).toEqual([]);
   });
 
-  // Attachments reach the server as the persisted display fields only: the
-  // composer's `File` never survives the queue row's JSON round trip, and its
-  // image preview is a `blob:` object URL the server cannot read. These pin the
-  // fields the delivery must use instead — `dataBase64` for the bytes, `type`
-  // for the media type, `content` for an inline-able text file.
-  test('a queued image rides its persisted base64 payload', async () => {
-    const item = queueItem('with image', null);
-    item.attachments = [
-      { id: 'i', name: 'shot.png', type: 'image/png', size: 4, preview: 'blob:http://x/1', dataBase64: 'iVBORw0KGgo=' },
-    ] as QueuedMessage['attachments'];
-    queue.set(sessionId, [item]);
+  test('repeated triggers deliver the head exactly once', async () => {
+    const sessionId = beginQueueTest();
+    queue.set(sessionId, [queueItem('once')]);
     const { host, sent } = makeHost(sessionId);
 
+    // Three triggers land before the settle delay elapses — the head must be
+    // claimed once, not once per trigger.
     scheduleQueueDelivery(host);
-    jest.advanceTimersByTime(600);
-    await flush();
-
-    expect(sent).toEqual([
-      {
-        type: 'prompt',
-        message: 'with image',
-        images: [{ type: 'image', data: 'iVBORw0KGgo=', mimeType: 'image/png' }],
-      },
-    ]);
-  });
-
-  test('a queued text file is inlined into the delivered prompt', async () => {
-    const item = queueItem('review this', null);
-    item.attachments = [
-      { id: 't', name: 'notes.md', type: 'text/markdown', size: 5, preview: '', content: '# hi' },
-    ] as QueuedMessage['attachments'];
-    queue.set(sessionId, [item]);
-    const { host, sent } = makeHost(sessionId);
-
     scheduleQueueDelivery(host);
-    jest.advanceTimersByTime(600);
-    await flush();
-
-    const prompt = sent.find((c) => c.type === 'prompt');
-    expect(prompt?.message).toBe('review this\n\nAttached file: notes.md\n```markdown\n# hi\n```');
-  });
-
-  test('an attachment without a payload is not sent as a blank image', async () => {
-    const item = queueItem('no payload', null);
-    item.attachments = [
-      { id: 'i', name: 'broken.png', type: 'image/png', size: 4, preview: 'blob:http://x/1', dataBase64: '' },
-    ] as QueuedMessage['attachments'];
-    queue.set(sessionId, [item]);
-    const { host, sent } = makeHost(sessionId);
-
     scheduleQueueDelivery(host);
     jest.advanceTimersByTime(600);
     await flush();
 
-    expect(sent).toEqual([{ type: 'prompt', message: 'no payload' }]);
+    expect(sent.filter((c) => c.type === 'prompt').length).toBe(1);
+    expect(queue.get(sessionId)).toEqual([]);
   });
 
   test('a lost timer delays the queue instead of wedging it', async () => {
+    const sessionId = beginQueueTest();
     queue.set(sessionId, [queueItem('recovered')]);
     const { host, sent } = makeHost(sessionId);
 

package/src/server/lib/queue/delivery.server.ts

@@ -20,7 +20,7 @@
  *   semantics live on the server now.
  */
 
-import { claimHeadQueueItem, requeueHeadQueueItem } from '@/server/lib/queue/store.server';
+import { claimHeadQueueItem, hasQueuedItem, requeueHeadQueueItem } from '@/server/lib/queue/store.server';
 import { composeMessageWithTextAttachments } from '@/shared/lib/chat/attachments';
 import type { QueuedMessage } from '@/shared/types/chat';
 
@@ -35,26 +35,57 @@ const DELIVERY_DELAY_MS = 500;
  *
  * Without this, one lost timer wedges the session's queue FOREVER: the guard
  * below early-returns on any existing entry, so nothing would ever schedule
- * again. A timer can be lost without the process dying — `bun --hot`
- * re-evaluating this module discards the pending closure, and a wedged event
- * loop drops the callback — and the symptom is silent: the queue simply never
+ * again. A timer can be lost without the process dying — a wedged event loop
+ * drops the callback — and the symptom is silent: the queue simply never
  * drains and no error is reported. Past this bound the entry is presumed lost
  * and rescheduled, so a client nudge (sent whenever a session with queued
  * items mounts or regains focus) always recovers the queue.
+ *
+ * This covers only a timer that never RUNS. A delivery that runs and cannot
+ * proceed re-arms itself (see {@link DELIVERY_RETRY_MS}); the two paths are
+ * independent, so neither depends on a client trigger arriving.
  */
 const DELIVERY_STALE_MS = 5_000;
 
+/**
+ * Retry interval for a delivery that could not run: the session was busy at its
+ * tick, or the attempt itself failed.
+ *
+ * A busy session is the common case, not an edge one — a subagent outlives the
+ * parent turn's `agent_end`, a compaction runs after it, and an ask/approval
+ * dialog parks it indefinitely — and every one of those is exactly when the
+ * run-end trigger fires. `isBusy()` is an in-memory flag read, so polling it
+ * costs nothing; the alternative (a wake hook per busy source) would need
+ * re-wiring every time a new source is added.
+ */
+const DELIVERY_RETRY_MS = 2_000;
+
 interface PendingDelivery {
   timer: ReturnType<typeof setTimeout>;
   /** Wall-clock after which this entry is presumed lost and superseded. */
   expiresAt: number;
 }
 
+declare global {
+  // eslint-disable-next-line no-var
+  var __ompQueueDeliveryTimers: Map<string, PendingDelivery> | undefined;
+}
+
 /** Per-session pending delivery timer, so a second agent_end while one is
- *  already scheduled cannot double-book the head. */
-const pendingTimers = new Map<string, PendingDelivery>();
+ *  already scheduled cannot double-book the head.
+ *
+ *  On `globalThis` for the same reason the session registry is: a `bun --hot`
+ *  soft reload re-evaluates this module but keeps globalThis, so a module-local
+ *  map would silently orphan every pending entry — the reloaded module would
+ *  see an empty map, `clearPendingTimer` could no longer cancel the orphan, and
+ *  the session's queue would stall with nothing logged. */
+function getPendingTimers(): Map<string, PendingDelivery> {
+  if (!globalThis.__ompQueueDeliveryTimers) globalThis.__ompQueueDeliveryTimers = new Map();
+  return globalThis.__ompQueueDeliveryTimers;
+}
 
 function clearPendingTimer(sessionId: string): void {
+  const pendingTimers = getPendingTimers();
   const entry = pendingTimers.get(sessionId);
   if (entry === undefined) return;
   clearTimeout(entry.timer);
@@ -129,26 +160,87 @@ export interface QueueDeliveryHost {
 export function scheduleQueueDelivery(session: QueueDeliveryHost): void {
   const sessionId = session.sessionId;
   if (!sessionId) return;
+  const pendingTimers = getPendingTimers();
   const pending = pendingTimers.get(sessionId);
   if (pending !== undefined) {
     if (Date.now() < pending.expiresAt) return;
     // Lost timer: clear it so its callback cannot also fire and double-deliver.
     clearTimeout(pending.timer);
   }
+  armDelivery(session, DELIVERY_DELAY_MS);
+}
+
+/**
+ * Deliver the head right now, with no timer involved. Returns whether an item
+ * was actually sent.
+ *
+ * The nudge route's entry point: the failure a nudge recovers from is a lost
+ * timer, so the nudge itself must not depend on one. Busy sessions re-arm the
+ * normal retry (which is timer-based, and harmless — the nudge is a bonus path,
+ * not the only one), and an empty queue is a no-op.
+ */
+export async function deliverQueueNow(session: QueueDeliveryHost): Promise<boolean> {
+  return deliverNext(session);
+}
+
+/** Arm (or re-arm) this session's delivery tick. */
+function armDelivery(session: QueueDeliveryHost, delayMs: number): void {
+  const sessionId = session.sessionId;
+  const pendingTimers = getPendingTimers();
   const timer = setTimeout(() => {
     pendingTimers.delete(sessionId);
-    void deliverNext(session);
-  }, DELIVERY_DELAY_MS);
+    // The attempt is fire-and-forget from the timer's perspective, but it must
+    // never reject silently: an unhandled rejection here is how a throwing
+    // `claimHeadQueueItem` (a `getDbSync()` before the db is resolved) or a
+    // failing RPC turned into a queue that simply stopped draining with nothing
+    // in the log. `deliverNext` owns its own failure handling; this catch is
+    // the last resort so the failure is at least reported.
+    deliverNext(session).catch((error) => {
+      console.error(`[queue] delivery attempt failed for session ${sessionId}:`, error);
+    });
+  }, delayMs);
   pendingTimers.set(sessionId, { timer, expiresAt: Date.now() + DELIVERY_STALE_MS });
 }
 
-async function deliverNext(session: QueueDeliveryHost): Promise<void> {
+/**
+ * Re-arm the retry, but only while there is still something to deliver.
+ *
+ * A busy session whose queue emptied in the meantime (another tab took the
+ * head, or the user deleted the row) must not leave a poll loop running
+ * forever.
+ */
+async function armRetryIfQueued(session: QueueDeliveryHost): Promise<void> {
+  if (await hasQueuedItem(session.sessionId)) armDelivery(session, DELIVERY_RETRY_MS);
+}
+
+/**
+ * Deliver the head, or re-arm and try again later. True when an item was sent.
+ *
+ * A busy session is the normal case at the moment this runs — the run-end that
+ * scheduled it fires while a subagent, a compaction or an ask dialog is still
+ * live, and `isBusy()` is true for all three. Returning there is what used to
+ * strand the queue permanently: the timer had already been deleted from the map
+ * by its own callback, so nothing remained to retry it and no later trigger was
+ * guaranteed to arrive. Re-arming keeps the retry owned by this module.
+ */
+async function deliverNext(session: QueueDeliveryHost): Promise<boolean> {
   const sessionId = session.sessionId;
-  if (!session.isAlive() || session.isBusy()) return;
-  const item = claimHeadQueueItem(sessionId);
-  if (!item) return;
+  if (!session.isAlive()) return false;
+  if (session.isBusy()) {
+    await armRetryIfQueued(session);
+    return false;
+  }
+  const item = await claimHeadQueueItem(sessionId);
+  if (!item) return false;
   const ok = await dispatchQueuedItem(session, item);
-  if (!ok) await requeueHeadQueueItem(sessionId, item);
+  if (!ok) {
+    await requeueHeadQueueItem(sessionId, item);
+    // The dispatch threw (provider error, dead child). Retry on the same cadence
+    // a busy session gets, so a transient failure drains without user action.
+    await armRetryIfQueued(session);
+    return false;
+  }
+  return true;
 }
 
 /**

package/src/server/lib/queue/store.server.ts

@@ -19,7 +19,7 @@
  * concurrent nudges. A failed send re-inserts the claimed row at the head.
  */
 
-import { getDb, getDbSync } from '@/server/db.server';
+import { getDb } from '@/server/db.server';
 import { withTransaction } from '@/shared/lib/db/transaction.server';
 import { isApprovalMode } from '@/shared/lib/omp/config/access-mode';
 import type { QueuedMessage, QueuedMessageModel } from '@/shared/types/chat';
@@ -154,11 +154,20 @@ export async function reorderQueue(sessionId: string, orderedIds: string[]): Pro
  * Atomically claim the head for delivery: read + delete inside one
  * transaction, so a concurrent claimant (second tab, reload nudge racing a
  * run end) can never observe — and send — the same item twice. Null when the
- * queue is empty. Synchronous by construction: `bun:sqlite` is sync, so the
- * BEGIN…COMMIT block cannot interleave with another request.
+ * queue is empty.
+ *
+ * Uses the ASYNC `getDb()` handle rather than `getDbSync()`, even though the
+ * transaction block itself is synchronous (`bun:sqlite` is sync, so BEGIN…COMMIT
+ * cannot interleave with another request and the atomicity above is unaffected
+ * by the await before it). `getDbSync()` throws when its module-local resolved
+ * handle is unset, which a `bun --hot` reload of `db.server.ts` does until some
+ * other request calls `getDb()` — and the delivery path runs from a timer, so
+ * nothing else necessarily would. That throw was swallowed by a fire-and-forget
+ * call and stranded the queue silently, with nudges failing the same way.
+ * `getDb()` re-initializes instead, so a delivery heals the state it needs.
  */
-export function claimHeadQueueItem(sessionId: string): QueuedMessage | null {
-  const db = getDbSync();
+export async function claimHeadQueueItem(sessionId: string): Promise<QueuedMessage | null> {
+  const db = await getDb();
   return withTransaction(db, () => {
     const row = db.raw
       .query('SELECT * FROM queued_messages WHERE session_id = ? ORDER BY position ASC LIMIT 1')
@@ -169,6 +178,22 @@ export function claimHeadQueueItem(sessionId: string): QueuedMessage | null {
   });
 }
 
+/**
+ * Whether this session has anything queued, without claiming it.
+ *
+ * Used by the delivery retry to decide whether a retry is still worth arming:
+ * a busy session whose queue emptied (delivered elsewhere, or deleted by the
+ * user) must not leave a poll loop running. Async for the same reason the claim
+ * is — `getDb()` heals a hot-reloaded module where `getDbSync()` would throw.
+ */
+export async function hasQueuedItem(sessionId: string): Promise<boolean> {
+  const db = await getDb();
+  const row = db.raw
+    .query('SELECT 1 AS present FROM queued_messages WHERE session_id = ? LIMIT 1')
+    .get(sessionId) as { present?: number } | undefined;
+  return row?.present === 1;
+}
+
 /**
  * Put a failed delivery back at the head (position below the current minimum)
  * so it is retried before anything else. The next run end or mount nudge