@haverstack/server 0.8.0

package/tests/fixtures/blockingWorker.ts

@@ -0,0 +1,27 @@
+/**
+ * Stand-in for src/lib/queryWorker/worker.ts that blocks on command, so
+ * pool behavior can be tested against controlled timing rather than
+ * inferred from how long a real search happens to take.
+ *
+ * `filter.search === '__block__'` blocks the thread outright via
+ * Atomics.wait — the closest honest simulation of node:sqlite's
+ * uncancellable synchronous call, and unlike a spin loop it costs no CPU
+ * while the test waits for the deadline to fire.
+ */
+import { parentPort } from 'node:worker_threads';
+import type { QueryRequest, QueryResponse } from '../../src/lib/queryWorker/protocol.js';
+
+if (!parentPort) throw new Error('fixture worker must run as a worker_thread');
+const port = parentPort;
+
+port.on('message', (req: QueryRequest) => {
+  if ((req.query.filter as { search?: string } | undefined)?.search === '__block__') {
+    Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0);
+    return; // unreachable: only terminate() ends the wait
+  }
+  port.postMessage({
+    id: req.id,
+    ok: true,
+    result: { records: [], cursor: null },
+  } satisfies QueryResponse);
+});

package/tests/fixtures/brokenWorker.ts

@@ -0,0 +1,7 @@
+/**
+ * Stand-in for a worker that cannot boot at all — the shape of a fatal
+ * misconfiguration (an unreadable or missing DB_PATH). Pins the pool's
+ * respawn backoff: without one, a worker failing this way would respawn as
+ * fast as the loop allows, forever.
+ */
+throw new Error('__fixture_boot_failure__');

package/tests/lib/changeFeedClient.test.ts

@@ -0,0 +1,126 @@
+import { describe, test, expect } from 'vitest';
+import { Hono } from 'hono';
+import { streamSSE } from 'hono/streaming';
+import { EventEmitter } from 'node:events';
+import { SSEDecoder, openChangeFeed } from '../changeFeedClient.js';
+import type { AppEnv } from '../../src/app.js';
+
+describe('SSEDecoder', () => {
+  test('decodes a single frame delivered in one chunk', () => {
+    const decoder = new SSEDecoder();
+    const frames = decoder.push('id: AA3f1R\nevent: record\ndata: {"kind":"created"}\n\n');
+    expect(frames).toEqual([{ id: 'AA3f1R', event: 'record', data: { kind: 'created' } }]);
+  });
+
+  test('decodes multiple frames from one chunk', () => {
+    const decoder = new SSEDecoder();
+    const frames = decoder.push(
+      'event: ready\ndata: {"seq":"AA3f1Q"}\n\nid: AA3f1R\nevent: record\ndata: {"kind":"created"}\n\n',
+    );
+    expect(frames).toEqual([
+      { event: 'ready', data: { seq: 'AA3f1Q' } },
+      { id: 'AA3f1R', event: 'record', data: { kind: 'created' } },
+    ]);
+  });
+
+  test('decodes a frame split across chunks, mid-line', () => {
+    const decoder = new SSEDecoder();
+    expect(decoder.push('event: rea')).toEqual([]);
+    expect(decoder.push('dy\ndata: {"se')).toEqual([]);
+    const frames = decoder.push('q":"AA3f1Q"}\n\n');
+    expect(frames).toEqual([{ event: 'ready', data: { seq: 'AA3f1Q' } }]);
+  });
+
+  test('ignores comment lines (keepalives)', () => {
+    const decoder = new SSEDecoder();
+    const frames = decoder.push(': keepalive\n\nevent: ready\ndata: {}\n\n');
+    expect(frames).toEqual([{ event: 'ready', data: {} }]);
+  });
+
+  test('a control frame with no id decodes with id absent, even right after a frame that had one', () => {
+    const decoder = new SSEDecoder();
+    const frames = decoder.push(
+      'id: AA3f1R\nevent: record\ndata: {}\n\nevent: reset\ndata: {"reason":"not_supported"}\n\n',
+    );
+    expect(frames[0]!.id).toBe('AA3f1R');
+    expect(frames[1]!.id).toBeUndefined();
+  });
+
+  test('an unrecognized event name is preserved, not dropped', () => {
+    const decoder = new SSEDecoder();
+    const frames = decoder.push('event: something-new\ndata: {}\n\n');
+    expect(frames).toEqual([{ event: 'something-new', data: {} }]);
+  });
+
+  test('non-JSON data is preserved as raw text rather than throwing', () => {
+    const decoder = new SSEDecoder();
+    const frames = decoder.push('event: ready\ndata: not-json\n\n');
+    expect(frames).toEqual([{ event: 'ready', data: 'not-json' }]);
+  });
+});
+
+/**
+ * A throwaway SSE server, deliberately not this repo's GET /changes, so
+ * these pin openChangeFeed's own mechanics rather than the feed's: a
+ * connection opens, a mutation dispatched on a *second*, concurrent request
+ * reaches the still-open stream, and waitForFrames/close behave around it.
+ */
+function buildSseTestApp() {
+  const emitter = new EventEmitter();
+  const app = new Hono<AppEnv>();
+  app.get('/sse-test', (c) => {
+    return streamSSE(c, async (stream) => {
+      await stream.writeSSE({ event: 'ready', data: '{}' });
+      for (let i = 0; i < 2; i++) {
+        const payload = await new Promise<{ id: string; data: unknown }>((resolve) => {
+          emitter.once('mutate', resolve);
+        });
+        await stream.writeSSE({
+          event: 'record',
+          id: payload.id,
+          data: JSON.stringify(payload.data),
+        });
+      }
+    });
+  });
+  app.post('/mutate', async (c) => {
+    const body = await c.req.json();
+    emitter.emit('mutate', body);
+    return c.json({ ok: true });
+  });
+  return app;
+}
+
+describe('openChangeFeed', () => {
+  test('captures the opening frame immediately, then a mutation made while open', async () => {
+    const app = buildSseTestApp();
+    const conn = await openChangeFeed(app, '/sse-test');
+    try {
+      expect(conn.status).toBe(200);
+      await conn.waitForFrames(1);
+      expect(conn.frames[0]).toEqual({ event: 'ready', data: {} });
+
+      await app.request('/mutate', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ id: 'AA3f1R', data: { kind: 'created' } }),
+      });
+
+      const frames = await conn.waitForFrames(2);
+      expect(frames[1]).toEqual({ id: 'AA3f1R', event: 'record', data: { kind: 'created' } });
+    } finally {
+      await conn.close();
+    }
+  });
+
+  test('waitForFrames rejects rather than hanging when too few frames arrive', async () => {
+    const app = buildSseTestApp();
+    const conn = await openChangeFeed(app, '/sse-test');
+    try {
+      await conn.waitForFrames(1);
+      await expect(conn.waitForFrames(5, 50)).rejects.toThrow(/Timed out/);
+    } finally {
+      await conn.close();
+    }
+  });
+});

package/tests/lib/frameGate.test.ts

@@ -0,0 +1,93 @@
+import { describe, it, expect } from 'vitest';
+import { FrameGate } from '../../src/lib/frameGate.js';
+
+/** A write that never settles on its own — stands in for a stalled client. */
+function neverSettles(): Promise<void> {
+  return new Promise(() => {});
+}
+
+/** A deferred write the test controls the resolution of. */
+function deferred(): { promise: Promise<void>; resolve: () => void } {
+  let resolve!: () => void;
+  const promise = new Promise<void>((r) => {
+    resolve = r;
+  });
+  return { promise, resolve };
+}
+
+describe('FrameGate', () => {
+  it('accepts sends up to the cap without triggering overflow', () => {
+    let overflowed = false;
+    const gate = new FrameGate(2, () => {
+      overflowed = true;
+    });
+
+    expect(gate.send(neverSettles)).toBe(true);
+    expect(gate.send(neverSettles)).toBe(true);
+    expect(overflowed).toBe(false);
+  });
+
+  it('reports overflow exactly once a send would exceed the cap', () => {
+    let overflowCount = 0;
+    const gate = new FrameGate(1, () => {
+      overflowCount++;
+    });
+
+    expect(gate.send(neverSettles)).toBe(true); // fills the one slot, never settles
+    expect(gate.send(neverSettles)).toBe(false); // would be the 2nd in flight — over cap
+    expect(overflowCount).toBe(1);
+  });
+
+  it('never calls onOverflow more than once, and refuses every send after', () => {
+    let overflowCount = 0;
+    const gate = new FrameGate(1, () => {
+      overflowCount++;
+    });
+
+    gate.send(neverSettles);
+    gate.send(neverSettles); // triggers overflow
+    expect(gate.send(neverSettles)).toBe(false);
+    expect(gate.send(neverSettles)).toBe(false);
+    expect(overflowCount).toBe(1);
+  });
+
+  it('frees a slot once an in-flight send settles, admitting the next one', async () => {
+    const gate = new FrameGate(1, () => {
+      throw new Error('should not overflow');
+    });
+    const first = deferred();
+
+    expect(gate.send(() => first.promise)).toBe(true);
+    first.resolve();
+    // Let the write's .then/.finally microtasks run.
+    await Promise.resolve();
+    await Promise.resolve();
+
+    let secondSent = false;
+    expect(
+      gate.send(() => {
+        secondSent = true;
+        return Promise.resolve();
+      }),
+    ).toBe(true);
+    expect(secondSent).toBe(true);
+  });
+
+  it('frees a slot even when the in-flight write rejects', async () => {
+    const gate = new FrameGate(1, () => {
+      throw new Error('should not overflow');
+    });
+    const first = deferred();
+    const rejecting = first.promise.then(() => {
+      throw new Error('write failed');
+    });
+
+    expect(gate.send(() => rejecting)).toBe(true);
+    first.resolve();
+    await Promise.resolve();
+    await Promise.resolve();
+    await Promise.resolve();
+
+    expect(gate.send(() => Promise.resolve())).toBe(true);
+  });
+});

package/tests/lib/queryLimit.test.ts

@@ -0,0 +1,33 @@
+/**
+ * @haverstack/core/wire's parsers report `limit` as requested and never
+ * clamp it — a ceiling is deployment policy, not wire contract (core#240 /
+ * server#114). This server's ceiling is applied here, at the call site in
+ * records.ts, and pins the one behavioural detail worth being careful
+ * about in that migration: `limit=5000` in must still yield `1000` out.
+ */
+import { describe, it, expect } from 'vitest';
+import { clampLimit, MAX_QUERY_LIMIT } from '../../src/lib/queryLimit.js';
+
+describe('clampLimit', () => {
+  it('clamps a limit above the ceiling down to it', () => {
+    expect(clampLimit({ limit: 5000 })).toEqual({ limit: MAX_QUERY_LIMIT });
+  });
+
+  it('leaves a limit at or under the ceiling untouched', () => {
+    expect(clampLimit({ limit: MAX_QUERY_LIMIT })).toEqual({ limit: MAX_QUERY_LIMIT });
+    expect(clampLimit({ limit: 10 })).toEqual({ limit: 10 });
+  });
+
+  it('leaves an absent limit absent, not defaulted', () => {
+    expect(clampLimit({})).toEqual({});
+  });
+
+  it('preserves the rest of the query', () => {
+    const query = { filter: { typeId: 'note@1' }, limit: 5000, cursor: 'abc' };
+    expect(clampLimit(query)).toEqual({
+      filter: { typeId: 'note@1' },
+      limit: MAX_QUERY_LIMIT,
+      cursor: 'abc',
+    });
+  });
+});

package/tests/lib/queryWorker.test.ts

@@ -0,0 +1,60 @@
+import { describe, it, expect, afterEach } from 'vitest';
+import { rm } from 'node:fs/promises';
+import { dirname } from 'node:path';
+import { createApp } from '../../src/app.js';
+import type { StackContext } from '../../src/stack.js';
+import { createTestContext, testConfig, tempDbPath, logger, req } from '../setup.js';
+
+/**
+ * Covers docs/spec/wire-format.md § Bounding query cost: a query that
+ * outlives its deadline answers 503 `timeout`, and — the part a shared,
+ * synchronous-in-process adapter can't otherwise offer — the server goes
+ * on serving other requests afterward rather than staying wedged behind
+ * the slow one. See src/lib/queryWorker/pool.ts.
+ */
+describe('Query worker deadline', () => {
+  let dbPath: string;
+  let ctx: StackContext;
+
+  afterEach(async () => {
+    await ctx.queryWorker.close();
+    await ctx.stack.close();
+    await ctx.tokens.close();
+    ctx.nonces.close();
+    await rm(dirname(dbPath), { recursive: true, force: true }).catch(() => {});
+  });
+
+  it('answers 503 timeout when the deadline is exceeded, and keeps serving after', async () => {
+    dbPath = tempDbPath();
+    ctx = await createTestContext(dbPath);
+
+    // A 1ms deadline is exceeded by any real query — spawning/dispatching
+    // to a worker alone takes longer — so this is deterministic without
+    // needing a slow query to actually construct.
+    const impatientApp = createApp(ctx, { ...testConfig(dbPath), queryTimeoutMs: 1 }, logger);
+    const timedOut = await req(impatientApp, 'GET', '/records');
+    expect(timedOut.status).toBe(503);
+    expect((timedOut.data as { error: { code: string } }).error.code).toBe('timeout');
+
+    // Same underlying queryWorker pool, a normal deadline this time: the
+    // worker the timeout terminated must have been replaced, not left
+    // wedging the pool for every query dispatched after it.
+    const patientApp = createApp(ctx, testConfig(dbPath), logger);
+    const ok = await req(patientApp, 'GET', '/records');
+    expect(ok.status).toBe(200);
+    expect((ok.data as { records: unknown[] }).records).toEqual([]);
+
+    // Never routed through the worker at all — just confirms the rest of
+    // the app stayed responsive throughout.
+    const health = await req(patientApp, 'GET', '/health');
+    expect(health.status).toBe(200);
+  });
+
+  it('rejects new work once closed, and tolerates being closed twice', async () => {
+    dbPath = tempDbPath();
+    ctx = await createTestContext(dbPath);
+    await ctx.queryWorker.close();
+    await expect(ctx.queryWorker.query(null, {}, 1000)).rejects.toThrow(/closed/i);
+    // afterEach calls close() again — must be a no-op, not a throw.
+  });
+});

package/tests/lib/queryWorkerPool.test.ts

@@ -0,0 +1,129 @@
+import { describe, it, expect, afterEach, vi } from 'vitest';
+import { StackTimeoutError } from '@haverstack/core';
+import type { Logger } from 'pino';
+import { QueryWorkerPool } from '../../src/lib/queryWorker/pool.js';
+
+/**
+ * Pool-level behavior, driven through fixture workers (tests/fixtures/)
+ * whose timing the test controls. tests/lib/queryWorker.test.ts covers
+ * the same boundary end-to-end over HTTP; this file covers the parts that
+ * only show up under contention and failure, where inferring behavior
+ * from real query latency would be guesswork.
+ */
+const blockingWorkerUrl = new URL('../fixtures/blockingWorker.ts', import.meta.url);
+const brokenWorkerUrl = new URL('../fixtures/brokenWorker.ts', import.meta.url);
+
+/** A query the fixture worker blocks on forever, until terminated. */
+const BLOCKING = { filter: { search: '__block__' } };
+
+function stubLogger(): Logger & { errorCount: () => number } {
+  const error = vi.fn();
+  const logger = { error, info: vi.fn(), warn: vi.fn(), debug: vi.fn() } as unknown as Logger;
+  return Object.assign(logger, { errorCount: () => error.mock.calls.length });
+}
+
+let pool: QueryWorkerPool | null = null;
+
+afterEach(async () => {
+  await pool?.close();
+  pool = null;
+});
+
+describe('QueryWorkerPool — execution deadline', () => {
+  it('terminates a blocked query at the deadline and keeps serving on a fresh worker', async () => {
+    pool = new QueryWorkerPool({
+      init: { dbPath: '/unused-by-fixture' },
+      poolSize: 1,
+      queueLimit: 16,
+      logger: stubLogger(),
+      workerUrl: blockingWorkerUrl,
+    });
+
+    await expect(pool.query(null, BLOCKING, 300)).rejects.toBeInstanceOf(StackTimeoutError);
+
+    // The blocked worker was terminated and replaced, not left holding the
+    // only slot in the pool.
+    await expect(pool.query(null, {}, 2000)).resolves.toMatchObject({ records: [] });
+  });
+});
+
+describe('QueryWorkerPool — queueing is slow, not fatal', () => {
+  it('does not spend a queued deadline while the request waits for a slot', async () => {
+    pool = new QueryWorkerPool({
+      init: { dbPath: '/unused-by-fixture' },
+      poolSize: 1,
+      queueLimit: 16,
+      logger: stubLogger(),
+      workerUrl: blockingWorkerUrl,
+    });
+
+    // Occupies the only slot in the pool for a full 400ms deadline.
+    const blocked = pool.query(null, BLOCKING, 400);
+    // Queued behind it with a 400ms deadline of its own. If that clock ran
+    // from acceptance — as it would if queue wait counted against the
+    // budget — this would time out at ~400ms having never reached a
+    // worker. It must instead wait, then run, then succeed.
+    const queued = pool.query(null, {}, 400);
+
+    await expect(blocked).rejects.toBeInstanceOf(StackTimeoutError);
+    await expect(queued).resolves.toMatchObject({ records: [] });
+  });
+
+  it('sheds load with a retryable timeout once the queue is full', async () => {
+    pool = new QueryWorkerPool({
+      init: { dbPath: '/unused-by-fixture' },
+      poolSize: 1,
+      queueLimit: 1,
+      logger: stubLogger(),
+      workerUrl: blockingWorkerUrl,
+    });
+
+    const blocked = pool.query(null, BLOCKING, 400); // takes the slot
+    const queued = pool.query(null, {}, 2000); // fills the queue (limit 1)
+    // Third request has nowhere to go: rejected up front rather than
+    // growing the queue without bound.
+    await expect(pool.query(null, {}, 2000)).rejects.toThrow(/queue is full/i);
+    await expect(pool.query(null, {}, 2000)).rejects.toBeInstanceOf(StackTimeoutError);
+
+    await expect(blocked).rejects.toBeInstanceOf(StackTimeoutError);
+    await expect(queued).resolves.toMatchObject({ records: [] });
+  });
+});
+
+describe('QueryWorkerPool — a worker that cannot boot', () => {
+  it('backs off instead of respawning forever, and reports the real cause', async () => {
+    const logger = stubLogger();
+    pool = new QueryWorkerPool({
+      init: { dbPath: '/unused-by-fixture' },
+      poolSize: 1,
+      queueLimit: 16,
+      logger,
+      workerUrl: brokenWorkerUrl,
+    });
+
+    // Backoff is 100/200/400/800/1600ms, then the pool marks itself
+    // unhealthy and stops hot-respawning. Without it this loop respawned
+    // as fast as a thread could be created — ~19 times in 3s, forever.
+    await new Promise((r) => setTimeout(r, 1500));
+    expect(logger.errorCount()).toBeLessThanOrEqual(6);
+
+    // A standing spawn fault surfaces as itself, not as a pile of
+    // deadline timeouts that hide the cause.
+    await expect(pool.query(null, {}, 5000)).rejects.toThrow(/__fixture_boot_failure__/);
+  });
+});
+
+describe('QueryWorkerPool — lifecycle', () => {
+  it('rejects new work once closed, and tolerates being closed twice', async () => {
+    pool = new QueryWorkerPool({
+      init: { dbPath: '/unused-by-fixture' },
+      poolSize: 1,
+      queueLimit: 16,
+      logger: stubLogger(),
+      workerUrl: blockingWorkerUrl,
+    });
+    await pool.close();
+    await expect(pool.query(null, {}, 1000)).rejects.toThrow(/closed/i);
+    // afterEach closes again — must be a no-op, not a throw.
+  });
+});