@openparachute/vault 0.5.3-rc.3 → 0.6.0

package/src/subscribe.ts

@@ -0,0 +1,248 @@
+/**
+ * Live-query SSE subscribe route (live-query SSE — design
+ * `design/2026-06-08-live-query-sse.md`).
+ *
+ *   GET /vault/<name>/api/subscribe?<query params>
+ *
+ * Sends an `event: snapshot` of the currently-matching (scoped) notes, then
+ * live `upsert`/`remove` events as notes change. Auth + tag-scope are already
+ * resolved by the caller (routing.ts) and threaded in — the same `?key=` /
+ * header credential and the same `tagScope` the notes path uses. This route
+ * adds NO new auth plumbing.
+ *
+ * The query string is parsed by the SAME `parseNotesQueryOpts` the structured
+ * notes-query branch uses, so the snapshot predicate and the live matcher
+ * evaluate an identical `QueryOpts`. `search` (FTS) and `near` (graph BFS) are
+ * not evaluable against a single in-memory note, so a subscribe request using
+ * them is rejected with 400 BEFORE any stream opens.
+ */
+
+import type { Store, QueryOpts } from "../core/src/types.ts";
+import { parseNotesQueryOpts, type TagScopeCtx } from "./routes.ts";
+import { filterNotesByTagScope } from "./tag-scope.ts";
+import { buildLiveMatcher, unsupportedSubscriptionReason } from "./live-match.ts";
+import {
+  snapshotFrame,
+  subscriptionManager,
+  type SubscriptionManager,
+  type SubscriptionSink,
+} from "./subscriptions.ts";
+
+/** Keepalive interval — `:` comment every ~25s to defeat idle-proxy timeouts. */
+const KEEPALIVE_MS = 25_000;
+
+function json(data: unknown, status = 200): Response {
+  return Response.json(data, { status });
+}
+
+/**
+ * Handle `GET /api/subscribe`. `vaultName` + `tagScope` come from routing.ts
+ * (auth already validated; tag-scope already expanded). `manager` is injectable
+ * for tests; defaults to the process-wide singleton.
+ */
+export async function handleSubscribe(
+  req: Request,
+  store: Store,
+  vaultName: string,
+  tagScope: TagScopeCtx,
+  manager: SubscriptionManager = subscriptionManager,
+): Promise<Response> {
+  if (req.method !== "GET") {
+    return json({ error: "Method not allowed", message: "subscribe is GET-only" }, 405);
+  }
+
+  const url = new URL(req.url);
+
+  // Reject the un-live-evaluable query shapes up front (before any stream).
+  if (url.searchParams.get("search")) {
+    return json(
+      {
+        error: "search (full-text) is not supported for live subscriptions — FTS can't be evaluated against a single changed note. Drop `search` or poll GET /notes?search=.",
+        code: "UNSUPPORTED_SUBSCRIPTION_QUERY",
+      },
+      400,
+    );
+  }
+  if (url.searchParams.get("near[note_id]")) {
+    return json(
+      {
+        error: "near (graph neighborhood) is not supported for live subscriptions — BFS can't be evaluated against a single changed note. Drop `near` or poll GET /notes?near[note_id]=.",
+        code: "UNSUPPORTED_SUBSCRIPTION_QUERY",
+      },
+      400,
+    );
+  }
+
+  const parsed = parseNotesQueryOpts(url);
+  if (parsed.error) return parsed.error;
+  const queryOpts = parsed.queryOpts!;
+
+  // Belt-and-suspenders: reject cursor (paging) too — meaningless for a live set.
+  const unsupported = unsupportedSubscriptionReason(queryOpts);
+  if (unsupported) {
+    return json({ error: unsupported, code: "UNSUPPORTED_SUBSCRIPTION_QUERY" }, 400);
+  }
+
+  // Build the in-process matcher (resolves tag descendants once, same
+  // hierarchy the snapshot query uses) — may throw QueryError on a malformed
+  // metadata filter shape; surface as 400, same as the notes path.
+  let matcher;
+  try {
+    matcher = await buildLiveMatcher(store, queryOpts);
+  } catch (e: any) {
+    if (e && e.name === "QueryError") {
+      return json({ error: e.message, code: e.code ?? "INVALID_QUERY" }, 400);
+    }
+    throw e;
+  }
+
+  // Snapshot: the scoped query result. queryNotes throws QueryError on e.g. a
+  // non-indexed metadata operator field — surface as 400 (no stream opened).
+  //
+  // Strip paging (M3): the live matcher ignores limit/offset, so a default
+  // limit:50 would truncate the snapshot while live events deliver the full
+  // set — snapshot ⊊ live. The snapshot must be the COMPLETE matching set.
+  // queryNotes defaults an ABSENT limit to 100 (not unlimited), so pass a
+  // large sentinel to fetch every matching row.
+  const SNAPSHOT_UNBOUNDED = Number.MAX_SAFE_INTEGER;
+  const snapshotOpts: QueryOpts = { ...queryOpts, limit: SNAPSHOT_UNBOUNDED, offset: undefined };
+  let snapshotNotes;
+  try {
+    const raw = await store.queryNotes(snapshotOpts);
+    snapshotNotes = filterNotesByTagScope(raw, tagScope.allowed, tagScope.raw);
+  } catch (e: any) {
+    if (e && e.name === "QueryError") {
+      return json({ error: e.message, code: e.code ?? "INVALID_QUERY" }, 400);
+    }
+    throw e;
+  }
+
+  // Cap check BEFORE opening a stream so we can return a real 503. (The
+  // in-`start` re-check below only guards the rare interleave race where two
+  // subscribes pass this check before either registers.)
+  if (!manager.hasCapacity(vaultName)) {
+    return json(
+      {
+        error: "subscription cap reached for this vault — too many concurrent live subscriptions. Retry shortly or close idle streams.",
+        code: "SUBSCRIPTION_CAP_REACHED",
+      },
+      503,
+    );
+  }
+
+  // ---- Build the SSE stream ----
+  //
+  // A pull ReadableStream with an internal frame queue. The manager's sink
+  // writes frames into the queue; `pull` drains them to the controller and
+  // notifies the manager (so the backpressure counter decrements). When the
+  // client disconnects, `cancel` fires → we unregister the subscription and
+  // clear the keepalive timer.
+  let handle: { flushed: () => void; close: () => void } | null = null;
+  let keepalive: ReturnType<typeof setInterval> | null = null;
+
+  const queue: string[] = [];
+  let controllerRef: ReadableStreamDefaultController<Uint8Array> | null = null;
+  let cancelled = false;
+  const encoder = new TextEncoder();
+
+  const flushQueue = () => {
+    if (!controllerRef || cancelled) return;
+    while (queue.length > 0) {
+      const frame = queue.shift()!;
+      try {
+        controllerRef.enqueue(encoder.encode(frame));
+      } catch {
+        // Controller closed underneath us — stop.
+        cancelled = true;
+        return;
+      }
+      handle?.flushed();
+    }
+  };
+
+  const sink: SubscriptionSink = {
+    write(frame: string): boolean {
+      if (cancelled) return false;
+      queue.push(frame);
+      flushQueue();
+      return true;
+    },
+    close(): void {
+      if (cancelled) return;
+      cancelled = true;
+      if (keepalive) {
+        clearInterval(keepalive);
+        keepalive = null;
+      }
+      try {
+        controllerRef?.close();
+      } catch {
+        /* already closed */
+      }
+    },
+  };
+
+  const stream = new ReadableStream<Uint8Array>({
+    start(controller) {
+      controllerRef = controller;
+      // 1. Snapshot first — written into the queue and flushed on this tick.
+      queue.push(snapshotFrame(snapshotNotes));
+
+      // 2. Register the live subscription. Hook dispatch is deferred to a
+      //    microtask, and we register synchronously here within start(), so no
+      //    live event can slip in front of the snapshot. Over-cap → 503; we
+      //    can't return a Response from inside the stream, so the cap is also
+      //    checked below before constructing the Response. (Re-check here for
+      //    the race where two subscribes interleave.)
+      handle = manager.register({
+        vaultName,
+        matcher,
+        tagScopeAllowed: tagScope.allowed,
+        tagScopeRaw: tagScope.raw,
+        sink,
+      });
+      if (!handle) {
+        // Lost the cap race. Emit nothing further and close; the pre-check
+        // below normally catches this, so this path is rare.
+        flushQueue();
+        try {
+          controller.close();
+        } catch {
+          /* noop */
+        }
+        cancelled = true;
+        return;
+      }
+
+      flushQueue();
+
+      // 3. Keepalive comments.
+      keepalive = setInterval(() => {
+        if (cancelled) return;
+        sink.write(":\n\n");
+      }, KEEPALIVE_MS);
+    },
+    pull() {
+      flushQueue();
+    },
+    cancel() {
+      cancelled = true;
+      if (keepalive) {
+        clearInterval(keepalive);
+        keepalive = null;
+      }
+      handle?.close();
+    },
+  });
+
+  return new Response(stream, {
+    status: 200,
+    headers: {
+      "Content-Type": "text/event-stream",
+      "Cache-Control": "no-cache, no-transform",
+      Connection: "keep-alive",
+      // Defeat nginx-style proxy buffering of the event stream.
+      "X-Accel-Buffering": "no",
+    },
+  });
+}

package/src/subscriptions.ts

@@ -0,0 +1,295 @@
+/**
+ * Live-query subscription registry (live-query SSE — design
+ * `design/2026-06-08-live-query-sse.md`).
+ *
+ * A second consumer of the post-commit hook dispatcher (`core/src/hooks.ts`),
+ * alongside the durable webhook-trigger sink. Where a trigger survives across
+ * connections (wakes an offline session), a subscription is ephemeral —
+ * connection-scoped, driving a live UI. Same event source, same predicate,
+ * different durability.
+ *
+ * ## How fan-out works
+ *
+ * All vault stores share the process-wide `defaultHookRegistry`
+ * (`vault-store.ts`). We register exactly ONE broad hook per event type
+ * (`created`/`updated`/`deleted`) — with NO `when` filter, because a
+ * subscription must see EVERY mutation to detect a note LEAVING its set (a
+ * `when` that pre-filters to the query would hide the just-left-the-set
+ * update). On each event the manager:
+ *
+ *   1. resolves the event's vault from the store handle
+ *      (`getVaultNameForStore`) — the shared registry fires for all vaults;
+ *   2. iterates only the subscriptions on THAT vault;
+ *   3. created/updated → `matcher.match(note) && noteWithinTagScope(...)` ?
+ *      emit `upsert{note}` : (updated only) emit `remove{id}` (left-the-set,
+ *      idempotent on the client);
+ *   4. deleted → broadcast `remove{id}` to all that vault's subs (the delete
+ *      payload is a thin `{id, path?}` ref — no tags/metadata — so it can't
+ *      be scope-matched; see the design's "Auth / scope intersection" §).
+ *
+ * O(writes × subscriptions). At vault scale (hundreds of notes, single-digit
+ * open tabs) this is free; documented ceiling, not a silent cap.
+ *
+ * ## Security: scope intersection (load-bearing)
+ *
+ * A subscription MUST NOT emit a note its token can't read. Every `upsert`
+ * passes BOTH the subscription predicate AND `noteWithinTagScope(note,
+ * allowed, rawRoots)` — the SAME check the REST notes path uses. The scope
+ * check is ANDed with the predicate and is not bypassable by query shape.
+ * The snapshot is separately scope-filtered at the route via
+ * `filterNotesByTagScope`.
+ */
+
+import type { Note, Store } from "../core/src/types.ts";
+import type { DeletedNoteRef, HookEvent, NoteHookPayload } from "../core/src/hooks.ts";
+import { defaultHookRegistry } from "../core/src/hooks.ts";
+import { getVaultNameForStore } from "./vault-store.ts";
+import { noteWithinTagScope } from "./tag-scope.ts";
+import type { LiveMatcher } from "./live-match.ts";
+
+/** Default per-vault concurrent-subscription cap. Over it → 503. */
+export const DEFAULT_MAX_SUBSCRIPTIONS_PER_VAULT = 100;
+
+/**
+ * Default bound on a single subscription's pending (unflushed) event buffer.
+ * If a slow client lets the buffer grow past this, the stream is closed — it
+ * reconnects and re-snapshots rather than the server growing memory unbounded.
+ */
+export const DEFAULT_MAX_BUFFERED_EVENTS = 1000;
+
+/** A serialized SSE frame ready to write to the wire. */
+type SseFrame = string;
+
+export interface SubscriptionSink {
+  /** Enqueue a serialized SSE frame. Returns false if the sink is closed. */
+  write(frame: SseFrame): boolean;
+  /** Close the underlying stream (teardown). Idempotent. */
+  close(): void;
+}
+
+interface Subscription {
+  readonly vaultName: string;
+  readonly matcher: LiveMatcher;
+  /** Expanded tag-scope allowlist (null = unscoped). */
+  readonly tagScopeAllowed: Set<string> | null;
+  /** Raw root-tag allowlist (null = unscoped) — `noteWithinTagScope` arg. */
+  readonly tagScopeRaw: string[] | null;
+  readonly sink: SubscriptionSink;
+  /** Pending unflushed frame count (backpressure bound). */
+  buffered: number;
+  readonly maxBuffered: number;
+  closed: boolean;
+}
+
+function sseEvent(event: string, data: unknown): SseFrame {
+  return `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`;
+}
+
+export class SubscriptionManager {
+  private subs = new Set<Subscription>();
+  private perVaultCount = new Map<string, number>();
+  private hooksRegistered = false;
+  private unregisters: Array<() => void> = [];
+  private readonly maxPerVault: number;
+  private readonly resolveVault: (store: Store) => string | undefined;
+
+  constructor(
+    private readonly registry = defaultHookRegistry,
+    opts: { maxPerVault?: number; resolveVault?: (store: Store) => string | undefined } = {},
+  ) {
+    this.maxPerVault = opts.maxPerVault ?? DEFAULT_MAX_SUBSCRIPTIONS_PER_VAULT;
+    // Resolve the event's vault from the store handle. Defaults to the
+    // process-wide WeakMap (vault-store.ts); injectable for tests that build
+    // a store directly without going through `getVaultStore`.
+    this.resolveVault = opts.resolveVault ?? getVaultNameForStore;
+  }
+
+  /** Active subscription count for a vault (for the cap check + tests). */
+  countForVault(vaultName: string): number {
+    return this.perVaultCount.get(vaultName) ?? 0;
+  }
+
+  /** The configured per-vault concurrent-subscription cap. */
+  get maxSubscriptionsPerVault(): number {
+    return this.maxPerVault;
+  }
+
+  /** True iff a new subscription on this vault would be under the cap. */
+  hasCapacity(vaultName: string): boolean {
+    return this.countForVault(vaultName) < this.maxPerVault;
+  }
+
+  /** Total active subscriptions (tests / diagnostics). */
+  get size(): number {
+    return this.subs.size;
+  }
+
+  /**
+   * Register a subscription. Returns the handle, or `null` if the vault is at
+   * its concurrent-subscription cap (the route maps null → 503). The caller
+   * is responsible for sending the initial `snapshot` frame BEFORE calling
+   * this — registration only wires the live stream so no live event can be
+   * missed between snapshot and registration (the caller registers
+   * synchronously after computing the snapshot; hook dispatch is deferred to
+   * a microtask, so a same-tick write can't slip in front of registration).
+   */
+  register(args: {
+    vaultName: string;
+    matcher: LiveMatcher;
+    tagScopeAllowed: Set<string> | null;
+    tagScopeRaw: string[] | null;
+    sink: SubscriptionSink;
+    maxBuffered?: number;
+  }): SubscriptionHandle | null {
+    const current = this.countForVault(args.vaultName);
+    if (current >= this.maxPerVault) return null;
+
+    this.ensureHooks();
+
+    const sub: Subscription = {
+      vaultName: args.vaultName,
+      matcher: args.matcher,
+      tagScopeAllowed: args.tagScopeAllowed,
+      tagScopeRaw: args.tagScopeRaw,
+      sink: args.sink,
+      buffered: 0,
+      maxBuffered: args.maxBuffered ?? DEFAULT_MAX_BUFFERED_EVENTS,
+      closed: false,
+    };
+    this.subs.add(sub);
+    this.perVaultCount.set(args.vaultName, current + 1);
+
+    return {
+      /** Note that a previously-buffered frame flushed to the wire. */
+      flushed: () => {
+        if (sub.buffered > 0) sub.buffered--;
+      },
+      close: () => this.remove(sub),
+    };
+  }
+
+  private remove(sub: Subscription): void {
+    if (sub.closed) return;
+    sub.closed = true;
+    this.subs.delete(sub);
+    const n = this.perVaultCount.get(sub.vaultName) ?? 0;
+    if (n <= 1) this.perVaultCount.delete(sub.vaultName);
+    else this.perVaultCount.set(sub.vaultName, n - 1);
+    try {
+      sub.sink.close();
+    } catch {
+      /* sink may already be torn down */
+    }
+  }
+
+  /**
+   * Send a keepalive comment to every open subscription (the route's timer
+   * calls this). A `:` comment defeats idle-proxy timeouts; it's not an event
+   * so it never counts against the buffer bound.
+   */
+  keepaliveAll(): void {
+    for (const sub of this.subs) {
+      if (sub.closed) continue;
+      const ok = sub.sink.write(":\n\n");
+      if (!ok) this.remove(sub);
+    }
+  }
+
+  /** Emit a frame to one subscription, enforcing the buffer bound. */
+  private emit(sub: Subscription, frame: SseFrame): void {
+    if (sub.closed) return;
+    if (sub.buffered >= sub.maxBuffered) {
+      // Backpressure: client can't keep up. Close → it reconnects + re-snapshots.
+      this.remove(sub);
+      return;
+    }
+    const ok = sub.sink.write(frame);
+    if (!ok) {
+      this.remove(sub);
+      return;
+    }
+    sub.buffered++;
+  }
+
+  /** Lazily register the three broad hooks (once per manager). */
+  private ensureHooks(): void {
+    if (this.hooksRegistered) return;
+    this.hooksRegistered = true;
+    const onNoteEvent = (event: HookEvent) => (payload: NoteHookPayload, store: Store) => {
+      this.dispatch(event, payload, store);
+    };
+    // NO `when` — a subscription must see all events to detect set-exit.
+    this.unregisters.push(
+      this.registry.onNote({ name: "live-subscribe:created", event: "created", handler: onNoteEvent("created") }),
+      this.registry.onNote({ name: "live-subscribe:updated", event: "updated", handler: onNoteEvent("updated") }),
+      this.registry.onNote({ name: "live-subscribe:deleted", event: "deleted", handler: onNoteEvent("deleted") }),
+    );
+  }
+
+  /**
+   * Core fan-out. `payload` is the full `Note` for created/updated (re-read
+   * fresh by the hook runner) or a `DeletedNoteRef` for deleted.
+   */
+  private dispatch(event: HookEvent, payload: NoteHookPayload, store: Store): void {
+    const vaultName = this.resolveVault(store);
+    if (!vaultName) return; // store not tracked → can't scope; drop safely
+    if (this.subs.size === 0) return;
+
+    for (const sub of this.subs) {
+      if (sub.closed) continue;
+      if (sub.vaultName !== vaultName) continue;
+
+      if (event === "deleted") {
+        // Thin ref ({id, path?}) — un-scope-matchable. Broadcast remove{id};
+        // the client ignores ids it never held. Documented low-sensitivity
+        // existence leak (see design §scope-intersection).
+        const ref = payload as DeletedNoteRef;
+        this.emit(sub, sseEvent("remove", { id: ref.id }));
+        continue;
+      }
+
+      // created / updated: full Note in hand. Compute scope ONCE and gate
+      // BOTH the upsert and the left-the-set remove on it — emitting a
+      // `remove{id}` for an out-of-scope note would leak its UUID to a token
+      // that could never have held it (M2).
+      const note = payload as Note;
+      const inScope = noteWithinTagScope(note, sub.tagScopeAllowed, sub.tagScopeRaw);
+      const matches = sub.matcher.match(note) && inScope;
+
+      if (matches) {
+        this.emit(sub, sseEvent("upsert", { note }));
+      } else if (event === "updated" && inScope) {
+        // Left the set (predicate no longer true) BUT still within this
+        // token's scope, so the sub could have held it — idempotent remove
+        // (client drops the id if held, no-op otherwise). When the note is
+        // OUT of scope the sub never had it; emitting would leak its id, so
+        // we stay silent.
+        this.emit(sub, sseEvent("remove", { id: note.id }));
+      }
+      // created that doesn't match → nothing (it was never in the set).
+    }
+  }
+
+  /** Tear down all hooks + close all streams. For shutdown / tests. */
+  shutdown(): void {
+    for (const u of this.unregisters) u();
+    this.unregisters = [];
+    this.hooksRegistered = false;
+    for (const sub of Array.from(this.subs)) this.remove(sub);
+  }
+}
+
+export interface SubscriptionHandle {
+  /** Decrement the pending-frame counter when a frame flushes to the wire. */
+  flushed: () => void;
+  /** Unregister + close. Called on stream cancel/close. Idempotent. */
+  close: () => void;
+}
+
+/** Serialize a snapshot frame (exported for the route + tests). */
+export function snapshotFrame(notes: Note[]): SseFrame {
+  return sseEvent("snapshot", { notes });
+}
+
+/** Process-wide manager — shared like `defaultHookRegistry`. */
+export const subscriptionManager = new SubscriptionManager();

package/src/tag-expand-routes.test.ts

@@ -0,0 +1,45 @@
+/**
+ * REST `?expand=` parsing for `parseNotesQueryOpts` — vault tag `expand` axis
+ * (design 2026-06-09). Validates the four-value enum, the 400 on a bad value,
+ * and that an ABSENT param leaves `expand` undefined (so the store resolves it
+ * to "subtypes" → byte-identical to pre-axis behavior).
+ */
+
+import { describe, it, expect } from "bun:test";
+import { parseNotesQueryOpts } from "./routes.ts";
+
+function parse(qs: string) {
+  return parseNotesQueryOpts(new URL(`http://localhost/api/notes?${qs}`));
+}
+
+describe("parseNotesQueryOpts — expand axis", () => {
+  it("absent → queryOpts.expand is undefined (default = subtypes at the store)", () => {
+    const r = parse("tag=entity");
+    expect(r.error).toBeUndefined();
+    expect(r.queryOpts!.expand).toBeUndefined();
+  });
+
+  for (const mode of ["subtypes", "namespace", "both", "exact"] as const) {
+    it(`expand=${mode} parses through`, () => {
+      const r = parse(`tag=entity&expand=${mode}`);
+      expect(r.error).toBeUndefined();
+      expect(r.queryOpts!.expand).toBe(mode);
+    });
+  }
+
+  it("empty expand= is treated as absent (undefined)", () => {
+    const r = parse("tag=entity&expand=");
+    expect(r.error).toBeUndefined();
+    expect(r.queryOpts!.expand).toBeUndefined();
+  });
+
+  it("unknown expand value → 400 with INVALID_QUERY", async () => {
+    const r = parse("tag=entity&expand=bogus");
+    expect(r.queryOpts).toBeUndefined();
+    expect(r.error).toBeDefined();
+    expect(r.error!.status).toBe(400);
+    const body = await r.error!.json();
+    expect(body.code).toBe("INVALID_QUERY");
+    expect(body.error).toContain("expand");
+  });
+});