@arcote.tech/arc-host 0.7.14 → 0.7.16

package/src/middleware/live-query.integration.test.ts

@@ -0,0 +1,432 @@
+import { beforeEach, describe, expect, it } from "bun:test";
+import {
+  MasterDataStorage,
+  aggregate,
+  applyQueryChanges,
+  boolean,
+  context,
+  id,
+  string,
+  token,
+} from "@arcote.tech/arc";
+import type { ConnectedClient } from "../types";
+import { cleanupClientSubs, querySubscriptionHandler } from "./ws";
+
+// ---------------------------------------------------------------------------
+// Test rig: real MasterDataStorage + a real aggregate with custom
+// clientQuery handlers (the NDT-shaped case: handler runs $query.find with
+// protectBy restrictions) + the real querySubscriptionHandler. The fake
+// connection manager collects outgoing messages.
+//
+// Covers the full server pipeline: subscribe-query → LiveQuery tracks the
+// handler's inner finds → store commit → resolveQueryChange → re-execute
+// from cache → diff → query-changes / query-snapshot.
+// ---------------------------------------------------------------------------
+
+/** Persistent in-memory DatabaseAdapter (shared tables across transactions). */
+function memoryAdapter() {
+  const tables = new Map<string, Map<string, any>>();
+  const table = (s: string) => {
+    if (!tables.has(s)) tables.set(s, new Map());
+    return tables.get(s)!;
+  };
+  const matches = (row: any, where?: Record<string, any>) =>
+    !where || Object.entries(where).every(([k, v]) => row[k] === v);
+  const tx = {
+    async find(store: string, options: any) {
+      let rows = Array.from(table(store).values()).filter((r) =>
+        matches(r, options?.where),
+      );
+      if (options?.orderBy) {
+        const entries = Object.entries(options.orderBy) as [string, string][];
+        rows = [...rows].sort((a, b) => {
+          for (const [k, dir] of entries) {
+            if (a[k] < b[k]) return dir === "asc" ? -1 : 1;
+            if (a[k] > b[k]) return dir === "asc" ? 1 : -1;
+          }
+          return 0;
+        });
+      }
+      if (options?.limit !== undefined) rows = rows.slice(0, options.limit);
+      return rows;
+    },
+    async set(store: string, data: any) {
+      table(store).set(data._id, structuredClone(data));
+    },
+    async remove(store: string, id: any) {
+      table(store).delete(String(id));
+    },
+    async commit() {},
+  };
+  return {
+    readWriteTransaction: () => tx,
+    readTransaction: () => tx,
+    executeReinitTables: async () => {},
+  } as any;
+}
+
+const wsToken = token("workspace", {});
+const taskId = id("task");
+
+const tasks = aggregate("tasks", taskId, {
+  title: string(),
+  workspaceId: string(),
+  done: boolean(),
+})
+  .clientQuery("getAll", (fn: any) =>
+    fn.handle(async (ctx: any) =>
+      ctx.$query.find({ orderBy: { title: "asc" } }),
+    ),
+  )
+  .clientQuery("getDone", (fn: any) =>
+    fn.handle(async (ctx: any) =>
+      ctx.$query.find({ where: { done: true }, orderBy: { title: "asc" } }),
+    ),
+  )
+  .clientQuery("getTop2", (fn: any) =>
+    fn.handle(async (ctx: any) =>
+      ctx.$query.find({ orderBy: { title: "asc" }, limit: 2 }),
+    ),
+  )
+  .clientQuery("countAll", (fn: any) =>
+    fn.handle(async (ctx: any) => {
+      const rows = await ctx.$query.find({});
+      return { count: rows.length };
+    }),
+  )
+  .protectBy(wsToken as any, (p: any) => ({ workspaceId: p.workspaceId }));
+
+function fakeJwt(tokenName: string, params: Record<string, any>): string {
+  const b64 = (o: any) => Buffer.from(JSON.stringify(o)).toString("base64");
+  return `${b64({ alg: "none" })}.${b64({ tokenName, params })}.sig`;
+}
+
+function fakeClient(
+  clientId: string,
+  scope: string,
+  params: Record<string, any>,
+): ConnectedClient {
+  return {
+    id: clientId,
+    scopeTokens: new Map([
+      [
+        scope,
+        {
+          decoded: { tokenType: "workspace", params },
+          raw: fakeJwt("workspace", params),
+        },
+      ],
+    ]),
+    lastHostEventId: null,
+    ws: null,
+  };
+}
+
+/** Microtask + async re-execute settle. */
+const settle = () => new Promise((r) => setTimeout(r, 10));
+
+function rig() {
+  const storage = new MasterDataStorage(memoryAdapter());
+  const testContext = context([tasks as any]);
+  const fakeModel = {
+    context: testContext,
+    getAdapters: () => ({ dataStorage: storage }),
+    getEnvironment: () => "server" as const,
+  };
+
+  const sent: Array<{ clientId: string; message: any }> = [];
+  const wsCtx = {
+    contextHandler: { getModel: () => fakeModel },
+    connectionManager: {
+      sendToClient: (clientId: string, message: any) => {
+        sent.push({ clientId, message });
+        return true;
+      },
+    },
+    verifyToken: () => null,
+  } as any;
+
+  const handler = querySubscriptionHandler();
+
+  const subscribe = (
+    client: ConnectedClient,
+    subscriptionId: string,
+    method: string,
+    scope = "workspace",
+  ) =>
+    handler(
+      client,
+      {
+        type: "subscribe-query",
+        subscriptionId,
+        descriptor: { element: "tasks", method, args: [] },
+        scope,
+      },
+      wsCtx,
+    );
+
+  const seed = (rows: any[]) =>
+    storage
+      .getStore<any>("tasks")
+      .applyChanges(rows.map((data) => ({ type: "set" as const, data })));
+
+  const messagesFor = (clientId: string, type?: string) =>
+    sent
+      .filter((s) => s.clientId === clientId)
+      .map((s) => s.message)
+      .filter((m) => (type ? m.type === type : true));
+
+  return { storage, wsCtx, handler, subscribe, seed, sent, messagesFor };
+}
+
+const row = (id: string, title: string, workspaceId: string, done = false) => ({
+  _id: id,
+  title,
+  workspaceId,
+  done,
+});
+
+let seq = 0;
+const uid = (p: string) => `${p}_${++seq}`;
+
+describe("live query subscriptions — end-to-end host pipeline", () => {
+  beforeEach(() => {
+    for (let i = 0; i <= seq; i++) {
+      cleanupClientSubs(`A_${i}`);
+      cleanupClientSubs(`B_${i}`);
+    }
+  });
+
+  it("snapshot contains the QUERY result for the subscriber's scope only", async () => {
+    const r = rig();
+    await r.seed([
+      row("t1", "Beta", "w1"),
+      row("t2", "Alpha", "w1"),
+      row("t3", "Other", "w2"),
+    ]);
+
+    const a = fakeClient(uid("A"), "workspace", { workspaceId: "w1" });
+    await r.subscribe(a, "s1", "getAll");
+    await settle();
+
+    const snapshots = r.messagesFor(a.id, "query-snapshot");
+    expect(snapshots).toHaveLength(1);
+    expect(snapshots[0].subscriptionId).toBe("s1");
+    // orderBy from the HANDLER (title asc), restrictions from protectBy
+    expect(snapshots[0].result.map((t: any) => t._id)).toEqual(["t2", "t1"]);
+  });
+
+  it("matching insert → query-changes with the correct position", async () => {
+    const r = rig();
+    await r.seed([row("t1", "Beta", "w1"), row("t2", "Delta", "w1")]);
+    const a = fakeClient(uid("A"), "workspace", { workspaceId: "w1" });
+    await r.subscribe(a, "s1", "getAll");
+    await settle();
+    r.sent.length = 0;
+
+    await r.storage
+      .getStore<any>("tasks")
+      .applyChanges([{ type: "set", data: row("t3", "Charlie", "w1") }]);
+    await settle();
+
+    const changes = r.messagesFor(a.id, "query-changes");
+    expect(changes).toHaveLength(1);
+    expect(changes[0].changes).toEqual([
+      {
+        type: "set",
+        id: "t3",
+        item: row("t3", "Charlie", "w1"),
+        index: 1, // Beta, Charlie, Delta
+      },
+    ]);
+  });
+
+  it("tenant isolation: a change in another workspace produces NO message", async () => {
+    const r = rig();
+    await r.seed([row("t1", "Beta", "w1")]);
+    const a = fakeClient(uid("A"), "workspace", { workspaceId: "w1" });
+    await r.subscribe(a, "s1", "getAll");
+    await settle();
+    r.sent.length = 0;
+
+    await r.storage
+      .getStore<any>("tasks")
+      .applyChanges([{ type: "set", data: row("t9", "Foreign", "w2") }]);
+    await settle();
+
+    expect(r.messagesFor(a.id)).toHaveLength(0);
+  });
+
+  it("item leaving the scope → delete for the old tenant, set for the new one", async () => {
+    const r = rig();
+    await r.seed([row("t1", "Task", "w1"), row("t2", "Keep", "w1")]);
+    const a = fakeClient(uid("A"), "workspace", { workspaceId: "w1" });
+    const b = fakeClient(uid("B"), "workspace", { workspaceId: "w2" });
+    await r.subscribe(a, "sa", "getAll");
+    await r.subscribe(b, "sb", "getAll");
+    await settle();
+    r.sent.length = 0;
+
+    // Move t1 from w1 to w2 (modify → full-item set event)
+    await r.storage
+      .getStore<any>("tasks")
+      .applyChanges([{ type: "modify", id: "t1", data: { workspaceId: "w2" } }]);
+    await settle();
+
+    const toA = r.messagesFor(a.id, "query-changes");
+    expect(toA).toHaveLength(1);
+    expect(toA[0].changes).toEqual([{ type: "delete", id: "t1" }]);
+
+    const toB = r.messagesFor(b.id, "query-changes");
+    expect(toB).toHaveLength(1);
+    expect(toB[0].changes).toEqual([
+      { type: "set", id: "t1", item: row("t1", "Task", "w2"), index: 0 },
+    ]);
+  });
+
+  it("where inside the HANDLER: toggling done moves items in/out of the result", async () => {
+    const r = rig();
+    await r.seed([row("t1", "A", "w1", true), row("t2", "B", "w1", false)]);
+    const a = fakeClient(uid("A"), "workspace", { workspaceId: "w1" });
+    await r.subscribe(a, "s1", "getDone");
+    await settle();
+
+    expect(r.messagesFor(a.id, "query-snapshot")[0].result).toEqual([
+      row("t1", "A", "w1", true),
+    ]);
+    r.sent.length = 0;
+
+    // t2 becomes done → enters the result
+    await r.storage
+      .getStore<any>("tasks")
+      .applyChanges([{ type: "modify", id: "t2", data: { done: true } }]);
+    await settle();
+    expect(r.messagesFor(a.id, "query-changes")[0].changes).toEqual([
+      { type: "set", id: "t2", item: row("t2", "B", "w1", true), index: 1 },
+    ]);
+    r.sent.length = 0;
+
+    // t1 becomes undone → leaves the result
+    await r.storage
+      .getStore<any>("tasks")
+      .applyChanges([{ type: "modify", id: "t1", data: { done: false } }]);
+    await settle();
+    expect(r.messagesFor(a.id, "query-changes")[0].changes).toEqual([
+      { type: "delete", id: "t1" },
+    ]);
+  });
+
+  it("limit underflow: removing from a full top-N refills from the store", async () => {
+    const r = rig();
+    await r.seed([
+      row("t1", "A", "w1"),
+      row("t2", "B", "w1"),
+      row("t3", "C", "w1"),
+    ]);
+    const a = fakeClient(uid("A"), "workspace", { workspaceId: "w1" });
+    await r.subscribe(a, "s1", "getTop2");
+    await settle();
+
+    expect(
+      r.messagesFor(a.id, "query-snapshot")[0].result.map((t: any) => t._id),
+    ).toEqual(["t1", "t2"]);
+    r.sent.length = 0;
+
+    // Remove t1 from the full top-2 → t3 must be pulled in from the store.
+    // The refill may arrive as a snapshot or as deltas — verify the FINAL
+    // result the client would hold either way.
+    await r.storage
+      .getStore<any>("tasks")
+      .applyChanges([{ type: "delete", id: "t1" }]);
+    await settle();
+
+    const all = r.messagesFor(a.id);
+    expect(all).toHaveLength(1);
+    const result =
+      all[0].type === "query-snapshot"
+        ? all[0].result
+        : applyQueryChanges(
+            [row("t1", "A", "w1"), row("t2", "B", "w1")],
+            all[0].changes,
+          );
+    expect(result.map((t: any) => t._id)).toEqual(["t2", "t3"]);
+  });
+
+  it("non-list result (count) → full snapshot on every change", async () => {
+    const r = rig();
+    await r.seed([row("t1", "A", "w1")]);
+    const a = fakeClient(uid("A"), "workspace", { workspaceId: "w1" });
+    await r.subscribe(a, "s1", "countAll");
+    await settle();
+
+    expect(r.messagesFor(a.id, "query-snapshot")[0].result).toEqual({
+      count: 1,
+    });
+    r.sent.length = 0;
+
+    await r.storage
+      .getStore<any>("tasks")
+      .applyChanges([{ type: "set", data: row("t2", "B", "w1") }]);
+    await settle();
+
+    const snapshots = r.messagesFor(a.id, "query-snapshot");
+    expect(snapshots).toHaveLength(1);
+    expect(snapshots[0].result).toEqual({ count: 2 });
+    expect(r.messagesFor(a.id, "query-changes")).toHaveLength(0);
+  });
+
+  it("unsubscribe-query stops deltas", async () => {
+    const r = rig();
+    await r.seed([row("t1", "A", "w1")]);
+    const a = fakeClient(uid("A"), "workspace", { workspaceId: "w1" });
+    await r.subscribe(a, "s1", "getAll");
+    await settle();
+    await r.handler(
+      a,
+      { type: "unsubscribe-query", subscriptionId: "s1" },
+      r.wsCtx,
+    );
+    r.sent.length = 0;
+
+    await r.storage
+      .getStore<any>("tasks")
+      .applyChanges([{ type: "set", data: row("t2", "B", "w1") }]);
+    await settle();
+    expect(r.messagesFor(a.id)).toHaveLength(0);
+  });
+
+  it("disconnect cleanup stops deltas", async () => {
+    const r = rig();
+    await r.seed([row("t1", "A", "w1")]);
+    const a = fakeClient(uid("A"), "workspace", { workspaceId: "w1" });
+    await r.subscribe(a, "s1", "getAll");
+    await settle();
+    cleanupClientSubs(a.id);
+    r.sent.length = 0;
+
+    await r.storage
+      .getStore<any>("tasks")
+      .applyChanges([{ type: "set", data: row("t2", "B", "w1") }]);
+    await settle();
+    expect(r.messagesFor(a.id)).toHaveLength(0);
+  });
+
+  it("re-subscribe with the same id (reconnect resend) replaces the old subscription", async () => {
+    const r = rig();
+    await r.seed([row("t1", "A", "w1")]);
+    const a = fakeClient(uid("A"), "workspace", { workspaceId: "w1" });
+    await r.subscribe(a, "s1", "getAll");
+    await settle();
+    await r.subscribe(a, "s1", "getAll"); // resend after reconnect
+    await settle();
+    r.sent.length = 0;
+
+    await r.storage
+      .getStore<any>("tasks")
+      .applyChanges([{ type: "set", data: row("t2", "B", "w1") }]);
+    await settle();
+
+    // Exactly ONE delta (old subscription was stopped, not duplicated)
+    expect(r.messagesFor(a.id, "query-changes")).toHaveLength(1);
+  });
+});

package/src/middleware/ws.ts

@@ -1,19 +1,23 @@
-import { ScopedModel } from "@arcote.tech/arc";
+import { LiveQuery } from "@arcote.tech/arc";
 import { filterEventsForTokens } from "../event-auth";
-import type { ConnectedClient } from "../types";
-import type { ArcWsContext, ArcWsHandler } from "./types";
+import type { ArcWsHandler } from "./types";
 
 // ---------------------------------------------------------------------------
-// View subscription tracking (per client)
+// Live query subscriptions
+//
+// One LiveQuery per (client, subscriptionId). ALL query logic — execution,
+// token restrictions, change detection, delta computation — lives in core's
+// LiveQuery (query layer); this module only forwards its updates over WS.
 // ---------------------------------------------------------------------------
 
-const clientViewSubs = new Map<string, Map<string, () => void>>();
+/** clientId → subscriptionId → LiveQuery */
+const clientQuerySubs = new Map<string, Map<string, LiveQuery>>();
 
 export function cleanupClientSubs(clientId: string): void {
-  const subs = clientViewSubs.get(clientId);
+  const subs = clientQuerySubs.get(clientId);
   if (subs) {
-    for (const unsub of subs.values()) unsub();
-    clientViewSubs.delete(clientId);
+    for (const live of subs.values()) live.stop();
+    clientQuerySubs.delete(clientId);
   }
 }
 
@@ -45,6 +49,10 @@ export function syncEventsHandler(): ArcWsHandler {
   return async (client, message, ctx) => {
     if (message.type !== "sync-events") return false;
 
+    // NOTE: sync-events alone does NOT mark the client as local-mode —
+    // streaming clients also push their optimistic mutations through it.
+    // Only request-sync (local-mode catch-up) sets `wantsEventSync`.
+
     const allTokens = ctx.connectionManager.getAllScopeTokens(client.id);
     const token = allTokens.length > 0 ? allTokens[0] : null;
 
@@ -55,9 +63,10 @@ export function syncEventsHandler(): ArcWsHandler {
     );
     if (persisted.length === 0) return true;
 
-    // Broadcast to other authorized clients
+    // Broadcast to other authorized LOCAL-MODE clients
     for (const c of ctx.connectionManager.getAllClients()) {
       if (c.id === client.id) continue;
+      if (!c.wantsEventSync) continue;
       const clientTokens = ctx.connectionManager.getAllScopeTokens(c.id);
       const authorized = filterEventsForTokens(
         clientTokens,
@@ -90,6 +99,10 @@ export function requestSyncHandler(): ArcWsHandler {
   return async (client, message, ctx) => {
     if (message.type !== "request-sync") return false;
 
+    // Only local-mode clients request the event log (streaming clients use
+    // view replicas) — mark so domain-event broadcasts reach this client.
+    client.wantsEventSync = true;
+
     const allTokens = ctx.connectionManager.getAllScopeTokens(client.id);
     const token = allTokens.length > 0 ? allTokens[0] : null;
     const events = await ctx.contextHandler.getEventsSince(
@@ -155,95 +168,85 @@ export function executeCommandHandler(): ArcWsHandler {
 }
 
 // ---------------------------------------------------------------------------
-// subscribe-view / unsubscribe-view — with scope-aware token selection
+// subscribe-query / unsubscribe-query — live query subscriptions
 // ---------------------------------------------------------------------------
 
 export function querySubscriptionHandler(): ArcWsHandler {
   return async (client, message, ctx) => {
     if (message.type === "subscribe-query") {
-      const { subscriptionId, descriptor, scope } = message;
+      const { subscriptionId, descriptor } = message;
+      const scope: string = message.scope ?? "default";
 
       // Pick the token matching the requested scope
-      const scopeToken = scope ? client.scopeTokens.get(scope) : null;
+      const scopeToken = client.scopeTokens.get(scope) ?? null;
       let rawToken = scopeToken?.raw ?? null;
 
-      // Fallback to first available token if no scope specified
+      // Fallback to first available token if no scope match
       if (!rawToken && client.scopeTokens.size > 0) {
         rawToken = client.scopeTokens.values().next().value!.raw;
       }
 
-      // Per-request scoped model with the right token
-      const scoped = new ScopedModel(ctx.contextHandler.getModel(), scope ?? "default");
-      if (rawToken) scoped.setToken(rawToken);
-
-      // Cache last result to avoid re-sending unchanged data
-      let lastResultJson = "";
-
-      const sendData = async () => {
-        try {
-          const data = await scoped.callQuery(descriptor);
-          const json = JSON.stringify(data ?? null);
-          if (json === lastResultJson) return; // No change — skip
-          lastResultJson = json;
-          ctx.connectionManager.sendToClient(client.id, {
-            type: "query-data",
-            subscriptionId,
-            data: data ?? null,
-          } as any);
-        } catch (err) {
-          console.error(`[Arc] Query subscription error:`, err);
-        }
-      };
-
-      // Initial data
-      sendData();
-
-      // Subscribe to SPECIFIC event types this element handles (namespaced keys)
-      const element = ctx.contextHandler.getModel().context.get(descriptor.element);
-      const handlers = (element as any)?.getHandlers?.();
-      const eventTypes: string[] = handlers ? Object.keys(handlers) : [];
-
-      let debounceTimer: ReturnType<typeof setTimeout> | undefined;
-      const debouncedSend = () => {
-        if (debounceTimer) clearTimeout(debounceTimer);
-        debounceTimer = setTimeout(() => sendData(), 50);
-      };
-
-      const unsubscribes: (() => void)[] = [];
-      if (eventTypes.length > 0) {
-        for (const eventType of eventTypes) {
-          unsubscribes.push(
-            ctx.contextHandler.getEventPublisher().subscribe(eventType, debouncedSend),
-          );
-        }
-      } else {
-        // Fallback to wildcard if no event types declared
-        unsubscribes.push(
-          ctx.contextHandler.getEventPublisher().subscribe("*", debouncedSend),
-        );
-      }
+      // Re-subscribe with the same id (reconnect resend) — drop the old one
+      clientQuerySubs.get(client.id)?.get(subscriptionId)?.stop();
 
-      const cleanup = () => {
-        if (debounceTimer) clearTimeout(debounceTimer);
-        for (const unsub of unsubscribes) unsub();
-      };
+      const live = new LiveQuery(
+        ctx.contextHandler.getModel(),
+        descriptor,
+        scope,
+        rawToken,
+        (update) => {
+          if (update.type === "changes") {
+            ctx.connectionManager.sendToClient(client.id, {
+              type: "query-changes",
+              subscriptionId,
+              changes: update.changes,
+            });
+          } else {
+            ctx.connectionManager.sendToClient(client.id, {
+              type: "query-snapshot",
+              subscriptionId,
+              result: update.result ?? null,
+            });
+          }
+        },
+      );
 
-      // Track for cleanup on disconnect
-      if (!clientViewSubs.has(client.id)) {
-        clientViewSubs.set(client.id, new Map());
+      if (!clientQuerySubs.has(client.id)) {
+        clientQuerySubs.set(client.id, new Map());
+      }
+      clientQuerySubs.get(client.id)!.set(subscriptionId, live);
+
+      try {
+        const result = await live.start();
+        ctx.connectionManager.sendToClient(client.id, {
+          type: "query-snapshot",
+          subscriptionId,
+          result: result ?? null,
+        });
+        // Catch-up pass AFTER the snapshot went out — diffs out any commit
+        // that slipped into the initial-execute window.
+        live.flush();
+      } catch (err) {
+        live.stop();
+        clientQuerySubs.get(client.id)?.delete(subscriptionId);
+        console.error(
+          `[Arc] Query subscription error (${descriptor?.element}.${descriptor?.method}):`,
+          err,
+        );
+        ctx.connectionManager.sendToClient(client.id, {
+          type: "error",
+          message: `Failed to subscribe to query "${descriptor?.element}.${descriptor?.method}"`,
+        });
       }
-      clientViewSubs.get(client.id)!.set(subscriptionId, cleanup);
       return true;
     }
 
     if (message.type === "unsubscribe-query") {
-      const subs = clientViewSubs.get(client.id);
-      if (subs) {
-        const unsub = subs.get(message.subscriptionId);
-        if (unsub) {
-          unsub();
-          subs.delete(message.subscriptionId);
-        }
+      const subs = clientQuerySubs.get(client.id);
+      const live = subs?.get(message.subscriptionId);
+      if (live) {
+        live.stop();
+        subs!.delete(message.subscriptionId);
       }
       return true;
     }

package/src/types.ts

@@ -42,6 +42,12 @@ export interface ConnectedClient {
   lastHostEventId: string | null;
   /** WebSocket instance */
   ws: any;
+  /**
+   * True for local-mode clients (they sent request-sync / sync-events).
+   * Only these receive domain-event broadcasts — streaming clients are
+   * served by view snapshots + deltas instead.
+   */
+  wantsEventSync?: boolean;
 }
 
 /**
@@ -137,7 +143,15 @@ export type HostToClientMessage =
       message: string;
     }
   | {
-      type: "query-data";
+      type: "query-snapshot";
       subscriptionId: string;
-      data: any[];
+      result: any;
+    }
+  | {
+      type: "query-changes";
+      subscriptionId: string;
+      changes: Array<
+        | { type: "set"; id: string; item: any; index: number }
+        | { type: "delete"; id: string }
+      >;
     };