@absolutejs/sync 1.5.0 → 1.7.0

package/dist/engine/index.js

@@ -569,6 +569,7 @@ var createPollingChangeSource = (options) => {
   const onError = options.onError ?? ((error) => {
     console.warn("[sync] polling change source error:", error);
   });
+  const onSkip = options.onSkip;
   let cursor = options.startSeq ?? 0;
   let running = false;
   let timer;
@@ -580,7 +581,9 @@ var createPollingChangeSource = (options) => {
       const rows = await options.poll(cursor);
       for (const row of rows) {
         const parsed = parse(row);
-        if (parsed !== undefined) {
+        if (parsed === undefined) {
+          onSkip?.(row, "parse-failed");
+        } else {
           await emit(parsed.table, parsed.change);
         }
         if (typeof row.seq === "number" && row.seq > cursor) {
@@ -1110,6 +1113,7 @@ var wrap = (source) => `
 var compile = async (source, config) => {
   const { createIsolate } = await loadIsolatedJsc();
   const isolate = await createIsolate({
+    backend: config.backend ?? "worker",
     memoryLimit: config.memoryLimit ?? 32
   });
   const script = await isolate.compileScript(wrap(source));
@@ -1161,6 +1165,28 @@ class SchemaError extends Error {
     this.name = "SchemaError";
   }
 }
+
+class MissedChangesError extends Error {
+  requestedSince;
+  availableSince;
+  constructor(requestedSince, availableSince) {
+    super(`Change log no longer covers version ${requestedSince}; oldest available is ${availableSince}. ` + `Re-bootstrap and resume from ${availableSince}.`);
+    this.name = "MissedChangesError";
+    this.requestedSince = requestedSince;
+    this.availableSince = availableSince;
+  }
+}
+
+class CdcConsumerSlowError extends Error {
+  maxBuffer;
+  lastDeliveredVersion;
+  constructor(maxBuffer, lastDeliveredVersion) {
+    super(`CDC stream buffer overflowed (max ${maxBuffer}); consumer fell behind. ` + `Last delivered version: ${lastDeliveredVersion}. Resubscribe with since=${lastDeliveredVersion}.`);
+    this.name = "CdcConsumerSlowError";
+    this.maxBuffer = maxBuffer;
+    this.lastDeliveredVersion = lastDeliveredVersion;
+  }
+}
 var defaultKey = (row) => row.id;
 var shallowEqual4 = (a, b) => {
   if (a === b) {
@@ -1306,6 +1332,7 @@ var createSyncEngine = (options = {}) => {
       listener(event);
     }
   };
+  const streamSubscribers = new Set;
   const runInTransaction = options.transaction;
   const instanceId = globalThis.crypto?.randomUUID?.() ?? `i${Math.random()}`;
   let clusterBus;
@@ -1677,6 +1704,9 @@ var createSyncEngine = (options = {}) => {
     if (changeLog.length > changeLogSize) {
       changeLog.shift();
     }
+    for (const subscriber of streamSubscribers) {
+      subscriber(entry);
+    }
   };
   const applyChange = async (table, change, shouldBroadcast = true) => {
     version += 1;
@@ -2276,9 +2306,170 @@ var createSyncEngine = (options = {}) => {
       return () => {
         activityListeners.delete(listener);
       };
+    },
+    streamChanges: ({
+      since = 0,
+      signal,
+      maxBuffer = 1e4
+    } = {}) => {
+      const oldest = changeLog[0];
+      if (since > 0 && oldest !== undefined && oldest.version > since + 1) {
+        const err = new MissedChangesError(since, oldest.version);
+        return {
+          [Symbol.asyncIterator]() {
+            return {
+              next: () => Promise.reject(err)
+            };
+          }
+        };
+      }
+      const buffer = [];
+      let waiter = null;
+      let overflow = false;
+      const wake = () => {
+        if (waiter !== null) {
+          const resume = waiter;
+          waiter = null;
+          resume();
+        }
+      };
+      const subscriber = (entry) => {
+        if (buffer.length >= maxBuffer) {
+          overflow = true;
+          wake();
+          return;
+        }
+        buffer.push(entry);
+        wake();
+      };
+      streamSubscribers.add(subscriber);
+      const onAbort = () => wake();
+      signal?.addEventListener("abort", onAbort, { once: true });
+      let lastDelivered = since;
+      return {
+        async* [Symbol.asyncIterator]() {
+          try {
+            const history = [...changeLog];
+            const headVersion = history.length > 0 ? history[history.length - 1].version : since;
+            for (const entry of history) {
+              if (signal?.aborted)
+                return;
+              if (entry.version > since) {
+                lastDelivered = entry.version;
+                yield entry;
+              }
+            }
+            while (!signal?.aborted) {
+              while (buffer.length > 0) {
+                const entry = buffer.shift();
+                if (entry.version > headVersion) {
+                  lastDelivered = entry.version;
+                  yield entry;
+                }
+              }
+              if (overflow) {
+                throw new CdcConsumerSlowError(maxBuffer, lastDelivered);
+              }
+              if (signal?.aborted)
+                return;
+              await new Promise((resolve) => {
+                waiter = resolve;
+              });
+            }
+          } finally {
+            streamSubscribers.delete(subscriber);
+            signal?.removeEventListener("abort", onAbort);
+          }
+        }
+      };
     }
   };
 };
+// src/engine/cdc.ts
+import { Elysia } from "elysia";
+var parseSince = (query2, lastEventId) => {
+  const raw = query2.since ?? lastEventId ?? "0";
+  const parsed = Number(raw);
+  return Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : 0;
+};
+var encodeEvent = (event, id, data) => {
+  const parts = [];
+  if (id !== null)
+    parts.push(`id: ${id}`);
+  parts.push(`event: ${event}`);
+  parts.push(`data: ${JSON.stringify(data)}`);
+  return `${parts.join(`
+`)}
+
+`;
+};
+var syncCdc = ({
+  engine,
+  path = "/sync/cdc",
+  heartbeatMs = 25000,
+  maxBuffer = 1e4
+}) => new Elysia({ name: "@absolutejs/sync/cdc" }).get(path, (context) => {
+  const lastEventId = context.request.headers.get("last-event-id");
+  const since = parseSince(context.query, lastEventId);
+  const encoder = new TextEncoder;
+  const stream = new ReadableStream({
+    async start(controller) {
+      const write = (chunk) => {
+        try {
+          controller.enqueue(encoder.encode(chunk));
+        } catch {}
+      };
+      write(encodeEvent("open", null, {
+        since,
+        at: Date.now()
+      }));
+      const heartbeat = setInterval(() => write(`: ping
+
+`), heartbeatMs);
+      try {
+        for await (const entry of engine.streamChanges({
+          since,
+          signal: context.request.signal,
+          maxBuffer
+        })) {
+          write(encodeEvent("change", entry.version, entry));
+        }
+      } catch (error) {
+        if (error instanceof MissedChangesError) {
+          write(encodeEvent("error", null, {
+            name: "MissedChangesError",
+            message: error.message,
+            requestedSince: error.requestedSince,
+            availableSince: error.availableSince
+          }));
+        } else if (error instanceof CdcConsumerSlowError) {
+          write(encodeEvent("error", null, {
+            name: "CdcConsumerSlowError",
+            message: error.message,
+            lastDeliveredVersion: error.lastDeliveredVersion
+          }));
+        } else {
+          write(encodeEvent("error", null, {
+            name: error instanceof Error ? error.name : "Error",
+            message: error instanceof Error ? error.message : String(error)
+          }));
+        }
+      } finally {
+        clearInterval(heartbeat);
+        try {
+          controller.close();
+        } catch {}
+      }
+    }
+  });
+  return new Response(stream, {
+    headers: {
+      "cache-control": "no-cache, no-transform",
+      connection: "keep-alive",
+      "content-type": "text/event-stream"
+    }
+  });
+});
 // src/engine/schema.ts
 var defineSchema = (schemas) => schemas;
 var isFiniteNumber = (value) => typeof value === "number" && Number.isFinite(value);
@@ -2527,6 +2718,7 @@ var createSyncConnection = ({
   return { handle, close };
 };
 export {
+  syncCdc,
   query,
   parseOutboxRow,
   orderByOp,
@@ -2565,8 +2757,10 @@ export {
   UnauthorizedError,
   SchemaError,
   SEARCH_SCORE_FIELD,
-  RetriesExhaustedError
+  RetriesExhaustedError,
+  MissedChangesError,
+  CdcConsumerSlowError
 };
 
-//# debugId=000A0D740DBD7CBA64756E2164756E21
+//# debugId=EE73D3886B04BDA964756E2164756E21
 //# sourceMappingURL=index.js.map