@defensestation/ysync-go 0.1.0-dev.12.d04ad84

package/dist/compaction/compaction.js

@@ -0,0 +1,153 @@
+// Compaction adapter for ysync-go servers.
+//
+// The server's update log grows until the host folds it (see the repository
+// README's Compaction section). This module implements the recommended
+// policy — once a document exceeds `threshold` log records, fold everything
+// except the newest `keep` records into one snapshot — using **real Yjs** to
+// build the snapshot. That matters: Go-side merges corrupt documents with
+// nested XML content (BlockNote, ProseMirror, Tiptap); see
+// adapters/yjs/ygo/README.md and its xmlrepro/ cross-implementation check.
+//
+// The functions are runtime-agnostic on purpose: call `compactDocuments`
+// from an AWS Lambda handler, a Cloud Run job, a Kubernetes CronJob (the
+// bundled `ysync-compact` CLI), or any Node 18+ process with `fetch`.
+//
+// Security: `CompactDocument` rewrites stored history, so gate it with
+// `Authz.CanCompact` on the server and give **only** this job's identity
+// that permission. Pass the credential via `headers`.
+import * as Y from "yjs";
+const DEFAULT_THRESHOLD = 200;
+const DEFAULT_KEEP = 50;
+/**
+ * Compacts one document if its update log exceeds the policy threshold:
+ * lists the stored log, replays all but the newest `keep` records into a
+ * fresh Y.Doc, encodes the state with `Y.encodeStateAsUpdate`, and calls
+ * `CompactDocument` for the covered sequence. Idempotent — re-running
+ * converges to the same folded log, and updates pushed concurrently are
+ * unaffected (they sort after `compactThroughSequence`).
+ */
+export async function compactDocument(options, docId) {
+    if (!docId) {
+        throw new Error("docId is required");
+    }
+    const threshold = options.threshold ?? DEFAULT_THRESHOLD;
+    const keep = options.keep ?? DEFAULT_KEEP;
+    if (threshold < 1 || keep < 0) {
+        throw new Error(`invalid policy: threshold ${threshold}, keep ${keep}`);
+    }
+    const listed = (await rpc(options, "ListDocumentUpdates", { docId }));
+    const updates = listed.updates ?? [];
+    const skip = {
+        docId,
+        compacted: false,
+        records: updates.length,
+    };
+    if (updates.length <= threshold) {
+        return skip;
+    }
+    // Records arrive in sequence order; the first may already be a snapshot
+    // from a prior run. Folding only that snapshot again would be a no-op.
+    const cutoff = updates.length - keep;
+    if (cutoff < 1 || (cutoff === 1 && updates[0]?.isSnapshot)) {
+        return skip;
+    }
+    const target = updates[cutoff - 1];
+    if (target?.sequence === undefined) {
+        throw new Error(`${docId}: log record ${cutoff - 1} has no sequence`);
+    }
+    const doc = new Y.Doc();
+    for (const record of updates.slice(0, cutoff)) {
+        if (record.yjsUpdate) {
+            Y.applyUpdate(doc, fromBase64(record.yjsUpdate));
+        }
+    }
+    const snapshot = Y.encodeStateAsUpdate(doc);
+    doc.destroy();
+    await rpc(options, "CompactDocument", {
+        docId,
+        compactUpdate: toBase64(snapshot),
+        compactThroughSequence: target.sequence,
+    });
+    return {
+        docId,
+        compacted: true,
+        records: updates.length,
+        foldedRecords: cutoff,
+        foldedThroughSequence: String(target.sequence),
+        snapshotBytes: snapshot.byteLength,
+    };
+}
+/**
+ * Compacts many documents sequentially (compaction is background work;
+ * predictable server load beats speed here). Per-document failures are
+ * collected in `errors` instead of aborting the run.
+ */
+export async function compactDocuments(options, docIds) {
+    const run = {
+        results: [],
+        errors: [],
+        compacted: 0,
+        skipped: 0,
+    };
+    for (const docId of docIds) {
+        try {
+            const result = await compactDocument(options, docId);
+            run.results.push(result);
+            if (result.compacted) {
+                run.compacted++;
+            }
+            else {
+                run.skipped++;
+            }
+        }
+        catch (error) {
+            run.errors.push({ docId, error });
+        }
+    }
+    return run;
+}
+/** Plain Connect JSON: POST /<service>/<method> with a JSON body. */
+async function rpc(options, method, body) {
+    const headers = typeof options.headers === "function"
+        ? await options.headers()
+        : (options.headers ?? {});
+    const doFetch = options.fetch ?? globalThis.fetch;
+    const base = options.baseUrl.replace(/\/+$/, "");
+    const response = await doFetch(`${base}/ysync.v1.YjsSyncService/${method}`, {
+        method: "POST",
+        headers: { "content-type": "application/json", ...headers },
+        body: JSON.stringify(body),
+    });
+    if (!response.ok) {
+        const detail = await response.text().catch(() => "");
+        throw new Error(`${method}: HTTP ${response.status}${detail ? `: ${detail}` : ""}`);
+    }
+    return response.json();
+}
+function nodeBuffer() {
+    return globalThis.Buffer;
+}
+function toBase64(bytes) {
+    const buffer = nodeBuffer();
+    if (buffer) {
+        return buffer.from(bytes).toString("base64");
+    }
+    let binary = "";
+    for (const byte of bytes) {
+        binary += String.fromCharCode(byte);
+    }
+    return btoa(binary);
+}
+function fromBase64(encoded) {
+    const buffer = nodeBuffer();
+    if (buffer) {
+        return Uint8Array.from(buffer.from(encoded, "base64"));
+    }
+    const binary = atob(encoded);
+    const out = new Uint8Array(binary.length);
+    for (let i = 0; i < binary.length; i++) {
+        out[i] = binary.charCodeAt(i);
+    }
+    return out;
+}
+//# sourceMappingURL=compaction.js.map

package/dist/compaction/compaction.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"compaction.js","sourceRoot":"","sources":["../../src/compaction/compaction.ts"],"names":[],"mappings":"AAAA,2CAA2C;AAC3C,EAAE;AACF,4EAA4E;AAC5E,uEAAuE;AACvE,4EAA4E;AAC5E,6EAA6E;AAC7E,0EAA0E;AAC1E,2DAA2D;AAC3D,2EAA2E;AAC3E,EAAE;AACF,yEAAyE;AACzE,yEAAyE;AACzE,sEAAsE;AACtE,EAAE;AACF,uEAAuE;AACvE,yEAAyE;AACzE,sDAAsD;AACtD,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAgEzB,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAC9B,MAAM,YAAY,GAAG,EAAE,CAAC;AAExB;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,OAA+B,EAC/B,KAAa;IAEb,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;IACvC,CAAC;IACD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,iBAAiB,CAAC;IACzD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,YAAY,CAAC;IAC1C,IAAI,SAAS,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,6BAA6B,SAAS,UAAU,IAAI,EAAE,CAAC,CAAC;IAC1E,CAAC;IAED,MAAM,MAAM,GAAG,CAAC,MAAM,GAAG,CAAC,OAAO,EAAE,qBAAqB,EAAE,EAAE,KAAK,EAAE,CAAC,CAEnE,CAAC;IACF,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;IACrC,MAAM,IAAI,GAA0B;QAClC,KAAK;QACL,SAAS,EAAE,KAAK;QAChB,OAAO,EAAE,OAAO,CAAC,MAAM;KACxB,CAAC;IACF,IAAI,OAAO,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;QAChC,OAAO,IAAI,CAAC;IACd,CAAC;IACD,wEAAwE;IACxE,uEAAuE;IACvE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IACrC,IAAI,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,EAAE,CAAC;QAC3D,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACnC,IAAI,MAAM,EAAE,QAAQ,KAAK,SAAS,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,gBAAgB,MAAM,GAAG,CAAC,kBAAkB,CAAC,CAAC;IACxE,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC;IACxB,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC;QAC9C,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACrB,CAAC,CAAC,WAAW,CAAC,GAAG,EAAE,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;QACnD,CAAC;IACH,CAAC;IACD,MAAM,QAAQ,GAAG,CAAC,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;IAC5C,GAAG,CAAC,OAAO,EAAE,CAAC;IAEd,MAAM,GAAG,CAAC,OAAO,EAAE,iBAAiB,EAAE;QACpC,KAAK;QACL,aAAa,EAAE,QAAQ,CAAC,QAAQ,CAAC;QACjC,sBAAsB,EAAE,MAAM,CAAC,QAAQ;KACxC,CAAC,CAAC;IACH,OAAO;QACL,KAAK;QACL,SAAS,EAAE,IAAI;QACf,OAAO,EAAE,OAAO,CAAC,MAAM;QACvB,aAAa,EAAE,MAAM;QACrB,qBAAqB,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC;QAC9C,aAAa,EAAE,QAAQ,CAAC,UAAU;KACnC,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,OAA+B,EAC/B,MAAwB;IAExB,MAAM,GAAG,GAAwB;QAC/B,OAAO,EAAE,EAAE;QACX,MAAM,EAAE,EAAE;QACV,SAAS,EAAE,CAAC;QACZ,OAAO,EAAE,CAAC;KACX,CAAC;IACF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YACrD,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACzB,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;gBACrB,GAAG,CAAC,SAAS,EAAE,CAAC;YAClB,CAAC;iBAAM,CAAC;gBACN,GAAG,CAAC,OAAO,EAAE,CAAC;YAChB,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,qEAAqE;AACrE,KAAK,UAAU,GAAG,CAChB,OAAgC,EAChC,MAAc,EACd,IAAa;IAEb,MAAM,OAAO,GACX,OAAO,OAAO,CAAC,OAAO,KAAK,UAAU;QACnC,CAAC,CAAC,MAAM,OAAO,CAAC,OAAO,EAAE;QACzB,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;IAC9B,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,IAAI,UAAU,CAAC,KAAK,CAAC;IAClD,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACjD,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,IAAI,4BAA4B,MAAM,EAAE,EAAE;QAC1E,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,GAAG,OAAO,EAAE;QAC3D,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;KAC3B,CAAC,CAAC;IACH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACrD,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,UAAU,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACtF,CAAC;IACD,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;AACzB,CAAC;AAUD,SAAS,UAAU;IACjB,OAAQ,UAAsC,CAAC,MAAM,CAAC;AACxD,CAAC;AAED,SAAS,QAAQ,CAAC,KAAiB;IACjC,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;IAC5B,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC/C,CAAC;IACD,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC;AACtB,CAAC;AAED,SAAS,UAAU,CAAC,OAAe;IACjC,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;IAC5B,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;IACzD,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC;IAC7B,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,GAAG,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}

package/dist/compaction/index.d.ts

@@ -0,0 +1,3 @@
+export { compactDocument, compactDocuments } from "./compaction.js";
+export type { CompactionClientOptions, CompactionPolicy, CompactDocumentOptions, CompactDocumentResult, CompactionRunResult, } from "./compaction.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/compaction/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/compaction/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACpE,YAAY,EACV,uBAAuB,EACvB,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,mBAAmB,GACpB,MAAM,iBAAiB,CAAC"}

package/dist/compaction/index.js

@@ -0,0 +1,2 @@
+export { compactDocument, compactDocuments } from "./compaction.js";
+//# sourceMappingURL=index.js.map

package/dist/compaction/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/compaction/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC"}