@syncular/server 0.32.1 → 0.34.0

package/dist/blob-handlers.js

@@ -100,7 +100,6 @@ async function downloadBlob(ctx, blobId) {
     // the bytes. Inline delivery loads the bytes; presigned delivery does not.
     const presignConfigured = ctx.blobSignedUrls !== undefined;
     let entry;
-    let mediaType;
     if (presignConfigured) {
         if (!(await store.has(ctx.partition, blobId))) {
             throw syncError('blob.not_found', 'unknown blob (§5.9.5)');
@@ -111,7 +110,6 @@ async function downloadBlob(ctx, blobId) {
         if (entry === undefined) {
             throw syncError('blob.not_found', 'unknown blob (§5.9.5)');
         }
-        mediaType = entry.record.mediaType;
     }
     // §5.9.5 authorization: resolve once, then test referencing rows.
     const listRows = ctx.storage.listRowsReferencingBlob;
@@ -191,10 +189,12 @@ async function downloadBlob(ctx, blobId) {
         // Unreachable: presign not configured ⇒ `entry` was loaded above.
         throw syncError('blob.not_found', 'blob bytes unavailable (§5.9.5)');
     }
+    // §5.9.5: inline bytes are always octet-stream. The stored media type
+    // would make a JSON blob indistinguishable from the signed-URL envelope.
     return {
         bytes,
         headers: {
-            'Content-Type': mediaType ?? 'application/octet-stream',
+            'Content-Type': 'application/octet-stream',
             ETag: `"${blobId}"`,
             'Cache-Control': 'private, max-age=0',
             Vary: 'Authorization, Accept-Encoding',

package/dist/bucket-blob-store.d.ts

@@ -0,0 +1,77 @@
+/**
+ * Content-addressed blob store over the `ObjectBucket` seam (§5.9.2 durable
+ * objects) — the store logic shared by `S3BlobStore` (S3 REST + SigV4) and
+ * `R2BlobStore` (`@syncular/server-workers`, the R2 binding). The blob twin of
+ * `bucket-segment-store.ts`: same content-addressed key layout, same
+ * LIST-free-on-the-hot-path CAS stats accumulator.
+ *
+ * Key layout (deterministic — every point lookup is a GET/HEAD, never a LIST):
+ *
+ *   {keyPrefix}blob/{partition}/sha256/{hex}
+ *     The blob bytes, VERBATIM (the object body MUST be exactly the
+ *     content-addressed bytes so a presigned GET or the inline download route
+ *     serves them directly and the client's §5.9.1 hash check passes — blobs
+ *     are never re-compressed at rest, §5.8/§5.9.5). The `byteLength` +
+ *     optional `mediaType` ride along as user metadata `syncular-blob` =
+ *     base64url(JSON), so `get` is a single GET. `createdAtMs` is stored there
+ *     too so the orphan sweep (§5.9.2) can read the upload age without a
+ *     separate index.
+ *
+ * DURABILITY vs SEGMENT TTL — the honest interface difference (§5.9.2): a
+ * blob referenced by a live row must stay downloadable INDEFINITELY. So,
+ * unlike the segment store, this store writes **no `expiresAtMs`, no bucket
+ * lifecycle-expiration mapping, and no TTL config**. Expiry is
+ * REFERENCE-driven, not time-driven: reclamation is the host-scheduled
+ * `sweepOrphanBlobs` (blob-store.ts) deleting only objects no live row
+ * references after a grace period — never a bucket lifecycle rule. Do NOT
+ * put a lifecycle expiration on the `blob/` prefix; it would delete
+ * still-referenced attachments.
+ *
+ * The orphan sweep is the ONLY operation that LISTs: `sweepOrphans` pages a
+ * prefix listing over the `blob/` prefix (an admin/GC path, off the hot path)
+ * to find candidates, reads each object's `createdAtMs` from its metadata for
+ * the grace check, and DELETEs the unreferenced-and-old ones.
+ */
+import type { BlobRecord, BlobStore, BlobStoreStats } from './blob-store.js';
+import type { ObjectBucket } from './object-bucket.js';
+export interface BucketBlobStoreOptions {
+    /** Key namespace inside the bucket, e.g. `syncular/`. Default: none. */
+    readonly keyPrefix?: string | undefined;
+}
+export declare class BucketBlobStore implements BlobStore {
+    #private;
+    constructor(bucket: ObjectBucket, options?: BucketBlobStoreOptions);
+    /**
+     * `{keyPrefix}blob/{partition}/sha256/{hex}` — the §5.9.5 "signed key embeds
+     * the blobId". Blobs are content-addressed, but download authorization and
+     * the reference index are per-partition, so the same bytes uploaded under
+     * two partitions are two objects (a partition MUST NOT read another's
+     * attachment by guessing a content address). The partition is folded into
+     * the key as a path segment.
+     */
+    objectKeyFor(partition: string, blobId: string): string;
+    put(partition: string, blobId: string, bytes: Uint8Array, nowMs: number, mediaType?: string): Promise<BlobRecord>;
+    has(partition: string, blobId: string): Promise<boolean>;
+    get(partition: string, blobId: string): Promise<{
+        record: BlobRecord;
+        bytes: Uint8Array;
+    } | undefined>;
+    /**
+     * §5.9.2 orphan sweep — the ONLY LISTing operation. Page the `{prefix}blob/
+     * {partition}/` prefix, HEAD each object for its `createdAtMs`, and DELETE
+     * the ones both older than `olderThanMs` AND absent from
+     * `referencedBlobIds`. Returns the deleted blobIds. Never deletes a
+     * referenced blob (§5.9.2). Off the hot path — a host-scheduled GC job.
+     */
+    sweepOrphans(partition: string, olderThanMs: number, referencedBlobIds: ReadonlySet<string>): Promise<string[]>;
+    /**
+     * Store-wide counters from the pointer-object accumulator (never a LIST on
+     * the read path). `approximate: true` for the same reasons the segment
+     * store documents: best-effort ETag-CAS maintenance and sweep deletes that
+     * may miss the decrement. Exact within a single writer; a health gauge
+     * under concurrency. Note this is store-WIDE, not partition-scoped, so a
+     * shared bucket reports the whole bucket — the parameter is accepted for
+     * interface parity but the accumulator is one object per store.
+     */
+    stats(_partition: string): Promise<BlobStoreStats>;
+}

package/dist/bucket-blob-store.js

@@ -0,0 +1,266 @@
+const BLOB_ID_PATTERN = /^sha256:[0-9a-f]{64}$/;
+const BLOB_META_KEY = 'syncular-blob';
+/**
+ * Stats accumulator (mirrors the segment store): a small counter object under
+ * a fixed key, updated read-modify-write on `put` under an ETag
+ * compare-and-swap. A LIST-free point-lookup store cannot count objects on
+ * demand cheaply, so `stats()` reads the accumulator. The counters are
+ * APPROXIMATE — a crash between the blob PUT and the accumulator CAS, or a
+ * sweep DELETE the accumulator does not decrement, drifts them — so the
+ * admin surface labels these stats `approximate: true`. A health gauge,
+ * not an invoice.
+ */
+const STATS_KEY_SUFFIX = 'stats/blobs.json';
+const STATS_CAS_ATTEMPTS = 16;
+function emptyStats() {
+    return { count: 0, bytes: 0 };
+}
+function parseStatsJson(json) {
+    const parsed = JSON.parse(json);
+    return {
+        count: Number(parsed.count ?? 0),
+        bytes: Number(parsed.bytes ?? 0),
+    };
+}
+// Runtime-neutral base64url: `Buffer` is not present on
+// Cloudflare Workers without `nodejs_compat`, so the metadata header is
+// (de)coded with `btoa`/`atob`, available in every runtime.
+function utf8ToBase64url(text) {
+    let binary = '';
+    for (const byte of new TextEncoder().encode(text)) {
+        binary += String.fromCharCode(byte);
+    }
+    return btoa(binary)
+        .replaceAll('+', '-')
+        .replaceAll('/', '_')
+        .replace(/=+$/, '');
+}
+function base64urlToUtf8(value) {
+    const binary = atob(value.replaceAll('-', '+').replaceAll('_', '/'));
+    const bytes = new Uint8Array(binary.length);
+    for (let i = 0; i < binary.length; i++)
+        bytes[i] = binary.charCodeAt(i);
+    return new TextDecoder().decode(bytes);
+}
+function metaToJson(meta) {
+    return JSON.stringify(meta);
+}
+function parseMetaJson(json, source, label) {
+    let parsed;
+    try {
+        parsed = JSON.parse(json);
+    }
+    catch {
+        throw new Error(`${label}: corrupt blob metadata in ${source}`);
+    }
+    if (typeof parsed !== 'object' || parsed === null) {
+        throw new Error(`${label}: corrupt blob metadata in ${source}`);
+    }
+    const r = parsed;
+    if (typeof r.byteLength !== 'number' || typeof r.createdAtMs !== 'number') {
+        throw new Error(`${label}: bad blob metadata in ${source}`);
+    }
+    return {
+        byteLength: r.byteLength,
+        ...(typeof r.mediaType === 'string' ? { mediaType: r.mediaType } : {}),
+        createdAtMs: r.createdAtMs,
+    };
+}
+export class BucketBlobStore {
+    #bucket;
+    #prefix;
+    /** Error prefix: the concrete store class the host constructed. */
+    #label;
+    constructor(bucket, options) {
+        this.#bucket = bucket;
+        this.#prefix = options?.keyPrefix ?? '';
+        this.#label = this.constructor.name;
+    }
+    /**
+     * `{keyPrefix}blob/{partition}/sha256/{hex}` — the §5.9.5 "signed key embeds
+     * the blobId". Blobs are content-addressed, but download authorization and
+     * the reference index are per-partition, so the same bytes uploaded under
+     * two partitions are two objects (a partition MUST NOT read another's
+     * attachment by guessing a content address). The partition is folded into
+     * the key as a path segment.
+     */
+    objectKeyFor(partition, blobId) {
+        return `${this.#prefix}blob/${partition}/${blobId.replace(':', '/')}`;
+    }
+    #statsKey() {
+        return `${this.#prefix}${STATS_KEY_SUFFIX}`;
+    }
+    async #readStats() {
+        const object = await this.#bucket.get(this.#statsKey());
+        if (object === undefined) {
+            return { stats: emptyStats(), etag: undefined };
+        }
+        return { stats: parseStatsJson(await object.text()), etag: object.etag };
+    }
+    /**
+     * Conditionally write the stats accumulator. `etag` present ⇒ `ifMatch`
+     * (update the object we read); absent ⇒ `createOnly` (create only). Returns
+     * `false` on a failed precondition so the caller retries.
+     */
+    async #writeStatsCas(stats, etag) {
+        return this.#bucket.put(this.#statsKey(), {
+            body: new TextEncoder().encode(JSON.stringify(stats)),
+            contentType: 'application/json',
+            ...(etag === undefined ? { createOnly: true } : { ifMatch: etag }),
+        });
+    }
+    /**
+     * Fold `delta` into the stats accumulator under an ETag CAS retry loop.
+     * Best-effort: exhausting the budget leaves the counters drifted (they are
+     * documented APPROXIMATE) rather than failing an op that already touched
+     * the bytes.
+     */
+    async #bumpStats(delta) {
+        for (let attempt = 0; attempt < STATS_CAS_ATTEMPTS; attempt++) {
+            const { stats, etag } = await this.#readStats();
+            const next = {
+                count: stats.count + delta.count,
+                bytes: stats.bytes + delta.bytes,
+            };
+            if (await this.#writeStatsCas(next, etag))
+                return;
+        }
+        // Lost the CAS race `STATS_CAS_ATTEMPTS` times — leave counters as-is;
+        // `stats()` stays APPROXIMATE by contract.
+    }
+    async put(partition, blobId, bytes, nowMs, mediaType) {
+        const record = {
+            blobId,
+            partition,
+            byteLength: bytes.length,
+            ...(mediaType !== undefined ? { mediaType } : {}),
+            createdAtMs: nowMs,
+        };
+        const objectKey = this.objectKeyFor(partition, blobId);
+        // Idempotent re-put (same content-address ⇒ same key): count each distinct
+        // blob once, and preserve the FIRST createdAtMs so the sweep's grace clock
+        // is not reset by a re-upload (§5.9.2 upload age).
+        const preexisting = await this.#bucket.head(objectKey);
+        if (preexisting !== undefined) {
+            const meta = preexisting.metadata[BLOB_META_KEY];
+            if (meta !== undefined) {
+                const existing = parseMetaJson(base64urlToUtf8(meta), blobId, this.#label);
+                return {
+                    blobId,
+                    partition,
+                    byteLength: existing.byteLength,
+                    ...(existing.mediaType !== undefined
+                        ? { mediaType: existing.mediaType }
+                        : {}),
+                    createdAtMs: existing.createdAtMs,
+                };
+            }
+        }
+        const meta = {
+            byteLength: bytes.length,
+            ...(mediaType !== undefined ? { mediaType } : {}),
+            createdAtMs: nowMs,
+        };
+        // Conditional write: of two concurrent uploads of the same blob exactly
+        // one lands, so the first createdAtMs survives and the blob counts once.
+        // The loser re-reads and returns the winner's record.
+        const written = await this.#bucket.put(objectKey, {
+            body: bytes,
+            contentType: mediaType ?? 'application/octet-stream',
+            metadata: { [BLOB_META_KEY]: utf8ToBase64url(metaToJson(meta)) },
+            ...(preexisting === undefined
+                ? { createOnly: true }
+                : { ifMatch: preexisting.etag }),
+        });
+        if (!written)
+            return this.put(partition, blobId, bytes, nowMs, mediaType);
+        if (preexisting === undefined)
+            await this.#bumpStats({ count: 1, bytes: bytes.length });
+        return record;
+    }
+    async has(partition, blobId) {
+        if (!BLOB_ID_PATTERN.test(blobId))
+            return false;
+        const head = await this.#bucket.head(this.objectKeyFor(partition, blobId));
+        return head !== undefined;
+    }
+    async get(partition, blobId) {
+        if (!BLOB_ID_PATTERN.test(blobId))
+            return undefined;
+        const object = await this.#bucket.get(this.objectKeyFor(partition, blobId));
+        if (object === undefined)
+            return undefined;
+        const bytes = new Uint8Array(await object.arrayBuffer());
+        const rawMeta = object.metadata[BLOB_META_KEY];
+        const meta = rawMeta !== undefined
+            ? parseMetaJson(base64urlToUtf8(rawMeta), blobId, this.#label)
+            : { byteLength: bytes.length, createdAtMs: 0 };
+        return {
+            record: {
+                blobId,
+                partition,
+                byteLength: meta.byteLength,
+                ...(meta.mediaType !== undefined ? { mediaType: meta.mediaType } : {}),
+                createdAtMs: meta.createdAtMs,
+            },
+            bytes,
+        };
+    }
+    /**
+     * §5.9.2 orphan sweep — the ONLY LISTing operation. Page the `{prefix}blob/
+     * {partition}/` prefix, HEAD each object for its `createdAtMs`, and DELETE
+     * the ones both older than `olderThanMs` AND absent from
+     * `referencedBlobIds`. Returns the deleted blobIds. Never deletes a
+     * referenced blob (§5.9.2). Off the hot path — a host-scheduled GC job.
+     */
+    async sweepOrphans(partition, olderThanMs, referencedBlobIds) {
+        const listPrefix = `${this.#prefix}blob/${partition}/`;
+        const swept = [];
+        let deletedBytes = 0;
+        let cursor;
+        do {
+            const page = await this.#bucket.list(listPrefix, cursor);
+            cursor = page.cursor;
+            for (const key of page.keys) {
+                // `{listPrefix}sha256/{hex}` → `sha256:{hex}`.
+                const rest = key.slice(listPrefix.length);
+                if (!rest.startsWith('sha256/'))
+                    continue;
+                const blobId = `sha256:${rest.slice('sha256/'.length)}`;
+                if (!BLOB_ID_PATTERN.test(blobId))
+                    continue;
+                if (referencedBlobIds.has(blobId))
+                    continue;
+                const head = await this.#bucket.head(key);
+                if (head === undefined)
+                    continue;
+                const rawMeta = head.metadata[BLOB_META_KEY];
+                const meta = rawMeta !== undefined
+                    ? parseMetaJson(base64urlToUtf8(rawMeta), blobId, this.#label)
+                    : { byteLength: head.size ?? 0, createdAtMs: 0 };
+                if (meta.createdAtMs >= olderThanMs)
+                    continue;
+                await this.#bucket.delete(key);
+                swept.push(blobId);
+                deletedBytes += meta.byteLength;
+            }
+        } while (cursor !== undefined);
+        if (swept.length > 0) {
+            await this.#bumpStats({ count: -swept.length, bytes: -deletedBytes });
+        }
+        return swept;
+    }
+    /**
+     * Store-wide counters from the pointer-object accumulator (never a LIST on
+     * the read path). `approximate: true` for the same reasons the segment
+     * store documents: best-effort ETag-CAS maintenance and sweep deletes that
+     * may miss the decrement. Exact within a single writer; a health gauge
+     * under concurrency. Note this is store-WIDE, not partition-scoped, so a
+     * shared bucket reports the whole bucket — the parameter is accepted for
+     * interface parity but the accumulator is one object per store.
+     */
+    async stats(_partition) {
+        const { stats } = await this.#readStats();
+        return { count: stats.count, bytes: stats.bytes, approximate: true };
+    }
+}

package/dist/bucket-segment-store.d.ts

@@ -0,0 +1,77 @@
+/**
+ * Content-addressed segment store over the `ObjectBucket` seam (§5.1 cache
+ * semantics) — the store logic shared by `S3SegmentStore` (S3 REST + SigV4)
+ * and `R2SegmentStore` (`@syncular/server-workers`, the R2 binding).
+ *
+ * Key layout (deterministic — every lookup is a GET/HEAD, never a LIST):
+ *
+ *   {keyPrefix}seg/sha256/{hex}
+ *     The segment bytes, verbatim and immutable (the object body MUST be
+ *     exactly the content-addressed bytes so a presigned GET or the inline
+ *     download route serves them directly and the client's §5.1 hash check
+ *     passes).
+ *
+ *   {keyPrefix}rec/sha256/{hex}.json
+ *     The `SegmentRecord` (§5.1). It is the one MUTABLE object of the two:
+ *     the scope-digest set is a union that every publication extends, so
+ *     the merge is a read-modify-write. Keeping the record in the body (not
+ *     in the bytes object's user metadata) is what makes that write
+ *     conditional — a body ETag changes when the union changes, while the
+ *     bytes ETag is a content hash and is identical for identical bytes. A
+ *     failed precondition re-reads and merges again, so no concurrent
+ *     publication can drop another one's digest. `get` therefore reads two
+ *     objects.
+ *
+ *   {keyPrefix}find/{sha256Hex(canonical reuse key)}.json
+ *     Whole-table reuse pointer (§5.3): written only when
+ *     `rowCursor === null`, body = the record JSON. The reuse key is the
+ *     canonical JSON array `[partition, logEpoch, table, schemaVersion,
+ *     mediaType, scopeDigest, asOfCommitSeq]`, so `find` is one GET (plus a
+ *     HEAD to confirm the segment object itself still exists — lifecycle GC
+ *     may remove objects independently of pointers).
+ *
+ * Records written before the split kept the record in the bytes object's
+ * user metadata `syncular-record`; `get` still reads that shape and the next
+ * `put` materializes the record object from it, so an upgrade loses no cache
+ * entry.
+ *
+ * TTL mapping: expiry is **store-side and authoritative** — `expiresAtMs`
+ * (put-time + `ttlMs`, default 24 h) is recorded in the record and the
+ * pointer; `get` returns expired records so §5.5 can answer
+ * `sync.segment_expired`, and `find` filters them out itself. Bucket
+ * lifecycle expiration is garbage collection only: configure it comfortably
+ * ABOVE `ttlMs` (e.g. 2 days for the 24 h default) so clients normally see
+ * the precise, retryable `sync.segment_expired` and hit `sync.not_found` only
+ * long after. Never set lifecycle below `ttlMs`.
+ */
+import type { ObjectBucket } from './object-bucket.js';
+import type { SegmentFindKey, SegmentMetadata, SegmentRecord, SegmentStore, SegmentStoreStats, SegmentStream } from './segment-store.js';
+export interface BucketSegmentStoreOptions {
+    /** Key namespace inside the bucket, e.g. `syncular/`. Default: none. */
+    readonly keyPrefix?: string | undefined;
+    /** Segment TTL (§5.1 cache semantics). Default 24 h. */
+    readonly ttlMs?: number | undefined;
+}
+export declare class BucketSegmentStore implements SegmentStore {
+    #private;
+    constructor(bucket: ObjectBucket, options?: BucketSegmentStoreOptions);
+    /** `{keyPrefix}seg/sha256/{hex}` — the §5.4 "key embeds the segmentId". */
+    objectKeyFor(segmentId: string): string;
+    put(metadata: SegmentMetadata, bytes: Uint8Array, nowMs: number): Promise<SegmentRecord>;
+    get(segmentId: string): Promise<{
+        record: SegmentRecord;
+        bytes: Uint8Array;
+    } | undefined>;
+    /** The record plus the object GET's body, never buffered (see `get`). */
+    open(segmentId: string): Promise<SegmentStream | undefined>;
+    find(key: SegmentFindKey, nowMs: number): Promise<SegmentRecord | undefined>;
+    /**
+     * Store-wide counters from the pointer-object accumulator (never a LIST).
+     * Marked `approximate: true`: the counters are maintained by a best-effort
+     * ETag-CAS read-modify-write on `put`, so a crash between the segment PUT
+     * and the accumulator write, or lifecycle GC that deletes objects the
+     * accumulator still counts, can drift them. Exact within a single writer;
+     * a health gauge under concurrency. See the README "S3 stats".
+     */
+    stats(): Promise<SegmentStoreStats>;
+}