@syncular/server 0.1.3 → 0.2.1

package/dist/s3-segment-store.js

@@ -0,0 +1,380 @@
+import { segmentIdFor } from './segment-store.js';
+import { EMPTY_PAYLOAD_SHA256, presignUrl, sha256Hex, signRequest, } from './sigv4.js';
+const DEFAULT_TTL_MS = 24 * 60 * 60 * 1000;
+const SEGMENT_ID_PATTERN = /^sha256:[0-9a-f]{64}$/;
+const RECORD_META_HEADER = 'x-amz-meta-syncular-record';
+/**
+ * Stats accumulator (§ "S3 stats" in the README). The store keeps a small
+ * counter object under a fixed key, updated read-modify-write on `put`. It
+ * exists because a LIST-free store (every lookup is a GET/HEAD) has no cheap
+ * way to count objects on demand — so `stats()` reads the accumulator instead
+ * of enumerating the bucket.
+ *
+ * The write is guarded by an ETag compare-and-swap (`If-Match`, or
+ * `If-None-Match: *` to create): the loop re-reads and retries on a `412`
+ * precondition failure, so concurrent writers do not silently clobber each
+ * other's increment. The counters are still APPROXIMATE — a crash between the
+ * segment PUT and the accumulator CAS, or a writer against a backend that
+ * ignores conditional headers, can drift them — so the admin surface labels
+ * S3 stats `approximate: true`. They are a health gauge, not an invoice.
+ */
+const STATS_KEY_SUFFIX = 'stats/segments.json';
+const STATS_CAS_ATTEMPTS = 16;
+function emptyStats() {
+    return { count: 0, bytes: 0, rowsSegments: 0, sqliteSegments: 0 };
+}
+function parseStatsJson(json) {
+    const parsed = JSON.parse(json);
+    return {
+        count: Number(parsed.count ?? 0),
+        bytes: Number(parsed.bytes ?? 0),
+        rowsSegments: Number(parsed.rowsSegments ?? 0),
+        sqliteSegments: Number(parsed.sqliteSegments ?? 0),
+    };
+}
+// Runtime-neutral base64url (TODO §4.2): `Buffer` is not present on
+// Cloudflare Workers without `nodejs_compat`, so the object-metadata record
+// header is (de)coded with `btoa`/`atob`, available in every runtime.
+function utf8ToBase64url(text) {
+    let binary = '';
+    for (const byte of new TextEncoder().encode(text)) {
+        binary += String.fromCharCode(byte);
+    }
+    return btoa(binary)
+        .replaceAll('+', '-')
+        .replaceAll('/', '_')
+        .replace(/=+$/, '');
+}
+function base64urlToUtf8(value) {
+    const binary = atob(value.replaceAll('-', '+').replaceAll('_', '/'));
+    const bytes = new Uint8Array(binary.length);
+    for (let i = 0; i < binary.length; i++)
+        bytes[i] = binary.charCodeAt(i);
+    return new TextDecoder().decode(bytes);
+}
+function recordToJson(record) {
+    return JSON.stringify(record);
+}
+function isMediaType(value) {
+    return value === 'rows' || value === 'sqlite';
+}
+function parseRecordJson(json, source) {
+    let parsed;
+    try {
+        parsed = JSON.parse(json);
+    }
+    catch {
+        throw new Error(`S3SegmentStore: corrupt record JSON in ${source}`);
+    }
+    if (typeof parsed !== 'object' || parsed === null) {
+        throw new Error(`S3SegmentStore: corrupt record in ${source}`);
+    }
+    const r = parsed;
+    const strings = ['segmentId', 'partition', 'table', 'scopeDigest'];
+    const numbers = [
+        'schemaVersion',
+        'asOfCommitSeq',
+        'rowCount',
+        'byteLength',
+        'createdAtMs',
+        'expiresAtMs',
+    ];
+    for (const field of strings) {
+        if (typeof r[field] !== 'string') {
+            throw new Error(`S3SegmentStore: bad record field ${field} in ${source}`);
+        }
+    }
+    for (const field of numbers) {
+        if (typeof r[field] !== 'number') {
+            throw new Error(`S3SegmentStore: bad record field ${field} in ${source}`);
+        }
+    }
+    if (!isMediaType(r.mediaType)) {
+        throw new Error(`S3SegmentStore: bad record mediaType in ${source}`);
+    }
+    const cursorOk = (v) => v === null || typeof v === 'string';
+    if (!cursorOk(r.rowCursor) || !cursorOk(r.nextRowCursor)) {
+        throw new Error(`S3SegmentStore: bad record cursor in ${source}`);
+    }
+    return {
+        segmentId: r.segmentId,
+        partition: r.partition,
+        table: r.table,
+        schemaVersion: r.schemaVersion,
+        mediaType: r.mediaType,
+        scopeDigest: r.scopeDigest,
+        asOfCommitSeq: r.asOfCommitSeq,
+        rowCount: r.rowCount,
+        rowCursor: r.rowCursor,
+        nextRowCursor: r.nextRowCursor,
+        byteLength: r.byteLength,
+        createdAtMs: r.createdAtMs,
+        expiresAtMs: r.expiresAtMs,
+    };
+}
+export class S3SegmentStore {
+    #config;
+    #credentials;
+    #ttlMs;
+    #prefix;
+    constructor(config) {
+        this.#config = config;
+        this.#credentials = {
+            accessKeyId: config.accessKeyId,
+            secretAccessKey: config.secretAccessKey,
+            ...(config.sessionToken !== undefined
+                ? { sessionToken: config.sessionToken }
+                : {}),
+        };
+        this.#ttlMs = config.ttlMs ?? DEFAULT_TTL_MS;
+        this.#prefix = config.keyPrefix ?? '';
+    }
+    /** `{keyPrefix}seg/sha256/{hex}` — the §5.4 "key embeds the segmentId". */
+    objectKeyFor(segmentId) {
+        return `${this.#prefix}seg/${segmentId.replace(':', '/')}`;
+    }
+    async #findKeyFor(key) {
+        const canonical = JSON.stringify([
+            key.partition,
+            key.table,
+            key.schemaVersion,
+            key.mediaType,
+            key.scopeDigest,
+            key.asOfCommitSeq,
+        ]);
+        return `${this.#prefix}find/${await sha256Hex(canonical)}.json`;
+    }
+    #statsKey() {
+        return `${this.#prefix}${STATS_KEY_SUFFIX}`;
+    }
+    #urlFor(key) {
+        const endpoint = this.#config.endpoint.replace(/\/+$/, '');
+        return new URL(`${endpoint}/${this.#config.bucket}/${key}`);
+    }
+    async #request(method, key, options) {
+        const url = this.#urlFor(key);
+        const body = options?.body;
+        const headers = await signRequest({
+            method,
+            url,
+            region: this.#config.region,
+            credentials: this.#credentials,
+            nowMs: Date.now(),
+            payloadHash: body === undefined ? EMPTY_PAYLOAD_SHA256 : await sha256Hex(body),
+            ...(options?.headers !== undefined ? { headers: options.headers } : {}),
+        });
+        const response = await fetch(url, {
+            method,
+            headers,
+            ...(body !== undefined ? { body: body } : {}),
+        });
+        if (response.status === 404) {
+            await response.arrayBuffer();
+            return undefined;
+        }
+        if (!response.ok) {
+            const detail = (await response.text()).slice(0, 300);
+            throw new Error(`S3SegmentStore: ${method} ${key} failed with ${response.status}: ${detail}`);
+        }
+        return response;
+    }
+    /** Read the stats accumulator + its ETag (for the CAS write), or none. */
+    async #readStats() {
+        const response = await this.#request('GET', this.#statsKey());
+        if (response === undefined) {
+            return { stats: emptyStats(), etag: undefined };
+        }
+        const etag = response.headers.get('etag') ?? undefined;
+        const stats = parseStatsJson(await response.text());
+        return { stats, etag };
+    }
+    /**
+     * Conditionally write the stats accumulator. `etag` present ⇒ `If-Match`
+     * (update the object we read); absent ⇒ `If-None-Match: *` (create only).
+     * Returns `false` on a `412` precondition failure so the caller retries.
+     */
+    async #writeStatsCas(stats, etag) {
+        const url = this.#urlFor(this.#statsKey());
+        const body = new TextEncoder().encode(JSON.stringify(stats));
+        const conditional = etag === undefined ? { 'if-none-match': '*' } : { 'if-match': etag };
+        const headers = await signRequest({
+            method: 'PUT',
+            url,
+            region: this.#config.region,
+            credentials: this.#credentials,
+            nowMs: Date.now(),
+            payloadHash: await sha256Hex(body),
+            headers: { 'content-type': 'application/json', ...conditional },
+        });
+        const response = await fetch(url, {
+            method: 'PUT',
+            headers,
+            body: body,
+        });
+        if (response.status === 412) {
+            await response.arrayBuffer();
+            return false;
+        }
+        if (!response.ok) {
+            const detail = (await response.text()).slice(0, 300);
+            throw new Error(`S3SegmentStore: stats PUT failed with ${response.status}: ${detail}`);
+        }
+        await response.arrayBuffer();
+        return true;
+    }
+    /**
+     * Fold `delta` into the stats accumulator under an ETag compare-and-swap
+     * retry loop. Best-effort: exhausting the retry budget leaves the counters
+     * drifted (they are documented APPROXIMATE) rather than failing the `put`
+     * that already stored the bytes.
+     */
+    async #bumpStats(delta) {
+        for (let attempt = 0; attempt < STATS_CAS_ATTEMPTS; attempt++) {
+            const { stats, etag } = await this.#readStats();
+            const next = {
+                count: stats.count + delta.count,
+                bytes: stats.bytes + delta.bytes,
+                rowsSegments: stats.rowsSegments + delta.rowsSegments,
+                sqliteSegments: stats.sqliteSegments + delta.sqliteSegments,
+            };
+            if (await this.#writeStatsCas(next, etag))
+                return;
+        }
+        // Lost the CAS race `STATS_CAS_ATTEMPTS` times running — leave the
+        // counters as-is. `stats()` stays APPROXIMATE by contract.
+    }
+    async put(metadata, bytes, nowMs) {
+        const segmentId = await segmentIdFor(bytes);
+        const record = {
+            ...metadata,
+            segmentId,
+            byteLength: bytes.length,
+            createdAtMs: nowMs,
+            expiresAtMs: nowMs + this.#ttlMs,
+        };
+        const recordJson = recordToJson(record);
+        // Detect an idempotent re-put (same content-address ⇒ same key) so the
+        // stats accumulator counts each distinct segment once, not once per PUT.
+        const objectKey = this.objectKeyFor(segmentId);
+        const preexisting = await this.#request('HEAD', objectKey);
+        await preexisting?.arrayBuffer();
+        const isNew = preexisting === undefined;
+        const response = await this.#request('PUT', objectKey, {
+            body: bytes,
+            headers: {
+                'content-type': 'application/octet-stream',
+                [RECORD_META_HEADER]: utf8ToBase64url(recordJson),
+            },
+        });
+        await response?.arrayBuffer();
+        if (isNew) {
+            await this.#bumpStats({
+                count: 1,
+                bytes: bytes.length,
+                rowsSegments: metadata.mediaType === 'rows' ? 1 : 0,
+                sqliteSegments: metadata.mediaType === 'sqlite' ? 1 : 0,
+            });
+        }
+        if (metadata.rowCursor === null) {
+            const pointer = await this.#request('PUT', await this.#findKeyFor(metadata), {
+                body: new TextEncoder().encode(recordJson),
+                headers: { 'content-type': 'application/json' },
+            });
+            await pointer?.arrayBuffer();
+        }
+        return record;
+    }
+    async get(segmentId) {
+        if (!SEGMENT_ID_PATTERN.test(segmentId))
+            return undefined;
+        const response = await this.#request('GET', this.objectKeyFor(segmentId));
+        if (response === undefined)
+            return undefined;
+        const bytes = new Uint8Array(await response.arrayBuffer());
+        const meta = response.headers.get(RECORD_META_HEADER);
+        if (meta === null) {
+            throw new Error(`S3SegmentStore: object for ${segmentId} lacks ${RECORD_META_HEADER}`);
+        }
+        const record = parseRecordJson(base64urlToUtf8(meta), segmentId);
+        return { record, bytes };
+    }
+    async find(key, nowMs) {
+        const response = await this.#request('GET', await this.#findKeyFor(key));
+        if (response === undefined)
+            return undefined;
+        const record = parseRecordJson(await response.text(), 'reuse pointer');
+        if (record.partition !== key.partition ||
+            record.table !== key.table ||
+            record.schemaVersion !== key.schemaVersion ||
+            record.mediaType !== key.mediaType ||
+            record.scopeDigest !== key.scopeDigest ||
+            record.asOfCommitSeq !== key.asOfCommitSeq ||
+            record.rowCursor !== null) {
+            return undefined;
+        }
+        if (record.expiresAtMs <= nowMs)
+            return undefined;
+        // Lifecycle GC may have removed the object while the pointer survived.
+        const head = await this.#request('HEAD', this.objectKeyFor(record.segmentId));
+        if (head === undefined)
+            return undefined;
+        await head.arrayBuffer();
+        return record;
+    }
+    /**
+     * Store-wide counters from the pointer-object accumulator (never a LIST).
+     * Marked `approximate: true`: the counters are maintained by a best-effort
+     * ETag-CAS read-modify-write on `put`, so a crash between the segment PUT
+     * and the accumulator write, or lifecycle GC that deletes objects the
+     * accumulator still counts, can drift them. Exact within a single writer;
+     * a health gauge under concurrency. See the README "S3 stats".
+     */
+    async stats() {
+        const { stats } = await this.#readStats();
+        return {
+            count: stats.count,
+            bytes: stats.bytes,
+            rowsSegments: stats.rowsSegments,
+            sqliteSegments: stats.sqliteSegments,
+            approximate: true,
+        };
+    }
+    /**
+     * SigV4 presigned GET for a segment object (§5.4 delegated presign).
+     * The signed key embeds the `segmentId` (equivalence rule) and
+     * `urlExpiresAtMs` is the provider-enforced expiry
+     * (`X-Amz-Date + X-Amz-Expires`). TTL SHOULD be ≤ 15 minutes (§5.4);
+     * default 900 s.
+     */
+    async presignSegmentGet(segmentId, options) {
+        if (!SEGMENT_ID_PATTERN.test(segmentId)) {
+            throw new Error(`S3SegmentStore: malformed segmentId ${segmentId}`);
+        }
+        const ttlSeconds = options?.ttlSeconds ?? 900;
+        const nowMs = options?.nowMs ?? Date.now();
+        const url = await presignUrl({
+            url: this.#urlFor(this.objectKeyFor(segmentId)),
+            region: this.#config.region,
+            credentials: this.#credentials,
+            nowMs,
+            expiresSeconds: ttlSeconds,
+        });
+        return {
+            url: url.toString(),
+            urlExpiresAtMs: (Math.floor(nowMs / 1000) + ttlSeconds) * 1000,
+        };
+    }
+}
+/**
+ * Wire an `S3SegmentStore` into `SyncServerConfig.signedUrls` as the §5.4
+ * delegated-presign scheme:
+ * `signedUrls: s3PresignedUrls(store, { ttlSeconds: 900 })`.
+ */
+export function s3PresignedUrls(store, options) {
+    return {
+        ...(options?.ttlSeconds !== undefined
+            ? { ttlSeconds: options.ttlSeconds }
+            : {}),
+        presign: ({ segmentId, ttlSeconds, nowMs }) => store.presignSegmentGet(segmentId, { ttlSeconds, nowMs }),
+    };
+}