@syncular/server 0.32.1 → 0.33.0

package/dist/s3-object-bucket.js

@@ -0,0 +1,164 @@
+import { EMPTY_PAYLOAD_SHA256, sha256Hex, signRequest, } from './sigv4.js';
+const META_HEADER_PREFIX = 'x-amz-meta-';
+/** User metadata carried as `x-amz-meta-*` headers (lowercased by `Headers`). */
+function metadataOf(headers) {
+    const metadata = {};
+    for (const [name, value] of headers) {
+        if (name.startsWith(META_HEADER_PREFIX)) {
+            metadata[name.slice(META_HEADER_PREFIX.length)] = value;
+        }
+    }
+    return metadata;
+}
+function sizeOf(headers) {
+    const value = headers.get('content-length');
+    if (value === null)
+        return undefined;
+    const size = Number(value);
+    return Number.isFinite(size) ? size : undefined;
+}
+/**
+ * Extract `<Key>` values (and the `<NextContinuationToken>`, if the listing
+ * is truncated) from a `ListObjectsV2` XML body. A tiny regex reader; the
+ * only XML the stores ever parse, and only on the GC path. `ListObjectsV2`
+ * returns keys XML-escaped for `& < >`; we unescape those.
+ */
+function parseListObjectsV2(xml) {
+    const objects = [];
+    for (const m of xml.matchAll(/<Key>([\s\S]*?)<\/Key>/g)) {
+        objects.push({ key: unescapeXml(m[1] ?? '') });
+    }
+    const truncated = /<IsTruncated>\s*true\s*<\/IsTruncated>/i.test(xml);
+    const tokenMatch = /<NextContinuationToken>([\s\S]*?)<\/NextContinuationToken>/.exec(xml);
+    const nextToken = truncated && tokenMatch !== null
+        ? unescapeXml(tokenMatch[1] ?? '')
+        : undefined;
+    return { objects, nextToken };
+}
+function unescapeXml(value) {
+    return value
+        .replaceAll('&lt;', '<')
+        .replaceAll('&gt;', '>')
+        .replaceAll('&quot;', '"')
+        .replaceAll('&apos;', "'")
+        .replaceAll('&amp;', '&');
+}
+export class S3ObjectBucket {
+    #config;
+    #credentials;
+    constructor(config) {
+        this.#config = config;
+        this.#credentials = {
+            accessKeyId: config.accessKeyId,
+            secretAccessKey: config.secretAccessKey,
+            ...(config.sessionToken !== undefined
+                ? { sessionToken: config.sessionToken }
+                : {}),
+        };
+    }
+    /** The object URL the presign helpers sign (`{endpoint}/{bucket}/{key}`). */
+    urlFor(key) {
+        const endpoint = this.#config.endpoint.replace(/\/+$/, '');
+        return new URL(`${endpoint}/${this.#config.bucket}/${key}`);
+    }
+    async #request(method, key, options) {
+        const url = this.urlFor(key);
+        if (options?.query !== undefined) {
+            for (const [name, value] of options.query) {
+                url.searchParams.set(name, value);
+            }
+        }
+        const body = options?.body;
+        const headers = await signRequest({
+            method,
+            url,
+            region: this.#config.region,
+            credentials: this.#credentials,
+            nowMs: Date.now(),
+            payloadHash: body === undefined ? EMPTY_PAYLOAD_SHA256 : await sha256Hex(body),
+            ...(options?.headers !== undefined ? { headers: options.headers } : {}),
+        });
+        const response = await fetch(url, {
+            method,
+            headers,
+            ...(body !== undefined ? { body: body } : {}),
+        });
+        if (response.status === 404) {
+            await response.arrayBuffer();
+            return undefined;
+        }
+        if (options?.allow412 === true && response.status === 412)
+            return response;
+        if (!response.ok) {
+            const detail = (await response.text()).slice(0, 300);
+            throw new Error(`S3ObjectBucket: ${method} ${key} failed with ${response.status}: ${detail}`);
+        }
+        return response;
+    }
+    async get(key) {
+        const response = await this.#request('GET', key);
+        if (response === undefined)
+            return undefined;
+        return {
+            body: response.body,
+            etag: response.headers.get('etag') ?? undefined,
+            size: sizeOf(response.headers),
+            metadata: metadataOf(response.headers),
+            arrayBuffer: () => response.arrayBuffer(),
+            text: () => response.text(),
+        };
+    }
+    async head(key) {
+        const response = await this.#request('HEAD', key);
+        if (response === undefined)
+            return undefined;
+        await response.arrayBuffer();
+        return {
+            etag: response.headers.get('etag') ?? undefined,
+            size: sizeOf(response.headers),
+            metadata: metadataOf(response.headers),
+        };
+    }
+    async put(key, options) {
+        const headers = {
+            'content-type': options.contentType,
+        };
+        if (options.createOnly === true)
+            headers['if-none-match'] = '*';
+        if (options.ifMatch !== undefined)
+            headers['if-match'] = options.ifMatch;
+        for (const [name, value] of Object.entries(options.metadata ?? {})) {
+            headers[`${META_HEADER_PREFIX}${name}`] = value;
+        }
+        const response = await this.#request('PUT', key, {
+            body: options.body,
+            headers,
+            allow412: true,
+        });
+        if (response === undefined) {
+            throw new Error(`S3ObjectBucket: PUT ${key} failed with 404`);
+        }
+        await response.arrayBuffer();
+        // A failed `If-Match` / `If-None-Match: *` is a `412`: the caller's CAS
+        // read superseded, so it re-reads instead of overwriting.
+        return response.status !== 412;
+    }
+    async delete(key) {
+        const response = await this.#request('DELETE', key);
+        await response?.arrayBuffer();
+    }
+    async list(prefix, cursor) {
+        const query = [
+            ['list-type', '2'],
+            ['prefix', prefix],
+        ];
+        if (cursor !== undefined)
+            query.push(['continuation-token', cursor]);
+        // ListObjectsV2 is a bucket-level GET (`GET /{bucket}?list-type=2`).
+        const response = await this.#request('GET', '', { query });
+        if (response === undefined)
+            return { keys: [], cursor: undefined };
+        const { objects, nextToken } = parseListObjectsV2(await response.text());
+        return { keys: objects.map((object) => object.key), cursor: nextToken };
+    }
+}

package/dist/s3-segment-store.d.ts

@@ -1,48 +1,15 @@
 /**
  * S3-compatible segment store (SPEC.md §5.1 cache semantics, §5.4
- * delegated presign) — AWS S3, Cloudflare R2, MinIO. Zero dependencies:
- * hand-rolled SigV4 over `fetch` (sigv4.ts).
+ * delegated presign) — AWS S3, Cloudflare R2's S3 endpoint, MinIO. Zero
+ * dependencies: hand-rolled SigV4 over `fetch` (sigv4.ts).
  *
- * Key layout (deterministic — every lookup is a GET/HEAD, never a LIST):
- *
- *   {keyPrefix}seg/sha256/{hex}
- *     The segment bytes, verbatim and immutable (the object body MUST be
- *     exactly the content-addressed bytes so presigned GETs serve them
- *     directly and the client's §5.1 hash check passes).
- *
- *   {keyPrefix}rec/sha256/{hex}.json
- *     The `SegmentRecord` (§5.1). It is the one MUTABLE object of the two:
- *     the scope-digest set is a union that every publication extends, so
- *     the merge is a read-modify-write. Keeping the record in the body (not
- *     in the bytes object's user metadata) is what makes that write
- *     conditional — a body ETag changes when the union changes, while the
- *     bytes ETag is a content hash and is identical for identical bytes. A
- *     `412` re-reads and merges again, so no concurrent publication can drop
- *     another one's digest. `get` therefore reads two objects.
- *
- *   {keyPrefix}find/{sha256Hex(canonical reuse key)}.json
- *     Whole-table reuse pointer (§5.3): written only when
- *     `rowCursor === null`, body = the record JSON. The reuse key is the
- *     canonical JSON array `[partition, table, schemaVersion, mediaType,
- *     scopeDigest, asOfCommitSeq]`, so `find` is one GET (plus a HEAD to
- *     confirm the segment object itself still exists — lifecycle GC may
- *     remove objects independently of pointers).
- *
- * Records written before the split kept the record in the bytes object's
- * user metadata `x-amz-meta-syncular-record`; `get` still reads that shape
- * and the next `put` materializes the record object from it, so an upgrade
- * loses no cache entry.
- *
- * TTL mapping: expiry is **store-side and authoritative** — `expiresAtMs`
- * (put-time + `ttlMs`, default 24 h) is recorded in the record and the
- * pointer; `get` returns expired records so §5.5 can answer
- * `sync.segment_expired`, and `find` filters them out itself. S3 lifecycle
- * expiration is garbage collection only: configure it comfortably ABOVE
- * `ttlMs` (e.g. 2 days for the 24 h default) so clients normally see the
- * precise, retryable `sync.segment_expired` and hit `sync.not_found` only
- * long after. Never set lifecycle below `ttlMs`.
+ * The store logic (key layout, record CAS, reuse pointers, stats
+ * accumulator) is `BucketSegmentStore`, shared with `R2SegmentStore` in
+ * `@syncular/server-workers`; this class adds the S3 REST transport
+ * (`S3ObjectBucket`) and the SigV4 presign methods, which a bucket binding
+ * cannot mint.
  */
-import type { SegmentFindKey, SegmentMetadata, SegmentRecord, SegmentStore, SegmentStoreStats, SegmentStream } from './segment-store.js';
+import { BucketSegmentStore } from './bucket-segment-store.js';
 import type { DelegatedPresignConfig, SegmentUrlIssue } from './signed-url.js';
 export interface S3SegmentStoreConfig {
     /**
@@ -65,28 +32,9 @@ export interface S3SegmentStoreConfig {
     /** Segment TTL (§5.1 cache semantics). Default 24 h. */
     readonly ttlMs?: number;
 }
-export declare class S3SegmentStore implements SegmentStore {
+export declare class S3SegmentStore extends BucketSegmentStore {
     #private;
     constructor(config: S3SegmentStoreConfig);
-    /** `{keyPrefix}seg/sha256/{hex}` — the §5.4 "key embeds the segmentId". */
-    objectKeyFor(segmentId: string): string;
-    put(metadata: SegmentMetadata, bytes: Uint8Array, nowMs: number): Promise<SegmentRecord>;
-    get(segmentId: string): Promise<{
-        record: SegmentRecord;
-        bytes: Uint8Array;
-    } | undefined>;
-    /** The record plus the object GET's body, never buffered (see `get`). */
-    open(segmentId: string): Promise<SegmentStream | undefined>;
-    find(key: SegmentFindKey, nowMs: number): Promise<SegmentRecord | undefined>;
-    /**
-     * Store-wide counters from the pointer-object accumulator (never a LIST).
-     * Marked `approximate: true`: the counters are maintained by a best-effort
-     * ETag-CAS read-modify-write on `put`, so a crash between the segment PUT
-     * and the accumulator write, or lifecycle GC that deletes objects the
-     * accumulator still counts, can drift them. Exact within a single writer;
-     * a health gauge under concurrency. See the README "S3 stats".
-     */
-    stats(): Promise<SegmentStoreStats>;
     /**
      * SigV4 presigned GET for a segment object (§5.4 delegated presign).
      * The signed key embeds the `segmentId` (equivalence rule) and