@platformatic/sqlite-js-stream-s3 0.0.0-stage → 0.1.4

package/dist/replica-client.js

@@ -0,0 +1,316 @@
+/**
+ * S3 replica client — replicates LTX files to an Amazon S3 bucket.
+ *
+ * Object layout mirrors the file backend:
+ *   `{prefix}ltx/{level}/{minTXID}-{maxTXID}.ltx`
+ *
+ * The implementation talks to S3 through the AWS SDK v3 {@link S3Client}. A
+ * minimal `send`-shaped client can be injected (e.g. an in-memory fake in
+ * tests) so the upload strategy is testable without network access.
+ *
+ * Creation timestamps are preserved on upload as the `ltx-timestamp` object
+ * metadata; listing falls back to the object's `LastModified`.
+ */
+import { Readable } from 'node:stream';
+import { S3Client, AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, DeleteObjectsCommand, GetObjectCommand, HeadBucketCommand, HeadObjectCommand, ListObjectsV2Command, PutObjectCommand, UploadPartCommand, } from '@aws-sdk/client-s3';
+import { HeaderSize, LTXError, formatFilename, noopLogger, parseFilename, unmarshalHeader, } from '@platformatic/sqlite-js-stream';
+/** The strategy identifier reported by {@link S3ReplicaClient.type}. */
+export const ReplicaClientType = 's3';
+/** Normalized base prefix (no leading/trailing slash, `''` when empty). */
+function normalizePrefix(prefix) {
+    return prefix.replace(/^\/+/, '').replace(/\/+$/, '');
+}
+/** Concatenate byte chunks into a single Uint8Array. */
+function concatBytes(chunks) {
+    let total = 0;
+    for (const c of chunks)
+        total += c.length;
+    const out = new Uint8Array(total);
+    let o = 0;
+    for (const c of chunks) {
+        out.set(c, o);
+        o += c.length;
+    }
+    return out;
+}
+/** S3 requires every multipart part (except the last) to be at least 5 MiB. */
+const MULTIPART_MIN_PART_SIZE = 5 * 1024 * 1024;
+/** S3ReplicaClient writes LTX files to an Amazon S3 bucket. */
+export class S3ReplicaClient {
+    bucket;
+    prefix;
+    client;
+    logger = noopLogger;
+    constructor(opts) {
+        if (!opts.bucket) {
+            throw new Error('S3 replica client: bucket is required');
+        }
+        this.bucket = opts.bucket;
+        this.prefix = normalizePrefix(opts.prefix ?? '');
+        this.client =
+            opts.client ??
+                new S3Client({
+                    region: opts.region,
+                    endpoint: opts.endpoint,
+                    credentials: opts.accessKeyId
+                        ? {
+                            accessKeyId: opts.accessKeyId,
+                            secretAccessKey: opts.secretAccessKey ?? '',
+                            sessionToken: opts.sessionToken,
+                        }
+                        : undefined,
+                });
+    }
+    type() {
+        return ReplicaClientType;
+    }
+    /** Best-effort connectivity check; real I/O errors surface on first use. */
+    async init() {
+        try {
+            await this.client.send(new HeadBucketCommand({ Bucket: this.bucket }));
+        }
+        catch (err) {
+            this.logger.debug('s3 head bucket failed', err);
+        }
+    }
+    setLogger(logger) {
+        this.logger = logger;
+    }
+    /** Returns the object key prefix (empty string if none). */
+    prefixKey() {
+        return this.prefix;
+    }
+    /** Returns `{prefix}ltx/{level}/`. */
+    levelPrefix(level) {
+        return `${this.prefix ? `${this.prefix}/` : ''}ltx/${level}/`;
+    }
+    /** Returns the object key for a level and TXID range. */
+    fileKey(level, minTXID, maxTXID) {
+        return `${this.levelPrefix(level)}${formatFilename(minTXID, maxTXID)}`;
+    }
+    async *ltxFiles(opts) {
+        const prefix = this.levelPrefix(opts.level);
+        let token;
+        do {
+            const res = await this.client.send(new ListObjectsV2Command({
+                Bucket: this.bucket,
+                Prefix: prefix,
+                ContinuationToken: token,
+            }));
+            for (const obj of res.Contents ?? []) {
+                if (!obj.Key)
+                    continue;
+                const name = obj.Key.slice(prefix.length);
+                let minTXID;
+                let maxTXID;
+                try {
+                    ({ minTXID, maxTXID } = parseFilename(name));
+                }
+                catch {
+                    continue; // skip non-conforming keys
+                }
+                if (minTXID < opts.seek)
+                    continue;
+                yield {
+                    level: opts.level,
+                    minTXID,
+                    maxTXID,
+                    size: obj.Size ?? 0,
+                    createdAt: (obj.LastModified ?? new Date(0)).getTime(),
+                };
+            }
+            token = res.IsTruncated ? (res.NextContinuationToken ?? undefined) : undefined;
+        } while (token);
+    }
+    async openLtxFile(opts) {
+        const key = this.fileKey(opts.level, opts.minTXID, opts.maxTXID);
+        // A zero offset/size is a full read; no HeadObject round-trip needed.
+        if (opts.offset === 0 && opts.size === 0) {
+            try {
+                const res = await this.client.send(new GetObjectCommand({ Bucket: this.bucket, Key: key }));
+                return res.Body;
+            }
+            catch (err) {
+                throw new LTXError('open', key, opts.level, opts.minTXID, opts.maxTXID, err);
+            }
+        }
+        // Need the object size to build a range and to handle offset >= size.
+        let head;
+        try {
+            head = await this.client.send(new HeadObjectCommand({ Bucket: this.bucket, Key: key }));
+        }
+        catch (err) {
+            throw new LTXError('open', key, opts.level, opts.minTXID, opts.maxTXID, err);
+        }
+        const len = Number(head.ContentLength ?? 0);
+        if (opts.offset >= len)
+            return Readable.from([]);
+        const end = opts.size > 0 ? Math.min(opts.offset + opts.size - 1, len - 1) : len - 1;
+        if (end < opts.offset)
+            return Readable.from([]);
+        try {
+            const res = await this.client.send(new GetObjectCommand({
+                Bucket: this.bucket,
+                Key: key,
+                Range: `bytes=${opts.offset}-${end}`,
+            }));
+            return res.Body;
+        }
+        catch (err) {
+            throw new LTXError('open', key, opts.level, opts.minTXID, opts.maxTXID, err);
+        }
+    }
+    async writeLtxFile(opts, body) {
+        const key = this.fileKey(opts.level, opts.minTXID, opts.maxTXID);
+        // Capture the LTX header (timestamp) from the leading bytes, and stream the
+        // rest. Small objects use a single PutObject; large objects (> 5 MiB) are
+        // uploaded with the multipart API so the whole file is never buffered.
+        let headerBytes = null;
+        let head = [];
+        let headLen = 0;
+        let acc = [];
+        let accLen = 0;
+        let size = 0;
+        let uploadId = null;
+        let partNumber = 0;
+        const uploadedParts = [];
+        const uploadPart = async (buf) => {
+            partNumber++;
+            const res = await this.client.send(new UploadPartCommand({
+                Bucket: this.bucket,
+                Key: key,
+                UploadId: uploadId,
+                PartNumber: partNumber,
+                Body: buf,
+            }));
+            uploadedParts.push({ PartNumber: partNumber, ETag: res.ETag ?? `etag-${partNumber}` });
+        };
+        try {
+            for await (const chunk of body) {
+                const bytes = typeof chunk === 'string' ? new TextEncoder().encode(chunk) : Uint8Array.from(chunk);
+                if (bytes.length === 0)
+                    continue;
+                size += bytes.length;
+                if (headerBytes === null) {
+                    head.push(bytes);
+                    headLen += bytes.length;
+                    if (headLen >= HeaderSize) {
+                        headerBytes = concatBytes(head).subarray(0, HeaderSize);
+                        head = [];
+                        headLen = 0;
+                    }
+                }
+                acc.push(bytes);
+                accLen += bytes.length;
+                if (uploadId === null && accLen >= MULTIPART_MIN_PART_SIZE) {
+                    const res = await this.client.send(new CreateMultipartUploadCommand({
+                        Bucket: this.bucket,
+                        Key: key,
+                        Metadata: { 'ltx-timestamp': String(Number(unmarshalHeader(headerBytes).timestamp)) },
+                    }));
+                    uploadId = res.UploadId;
+                    await uploadPart(concatBytes(acc));
+                    acc = [];
+                    accLen = 0;
+                }
+                else if (uploadId !== null && accLen >= MULTIPART_MIN_PART_SIZE) {
+                    await uploadPart(concatBytes(acc));
+                    acc = [];
+                    accLen = 0;
+                }
+            }
+            if (headerBytes === null) {
+                throw new Error('write ltx file: input too short to contain a header');
+            }
+            const timestamp = Number(unmarshalHeader(headerBytes).timestamp);
+            const metadata = { 'ltx-timestamp': String(timestamp) };
+            if (uploadId === null) {
+                // Single PutObject for small objects.
+                const buf = concatBytes(acc);
+                await this.client.send(new PutObjectCommand({
+                    Bucket: this.bucket,
+                    Key: key,
+                    Body: buf,
+                    ContentLength: size,
+                    Metadata: metadata,
+                }));
+            }
+            else {
+                // Multipart: upload any remaining bytes as the final part, then complete.
+                if (accLen > 0)
+                    await uploadPart(concatBytes(acc));
+                await this.client.send(new CompleteMultipartUploadCommand({
+                    Bucket: this.bucket,
+                    Key: key,
+                    UploadId: uploadId,
+                    MultipartUpload: { Parts: uploadedParts },
+                }));
+            }
+            return {
+                level: opts.level,
+                minTXID: opts.minTXID,
+                maxTXID: opts.maxTXID,
+                size,
+                createdAt: timestamp,
+            };
+        }
+        catch (err) {
+            if (uploadId !== null) {
+                await this.client
+                    .send(new AbortMultipartUploadCommand({
+                    Bucket: this.bucket,
+                    Key: key,
+                    UploadId: uploadId,
+                }))
+                    .catch(() => { });
+            }
+            throw new LTXError('write', key, opts.level, opts.minTXID, opts.maxTXID, err);
+        }
+    }
+    async deleteLtxFiles(infos) {
+        const objects = infos.map((info) => ({
+            Key: this.fileKey(info.level, info.minTXID, info.maxTXID),
+        }));
+        for (let i = 0; i < objects.length; i += 1000) {
+            await this.client.send(new DeleteObjectsCommand({
+                Bucket: this.bucket,
+                Delete: { Objects: objects.slice(i, i + 1000), Quiet: true },
+            }));
+        }
+    }
+    async deleteAll() {
+        const prefix = this.prefix ? `${this.prefix}/` : '';
+        let token;
+        do {
+            const res = await this.client.send(new ListObjectsV2Command({ Bucket: this.bucket, Prefix: prefix, ContinuationToken: token }));
+            const contents = (res.Contents ?? []);
+            const objects = contents.filter((o) => o.Key).map((o) => ({ Key: o.Key }));
+            for (let i = 0; i < objects.length; i += 1000) {
+                await this.client.send(new DeleteObjectsCommand({
+                    Bucket: this.bucket,
+                    Delete: { Objects: objects.slice(i, i + 1000), Quiet: true },
+                }));
+            }
+            token = res.IsTruncated ? (res.NextContinuationToken ?? undefined) : undefined;
+        } while (token);
+    }
+}
+/**
+ * Builds an S3ReplicaClient from parsed replica URL parts. `host` is the bucket,
+ * `urlPath` is the prefix, `query` may carry `region`/`endpoint`, and `userinfo`
+ * carries `accessKeyId:secretAccessKey`.
+ */
+export function newReplicaClientFromURL(_scheme, host, urlPath, query, userinfo) {
+    const opts = {
+        bucket: host,
+        prefix: normalizePrefix(urlPath),
+        region: query?.get('region') ?? undefined,
+        endpoint: query?.get('endpoint') ?? undefined,
+    };
+    if (userinfo) {
+        const [accessKeyId, secretAccessKey] = userinfo.split(':');
+        opts.accessKeyId = accessKeyId;
+        opts.secretAccessKey = secretAccessKey;
+    }
+    return new S3ReplicaClient(opts);
+}

package/package.json

@@ -1,6 +1,50 @@
 {
   "name": "@platformatic/sqlite-js-stream-s3",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
+  "version": "0.1.4",
+  "description": "Amazon S3 upload strategy (replica client) for @platformatic/sqlite-js-stream",
+  "type": "module",
+  "engines": {
+    "node": ">=24"
+  },
+  "license": "Apache-2.0",
+  "author": "Matteo Collina",
+  "homepage": "https://github.com/platformatic/sqlite-js-stream/tree/main/packages/backend-s3",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/platformatic/sqlite-js-stream.git",
+    "directory": "packages/backend-s3"
+  },
+  "bugs": {
+    "url": "https://github.com/platformatic/sqlite-js-stream/issues"
+  },
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org"
+  },
+  "dependencies": {
+    "@aws-sdk/client-s3": "^3.1135.0",
+    "@platformatic/sqlite-js-stream": "0.1.4"
+  },
+  "devDependencies": {
+    "@types/node": "^24.0.0",
+    "typescript": "^5.7.0"
+  },
+  "scripts": {
+    "test": "node --test 'test/**/*.test.ts'",
+    "build": "tsc -p tsconfig.build.json"
+  }
 }

package/src/index.ts

@@ -0,0 +1,23 @@
+/**
+ * @platformatic/sqlite-js-stream-s3 — Amazon S3 upload strategy.
+ *
+ * Importing this module registers the `s3` scheme with the core's replica URL
+ * factory, so `newReplicaClientFromURL('s3://bucket/prefix')` resolves after
+ * this addon is loaded.
+ */
+
+export {
+  S3ReplicaClient,
+  newReplicaClientFromURL,
+  ReplicaClientType,
+  type S3ClientLike,
+  type S3ReplicaClientOptions,
+} from './replica-client.ts';
+
+export { S3Leaser } from './leaser.ts';
+
+import { registerReplicaClientFactory } from '@platformatic/sqlite-js-stream';
+import { newReplicaClientFromURL } from './replica-client.ts';
+
+// Register the 's3' URL scheme when the addon is imported.
+registerReplicaClientFactory('s3', newReplicaClientFromURL);

package/src/leaser.ts

@@ -0,0 +1,202 @@
+/**
+ * S3 Leaser — distributed lease backed by an S3 `lock.json` object.
+ *
+ * Mirrors litestream's `s3/leaser.go`. Acquire uses `If-None-Match: *` when no
+ * lease exists (and `If-Match: <etag>` when replacing an expired lease), so two
+ * racing acquirers cannot both win; one gets `412 PreconditionFailed`.
+ */
+
+import {
+  DeleteObjectCommand,
+  GetObjectCommand,
+  PutObjectCommand,
+} from '@aws-sdk/client-s3';
+
+import {
+  DefaultLeasePath,
+  DefaultLeaseTTL,
+  ErrLeaseNotHeld,
+  Lease,
+  LeaseExistsError,
+  type Leaser,
+} from '@platformatic/sqlite-js-stream';
+
+import type { S3ClientLike } from './replica-client.ts';
+
+/** Reads all bytes from an S3 body (stream or Buffer). */
+async function readBody(body: any): Promise<Uint8Array> {
+  // AWS SDK v3 returns a mix; normalize to a Uint8Array.
+  if (body instanceof Uint8Array) return body;
+  if (typeof body?.on === 'function') {
+    // Node stream.
+    const chunks: Uint8Array[] = [];
+    for await (const chunk of body as AsyncIterable<Uint8Array | string>) {
+      chunks.push(typeof chunk === 'string' ? new TextEncoder().encode(chunk) : chunk);
+    }
+    let total = 0;
+    for (const c of chunks) total += c.length;
+    const out = new Uint8Array(total);
+    let o = 0;
+    for (const c of chunks) {
+      out.set(c, o);
+      o += c.length;
+    }
+    return out;
+  }
+  // Web ReadableStream or Buffer.
+  if (typeof body?.getReader === 'function') {
+    const reader = (body as ReadableStream<Uint8Array>).getReader();
+    const chunks: Uint8Array[] = [];
+    for (;;) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      chunks.push(value);
+    }
+    let total = 0;
+    for (const c of chunks) total += c.length;
+    const out = new Uint8Array(total);
+    let o = 0;
+    for (const c of chunks) {
+      out.set(c, o);
+      o += c.length;
+    }
+    return out;
+  }
+  return new Uint8Array(body as Uint8Array);
+}
+
+function isPreconditionFailed(err: unknown): boolean {
+  const code = (err as { $metadata?: { httpStatusCode?: number } })?.$metadata?.httpStatusCode;
+  const name = (err as { name?: string })?.name ?? '';
+  return code === 412 || name === 'PreconditionFailed';
+}
+
+function isNotFound(err: unknown): boolean {
+  const code = (err as { $metadata?: { httpStatusCode?: number } })?.$metadata?.httpStatusCode;
+  const name = (err as { name?: string })?.name ?? '';
+  return code === 404 || name === 'NoSuchKey' || name === 'NotFound';
+}
+
+/** S3Leaser acquires/renews/releases a lease object in an S3 bucket. */
+export class S3Leaser implements Leaser {
+  readonly bucket: string;
+  readonly prefix: string;
+  readonly key: string;
+  private client: S3ClientLike;
+  private owner: string;
+  private ttlMs: number;
+
+  constructor(opts: { bucket: string; prefix?: string; client: S3ClientLike; owner?: string; ttlMs?: number }) {
+    if (!opts.bucket) throw new Error('S3 leaser: bucket is required');
+    this.bucket = opts.bucket;
+    this.prefix = opts.prefix?.replace(/^\/+/, '').replace(/\/+$/, '') ?? '';
+    this.key = this.prefix ? `${this.prefix}/${DefaultLeasePath}` : DefaultLeasePath;
+    this.client = opts.client;
+    this.owner = opts.owner ?? `${process.pid}`;
+    this.ttlMs = opts.ttlMs ?? DefaultLeaseTTL;
+  }
+
+  async acquireLease(): Promise<Lease> {
+    const existing = await this.readLease();
+    if (existing !== null && !existing.isExpired()) {
+      throw new LeaseExistsError(existing.owner, existing.expiresAt);
+    }
+
+    let generation = 1;
+    if (existing !== null) generation = existing.generation + 1;
+
+    const lease = new Lease(generation, Date.now() + this.ttlMs, this.owner);
+    const data = new TextEncoder().encode(
+      JSON.stringify({
+        generation: lease.generation,
+        expires_at: new Date(lease.expiresAt).toISOString(),
+        owner: lease.owner,
+      }),
+    );
+
+    try {
+      await this.client.send(
+        new PutObjectCommand({
+          Bucket: this.bucket,
+          Key: this.key,
+          Body: data,
+          ContentType: 'application/json',
+          ...(existing === null ? { IfNoneMatch: '*' } : { IfMatch: existing.etag ?? '' }),
+        }),
+      );
+    } catch (err) {
+      if (isPreconditionFailed(err)) {
+        const current = await this.readLease();
+        if (current !== null) throw new LeaseExistsError(current.owner, current.expiresAt);
+      }
+      throw err;
+    }
+
+    return (await this.readLease()) ?? lease;
+  }
+
+  async renewLease(lease: Lease): Promise<Lease> {
+    if (lease.etag == null) throw new ErrLeaseNotHeld();
+    const renewed = new Lease(lease.generation, Date.now() + this.ttlMs, lease.owner);
+    const data = new TextEncoder().encode(
+      JSON.stringify({
+        generation: renewed.generation,
+        expires_at: new Date(renewed.expiresAt).toISOString(),
+        owner: renewed.owner,
+      }),
+    );
+    try {
+      await this.client.send(
+        new PutObjectCommand({
+          Bucket: this.bucket,
+          Key: this.key,
+          Body: data,
+          ContentType: 'application/json',
+          IfMatch: lease.etag,
+        }),
+      );
+    } catch (err) {
+      if (isPreconditionFailed(err)) throw new ErrLeaseNotHeld();
+      throw err;
+    }
+    return (await this.readLease()) ?? renewed;
+  }
+
+  async releaseLease(lease: Lease): Promise<void> {
+    if (lease.etag == null) throw new ErrLeaseNotHeld();
+    try {
+      await this.client.send(
+        new DeleteObjectCommand({
+          Bucket: this.bucket,
+          Key: this.key,
+          IfMatch: lease.etag,
+        }),
+      );
+    } catch (err) {
+      if (isNotFound(err)) return; // already released
+      if (isPreconditionFailed(err)) throw new ErrLeaseNotHeld();
+      throw err;
+    }
+  }
+
+  private async readLease(): Promise<Lease | null> {
+    try {
+      const res = await this.client.send(
+        new GetObjectCommand({ Bucket: this.bucket, Key: this.key }),
+      );
+      const data = await readBody(res.Body);
+      const parsed = JSON.parse(new TextDecoder().decode(data)) as {
+        generation?: number;
+        expires_at?: string;
+        owner?: string;
+      };
+      const expiresAt = Date.parse(parsed.expires_at ?? '');
+      if (Number.isNaN(expiresAt)) throw new Error('invalid lease file: bad expires_at');
+      const etag = typeof res.ETag === 'string' ? res.ETag : null;
+      return new Lease(parsed.generation ?? 1, expiresAt, parsed.owner ?? '', etag);
+    } catch (err) {
+      if (isNotFound(err)) return null;
+      throw err;
+    }
+  }
+}