@cloudbitmaps/core 0.1.0-rc.0

package/dist/s3/index.js

@@ -0,0 +1,555 @@
+import { mapWithConcurrency } from '../chunk-SNJVZ227.js';
+import { errorName, httpStatus, isServerSide, isNetworkOrTimeout, isSdkRetryable } from '../chunk-7LMLYSVJ.js';
+import { validateNewRegistryRecord, registryCounterOf, recordFromNew, validateRegistryPatch, applyRegistryPatch, parseRegistryEnvelope, serializeRegistryEnvelope } from '../chunk-2YDULGXS.js';
+import { WriteConflictError, isValidationError, isWriteConflictError, isNotFoundError, ValidationError, NotFoundError, TransientError, IntegrityError, validateSegmentRef, namespacePart, DEFAULT_NAMESPACE } from '../chunk-NUIDEEFZ.js';
+import { createHash } from 'crypto';
+import { GetObjectCommand, HeadObjectCommand, DeleteObjectCommand, ListObjectsV2Command, CreateMultipartUploadCommand, UploadPartCommand, PutObjectCommand, CompleteMultipartUploadCommand, AbortMultipartUploadCommand } from '@aws-sdk/client-s3';
+
+// src/drivers/s3/keys.ts
+var SUFFIX = ".crbm";
+var REGISTRY_SUFFIX = ".reg";
+function prefixPart(prefix) {
+  if (prefix === void 0) return "";
+  const trimmed = prefix.replace(/^\/+|\/+$/g, "");
+  return trimmed === "" ? "" : `${trimmed}/`;
+}
+function normalizeS3Prefix(prefix) {
+  if (prefix === void 0) return void 0;
+  for (const ch of prefix) {
+    if (ch.charCodeAt(0) < 32) {
+      throw new ValidationError("prefix must not contain control characters");
+    }
+  }
+  for (const segment of prefix.split("/")) {
+    if (segment === "." || segment === "..") {
+      throw new ValidationError(
+        `prefix must not contain "." or ".." path segments: ${JSON.stringify(prefix)}`
+      );
+    }
+  }
+  return prefix;
+}
+function segmentObjectPrefix(prefix, ref) {
+  validateSegmentRef(ref);
+  return `${prefixPart(prefix)}${namespacePart(ref.namespace)}/segments/${ref.segment}.`;
+}
+function coldObjectKey(prefix, key) {
+  if (!Number.isInteger(key.generation) || key.generation < 0) {
+    throw new ValidationError(`generation must be a non-negative integer; got ${key.generation}`);
+  }
+  return `${segmentObjectPrefix(prefix, key)}${key.generation}${SUFFIX}`;
+}
+function registryPrefix(prefix) {
+  return `${prefixPart(prefix)}registry/`;
+}
+function registryObjectKey(prefix, ref) {
+  validateSegmentRef(ref);
+  return `${registryPrefix(prefix)}${namespacePart(ref.namespace)}/${ref.segment}${REGISTRY_SUFFIX}`;
+}
+function registryListPrefix(prefix, namespace) {
+  const base = registryPrefix(prefix);
+  return namespace === void 0 ? base : `${base}${namespacePart(namespace)}/`;
+}
+function parseRegistryKey(prefix, objectKey) {
+  const base = registryPrefix(prefix);
+  if (!objectKey.startsWith(base) || !objectKey.endsWith(REGISTRY_SUFFIX)) return null;
+  const rest = objectKey.slice(base.length, objectKey.length - REGISTRY_SUFFIX.length);
+  const slash = rest.indexOf("/");
+  if (slash < 0) return null;
+  const nsPart = rest.slice(0, slash);
+  const segment = rest.slice(slash + 1);
+  if (segment.length === 0 || segment.includes("/")) return null;
+  const namespace = nsPart === DEFAULT_NAMESPACE ? void 0 : nsPart;
+  const ref = { segment, namespace };
+  try {
+    validateSegmentRef(ref);
+  } catch {
+    return null;
+  }
+  return ref;
+}
+function parseGenerationFromKey(segmentPrefix, objectKey) {
+  if (!objectKey.startsWith(segmentPrefix) || !objectKey.endsWith(SUFFIX)) return null;
+  const middle = objectKey.slice(segmentPrefix.length, objectKey.length - SUFFIX.length);
+  if (!/^(0|[1-9]\d*)$/.test(middle)) return null;
+  const generation = Number(middle);
+  return Number.isSafeInteger(generation) ? generation : null;
+}
+
+// src/drivers/s3/s3-errors.ts
+function isPreconditionFailed(err) {
+  return errorName(err) === "PreconditionFailed" || httpStatus(err) === 412;
+}
+function isConditionalConflict(err) {
+  return isPreconditionFailed(err) || errorName(err) === "ConditionalRequestConflict" || httpStatus(err) === 409;
+}
+function isNotFound(err) {
+  const name = errorName(err);
+  return name === "NoSuchKey" || name === "NotFound" || httpStatus(err) === 404;
+}
+function isInvalidRange(err) {
+  return errorName(err) === "InvalidRange" || httpStatus(err) === 416;
+}
+function isTransient(err) {
+  if (isConditionalConflict(err) || isNotFound(err) || isInvalidRange(err)) return false;
+  return errorName(err) === "SlowDown" || isServerSide(err) || isNetworkOrTimeout(err) || isSdkRetryable(err);
+}
+function totalFromContentRange(contentRange) {
+  if (contentRange === void 0) return void 0;
+  const match = /\/(\d+)\s*$/.exec(contentRange);
+  if (match === null) return void 0;
+  const total = Number(match[1]);
+  return Number.isSafeInteger(total) ? total : void 0;
+}
+
+// src/drivers/s3/cold.ts
+var S3_PART_BYTES = 8 * 1024 * 1024;
+var S3_MAX_PARTS = 1e4;
+var S3ColdDriver = class {
+  client;
+  bucket;
+  prefix;
+  maxObjectBytes;
+  partBytes;
+  constructor(options) {
+    this.client = options.client;
+    this.bucket = options.bucket;
+    this.prefix = normalizeS3Prefix(options.prefix);
+    const requestedPart = Math.max(options.partBytes ?? S3_PART_BYTES, 5 * 1024 * 1024);
+    this.maxObjectBytes = options.maxObjectBytes ?? requestedPart * S3_MAX_PARTS;
+    this.partBytes = Math.max(requestedPart, Math.ceil(this.maxObjectBytes / S3_MAX_PARTS));
+  }
+  capabilities() {
+    return { rangeRead: true, maxObjectBytes: this.maxObjectBytes, conditionalPut: true };
+  }
+  async putImmutable(key, write) {
+    const objectKey = coldObjectKey(this.prefix, key);
+    const sink = new S3MultipartSink(
+      this.client,
+      this.bucket,
+      objectKey,
+      this.partBytes,
+      this.maxObjectBytes
+    );
+    try {
+      await write(sink);
+      return await sink.finish();
+    } catch (err) {
+      await sink.abort();
+      if (isConditionalConflict(err)) {
+        throw new WriteConflictError(
+          `generation already exists (write-once): ${key.segment}.${key.generation}`
+        );
+      }
+      if (isValidationError(err) || isWriteConflictError(err) || isNotFoundError(err)) {
+        throw err;
+      }
+      throw this.mapError(err);
+    }
+  }
+  async getRange(key, offset, length) {
+    if (!Number.isInteger(offset) || !Number.isInteger(length) || offset < 0 || length < 0) {
+      throw new ValidationError(`invalid range offset=${offset} length=${length}`);
+    }
+    const objectKey = coldObjectKey(this.prefix, key);
+    if (length === 0) return new Uint8Array(0);
+    try {
+      const res = await this.client.send(
+        new GetObjectCommand({
+          Bucket: this.bucket,
+          Key: objectKey,
+          Range: `bytes=${offset}-${offset + length - 1}`
+        })
+      );
+      const bytes = await collect(res.Body);
+      if (bytes.length !== length) {
+        throw new ValidationError(
+          `range [${offset}, ${offset + length}) out of bounds (got ${bytes.length}B)`
+        );
+      }
+      return bytes;
+    } catch (err) {
+      throw this.mapReadError(err, key);
+    }
+  }
+  async getTail(key, maxBytes) {
+    const objectKey = coldObjectKey(this.prefix, key);
+    if (maxBytes <= 0) {
+      try {
+        const head = await this.client.send(
+          new HeadObjectCommand({ Bucket: this.bucket, Key: objectKey })
+        );
+        return { bytes: new Uint8Array(0), size: head.ContentLength ?? 0 };
+      } catch (err) {
+        throw this.mapReadError(err, key);
+      }
+    }
+    try {
+      const res = await this.client.send(
+        new GetObjectCommand({ Bucket: this.bucket, Key: objectKey, Range: `bytes=-${maxBytes}` })
+      );
+      const bytes = await collect(res.Body);
+      let size = totalFromContentRange(res.ContentRange);
+      if (size === void 0) {
+        if (bytes.length === maxBytes) {
+          const head = await this.client.send(
+            new HeadObjectCommand({ Bucket: this.bucket, Key: objectKey })
+          );
+          size = head.ContentLength ?? bytes.length;
+        } else {
+          size = bytes.length;
+        }
+      }
+      return { bytes, size };
+    } catch (err) {
+      throw this.mapReadError(err, key);
+    }
+  }
+  async delete(key) {
+    try {
+      await this.client.send(
+        new DeleteObjectCommand({ Bucket: this.bucket, Key: coldObjectKey(this.prefix, key) })
+      );
+    } catch (err) {
+      throw this.mapError(err);
+    }
+  }
+  async *list(ref) {
+    const prefix = segmentObjectPrefix(this.prefix, ref);
+    let token;
+    do {
+      let res;
+      try {
+        res = await this.client.send(
+          new ListObjectsV2Command({
+            Bucket: this.bucket,
+            Prefix: prefix,
+            ContinuationToken: token
+          })
+        );
+      } catch (err) {
+        throw this.mapError(err);
+      }
+      for (const obj of res.Contents ?? []) {
+        if (obj.Key === void 0) continue;
+        const generation = parseGenerationFromKey(prefix, obj.Key);
+        if (generation !== null) {
+          yield { namespace: ref.namespace, segment: ref.segment, generation };
+        }
+      }
+      token = res.IsTruncated === true ? res.NextContinuationToken : void 0;
+    } while (token !== void 0);
+  }
+  /** Map S3 read errors to the driver vocabulary; pass everything else through {@link mapError}. */
+  mapReadError(err, key) {
+    if (isValidationError(err)) return err;
+    if (isNotFound(err)) {
+      return new NotFoundError(`no such generation: ${key.segment}.${key.generation}`);
+    }
+    if (isInvalidRange(err)) {
+      return new ValidationError(`range out of bounds for ${key.segment}.${key.generation}`);
+    }
+    return this.mapError(err);
+  }
+  /**
+   * Reclassify a transient S3 fault (throttle/5xx/dropped connection) as a retryable {@link TransientError}
+   * so the retry decorator can ride it out; everything else propagates unchanged. The final fallback at every
+   * `client.send` site, so callers and the decorator only ever see typed errors.
+   */
+  mapError(err) {
+    if (isTransient(err)) {
+      return new TransientError(
+        `transient S3 fault: ${err?.name ?? "unknown"}`,
+        { cause: err }
+      );
+    }
+    return err;
+  }
+};
+function concatBytes(parts, total) {
+  const out = new Uint8Array(total);
+  let offset = 0;
+  for (const p of parts) {
+    out.set(p, offset);
+    offset += p.length;
+  }
+  return out;
+}
+var S3MultipartSink = class {
+  constructor(client, bucket, objectKey, partBytes, maxObjectBytes) {
+    this.client = client;
+    this.bucket = bucket;
+    this.objectKey = objectKey;
+    this.partBytes = partBytes;
+    this.maxObjectBytes = maxObjectBytes;
+  }
+  client;
+  bucket;
+  objectKey;
+  partBytes;
+  maxObjectBytes;
+  hash = createHash("sha256");
+  pending = [];
+  pendingLen = 0;
+  total = 0;
+  uploadId;
+  partNumber = 0;
+  parts = [];
+  async write(bytes) {
+    if (bytes.length === 0) return;
+    this.total += bytes.length;
+    if (this.total > this.maxObjectBytes) {
+      throw new ValidationError(`object exceeds maxObjectBytes ${this.maxObjectBytes}`);
+    }
+    this.hash.update(bytes);
+    this.pending.push(bytes);
+    this.pendingLen += bytes.length;
+    if (this.pendingLen >= this.partBytes) await this.flushPart();
+  }
+  /** Upload the buffered bytes (≥ one part) as a single part, freeing them. Starts the upload on first call. */
+  async flushPart() {
+    if (this.uploadId === void 0) {
+      const res2 = await this.client.send(
+        new CreateMultipartUploadCommand({ Bucket: this.bucket, Key: this.objectKey })
+      );
+      if (res2.UploadId === void 0) {
+        throw new TransientError("S3 CreateMultipartUpload returned no UploadId");
+      }
+      this.uploadId = res2.UploadId;
+    }
+    const body = concatBytes(this.pending, this.pendingLen);
+    this.pending.length = 0;
+    this.pendingLen = 0;
+    this.partNumber += 1;
+    if (this.partNumber > S3_MAX_PARTS) {
+      throw new ValidationError(`multipart upload exceeded the S3 ${S3_MAX_PARTS}-part limit`);
+    }
+    const res = await this.client.send(
+      new UploadPartCommand({
+        Bucket: this.bucket,
+        Key: this.objectKey,
+        UploadId: this.uploadId,
+        PartNumber: this.partNumber,
+        Body: body
+      })
+    );
+    this.parts.push({ ETag: res.ETag, PartNumber: this.partNumber });
+  }
+  /** Commit the object: a single conditional PUT if it fit in one part, else complete the multipart upload. */
+  async finish() {
+    const sha256 = this.hash.digest("hex");
+    if (this.uploadId === void 0) {
+      await this.client.send(
+        new PutObjectCommand({
+          Bucket: this.bucket,
+          Key: this.objectKey,
+          Body: concatBytes(this.pending, this.pendingLen),
+          IfNoneMatch: "*"
+          // write-once
+        })
+      );
+      return { size: this.total, sha256 };
+    }
+    if (this.pendingLen > 0) await this.flushPart();
+    await this.client.send(
+      new CompleteMultipartUploadCommand({
+        Bucket: this.bucket,
+        Key: this.objectKey,
+        UploadId: this.uploadId,
+        MultipartUpload: { Parts: this.parts },
+        IfNoneMatch: "*"
+        // write-once: fail if the object already exists
+      })
+    );
+    this.uploadId = void 0;
+    return { size: this.total, sha256 };
+  }
+  /** Best-effort cleanup of an in-flight multipart upload after an error (a leaked MPU is reaped by a bucket
+   * lifecycle rule; never a correctness issue). No-op if nothing was started or it already completed. */
+  async abort() {
+    if (this.uploadId === void 0) return;
+    const id = this.uploadId;
+    this.uploadId = void 0;
+    try {
+      await this.client.send(
+        new AbortMultipartUploadCommand({ Bucket: this.bucket, Key: this.objectKey, UploadId: id })
+      );
+    } catch {
+    }
+  }
+};
+async function collect(body) {
+  if (body === void 0) {
+    throw new NotFoundError("S3 GetObject returned an empty body");
+  }
+  return body.transformToByteArray();
+}
+var MAX_ROW_BYTES = 1 * 1024 * 1024;
+var MAX_DELETE_ATTEMPTS = 8;
+var LIST_READ_CONCURRENCY = 16;
+var S3RegistryDriver = class {
+  client;
+  bucket;
+  prefix;
+  now;
+  constructor(options) {
+    this.client = options.client;
+    this.bucket = options.bucket;
+    this.prefix = normalizeS3Prefix(options.prefix);
+    this.now = options.now ?? (() => Date.now());
+  }
+  capabilities() {
+    return { strongRead: true };
+  }
+  async get(ref) {
+    const current = await this.readRow(registryObjectKey(this.prefix, ref));
+    return current && !current.env.deleted ? current.env.record : null;
+  }
+  async create(ref, record) {
+    validateNewRegistryRecord(record);
+    const key = registryObjectKey(this.prefix, ref);
+    const current = await this.readRow(key);
+    if (current !== null && !current.env.deleted) {
+      throw new WriteConflictError(`registry row already exists for segment ${ref.segment}`);
+    }
+    const token = String(current ? registryCounterOf(current.env.record) + 1 : 0);
+    const env = {
+      deleted: false,
+      record: recordFromNew(ref, record, this.now(), token)
+    };
+    await this.putRow(key, env, current ? { ifMatch: current.etag } : { ifNoneMatch: "*" });
+    return { token };
+  }
+  async compareAndSwap(ref, expected, patch) {
+    validateRegistryPatch(patch);
+    const key = registryObjectKey(this.prefix, ref);
+    const current = await this.readRow(key);
+    if (current === null || current.env.deleted || current.env.record.token !== expected) {
+      throw new WriteConflictError(`OCC token mismatch for registry row ${ref.segment}`);
+    }
+    const token = String(registryCounterOf(current.env.record) + 1);
+    const env = {
+      deleted: false,
+      record: applyRegistryPatch(current.env.record, patch, this.now(), token)
+    };
+    await this.putRow(key, env, { ifMatch: current.etag });
+    return { token };
+  }
+  async *list(namespace) {
+    const prefix = registryListPrefix(this.prefix, namespace);
+    let token;
+    do {
+      let res;
+      try {
+        res = await this.client.send(
+          new ListObjectsV2Command({
+            Bucket: this.bucket,
+            Prefix: prefix,
+            ContinuationToken: token
+          })
+        );
+      } catch (err) {
+        throw this.mapError(err);
+      }
+      const keys = (res.Contents ?? []).map((obj) => obj.Key).filter((k) => k !== void 0 && parseRegistryKey(this.prefix, k) !== null);
+      const rows = await mapWithConcurrency(
+        keys,
+        LIST_READ_CONCURRENCY,
+        (key) => this.readRow(key)
+      );
+      for (const current of rows) {
+        if (current && !current.env.deleted) yield current.env.record;
+      }
+      token = res.IsTruncated === true ? res.NextContinuationToken : void 0;
+    } while (token !== void 0);
+  }
+  async delete(ref) {
+    const key = registryObjectKey(this.prefix, ref);
+    for (let attempt = 0; attempt < MAX_DELETE_ATTEMPTS; attempt++) {
+      const current = await this.readRow(key);
+      if (current === null || current.env.deleted) return;
+      const token = String(registryCounterOf(current.env.record) + 1);
+      const env = {
+        deleted: true,
+        record: { ...current.env.record, token, updatedAt: this.now() }
+      };
+      try {
+        await this.putRow(key, env, { ifMatch: current.etag });
+        return;
+      } catch (err) {
+        if (isWriteConflictError(err)) continue;
+        throw err;
+      }
+    }
+    throw new WriteConflictError(
+      `registry delete: contention tombstoning "${ref.segment}" \u2014 retry`
+    );
+  }
+  /** GET + parse a registry object, returning its envelope + ETag (for a later If-Match), or null if absent. */
+  async readRow(objectKey) {
+    let res;
+    try {
+      res = await this.client.send(new GetObjectCommand({ Bucket: this.bucket, Key: objectKey }));
+    } catch (err) {
+      if (isNotFound(err)) return null;
+      throw this.mapError(err);
+    }
+    if ((res.ContentLength ?? 0) > MAX_ROW_BYTES) {
+      throw new IntegrityError(
+        `registry object ${res.ContentLength}B exceeds cap ${MAX_ROW_BYTES}B`
+      );
+    }
+    if (res.Body === void 0) {
+      throw new IntegrityError(`registry object has an empty body: ${objectKey}`);
+    }
+    const bytes = await res.Body.transformToByteArray();
+    if (bytes.length > MAX_ROW_BYTES) {
+      throw new IntegrityError(`registry object ${bytes.length}B exceeds cap ${MAX_ROW_BYTES}B`);
+    }
+    const env = parseRegistryEnvelope(Buffer.from(bytes).toString("utf8"), objectKey);
+    if (res.ETag === void 0 || res.ETag.length === 0) {
+      throw new IntegrityError(
+        `registry object has no ETag (needed for compare-and-swap): ${objectKey}`
+      );
+    }
+    return { env, etag: res.ETag };
+  }
+  /** Conditional PUT of an envelope; maps a `412` precondition failure to {@link WriteConflictError}. */
+  async putRow(objectKey, env, cond) {
+    const Body = Buffer.from(serializeRegistryEnvelope(env), "utf8");
+    if (Body.length > MAX_ROW_BYTES) {
+      throw new ValidationError(`registry object ${Body.length}B exceeds cap ${MAX_ROW_BYTES}B`);
+    }
+    try {
+      await this.client.send(
+        new PutObjectCommand({
+          Bucket: this.bucket,
+          Key: objectKey,
+          Body,
+          ContentType: "application/json",
+          IfNoneMatch: "ifNoneMatch" in cond ? cond.ifNoneMatch : void 0,
+          IfMatch: "ifMatch" in cond ? cond.ifMatch : void 0
+        })
+      );
+    } catch (err) {
+      if (isConditionalConflict(err)) {
+        throw new WriteConflictError(`registry OCC conflict for ${objectKey}`);
+      }
+      throw this.mapError(err);
+    }
+  }
+  /** Reclassify a transient S3 fault as a retryable {@link TransientError}; pass everything else through. */
+  mapError(err) {
+    if (isTransient(err)) {
+      return new TransientError(
+        `transient S3 fault: ${err?.name ?? "unknown"}`,
+        { cause: err }
+      );
+    }
+    return err;
+  }
+};
+
+export { S3ColdDriver, S3RegistryDriver };
+//# sourceMappingURL=index.js.map
+//# sourceMappingURL=index.js.map