@hyperframes/aws-lambda 0.8.177 → 0.8.179

package/dist/_chunks/chunk-ZY5YIH5W.js

@@ -0,0 +1,380 @@
+import {
+  CONTENT_STORE_REFRESH_AFTER_DAYS,
+  contentStoreUri
+} from "./chunk-YS3KY3C5.js";
+
+// src/s3Transport.ts
+import {
+  createReadStream,
+  createWriteStream,
+  existsSync,
+  mkdirSync,
+  rmSync,
+  statSync
+} from "node:fs";
+import { createHash } from "node:crypto";
+import { dirname } from "node:path";
+import { pipeline } from "node:stream/promises";
+import {
+  CopyObjectCommand,
+  GetObjectCommand,
+  HeadObjectCommand,
+  PutObjectCommand
+} from "@aws-sdk/client-s3";
+import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
+import * as tar from "tar";
+function parseS3Uri(uri) {
+  if (!uri.startsWith("s3://")) {
+    throw new Error(`[s3Transport] expected s3:// URI, got: ${JSON.stringify(uri)}`);
+  }
+  const rest = uri.slice("s3://".length);
+  const slash = rest.indexOf("/");
+  if (slash === -1) {
+    throw new Error(`[s3Transport] missing key in s3 URI: ${JSON.stringify(uri)}`);
+  }
+  const bucket = rest.slice(0, slash);
+  const key = rest.slice(slash + 1);
+  if (!bucket || !key) {
+    throw new Error(`[s3Transport] empty bucket or key in s3 URI: ${JSON.stringify(uri)}`);
+  }
+  return { bucket, key };
+}
+function formatS3Uri(loc) {
+  return `s3://${loc.bucket}/${loc.key}`;
+}
+async function downloadS3ObjectToFile(client, uri, destPath) {
+  const { bucket, key } = parseS3Uri(uri);
+  const response = await client.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
+  const body = response.Body;
+  if (!body) {
+    throw new Error(`[s3Transport] s3 GetObject returned empty body for ${uri}`);
+  }
+  mkdirSync(dirname(destPath), { recursive: true });
+  await pipeline(body, createWriteStream(destPath));
+}
+async function downloadS3ObjectToFileVerified(client, uri, destPath, expectedSha256) {
+  assertSha256(expectedSha256);
+  await downloadS3ObjectToFile(client, uri, destPath);
+  const actual = await sha256File(destPath);
+  if (actual !== expectedSha256) {
+    rmSync(destPath, { force: true });
+    const error = new Error(
+      `[s3Transport] PLAN_ARTIFACT_DIGEST_MISMATCH: ${uri} expected ${expectedSha256}, got ${actual}`
+    );
+    error.name = "PLAN_ARTIFACT_DIGEST_MISMATCH";
+    throw error;
+  }
+}
+async function presignVerifiedS3GetUrl(client, uri, expectedSha256, expectedSize, expiresInSeconds) {
+  assertSha256(expectedSha256);
+  const { bucket, key } = parseS3Uri(uri);
+  const head = await client.send(
+    new HeadObjectCommand({ Bucket: bucket, Key: key, ChecksumMode: "ENABLED" })
+  );
+  const expectedChecksum = Buffer.from(expectedSha256, "hex").toString("base64");
+  if (head.ChecksumSHA256 !== expectedChecksum || head.ContentLength !== expectedSize) {
+    const error = new Error(
+      `[s3Transport] PLAN_ARTIFACT_DIGEST_MISMATCH: ${uri} expected ${expectedSha256} (${expectedSize} bytes), S3 holds checksum ${String(head.ChecksumSHA256)} (${String(head.ContentLength)} bytes)`
+    );
+    error.name = "PLAN_ARTIFACT_DIGEST_MISMATCH";
+    throw error;
+  }
+  return getSignedUrl(client, new GetObjectCommand({ Bucket: bucket, Key: key }), {
+    expiresIn: expiresInSeconds
+  });
+}
+async function uploadFileToS3(client, localPath, uri, contentType) {
+  if (!existsSync(localPath)) {
+    throw new Error(`[s3Transport] upload source missing: ${localPath}`);
+  }
+  const { bucket, key } = parseS3Uri(uri);
+  const size = statSync(localPath).size;
+  await client.send(
+    new PutObjectCommand({
+      Bucket: bucket,
+      Key: key,
+      Body: createReadStream(localPath),
+      ContentType: contentType,
+      ContentLength: size
+    })
+  );
+}
+async function uploadContentAddressedFileToS3(client, localPath, uri, expectedSha256, contentType) {
+  return (await putImmutableFile(client, localPath, uri, expectedSha256, contentType)).outcome;
+}
+async function putImmutableFile(client, localPath, uri, expectedSha256, contentType) {
+  assertSha256(expectedSha256);
+  if (!existsSync(localPath)) {
+    throw new Error(`[s3Transport] upload source missing: ${localPath}`);
+  }
+  const { bucket, key } = parseS3Uri(uri);
+  const size = statSync(localPath).size;
+  const existing = await inspectContentAddressedObject(client, bucket, key, size, expectedSha256);
+  if (existing.state === "matching")
+    return { outcome: "reused", lastModified: existing.lastModified };
+  if (existing.state === "conflict") throwImmutableObjectConflict(uri);
+  const body = createReadStream(localPath);
+  try {
+    await client.send(
+      new PutObjectCommand({
+        Bucket: bucket,
+        Key: key,
+        Body: body,
+        ContentType: contentType,
+        ContentLength: size,
+        Metadata: { sha256: expectedSha256 },
+        ChecksumSHA256: Buffer.from(expectedSha256, "hex").toString("base64"),
+        // HEAD followed by an unconditional PUT can overwrite a conflicting
+        // object published by a concurrent planner. Conditional create makes
+        // immutable CAS and fixed-key manifest publication race-safe.
+        IfNoneMatch: "*"
+      })
+    );
+    return { outcome: "uploaded" };
+  } catch (error) {
+    if (!isS3PreconditionFailed(error)) throw error;
+    const raced = await inspectContentAddressedObject(client, bucket, key, size, expectedSha256);
+    if (raced.state === "matching") return { outcome: "reused" };
+    if (raced.state === "conflict") throwImmutableObjectConflict(uri);
+    throw error;
+  } finally {
+    body.destroy();
+  }
+}
+async function mapConcurrent(values, concurrency, fn) {
+  let cursor = 0;
+  async function worker() {
+    while (cursor < values.length) {
+      const index = cursor++;
+      await fn(values[index]);
+    }
+  }
+  const results = await Promise.allSettled(
+    Array.from({ length: Math.min(concurrency, values.length) }, () => worker())
+  );
+  const failure = results.find(
+    (result) => result.status === "rejected"
+  );
+  if (failure) throw failure.reason;
+}
+async function sha256File(path) {
+  const hash = createHash("sha256");
+  for await (const chunk of createReadStream(path)) {
+    hash.update(chunk);
+  }
+  return hash.digest("hex");
+}
+function assertSha256(value) {
+  if (!/^[a-f0-9]{64}$/.test(value)) {
+    throw new Error(
+      `[s3Transport] expected lowercase SHA-256 digest, got ${JSON.stringify(value)}`
+    );
+  }
+}
+async function inspectContentAddressedObject(client, bucket, key, expectedSize, expectedSha256) {
+  try {
+    const existing = await client.send(
+      new HeadObjectCommand({ Bucket: bucket, Key: key, ChecksumMode: "ENABLED" })
+    );
+    return existing.ContentLength === expectedSize && existing.Metadata?.sha256 === expectedSha256 ? { state: "matching", lastModified: existing.LastModified } : { state: "conflict" };
+  } catch (error) {
+    if (isS3NotFound(error)) return { state: "missing" };
+    throw error;
+  }
+}
+async function putContentStoreFile(client, localPath, bucket, sha256) {
+  const uri = contentStoreUri(bucket, sha256);
+  const { outcome, lastModified } = await putImmutableFile(client, localPath, uri, sha256);
+  if (lastModified) await refreshAgingObject(client, uri, sha256, lastModified);
+  return outcome;
+}
+async function ensureContentStoreObject(client, bucket, sha256, sizeBytes) {
+  const uri = contentStoreUri(bucket, sha256);
+  const { key } = parseS3Uri(uri);
+  const existing = await inspectContentAddressedObject(client, bucket, key, sizeBytes, sha256);
+  if (existing.state === "matching") {
+    if (existing.lastModified) await refreshAgingObject(client, uri, sha256, existing.lastModified);
+    return;
+  }
+  if (existing.state === "conflict") throwImmutableObjectConflict(uri);
+  const error = new Error(
+    `[s3Transport] PLAN_ARTIFACT_MISSING: ${uri} is not in the content store`
+  );
+  error.name = "PLAN_ARTIFACT_MISSING";
+  throw error;
+}
+async function presignS3GetUrl(client, uri, expiresInSeconds) {
+  const { bucket, key } = parseS3Uri(uri);
+  return getSignedUrl(client, new GetObjectCommand({ Bucket: bucket, Key: key }), {
+    expiresIn: expiresInSeconds
+  });
+}
+async function refreshAgingObject(client, uri, sha256, lastModified) {
+  const { bucket, key } = parseS3Uri(uri);
+  const ageMs = Date.now() - lastModified.getTime();
+  if (ageMs < CONTENT_STORE_REFRESH_AFTER_DAYS * 864e5) return;
+  await client.send(
+    new CopyObjectCommand({
+      Bucket: bucket,
+      Key: key,
+      CopySource: `${bucket}/${key}`,
+      MetadataDirective: "REPLACE",
+      Metadata: { sha256 },
+      ChecksumAlgorithm: "SHA256"
+    })
+  );
+}
+function throwImmutableObjectConflict(uri) {
+  const error = new Error(
+    `[s3Transport] PLAN_ARTIFACT_DIGEST_MISMATCH: immutable object ${uri} already exists with different digest metadata or size`
+  );
+  error.name = "PLAN_ARTIFACT_DIGEST_MISMATCH";
+  throw error;
+}
+async function s3ObjectExists(client, uri) {
+  const { bucket, key } = parseS3Uri(uri);
+  try {
+    await client.send(new HeadObjectCommand({ Bucket: bucket, Key: key }));
+    return true;
+  } catch (error) {
+    if (isS3NotFound(error)) return false;
+    throw error;
+  }
+}
+function isS3NotFound(error) {
+  if (!isRecord(error)) return false;
+  const metadata = isRecord(error.$metadata) ? error.$metadata : void 0;
+  return error.name === "NotFound" || error.name === "NoSuchKey" || metadata?.httpStatusCode === 404;
+}
+function isS3PreconditionFailed(error) {
+  if (!isRecord(error)) return false;
+  const metadata = isRecord(error.$metadata) ? error.$metadata : void 0;
+  return error.name === "PreconditionFailed" || metadata?.httpStatusCode === 412;
+}
+function isRecord(value) {
+  return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+async function tarDirectory(sourceDir, destTarball, files) {
+  if (!existsSync(sourceDir) || !statSync(sourceDir).isDirectory()) {
+    throw new Error(`[s3Transport] tar source must be an existing directory: ${sourceDir}`);
+  }
+  mkdirSync(dirname(destTarball), { recursive: true });
+  await tar.create(
+    { gzip: true, file: destTarball, cwd: sourceDir, follow: files !== void 0 },
+    files ?? ["."]
+  );
+}
+async function untarDirectory(tarballPath, destDir) {
+  if (!existsSync(tarballPath)) {
+    throw new Error(`[s3Transport] tarball missing: ${tarballPath}`);
+  }
+  if (existsSync(destDir)) {
+    rmSync(destDir, { recursive: true, force: true });
+  }
+  mkdirSync(destDir, { recursive: true });
+  await tar.extract({ file: tarballPath, cwd: destDir });
+}
+
+// src/formatExtension.ts
+var FORMAT_EXTENSIONS = {
+  mp4: ".mp4",
+  mov: ".mov",
+  webm: ".webm",
+  "png-sequence": ""
+};
+function formatExtension(format) {
+  return FORMAT_EXTENSIONS[format];
+}
+
+// src/siteManifest.ts
+import { readFileSync } from "node:fs";
+import { extname, join } from "node:path";
+var SITE_MANIFEST_NAME = "files.json";
+var SITE_MANIFEST_INVALID = "SITE_MANIFEST_INVALID";
+var VIDEO_EXTENSIONS_PLANNED_IN_PLACE = /* @__PURE__ */ new Set([".mp4", ".mov", ".m4v", ".webm", ".mkv"]);
+var REMOTE_URL_TTL_SECONDS = 3600;
+function siteManifestUri(bucket, siteId) {
+  return formatS3Uri({ bucket, key: `sites/${siteId}/${SITE_MANIFEST_NAME}` });
+}
+function siteTarballUri(bucket, siteId) {
+  return formatS3Uri({ bucket, key: `sites/${siteId}/project.tar.gz` });
+}
+function isSiteManifestUri(uri) {
+  return uri.endsWith(`/${SITE_MANIFEST_NAME}`);
+}
+function serializeSiteManifest(files) {
+  return JSON.stringify({ files });
+}
+function parseSiteManifest(text) {
+  const value = JSON.parse(text);
+  if (!Array.isArray(value.files)) throw siteManifestError("files must be an array");
+  const seen = /* @__PURE__ */ new Set();
+  return value.files.map((entry, index) => {
+    const { path, sha256, sizeBytes } = entry ?? {};
+    if (typeof path !== "string" || !isSafeRelativePath(path) || seen.has(path)) {
+      throw siteManifestError(`files[${index}].path is not a unique relative path`);
+    }
+    if (typeof sha256 !== "string" || !/^[a-f0-9]{64}$/.test(sha256)) {
+      throw siteManifestError(`files[${index}].sha256 is not a lowercase SHA-256 digest`);
+    }
+    if (!Number.isSafeInteger(sizeBytes) || sizeBytes < 0) {
+      throw siteManifestError(`files[${index}].sizeBytes is not a byte count`);
+    }
+    seen.add(path);
+    return { path, sha256, sizeBytes };
+  });
+}
+async function downloadSiteFiles(s3, manifestUri, projectDir, leaveVideosRemote) {
+  const manifestPath = `${projectDir}.${SITE_MANIFEST_NAME}`;
+  await downloadS3ObjectToFile(s3, manifestUri, manifestPath);
+  const files = parseSiteManifest(readFileSync(manifestPath, "utf-8"));
+  const { bucket } = parseS3Uri(manifestUri);
+  const remoteFiles = {};
+  await mapConcurrent(files, 16, async ({ path, sha256, sizeBytes }) => {
+    const uri = contentStoreUri(bucket, sha256);
+    if (leaveVideosRemote && VIDEO_EXTENSIONS_PLANNED_IN_PLACE.has(extname(path).toLowerCase())) {
+      remoteFiles[path] = {
+        url: await presignS3GetUrl(s3, uri, REMOTE_URL_TTL_SECONDS),
+        sha256,
+        sizeBytes
+      };
+      return;
+    }
+    await downloadS3ObjectToFileVerified(s3, uri, join(projectDir, ...path.split("/")), sha256);
+  });
+  return remoteFiles;
+}
+function isSafeRelativePath(path) {
+  return path.split("/").every((segment) => segment !== "" && segment !== "." && segment !== "..") && !path.includes("\\");
+}
+function siteManifestError(detail) {
+  const error = new Error(`[siteManifest] ${SITE_MANIFEST_INVALID}: ${detail}`);
+  error.name = SITE_MANIFEST_INVALID;
+  return error;
+}
+
+export {
+  parseS3Uri,
+  formatS3Uri,
+  downloadS3ObjectToFile,
+  downloadS3ObjectToFileVerified,
+  presignVerifiedS3GetUrl,
+  uploadFileToS3,
+  uploadContentAddressedFileToS3,
+  mapConcurrent,
+  sha256File,
+  putContentStoreFile,
+  ensureContentStoreObject,
+  s3ObjectExists,
+  isS3NotFound,
+  tarDirectory,
+  untarDirectory,
+  formatExtension,
+  SITE_MANIFEST_INVALID,
+  siteManifestUri,
+  siteTarballUri,
+  isSiteManifestUri,
+  serializeSiteManifest,
+  parseSiteManifest,
+  downloadSiteFiles
+};

package/dist/cdk/index.js

@@ -1,3 +1,8 @@
+import {
+  CONTENT_STORE_PREFIX,
+  CONTENT_STORE_RETENTION_DAYS
+} from "../_chunks/chunk-YS3KY3C5.js";
+
 // src/cdk/HyperframesRenderStack.ts
 import { dirname, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
@@ -42,6 +47,12 @@ var HyperframesRenderStack = class extends Construct {
           enabled: true,
           prefix: "renders/",
           expiration: Duration.days(7)
+        },
+        {
+          id: "ExpireSharedContent",
+          enabled: true,
+          prefix: `${CONTENT_STORE_PREFIX}/`,
+          expiration: Duration.days(CONTENT_STORE_RETENTION_DAYS)
         }
       ]
     });
@@ -149,7 +160,8 @@ var HyperframesRenderStack = class extends Construct {
       "PLAN_ARTIFACT_DIGEST_MISMATCH",
       "FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED",
       "CHUNK_EXCEEDS_WORKER_TIMEOUT",
-      "ChromeBinaryUnavailableError"
+      "ChromeBinaryUnavailableError",
+      "SITE_MANIFEST_INVALID"
     ];
     const NON_RETRYABLE_CHUNK = [
       "FFMPEG_VERSION_MISMATCH",

package/dist/contentStore.d.ts

@@ -0,0 +1,5 @@
+export declare const CONTENT_STORE_PREFIX = "cas/sha256";
+export declare const CONTENT_STORE_RETENTION_DAYS = 30;
+/** Reusing an object at least this old rewrites it, so a plan's objects outlive it by 23+ days. */
+export declare const CONTENT_STORE_REFRESH_AFTER_DAYS = 7;
+export declare function contentStoreUri(bucket: string, sha256: string): string;