@hyperframes/aws-lambda 0.7.70 → 0.7.72

package/dist/index.js

@@ -5,7 +5,11 @@ import { basename, join } from "node:path";
 import { S3Client } from "@aws-sdk/client-s3";
 import {
   assemble,
+  listPlanV2ArtifactsForTarget,
+  materializePlanV2Target,
   plan,
+  planV2,
+  readPlanV2Manifest,
   renderChunk
 } from "@hyperframes/producer/distributed";
 
@@ -95,9 +99,14 @@ import {
   rmSync,
   statSync
 } from "node:fs";
+import { createHash } from "node:crypto";
 import { dirname } from "node:path";
 import { pipeline } from "node:stream/promises";
-import { GetObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3";
+import {
+  GetObjectCommand,
+  HeadObjectCommand,
+  PutObjectCommand
+} from "@aws-sdk/client-s3";
 import * as tar from "tar";
 function parseS3Uri(uri) {
   if (!uri.startsWith("s3://")) {
@@ -128,6 +137,19 @@ async function downloadS3ObjectToFile(client, uri, destPath) {
   mkdirSync(dirname(destPath), { recursive: true });
   await pipeline(body, createWriteStream(destPath));
 }
+async function downloadS3ObjectToFileVerified(client, uri, destPath, expectedSha256) {
+  assertSha256(expectedSha256);
+  await downloadS3ObjectToFile(client, uri, destPath);
+  const actual = await sha256File(destPath);
+  if (actual !== expectedSha256) {
+    rmSync(destPath, { force: true });
+    const error = new Error(
+      `[s3Transport] PLAN_ARTIFACT_DIGEST_MISMATCH: ${uri} expected ${expectedSha256}, got ${actual}`
+    );
+    error.name = "PLAN_ARTIFACT_DIGEST_MISMATCH";
+    throw error;
+  }
+}
 async function uploadFileToS3(client, localPath, uri, contentType) {
   if (!existsSync2(localPath)) {
     throw new Error(`[s3Transport] upload source missing: ${localPath}`);
@@ -144,6 +166,68 @@ async function uploadFileToS3(client, localPath, uri, contentType) {
     })
   );
 }
+async function uploadContentAddressedFileToS3(client, localPath, uri, expectedSha256, contentType) {
+  assertSha256(expectedSha256);
+  if (!existsSync2(localPath)) {
+    throw new Error(`[s3Transport] upload source missing: ${localPath}`);
+  }
+  const actualSha256 = await sha256File(localPath);
+  if (actualSha256 !== expectedSha256) {
+    const error = new Error(
+      `[s3Transport] PLAN_ARTIFACT_DIGEST_MISMATCH: local artifact ${localPath} expected ${expectedSha256}, got ${actualSha256}`
+    );
+    error.name = "PLAN_ARTIFACT_DIGEST_MISMATCH";
+    throw error;
+  }
+  const { bucket, key } = parseS3Uri(uri);
+  const size = statSync(localPath).size;
+  try {
+    const existing = await client.send(
+      new HeadObjectCommand({ Bucket: bucket, Key: key, ChecksumMode: "ENABLED" })
+    );
+    if (existing.ContentLength === size && existing.Metadata?.sha256 === expectedSha256) {
+      return "reused";
+    }
+    const error = new Error(
+      `[s3Transport] PLAN_ARTIFACT_DIGEST_MISMATCH: immutable object ${uri} already exists with different digest metadata or size`
+    );
+    error.name = "PLAN_ARTIFACT_DIGEST_MISMATCH";
+    throw error;
+  } catch (error) {
+    if (!isS3NotFound(error)) throw error;
+  }
+  await client.send(
+    new PutObjectCommand({
+      Bucket: bucket,
+      Key: key,
+      Body: createReadStream(localPath),
+      ContentType: contentType,
+      ContentLength: size,
+      Metadata: { sha256: expectedSha256 },
+      ChecksumSHA256: Buffer.from(expectedSha256, "hex").toString("base64")
+    })
+  );
+  return "uploaded";
+}
+async function sha256File(path) {
+  const hash = createHash("sha256");
+  for await (const chunk of createReadStream(path)) {
+    hash.update(chunk);
+  }
+  return hash.digest("hex");
+}
+function assertSha256(value) {
+  if (!/^[a-f0-9]{64}$/.test(value)) {
+    throw new Error(
+      `[s3Transport] expected lowercase SHA-256 digest, got ${JSON.stringify(value)}`
+    );
+  }
+}
+function isS3NotFound(error) {
+  if (!error || typeof error !== "object") return false;
+  const candidate = error;
+  return candidate.name === "NotFound" || candidate.name === "NoSuchKey" || candidate.$metadata?.httpStatusCode === 404;
+}
 async function tarDirectory(sourceDir, destTarball) {
   if (!existsSync2(sourceDir) || !statSync(sourceDir).isDirectory()) {
     throw new Error(`[s3Transport] tar source must be an existing directory: ${sourceDir}`);
@@ -192,15 +276,24 @@ async function handler(event, deps) {
       }
     }
   } catch (err) {
+    normalizeTerminalErrorName(err);
     logEvent({
       event: "handler_error",
       action: unwrapped.Action,
+      input: summarizeEvent(unwrapped),
       message: err instanceof Error ? err.message : String(err),
       name: err instanceof Error ? err.name : void 0
     });
     throw err;
   }
 }
+function normalizeTerminalErrorName(error) {
+  if (!error || typeof error !== "object") return;
+  const candidate = error;
+  if (candidate.code === "PLAN_PROTOCOL_UNSUPPORTED" || candidate.code === "PLAN_TOO_LARGE" || candidate.code === "PLAN_V2_INTEGRITY_UNRECOVERABLE") {
+    candidate.name = candidate.code;
+  }
+}
 var MAX_ENVELOPE_DEPTH = 4;
 function unwrapEvent(event) {
   let cursor = event;
@@ -237,18 +330,21 @@ function summarizeEvent(event) {
       return {
         projectS3Uri: event.ProjectS3Uri,
         planOutputS3Prefix: event.PlanOutputS3Prefix,
+        planProtocol: event.PlanProtocol ?? "v1",
         format: event.Config.format,
         fps: event.Config.fps
       };
     case "renderChunk":
       return {
-        planS3Uri: event.PlanS3Uri,
+        planProtocol: event.PlanProtocol ?? "v1",
+        ...event.PlanProtocol === "v2" ? { planV2ManifestS3Uri: event.PlanV2ManifestS3Uri } : { planS3Uri: event.PlanS3Uri },
         chunkIndex: event.ChunkIndex,
         format: event.Format
       };
     case "assemble":
       return {
-        planS3Uri: event.PlanS3Uri,
+        planProtocol: event.PlanProtocol ?? "v1",
+        ...event.PlanProtocol === "v2" ? { planV2ManifestS3Uri: event.PlanV2ManifestS3Uri } : { planS3Uri: event.PlanS3Uri },
         chunkCount: event.ChunkS3Uris.length,
         hasAudio: event.AudioS3Uri !== null,
         outputS3Uri: event.OutputS3Uri,
@@ -267,6 +363,9 @@ function primeRuntimeEnv() {
   }
 }
 async function handlePlan(event, deps) {
+  if (event.PlanProtocol === "v2") {
+    return handlePlanV2(event, deps);
+  }
   const started = Date.now();
   const s3 = deps?.s3 ?? getS3Client();
   const primitive = deps?.primitives?.plan ?? plan;
@@ -315,7 +414,73 @@ async function handlePlan(event, deps) {
     cleanupDir(work);
   }
 }
+async function handlePlanV2(event, deps) {
+  const started = Date.now();
+  const s3 = deps?.s3 ?? getS3Client();
+  const primitive = deps?.primitives?.planV2 ?? planV2;
+  if (!deps?.skipChromeResolution && !process.env.PRODUCER_HEADLESS_SHELL_PATH) {
+    process.env.PRODUCER_HEADLESS_SHELL_PATH = await resolveChromeExecutablePath();
+  }
+  const work = mkdtempSync(join(deps?.tmpRoot ?? tmpdir(), "hf-lambda-plan-v2-"));
+  const projectArchive = join(work, "project.tar.gz");
+  const projectDir = join(work, "project");
+  const planV2Dir = join(work, "plan-v2");
+  try {
+    await downloadS3ObjectToFile(s3, event.ProjectS3Uri, projectArchive);
+    await untarDirectory(projectArchive, projectDir);
+    const result = await primitive(projectDir, { ...event.Config }, planV2Dir);
+    const manifest = readPlanV2Manifest(planV2Dir);
+    if (manifest.planHash !== result.planHash) {
+      throwPlanHashMismatch(result.planHash, manifest.planHash);
+    }
+    const outputPrefix = `${trimTrailingSlash(event.PlanOutputS3Prefix)}/v2`;
+    const artifactPrefix = `${outputPrefix}/artifacts/sha256`;
+    const uniqueArtifacts = [
+      ...new Map(manifest.artifacts.map((artifact) => [artifact.sha256, artifact])).values()
+    ];
+    await mapConcurrent(uniqueArtifacts, 16, async (artifact) => {
+      const localPath = planV2BlobPath(planV2Dir, artifact.sha256);
+      await uploadContentAddressedFileToS3(
+        s3,
+        localPath,
+        planV2BlobUri(artifactPrefix, artifact.sha256),
+        artifact.sha256
+      );
+    });
+    const manifestUri = `${outputPrefix}/manifest.json`;
+    await uploadContentAddressedFileToS3(
+      s3,
+      result.manifestPath,
+      manifestUri,
+      await sha256File(result.manifestPath),
+      "application/json"
+    );
+    return {
+      Action: "plan",
+      PlanProtocol: "v2",
+      PlanV2ManifestS3Uri: manifestUri,
+      PlanV2ArtifactS3Prefix: artifactPrefix,
+      PlanHash: result.planHash,
+      ChunkCount: result.chunkCount,
+      TotalFrames: result.totalFrames,
+      Fps: result.fps,
+      Width: result.width,
+      Height: result.height,
+      Format: result.format,
+      HasAudio: manifest.artifacts.some((artifact) => artifact.path === "audio.aac"),
+      AudioS3Uri: null,
+      FfmpegVersion: result.ffmpegVersion,
+      ProducerVersion: result.producerVersion,
+      DurationMs: Date.now() - started
+    };
+  } finally {
+    cleanupDir(work);
+  }
+}
 async function handleRenderChunk(event, deps) {
+  if (event.PlanProtocol === "v2") {
+    return handleRenderChunkV2(event, deps);
+  }
   const started = Date.now();
   const s3 = deps?.s3 ?? getS3Client();
   const primitive = deps?.primitives?.renderChunk ?? renderChunk;
@@ -353,6 +518,44 @@ async function handleRenderChunk(event, deps) {
     cleanupDir(work);
   }
 }
+async function handleRenderChunkV2(event, deps) {
+  const started = Date.now();
+  const s3 = deps?.s3 ?? getS3Client();
+  const primitive = deps?.primitives?.renderChunk ?? renderChunk;
+  if (!deps?.skipChromeResolution && !process.env.PRODUCER_HEADLESS_SHELL_PATH) {
+    process.env.PRODUCER_HEADLESS_SHELL_PATH = await resolveChromeExecutablePath();
+  }
+  const work = mkdtempSync(join(deps?.tmpRoot ?? tmpdir(), "hf-lambda-chunk-v2-"));
+  try {
+    const planDir = await downloadAndMaterializePlanV2(
+      s3,
+      event,
+      { role: "chunk", chunkIndex: event.ChunkIndex },
+      work
+    );
+    const chunkOutputBase = join(
+      work,
+      event.Format === "png-sequence" ? `chunk-${pad(event.ChunkIndex)}` : `chunk-${pad(event.ChunkIndex)}${formatExtension(event.Format)}`
+    );
+    const result = await primitive(planDir, event.ChunkIndex, chunkOutputBase);
+    const chunkUri = await uploadChunkOutput(
+      s3,
+      result,
+      event.ChunkOutputS3Prefix,
+      event.ChunkIndex
+    );
+    return {
+      Action: "renderChunk",
+      ChunkS3Uri: chunkUri,
+      ChunkIndex: event.ChunkIndex,
+      Sha256: result.sha256,
+      FramesEncoded: result.framesEncoded,
+      DurationMs: Date.now() - started
+    };
+  } finally {
+    cleanupDir(work);
+  }
+}
 async function uploadChunkOutput(s3, result, prefix, chunkIndex) {
   const trimmed = trimTrailingSlash(prefix);
   if (result.outputKind === "file") {
@@ -368,6 +571,9 @@ async function uploadChunkOutput(s3, result, prefix, chunkIndex) {
   return uri;
 }
 async function handleAssemble(event, deps) {
+  if (event.PlanProtocol === "v2") {
+    return handleAssembleV2(event, deps);
+  }
   const started = Date.now();
   const s3 = deps?.s3 ?? getS3Client();
   const primitive = deps?.primitives?.assemble ?? assemble;
@@ -405,6 +611,87 @@ async function handleAssemble(event, deps) {
     cleanupDir(work);
   }
 }
+async function handleAssembleV2(event, deps) {
+  const started = Date.now();
+  const s3 = deps?.s3 ?? getS3Client();
+  const primitive = deps?.primitives?.assemble ?? assemble;
+  const work = mkdtempSync(join(deps?.tmpRoot ?? tmpdir(), "hf-lambda-assemble-v2-"));
+  try {
+    const planDir = await downloadAndMaterializePlanV2(s3, event, { role: "assembler" }, work);
+    const audioPath = existsSync3(join(planDir, "audio.aac")) ? join(planDir, "audio.aac") : null;
+    const chunkPaths = await downloadChunkObjects(s3, event.ChunkS3Uris, work, event.Format);
+    const finalOutput = event.Format === "png-sequence" ? join(work, "output-frames") : join(work, `output${formatExtension(event.Format)}`);
+    const result = await primitive(planDir, chunkPaths, audioPath, finalOutput, {
+      cfr: event.Cfr === true
+    });
+    if (event.Format === "png-sequence") {
+      const tarball = `${finalOutput}.tar.gz`;
+      await tarDirectory(finalOutput, tarball);
+      await uploadFileToS3(s3, tarball, event.OutputS3Uri, "application/gzip");
+    } else {
+      await uploadFileToS3(s3, finalOutput, event.OutputS3Uri);
+    }
+    return {
+      Action: "assemble",
+      OutputS3Uri: event.OutputS3Uri,
+      FramesEncoded: result.framesEncoded,
+      FileSize: result.fileSize,
+      DurationMs: Date.now() - started
+    };
+  } finally {
+    cleanupDir(work);
+  }
+}
+async function downloadAndMaterializePlanV2(s3, event, target, work) {
+  const transportDir = join(work, "plan-v2");
+  mkdirSync2(transportDir, { recursive: true });
+  await downloadS3ObjectToFile(s3, event.PlanV2ManifestS3Uri, join(transportDir, "plan.json"));
+  const manifest = readPlanV2Manifest(transportDir);
+  if (manifest.planHash !== event.PlanHash) {
+    throwPlanHashMismatch(event.PlanHash, manifest.planHash);
+  }
+  const artifacts = listPlanV2ArtifactsForTarget(manifest, target);
+  const uniqueArtifacts = [
+    ...new Map(artifacts.map((artifact) => [artifact.sha256, artifact])).values()
+  ];
+  await mapConcurrent(uniqueArtifacts, 16, async (artifact) => {
+    await downloadPlanV2Artifact(s3, event.PlanV2ArtifactS3Prefix, transportDir, artifact);
+  });
+  const planDir = join(work, "plan");
+  materializePlanV2Target(transportDir, target, planDir);
+  return planDir;
+}
+async function downloadPlanV2Artifact(s3, artifactPrefix, planV2Dir, artifact) {
+  await downloadS3ObjectToFileVerified(
+    s3,
+    planV2BlobUri(artifactPrefix, artifact.sha256),
+    planV2BlobPath(planV2Dir, artifact.sha256),
+    artifact.sha256
+  );
+}
+function planV2BlobPath(planV2Dir, digest) {
+  return join(planV2Dir, "artifacts", "sha256", digest.slice(0, 2), digest);
+}
+function planV2BlobUri(prefix, digest) {
+  return `${trimTrailingSlash(prefix)}/${digest.slice(0, 2)}/${digest}`;
+}
+function throwPlanHashMismatch(expected, actual) {
+  const error = new Error(
+    `PLAN_HASH_MISMATCH: event PlanHash=${expected} did not match v2 manifest planHash=${actual}`
+  );
+  error.name = "PLAN_HASH_MISMATCH";
+  throw error;
+}
+async function mapConcurrent(values, concurrency, fn) {
+  let cursor = 0;
+  async function worker() {
+    while (cursor < values.length) {
+      const index = cursor++;
+      await fn(values[index]);
+    }
+  }
+  await Promise.all(Array.from({ length: Math.min(concurrency, values.length) }, () => worker()));
+}
 async function downloadChunkObjects(s3, uris, workDir, format) {
   const chunksDir = join(workDir, "chunks");
   mkdirSync2(chunksDir, { recursive: true });
@@ -433,11 +720,14 @@ function getEventS3Uris(event) {
     case "plan":
       return [event.ProjectS3Uri, event.PlanOutputS3Prefix];
     case "renderChunk":
-      return [event.PlanS3Uri, event.ChunkOutputS3Prefix];
+      return event.PlanProtocol === "v2" ? [event.PlanV2ManifestS3Uri, event.PlanV2ArtifactS3Prefix, event.ChunkOutputS3Prefix] : [event.PlanS3Uri, event.ChunkOutputS3Prefix];
     case "assemble":
-      return [event.PlanS3Uri, ...event.ChunkS3Uris, event.OutputS3Uri, event.AudioS3Uri].filter(
-        (u) => u != null
-      );
+      return [
+        ...event.PlanProtocol === "v2" ? [event.PlanV2ManifestS3Uri, event.PlanV2ArtifactS3Prefix] : [event.PlanS3Uri],
+        ...event.ChunkS3Uris,
+        event.OutputS3Uri,
+        event.AudioS3Uri
+      ].filter((u) => u != null);
   }
 }
 function validateEventS3Uris(event) {
@@ -491,7 +781,7 @@ function verifyPlanHash(planDir, expected) {
 import { mkdtempSync as mkdtempSync2, rmSync as rmSync3, statSync as statSync3 } from "node:fs";
 import { tmpdir as tmpdir2 } from "node:os";
 import { join as join2 } from "node:path";
-import { HeadObjectCommand, S3Client as S3Client2 } from "@aws-sdk/client-s3";
+import { HeadObjectCommand as HeadObjectCommand2, S3Client as S3Client2 } from "@aws-sdk/client-s3";
 import { hashProjectDir } from "@hyperframes/producer/distributed";
 async function deploySite(opts) {
   if (!statSync3(opts.projectDir).isDirectory()) {
@@ -532,7 +822,7 @@ async function deploySite(opts) {
 }
 async function headObject(s3, bucket, key) {
   try {
-    const res = await s3.send(new HeadObjectCommand({ Bucket: bucket, Key: key }));
+    const res = await s3.send(new HeadObjectCommand2({ Bucket: bucket, Key: key }));
     return {
       bytes: typeof res.ContentLength === "number" ? res.ContentLength : 0,
       lastModified: res.LastModified instanceof Date ? res.LastModified.toISOString() : (/* @__PURE__ */ new Date()).toISOString()
@@ -614,7 +904,8 @@ async function renderToLambda(opts) {
     ProjectS3Uri: site.projectS3Uri,
     PlanOutputS3Prefix: planOutputS3Prefix,
     OutputS3Uri: outputS3Uri,
-    Config: opts.config
+    Config: opts.config,
+    PlanProtocol: opts.planProtocol ?? "v1"
   };
   validateStepFunctionsInputSize(input);
   const sfn = opts.sfn ?? new SFNClient({ region: opts.region });
@@ -804,7 +1095,7 @@ function summarizeHistory(events, memoryMb) {
       }
       case "TaskStateExited":
       case "MapStateExited":
-        if (ev.stateExitedEventDetails?.name === "Assemble") {
+        if (isAssembleState(ev.stateExitedEventDetails?.name)) {
           assembleComplete = true;
           const exitPayload = parseJson(ev.stateExitedEventDetails?.output);
           if (exitPayload && typeof exitPayload === "object") {
@@ -883,11 +1174,17 @@ function unwrapLambdaPayload(payload) {
   return payload;
 }
 function applyPayloadFrameCounts(payload, currentLambdaState, bump) {
-  if (currentLambdaState !== "RenderChunk") return;
+  if (!isRenderChunkState(currentLambdaState)) return;
   if (!payload || typeof payload !== "object") return;
   const obj = payload;
   if (typeof obj.FramesEncoded === "number") bump(obj.FramesEncoded);
 }
+function isRenderChunkState(name) {
+  return name === "RenderChunk" || name === "RenderChunkV2";
+}
+function isAssembleState(name) {
+  return name === "Assemble" || name === "AssembleV2";
+}
 function inferBilledMs(payload) {
   if (!payload || typeof payload !== "object") return 0;
   const obj = payload;
@@ -915,6 +1212,7 @@ export {
   computeRenderCost,
   deploySite,
   downloadS3ObjectToFile,
+  downloadS3ObjectToFileVerified,
   formatS3Uri,
   getRenderProgress,
   handler,
@@ -926,6 +1224,7 @@ export {
   tarDirectory,
   untarDirectory,
   unwrapEvent,
+  uploadContentAddressedFileToS3,
   uploadFileToS3,
   validateDistributedRenderConfig
 };