@hyperframes/gcp-cloud-run 0.7.70 → 0.7.72

package/dist/index.js

@@ -8,7 +8,11 @@ import { Storage } from "@google-cloud/storage";
 import { Hono } from "hono";
 import {
   assemble,
+  listPlanV2ArtifactsForTarget,
+  materializePlanV2Target,
   plan,
+  planV2,
+  readPlanV2Manifest,
   renderChunk
 } from "@hyperframes/producer/distributed";
 
@@ -74,7 +78,15 @@ function formatExtension(format) {
 }
 
 // src/gcsTransport.ts
-import { createWriteStream, existsSync as existsSync2, mkdirSync, rmSync, statSync } from "node:fs";
+import {
+  createReadStream,
+  createWriteStream,
+  existsSync as existsSync2,
+  mkdirSync,
+  rmSync,
+  statSync
+} from "node:fs";
+import { createHash } from "node:crypto";
 import { dirname } from "node:path";
 import { pipeline } from "node:stream/promises";
 import * as tar from "tar";
@@ -103,6 +115,19 @@ async function downloadGcsObjectToFile(storage, uri, destPath) {
   const file = storage.bucket(bucket).file(key);
   await pipeline(file.createReadStream(), createWriteStream(destPath));
 }
+async function downloadGcsObjectToFileVerified(storage, uri, destPath, expectedSha256) {
+  assertSha256(expectedSha256);
+  await downloadGcsObjectToFile(storage, uri, destPath);
+  const actual = await sha256File(destPath);
+  if (actual !== expectedSha256) {
+    rmSync(destPath, { force: true });
+    const error = new Error(
+      `[gcsTransport] PLAN_ARTIFACT_DIGEST_MISMATCH: ${uri} expected ${expectedSha256}, got ${actual}`
+    );
+    error.name = "PLAN_ARTIFACT_DIGEST_MISMATCH";
+    throw error;
+  }
+}
 async function uploadFileToGcs(storage, localPath, uri, contentType) {
   if (!existsSync2(localPath)) {
     throw new Error(`[gcsTransport] upload source missing: ${localPath}`);
@@ -117,6 +142,73 @@ async function uploadFileToGcs(storage, localPath, uri, contentType) {
     contentType
   });
 }
+async function uploadContentAddressedFileToGcs(storage, localPath, uri, expectedSha256, contentType) {
+  assertSha256(expectedSha256);
+  if (!existsSync2(localPath)) {
+    throw new Error(`[gcsTransport] upload source missing: ${localPath}`);
+  }
+  const actualSha256 = await sha256File(localPath);
+  if (actualSha256 !== expectedSha256) {
+    throwDigestMismatch(
+      `local artifact ${localPath} expected ${expectedSha256}, got ${actualSha256}`
+    );
+  }
+  const { bucket, key } = parseGcsUri(uri);
+  const bucketHandle = storage.bucket(bucket);
+  const file = bucketHandle.file(key);
+  const size = statSync(localPath).size;
+  if (await isReusableContentAddressedObject(file, uri, size, expectedSha256)) {
+    return "reused";
+  }
+  try {
+    await bucketHandle.upload(localPath, {
+      destination: key,
+      contentType,
+      metadata: { metadata: { sha256: expectedSha256 } },
+      preconditionOpts: { ifGenerationMatch: 0 }
+    });
+    return "uploaded";
+  } catch (error) {
+    if (isGcsPreconditionFailed(error) && await isReusableContentAddressedObject(file, uri, size, expectedSha256)) {
+      return "reused";
+    }
+    throw error;
+  }
+}
+async function isReusableContentAddressedObject(file, uri, expectedSize, expectedSha256) {
+  const [exists] = await file.exists();
+  if (!exists) return false;
+  const [metadata] = await file.getMetadata();
+  if (Number(metadata.size) === expectedSize && metadata.metadata?.sha256 === expectedSha256) {
+    return true;
+  }
+  throwDigestMismatch(
+    `immutable object ${uri} already exists with different digest metadata or size`
+  );
+}
+async function sha256File(path) {
+  const hash = createHash("sha256");
+  for await (const chunk of createReadStream(path)) {
+    hash.update(chunk);
+  }
+  return hash.digest("hex");
+}
+function assertSha256(value) {
+  if (!/^[a-f0-9]{64}$/.test(value)) {
+    throw new Error(
+      `[gcsTransport] expected lowercase SHA-256 digest, got ${JSON.stringify(value)}`
+    );
+  }
+}
+function throwDigestMismatch(detail) {
+  const error = new Error(`[gcsTransport] PLAN_ARTIFACT_DIGEST_MISMATCH: ${detail}`);
+  error.name = "PLAN_ARTIFACT_DIGEST_MISMATCH";
+  throw error;
+}
+function isGcsPreconditionFailed(error) {
+  if (!error || typeof error !== "object") return false;
+  return error.code === 412;
+}
 async function tarDirectory(sourceDir, destTarball) {
   if (!existsSync2(sourceDir) || !statSync(sourceDir).isDirectory()) {
     throw new Error(`[gcsTransport] tar source must be an existing directory: ${sourceDir}`);
@@ -144,6 +236,7 @@ function getStorage() {
 }
 async function dispatch(event, deps) {
   const unwrapped = unwrapEvent(event);
+  validatePlanProtocolShape(unwrapped);
   validateEventGcsUris(unwrapped);
   logEvent({ event: "handler_start", action: unwrapped.Action, input: summarizeEvent(unwrapped) });
   try {
@@ -164,15 +257,52 @@ async function dispatch(event, deps) {
       }
     }
   } catch (err) {
+    normalizeTerminalErrorName(err);
     logEvent({
       event: "handler_error",
       action: unwrapped.Action,
+      input: summarizeEvent(unwrapped),
       message: err instanceof Error ? err.message : String(err),
       name: err instanceof Error ? err.name : void 0
     });
     throw err;
   }
 }
+function validatePlanProtocolShape(event) {
+  const raw = event;
+  const protocol = raw.PlanProtocol;
+  if (protocol !== void 0 && protocol !== "v1" && protocol !== "v2") {
+    const error = new Error(
+      `[handler] unsupported PlanProtocol ${JSON.stringify(protocol)}; expected "v1", "v2", or absent`
+    );
+    error.name = "PLAN_PROTOCOL_UNSUPPORTED";
+    throw error;
+  }
+  if (event.Action === "plan") return;
+  const hasV1Locator = typeof raw.PlanGcsUri === "string";
+  const hasV2Manifest = typeof raw.PlanV2ManifestGcsUri === "string";
+  const hasV2Prefix = typeof raw.PlanV2ArtifactGcsPrefix === "string";
+  const valid = protocol === "v2" ? !hasV1Locator && hasV2Manifest && hasV2Prefix : hasV1Locator && !hasV2Manifest && !hasV2Prefix;
+  if (!valid) {
+    const error = new Error(
+      `[handler] ${protocol === "v2" ? "v2" : "v1"} ${event.Action} event has mixed or missing plan locators`
+    );
+    error.name = "PLAN_PROTOCOL_UNSUPPORTED";
+    throw error;
+  }
+  if (protocol === "v2" && event.Action === "assemble" && event.AudioGcsUri !== null) {
+    const error = new Error("[handler] v2 assemble audio must be materialized from the manifest");
+    error.name = "PLAN_PROTOCOL_UNSUPPORTED";
+    throw error;
+  }
+}
+function normalizeTerminalErrorName(error) {
+  if (!error || typeof error !== "object") return;
+  const candidate = error;
+  if (candidate.code === "PLAN_PROTOCOL_UNSUPPORTED" || candidate.code === "PLAN_TOO_LARGE" || candidate.code === "PLAN_V2_INTEGRITY_UNRECOVERABLE") {
+    candidate.name = candidate.code;
+  }
+}
 var MAX_ENVELOPE_DEPTH = 4;
 function unwrapEvent(event) {
   let cursor = event;
@@ -209,18 +339,21 @@ function summarizeEvent(event) {
       return {
         projectGcsUri: event.ProjectGcsUri,
         planOutputGcsPrefix: event.PlanOutputGcsPrefix,
+        planProtocol: event.PlanProtocol ?? "v1",
         format: event.Config.format,
         fps: event.Config.fps
       };
     case "renderChunk":
       return {
-        planGcsUri: event.PlanGcsUri,
+        planProtocol: event.PlanProtocol ?? "v1",
+        ...event.PlanProtocol === "v2" ? { planV2ManifestGcsUri: event.PlanV2ManifestGcsUri } : { planGcsUri: event.PlanGcsUri },
         chunkIndex: event.ChunkIndex,
         format: event.Format
       };
     case "assemble":
       return {
-        planGcsUri: event.PlanGcsUri,
+        planProtocol: event.PlanProtocol ?? "v1",
+        ...event.PlanProtocol === "v2" ? { planV2ManifestGcsUri: event.PlanV2ManifestGcsUri } : { planGcsUri: event.PlanGcsUri },
         chunkCount: event.ChunkGcsUris.length,
         hasAudio: event.AudioGcsUri !== null,
         outputGcsUri: event.OutputGcsUri,
@@ -234,6 +367,9 @@ function primeChrome(deps) {
   process.env.PRODUCER_HEADLESS_SHELL_PATH = resolveChromeExecutablePath();
 }
 async function handlePlan(event, deps) {
+  if (event.PlanProtocol === "v2") {
+    return handlePlanV2(event, deps);
+  }
   const started = Date.now();
   const storage = deps?.storage ?? getStorage();
   const primitive = deps?.primitives?.plan ?? plan;
@@ -275,7 +411,70 @@ async function handlePlan(event, deps) {
     cleanupDir(work);
   }
 }
+async function handlePlanV2(event, deps) {
+  const started = Date.now();
+  const storage = deps?.storage ?? getStorage();
+  const primitive = deps?.primitives?.planV2 ?? planV2;
+  primeChrome(deps);
+  const work = mkdtempSync(join(deps?.tmpRoot ?? tmpdir(), "hf-cr-plan-v2-"));
+  const projectArchive = join(work, "project.tar.gz");
+  const projectDir = join(work, "project");
+  const planV2Dir = join(work, "plan-v2");
+  try {
+    await downloadGcsObjectToFile(storage, event.ProjectGcsUri, projectArchive);
+    await untarDirectory(projectArchive, projectDir);
+    const result = await primitive(projectDir, { ...event.Config }, planV2Dir);
+    const manifest = readPlanV2Manifest(planV2Dir);
+    if (manifest.planHash !== result.planHash) {
+      throwPlanHashMismatch(result.planHash, manifest.planHash);
+    }
+    const outputPrefix = `${trimTrailingSlash(event.PlanOutputGcsPrefix)}/v2`;
+    const artifactPrefix = `${outputPrefix}/artifacts/sha256`;
+    const uniqueArtifacts = [
+      ...new Map(manifest.artifacts.map((artifact) => [artifact.sha256, artifact])).values()
+    ];
+    await mapConcurrent(uniqueArtifacts, 16, async (artifact) => {
+      await uploadContentAddressedFileToGcs(
+        storage,
+        planV2BlobPath(planV2Dir, artifact.sha256),
+        planV2BlobUri(artifactPrefix, artifact.sha256),
+        artifact.sha256
+      );
+    });
+    const manifestUri = `${outputPrefix}/manifest.json`;
+    await uploadContentAddressedFileToGcs(
+      storage,
+      result.manifestPath,
+      manifestUri,
+      await sha256File(result.manifestPath),
+      "application/json"
+    );
+    return {
+      Action: "plan",
+      PlanProtocol: "v2",
+      PlanV2ManifestGcsUri: manifestUri,
+      PlanV2ArtifactGcsPrefix: artifactPrefix,
+      PlanHash: result.planHash,
+      ChunkCount: result.chunkCount,
+      TotalFrames: result.totalFrames,
+      Fps: result.fps,
+      Width: result.width,
+      Height: result.height,
+      Format: result.format,
+      HasAudio: manifest.artifacts.some((artifact) => artifact.path === "audio.aac"),
+      AudioGcsUri: null,
+      FfmpegVersion: result.ffmpegVersion,
+      ProducerVersion: result.producerVersion,
+      DurationMs: Date.now() - started
+    };
+  } finally {
+    cleanupDir(work);
+  }
+}
 async function handleRenderChunk(event, deps) {
+  if (event.PlanProtocol === "v2") {
+    return handleRenderChunkV2(event, deps);
+  }
   const started = Date.now();
   const storage = deps?.storage ?? getStorage();
   const primitive = deps?.primitives?.renderChunk ?? renderChunk;
@@ -310,6 +509,42 @@ async function handleRenderChunk(event, deps) {
     cleanupDir(work);
   }
 }
+async function handleRenderChunkV2(event, deps) {
+  const started = Date.now();
+  const storage = deps?.storage ?? getStorage();
+  const primitive = deps?.primitives?.renderChunk ?? renderChunk;
+  primeChrome(deps);
+  const work = mkdtempSync(join(deps?.tmpRoot ?? tmpdir(), "hf-cr-chunk-v2-"));
+  try {
+    const planDir = await downloadAndMaterializePlanV2(
+      storage,
+      event,
+      { role: "chunk", chunkIndex: event.ChunkIndex },
+      work
+    );
+    const chunkOutputBase = join(
+      work,
+      event.Format === "png-sequence" ? `chunk-${pad(event.ChunkIndex)}` : `chunk-${pad(event.ChunkIndex)}${formatExtension(event.Format)}`
+    );
+    const result = await primitive(planDir, event.ChunkIndex, chunkOutputBase);
+    const chunkUri = await uploadChunkOutput(
+      storage,
+      result,
+      event.ChunkOutputGcsPrefix,
+      event.ChunkIndex
+    );
+    return {
+      Action: "renderChunk",
+      ChunkGcsUri: chunkUri,
+      ChunkIndex: event.ChunkIndex,
+      Sha256: result.sha256,
+      FramesEncoded: result.framesEncoded,
+      DurationMs: Date.now() - started
+    };
+  } finally {
+    cleanupDir(work);
+  }
+}
 async function uploadChunkOutput(storage, result, prefix, chunkIndex) {
   const trimmed = trimTrailingSlash(prefix);
   if (result.outputKind === "file") {
@@ -325,6 +560,9 @@ async function uploadChunkOutput(storage, result, prefix, chunkIndex) {
   return uri;
 }
 async function handleAssemble(event, deps) {
+  if (event.PlanProtocol === "v2") {
+    return handleAssembleV2(event, deps);
+  }
   const started = Date.now();
   const storage = deps?.storage ?? getStorage();
   const primitive = deps?.primitives?.assemble ?? assemble;
@@ -365,6 +603,91 @@ async function handleAssemble(event, deps) {
     cleanupDir(work);
   }
 }
+async function handleAssembleV2(event, deps) {
+  const started = Date.now();
+  const storage = deps?.storage ?? getStorage();
+  const primitive = deps?.primitives?.assemble ?? assemble;
+  const work = mkdtempSync(join(deps?.tmpRoot ?? tmpdir(), "hf-cr-assemble-v2-"));
+  try {
+    const planDir = await downloadAndMaterializePlanV2(storage, event, { role: "assembler" }, work);
+    const audioPath = existsSync3(join(planDir, "audio.aac")) ? join(planDir, "audio.aac") : null;
+    const chunkPaths = await downloadChunkObjects(storage, event.ChunkGcsUris, work, event.Format);
+    const finalOutput = event.Format === "png-sequence" ? join(work, "output-frames") : join(work, `output${formatExtension(event.Format)}`);
+    const result = await primitive(planDir, chunkPaths, audioPath, finalOutput, {
+      cfr: event.Cfr === true
+    });
+    if (event.Format === "png-sequence") {
+      const tarball = `${finalOutput}.tar.gz`;
+      await tarDirectory(finalOutput, tarball);
+      await uploadFileToGcs(storage, tarball, event.OutputGcsUri, "application/gzip");
+    } else {
+      await uploadFileToGcs(storage, finalOutput, event.OutputGcsUri);
+    }
+    return {
+      Action: "assemble",
+      OutputGcsUri: event.OutputGcsUri,
+      FramesEncoded: result.framesEncoded,
+      FileSize: result.fileSize,
+      DurationMs: Date.now() - started
+    };
+  } finally {
+    cleanupDir(work);
+  }
+}
+async function downloadAndMaterializePlanV2(storage, event, target, work) {
+  const transportDir = join(work, "plan-v2");
+  mkdirSync2(transportDir, { recursive: true });
+  await downloadGcsObjectToFile(
+    storage,
+    event.PlanV2ManifestGcsUri,
+    join(transportDir, "plan.json")
+  );
+  const manifest = readPlanV2Manifest(transportDir);
+  if (manifest.planHash !== event.PlanHash) {
+    throwPlanHashMismatch(event.PlanHash, manifest.planHash);
+  }
+  const artifacts = listPlanV2ArtifactsForTarget(manifest, target);
+  const uniqueArtifacts = [
+    ...new Map(artifacts.map((artifact) => [artifact.sha256, artifact])).values()
+  ];
+  await mapConcurrent(uniqueArtifacts, 16, async (artifact) => {
+    await downloadPlanV2Artifact(storage, event.PlanV2ArtifactGcsPrefix, transportDir, artifact);
+  });
+  const planDir = join(work, "plan");
+  materializePlanV2Target(transportDir, target, planDir);
+  return planDir;
+}
+async function downloadPlanV2Artifact(storage, artifactPrefix, planV2Dir, artifact) {
+  await downloadGcsObjectToFileVerified(
+    storage,
+    planV2BlobUri(artifactPrefix, artifact.sha256),
+    planV2BlobPath(planV2Dir, artifact.sha256),
+    artifact.sha256
+  );
+}
+function planV2BlobPath(planV2Dir, digest) {
+  return join(planV2Dir, "artifacts", "sha256", digest.slice(0, 2), digest);
+}
+function planV2BlobUri(prefix, digest) {
+  return `${trimTrailingSlash(prefix)}/${digest.slice(0, 2)}/${digest}`;
+}
+function throwPlanHashMismatch(expected, actual) {
+  const error = new Error(
+    `PLAN_HASH_MISMATCH: event PlanHash=${expected} did not match v2 manifest planHash=${actual}`
+  );
+  error.name = "PLAN_HASH_MISMATCH";
+  throw error;
+}
+async function mapConcurrent(values, concurrency, fn) {
+  let cursor = 0;
+  async function worker() {
+    while (cursor < values.length) {
+      const index = cursor++;
+      await fn(values[index]);
+    }
+  }
+  await Promise.all(Array.from({ length: Math.min(concurrency, values.length) }, () => worker()));
+}
 async function downloadChunkObjects(storage, uris, workDir, format) {
   const chunksDir = join(workDir, "chunks");
   mkdirSync2(chunksDir, { recursive: true });
@@ -393,10 +716,10 @@ function getEventGcsUris(event) {
     case "plan":
       return [event.ProjectGcsUri, event.PlanOutputGcsPrefix];
     case "renderChunk":
-      return [event.PlanGcsUri, event.ChunkOutputGcsPrefix];
+      return event.PlanProtocol === "v2" ? [event.PlanV2ManifestGcsUri, event.PlanV2ArtifactGcsPrefix, event.ChunkOutputGcsPrefix] : [event.PlanGcsUri, event.ChunkOutputGcsPrefix];
     case "assemble":
       return [
-        event.PlanGcsUri,
+        ...event.PlanProtocol === "v2" ? [event.PlanV2ManifestGcsUri, event.PlanV2ArtifactGcsPrefix] : [event.PlanGcsUri],
         ...event.ChunkGcsUris,
         event.OutputGcsUri,
         event.AudioGcsUri
@@ -465,12 +788,17 @@ var NON_RETRYABLE_ERROR_NAMES = /* @__PURE__ */ new Set([
   // Handler-boundary guards.
   "GCS_URI_NOT_ALLOWED",
   "PLAN_HASH_MISMATCH",
+  "PLAN_ARTIFACT_DIGEST_MISMATCH",
+  "PLAN_PROTOCOL_UNSUPPORTED",
+  "PLAN_V2_INTEGRITY_UNRECOVERABLE",
   // Producer error class names (`.name`) + their string code aliases — the
   // class sets `.name` to the class name but wraps a `code`; cover both so a
   // raw-code throw is caught too. Mirrors the AWS state machine's
   // non-retryable list.
   "FormatNotSupportedInDistributedError",
   "PlanTooLargeError",
+  "PlanProtocolUnsupportedError",
+  "PlanV2IntegrityError",
   "RenderChunkValidationError",
   "FFMPEG_VERSION_MISMATCH",
   "FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED",
@@ -640,7 +968,8 @@ async function renderToCloudRun(opts) {
     PlanOutputGcsPrefix: planOutputGcsPrefix,
     OutputGcsUri: outputGcsUri,
     ServiceUrl: opts.serviceUrl,
-    Config: opts.config
+    Config: opts.config,
+    PlanProtocol: opts.planProtocol ?? "v1"
   };
   validateWorkflowsInputSize(argument);
   const executions = opts.executions ?? await defaultExecutionsClient();
@@ -847,16 +1176,19 @@ export {
   deploySite,
   dispatch,
   downloadGcsObjectToFile,
+  downloadGcsObjectToFileVerified,
   formatGcsUri,
   getRenderProgress,
   getTerraformModuleDir,
   parseGcsUri,
   renderToCloudRun,
   resolveChromeExecutablePath,
+  sha256File,
   startServer,
   tarDirectory,
   untarDirectory,
   unwrapEvent,
+  uploadContentAddressedFileToGcs,
   uploadFileToGcs,
   validateDistributedRenderConfig
 };