@deepblend/dsh-blender-host 0.1.0

package/lib/delivery-manifest.js

@@ -0,0 +1,177 @@
+/**
+ * Delivery manifest — what a delivered package contains, without opening it.
+ *
+ * SPEC §10.4 and §20 M3 require the final package to carry the SceneSpec, the
+ * `.blend`, the video, the manifest and QA, and the M3 brief's design question 5
+ * states the test: "它要能让人**不看磁盘**就判断交付是否完整" — a reader must be
+ * able to judge completeness from the manifest alone. So this module writes down
+ * everything that judgement needs: the frame range and per-frame inventory, the
+ * video's CLAIMED properties, the MEASURED properties ffprobe reported, the
+ * disagreements between them, and the digests that tie the video back to the exact
+ * frames and the exact scene.
+ *
+ * WHY THE CLAIM AND THE MEASUREMENT BOTH APPEAR
+ * ---------------------------------------------
+ * A manifest that only recorded what the encoder intended would be a manifest
+ * that agrees with itself. Recording `video.probed` beside `video.expected`, plus
+ * the problems found between them, is what makes the file falsifiable by anyone
+ * who reads it — and it is why `verified` is a fact about a comparison rather than
+ * a boolean the encoder sets.
+ *
+ * Owner: DeepBlend Studio — M3
+ * Plane: Host composition
+ */
+
+import { basename } from 'node:path'
+
+import {
+  DELIVERY_MANIFEST_VERSION,
+  deliveryCompleteness,
+  verifyVideoProperties,
+} from '@deepblend/dsh-blender-contracts'
+
+import { fileSha256, fileSize } from './paths.js'
+
+/**
+ * Build the delivery manifest for a finished render job.
+ *
+ * @param {object} input
+ * @param {object} input.record - the render job record
+ * @param {object} input.ledger - the verified frame ledger
+ * @param {object} input.probed - the ffprobe measurement
+ * @param {object} input.encode - `{ outputPath, bytes, argv, durationMs }`
+ * @param {object} input.publish - `{ videoPath, relativeTo }`
+ * @param {object} input.sources - `{ sceneSpec, checkpoint, qa }` absolute paths
+ * @param {object} input.runtimeIdentity - Blender/runtime identity (SPEC §9.5)
+ * @returns {object}
+ */
+export function buildDeliveryManifest(input) {
+  const record = input.record
+  const frameCount = record.frameEnd - record.frameStart + 1
+  const relative = absolute => relativeTo(input.publish.relativeTo, absolute)
+
+  const claimed = {
+    frameStart: record.frameStart,
+    frameEnd: record.frameEnd,
+    frameCount,
+    fps: record.fps,
+    width: record.renderConfig?.resolution?.[0],
+    height: record.renderConfig?.resolution?.[1],
+  }
+  const verdict = verifyVideoProperties({
+    claimed,
+    probed: {
+      durationSeconds: input.probed.durationSeconds,
+      fps: input.probed.fps,
+      width: input.probed.width,
+      height: input.probed.height,
+      nbFrames: input.probed.nbFrames,
+      codec: input.probed.codec,
+    },
+  })
+
+  const manifest = {
+    schemaVersion: DELIVERY_MANIFEST_VERSION,
+    generatedAt: new Date().toISOString(),
+    projectId: record.projectId,
+    revisionId: record.revisionId,
+    jobId: record.jobId,
+    attempt: record.attempt,
+
+    source: {
+      sceneSpec: sourceEntry(input.sources.sceneSpec, relative),
+      checkpoint: sourceEntry(input.sources.checkpoint, relative),
+      qa: sourceEntry(input.sources.qa, relative),
+      sceneSpecDigest: record.sceneSpecDigest ?? null,
+    },
+
+    frames: {
+      directory: relative(record.framesDirectory ?? null),
+      prefix: record.filePrefix ?? 'frame_',
+      padding: record.filePadding ?? 4,
+      first: record.frameStart,
+      last: record.frameEnd,
+      expected: frameCount,
+      rendered: input.ledger.presentCount,
+      // The per-frame inventory is a list of NUMBERS, not of digests: 450 sha256
+      // lines would triple the manifest to answer a question ("is every frame
+      // there") that the count plus the ledger verdict already answers.
+      renderedFrames: input.ledger.present.map(entry => entry.frame),
+      corrupt: input.ledger.corrupt.map(entry => ({ frame: entry.frame, reason: entry.reason })),
+      bytes: input.ledger.present.reduce((total, entry) => total + entry.bytes, 0),
+    },
+
+    render: {
+      profileName: record.profileName ?? null,
+      config: record.renderConfig ?? null,
+      cameraId: record.cameraId ?? null,
+      meanMsPerFrame: record.meanMsPerFrame ?? null,
+      renderDurationMs: record.renderDurationMs ?? null,
+      blender: input.runtimeIdentity ?? null,
+    },
+
+    video: {
+      path: relative(input.publish.videoPath),
+      container: 'mp4',
+      bytes: input.encode.bytes,
+      sha256: fileSha256(input.publish.videoPath),
+      expected: claimed,
+      probed: {
+        durationSeconds: input.probed.durationSeconds,
+        fps: input.probed.fps,
+        width: input.probed.width,
+        height: input.probed.height,
+        frameCount: input.probed.nbFrames,
+        // Kept separate from `frameCount` on purpose: the container's own claim and
+        // the decoded count are different facts, and a delivery where they disagree
+        // is worth seeing even when the decoded count is right.
+        containerClaimedFrameCount: input.probed.nbClaimedFrames ?? null,
+        codec: input.probed.codec,
+      },
+      problems: verdict.problems,
+      verified: verdict.ok,
+      encode: {
+        durationMs: input.encode.durationMs,
+        argv: input.encode.argv,
+      },
+    },
+
+    // The QA verdict travels WITH the manifest, for the same reason the video's
+    // measured properties do: a reader must be able to judge the package without
+    // opening it, and "the QA report exists at this path" cannot answer "did it
+    // pass". The path is recorded beside it (`source.qa`) for the audit trail.
+    qa: {
+      path: relative(input.sources.qa ?? null),
+      report: input.qa ?? record.qa ?? null,
+    },
+  }
+
+  manifest.completeness = deliveryCompleteness(manifest)
+  return manifest
+}
+
+function sourceEntry(path, relative) {
+  if (path === null || path === undefined) return { path: null, bytes: null, sha256: null }
+  return { path: relative(path), bytes: fileSize(path), sha256: fileSha256(path) }
+}
+
+/**
+ * Express a path relative to the project root, or in full when it is outside it.
+ *
+ * A stored manifest must survive the workspace being moved (the same rule the
+ * M1 artifact records follow), so paths are project-relative. A path that cannot
+ * be made relative is reported in full rather than as a plausible-looking relative
+ * one: a silently wrong path in a delivery manifest is worse than an absolute one.
+ *
+ * @param {string} root
+ * @param {string|null|undefined} absolute
+ * @returns {string|null}
+ */
+export function relativeTo(root, absolute) {
+  if (absolute === null || absolute === undefined) return null
+  if (root === null || root === undefined) return absolute
+  const normalizedRoot = root.endsWith('/') ? root.slice(0, -1) : root
+  if (absolute === normalizedRoot) return basename(absolute)
+  if (absolute.startsWith(`${normalizedRoot}/`)) return absolute.slice(normalizedRoot.length + 1)
+  return absolute
+}

package/lib/frame-ledger.js

@@ -0,0 +1,189 @@
+/**
+ * Frame ledger — what a render job has actually delivered, read off disk.
+ *
+ * THE RULE THIS MODULE EXISTS FOR
+ * -------------------------------
+ * **The frames are the authority; the job record's `completedFrames[]` is a cache
+ * of that authority and may never overrule it.**
+ *
+ * The M3 restart probe measured why this matters. A `kill -9` of the process that
+ * started a render left three facts behind:
+ *
+ *   1. the Blender child SURVIVED (it is spawned `detached`, so it leads its own
+ *      process group) and was still writing into the directory a recovery was
+ *      about to describe;
+ *   2. the child's `events.jsonl` journal was intact up to its last fsynced line —
+ *      a fact about the journal, not about the frames;
+ *   3. a process killed between `create` and `finish` leaves a file that EXISTS
+ *      and is not a frame.
+ *
+ * So a record that said "frames 1..3 are done" would have been right by luck, and
+ * a record that trusted file existence would have been wrong whenever the kill
+ * landed mid-write. This module reads the frames.
+ *
+ * WHY IT SAMPLES RATHER THAN READS
+ * --------------------------------
+ * A 450-frame delivery is ~424 MiB. The ledger is rebuilt on every recovery and
+ * on every progress tick, so reading all of it would cost more than the render.
+ * `inspectFrameSample` in the contracts package decides the question from a stat
+ * plus 45 bytes per frame — the PNG signature and IHDR at the front, the IEND
+ * trailer at the back — which is the same question answered from the artifact.
+ *
+ * Owner: DeepBlend Studio — M3
+ * Plane: Host composition
+ */
+
+import { closeSync, existsSync, openSync, readSync, readdirSync, statSync } from 'node:fs'
+import { join } from 'node:path'
+
+import {
+  frameFileName,
+  inspectFrameSample as inspectFrameSampleFromContracts,
+  resolveFrameLedger,
+} from '@deepblend/dsh-blender-contracts'
+
+/** Bytes of the file read from the front. Covers the signature and the IHDR. */
+const HEAD_BYTES = 33
+/** Bytes of the file read from the back. Covers a whole IEND chunk and its CRC. */
+const TAIL_BYTES = 12
+
+/**
+ * Sample one frame without reading it.
+ *
+ * `exists` is part of the result and not inferable from `size`, and that is a
+ * measured fix rather than tidiness: an ABSENT file and an EMPTY file both have
+ * size 0, so a reader that only saw the size reported every missing frame as
+ * "empty" — a corrupt frame. The render set came out right by luck (both are
+ * re-rendered), but the diagnosis was wrong, so a status line saying "3 absent"
+ * where the truth was "3 corrupt" was a lie the type system could not catch.
+ *
+ * @param {string} path
+ * @returns {{exists: boolean, size: number, header?: Buffer, tail?: Buffer}}
+ */
+export function sampleFrame(path) {
+  let size
+  try {
+    size = statSync(path).size
+  } catch {
+    return { exists: false, size: 0 }
+  }
+  let descriptor = null
+  try {
+    descriptor = openSync(path, 'r')
+    const header = Buffer.alloc(Math.min(HEAD_BYTES, size))
+    if (header.length > 0) readSync(descriptor, header, 0, header.length, 0)
+    const tailLength = Math.min(TAIL_BYTES, size)
+    const tail = Buffer.alloc(tailLength)
+    if (tailLength > 0) readSync(descriptor, tail, 0, tailLength, size - tailLength)
+    return { exists: true, size, header, tail }
+  } catch {
+    // An unreadable file is a fact to report, not an exception to escape this
+    // function: the ledger's whole job is to describe the directory truthfully.
+    // It EXISTS — the stat succeeded — and its bytes could not be read.
+    return { exists: true, size }
+  } finally {
+    if (descriptor !== null) {
+      try {
+        closeSync(descriptor)
+      } catch {
+        // Not for "already closed" — the descriptor was opened here and nothing else closes it. It is here
+        // because a throw inside `finally` REPLACES the error being propagated, and the caller needs the read
+        // failure rather than a close failure.
+      }
+    }
+  }
+}
+
+/**
+ * Build the ledger for one job's frames.
+ *
+ * @param {object} input
+ * @param {string} input.framesDirectory - absolute path of the frames directory.
+ * @param {number[]} input.expected - every frame this job is responsible for.
+ * @param {{width?: number, height?: number, prefix?: string, padding?: number}} [input.expectedSize]
+ * @returns {{present: object[], corrupt: object[], missing: number[], toRender: number[],
+ *   presentCount: number, corruptCount: number, missingCount: number, toRenderCount: number}}
+ */
+/**
+ * The same decision applied to one frame, re-exported so a caller that is
+ * verifying a single claimed frame (the progress path) does not have to build a
+ * whole ledger to ask about it.
+ *
+ * @param {{size: number, header?: Buffer, tail?: Buffer}} sample
+ * @param {{width?: number, height?: number}} [expected]
+ */
+export function inspectFrameSample(sample, expected = {}) {
+  return inspectFrameSampleFromContracts(sample, expected)
+}
+
+/**
+ * Build the ledger for one job's frames.
+ *
+ * @param {object} input
+ * @param {string} input.framesDirectory - absolute path of the frames directory.
+ * @param {number[]} input.expected - every frame this job is responsible for.
+ * @param {{width?: number, height?: number, prefix?: string, padding?: number}} [input.expectedSize]
+ * @returns {{present: object[], corrupt: object[], missing: number[], toRender: number[],
+ *   presentCount: number, corruptCount: number, missingCount: number, toRenderCount: number}}
+ */
+export function readFrameLedger(input) {
+  const prefix = input.expectedSize?.prefix ?? 'frame_'
+  const padding = input.expectedSize?.padding ?? 4
+  /** @type {Map<number, {ok: boolean, reason?: string|null, bytes?: number}>} */
+  const observed = new Map()
+  for (const frame of input.expected) {
+    const path = join(input.framesDirectory, frameFileName(frame, prefix, padding))
+    const sample = sampleFrame(path)
+    // An absent frame is left OUT of the observation map rather than recorded as a
+    // zero-byte file: `resolveFrameLedger` reports what it does not observe as
+    // MISSING, which is a different fact from a present-but-incomplete frame and
+    // the two lead a reader to different actions.
+    if (sample.exists !== true) continue
+    const verdict = inspectFrameSample(sample, {
+      width: input.expectedSize?.width,
+      height: input.expectedSize?.height,
+    })
+    observed.set(frame, { ok: verdict.ok, reason: verdict.reason, bytes: verdict.bytes })
+  }
+  const ledger = resolveFrameLedger({ expected: input.expected, observed })
+  return {
+    ...ledger,
+    presentCount: ledger.present.length,
+    corruptCount: ledger.corrupt.length,
+    missingCount: ledger.missing.length,
+    toRenderCount: ledger.toRender.length,
+  }
+}
+
+/**
+ * Every PNG present in a frames directory, as frame numbers.
+ *
+ * Used only for DIAGNOSIS — reporting frames on disk that this job does not
+ * expect. It is deliberately not part of the ledger: a directory listing cannot
+ * tell a complete frame from a torn one, and it cannot tell a frame of THIS job
+ * from a leftover of another.
+ *
+ * Directory ENTRIES are skipped, which is a measured fix: a directory named
+ * `frame_0004.png` was listed here as a frame on disk, and the listing is the one
+ * place a reader looks to see whether the frames are really there. What the entry
+ * RESOLVES to is a question for the ledger (`sampleFrame` follows links and reads
+ * bytes), so a symlink to a PNG is still listed — it is a usable frame.
+ *
+ * @param {string} framesDirectory
+ * @param {{prefix?: string}} [options]
+ * @returns {number[]}
+ */
+export function framesOnDisk(framesDirectory, options = {}) {
+  if (!existsSync(framesDirectory)) return []
+  const prefix = options.prefix ?? 'frame_'
+  const found = []
+  for (const entry of readdirSync(framesDirectory, { withFileTypes: true })) {
+    if (entry.isDirectory()) continue
+    const name = entry.name
+    if (!name.startsWith(prefix) || !name.endsWith('.png')) continue
+    const digits = name.slice(prefix.length, -4)
+    if (!/^\d+$/.test(digits)) continue
+    found.push(Number.parseInt(digits, 10))
+  }
+  return found.sort((left, right) => left - right)
+}