@morshoto/framekit 0.2.3 → 0.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -23,3 +23,5 @@ export declare class FfmpegTimelineRenderer implements FramekitRenderProvider {
23
23
  capabilities(request: FramekitRenderRequest): FramekitRenderCapabilityReport;
24
24
  render(plan: FramekitRenderPlan): Promise<FramekitRenderProviderResult>;
25
25
  }
26
+ export declare function escapeFilterValue(value: string): string;
27
+ export declare function resolveFontFile(preferred?: string): string | undefined;
@@ -570,10 +570,10 @@ function clamp(value, minimum, maximum) {
570
570
  function formatNumber(value) {
571
571
  return Number(value.toFixed(6)).toString();
572
572
  }
573
- function escapeFilterValue(value) {
573
+ export function escapeFilterValue(value) {
574
574
  return value.replaceAll("\\", "\\\\").replaceAll("'", "'\\''").replaceAll(":", "\\:").replaceAll(",", "\\,").replaceAll("[", "\\[").replaceAll("]", "\\]").replaceAll("\n", "\\n");
575
575
  }
576
- function resolveFontFile(preferred) {
576
+ export function resolveFontFile(preferred) {
577
577
  const candidates = [
578
578
  preferred,
579
579
  "/System/Library/Fonts/HelveticaNeue.ttc",
@@ -0,0 +1,28 @@
1
+ import type { TimelineIr } from "../../../../packages/runtime/src/index.js";
2
+ type Sample = {
3
+ kind: "shot" | "title";
4
+ id: string;
5
+ occurrenceId: string;
6
+ mediaId: string;
7
+ sourceDigest: string;
8
+ timelineSeconds: number;
9
+ sourceSeconds: number;
10
+ };
11
+ /** Renders a bounded set of individual frames, without writing a movie or changing sources. */
12
+ export declare class FfmpegStoryboardRenderer {
13
+ private readonly options;
14
+ constructor(options?: {
15
+ ffmpegPath?: string;
16
+ temporaryDirectory?: string;
17
+ });
18
+ render(timeline: TimelineIr, dimensions: {
19
+ width: number;
20
+ height: number;
21
+ }, signal?: AbortSignal): Promise<{
22
+ data: string;
23
+ mimeType: "image/png";
24
+ samples: Sample[];
25
+ limitations: string[];
26
+ }>;
27
+ }
28
+ export {};
@@ -0,0 +1,120 @@
1
+ import { execFile } from "node:child_process";
2
+ import { createHash } from "node:crypto";
3
+ import { createReadStream } from "node:fs";
4
+ import { lstat, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
5
+ import { tmpdir } from "node:os";
6
+ import { join, resolve } from "node:path";
7
+ import { promisify } from "node:util";
8
+ import { escapeFilterValue, resolveFontFile } from "./ffmpeg-renderer.js";
9
+ const run = promisify(execFile);
10
+ const MAX_SAMPLES = 12;
11
+ const MAX_WIDTH = 640;
12
+ const MAX_HEIGHT = 360;
13
+ /** Renders a bounded set of individual frames, without writing a movie or changing sources. */
14
+ export class FfmpegStoryboardRenderer {
15
+ options;
16
+ constructor(options = {}) {
17
+ this.options = options;
18
+ }
19
+ async render(timeline, dimensions, signal) {
20
+ signal?.throwIfAborted();
21
+ const sequence = timeline.sequence;
22
+ if (sequence.transitions?.length || sequence.storyElements.length || sequence.captions.length || sequence.occurrences.some((item) => item.track === 0 && (item.transform || item.enabled === false))) {
23
+ throw new Error("HEADLESS_STORYBOARD_UNSUPPORTED: transitions, gaps, disabled clips and transforms cannot be sampled faithfully");
24
+ }
25
+ const clips = sequence.occurrences.filter((item) => item.track === 0 && (!item.role || item.role === "video")).sort((a, b) => seconds(a.startTime) - seconds(b.startTime));
26
+ const titles = sequence.titles ?? [];
27
+ if (!clips.length || clips.length + titles.length > MAX_SAMPLES)
28
+ throw new Error(`HEADLESS_STORYBOARD_LIMIT: expected 1 to ${MAX_SAMPLES} total shots and titles`);
29
+ if (sequence.occurrences.some((item) => item.track === 0 && item.role && item.role !== "video") || clips.some((item, index) => Math.abs(seconds(item.startTime) - (index ? seconds(clips[index - 1].startTime) + seconds(clips[index - 1].durationTime) : 0)) > 1e-7))
30
+ throw new Error("HEADLESS_STORYBOARD_UNSUPPORTED: primary clips must form a contiguous storyline");
31
+ if (sequence.occurrences.some((item) => item.track !== 0 && item.role !== "audio" && item.role !== "music"))
32
+ throw new Error("HEADLESS_STORYBOARD_UNSUPPORTED: visual overlays cannot be sampled faithfully");
33
+ const frameDuration = seconds(sequence.frameDuration);
34
+ const scale = Math.min(MAX_WIDTH / dimensions.width, MAX_HEIGHT / dimensions.height);
35
+ const width = Math.max(2, Math.floor(dimensions.width * scale / 2) * 2);
36
+ const height = Math.max(2, Math.floor(dimensions.height * scale / 2) * 2);
37
+ const resources = new Map(timeline.resources.map((resource) => [resource.id, resource]));
38
+ const samples = [];
39
+ const add = (kind, id, time) => {
40
+ const clip = clips.find((item) => time >= seconds(item.startTime) - 1e-8 && time < seconds(item.startTime) + seconds(item.durationTime) - 1e-8);
41
+ const resource = clip?.mediaId ? resources.get(clip.mediaId) : undefined;
42
+ if (!clip || !resource?.source || resource.sourceKind !== "local-file" || !resource.sourceDigest)
43
+ throw new Error("HEADLESS_STORYBOARD_UNSUPPORTED: sample has no registered video source");
44
+ samples.push({ kind, id, occurrenceId: clip.id, mediaId: resource.id, sourceDigest: resource.sourceDigest, timelineSeconds: time, sourceSeconds: seconds(clip.sourceStartTime ?? { value: "0", timescale: "1" }) + time - seconds(clip.startTime) });
45
+ };
46
+ for (const clip of clips)
47
+ add("shot", clip.id, seconds(clip.startTime));
48
+ for (const title of titles) {
49
+ const midpoint = seconds(title.startTime) + seconds(title.durationTime) / 2;
50
+ const sampleTime = Math.floor((midpoint + 1e-8) / frameDuration) * frameDuration;
51
+ if (sampleTime < seconds(title.startTime) || sampleTime >= seconds(title.startTime) + seconds(title.durationTime)) {
52
+ throw new Error("HEADLESS_STORYBOARD_UNSUPPORTED: title midpoint has no sample on the sequence frame grid");
53
+ }
54
+ add("title", title.id, sampleTime);
55
+ }
56
+ samples.sort((a, b) => a.timelineSeconds - b.timelineSeconds || a.kind.localeCompare(b.kind));
57
+ const identities = new Map();
58
+ for (const sample of samples) {
59
+ const resource = resources.get(sample.mediaId);
60
+ identities.set(resolve(resource.source), sample.sourceDigest);
61
+ }
62
+ const checkSources = async () => {
63
+ for (const [path, expected] of identities) {
64
+ signal?.throwIfAborted();
65
+ const info = await lstat(path);
66
+ if (!info.isFile() || info.isSymbolicLink() || await digest(path, signal) !== expected)
67
+ throw new Error("HEADLESS_STORYBOARD_SOURCE_CHANGED: source identity does not match registered media");
68
+ }
69
+ };
70
+ await checkSources();
71
+ const dir = await mkdtemp(join(this.options.temporaryDirectory ?? tmpdir(), "framekit-storyboard-"));
72
+ try {
73
+ const frames = [];
74
+ for (const [index, sample] of samples.entries()) {
75
+ signal?.throwIfAborted();
76
+ const resource = resources.get(sample.mediaId);
77
+ // The render provider's drawtext filter includes both endpoints.
78
+ const activeTitles = titles.filter((title) => sample.timelineSeconds >= seconds(title.startTime) && sample.timelineSeconds <= seconds(title.startTime) + seconds(title.durationTime));
79
+ const filters = [`scale=${width}:${height}:force_original_aspect_ratio=disable`];
80
+ for (const [titleIndex, title] of activeTitles.entries()) {
81
+ if (title.style || title.position)
82
+ throw new Error("HEADLESS_STORYBOARD_UNSUPPORTED: styled titles are not supported");
83
+ const filename = `title-${index}-${titleIndex}.txt`;
84
+ await writeFile(join(dir, filename), title.text, "utf8");
85
+ const fontSize = Math.max(1, Math.round(32 * scale));
86
+ const font = resolveFontFile(process.env.FRAMEKIT_RENDER_FONT);
87
+ filters.push(`drawtext=${font ? `fontfile='${escapeFilterValue(font)}':` : ""}textfile='${filename}':expansion=none:fontcolor=white:fontsize=${fontSize}:x=(w-text_w)/2:y=(h-text_h)/2`);
88
+ }
89
+ const frame = join(dir, `${index}.png`);
90
+ await run(this.options.ffmpegPath ?? process.env.FFMPEG_BIN ?? "ffmpeg", ["-hide_banner", "-loglevel", "error", "-nostdin", "-ss", String(sample.sourceSeconds), "-i", resolve(resource.source), "-vf", filters.join(","), "-frames:v", "1", "-threads", "1", "-y", frame], { cwd: dir, signal, timeout: 30000, maxBuffer: 1024 * 1024 });
91
+ frames.push(frame);
92
+ }
93
+ const columns = Math.min(3, frames.length);
94
+ const rows = Math.ceil(frames.length / columns);
95
+ const sheet = join(dir, "sheet.png");
96
+ const input = frames.flatMap((frame) => ["-i", frame]);
97
+ const layout = frames.map((_, index) => `${(index % columns) * width}_${Math.floor(index / columns) * height}`).join("|");
98
+ const filter = frames.length === 1 ? "[0:v]null[out]" : `${frames.map((_, index) => `[${index}:v]`).join("")}xstack=inputs=${frames.length}:layout=${layout}:fill=black[out]`;
99
+ await run(this.options.ffmpegPath ?? process.env.FFMPEG_BIN ?? "ffmpeg", ["-hide_banner", "-loglevel", "error", "-nostdin", ...input, "-filter_complex", filter, "-map", "[out]", "-frames:v", "1", "-threads", "1", "-y", sheet], { cwd: dir, signal, timeout: 30000, maxBuffer: 1024 * 1024 });
100
+ const bytes = await readFile(sheet);
101
+ if (bytes.length > 8 * 1024 * 1024)
102
+ throw new Error("HEADLESS_STORYBOARD_LIMIT: contact sheet exceeds 8 MiB");
103
+ await checkSources();
104
+ signal?.throwIfAborted();
105
+ return { data: bytes.toString("base64"), mimeType: "image/png", samples, limitations: [`${width}x${height} pixels per sample; ${columns} columns and ${rows} rows in sample order`, "Only shot starts and title midpoints are sampled; motion, audio and unsampled frames are not represented.", "Source frames are sought independently; source and output frame-rate conversion can select different nearby frames."] };
106
+ }
107
+ finally {
108
+ await rm(dir, { recursive: true, force: true });
109
+ }
110
+ }
111
+ }
112
+ function seconds(value) {
113
+ return Number(value.value) / Number(value.timescale);
114
+ }
115
+ async function digest(path, signal) {
116
+ const hash = createHash("sha256");
117
+ for await (const chunk of createReadStream(path, { signal }))
118
+ hash.update(chunk);
119
+ return hash.digest("hex");
120
+ }
@@ -1,3 +1,4 @@
1
1
  export * from "./ffmpeg-renderer.js";
2
2
  export * from "./ffmpeg-verifier.js";
3
3
  export * from "./ffmpeg-media-probe.js";
4
+ export * from "./ffmpeg-storyboard.js";
@@ -1,3 +1,4 @@
1
1
  export * from "./ffmpeg-renderer.js";
2
2
  export * from "./ffmpeg-verifier.js";
3
3
  export * from "./ffmpeg-media-probe.js";
4
+ export * from "./ffmpeg-storyboard.js";
@@ -1,5 +1,5 @@
1
1
  {
2
2
  "schemaVersion": 1,
3
- "version": "0.2.3",
4
- "commit": "da243be36c46e0471ef25c6f2c50c2306aed2c93"
3
+ "version": "0.2.4",
4
+ "commit": "2d961b6132a355e83c067b440bccc83c8ae4309b"
5
5
  }
@@ -9,7 +9,7 @@ import { HeadlessProjectService } from "../../../packages/headless-application/s
9
9
  import { createMcpServer } from "./server.js";
10
10
  import { createCanonicalSessionChangeSource } from "./headless-sessions.js";
11
11
  import { createCanonicalNativeSnapshotReader } from "./canonical-native-snapshot.js";
12
- import { FfmpegMediaMetadataProbe, FfmpegRenderVerifier, FfmpegTimelineRenderer } from "../../../adapters/headless/typescript/src/index.js";
12
+ import { FfmpegMediaMetadataProbe, FfmpegRenderVerifier, FfmpegTimelineRenderer, FfmpegStoryboardRenderer } from "../../../adapters/headless/typescript/src/index.js";
13
13
  import { shouldConfigureFinalCutProjectPublisher } from "./final-cut-runtime-config.js";
14
14
  const fixture = new InMemoryEditorAdapter({
15
15
  projectId: "project-1",
@@ -190,6 +190,7 @@ const headlessProjects = new HeadlessProjectService({
190
190
  protectedStateDirectory: framekitStateDirectory,
191
191
  mediaProbe: new FfmpegMediaMetadataProbe(),
192
192
  renderer: new FfmpegTimelineRenderer(),
193
+ storyboard: new FfmpegStoryboardRenderer(),
193
194
  verifier: new FfmpegRenderVerifier(),
194
195
  });
195
196
  const disposableNative = liveMode && !headlessFinalCut && !fcpxmlPath && nativeEditor
@@ -1394,9 +1394,23 @@ export function createMcpServer(runtime, options = {}) {
1394
1394
  },
1395
1395
  }, async ({ projectId, sequenceId, expectedRevision, parameters }) => sessionResult(() => requireHeadlessProjects().render(projectId, sequenceId, parameters, expectedRevision)));
1396
1396
  server.registerTool("headless.rough-cut.preview", {
1397
- description: "Preview a semantic rough-cut plan with optional Timeline IR operations; the digest binds both, and preview returns provenance and FCPXML output intent without mutation.",
1397
+ description: "Preview a semantic rough-cut plan and a bounded storyboard; the digest binds the plan and operations without mutating the timeline.",
1398
1398
  inputSchema: headlessRoughCutRequestSchema.shape,
1399
- }, async (request) => sessionResult(() => requireHeadlessProjects().previewRoughCut(request)));
1399
+ }, async (request, extra) => {
1400
+ try {
1401
+ const preview = await requireHeadlessProjects().previewRoughCut(request, extra.signal, true);
1402
+ if (!preview.storyboard)
1403
+ throw new Error("HEADLESS_STORYBOARD_UNAVAILABLE: storyboard renderer returned no image");
1404
+ const { data, mimeType, ...metadata } = preview.storyboard;
1405
+ return { content: [
1406
+ { type: "text", text: JSON.stringify({ ...preview, storyboard: { mimeType, ...metadata } }) },
1407
+ { type: "image", data, mimeType },
1408
+ ] };
1409
+ }
1410
+ catch (error) {
1411
+ return sessionResult(async () => { throw error; });
1412
+ }
1413
+ });
1400
1414
  server.registerTool("headless.rough-cut.execute", {
1401
1415
  description: "Execute an explicitly approved semantic rough-cut plan and its digest-bound Timeline IR operations headlessly, verify MP4 output, and write equivalent editable FCPXML.",
1402
1416
  inputSchema: {
@@ -6,8 +6,29 @@ export interface HeadlessProjectServiceOptions {
6
6
  mediaProbe?: LocalMediaMetadataProbe;
7
7
  renderer?: FramekitRenderProvider;
8
8
  verifier?: FramekitRenderArtifactVerifier;
9
+ storyboard?: StoryboardRenderer;
9
10
  clock?: () => string;
10
11
  }
12
+ export interface StoryboardRenderer {
13
+ render(timeline: TimelineIr, dimensions: {
14
+ width: number;
15
+ height: number;
16
+ }, signal?: AbortSignal): Promise<StoryboardImage>;
17
+ }
18
+ export interface StoryboardImage {
19
+ data: string;
20
+ mimeType: "image/png";
21
+ samples: Array<{
22
+ kind: "shot" | "title";
23
+ id: string;
24
+ occurrenceId: string;
25
+ mediaId: string;
26
+ sourceDigest: string;
27
+ timelineSeconds: number;
28
+ sourceSeconds: number;
29
+ }>;
30
+ limitations: string[];
31
+ }
11
32
  export type HeadlessProjectEditCommand = Omit<ProjectEditCommand, "target"> & {
12
33
  target?: ProjectEditCommand["target"];
13
34
  };
@@ -36,7 +57,7 @@ export interface HeadlessProjectWorkflow {
36
57
  provenance: TimelineIrToFcpxmlResult["provenance"];
37
58
  coverage: TimelineIrToFcpxmlResult["coverage"];
38
59
  }>;
39
- previewRoughCut(request: HeadlessRoughCutRequest): Promise<HeadlessRoughCutPreview>;
60
+ previewRoughCut(request: HeadlessRoughCutRequest, signal?: AbortSignal, requireStoryboard?: boolean): Promise<HeadlessRoughCutPreview>;
40
61
  executeRoughCut(request: HeadlessRoughCutExecutionRequest): Promise<HeadlessRoughCutExecution>;
41
62
  verifyPublishableRoughCut(executionId: string, artifactPath: string, artifactDigest?: string): Promise<HeadlessRoughCutPublishProof>;
42
63
  verifyPublishableRoughCutTransaction(request: HeadlessRoughCutPublishVerificationRequest): Promise<HeadlessRoughCutPublishProof>;
@@ -115,6 +136,10 @@ export interface HeadlessRoughCutPreview extends ProjectEditPreview {
115
136
  plan: RoughCutPlan;
116
137
  planDigest: string;
117
138
  provenance: HeadlessRoughCutShotProvenance[];
139
+ storyboard?: StoryboardImage & {
140
+ planDigest: string;
141
+ revision: ContextRevision;
142
+ };
118
143
  outputIntent: {
119
144
  sourceAudioPolicy: "preserve-and-mix";
120
145
  render: FramekitRenderParameters;
@@ -175,6 +200,7 @@ export declare class HeadlessProjectService implements HeadlessProjectWorkflow {
175
200
  private readonly mediaProbe?;
176
201
  private readonly renderer?;
177
202
  private readonly verifier?;
203
+ private readonly storyboard?;
178
204
  private readonly clock;
179
205
  constructor(options: HeadlessProjectServiceOptions);
180
206
  create(timeline: TimelineIr): Promise<FramekitProjectDocument>;
@@ -211,7 +237,7 @@ export declare class HeadlessProjectService implements HeadlessProjectWorkflow {
211
237
  provenance: import("../../../adapters/final-cut/typescript/src/documents.js").TimelineIrMaterializationProvenance;
212
238
  coverage: import("../../../adapters/final-cut/typescript/src/documents.js").TimelineIrMaterializationCoverage;
213
239
  }>;
214
- previewRoughCut(request: HeadlessRoughCutRequest): Promise<HeadlessRoughCutPreview>;
240
+ previewRoughCut(request: HeadlessRoughCutRequest, signal?: AbortSignal, requireStoryboard?: boolean): Promise<HeadlessRoughCutPreview>;
215
241
  executeRoughCut(request: HeadlessRoughCutExecutionRequest): Promise<HeadlessRoughCutExecution>;
216
242
  verifyPublishableRoughCut(executionId: string, artifactPath: string, artifactDigest?: string): Promise<HeadlessRoughCutPublishProof>;
217
243
  verifyPublishableRoughCutTransaction(request: HeadlessRoughCutPublishVerificationRequest): Promise<HeadlessRoughCutPublishProof>;
@@ -9,6 +9,7 @@ export class HeadlessProjectService {
9
9
  mediaProbe;
10
10
  renderer;
11
11
  verifier;
12
+ storyboard;
12
13
  clock;
13
14
  constructor(options) {
14
15
  this.directory = options.directory;
@@ -16,6 +17,7 @@ export class HeadlessProjectService {
16
17
  this.mediaProbe = options.mediaProbe;
17
18
  this.renderer = options.renderer;
18
19
  this.verifier = options.verifier;
20
+ this.storyboard = options.storyboard;
19
21
  this.clock = options.clock ?? (() => new Date().toISOString());
20
22
  }
21
23
  async create(timeline) {
@@ -179,7 +181,10 @@ export class HeadlessProjectService {
179
181
  const artifact = await writeVerifiedFcpxml(resolve(outputPath), compiled.xml, compiled.digest, false);
180
182
  return { artifact, provenance: compiled.provenance, coverage: compiled.coverage };
181
183
  }
182
- async previewRoughCut(request) {
184
+ async previewRoughCut(request, signal, requireStoryboard = false) {
185
+ signal?.throwIfAborted();
186
+ if (requireStoryboard && !this.storyboard)
187
+ throw new Error("HEADLESS_STORYBOARD_UNAVAILABLE: storyboard renderer is not configured");
183
188
  const prepared = await this.prepareRoughCut(request);
184
189
  const timestamp = this.clock();
185
190
  const { preview, fcpxml } = await this.previewPreparedRoughCut(request, prepared, timestamp);
@@ -193,12 +198,20 @@ export class HeadlessProjectService {
193
198
  artifactDigest: fcpxml.digest,
194
199
  outputIntent: prepared.outputIntent,
195
200
  });
201
+ const storyboard = this.storyboard && await this.storyboard.render(preview.after.timeline, request.render, signal);
202
+ signal?.throwIfAborted();
203
+ const current = await this.open(request.projectId);
204
+ if (timelineIrDigest(current.timeline) !== timelineIrDigest(prepared.project.timeline)) {
205
+ throw new Error("HEADLESS_ROUGH_CUT_STALE_REVISION: project changed during storyboard generation");
206
+ }
207
+ signal?.throwIfAborted();
196
208
  await this.saveRoughCutApproval(approval);
197
209
  return {
198
210
  ...preview,
199
211
  plan: structuredClone(request.plan),
200
212
  planDigest: approval.planDigest,
201
213
  provenance: prepared.provenance,
214
+ ...(storyboard ? { storyboard: { ...storyboard, planDigest: approval.planDigest, revision: preview.after.timeline.revision } } : {}),
202
215
  outputIntent: prepared.outputIntent,
203
216
  fcpxml: selectFcpxmlMetadata(fcpxml),
204
217
  };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@morshoto/framekit",
3
- "version": "0.2.3",
3
+ "version": "0.2.4",
4
4
  "packageManager": "pnpm@12.18.0",
5
5
  "private": false,
6
6
  "type": "module",