@speclip/pi-talking-head 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +97 -16
- package/extensions/talking-head/index.ts +171 -7
- package/package.json +1 -1
- package/prompts/edit-talking-head.md +1 -1
- package/skills/talking-head-edit/SKILL.md +11 -2
- package/skills/talking-head-edit/references/cut-craft.md +5 -0
- package/src/broll.ts +406 -0
- package/src/continuity.ts +355 -0
- package/src/contracts.ts +108 -1
- package/src/project.ts +58 -8
- package/src/workspace.ts +18 -4
|
@@ -0,0 +1,355 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import type {
|
|
3
|
+
ArollJumpCut,
|
|
4
|
+
ArollSegment,
|
|
5
|
+
BrollContinuityBridge,
|
|
6
|
+
BrollContinuityPlan,
|
|
7
|
+
BrollContinuityPlanReceipt,
|
|
8
|
+
BrollNeed,
|
|
9
|
+
} from "./contracts.ts";
|
|
10
|
+
import { timelineDuration } from "./transcript.ts";
|
|
11
|
+
|
|
12
|
+
export interface PlanBrollContinuityInput {
|
|
13
|
+
aroll: ArollSegment[];
|
|
14
|
+
needs: BrollNeed[];
|
|
15
|
+
shortGapMs?: number;
|
|
16
|
+
cutCoverBeforeMs?: number;
|
|
17
|
+
cutCoverAfterMs?: number;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export interface BrollOutputWindow {
|
|
21
|
+
id: string;
|
|
22
|
+
outputStartMs: number;
|
|
23
|
+
outputEndMs: number;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export interface ShortArollFlashGap {
|
|
27
|
+
fromBrollId: string;
|
|
28
|
+
toBrollId: string;
|
|
29
|
+
startMs: number;
|
|
30
|
+
endMs: number;
|
|
31
|
+
durationMs: number;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export interface OverlappingBrollWindows {
|
|
35
|
+
underBrollId: string;
|
|
36
|
+
overBrollId: string;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function boundedMilliseconds(value: number | undefined, fallback: number, name: string, maximum: number): number {
|
|
40
|
+
const selected = value ?? fallback;
|
|
41
|
+
if (!Number.isInteger(selected) || selected < 0 || selected > maximum) {
|
|
42
|
+
throw new Error(`${name} must be an integer within 0-${maximum}`);
|
|
43
|
+
}
|
|
44
|
+
return selected;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export function findShortArollFlashGaps(
|
|
48
|
+
windows: BrollOutputWindow[],
|
|
49
|
+
shortGapMs = 500,
|
|
50
|
+
): ShortArollFlashGap[] {
|
|
51
|
+
const threshold = boundedMilliseconds(shortGapMs, 500, "shortGapMs", 2_000);
|
|
52
|
+
const sorted = [...windows].sort((left, right) => (
|
|
53
|
+
left.outputStartMs - right.outputStartMs
|
|
54
|
+
|| left.outputEndMs - right.outputEndMs
|
|
55
|
+
|| (left.id < right.id ? -1 : left.id > right.id ? 1 : 0)
|
|
56
|
+
));
|
|
57
|
+
const gaps: ShortArollFlashGap[] = [];
|
|
58
|
+
let frontier = sorted[0];
|
|
59
|
+
let coveredUntilMs = frontier?.outputEndMs ?? 0;
|
|
60
|
+
for (let index = 1; index < sorted.length; index += 1) {
|
|
61
|
+
const next = sorted[index]!;
|
|
62
|
+
if (next.outputStartMs <= coveredUntilMs) {
|
|
63
|
+
if (next.outputEndMs > coveredUntilMs) {
|
|
64
|
+
frontier = next;
|
|
65
|
+
coveredUntilMs = next.outputEndMs;
|
|
66
|
+
}
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
const durationMs = next.outputStartMs - coveredUntilMs;
|
|
70
|
+
if (durationMs > threshold) {
|
|
71
|
+
frontier = next;
|
|
72
|
+
coveredUntilMs = next.outputEndMs;
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
gaps.push({
|
|
76
|
+
fromBrollId: frontier!.id,
|
|
77
|
+
toBrollId: next.id,
|
|
78
|
+
startMs: coveredUntilMs,
|
|
79
|
+
endMs: next.outputStartMs,
|
|
80
|
+
durationMs,
|
|
81
|
+
});
|
|
82
|
+
frontier = next;
|
|
83
|
+
coveredUntilMs = next.outputEndMs;
|
|
84
|
+
}
|
|
85
|
+
return gaps;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export function findOverlappingBrollWindows(windows: BrollOutputWindow[]): OverlappingBrollWindows[] {
|
|
89
|
+
const sorted = [...windows].sort((left, right) => (
|
|
90
|
+
left.outputStartMs - right.outputStartMs
|
|
91
|
+
|| right.outputEndMs - left.outputEndMs
|
|
92
|
+
|| (left.id < right.id ? -1 : left.id > right.id ? 1 : 0)
|
|
93
|
+
));
|
|
94
|
+
const overlaps: OverlappingBrollWindows[] = [];
|
|
95
|
+
let frontier = sorted[0];
|
|
96
|
+
for (let index = 1; index < sorted.length; index += 1) {
|
|
97
|
+
const next = sorted[index]!;
|
|
98
|
+
if (frontier && next.outputStartMs < frontier.outputEndMs) {
|
|
99
|
+
overlaps.push({ underBrollId: frontier.id, overBrollId: next.id });
|
|
100
|
+
if (next.outputEndMs > frontier.outputEndMs) frontier = next;
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
frontier = next;
|
|
104
|
+
}
|
|
105
|
+
return overlaps;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function validateSegments(aroll: ArollSegment[]): void {
|
|
109
|
+
if (aroll.length === 0 || aroll.length > 1_000) throw new Error("A-roll must contain 1-1000 segments");
|
|
110
|
+
const ids = new Set<string>();
|
|
111
|
+
for (const segment of aroll) {
|
|
112
|
+
if (ids.has(segment.id)) throw new Error(`Duplicate A-roll segment ID: ${segment.id}`);
|
|
113
|
+
ids.add(segment.id);
|
|
114
|
+
if (!Number.isFinite(segment.sourceStartMs) || !Number.isFinite(segment.sourceEndMs)
|
|
115
|
+
|| segment.sourceStartMs < 0 || segment.sourceEndMs <= segment.sourceStartMs) {
|
|
116
|
+
throw new Error(`Invalid A-roll segment range: ${segment.id}`);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function validateNeeds(needs: BrollNeed[], outputDurationMs: number): void {
|
|
122
|
+
if (needs.length > 500) throw new Error("B-roll plan must contain at most 500 needs");
|
|
123
|
+
const ids = new Set<string>();
|
|
124
|
+
for (const need of needs) {
|
|
125
|
+
if (ids.has(need.id)) throw new Error(`Duplicate B-roll need ID: ${need.id}`);
|
|
126
|
+
ids.add(need.id);
|
|
127
|
+
if (!Number.isFinite(need.outputStartMs) || !Number.isFinite(need.outputEndMs)
|
|
128
|
+
|| need.outputStartMs < 0 || need.outputEndMs <= need.outputStartMs
|
|
129
|
+
|| need.outputEndMs > outputDurationMs) {
|
|
130
|
+
throw new Error(`Invalid B-roll need output range: ${need.id}`);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
const overlap = findOverlappingBrollWindows(needs)[0];
|
|
134
|
+
if (overlap) throw new Error(`B-roll needs ${overlap.underBrollId} and ${overlap.overBrollId} overlap; z-order is not supported`);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function bridgeShortGaps(needs: BrollNeed[], shortGapMs: number): {
|
|
138
|
+
needs: BrollNeed[];
|
|
139
|
+
bridges: BrollContinuityBridge[];
|
|
140
|
+
} {
|
|
141
|
+
const sorted = structuredClone(needs).sort((left, right) => (
|
|
142
|
+
left.outputStartMs - right.outputStartMs
|
|
143
|
+
|| left.outputEndMs - right.outputEndMs
|
|
144
|
+
|| (left.id < right.id ? -1 : left.id > right.id ? 1 : 0)
|
|
145
|
+
));
|
|
146
|
+
const bridges: BrollContinuityBridge[] = [];
|
|
147
|
+
let frontier = sorted[0];
|
|
148
|
+
let coveredUntilMs = frontier?.outputEndMs ?? 0;
|
|
149
|
+
for (let index = 1; index < sorted.length; index += 1) {
|
|
150
|
+
const next = sorted[index]!;
|
|
151
|
+
if (next.outputStartMs <= coveredUntilMs) {
|
|
152
|
+
if (next.outputEndMs > coveredUntilMs) {
|
|
153
|
+
frontier = next;
|
|
154
|
+
coveredUntilMs = next.outputEndMs;
|
|
155
|
+
}
|
|
156
|
+
continue;
|
|
157
|
+
}
|
|
158
|
+
const durationMs = next.outputStartMs - coveredUntilMs;
|
|
159
|
+
if (durationMs > shortGapMs) {
|
|
160
|
+
frontier = next;
|
|
161
|
+
coveredUntilMs = next.outputEndMs;
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
164
|
+
bridges.push({
|
|
165
|
+
fromNeedId: frontier!.id,
|
|
166
|
+
toNeedId: next.id,
|
|
167
|
+
gapStartMs: coveredUntilMs,
|
|
168
|
+
gapEndMs: next.outputStartMs,
|
|
169
|
+
durationMs,
|
|
170
|
+
strategy: "extend-previous-broll",
|
|
171
|
+
});
|
|
172
|
+
frontier!.outputEndMs = next.outputStartMs;
|
|
173
|
+
frontier = next;
|
|
174
|
+
coveredUntilMs = next.outputEndMs;
|
|
175
|
+
}
|
|
176
|
+
return { needs: sorted, bridges };
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
function coverageForRange(needs: BrollOutputWindow[], startMs: number, endMs: number): {
|
|
180
|
+
covered: boolean;
|
|
181
|
+
needIds: string[];
|
|
182
|
+
} {
|
|
183
|
+
const overlapping = needs.filter((need) => need.outputEndMs > startMs && need.outputStartMs < endMs).sort((left, right) => (
|
|
184
|
+
left.outputStartMs - right.outputStartMs
|
|
185
|
+
|| left.outputEndMs - right.outputEndMs
|
|
186
|
+
|| (left.id < right.id ? -1 : left.id > right.id ? 1 : 0)
|
|
187
|
+
));
|
|
188
|
+
let cursor = startMs;
|
|
189
|
+
const needIds: string[] = [];
|
|
190
|
+
for (const need of overlapping) {
|
|
191
|
+
if (need.outputStartMs > cursor) return { covered: false, needIds };
|
|
192
|
+
needIds.push(need.id);
|
|
193
|
+
cursor = Math.max(cursor, need.outputEndMs);
|
|
194
|
+
if (cursor >= endMs) return { covered: true, needIds };
|
|
195
|
+
}
|
|
196
|
+
return { covered: false, needIds };
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
export function analyzeArollJumpCuts(
|
|
200
|
+
aroll: ArollSegment[],
|
|
201
|
+
needs: BrollOutputWindow[],
|
|
202
|
+
outputDurationMs: number,
|
|
203
|
+
cutCoverBeforeMs: number,
|
|
204
|
+
cutCoverAfterMs: number,
|
|
205
|
+
): ArollJumpCut[] {
|
|
206
|
+
const jumpCuts: ArollJumpCut[] = [];
|
|
207
|
+
let outputTimeMs = 0;
|
|
208
|
+
for (let index = 0; index < aroll.length - 1; index += 1) {
|
|
209
|
+
const current = aroll[index]!;
|
|
210
|
+
const next = aroll[index + 1]!;
|
|
211
|
+
outputTimeMs += current.sourceEndMs - current.sourceStartMs;
|
|
212
|
+
const sourceDeltaMs = next.sourceStartMs - current.sourceEndMs;
|
|
213
|
+
if (sourceDeltaMs === 0) continue;
|
|
214
|
+
const suggestedOutputStartMs = Math.max(0, outputTimeMs - cutCoverBeforeMs);
|
|
215
|
+
const suggestedOutputEndMs = Math.min(outputDurationMs, outputTimeMs + cutCoverAfterMs);
|
|
216
|
+
const coverage = coverageForRange(needs, suggestedOutputStartMs, suggestedOutputEndMs);
|
|
217
|
+
jumpCuts.push({
|
|
218
|
+
fromSegmentId: current.id,
|
|
219
|
+
toSegmentId: next.id,
|
|
220
|
+
outputTimeMs,
|
|
221
|
+
sourceDeltaMs,
|
|
222
|
+
suggestedOutputStartMs,
|
|
223
|
+
suggestedOutputEndMs,
|
|
224
|
+
coveredByNeedIds: coverage.needIds,
|
|
225
|
+
status: coverage.covered ? "covered" : "needs-broll",
|
|
226
|
+
});
|
|
227
|
+
}
|
|
228
|
+
return jumpCuts;
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
export function planBrollContinuity(input: PlanBrollContinuityInput): BrollContinuityPlan {
|
|
232
|
+
validateSegments(input.aroll);
|
|
233
|
+
const outputDurationMs = timelineDuration(input.aroll);
|
|
234
|
+
validateNeeds(input.needs, outputDurationMs);
|
|
235
|
+
const shortGapMs = boundedMilliseconds(input.shortGapMs, 500, "shortGapMs", 2_000);
|
|
236
|
+
const cutCoverBeforeMs = boundedMilliseconds(input.cutCoverBeforeMs, 250, "cutCoverBeforeMs", 2_000);
|
|
237
|
+
const cutCoverAfterMs = boundedMilliseconds(input.cutCoverAfterMs, 500, "cutCoverAfterMs", 2_000);
|
|
238
|
+
if (cutCoverBeforeMs === 0 && cutCoverAfterMs === 0) {
|
|
239
|
+
throw new Error("cutCoverBeforeMs and cutCoverAfterMs cannot both be zero");
|
|
240
|
+
}
|
|
241
|
+
const bridged = bridgeShortGaps(input.needs, shortGapMs);
|
|
242
|
+
return {
|
|
243
|
+
outputDurationMs,
|
|
244
|
+
shortGapMs,
|
|
245
|
+
cutCoverBeforeMs,
|
|
246
|
+
cutCoverAfterMs,
|
|
247
|
+
needs: bridged.needs,
|
|
248
|
+
bridges: bridged.bridges,
|
|
249
|
+
jumpCuts: analyzeArollJumpCuts(
|
|
250
|
+
input.aroll,
|
|
251
|
+
bridged.needs,
|
|
252
|
+
outputDurationMs,
|
|
253
|
+
cutCoverBeforeMs,
|
|
254
|
+
cutCoverAfterMs,
|
|
255
|
+
),
|
|
256
|
+
};
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
function planReceiptPayload(receipt: Omit<BrollContinuityPlanReceipt, "planSha256">): string {
|
|
260
|
+
return canonicalJson(receipt);
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
function canonicalJson(value: unknown): string {
|
|
264
|
+
if (value === null || typeof value !== "object") {
|
|
265
|
+
const serialized = JSON.stringify(value);
|
|
266
|
+
if (serialized === undefined) throw new Error("Cannot canonicalize an undefined value");
|
|
267
|
+
return serialized;
|
|
268
|
+
}
|
|
269
|
+
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
|
|
270
|
+
const object = value as Record<string, unknown>;
|
|
271
|
+
return `{${Object.keys(object).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(object[key])}`).join(",")}}`;
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
function sha256(value: string): string {
|
|
275
|
+
return createHash("sha256").update(value).digest("hex");
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
function arollSha256(aroll: ArollSegment[]): string {
|
|
279
|
+
return sha256(canonicalJson(aroll));
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
function plannedRanges(needs: BrollNeed[]): BrollContinuityPlanReceipt["needs"] {
|
|
283
|
+
return needs.map((need) => ({
|
|
284
|
+
id: need.id,
|
|
285
|
+
outputStartMs: need.outputStartMs,
|
|
286
|
+
outputEndMs: need.outputEndMs,
|
|
287
|
+
}));
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
export function createBrollContinuityPlanReceipt(
|
|
291
|
+
projectId: string,
|
|
292
|
+
revision: number,
|
|
293
|
+
aroll: ArollSegment[],
|
|
294
|
+
plan: BrollContinuityPlan,
|
|
295
|
+
): BrollContinuityPlanReceipt {
|
|
296
|
+
const payload: Omit<BrollContinuityPlanReceipt, "planSha256"> = {
|
|
297
|
+
schemaVersion: 1,
|
|
298
|
+
projectId,
|
|
299
|
+
revision,
|
|
300
|
+
arollSha256: arollSha256(aroll),
|
|
301
|
+
shortGapMs: plan.shortGapMs,
|
|
302
|
+
cutCoverBeforeMs: plan.cutCoverBeforeMs,
|
|
303
|
+
cutCoverAfterMs: plan.cutCoverAfterMs,
|
|
304
|
+
needs: plannedRanges(plan.needs),
|
|
305
|
+
jumpCuts: structuredClone(plan.jumpCuts),
|
|
306
|
+
};
|
|
307
|
+
return { ...payload, planSha256: sha256(planReceiptPayload(payload)) };
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
export function verifyBrollContinuityPlanReceipt(
|
|
311
|
+
projectId: string,
|
|
312
|
+
revision: number,
|
|
313
|
+
aroll: ArollSegment[],
|
|
314
|
+
placements: BrollOutputWindow[],
|
|
315
|
+
receipt: BrollContinuityPlanReceipt,
|
|
316
|
+
): void {
|
|
317
|
+
if (receipt.schemaVersion !== 1 || receipt.projectId !== projectId || receipt.revision !== revision) {
|
|
318
|
+
throw new Error("B-roll continuity receipt does not match the current project revision");
|
|
319
|
+
}
|
|
320
|
+
if (receipt.arollSha256 !== arollSha256(aroll)) {
|
|
321
|
+
throw new Error("B-roll continuity receipt does not match the planned A-roll");
|
|
322
|
+
}
|
|
323
|
+
const { planSha256, ...payload } = receipt;
|
|
324
|
+
if (planSha256 !== sha256(planReceiptPayload(payload))) {
|
|
325
|
+
throw new Error("B-roll continuity receipt hash is invalid");
|
|
326
|
+
}
|
|
327
|
+
const dummyNeeds: BrollNeed[] = receipt.needs.map((need) => ({
|
|
328
|
+
...need,
|
|
329
|
+
speechText: "continuity receipt",
|
|
330
|
+
purpose: "mask-cut",
|
|
331
|
+
searchTerms: ["continuity"],
|
|
332
|
+
reason: "continuity receipt verification",
|
|
333
|
+
}));
|
|
334
|
+
const recomputed = planBrollContinuity({
|
|
335
|
+
aroll,
|
|
336
|
+
needs: dummyNeeds,
|
|
337
|
+
shortGapMs: receipt.shortGapMs,
|
|
338
|
+
cutCoverBeforeMs: receipt.cutCoverBeforeMs,
|
|
339
|
+
cutCoverAfterMs: receipt.cutCoverAfterMs,
|
|
340
|
+
});
|
|
341
|
+
if (canonicalJson(plannedRanges(recomputed.needs)) !== canonicalJson(receipt.needs)
|
|
342
|
+
|| canonicalJson(recomputed.jumpCuts) !== canonicalJson(receipt.jumpCuts)) {
|
|
343
|
+
throw new Error("B-roll continuity receipt does not match the recomputed plan");
|
|
344
|
+
}
|
|
345
|
+
const actualRanges = [...placements].sort((left, right) => (
|
|
346
|
+
left.outputStartMs - right.outputStartMs || left.outputEndMs - right.outputEndMs
|
|
347
|
+
)).map((placement) => ({
|
|
348
|
+
id: placement.id,
|
|
349
|
+
outputStartMs: placement.outputStartMs,
|
|
350
|
+
outputEndMs: placement.outputEndMs,
|
|
351
|
+
}));
|
|
352
|
+
if (canonicalJson(actualRanges) !== canonicalJson(receipt.needs)) {
|
|
353
|
+
throw new Error("B-roll placements do not match the planned ranges");
|
|
354
|
+
}
|
|
355
|
+
}
|
package/src/contracts.ts
CHANGED
|
@@ -95,13 +95,118 @@ export interface BrollPlacement {
|
|
|
95
95
|
outputStartMs: number;
|
|
96
96
|
outputEndMs: number;
|
|
97
97
|
assetStartMs?: number;
|
|
98
|
+
selectionReceipt?: BrollSelectionReceipt;
|
|
98
99
|
fit: "cover" | "contain";
|
|
99
100
|
audio: "keep-primary";
|
|
100
101
|
query?: string;
|
|
101
102
|
reason?: string;
|
|
102
103
|
}
|
|
103
104
|
|
|
104
|
-
export
|
|
105
|
+
export interface BrollSelectionReceipt {
|
|
106
|
+
assetBytes: number;
|
|
107
|
+
assetSha256: string;
|
|
108
|
+
manifestPath: string;
|
|
109
|
+
manifestSha256: string;
|
|
110
|
+
selectedEndMs: number;
|
|
111
|
+
evidenceTimestampsMs: number[];
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
export type BrollPlacementInput = Omit<
|
|
115
|
+
BrollPlacement,
|
|
116
|
+
"assetBytes" | "assetSha256" | "assetStartMs" | "selectionReceipt"
|
|
117
|
+
> & {
|
|
118
|
+
assetStartMs: number;
|
|
119
|
+
selectionReceipt: BrollSelectionReceipt;
|
|
120
|
+
};
|
|
121
|
+
|
|
122
|
+
export type BrollPurpose = "demonstrate" | "explain" | "evidence" | "establish" | "transition" | "mask-cut";
|
|
123
|
+
|
|
124
|
+
export interface BrollNeed {
|
|
125
|
+
id: string;
|
|
126
|
+
outputStartMs: number;
|
|
127
|
+
outputEndMs: number;
|
|
128
|
+
speechText: string;
|
|
129
|
+
purpose: BrollPurpose;
|
|
130
|
+
searchTerms: string[];
|
|
131
|
+
reason: string;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export interface BrollAssetCandidate {
|
|
135
|
+
assetPath: string;
|
|
136
|
+
fileName: string;
|
|
137
|
+
matchedTerms: string[];
|
|
138
|
+
score: number;
|
|
139
|
+
confidence: "high" | "medium" | "low";
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
interface BrollAssetMatchBase {
|
|
143
|
+
requiredDurationMs: number;
|
|
144
|
+
totalCandidates: number;
|
|
145
|
+
candidateOffset: number;
|
|
146
|
+
nextCandidateOffset: number | null;
|
|
147
|
+
shortlist: BrollAssetCandidate[];
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
export type BrollAssetMatch =
|
|
151
|
+
| BrollAssetMatchBase & {
|
|
152
|
+
selectionMode: "filename-direct";
|
|
153
|
+
nextStep: { action: "inspect-selected-asset"; reason: string };
|
|
154
|
+
}
|
|
155
|
+
| BrollAssetMatchBase & {
|
|
156
|
+
selectionMode: "filename-shortlist";
|
|
157
|
+
nextStep: { action: "inspect-shortlist"; reason: string };
|
|
158
|
+
}
|
|
159
|
+
| BrollAssetMatchBase & {
|
|
160
|
+
selectionMode: "visual-fallback";
|
|
161
|
+
nextStep: { action: "visual-fallback"; reason: string };
|
|
162
|
+
};
|
|
163
|
+
|
|
164
|
+
export interface BrollWindowSelection {
|
|
165
|
+
placement: BrollPlacementInput;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
export interface BrollContinuityBridge {
|
|
169
|
+
fromNeedId: string;
|
|
170
|
+
toNeedId: string;
|
|
171
|
+
gapStartMs: number;
|
|
172
|
+
gapEndMs: number;
|
|
173
|
+
durationMs: number;
|
|
174
|
+
strategy: "extend-previous-broll";
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
export interface ArollJumpCut {
|
|
178
|
+
fromSegmentId: string;
|
|
179
|
+
toSegmentId: string;
|
|
180
|
+
outputTimeMs: number;
|
|
181
|
+
sourceDeltaMs: number;
|
|
182
|
+
suggestedOutputStartMs: number;
|
|
183
|
+
suggestedOutputEndMs: number;
|
|
184
|
+
coveredByNeedIds: string[];
|
|
185
|
+
status: "covered" | "needs-broll";
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
export interface BrollContinuityPlan {
|
|
189
|
+
outputDurationMs: number;
|
|
190
|
+
shortGapMs: number;
|
|
191
|
+
cutCoverBeforeMs: number;
|
|
192
|
+
cutCoverAfterMs: number;
|
|
193
|
+
needs: BrollNeed[];
|
|
194
|
+
bridges: BrollContinuityBridge[];
|
|
195
|
+
jumpCuts: ArollJumpCut[];
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
export interface BrollContinuityPlanReceipt {
|
|
199
|
+
schemaVersion: 1;
|
|
200
|
+
projectId: string;
|
|
201
|
+
revision: number;
|
|
202
|
+
arollSha256: string;
|
|
203
|
+
shortGapMs: number;
|
|
204
|
+
cutCoverBeforeMs: number;
|
|
205
|
+
cutCoverAfterMs: number;
|
|
206
|
+
needs: Array<{ id: string; outputStartMs: number; outputEndMs: number }>;
|
|
207
|
+
jumpCuts: ArollJumpCut[];
|
|
208
|
+
planSha256: string;
|
|
209
|
+
}
|
|
105
210
|
|
|
106
211
|
export interface TranscriptAnalysis {
|
|
107
212
|
schemaVersion: 2;
|
|
@@ -147,6 +252,8 @@ export interface TalkingHeadSnapshot {
|
|
|
147
252
|
policy: TalkingHeadPolicy;
|
|
148
253
|
aroll: ArollSegment[];
|
|
149
254
|
broll: BrollPlacement[];
|
|
255
|
+
continuityPlanReceipt?: BrollContinuityPlanReceipt;
|
|
256
|
+
jumpCuts?: ArollJumpCut[];
|
|
150
257
|
outputDurationMs: number;
|
|
151
258
|
}
|
|
152
259
|
|
package/src/project.ts
CHANGED
|
@@ -5,6 +5,7 @@ import type {
|
|
|
5
5
|
ArollSegment,
|
|
6
6
|
BrollPlacement,
|
|
7
7
|
BrollPlacementInput,
|
|
8
|
+
BrollContinuityPlanReceipt,
|
|
8
9
|
FillerCandidate,
|
|
9
10
|
PauseCandidate,
|
|
10
11
|
RepetitionCandidate,
|
|
@@ -16,6 +17,13 @@ import type {
|
|
|
16
17
|
TranscriptWord,
|
|
17
18
|
WordTranscript,
|
|
18
19
|
} from "./contracts.ts";
|
|
20
|
+
import { verifyBrollPlacementSelection } from "./broll.ts";
|
|
21
|
+
import {
|
|
22
|
+
analyzeArollJumpCuts,
|
|
23
|
+
findOverlappingBrollWindows,
|
|
24
|
+
findShortArollFlashGaps,
|
|
25
|
+
verifyBrollContinuityPlanReceipt,
|
|
26
|
+
} from "./continuity.ts";
|
|
19
27
|
import { analyzeTranscript, DEFAULT_POLICY, timelineDuration } from "./transcript.ts";
|
|
20
28
|
import { resolveExistingWorkspaceFile, resolveWorkspacePath, snapshotFile, workspaceRelativePath } from "./workspace.ts";
|
|
21
29
|
|
|
@@ -33,6 +41,7 @@ export interface ApplyTimelineInput {
|
|
|
33
41
|
expectedRevision: number;
|
|
34
42
|
aroll: ArollSegment[];
|
|
35
43
|
broll: BrollPlacementInput[];
|
|
44
|
+
continuityPlanReceipt?: BrollContinuityPlanReceipt;
|
|
36
45
|
}
|
|
37
46
|
|
|
38
47
|
function assertProjectId(projectId: string): void {
|
|
@@ -77,7 +86,11 @@ async function validateTimeline(
|
|
|
77
86
|
cwd: string,
|
|
78
87
|
aroll: ArollSegment[],
|
|
79
88
|
broll: BrollPlacementInput[],
|
|
80
|
-
|
|
89
|
+
projectId: string,
|
|
90
|
+
revision: number,
|
|
91
|
+
continuityPlanReceipt: BrollContinuityPlanReceipt | undefined,
|
|
92
|
+
signal?: AbortSignal,
|
|
93
|
+
): Promise<{ outputDurationMs: number; broll: BrollPlacement[]; jumpCuts: ReturnType<typeof analyzeArollJumpCuts> }> {
|
|
81
94
|
if (aroll.length === 0 || aroll.length > 1_000) throw new Error("A-roll must contain 1-1000 segments");
|
|
82
95
|
uniqueIds(aroll, "A-roll segment");
|
|
83
96
|
for (const segment of aroll) {
|
|
@@ -89,7 +102,6 @@ async function validateTimeline(
|
|
|
89
102
|
const outputDurationMs = timelineDuration(aroll);
|
|
90
103
|
if (broll.length > 500) throw new Error("B-roll must contain at most 500 placements");
|
|
91
104
|
uniqueIds(broll, "B-roll placement");
|
|
92
|
-
const normalizedBroll: BrollPlacement[] = [];
|
|
93
105
|
for (const placement of broll) {
|
|
94
106
|
if (!Number.isFinite(placement.outputStartMs) || !Number.isFinite(placement.outputEndMs)
|
|
95
107
|
|| placement.outputStartMs < 0 || placement.outputEndMs <= placement.outputStartMs) {
|
|
@@ -98,11 +110,30 @@ async function validateTimeline(
|
|
|
98
110
|
if (placement.outputEndMs > outputDurationMs) {
|
|
99
111
|
throw new Error(`B-roll placement ${placement.id} exceeds output duration ${outputDurationMs}ms`);
|
|
100
112
|
}
|
|
101
|
-
if (
|
|
102
|
-
throw new Error(`
|
|
113
|
+
if (!Number.isFinite(placement.assetStartMs) || placement.assetStartMs < 0) {
|
|
114
|
+
throw new Error(`B-roll assetStartMs is required and must be valid: ${placement.id}`);
|
|
103
115
|
}
|
|
116
|
+
if (!placement.selectionReceipt) throw new Error(`B-roll selectionReceipt is required: ${placement.id}`);
|
|
104
117
|
if (placement.audio !== "keep-primary") throw new Error("B-roll audio must keep the primary A-roll audio");
|
|
105
|
-
|
|
118
|
+
}
|
|
119
|
+
const overlap = findOverlappingBrollWindows(broll)[0];
|
|
120
|
+
if (overlap) {
|
|
121
|
+
throw new Error(`B-roll placements ${overlap.underBrollId} and ${overlap.overBrollId} overlap; ambiguous z-order is not supported`);
|
|
122
|
+
}
|
|
123
|
+
const flashGap = findShortArollFlashGaps(broll, 500)[0];
|
|
124
|
+
if (flashGap) {
|
|
125
|
+
throw new Error(`B-roll placements ${flashGap.fromBrollId} and ${flashGap.toBrollId} leave a brief ${flashGap.durationMs}ms A-roll flash; run talking_head_broll_plan and reselect the bridged window`);
|
|
126
|
+
}
|
|
127
|
+
if (broll.length > 0 && !continuityPlanReceipt) {
|
|
128
|
+
throw new Error("B-roll continuityPlanReceipt is required; run talking_head_broll_plan first");
|
|
129
|
+
}
|
|
130
|
+
if (continuityPlanReceipt) {
|
|
131
|
+
verifyBrollContinuityPlanReceipt(projectId, revision, aroll, broll, continuityPlanReceipt);
|
|
132
|
+
}
|
|
133
|
+
const normalizedBroll: BrollPlacement[] = [];
|
|
134
|
+
for (const placement of broll) {
|
|
135
|
+
signal?.throwIfAborted();
|
|
136
|
+
const asset = await verifyBrollPlacementSelection(cwd, placement, signal);
|
|
106
137
|
normalizedBroll.push({
|
|
107
138
|
...structuredClone(placement),
|
|
108
139
|
assetPath: asset.path,
|
|
@@ -110,7 +141,13 @@ async function validateTimeline(
|
|
|
110
141
|
assetSha256: asset.sha256,
|
|
111
142
|
});
|
|
112
143
|
}
|
|
113
|
-
return {
|
|
144
|
+
return {
|
|
145
|
+
outputDurationMs,
|
|
146
|
+
broll: normalizedBroll,
|
|
147
|
+
jumpCuts: continuityPlanReceipt
|
|
148
|
+
? structuredClone(continuityPlanReceipt.jumpCuts)
|
|
149
|
+
: analyzeArollJumpCuts(aroll, broll, outputDurationMs, 250, 500),
|
|
150
|
+
};
|
|
114
151
|
}
|
|
115
152
|
|
|
116
153
|
function summary(analysis: TranscriptAnalysis) {
|
|
@@ -177,6 +214,7 @@ export async function createTalkingHeadProject(cwd: string, input: CreateTalking
|
|
|
177
214
|
policy,
|
|
178
215
|
aroll: analysis.segments,
|
|
179
216
|
broll: [],
|
|
217
|
+
jumpCuts: analyzeArollJumpCuts(analysis.segments, [], analysis.outputDurationMs, 250, 500),
|
|
180
218
|
outputDurationMs: analysis.outputDurationMs,
|
|
181
219
|
};
|
|
182
220
|
try {
|
|
@@ -233,7 +271,7 @@ async function acquireLock(directory: string): Promise<() => Promise<void>> {
|
|
|
233
271
|
return async () => { await unlink(lockPath).catch(() => undefined); };
|
|
234
272
|
}
|
|
235
273
|
|
|
236
|
-
export async function applyTimeline(cwd: string, input: ApplyTimelineInput): Promise<TalkingHeadSnapshot> {
|
|
274
|
+
export async function applyTimeline(cwd: string, input: ApplyTimelineInput, signal?: AbortSignal): Promise<TalkingHeadSnapshot> {
|
|
237
275
|
const directory = await projectDirectory(cwd, input.projectId);
|
|
238
276
|
const release = await acquireLock(directory);
|
|
239
277
|
try {
|
|
@@ -244,7 +282,15 @@ export async function applyTimeline(cwd: string, input: ApplyTimelineInput): Pro
|
|
|
244
282
|
await assertProjectSourcesUnchanged(cwd, project);
|
|
245
283
|
await assertSnapshotAssetsUnchanged(cwd, current);
|
|
246
284
|
assertWordSafeSegments(input.aroll, await getAnalysis(cwd, project));
|
|
247
|
-
const validated = await validateTimeline(
|
|
285
|
+
const validated = await validateTimeline(
|
|
286
|
+
cwd,
|
|
287
|
+
input.aroll,
|
|
288
|
+
input.broll,
|
|
289
|
+
input.projectId,
|
|
290
|
+
input.expectedRevision,
|
|
291
|
+
input.continuityPlanReceipt,
|
|
292
|
+
signal,
|
|
293
|
+
);
|
|
248
294
|
const revision = project.currentRevision + 1;
|
|
249
295
|
const now = new Date().toISOString();
|
|
250
296
|
const snapshot: TalkingHeadSnapshot = {
|
|
@@ -256,6 +302,10 @@ export async function applyTimeline(cwd: string, input: ApplyTimelineInput): Pro
|
|
|
256
302
|
policy: current.policy,
|
|
257
303
|
aroll: structuredClone(input.aroll),
|
|
258
304
|
broll: validated.broll,
|
|
305
|
+
jumpCuts: validated.jumpCuts,
|
|
306
|
+
...(input.continuityPlanReceipt === undefined
|
|
307
|
+
? {}
|
|
308
|
+
: { continuityPlanReceipt: structuredClone(input.continuityPlanReceipt) }),
|
|
259
309
|
outputDurationMs: validated.outputDurationMs,
|
|
260
310
|
};
|
|
261
311
|
await writeJson(join(directory, "snapshots", `${revision}.json`), snapshot, true);
|
package/src/workspace.ts
CHANGED
|
@@ -39,6 +39,17 @@ export async function resolveExistingWorkspaceFile(cwd: string, inputPath: strin
|
|
|
39
39
|
return canonical;
|
|
40
40
|
}
|
|
41
41
|
|
|
42
|
+
export async function resolveExistingWorkspaceDirectory(cwd: string, inputPath: string): Promise<string> {
|
|
43
|
+
const root = await workspaceRoot(cwd);
|
|
44
|
+
const lexical = resolve(root, inputPath);
|
|
45
|
+
if (!isWithin(root, lexical)) throw new Error(`Path is outside the workspace: ${inputPath}`);
|
|
46
|
+
if ((await lstat(lexical)).isSymbolicLink()) throw new Error(`Unsafe workspace-directory symlink: ${inputPath}`);
|
|
47
|
+
const canonical = await realpath(lexical);
|
|
48
|
+
if (!isWithin(root, canonical)) throw new Error(`Path resolves outside the workspace: ${inputPath}`);
|
|
49
|
+
if (!(await stat(canonical)).isDirectory()) throw new Error(`Path is not a directory: ${inputPath}`);
|
|
50
|
+
return canonical;
|
|
51
|
+
}
|
|
52
|
+
|
|
42
53
|
export async function resolveWorkspacePath(cwd: string, inputPath: string): Promise<string> {
|
|
43
54
|
const root = await workspaceRoot(cwd);
|
|
44
55
|
const lexical = resolve(root, inputPath);
|
|
@@ -55,17 +66,20 @@ export async function workspaceRelativePath(cwd: string, absolutePath: string):
|
|
|
55
66
|
return relative(root, absolutePath).split(sep).join("/");
|
|
56
67
|
}
|
|
57
68
|
|
|
58
|
-
async function sha256File(path: string): Promise<string> {
|
|
69
|
+
async function sha256File(path: string, signal?: AbortSignal): Promise<string> {
|
|
70
|
+
signal?.throwIfAborted();
|
|
59
71
|
const hash = createHash("sha256");
|
|
60
|
-
for await (const chunk of createReadStream(path)) hash.update(chunk);
|
|
72
|
+
for await (const chunk of createReadStream(path, signal === undefined ? {} : { signal })) hash.update(chunk);
|
|
73
|
+
signal?.throwIfAborted();
|
|
61
74
|
return hash.digest("hex");
|
|
62
75
|
}
|
|
63
76
|
|
|
64
|
-
export async function snapshotFile(cwd: string, inputPath: string): Promise<FileRef> {
|
|
77
|
+
export async function snapshotFile(cwd: string, inputPath: string, signal?: AbortSignal): Promise<FileRef> {
|
|
78
|
+
signal?.throwIfAborted();
|
|
65
79
|
const absolute = await resolveExistingWorkspaceFile(cwd, inputPath);
|
|
66
80
|
return {
|
|
67
81
|
path: await workspaceRelativePath(cwd, absolute),
|
|
68
82
|
bytes: (await stat(absolute)).size,
|
|
69
|
-
sha256: await sha256File(absolute),
|
|
83
|
+
sha256: await sha256File(absolute, signal),
|
|
70
84
|
};
|
|
71
85
|
}
|