@speclip/pi-talking-head 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -16
- package/extensions/talking-head/index.ts +70 -3
- package/package.json +1 -1
- package/prompts/edit-talking-head.md +1 -1
- package/skills/talking-head-edit/SKILL.md +7 -6
- package/skills/talking-head-edit/references/cut-craft.md +1 -0
- package/src/continuity.ts +432 -0
- package/src/contracts.ts +58 -0
- package/src/project.ts +61 -5
package/README.md
CHANGED
|
@@ -71,9 +71,55 @@ talking_head_get {
|
|
|
71
71
|
|
|
72
72
|
只有调用这个工具时才会把这些证据放进当前会话上下文;安装 package 不会把整份转录常驻注入上下文。
|
|
73
73
|
|
|
74
|
-
### 3.
|
|
74
|
+
### 3. 先规划 B-roll 视觉连续性
|
|
75
75
|
|
|
76
|
-
|
|
76
|
+
在查找素材前,把最终 A-roll 和所有 B-roll 需求交给连续性规划工具:
|
|
77
|
+
|
|
78
|
+
```js
|
|
79
|
+
talking_head_broll_plan {
|
|
80
|
+
projectId: "launch-video",
|
|
81
|
+
aroll: [
|
|
82
|
+
{ id: "hook", sourceStartMs: 50, sourceEndMs: 4120 },
|
|
83
|
+
{ id: "answer", sourceStartMs: 4860, sourceEndMs: 13200 }
|
|
84
|
+
],
|
|
85
|
+
needs: [
|
|
86
|
+
{
|
|
87
|
+
id: "product-a",
|
|
88
|
+
outputStartMs: 1000,
|
|
89
|
+
outputEndMs: 3000,
|
|
90
|
+
speechText: "第一步打开设置",
|
|
91
|
+
purpose: "demonstrate",
|
|
92
|
+
searchTerms: ["设置", "打开"],
|
|
93
|
+
reason: "展示第一步"
|
|
94
|
+
},
|
|
95
|
+
{
|
|
96
|
+
id: "product-b",
|
|
97
|
+
outputStartMs: 3300,
|
|
98
|
+
outputEndMs: 5200,
|
|
99
|
+
speechText: "第二步选择设备",
|
|
100
|
+
purpose: "demonstrate",
|
|
101
|
+
searchTerms: ["设备", "选择"],
|
|
102
|
+
reason: "展示第二步"
|
|
103
|
+
}
|
|
104
|
+
],
|
|
105
|
+
cutCoverBeforeMs: 250,
|
|
106
|
+
cutCoverAfterMs: 500
|
|
107
|
+
}
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
规划结果同时返回 `plan.coverage` 和 `continuityPlanReceipt`。`plan.coverage` 会列出 B-roll/A-roll 时长、B-roll 占比和最长连续 B-roll;凭证则把当前项目 revision、A-roll、规划后的 B-roll 区间和跳点审阅结果绑定在一起。后续 `talking_head_apply` 会重新计算并核对,不能跳过规划或把已经衔接的区间手动缩短。
|
|
111
|
+
|
|
112
|
+
规划结果处理两类问题:
|
|
113
|
+
|
|
114
|
+
- **B-roll 闪屏**:两个 B-roll 之间只露出不超过 500ms 的 A-roll 时,`plan.needs` 会把前一个需求延长到后一个需求的起点。后续分别选段后,两段 B-roll 会直接交接,不再闪回 A-roll。
|
|
115
|
+
- **气口剪辑跳转**:A-roll 相邻片段的源时间不连续时,会生成 `jumpCuts`。工具检查建议遮盖区间是否已被 B-roll 连续覆盖;未覆盖项标为 `needs-broll`,Agent 应结合口播语义补充 `purpose: "mask-cut"` 的需求,再重新规划。
|
|
116
|
+
- **A-roll 出镜预算**:B-roll 总覆盖最多占成片 50%,单次连续覆盖最多 8000ms。规划和最终应用都会独立校验;超限时应删除或缩短低价值需求,而不是继续遮盖 A-roll。
|
|
117
|
+
|
|
118
|
+
规划只提供时间轴证据,不会假定每个剪辑点都必须加 B-roll。`needs-broll` 表示“值得检查”,不表示“必须遮盖”;是否添加仍需结合实际画面判断。
|
|
119
|
+
|
|
120
|
+
### 4. 文件名优先筛选 B-roll
|
|
121
|
+
|
|
122
|
+
使用 `plan.needs` 中已经调整过的完整区间,再扫描指定素材目录:
|
|
77
123
|
|
|
78
124
|
```js
|
|
79
125
|
talking_head_broll_match {
|
|
@@ -85,15 +131,7 @@ talking_head_broll_match {
|
|
|
85
131
|
maxDepth: 12,
|
|
86
132
|
maxCandidates: 5,
|
|
87
133
|
candidateOffset: 0,
|
|
88
|
-
need:
|
|
89
|
-
id: "mouse-demo",
|
|
90
|
-
outputStartMs: 5200,
|
|
91
|
-
outputEndMs: 8200,
|
|
92
|
-
speechText: "电脑除了键盘,还需要鼠标",
|
|
93
|
-
purpose: "demonstrate",
|
|
94
|
-
searchTerms: ["鼠标", "操作"],
|
|
95
|
-
reason: "展示口播提到的具体对象和动作"
|
|
96
|
-
}
|
|
134
|
+
need: plan.needs[0]
|
|
97
135
|
}
|
|
98
136
|
```
|
|
99
137
|
|
|
@@ -121,23 +159,24 @@ talking_head_broll_select {
|
|
|
121
159
|
|
|
122
160
|
该工具会验证素材 SHA-256、manifest 分析范围、起始帧和覆盖片段尾部的证据时间戳,返回可直接交给 `talking_head_apply` 的完整 placement。placement 内含选择凭证;不要手写或删除其中字段。
|
|
123
161
|
|
|
124
|
-
###
|
|
162
|
+
### 5. 写入人工确认后的时间线
|
|
125
163
|
|
|
126
164
|
```js
|
|
127
165
|
talking_head_apply {
|
|
128
166
|
projectId: "launch-video",
|
|
129
167
|
expectedRevision: 1,
|
|
130
|
-
|
|
168
|
+
aroll: [
|
|
131
169
|
{ id: "hook", sourceStartMs: 50, sourceEndMs: 4120 },
|
|
132
170
|
{ id: "answer", sourceStartMs: 4860, sourceEndMs: 13200 }
|
|
133
171
|
],
|
|
134
|
-
broll: [
|
|
172
|
+
broll: [selectionA.placement, selectionB.placement],
|
|
173
|
+
continuityPlanReceipt
|
|
135
174
|
}
|
|
136
175
|
```
|
|
137
176
|
|
|
138
|
-
B-roll 使用成片时间轴定位,`assetStartMs` 来自联络表 manifest,并永远保留主口播音轨。`talking_head_apply`
|
|
177
|
+
B-roll 使用成片时间轴定位,`assetStartMs` 来自联络表 manifest,并永远保留主口播音轨。`talking_head_apply` 会重新校验连续性规划凭证、选段凭证、素材与 manifest 的 SHA-256;规划区间或时间点被手改、素材被替换都会拒绝写入新修订。若两个 B-roll 之间仍存在不超过 500ms 的 A-roll,也会拒绝写入并要求重新规划。当前不支持 B-roll 相互重叠,因为未定义 z-order。
|
|
139
178
|
|
|
140
|
-
###
|
|
179
|
+
### 6. 交给 pi-media 渲染
|
|
141
180
|
|
|
142
181
|
先用同一个源文件创建 `pi-media` 项目,再把上一步的 `mediaOperation` 原样传入:
|
|
143
182
|
|
|
@@ -165,6 +204,7 @@ review { path: "out/launch-final.mp4" }
|
|
|
165
204
|
| --- | --- |
|
|
166
205
|
| `talking_head_create` | 从视频和词级转录建立 revision 1,分析句子、语气词、重复和停顿并生成默认 EDL |
|
|
167
206
|
| `talking_head_get` | 读取指定 revision,分页返回整句与编辑候选,可选导出 pi-media EDL |
|
|
207
|
+
| `talking_head_broll_plan` | 按 A-roll 优先预算规划 B-roll、衔接相邻片段并报告待审阅跳点 |
|
|
168
208
|
| `talking_head_broll_match` | 只用文件名和目录名匹配本地 B-roll,返回受限 shortlist 与视觉升级建议 |
|
|
169
209
|
| `talking_head_broll_select` | 用 pi-media 联络表 manifest 验证具体源时间段并生成 placement |
|
|
170
210
|
| `talking_head_apply` | 写入新的不可变口播 revision,并返回 pi-media EDL |
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
2
2
|
import { Type } from "typebox";
|
|
3
3
|
import { matchWorkspaceBrollAssets, selectBrollWindow } from "../../src/broll.ts";
|
|
4
|
+
import { createBrollContinuityPlanReceipt, planBrollContinuity } from "../../src/continuity.ts";
|
|
4
5
|
import { toMediaTimelineOperation } from "../../src/edl.ts";
|
|
5
6
|
import {
|
|
6
7
|
applyTimeline,
|
|
@@ -23,7 +24,7 @@ const stableId = Type.String({
|
|
|
23
24
|
pattern: "^[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?$",
|
|
24
25
|
});
|
|
25
26
|
|
|
26
|
-
const
|
|
27
|
+
const arollSegment = Type.Object({
|
|
27
28
|
id: stableId,
|
|
28
29
|
sourceStartMs: Type.Number({ minimum: 0 }),
|
|
29
30
|
sourceEndMs: Type.Number({ exclusiveMinimum: 0 }),
|
|
@@ -66,6 +67,34 @@ const brollNeed = Type.Object({
|
|
|
66
67
|
reason: Type.String({ minLength: 1, maxLength: 1_000 }),
|
|
67
68
|
}, { additionalProperties: false });
|
|
68
69
|
|
|
70
|
+
const arollJumpCut = Type.Object({
|
|
71
|
+
fromSegmentId: stableId,
|
|
72
|
+
toSegmentId: stableId,
|
|
73
|
+
outputTimeMs: Type.Number({ minimum: 0 }),
|
|
74
|
+
sourceDeltaMs: Type.Number(),
|
|
75
|
+
suggestedOutputStartMs: Type.Number({ minimum: 0 }),
|
|
76
|
+
suggestedOutputEndMs: Type.Number({ minimum: 0 }),
|
|
77
|
+
coveredByNeedIds: Type.Array(stableId, { maxItems: 500 }),
|
|
78
|
+
status: Type.Union([Type.Literal("covered"), Type.Literal("needs-broll")]),
|
|
79
|
+
}, { additionalProperties: false });
|
|
80
|
+
|
|
81
|
+
const continuityPlanReceipt = Type.Object({
|
|
82
|
+
schemaVersion: Type.Literal(1),
|
|
83
|
+
projectId: stableId,
|
|
84
|
+
revision: Type.Integer({ minimum: 1 }),
|
|
85
|
+
arollSha256: Type.String({ pattern: "^[a-f0-9]{64}$" }),
|
|
86
|
+
shortGapMs: Type.Integer({ minimum: 0, maximum: 2_000 }),
|
|
87
|
+
cutCoverBeforeMs: Type.Integer({ minimum: 0, maximum: 2_000 }),
|
|
88
|
+
cutCoverAfterMs: Type.Integer({ minimum: 0, maximum: 2_000 }),
|
|
89
|
+
needs: Type.Array(Type.Object({
|
|
90
|
+
id: stableId,
|
|
91
|
+
outputStartMs: Type.Number({ minimum: 0 }),
|
|
92
|
+
outputEndMs: Type.Number({ exclusiveMinimum: 0 }),
|
|
93
|
+
}, { additionalProperties: false }), { maxItems: 500 }),
|
|
94
|
+
jumpCuts: Type.Array(arollJumpCut, { maxItems: 999 }),
|
|
95
|
+
planSha256: Type.String({ pattern: "^[a-f0-9]{64}$" }),
|
|
96
|
+
}, { additionalProperties: false });
|
|
97
|
+
|
|
69
98
|
export default function talkingHead(pi: ExtensionAPI): void {
|
|
70
99
|
pi.registerTool({
|
|
71
100
|
name: "talking_head_create",
|
|
@@ -168,6 +197,43 @@ export default function talkingHead(pi: ExtensionAPI): void {
|
|
|
168
197
|
},
|
|
169
198
|
});
|
|
170
199
|
|
|
200
|
+
pi.registerTool({
|
|
201
|
+
name: "talking_head_broll_plan",
|
|
202
|
+
label: "Plan B-roll visual continuity",
|
|
203
|
+
description: "Plan A-roll-first B-roll windows before asset matching. Keeps total B-roll at or below 50% and each continuous run at or below 8 seconds, bridges brief A-roll flashes, and reports source jump cuts for visual review.",
|
|
204
|
+
parameters: Type.Object({
|
|
205
|
+
projectId: stableId,
|
|
206
|
+
revision: Type.Optional(Type.Integer({ minimum: 1 })),
|
|
207
|
+
aroll: Type.Array(arollSegment, { minItems: 1, maxItems: 1_000 }),
|
|
208
|
+
needs: Type.Array(brollNeed, { maxItems: 500 }),
|
|
209
|
+
cutCoverBeforeMs: Type.Optional(Type.Integer({ minimum: 0, maximum: 2_000, default: 250 })),
|
|
210
|
+
cutCoverAfterMs: Type.Optional(Type.Integer({ minimum: 0, maximum: 2_000, default: 500 })),
|
|
211
|
+
}, { additionalProperties: false }),
|
|
212
|
+
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
|
213
|
+
const { snapshot } = await getTalkingHeadProject(ctx.cwd, params.projectId, params.revision);
|
|
214
|
+
const plan = planBrollContinuity({
|
|
215
|
+
aroll: params.aroll,
|
|
216
|
+
needs: params.needs,
|
|
217
|
+
...(params.cutCoverBeforeMs === undefined ? {} : { cutCoverBeforeMs: params.cutCoverBeforeMs }),
|
|
218
|
+
...(params.cutCoverAfterMs === undefined ? {} : { cutCoverAfterMs: params.cutCoverAfterMs }),
|
|
219
|
+
});
|
|
220
|
+
return result({
|
|
221
|
+
projectId: params.projectId,
|
|
222
|
+
revision: snapshot.revision,
|
|
223
|
+
plan,
|
|
224
|
+
continuityPlanReceipt: createBrollContinuityPlanReceipt(
|
|
225
|
+
params.projectId,
|
|
226
|
+
snapshot.revision,
|
|
227
|
+
params.aroll,
|
|
228
|
+
plan,
|
|
229
|
+
),
|
|
230
|
+
nextStep: plan.jumpCuts.some((cut) => cut.status === "needs-broll")
|
|
231
|
+
? "Create purpose=mask-cut needs for uncovered jump cuts, then run this planner again before asset matching."
|
|
232
|
+
: "Use plan.needs for B-roll asset matching and source-window selection.",
|
|
233
|
+
});
|
|
234
|
+
},
|
|
235
|
+
});
|
|
236
|
+
|
|
171
237
|
pi.registerTool({
|
|
172
238
|
name: "talking_head_broll_match",
|
|
173
239
|
label: "Match local B-roll assets",
|
|
@@ -242,12 +308,13 @@ export default function talkingHead(pi: ExtensionAPI): void {
|
|
|
242
308
|
pi.registerTool({
|
|
243
309
|
name: "talking_head_apply",
|
|
244
310
|
label: "Apply talking-head timeline",
|
|
245
|
-
description: "Create a new immutable talking-head revision from word-boundary A-roll ranges and optional B-roll placements. Returns a generic pi-media timeline operation ready for edit_apply.",
|
|
311
|
+
description: "Create a new immutable talking-head revision from word-boundary A-roll ranges and optional B-roll placements. B-roll requires the matching continuity plan receipt; coverage above 50%, continuous runs above 8 seconds, brief A-roll flashes, unplanned ranges, and overlaps are rejected. Returns a generic pi-media timeline operation ready for edit_apply.",
|
|
246
312
|
parameters: Type.Object({
|
|
247
313
|
projectId: stableId,
|
|
248
314
|
expectedRevision: Type.Integer({ minimum: 1 }),
|
|
249
|
-
aroll: Type.Array(
|
|
315
|
+
aroll: Type.Array(arollSegment, { minItems: 1, maxItems: 1_000 }),
|
|
250
316
|
broll: Type.Array(brollPlacement, { maxItems: 500 }),
|
|
317
|
+
continuityPlanReceipt: Type.Optional(continuityPlanReceipt),
|
|
251
318
|
}, { additionalProperties: false }),
|
|
252
319
|
async execute(_toolCallId, params, signal, _onUpdate, ctx) {
|
|
253
320
|
const current = await getTalkingHeadProject(ctx.cwd, params.projectId);
|
package/package.json
CHANGED
|
@@ -2,6 +2,6 @@
|
|
|
2
2
|
description: Tighten a talking-head video without destroying natural speech rhythm
|
|
3
3
|
---
|
|
4
4
|
|
|
5
|
-
Use the `talking-head-edit` skill to review full-sentence context, word-level pauses, filler words, adjacent repetitions, and low-confidence delivery cues; stabilize A-roll, match B-roll by filename before escalating through paged contact-sheet batches, validate the selected manifest-backed source window, and export a generic pi-media EDL for this request: $@
|
|
5
|
+
Use the `talking-head-edit` skill to review full-sentence context, word-level pauses, filler words, adjacent repetitions, and low-confidence delivery cues; stabilize A-roll, keep A-roll visible for at least half of the output, limit each continuous B-roll run to 8 seconds, bridge brief A-roll flashes, mask only visually objectionable jump cuts, match B-roll by filename before escalating through paged contact-sheet batches, validate the selected manifest-backed source window, and export a generic pi-media EDL for this request: $@
|
|
6
6
|
|
|
7
7
|
Show the proposed rhythm before writing a new revision. Never overwrite the source or an existing render.
|
|
@@ -12,16 +12,17 @@ Use `pi-speech` for word evidence, this package for editorial decisions, and `pi
|
|
|
12
12
|
3. Inspect bounded `talking_head_get` pages. Review the full sentence context, filler candidates, and repetition candidates together with pauses. Treat `safe` and every recommendation as evidence, not an instruction. Preserve pauses that carry emphasis, emotion, topic boundaries, or a deliberate breath.
|
|
13
13
|
4. Never delete a filler token automatically. Decide whether `啊`, `额`, `嗯`, or a repeated token is a false start, a discourse marker, or intentional emphasis from its sentence context. A text-only delivery cue is low-confidence evidence; review audio and picture before relying on emotion or performance intent.
|
|
14
14
|
5. Before changing the revision, summarize the proposed rhythm: what words and gaps will be removed, which pauses or fillers will remain, and why. Get the user's approval unless they explicitly delegated editorial judgment.
|
|
15
|
-
6. Stabilize A-roll before B-roll. For every B-roll window, record the output range, complete spoken text, one purpose (`demonstrate`, `explain`, `evidence`, `establish`, `transition`, or `mask-cut`), 1-20 concise search terms, and a concrete reason.
|
|
16
|
-
7. Call `
|
|
15
|
+
6. Stabilize A-roll before B-roll. A-roll is the primary picture, not leftover filler: keep it visible for at least half of the final duration and reintroduce it within every 8 seconds of continuous B-roll. For every B-roll window, record the output range, complete spoken text, one purpose (`demonstrate`, `explain`, `evidence`, `establish`, `transition`, or `mask-cut`), 1-20 concise search terms, and a concrete reason.
|
|
16
|
+
7. Call `talking_head_broll_plan` with the final proposed A-roll and all semantic B-roll needs before matching assets. Keep its `continuityPlanReceipt` unchanged. Use the returned `plan.needs` and inspect `plan.coverage`: the planner rejects total B-roll above 50% or any continuous run above 8000ms. It extends the previous B-roll across A-roll gaps of 500ms or less so neighboring B-roll clips hand off directly. Review every reported A-roll jump cut, but add `purpose: mask-cut` only when picture review shows a visually objectionable cut and the spoken context supports B-roll. Never cover every reported jump cut by default. If the budget is exceeded, remove or shorten lower-value needs instead of hiding more A-roll, then rerun the plan.
|
|
17
|
+
8. Call `talking_head_broll_match` before generating contact sheets. Follow its bounded result:
|
|
17
18
|
- `filename-direct`: probe and visually inspect only the selected asset to choose its source window. Do not call `media_contact_sheet` for unselected assets.
|
|
18
19
|
- `filename-shortlist`: inspect only the returned shortlist and stop as soon as one asset satisfies the need. If none does and `nextCandidateOffset` is non-null, request that next page; do not repeat the first batch.
|
|
19
20
|
- `visual-fallback`: use low-cost contact sheets on the bounded shortlist because filenames supplied no useful evidence. Exhaust that batch before requesting a non-null `nextCandidateOffset`.
|
|
20
21
|
- If the scan is truncated, do not paginate it: narrow the asset directory and rescan so the candidate inventory is stable.
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
22
|
+
9. After selecting one asset, call `media_probe`, then use `media_contact_sheet` only as densely as timing requires: one high-density pass for a short ambiguous clip, medium for a normal clip, or low over a long clip followed by high density over the narrowed range. Read `contact_sheet_manifest.json` timestamps instead of guessing times from the PNG. Verify a continuous source window at least as long as the planned output window, preferably with 500ms handles on both sides.
|
|
23
|
+
10. Call `talking_head_broll_select` with each planned need, selected asset, manifest path, exact source start, and the manifest timestamps actually reviewed. Use its complete returned `placement`, including `selectionReceipt`; do not shorten bridged output ranges or hand-author selection fields.
|
|
24
|
+
11. Call `talking_head_apply` using the same planned A-roll, every validated B-roll placement, and the unchanged `continuityPlanReceipt`. The tool independently rejects B-roll above the 50%/8000ms A-roll-first budget, unplanned ranges, overlaps, and A-roll flashes of 500ms or less. Every B-roll window must have a concrete visual purpose in `reason`; keep `audio: keep-primary`.
|
|
25
|
+
12. Create/read a `pi-media` project for the same source. Pass the returned `mediaOperation` unchanged to `edit_apply`, then `render` the exact new revision and call `review`.
|
|
25
26
|
|
|
26
27
|
Never overwrite source media or outputs. If a revision conflict occurs, re-read both projects and reconcile intent. If the source, transcript, or B-roll hash changed, stop and ask whether to create a new project rather than silently adopting new bytes.
|
|
27
28
|
|
|
@@ -23,6 +23,7 @@
|
|
|
23
23
|
- Start with filename and directory metadata. Generate contact sheets only for the selected asset, a bounded ambiguous shortlist, or a bounded fallback batch whose names carry no meaning.
|
|
24
24
|
- For a long selected asset, locate a broad range cheaply and then generate a denser contact sheet only for that range. Use manifest timestamps, not visual timestamp transcription.
|
|
25
25
|
- Ensure the selected source window covers the complete spoken beat. Prefer extra source handles so a later timing adjustment does not require rescanning.
|
|
26
|
+
- If neighboring B-roll windows expose 500ms or less of A-roll, plan a direct handoff by extending the previous B-roll need before source-window selection. Do not let a few frames of A-roll flash between overlays.
|
|
26
27
|
- Cover visible jump cuts when appropriate, but do not hide continuity errors that change meaning.
|
|
27
28
|
- Keep the primary voice track. B-roll audio replacement is outside the current contract.
|
|
28
29
|
- Record the search query and editorial reason so a later agent can replace the asset without guessing intent.
|
|
@@ -0,0 +1,432 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import type {
|
|
3
|
+
ArollJumpCut,
|
|
4
|
+
ArollSegment,
|
|
5
|
+
BrollContinuityBridge,
|
|
6
|
+
BrollContinuityPlan,
|
|
7
|
+
BrollContinuityPlanReceipt,
|
|
8
|
+
BrollCoverageSummary,
|
|
9
|
+
BrollNeed,
|
|
10
|
+
} from "./contracts.ts";
|
|
11
|
+
import { timelineDuration } from "./transcript.ts";
|
|
12
|
+
|
|
13
|
+
export const MAX_BROLL_COVERAGE_RATIO = 0.5;
|
|
14
|
+
export const MAX_CONTINUOUS_BROLL_MS = 8_000;
|
|
15
|
+
|
|
16
|
+
export interface PlanBrollContinuityInput {
|
|
17
|
+
aroll: ArollSegment[];
|
|
18
|
+
needs: BrollNeed[];
|
|
19
|
+
shortGapMs?: number;
|
|
20
|
+
cutCoverBeforeMs?: number;
|
|
21
|
+
cutCoverAfterMs?: number;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export interface BrollOutputWindow {
|
|
25
|
+
id: string;
|
|
26
|
+
outputStartMs: number;
|
|
27
|
+
outputEndMs: number;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export interface ShortArollFlashGap {
|
|
31
|
+
fromBrollId: string;
|
|
32
|
+
toBrollId: string;
|
|
33
|
+
startMs: number;
|
|
34
|
+
endMs: number;
|
|
35
|
+
durationMs: number;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface OverlappingBrollWindows {
|
|
39
|
+
underBrollId: string;
|
|
40
|
+
overBrollId: string;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function boundedMilliseconds(value: number | undefined, fallback: number, name: string, maximum: number): number {
|
|
44
|
+
const selected = value ?? fallback;
|
|
45
|
+
if (!Number.isInteger(selected) || selected < 0 || selected > maximum) {
|
|
46
|
+
throw new Error(`${name} must be an integer within 0-${maximum}`);
|
|
47
|
+
}
|
|
48
|
+
return selected;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export function findShortArollFlashGaps(
|
|
52
|
+
windows: BrollOutputWindow[],
|
|
53
|
+
shortGapMs = 500,
|
|
54
|
+
): ShortArollFlashGap[] {
|
|
55
|
+
const threshold = boundedMilliseconds(shortGapMs, 500, "shortGapMs", 2_000);
|
|
56
|
+
const sorted = [...windows].sort((left, right) => (
|
|
57
|
+
left.outputStartMs - right.outputStartMs
|
|
58
|
+
|| left.outputEndMs - right.outputEndMs
|
|
59
|
+
|| (left.id < right.id ? -1 : left.id > right.id ? 1 : 0)
|
|
60
|
+
));
|
|
61
|
+
const gaps: ShortArollFlashGap[] = [];
|
|
62
|
+
let frontier = sorted[0];
|
|
63
|
+
let coveredUntilMs = frontier?.outputEndMs ?? 0;
|
|
64
|
+
for (let index = 1; index < sorted.length; index += 1) {
|
|
65
|
+
const next = sorted[index]!;
|
|
66
|
+
if (next.outputStartMs <= coveredUntilMs) {
|
|
67
|
+
if (next.outputEndMs > coveredUntilMs) {
|
|
68
|
+
frontier = next;
|
|
69
|
+
coveredUntilMs = next.outputEndMs;
|
|
70
|
+
}
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
const durationMs = next.outputStartMs - coveredUntilMs;
|
|
74
|
+
if (durationMs > threshold) {
|
|
75
|
+
frontier = next;
|
|
76
|
+
coveredUntilMs = next.outputEndMs;
|
|
77
|
+
continue;
|
|
78
|
+
}
|
|
79
|
+
gaps.push({
|
|
80
|
+
fromBrollId: frontier!.id,
|
|
81
|
+
toBrollId: next.id,
|
|
82
|
+
startMs: coveredUntilMs,
|
|
83
|
+
endMs: next.outputStartMs,
|
|
84
|
+
durationMs,
|
|
85
|
+
});
|
|
86
|
+
frontier = next;
|
|
87
|
+
coveredUntilMs = next.outputEndMs;
|
|
88
|
+
}
|
|
89
|
+
return gaps;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export function findOverlappingBrollWindows(windows: BrollOutputWindow[]): OverlappingBrollWindows[] {
|
|
93
|
+
const sorted = [...windows].sort((left, right) => (
|
|
94
|
+
left.outputStartMs - right.outputStartMs
|
|
95
|
+
|| right.outputEndMs - left.outputEndMs
|
|
96
|
+
|| (left.id < right.id ? -1 : left.id > right.id ? 1 : 0)
|
|
97
|
+
));
|
|
98
|
+
const overlaps: OverlappingBrollWindows[] = [];
|
|
99
|
+
let frontier = sorted[0];
|
|
100
|
+
for (let index = 1; index < sorted.length; index += 1) {
|
|
101
|
+
const next = sorted[index]!;
|
|
102
|
+
if (frontier && next.outputStartMs < frontier.outputEndMs) {
|
|
103
|
+
overlaps.push({ underBrollId: frontier.id, overBrollId: next.id });
|
|
104
|
+
if (next.outputEndMs > frontier.outputEndMs) frontier = next;
|
|
105
|
+
continue;
|
|
106
|
+
}
|
|
107
|
+
frontier = next;
|
|
108
|
+
}
|
|
109
|
+
return overlaps;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export function summarizeBrollCoverage(
|
|
113
|
+
windows: BrollOutputWindow[],
|
|
114
|
+
outputDurationMs: number,
|
|
115
|
+
): BrollCoverageSummary {
|
|
116
|
+
if (!Number.isFinite(outputDurationMs) || outputDurationMs <= 0) {
|
|
117
|
+
throw new Error("outputDurationMs must be a positive finite number");
|
|
118
|
+
}
|
|
119
|
+
for (const window of windows) {
|
|
120
|
+
if (!Number.isFinite(window.outputStartMs) || !Number.isFinite(window.outputEndMs)
|
|
121
|
+
|| window.outputStartMs < 0 || window.outputEndMs <= window.outputStartMs
|
|
122
|
+
|| window.outputEndMs > outputDurationMs) {
|
|
123
|
+
throw new Error(`Invalid B-roll coverage window: ${window.id}`);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
const sorted = [...windows].sort((left, right) => (
|
|
127
|
+
left.outputStartMs - right.outputStartMs
|
|
128
|
+
|| left.outputEndMs - right.outputEndMs
|
|
129
|
+
|| (left.id < right.id ? -1 : left.id > right.id ? 1 : 0)
|
|
130
|
+
));
|
|
131
|
+
let brollDurationMs = 0;
|
|
132
|
+
let longestContinuousBrollMs = 0;
|
|
133
|
+
let runStartMs: number | undefined;
|
|
134
|
+
let runEndMs: number | undefined;
|
|
135
|
+
for (const window of sorted) {
|
|
136
|
+
if (runStartMs === undefined || runEndMs === undefined) {
|
|
137
|
+
runStartMs = window.outputStartMs;
|
|
138
|
+
runEndMs = window.outputEndMs;
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
if (window.outputStartMs <= runEndMs) {
|
|
142
|
+
runEndMs = Math.max(runEndMs, window.outputEndMs);
|
|
143
|
+
continue;
|
|
144
|
+
}
|
|
145
|
+
const durationMs = runEndMs - runStartMs;
|
|
146
|
+
brollDurationMs += durationMs;
|
|
147
|
+
longestContinuousBrollMs = Math.max(longestContinuousBrollMs, durationMs);
|
|
148
|
+
runStartMs = window.outputStartMs;
|
|
149
|
+
runEndMs = window.outputEndMs;
|
|
150
|
+
}
|
|
151
|
+
if (runStartMs !== undefined && runEndMs !== undefined) {
|
|
152
|
+
const durationMs = runEndMs - runStartMs;
|
|
153
|
+
brollDurationMs += durationMs;
|
|
154
|
+
longestContinuousBrollMs = Math.max(longestContinuousBrollMs, durationMs);
|
|
155
|
+
}
|
|
156
|
+
return {
|
|
157
|
+
brollDurationMs,
|
|
158
|
+
arollDurationMs: outputDurationMs - brollDurationMs,
|
|
159
|
+
brollCoverageRatio: Math.round((brollDurationMs / outputDurationMs) * 10_000) / 10_000,
|
|
160
|
+
longestContinuousBrollMs,
|
|
161
|
+
limits: {
|
|
162
|
+
maxBrollCoverageRatio: MAX_BROLL_COVERAGE_RATIO,
|
|
163
|
+
maxContinuousBrollMs: MAX_CONTINUOUS_BROLL_MS,
|
|
164
|
+
},
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
export function assertBrollCoverageBudget(
|
|
169
|
+
windows: BrollOutputWindow[],
|
|
170
|
+
outputDurationMs: number,
|
|
171
|
+
): BrollCoverageSummary {
|
|
172
|
+
const coverage = summarizeBrollCoverage(windows, outputDurationMs);
|
|
173
|
+
if (coverage.brollDurationMs > outputDurationMs * MAX_BROLL_COVERAGE_RATIO) {
|
|
174
|
+
const percent = ((coverage.brollDurationMs / outputDurationMs) * 100).toFixed(1);
|
|
175
|
+
throw new Error(`B-roll coverage ${percent}% exceeds the 50% A-roll-first limit; remove or shorten lower-priority B-roll needs`);
|
|
176
|
+
}
|
|
177
|
+
if (coverage.longestContinuousBrollMs > MAX_CONTINUOUS_BROLL_MS) {
|
|
178
|
+
throw new Error(`continuous B-roll run ${coverage.longestContinuousBrollMs}ms exceeds the 8000ms limit; split it with a meaningful A-roll appearance`);
|
|
179
|
+
}
|
|
180
|
+
return coverage;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
function validateSegments(aroll: ArollSegment[]): void {
|
|
184
|
+
if (aroll.length === 0 || aroll.length > 1_000) throw new Error("A-roll must contain 1-1000 segments");
|
|
185
|
+
const ids = new Set<string>();
|
|
186
|
+
for (const segment of aroll) {
|
|
187
|
+
if (ids.has(segment.id)) throw new Error(`Duplicate A-roll segment ID: ${segment.id}`);
|
|
188
|
+
ids.add(segment.id);
|
|
189
|
+
if (!Number.isFinite(segment.sourceStartMs) || !Number.isFinite(segment.sourceEndMs)
|
|
190
|
+
|| segment.sourceStartMs < 0 || segment.sourceEndMs <= segment.sourceStartMs) {
|
|
191
|
+
throw new Error(`Invalid A-roll segment range: ${segment.id}`);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
function validateNeeds(needs: BrollNeed[], outputDurationMs: number): void {
|
|
197
|
+
if (needs.length > 500) throw new Error("B-roll plan must contain at most 500 needs");
|
|
198
|
+
const ids = new Set<string>();
|
|
199
|
+
for (const need of needs) {
|
|
200
|
+
if (ids.has(need.id)) throw new Error(`Duplicate B-roll need ID: ${need.id}`);
|
|
201
|
+
ids.add(need.id);
|
|
202
|
+
if (!Number.isFinite(need.outputStartMs) || !Number.isFinite(need.outputEndMs)
|
|
203
|
+
|| need.outputStartMs < 0 || need.outputEndMs <= need.outputStartMs
|
|
204
|
+
|| need.outputEndMs > outputDurationMs) {
|
|
205
|
+
throw new Error(`Invalid B-roll need output range: ${need.id}`);
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
const overlap = findOverlappingBrollWindows(needs)[0];
|
|
209
|
+
if (overlap) throw new Error(`B-roll needs ${overlap.underBrollId} and ${overlap.overBrollId} overlap; z-order is not supported`);
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
function bridgeShortGaps(needs: BrollNeed[], shortGapMs: number): {
|
|
213
|
+
needs: BrollNeed[];
|
|
214
|
+
bridges: BrollContinuityBridge[];
|
|
215
|
+
} {
|
|
216
|
+
const sorted = structuredClone(needs).sort((left, right) => (
|
|
217
|
+
left.outputStartMs - right.outputStartMs
|
|
218
|
+
|| left.outputEndMs - right.outputEndMs
|
|
219
|
+
|| (left.id < right.id ? -1 : left.id > right.id ? 1 : 0)
|
|
220
|
+
));
|
|
221
|
+
const bridges: BrollContinuityBridge[] = [];
|
|
222
|
+
let frontier = sorted[0];
|
|
223
|
+
let coveredUntilMs = frontier?.outputEndMs ?? 0;
|
|
224
|
+
for (let index = 1; index < sorted.length; index += 1) {
|
|
225
|
+
const next = sorted[index]!;
|
|
226
|
+
if (next.outputStartMs <= coveredUntilMs) {
|
|
227
|
+
if (next.outputEndMs > coveredUntilMs) {
|
|
228
|
+
frontier = next;
|
|
229
|
+
coveredUntilMs = next.outputEndMs;
|
|
230
|
+
}
|
|
231
|
+
continue;
|
|
232
|
+
}
|
|
233
|
+
const durationMs = next.outputStartMs - coveredUntilMs;
|
|
234
|
+
if (durationMs > shortGapMs) {
|
|
235
|
+
frontier = next;
|
|
236
|
+
coveredUntilMs = next.outputEndMs;
|
|
237
|
+
continue;
|
|
238
|
+
}
|
|
239
|
+
bridges.push({
|
|
240
|
+
fromNeedId: frontier!.id,
|
|
241
|
+
toNeedId: next.id,
|
|
242
|
+
gapStartMs: coveredUntilMs,
|
|
243
|
+
gapEndMs: next.outputStartMs,
|
|
244
|
+
durationMs,
|
|
245
|
+
strategy: "extend-previous-broll",
|
|
246
|
+
});
|
|
247
|
+
frontier!.outputEndMs = next.outputStartMs;
|
|
248
|
+
frontier = next;
|
|
249
|
+
coveredUntilMs = next.outputEndMs;
|
|
250
|
+
}
|
|
251
|
+
return { needs: sorted, bridges };
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
function coverageForRange(needs: BrollOutputWindow[], startMs: number, endMs: number): {
|
|
255
|
+
covered: boolean;
|
|
256
|
+
needIds: string[];
|
|
257
|
+
} {
|
|
258
|
+
const overlapping = needs.filter((need) => need.outputEndMs > startMs && need.outputStartMs < endMs).sort((left, right) => (
|
|
259
|
+
left.outputStartMs - right.outputStartMs
|
|
260
|
+
|| left.outputEndMs - right.outputEndMs
|
|
261
|
+
|| (left.id < right.id ? -1 : left.id > right.id ? 1 : 0)
|
|
262
|
+
));
|
|
263
|
+
let cursor = startMs;
|
|
264
|
+
const needIds: string[] = [];
|
|
265
|
+
for (const need of overlapping) {
|
|
266
|
+
if (need.outputStartMs > cursor) return { covered: false, needIds };
|
|
267
|
+
needIds.push(need.id);
|
|
268
|
+
cursor = Math.max(cursor, need.outputEndMs);
|
|
269
|
+
if (cursor >= endMs) return { covered: true, needIds };
|
|
270
|
+
}
|
|
271
|
+
return { covered: false, needIds };
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
export function analyzeArollJumpCuts(
|
|
275
|
+
aroll: ArollSegment[],
|
|
276
|
+
needs: BrollOutputWindow[],
|
|
277
|
+
outputDurationMs: number,
|
|
278
|
+
cutCoverBeforeMs: number,
|
|
279
|
+
cutCoverAfterMs: number,
|
|
280
|
+
): ArollJumpCut[] {
|
|
281
|
+
const jumpCuts: ArollJumpCut[] = [];
|
|
282
|
+
let outputTimeMs = 0;
|
|
283
|
+
for (let index = 0; index < aroll.length - 1; index += 1) {
|
|
284
|
+
const current = aroll[index]!;
|
|
285
|
+
const next = aroll[index + 1]!;
|
|
286
|
+
outputTimeMs += current.sourceEndMs - current.sourceStartMs;
|
|
287
|
+
const sourceDeltaMs = next.sourceStartMs - current.sourceEndMs;
|
|
288
|
+
if (sourceDeltaMs === 0) continue;
|
|
289
|
+
const suggestedOutputStartMs = Math.max(0, outputTimeMs - cutCoverBeforeMs);
|
|
290
|
+
const suggestedOutputEndMs = Math.min(outputDurationMs, outputTimeMs + cutCoverAfterMs);
|
|
291
|
+
const coverage = coverageForRange(needs, suggestedOutputStartMs, suggestedOutputEndMs);
|
|
292
|
+
jumpCuts.push({
|
|
293
|
+
fromSegmentId: current.id,
|
|
294
|
+
toSegmentId: next.id,
|
|
295
|
+
outputTimeMs,
|
|
296
|
+
sourceDeltaMs,
|
|
297
|
+
suggestedOutputStartMs,
|
|
298
|
+
suggestedOutputEndMs,
|
|
299
|
+
coveredByNeedIds: coverage.needIds,
|
|
300
|
+
status: coverage.covered ? "covered" : "needs-broll",
|
|
301
|
+
});
|
|
302
|
+
}
|
|
303
|
+
return jumpCuts;
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
export function planBrollContinuity(input: PlanBrollContinuityInput): BrollContinuityPlan {
|
|
307
|
+
validateSegments(input.aroll);
|
|
308
|
+
const outputDurationMs = timelineDuration(input.aroll);
|
|
309
|
+
validateNeeds(input.needs, outputDurationMs);
|
|
310
|
+
const shortGapMs = boundedMilliseconds(input.shortGapMs, 500, "shortGapMs", 2_000);
|
|
311
|
+
const cutCoverBeforeMs = boundedMilliseconds(input.cutCoverBeforeMs, 250, "cutCoverBeforeMs", 2_000);
|
|
312
|
+
const cutCoverAfterMs = boundedMilliseconds(input.cutCoverAfterMs, 500, "cutCoverAfterMs", 2_000);
|
|
313
|
+
if (cutCoverBeforeMs === 0 && cutCoverAfterMs === 0) {
|
|
314
|
+
throw new Error("cutCoverBeforeMs and cutCoverAfterMs cannot both be zero");
|
|
315
|
+
}
|
|
316
|
+
const bridged = bridgeShortGaps(input.needs, shortGapMs);
|
|
317
|
+
const coverage = assertBrollCoverageBudget(bridged.needs, outputDurationMs);
|
|
318
|
+
return {
|
|
319
|
+
outputDurationMs,
|
|
320
|
+
shortGapMs,
|
|
321
|
+
cutCoverBeforeMs,
|
|
322
|
+
cutCoverAfterMs,
|
|
323
|
+
needs: bridged.needs,
|
|
324
|
+
bridges: bridged.bridges,
|
|
325
|
+
coverage,
|
|
326
|
+
jumpCuts: analyzeArollJumpCuts(
|
|
327
|
+
input.aroll,
|
|
328
|
+
bridged.needs,
|
|
329
|
+
outputDurationMs,
|
|
330
|
+
cutCoverBeforeMs,
|
|
331
|
+
cutCoverAfterMs,
|
|
332
|
+
),
|
|
333
|
+
};
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
function planReceiptPayload(receipt: Omit<BrollContinuityPlanReceipt, "planSha256">): string {
|
|
337
|
+
return canonicalJson(receipt);
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
function canonicalJson(value: unknown): string {
|
|
341
|
+
if (value === null || typeof value !== "object") {
|
|
342
|
+
const serialized = JSON.stringify(value);
|
|
343
|
+
if (serialized === undefined) throw new Error("Cannot canonicalize an undefined value");
|
|
344
|
+
return serialized;
|
|
345
|
+
}
|
|
346
|
+
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
|
|
347
|
+
const object = value as Record<string, unknown>;
|
|
348
|
+
return `{${Object.keys(object).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(object[key])}`).join(",")}}`;
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
function sha256(value: string): string {
|
|
352
|
+
return createHash("sha256").update(value).digest("hex");
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
function arollSha256(aroll: ArollSegment[]): string {
|
|
356
|
+
return sha256(canonicalJson(aroll));
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
function plannedRanges(needs: BrollNeed[]): BrollContinuityPlanReceipt["needs"] {
|
|
360
|
+
return needs.map((need) => ({
|
|
361
|
+
id: need.id,
|
|
362
|
+
outputStartMs: need.outputStartMs,
|
|
363
|
+
outputEndMs: need.outputEndMs,
|
|
364
|
+
}));
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
export function createBrollContinuityPlanReceipt(
|
|
368
|
+
projectId: string,
|
|
369
|
+
revision: number,
|
|
370
|
+
aroll: ArollSegment[],
|
|
371
|
+
plan: BrollContinuityPlan,
|
|
372
|
+
): BrollContinuityPlanReceipt {
|
|
373
|
+
const payload: Omit<BrollContinuityPlanReceipt, "planSha256"> = {
|
|
374
|
+
schemaVersion: 1,
|
|
375
|
+
projectId,
|
|
376
|
+
revision,
|
|
377
|
+
arollSha256: arollSha256(aroll),
|
|
378
|
+
shortGapMs: plan.shortGapMs,
|
|
379
|
+
cutCoverBeforeMs: plan.cutCoverBeforeMs,
|
|
380
|
+
cutCoverAfterMs: plan.cutCoverAfterMs,
|
|
381
|
+
needs: plannedRanges(plan.needs),
|
|
382
|
+
jumpCuts: structuredClone(plan.jumpCuts),
|
|
383
|
+
};
|
|
384
|
+
return { ...payload, planSha256: sha256(planReceiptPayload(payload)) };
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
export function verifyBrollContinuityPlanReceipt(
|
|
388
|
+
projectId: string,
|
|
389
|
+
revision: number,
|
|
390
|
+
aroll: ArollSegment[],
|
|
391
|
+
placements: BrollOutputWindow[],
|
|
392
|
+
receipt: BrollContinuityPlanReceipt,
|
|
393
|
+
): void {
|
|
394
|
+
if (receipt.schemaVersion !== 1 || receipt.projectId !== projectId || receipt.revision !== revision) {
|
|
395
|
+
throw new Error("B-roll continuity receipt does not match the current project revision");
|
|
396
|
+
}
|
|
397
|
+
if (receipt.arollSha256 !== arollSha256(aroll)) {
|
|
398
|
+
throw new Error("B-roll continuity receipt does not match the planned A-roll");
|
|
399
|
+
}
|
|
400
|
+
const { planSha256, ...payload } = receipt;
|
|
401
|
+
if (planSha256 !== sha256(planReceiptPayload(payload))) {
|
|
402
|
+
throw new Error("B-roll continuity receipt hash is invalid");
|
|
403
|
+
}
|
|
404
|
+
const dummyNeeds: BrollNeed[] = receipt.needs.map((need) => ({
|
|
405
|
+
...need,
|
|
406
|
+
speechText: "continuity receipt",
|
|
407
|
+
purpose: "mask-cut",
|
|
408
|
+
searchTerms: ["continuity"],
|
|
409
|
+
reason: "continuity receipt verification",
|
|
410
|
+
}));
|
|
411
|
+
const recomputed = planBrollContinuity({
|
|
412
|
+
aroll,
|
|
413
|
+
needs: dummyNeeds,
|
|
414
|
+
shortGapMs: receipt.shortGapMs,
|
|
415
|
+
cutCoverBeforeMs: receipt.cutCoverBeforeMs,
|
|
416
|
+
cutCoverAfterMs: receipt.cutCoverAfterMs,
|
|
417
|
+
});
|
|
418
|
+
if (canonicalJson(plannedRanges(recomputed.needs)) !== canonicalJson(receipt.needs)
|
|
419
|
+
|| canonicalJson(recomputed.jumpCuts) !== canonicalJson(receipt.jumpCuts)) {
|
|
420
|
+
throw new Error("B-roll continuity receipt does not match the recomputed plan");
|
|
421
|
+
}
|
|
422
|
+
const actualRanges = [...placements].sort((left, right) => (
|
|
423
|
+
left.outputStartMs - right.outputStartMs || left.outputEndMs - right.outputEndMs
|
|
424
|
+
)).map((placement) => ({
|
|
425
|
+
id: placement.id,
|
|
426
|
+
outputStartMs: placement.outputStartMs,
|
|
427
|
+
outputEndMs: placement.outputEndMs,
|
|
428
|
+
}));
|
|
429
|
+
if (canonicalJson(actualRanges) !== canonicalJson(receipt.needs)) {
|
|
430
|
+
throw new Error("B-roll placements do not match the planned ranges");
|
|
431
|
+
}
|
|
432
|
+
}
|
package/src/contracts.ts
CHANGED
|
@@ -165,6 +165,61 @@ export interface BrollWindowSelection {
|
|
|
165
165
|
placement: BrollPlacementInput;
|
|
166
166
|
}
|
|
167
167
|
|
|
168
|
+
export interface BrollContinuityBridge {
|
|
169
|
+
fromNeedId: string;
|
|
170
|
+
toNeedId: string;
|
|
171
|
+
gapStartMs: number;
|
|
172
|
+
gapEndMs: number;
|
|
173
|
+
durationMs: number;
|
|
174
|
+
strategy: "extend-previous-broll";
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
export interface ArollJumpCut {
|
|
178
|
+
fromSegmentId: string;
|
|
179
|
+
toSegmentId: string;
|
|
180
|
+
outputTimeMs: number;
|
|
181
|
+
sourceDeltaMs: number;
|
|
182
|
+
suggestedOutputStartMs: number;
|
|
183
|
+
suggestedOutputEndMs: number;
|
|
184
|
+
coveredByNeedIds: string[];
|
|
185
|
+
status: "covered" | "needs-broll";
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
export interface BrollCoverageSummary {
|
|
189
|
+
brollDurationMs: number;
|
|
190
|
+
arollDurationMs: number;
|
|
191
|
+
brollCoverageRatio: number;
|
|
192
|
+
longestContinuousBrollMs: number;
|
|
193
|
+
limits: {
|
|
194
|
+
maxBrollCoverageRatio: number;
|
|
195
|
+
maxContinuousBrollMs: number;
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
export interface BrollContinuityPlan {
|
|
200
|
+
outputDurationMs: number;
|
|
201
|
+
shortGapMs: number;
|
|
202
|
+
cutCoverBeforeMs: number;
|
|
203
|
+
cutCoverAfterMs: number;
|
|
204
|
+
needs: BrollNeed[];
|
|
205
|
+
bridges: BrollContinuityBridge[];
|
|
206
|
+
jumpCuts: ArollJumpCut[];
|
|
207
|
+
coverage: BrollCoverageSummary;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
export interface BrollContinuityPlanReceipt {
|
|
211
|
+
schemaVersion: 1;
|
|
212
|
+
projectId: string;
|
|
213
|
+
revision: number;
|
|
214
|
+
arollSha256: string;
|
|
215
|
+
shortGapMs: number;
|
|
216
|
+
cutCoverBeforeMs: number;
|
|
217
|
+
cutCoverAfterMs: number;
|
|
218
|
+
needs: Array<{ id: string; outputStartMs: number; outputEndMs: number }>;
|
|
219
|
+
jumpCuts: ArollJumpCut[];
|
|
220
|
+
planSha256: string;
|
|
221
|
+
}
|
|
222
|
+
|
|
168
223
|
export interface TranscriptAnalysis {
|
|
169
224
|
schemaVersion: 2;
|
|
170
225
|
text: string;
|
|
@@ -209,6 +264,9 @@ export interface TalkingHeadSnapshot {
|
|
|
209
264
|
policy: TalkingHeadPolicy;
|
|
210
265
|
aroll: ArollSegment[];
|
|
211
266
|
broll: BrollPlacement[];
|
|
267
|
+
continuityPlanReceipt?: BrollContinuityPlanReceipt;
|
|
268
|
+
jumpCuts?: ArollJumpCut[];
|
|
269
|
+
brollCoverage?: BrollCoverageSummary;
|
|
212
270
|
outputDurationMs: number;
|
|
213
271
|
}
|
|
214
272
|
|
package/src/project.ts
CHANGED
|
@@ -5,6 +5,7 @@ import type {
|
|
|
5
5
|
ArollSegment,
|
|
6
6
|
BrollPlacement,
|
|
7
7
|
BrollPlacementInput,
|
|
8
|
+
BrollContinuityPlanReceipt,
|
|
8
9
|
FillerCandidate,
|
|
9
10
|
PauseCandidate,
|
|
10
11
|
RepetitionCandidate,
|
|
@@ -17,6 +18,13 @@ import type {
|
|
|
17
18
|
WordTranscript,
|
|
18
19
|
} from "./contracts.ts";
|
|
19
20
|
import { verifyBrollPlacementSelection } from "./broll.ts";
|
|
21
|
+
import {
|
|
22
|
+
analyzeArollJumpCuts,
|
|
23
|
+
assertBrollCoverageBudget,
|
|
24
|
+
findOverlappingBrollWindows,
|
|
25
|
+
findShortArollFlashGaps,
|
|
26
|
+
verifyBrollContinuityPlanReceipt,
|
|
27
|
+
} from "./continuity.ts";
|
|
20
28
|
import { analyzeTranscript, DEFAULT_POLICY, timelineDuration } from "./transcript.ts";
|
|
21
29
|
import { resolveExistingWorkspaceFile, resolveWorkspacePath, snapshotFile, workspaceRelativePath } from "./workspace.ts";
|
|
22
30
|
|
|
@@ -34,6 +42,7 @@ export interface ApplyTimelineInput {
|
|
|
34
42
|
expectedRevision: number;
|
|
35
43
|
aroll: ArollSegment[];
|
|
36
44
|
broll: BrollPlacementInput[];
|
|
45
|
+
continuityPlanReceipt?: BrollContinuityPlanReceipt;
|
|
37
46
|
}
|
|
38
47
|
|
|
39
48
|
function assertProjectId(projectId: string): void {
|
|
@@ -78,8 +87,16 @@ async function validateTimeline(
|
|
|
78
87
|
cwd: string,
|
|
79
88
|
aroll: ArollSegment[],
|
|
80
89
|
broll: BrollPlacementInput[],
|
|
90
|
+
projectId: string,
|
|
91
|
+
revision: number,
|
|
92
|
+
continuityPlanReceipt: BrollContinuityPlanReceipt | undefined,
|
|
81
93
|
signal?: AbortSignal,
|
|
82
|
-
): Promise<{
|
|
94
|
+
): Promise<{
|
|
95
|
+
outputDurationMs: number;
|
|
96
|
+
broll: BrollPlacement[];
|
|
97
|
+
brollCoverage: ReturnType<typeof assertBrollCoverageBudget>;
|
|
98
|
+
jumpCuts: ReturnType<typeof analyzeArollJumpCuts>;
|
|
99
|
+
}> {
|
|
83
100
|
if (aroll.length === 0 || aroll.length > 1_000) throw new Error("A-roll must contain 1-1000 segments");
|
|
84
101
|
uniqueIds(aroll, "A-roll segment");
|
|
85
102
|
for (const segment of aroll) {
|
|
@@ -91,9 +108,7 @@ async function validateTimeline(
|
|
|
91
108
|
const outputDurationMs = timelineDuration(aroll);
|
|
92
109
|
if (broll.length > 500) throw new Error("B-roll must contain at most 500 placements");
|
|
93
110
|
uniqueIds(broll, "B-roll placement");
|
|
94
|
-
const normalizedBroll: BrollPlacement[] = [];
|
|
95
111
|
for (const placement of broll) {
|
|
96
|
-
signal?.throwIfAborted();
|
|
97
112
|
if (!Number.isFinite(placement.outputStartMs) || !Number.isFinite(placement.outputEndMs)
|
|
98
113
|
|| placement.outputStartMs < 0 || placement.outputEndMs <= placement.outputStartMs) {
|
|
99
114
|
throw new Error(`Invalid B-roll output range: ${placement.id}`);
|
|
@@ -106,6 +121,25 @@ async function validateTimeline(
|
|
|
106
121
|
}
|
|
107
122
|
if (!placement.selectionReceipt) throw new Error(`B-roll selectionReceipt is required: ${placement.id}`);
|
|
108
123
|
if (placement.audio !== "keep-primary") throw new Error("B-roll audio must keep the primary A-roll audio");
|
|
124
|
+
}
|
|
125
|
+
const overlap = findOverlappingBrollWindows(broll)[0];
|
|
126
|
+
if (overlap) {
|
|
127
|
+
throw new Error(`B-roll placements ${overlap.underBrollId} and ${overlap.overBrollId} overlap; ambiguous z-order is not supported`);
|
|
128
|
+
}
|
|
129
|
+
const flashGap = findShortArollFlashGaps(broll, 500)[0];
|
|
130
|
+
if (flashGap) {
|
|
131
|
+
throw new Error(`B-roll placements ${flashGap.fromBrollId} and ${flashGap.toBrollId} leave a brief ${flashGap.durationMs}ms A-roll flash; run talking_head_broll_plan and reselect the bridged window`);
|
|
132
|
+
}
|
|
133
|
+
const brollCoverage = assertBrollCoverageBudget(broll, outputDurationMs);
|
|
134
|
+
if (broll.length > 0 && !continuityPlanReceipt) {
|
|
135
|
+
throw new Error("B-roll continuityPlanReceipt is required; run talking_head_broll_plan first");
|
|
136
|
+
}
|
|
137
|
+
if (continuityPlanReceipt) {
|
|
138
|
+
verifyBrollContinuityPlanReceipt(projectId, revision, aroll, broll, continuityPlanReceipt);
|
|
139
|
+
}
|
|
140
|
+
const normalizedBroll: BrollPlacement[] = [];
|
|
141
|
+
for (const placement of broll) {
|
|
142
|
+
signal?.throwIfAborted();
|
|
109
143
|
const asset = await verifyBrollPlacementSelection(cwd, placement, signal);
|
|
110
144
|
normalizedBroll.push({
|
|
111
145
|
...structuredClone(placement),
|
|
@@ -114,7 +148,14 @@ async function validateTimeline(
|
|
|
114
148
|
assetSha256: asset.sha256,
|
|
115
149
|
});
|
|
116
150
|
}
|
|
117
|
-
return {
|
|
151
|
+
return {
|
|
152
|
+
outputDurationMs,
|
|
153
|
+
broll: normalizedBroll,
|
|
154
|
+
brollCoverage,
|
|
155
|
+
jumpCuts: continuityPlanReceipt
|
|
156
|
+
? structuredClone(continuityPlanReceipt.jumpCuts)
|
|
157
|
+
: analyzeArollJumpCuts(aroll, broll, outputDurationMs, 250, 500),
|
|
158
|
+
};
|
|
118
159
|
}
|
|
119
160
|
|
|
120
161
|
function summary(analysis: TranscriptAnalysis) {
|
|
@@ -181,6 +222,8 @@ export async function createTalkingHeadProject(cwd: string, input: CreateTalking
|
|
|
181
222
|
policy,
|
|
182
223
|
aroll: analysis.segments,
|
|
183
224
|
broll: [],
|
|
225
|
+
brollCoverage: assertBrollCoverageBudget([], analysis.outputDurationMs),
|
|
226
|
+
jumpCuts: analyzeArollJumpCuts(analysis.segments, [], analysis.outputDurationMs, 250, 500),
|
|
184
227
|
outputDurationMs: analysis.outputDurationMs,
|
|
185
228
|
};
|
|
186
229
|
try {
|
|
@@ -248,7 +291,15 @@ export async function applyTimeline(cwd: string, input: ApplyTimelineInput, sign
|
|
|
248
291
|
await assertProjectSourcesUnchanged(cwd, project);
|
|
249
292
|
await assertSnapshotAssetsUnchanged(cwd, current);
|
|
250
293
|
assertWordSafeSegments(input.aroll, await getAnalysis(cwd, project));
|
|
251
|
-
const validated = await validateTimeline(
|
|
294
|
+
const validated = await validateTimeline(
|
|
295
|
+
cwd,
|
|
296
|
+
input.aroll,
|
|
297
|
+
input.broll,
|
|
298
|
+
input.projectId,
|
|
299
|
+
input.expectedRevision,
|
|
300
|
+
input.continuityPlanReceipt,
|
|
301
|
+
signal,
|
|
302
|
+
);
|
|
252
303
|
const revision = project.currentRevision + 1;
|
|
253
304
|
const now = new Date().toISOString();
|
|
254
305
|
const snapshot: TalkingHeadSnapshot = {
|
|
@@ -260,6 +311,11 @@ export async function applyTimeline(cwd: string, input: ApplyTimelineInput, sign
|
|
|
260
311
|
policy: current.policy,
|
|
261
312
|
aroll: structuredClone(input.aroll),
|
|
262
313
|
broll: validated.broll,
|
|
314
|
+
brollCoverage: validated.brollCoverage,
|
|
315
|
+
jumpCuts: validated.jumpCuts,
|
|
316
|
+
...(input.continuityPlanReceipt === undefined
|
|
317
|
+
? {}
|
|
318
|
+
: { continuityPlanReceipt: structuredClone(input.continuityPlanReceipt) }),
|
|
263
319
|
outputDurationMs: validated.outputDurationMs,
|
|
264
320
|
};
|
|
265
321
|
await writeJson(join(directory, "snapshots", `${revision}.json`), snapshot, true);
|