@speclip/pi-talking-head 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -4
- package/extensions/talking-head/index.ts +157 -17
- package/package.json +1 -1
- package/prompts/edit-talking-head.md +1 -1
- package/skills/talking-head-edit/SKILL.md +5 -5
- package/skills/talking-head-edit/references/cut-craft.md +3 -1
- package/src/continuity.ts +186 -20
- package/src/contracts.ts +62 -7
- package/src/project.ts +52 -5
package/README.md
CHANGED
|
@@ -107,14 +107,16 @@ talking_head_broll_plan {
|
|
|
107
107
|
}
|
|
108
108
|
```
|
|
109
109
|
|
|
110
|
-
规划结果同时返回 `continuityPlanReceipt
|
|
110
|
+
规划结果同时返回 `plan.coverage` 和 `continuityPlanReceipt`。`plan.coverage` 会列出 B-roll/A-roll 时长、B-roll 占比、最长连续 B-roll 和审阅提醒;凭证则把当前项目 revision、A-roll、规划后的 B-roll 区间和跳点审阅结果绑定在一起。后续 `talking_head_apply` 会重新计算并核对,不能跳过规划或把已经衔接的区间手动缩短。
|
|
111
111
|
|
|
112
112
|
规划结果处理两类问题:
|
|
113
113
|
|
|
114
114
|
- **B-roll 闪屏**:两个 B-roll 之间只露出不超过 500ms 的 A-roll 时,`plan.needs` 会把前一个需求延长到后一个需求的起点。后续分别选段后,两段 B-roll 会直接交接,不再闪回 A-roll。
|
|
115
|
-
- **气口剪辑跳转**:A-roll
|
|
115
|
+
- **气口剪辑跳转**:A-roll 相邻片段的源时间不连续时会生成 `jumpCuts`。未覆盖项标为 `review`,只表示 Agent 需要查看实际画面。只有确认跳点视觉上突兀时才能添加 `purpose: "mask-cut"`,保存工作区内的审阅图片或视频,并提交 `visualReview: { decision: "mask-with-broll", reviewedJumpCutOutputMs, artifactPath }`;规划器会计算文件哈希并绑定到 v2 凭证,应用时再次校验。否则继续保留 A-roll。
|
|
116
|
+
- **时长与覆盖审阅**:B-roll 时长由完整口播语义和说话节奏决定,不能用固定三秒规则,也不能在一句话没有表达完时提前结束。总覆盖超过 50% 或连续覆盖超过 8000ms 会生成 warning,但不会硬性拒绝合理的长演示;Agent 应删除纯装饰画面,或把同一语义段落组织成不同角度、景别和素材组成的镜头组。
|
|
117
|
+
- **避免重复**:最终应用会拒绝同一素材的源时间段发生任何重叠,避免重复出现相同画面;应改用不同角度、不同素材或同一素材中的其他非重叠有效时刻。
|
|
116
118
|
|
|
117
|
-
规划只提供时间轴证据,不会假定每个剪辑点都必须加 B-roll
|
|
119
|
+
规划只提供时间轴证据,不会假定每个剪辑点都必须加 B-roll。语义匹配、完整表达和实际画面审阅始终优先于覆盖率数字。
|
|
118
120
|
|
|
119
121
|
### 4. 文件名优先筛选 B-roll
|
|
120
122
|
|
|
@@ -203,7 +205,7 @@ review { path: "out/launch-final.mp4" }
|
|
|
203
205
|
| --- | --- |
|
|
204
206
|
| `talking_head_create` | 从视频和词级转录建立 revision 1,分析句子、语气词、重复和停顿并生成默认 EDL |
|
|
205
207
|
| `talking_head_get` | 读取指定 revision,分页返回整句与编辑候选,可选导出 pi-media EDL |
|
|
206
|
-
| `talking_head_broll_plan` |
|
|
208
|
+
| `talking_head_broll_plan` | 按 A-roll 优先预算规划 B-roll、衔接相邻片段并报告待审阅跳点 |
|
|
207
209
|
| `talking_head_broll_match` | 只用文件名和目录名匹配本地 B-roll,返回受限 shortlist 与视觉升级建议 |
|
|
208
210
|
| `talking_head_broll_select` | 用 pi-media 联络表 manifest 验证具体源时间段并生成 placement |
|
|
209
211
|
| `talking_head_apply` | 写入新的不可变口播 revision,并返回 pi-media EDL |
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
2
|
+
import { open } from "node:fs/promises";
|
|
3
|
+
import { extname } from "node:path";
|
|
2
4
|
import { Type } from "typebox";
|
|
5
|
+
import type { BrollNeed } from "../../src/contracts.ts";
|
|
3
6
|
import { matchWorkspaceBrollAssets, selectBrollWindow } from "../../src/broll.ts";
|
|
4
7
|
import { createBrollContinuityPlanReceipt, planBrollContinuity } from "../../src/continuity.ts";
|
|
5
8
|
import { toMediaTimelineOperation } from "../../src/edl.ts";
|
|
@@ -11,6 +14,7 @@ import {
|
|
|
11
14
|
getAnalysis,
|
|
12
15
|
getTalkingHeadProject,
|
|
13
16
|
} from "../../src/project.ts";
|
|
17
|
+
import { resolveExistingWorkspaceFile, snapshotFile } from "../../src/workspace.ts";
|
|
14
18
|
|
|
15
19
|
function result(details: unknown) {
|
|
16
20
|
return {
|
|
@@ -50,23 +54,115 @@ const brollPlacement = Type.Object({
|
|
|
50
54
|
}, { additionalProperties: false }),
|
|
51
55
|
}, { additionalProperties: false });
|
|
52
56
|
|
|
53
|
-
const
|
|
57
|
+
const brollNeedFields = {
|
|
54
58
|
id: stableId,
|
|
55
59
|
outputStartMs: Type.Number({ minimum: 0 }),
|
|
56
60
|
outputEndMs: Type.Number({ exclusiveMinimum: 0 }),
|
|
57
61
|
speechText: Type.String({ minLength: 1, maxLength: 2_000 }),
|
|
62
|
+
searchTerms: Type.Array(Type.String({ minLength: 1, maxLength: 100 }), { minItems: 1, maxItems: 20 }),
|
|
63
|
+
reason: Type.String({ minLength: 1, maxLength: 1_000 }),
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
const semanticBrollNeed = Type.Object({
|
|
67
|
+
...brollNeedFields,
|
|
58
68
|
purpose: Type.Union([
|
|
59
69
|
Type.Literal("demonstrate"),
|
|
60
70
|
Type.Literal("explain"),
|
|
61
71
|
Type.Literal("evidence"),
|
|
62
72
|
Type.Literal("establish"),
|
|
63
73
|
Type.Literal("transition"),
|
|
64
|
-
Type.Literal("mask-cut"),
|
|
65
74
|
]),
|
|
66
|
-
searchTerms: Type.Array(Type.String({ minLength: 1, maxLength: 100 }), { minItems: 1, maxItems: 20 }),
|
|
67
|
-
reason: Type.String({ minLength: 1, maxLength: 1_000 }),
|
|
68
75
|
}, { additionalProperties: false });
|
|
69
76
|
|
|
77
|
+
const maskCutReviewFields = {
|
|
78
|
+
decision: Type.Literal("mask-with-broll"),
|
|
79
|
+
reviewedJumpCutOutputMs: Type.Number({ minimum: 0 }),
|
|
80
|
+
artifactPath: Type.String({ minLength: 1, description: "Workspace-relative image or video reviewed at this A-roll jump." }),
|
|
81
|
+
};
|
|
82
|
+
|
|
83
|
+
const maskCutBrollNeedInput = Type.Object({
|
|
84
|
+
...brollNeedFields,
|
|
85
|
+
purpose: Type.Literal("mask-cut"),
|
|
86
|
+
visualReview: Type.Object(maskCutReviewFields, { additionalProperties: false }),
|
|
87
|
+
}, { additionalProperties: false });
|
|
88
|
+
|
|
89
|
+
const maskCutBrollNeed = Type.Object({
|
|
90
|
+
...brollNeedFields,
|
|
91
|
+
purpose: Type.Literal("mask-cut"),
|
|
92
|
+
visualReview: Type.Object({
|
|
93
|
+
...maskCutReviewFields,
|
|
94
|
+
artifactSha256: Type.String({ pattern: "^[a-f0-9]{64}$" }),
|
|
95
|
+
}, { additionalProperties: false }),
|
|
96
|
+
}, { additionalProperties: false });
|
|
97
|
+
|
|
98
|
+
const brollNeedInput = Type.Union([semanticBrollNeed, maskCutBrollNeedInput]);
|
|
99
|
+
const brollNeed = Type.Union([
|
|
100
|
+
semanticBrollNeed,
|
|
101
|
+
maskCutBrollNeed,
|
|
102
|
+
]);
|
|
103
|
+
|
|
104
|
+
const visualReviewArtifactExtensions = new Set([
|
|
105
|
+
".png", ".jpg", ".jpeg", ".webp", ".avif", ".mp4", ".mov", ".m4v", ".webm", ".mkv",
|
|
106
|
+
]);
|
|
107
|
+
|
|
108
|
+
function hasVisualMediaSignature(bytes: Buffer): boolean {
|
|
109
|
+
const startsWith = (signature: number[]) => signature.every((byte, index) => bytes[index] === byte);
|
|
110
|
+
const ascii = (start: number, value: string) => bytes.subarray(start, start + value.length).toString("ascii") === value;
|
|
111
|
+
return startsWith([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])
|
|
112
|
+
|| startsWith([0xff, 0xd8, 0xff])
|
|
113
|
+
|| (ascii(0, "RIFF") && ascii(8, "WEBP"))
|
|
114
|
+
|| ascii(4, "ftyp")
|
|
115
|
+
|| startsWith([0x1a, 0x45, 0xdf, 0xa3]);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
async function assertVisualReviewArtifactBytes(cwd: string, artifactPath: string, signal?: AbortSignal): Promise<void> {
|
|
119
|
+
signal?.throwIfAborted();
|
|
120
|
+
const absolutePath = await resolveExistingWorkspaceFile(cwd, artifactPath);
|
|
121
|
+
const handle = await open(absolutePath, "r");
|
|
122
|
+
try {
|
|
123
|
+
const bytes = Buffer.alloc(16);
|
|
124
|
+
const { bytesRead } = await handle.read(bytes, 0, bytes.length, 0);
|
|
125
|
+
if (!hasVisualMediaSignature(bytes.subarray(0, bytesRead))) {
|
|
126
|
+
throw new Error("visual review artifact must contain recognizable image or video bytes");
|
|
127
|
+
}
|
|
128
|
+
} finally {
|
|
129
|
+
await handle.close();
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
async function bindVisualReviewArtifacts(
|
|
134
|
+
cwd: string,
|
|
135
|
+
needs: BrollNeed[],
|
|
136
|
+
signal?: AbortSignal,
|
|
137
|
+
): Promise<BrollNeed[]> {
|
|
138
|
+
const verifiedNeeds: BrollNeed[] = [];
|
|
139
|
+
for (const need of needs) {
|
|
140
|
+
signal?.throwIfAborted();
|
|
141
|
+
if (need.purpose !== "mask-cut") {
|
|
142
|
+
verifiedNeeds.push(structuredClone(need));
|
|
143
|
+
continue;
|
|
144
|
+
}
|
|
145
|
+
if (!visualReviewArtifactExtensions.has(extname(need.visualReview.artifactPath).toLowerCase())) {
|
|
146
|
+
throw new Error(`B-roll mask-cut need ${need.id} visual review artifact must be an image or video`);
|
|
147
|
+
}
|
|
148
|
+
try {
|
|
149
|
+
await assertVisualReviewArtifactBytes(cwd, need.visualReview.artifactPath, signal);
|
|
150
|
+
} catch (error) {
|
|
151
|
+
throw new Error(`B-roll mask-cut need ${need.id} ${(error as Error).message}`);
|
|
152
|
+
}
|
|
153
|
+
const artifact = await snapshotFile(cwd, need.visualReview.artifactPath, signal);
|
|
154
|
+
verifiedNeeds.push({
|
|
155
|
+
...structuredClone(need),
|
|
156
|
+
visualReview: {
|
|
157
|
+
...structuredClone(need.visualReview),
|
|
158
|
+
artifactPath: artifact.path,
|
|
159
|
+
artifactSha256: artifact.sha256,
|
|
160
|
+
},
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
return verifiedNeeds;
|
|
164
|
+
}
|
|
165
|
+
|
|
70
166
|
const arollJumpCut = Type.Object({
|
|
71
167
|
fromSegmentId: stableId,
|
|
72
168
|
toSegmentId: stableId,
|
|
@@ -75,26 +171,65 @@ const arollJumpCut = Type.Object({
|
|
|
75
171
|
suggestedOutputStartMs: Type.Number({ minimum: 0 }),
|
|
76
172
|
suggestedOutputEndMs: Type.Number({ minimum: 0 }),
|
|
77
173
|
coveredByNeedIds: Type.Array(stableId, { maxItems: 500 }),
|
|
78
|
-
status: Type.Union([
|
|
174
|
+
status: Type.Union([
|
|
175
|
+
Type.Literal("covered"),
|
|
176
|
+
Type.Literal("review"),
|
|
177
|
+
Type.Literal("needs-broll", { description: "Legacy v0.1.5 receipt value; new plans emit review." }),
|
|
178
|
+
]),
|
|
79
179
|
}, { additionalProperties: false });
|
|
80
180
|
|
|
81
|
-
const
|
|
82
|
-
schemaVersion: Type.Literal(1),
|
|
181
|
+
const continuityPlanReceiptBase = {
|
|
83
182
|
projectId: stableId,
|
|
84
183
|
revision: Type.Integer({ minimum: 1 }),
|
|
85
184
|
arollSha256: Type.String({ pattern: "^[a-f0-9]{64}$" }),
|
|
86
185
|
shortGapMs: Type.Integer({ minimum: 0, maximum: 2_000 }),
|
|
87
186
|
cutCoverBeforeMs: Type.Integer({ minimum: 0, maximum: 2_000 }),
|
|
88
187
|
cutCoverAfterMs: Type.Integer({ minimum: 0, maximum: 2_000 }),
|
|
188
|
+
jumpCuts: Type.Array(arollJumpCut, { maxItems: 999 }),
|
|
189
|
+
planSha256: Type.String({ pattern: "^[a-f0-9]{64}$" }),
|
|
190
|
+
};
|
|
191
|
+
|
|
192
|
+
const continuityPlanReceiptV1 = Type.Object({
|
|
193
|
+
...continuityPlanReceiptBase,
|
|
194
|
+
schemaVersion: Type.Literal(1),
|
|
89
195
|
needs: Type.Array(Type.Object({
|
|
90
196
|
id: stableId,
|
|
91
197
|
outputStartMs: Type.Number({ minimum: 0 }),
|
|
92
198
|
outputEndMs: Type.Number({ exclusiveMinimum: 0 }),
|
|
93
199
|
}, { additionalProperties: false }), { maxItems: 500 }),
|
|
94
|
-
jumpCuts: Type.Array(arollJumpCut, { maxItems: 999 }),
|
|
95
|
-
planSha256: Type.String({ pattern: "^[a-f0-9]{64}$" }),
|
|
96
200
|
}, { additionalProperties: false });
|
|
97
201
|
|
|
202
|
+
const continuityPlanReceiptV2 = Type.Object({
|
|
203
|
+
...continuityPlanReceiptBase,
|
|
204
|
+
schemaVersion: Type.Literal(2),
|
|
205
|
+
needs: Type.Array(Type.Union([
|
|
206
|
+
Type.Object({
|
|
207
|
+
id: stableId,
|
|
208
|
+
outputStartMs: Type.Number({ minimum: 0 }),
|
|
209
|
+
outputEndMs: Type.Number({ exclusiveMinimum: 0 }),
|
|
210
|
+
purpose: Type.Union([
|
|
211
|
+
Type.Literal("demonstrate"),
|
|
212
|
+
Type.Literal("explain"),
|
|
213
|
+
Type.Literal("evidence"),
|
|
214
|
+
Type.Literal("establish"),
|
|
215
|
+
Type.Literal("transition"),
|
|
216
|
+
]),
|
|
217
|
+
}, { additionalProperties: false }),
|
|
218
|
+
Type.Object({
|
|
219
|
+
id: stableId,
|
|
220
|
+
outputStartMs: Type.Number({ minimum: 0 }),
|
|
221
|
+
outputEndMs: Type.Number({ exclusiveMinimum: 0 }),
|
|
222
|
+
purpose: Type.Literal("mask-cut"),
|
|
223
|
+
visualReview: Type.Object({
|
|
224
|
+
...maskCutReviewFields,
|
|
225
|
+
artifactSha256: Type.String({ pattern: "^[a-f0-9]{64}$" }),
|
|
226
|
+
}, { additionalProperties: false }),
|
|
227
|
+
}, { additionalProperties: false }),
|
|
228
|
+
]), { maxItems: 500 }),
|
|
229
|
+
}, { additionalProperties: false });
|
|
230
|
+
|
|
231
|
+
const continuityPlanReceipt = Type.Union([continuityPlanReceiptV1, continuityPlanReceiptV2]);
|
|
232
|
+
|
|
98
233
|
export default function talkingHead(pi: ExtensionAPI): void {
|
|
99
234
|
pi.registerTool({
|
|
100
235
|
name: "talking_head_create",
|
|
@@ -200,20 +335,25 @@ export default function talkingHead(pi: ExtensionAPI): void {
|
|
|
200
335
|
pi.registerTool({
|
|
201
336
|
name: "talking_head_broll_plan",
|
|
202
337
|
label: "Plan B-roll visual continuity",
|
|
203
|
-
description: "Plan B-roll windows
|
|
338
|
+
description: "Plan meaning- and rhythm-led B-roll windows after A-roll is stable. Coverage above 50% or a continuous run above 8 seconds produces review warnings rather than fixed rejection. Brief A-roll flashes are bridged, while uncovered source jumps are reported only for picture review.",
|
|
204
339
|
parameters: Type.Object({
|
|
205
340
|
projectId: stableId,
|
|
206
341
|
revision: Type.Optional(Type.Integer({ minimum: 1 })),
|
|
207
342
|
aroll: Type.Array(arollSegment, { minItems: 1, maxItems: 1_000 }),
|
|
208
|
-
needs: Type.Array(
|
|
343
|
+
needs: Type.Array(brollNeedInput, { maxItems: 500 }),
|
|
209
344
|
cutCoverBeforeMs: Type.Optional(Type.Integer({ minimum: 0, maximum: 2_000, default: 250 })),
|
|
210
345
|
cutCoverAfterMs: Type.Optional(Type.Integer({ minimum: 0, maximum: 2_000, default: 500 })),
|
|
211
346
|
}, { additionalProperties: false }),
|
|
212
|
-
async execute(_toolCallId, params,
|
|
347
|
+
async execute(_toolCallId, params, signal, _onUpdate, ctx) {
|
|
213
348
|
const { snapshot } = await getTalkingHeadProject(ctx.cwd, params.projectId, params.revision);
|
|
349
|
+
const needs = await bindVisualReviewArtifacts(
|
|
350
|
+
ctx.cwd,
|
|
351
|
+
params.needs as BrollNeed[],
|
|
352
|
+
signal,
|
|
353
|
+
);
|
|
214
354
|
const plan = planBrollContinuity({
|
|
215
355
|
aroll: params.aroll,
|
|
216
|
-
needs
|
|
356
|
+
needs,
|
|
217
357
|
...(params.cutCoverBeforeMs === undefined ? {} : { cutCoverBeforeMs: params.cutCoverBeforeMs }),
|
|
218
358
|
...(params.cutCoverAfterMs === undefined ? {} : { cutCoverAfterMs: params.cutCoverAfterMs }),
|
|
219
359
|
});
|
|
@@ -227,9 +367,9 @@ export default function talkingHead(pi: ExtensionAPI): void {
|
|
|
227
367
|
params.aroll,
|
|
228
368
|
plan,
|
|
229
369
|
),
|
|
230
|
-
nextStep: plan.jumpCuts.some((cut) => cut.status === "
|
|
231
|
-
? "
|
|
232
|
-
: "
|
|
370
|
+
nextStep: plan.jumpCuts.some((cut) => cut.status === "review")
|
|
371
|
+
? "Review the actual picture at every status=review jump. Keep A-roll unless the cut is visibly objectionable; purpose=mask-cut requires visualReview with that exact output timestamp. Then resolve any plan.coverage warnings before matching assets."
|
|
372
|
+
: "Resolve any plan.coverage warnings, then use plan.needs for B-roll asset matching and source-window selection.",
|
|
233
373
|
});
|
|
234
374
|
},
|
|
235
375
|
});
|
|
@@ -308,7 +448,7 @@ export default function talkingHead(pi: ExtensionAPI): void {
|
|
|
308
448
|
pi.registerTool({
|
|
309
449
|
name: "talking_head_apply",
|
|
310
450
|
label: "Apply talking-head timeline",
|
|
311
|
-
description: "Create a new immutable talking-head revision from word-boundary A-roll ranges and optional B-roll placements. B-roll requires the matching continuity plan receipt; brief A-roll flashes,
|
|
451
|
+
description: "Create a new immutable talking-head revision from word-boundary A-roll ranges and optional B-roll placements. B-roll requires the matching continuity plan receipt; unplanned ranges, repeated source windows, brief A-roll flashes, and overlaps are rejected. Coverage and continuous-run concerns remain visible as editorial warnings. Returns a generic pi-media timeline operation ready for edit_apply.",
|
|
312
452
|
parameters: Type.Object({
|
|
313
453
|
projectId: stableId,
|
|
314
454
|
expectedRevision: Type.Integer({ minimum: 1 }),
|
package/package.json
CHANGED
|
@@ -2,6 +2,6 @@
|
|
|
2
2
|
description: Tighten a talking-head video without destroying natural speech rhythm
|
|
3
3
|
---
|
|
4
4
|
|
|
5
|
-
Use the `talking-head-edit` skill to review full-sentence context, word-level pauses, filler words, adjacent repetitions, and low-confidence delivery cues; stabilize A-roll,
|
|
5
|
+
Use the `talking-head-edit` skill to review full-sentence context, word-level pauses, filler words, adjacent repetitions, and low-confidence delivery cues; stabilize A-roll, let each complete spoken idea determine its B-roll duration, treat high coverage and long runs as review warnings rather than fixed limits, bridge brief A-roll flashes, mask only picture-reviewed objectionable jump cuts, avoid repeated source windows, match B-roll by filename before escalating through paged contact-sheet batches, validate the selected manifest-backed source window, and export a generic pi-media EDL for this request: $@
|
|
6
6
|
|
|
7
7
|
Show the proposed rhythm before writing a new revision. Never overwrite the source or an existing render.
|
|
@@ -12,16 +12,16 @@ Use `pi-speech` for word evidence, this package for editorial decisions, and `pi
|
|
|
12
12
|
3. Inspect bounded `talking_head_get` pages. Review the full sentence context, filler candidates, and repetition candidates together with pauses. Treat `safe` and every recommendation as evidence, not an instruction. Preserve pauses that carry emphasis, emotion, topic boundaries, or a deliberate breath.
|
|
13
13
|
4. Never delete a filler token automatically. Decide whether `啊`, `额`, `嗯`, or a repeated token is a false start, a discourse marker, or intentional emphasis from its sentence context. A text-only delivery cue is low-confidence evidence; review audio and picture before relying on emotion or performance intent.
|
|
14
14
|
5. Before changing the revision, summarize the proposed rhythm: what words and gaps will be removed, which pauses or fillers will remain, and why. Get the user's approval unless they explicitly delegated editorial judgment.
|
|
15
|
-
6. Stabilize A-roll before B-roll.
|
|
16
|
-
7. Call `talking_head_broll_plan` with the
|
|
15
|
+
6. Stabilize A-roll before B-roll. A-roll carries the primary message; B-roll must show or clarify the object, action, place, comparison, or evidence being spoken about. For every window, record the complete spoken idea in `speechText`, choose one purpose (`demonstrate`, `explain`, `evidence`, `establish`, `transition`, or `mask-cut`), add 1-20 concise search terms, and explain what visual information it contributes. Let the spoken idea determine the window length; never use a fixed three-second rule or end B-roll before the thought finishes.
|
|
16
|
+
7. Call `talking_head_broll_plan` with the stable A-roll and all semantic B-roll needs before matching assets. Keep its `continuityPlanReceipt` unchanged. Use `plan.needs` and inspect `plan.coverage`. Coverage above 50% and a continuous run above 8000ms are review signals, not fixed limits: remove decorative windows, but retain a longer demonstration when it remains useful for the complete spoken idea. The planner extends the previous B-roll across A-roll gaps of 500ms or less so neighboring clips hand off directly. A jump with `status: review` only asks for picture review. Keep A-roll by default; create `purpose: mask-cut` only after the actual picture is visibly objectionable. Save the reviewed image or video inside the workspace and submit `visualReview: { decision: "mask-with-broll", reviewedJumpCutOutputMs, artifactPath }`; the planner binds its hash into the v2 receipt and apply rejects changed evidence.
|
|
17
17
|
8. Call `talking_head_broll_match` before generating contact sheets. Follow its bounded result:
|
|
18
18
|
- `filename-direct`: probe and visually inspect only the selected asset to choose its source window. Do not call `media_contact_sheet` for unselected assets.
|
|
19
19
|
- `filename-shortlist`: inspect only the returned shortlist and stop as soon as one asset satisfies the need. If none does and `nextCandidateOffset` is non-null, request that next page; do not repeat the first batch.
|
|
20
20
|
- `visual-fallback`: use low-cost contact sheets on the bounded shortlist because filenames supplied no useful evidence. Exhaust that batch before requesting a non-null `nextCandidateOffset`.
|
|
21
21
|
- If the scan is truncated, do not paginate it: narrow the asset directory and rescan so the candidate inventory is stable.
|
|
22
|
-
9. After selecting one asset, call `media_probe`, then use `media_contact_sheet` only as densely as timing requires: one high-density pass for a short ambiguous clip, medium for a normal clip, or low over a long clip followed by high density over the narrowed range. Read `contact_sheet_manifest.json` timestamps instead of guessing times from the PNG. Verify a continuous source window at least as long as the planned
|
|
23
|
-
10. Call `talking_head_broll_select`
|
|
24
|
-
11. Call `talking_head_apply` using the same planned A-roll, every validated B-roll placement, and the unchanged `continuityPlanReceipt`. The tool rejects unplanned ranges,
|
|
22
|
+
9. After selecting one asset, call `media_probe`, then use `media_contact_sheet` only as densely as timing requires: one high-density pass for a short ambiguous clip, medium for a normal clip, or low over a long clip followed by high density over the narrowed range. Read `contact_sheet_manifest.json` timestamps instead of guessing times from the PNG. Verify a continuous source window at least as long as the complete planned spoken idea, preferably with 500ms handles on both sides. If it is too short, choose a longer source window or another asset instead of cutting the idea short.
|
|
23
|
+
10. Build a shot group when one spoken idea benefits from several views: use different angles, scales, assets, or source moments and let neighboring planned windows hand off directly. Call `talking_head_broll_select` for each need with the selected asset, manifest path, exact source start, and reviewed manifest timestamps. Use its complete returned `placement`, including `selectionReceipt`; do not shorten bridged output ranges or hand-author selection fields.
|
|
24
|
+
11. Call `talking_head_apply` using the same planned A-roll, every validated B-roll placement, and the unchanged `continuityPlanReceipt`. The tool rejects unplanned ranges, any overlap between source windows from the same asset, output overlaps, and A-roll flashes of 500ms or less; coverage concerns are persisted as review warnings. Every B-roll window must have a concrete visual purpose in `reason`; keep `audio: keep-primary`.
|
|
25
25
|
12. Create/read a `pi-media` project for the same source. Pass the returned `mediaOperation` unchanged to `edit_apply`, then `render` the exact new revision and call `review`.
|
|
26
26
|
|
|
27
27
|
Never overwrite source media or outputs. If a revision conflict occurs, re-read both projects and reconcile intent. If the source, transcript, or B-roll hash changed, stop and ask whether to create a new project rather than silently adopting new bytes.
|
|
@@ -23,7 +23,9 @@
|
|
|
23
23
|
- Start with filename and directory metadata. Generate contact sheets only for the selected asset, a bounded ambiguous shortlist, or a bounded fallback batch whose names carry no meaning.
|
|
24
24
|
- For a long selected asset, locate a broad range cheaply and then generate a denser contact sheet only for that range. Use manifest timestamps, not visual timestamp transcription.
|
|
25
25
|
- Ensure the selected source window covers the complete spoken beat. Prefer extra source handles so a later timing adjustment does not require rescanning.
|
|
26
|
+
- Let meaning and delivery rhythm set the duration. Coverage above 50% or one run above eight seconds deserves review, but is not automatically wrong when a long demonstration remains informative.
|
|
27
|
+
- For one semantic beat, prefer a shot group of different angles, scales, assets, or source moments over repeating an identical clip.
|
|
26
28
|
- If neighboring B-roll windows expose 500ms or less of A-roll, plan a direct handoff by extending the previous B-roll need before source-window selection. Do not let a few frames of A-roll flash between overlays.
|
|
27
|
-
-
|
|
29
|
+
- Treat every uncovered jump cut as a picture-review candidate, not a request for B-roll. Add a mask-cut only after confirming the actual jump is objectionable, recording its output timestamp, and binding the reviewed workspace image or video into the continuity receipt.
|
|
28
30
|
- Keep the primary voice track. B-roll audio replacement is outside the current contract.
|
|
29
31
|
- Record the search query and editorial reason so a later agent can replace the asset without guessing intent.
|
package/src/continuity.ts
CHANGED
|
@@ -5,10 +5,15 @@ import type {
|
|
|
5
5
|
BrollContinuityBridge,
|
|
6
6
|
BrollContinuityPlan,
|
|
7
7
|
BrollContinuityPlanReceipt,
|
|
8
|
+
BrollContinuityPlanReceiptV2,
|
|
9
|
+
BrollCoverageSummary,
|
|
8
10
|
BrollNeed,
|
|
9
11
|
} from "./contracts.ts";
|
|
10
12
|
import { timelineDuration } from "./transcript.ts";
|
|
11
13
|
|
|
14
|
+
export const HIGH_BROLL_COVERAGE_REVIEW_RATIO = 0.5;
|
|
15
|
+
export const LONG_CONTINUOUS_BROLL_REVIEW_MS = 8_000;
|
|
16
|
+
|
|
12
17
|
export interface PlanBrollContinuityInput {
|
|
13
18
|
aroll: ArollSegment[];
|
|
14
19
|
needs: BrollNeed[];
|
|
@@ -105,6 +110,83 @@ export function findOverlappingBrollWindows(windows: BrollOutputWindow[]): Overl
|
|
|
105
110
|
return overlaps;
|
|
106
111
|
}
|
|
107
112
|
|
|
113
|
+
export function summarizeBrollCoverage(
|
|
114
|
+
windows: BrollOutputWindow[],
|
|
115
|
+
outputDurationMs: number,
|
|
116
|
+
): BrollCoverageSummary {
|
|
117
|
+
if (!Number.isFinite(outputDurationMs) || outputDurationMs <= 0) {
|
|
118
|
+
throw new Error("outputDurationMs must be a positive finite number");
|
|
119
|
+
}
|
|
120
|
+
for (const window of windows) {
|
|
121
|
+
if (!Number.isFinite(window.outputStartMs) || !Number.isFinite(window.outputEndMs)
|
|
122
|
+
|| window.outputStartMs < 0 || window.outputEndMs <= window.outputStartMs
|
|
123
|
+
|| window.outputEndMs > outputDurationMs) {
|
|
124
|
+
throw new Error(`Invalid B-roll coverage window: ${window.id}`);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
const sorted = [...windows].sort((left, right) => (
|
|
128
|
+
left.outputStartMs - right.outputStartMs
|
|
129
|
+
|| left.outputEndMs - right.outputEndMs
|
|
130
|
+
|| (left.id < right.id ? -1 : left.id > right.id ? 1 : 0)
|
|
131
|
+
));
|
|
132
|
+
let brollDurationMs = 0;
|
|
133
|
+
let longestContinuousBrollMs = 0;
|
|
134
|
+
let runStartMs: number | undefined;
|
|
135
|
+
let runEndMs: number | undefined;
|
|
136
|
+
for (const window of sorted) {
|
|
137
|
+
if (runStartMs === undefined || runEndMs === undefined) {
|
|
138
|
+
runStartMs = window.outputStartMs;
|
|
139
|
+
runEndMs = window.outputEndMs;
|
|
140
|
+
continue;
|
|
141
|
+
}
|
|
142
|
+
if (window.outputStartMs <= runEndMs) {
|
|
143
|
+
runEndMs = Math.max(runEndMs, window.outputEndMs);
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
const durationMs = runEndMs - runStartMs;
|
|
147
|
+
brollDurationMs += durationMs;
|
|
148
|
+
longestContinuousBrollMs = Math.max(longestContinuousBrollMs, durationMs);
|
|
149
|
+
runStartMs = window.outputStartMs;
|
|
150
|
+
runEndMs = window.outputEndMs;
|
|
151
|
+
}
|
|
152
|
+
if (runStartMs !== undefined && runEndMs !== undefined) {
|
|
153
|
+
const durationMs = runEndMs - runStartMs;
|
|
154
|
+
brollDurationMs += durationMs;
|
|
155
|
+
longestContinuousBrollMs = Math.max(longestContinuousBrollMs, durationMs);
|
|
156
|
+
}
|
|
157
|
+
const exactBrollCoverageRatio = brollDurationMs / outputDurationMs;
|
|
158
|
+
const brollCoverageRatio = Math.round(exactBrollCoverageRatio * 10_000) / 10_000;
|
|
159
|
+
const warnings: BrollCoverageSummary["warnings"] = [];
|
|
160
|
+
if (exactBrollCoverageRatio > HIGH_BROLL_COVERAGE_REVIEW_RATIO) {
|
|
161
|
+
const percent = ((brollDurationMs / outputDurationMs) * 100).toFixed(1);
|
|
162
|
+
warnings.push({
|
|
163
|
+
code: "high-total-coverage",
|
|
164
|
+
message: `B-roll covers ${percent}% of the output; review whether every window supports a complete spoken idea and keep A-roll whenever the picture adds no information.`,
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
if (longestContinuousBrollMs > LONG_CONTINUOUS_BROLL_REVIEW_MS) {
|
|
168
|
+
warnings.push({
|
|
169
|
+
code: "long-continuous-run",
|
|
170
|
+
message: `One continuous B-roll run lasts ${longestContinuousBrollMs}ms; confirm that its source window remains useful for the full spoken idea or split it into a varied shot group.`,
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
return {
|
|
174
|
+
brollDurationMs,
|
|
175
|
+
arollDurationMs: outputDurationMs - brollDurationMs,
|
|
176
|
+
brollCoverageRatio,
|
|
177
|
+
longestContinuousBrollMs,
|
|
178
|
+
limits: {
|
|
179
|
+
maxBrollCoverageRatio: HIGH_BROLL_COVERAGE_REVIEW_RATIO,
|
|
180
|
+
maxContinuousBrollMs: LONG_CONTINUOUS_BROLL_REVIEW_MS,
|
|
181
|
+
},
|
|
182
|
+
reviewThresholds: {
|
|
183
|
+
highBrollCoverageRatio: HIGH_BROLL_COVERAGE_REVIEW_RATIO,
|
|
184
|
+
longContinuousBrollMs: LONG_CONTINUOUS_BROLL_REVIEW_MS,
|
|
185
|
+
},
|
|
186
|
+
warnings,
|
|
187
|
+
};
|
|
188
|
+
}
|
|
189
|
+
|
|
108
190
|
function validateSegments(aroll: ArollSegment[]): void {
|
|
109
191
|
if (aroll.length === 0 || aroll.length > 1_000) throw new Error("A-roll must contain 1-1000 segments");
|
|
110
192
|
const ids = new Set<string>();
|
|
@@ -118,10 +200,24 @@ function validateSegments(aroll: ArollSegment[]): void {
|
|
|
118
200
|
}
|
|
119
201
|
}
|
|
120
202
|
|
|
121
|
-
function
|
|
203
|
+
function arollJumpCutOutputTimes(aroll: ArollSegment[]): Set<number> {
|
|
204
|
+
const outputTimes = new Set<number>();
|
|
205
|
+
let outputTimeMs = 0;
|
|
206
|
+
for (let index = 0; index < aroll.length - 1; index += 1) {
|
|
207
|
+
const current = aroll[index]!;
|
|
208
|
+
const next = aroll[index + 1]!;
|
|
209
|
+
outputTimeMs += current.sourceEndMs - current.sourceStartMs;
|
|
210
|
+
if (next.sourceStartMs !== current.sourceEndMs) outputTimes.add(outputTimeMs);
|
|
211
|
+
}
|
|
212
|
+
return outputTimes;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
function validateNeeds(needs: BrollNeed[], outputDurationMs: number, aroll: ArollSegment[]): void {
|
|
122
216
|
if (needs.length > 500) throw new Error("B-roll plan must contain at most 500 needs");
|
|
123
217
|
const ids = new Set<string>();
|
|
218
|
+
const jumpCutOutputTimes = arollJumpCutOutputTimes(aroll);
|
|
124
219
|
for (const need of needs) {
|
|
220
|
+
const unexpectedVisualReview = (need as { visualReview?: unknown }).visualReview;
|
|
125
221
|
if (ids.has(need.id)) throw new Error(`Duplicate B-roll need ID: ${need.id}`);
|
|
126
222
|
ids.add(need.id);
|
|
127
223
|
if (!Number.isFinite(need.outputStartMs) || !Number.isFinite(need.outputEndMs)
|
|
@@ -129,6 +225,27 @@ function validateNeeds(needs: BrollNeed[], outputDurationMs: number): void {
|
|
|
129
225
|
|| need.outputEndMs > outputDurationMs) {
|
|
130
226
|
throw new Error(`Invalid B-roll need output range: ${need.id}`);
|
|
131
227
|
}
|
|
228
|
+
if (!need.speechText.trim()) throw new Error(`B-roll need ${need.id} must name the complete spoken idea it supports`);
|
|
229
|
+
if (!need.reason.trim()) throw new Error(`B-roll need ${need.id} must explain why the picture adds information`);
|
|
230
|
+
if (need.purpose === "mask-cut") {
|
|
231
|
+
if (need.visualReview?.decision !== "mask-with-broll"
|
|
232
|
+
|| !Number.isFinite(need.visualReview.reviewedJumpCutOutputMs)) {
|
|
233
|
+
throw new Error(`B-roll mask-cut need ${need.id} requires visualReview evidence for an objectionable A-roll jump`);
|
|
234
|
+
}
|
|
235
|
+
const reviewedOutputMs = need.visualReview.reviewedJumpCutOutputMs;
|
|
236
|
+
if (!jumpCutOutputTimes.has(reviewedOutputMs)) {
|
|
237
|
+
throw new Error(`B-roll mask-cut need ${need.id} visualReview does not identify an A-roll source jump`);
|
|
238
|
+
}
|
|
239
|
+
if (reviewedOutputMs <= need.outputStartMs || reviewedOutputMs >= need.outputEndMs) {
|
|
240
|
+
throw new Error(`B-roll mask-cut need ${need.id} must span the reviewed A-roll jump`);
|
|
241
|
+
}
|
|
242
|
+
if (!need.visualReview.artifactPath.trim()
|
|
243
|
+
|| !/^[a-f0-9]{64}$/.test(need.visualReview.artifactSha256 ?? "")) {
|
|
244
|
+
throw new Error(`B-roll mask-cut need ${need.id} requires a verified visual review artifact`);
|
|
245
|
+
}
|
|
246
|
+
} else if (unexpectedVisualReview !== undefined) {
|
|
247
|
+
throw new Error(`B-roll visualReview is only valid for purpose=mask-cut: ${need.id}`);
|
|
248
|
+
}
|
|
132
249
|
}
|
|
133
250
|
const overlap = findOverlappingBrollWindows(needs)[0];
|
|
134
251
|
if (overlap) throw new Error(`B-roll needs ${overlap.underBrollId} and ${overlap.overBrollId} overlap; z-order is not supported`);
|
|
@@ -222,7 +339,7 @@ export function analyzeArollJumpCuts(
|
|
|
222
339
|
suggestedOutputStartMs,
|
|
223
340
|
suggestedOutputEndMs,
|
|
224
341
|
coveredByNeedIds: coverage.needIds,
|
|
225
|
-
status: coverage.covered ? "covered" : "
|
|
342
|
+
status: coverage.covered ? "covered" : "review",
|
|
226
343
|
});
|
|
227
344
|
}
|
|
228
345
|
return jumpCuts;
|
|
@@ -231,7 +348,7 @@ export function analyzeArollJumpCuts(
|
|
|
231
348
|
export function planBrollContinuity(input: PlanBrollContinuityInput): BrollContinuityPlan {
|
|
232
349
|
validateSegments(input.aroll);
|
|
233
350
|
const outputDurationMs = timelineDuration(input.aroll);
|
|
234
|
-
validateNeeds(input.needs, outputDurationMs);
|
|
351
|
+
validateNeeds(input.needs, outputDurationMs, input.aroll);
|
|
235
352
|
const shortGapMs = boundedMilliseconds(input.shortGapMs, 500, "shortGapMs", 2_000);
|
|
236
353
|
const cutCoverBeforeMs = boundedMilliseconds(input.cutCoverBeforeMs, 250, "cutCoverBeforeMs", 2_000);
|
|
237
354
|
const cutCoverAfterMs = boundedMilliseconds(input.cutCoverAfterMs, 500, "cutCoverAfterMs", 2_000);
|
|
@@ -239,6 +356,7 @@ export function planBrollContinuity(input: PlanBrollContinuityInput): BrollConti
|
|
|
239
356
|
throw new Error("cutCoverBeforeMs and cutCoverAfterMs cannot both be zero");
|
|
240
357
|
}
|
|
241
358
|
const bridged = bridgeShortGaps(input.needs, shortGapMs);
|
|
359
|
+
const coverage = summarizeBrollCoverage(bridged.needs, outputDurationMs);
|
|
242
360
|
return {
|
|
243
361
|
outputDurationMs,
|
|
244
362
|
shortGapMs,
|
|
@@ -246,6 +364,7 @@ export function planBrollContinuity(input: PlanBrollContinuityInput): BrollConti
|
|
|
246
364
|
cutCoverAfterMs,
|
|
247
365
|
needs: bridged.needs,
|
|
248
366
|
bridges: bridged.bridges,
|
|
367
|
+
coverage,
|
|
249
368
|
jumpCuts: analyzeArollJumpCuts(
|
|
250
369
|
input.aroll,
|
|
251
370
|
bridged.needs,
|
|
@@ -256,7 +375,9 @@ export function planBrollContinuity(input: PlanBrollContinuityInput): BrollConti
|
|
|
256
375
|
};
|
|
257
376
|
}
|
|
258
377
|
|
|
259
|
-
|
|
378
|
+
type BrollContinuityPlanReceiptPayload = Omit<BrollContinuityPlanReceipt, "planSha256">;
|
|
379
|
+
|
|
380
|
+
function planReceiptPayload(receipt: BrollContinuityPlanReceiptPayload): string {
|
|
260
381
|
return canonicalJson(receipt);
|
|
261
382
|
}
|
|
262
383
|
|
|
@@ -279,7 +400,11 @@ function arollSha256(aroll: ArollSegment[]): string {
|
|
|
279
400
|
return sha256(canonicalJson(aroll));
|
|
280
401
|
}
|
|
281
402
|
|
|
282
|
-
function plannedRanges(needs: BrollNeed
|
|
403
|
+
function plannedRanges(needs: Array<Pick<BrollNeed, "id" | "outputStartMs" | "outputEndMs">>): Array<{
|
|
404
|
+
id: string;
|
|
405
|
+
outputStartMs: number;
|
|
406
|
+
outputEndMs: number;
|
|
407
|
+
}> {
|
|
283
408
|
return needs.map((need) => ({
|
|
284
409
|
id: need.id,
|
|
285
410
|
outputStartMs: need.outputStartMs,
|
|
@@ -287,21 +412,51 @@ function plannedRanges(needs: BrollNeed[]): BrollContinuityPlanReceipt["needs"]
|
|
|
287
412
|
}));
|
|
288
413
|
}
|
|
289
414
|
|
|
415
|
+
function plannedNeedsV2(needs: BrollNeed[]): BrollContinuityPlanReceiptV2["needs"] {
|
|
416
|
+
return needs.map((need) => {
|
|
417
|
+
const range = {
|
|
418
|
+
id: need.id,
|
|
419
|
+
outputStartMs: need.outputStartMs,
|
|
420
|
+
outputEndMs: need.outputEndMs,
|
|
421
|
+
};
|
|
422
|
+
if (need.purpose !== "mask-cut") return { ...range, purpose: need.purpose };
|
|
423
|
+
const artifactSha256 = need.visualReview.artifactSha256;
|
|
424
|
+
if (artifactSha256 === undefined) {
|
|
425
|
+
throw new Error(`B-roll mask-cut need ${need.id} requires a verified visual review artifact`);
|
|
426
|
+
}
|
|
427
|
+
return {
|
|
428
|
+
...range,
|
|
429
|
+
purpose: "mask-cut",
|
|
430
|
+
visualReview: {
|
|
431
|
+
...structuredClone(need.visualReview),
|
|
432
|
+
artifactSha256,
|
|
433
|
+
},
|
|
434
|
+
};
|
|
435
|
+
});
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
export function normalizeArollJumpCutStatuses(jumpCuts: ArollJumpCut[]): ArollJumpCut[] {
|
|
439
|
+
return jumpCuts.map((jumpCut) => ({
|
|
440
|
+
...jumpCut,
|
|
441
|
+
status: jumpCut.status === "needs-broll" ? "review" : jumpCut.status,
|
|
442
|
+
}));
|
|
443
|
+
}
|
|
444
|
+
|
|
290
445
|
export function createBrollContinuityPlanReceipt(
|
|
291
446
|
projectId: string,
|
|
292
447
|
revision: number,
|
|
293
448
|
aroll: ArollSegment[],
|
|
294
449
|
plan: BrollContinuityPlan,
|
|
295
|
-
):
|
|
296
|
-
const payload: Omit<
|
|
297
|
-
schemaVersion:
|
|
450
|
+
): BrollContinuityPlanReceiptV2 {
|
|
451
|
+
const payload: Omit<BrollContinuityPlanReceiptV2, "planSha256"> = {
|
|
452
|
+
schemaVersion: 2,
|
|
298
453
|
projectId,
|
|
299
454
|
revision,
|
|
300
455
|
arollSha256: arollSha256(aroll),
|
|
301
456
|
shortGapMs: plan.shortGapMs,
|
|
302
457
|
cutCoverBeforeMs: plan.cutCoverBeforeMs,
|
|
303
458
|
cutCoverAfterMs: plan.cutCoverAfterMs,
|
|
304
|
-
needs:
|
|
459
|
+
needs: plannedNeedsV2(plan.needs),
|
|
305
460
|
jumpCuts: structuredClone(plan.jumpCuts),
|
|
306
461
|
};
|
|
307
462
|
return { ...payload, planSha256: sha256(planReceiptPayload(payload)) };
|
|
@@ -314,7 +469,8 @@ export function verifyBrollContinuityPlanReceipt(
|
|
|
314
469
|
placements: BrollOutputWindow[],
|
|
315
470
|
receipt: BrollContinuityPlanReceipt,
|
|
316
471
|
): void {
|
|
317
|
-
if (receipt.schemaVersion !== 1
|
|
472
|
+
if ((receipt.schemaVersion !== 1 && receipt.schemaVersion !== 2)
|
|
473
|
+
|| receipt.projectId !== projectId || receipt.revision !== revision) {
|
|
318
474
|
throw new Error("B-roll continuity receipt does not match the current project revision");
|
|
319
475
|
}
|
|
320
476
|
if (receipt.arollSha256 !== arollSha256(aroll)) {
|
|
@@ -324,13 +480,20 @@ export function verifyBrollContinuityPlanReceipt(
|
|
|
324
480
|
if (planSha256 !== sha256(planReceiptPayload(payload))) {
|
|
325
481
|
throw new Error("B-roll continuity receipt hash is invalid");
|
|
326
482
|
}
|
|
327
|
-
const dummyNeeds: BrollNeed[] = receipt.
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
483
|
+
const dummyNeeds: BrollNeed[] = receipt.schemaVersion === 1
|
|
484
|
+
? receipt.needs.map((need) => ({
|
|
485
|
+
...need,
|
|
486
|
+
speechText: "continuity receipt",
|
|
487
|
+
purpose: "demonstrate",
|
|
488
|
+
searchTerms: ["continuity"],
|
|
489
|
+
reason: "continuity receipt verification",
|
|
490
|
+
}))
|
|
491
|
+
: receipt.needs.map((need) => ({
|
|
492
|
+
...need,
|
|
493
|
+
speechText: "continuity receipt",
|
|
494
|
+
searchTerms: ["continuity"],
|
|
495
|
+
reason: "continuity receipt verification",
|
|
496
|
+
}));
|
|
334
497
|
const recomputed = planBrollContinuity({
|
|
335
498
|
aroll,
|
|
336
499
|
needs: dummyNeeds,
|
|
@@ -338,8 +501,11 @@ export function verifyBrollContinuityPlanReceipt(
|
|
|
338
501
|
cutCoverBeforeMs: receipt.cutCoverBeforeMs,
|
|
339
502
|
cutCoverAfterMs: receipt.cutCoverAfterMs,
|
|
340
503
|
});
|
|
341
|
-
|
|
342
|
-
|
|
504
|
+
const recomputedNeeds = receipt.schemaVersion === 1
|
|
505
|
+
? plannedRanges(recomputed.needs)
|
|
506
|
+
: plannedNeedsV2(recomputed.needs);
|
|
507
|
+
if (canonicalJson(recomputedNeeds) !== canonicalJson(receipt.needs)
|
|
508
|
+
|| canonicalJson(normalizeArollJumpCutStatuses(recomputed.jumpCuts)) !== canonicalJson(normalizeArollJumpCutStatuses(receipt.jumpCuts))) {
|
|
343
509
|
throw new Error("B-roll continuity receipt does not match the recomputed plan");
|
|
344
510
|
}
|
|
345
511
|
const actualRanges = [...placements].sort((left, right) => (
|
|
@@ -349,7 +515,7 @@ export function verifyBrollContinuityPlanReceipt(
|
|
|
349
515
|
outputStartMs: placement.outputStartMs,
|
|
350
516
|
outputEndMs: placement.outputEndMs,
|
|
351
517
|
}));
|
|
352
|
-
if (canonicalJson(actualRanges) !== canonicalJson(receipt.needs)) {
|
|
518
|
+
if (canonicalJson(actualRanges) !== canonicalJson(plannedRanges(receipt.needs))) {
|
|
353
519
|
throw new Error("B-roll placements do not match the planned ranges");
|
|
354
520
|
}
|
|
355
521
|
}
|
package/src/contracts.ts
CHANGED
|
@@ -121,16 +121,32 @@ export type BrollPlacementInput = Omit<
|
|
|
121
121
|
|
|
122
122
|
export type BrollPurpose = "demonstrate" | "explain" | "evidence" | "establish" | "transition" | "mask-cut";
|
|
123
123
|
|
|
124
|
-
export interface
|
|
124
|
+
export interface BrollVisualReview {
|
|
125
|
+
decision: "mask-with-broll";
|
|
126
|
+
reviewedJumpCutOutputMs: number;
|
|
127
|
+
artifactPath: string;
|
|
128
|
+
artifactSha256?: string;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
interface BrollNeedBase {
|
|
125
132
|
id: string;
|
|
126
133
|
outputStartMs: number;
|
|
127
134
|
outputEndMs: number;
|
|
128
135
|
speechText: string;
|
|
129
|
-
purpose: BrollPurpose;
|
|
130
136
|
searchTerms: string[];
|
|
131
137
|
reason: string;
|
|
132
138
|
}
|
|
133
139
|
|
|
140
|
+
export type BrollNeed =
|
|
141
|
+
| BrollNeedBase & {
|
|
142
|
+
purpose: Exclude<BrollPurpose, "mask-cut">;
|
|
143
|
+
visualReview?: never;
|
|
144
|
+
}
|
|
145
|
+
| BrollNeedBase & {
|
|
146
|
+
purpose: "mask-cut";
|
|
147
|
+
visualReview: BrollVisualReview;
|
|
148
|
+
};
|
|
149
|
+
|
|
134
150
|
export interface BrollAssetCandidate {
|
|
135
151
|
assetPath: string;
|
|
136
152
|
fileName: string;
|
|
@@ -182,7 +198,27 @@ export interface ArollJumpCut {
|
|
|
182
198
|
suggestedOutputStartMs: number;
|
|
183
199
|
suggestedOutputEndMs: number;
|
|
184
200
|
coveredByNeedIds: string[];
|
|
185
|
-
status: "covered" | "needs-broll";
|
|
201
|
+
status: "covered" | "review" | "needs-broll";
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
export interface BrollCoverageSummary {
|
|
205
|
+
brollDurationMs: number;
|
|
206
|
+
arollDurationMs: number;
|
|
207
|
+
brollCoverageRatio: number;
|
|
208
|
+
longestContinuousBrollMs: number;
|
|
209
|
+
/** @deprecated Compatibility alias for v0.1.5 clients; these values are review thresholds, not hard limits. */
|
|
210
|
+
limits: {
|
|
211
|
+
maxBrollCoverageRatio: number;
|
|
212
|
+
maxContinuousBrollMs: number;
|
|
213
|
+
};
|
|
214
|
+
reviewThresholds?: {
|
|
215
|
+
highBrollCoverageRatio: number;
|
|
216
|
+
longContinuousBrollMs: number;
|
|
217
|
+
};
|
|
218
|
+
warnings?: Array<{
|
|
219
|
+
code: "high-total-coverage" | "long-continuous-run";
|
|
220
|
+
message: string;
|
|
221
|
+
}>;
|
|
186
222
|
}
|
|
187
223
|
|
|
188
224
|
export interface BrollContinuityPlan {
|
|
@@ -193,21 +229,39 @@ export interface BrollContinuityPlan {
|
|
|
193
229
|
needs: BrollNeed[];
|
|
194
230
|
bridges: BrollContinuityBridge[];
|
|
195
231
|
jumpCuts: ArollJumpCut[];
|
|
232
|
+
coverage: BrollCoverageSummary;
|
|
196
233
|
}
|
|
197
234
|
|
|
198
|
-
|
|
199
|
-
schemaVersion: 1;
|
|
235
|
+
interface BrollContinuityPlanReceiptBase {
|
|
200
236
|
projectId: string;
|
|
201
237
|
revision: number;
|
|
202
238
|
arollSha256: string;
|
|
203
239
|
shortGapMs: number;
|
|
204
240
|
cutCoverBeforeMs: number;
|
|
205
241
|
cutCoverAfterMs: number;
|
|
206
|
-
needs: Array<{ id: string; outputStartMs: number; outputEndMs: number }>;
|
|
207
242
|
jumpCuts: ArollJumpCut[];
|
|
208
243
|
planSha256: string;
|
|
209
244
|
}
|
|
210
245
|
|
|
246
|
+
export interface BrollContinuityPlanReceiptV1 extends BrollContinuityPlanReceiptBase {
|
|
247
|
+
schemaVersion: 1;
|
|
248
|
+
needs: Array<{ id: string; outputStartMs: number; outputEndMs: number }>;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
export interface BrollContinuityPlanReceiptV2 extends BrollContinuityPlanReceiptBase {
|
|
252
|
+
schemaVersion: 2;
|
|
253
|
+
needs: Array<({
|
|
254
|
+
id: string;
|
|
255
|
+
outputStartMs: number;
|
|
256
|
+
outputEndMs: number;
|
|
257
|
+
} & (
|
|
258
|
+
| { purpose: Exclude<BrollPurpose, "mask-cut">; visualReview?: never }
|
|
259
|
+
| { purpose: "mask-cut"; visualReview: BrollVisualReview & { artifactSha256: string } }
|
|
260
|
+
))>;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
export type BrollContinuityPlanReceipt = BrollContinuityPlanReceiptV1 | BrollContinuityPlanReceiptV2;
|
|
264
|
+
|
|
211
265
|
export interface TranscriptAnalysis {
|
|
212
266
|
schemaVersion: 2;
|
|
213
267
|
text: string;
|
|
@@ -244,7 +298,7 @@ export interface TalkingHeadProject {
|
|
|
244
298
|
}
|
|
245
299
|
|
|
246
300
|
export interface TalkingHeadSnapshot {
|
|
247
|
-
schemaVersion: 1;
|
|
301
|
+
schemaVersion: 1 | 2;
|
|
248
302
|
projectId: string;
|
|
249
303
|
revision: number;
|
|
250
304
|
parentRevision: number | null;
|
|
@@ -254,6 +308,7 @@ export interface TalkingHeadSnapshot {
|
|
|
254
308
|
broll: BrollPlacement[];
|
|
255
309
|
continuityPlanReceipt?: BrollContinuityPlanReceipt;
|
|
256
310
|
jumpCuts?: ArollJumpCut[];
|
|
311
|
+
brollCoverage?: BrollCoverageSummary;
|
|
257
312
|
outputDurationMs: number;
|
|
258
313
|
}
|
|
259
314
|
|
package/src/project.ts
CHANGED
|
@@ -22,6 +22,8 @@ import {
|
|
|
22
22
|
analyzeArollJumpCuts,
|
|
23
23
|
findOverlappingBrollWindows,
|
|
24
24
|
findShortArollFlashGaps,
|
|
25
|
+
normalizeArollJumpCutStatuses,
|
|
26
|
+
summarizeBrollCoverage,
|
|
25
27
|
verifyBrollContinuityPlanReceipt,
|
|
26
28
|
} from "./continuity.ts";
|
|
27
29
|
import { analyzeTranscript, DEFAULT_POLICY, timelineDuration } from "./transcript.ts";
|
|
@@ -82,6 +84,39 @@ function uniqueIds(values: Array<{ id: string }>, label: string): void {
|
|
|
82
84
|
}
|
|
83
85
|
}
|
|
84
86
|
|
|
87
|
+
function assertDistinctBrollSourceWindows(broll: BrollPlacementInput[]): void {
|
|
88
|
+
for (let leftIndex = 0; leftIndex < broll.length; leftIndex += 1) {
|
|
89
|
+
const left = broll[leftIndex]!;
|
|
90
|
+
const leftEndMs = left.assetStartMs + (left.outputEndMs - left.outputStartMs);
|
|
91
|
+
for (let rightIndex = leftIndex + 1; rightIndex < broll.length; rightIndex += 1) {
|
|
92
|
+
const right = broll[rightIndex]!;
|
|
93
|
+
if (left.selectionReceipt.assetSha256 !== right.selectionReceipt.assetSha256) continue;
|
|
94
|
+
const rightEndMs = right.assetStartMs + (right.outputEndMs - right.outputStartMs);
|
|
95
|
+
if (left.assetStartMs === right.assetStartMs && leftEndMs === rightEndMs) {
|
|
96
|
+
throw new Error(`B-roll placement ${right.id} reuses the same B-roll source window as ${left.id}; select a different source moment or angle`);
|
|
97
|
+
}
|
|
98
|
+
if (Math.max(left.assetStartMs, right.assetStartMs) < Math.min(leftEndMs, rightEndMs)) {
|
|
99
|
+
throw new Error(`B-roll placement ${right.id} overlaps the B-roll source window used by ${left.id}; select non-repeating frames`);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
async function assertVisualReviewArtifactsUnchanged(
|
|
106
|
+
cwd: string,
|
|
107
|
+
receipt: BrollContinuityPlanReceipt,
|
|
108
|
+
signal?: AbortSignal,
|
|
109
|
+
): Promise<void> {
|
|
110
|
+
if (receipt.schemaVersion !== 2) return;
|
|
111
|
+
for (const need of receipt.needs) {
|
|
112
|
+
if (!need.visualReview) continue;
|
|
113
|
+
const artifact = await snapshotFile(cwd, need.visualReview.artifactPath, signal);
|
|
114
|
+
if (artifact.sha256 !== need.visualReview.artifactSha256) {
|
|
115
|
+
throw new Error(`B-roll visual review artifact changed after continuity planning: ${need.visualReview.artifactPath}`);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
85
120
|
async function validateTimeline(
|
|
86
121
|
cwd: string,
|
|
87
122
|
aroll: ArollSegment[],
|
|
@@ -90,7 +125,12 @@ async function validateTimeline(
|
|
|
90
125
|
revision: number,
|
|
91
126
|
continuityPlanReceipt: BrollContinuityPlanReceipt | undefined,
|
|
92
127
|
signal?: AbortSignal,
|
|
93
|
-
): Promise<{
|
|
128
|
+
): Promise<{
|
|
129
|
+
outputDurationMs: number;
|
|
130
|
+
broll: BrollPlacement[];
|
|
131
|
+
brollCoverage: ReturnType<typeof summarizeBrollCoverage>;
|
|
132
|
+
jumpCuts: ReturnType<typeof analyzeArollJumpCuts>;
|
|
133
|
+
}> {
|
|
94
134
|
if (aroll.length === 0 || aroll.length > 1_000) throw new Error("A-roll must contain 1-1000 segments");
|
|
95
135
|
uniqueIds(aroll, "A-roll segment");
|
|
96
136
|
for (const segment of aroll) {
|
|
@@ -124,11 +164,14 @@ async function validateTimeline(
|
|
|
124
164
|
if (flashGap) {
|
|
125
165
|
throw new Error(`B-roll placements ${flashGap.fromBrollId} and ${flashGap.toBrollId} leave a brief ${flashGap.durationMs}ms A-roll flash; run talking_head_broll_plan and reselect the bridged window`);
|
|
126
166
|
}
|
|
167
|
+
assertDistinctBrollSourceWindows(broll);
|
|
168
|
+
const brollCoverage = summarizeBrollCoverage(broll, outputDurationMs);
|
|
127
169
|
if (broll.length > 0 && !continuityPlanReceipt) {
|
|
128
170
|
throw new Error("B-roll continuityPlanReceipt is required; run talking_head_broll_plan first");
|
|
129
171
|
}
|
|
130
172
|
if (continuityPlanReceipt) {
|
|
131
173
|
verifyBrollContinuityPlanReceipt(projectId, revision, aroll, broll, continuityPlanReceipt);
|
|
174
|
+
await assertVisualReviewArtifactsUnchanged(cwd, continuityPlanReceipt, signal);
|
|
132
175
|
}
|
|
133
176
|
const normalizedBroll: BrollPlacement[] = [];
|
|
134
177
|
for (const placement of broll) {
|
|
@@ -144,8 +187,9 @@ async function validateTimeline(
|
|
|
144
187
|
return {
|
|
145
188
|
outputDurationMs,
|
|
146
189
|
broll: normalizedBroll,
|
|
190
|
+
brollCoverage,
|
|
147
191
|
jumpCuts: continuityPlanReceipt
|
|
148
|
-
?
|
|
192
|
+
? normalizeArollJumpCutStatuses(continuityPlanReceipt.jumpCuts)
|
|
149
193
|
: analyzeArollJumpCuts(aroll, broll, outputDurationMs, 250, 500),
|
|
150
194
|
};
|
|
151
195
|
}
|
|
@@ -206,7 +250,7 @@ export async function createTalkingHeadProject(cwd: string, input: CreateTalking
|
|
|
206
250
|
updatedAt: now,
|
|
207
251
|
};
|
|
208
252
|
const snapshot: TalkingHeadSnapshot = {
|
|
209
|
-
schemaVersion:
|
|
253
|
+
schemaVersion: 2,
|
|
210
254
|
projectId: input.projectId,
|
|
211
255
|
revision: 1,
|
|
212
256
|
parentRevision: null,
|
|
@@ -214,6 +258,7 @@ export async function createTalkingHeadProject(cwd: string, input: CreateTalking
|
|
|
214
258
|
policy,
|
|
215
259
|
aroll: analysis.segments,
|
|
216
260
|
broll: [],
|
|
261
|
+
brollCoverage: summarizeBrollCoverage([], analysis.outputDurationMs),
|
|
217
262
|
jumpCuts: analyzeArollJumpCuts(analysis.segments, [], analysis.outputDurationMs, 250, 500),
|
|
218
263
|
outputDurationMs: analysis.outputDurationMs,
|
|
219
264
|
};
|
|
@@ -250,7 +295,8 @@ export async function getTalkingHeadProject(cwd: string, projectId: string, revi
|
|
|
250
295
|
join(directory, "snapshots", `${selectedRevision}.json`),
|
|
251
296
|
`talking-head snapshot ${projectId}@${selectedRevision}`,
|
|
252
297
|
);
|
|
253
|
-
if (snapshot.schemaVersion !== 1
|
|
298
|
+
if ((snapshot.schemaVersion !== 1 && snapshot.schemaVersion !== 2)
|
|
299
|
+
|| snapshot.projectId !== projectId || snapshot.revision !== selectedRevision) {
|
|
254
300
|
throw new Error(`Invalid talking-head snapshot: ${projectId}@${selectedRevision}`);
|
|
255
301
|
}
|
|
256
302
|
return { project, snapshot };
|
|
@@ -294,7 +340,7 @@ export async function applyTimeline(cwd: string, input: ApplyTimelineInput, sign
|
|
|
294
340
|
const revision = project.currentRevision + 1;
|
|
295
341
|
const now = new Date().toISOString();
|
|
296
342
|
const snapshot: TalkingHeadSnapshot = {
|
|
297
|
-
schemaVersion:
|
|
343
|
+
schemaVersion: 2,
|
|
298
344
|
projectId: input.projectId,
|
|
299
345
|
revision,
|
|
300
346
|
parentRevision: current.revision,
|
|
@@ -302,6 +348,7 @@ export async function applyTimeline(cwd: string, input: ApplyTimelineInput, sign
|
|
|
302
348
|
policy: current.policy,
|
|
303
349
|
aroll: structuredClone(input.aroll),
|
|
304
350
|
broll: validated.broll,
|
|
351
|
+
brollCoverage: validated.brollCoverage,
|
|
305
352
|
jumpCuts: validated.jumpCuts,
|
|
306
353
|
...(input.continuityPlanReceipt === undefined
|
|
307
354
|
? {}
|