@speclip/pi-talking-head 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -23
- package/extensions/talking-head/index.ts +142 -10
- package/package.json +1 -1
- package/prompts/edit-talking-head.md +1 -1
- package/skills/talking-head-edit/SKILL.md +13 -4
- package/skills/talking-head-edit/references/cut-craft.md +11 -0
- package/src/broll.ts +406 -0
- package/src/contracts.ts +119 -1
- package/src/project.ts +73 -6
- package/src/transcript.ts +198 -6
- package/src/workspace.ts +18 -4
package/src/broll.ts
ADDED
|
@@ -0,0 +1,406 @@
|
|
|
1
|
+
import { opendir, readFile, stat } from "node:fs/promises";
|
|
2
|
+
import { basename, dirname, extname, join, relative, sep } from "node:path";
|
|
3
|
+
import type {
|
|
4
|
+
BrollAssetCandidate,
|
|
5
|
+
BrollAssetMatch,
|
|
6
|
+
BrollNeed,
|
|
7
|
+
BrollPlacementInput,
|
|
8
|
+
BrollSelectionReceipt,
|
|
9
|
+
BrollWindowSelection,
|
|
10
|
+
FileRef,
|
|
11
|
+
} from "./contracts.ts";
|
|
12
|
+
import { resolveExistingWorkspaceDirectory, resolveExistingWorkspaceFile, snapshotFile, workspaceRoot } from "./workspace.ts";
|
|
13
|
+
|
|
14
|
+
const VIDEO_EXTENSIONS = new Set([".mp4", ".mov", ".m4v", ".webm", ".mkv", ".avi"]);
|
|
15
|
+
|
|
16
|
+
export interface RankBrollAssetsInput {
|
|
17
|
+
need: BrollNeed;
|
|
18
|
+
assetPaths: string[];
|
|
19
|
+
maxCandidates?: number;
|
|
20
|
+
candidateOffset?: number;
|
|
21
|
+
completeInventory?: boolean;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export interface MatchWorkspaceBrollAssetsInput {
|
|
25
|
+
assetDirectory: string;
|
|
26
|
+
recursive?: boolean;
|
|
27
|
+
maxFiles?: number;
|
|
28
|
+
maxEntries?: number;
|
|
29
|
+
maxDepth?: number;
|
|
30
|
+
maxCandidates?: number;
|
|
31
|
+
candidateOffset?: number;
|
|
32
|
+
need: BrollNeed;
|
|
33
|
+
signal?: AbortSignal;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export type WorkspaceBrollAssetMatch = BrollAssetMatch & {
|
|
37
|
+
assetDirectory: string;
|
|
38
|
+
scannedFileCount: number;
|
|
39
|
+
scannedEntryCount: number;
|
|
40
|
+
truncated: boolean;
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
export interface SelectBrollWindowInput {
|
|
44
|
+
need: BrollNeed;
|
|
45
|
+
assetPath: string;
|
|
46
|
+
manifestPath: string;
|
|
47
|
+
selectedStartMs: number;
|
|
48
|
+
evidenceTimestampsMs: number[];
|
|
49
|
+
fit: "cover" | "contain";
|
|
50
|
+
signal?: AbortSignal;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
interface ContactSheetManifest {
|
|
54
|
+
schemaVersion: number;
|
|
55
|
+
source?: { path?: string; bytes?: number; sha256?: string };
|
|
56
|
+
range?: { startSeconds?: number; endSeconds?: number };
|
|
57
|
+
artifacts?: Array<{ timestampsSeconds?: number[]; sourceSha256?: string }>;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function normalized(value: string): string {
|
|
61
|
+
return value.normalize("NFKC").toLowerCase().replace(/[\p{P}\p{S}\s]+/gu, "");
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function normalizedTerms(terms: string[]): string[] {
|
|
65
|
+
const unique = new Set<string>();
|
|
66
|
+
for (const term of terms) {
|
|
67
|
+
const value = normalized(term);
|
|
68
|
+
if (value) unique.add(value);
|
|
69
|
+
}
|
|
70
|
+
if (unique.size === 0) throw new Error("B-roll searchTerms must contain at least one meaningful term");
|
|
71
|
+
return [...unique];
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function candidateFor(assetPath: string, terms: string[]): BrollAssetCandidate {
|
|
75
|
+
const fileName = basename(assetPath);
|
|
76
|
+
const normalizedName = normalized(fileName.replace(/\.[^.]+$/u, ""));
|
|
77
|
+
const normalizedDirectory = normalized(dirname(assetPath));
|
|
78
|
+
const matchedTerms: string[] = [];
|
|
79
|
+
let score = 0;
|
|
80
|
+
for (const term of terms) {
|
|
81
|
+
if (normalizedName.includes(term)) {
|
|
82
|
+
matchedTerms.push(term);
|
|
83
|
+
score += 4;
|
|
84
|
+
} else if (normalizedDirectory.includes(term)) {
|
|
85
|
+
matchedTerms.push(term);
|
|
86
|
+
score += 2;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
const highThreshold = Math.min(2, terms.length);
|
|
90
|
+
return {
|
|
91
|
+
assetPath,
|
|
92
|
+
fileName,
|
|
93
|
+
matchedTerms,
|
|
94
|
+
score,
|
|
95
|
+
confidence: matchedTerms.length >= highThreshold ? "high" : matchedTerms.length > 0 ? "medium" : "low",
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function validateNeed(need: BrollNeed): void {
|
|
100
|
+
if (!Number.isFinite(need.outputStartMs) || !Number.isFinite(need.outputEndMs)
|
|
101
|
+
|| need.outputStartMs < 0 || need.outputEndMs <= need.outputStartMs) {
|
|
102
|
+
throw new Error("B-roll need must contain a valid output range");
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export function rankBrollAssets(input: RankBrollAssetsInput): BrollAssetMatch {
|
|
107
|
+
validateNeed(input.need);
|
|
108
|
+
if (input.assetPaths.length === 0) throw new Error("No candidate B-roll video assets were found");
|
|
109
|
+
const terms = normalizedTerms(input.need.searchTerms);
|
|
110
|
+
const maxCandidates = input.maxCandidates ?? 5;
|
|
111
|
+
if (!Number.isInteger(maxCandidates) || maxCandidates < 1 || maxCandidates > 20) {
|
|
112
|
+
throw new Error("maxCandidates must be within 1-20");
|
|
113
|
+
}
|
|
114
|
+
const ranked = [...new Set(input.assetPaths)].map((assetPath) => candidateFor(assetPath, terms)).sort((left, right) => (
|
|
115
|
+
right.score - left.score || (left.assetPath < right.assetPath ? -1 : left.assetPath > right.assetPath ? 1 : 0)
|
|
116
|
+
));
|
|
117
|
+
const candidateOffset = input.candidateOffset ?? 0;
|
|
118
|
+
if (!Number.isInteger(candidateOffset) || candidateOffset < 0 || candidateOffset >= ranked.length) {
|
|
119
|
+
throw new Error(`candidateOffset must be within 0-${ranked.length - 1}`);
|
|
120
|
+
}
|
|
121
|
+
const top = ranked[0];
|
|
122
|
+
const second = ranked[1];
|
|
123
|
+
const direct = (input.completeInventory ?? true) && candidateOffset === 0
|
|
124
|
+
&& top?.confidence === "high" && (second === undefined || top.score > second.score);
|
|
125
|
+
if (direct) {
|
|
126
|
+
return {
|
|
127
|
+
requiredDurationMs: input.need.outputEndMs - input.need.outputStartMs,
|
|
128
|
+
totalCandidates: ranked.length,
|
|
129
|
+
candidateOffset: 0,
|
|
130
|
+
nextCandidateOffset: null,
|
|
131
|
+
selectionMode: "filename-direct",
|
|
132
|
+
shortlist: [top],
|
|
133
|
+
nextStep: {
|
|
134
|
+
action: "inspect-selected-asset",
|
|
135
|
+
reason: "One filename is the unique high-confidence match; inspect only this asset to choose the source window.",
|
|
136
|
+
},
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
const hasLexicalMatch = ranked.some((candidate) => candidate.confidence !== "low");
|
|
140
|
+
const shortlist = ranked.slice(candidateOffset, candidateOffset + maxCandidates);
|
|
141
|
+
const nextCandidateOffset = (input.completeInventory ?? true) && candidateOffset + shortlist.length < ranked.length
|
|
142
|
+
? candidateOffset + shortlist.length
|
|
143
|
+
: null;
|
|
144
|
+
const shared = {
|
|
145
|
+
requiredDurationMs: input.need.outputEndMs - input.need.outputStartMs,
|
|
146
|
+
totalCandidates: ranked.length,
|
|
147
|
+
candidateOffset,
|
|
148
|
+
nextCandidateOffset,
|
|
149
|
+
shortlist,
|
|
150
|
+
};
|
|
151
|
+
if (hasLexicalMatch) return {
|
|
152
|
+
...shared,
|
|
153
|
+
selectionMode: "filename-shortlist",
|
|
154
|
+
nextStep: {
|
|
155
|
+
action: "inspect-shortlist",
|
|
156
|
+
reason: "Filename evidence is ambiguous; visually inspect only the bounded shortlist.",
|
|
157
|
+
},
|
|
158
|
+
};
|
|
159
|
+
return {
|
|
160
|
+
...shared,
|
|
161
|
+
selectionMode: "visual-fallback",
|
|
162
|
+
nextStep: {
|
|
163
|
+
action: "visual-fallback",
|
|
164
|
+
reason: "Filenames provide no useful evidence; use low-cost visual screening on the bounded shortlist.",
|
|
165
|
+
},
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
interface ScanState {
|
|
170
|
+
paths: string[];
|
|
171
|
+
visitedEntries: number;
|
|
172
|
+
truncated: boolean;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
interface ScanLimits {
|
|
176
|
+
recursive: boolean;
|
|
177
|
+
maxFiles: number;
|
|
178
|
+
maxEntries: number;
|
|
179
|
+
maxDepth: number;
|
|
180
|
+
signal?: AbortSignal;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
function codePointCompare(left: string, right: string): number {
|
|
184
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
async function collectVideoPaths(directory: string, depth: number, limits: ScanLimits, state: ScanState): Promise<void> {
|
|
188
|
+
limits.signal?.throwIfAborted();
|
|
189
|
+
const entries = [];
|
|
190
|
+
const handle = await opendir(directory);
|
|
191
|
+
for await (const entry of handle) {
|
|
192
|
+
limits.signal?.throwIfAborted();
|
|
193
|
+
if (state.visitedEntries >= limits.maxEntries) {
|
|
194
|
+
state.truncated = true;
|
|
195
|
+
break;
|
|
196
|
+
}
|
|
197
|
+
state.visitedEntries += 1;
|
|
198
|
+
entries.push(entry);
|
|
199
|
+
}
|
|
200
|
+
entries.sort((left, right) => codePointCompare(left.name, right.name));
|
|
201
|
+
for (const entry of entries) {
|
|
202
|
+
limits.signal?.throwIfAborted();
|
|
203
|
+
if (state.paths.length > limits.maxFiles) {
|
|
204
|
+
state.truncated = true;
|
|
205
|
+
return;
|
|
206
|
+
}
|
|
207
|
+
if (entry.isSymbolicLink()) continue;
|
|
208
|
+
const absolute = join(directory, entry.name);
|
|
209
|
+
if (entry.isDirectory()) {
|
|
210
|
+
if (limits.recursive) {
|
|
211
|
+
if (depth >= limits.maxDepth) state.truncated = true;
|
|
212
|
+
else await collectVideoPaths(absolute, depth + 1, limits, state);
|
|
213
|
+
}
|
|
214
|
+
continue;
|
|
215
|
+
}
|
|
216
|
+
if (entry.isFile() && VIDEO_EXTENSIONS.has(extname(entry.name).toLowerCase())) state.paths.push(absolute);
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
export async function matchWorkspaceBrollAssets(
|
|
221
|
+
cwd: string,
|
|
222
|
+
input: MatchWorkspaceBrollAssetsInput,
|
|
223
|
+
): Promise<WorkspaceBrollAssetMatch> {
|
|
224
|
+
const maxFiles = input.maxFiles ?? 1_000;
|
|
225
|
+
if (!Number.isInteger(maxFiles) || maxFiles < 1 || maxFiles > 5_000) {
|
|
226
|
+
throw new Error("maxFiles must be within 1-5000");
|
|
227
|
+
}
|
|
228
|
+
const maxEntries = input.maxEntries ?? 20_000;
|
|
229
|
+
if (!Number.isInteger(maxEntries) || maxEntries < 1 || maxEntries > 100_000) {
|
|
230
|
+
throw new Error("maxEntries must be within 1-100000");
|
|
231
|
+
}
|
|
232
|
+
const maxDepth = input.maxDepth ?? 12;
|
|
233
|
+
if (!Number.isInteger(maxDepth) || maxDepth < 0 || maxDepth > 50) {
|
|
234
|
+
throw new Error("maxDepth must be within 0-50");
|
|
235
|
+
}
|
|
236
|
+
const directory = await resolveExistingWorkspaceDirectory(cwd, input.assetDirectory);
|
|
237
|
+
const state: ScanState = { paths: [], visitedEntries: 0, truncated: false };
|
|
238
|
+
await collectVideoPaths(directory, 0, {
|
|
239
|
+
recursive: input.recursive ?? true,
|
|
240
|
+
maxFiles,
|
|
241
|
+
maxEntries,
|
|
242
|
+
maxDepth,
|
|
243
|
+
...(input.signal === undefined ? {} : { signal: input.signal }),
|
|
244
|
+
}, state);
|
|
245
|
+
const truncated = state.truncated || state.paths.length > maxFiles;
|
|
246
|
+
if (truncated && (input.candidateOffset ?? 0) !== 0) {
|
|
247
|
+
throw new Error("candidateOffset cannot be reused after a truncated scan; narrow the asset directory and rescan");
|
|
248
|
+
}
|
|
249
|
+
const included = state.paths.slice(0, maxFiles);
|
|
250
|
+
const root = await workspaceRoot(cwd);
|
|
251
|
+
const toWorkspacePath = (path: string) => relative(root, path).split(sep).join("/");
|
|
252
|
+
const assetPaths = included.map(toWorkspacePath);
|
|
253
|
+
return {
|
|
254
|
+
assetDirectory: toWorkspacePath(directory),
|
|
255
|
+
scannedFileCount: included.length,
|
|
256
|
+
scannedEntryCount: state.visitedEntries,
|
|
257
|
+
truncated,
|
|
258
|
+
...rankBrollAssets({
|
|
259
|
+
need: input.need,
|
|
260
|
+
assetPaths,
|
|
261
|
+
...(input.maxCandidates === undefined ? {} : { maxCandidates: input.maxCandidates }),
|
|
262
|
+
...(input.candidateOffset === undefined ? {} : { candidateOffset: input.candidateOffset }),
|
|
263
|
+
completeInventory: !truncated,
|
|
264
|
+
}),
|
|
265
|
+
};
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
function parseContactSheetManifest(payload: string, manifestPath: string): ContactSheetManifest {
|
|
269
|
+
try {
|
|
270
|
+
return JSON.parse(payload) as ContactSheetManifest;
|
|
271
|
+
} catch (error) {
|
|
272
|
+
throw new Error(`Invalid contact-sheet manifest ${manifestPath}: ${(error as Error).message}`);
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
interface SelectionWindow {
|
|
277
|
+
assetStartMs: number;
|
|
278
|
+
outputStartMs: number;
|
|
279
|
+
outputEndMs: number;
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
function validateManifestSelection(
|
|
283
|
+
asset: FileRef,
|
|
284
|
+
manifest: FileRef,
|
|
285
|
+
payload: ContactSheetManifest,
|
|
286
|
+
window: SelectionWindow,
|
|
287
|
+
receipt: BrollSelectionReceipt,
|
|
288
|
+
): void {
|
|
289
|
+
if (payload.schemaVersion !== 2 || !payload.source || !payload.range || !Array.isArray(payload.artifacts)) {
|
|
290
|
+
throw new Error("Contact-sheet manifest must use pi-media schemaVersion 2");
|
|
291
|
+
}
|
|
292
|
+
if (receipt.assetSha256 !== asset.sha256 || receipt.assetBytes !== asset.bytes) {
|
|
293
|
+
throw new Error("B-roll selection receipt does not match the current asset");
|
|
294
|
+
}
|
|
295
|
+
if (receipt.manifestPath !== manifest.path || receipt.manifestSha256 !== manifest.sha256) {
|
|
296
|
+
throw new Error("B-roll selection receipt does not match the current contact-sheet manifest");
|
|
297
|
+
}
|
|
298
|
+
if (payload.source.path !== asset.path || payload.source.sha256 !== asset.sha256 || payload.source.bytes !== asset.bytes) {
|
|
299
|
+
throw new Error("Contact-sheet manifest source does not match the selected B-roll asset");
|
|
300
|
+
}
|
|
301
|
+
const requiredDurationMs = window.outputEndMs - window.outputStartMs;
|
|
302
|
+
const selectedEndMs = window.assetStartMs + requiredDurationMs;
|
|
303
|
+
if (receipt.selectedEndMs !== selectedEndMs) {
|
|
304
|
+
throw new Error("B-roll selection receipt duration does not match the output placement");
|
|
305
|
+
}
|
|
306
|
+
const rangeStartMs = Number(payload.range.startSeconds) * 1_000;
|
|
307
|
+
const rangeEndMs = Number(payload.range.endSeconds) * 1_000;
|
|
308
|
+
if (!Number.isFinite(rangeStartMs) || !Number.isFinite(rangeEndMs)
|
|
309
|
+
|| window.assetStartMs < rangeStartMs || selectedEndMs > rangeEndMs) {
|
|
310
|
+
throw new Error("Selected B-roll source window is outside the analyzed contact-sheet range");
|
|
311
|
+
}
|
|
312
|
+
const availableTimestamps = new Set<number>();
|
|
313
|
+
for (const artifact of payload.artifacts) {
|
|
314
|
+
if (artifact.sourceSha256 !== asset.sha256 || !Array.isArray(artifact.timestampsSeconds)) {
|
|
315
|
+
throw new Error("Contact-sheet artifact provenance does not match the selected B-roll asset");
|
|
316
|
+
}
|
|
317
|
+
for (const timestamp of artifact.timestampsSeconds) {
|
|
318
|
+
if (Number.isFinite(timestamp)) availableTimestamps.add(Math.round(timestamp * 1_000));
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
if (!availableTimestamps.has(window.assetStartMs)) {
|
|
322
|
+
throw new Error("Selected B-roll source window must start on a manifest timestamp");
|
|
323
|
+
}
|
|
324
|
+
const evidenceTimestampsMs = [...new Set(receipt.evidenceTimestampsMs)];
|
|
325
|
+
const minimumEvidence = requiredDurationMs >= 1_000 ? 2 : 1;
|
|
326
|
+
if (evidenceTimestampsMs.length < minimumEvidence || !evidenceTimestampsMs.includes(window.assetStartMs)
|
|
327
|
+
|| evidenceTimestampsMs.some((timestamp) => (
|
|
328
|
+
!Number.isFinite(timestamp)
|
|
329
|
+
|| !availableTimestamps.has(timestamp)
|
|
330
|
+
|| timestamp < window.assetStartMs
|
|
331
|
+
|| timestamp > selectedEndMs
|
|
332
|
+
))) {
|
|
333
|
+
throw new Error(`B-roll selection requires ${minimumEvidence} or more manifest-backed evidence timestamps, including its source start`);
|
|
334
|
+
}
|
|
335
|
+
const tailThresholdMs = selectedEndMs - Math.max(500, requiredDurationMs * 0.25);
|
|
336
|
+
if (Math.max(...evidenceTimestampsMs) < tailThresholdMs) {
|
|
337
|
+
throw new Error("B-roll selection evidence must cover the end of the source window");
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
async function readVerifiedManifest(
|
|
342
|
+
cwd: string,
|
|
343
|
+
manifestPath: string,
|
|
344
|
+
signal?: AbortSignal,
|
|
345
|
+
): Promise<{ manifest: FileRef; payload: ContactSheetManifest }> {
|
|
346
|
+
signal?.throwIfAborted();
|
|
347
|
+
const manifestAbsolute = await resolveExistingWorkspaceFile(cwd, manifestPath);
|
|
348
|
+
if ((await stat(manifestAbsolute)).size > 5 * 1024 * 1024) throw new Error("Contact-sheet manifest exceeds the 5MB limit");
|
|
349
|
+
const manifest = await snapshotFile(cwd, manifestPath, signal);
|
|
350
|
+
const payload = parseContactSheetManifest(
|
|
351
|
+
await readFile(manifestAbsolute, signal === undefined ? "utf8" : { encoding: "utf8", signal }),
|
|
352
|
+
manifest.path,
|
|
353
|
+
);
|
|
354
|
+
return { manifest, payload };
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
export async function verifyBrollPlacementSelection(
|
|
358
|
+
cwd: string,
|
|
359
|
+
placement: BrollPlacementInput,
|
|
360
|
+
signal?: AbortSignal,
|
|
361
|
+
): Promise<FileRef> {
|
|
362
|
+
if (!Number.isFinite(placement.assetStartMs) || placement.assetStartMs < 0) {
|
|
363
|
+
throw new Error(`B-roll assetStartMs is required and must be valid: ${placement.id}`);
|
|
364
|
+
}
|
|
365
|
+
if (!placement.selectionReceipt) {
|
|
366
|
+
throw new Error(`B-roll selectionReceipt is required: ${placement.id}`);
|
|
367
|
+
}
|
|
368
|
+
const asset = await snapshotFile(cwd, placement.assetPath, signal);
|
|
369
|
+
const { manifest, payload } = await readVerifiedManifest(cwd, placement.selectionReceipt.manifestPath, signal);
|
|
370
|
+
validateManifestSelection(asset, manifest, payload, placement, placement.selectionReceipt);
|
|
371
|
+
return asset;
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
export async function selectBrollWindow(cwd: string, input: SelectBrollWindowInput): Promise<BrollWindowSelection> {
|
|
375
|
+
validateNeed(input.need);
|
|
376
|
+
input.signal?.throwIfAborted();
|
|
377
|
+
if (!Number.isFinite(input.selectedStartMs) || input.selectedStartMs < 0) {
|
|
378
|
+
throw new Error("selectedStartMs must be a finite non-negative timestamp");
|
|
379
|
+
}
|
|
380
|
+
const requiredDurationMs = input.need.outputEndMs - input.need.outputStartMs;
|
|
381
|
+
const selectedEndMs = input.selectedStartMs + requiredDurationMs;
|
|
382
|
+
const evidenceTimestampsMs = [...new Set(input.evidenceTimestampsMs)];
|
|
383
|
+
const asset = await snapshotFile(cwd, input.assetPath, input.signal);
|
|
384
|
+
const { manifest, payload } = await readVerifiedManifest(cwd, input.manifestPath, input.signal);
|
|
385
|
+
const placement: BrollPlacementInput = {
|
|
386
|
+
id: input.need.id,
|
|
387
|
+
assetPath: asset.path,
|
|
388
|
+
outputStartMs: input.need.outputStartMs,
|
|
389
|
+
outputEndMs: input.need.outputEndMs,
|
|
390
|
+
assetStartMs: input.selectedStartMs,
|
|
391
|
+
fit: input.fit,
|
|
392
|
+
audio: "keep-primary",
|
|
393
|
+
query: input.need.searchTerms.join(" "),
|
|
394
|
+
reason: input.need.reason,
|
|
395
|
+
selectionReceipt: {
|
|
396
|
+
assetBytes: asset.bytes,
|
|
397
|
+
assetSha256: asset.sha256,
|
|
398
|
+
manifestPath: manifest.path,
|
|
399
|
+
manifestSha256: manifest.sha256,
|
|
400
|
+
selectedEndMs,
|
|
401
|
+
evidenceTimestampsMs,
|
|
402
|
+
},
|
|
403
|
+
};
|
|
404
|
+
validateManifestSelection(asset, manifest, payload, placement, placement.selectionReceipt);
|
|
405
|
+
return { placement };
|
|
406
|
+
}
|
package/src/contracts.ts
CHANGED
|
@@ -28,6 +28,57 @@ export interface PauseCandidate {
|
|
|
28
28
|
classification: PauseClassification;
|
|
29
29
|
beforeText: string;
|
|
30
30
|
afterText: string;
|
|
31
|
+
boundary: "within-sentence" | "between-sentences";
|
|
32
|
+
context: {
|
|
33
|
+
before: string;
|
|
34
|
+
after: string;
|
|
35
|
+
};
|
|
36
|
+
adjacentFillerIds: string[];
|
|
37
|
+
recommendation: EditorialRecommendation;
|
|
38
|
+
reasons: string[];
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export type EditorialRecommendation = "cut" | "review" | "keep";
|
|
42
|
+
export type DeliveryCue = "neutral" | "hesitation" | "emphasis" | "question";
|
|
43
|
+
|
|
44
|
+
export interface SentenceAnalysis {
|
|
45
|
+
sentenceIndex: number;
|
|
46
|
+
sentenceId: number;
|
|
47
|
+
beginMs: number;
|
|
48
|
+
endMs: number;
|
|
49
|
+
text: string;
|
|
50
|
+
wordStartIndex: number;
|
|
51
|
+
wordEndIndex: number;
|
|
52
|
+
deliveryCues: DeliveryCue[];
|
|
53
|
+
confidence: "low";
|
|
54
|
+
evidence: string[];
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export interface FillerCandidate {
|
|
58
|
+
id: string;
|
|
59
|
+
wordIndex: number;
|
|
60
|
+
sentenceIndex: number;
|
|
61
|
+
text: string;
|
|
62
|
+
startMs: number;
|
|
63
|
+
endMs: number;
|
|
64
|
+
kind: "hesitation" | "discourse";
|
|
65
|
+
matchConfidence: "exact" | "contextual";
|
|
66
|
+
recommendation: "review";
|
|
67
|
+
contextText: string;
|
|
68
|
+
reasons: string[];
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export interface RepetitionCandidate {
|
|
72
|
+
id: string;
|
|
73
|
+
sentenceIndex: number;
|
|
74
|
+
text: string;
|
|
75
|
+
firstWordIndex: number;
|
|
76
|
+
secondWordIndex: number;
|
|
77
|
+
startMs: number;
|
|
78
|
+
endMs: number;
|
|
79
|
+
recommendation: "review";
|
|
80
|
+
contextText: string;
|
|
81
|
+
reasons: string[];
|
|
31
82
|
}
|
|
32
83
|
|
|
33
84
|
export interface ArollSegment {
|
|
@@ -44,16 +95,83 @@ export interface BrollPlacement {
|
|
|
44
95
|
outputStartMs: number;
|
|
45
96
|
outputEndMs: number;
|
|
46
97
|
assetStartMs?: number;
|
|
98
|
+
selectionReceipt?: BrollSelectionReceipt;
|
|
47
99
|
fit: "cover" | "contain";
|
|
48
100
|
audio: "keep-primary";
|
|
49
101
|
query?: string;
|
|
50
102
|
reason?: string;
|
|
51
103
|
}
|
|
52
104
|
|
|
53
|
-
export
|
|
105
|
+
export interface BrollSelectionReceipt {
|
|
106
|
+
assetBytes: number;
|
|
107
|
+
assetSha256: string;
|
|
108
|
+
manifestPath: string;
|
|
109
|
+
manifestSha256: string;
|
|
110
|
+
selectedEndMs: number;
|
|
111
|
+
evidenceTimestampsMs: number[];
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
export type BrollPlacementInput = Omit<
|
|
115
|
+
BrollPlacement,
|
|
116
|
+
"assetBytes" | "assetSha256" | "assetStartMs" | "selectionReceipt"
|
|
117
|
+
> & {
|
|
118
|
+
assetStartMs: number;
|
|
119
|
+
selectionReceipt: BrollSelectionReceipt;
|
|
120
|
+
};
|
|
121
|
+
|
|
122
|
+
export type BrollPurpose = "demonstrate" | "explain" | "evidence" | "establish" | "transition" | "mask-cut";
|
|
123
|
+
|
|
124
|
+
export interface BrollNeed {
|
|
125
|
+
id: string;
|
|
126
|
+
outputStartMs: number;
|
|
127
|
+
outputEndMs: number;
|
|
128
|
+
speechText: string;
|
|
129
|
+
purpose: BrollPurpose;
|
|
130
|
+
searchTerms: string[];
|
|
131
|
+
reason: string;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export interface BrollAssetCandidate {
|
|
135
|
+
assetPath: string;
|
|
136
|
+
fileName: string;
|
|
137
|
+
matchedTerms: string[];
|
|
138
|
+
score: number;
|
|
139
|
+
confidence: "high" | "medium" | "low";
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
interface BrollAssetMatchBase {
|
|
143
|
+
requiredDurationMs: number;
|
|
144
|
+
totalCandidates: number;
|
|
145
|
+
candidateOffset: number;
|
|
146
|
+
nextCandidateOffset: number | null;
|
|
147
|
+
shortlist: BrollAssetCandidate[];
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
export type BrollAssetMatch =
|
|
151
|
+
| BrollAssetMatchBase & {
|
|
152
|
+
selectionMode: "filename-direct";
|
|
153
|
+
nextStep: { action: "inspect-selected-asset"; reason: string };
|
|
154
|
+
}
|
|
155
|
+
| BrollAssetMatchBase & {
|
|
156
|
+
selectionMode: "filename-shortlist";
|
|
157
|
+
nextStep: { action: "inspect-shortlist"; reason: string };
|
|
158
|
+
}
|
|
159
|
+
| BrollAssetMatchBase & {
|
|
160
|
+
selectionMode: "visual-fallback";
|
|
161
|
+
nextStep: { action: "visual-fallback"; reason: string };
|
|
162
|
+
};
|
|
163
|
+
|
|
164
|
+
export interface BrollWindowSelection {
|
|
165
|
+
placement: BrollPlacementInput;
|
|
166
|
+
}
|
|
54
167
|
|
|
55
168
|
export interface TranscriptAnalysis {
|
|
169
|
+
schemaVersion: 2;
|
|
170
|
+
text: string;
|
|
56
171
|
words: TranscriptWord[];
|
|
172
|
+
sentences: SentenceAnalysis[];
|
|
173
|
+
fillers: FillerCandidate[];
|
|
174
|
+
repetitions: RepetitionCandidate[];
|
|
57
175
|
candidates: PauseCandidate[];
|
|
58
176
|
segments: ArollSegment[];
|
|
59
177
|
outputDurationMs: number;
|
package/src/project.ts
CHANGED
|
@@ -5,12 +5,18 @@ import type {
|
|
|
5
5
|
ArollSegment,
|
|
6
6
|
BrollPlacement,
|
|
7
7
|
BrollPlacementInput,
|
|
8
|
+
FillerCandidate,
|
|
9
|
+
PauseCandidate,
|
|
10
|
+
RepetitionCandidate,
|
|
11
|
+
SentenceAnalysis,
|
|
8
12
|
TalkingHeadPolicy,
|
|
9
13
|
TalkingHeadProject,
|
|
10
14
|
TalkingHeadSnapshot,
|
|
11
15
|
TranscriptAnalysis,
|
|
16
|
+
TranscriptWord,
|
|
12
17
|
WordTranscript,
|
|
13
18
|
} from "./contracts.ts";
|
|
19
|
+
import { verifyBrollPlacementSelection } from "./broll.ts";
|
|
14
20
|
import { analyzeTranscript, DEFAULT_POLICY, timelineDuration } from "./transcript.ts";
|
|
15
21
|
import { resolveExistingWorkspaceFile, resolveWorkspacePath, snapshotFile, workspaceRelativePath } from "./workspace.ts";
|
|
16
22
|
|
|
@@ -72,6 +78,7 @@ async function validateTimeline(
|
|
|
72
78
|
cwd: string,
|
|
73
79
|
aroll: ArollSegment[],
|
|
74
80
|
broll: BrollPlacementInput[],
|
|
81
|
+
signal?: AbortSignal,
|
|
75
82
|
): Promise<{ outputDurationMs: number; broll: BrollPlacement[] }> {
|
|
76
83
|
if (aroll.length === 0 || aroll.length > 1_000) throw new Error("A-roll must contain 1-1000 segments");
|
|
77
84
|
uniqueIds(aroll, "A-roll segment");
|
|
@@ -86,6 +93,7 @@ async function validateTimeline(
|
|
|
86
93
|
uniqueIds(broll, "B-roll placement");
|
|
87
94
|
const normalizedBroll: BrollPlacement[] = [];
|
|
88
95
|
for (const placement of broll) {
|
|
96
|
+
signal?.throwIfAborted();
|
|
89
97
|
if (!Number.isFinite(placement.outputStartMs) || !Number.isFinite(placement.outputEndMs)
|
|
90
98
|
|| placement.outputStartMs < 0 || placement.outputEndMs <= placement.outputStartMs) {
|
|
91
99
|
throw new Error(`Invalid B-roll output range: ${placement.id}`);
|
|
@@ -93,11 +101,12 @@ async function validateTimeline(
|
|
|
93
101
|
if (placement.outputEndMs > outputDurationMs) {
|
|
94
102
|
throw new Error(`B-roll placement ${placement.id} exceeds output duration ${outputDurationMs}ms`);
|
|
95
103
|
}
|
|
96
|
-
if (
|
|
97
|
-
throw new Error(`
|
|
104
|
+
if (!Number.isFinite(placement.assetStartMs) || placement.assetStartMs < 0) {
|
|
105
|
+
throw new Error(`B-roll assetStartMs is required and must be valid: ${placement.id}`);
|
|
98
106
|
}
|
|
107
|
+
if (!placement.selectionReceipt) throw new Error(`B-roll selectionReceipt is required: ${placement.id}`);
|
|
99
108
|
if (placement.audio !== "keep-primary") throw new Error("B-roll audio must keep the primary A-roll audio");
|
|
100
|
-
const asset = await
|
|
109
|
+
const asset = await verifyBrollPlacementSelection(cwd, placement, signal);
|
|
101
110
|
normalizedBroll.push({
|
|
102
111
|
...structuredClone(placement),
|
|
103
112
|
assetPath: asset.path,
|
|
@@ -111,10 +120,15 @@ async function validateTimeline(
|
|
|
111
120
|
function summary(analysis: TranscriptAnalysis) {
|
|
112
121
|
return {
|
|
113
122
|
wordCount: analysis.words.length,
|
|
123
|
+
sentenceCount: analysis.sentences.length,
|
|
124
|
+
fillerCount: analysis.fillers.length,
|
|
125
|
+
repetitionCount: analysis.repetitions.length,
|
|
114
126
|
pauseCount: analysis.candidates.length,
|
|
115
127
|
safePauses: analysis.candidates.filter((candidate) => candidate.classification === "safe").length,
|
|
116
128
|
reviewPauses: analysis.candidates.filter((candidate) => candidate.classification === "review").length,
|
|
117
129
|
unsafePauses: analysis.candidates.filter((candidate) => candidate.classification === "unsafe").length,
|
|
130
|
+
automaticCutPauses: analysis.candidates.filter((candidate) => candidate.recommendation === "cut").length,
|
|
131
|
+
editorialReviewPauses: analysis.candidates.filter((candidate) => candidate.recommendation === "review").length,
|
|
118
132
|
defaultSegmentCount: analysis.segments.length,
|
|
119
133
|
defaultOutputDurationMs: analysis.outputDurationMs,
|
|
120
134
|
};
|
|
@@ -223,7 +237,7 @@ async function acquireLock(directory: string): Promise<() => Promise<void>> {
|
|
|
223
237
|
return async () => { await unlink(lockPath).catch(() => undefined); };
|
|
224
238
|
}
|
|
225
239
|
|
|
226
|
-
export async function applyTimeline(cwd: string, input: ApplyTimelineInput): Promise<TalkingHeadSnapshot> {
|
|
240
|
+
export async function applyTimeline(cwd: string, input: ApplyTimelineInput, signal?: AbortSignal): Promise<TalkingHeadSnapshot> {
|
|
227
241
|
const directory = await projectDirectory(cwd, input.projectId);
|
|
228
242
|
const release = await acquireLock(directory);
|
|
229
243
|
try {
|
|
@@ -234,7 +248,7 @@ export async function applyTimeline(cwd: string, input: ApplyTimelineInput): Pro
|
|
|
234
248
|
await assertProjectSourcesUnchanged(cwd, project);
|
|
235
249
|
await assertSnapshotAssetsUnchanged(cwd, current);
|
|
236
250
|
assertWordSafeSegments(input.aroll, await getAnalysis(cwd, project));
|
|
237
|
-
const validated = await validateTimeline(cwd, input.aroll, input.broll);
|
|
251
|
+
const validated = await validateTimeline(cwd, input.aroll, input.broll, signal);
|
|
238
252
|
const revision = project.currentRevision + 1;
|
|
239
253
|
const now = new Date().toISOString();
|
|
240
254
|
const snapshot: TalkingHeadSnapshot = {
|
|
@@ -261,7 +275,60 @@ export async function applyTimeline(cwd: string, input: ApplyTimelineInput): Pro
|
|
|
261
275
|
|
|
262
276
|
export async function getAnalysis(cwd: string, project: TalkingHeadProject): Promise<TranscriptAnalysis> {
|
|
263
277
|
const absolute = await resolveExistingWorkspaceFile(cwd, project.analysisPath);
|
|
264
|
-
|
|
278
|
+
const persisted = await readJson<{
|
|
279
|
+
schemaVersion?: number;
|
|
280
|
+
text?: string;
|
|
281
|
+
words: TranscriptWord[];
|
|
282
|
+
sentences?: SentenceAnalysis[];
|
|
283
|
+
fillers?: FillerCandidate[];
|
|
284
|
+
repetitions?: RepetitionCandidate[];
|
|
285
|
+
candidates: Array<Partial<PauseCandidate> & Pick<PauseCandidate, "id" | "startMs" | "endMs" | "durationMs" | "classification" | "beforeText" | "afterText">>;
|
|
286
|
+
segments: ArollSegment[];
|
|
287
|
+
outputDurationMs: number;
|
|
288
|
+
}>(absolute, `talking-head analysis ${project.projectId}`);
|
|
289
|
+
if (!Array.isArray(persisted.words) || !Array.isArray(persisted.candidates)
|
|
290
|
+
|| !Array.isArray(persisted.segments) || !Number.isFinite(persisted.outputDurationMs)) {
|
|
291
|
+
throw new Error(`Invalid talking-head analysis: ${project.projectId}`);
|
|
292
|
+
}
|
|
293
|
+
const completeV2 = persisted.schemaVersion === 2
|
|
294
|
+
&& typeof persisted.text === "string"
|
|
295
|
+
&& Array.isArray(persisted.sentences)
|
|
296
|
+
&& Array.isArray(persisted.fillers)
|
|
297
|
+
&& Array.isArray(persisted.repetitions);
|
|
298
|
+
if (!completeV2) {
|
|
299
|
+
const currentTranscript = await snapshotFile(cwd, project.transcript.path);
|
|
300
|
+
if (currentTranscript.sha256 !== project.transcript.sha256 || currentTranscript.bytes !== project.transcript.bytes) {
|
|
301
|
+
throw new Error(`Transcript changed before legacy analysis migration: ${project.transcript.path}`);
|
|
302
|
+
}
|
|
303
|
+
const transcriptAbsolute = await resolveExistingWorkspaceFile(cwd, project.transcript.path);
|
|
304
|
+
const transcriptPayload = await readJson<WordTranscript>(transcriptAbsolute, "word transcript");
|
|
305
|
+
const { snapshot } = await getTalkingHeadProject(cwd, project.projectId);
|
|
306
|
+
return analyzeTranscript(transcriptPayload, snapshot.policy);
|
|
307
|
+
}
|
|
308
|
+
return {
|
|
309
|
+
schemaVersion: 2,
|
|
310
|
+
text: persisted.text ?? "",
|
|
311
|
+
words: persisted.words,
|
|
312
|
+
sentences: persisted.sentences ?? [],
|
|
313
|
+
fillers: persisted.fillers ?? [],
|
|
314
|
+
repetitions: persisted.repetitions ?? [],
|
|
315
|
+
candidates: persisted.candidates.map((candidate) => ({
|
|
316
|
+
id: candidate.id,
|
|
317
|
+
startMs: candidate.startMs,
|
|
318
|
+
endMs: candidate.endMs,
|
|
319
|
+
durationMs: candidate.durationMs,
|
|
320
|
+
classification: candidate.classification,
|
|
321
|
+
beforeText: candidate.beforeText,
|
|
322
|
+
afterText: candidate.afterText,
|
|
323
|
+
boundary: candidate.boundary ?? "within-sentence",
|
|
324
|
+
context: candidate.context ?? { before: candidate.beforeText, after: candidate.afterText },
|
|
325
|
+
adjacentFillerIds: candidate.adjacentFillerIds ?? [],
|
|
326
|
+
recommendation: candidate.recommendation ?? "review",
|
|
327
|
+
reasons: candidate.reasons ?? ["Legacy pause analysis requires editorial review before cutting."],
|
|
328
|
+
})),
|
|
329
|
+
segments: persisted.segments,
|
|
330
|
+
outputDurationMs: persisted.outputDurationMs,
|
|
331
|
+
};
|
|
265
332
|
}
|
|
266
333
|
|
|
267
334
|
export async function assertProjectSourcesUnchanged(cwd: string, project: TalkingHeadProject): Promise<void> {
|