@morshoto/framekit 0.1.0 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -17
- package/dist-package/adapters/final-cut/typescript/src/analyzers.js +57 -0
- package/dist-package/adapters/final-cut/typescript/src/connection.js +5 -1
- package/dist-package/adapters/final-cut/typescript/src/export.js +231 -8
- package/dist-package/adapters/final-cut/typescript/src/fcpxml.js +55 -5
- package/dist-package/adapters/final-cut/typescript/src/live.js +2 -2
- package/dist-package/adapters/final-cut/typescript/src/native.js +1000 -100
- package/dist-package/adapters/final-cut/typescript/src/publisher.js +40 -5
- package/dist-package/adapters/final-cut/typescript/src/session.js +21 -5
- package/dist-package/apps/mcp-server/src/main.js +9 -1
- package/dist-package/apps/mcp-server/src/routing.js +172 -0
- package/dist-package/apps/mcp-server/src/server.js +780 -33
- package/dist-package/packages/runtime/src/application/duration-policy-service.js +6 -0
- package/dist-package/packages/runtime/src/application/media-analysis-service.js +370 -0
- package/dist-package/packages/runtime/src/application/project-service.js +127 -0
- package/dist-package/packages/runtime/src/application/runtime-options.js +1 -0
- package/dist-package/packages/runtime/src/application/transaction-store.js +12 -0
- package/dist-package/packages/runtime/src/audio/dialogue-normalization.js +64 -0
- package/dist-package/packages/runtime/src/audio/noise-reduction.js +46 -0
- package/dist-package/packages/runtime/src/capabilities.js +112 -1
- package/dist-package/packages/runtime/src/color-correction.js +48 -0
- package/dist-package/packages/runtime/src/context/context-engine.js +11 -6
- package/dist-package/packages/runtime/src/context/context-service.js +30 -0
- package/dist-package/packages/runtime/src/context/revision.js +3 -0
- package/dist-package/packages/runtime/src/domain/capabilities.js +1 -0
- package/dist-package/packages/runtime/src/domain/context.js +1 -0
- package/dist-package/packages/runtime/src/domain/diff.js +1 -0
- package/dist-package/packages/runtime/src/domain/duration.js +1 -0
- package/dist-package/packages/runtime/src/domain/editing.js +1 -0
- package/dist-package/packages/runtime/src/domain/index.js +12 -0
- package/dist-package/packages/runtime/src/domain/media.js +7 -0
- package/dist-package/packages/runtime/src/domain/ports.js +1 -0
- package/dist-package/packages/runtime/src/domain/primitives.js +1 -0
- package/dist-package/packages/runtime/src/domain/project.js +1 -0
- package/dist-package/packages/runtime/src/domain/rough-cut.js +69 -0
- package/dist-package/packages/runtime/src/domain/skills.js +33 -0
- package/dist-package/packages/runtime/src/domain/types.js +5 -1
- package/dist-package/packages/runtime/src/domain/verification.js +1 -0
- package/dist-package/packages/runtime/src/editing/dialogue-normalization-service.js +141 -0
- package/dist-package/packages/runtime/src/editing/duration-policy.js +187 -0
- package/dist-package/packages/runtime/src/editing/edit-service.js +413 -0
- package/dist-package/packages/runtime/src/editing/filler-removal-service.js +337 -0
- package/dist-package/packages/runtime/src/editing/music-service.js +85 -0
- package/dist-package/packages/runtime/src/index.js +12 -3
- package/dist-package/packages/runtime/src/planning/rough-cut.js +65 -0
- package/dist-package/packages/runtime/src/runtime.js +130 -825
- package/dist-package/packages/runtime/src/skills/builtin.js +339 -0
- package/dist-package/packages/runtime/src/skills/index.js +3 -0
- package/dist-package/packages/runtime/src/skills/registry.js +74 -0
- package/dist-package/packages/runtime/src/skills/requirements.js +170 -0
- package/dist-package/packages/runtime/src/skills/runtime.js +239 -0
- package/dist-package/packages/runtime/src/speech/filler-detector.js +132 -0
- package/dist-package/packages/runtime/src/speech/safe-cut-resolver.js +314 -0
- package/dist-package/packages/runtime/src/timeline/rational-time.js +58 -0
- package/dist-package/packages/runtime/src/verification/verification.js +712 -1
- package/dist-package/packages/testkit/src/fixture-analyzers.js +27 -0
- package/dist-package/packages/testkit/src/in-memory-editor.js +286 -3
- package/dist-package/packages/testkit/src/index.js +1 -0
- package/dist-package/packages/testkit/src/skill-fixtures.js +64 -0
- package/package.json +15 -7
- /package/dist-package/packages/runtime/src/{diff/diff.js → timeline/snapshot-diff.js} +0 -0
- /package/dist-package/packages/runtime/src/{snapshot-digest.js → timeline/snapshot-digest.js} +0 -0
|
@@ -0,0 +1,337 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { diffSnapshots } from "../timeline/snapshot-diff.js";
|
|
3
|
+
import { sameRevision } from "../context/revision.js";
|
|
4
|
+
import { translateRationalRange } from "../timeline/rational-time.js";
|
|
5
|
+
import { canonicalSnapshotDigest } from "../timeline/snapshot-digest.js";
|
|
6
|
+
import { planFillerRemoval } from "../speech/filler-removal.js";
|
|
7
|
+
export class FillerRemovalService {
|
|
8
|
+
adapter;
|
|
9
|
+
project;
|
|
10
|
+
verificationEngine;
|
|
11
|
+
options;
|
|
12
|
+
transactions;
|
|
13
|
+
previews = new Map();
|
|
14
|
+
constructor(adapter, project, verificationEngine, options, transactions) {
|
|
15
|
+
this.adapter = adapter;
|
|
16
|
+
this.project = project;
|
|
17
|
+
this.verificationEngine = verificationEngine;
|
|
18
|
+
this.options = options;
|
|
19
|
+
this.transactions = transactions;
|
|
20
|
+
}
|
|
21
|
+
async previewFillerRemoval(request) {
|
|
22
|
+
const before = await this.project.inspectProject();
|
|
23
|
+
if (!sameRevision(request.baseRevision, before.revision)) {
|
|
24
|
+
throw new Error("STALE_CONTEXT: filler preview base revision does not match current editor state");
|
|
25
|
+
}
|
|
26
|
+
await this.assertCapabilities();
|
|
27
|
+
const selectedRange = validateFillerSelection(request.range);
|
|
28
|
+
const candidates = [];
|
|
29
|
+
const analysisRangesByClip = new Map();
|
|
30
|
+
const selectedClips = before.timeline.clips.filter((clip) => clip.mediaId && rangesOverlap(clip.start, clip.start + clip.duration, selectedRange.start, selectedRange.end));
|
|
31
|
+
const selectedMediaIds = new Set();
|
|
32
|
+
for (const clip of selectedClips) {
|
|
33
|
+
if (clip.mediaId && selectedMediaIds.has(clip.mediaId)) {
|
|
34
|
+
throw new Error("CAPABILITY_UNAVAILABLE: filler removal cannot verify repeated timeline occurrences of the same media item");
|
|
35
|
+
}
|
|
36
|
+
if (clip.mediaId)
|
|
37
|
+
selectedMediaIds.add(clip.mediaId);
|
|
38
|
+
}
|
|
39
|
+
for (const clip of selectedClips) {
|
|
40
|
+
if (!clip.mediaId)
|
|
41
|
+
continue;
|
|
42
|
+
const media = before.media.find((candidate) => candidate.mediaId === clip.mediaId);
|
|
43
|
+
if (!media)
|
|
44
|
+
throw new Error(`MEDIA_NOT_FOUND: ${clip.mediaId}`);
|
|
45
|
+
const localRange = {
|
|
46
|
+
start: Math.max(0, selectedRange.start - clip.start),
|
|
47
|
+
end: Math.min(clip.duration, selectedRange.end - clip.start),
|
|
48
|
+
};
|
|
49
|
+
if (localRange.end <= localRange.start)
|
|
50
|
+
continue;
|
|
51
|
+
const speech = await this.options.speechAnalyzer.analyze({ project: before, media }, localRange);
|
|
52
|
+
analysisRangesByClip.set(clip.id, structuredClone(localRange));
|
|
53
|
+
for (const candidate of planFillerRemoval(speech.words, localRange, request)) {
|
|
54
|
+
candidates.push({
|
|
55
|
+
...candidate,
|
|
56
|
+
clipId: clip.id,
|
|
57
|
+
mediaId: clip.mediaId,
|
|
58
|
+
sourceRange: structuredClone(candidate.range),
|
|
59
|
+
range: translateRationalRange(clip.startTime, clip.start, candidate.range),
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
if (candidates.length === 0) {
|
|
64
|
+
throw new Error("NO_FILLERS_FOUND: no high-confidence filler words in selected range");
|
|
65
|
+
}
|
|
66
|
+
candidates.sort((left, right) => right.range.start - left.range.start);
|
|
67
|
+
const operations = candidates.map((candidate) => ({
|
|
68
|
+
type: "ripple-delete",
|
|
69
|
+
timelineId: before.timeline.id,
|
|
70
|
+
range: structuredClone(candidate.range),
|
|
71
|
+
reason: `remove high-confidence filler word: ${candidate.word.text}`,
|
|
72
|
+
}));
|
|
73
|
+
const previewToken = `filler-preview-${randomUUID()}`;
|
|
74
|
+
const expiresAt = new Date(this.now() + (this.options.previewTtlMs ?? 30_000)).toISOString();
|
|
75
|
+
const expectedDiff = this.adapter.previewTransaction
|
|
76
|
+
? diffSnapshots(before, await this.adapter.previewTransaction(operations, before.revision))
|
|
77
|
+
: undefined;
|
|
78
|
+
const preview = {
|
|
79
|
+
previewToken,
|
|
80
|
+
baseRevision: structuredClone(before.revision),
|
|
81
|
+
range: structuredClone(selectedRange),
|
|
82
|
+
candidates: structuredClone(candidates),
|
|
83
|
+
operations: structuredClone(operations),
|
|
84
|
+
...(expectedDiff ? { expectedDiff } : {}),
|
|
85
|
+
warnings: expectedDiff ? [] : ["The selected canonical editor has no non-mutating preview provider; the diff will be observed after execution."],
|
|
86
|
+
expiresAt,
|
|
87
|
+
};
|
|
88
|
+
this.prunePreviews();
|
|
89
|
+
const maxActivePreviews = Number.isInteger(this.options.maxActivePreviews) && this.options.maxActivePreviews > 0
|
|
90
|
+
? this.options.maxActivePreviews
|
|
91
|
+
: 128;
|
|
92
|
+
while (this.previews.size >= maxActivePreviews) {
|
|
93
|
+
const oldestToken = this.previews.keys().next().value;
|
|
94
|
+
if (oldestToken === undefined)
|
|
95
|
+
break;
|
|
96
|
+
this.previews.delete(oldestToken);
|
|
97
|
+
}
|
|
98
|
+
this.previews.set(previewToken, { preview, analysisRangesByClip });
|
|
99
|
+
return structuredClone(preview);
|
|
100
|
+
}
|
|
101
|
+
async executeFillerRemoval(previewToken) {
|
|
102
|
+
const record = this.previews.get(previewToken);
|
|
103
|
+
if (!record)
|
|
104
|
+
throw new Error(`PREVIEW_TOKEN_INVALID: unknown or already used filler preview ${previewToken}`);
|
|
105
|
+
this.previews.delete(previewToken);
|
|
106
|
+
if (this.now() > Date.parse(record.preview.expiresAt)) {
|
|
107
|
+
throw new Error("PREVIEW_TOKEN_EXPIRED: filler removal preview has expired");
|
|
108
|
+
}
|
|
109
|
+
const before = await this.project.inspectProject();
|
|
110
|
+
if (!sameRevision(record.preview.baseRevision, before.revision)) {
|
|
111
|
+
throw new Error("STALE_CONTEXT: filler preview base revision does not match current editor state");
|
|
112
|
+
}
|
|
113
|
+
await this.assertCapabilities();
|
|
114
|
+
let current = before;
|
|
115
|
+
let currentRevision = before.revision;
|
|
116
|
+
try {
|
|
117
|
+
for (const operation of record.preview.operations) {
|
|
118
|
+
currentRevision = await this.adapter.apply(operation, currentRevision);
|
|
119
|
+
current = await this.project.inspectProject();
|
|
120
|
+
if (!sameRevision(current.revision, currentRevision)) {
|
|
121
|
+
throw new Error("READ_AFTER_WRITE_FAILED: editor returned an unexpected filler-removal revision");
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
const transaction = {
|
|
125
|
+
id: `txn-${randomUUID()}`,
|
|
126
|
+
intent: "remove-filler-words",
|
|
127
|
+
planned: structuredClone(record.preview.operations),
|
|
128
|
+
applied: structuredClone(record.preview.operations),
|
|
129
|
+
baseRevision: before.revision,
|
|
130
|
+
before,
|
|
131
|
+
after: current,
|
|
132
|
+
attemptedAfter: current,
|
|
133
|
+
diff: diffSnapshots(before, current),
|
|
134
|
+
status: "APPLIED",
|
|
135
|
+
};
|
|
136
|
+
transaction.attemptedAfter = await this.reanalyzeFillerTargets(transaction, record);
|
|
137
|
+
transaction.after = transaction.attemptedAfter;
|
|
138
|
+
const structuralVerification = await this.verificationEngine.verify(transaction, { requireExpectedChange: true });
|
|
139
|
+
const continuity = verifyFillerSpeechContinuity(transaction, record);
|
|
140
|
+
transaction.verification = {
|
|
141
|
+
passed: structuralVerification.passed && continuity.passed,
|
|
142
|
+
checks: [...structuralVerification.checks, continuity],
|
|
143
|
+
};
|
|
144
|
+
if (transaction.verification.passed) {
|
|
145
|
+
transaction.status = "VERIFIED";
|
|
146
|
+
}
|
|
147
|
+
else {
|
|
148
|
+
await this.adapter.restore(before, current.revision);
|
|
149
|
+
transaction.after = await this.project.inspectProject();
|
|
150
|
+
this.assertRestored(before, transaction.after);
|
|
151
|
+
transaction.status = "ROLLED_BACK";
|
|
152
|
+
}
|
|
153
|
+
this.transactions.set(transaction);
|
|
154
|
+
return transaction;
|
|
155
|
+
}
|
|
156
|
+
catch (error) {
|
|
157
|
+
let partiallyApplied;
|
|
158
|
+
try {
|
|
159
|
+
partiallyApplied = await this.project.inspectProject();
|
|
160
|
+
}
|
|
161
|
+
catch (readError) {
|
|
162
|
+
throw new Error(`FILLER_REMOVAL_FAILED: ${String(error)}; rollback failed: ${String(readError)}`);
|
|
163
|
+
}
|
|
164
|
+
if (!sameRevision(partiallyApplied.revision, before.revision)) {
|
|
165
|
+
try {
|
|
166
|
+
await this.adapter.restore(before, partiallyApplied.revision);
|
|
167
|
+
this.assertRestored(before, await this.project.inspectProject());
|
|
168
|
+
}
|
|
169
|
+
catch (rollbackError) {
|
|
170
|
+
throw new Error(`FILLER_REMOVAL_FAILED: ${String(error)}; rollback failed: ${String(rollbackError)}`);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
throw new Error(`FILLER_REMOVAL_FAILED: ${String(error)}`);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
async assertCapabilities() {
|
|
177
|
+
if (!this.options.speechAnalyzer)
|
|
178
|
+
throw new Error("CAPABILITY_UNAVAILABLE: speech analysis");
|
|
179
|
+
const capabilities = (await this.adapter.getCapabilities()).editor;
|
|
180
|
+
if (!capabilities.timelineSnapshotRead || !capabilities.timelineWrite
|
|
181
|
+
|| !capabilities.readAfterWrite || !capabilities.rollback) {
|
|
182
|
+
throw new Error("CAPABILITY_UNAVAILABLE: filler removal requires canonical timeline write, read-after-write, and rollback");
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
assertRestored(expected, actual) {
|
|
186
|
+
if (canonicalSnapshotDigest(expected) !== canonicalSnapshotDigest(actual)) {
|
|
187
|
+
throw new Error("ROLLBACK_FAILED: restored canonical digest does not match pre-edit state");
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
async reanalyzeFillerTargets(transaction, record) {
|
|
191
|
+
const next = structuredClone(transaction.attemptedAfter);
|
|
192
|
+
const targets = new Map(record.preview.candidates.map((candidate) => [candidate.clipId, candidate]));
|
|
193
|
+
for (const clipId of targets.keys()) {
|
|
194
|
+
const beforeClip = transaction.before.timeline.clips.find((clip) => clip.id === clipId);
|
|
195
|
+
const afterClip = next.timeline.clips.find((clip) => clip.id === clipId);
|
|
196
|
+
if (!beforeClip || !afterClip || !beforeClip.mediaId) {
|
|
197
|
+
throw new Error(`ANALYSIS_FAILED: filler target clip ${clipId} disappeared after the edit`);
|
|
198
|
+
}
|
|
199
|
+
const media = next.media.find((candidate) => candidate.mediaId === beforeClip.mediaId);
|
|
200
|
+
if (!media)
|
|
201
|
+
throw new Error(`MEDIA_NOT_FOUND: ${beforeClip.mediaId}`);
|
|
202
|
+
const analysisRange = record.analysisRangesByClip.get(clipId);
|
|
203
|
+
if (!analysisRange)
|
|
204
|
+
throw new Error(`ANALYSIS_FAILED: filler analysis range for clip ${clipId} is unavailable`);
|
|
205
|
+
const fillerCandidates = record.preview.candidates.filter((candidate) => candidate.clipId === clipId);
|
|
206
|
+
const deletes = fillerCandidates.map((candidate) => candidate.sourceRange);
|
|
207
|
+
const postEditRange = translateRangeAfterDeletes(analysisRange, deletes);
|
|
208
|
+
const speech = await this.options.speechAnalyzer.analyze({ project: next, media }, postEditRange);
|
|
209
|
+
const retainedWords = transaction.before.media
|
|
210
|
+
.find((candidate) => candidate.mediaId === beforeClip.mediaId)
|
|
211
|
+
?.speech?.words ?? [];
|
|
212
|
+
const translatedRetainedWords = retainedWords
|
|
213
|
+
.filter((word) => !fillerCandidates.some((candidate) => sameSpeechWord(word, candidate.word)))
|
|
214
|
+
.map((word) => translateSpeechWordAfterDeletes(word, deletes));
|
|
215
|
+
media.speech = {
|
|
216
|
+
words: [
|
|
217
|
+
...translatedRetainedWords.filter((word) => !rangesOverlap(word.start, word.end, postEditRange.start, postEditRange.end)),
|
|
218
|
+
...speech.words.filter((word) => word.start >= postEditRange.start && word.end <= postEditRange.end),
|
|
219
|
+
].sort((left, right) => left.start - right.start || left.end - right.end),
|
|
220
|
+
};
|
|
221
|
+
media.analysisRevision = next.revision.id;
|
|
222
|
+
}
|
|
223
|
+
return next;
|
|
224
|
+
}
|
|
225
|
+
now() {
|
|
226
|
+
return this.options.now?.() ?? Date.now();
|
|
227
|
+
}
|
|
228
|
+
prunePreviews() {
|
|
229
|
+
const now = this.now();
|
|
230
|
+
for (const [previewToken, record] of this.previews) {
|
|
231
|
+
if (now > Date.parse(record.preview.expiresAt))
|
|
232
|
+
this.previews.delete(previewToken);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
function validateFillerSelection(range) {
|
|
237
|
+
if (!Number.isFinite(range.start) || !Number.isFinite(range.end)
|
|
238
|
+
|| range.start < 0 || range.end <= range.start) {
|
|
239
|
+
throw new Error("INVALID_OPERATION: filler selection range must be positive");
|
|
240
|
+
}
|
|
241
|
+
return {
|
|
242
|
+
start: range.start,
|
|
243
|
+
end: range.end,
|
|
244
|
+
...(range.startTime ? { startTime: { ...range.startTime } } : {}),
|
|
245
|
+
...(range.durationTime ? { durationTime: { ...range.durationTime } } : {}),
|
|
246
|
+
};
|
|
247
|
+
}
|
|
248
|
+
function rangesOverlap(leftStart, leftEnd, rightStart, rightEnd) {
|
|
249
|
+
return leftStart < rightEnd && leftEnd > rightStart;
|
|
250
|
+
}
|
|
251
|
+
function translateRangeAfterDeletes(range, deletes) {
|
|
252
|
+
const orderedDeletes = orderDeletes(deletes);
|
|
253
|
+
return {
|
|
254
|
+
start: translateBoundaryAfterDeletes(range.start, orderedDeletes),
|
|
255
|
+
end: translateBoundaryAfterDeletes(range.end, orderedDeletes),
|
|
256
|
+
};
|
|
257
|
+
}
|
|
258
|
+
function orderDeletes(deletes) {
|
|
259
|
+
return [...deletes].sort((left, right) => left.start - right.start);
|
|
260
|
+
}
|
|
261
|
+
function translateBoundaryAfterDeletes(boundary, deletes) {
|
|
262
|
+
let translated = boundary;
|
|
263
|
+
let removed = 0;
|
|
264
|
+
for (const deletion of deletes) {
|
|
265
|
+
if (boundary <= deletion.start)
|
|
266
|
+
break;
|
|
267
|
+
if (boundary < deletion.end)
|
|
268
|
+
return deletion.start - removed;
|
|
269
|
+
const duration = deletion.end - deletion.start;
|
|
270
|
+
translated -= duration;
|
|
271
|
+
removed += duration;
|
|
272
|
+
}
|
|
273
|
+
return translated;
|
|
274
|
+
}
|
|
275
|
+
function translateSpeechWordAfterDeletes(word, deletes) {
|
|
276
|
+
const orderedDeletes = orderDeletes(deletes);
|
|
277
|
+
return {
|
|
278
|
+
...word,
|
|
279
|
+
start: translateBoundaryAfterDeletes(word.start, orderedDeletes),
|
|
280
|
+
end: translateBoundaryAfterDeletes(word.end, orderedDeletes),
|
|
281
|
+
};
|
|
282
|
+
}
|
|
283
|
+
function verifyFillerSpeechContinuity(transaction, record) {
|
|
284
|
+
const clipIds = new Set(record.preview.candidates.map((candidate) => candidate.clipId));
|
|
285
|
+
for (const clipId of clipIds) {
|
|
286
|
+
const beforeClip = transaction.before.timeline.clips.find((clip) => clip.id === clipId);
|
|
287
|
+
const afterClip = transaction.attemptedAfter.timeline.clips.find((clip) => clip.id === clipId);
|
|
288
|
+
const mediaId = beforeClip?.mediaId;
|
|
289
|
+
const beforeWords = mediaId
|
|
290
|
+
? transaction.before.media.find((media) => media.mediaId === mediaId)?.speech?.words
|
|
291
|
+
: undefined;
|
|
292
|
+
const actualWords = mediaId
|
|
293
|
+
? transaction.attemptedAfter.media.find((media) => media.mediaId === mediaId)?.speech?.words
|
|
294
|
+
: undefined;
|
|
295
|
+
if (!beforeClip || !afterClip || !beforeWords || !actualWords) {
|
|
296
|
+
return {
|
|
297
|
+
name: "filler-speech-continuity",
|
|
298
|
+
passed: false,
|
|
299
|
+
detail: `complete pre- and post-edit speech analysis is unavailable for filler target clip ${clipId}`,
|
|
300
|
+
};
|
|
301
|
+
}
|
|
302
|
+
const candidates = record.preview.candidates.filter((candidate) => candidate.clipId === clipId);
|
|
303
|
+
const deletes = candidates.map((candidate) => candidate.sourceRange);
|
|
304
|
+
const expectedWords = beforeWords
|
|
305
|
+
.filter((word) => !candidates.some((candidate) => sameSpeechWord(candidate.word, word)))
|
|
306
|
+
.map((word) => translateSpeechWordAfterDeletes(word, deletes));
|
|
307
|
+
if (expectedWords.length !== actualWords.length) {
|
|
308
|
+
return {
|
|
309
|
+
name: "filler-speech-continuity",
|
|
310
|
+
passed: false,
|
|
311
|
+
detail: `post-edit transcript has ${actualWords.length} words; expected ${expectedWords.length} adjacent words after filler removal`,
|
|
312
|
+
};
|
|
313
|
+
}
|
|
314
|
+
for (let index = 0; index < expectedWords.length; index += 1) {
|
|
315
|
+
const expected = expectedWords[index];
|
|
316
|
+
const actual = actualWords[index];
|
|
317
|
+
if (expected.text.trim().toLowerCase() !== actual.text.trim().toLowerCase()
|
|
318
|
+
|| Math.abs(expected.start - actual.start) > 0.02
|
|
319
|
+
|| Math.abs(expected.end - actual.end) > 0.02
|
|
320
|
+
|| actual.end > afterClip.duration + 0.02) {
|
|
321
|
+
return {
|
|
322
|
+
name: "filler-speech-continuity",
|
|
323
|
+
passed: false,
|
|
324
|
+
detail: `post-edit transcript boundary ${index + 1} does not preserve adjacent speech around the removed fillers`,
|
|
325
|
+
};
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
return {
|
|
330
|
+
name: "filler-speech-continuity",
|
|
331
|
+
passed: true,
|
|
332
|
+
detail: "post-edit speech re-analysis preserves adjacent words and clip bounds",
|
|
333
|
+
};
|
|
334
|
+
}
|
|
335
|
+
function sameSpeechWord(left, right) {
|
|
336
|
+
return left.text === right.text && left.start === right.start && left.end === right.end;
|
|
337
|
+
}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
export class MusicService {
|
|
2
|
+
project;
|
|
3
|
+
edits;
|
|
4
|
+
constructor(project, edits) {
|
|
5
|
+
this.project = project;
|
|
6
|
+
this.edits = edits;
|
|
7
|
+
}
|
|
8
|
+
async previewMusic(request) {
|
|
9
|
+
if (request.ducking?.enabled) {
|
|
10
|
+
throw new Error("CAPABILITY_UNAVAILABLE: dialogue ducking");
|
|
11
|
+
}
|
|
12
|
+
const before = await this.project.inspectProject();
|
|
13
|
+
const operations = [];
|
|
14
|
+
const hasMediaId = request.mediaId !== undefined;
|
|
15
|
+
const hasImport = request.import !== undefined;
|
|
16
|
+
if (hasMediaId === hasImport) {
|
|
17
|
+
throw new Error("INVALID_OPERATION: music requires exactly one mediaId or import source");
|
|
18
|
+
}
|
|
19
|
+
if (!Number.isInteger(request.targetLane) || request.targetLane === 0) {
|
|
20
|
+
throw new Error("INVALID_OPERATION: music requires a non-primary target lane");
|
|
21
|
+
}
|
|
22
|
+
if (request.placement === "append" && request.start !== undefined) {
|
|
23
|
+
throw new Error("INVALID_OPERATION: appended music cannot specify a start position");
|
|
24
|
+
}
|
|
25
|
+
const mediaId = request.import?.mediaId ?? request.mediaId;
|
|
26
|
+
if (request.import) {
|
|
27
|
+
operations.push({
|
|
28
|
+
type: "media.import",
|
|
29
|
+
mediaId: request.import.mediaId,
|
|
30
|
+
source: request.import.source,
|
|
31
|
+
mediaKind: "audio",
|
|
32
|
+
duration: request.import.duration,
|
|
33
|
+
sourceDigest: request.import.sourceDigest,
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
const media = before.media.find((candidate) => candidate.mediaId === mediaId);
|
|
37
|
+
const duration = request.duration ?? request.import?.duration ?? media?.duration;
|
|
38
|
+
const sourceDuration = request.import?.duration ?? media?.duration;
|
|
39
|
+
if (duration === undefined) {
|
|
40
|
+
throw new Error("INVALID_OPERATION: music duration is required when media duration is unavailable");
|
|
41
|
+
}
|
|
42
|
+
if (!Number.isFinite(duration) || duration <= 0) {
|
|
43
|
+
throw new Error("INVALID_OPERATION: music duration must be positive");
|
|
44
|
+
}
|
|
45
|
+
if (sourceDuration !== undefined && duration > sourceDuration) {
|
|
46
|
+
throw new Error("INVALID_OPERATION: music duration exceeds the source duration");
|
|
47
|
+
}
|
|
48
|
+
const start = request.placement === "append" ? before.timeline.duration : request.start;
|
|
49
|
+
if (start === undefined || !Number.isFinite(start) || start < 0) {
|
|
50
|
+
throw new Error("INVALID_OPERATION: inserted music requires a non-negative start position");
|
|
51
|
+
}
|
|
52
|
+
operations.push({
|
|
53
|
+
type: "timeline.media.add",
|
|
54
|
+
occurrenceId: request.occurrenceId,
|
|
55
|
+
mediaId,
|
|
56
|
+
role: "music",
|
|
57
|
+
start,
|
|
58
|
+
duration,
|
|
59
|
+
targetLane: request.targetLane,
|
|
60
|
+
});
|
|
61
|
+
if (request.gainDb !== undefined) {
|
|
62
|
+
if (!Number.isFinite(request.gainDb))
|
|
63
|
+
throw new Error("INVALID_OPERATION: music gain must be finite");
|
|
64
|
+
operations.push({ type: "set-gain", clipId: request.occurrenceId, gainDb: request.gainDb });
|
|
65
|
+
}
|
|
66
|
+
if (request.fadeIn !== undefined || request.fadeOut !== undefined) {
|
|
67
|
+
const fadeIn = request.fadeIn ?? 0;
|
|
68
|
+
const fadeOut = request.fadeOut ?? 0;
|
|
69
|
+
if (!Number.isFinite(fadeIn) || !Number.isFinite(fadeOut) || fadeIn < 0 || fadeOut < 0 || fadeIn + fadeOut > duration) {
|
|
70
|
+
throw new Error("INVALID_OPERATION: music fades must be non-negative and fit within the music duration");
|
|
71
|
+
}
|
|
72
|
+
operations.push({
|
|
73
|
+
type: "timeline.audio.fades",
|
|
74
|
+
clipId: request.occurrenceId,
|
|
75
|
+
fadeIn,
|
|
76
|
+
fadeOut,
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
return this.edits.previewEdit({
|
|
80
|
+
baseRevision: request.baseRevision,
|
|
81
|
+
operations,
|
|
82
|
+
verification: request.verification,
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
}
|
|
@@ -1,8 +1,17 @@
|
|
|
1
|
-
export * from "./domain/
|
|
1
|
+
export * from "./domain/index.js";
|
|
2
2
|
export * from "./runtime.js";
|
|
3
|
-
export * from "./
|
|
3
|
+
export * from "./timeline/snapshot-diff.js";
|
|
4
4
|
export * from "./verification/verification.js";
|
|
5
5
|
export * from "./editing/intent.js";
|
|
6
|
+
export * from "./editing/duration-policy.js";
|
|
6
7
|
export * from "./capabilities.js";
|
|
7
|
-
export * from "./snapshot-digest.js";
|
|
8
|
+
export * from "./timeline/snapshot-digest.js";
|
|
8
9
|
export * from "./speech/filler-removal.js";
|
|
10
|
+
export * from "./audio/dialogue-normalization.js";
|
|
11
|
+
export * from "./audio/noise-reduction.js";
|
|
12
|
+
export * from "./color-correction.js";
|
|
13
|
+
export * from "./domain/skills.js";
|
|
14
|
+
export * from "./skills/requirements.js";
|
|
15
|
+
export * from "./skills/index.js";
|
|
16
|
+
export * from "./speech/filler-detector.js";
|
|
17
|
+
export * from "./speech/safe-cut-resolver.js";
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
const DEFAULT_MAX_SHOTS = 50;
|
|
2
|
+
export function planRoughCut(entries, revision, request) {
|
|
3
|
+
const maxShots = request.maxShots ?? DEFAULT_MAX_SHOTS;
|
|
4
|
+
if (!Number.isInteger(maxShots) || maxShots <= 0) {
|
|
5
|
+
throw new Error("INVALID_ROUGH_CUT_REQUEST: maxShots must be a positive integer");
|
|
6
|
+
}
|
|
7
|
+
const candidates = entries
|
|
8
|
+
.flatMap((entry) => entry.semantic.usableRanges.map((range) => ({ entry, range })))
|
|
9
|
+
.sort((left, right) => {
|
|
10
|
+
const mediaOrder = left.entry.sourceIdentity.mediaId.localeCompare(right.entry.sourceIdentity.mediaId);
|
|
11
|
+
if (mediaOrder !== 0)
|
|
12
|
+
return mediaOrder;
|
|
13
|
+
if (left.range.start !== right.range.start)
|
|
14
|
+
return left.range.start - right.range.start;
|
|
15
|
+
return left.range.end - right.range.end;
|
|
16
|
+
});
|
|
17
|
+
const shots = candidates.slice(0, maxShots).map(({ entry, range }, index) => shotFor(entry, range, request, index + 1));
|
|
18
|
+
const warnings = entries.length > 0 && candidates.length === 0
|
|
19
|
+
? ["No matching media has an explicitly analyzed usable range"]
|
|
20
|
+
: [];
|
|
21
|
+
return {
|
|
22
|
+
planner: { id: "framekit.rough-cut", version: 1 },
|
|
23
|
+
revision: structuredClone(revision),
|
|
24
|
+
query: structuredClone(request),
|
|
25
|
+
shots,
|
|
26
|
+
warnings,
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
function shotFor(entry, range, request, order) {
|
|
30
|
+
const matchedProperties = [
|
|
31
|
+
...(request.query ? [`query:${request.query}`] : []),
|
|
32
|
+
...(request.subject ? [`subject:${request.subject}`] : []),
|
|
33
|
+
...(request.scene ? [`scene:${request.scene}`] : []),
|
|
34
|
+
...(request.environment ? [`environment:${request.environment}`] : []),
|
|
35
|
+
...(request.timeOfDay ? [`timeOfDay:${request.timeOfDay}`] : []),
|
|
36
|
+
...(request.mood ? [`mood:${request.mood}`] : []),
|
|
37
|
+
...(request.motion ? [`motion:${request.motion}`] : []),
|
|
38
|
+
];
|
|
39
|
+
const semanticAnnotations = [
|
|
40
|
+
{ query: request.subject, tags: entry.semantic.subjects },
|
|
41
|
+
{ query: request.scene, tags: entry.semantic.scenes },
|
|
42
|
+
{ query: request.environment, tags: entry.semantic.environments },
|
|
43
|
+
{ query: request.timeOfDay, tags: entry.semantic.timeOfDay },
|
|
44
|
+
{ query: request.mood, tags: entry.semantic.moods },
|
|
45
|
+
];
|
|
46
|
+
const activeAnnotations = semanticAnnotations.filter(({ query }) => Boolean(query?.trim()));
|
|
47
|
+
const confidenceTags = activeAnnotations.length > 0
|
|
48
|
+
? activeAnnotations.flatMap(({ query, tags }) => tags.filter((tag) => tag.value.toLowerCase() === query.trim().toLowerCase()))
|
|
49
|
+
: semanticAnnotations.flatMap(({ tags }) => tags);
|
|
50
|
+
const confidence = Math.max(0, ...confidenceTags.map((tag) => tag.confidence));
|
|
51
|
+
const reason = matchedProperties.length > 0
|
|
52
|
+
? `matches ${matchedProperties.map((property) => {
|
|
53
|
+
const [kind, ...value] = property.split(":");
|
|
54
|
+
return `${kind} "${value.join(":")}"`;
|
|
55
|
+
}).join(", ")}`
|
|
56
|
+
: "has an explicitly analyzed usable range";
|
|
57
|
+
return {
|
|
58
|
+
order,
|
|
59
|
+
sourceIdentity: structuredClone(entry.sourceIdentity),
|
|
60
|
+
range: structuredClone(range),
|
|
61
|
+
confidence,
|
|
62
|
+
matchedProperties,
|
|
63
|
+
rationale: `Selected ${entry.sourceIdentity.mediaId} because it ${reason}.`,
|
|
64
|
+
};
|
|
65
|
+
}
|