@morshoto/framekit 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist-package/adapters/final-cut/typescript/src/analyzers.js +29 -2
- package/dist-package/adapters/final-cut/typescript/src/assets.js +89 -3
- package/dist-package/adapters/final-cut/typescript/src/canonical.js +431 -0
- package/dist-package/adapters/final-cut/typescript/src/connection.js +19 -4
- package/dist-package/adapters/final-cut/typescript/src/fcpxml.js +212 -13
- package/dist-package/adapters/final-cut/typescript/src/index.js +1 -0
- package/dist-package/adapters/final-cut/typescript/src/native.js +1026 -7
- package/dist-package/adapters/final-cut/typescript/src/publisher.js +207 -30
- package/dist-package/adapters/final-cut/typescript/src/session.js +69 -8
- package/dist-package/apps/mcp-server/src/main.js +49 -11
- package/dist-package/apps/mcp-server/src/routing.js +15 -0
- package/dist-package/apps/mcp-server/src/server.js +289 -68
- package/dist-package/apps/mcp-server/src/version.js +34 -0
- package/dist-package/packages/runtime/src/application/media-analysis-service.js +119 -15
- package/dist-package/packages/runtime/src/application/project-service.js +21 -2
- package/dist-package/packages/runtime/src/audio/dialogue-normalization.js +9 -0
- package/dist-package/packages/runtime/src/capabilities.js +90 -10
- package/dist-package/packages/runtime/src/context/context-engine.js +10 -1
- package/dist-package/packages/runtime/src/domain/editing.js +27 -1
- package/dist-package/packages/runtime/src/domain/media.js +1 -0
- package/dist-package/packages/runtime/src/domain/skills.js +2 -0
- package/dist-package/packages/runtime/src/editing/edit-service.js +82 -4
- package/dist-package/packages/runtime/src/editing/filler-removal-service.js +31 -7
- package/dist-package/packages/runtime/src/index.js +2 -0
- package/dist-package/packages/runtime/src/planning/rough-cut.js +9 -3
- package/dist-package/packages/runtime/src/runtime.js +2 -2
- package/dist-package/packages/runtime/src/skills/builtin.js +314 -39
- package/dist-package/packages/runtime/src/skills/requirements.js +1 -0
- package/dist-package/packages/runtime/src/skills/runtime.js +33 -8
- package/dist-package/packages/runtime/src/speech/analysis.js +301 -0
- package/dist-package/packages/runtime/src/speech/mapping.js +227 -0
- package/dist-package/packages/runtime/src/timeline/rational-time.js +5 -0
- package/dist-package/packages/runtime/src/verification/verification.js +81 -4
- package/dist-package/packages/testkit/src/fixture-analyzers.js +1 -0
- package/dist-package/packages/testkit/src/in-memory-editor.js +126 -0
- package/package.json +8 -2
|
@@ -2,7 +2,9 @@ import { randomUUID } from "node:crypto";
|
|
|
2
2
|
import { resolve as resolvePath } from "node:path";
|
|
3
3
|
import { diffSnapshots } from "../timeline/snapshot-diff.js";
|
|
4
4
|
import { canonicalSnapshotDigest } from "../timeline/snapshot-digest.js";
|
|
5
|
+
import { assertValidMaskConfiguration } from "../domain/editing.js";
|
|
5
6
|
import { sameRevision } from "../context/revision.js";
|
|
7
|
+
import { parseRational } from "../timeline/rational-time.js";
|
|
6
8
|
import { assertValidVerificationPolicy } from "../verification/verification.js";
|
|
7
9
|
export class EditService {
|
|
8
10
|
adapter;
|
|
@@ -47,6 +49,7 @@ export class EditService {
|
|
|
47
49
|
if (operation.baseRevision && !sameRevision(operation.baseRevision, before.revision)) {
|
|
48
50
|
throw new Error("STALE_CONTEXT: operation base revision does not match current editor state");
|
|
49
51
|
}
|
|
52
|
+
assertValidWorkflowOperation(operation);
|
|
50
53
|
this.assertSurfaceCapability(target, capabilities.editor);
|
|
51
54
|
if (!capabilities.editor.timelineSnapshotRead || !capabilities.editor.readAfterWrite || !capabilities.editor.rollback) {
|
|
52
55
|
throw new Error(`CAPABILITY_UNAVAILABLE: ${target.kind === "artifact" ? "artifact" : "editor timeline"} mutation requires snapshot, read-after-write, and rollback`);
|
|
@@ -84,8 +87,7 @@ export class EditService {
|
|
|
84
87
|
status: "APPLIED",
|
|
85
88
|
};
|
|
86
89
|
try {
|
|
87
|
-
|
|
88
|
-
transaction.after = transaction.attemptedAfter;
|
|
90
|
+
await this.reanalyzeForVerification(transaction, verificationPolicy);
|
|
89
91
|
}
|
|
90
92
|
catch (error) {
|
|
91
93
|
await this.adapter.restore(before, attemptedAfter.revision);
|
|
@@ -147,6 +149,7 @@ export class EditService {
|
|
|
147
149
|
}
|
|
148
150
|
if (request.operations.length === 0)
|
|
149
151
|
throw new Error("INVALID_OPERATION: composite edit requires operations");
|
|
152
|
+
request.operations.forEach(assertValidWorkflowOperation);
|
|
150
153
|
await this.assertCompositeCapabilities(request.operations, target.kind, capabilities.editor);
|
|
151
154
|
if (!this.adapter.previewTransaction) {
|
|
152
155
|
throw new Error("CAPABILITY_UNAVAILABLE: editor composite transaction preview");
|
|
@@ -191,6 +194,7 @@ export class EditService {
|
|
|
191
194
|
throw new Error("STALE_CONTEXT: preview base revision does not match current editor state");
|
|
192
195
|
}
|
|
193
196
|
this.assertTarget(preview.target, before);
|
|
197
|
+
preview.operations.forEach(assertValidWorkflowOperation);
|
|
194
198
|
await this.assertCompositeCapabilities(preview.operations, preview.target.kind);
|
|
195
199
|
if (!this.adapter.applyTransaction) {
|
|
196
200
|
throw new Error("CAPABILITY_UNAVAILABLE: editor composite transaction execution");
|
|
@@ -231,9 +235,19 @@ export class EditService {
|
|
|
231
235
|
verificationPolicy,
|
|
232
236
|
status: "APPLIED",
|
|
233
237
|
};
|
|
238
|
+
const authorizedDiffCheck = sameDiffContent(transaction.diff, preview.expectedDiff)
|
|
239
|
+
? undefined
|
|
240
|
+
: {
|
|
241
|
+
name: "authorized-diff",
|
|
242
|
+
passed: false,
|
|
243
|
+
status: "failed",
|
|
244
|
+
reason: "UNAUTHORIZED_DIFF",
|
|
245
|
+
expected: structuredClone(preview.expectedDiff),
|
|
246
|
+
observed: structuredClone(transaction.diff),
|
|
247
|
+
detail: "canonical diff contains changes outside the preview-authorized operations",
|
|
248
|
+
};
|
|
234
249
|
try {
|
|
235
|
-
|
|
236
|
-
transaction.after = transaction.attemptedAfter;
|
|
250
|
+
await this.reanalyzeForVerification(transaction, verificationPolicy);
|
|
237
251
|
}
|
|
238
252
|
catch (error) {
|
|
239
253
|
await this.adapter.restore(before, attemptedAfter.revision);
|
|
@@ -256,6 +270,13 @@ export class EditService {
|
|
|
256
270
|
}
|
|
257
271
|
throw new Error(`VERIFICATION_FAILED: canonical state was restored (${String(verificationError)})`);
|
|
258
272
|
}
|
|
273
|
+
if (authorizedDiffCheck && transaction.verification) {
|
|
274
|
+
transaction.verification = {
|
|
275
|
+
...transaction.verification,
|
|
276
|
+
passed: false,
|
|
277
|
+
checks: [...transaction.verification.checks, authorizedDiffCheck],
|
|
278
|
+
};
|
|
279
|
+
}
|
|
259
280
|
if (transaction.verification.passed) {
|
|
260
281
|
transaction.status = "VERIFIED";
|
|
261
282
|
}
|
|
@@ -277,12 +298,22 @@ export class EditService {
|
|
|
277
298
|
async verifyTransaction(transactionId) {
|
|
278
299
|
return this.getTransaction(transactionId).verification;
|
|
279
300
|
}
|
|
301
|
+
async reanalyzeForVerification(transaction, policy) {
|
|
302
|
+
const requirements = postWriteAnalysisRequirements(policy);
|
|
303
|
+
if (!Object.values(requirements).some(Boolean))
|
|
304
|
+
return;
|
|
305
|
+
transaction.attemptedAfter = await this.analysis.reanalyzeAffectedRanges(transaction, requirements);
|
|
306
|
+
transaction.after = transaction.attemptedAfter;
|
|
307
|
+
}
|
|
280
308
|
async undo(transactionId) {
|
|
281
309
|
const transaction = this.getTransaction(transactionId);
|
|
282
310
|
const current = await this.project.inspectProject();
|
|
283
311
|
if (current.projectId !== transaction.before.projectId || current.timeline.id !== transaction.before.timeline.id) {
|
|
284
312
|
throw new Error(`TARGET_MISMATCH: cannot undo ${transaction.before.projectId}/${transaction.before.timeline.id} while ${current.projectId}/${current.timeline.id} is active`);
|
|
285
313
|
}
|
|
314
|
+
if (!sameRevision(current.revision, transaction.after.revision)) {
|
|
315
|
+
throw new Error(`STALE_CONTEXT: transaction ${transactionId} changed after edit`);
|
|
316
|
+
}
|
|
286
317
|
await this.adapter.restore(transaction.before, current.revision);
|
|
287
318
|
const restored = await this.project.inspectProject();
|
|
288
319
|
this.assertRestored(transaction.before, restored);
|
|
@@ -304,6 +335,9 @@ export class EditService {
|
|
|
304
335
|
if (operations.some((operation) => operation.type === "timeline.media.add") && !capabilities.mediaPlacement) {
|
|
305
336
|
throw new Error("CAPABILITY_UNAVAILABLE: timeline media placement");
|
|
306
337
|
}
|
|
338
|
+
if (operations.some((operation) => operation.type === "timeline.picture-in-picture.add") && !capabilities.pictureInPicture) {
|
|
339
|
+
throw new Error("CAPABILITY_UNAVAILABLE: picture-in-picture placement");
|
|
340
|
+
}
|
|
307
341
|
if (operations.some((operation) => operation.type === "timeline.title.add")
|
|
308
342
|
&& (!capabilities.titlePlacement || !capabilities.assetDiscovery)) {
|
|
309
343
|
throw new Error("CAPABILITY_UNAVAILABLE: timeline title placement");
|
|
@@ -332,6 +366,13 @@ export class EditService {
|
|
|
332
366
|
&& (!capabilities.transitionPlacement || !capabilities.assetDiscovery)) {
|
|
333
367
|
throw new Error("CAPABILITY_UNAVAILABLE: timeline transition placement");
|
|
334
368
|
}
|
|
369
|
+
const maskOperations = operations.filter((operation) => operation.type === "timeline.mask.add");
|
|
370
|
+
if (maskOperations.some((operation) => operation.mask.mode !== "person-cutout") && !capabilities.masking) {
|
|
371
|
+
throw new Error("CAPABILITY_UNAVAILABLE: masking");
|
|
372
|
+
}
|
|
373
|
+
if (maskOperations.some((operation) => operation.mask.mode === "person-cutout") && !capabilities.personCutout) {
|
|
374
|
+
throw new Error("CAPABILITY_UNAVAILABLE: person cutout");
|
|
375
|
+
}
|
|
335
376
|
if (operations.some((operation) => operation.type === "timeline.audio.attach") && !capabilities.audioAttachment) {
|
|
336
377
|
throw new Error("CAPABILITY_UNAVAILABLE: timeline audio attachment");
|
|
337
378
|
}
|
|
@@ -411,3 +452,40 @@ export class EditService {
|
|
|
411
452
|
}
|
|
412
453
|
}
|
|
413
454
|
}
|
|
455
|
+
function postWriteAnalysisRequirements(policy) {
|
|
456
|
+
const assertions = policy.assertions ?? [];
|
|
457
|
+
return {
|
|
458
|
+
speech: policy.requireSpeechContinuity === true,
|
|
459
|
+
audio: policy.maxTruePeakDb !== undefined
|
|
460
|
+
|| policy.targetLufs !== undefined
|
|
461
|
+
|| assertions.some((assertion) => assertion.type === "audio-audibility" || assertion.type === "audio-loudness"),
|
|
462
|
+
noise: assertions.some((assertion) => assertion.type === "audio-noise"),
|
|
463
|
+
visual: assertions.some((assertion) => assertion.type === "visual-content"),
|
|
464
|
+
};
|
|
465
|
+
}
|
|
466
|
+
function sameDiffContent(left, right) {
|
|
467
|
+
const comparable = (diff) => {
|
|
468
|
+
const { from: _from, to: _to, ...content } = diff;
|
|
469
|
+
return content;
|
|
470
|
+
};
|
|
471
|
+
return JSON.stringify(comparable(left)) === JSON.stringify(comparable(right));
|
|
472
|
+
}
|
|
473
|
+
function assertValidWorkflowOperation(operation) {
|
|
474
|
+
if (operation.type === "timeline.mask.add") {
|
|
475
|
+
if (!operation.occurrenceId.trim())
|
|
476
|
+
throw new Error("INVALID_OPERATION: mask occurrenceId is required");
|
|
477
|
+
assertValidMaskConfiguration(operation.mask);
|
|
478
|
+
return;
|
|
479
|
+
}
|
|
480
|
+
if (operation.type !== "trim-clip")
|
|
481
|
+
return;
|
|
482
|
+
if (!Number.isFinite(operation.duration) || operation.duration <= 0) {
|
|
483
|
+
throw new Error("INVALID_OPERATION: clip duration must be positive");
|
|
484
|
+
}
|
|
485
|
+
if (!operation.durationTime)
|
|
486
|
+
return;
|
|
487
|
+
const duration = parseRational(operation.durationTime, "INVALID_OPERATION");
|
|
488
|
+
if (duration.value <= 0n) {
|
|
489
|
+
throw new Error("INVALID_OPERATION: clip duration must be positive");
|
|
490
|
+
}
|
|
491
|
+
}
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { diffSnapshots } from "../timeline/snapshot-diff.js";
|
|
3
3
|
import { sameRevision } from "../context/revision.js";
|
|
4
|
-
import { translateRationalRange } from "../timeline/rational-time.js";
|
|
5
4
|
import { canonicalSnapshotDigest } from "../timeline/snapshot-digest.js";
|
|
6
5
|
import { planFillerRemoval } from "../speech/filler-removal.js";
|
|
6
|
+
import { bindSpeechAnalysis } from "../speech/analysis.js";
|
|
7
|
+
import { mapSourceRangeToSequenceRange } from "../speech/mapping.js";
|
|
7
8
|
export class FillerRemovalService {
|
|
8
9
|
adapter;
|
|
9
10
|
project;
|
|
@@ -48,15 +49,20 @@ export class FillerRemovalService {
|
|
|
48
49
|
};
|
|
49
50
|
if (localRange.end <= localRange.start)
|
|
50
51
|
continue;
|
|
51
|
-
const
|
|
52
|
-
|
|
53
|
-
|
|
52
|
+
const sourceStart = clip.sourceStart ?? 0;
|
|
53
|
+
const sourceRange = {
|
|
54
|
+
start: sourceStart + localRange.start,
|
|
55
|
+
end: sourceStart + localRange.end,
|
|
56
|
+
};
|
|
57
|
+
const speech = bindSpeechAnalysis(await this.options.speechAnalyzer.analyze({ project: before, media }, sourceRange), { input: { project: before, media }, range: sourceRange, provider: this.options.speechAnalyzer.descriptor });
|
|
58
|
+
analysisRangesByClip.set(clip.id, structuredClone(sourceRange));
|
|
59
|
+
for (const candidate of planFillerRemoval(speech.words, sourceRange, request)) {
|
|
54
60
|
candidates.push({
|
|
55
61
|
...candidate,
|
|
56
62
|
clipId: clip.id,
|
|
57
63
|
mediaId: clip.mediaId,
|
|
58
64
|
sourceRange: structuredClone(candidate.range),
|
|
59
|
-
range:
|
|
65
|
+
range: translateSourceRangeToSequence(clip, candidate.range),
|
|
60
66
|
});
|
|
61
67
|
}
|
|
62
68
|
}
|
|
@@ -205,7 +211,7 @@ export class FillerRemovalService {
|
|
|
205
211
|
const fillerCandidates = record.preview.candidates.filter((candidate) => candidate.clipId === clipId);
|
|
206
212
|
const deletes = fillerCandidates.map((candidate) => candidate.sourceRange);
|
|
207
213
|
const postEditRange = translateRangeAfterDeletes(analysisRange, deletes);
|
|
208
|
-
const speech = await this.options.speechAnalyzer.analyze({ project: next, media }, postEditRange);
|
|
214
|
+
const speech = bindSpeechAnalysis(await this.options.speechAnalyzer.analyze({ project: next, media }, postEditRange), { input: { project: next, media }, range: postEditRange, provider: this.options.speechAnalyzer.descriptor });
|
|
209
215
|
const retainedWords = transaction.before.media
|
|
210
216
|
.find((candidate) => candidate.mediaId === beforeClip.mediaId)
|
|
211
217
|
?.speech?.words ?? [];
|
|
@@ -213,6 +219,7 @@ export class FillerRemovalService {
|
|
|
213
219
|
.filter((word) => !fillerCandidates.some((candidate) => sameSpeechWord(word, candidate.word)))
|
|
214
220
|
.map((word) => translateSpeechWordAfterDeletes(word, deletes));
|
|
215
221
|
media.speech = {
|
|
222
|
+
...speech,
|
|
216
223
|
words: [
|
|
217
224
|
...translatedRetainedWords.filter((word) => !rangesOverlap(word.start, word.end, postEditRange.start, postEditRange.end)),
|
|
218
225
|
...speech.words.filter((word) => word.start >= postEditRange.start && word.end <= postEditRange.end),
|
|
@@ -317,7 +324,7 @@ function verifyFillerSpeechContinuity(transaction, record) {
|
|
|
317
324
|
if (expected.text.trim().toLowerCase() !== actual.text.trim().toLowerCase()
|
|
318
325
|
|| Math.abs(expected.start - actual.start) > 0.02
|
|
319
326
|
|| Math.abs(expected.end - actual.end) > 0.02
|
|
320
|
-
|| actual.end > afterClip.duration + 0.02) {
|
|
327
|
+
|| actual.end > (afterClip.sourceStart ?? 0) + afterClip.duration + 0.02) {
|
|
321
328
|
return {
|
|
322
329
|
name: "filler-speech-continuity",
|
|
323
330
|
passed: false,
|
|
@@ -335,3 +342,20 @@ function verifyFillerSpeechContinuity(transaction, record) {
|
|
|
335
342
|
function sameSpeechWord(left, right) {
|
|
336
343
|
return left.text === right.text && left.start === right.start && left.end === right.end;
|
|
337
344
|
}
|
|
345
|
+
function translateSourceRangeToSequence(clip, sourceRange) {
|
|
346
|
+
const sourceStart = clip.sourceStart ?? 0;
|
|
347
|
+
return mapSourceRangeToSequenceRange(sourceRange, {
|
|
348
|
+
sourceRange: {
|
|
349
|
+
start: sourceStart,
|
|
350
|
+
end: sourceStart + clip.duration,
|
|
351
|
+
startTime: clip.sourceStartTime,
|
|
352
|
+
durationTime: clip.durationTime,
|
|
353
|
+
},
|
|
354
|
+
sequenceRange: {
|
|
355
|
+
start: clip.start,
|
|
356
|
+
end: clip.start + clip.duration,
|
|
357
|
+
startTime: clip.startTime,
|
|
358
|
+
durationTime: clip.durationTime,
|
|
359
|
+
},
|
|
360
|
+
});
|
|
361
|
+
}
|
|
@@ -5,6 +5,7 @@ export function planRoughCut(entries, revision, request) {
|
|
|
5
5
|
throw new Error("INVALID_ROUGH_CUT_REQUEST: maxShots must be a positive integer");
|
|
6
6
|
}
|
|
7
7
|
const candidates = entries
|
|
8
|
+
.filter((entry) => entry.sourceIdentity.mediaKind !== "audio")
|
|
8
9
|
.flatMap((entry) => entry.semantic.usableRanges.map((range) => ({ entry, range })))
|
|
9
10
|
.sort((left, right) => {
|
|
10
11
|
const mediaOrder = left.entry.sourceIdentity.mediaId.localeCompare(right.entry.sourceIdentity.mediaId);
|
|
@@ -15,9 +16,14 @@ export function planRoughCut(entries, revision, request) {
|
|
|
15
16
|
return left.range.end - right.range.end;
|
|
16
17
|
});
|
|
17
18
|
const shots = candidates.slice(0, maxShots).map(({ entry, range }, index) => shotFor(entry, range, request, index + 1));
|
|
18
|
-
const
|
|
19
|
-
|
|
20
|
-
|
|
19
|
+
const excludedAudioMediaIds = entries
|
|
20
|
+
.filter((entry) => entry.sourceIdentity.mediaKind === "audio" && entry.semantic.usableRanges.length > 0)
|
|
21
|
+
.map((entry) => entry.sourceIdentity.mediaId);
|
|
22
|
+
const warnings = excludedAudioMediaIds.length > 0
|
|
23
|
+
? [`Excluded audio-only media from rough-cut shot candidates: ${excludedAudioMediaIds.join(", ")}`]
|
|
24
|
+
: entries.length > 0 && candidates.length === 0
|
|
25
|
+
? ["No matching media has an explicitly analyzed usable range"]
|
|
26
|
+
: [];
|
|
21
27
|
return {
|
|
22
28
|
planner: { id: "framekit.rough-cut", version: 1 },
|
|
23
29
|
revision: structuredClone(revision),
|
|
@@ -147,8 +147,8 @@ export class AgentVideoRuntime {
|
|
|
147
147
|
async contextChangesSince(revision, waitMs = 0) {
|
|
148
148
|
return this.contexts.contextChangesSince(revision, waitMs);
|
|
149
149
|
}
|
|
150
|
-
async analyzeSpeech(mediaId) {
|
|
151
|
-
return this.media.analyzeSpeech(mediaId);
|
|
150
|
+
async analyzeSpeech(mediaId, range) {
|
|
151
|
+
return this.media.analyzeSpeech(mediaId, range);
|
|
152
152
|
}
|
|
153
153
|
async analyzeAudio(mediaId) {
|
|
154
154
|
return this.media.analyzeAudio(mediaId);
|