@morshoto/framekit 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist-package/adapters/final-cut/typescript/src/analyzers.js +29 -2
- package/dist-package/adapters/final-cut/typescript/src/fcpxml.js +16 -5
- package/dist-package/adapters/final-cut/typescript/src/publisher.js +207 -30
- package/dist-package/adapters/final-cut/typescript/src/session.js +69 -8
- package/dist-package/apps/mcp-server/src/main.js +1 -0
- package/dist-package/apps/mcp-server/src/server.js +136 -68
- package/dist-package/apps/mcp-server/src/version.js +34 -0
- package/dist-package/packages/runtime/src/application/media-analysis-service.js +117 -14
- package/dist-package/packages/runtime/src/application/project-service.js +21 -2
- package/dist-package/packages/runtime/src/capabilities.js +85 -10
- package/dist-package/packages/runtime/src/context/context-engine.js +10 -1
- package/dist-package/packages/runtime/src/domain/media.js +1 -0
- package/dist-package/packages/runtime/src/editing/edit-service.js +40 -4
- package/dist-package/packages/runtime/src/editing/filler-removal-service.js +31 -7
- package/dist-package/packages/runtime/src/index.js +2 -0
- package/dist-package/packages/runtime/src/planning/rough-cut.js +9 -3
- package/dist-package/packages/runtime/src/runtime.js +2 -2
- package/dist-package/packages/runtime/src/speech/analysis.js +301 -0
- package/dist-package/packages/runtime/src/speech/mapping.js +227 -0
- package/dist-package/packages/runtime/src/timeline/rational-time.js +5 -0
- package/dist-package/packages/testkit/src/fixture-analyzers.js +1 -0
- package/dist-package/packages/testkit/src/in-memory-editor.js +1 -0
- package/package.json +3 -2
|
@@ -25,18 +25,48 @@ export function withCanonicalTimelineMode(capabilities) {
|
|
|
25
25
|
const previous = capabilities.families;
|
|
26
26
|
if (!previous)
|
|
27
27
|
return normalized;
|
|
28
|
+
const canonicalRead = normalized.editor.canonicalTimelineMode === "canonical-read"
|
|
29
|
+
|| normalized.editor.canonicalTimelineMode === "canonical-write";
|
|
30
|
+
const canonicalWrite = normalized.editor.canonicalTimelineMode === "canonical-write";
|
|
31
|
+
const compositeTransactions = Boolean(normalized.editor.compositeTransactions
|
|
32
|
+
&& normalized.editor.timelineSnapshotRead
|
|
33
|
+
&& normalized.editor.readAfterWrite
|
|
34
|
+
&& normalized.editor.rollback
|
|
35
|
+
&& (normalized.editor.timelineArtifactWrite || canonicalWrite));
|
|
28
36
|
return withCapabilityFamilies(normalized, {
|
|
29
37
|
backend: previous.connection.status.backend,
|
|
30
38
|
nativeBackend: previous.native.selectionWrite.backend,
|
|
31
39
|
publishingBackend: previous.publishing.projectCreation.backend,
|
|
32
40
|
exportBackend: previous.export.timeline.backend,
|
|
33
|
-
|
|
41
|
+
analyzerBackends: {
|
|
42
|
+
speechTranscribe: previous.analyzers.speechTranscribe.backend,
|
|
43
|
+
speechVad: previous.analyzers.speechVad.backend,
|
|
44
|
+
audioLoudness: previous.analyzers.audioLoudness.backend,
|
|
45
|
+
audioNoise: previous.analyzers.audioNoise?.backend,
|
|
46
|
+
visualTrack: previous.analyzers.visualTrack.backend,
|
|
47
|
+
},
|
|
34
48
|
connection: previous.connection.status,
|
|
49
|
+
canonicalDocument: {
|
|
50
|
+
read: refreshDescriptor(canonicalRead, previous.canonicalDocument.read, "canonical-read", "canonical timeline reads are unavailable"),
|
|
51
|
+
write: refreshDescriptor(canonicalWrite, previous.canonicalDocument.write, "canonical-write", "canonical timeline writes are unavailable"),
|
|
52
|
+
artifactWrite: refreshDescriptor(normalized.editor.timelineArtifactWrite, previous.canonicalDocument.artifactWrite, "artifact-write", "canonical artifact writes are unavailable"),
|
|
53
|
+
},
|
|
54
|
+
editing: {
|
|
55
|
+
compositeTransactions: refreshDescriptor(compositeTransactions, previous.editing.compositeTransactions, "verified", "composite editing transactions are unavailable"),
|
|
56
|
+
titlePlacement: refreshDescriptor(Boolean(normalized.editor.titlePlacement), previous.editing.titlePlacement, "verified", "title placement is unavailable"),
|
|
57
|
+
pictureInPicture: previous.editing.pictureInPicture,
|
|
58
|
+
masking: previous.editing.masking,
|
|
59
|
+
},
|
|
35
60
|
native: nativeAvailability(previous.native),
|
|
36
61
|
publishing: previous.publishing.projectCreation,
|
|
37
62
|
export: previous.export.timeline,
|
|
38
63
|
});
|
|
39
64
|
}
|
|
65
|
+
function refreshDescriptor(available, previous, guarantee, unavailableReason) {
|
|
66
|
+
if (available && previous.available)
|
|
67
|
+
return previous;
|
|
68
|
+
return descriptorFrom(available, previous.backend, guarantee, unavailableReason);
|
|
69
|
+
}
|
|
40
70
|
/**
|
|
41
71
|
* Adds the versioned operation-level contract while retaining the legacy
|
|
42
72
|
* boolean fields used by existing runtime consumers.
|
|
@@ -53,10 +83,22 @@ export function withCapabilityFamilies(capabilities, options = {}) {
|
|
|
53
83
|
},
|
|
54
84
|
};
|
|
55
85
|
const editor = normalized.editor;
|
|
86
|
+
const analyzerBackend = (operation) => options.analyzerBackends?.[operation]
|
|
87
|
+
?? options.analyzerBackend
|
|
88
|
+
?? previous?.analyzers[operation]?.backend
|
|
89
|
+
?? backend;
|
|
56
90
|
const native = {
|
|
57
91
|
...nativeAvailability(previous?.native),
|
|
58
92
|
...options.native,
|
|
59
93
|
};
|
|
94
|
+
const canonicalDocument = {
|
|
95
|
+
...previous?.canonicalDocument,
|
|
96
|
+
...options.canonicalDocument,
|
|
97
|
+
};
|
|
98
|
+
const editing = {
|
|
99
|
+
...previous?.editing,
|
|
100
|
+
...options.editing,
|
|
101
|
+
};
|
|
60
102
|
const families = {
|
|
61
103
|
connection: {
|
|
62
104
|
status: descriptorFrom(options.connection ?? previous?.connection.status ?? true, options.connectionBackend ?? backend, "observed", "editor connection is unavailable"),
|
|
@@ -66,11 +108,12 @@ export function withCapabilityFamilies(capabilities, options = {}) {
|
|
|
66
108
|
media: descriptorFrom(editor.timelineSnapshotRead && editor.projectRead, backend, "canonical-read", "media observation is unavailable"),
|
|
67
109
|
},
|
|
68
110
|
canonicalDocument: {
|
|
69
|
-
read: descriptorFrom(editor.canonicalTimelineMode === "canonical-read"
|
|
70
|
-
|| editor.canonicalTimelineMode === "canonical-write", backend, "canonical-read", "canonical timeline reads are unavailable"),
|
|
71
|
-
write: descriptorFrom(editor.canonicalTimelineMode === "canonical-write", backend, "canonical-write", "canonical timeline writes are unavailable"),
|
|
72
|
-
artifactWrite: descriptorFrom(editor.timelineArtifactWrite, backend, "artifact-write", "canonical artifact writes are unavailable"),
|
|
111
|
+
read: descriptorFrom(canonicalDocument.read ?? (editor.canonicalTimelineMode === "canonical-read"
|
|
112
|
+
|| editor.canonicalTimelineMode === "canonical-write"), backend, "canonical-read", "canonical timeline reads are unavailable"),
|
|
113
|
+
write: descriptorFrom(canonicalDocument.write ?? (editor.canonicalTimelineMode === "canonical-write"), backend, "canonical-write", "canonical timeline writes are unavailable"),
|
|
114
|
+
artifactWrite: descriptorFrom(canonicalDocument.artifactWrite ?? editor.timelineArtifactWrite, backend, "artifact-write", "canonical artifact writes are unavailable"),
|
|
73
115
|
},
|
|
116
|
+
editing: editingFamily(editor, editing, options.editingBackend ?? previous?.editing?.compositeTransactions.backend ?? backend),
|
|
74
117
|
native: nativeFamily(native, options.nativeBackend ?? previous?.native.selectionWrite.backend ?? backend),
|
|
75
118
|
publishing: {
|
|
76
119
|
projectCreation: descriptorFrom(options.publishing ?? previous?.publishing.projectCreation ?? false, options.publishingBackend ?? backend, "verified", "new project publishing is unavailable"),
|
|
@@ -79,15 +122,47 @@ export function withCapabilityFamilies(capabilities, options = {}) {
|
|
|
79
122
|
timeline: descriptorFrom(options.export ?? previous?.export.timeline ?? false, options.exportBackend ?? backend, "verified", "timeline export is unavailable"),
|
|
80
123
|
},
|
|
81
124
|
analyzers: {
|
|
82
|
-
speechTranscribe: analyzerDescriptor(capabilities.analyzers.speechTranscribe,
|
|
83
|
-
speechVad: analyzerDescriptor(capabilities.analyzers.speechVad,
|
|
84
|
-
audioLoudness: analyzerDescriptor(capabilities.analyzers.audioLoudness,
|
|
85
|
-
audioNoise: analyzerDescriptor(Boolean(capabilities.analyzers.audioNoise),
|
|
86
|
-
visualTrack: analyzerDescriptor(capabilities.analyzers.visualTrack,
|
|
125
|
+
speechTranscribe: analyzerDescriptor(capabilities.analyzers.speechTranscribe, analyzerBackend("speechTranscribe"), "speech transcription"),
|
|
126
|
+
speechVad: analyzerDescriptor(capabilities.analyzers.speechVad, analyzerBackend("speechVad"), "speech VAD"),
|
|
127
|
+
audioLoudness: analyzerDescriptor(capabilities.analyzers.audioLoudness, analyzerBackend("audioLoudness"), "audio loudness analysis"),
|
|
128
|
+
audioNoise: analyzerDescriptor(Boolean(capabilities.analyzers.audioNoise), analyzerBackend("audioNoise"), "audio noise analysis"),
|
|
129
|
+
visualTrack: analyzerDescriptor(capabilities.analyzers.visualTrack, analyzerBackend("visualTrack"), "visual analysis"),
|
|
87
130
|
},
|
|
88
131
|
};
|
|
89
132
|
return { ...normalized, families };
|
|
90
133
|
}
|
|
134
|
+
function editingFamily(editor, overrides, backend) {
|
|
135
|
+
return {
|
|
136
|
+
compositeTransactions: descriptorFrom(overrides.compositeTransactions ?? Boolean(editor.compositeTransactions), backend, "verified", "composite editing transactions are unavailable"),
|
|
137
|
+
titlePlacement: descriptorFrom(overrides.titlePlacement ?? Boolean(editor.titlePlacement), backend, "verified", "title placement is unavailable"),
|
|
138
|
+
pictureInPicture: descriptorFrom(overrides.pictureInPicture ?? false, backend, "verified", "picture-in-picture editing is unavailable"),
|
|
139
|
+
masking: descriptorFrom(overrides.masking ?? false, backend, "verified", "masking is unavailable"),
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
export function createCapabilityPreflight(identity, capabilities, options = {}) {
|
|
143
|
+
const normalized = capabilities.families
|
|
144
|
+
? capabilities
|
|
145
|
+
: withCapabilityFamilies(capabilities, { backend: identity.backend });
|
|
146
|
+
const editor = normalized.editor;
|
|
147
|
+
const documentMode = identity.backend === "fixture"
|
|
148
|
+
? "fixture"
|
|
149
|
+
: editor.timelineArtifactWrite && !editor.timelineWrite
|
|
150
|
+
? "fcpxml-artifact"
|
|
151
|
+
: editor.canonicalTimelineMode === "metadata-only"
|
|
152
|
+
? "metadata-only"
|
|
153
|
+
: "canonical-live";
|
|
154
|
+
const processMode = options.processMode ?? "headless";
|
|
155
|
+
const mode = options.nativeWrite && processMode === "headed"
|
|
156
|
+
? "native-write"
|
|
157
|
+
: documentMode;
|
|
158
|
+
return {
|
|
159
|
+
mode,
|
|
160
|
+
documentMode,
|
|
161
|
+
processMode,
|
|
162
|
+
backend: identity.backend,
|
|
163
|
+
capabilities: normalized.families,
|
|
164
|
+
};
|
|
165
|
+
}
|
|
91
166
|
function descriptorFrom(value, backend, guarantee, unavailableReason) {
|
|
92
167
|
if (typeof value !== "boolean")
|
|
93
168
|
return value;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { diffSnapshots } from "../timeline/snapshot-diff.js";
|
|
2
2
|
import { sameMediaSourceIdentity } from "../domain/media.js";
|
|
3
|
+
import { sameRevision } from "./revision.js";
|
|
3
4
|
export class ContextEngine {
|
|
4
5
|
editor;
|
|
5
6
|
snapshots = new Map();
|
|
@@ -110,8 +111,12 @@ export class ContextEngine {
|
|
|
110
111
|
const next = structuredClone(snapshot);
|
|
111
112
|
next.media = next.media.map((media) => {
|
|
112
113
|
const understanding = this.mediaUnderstanding.get(media.mediaId);
|
|
113
|
-
if (!understanding
|
|
114
|
+
if (!understanding)
|
|
114
115
|
return media;
|
|
116
|
+
if (!isCurrentUnderstanding(understanding, media, snapshot.revision)) {
|
|
117
|
+
this.mediaUnderstanding.delete(media.mediaId);
|
|
118
|
+
return media;
|
|
119
|
+
}
|
|
115
120
|
const attached = structuredClone(understanding);
|
|
116
121
|
return {
|
|
117
122
|
...media,
|
|
@@ -127,6 +132,10 @@ export class ContextEngine {
|
|
|
127
132
|
return next;
|
|
128
133
|
}
|
|
129
134
|
}
|
|
135
|
+
function isCurrentUnderstanding(understanding, media, revision) {
|
|
136
|
+
return sameMediaSourceIdentity(understanding.sourceIdentity, media)
|
|
137
|
+
&& sameRevision(understanding.analysisRevision, revision);
|
|
138
|
+
}
|
|
130
139
|
function isOptionalLiveUnavailable(error) {
|
|
131
140
|
const message = String(error);
|
|
132
141
|
return message.includes("FINAL_CUT_LIVE_UNAVAILABLE") || message.includes("CAPABILITY_UNAVAILABLE: live Final Cut editor state");
|
|
@@ -3,6 +3,7 @@ import { resolve as resolvePath } from "node:path";
|
|
|
3
3
|
import { diffSnapshots } from "../timeline/snapshot-diff.js";
|
|
4
4
|
import { canonicalSnapshotDigest } from "../timeline/snapshot-digest.js";
|
|
5
5
|
import { sameRevision } from "../context/revision.js";
|
|
6
|
+
import { parseRational } from "../timeline/rational-time.js";
|
|
6
7
|
import { assertValidVerificationPolicy } from "../verification/verification.js";
|
|
7
8
|
export class EditService {
|
|
8
9
|
adapter;
|
|
@@ -47,6 +48,7 @@ export class EditService {
|
|
|
47
48
|
if (operation.baseRevision && !sameRevision(operation.baseRevision, before.revision)) {
|
|
48
49
|
throw new Error("STALE_CONTEXT: operation base revision does not match current editor state");
|
|
49
50
|
}
|
|
51
|
+
assertValidWorkflowOperation(operation);
|
|
50
52
|
this.assertSurfaceCapability(target, capabilities.editor);
|
|
51
53
|
if (!capabilities.editor.timelineSnapshotRead || !capabilities.editor.readAfterWrite || !capabilities.editor.rollback) {
|
|
52
54
|
throw new Error(`CAPABILITY_UNAVAILABLE: ${target.kind === "artifact" ? "artifact" : "editor timeline"} mutation requires snapshot, read-after-write, and rollback`);
|
|
@@ -84,8 +86,7 @@ export class EditService {
|
|
|
84
86
|
status: "APPLIED",
|
|
85
87
|
};
|
|
86
88
|
try {
|
|
87
|
-
|
|
88
|
-
transaction.after = transaction.attemptedAfter;
|
|
89
|
+
await this.reanalyzeForVerification(transaction, verificationPolicy);
|
|
89
90
|
}
|
|
90
91
|
catch (error) {
|
|
91
92
|
await this.adapter.restore(before, attemptedAfter.revision);
|
|
@@ -147,6 +148,7 @@ export class EditService {
|
|
|
147
148
|
}
|
|
148
149
|
if (request.operations.length === 0)
|
|
149
150
|
throw new Error("INVALID_OPERATION: composite edit requires operations");
|
|
151
|
+
request.operations.forEach(assertValidWorkflowOperation);
|
|
150
152
|
await this.assertCompositeCapabilities(request.operations, target.kind, capabilities.editor);
|
|
151
153
|
if (!this.adapter.previewTransaction) {
|
|
152
154
|
throw new Error("CAPABILITY_UNAVAILABLE: editor composite transaction preview");
|
|
@@ -191,6 +193,7 @@ export class EditService {
|
|
|
191
193
|
throw new Error("STALE_CONTEXT: preview base revision does not match current editor state");
|
|
192
194
|
}
|
|
193
195
|
this.assertTarget(preview.target, before);
|
|
196
|
+
preview.operations.forEach(assertValidWorkflowOperation);
|
|
194
197
|
await this.assertCompositeCapabilities(preview.operations, preview.target.kind);
|
|
195
198
|
if (!this.adapter.applyTransaction) {
|
|
196
199
|
throw new Error("CAPABILITY_UNAVAILABLE: editor composite transaction execution");
|
|
@@ -232,8 +235,7 @@ export class EditService {
|
|
|
232
235
|
status: "APPLIED",
|
|
233
236
|
};
|
|
234
237
|
try {
|
|
235
|
-
|
|
236
|
-
transaction.after = transaction.attemptedAfter;
|
|
238
|
+
await this.reanalyzeForVerification(transaction, verificationPolicy);
|
|
237
239
|
}
|
|
238
240
|
catch (error) {
|
|
239
241
|
await this.adapter.restore(before, attemptedAfter.revision);
|
|
@@ -277,12 +279,22 @@ export class EditService {
|
|
|
277
279
|
async verifyTransaction(transactionId) {
|
|
278
280
|
return this.getTransaction(transactionId).verification;
|
|
279
281
|
}
|
|
282
|
+
async reanalyzeForVerification(transaction, policy) {
|
|
283
|
+
const requirements = postWriteAnalysisRequirements(policy);
|
|
284
|
+
if (!Object.values(requirements).some(Boolean))
|
|
285
|
+
return;
|
|
286
|
+
transaction.attemptedAfter = await this.analysis.reanalyzeAffectedRanges(transaction, requirements);
|
|
287
|
+
transaction.after = transaction.attemptedAfter;
|
|
288
|
+
}
|
|
280
289
|
async undo(transactionId) {
|
|
281
290
|
const transaction = this.getTransaction(transactionId);
|
|
282
291
|
const current = await this.project.inspectProject();
|
|
283
292
|
if (current.projectId !== transaction.before.projectId || current.timeline.id !== transaction.before.timeline.id) {
|
|
284
293
|
throw new Error(`TARGET_MISMATCH: cannot undo ${transaction.before.projectId}/${transaction.before.timeline.id} while ${current.projectId}/${current.timeline.id} is active`);
|
|
285
294
|
}
|
|
295
|
+
if (!sameRevision(current.revision, transaction.after.revision)) {
|
|
296
|
+
throw new Error(`STALE_CONTEXT: transaction ${transactionId} changed after edit`);
|
|
297
|
+
}
|
|
286
298
|
await this.adapter.restore(transaction.before, current.revision);
|
|
287
299
|
const restored = await this.project.inspectProject();
|
|
288
300
|
this.assertRestored(transaction.before, restored);
|
|
@@ -411,3 +423,27 @@ export class EditService {
|
|
|
411
423
|
}
|
|
412
424
|
}
|
|
413
425
|
}
|
|
426
|
+
function postWriteAnalysisRequirements(policy) {
|
|
427
|
+
const assertions = policy.assertions ?? [];
|
|
428
|
+
return {
|
|
429
|
+
speech: policy.requireSpeechContinuity === true,
|
|
430
|
+
audio: policy.maxTruePeakDb !== undefined
|
|
431
|
+
|| policy.targetLufs !== undefined
|
|
432
|
+
|| assertions.some((assertion) => assertion.type === "audio-audibility" || assertion.type === "audio-loudness"),
|
|
433
|
+
noise: assertions.some((assertion) => assertion.type === "audio-noise"),
|
|
434
|
+
visual: assertions.some((assertion) => assertion.type === "visual-content"),
|
|
435
|
+
};
|
|
436
|
+
}
|
|
437
|
+
function assertValidWorkflowOperation(operation) {
|
|
438
|
+
if (operation.type !== "trim-clip")
|
|
439
|
+
return;
|
|
440
|
+
if (!Number.isFinite(operation.duration) || operation.duration <= 0) {
|
|
441
|
+
throw new Error("INVALID_OPERATION: clip duration must be positive");
|
|
442
|
+
}
|
|
443
|
+
if (!operation.durationTime)
|
|
444
|
+
return;
|
|
445
|
+
const duration = parseRational(operation.durationTime, "INVALID_OPERATION");
|
|
446
|
+
if (duration.value <= 0n) {
|
|
447
|
+
throw new Error("INVALID_OPERATION: clip duration must be positive");
|
|
448
|
+
}
|
|
449
|
+
}
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { diffSnapshots } from "../timeline/snapshot-diff.js";
|
|
3
3
|
import { sameRevision } from "../context/revision.js";
|
|
4
|
-
import { translateRationalRange } from "../timeline/rational-time.js";
|
|
5
4
|
import { canonicalSnapshotDigest } from "../timeline/snapshot-digest.js";
|
|
6
5
|
import { planFillerRemoval } from "../speech/filler-removal.js";
|
|
6
|
+
import { bindSpeechAnalysis } from "../speech/analysis.js";
|
|
7
|
+
import { mapSourceRangeToSequenceRange } from "../speech/mapping.js";
|
|
7
8
|
export class FillerRemovalService {
|
|
8
9
|
adapter;
|
|
9
10
|
project;
|
|
@@ -48,15 +49,20 @@ export class FillerRemovalService {
|
|
|
48
49
|
};
|
|
49
50
|
if (localRange.end <= localRange.start)
|
|
50
51
|
continue;
|
|
51
|
-
const
|
|
52
|
-
|
|
53
|
-
|
|
52
|
+
const sourceStart = clip.sourceStart ?? 0;
|
|
53
|
+
const sourceRange = {
|
|
54
|
+
start: sourceStart + localRange.start,
|
|
55
|
+
end: sourceStart + localRange.end,
|
|
56
|
+
};
|
|
57
|
+
const speech = bindSpeechAnalysis(await this.options.speechAnalyzer.analyze({ project: before, media }, sourceRange), { input: { project: before, media }, range: sourceRange, provider: this.options.speechAnalyzer.descriptor });
|
|
58
|
+
analysisRangesByClip.set(clip.id, structuredClone(sourceRange));
|
|
59
|
+
for (const candidate of planFillerRemoval(speech.words, sourceRange, request)) {
|
|
54
60
|
candidates.push({
|
|
55
61
|
...candidate,
|
|
56
62
|
clipId: clip.id,
|
|
57
63
|
mediaId: clip.mediaId,
|
|
58
64
|
sourceRange: structuredClone(candidate.range),
|
|
59
|
-
range:
|
|
65
|
+
range: translateSourceRangeToSequence(clip, candidate.range),
|
|
60
66
|
});
|
|
61
67
|
}
|
|
62
68
|
}
|
|
@@ -205,7 +211,7 @@ export class FillerRemovalService {
|
|
|
205
211
|
const fillerCandidates = record.preview.candidates.filter((candidate) => candidate.clipId === clipId);
|
|
206
212
|
const deletes = fillerCandidates.map((candidate) => candidate.sourceRange);
|
|
207
213
|
const postEditRange = translateRangeAfterDeletes(analysisRange, deletes);
|
|
208
|
-
const speech = await this.options.speechAnalyzer.analyze({ project: next, media }, postEditRange);
|
|
214
|
+
const speech = bindSpeechAnalysis(await this.options.speechAnalyzer.analyze({ project: next, media }, postEditRange), { input: { project: next, media }, range: postEditRange, provider: this.options.speechAnalyzer.descriptor });
|
|
209
215
|
const retainedWords = transaction.before.media
|
|
210
216
|
.find((candidate) => candidate.mediaId === beforeClip.mediaId)
|
|
211
217
|
?.speech?.words ?? [];
|
|
@@ -213,6 +219,7 @@ export class FillerRemovalService {
|
|
|
213
219
|
.filter((word) => !fillerCandidates.some((candidate) => sameSpeechWord(word, candidate.word)))
|
|
214
220
|
.map((word) => translateSpeechWordAfterDeletes(word, deletes));
|
|
215
221
|
media.speech = {
|
|
222
|
+
...speech,
|
|
216
223
|
words: [
|
|
217
224
|
...translatedRetainedWords.filter((word) => !rangesOverlap(word.start, word.end, postEditRange.start, postEditRange.end)),
|
|
218
225
|
...speech.words.filter((word) => word.start >= postEditRange.start && word.end <= postEditRange.end),
|
|
@@ -317,7 +324,7 @@ function verifyFillerSpeechContinuity(transaction, record) {
|
|
|
317
324
|
if (expected.text.trim().toLowerCase() !== actual.text.trim().toLowerCase()
|
|
318
325
|
|| Math.abs(expected.start - actual.start) > 0.02
|
|
319
326
|
|| Math.abs(expected.end - actual.end) > 0.02
|
|
320
|
-
|| actual.end > afterClip.duration + 0.02) {
|
|
327
|
+
|| actual.end > (afterClip.sourceStart ?? 0) + afterClip.duration + 0.02) {
|
|
321
328
|
return {
|
|
322
329
|
name: "filler-speech-continuity",
|
|
323
330
|
passed: false,
|
|
@@ -335,3 +342,20 @@ function verifyFillerSpeechContinuity(transaction, record) {
|
|
|
335
342
|
function sameSpeechWord(left, right) {
|
|
336
343
|
return left.text === right.text && left.start === right.start && left.end === right.end;
|
|
337
344
|
}
|
|
345
|
+
function translateSourceRangeToSequence(clip, sourceRange) {
|
|
346
|
+
const sourceStart = clip.sourceStart ?? 0;
|
|
347
|
+
return mapSourceRangeToSequenceRange(sourceRange, {
|
|
348
|
+
sourceRange: {
|
|
349
|
+
start: sourceStart,
|
|
350
|
+
end: sourceStart + clip.duration,
|
|
351
|
+
startTime: clip.sourceStartTime,
|
|
352
|
+
durationTime: clip.durationTime,
|
|
353
|
+
},
|
|
354
|
+
sequenceRange: {
|
|
355
|
+
start: clip.start,
|
|
356
|
+
end: clip.start + clip.duration,
|
|
357
|
+
startTime: clip.startTime,
|
|
358
|
+
durationTime: clip.durationTime,
|
|
359
|
+
},
|
|
360
|
+
});
|
|
361
|
+
}
|
|
@@ -5,6 +5,7 @@ export function planRoughCut(entries, revision, request) {
|
|
|
5
5
|
throw new Error("INVALID_ROUGH_CUT_REQUEST: maxShots must be a positive integer");
|
|
6
6
|
}
|
|
7
7
|
const candidates = entries
|
|
8
|
+
.filter((entry) => entry.sourceIdentity.mediaKind !== "audio")
|
|
8
9
|
.flatMap((entry) => entry.semantic.usableRanges.map((range) => ({ entry, range })))
|
|
9
10
|
.sort((left, right) => {
|
|
10
11
|
const mediaOrder = left.entry.sourceIdentity.mediaId.localeCompare(right.entry.sourceIdentity.mediaId);
|
|
@@ -15,9 +16,14 @@ export function planRoughCut(entries, revision, request) {
|
|
|
15
16
|
return left.range.end - right.range.end;
|
|
16
17
|
});
|
|
17
18
|
const shots = candidates.slice(0, maxShots).map(({ entry, range }, index) => shotFor(entry, range, request, index + 1));
|
|
18
|
-
const
|
|
19
|
-
|
|
20
|
-
|
|
19
|
+
const excludedAudioMediaIds = entries
|
|
20
|
+
.filter((entry) => entry.sourceIdentity.mediaKind === "audio" && entry.semantic.usableRanges.length > 0)
|
|
21
|
+
.map((entry) => entry.sourceIdentity.mediaId);
|
|
22
|
+
const warnings = excludedAudioMediaIds.length > 0
|
|
23
|
+
? [`Excluded audio-only media from rough-cut shot candidates: ${excludedAudioMediaIds.join(", ")}`]
|
|
24
|
+
: entries.length > 0 && candidates.length === 0
|
|
25
|
+
? ["No matching media has an explicitly analyzed usable range"]
|
|
26
|
+
: [];
|
|
21
27
|
return {
|
|
22
28
|
planner: { id: "framekit.rough-cut", version: 1 },
|
|
23
29
|
revision: structuredClone(revision),
|
|
@@ -147,8 +147,8 @@ export class AgentVideoRuntime {
|
|
|
147
147
|
async contextChangesSince(revision, waitMs = 0) {
|
|
148
148
|
return this.contexts.contextChangesSince(revision, waitMs);
|
|
149
149
|
}
|
|
150
|
-
async analyzeSpeech(mediaId) {
|
|
151
|
-
return this.media.analyzeSpeech(mediaId);
|
|
150
|
+
async analyzeSpeech(mediaId, range) {
|
|
151
|
+
return this.media.analyzeSpeech(mediaId, range);
|
|
152
152
|
}
|
|
153
153
|
async analyzeAudio(mediaId) {
|
|
154
154
|
return this.media.analyzeAudio(mediaId);
|