@morshoto/framekit 0.1.0 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -17
- package/dist-package/adapters/final-cut/typescript/src/analyzers.js +57 -0
- package/dist-package/adapters/final-cut/typescript/src/connection.js +5 -1
- package/dist-package/adapters/final-cut/typescript/src/export.js +231 -8
- package/dist-package/adapters/final-cut/typescript/src/fcpxml.js +55 -5
- package/dist-package/adapters/final-cut/typescript/src/live.js +2 -2
- package/dist-package/adapters/final-cut/typescript/src/native.js +1000 -100
- package/dist-package/adapters/final-cut/typescript/src/publisher.js +40 -5
- package/dist-package/adapters/final-cut/typescript/src/session.js +21 -5
- package/dist-package/apps/mcp-server/src/main.js +9 -1
- package/dist-package/apps/mcp-server/src/routing.js +172 -0
- package/dist-package/apps/mcp-server/src/server.js +780 -33
- package/dist-package/packages/runtime/src/application/duration-policy-service.js +6 -0
- package/dist-package/packages/runtime/src/application/media-analysis-service.js +370 -0
- package/dist-package/packages/runtime/src/application/project-service.js +127 -0
- package/dist-package/packages/runtime/src/application/runtime-options.js +1 -0
- package/dist-package/packages/runtime/src/application/transaction-store.js +12 -0
- package/dist-package/packages/runtime/src/audio/dialogue-normalization.js +64 -0
- package/dist-package/packages/runtime/src/audio/noise-reduction.js +46 -0
- package/dist-package/packages/runtime/src/capabilities.js +112 -1
- package/dist-package/packages/runtime/src/color-correction.js +48 -0
- package/dist-package/packages/runtime/src/context/context-engine.js +11 -6
- package/dist-package/packages/runtime/src/context/context-service.js +30 -0
- package/dist-package/packages/runtime/src/context/revision.js +3 -0
- package/dist-package/packages/runtime/src/domain/capabilities.js +1 -0
- package/dist-package/packages/runtime/src/domain/context.js +1 -0
- package/dist-package/packages/runtime/src/domain/diff.js +1 -0
- package/dist-package/packages/runtime/src/domain/duration.js +1 -0
- package/dist-package/packages/runtime/src/domain/editing.js +1 -0
- package/dist-package/packages/runtime/src/domain/index.js +12 -0
- package/dist-package/packages/runtime/src/domain/media.js +7 -0
- package/dist-package/packages/runtime/src/domain/ports.js +1 -0
- package/dist-package/packages/runtime/src/domain/primitives.js +1 -0
- package/dist-package/packages/runtime/src/domain/project.js +1 -0
- package/dist-package/packages/runtime/src/domain/rough-cut.js +69 -0
- package/dist-package/packages/runtime/src/domain/skills.js +33 -0
- package/dist-package/packages/runtime/src/domain/types.js +5 -1
- package/dist-package/packages/runtime/src/domain/verification.js +1 -0
- package/dist-package/packages/runtime/src/editing/dialogue-normalization-service.js +141 -0
- package/dist-package/packages/runtime/src/editing/duration-policy.js +187 -0
- package/dist-package/packages/runtime/src/editing/edit-service.js +413 -0
- package/dist-package/packages/runtime/src/editing/filler-removal-service.js +337 -0
- package/dist-package/packages/runtime/src/editing/music-service.js +85 -0
- package/dist-package/packages/runtime/src/index.js +12 -3
- package/dist-package/packages/runtime/src/planning/rough-cut.js +65 -0
- package/dist-package/packages/runtime/src/runtime.js +130 -825
- package/dist-package/packages/runtime/src/skills/builtin.js +339 -0
- package/dist-package/packages/runtime/src/skills/index.js +3 -0
- package/dist-package/packages/runtime/src/skills/registry.js +74 -0
- package/dist-package/packages/runtime/src/skills/requirements.js +170 -0
- package/dist-package/packages/runtime/src/skills/runtime.js +239 -0
- package/dist-package/packages/runtime/src/speech/filler-detector.js +132 -0
- package/dist-package/packages/runtime/src/speech/safe-cut-resolver.js +314 -0
- package/dist-package/packages/runtime/src/timeline/rational-time.js +58 -0
- package/dist-package/packages/runtime/src/verification/verification.js +712 -1
- package/dist-package/packages/testkit/src/fixture-analyzers.js +27 -0
- package/dist-package/packages/testkit/src/in-memory-editor.js +286 -3
- package/dist-package/packages/testkit/src/index.js +1 -0
- package/dist-package/packages/testkit/src/skill-fixtures.js +64 -0
- package/package.json +15 -7
- /package/dist-package/packages/runtime/src/{diff/diff.js → timeline/snapshot-diff.js} +0 -0
- /package/dist-package/packages/runtime/src/{snapshot-digest.js → timeline/snapshot-digest.js} +0 -0
|
@@ -0,0 +1,370 @@
|
|
|
1
|
+
import { planRoughCut } from "../planning/rough-cut.js";
|
|
2
|
+
export class MediaAnalysisService {
|
|
3
|
+
project;
|
|
4
|
+
context;
|
|
5
|
+
options;
|
|
6
|
+
constructor(project, context, options) {
|
|
7
|
+
this.project = project;
|
|
8
|
+
this.context = context;
|
|
9
|
+
this.options = options;
|
|
10
|
+
}
|
|
11
|
+
async analyzeSpeech(mediaId, range) {
|
|
12
|
+
if (!this.options.speechAnalyzer)
|
|
13
|
+
throw new Error("CAPABILITY_UNAVAILABLE: speech analysis");
|
|
14
|
+
const project = await this.project.inspectProject();
|
|
15
|
+
const media = findMedia(project, mediaId);
|
|
16
|
+
return this.options.speechAnalyzer.analyze({ project, media }, range);
|
|
17
|
+
}
|
|
18
|
+
async analyzeAudio(mediaId) {
|
|
19
|
+
if (!this.options.audioAnalyzer)
|
|
20
|
+
throw new Error("CAPABILITY_UNAVAILABLE: audio analysis");
|
|
21
|
+
const project = await this.project.inspectProject();
|
|
22
|
+
const media = findMedia(project, mediaId);
|
|
23
|
+
return this.options.audioAnalyzer.analyze({ project, media });
|
|
24
|
+
}
|
|
25
|
+
async analyzeNoise(mediaId, range) {
|
|
26
|
+
if (!this.options.noiseAnalyzer)
|
|
27
|
+
throw new Error("CAPABILITY_UNAVAILABLE: audio noise analysis");
|
|
28
|
+
const project = await this.project.inspectProject();
|
|
29
|
+
const media = findMedia(project, mediaId);
|
|
30
|
+
return this.options.noiseAnalyzer.analyze({ project, media }, range);
|
|
31
|
+
}
|
|
32
|
+
async measureAudio(mediaId, occurrenceId) {
|
|
33
|
+
if (!this.options.audioAnalyzer)
|
|
34
|
+
throw new Error("CAPABILITY_UNAVAILABLE: audio analysis");
|
|
35
|
+
const project = await this.project.inspectProject();
|
|
36
|
+
const clip = project.timeline.clips.find((candidate) => candidate.id === occurrenceId);
|
|
37
|
+
if (!clip)
|
|
38
|
+
throw new Error(`OCCURRENCE_NOT_FOUND: ${occurrenceId}`);
|
|
39
|
+
if (clip.mediaId !== mediaId) {
|
|
40
|
+
throw new Error(`TARGET_MISMATCH: occurrence ${occurrenceId} does not reference media ${mediaId}`);
|
|
41
|
+
}
|
|
42
|
+
const media = findMedia(project, mediaId);
|
|
43
|
+
const requestedRange = { start: 0, end: clip.duration };
|
|
44
|
+
const analysis = await this.options.audioAnalyzer.analyze({ project, media }, requestedRange);
|
|
45
|
+
const analyzedDurationSeconds = analysis.analyzedDurationSeconds ?? clip.duration;
|
|
46
|
+
const valid = analysis.valid !== false
|
|
47
|
+
&& Number.isFinite(analysis.integratedLufs)
|
|
48
|
+
&& Number.isFinite(analysis.truePeakDb)
|
|
49
|
+
&& Number.isFinite(analysis.silenceMs)
|
|
50
|
+
&& Number.isFinite(analyzedDurationSeconds)
|
|
51
|
+
&& analyzedDurationSeconds > 0;
|
|
52
|
+
return {
|
|
53
|
+
mediaId,
|
|
54
|
+
occurrenceId,
|
|
55
|
+
requestedRange,
|
|
56
|
+
measuredRange: {
|
|
57
|
+
start: 0,
|
|
58
|
+
end: Number.isFinite(analyzedDurationSeconds) ? analyzedDurationSeconds : clip.duration,
|
|
59
|
+
},
|
|
60
|
+
revision: project.revision,
|
|
61
|
+
provider: this.options.audioAnalyzer.descriptor ?? { id: "framekit.audio", provider: "unknown" },
|
|
62
|
+
dialoguePresent: analysis.dialoguePresent
|
|
63
|
+
?? Boolean(media.speech?.words.some((word) => word.filler !== true)),
|
|
64
|
+
integratedLufs: analysis.integratedLufs,
|
|
65
|
+
truePeakDb: analysis.truePeakDb,
|
|
66
|
+
silenceMs: analysis.silenceMs,
|
|
67
|
+
analyzedDurationSeconds,
|
|
68
|
+
valid,
|
|
69
|
+
...(analysis.invalidReason ? { invalidReason: analysis.invalidReason } : {}),
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
async measureNoise(mediaId, occurrenceId) {
|
|
73
|
+
if (!this.options.noiseAnalyzer)
|
|
74
|
+
throw new Error("CAPABILITY_UNAVAILABLE: audio noise analysis");
|
|
75
|
+
const project = await this.project.inspectProject();
|
|
76
|
+
const clip = project.timeline.clips.find((candidate) => candidate.id === occurrenceId);
|
|
77
|
+
if (!clip)
|
|
78
|
+
throw new Error(`OCCURRENCE_NOT_FOUND: ${occurrenceId}`);
|
|
79
|
+
if (clip.mediaId !== mediaId) {
|
|
80
|
+
throw new Error(`TARGET_MISMATCH: occurrence ${occurrenceId} does not reference media ${mediaId}`);
|
|
81
|
+
}
|
|
82
|
+
const media = findMedia(project, mediaId);
|
|
83
|
+
const requestedRange = { start: 0, end: clip.duration };
|
|
84
|
+
const analysis = await this.options.noiseAnalyzer.analyze({ project, media }, requestedRange);
|
|
85
|
+
const measuredEnd = analysis.affectedRanges.reduce((end, range) => Math.max(end, range.end), 0);
|
|
86
|
+
const measuredRange = {
|
|
87
|
+
start: 0,
|
|
88
|
+
end: Math.max(analysis.affectedRanges.length > 0 ? measuredEnd : requestedRange.end, 0),
|
|
89
|
+
};
|
|
90
|
+
const valid = analysis.valid !== false
|
|
91
|
+
&& Number.isFinite(analysis.noiseFloorDb)
|
|
92
|
+
&& Number.isFinite(analysis.recommendedReductionDb)
|
|
93
|
+
&& Number.isFinite(analysis.confidence)
|
|
94
|
+
&& analysis.confidence >= 0
|
|
95
|
+
&& analysis.confidence <= 1
|
|
96
|
+
&& analysis.affectedRanges.every((range) => Number.isFinite(range.start)
|
|
97
|
+
&& Number.isFinite(range.end) && range.start >= 0 && range.end > range.start);
|
|
98
|
+
return {
|
|
99
|
+
...structuredClone(analysis),
|
|
100
|
+
mediaId,
|
|
101
|
+
occurrenceId,
|
|
102
|
+
requestedRange,
|
|
103
|
+
measuredRange,
|
|
104
|
+
revision: project.revision,
|
|
105
|
+
provider: this.options.noiseAnalyzer.descriptor ?? { id: "framekit.audio-noise", provider: "unknown" },
|
|
106
|
+
valid,
|
|
107
|
+
...(analysis.invalidReason ? { invalidReason: analysis.invalidReason } : {}),
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
async analyzeVisual(mediaId, range) {
|
|
111
|
+
if (!this.options.visualAnalyzer)
|
|
112
|
+
throw new Error("CAPABILITY_UNAVAILABLE: visual analysis");
|
|
113
|
+
const project = await this.project.inspectProject();
|
|
114
|
+
const media = findMedia(project, mediaId);
|
|
115
|
+
return this.options.visualAnalyzer.analyze({ project, media }, range);
|
|
116
|
+
}
|
|
117
|
+
async understandMedia(mediaId) {
|
|
118
|
+
const project = await this.project.inspectProject();
|
|
119
|
+
const media = findMedia(project, mediaId);
|
|
120
|
+
const input = { project, media };
|
|
121
|
+
const [speechResult, audioResult, noiseResult, visualResult, metadataResult] = await Promise.all([
|
|
122
|
+
settle(() => this.options.speechAnalyzer?.analyze(input)),
|
|
123
|
+
settle(() => this.options.audioAnalyzer?.analyze(input)),
|
|
124
|
+
settle(() => this.options.noiseAnalyzer?.analyze(input)),
|
|
125
|
+
settle(() => this.options.visualAnalyzer?.analyze(input)),
|
|
126
|
+
settle(() => this.options.metadataAnalyzer?.analyze(input)),
|
|
127
|
+
]);
|
|
128
|
+
const speech = fulfilledValue(speechResult);
|
|
129
|
+
const audio = fulfilledValue(audioResult);
|
|
130
|
+
const noise = fulfilledValue(noiseResult);
|
|
131
|
+
const visual = fulfilledValue(visualResult);
|
|
132
|
+
const metadata = fulfilledValue(metadataResult);
|
|
133
|
+
const sourceIdentity = sourceIdentityOf(media);
|
|
134
|
+
const understanding = {
|
|
135
|
+
mediaId: media.mediaId,
|
|
136
|
+
source: media.source,
|
|
137
|
+
sourceIdentity,
|
|
138
|
+
...(metadata ? { metadata } : {}),
|
|
139
|
+
...(speech ? { speech } : {}),
|
|
140
|
+
...(audio ? { audio } : {}),
|
|
141
|
+
...(noise ? { noise } : {}),
|
|
142
|
+
...(visual ? { visual } : {}),
|
|
143
|
+
semantic: semanticFromAnalyses(speech, audio, visual, metadata),
|
|
144
|
+
analysis: [
|
|
145
|
+
analysisStatus("speech", this.options.speechAnalyzer, sourceIdentity, Boolean(speech), [], failureReason(speechResult)),
|
|
146
|
+
analysisStatus("audio", this.options.audioAnalyzer, sourceIdentity, Boolean(audio), [], failureReason(audioResult)),
|
|
147
|
+
analysisStatus("noise", this.options.noiseAnalyzer, sourceIdentity, Boolean(noise), noise?.affectedRanges, failureReason(noiseResult)),
|
|
148
|
+
analysisStatus("visual", this.options.visualAnalyzer, sourceIdentity, Boolean(visual), [], failureReason(visualResult)),
|
|
149
|
+
analysisStatus("metadata", this.options.metadataAnalyzer, sourceIdentity, Boolean(metadata), metadata?.usableRanges, failureReason(metadataResult)),
|
|
150
|
+
],
|
|
151
|
+
analysisRevision: project.revision,
|
|
152
|
+
};
|
|
153
|
+
this.context.attachMediaUnderstanding(understanding);
|
|
154
|
+
return structuredClone(understanding);
|
|
155
|
+
}
|
|
156
|
+
async inspectMedia(mediaId) {
|
|
157
|
+
const project = await this.project.inspectProject();
|
|
158
|
+
return findMedia(project, mediaId);
|
|
159
|
+
}
|
|
160
|
+
async searchMedia(query) {
|
|
161
|
+
const project = await this.project.inspectProject();
|
|
162
|
+
const normalized = query.trim().toLowerCase();
|
|
163
|
+
return project.media.filter((media) => media.mediaId.toLowerCase().includes(normalized) || media.source.toLowerCase().includes(normalized));
|
|
164
|
+
}
|
|
165
|
+
async indexMedia(query = {}) {
|
|
166
|
+
const project = await this.project.inspectProject();
|
|
167
|
+
return this.indexFromProject(project, query);
|
|
168
|
+
}
|
|
169
|
+
async planRoughCut(request) {
|
|
170
|
+
const project = await this.project.inspectProject();
|
|
171
|
+
const { maxShots: _maxShots, ...query } = request;
|
|
172
|
+
return planRoughCut(this.indexFromProject(project, query), project.revision, request);
|
|
173
|
+
}
|
|
174
|
+
indexFromProject(project, query) {
|
|
175
|
+
return project.media
|
|
176
|
+
.map((media) => ({
|
|
177
|
+
sourceIdentity: sourceIdentityOf(media),
|
|
178
|
+
semantic: media.semantic ?? emptySemanticDescription(),
|
|
179
|
+
analysis: media.analysis ?? [
|
|
180
|
+
analysisStatus("speech", this.options.speechAnalyzer, sourceIdentityOf(media), false),
|
|
181
|
+
analysisStatus("audio", this.options.audioAnalyzer, sourceIdentityOf(media), false),
|
|
182
|
+
analysisStatus("noise", this.options.noiseAnalyzer, sourceIdentityOf(media), false),
|
|
183
|
+
analysisStatus("visual", this.options.visualAnalyzer, sourceIdentityOf(media), false),
|
|
184
|
+
analysisStatus("metadata", this.options.metadataAnalyzer, sourceIdentityOf(media), false),
|
|
185
|
+
],
|
|
186
|
+
...(media.analysisRevision ? { analysisRevision: media.analysisRevision } : {}),
|
|
187
|
+
}))
|
|
188
|
+
.filter((entry) => matchesMediaIndexQuery(entry, query));
|
|
189
|
+
}
|
|
190
|
+
async reanalyzeAffectedRanges(transaction) {
|
|
191
|
+
const affectedMediaRanges = transaction.diff.affectedRanges.flatMap((range) => transaction.attemptedAfter.timeline.clips.flatMap((clip) => {
|
|
192
|
+
const intersectionStart = Math.max(range.start, clip.start);
|
|
193
|
+
const intersectionEnd = Math.min(range.end, clip.start + clip.duration);
|
|
194
|
+
if (!clip.mediaId || intersectionStart >= intersectionEnd)
|
|
195
|
+
return [];
|
|
196
|
+
return [{
|
|
197
|
+
mediaId: clip.mediaId,
|
|
198
|
+
range: {
|
|
199
|
+
start: intersectionStart - clip.start,
|
|
200
|
+
end: intersectionEnd - clip.start,
|
|
201
|
+
},
|
|
202
|
+
}];
|
|
203
|
+
}));
|
|
204
|
+
const mediaIds = new Set(affectedMediaRanges.map(({ mediaId }) => mediaId));
|
|
205
|
+
if (mediaIds.size === 0)
|
|
206
|
+
return transaction.attemptedAfter;
|
|
207
|
+
const next = structuredClone(transaction.attemptedAfter);
|
|
208
|
+
for (const mediaId of mediaIds) {
|
|
209
|
+
const media = next.media.find((candidate) => candidate.mediaId === mediaId);
|
|
210
|
+
if (!media)
|
|
211
|
+
continue;
|
|
212
|
+
const ranges = affectedMediaRanges
|
|
213
|
+
.filter((affected) => affected.mediaId === mediaId)
|
|
214
|
+
.map((affected) => affected.range);
|
|
215
|
+
const input = { project: next, media };
|
|
216
|
+
if (this.options.speechAnalyzer) {
|
|
217
|
+
const analyses = await Promise.all(ranges.map((range) => this.options.speechAnalyzer.analyze(input, range)));
|
|
218
|
+
media.speech = { words: analyses.flatMap((analysis) => analysis.words) };
|
|
219
|
+
}
|
|
220
|
+
if (this.options.audioAnalyzer) {
|
|
221
|
+
const analyses = await Promise.all(ranges.map((range) => this.options.audioAnalyzer.analyze(input, range)));
|
|
222
|
+
if (analyses[analyses.length - 1])
|
|
223
|
+
media.audio = analyses[analyses.length - 1];
|
|
224
|
+
}
|
|
225
|
+
if (this.options.noiseAnalyzer) {
|
|
226
|
+
const analyses = await Promise.all(ranges.map((range) => this.options.noiseAnalyzer.analyze(input, range)));
|
|
227
|
+
if (analyses[analyses.length - 1])
|
|
228
|
+
media.noise = analyses[analyses.length - 1];
|
|
229
|
+
}
|
|
230
|
+
if (this.options.visualAnalyzer) {
|
|
231
|
+
const analyses = await Promise.all(ranges.map((range) => this.options.visualAnalyzer.analyze(input, range)));
|
|
232
|
+
media.visual = {
|
|
233
|
+
scenes: analyses.flatMap((analysis) => analysis.scenes),
|
|
234
|
+
subjects: analyses.flatMap((analysis) => analysis.subjects),
|
|
235
|
+
keyframes: analyses.flatMap((analysis) => analysis.keyframes),
|
|
236
|
+
motion: analyses[analyses.length - 1]?.motion,
|
|
237
|
+
};
|
|
238
|
+
}
|
|
239
|
+
if (this.options.speechAnalyzer || this.options.audioAnalyzer || this.options.noiseAnalyzer || this.options.visualAnalyzer) {
|
|
240
|
+
for (const candidate of next.media) {
|
|
241
|
+
if (candidate.mediaId === mediaId)
|
|
242
|
+
candidate.analysisRevision = next.revision.id;
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
return next;
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
function sourceIdentityOf(media) {
|
|
250
|
+
return {
|
|
251
|
+
mediaId: media.mediaId,
|
|
252
|
+
source: media.source,
|
|
253
|
+
...(media.sourceDigest ? { sourceDigest: media.sourceDigest } : {}),
|
|
254
|
+
...(media.mediaKind ? { mediaKind: media.mediaKind } : {}),
|
|
255
|
+
...(media.duration !== undefined ? { duration: media.duration } : {}),
|
|
256
|
+
};
|
|
257
|
+
}
|
|
258
|
+
function semanticFromAnalyses(speech, audio, visual, metadata) {
|
|
259
|
+
return {
|
|
260
|
+
subjects: [
|
|
261
|
+
...(visual?.subjects ?? []).map((subject) => ({ value: subject.label, confidence: subject.confidence })),
|
|
262
|
+
...(metadata?.subjects ?? []),
|
|
263
|
+
],
|
|
264
|
+
scenes: [
|
|
265
|
+
...(visual?.scenes ?? []).flatMap((scene) => scene.label ? [{ value: scene.label, confidence: scene.confidence ?? 1 }] : []),
|
|
266
|
+
...(metadata?.scenes ?? []),
|
|
267
|
+
],
|
|
268
|
+
environments: metadata?.environments ? structuredClone(metadata.environments) : [],
|
|
269
|
+
timeOfDay: metadata?.timeOfDay ? structuredClone(metadata.timeOfDay) : [],
|
|
270
|
+
moods: metadata?.moods ? structuredClone(metadata.moods) : [],
|
|
271
|
+
...(visual?.motion ? { motion: structuredClone(visual.motion) } : {}),
|
|
272
|
+
usableRanges: metadata?.usableRanges ? structuredClone(metadata.usableRanges) : [],
|
|
273
|
+
...(speech ? { transcript: speech.words.map((word) => word.text).join(" ") } : {}),
|
|
274
|
+
...(audio ? {
|
|
275
|
+
audio: {
|
|
276
|
+
present: true,
|
|
277
|
+
integratedLufs: audio.integratedLufs,
|
|
278
|
+
truePeakDb: audio.truePeakDb,
|
|
279
|
+
silenceMs: audio.silenceMs,
|
|
280
|
+
},
|
|
281
|
+
} : {}),
|
|
282
|
+
};
|
|
283
|
+
}
|
|
284
|
+
function emptySemanticDescription() {
|
|
285
|
+
return {
|
|
286
|
+
subjects: [],
|
|
287
|
+
scenes: [],
|
|
288
|
+
environments: [],
|
|
289
|
+
timeOfDay: [],
|
|
290
|
+
moods: [],
|
|
291
|
+
usableRanges: [],
|
|
292
|
+
};
|
|
293
|
+
}
|
|
294
|
+
function matchesMediaIndexQuery(entry, query) {
|
|
295
|
+
const text = query.query?.trim().toLowerCase();
|
|
296
|
+
const values = [
|
|
297
|
+
entry.sourceIdentity.mediaId,
|
|
298
|
+
entry.sourceIdentity.source,
|
|
299
|
+
entry.semantic.transcript ?? "",
|
|
300
|
+
...entry.semantic.subjects.map((tag) => tag.value),
|
|
301
|
+
...entry.semantic.scenes.map((tag) => tag.value),
|
|
302
|
+
...entry.semantic.environments.map((tag) => tag.value),
|
|
303
|
+
...entry.semantic.timeOfDay.map((tag) => tag.value),
|
|
304
|
+
...entry.semantic.moods.map((tag) => tag.value),
|
|
305
|
+
].map((value) => value.toLowerCase());
|
|
306
|
+
if (text && !values.some((value) => value.includes(text)))
|
|
307
|
+
return false;
|
|
308
|
+
if (query.subject && !matchesTag(entry.semantic.subjects, query.subject))
|
|
309
|
+
return false;
|
|
310
|
+
if (query.scene && !matchesTag(entry.semantic.scenes, query.scene))
|
|
311
|
+
return false;
|
|
312
|
+
if (query.environment && !matchesTag(entry.semantic.environments, query.environment))
|
|
313
|
+
return false;
|
|
314
|
+
if (query.timeOfDay && !matchesTag(entry.semantic.timeOfDay, query.timeOfDay))
|
|
315
|
+
return false;
|
|
316
|
+
if (query.mood && !matchesTag(entry.semantic.moods, query.mood))
|
|
317
|
+
return false;
|
|
318
|
+
if (query.motion && entry.semantic.motion?.label !== query.motion)
|
|
319
|
+
return false;
|
|
320
|
+
if (query.range && !entry.semantic.usableRanges.some((range) => range.end > query.range.start && range.start < query.range.end))
|
|
321
|
+
return false;
|
|
322
|
+
if (query.capabilities?.some((capability) => !entry.analysis.some((record) => record.capability === capability && record.status !== "unavailable"))) {
|
|
323
|
+
return false;
|
|
324
|
+
}
|
|
325
|
+
return true;
|
|
326
|
+
}
|
|
327
|
+
function matchesTag(tags, query) {
|
|
328
|
+
const normalized = query.trim().toLowerCase();
|
|
329
|
+
return tags.some((tag) => tag.value.toLowerCase() === normalized);
|
|
330
|
+
}
|
|
331
|
+
function analysisStatus(capability, analyzer, source, analyzed, ranges = [], failureReason) {
|
|
332
|
+
if (!analyzer) {
|
|
333
|
+
return { capability, status: "unavailable", reason: `${capability} analyzer is not configured` };
|
|
334
|
+
}
|
|
335
|
+
if (failureReason)
|
|
336
|
+
return { capability, status: "unavailable", reason: failureReason };
|
|
337
|
+
if (!analyzed) {
|
|
338
|
+
return { capability, status: "available", reason: `${capability} analysis is not attached` };
|
|
339
|
+
}
|
|
340
|
+
return {
|
|
341
|
+
capability,
|
|
342
|
+
status: "analyzed",
|
|
343
|
+
provenance: {
|
|
344
|
+
analyzer: analyzer.descriptor ?? { id: `framekit.${capability}`, provider: "unknown" },
|
|
345
|
+
source,
|
|
346
|
+
ranges: structuredClone(ranges),
|
|
347
|
+
},
|
|
348
|
+
};
|
|
349
|
+
}
|
|
350
|
+
async function settle(operation) {
|
|
351
|
+
try {
|
|
352
|
+
const promise = operation();
|
|
353
|
+
return promise ? await Promise.resolve(promise).then((value) => ({ status: "fulfilled", value }), (reason) => ({ status: "rejected", reason })) : undefined;
|
|
354
|
+
}
|
|
355
|
+
catch (reason) {
|
|
356
|
+
return { status: "rejected", reason };
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
function fulfilledValue(result) {
|
|
360
|
+
return result?.status === "fulfilled" ? result.value : undefined;
|
|
361
|
+
}
|
|
362
|
+
function failureReason(result) {
|
|
363
|
+
return result?.status === "rejected" ? String(result.reason) : undefined;
|
|
364
|
+
}
|
|
365
|
+
function findMedia(project, mediaId) {
|
|
366
|
+
const media = project.media.find((candidate) => candidate.mediaId === mediaId);
|
|
367
|
+
if (!media)
|
|
368
|
+
throw new Error(`MEDIA_NOT_FOUND: ${mediaId}`);
|
|
369
|
+
return media;
|
|
370
|
+
}
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import { withCapabilityFamilies } from "../capabilities.js";
|
|
2
|
+
import { isWithinClip, parseRational, rationalDifferenceSeconds } from "../timeline/rational-time.js";
|
|
3
|
+
export class ProjectService {
|
|
4
|
+
adapter;
|
|
5
|
+
context;
|
|
6
|
+
options;
|
|
7
|
+
constructor(adapter, context, options) {
|
|
8
|
+
this.adapter = adapter;
|
|
9
|
+
this.context = context;
|
|
10
|
+
this.options = options;
|
|
11
|
+
}
|
|
12
|
+
async inspectProject() {
|
|
13
|
+
return this.context.inspectProject();
|
|
14
|
+
}
|
|
15
|
+
async inspectTimeline() {
|
|
16
|
+
const project = await this.inspectProject();
|
|
17
|
+
return project.timeline;
|
|
18
|
+
}
|
|
19
|
+
async inspectArtifact() {
|
|
20
|
+
if (!this.adapter.getManagedArtifact) {
|
|
21
|
+
throw new Error("CAPABILITY_UNAVAILABLE: managed FCPXML artifact");
|
|
22
|
+
}
|
|
23
|
+
return this.adapter.getManagedArtifact();
|
|
24
|
+
}
|
|
25
|
+
async captureFrame(position, options = {}) {
|
|
26
|
+
const exactPosition = parseRational(position, "INVALID_TIMELINE_POSITION");
|
|
27
|
+
const capabilities = await this.adapter.getCapabilities();
|
|
28
|
+
if (!capabilities.editor.frameCapture || !this.adapter.captureFrame) {
|
|
29
|
+
throw new Error("CAPABILITY_UNAVAILABLE: timeline frame capture");
|
|
30
|
+
}
|
|
31
|
+
if (options.analyze && !this.options.visualAnalyzer) {
|
|
32
|
+
throw new Error("CAPABILITY_UNAVAILABLE: visual analysis");
|
|
33
|
+
}
|
|
34
|
+
const project = await this.inspectProject();
|
|
35
|
+
const source = await this.adapter.captureFrame(position, project.revision);
|
|
36
|
+
const clip = project.timeline.clips
|
|
37
|
+
.filter((candidate) => isWithinClip(exactPosition, candidate.startTime, candidate.durationTime))
|
|
38
|
+
.sort((left, right) => right.track - left.track)[0];
|
|
39
|
+
let analysis;
|
|
40
|
+
if (options.analyze) {
|
|
41
|
+
const media = clip?.mediaId
|
|
42
|
+
? project.media.find((candidate) => candidate.mediaId === clip.mediaId)
|
|
43
|
+
: undefined;
|
|
44
|
+
if (!clip || !media) {
|
|
45
|
+
throw new Error("CAPABILITY_UNAVAILABLE: visual analysis requires media at the captured position");
|
|
46
|
+
}
|
|
47
|
+
const mediaTime = rationalDifferenceSeconds(exactPosition, parseRational(clip.startTime, "INVALID_PROJECT_STATE"));
|
|
48
|
+
analysis = await this.options.visualAnalyzer.analyze({ project, media }, { start: mediaTime, end: mediaTime });
|
|
49
|
+
}
|
|
50
|
+
return {
|
|
51
|
+
image: structuredClone(source.image),
|
|
52
|
+
position: { ...position },
|
|
53
|
+
timecode: source.timecode,
|
|
54
|
+
project: { id: project.projectId, name: project.projectName },
|
|
55
|
+
sequence: { id: project.timeline.id, name: project.timeline.name },
|
|
56
|
+
...(clip ? {
|
|
57
|
+
clip: {
|
|
58
|
+
id: clip.id,
|
|
59
|
+
...(clip.mediaId ? { mediaId: clip.mediaId } : {}),
|
|
60
|
+
name: clip.name,
|
|
61
|
+
startTime: { ...clip.startTime },
|
|
62
|
+
durationTime: { ...clip.durationTime },
|
|
63
|
+
track: clip.track,
|
|
64
|
+
},
|
|
65
|
+
} : {}),
|
|
66
|
+
...(analysis ? { analysis } : {}),
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
async listProjects() {
|
|
70
|
+
const capabilities = await this.adapter.getCapabilities();
|
|
71
|
+
if (!capabilities.editor.projectCatalogRead || !this.adapter.listProjects) {
|
|
72
|
+
throw new Error("CAPABILITY_UNAVAILABLE: editor project catalog");
|
|
73
|
+
}
|
|
74
|
+
return this.adapter.listProjects();
|
|
75
|
+
}
|
|
76
|
+
async selectProject(selection) {
|
|
77
|
+
const capabilities = await this.adapter.getCapabilities();
|
|
78
|
+
if (!capabilities.editor.projectSelection || !this.adapter.selectProject) {
|
|
79
|
+
throw new Error("CAPABILITY_UNAVAILABLE: editor project selection");
|
|
80
|
+
}
|
|
81
|
+
if (!selection.projectId.trim())
|
|
82
|
+
throw new Error("INVALID_PROJECT_SELECTION: projectId is required");
|
|
83
|
+
if (selection.sequenceId !== undefined && !selection.sequenceId.trim()) {
|
|
84
|
+
throw new Error("INVALID_PROJECT_SELECTION: sequenceId cannot be empty");
|
|
85
|
+
}
|
|
86
|
+
return this.adapter.selectProject(selection);
|
|
87
|
+
}
|
|
88
|
+
async inspectEditor() {
|
|
89
|
+
const identity = await this.adapter.getIdentity();
|
|
90
|
+
const capabilities = withCapabilityFamilies(await this.adapter.getCapabilities(), { backend: identity.backend });
|
|
91
|
+
const analyzers = {
|
|
92
|
+
...capabilities.analyzers,
|
|
93
|
+
speechTranscribe: capabilities.analyzers.speechTranscribe || Boolean(this.options.speechAnalyzer),
|
|
94
|
+
audioLoudness: capabilities.analyzers.audioLoudness || Boolean(this.options.audioAnalyzer),
|
|
95
|
+
audioNoise: capabilities.analyzers.audioNoise || Boolean(this.options.noiseAnalyzer),
|
|
96
|
+
visualTrack: capabilities.analyzers.visualTrack || Boolean(this.options.visualAnalyzer),
|
|
97
|
+
metadataDescribe: capabilities.analyzers.metadataDescribe || Boolean(this.options.metadataAnalyzer),
|
|
98
|
+
};
|
|
99
|
+
return {
|
|
100
|
+
identity,
|
|
101
|
+
capabilities: {
|
|
102
|
+
...withCapabilityFamilies({ ...capabilities, analyzers }, { backend: identity.backend }),
|
|
103
|
+
},
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
async listAssets(query) {
|
|
107
|
+
const capabilities = await this.adapter.getCapabilities();
|
|
108
|
+
if (!capabilities.editor.assetDiscovery || !this.adapter.listAssets) {
|
|
109
|
+
throw new Error("CAPABILITY_UNAVAILABLE: editor assets");
|
|
110
|
+
}
|
|
111
|
+
const assets = await this.context.listAssets(query);
|
|
112
|
+
return assets.filter((asset) => matchesAssetQuery(asset, query));
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
function matchesAssetQuery(asset, query) {
|
|
116
|
+
if (!query)
|
|
117
|
+
return true;
|
|
118
|
+
const normalized = query.query?.trim().toLowerCase();
|
|
119
|
+
if (normalized && ![asset.id, asset.name, asset.vendor].some((value) => value.toLowerCase().includes(normalized))) {
|
|
120
|
+
return false;
|
|
121
|
+
}
|
|
122
|
+
if (query.kind && asset.kind !== query.kind)
|
|
123
|
+
return false;
|
|
124
|
+
if (query.vendor && asset.vendor.toLowerCase() !== query.vendor.trim().toLowerCase())
|
|
125
|
+
return false;
|
|
126
|
+
return true;
|
|
127
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export class TransactionStore {
|
|
2
|
+
transactions = new Map();
|
|
3
|
+
set(transaction) {
|
|
4
|
+
this.transactions.set(transaction.id, transaction);
|
|
5
|
+
}
|
|
6
|
+
get(transactionId) {
|
|
7
|
+
const transaction = this.transactions.get(transactionId);
|
|
8
|
+
if (!transaction)
|
|
9
|
+
throw new Error(`TRANSACTION_NOT_FOUND: ${transactionId}`);
|
|
10
|
+
return transaction;
|
|
11
|
+
}
|
|
12
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/** Plan a bounded clip-level gain change without mutating an editor. */
|
|
2
|
+
export function planDialogueGain(measurement, options) {
|
|
3
|
+
validateOptions(options);
|
|
4
|
+
const currentLufs = finiteOrZero(measurement.integratedLufs);
|
|
5
|
+
const desiredGainDb = roundDb(options.targetLufs - currentLufs);
|
|
6
|
+
const clampedGainDb = clamp(desiredGainDb, options.minGainDb, options.maxGainDb);
|
|
7
|
+
const estimatedPeakDb = roundDb(finiteOrZero(measurement.truePeakDb) + clampedGainDb);
|
|
8
|
+
const base = {
|
|
9
|
+
...structuredClone(options),
|
|
10
|
+
currentLufs,
|
|
11
|
+
desiredGainDb,
|
|
12
|
+
clampedGainDb,
|
|
13
|
+
estimatedPeakDb,
|
|
14
|
+
};
|
|
15
|
+
if (!measurement.valid || !Number.isFinite(measurement.integratedLufs)
|
|
16
|
+
|| !Number.isFinite(measurement.truePeakDb) || !Number.isFinite(measurement.silenceMs)
|
|
17
|
+
|| !Number.isFinite(measurement.analyzedDurationSeconds)) {
|
|
18
|
+
return { ...base, decision: "SKIP", reasonCodes: ["MEASUREMENT_INVALID"] };
|
|
19
|
+
}
|
|
20
|
+
if (!measurement.dialoguePresent) {
|
|
21
|
+
return { ...base, decision: "SKIP", reasonCodes: ["NO_DIALOGUE"] };
|
|
22
|
+
}
|
|
23
|
+
if (measurement.silenceMs >= measurement.analyzedDurationSeconds * 1000) {
|
|
24
|
+
return { ...base, decision: "SKIP", reasonCodes: ["SILENCE"] };
|
|
25
|
+
}
|
|
26
|
+
if (measurement.analyzedDurationSeconds < options.minDialogueDurationSeconds) {
|
|
27
|
+
return { ...base, decision: "SKIP", reasonCodes: ["DIALOGUE_TOO_SHORT"] };
|
|
28
|
+
}
|
|
29
|
+
if (Math.abs(measurement.integratedLufs - options.targetLufs) <= options.toleranceDb) {
|
|
30
|
+
return {
|
|
31
|
+
...base,
|
|
32
|
+
decision: "NO_OP",
|
|
33
|
+
desiredGainDb: 0,
|
|
34
|
+
clampedGainDb: 0,
|
|
35
|
+
estimatedPeakDb: measurement.truePeakDb,
|
|
36
|
+
reasonCodes: ["ALREADY_IN_TOLERANCE"],
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
if (desiredGainDb < options.minGainDb || desiredGainDb > options.maxGainDb) {
|
|
40
|
+
return { ...base, decision: "SKIP", reasonCodes: ["GAIN_OUT_OF_BOUNDS"] };
|
|
41
|
+
}
|
|
42
|
+
if (estimatedPeakDb > options.maxTruePeakDb) {
|
|
43
|
+
return { ...base, decision: "SKIP", reasonCodes: ["PEAK_RISK"] };
|
|
44
|
+
}
|
|
45
|
+
return { ...base, decision: "APPLY", reasonCodes: ["APPLY_GAIN"] };
|
|
46
|
+
}
|
|
47
|
+
function validateOptions(options) {
|
|
48
|
+
if (!Number.isFinite(options.targetLufs) || !Number.isFinite(options.toleranceDb) || options.toleranceDb < 0
|
|
49
|
+
|| !Number.isFinite(options.maxTruePeakDb)
|
|
50
|
+
|| !Number.isFinite(options.minGainDb) || !Number.isFinite(options.maxGainDb)
|
|
51
|
+
|| options.minGainDb > options.maxGainDb
|
|
52
|
+
|| !Number.isFinite(options.minDialogueDurationSeconds) || options.minDialogueDurationSeconds < 0) {
|
|
53
|
+
throw new Error("INVALID_OPERATION: dialogue gain policy is invalid");
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
function finiteOrZero(value) {
|
|
57
|
+
return Number.isFinite(value) ? value : 0;
|
|
58
|
+
}
|
|
59
|
+
function clamp(value, minimum, maximum) {
|
|
60
|
+
return Math.min(Math.max(value, minimum), maximum);
|
|
61
|
+
}
|
|
62
|
+
function roundDb(value) {
|
|
63
|
+
return Number(value.toFixed(6));
|
|
64
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
export function planNoiseReduction(measurement, options) {
|
|
2
|
+
validateOptions(options);
|
|
3
|
+
const recommendedReductionDb = finiteOrZero(measurement.recommendedReductionDb);
|
|
4
|
+
const reductionDb = Math.min(Math.max(recommendedReductionDb, 0), options.maxReductionDb);
|
|
5
|
+
const base = {
|
|
6
|
+
...structuredClone(options),
|
|
7
|
+
noiseFloorDb: measurement.noiseFloorDb,
|
|
8
|
+
recommendedReductionDb,
|
|
9
|
+
reductionDb,
|
|
10
|
+
affectedRanges: clampRanges(measurement.affectedRanges, measurement.requestedRange),
|
|
11
|
+
};
|
|
12
|
+
if (measurement.valid !== true || !Number.isFinite(measurement.noiseFloorDb)) {
|
|
13
|
+
return { ...base, decision: "SKIP", reasonCodes: ["MEASUREMENT_INVALID"] };
|
|
14
|
+
}
|
|
15
|
+
if (measurement.noiseFloorDb <= options.noiseThresholdDb) {
|
|
16
|
+
return { ...base, decision: "NO_OP", reductionDb: 0, reasonCodes: ["ALREADY_BELOW_THRESHOLD"] };
|
|
17
|
+
}
|
|
18
|
+
if (measurement.confidence < options.minConfidence) {
|
|
19
|
+
return { ...base, decision: "SKIP", reasonCodes: ["CONFIDENCE_TOO_LOW"] };
|
|
20
|
+
}
|
|
21
|
+
if (base.affectedRanges.length === 0) {
|
|
22
|
+
return { ...base, decision: "SKIP", reasonCodes: ["NO_AFFECTED_RANGE"] };
|
|
23
|
+
}
|
|
24
|
+
if (!Number.isFinite(recommendedReductionDb) || recommendedReductionDb <= 0 || reductionDb <= 0) {
|
|
25
|
+
return { ...base, decision: "SKIP", reasonCodes: ["REDUCTION_INVALID"] };
|
|
26
|
+
}
|
|
27
|
+
return { ...base, decision: "APPLY", reasonCodes: ["APPLY_NOISE_REDUCTION"] };
|
|
28
|
+
}
|
|
29
|
+
function validateOptions(options) {
|
|
30
|
+
if (!Number.isFinite(options.noiseThresholdDb)
|
|
31
|
+
|| !Number.isFinite(options.maxReductionDb) || options.maxReductionDb <= 0
|
|
32
|
+
|| !Number.isFinite(options.minConfidence) || options.minConfidence < 0 || options.minConfidence > 1) {
|
|
33
|
+
throw new Error("INVALID_OPERATION: noise reduction policy is invalid");
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
function clampRanges(ranges, bounds) {
|
|
37
|
+
return ranges
|
|
38
|
+
.map((range) => ({
|
|
39
|
+
start: Math.max(bounds.start, range.start),
|
|
40
|
+
end: Math.min(bounds.end, range.end),
|
|
41
|
+
}))
|
|
42
|
+
.filter((range) => Number.isFinite(range.start) && Number.isFinite(range.end) && range.end > range.start);
|
|
43
|
+
}
|
|
44
|
+
function finiteOrZero(value) {
|
|
45
|
+
return Number.isFinite(value) ? value : 0;
|
|
46
|
+
}
|