@morshoto/framekit 0.1.0 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -17
- package/dist-package/adapters/final-cut/typescript/src/analyzers.js +86 -2
- package/dist-package/adapters/final-cut/typescript/src/connection.js +5 -1
- package/dist-package/adapters/final-cut/typescript/src/export.js +231 -8
- package/dist-package/adapters/final-cut/typescript/src/fcpxml.js +70 -9
- package/dist-package/adapters/final-cut/typescript/src/live.js +2 -2
- package/dist-package/adapters/final-cut/typescript/src/native.js +1000 -100
- package/dist-package/adapters/final-cut/typescript/src/publisher.js +242 -30
- package/dist-package/adapters/final-cut/typescript/src/session.js +86 -9
- package/dist-package/apps/mcp-server/src/main.js +10 -1
- package/dist-package/apps/mcp-server/src/routing.js +172 -0
- package/dist-package/apps/mcp-server/src/server.js +867 -52
- package/dist-package/apps/mcp-server/src/version.js +34 -0
- package/dist-package/packages/runtime/src/application/duration-policy-service.js +6 -0
- package/dist-package/packages/runtime/src/application/media-analysis-service.js +473 -0
- package/dist-package/packages/runtime/src/application/project-service.js +146 -0
- package/dist-package/packages/runtime/src/application/runtime-options.js +1 -0
- package/dist-package/packages/runtime/src/application/transaction-store.js +12 -0
- package/dist-package/packages/runtime/src/audio/dialogue-normalization.js +64 -0
- package/dist-package/packages/runtime/src/audio/noise-reduction.js +46 -0
- package/dist-package/packages/runtime/src/capabilities.js +187 -1
- package/dist-package/packages/runtime/src/color-correction.js +48 -0
- package/dist-package/packages/runtime/src/context/context-engine.js +20 -6
- package/dist-package/packages/runtime/src/context/context-service.js +30 -0
- package/dist-package/packages/runtime/src/context/revision.js +3 -0
- package/dist-package/packages/runtime/src/domain/capabilities.js +1 -0
- package/dist-package/packages/runtime/src/domain/context.js +1 -0
- package/dist-package/packages/runtime/src/domain/diff.js +1 -0
- package/dist-package/packages/runtime/src/domain/duration.js +1 -0
- package/dist-package/packages/runtime/src/domain/editing.js +1 -0
- package/dist-package/packages/runtime/src/domain/index.js +12 -0
- package/dist-package/packages/runtime/src/domain/media.js +8 -0
- package/dist-package/packages/runtime/src/domain/ports.js +1 -0
- package/dist-package/packages/runtime/src/domain/primitives.js +1 -0
- package/dist-package/packages/runtime/src/domain/project.js +1 -0
- package/dist-package/packages/runtime/src/domain/rough-cut.js +69 -0
- package/dist-package/packages/runtime/src/domain/skills.js +33 -0
- package/dist-package/packages/runtime/src/domain/types.js +5 -1
- package/dist-package/packages/runtime/src/domain/verification.js +1 -0
- package/dist-package/packages/runtime/src/editing/dialogue-normalization-service.js +141 -0
- package/dist-package/packages/runtime/src/editing/duration-policy.js +187 -0
- package/dist-package/packages/runtime/src/editing/edit-service.js +449 -0
- package/dist-package/packages/runtime/src/editing/filler-removal-service.js +361 -0
- package/dist-package/packages/runtime/src/editing/music-service.js +85 -0
- package/dist-package/packages/runtime/src/index.js +14 -3
- package/dist-package/packages/runtime/src/planning/rough-cut.js +71 -0
- package/dist-package/packages/runtime/src/runtime.js +131 -826
- package/dist-package/packages/runtime/src/skills/builtin.js +339 -0
- package/dist-package/packages/runtime/src/skills/index.js +3 -0
- package/dist-package/packages/runtime/src/skills/registry.js +74 -0
- package/dist-package/packages/runtime/src/skills/requirements.js +170 -0
- package/dist-package/packages/runtime/src/skills/runtime.js +239 -0
- package/dist-package/packages/runtime/src/speech/analysis.js +301 -0
- package/dist-package/packages/runtime/src/speech/filler-detector.js +132 -0
- package/dist-package/packages/runtime/src/speech/mapping.js +227 -0
- package/dist-package/packages/runtime/src/speech/safe-cut-resolver.js +314 -0
- package/dist-package/packages/runtime/src/timeline/rational-time.js +63 -0
- package/dist-package/packages/runtime/src/verification/verification.js +712 -1
- package/dist-package/packages/testkit/src/fixture-analyzers.js +28 -0
- package/dist-package/packages/testkit/src/in-memory-editor.js +287 -3
- package/dist-package/packages/testkit/src/index.js +1 -0
- package/dist-package/packages/testkit/src/skill-fixtures.js +64 -0
- package/package.json +17 -8
- /package/dist-package/packages/runtime/src/{diff/diff.js → timeline/snapshot-diff.js} +0 -0
- /package/dist-package/packages/runtime/src/{snapshot-digest.js → timeline/snapshot-digest.js} +0 -0
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { dirname, join } from "node:path";
|
|
3
|
+
import { fileURLToPath } from "node:url";
|
|
4
|
+
const FRAMEKIT_PACKAGE_NAME = "@morshoto/framekit";
|
|
5
|
+
export const FRAMEKIT_VERSION = readFramekitVersion();
|
|
6
|
+
function readFramekitVersion() {
|
|
7
|
+
let directory = dirname(fileURLToPath(import.meta.url));
|
|
8
|
+
while (true) {
|
|
9
|
+
const manifestPath = join(directory, "package.json");
|
|
10
|
+
try {
|
|
11
|
+
const manifest = JSON.parse(readFileSync(manifestPath, "utf8"));
|
|
12
|
+
if (manifest.name === FRAMEKIT_PACKAGE_NAME) {
|
|
13
|
+
if (typeof manifest.version !== "string" || manifest.version.length === 0) {
|
|
14
|
+
throw new Error(`FRAMEKIT_PACKAGE_VERSION_INVALID: ${manifestPath}`);
|
|
15
|
+
}
|
|
16
|
+
return manifest.version;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
catch (error) {
|
|
20
|
+
if (!isMissingFile(error))
|
|
21
|
+
throw error;
|
|
22
|
+
}
|
|
23
|
+
const parent = dirname(directory);
|
|
24
|
+
if (parent === directory)
|
|
25
|
+
break;
|
|
26
|
+
directory = parent;
|
|
27
|
+
}
|
|
28
|
+
throw new Error(`FRAMEKIT_PACKAGE_VERSION_UNAVAILABLE: ${FRAMEKIT_PACKAGE_NAME}`);
|
|
29
|
+
}
|
|
30
|
+
function isMissingFile(error) {
|
|
31
|
+
if (!error || typeof error !== "object" || !("code" in error))
|
|
32
|
+
return false;
|
|
33
|
+
return error.code === "ENOENT";
|
|
34
|
+
}
|
|
@@ -0,0 +1,473 @@
|
|
|
1
|
+
import { planRoughCut } from "../planning/rough-cut.js";
|
|
2
|
+
import { bindSpeechAnalysis } from "../speech/analysis.js";
|
|
3
|
+
import { parseRational } from "../timeline/rational-time.js";
|
|
4
|
+
export class MediaAnalysisService {
|
|
5
|
+
project;
|
|
6
|
+
context;
|
|
7
|
+
options;
|
|
8
|
+
constructor(project, context, options) {
|
|
9
|
+
this.project = project;
|
|
10
|
+
this.context = context;
|
|
11
|
+
this.options = options;
|
|
12
|
+
}
|
|
13
|
+
async analyzeSpeech(mediaId, range) {
|
|
14
|
+
if (!this.options.speechAnalyzer)
|
|
15
|
+
throw new Error("CAPABILITY_UNAVAILABLE: speech analysis");
|
|
16
|
+
const project = await this.project.inspectProject();
|
|
17
|
+
const media = findMedia(project, mediaId);
|
|
18
|
+
return this.analyzeSpeechForProject(project, media, range);
|
|
19
|
+
}
|
|
20
|
+
async analyzeAudio(mediaId) {
|
|
21
|
+
if (!this.options.audioAnalyzer)
|
|
22
|
+
throw new Error("CAPABILITY_UNAVAILABLE: audio analysis");
|
|
23
|
+
const project = await this.project.inspectProject();
|
|
24
|
+
const media = findMedia(project, mediaId);
|
|
25
|
+
return this.options.audioAnalyzer.analyze({ project, media });
|
|
26
|
+
}
|
|
27
|
+
async analyzeNoise(mediaId, range) {
|
|
28
|
+
if (!this.options.noiseAnalyzer)
|
|
29
|
+
throw new Error("CAPABILITY_UNAVAILABLE: audio noise analysis");
|
|
30
|
+
const project = await this.project.inspectProject();
|
|
31
|
+
const media = findMedia(project, mediaId);
|
|
32
|
+
return this.options.noiseAnalyzer.analyze({ project, media }, range);
|
|
33
|
+
}
|
|
34
|
+
async measureAudio(mediaId, occurrenceId) {
|
|
35
|
+
if (!this.options.audioAnalyzer)
|
|
36
|
+
throw new Error("CAPABILITY_UNAVAILABLE: audio analysis");
|
|
37
|
+
const project = await this.project.inspectProject();
|
|
38
|
+
const clip = project.timeline.clips.find((candidate) => candidate.id === occurrenceId);
|
|
39
|
+
if (!clip)
|
|
40
|
+
throw new Error(`OCCURRENCE_NOT_FOUND: ${occurrenceId}`);
|
|
41
|
+
if (clip.mediaId !== mediaId) {
|
|
42
|
+
throw new Error(`TARGET_MISMATCH: occurrence ${occurrenceId} does not reference media ${mediaId}`);
|
|
43
|
+
}
|
|
44
|
+
const media = findMedia(project, mediaId);
|
|
45
|
+
const requestedRange = { start: 0, end: clip.duration };
|
|
46
|
+
const analysis = await this.options.audioAnalyzer.analyze({ project, media }, requestedRange);
|
|
47
|
+
const analyzedDurationSeconds = analysis.analyzedDurationSeconds ?? clip.duration;
|
|
48
|
+
const valid = analysis.valid !== false
|
|
49
|
+
&& Number.isFinite(analysis.integratedLufs)
|
|
50
|
+
&& Number.isFinite(analysis.truePeakDb)
|
|
51
|
+
&& Number.isFinite(analysis.silenceMs)
|
|
52
|
+
&& Number.isFinite(analyzedDurationSeconds)
|
|
53
|
+
&& analyzedDurationSeconds > 0;
|
|
54
|
+
return {
|
|
55
|
+
mediaId,
|
|
56
|
+
occurrenceId,
|
|
57
|
+
requestedRange,
|
|
58
|
+
measuredRange: {
|
|
59
|
+
start: 0,
|
|
60
|
+
end: Number.isFinite(analyzedDurationSeconds) ? analyzedDurationSeconds : clip.duration,
|
|
61
|
+
},
|
|
62
|
+
revision: project.revision,
|
|
63
|
+
provider: this.options.audioAnalyzer.descriptor ?? { id: "framekit.audio", provider: "unknown" },
|
|
64
|
+
dialoguePresent: analysis.dialoguePresent
|
|
65
|
+
?? Boolean(media.speech?.words.some((word) => word.filler !== true)),
|
|
66
|
+
integratedLufs: analysis.integratedLufs,
|
|
67
|
+
truePeakDb: analysis.truePeakDb,
|
|
68
|
+
silenceMs: analysis.silenceMs,
|
|
69
|
+
analyzedDurationSeconds,
|
|
70
|
+
valid,
|
|
71
|
+
...(analysis.invalidReason ? { invalidReason: analysis.invalidReason } : {}),
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
async measureNoise(mediaId, occurrenceId) {
|
|
75
|
+
if (!this.options.noiseAnalyzer)
|
|
76
|
+
throw new Error("CAPABILITY_UNAVAILABLE: audio noise analysis");
|
|
77
|
+
const project = await this.project.inspectProject();
|
|
78
|
+
const clip = project.timeline.clips.find((candidate) => candidate.id === occurrenceId);
|
|
79
|
+
if (!clip)
|
|
80
|
+
throw new Error(`OCCURRENCE_NOT_FOUND: ${occurrenceId}`);
|
|
81
|
+
if (clip.mediaId !== mediaId) {
|
|
82
|
+
throw new Error(`TARGET_MISMATCH: occurrence ${occurrenceId} does not reference media ${mediaId}`);
|
|
83
|
+
}
|
|
84
|
+
const media = findMedia(project, mediaId);
|
|
85
|
+
const requestedRange = { start: 0, end: clip.duration };
|
|
86
|
+
const analysis = await this.options.noiseAnalyzer.analyze({ project, media }, requestedRange);
|
|
87
|
+
const measuredEnd = analysis.affectedRanges.reduce((end, range) => Math.max(end, range.end), 0);
|
|
88
|
+
const measuredRange = {
|
|
89
|
+
start: 0,
|
|
90
|
+
end: Math.max(analysis.affectedRanges.length > 0 ? measuredEnd : requestedRange.end, 0),
|
|
91
|
+
};
|
|
92
|
+
const valid = analysis.valid !== false
|
|
93
|
+
&& Number.isFinite(analysis.noiseFloorDb)
|
|
94
|
+
&& Number.isFinite(analysis.recommendedReductionDb)
|
|
95
|
+
&& Number.isFinite(analysis.confidence)
|
|
96
|
+
&& analysis.confidence >= 0
|
|
97
|
+
&& analysis.confidence <= 1
|
|
98
|
+
&& analysis.affectedRanges.every((range) => Number.isFinite(range.start)
|
|
99
|
+
&& Number.isFinite(range.end) && range.start >= 0 && range.end > range.start);
|
|
100
|
+
return {
|
|
101
|
+
...structuredClone(analysis),
|
|
102
|
+
mediaId,
|
|
103
|
+
occurrenceId,
|
|
104
|
+
requestedRange,
|
|
105
|
+
measuredRange,
|
|
106
|
+
revision: project.revision,
|
|
107
|
+
provider: this.options.noiseAnalyzer.descriptor ?? { id: "framekit.audio-noise", provider: "unknown" },
|
|
108
|
+
valid,
|
|
109
|
+
...(analysis.invalidReason ? { invalidReason: analysis.invalidReason } : {}),
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
async analyzeVisual(mediaId, range) {
|
|
113
|
+
if (!this.options.visualAnalyzer)
|
|
114
|
+
throw new Error("CAPABILITY_UNAVAILABLE: visual analysis");
|
|
115
|
+
const project = await this.project.inspectProject();
|
|
116
|
+
const media = findMedia(project, mediaId);
|
|
117
|
+
return this.options.visualAnalyzer.analyze({ project, media }, range);
|
|
118
|
+
}
|
|
119
|
+
async understandMedia(mediaId) {
|
|
120
|
+
const project = await this.project.inspectProject();
|
|
121
|
+
const media = findMedia(project, mediaId);
|
|
122
|
+
const input = { project, media };
|
|
123
|
+
const [speechResult, audioResult, noiseResult, visualResult, metadataResult] = await Promise.all([
|
|
124
|
+
settle(() => this.options.speechAnalyzer ? this.analyzeSpeechForProject(project, media) : undefined),
|
|
125
|
+
settle(() => this.options.audioAnalyzer?.analyze(input)),
|
|
126
|
+
settle(() => this.options.noiseAnalyzer?.analyze(input)),
|
|
127
|
+
settle(() => this.options.visualAnalyzer?.analyze(input)),
|
|
128
|
+
settle(() => this.options.metadataAnalyzer?.analyze(input)),
|
|
129
|
+
]);
|
|
130
|
+
const speech = fulfilledValue(speechResult);
|
|
131
|
+
const audio = fulfilledValue(audioResult);
|
|
132
|
+
const noise = fulfilledValue(noiseResult);
|
|
133
|
+
const visual = fulfilledValue(visualResult);
|
|
134
|
+
const metadata = fulfilledValue(metadataResult);
|
|
135
|
+
const sourceIdentity = sourceIdentityOf(media);
|
|
136
|
+
const understanding = {
|
|
137
|
+
mediaId: media.mediaId,
|
|
138
|
+
source: media.source,
|
|
139
|
+
sourceIdentity,
|
|
140
|
+
...(metadata ? { metadata } : {}),
|
|
141
|
+
...(speech ? { speech } : {}),
|
|
142
|
+
...(audio ? { audio } : {}),
|
|
143
|
+
...(noise ? { noise } : {}),
|
|
144
|
+
...(visual ? { visual } : {}),
|
|
145
|
+
semantic: semanticFromAnalyses(speech, audio, visual, metadata),
|
|
146
|
+
analysis: [
|
|
147
|
+
analysisStatus("speech", this.options.speechAnalyzer, sourceIdentity, Boolean(speech), [], failureReason(speechResult)),
|
|
148
|
+
analysisStatus("audio", this.options.audioAnalyzer, sourceIdentity, Boolean(audio), [], failureReason(audioResult)),
|
|
149
|
+
analysisStatus("noise", this.options.noiseAnalyzer, sourceIdentity, Boolean(noise), noise?.affectedRanges, failureReason(noiseResult)),
|
|
150
|
+
analysisStatus("visual", this.options.visualAnalyzer, sourceIdentity, Boolean(visual), [], failureReason(visualResult)),
|
|
151
|
+
analysisStatus("metadata", this.options.metadataAnalyzer, sourceIdentity, Boolean(metadata), metadata?.usableRanges, failureReason(metadataResult)),
|
|
152
|
+
],
|
|
153
|
+
analysisRevision: project.revision,
|
|
154
|
+
};
|
|
155
|
+
this.context.attachMediaUnderstanding(understanding);
|
|
156
|
+
return structuredClone(understanding);
|
|
157
|
+
}
|
|
158
|
+
async inspectMedia(mediaId) {
|
|
159
|
+
const project = await this.project.inspectProject();
|
|
160
|
+
return findMedia(project, mediaId);
|
|
161
|
+
}
|
|
162
|
+
async searchMedia(query) {
|
|
163
|
+
const project = await this.project.inspectProject();
|
|
164
|
+
const normalized = query.trim().toLowerCase();
|
|
165
|
+
return project.media.filter((media) => media.mediaId.toLowerCase().includes(normalized) || media.source.toLowerCase().includes(normalized));
|
|
166
|
+
}
|
|
167
|
+
async indexMedia(query = {}) {
|
|
168
|
+
const project = await this.project.inspectProject();
|
|
169
|
+
return this.indexFromProject(project, query);
|
|
170
|
+
}
|
|
171
|
+
async planRoughCut(request) {
|
|
172
|
+
const project = await this.project.inspectProject();
|
|
173
|
+
const { maxShots: _maxShots, ...query } = request;
|
|
174
|
+
return planRoughCut(this.indexFromProject(project, query), project.revision, request);
|
|
175
|
+
}
|
|
176
|
+
indexFromProject(project, query) {
|
|
177
|
+
return project.media
|
|
178
|
+
.map((media) => ({
|
|
179
|
+
sourceIdentity: sourceIdentityOf(media),
|
|
180
|
+
semantic: media.semantic ?? emptySemanticDescription(),
|
|
181
|
+
analysis: media.analysis ?? [
|
|
182
|
+
analysisStatus("speech", this.options.speechAnalyzer, sourceIdentityOf(media), false),
|
|
183
|
+
analysisStatus("audio", this.options.audioAnalyzer, sourceIdentityOf(media), false),
|
|
184
|
+
analysisStatus("noise", this.options.noiseAnalyzer, sourceIdentityOf(media), false),
|
|
185
|
+
analysisStatus("visual", this.options.visualAnalyzer, sourceIdentityOf(media), false),
|
|
186
|
+
analysisStatus("metadata", this.options.metadataAnalyzer, sourceIdentityOf(media), false),
|
|
187
|
+
],
|
|
188
|
+
...(media.analysisRevision ? { analysisRevision: media.analysisRevision } : {}),
|
|
189
|
+
}))
|
|
190
|
+
.filter((entry) => matchesMediaIndexQuery(entry, query));
|
|
191
|
+
}
|
|
192
|
+
async reanalyzeAffectedRanges(transaction, requirements) {
|
|
193
|
+
const affectedMediaRanges = transaction.diff.affectedRanges.flatMap((range) => transaction.attemptedAfter.timeline.clips.flatMap((clip) => {
|
|
194
|
+
const intersectionStart = Math.max(range.start, clip.start);
|
|
195
|
+
const intersectionEnd = Math.min(range.end, clip.start + clip.duration);
|
|
196
|
+
if (!clip.mediaId || intersectionStart >= intersectionEnd)
|
|
197
|
+
return [];
|
|
198
|
+
const sourceStart = clip.sourceStart ?? 0;
|
|
199
|
+
return [{
|
|
200
|
+
mediaId: clip.mediaId,
|
|
201
|
+
range: {
|
|
202
|
+
start: sourceStart + intersectionStart - clip.start,
|
|
203
|
+
end: sourceStart + intersectionEnd - clip.start,
|
|
204
|
+
},
|
|
205
|
+
}];
|
|
206
|
+
}));
|
|
207
|
+
const mediaIds = new Set(affectedMediaRanges.map(({ mediaId }) => mediaId));
|
|
208
|
+
if (mediaIds.size === 0)
|
|
209
|
+
return transaction.attemptedAfter;
|
|
210
|
+
const next = structuredClone(transaction.attemptedAfter);
|
|
211
|
+
for (const mediaId of mediaIds) {
|
|
212
|
+
const media = next.media.find((candidate) => candidate.mediaId === mediaId);
|
|
213
|
+
if (!media)
|
|
214
|
+
continue;
|
|
215
|
+
const ranges = mergeRanges(affectedMediaRanges
|
|
216
|
+
.filter((affected) => affected.mediaId === mediaId)
|
|
217
|
+
.map((affected) => affected.range));
|
|
218
|
+
const input = { project: next, media };
|
|
219
|
+
if (requirements.speech && this.options.speechAnalyzer) {
|
|
220
|
+
const analyses = await Promise.all(ranges.map(async (range) => bindSpeechAnalysis(await this.options.speechAnalyzer.analyze(input, range), { input, range, provider: this.options.speechAnalyzer.descriptor })));
|
|
221
|
+
const latest = analyses[analyses.length - 1];
|
|
222
|
+
if (latest) {
|
|
223
|
+
media.speech = mergeSpeechAnalyses(media.speech, analyses, ranges, {
|
|
224
|
+
input,
|
|
225
|
+
provider: this.options.speechAnalyzer.descriptor,
|
|
226
|
+
});
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
if (requirements.audio && this.options.audioAnalyzer) {
|
|
230
|
+
const analyses = await Promise.all(ranges.map((range) => this.options.audioAnalyzer.analyze(input, range)));
|
|
231
|
+
if (analyses[analyses.length - 1])
|
|
232
|
+
media.audio = analyses[analyses.length - 1];
|
|
233
|
+
}
|
|
234
|
+
if (requirements.noise && this.options.noiseAnalyzer) {
|
|
235
|
+
const analyses = await Promise.all(ranges.map((range) => this.options.noiseAnalyzer.analyze(input, range)));
|
|
236
|
+
if (analyses[analyses.length - 1])
|
|
237
|
+
media.noise = analyses[analyses.length - 1];
|
|
238
|
+
}
|
|
239
|
+
if (requirements.visual && this.options.visualAnalyzer) {
|
|
240
|
+
const analyses = await Promise.all(ranges.map((range) => this.options.visualAnalyzer.analyze(input, range)));
|
|
241
|
+
media.visual = {
|
|
242
|
+
scenes: analyses.flatMap((analysis) => analysis.scenes),
|
|
243
|
+
subjects: analyses.flatMap((analysis) => analysis.subjects),
|
|
244
|
+
keyframes: analyses.flatMap((analysis) => analysis.keyframes),
|
|
245
|
+
motion: analyses[analyses.length - 1]?.motion,
|
|
246
|
+
};
|
|
247
|
+
}
|
|
248
|
+
if ((requirements.speech && this.options.speechAnalyzer)
|
|
249
|
+
|| (requirements.audio && this.options.audioAnalyzer)
|
|
250
|
+
|| (requirements.noise && this.options.noiseAnalyzer)
|
|
251
|
+
|| (requirements.visual && this.options.visualAnalyzer)) {
|
|
252
|
+
for (const candidate of next.media) {
|
|
253
|
+
if (candidate.mediaId === mediaId)
|
|
254
|
+
candidate.analysisRevision = next.revision.id;
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
return next;
|
|
259
|
+
}
|
|
260
|
+
async analyzeSpeechForProject(project, media, range) {
|
|
261
|
+
const analyzer = this.options.speechAnalyzer;
|
|
262
|
+
if (!analyzer)
|
|
263
|
+
throw new Error("CAPABILITY_UNAVAILABLE: speech analysis");
|
|
264
|
+
const input = { project, media };
|
|
265
|
+
const analysis = await analyzer.analyze(input, range);
|
|
266
|
+
return bindSpeechAnalysis(analysis, {
|
|
267
|
+
input,
|
|
268
|
+
range,
|
|
269
|
+
provider: analyzer.descriptor,
|
|
270
|
+
});
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
function sourceIdentityOf(media) {
|
|
274
|
+
return {
|
|
275
|
+
mediaId: media.mediaId,
|
|
276
|
+
source: media.source,
|
|
277
|
+
...(media.sourceDigest ? { sourceDigest: media.sourceDigest } : {}),
|
|
278
|
+
...(media.mediaKind ? { mediaKind: media.mediaKind } : {}),
|
|
279
|
+
...(media.duration !== undefined ? { duration: media.duration } : {}),
|
|
280
|
+
};
|
|
281
|
+
}
|
|
282
|
+
function mergeSpeechAnalyses(previous, analyses, updatedRanges, context) {
|
|
283
|
+
const latest = analyses[analyses.length - 1];
|
|
284
|
+
if (!latest)
|
|
285
|
+
throw new Error("ANALYSIS_INVALID: speech reanalysis returned no results");
|
|
286
|
+
if (previous?.provider && !sameDescriptor(previous.provider, latest.provider)) {
|
|
287
|
+
throw new Error("ANALYSIS_INVALID: speech evidence providers cannot be combined");
|
|
288
|
+
}
|
|
289
|
+
if (previous?.sourceTimebase && !sameRational(previous.sourceTimebase, latest.sourceTimebase)) {
|
|
290
|
+
throw new Error("ANALYSIS_INVALID: speech evidence timebases cannot be combined");
|
|
291
|
+
}
|
|
292
|
+
const requestedRange = encompassingRange([
|
|
293
|
+
...(previous?.requestedRange ? [previous.requestedRange] : []),
|
|
294
|
+
...analyses.map((analysis) => analysis.requestedRange),
|
|
295
|
+
]);
|
|
296
|
+
const observedRange = encompassingRange([
|
|
297
|
+
...(previous?.observedRange ? [previous.observedRange] : []),
|
|
298
|
+
...analyses.map((analysis) => analysis.observedRange),
|
|
299
|
+
]);
|
|
300
|
+
const words = mergeSpeechEvidence(previous?.words, analyses.map((analysis) => analysis.words), updatedRanges);
|
|
301
|
+
const hasVad = previous?.vadSegments !== undefined || analyses.some((analysis) => analysis.vadSegments !== undefined);
|
|
302
|
+
const vadSegments = hasVad
|
|
303
|
+
? mergeSpeechEvidence(previous?.vadSegments, analyses.map((analysis) => analysis.vadSegments ?? []), updatedRanges)
|
|
304
|
+
: undefined;
|
|
305
|
+
const hasSilence = previous?.silenceSegments !== undefined || analyses.some((analysis) => analysis.silenceSegments !== undefined);
|
|
306
|
+
const silenceSegments = hasSilence
|
|
307
|
+
? mergeSpeechEvidence(previous?.silenceSegments, analyses.map((analysis) => analysis.silenceSegments ?? []), updatedRanges)
|
|
308
|
+
: undefined;
|
|
309
|
+
const hasProtected = previous?.protectedSegments !== undefined || analyses.some((analysis) => analysis.protectedSegments !== undefined);
|
|
310
|
+
const protectedSegments = hasProtected
|
|
311
|
+
? mergeSpeechEvidence(previous?.protectedSegments, analyses.map((analysis) => analysis.protectedSegments ?? []), updatedRanges)
|
|
312
|
+
: undefined;
|
|
313
|
+
return bindSpeechAnalysis({
|
|
314
|
+
...latest,
|
|
315
|
+
requestedRange,
|
|
316
|
+
observedRange,
|
|
317
|
+
capability: vadSegments ? "transcription-plus-vad" : latest.capability,
|
|
318
|
+
words,
|
|
319
|
+
...(vadSegments ? { vadSegments } : {}),
|
|
320
|
+
...(silenceSegments ? { silenceSegments } : {}),
|
|
321
|
+
...(protectedSegments ? { protectedSegments } : {}),
|
|
322
|
+
}, context);
|
|
323
|
+
}
|
|
324
|
+
function mergeSpeechEvidence(previous, fresh, updatedRanges) {
|
|
325
|
+
return [
|
|
326
|
+
...(previous ?? []).filter((evidence) => !updatedRanges.some((range) => rangesOverlap(evidence, range))),
|
|
327
|
+
...fresh.flat(),
|
|
328
|
+
].sort((left, right) => left.start - right.start || left.end - right.end);
|
|
329
|
+
}
|
|
330
|
+
function rangesOverlap(left, right) {
|
|
331
|
+
return left.end > right.start && left.start < right.end;
|
|
332
|
+
}
|
|
333
|
+
function encompassingRange(ranges) {
|
|
334
|
+
if (ranges.length === 0)
|
|
335
|
+
throw new Error("ANALYSIS_INVALID: speech evidence needs a range");
|
|
336
|
+
return {
|
|
337
|
+
start: Math.min(...ranges.map((range) => range.start)),
|
|
338
|
+
end: Math.max(...ranges.map((range) => range.end)),
|
|
339
|
+
};
|
|
340
|
+
}
|
|
341
|
+
function mergeRanges(ranges) {
|
|
342
|
+
const merged = [];
|
|
343
|
+
for (const range of [...ranges].sort((left, right) => left.start - right.start || left.end - right.end)) {
|
|
344
|
+
const previous = merged[merged.length - 1];
|
|
345
|
+
if (!previous || range.start > previous.end) {
|
|
346
|
+
merged.push({ start: range.start, end: range.end });
|
|
347
|
+
continue;
|
|
348
|
+
}
|
|
349
|
+
previous.end = Math.max(previous.end, range.end);
|
|
350
|
+
}
|
|
351
|
+
return merged;
|
|
352
|
+
}
|
|
353
|
+
function sameDescriptor(left, right) {
|
|
354
|
+
return left.id === right.id && left.provider === right.provider && left.version === right.version;
|
|
355
|
+
}
|
|
356
|
+
function sameRational(left, right) {
|
|
357
|
+
const leftParts = parseRational(left, "ANALYSIS_INVALID");
|
|
358
|
+
const rightParts = parseRational(right, "ANALYSIS_INVALID");
|
|
359
|
+
return leftParts.value * rightParts.timescale === rightParts.value * leftParts.timescale;
|
|
360
|
+
}
|
|
361
|
+
function semanticFromAnalyses(speech, audio, visual, metadata) {
|
|
362
|
+
return {
|
|
363
|
+
subjects: [
|
|
364
|
+
...(visual?.subjects ?? []).map((subject) => ({ value: subject.label, confidence: subject.confidence })),
|
|
365
|
+
...(metadata?.subjects ?? []),
|
|
366
|
+
],
|
|
367
|
+
scenes: [
|
|
368
|
+
...(visual?.scenes ?? []).flatMap((scene) => scene.label ? [{ value: scene.label, confidence: scene.confidence ?? 1 }] : []),
|
|
369
|
+
...(metadata?.scenes ?? []),
|
|
370
|
+
],
|
|
371
|
+
environments: metadata?.environments ? structuredClone(metadata.environments) : [],
|
|
372
|
+
timeOfDay: metadata?.timeOfDay ? structuredClone(metadata.timeOfDay) : [],
|
|
373
|
+
moods: metadata?.moods ? structuredClone(metadata.moods) : [],
|
|
374
|
+
...(visual?.motion ? { motion: structuredClone(visual.motion) } : {}),
|
|
375
|
+
usableRanges: metadata?.usableRanges ? structuredClone(metadata.usableRanges) : [],
|
|
376
|
+
...(speech ? { transcript: speech.words.map((word) => word.text).join(" ") } : {}),
|
|
377
|
+
...(audio ? {
|
|
378
|
+
audio: {
|
|
379
|
+
present: true,
|
|
380
|
+
integratedLufs: audio.integratedLufs,
|
|
381
|
+
truePeakDb: audio.truePeakDb,
|
|
382
|
+
silenceMs: audio.silenceMs,
|
|
383
|
+
},
|
|
384
|
+
} : {}),
|
|
385
|
+
};
|
|
386
|
+
}
|
|
387
|
+
function emptySemanticDescription() {
|
|
388
|
+
return {
|
|
389
|
+
subjects: [],
|
|
390
|
+
scenes: [],
|
|
391
|
+
environments: [],
|
|
392
|
+
timeOfDay: [],
|
|
393
|
+
moods: [],
|
|
394
|
+
usableRanges: [],
|
|
395
|
+
};
|
|
396
|
+
}
|
|
397
|
+
function matchesMediaIndexQuery(entry, query) {
|
|
398
|
+
const text = query.query?.trim().toLowerCase();
|
|
399
|
+
const values = [
|
|
400
|
+
entry.sourceIdentity.mediaId,
|
|
401
|
+
entry.sourceIdentity.source,
|
|
402
|
+
entry.semantic.transcript ?? "",
|
|
403
|
+
...entry.semantic.subjects.map((tag) => tag.value),
|
|
404
|
+
...entry.semantic.scenes.map((tag) => tag.value),
|
|
405
|
+
...entry.semantic.environments.map((tag) => tag.value),
|
|
406
|
+
...entry.semantic.timeOfDay.map((tag) => tag.value),
|
|
407
|
+
...entry.semantic.moods.map((tag) => tag.value),
|
|
408
|
+
].map((value) => value.toLowerCase());
|
|
409
|
+
if (text && !values.some((value) => value.includes(text)))
|
|
410
|
+
return false;
|
|
411
|
+
if (query.subject && !matchesTag(entry.semantic.subjects, query.subject))
|
|
412
|
+
return false;
|
|
413
|
+
if (query.scene && !matchesTag(entry.semantic.scenes, query.scene))
|
|
414
|
+
return false;
|
|
415
|
+
if (query.environment && !matchesTag(entry.semantic.environments, query.environment))
|
|
416
|
+
return false;
|
|
417
|
+
if (query.timeOfDay && !matchesTag(entry.semantic.timeOfDay, query.timeOfDay))
|
|
418
|
+
return false;
|
|
419
|
+
if (query.mood && !matchesTag(entry.semantic.moods, query.mood))
|
|
420
|
+
return false;
|
|
421
|
+
if (query.motion && entry.semantic.motion?.label !== query.motion)
|
|
422
|
+
return false;
|
|
423
|
+
if (query.range && !entry.semantic.usableRanges.some((range) => range.end > query.range.start && range.start < query.range.end))
|
|
424
|
+
return false;
|
|
425
|
+
if (query.capabilities?.some((capability) => !entry.analysis.some((record) => record.capability === capability && record.status !== "unavailable"))) {
|
|
426
|
+
return false;
|
|
427
|
+
}
|
|
428
|
+
return true;
|
|
429
|
+
}
|
|
430
|
+
function matchesTag(tags, query) {
|
|
431
|
+
const normalized = query.trim().toLowerCase();
|
|
432
|
+
return tags.some((tag) => tag.value.toLowerCase() === normalized);
|
|
433
|
+
}
|
|
434
|
+
function analysisStatus(capability, analyzer, source, analyzed, ranges = [], failureReason) {
|
|
435
|
+
if (!analyzer) {
|
|
436
|
+
return { capability, status: "unavailable", reason: `${capability} analyzer is not configured` };
|
|
437
|
+
}
|
|
438
|
+
if (failureReason)
|
|
439
|
+
return { capability, status: "unavailable", reason: failureReason };
|
|
440
|
+
if (!analyzed) {
|
|
441
|
+
return { capability, status: "available", reason: `${capability} analysis is not attached` };
|
|
442
|
+
}
|
|
443
|
+
return {
|
|
444
|
+
capability,
|
|
445
|
+
status: "analyzed",
|
|
446
|
+
provenance: {
|
|
447
|
+
analyzer: analyzer.descriptor ?? { id: `framekit.${capability}`, provider: "unknown" },
|
|
448
|
+
source,
|
|
449
|
+
ranges: structuredClone(ranges),
|
|
450
|
+
},
|
|
451
|
+
};
|
|
452
|
+
}
|
|
453
|
+
async function settle(operation) {
|
|
454
|
+
try {
|
|
455
|
+
const promise = operation();
|
|
456
|
+
return promise ? await Promise.resolve(promise).then((value) => ({ status: "fulfilled", value }), (reason) => ({ status: "rejected", reason })) : undefined;
|
|
457
|
+
}
|
|
458
|
+
catch (reason) {
|
|
459
|
+
return { status: "rejected", reason };
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
function fulfilledValue(result) {
|
|
463
|
+
return result?.status === "fulfilled" ? result.value : undefined;
|
|
464
|
+
}
|
|
465
|
+
function failureReason(result) {
|
|
466
|
+
return result?.status === "rejected" ? String(result.reason) : undefined;
|
|
467
|
+
}
|
|
468
|
+
function findMedia(project, mediaId) {
|
|
469
|
+
const media = project.media.find((candidate) => candidate.mediaId === mediaId);
|
|
470
|
+
if (!media)
|
|
471
|
+
throw new Error(`MEDIA_NOT_FOUND: ${mediaId}`);
|
|
472
|
+
return media;
|
|
473
|
+
}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import { withCapabilityFamilies } from "../capabilities.js";
|
|
2
|
+
import { isWithinClip, parseRational, rationalDifferenceSeconds } from "../timeline/rational-time.js";
|
|
3
|
+
export class ProjectService {
|
|
4
|
+
adapter;
|
|
5
|
+
context;
|
|
6
|
+
options;
|
|
7
|
+
constructor(adapter, context, options) {
|
|
8
|
+
this.adapter = adapter;
|
|
9
|
+
this.context = context;
|
|
10
|
+
this.options = options;
|
|
11
|
+
}
|
|
12
|
+
async inspectProject() {
|
|
13
|
+
return this.context.inspectProject();
|
|
14
|
+
}
|
|
15
|
+
async inspectTimeline() {
|
|
16
|
+
const project = await this.inspectProject();
|
|
17
|
+
return project.timeline;
|
|
18
|
+
}
|
|
19
|
+
async inspectArtifact() {
|
|
20
|
+
if (!this.adapter.getManagedArtifact) {
|
|
21
|
+
throw new Error("CAPABILITY_UNAVAILABLE: managed FCPXML artifact");
|
|
22
|
+
}
|
|
23
|
+
return this.adapter.getManagedArtifact();
|
|
24
|
+
}
|
|
25
|
+
async captureFrame(position, options = {}) {
|
|
26
|
+
const exactPosition = parseRational(position, "INVALID_TIMELINE_POSITION");
|
|
27
|
+
const capabilities = await this.adapter.getCapabilities();
|
|
28
|
+
if (!capabilities.editor.frameCapture || !this.adapter.captureFrame) {
|
|
29
|
+
throw new Error("CAPABILITY_UNAVAILABLE: timeline frame capture");
|
|
30
|
+
}
|
|
31
|
+
if (options.analyze && !this.options.visualAnalyzer) {
|
|
32
|
+
throw new Error("CAPABILITY_UNAVAILABLE: visual analysis");
|
|
33
|
+
}
|
|
34
|
+
const project = await this.inspectProject();
|
|
35
|
+
const source = await this.adapter.captureFrame(position, project.revision);
|
|
36
|
+
const clip = project.timeline.clips
|
|
37
|
+
.filter((candidate) => isWithinClip(exactPosition, candidate.startTime, candidate.durationTime))
|
|
38
|
+
.sort((left, right) => right.track - left.track)[0];
|
|
39
|
+
let analysis;
|
|
40
|
+
if (options.analyze) {
|
|
41
|
+
const media = clip?.mediaId
|
|
42
|
+
? project.media.find((candidate) => candidate.mediaId === clip.mediaId)
|
|
43
|
+
: undefined;
|
|
44
|
+
if (!clip || !media) {
|
|
45
|
+
throw new Error("CAPABILITY_UNAVAILABLE: visual analysis requires media at the captured position");
|
|
46
|
+
}
|
|
47
|
+
const mediaTime = rationalDifferenceSeconds(exactPosition, parseRational(clip.startTime, "INVALID_PROJECT_STATE"));
|
|
48
|
+
analysis = await this.options.visualAnalyzer.analyze({ project, media }, { start: mediaTime, end: mediaTime });
|
|
49
|
+
}
|
|
50
|
+
return {
|
|
51
|
+
image: structuredClone(source.image),
|
|
52
|
+
position: { ...position },
|
|
53
|
+
timecode: source.timecode,
|
|
54
|
+
project: { id: project.projectId, name: project.projectName },
|
|
55
|
+
sequence: { id: project.timeline.id, name: project.timeline.name },
|
|
56
|
+
...(clip ? {
|
|
57
|
+
clip: {
|
|
58
|
+
id: clip.id,
|
|
59
|
+
...(clip.mediaId ? { mediaId: clip.mediaId } : {}),
|
|
60
|
+
name: clip.name,
|
|
61
|
+
startTime: { ...clip.startTime },
|
|
62
|
+
durationTime: { ...clip.durationTime },
|
|
63
|
+
track: clip.track,
|
|
64
|
+
},
|
|
65
|
+
} : {}),
|
|
66
|
+
...(analysis ? { analysis } : {}),
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
async listProjects() {
|
|
70
|
+
const capabilities = await this.adapter.getCapabilities();
|
|
71
|
+
if (!capabilities.editor.projectCatalogRead || !this.adapter.listProjects) {
|
|
72
|
+
throw new Error("CAPABILITY_UNAVAILABLE: editor project catalog");
|
|
73
|
+
}
|
|
74
|
+
return this.adapter.listProjects();
|
|
75
|
+
}
|
|
76
|
+
async selectProject(selection) {
|
|
77
|
+
const capabilities = await this.adapter.getCapabilities();
|
|
78
|
+
if (!capabilities.editor.projectSelection || !this.adapter.selectProject) {
|
|
79
|
+
throw new Error("CAPABILITY_UNAVAILABLE: editor project selection");
|
|
80
|
+
}
|
|
81
|
+
if (!selection.projectId.trim())
|
|
82
|
+
throw new Error("INVALID_PROJECT_SELECTION: projectId is required");
|
|
83
|
+
if (selection.sequenceId !== undefined && !selection.sequenceId.trim()) {
|
|
84
|
+
throw new Error("INVALID_PROJECT_SELECTION: sequenceId cannot be empty");
|
|
85
|
+
}
|
|
86
|
+
return this.adapter.selectProject(selection);
|
|
87
|
+
}
|
|
88
|
+
async inspectEditor() {
|
|
89
|
+
const identity = await this.adapter.getIdentity();
|
|
90
|
+
const capabilities = withCapabilityFamilies(await this.adapter.getCapabilities(), { backend: identity.backend });
|
|
91
|
+
const speechAnalyzer = this.options.speechAnalyzer;
|
|
92
|
+
const speechTranscribe = capabilities.analyzers.speechTranscribe
|
|
93
|
+
|| (speechAnalyzer ? speechAnalyzer.capabilities?.transcription ?? true : false);
|
|
94
|
+
const speechVad = capabilities.analyzers.speechVad
|
|
95
|
+
|| (speechAnalyzer?.capabilities?.vad ?? false);
|
|
96
|
+
const analyzers = {
|
|
97
|
+
...capabilities.analyzers,
|
|
98
|
+
speechTranscribe,
|
|
99
|
+
speechVad,
|
|
100
|
+
speechCapability: speechTranscribe
|
|
101
|
+
? speechVad ? "transcription-plus-vad" : "transcription-only"
|
|
102
|
+
: "unavailable",
|
|
103
|
+
audioLoudness: capabilities.analyzers.audioLoudness || Boolean(this.options.audioAnalyzer),
|
|
104
|
+
audioNoise: capabilities.analyzers.audioNoise || Boolean(this.options.noiseAnalyzer),
|
|
105
|
+
visualTrack: capabilities.analyzers.visualTrack || Boolean(this.options.visualAnalyzer),
|
|
106
|
+
metadataDescribe: capabilities.analyzers.metadataDescribe || Boolean(this.options.metadataAnalyzer),
|
|
107
|
+
};
|
|
108
|
+
const analyzerBackends = {
|
|
109
|
+
speechTranscribe: speechAnalyzer?.descriptor?.provider,
|
|
110
|
+
speechVad: speechAnalyzer?.descriptor?.provider,
|
|
111
|
+
audioLoudness: this.options.audioAnalyzer?.descriptor?.provider,
|
|
112
|
+
audioNoise: this.options.noiseAnalyzer?.descriptor?.provider,
|
|
113
|
+
visualTrack: this.options.visualAnalyzer?.descriptor?.provider,
|
|
114
|
+
};
|
|
115
|
+
return {
|
|
116
|
+
identity,
|
|
117
|
+
capabilities: {
|
|
118
|
+
...withCapabilityFamilies({ ...capabilities, analyzers }, {
|
|
119
|
+
backend: identity.backend,
|
|
120
|
+
analyzerBackends,
|
|
121
|
+
}),
|
|
122
|
+
},
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
async listAssets(query) {
|
|
126
|
+
const capabilities = await this.adapter.getCapabilities();
|
|
127
|
+
if (!capabilities.editor.assetDiscovery || !this.adapter.listAssets) {
|
|
128
|
+
throw new Error("CAPABILITY_UNAVAILABLE: editor assets");
|
|
129
|
+
}
|
|
130
|
+
const assets = await this.context.listAssets(query);
|
|
131
|
+
return assets.filter((asset) => matchesAssetQuery(asset, query));
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
function matchesAssetQuery(asset, query) {
|
|
135
|
+
if (!query)
|
|
136
|
+
return true;
|
|
137
|
+
const normalized = query.query?.trim().toLowerCase();
|
|
138
|
+
if (normalized && ![asset.id, asset.name, asset.vendor].some((value) => value.toLowerCase().includes(normalized))) {
|
|
139
|
+
return false;
|
|
140
|
+
}
|
|
141
|
+
if (query.kind && asset.kind !== query.kind)
|
|
142
|
+
return false;
|
|
143
|
+
if (query.vendor && asset.vendor.toLowerCase() !== query.vendor.trim().toLowerCase())
|
|
144
|
+
return false;
|
|
145
|
+
return true;
|
|
146
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export class TransactionStore {
|
|
2
|
+
transactions = new Map();
|
|
3
|
+
set(transaction) {
|
|
4
|
+
this.transactions.set(transaction.id, transaction);
|
|
5
|
+
}
|
|
6
|
+
get(transactionId) {
|
|
7
|
+
const transaction = this.transactions.get(transactionId);
|
|
8
|
+
if (!transaction)
|
|
9
|
+
throw new Error(`TRANSACTION_NOT_FOUND: ${transactionId}`);
|
|
10
|
+
return transaction;
|
|
11
|
+
}
|
|
12
|
+
}
|