@morshoto/framekit 0.1.16 → 0.1.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -4
- package/dist-package/adapters/final-cut/typescript/src/canonical.js +19 -7
- package/dist-package/adapters/final-cut/typescript/src/connection.js +13 -8
- package/dist-package/adapters/final-cut/typescript/src/fcpxml.js +147 -34
- package/dist-package/adapters/final-cut/typescript/src/index.js +1 -0
- package/dist-package/adapters/final-cut/typescript/src/library-inspection.js +60 -1
- package/dist-package/adapters/final-cut/typescript/src/live.js +2 -2
- package/dist-package/adapters/final-cut/typescript/src/publish-readback.js +213 -0
- package/dist-package/adapters/final-cut/typescript/src/publisher.js +762 -238
- package/dist-package/adapters/final-cut/typescript/src/session.js +6 -1
- package/dist-package/adapters/final-cut/typescript/src/timeline-ir-fcpxml.js +293 -59
- package/dist-package/adapters/headless/typescript/src/ffmpeg-renderer.js +137 -19
- package/dist-package/adapters/headless/typescript/src/ffmpeg-verifier.js +2 -2
- package/dist-package/apps/cli/src/main.js +34 -19
- package/dist-package/apps/mcp-server/src/build-alignment.js +59 -0
- package/dist-package/apps/mcp-server/src/build-identity.js +30 -0
- package/dist-package/apps/mcp-server/src/build-identity.json +5 -0
- package/dist-package/apps/mcp-server/src/headless-project-snapshot.js +101 -0
- package/dist-package/apps/mcp-server/src/headless-projects.js +473 -3
- package/dist-package/apps/mcp-server/src/main.js +33 -6
- package/dist-package/apps/mcp-server/src/server.js +367 -52
- package/dist-package/apps/mcp-server/src/version.js +21 -2
- package/dist-package/packages/runtime/src/application/media-analysis-service.js +360 -30
- package/dist-package/packages/runtime/src/application/semantic-media-index-store.js +86 -0
- package/dist-package/packages/runtime/src/index.js +1 -0
- package/dist-package/packages/runtime/src/planning/rough-cut.js +103 -7
- package/dist-package/packages/runtime/src/render/render-contract.js +3 -0
- package/dist-package/packages/runtime/src/runtime.js +11 -2
- package/dist-package/packages/runtime/src/timeline/editing-session.js +7 -2
- package/package.json +9 -9
|
@@ -18,12 +18,12 @@ export class MediaAnalysisService {
|
|
|
18
18
|
const media = findMedia(project, mediaId);
|
|
19
19
|
return this.analyzeSpeechForProject(project, media, range);
|
|
20
20
|
}
|
|
21
|
-
async analyzeAudio(mediaId) {
|
|
21
|
+
async analyzeAudio(mediaId, range) {
|
|
22
22
|
if (!this.options.audioAnalyzer)
|
|
23
23
|
throw new Error("CAPABILITY_UNAVAILABLE: audio analysis");
|
|
24
24
|
const project = await this.project.inspectProject();
|
|
25
25
|
const media = findMedia(project, mediaId);
|
|
26
|
-
return this.
|
|
26
|
+
return this.analyzeAudioForProject(project, media, range);
|
|
27
27
|
}
|
|
28
28
|
async analyzeNoise(mediaId, range) {
|
|
29
29
|
if (!this.options.noiseAnalyzer)
|
|
@@ -45,19 +45,8 @@ export class MediaAnalysisService {
|
|
|
45
45
|
const media = findMedia(project, mediaId);
|
|
46
46
|
const sourceStart = clip.sourceStart ?? 0;
|
|
47
47
|
const requestedRange = { start: sourceStart, end: sourceStart + clip.duration };
|
|
48
|
-
const analysis = await this.
|
|
49
|
-
|
|
50
|
-
media,
|
|
51
|
-
mediaId,
|
|
52
|
-
project,
|
|
53
|
-
provider: this.options.audioAnalyzer.descriptor,
|
|
54
|
-
requestedRange,
|
|
55
|
-
});
|
|
56
|
-
const measuredRange = analysis.measuredRange ?? {
|
|
57
|
-
start: requestedRange.start,
|
|
58
|
-
end: requestedRange.start + (analysis.analyzedDurationSeconds ?? clip.duration),
|
|
59
|
-
};
|
|
60
|
-
validateAudioRange(measuredRange, requestedRange, "measured audio range");
|
|
48
|
+
const analysis = await this.analyzeAudioForProject(project, media, requestedRange);
|
|
49
|
+
const measuredRange = analysis.measuredRange ?? requestedRange;
|
|
61
50
|
const analyzedDurationSeconds = analysis.analyzedDurationSeconds ?? measuredRange.end - measuredRange.start;
|
|
62
51
|
if (Math.abs(analyzedDurationSeconds - (measuredRange.end - measuredRange.start)) > 0.000001) {
|
|
63
52
|
throw new Error("ANALYSIS_INVALID: audio measured duration does not match its measured range");
|
|
@@ -132,11 +121,20 @@ export class MediaAnalysisService {
|
|
|
132
121
|
}
|
|
133
122
|
async understandMedia(mediaId) {
|
|
134
123
|
const project = await this.project.inspectProject();
|
|
124
|
+
return this.understandMediaInProject(mediaId, project, true);
|
|
125
|
+
}
|
|
126
|
+
async understandMediaInProject(mediaId, project, attachToLiveContext = false) {
|
|
135
127
|
const media = findMedia(project, mediaId);
|
|
128
|
+
const cached = await this.loadCachedUnderstanding(project, media);
|
|
129
|
+
if (cached) {
|
|
130
|
+
if (attachToLiveContext)
|
|
131
|
+
this.context.attachMediaUnderstanding(cached);
|
|
132
|
+
return structuredClone(cached);
|
|
133
|
+
}
|
|
136
134
|
const input = { project, media };
|
|
137
135
|
const [speechResult, audioResult, noiseResult, visualResult, metadataResult] = await Promise.all([
|
|
138
136
|
settle(() => this.options.speechAnalyzer ? this.analyzeSpeechForProject(project, media) : undefined),
|
|
139
|
-
settle(() => this.options.audioAnalyzer
|
|
137
|
+
settle(() => this.options.audioAnalyzer ? this.analyzeAudioForProject(project, media) : undefined),
|
|
140
138
|
settle(() => this.options.noiseAnalyzer?.analyze(input)),
|
|
141
139
|
settle(() => this.options.visualAnalyzer?.analyze(input)),
|
|
142
140
|
settle(() => this.options.metadataAnalyzer?.analyze(input)),
|
|
@@ -166,7 +164,9 @@ export class MediaAnalysisService {
|
|
|
166
164
|
],
|
|
167
165
|
analysisRevision: project.revision,
|
|
168
166
|
};
|
|
169
|
-
|
|
167
|
+
if (attachToLiveContext)
|
|
168
|
+
this.context.attachMediaUnderstanding(understanding);
|
|
169
|
+
await this.options.semanticMediaIndexStore?.save(understanding);
|
|
170
170
|
return structuredClone(understanding);
|
|
171
171
|
}
|
|
172
172
|
async inspectMedia(mediaId) {
|
|
@@ -178,12 +178,115 @@ export class MediaAnalysisService {
|
|
|
178
178
|
}
|
|
179
179
|
async indexMedia(query = {}) {
|
|
180
180
|
const project = await this.project.inspectProject();
|
|
181
|
-
|
|
181
|
+
const current = this.indexFromProject(project, query);
|
|
182
|
+
const persisted = await this.persistedIndexEntries(query);
|
|
183
|
+
return mergeIndexEntries(persisted, current);
|
|
182
184
|
}
|
|
183
185
|
async planRoughCut(request) {
|
|
184
186
|
const project = await this.project.inspectProject();
|
|
187
|
+
return this.planRoughCutInProject(request, project);
|
|
188
|
+
}
|
|
189
|
+
async planRoughCutInProject(request, project) {
|
|
185
190
|
const { maxShots: _maxShots, ...query } = request;
|
|
186
|
-
|
|
191
|
+
const current = this.indexFromProject(project, query);
|
|
192
|
+
const persisted = await this.persistedIndexEntries(query);
|
|
193
|
+
const projectMedia = project.media.map(sourceIdentityOf);
|
|
194
|
+
const matchingPersisted = persisted.filter((entry) => projectMedia.some((source) => sameMediaSourceIdentity(source, entry.sourceIdentity)));
|
|
195
|
+
return planRoughCut(mergeIndexEntries(matchingPersisted, current), project.revision, request);
|
|
196
|
+
}
|
|
197
|
+
async inspectTimelineSemanticContext(query = {}) {
|
|
198
|
+
const project = await this.project.inspectProject();
|
|
199
|
+
const requestedOccurrenceIds = query.occurrenceIds ? new Set(query.occurrenceIds) : undefined;
|
|
200
|
+
const occurrences = [];
|
|
201
|
+
for (const clip of project.timeline.clips) {
|
|
202
|
+
if (requestedOccurrenceIds && !requestedOccurrenceIds.has(clip.id))
|
|
203
|
+
continue;
|
|
204
|
+
const context = await this.inspectTimelineOccurrence(project, clip, query);
|
|
205
|
+
if (context.status === "available" || requestedOccurrenceIds || !query.query) {
|
|
206
|
+
if (!query.query || context.status === "unavailable" || context.observations.length > 0) {
|
|
207
|
+
occurrences.push(context);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
return {
|
|
212
|
+
projectId: project.projectId,
|
|
213
|
+
timelineId: project.timeline.id,
|
|
214
|
+
revision: structuredClone(project.revision),
|
|
215
|
+
query: structuredClone(query),
|
|
216
|
+
occurrences,
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
async inspectTimelineOccurrence(project, clip, query) {
|
|
220
|
+
if (!clip.mediaId)
|
|
221
|
+
return unavailableTimelineOccurrence(clip.id, "timeline occurrence has no media binding");
|
|
222
|
+
const media = project.media.find((candidate) => candidate.mediaId === clip.mediaId);
|
|
223
|
+
if (!media)
|
|
224
|
+
return unavailableTimelineOccurrence(clip.id, `media binding is unavailable: ${clip.mediaId}`, clip.mediaId);
|
|
225
|
+
const sourceStart = clip.sourceStart ?? (clip.sourceStartTime ? rationalSeconds(clip.sourceStartTime) : 0);
|
|
226
|
+
const sourceDuration = clip.durationTime ? rationalSeconds(clip.durationTime) : clip.duration;
|
|
227
|
+
const sourceRange = {
|
|
228
|
+
start: sourceStart,
|
|
229
|
+
end: sourceStart + sourceDuration,
|
|
230
|
+
...(clip.sourceStartTime ? { startTime: structuredClone(clip.sourceStartTime) } : {}),
|
|
231
|
+
durationTime: structuredClone(clip.durationTime),
|
|
232
|
+
};
|
|
233
|
+
if (!Number.isFinite(sourceRange.start) || !Number.isFinite(sourceRange.end) || sourceRange.start < 0 || sourceRange.end <= sourceRange.start) {
|
|
234
|
+
return unavailableTimelineOccurrence(clip.id, "timeline occurrence source range is invalid", clip.mediaId, sourceRange);
|
|
235
|
+
}
|
|
236
|
+
if (media.duration !== undefined && sourceRange.end > media.duration + 0.000001) {
|
|
237
|
+
return unavailableTimelineOccurrence(clip.id, "timeline occurrence source range exceeds media duration", clip.mediaId, sourceRange);
|
|
238
|
+
}
|
|
239
|
+
const sourceIdentity = sourceIdentityOf(media);
|
|
240
|
+
const understanding = await this.options.semanticMediaIndexStore?.load(sourceIdentity)
|
|
241
|
+
?? understandingFromMediaContext(media, project.revision);
|
|
242
|
+
if (!understanding) {
|
|
243
|
+
return unavailableTimelineOccurrence(clip.id, "source-bound semantic index entry is unavailable", clip.mediaId, sourceRange, sourceIdentity);
|
|
244
|
+
}
|
|
245
|
+
const observations = timelineSemanticObservations(understanding, sourceRange);
|
|
246
|
+
const filtered = query.query
|
|
247
|
+
? observations.filter((observation) => timelineObservationMatches(observation, query.query))
|
|
248
|
+
: observations;
|
|
249
|
+
return {
|
|
250
|
+
occurrenceId: clip.id,
|
|
251
|
+
status: "available",
|
|
252
|
+
mediaId: clip.mediaId,
|
|
253
|
+
sourceIdentity: structuredClone(understanding.sourceIdentity),
|
|
254
|
+
sourceRange,
|
|
255
|
+
observations: filtered,
|
|
256
|
+
analysis: structuredClone(understanding.analysis),
|
|
257
|
+
analysisRevision: understanding.analysisRevision.id,
|
|
258
|
+
};
|
|
259
|
+
}
|
|
260
|
+
async loadCachedUnderstanding(project, media) {
|
|
261
|
+
const store = this.options.semanticMediaIndexStore;
|
|
262
|
+
if (!store)
|
|
263
|
+
return undefined;
|
|
264
|
+
const cached = await store.load(sourceIdentityOf(media));
|
|
265
|
+
if (!cached || !cacheMatchesAnalyzers(cached, this.options))
|
|
266
|
+
return undefined;
|
|
267
|
+
const rebound = structuredClone(cached);
|
|
268
|
+
const sourceIdentity = sourceIdentityOf(media);
|
|
269
|
+
rebound.sourceIdentity = sourceIdentity;
|
|
270
|
+
rebound.mediaId = media.mediaId;
|
|
271
|
+
rebound.source = media.source;
|
|
272
|
+
rebound.analysisRevision = structuredClone(project.revision);
|
|
273
|
+
if (rebound.speech) {
|
|
274
|
+
rebound.speech.mediaId = media.mediaId;
|
|
275
|
+
rebound.speech.sourceIdentity = sourceIdentity;
|
|
276
|
+
rebound.speech.revision = structuredClone(project.revision);
|
|
277
|
+
}
|
|
278
|
+
if (rebound.audio) {
|
|
279
|
+
rebound.audio.mediaId = media.mediaId;
|
|
280
|
+
rebound.audio.sourceIdentity = sourceIdentity;
|
|
281
|
+
rebound.audio.revision = structuredClone(project.revision);
|
|
282
|
+
}
|
|
283
|
+
return rebound;
|
|
284
|
+
}
|
|
285
|
+
async persistedIndexEntries(query) {
|
|
286
|
+
const entries = await this.options.semanticMediaIndexStore?.list() ?? [];
|
|
287
|
+
return entries
|
|
288
|
+
.map(understandingToIndexEntry)
|
|
289
|
+
.filter((entry) => matchesMediaIndexQuery(entry, query));
|
|
187
290
|
}
|
|
188
291
|
indexFromProject(project, query) {
|
|
189
292
|
return project.media
|
|
@@ -197,6 +300,12 @@ export class MediaAnalysisService {
|
|
|
197
300
|
analysisStatus("visual", this.options.visualAnalyzer, sourceIdentityOf(media), false),
|
|
198
301
|
analysisStatus("metadata", this.options.metadataAnalyzer, sourceIdentityOf(media), false),
|
|
199
302
|
],
|
|
303
|
+
...(media.analysis?.some((record) => record.capability === "speech" && record.status === "analyzed") && media.speech
|
|
304
|
+
? { speech: structuredClone(media.speech) }
|
|
305
|
+
: {}),
|
|
306
|
+
...(media.analysis?.some((record) => record.capability === "audio" && record.status === "analyzed") && media.audio
|
|
307
|
+
? { audio: structuredClone(media.audio) }
|
|
308
|
+
: {}),
|
|
200
309
|
...(media.analysisRevision ? { analysisRevision: media.analysisRevision } : {}),
|
|
201
310
|
}))
|
|
202
311
|
.filter((entry) => matchesMediaIndexQuery(entry, query));
|
|
@@ -239,7 +348,7 @@ export class MediaAnalysisService {
|
|
|
239
348
|
}
|
|
240
349
|
}
|
|
241
350
|
if (requirements.audio && this.options.audioAnalyzer) {
|
|
242
|
-
const analyses = await Promise.all(ranges.map((range) => this.
|
|
351
|
+
const analyses = await Promise.all(ranges.map((range) => this.analyzeAudioForProject(next, media, range)));
|
|
243
352
|
if (analyses[analyses.length - 1])
|
|
244
353
|
media.audio = analyses[analyses.length - 1];
|
|
245
354
|
}
|
|
@@ -281,6 +390,44 @@ export class MediaAnalysisService {
|
|
|
281
390
|
provider: analyzer.descriptor,
|
|
282
391
|
});
|
|
283
392
|
}
|
|
393
|
+
async analyzeAudioForProject(project, media, range) {
|
|
394
|
+
const analyzer = this.options.audioAnalyzer;
|
|
395
|
+
if (!analyzer)
|
|
396
|
+
throw new Error("CAPABILITY_UNAVAILABLE: audio analysis");
|
|
397
|
+
const requestedRange = range ?? (media.duration === undefined ? undefined : { start: 0, end: media.duration });
|
|
398
|
+
validateRequestedAudioRange(requestedRange, media.duration, "requested audio range");
|
|
399
|
+
const analysis = await analyzer.analyze({ project, media }, requestedRange);
|
|
400
|
+
validateAudioProvenance(analysis, {
|
|
401
|
+
media,
|
|
402
|
+
mediaId: media.mediaId,
|
|
403
|
+
project,
|
|
404
|
+
provider: analyzer.descriptor,
|
|
405
|
+
requestedRange,
|
|
406
|
+
});
|
|
407
|
+
const measuredRange = analysis.measuredRange ?? (requestedRange ? structuredClone(requestedRange) : undefined);
|
|
408
|
+
if (measuredRange && requestedRange)
|
|
409
|
+
validateAudioRange(measuredRange, requestedRange, "measured audio range");
|
|
410
|
+
if (measuredRange)
|
|
411
|
+
validateRequestedAudioRange(measuredRange, media.duration, "measured audio source range");
|
|
412
|
+
const analyzedDurationSeconds = measuredRange
|
|
413
|
+
? analysis.analyzedDurationSeconds ?? measuredRange.end - measuredRange.start
|
|
414
|
+
: analysis.analyzedDurationSeconds;
|
|
415
|
+
if (measuredRange && analyzedDurationSeconds !== undefined
|
|
416
|
+
&& Math.abs(analyzedDurationSeconds - (measuredRange.end - measuredRange.start)) > 0.000001) {
|
|
417
|
+
throw new Error("ANALYSIS_INVALID: audio measured duration does not match its measured range");
|
|
418
|
+
}
|
|
419
|
+
return {
|
|
420
|
+
...structuredClone(analysis),
|
|
421
|
+
schemaVersion: 1,
|
|
422
|
+
mediaId: media.mediaId,
|
|
423
|
+
sourceIdentity: sourceIdentityOf(media),
|
|
424
|
+
...(requestedRange ? { requestedRange: structuredClone(requestedRange) } : {}),
|
|
425
|
+
...(measuredRange ? { measuredRange: structuredClone(measuredRange) } : {}),
|
|
426
|
+
revision: structuredClone(project.revision),
|
|
427
|
+
provider: analysis.provider ?? analyzer.descriptor ?? { id: "framekit.audio", provider: "unknown" },
|
|
428
|
+
...(analyzedDurationSeconds !== undefined ? { analyzedDurationSeconds } : {}),
|
|
429
|
+
};
|
|
430
|
+
}
|
|
284
431
|
}
|
|
285
432
|
function sourceIdentityOf(media) {
|
|
286
433
|
return {
|
|
@@ -355,10 +502,14 @@ function mergeRanges(ranges) {
|
|
|
355
502
|
for (const range of [...ranges].sort((left, right) => left.start - right.start || left.end - right.end)) {
|
|
356
503
|
const previous = merged[merged.length - 1];
|
|
357
504
|
if (!previous || range.start > previous.end) {
|
|
358
|
-
merged.push(
|
|
505
|
+
merged.push(structuredClone(range));
|
|
359
506
|
continue;
|
|
360
507
|
}
|
|
361
|
-
|
|
508
|
+
if (range.end > previous.end) {
|
|
509
|
+
previous.end = range.end;
|
|
510
|
+
delete previous.startTime;
|
|
511
|
+
delete previous.durationTime;
|
|
512
|
+
}
|
|
362
513
|
}
|
|
363
514
|
return merged;
|
|
364
515
|
}
|
|
@@ -371,6 +522,11 @@ function sameRational(left, right) {
|
|
|
371
522
|
return leftParts.value * rightParts.timescale === rightParts.value * leftParts.timescale;
|
|
372
523
|
}
|
|
373
524
|
function semanticFromAnalyses(speech, audio, visual, metadata) {
|
|
525
|
+
const usableRanges = mergeRanges([
|
|
526
|
+
...(metadata?.usableRanges ?? []),
|
|
527
|
+
...(visual?.scenes ?? []).map((scene) => ({ start: scene.start, end: scene.end })),
|
|
528
|
+
...(speech?.vadSegments ?? []).filter((segment) => segment.kind === "speech").map((segment) => ({ start: segment.start, end: segment.end })),
|
|
529
|
+
]);
|
|
374
530
|
return {
|
|
375
531
|
subjects: [
|
|
376
532
|
...(visual?.subjects ?? []).map((subject) => ({ value: subject.label, confidence: subject.confidence })),
|
|
@@ -384,7 +540,7 @@ function semanticFromAnalyses(speech, audio, visual, metadata) {
|
|
|
384
540
|
timeOfDay: metadata?.timeOfDay ? structuredClone(metadata.timeOfDay) : [],
|
|
385
541
|
moods: metadata?.moods ? structuredClone(metadata.moods) : [],
|
|
386
542
|
...(visual?.motion ? { motion: structuredClone(visual.motion) } : {}),
|
|
387
|
-
usableRanges
|
|
543
|
+
usableRanges,
|
|
388
544
|
...(speech ? { transcript: speech.words.map((word) => word.text).join(" ") } : {}),
|
|
389
545
|
...(audio ? {
|
|
390
546
|
audio: {
|
|
@@ -396,6 +552,159 @@ function semanticFromAnalyses(speech, audio, visual, metadata) {
|
|
|
396
552
|
} : {}),
|
|
397
553
|
};
|
|
398
554
|
}
|
|
555
|
+
function understandingFromMediaContext(media, revision) {
|
|
556
|
+
if (!media.semantic && !media.analysis && !media.speech && !media.visual && !media.metadata)
|
|
557
|
+
return undefined;
|
|
558
|
+
const sourceIdentity = sourceIdentityOf(media);
|
|
559
|
+
return {
|
|
560
|
+
mediaId: media.mediaId,
|
|
561
|
+
source: media.source,
|
|
562
|
+
sourceIdentity,
|
|
563
|
+
...(media.metadata ? { metadata: structuredClone(media.metadata) } : {}),
|
|
564
|
+
...(media.speech ? { speech: structuredClone(media.speech) } : {}),
|
|
565
|
+
...(media.audio ? { audio: structuredClone(media.audio) } : {}),
|
|
566
|
+
...(media.noise ? { noise: structuredClone(media.noise) } : {}),
|
|
567
|
+
...(media.visual ? { visual: structuredClone(media.visual) } : {}),
|
|
568
|
+
semantic: structuredClone(media.semantic ?? emptySemanticDescription()),
|
|
569
|
+
analysis: structuredClone(media.analysis ?? []),
|
|
570
|
+
analysisRevision: {
|
|
571
|
+
id: media.analysisRevision ?? revision.id,
|
|
572
|
+
sequence: revision.sequence,
|
|
573
|
+
timestamp: revision.timestamp,
|
|
574
|
+
},
|
|
575
|
+
};
|
|
576
|
+
}
|
|
577
|
+
function timelineSemanticObservations(understanding, sourceRange) {
|
|
578
|
+
const observations = [];
|
|
579
|
+
const analyzerFor = (capability) => understanding.analysis.find((status) => status.capability === capability)?.provenance?.analyzer;
|
|
580
|
+
for (const word of understanding.speech?.words ?? []) {
|
|
581
|
+
const range = intersectTimeRanges({ start: word.start, end: word.end }, sourceRange);
|
|
582
|
+
if (!range)
|
|
583
|
+
continue;
|
|
584
|
+
observations.push({
|
|
585
|
+
capability: "speech",
|
|
586
|
+
range,
|
|
587
|
+
text: word.text,
|
|
588
|
+
confidence: word.confidence,
|
|
589
|
+
...(analyzerFor("speech") ? { analyzer: structuredClone(analyzerFor("speech")) } : {}),
|
|
590
|
+
});
|
|
591
|
+
}
|
|
592
|
+
for (const scene of understanding.visual?.scenes ?? []) {
|
|
593
|
+
const range = intersectTimeRanges({ start: scene.start, end: scene.end }, sourceRange);
|
|
594
|
+
if (!range)
|
|
595
|
+
continue;
|
|
596
|
+
observations.push({
|
|
597
|
+
capability: "visual",
|
|
598
|
+
range,
|
|
599
|
+
...(scene.label ? { label: scene.label } : {}),
|
|
600
|
+
...(scene.confidence !== undefined ? { confidence: scene.confidence } : {}),
|
|
601
|
+
...(analyzerFor("visual") ? { analyzer: structuredClone(analyzerFor("visual")) } : {}),
|
|
602
|
+
});
|
|
603
|
+
}
|
|
604
|
+
for (const subject of understanding.visual?.subjects ?? []) {
|
|
605
|
+
const range = intersectTimeRanges({
|
|
606
|
+
start: subject.start ?? sourceRange.start,
|
|
607
|
+
end: subject.end ?? sourceRange.end,
|
|
608
|
+
}, sourceRange);
|
|
609
|
+
if (!range)
|
|
610
|
+
continue;
|
|
611
|
+
observations.push({
|
|
612
|
+
capability: "visual",
|
|
613
|
+
range,
|
|
614
|
+
label: subject.label,
|
|
615
|
+
confidence: subject.confidence,
|
|
616
|
+
...(analyzerFor("visual") ? { analyzer: structuredClone(analyzerFor("visual")) } : {}),
|
|
617
|
+
});
|
|
618
|
+
}
|
|
619
|
+
for (const range of understanding.metadata?.usableRanges ?? []) {
|
|
620
|
+
const clipped = intersectTimeRanges(range, sourceRange);
|
|
621
|
+
if (!clipped)
|
|
622
|
+
continue;
|
|
623
|
+
observations.push({
|
|
624
|
+
capability: "metadata",
|
|
625
|
+
range: clipped,
|
|
626
|
+
...(analyzerFor("metadata") ? { analyzer: structuredClone(analyzerFor("metadata")) } : {}),
|
|
627
|
+
});
|
|
628
|
+
}
|
|
629
|
+
return observations.sort((left, right) => left.range.start - right.range.start
|
|
630
|
+
|| left.range.end - right.range.end
|
|
631
|
+
|| left.capability.localeCompare(right.capability)
|
|
632
|
+
|| (left.text ?? left.label ?? "").localeCompare(right.text ?? right.label ?? ""));
|
|
633
|
+
}
|
|
634
|
+
function intersectTimeRanges(left, right) {
|
|
635
|
+
const start = Math.max(left.start, right.start);
|
|
636
|
+
const end = Math.min(left.end, right.end);
|
|
637
|
+
return start < end ? { start, end } : undefined;
|
|
638
|
+
}
|
|
639
|
+
function timelineObservationMatches(observation, query) {
|
|
640
|
+
const normalized = query.trim().toLocaleLowerCase();
|
|
641
|
+
if (!normalized)
|
|
642
|
+
return true;
|
|
643
|
+
return [observation.text, observation.label]
|
|
644
|
+
.filter((value) => Boolean(value))
|
|
645
|
+
.some((value) => value.toLocaleLowerCase().includes(normalized));
|
|
646
|
+
}
|
|
647
|
+
function unavailableTimelineOccurrence(occurrenceId, reason, mediaId, sourceRange, sourceIdentity) {
|
|
648
|
+
return {
|
|
649
|
+
occurrenceId,
|
|
650
|
+
status: "unavailable",
|
|
651
|
+
...(mediaId ? { mediaId } : {}),
|
|
652
|
+
...(sourceIdentity ? { sourceIdentity: structuredClone(sourceIdentity) } : {}),
|
|
653
|
+
...(sourceRange ? { sourceRange: structuredClone(sourceRange) } : {}),
|
|
654
|
+
reason,
|
|
655
|
+
observations: [],
|
|
656
|
+
analysis: [],
|
|
657
|
+
};
|
|
658
|
+
}
|
|
659
|
+
function rationalSeconds(time) {
|
|
660
|
+
const parsed = parseRational(time, "TIMELINE_SEMANTIC_CONTEXT_INVALID");
|
|
661
|
+
const seconds = Number(parsed.value) / Number(parsed.timescale);
|
|
662
|
+
if (!Number.isFinite(seconds))
|
|
663
|
+
throw new Error("TIMELINE_SEMANTIC_CONTEXT_INVALID: rational time is outside supported range");
|
|
664
|
+
return seconds;
|
|
665
|
+
}
|
|
666
|
+
function understandingToIndexEntry(understanding) {
|
|
667
|
+
return {
|
|
668
|
+
sourceIdentity: structuredClone(understanding.sourceIdentity),
|
|
669
|
+
semantic: structuredClone(understanding.semantic),
|
|
670
|
+
analysis: structuredClone(understanding.analysis),
|
|
671
|
+
...(understanding.speech ? { speech: structuredClone(understanding.speech) } : {}),
|
|
672
|
+
...(understanding.audio ? { audio: structuredClone(understanding.audio) } : {}),
|
|
673
|
+
analysisRevision: understanding.analysisRevision.id,
|
|
674
|
+
};
|
|
675
|
+
}
|
|
676
|
+
function mergeIndexEntries(persisted, current) {
|
|
677
|
+
const merged = new Map();
|
|
678
|
+
for (const entry of [...persisted, ...current]) {
|
|
679
|
+
const key = JSON.stringify(entry.sourceIdentity);
|
|
680
|
+
const previous = merged.get(key);
|
|
681
|
+
if (!previous || indexEntryScore(entry) >= indexEntryScore(previous))
|
|
682
|
+
merged.set(key, structuredClone(entry));
|
|
683
|
+
}
|
|
684
|
+
return [...merged.values()].sort((left, right) => left.sourceIdentity.mediaId.localeCompare(right.sourceIdentity.mediaId));
|
|
685
|
+
}
|
|
686
|
+
function indexEntryScore(entry) {
|
|
687
|
+
return entry.analysis.filter((record) => record.status === "analyzed").length
|
|
688
|
+
+ entry.semantic.usableRanges.length
|
|
689
|
+
+ (entry.semantic.transcript?.trim() ? 1 : 0);
|
|
690
|
+
}
|
|
691
|
+
function cacheMatchesAnalyzers(understanding, options) {
|
|
692
|
+
const configured = [
|
|
693
|
+
["speech", options.speechAnalyzer],
|
|
694
|
+
["audio", options.audioAnalyzer],
|
|
695
|
+
["noise", options.noiseAnalyzer],
|
|
696
|
+
["visual", options.visualAnalyzer],
|
|
697
|
+
["metadata", options.metadataAnalyzer],
|
|
698
|
+
];
|
|
699
|
+
return configured.every(([capability, analyzer]) => {
|
|
700
|
+
if (!analyzer)
|
|
701
|
+
return true;
|
|
702
|
+
const record = understanding.analysis.find((candidate) => candidate.capability === capability);
|
|
703
|
+
return record?.status === "analyzed"
|
|
704
|
+
&& record.provenance?.analyzer !== undefined
|
|
705
|
+
&& sameDescriptor(record.provenance.analyzer, analyzer.descriptor ?? { id: `framekit.${capability}`, provider: "unknown" });
|
|
706
|
+
});
|
|
707
|
+
}
|
|
399
708
|
function emptySemanticDescription() {
|
|
400
709
|
return {
|
|
401
710
|
subjects: [],
|
|
@@ -407,7 +716,7 @@ function emptySemanticDescription() {
|
|
|
407
716
|
};
|
|
408
717
|
}
|
|
409
718
|
function matchesMediaIndexQuery(entry, query) {
|
|
410
|
-
const
|
|
719
|
+
const terms = query.query?.trim().toLowerCase().split(/\s+/u).filter(Boolean) ?? [];
|
|
411
720
|
const values = [
|
|
412
721
|
entry.sourceIdentity.mediaId,
|
|
413
722
|
entry.sourceIdentity.source,
|
|
@@ -418,7 +727,7 @@ function matchesMediaIndexQuery(entry, query) {
|
|
|
418
727
|
...entry.semantic.timeOfDay.map((tag) => tag.value),
|
|
419
728
|
...entry.semantic.moods.map((tag) => tag.value),
|
|
420
729
|
].map((value) => value.toLowerCase());
|
|
421
|
-
if (
|
|
730
|
+
if (terms.length > 0 && !values.some((value) => terms.some((term) => value.includes(term))))
|
|
422
731
|
return false;
|
|
423
732
|
if (query.subject && !matchesTag(entry.semantic.subjects, query.subject))
|
|
424
733
|
return false;
|
|
@@ -432,13 +741,21 @@ function matchesMediaIndexQuery(entry, query) {
|
|
|
432
741
|
return false;
|
|
433
742
|
if (query.motion && entry.semantic.motion?.label !== query.motion)
|
|
434
743
|
return false;
|
|
435
|
-
if (query.range && !entry.
|
|
744
|
+
if (query.range && !indexEntryRanges(entry).some((range) => range.end > query.range.start && range.start < query.range.end))
|
|
436
745
|
return false;
|
|
437
746
|
if (query.capabilities?.some((capability) => !entry.analysis.some((record) => record.capability === capability && record.status !== "unavailable"))) {
|
|
438
747
|
return false;
|
|
439
748
|
}
|
|
440
749
|
return true;
|
|
441
750
|
}
|
|
751
|
+
function indexEntryRanges(entry) {
|
|
752
|
+
const speechRanges = entry.speech?.vadSegments
|
|
753
|
+
?.filter((segment) => segment.kind === "speech")
|
|
754
|
+
.map(({ start, end }) => ({ start, end }))
|
|
755
|
+
?? entry.speech?.words.map(({ start, end }) => ({ start, end }))
|
|
756
|
+
?? [];
|
|
757
|
+
return [...entry.semantic.usableRanges, ...speechRanges];
|
|
758
|
+
}
|
|
442
759
|
function matchesTag(tags, query) {
|
|
443
760
|
const normalized = query.trim().toLowerCase();
|
|
444
761
|
return tags.some((tag) => tag.value.toLowerCase() === normalized);
|
|
@@ -494,11 +811,12 @@ function validateAudioProvenance(analysis, expected) {
|
|
|
494
811
|
&& !sameMediaSourceIdentity(analysis.sourceIdentity, sourceIdentityOf(expected.media))) {
|
|
495
812
|
throw new Error("TARGET_MISMATCH: audio analysis source identity does not match the requested media");
|
|
496
813
|
}
|
|
497
|
-
if (analysis.requestedRange !== undefined &&
|
|
814
|
+
if (analysis.requestedRange !== undefined && expected.requestedRange !== undefined
|
|
815
|
+
&& !sameRange(analysis.requestedRange, expected.requestedRange)) {
|
|
498
816
|
throw new Error("ANALYSIS_INVALID: audio requested range does not match the runtime request");
|
|
499
817
|
}
|
|
500
818
|
if (analysis.revision !== undefined
|
|
501
|
-
&& (analysis.revision
|
|
819
|
+
&& !sameContextRevision(analysis.revision, expected.project.revision)) {
|
|
502
820
|
throw new Error("ANALYSIS_STALE: audio analysis revision does not match the inspected project");
|
|
503
821
|
}
|
|
504
822
|
if (analysis.provider !== undefined && expected.provider !== undefined
|
|
@@ -507,11 +825,23 @@ function validateAudioProvenance(analysis, expected) {
|
|
|
507
825
|
}
|
|
508
826
|
}
|
|
509
827
|
function validateAudioRange(range, requested, label) {
|
|
510
|
-
|
|
511
|
-
|
|
828
|
+
validateRequestedAudioRange(range, undefined, label);
|
|
829
|
+
if (range.start < requested.start || range.end > requested.end) {
|
|
512
830
|
throw new Error(`ANALYSIS_INVALID: ${label} must fit inside the requested occurrence range`);
|
|
513
831
|
}
|
|
514
832
|
}
|
|
833
|
+
function validateRequestedAudioRange(range, duration, label) {
|
|
834
|
+
if (!range)
|
|
835
|
+
return;
|
|
836
|
+
if (!Number.isFinite(range.start) || !Number.isFinite(range.end)
|
|
837
|
+
|| range.start < 0 || range.end <= range.start
|
|
838
|
+
|| (duration !== undefined && range.end > duration)) {
|
|
839
|
+
throw new Error(`ANALYSIS_INVALID: ${label} must be finite, positive, and inside the source duration`);
|
|
840
|
+
}
|
|
841
|
+
}
|
|
842
|
+
function sameContextRevision(left, right) {
|
|
843
|
+
return left.id === right.id && left.sequence === right.sequence && left.timestamp === right.timestamp;
|
|
844
|
+
}
|
|
515
845
|
function sameRange(left, right) {
|
|
516
846
|
return Math.abs(left.start - right.start) <= 0.000001
|
|
517
847
|
&& Math.abs(left.end - right.end) <= 0.000001;
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { mkdir, readFile, rename, unlink, writeFile } from "node:fs/promises";
|
|
2
|
+
import { dirname } from "node:path";
|
|
3
|
+
import { randomUUID } from "node:crypto";
|
|
4
|
+
import { sameMediaSourceIdentity } from "../domain/media.js";
|
|
5
|
+
export const SEMANTIC_MEDIA_INDEX_SCHEMA_VERSION = 1;
|
|
6
|
+
/** Deterministic store for runtime and integration tests. */
|
|
7
|
+
export class InMemorySemanticMediaIndexStore {
|
|
8
|
+
entries = [];
|
|
9
|
+
async load(source) {
|
|
10
|
+
const entry = this.entries.find((candidate) => sameMediaSourceIdentity(candidate.sourceIdentity, source));
|
|
11
|
+
return entry ? structuredClone(entry) : undefined;
|
|
12
|
+
}
|
|
13
|
+
async list() {
|
|
14
|
+
return structuredClone(this.entries);
|
|
15
|
+
}
|
|
16
|
+
async save(understanding) {
|
|
17
|
+
const retained = this.entries.filter((candidate) => candidate.mediaId !== understanding.mediaId);
|
|
18
|
+
retained.push(structuredClone(understanding));
|
|
19
|
+
retained.sort((left, right) => left.mediaId.localeCompare(right.mediaId));
|
|
20
|
+
this.entries.splice(0, this.entries.length, ...retained);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
/** Atomic JSON persistence for a local semantic media index. */
|
|
24
|
+
export class JsonSemanticMediaIndexStore {
|
|
25
|
+
path;
|
|
26
|
+
constructor(path) {
|
|
27
|
+
this.path = path;
|
|
28
|
+
}
|
|
29
|
+
async load(source) {
|
|
30
|
+
const document = await this.readDocument();
|
|
31
|
+
const entry = document.entries.find((candidate) => sameMediaSourceIdentity(candidate.sourceIdentity, source));
|
|
32
|
+
return entry ? structuredClone(entry) : undefined;
|
|
33
|
+
}
|
|
34
|
+
async list() {
|
|
35
|
+
return structuredClone((await this.readDocument()).entries);
|
|
36
|
+
}
|
|
37
|
+
async save(understanding) {
|
|
38
|
+
const document = await this.readDocument();
|
|
39
|
+
document.entries = [
|
|
40
|
+
...document.entries.filter((candidate) => candidate.mediaId !== understanding.mediaId),
|
|
41
|
+
structuredClone(understanding),
|
|
42
|
+
].sort((left, right) => left.mediaId.localeCompare(right.mediaId));
|
|
43
|
+
await mkdir(dirname(this.path), { recursive: true });
|
|
44
|
+
const temporary = `${this.path}.${randomUUID()}.tmp`;
|
|
45
|
+
try {
|
|
46
|
+
await writeFile(temporary, `${JSON.stringify(document)}\n`, { encoding: "utf8", flag: "wx" });
|
|
47
|
+
await rename(temporary, this.path);
|
|
48
|
+
}
|
|
49
|
+
finally {
|
|
50
|
+
try {
|
|
51
|
+
await unlink(temporary);
|
|
52
|
+
}
|
|
53
|
+
catch (error) {
|
|
54
|
+
if (!isMissingFile(error))
|
|
55
|
+
throw error;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
async readDocument() {
|
|
60
|
+
let raw;
|
|
61
|
+
try {
|
|
62
|
+
raw = await readFile(this.path, "utf8");
|
|
63
|
+
}
|
|
64
|
+
catch (error) {
|
|
65
|
+
if (isMissingFile(error))
|
|
66
|
+
return { schemaVersion: SEMANTIC_MEDIA_INDEX_SCHEMA_VERSION, entries: [] };
|
|
67
|
+
throw error;
|
|
68
|
+
}
|
|
69
|
+
let value;
|
|
70
|
+
try {
|
|
71
|
+
value = JSON.parse(raw);
|
|
72
|
+
}
|
|
73
|
+
catch (error) {
|
|
74
|
+
throw new Error(`SEMANTIC_INDEX_CORRUPT: index JSON is malformed (${String(error)})`);
|
|
75
|
+
}
|
|
76
|
+
if (!value || typeof value !== "object"
|
|
77
|
+
|| value.schemaVersion !== SEMANTIC_MEDIA_INDEX_SCHEMA_VERSION
|
|
78
|
+
|| !Array.isArray(value.entries)) {
|
|
79
|
+
throw new Error("SEMANTIC_INDEX_CORRUPT: unsupported semantic index document");
|
|
80
|
+
}
|
|
81
|
+
return structuredClone(value);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
function isMissingFile(error) {
|
|
85
|
+
return Boolean(error && typeof error === "object" && "code" in error && error.code === "ENOENT");
|
|
86
|
+
}
|
|
@@ -14,6 +14,7 @@ export * from "./timeline/fast-observation.js";
|
|
|
14
14
|
export * from "./application/project-store.js";
|
|
15
15
|
export * from "./application/local-media.js";
|
|
16
16
|
export * from "./application/project-transactions.js";
|
|
17
|
+
export * from "./application/semantic-media-index-store.js";
|
|
17
18
|
export * from "./context/project-catalog.js";
|
|
18
19
|
export * from "./speech/filler-removal.js";
|
|
19
20
|
export * from "./audio/dialogue-normalization.js";
|