@morshoto/framekit 0.1.16 → 0.1.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist-package/adapters/final-cut/typescript/src/connection.js +8 -5
- package/dist-package/adapters/final-cut/typescript/src/fcpxml.js +21 -7
- package/dist-package/adapters/final-cut/typescript/src/timeline-ir-fcpxml.js +110 -28
- package/dist-package/adapters/headless/typescript/src/ffmpeg-renderer.js +120 -12
- package/dist-package/apps/mcp-server/src/headless-projects.js +233 -2
- package/dist-package/apps/mcp-server/src/main.js +6 -3
- package/dist-package/apps/mcp-server/src/server.js +97 -3
- package/dist-package/packages/runtime/src/application/media-analysis-service.js +347 -27
- package/dist-package/packages/runtime/src/application/semantic-media-index-store.js +86 -0
- package/dist-package/packages/runtime/src/index.js +1 -0
- package/dist-package/packages/runtime/src/planning/rough-cut.js +103 -7
- package/dist-package/packages/runtime/src/render/render-contract.js +3 -0
- package/dist-package/packages/runtime/src/runtime.js +5 -2
- package/package.json +1 -1
|
@@ -18,12 +18,12 @@ export class MediaAnalysisService {
|
|
|
18
18
|
const media = findMedia(project, mediaId);
|
|
19
19
|
return this.analyzeSpeechForProject(project, media, range);
|
|
20
20
|
}
|
|
21
|
-
async analyzeAudio(mediaId) {
|
|
21
|
+
async analyzeAudio(mediaId, range) {
|
|
22
22
|
if (!this.options.audioAnalyzer)
|
|
23
23
|
throw new Error("CAPABILITY_UNAVAILABLE: audio analysis");
|
|
24
24
|
const project = await this.project.inspectProject();
|
|
25
25
|
const media = findMedia(project, mediaId);
|
|
26
|
-
return this.
|
|
26
|
+
return this.analyzeAudioForProject(project, media, range);
|
|
27
27
|
}
|
|
28
28
|
async analyzeNoise(mediaId, range) {
|
|
29
29
|
if (!this.options.noiseAnalyzer)
|
|
@@ -45,19 +45,8 @@ export class MediaAnalysisService {
|
|
|
45
45
|
const media = findMedia(project, mediaId);
|
|
46
46
|
const sourceStart = clip.sourceStart ?? 0;
|
|
47
47
|
const requestedRange = { start: sourceStart, end: sourceStart + clip.duration };
|
|
48
|
-
const analysis = await this.
|
|
49
|
-
|
|
50
|
-
media,
|
|
51
|
-
mediaId,
|
|
52
|
-
project,
|
|
53
|
-
provider: this.options.audioAnalyzer.descriptor,
|
|
54
|
-
requestedRange,
|
|
55
|
-
});
|
|
56
|
-
const measuredRange = analysis.measuredRange ?? {
|
|
57
|
-
start: requestedRange.start,
|
|
58
|
-
end: requestedRange.start + (analysis.analyzedDurationSeconds ?? clip.duration),
|
|
59
|
-
};
|
|
60
|
-
validateAudioRange(measuredRange, requestedRange, "measured audio range");
|
|
48
|
+
const analysis = await this.analyzeAudioForProject(project, media, requestedRange);
|
|
49
|
+
const measuredRange = analysis.measuredRange ?? requestedRange;
|
|
61
50
|
const analyzedDurationSeconds = analysis.analyzedDurationSeconds ?? measuredRange.end - measuredRange.start;
|
|
62
51
|
if (Math.abs(analyzedDurationSeconds - (measuredRange.end - measuredRange.start)) > 0.000001) {
|
|
63
52
|
throw new Error("ANALYSIS_INVALID: audio measured duration does not match its measured range");
|
|
@@ -133,10 +122,15 @@ export class MediaAnalysisService {
|
|
|
133
122
|
async understandMedia(mediaId) {
|
|
134
123
|
const project = await this.project.inspectProject();
|
|
135
124
|
const media = findMedia(project, mediaId);
|
|
125
|
+
const cached = await this.loadCachedUnderstanding(project, media);
|
|
126
|
+
if (cached) {
|
|
127
|
+
this.context.attachMediaUnderstanding(cached);
|
|
128
|
+
return structuredClone(cached);
|
|
129
|
+
}
|
|
136
130
|
const input = { project, media };
|
|
137
131
|
const [speechResult, audioResult, noiseResult, visualResult, metadataResult] = await Promise.all([
|
|
138
132
|
settle(() => this.options.speechAnalyzer ? this.analyzeSpeechForProject(project, media) : undefined),
|
|
139
|
-
settle(() => this.options.audioAnalyzer
|
|
133
|
+
settle(() => this.options.audioAnalyzer ? this.analyzeAudioForProject(project, media) : undefined),
|
|
140
134
|
settle(() => this.options.noiseAnalyzer?.analyze(input)),
|
|
141
135
|
settle(() => this.options.visualAnalyzer?.analyze(input)),
|
|
142
136
|
settle(() => this.options.metadataAnalyzer?.analyze(input)),
|
|
@@ -167,6 +161,7 @@ export class MediaAnalysisService {
|
|
|
167
161
|
analysisRevision: project.revision,
|
|
168
162
|
};
|
|
169
163
|
this.context.attachMediaUnderstanding(understanding);
|
|
164
|
+
await this.options.semanticMediaIndexStore?.save(understanding);
|
|
170
165
|
return structuredClone(understanding);
|
|
171
166
|
}
|
|
172
167
|
async inspectMedia(mediaId) {
|
|
@@ -178,12 +173,110 @@ export class MediaAnalysisService {
|
|
|
178
173
|
}
|
|
179
174
|
async indexMedia(query = {}) {
|
|
180
175
|
const project = await this.project.inspectProject();
|
|
181
|
-
|
|
176
|
+
const current = this.indexFromProject(project, query);
|
|
177
|
+
const persisted = await this.persistedIndexEntries(query);
|
|
178
|
+
return mergeIndexEntries(persisted, current);
|
|
182
179
|
}
|
|
183
180
|
async planRoughCut(request) {
|
|
184
181
|
const project = await this.project.inspectProject();
|
|
185
182
|
const { maxShots: _maxShots, ...query } = request;
|
|
186
|
-
|
|
183
|
+
const current = this.indexFromProject(project, query);
|
|
184
|
+
const persisted = await this.persistedIndexEntries(query);
|
|
185
|
+
return planRoughCut(mergeIndexEntries(persisted, current), project.revision, request);
|
|
186
|
+
}
|
|
187
|
+
async inspectTimelineSemanticContext(query = {}) {
|
|
188
|
+
const project = await this.project.inspectProject();
|
|
189
|
+
const requestedOccurrenceIds = query.occurrenceIds ? new Set(query.occurrenceIds) : undefined;
|
|
190
|
+
const occurrences = [];
|
|
191
|
+
for (const clip of project.timeline.clips) {
|
|
192
|
+
if (requestedOccurrenceIds && !requestedOccurrenceIds.has(clip.id))
|
|
193
|
+
continue;
|
|
194
|
+
const context = await this.inspectTimelineOccurrence(project, clip, query);
|
|
195
|
+
if (context.status === "available" || requestedOccurrenceIds || !query.query) {
|
|
196
|
+
if (!query.query || context.status === "unavailable" || context.observations.length > 0) {
|
|
197
|
+
occurrences.push(context);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
return {
|
|
202
|
+
projectId: project.projectId,
|
|
203
|
+
timelineId: project.timeline.id,
|
|
204
|
+
revision: structuredClone(project.revision),
|
|
205
|
+
query: structuredClone(query),
|
|
206
|
+
occurrences,
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
async inspectTimelineOccurrence(project, clip, query) {
|
|
210
|
+
if (!clip.mediaId)
|
|
211
|
+
return unavailableTimelineOccurrence(clip.id, "timeline occurrence has no media binding");
|
|
212
|
+
const media = project.media.find((candidate) => candidate.mediaId === clip.mediaId);
|
|
213
|
+
if (!media)
|
|
214
|
+
return unavailableTimelineOccurrence(clip.id, `media binding is unavailable: ${clip.mediaId}`, clip.mediaId);
|
|
215
|
+
const sourceStart = clip.sourceStart ?? (clip.sourceStartTime ? rationalSeconds(clip.sourceStartTime) : 0);
|
|
216
|
+
const sourceDuration = clip.durationTime ? rationalSeconds(clip.durationTime) : clip.duration;
|
|
217
|
+
const sourceRange = {
|
|
218
|
+
start: sourceStart,
|
|
219
|
+
end: sourceStart + sourceDuration,
|
|
220
|
+
...(clip.sourceStartTime ? { startTime: structuredClone(clip.sourceStartTime) } : {}),
|
|
221
|
+
durationTime: structuredClone(clip.durationTime),
|
|
222
|
+
};
|
|
223
|
+
if (!Number.isFinite(sourceRange.start) || !Number.isFinite(sourceRange.end) || sourceRange.start < 0 || sourceRange.end <= sourceRange.start) {
|
|
224
|
+
return unavailableTimelineOccurrence(clip.id, "timeline occurrence source range is invalid", clip.mediaId, sourceRange);
|
|
225
|
+
}
|
|
226
|
+
if (media.duration !== undefined && sourceRange.end > media.duration + 0.000001) {
|
|
227
|
+
return unavailableTimelineOccurrence(clip.id, "timeline occurrence source range exceeds media duration", clip.mediaId, sourceRange);
|
|
228
|
+
}
|
|
229
|
+
const sourceIdentity = sourceIdentityOf(media);
|
|
230
|
+
const understanding = await this.options.semanticMediaIndexStore?.load(sourceIdentity)
|
|
231
|
+
?? understandingFromMediaContext(media, project.revision);
|
|
232
|
+
if (!understanding) {
|
|
233
|
+
return unavailableTimelineOccurrence(clip.id, "source-bound semantic index entry is unavailable", clip.mediaId, sourceRange, sourceIdentity);
|
|
234
|
+
}
|
|
235
|
+
const observations = timelineSemanticObservations(understanding, sourceRange);
|
|
236
|
+
const filtered = query.query
|
|
237
|
+
? observations.filter((observation) => timelineObservationMatches(observation, query.query))
|
|
238
|
+
: observations;
|
|
239
|
+
return {
|
|
240
|
+
occurrenceId: clip.id,
|
|
241
|
+
status: "available",
|
|
242
|
+
mediaId: clip.mediaId,
|
|
243
|
+
sourceIdentity: structuredClone(understanding.sourceIdentity),
|
|
244
|
+
sourceRange,
|
|
245
|
+
observations: filtered,
|
|
246
|
+
analysis: structuredClone(understanding.analysis),
|
|
247
|
+
analysisRevision: understanding.analysisRevision.id,
|
|
248
|
+
};
|
|
249
|
+
}
|
|
250
|
+
async loadCachedUnderstanding(project, media) {
|
|
251
|
+
const store = this.options.semanticMediaIndexStore;
|
|
252
|
+
if (!store)
|
|
253
|
+
return undefined;
|
|
254
|
+
const cached = await store.load(sourceIdentityOf(media));
|
|
255
|
+
if (!cached || !cacheMatchesAnalyzers(cached, this.options))
|
|
256
|
+
return undefined;
|
|
257
|
+
const rebound = structuredClone(cached);
|
|
258
|
+
const sourceIdentity = sourceIdentityOf(media);
|
|
259
|
+
rebound.sourceIdentity = sourceIdentity;
|
|
260
|
+
rebound.mediaId = media.mediaId;
|
|
261
|
+
rebound.source = media.source;
|
|
262
|
+
rebound.analysisRevision = structuredClone(project.revision);
|
|
263
|
+
if (rebound.speech) {
|
|
264
|
+
rebound.speech.mediaId = media.mediaId;
|
|
265
|
+
rebound.speech.sourceIdentity = sourceIdentity;
|
|
266
|
+
rebound.speech.revision = structuredClone(project.revision);
|
|
267
|
+
}
|
|
268
|
+
if (rebound.audio) {
|
|
269
|
+
rebound.audio.mediaId = media.mediaId;
|
|
270
|
+
rebound.audio.sourceIdentity = sourceIdentity;
|
|
271
|
+
rebound.audio.revision = structuredClone(project.revision);
|
|
272
|
+
}
|
|
273
|
+
return rebound;
|
|
274
|
+
}
|
|
275
|
+
async persistedIndexEntries(query) {
|
|
276
|
+
const entries = await this.options.semanticMediaIndexStore?.list() ?? [];
|
|
277
|
+
return entries
|
|
278
|
+
.map(understandingToIndexEntry)
|
|
279
|
+
.filter((entry) => matchesMediaIndexQuery(entry, query));
|
|
187
280
|
}
|
|
188
281
|
indexFromProject(project, query) {
|
|
189
282
|
return project.media
|
|
@@ -197,6 +290,12 @@ export class MediaAnalysisService {
|
|
|
197
290
|
analysisStatus("visual", this.options.visualAnalyzer, sourceIdentityOf(media), false),
|
|
198
291
|
analysisStatus("metadata", this.options.metadataAnalyzer, sourceIdentityOf(media), false),
|
|
199
292
|
],
|
|
293
|
+
...(media.analysis?.some((record) => record.capability === "speech" && record.status === "analyzed") && media.speech
|
|
294
|
+
? { speech: structuredClone(media.speech) }
|
|
295
|
+
: {}),
|
|
296
|
+
...(media.analysis?.some((record) => record.capability === "audio" && record.status === "analyzed") && media.audio
|
|
297
|
+
? { audio: structuredClone(media.audio) }
|
|
298
|
+
: {}),
|
|
200
299
|
...(media.analysisRevision ? { analysisRevision: media.analysisRevision } : {}),
|
|
201
300
|
}))
|
|
202
301
|
.filter((entry) => matchesMediaIndexQuery(entry, query));
|
|
@@ -239,7 +338,7 @@ export class MediaAnalysisService {
|
|
|
239
338
|
}
|
|
240
339
|
}
|
|
241
340
|
if (requirements.audio && this.options.audioAnalyzer) {
|
|
242
|
-
const analyses = await Promise.all(ranges.map((range) => this.
|
|
341
|
+
const analyses = await Promise.all(ranges.map((range) => this.analyzeAudioForProject(next, media, range)));
|
|
243
342
|
if (analyses[analyses.length - 1])
|
|
244
343
|
media.audio = analyses[analyses.length - 1];
|
|
245
344
|
}
|
|
@@ -281,6 +380,44 @@ export class MediaAnalysisService {
|
|
|
281
380
|
provider: analyzer.descriptor,
|
|
282
381
|
});
|
|
283
382
|
}
|
|
383
|
+
async analyzeAudioForProject(project, media, range) {
|
|
384
|
+
const analyzer = this.options.audioAnalyzer;
|
|
385
|
+
if (!analyzer)
|
|
386
|
+
throw new Error("CAPABILITY_UNAVAILABLE: audio analysis");
|
|
387
|
+
const requestedRange = range ?? (media.duration === undefined ? undefined : { start: 0, end: media.duration });
|
|
388
|
+
validateRequestedAudioRange(requestedRange, media.duration, "requested audio range");
|
|
389
|
+
const analysis = await analyzer.analyze({ project, media }, requestedRange);
|
|
390
|
+
validateAudioProvenance(analysis, {
|
|
391
|
+
media,
|
|
392
|
+
mediaId: media.mediaId,
|
|
393
|
+
project,
|
|
394
|
+
provider: analyzer.descriptor,
|
|
395
|
+
requestedRange,
|
|
396
|
+
});
|
|
397
|
+
const measuredRange = analysis.measuredRange ?? (requestedRange ? structuredClone(requestedRange) : undefined);
|
|
398
|
+
if (measuredRange && requestedRange)
|
|
399
|
+
validateAudioRange(measuredRange, requestedRange, "measured audio range");
|
|
400
|
+
if (measuredRange)
|
|
401
|
+
validateRequestedAudioRange(measuredRange, media.duration, "measured audio source range");
|
|
402
|
+
const analyzedDurationSeconds = measuredRange
|
|
403
|
+
? analysis.analyzedDurationSeconds ?? measuredRange.end - measuredRange.start
|
|
404
|
+
: analysis.analyzedDurationSeconds;
|
|
405
|
+
if (measuredRange && analyzedDurationSeconds !== undefined
|
|
406
|
+
&& Math.abs(analyzedDurationSeconds - (measuredRange.end - measuredRange.start)) > 0.000001) {
|
|
407
|
+
throw new Error("ANALYSIS_INVALID: audio measured duration does not match its measured range");
|
|
408
|
+
}
|
|
409
|
+
return {
|
|
410
|
+
...structuredClone(analysis),
|
|
411
|
+
schemaVersion: 1,
|
|
412
|
+
mediaId: media.mediaId,
|
|
413
|
+
sourceIdentity: sourceIdentityOf(media),
|
|
414
|
+
...(requestedRange ? { requestedRange: structuredClone(requestedRange) } : {}),
|
|
415
|
+
...(measuredRange ? { measuredRange: structuredClone(measuredRange) } : {}),
|
|
416
|
+
revision: structuredClone(project.revision),
|
|
417
|
+
provider: analysis.provider ?? analyzer.descriptor ?? { id: "framekit.audio", provider: "unknown" },
|
|
418
|
+
...(analyzedDurationSeconds !== undefined ? { analyzedDurationSeconds } : {}),
|
|
419
|
+
};
|
|
420
|
+
}
|
|
284
421
|
}
|
|
285
422
|
function sourceIdentityOf(media) {
|
|
286
423
|
return {
|
|
@@ -355,10 +492,14 @@ function mergeRanges(ranges) {
|
|
|
355
492
|
for (const range of [...ranges].sort((left, right) => left.start - right.start || left.end - right.end)) {
|
|
356
493
|
const previous = merged[merged.length - 1];
|
|
357
494
|
if (!previous || range.start > previous.end) {
|
|
358
|
-
merged.push(
|
|
495
|
+
merged.push(structuredClone(range));
|
|
359
496
|
continue;
|
|
360
497
|
}
|
|
361
|
-
|
|
498
|
+
if (range.end > previous.end) {
|
|
499
|
+
previous.end = range.end;
|
|
500
|
+
delete previous.startTime;
|
|
501
|
+
delete previous.durationTime;
|
|
502
|
+
}
|
|
362
503
|
}
|
|
363
504
|
return merged;
|
|
364
505
|
}
|
|
@@ -371,6 +512,11 @@ function sameRational(left, right) {
|
|
|
371
512
|
return leftParts.value * rightParts.timescale === rightParts.value * leftParts.timescale;
|
|
372
513
|
}
|
|
373
514
|
function semanticFromAnalyses(speech, audio, visual, metadata) {
|
|
515
|
+
const usableRanges = mergeRanges([
|
|
516
|
+
...(metadata?.usableRanges ?? []),
|
|
517
|
+
...(visual?.scenes ?? []).map((scene) => ({ start: scene.start, end: scene.end })),
|
|
518
|
+
...(speech?.vadSegments ?? []).filter((segment) => segment.kind === "speech").map((segment) => ({ start: segment.start, end: segment.end })),
|
|
519
|
+
]);
|
|
374
520
|
return {
|
|
375
521
|
subjects: [
|
|
376
522
|
...(visual?.subjects ?? []).map((subject) => ({ value: subject.label, confidence: subject.confidence })),
|
|
@@ -384,7 +530,7 @@ function semanticFromAnalyses(speech, audio, visual, metadata) {
|
|
|
384
530
|
timeOfDay: metadata?.timeOfDay ? structuredClone(metadata.timeOfDay) : [],
|
|
385
531
|
moods: metadata?.moods ? structuredClone(metadata.moods) : [],
|
|
386
532
|
...(visual?.motion ? { motion: structuredClone(visual.motion) } : {}),
|
|
387
|
-
usableRanges
|
|
533
|
+
usableRanges,
|
|
388
534
|
...(speech ? { transcript: speech.words.map((word) => word.text).join(" ") } : {}),
|
|
389
535
|
...(audio ? {
|
|
390
536
|
audio: {
|
|
@@ -396,6 +542,159 @@ function semanticFromAnalyses(speech, audio, visual, metadata) {
|
|
|
396
542
|
} : {}),
|
|
397
543
|
};
|
|
398
544
|
}
|
|
545
|
+
function understandingFromMediaContext(media, revision) {
|
|
546
|
+
if (!media.semantic && !media.analysis && !media.speech && !media.visual && !media.metadata)
|
|
547
|
+
return undefined;
|
|
548
|
+
const sourceIdentity = sourceIdentityOf(media);
|
|
549
|
+
return {
|
|
550
|
+
mediaId: media.mediaId,
|
|
551
|
+
source: media.source,
|
|
552
|
+
sourceIdentity,
|
|
553
|
+
...(media.metadata ? { metadata: structuredClone(media.metadata) } : {}),
|
|
554
|
+
...(media.speech ? { speech: structuredClone(media.speech) } : {}),
|
|
555
|
+
...(media.audio ? { audio: structuredClone(media.audio) } : {}),
|
|
556
|
+
...(media.noise ? { noise: structuredClone(media.noise) } : {}),
|
|
557
|
+
...(media.visual ? { visual: structuredClone(media.visual) } : {}),
|
|
558
|
+
semantic: structuredClone(media.semantic ?? emptySemanticDescription()),
|
|
559
|
+
analysis: structuredClone(media.analysis ?? []),
|
|
560
|
+
analysisRevision: {
|
|
561
|
+
id: media.analysisRevision ?? revision.id,
|
|
562
|
+
sequence: revision.sequence,
|
|
563
|
+
timestamp: revision.timestamp,
|
|
564
|
+
},
|
|
565
|
+
};
|
|
566
|
+
}
|
|
567
|
+
function timelineSemanticObservations(understanding, sourceRange) {
|
|
568
|
+
const observations = [];
|
|
569
|
+
const analyzerFor = (capability) => understanding.analysis.find((status) => status.capability === capability)?.provenance?.analyzer;
|
|
570
|
+
for (const word of understanding.speech?.words ?? []) {
|
|
571
|
+
const range = intersectTimeRanges({ start: word.start, end: word.end }, sourceRange);
|
|
572
|
+
if (!range)
|
|
573
|
+
continue;
|
|
574
|
+
observations.push({
|
|
575
|
+
capability: "speech",
|
|
576
|
+
range,
|
|
577
|
+
text: word.text,
|
|
578
|
+
confidence: word.confidence,
|
|
579
|
+
...(analyzerFor("speech") ? { analyzer: structuredClone(analyzerFor("speech")) } : {}),
|
|
580
|
+
});
|
|
581
|
+
}
|
|
582
|
+
for (const scene of understanding.visual?.scenes ?? []) {
|
|
583
|
+
const range = intersectTimeRanges({ start: scene.start, end: scene.end }, sourceRange);
|
|
584
|
+
if (!range)
|
|
585
|
+
continue;
|
|
586
|
+
observations.push({
|
|
587
|
+
capability: "visual",
|
|
588
|
+
range,
|
|
589
|
+
...(scene.label ? { label: scene.label } : {}),
|
|
590
|
+
...(scene.confidence !== undefined ? { confidence: scene.confidence } : {}),
|
|
591
|
+
...(analyzerFor("visual") ? { analyzer: structuredClone(analyzerFor("visual")) } : {}),
|
|
592
|
+
});
|
|
593
|
+
}
|
|
594
|
+
for (const subject of understanding.visual?.subjects ?? []) {
|
|
595
|
+
const range = intersectTimeRanges({
|
|
596
|
+
start: subject.start ?? sourceRange.start,
|
|
597
|
+
end: subject.end ?? sourceRange.end,
|
|
598
|
+
}, sourceRange);
|
|
599
|
+
if (!range)
|
|
600
|
+
continue;
|
|
601
|
+
observations.push({
|
|
602
|
+
capability: "visual",
|
|
603
|
+
range,
|
|
604
|
+
label: subject.label,
|
|
605
|
+
confidence: subject.confidence,
|
|
606
|
+
...(analyzerFor("visual") ? { analyzer: structuredClone(analyzerFor("visual")) } : {}),
|
|
607
|
+
});
|
|
608
|
+
}
|
|
609
|
+
for (const range of understanding.metadata?.usableRanges ?? []) {
|
|
610
|
+
const clipped = intersectTimeRanges(range, sourceRange);
|
|
611
|
+
if (!clipped)
|
|
612
|
+
continue;
|
|
613
|
+
observations.push({
|
|
614
|
+
capability: "metadata",
|
|
615
|
+
range: clipped,
|
|
616
|
+
...(analyzerFor("metadata") ? { analyzer: structuredClone(analyzerFor("metadata")) } : {}),
|
|
617
|
+
});
|
|
618
|
+
}
|
|
619
|
+
return observations.sort((left, right) => left.range.start - right.range.start
|
|
620
|
+
|| left.range.end - right.range.end
|
|
621
|
+
|| left.capability.localeCompare(right.capability)
|
|
622
|
+
|| (left.text ?? left.label ?? "").localeCompare(right.text ?? right.label ?? ""));
|
|
623
|
+
}
|
|
624
|
+
function intersectTimeRanges(left, right) {
|
|
625
|
+
const start = Math.max(left.start, right.start);
|
|
626
|
+
const end = Math.min(left.end, right.end);
|
|
627
|
+
return start < end ? { start, end } : undefined;
|
|
628
|
+
}
|
|
629
|
+
function timelineObservationMatches(observation, query) {
|
|
630
|
+
const normalized = query.trim().toLocaleLowerCase();
|
|
631
|
+
if (!normalized)
|
|
632
|
+
return true;
|
|
633
|
+
return [observation.text, observation.label]
|
|
634
|
+
.filter((value) => Boolean(value))
|
|
635
|
+
.some((value) => value.toLocaleLowerCase().includes(normalized));
|
|
636
|
+
}
|
|
637
|
+
function unavailableTimelineOccurrence(occurrenceId, reason, mediaId, sourceRange, sourceIdentity) {
|
|
638
|
+
return {
|
|
639
|
+
occurrenceId,
|
|
640
|
+
status: "unavailable",
|
|
641
|
+
...(mediaId ? { mediaId } : {}),
|
|
642
|
+
...(sourceIdentity ? { sourceIdentity: structuredClone(sourceIdentity) } : {}),
|
|
643
|
+
...(sourceRange ? { sourceRange: structuredClone(sourceRange) } : {}),
|
|
644
|
+
reason,
|
|
645
|
+
observations: [],
|
|
646
|
+
analysis: [],
|
|
647
|
+
};
|
|
648
|
+
}
|
|
649
|
+
function rationalSeconds(time) {
|
|
650
|
+
const parsed = parseRational(time, "TIMELINE_SEMANTIC_CONTEXT_INVALID");
|
|
651
|
+
const seconds = Number(parsed.value) / Number(parsed.timescale);
|
|
652
|
+
if (!Number.isFinite(seconds))
|
|
653
|
+
throw new Error("TIMELINE_SEMANTIC_CONTEXT_INVALID: rational time is outside supported range");
|
|
654
|
+
return seconds;
|
|
655
|
+
}
|
|
656
|
+
function understandingToIndexEntry(understanding) {
|
|
657
|
+
return {
|
|
658
|
+
sourceIdentity: structuredClone(understanding.sourceIdentity),
|
|
659
|
+
semantic: structuredClone(understanding.semantic),
|
|
660
|
+
analysis: structuredClone(understanding.analysis),
|
|
661
|
+
...(understanding.speech ? { speech: structuredClone(understanding.speech) } : {}),
|
|
662
|
+
...(understanding.audio ? { audio: structuredClone(understanding.audio) } : {}),
|
|
663
|
+
analysisRevision: understanding.analysisRevision.id,
|
|
664
|
+
};
|
|
665
|
+
}
|
|
666
|
+
function mergeIndexEntries(persisted, current) {
|
|
667
|
+
const merged = new Map();
|
|
668
|
+
for (const entry of [...persisted, ...current]) {
|
|
669
|
+
const key = JSON.stringify(entry.sourceIdentity);
|
|
670
|
+
const previous = merged.get(key);
|
|
671
|
+
if (!previous || indexEntryScore(entry) >= indexEntryScore(previous))
|
|
672
|
+
merged.set(key, structuredClone(entry));
|
|
673
|
+
}
|
|
674
|
+
return [...merged.values()].sort((left, right) => left.sourceIdentity.mediaId.localeCompare(right.sourceIdentity.mediaId));
|
|
675
|
+
}
|
|
676
|
+
function indexEntryScore(entry) {
|
|
677
|
+
return entry.analysis.filter((record) => record.status === "analyzed").length
|
|
678
|
+
+ entry.semantic.usableRanges.length
|
|
679
|
+
+ (entry.semantic.transcript?.trim() ? 1 : 0);
|
|
680
|
+
}
|
|
681
|
+
function cacheMatchesAnalyzers(understanding, options) {
|
|
682
|
+
const configured = [
|
|
683
|
+
["speech", options.speechAnalyzer],
|
|
684
|
+
["audio", options.audioAnalyzer],
|
|
685
|
+
["noise", options.noiseAnalyzer],
|
|
686
|
+
["visual", options.visualAnalyzer],
|
|
687
|
+
["metadata", options.metadataAnalyzer],
|
|
688
|
+
];
|
|
689
|
+
return configured.every(([capability, analyzer]) => {
|
|
690
|
+
if (!analyzer)
|
|
691
|
+
return true;
|
|
692
|
+
const record = understanding.analysis.find((candidate) => candidate.capability === capability);
|
|
693
|
+
return record?.status === "analyzed"
|
|
694
|
+
&& record.provenance?.analyzer !== undefined
|
|
695
|
+
&& sameDescriptor(record.provenance.analyzer, analyzer.descriptor ?? { id: `framekit.${capability}`, provider: "unknown" });
|
|
696
|
+
});
|
|
697
|
+
}
|
|
399
698
|
function emptySemanticDescription() {
|
|
400
699
|
return {
|
|
401
700
|
subjects: [],
|
|
@@ -432,13 +731,21 @@ function matchesMediaIndexQuery(entry, query) {
|
|
|
432
731
|
return false;
|
|
433
732
|
if (query.motion && entry.semantic.motion?.label !== query.motion)
|
|
434
733
|
return false;
|
|
435
|
-
if (query.range && !entry.
|
|
734
|
+
if (query.range && !indexEntryRanges(entry).some((range) => range.end > query.range.start && range.start < query.range.end))
|
|
436
735
|
return false;
|
|
437
736
|
if (query.capabilities?.some((capability) => !entry.analysis.some((record) => record.capability === capability && record.status !== "unavailable"))) {
|
|
438
737
|
return false;
|
|
439
738
|
}
|
|
440
739
|
return true;
|
|
441
740
|
}
|
|
741
|
+
function indexEntryRanges(entry) {
|
|
742
|
+
const speechRanges = entry.speech?.vadSegments
|
|
743
|
+
?.filter((segment) => segment.kind === "speech")
|
|
744
|
+
.map(({ start, end }) => ({ start, end }))
|
|
745
|
+
?? entry.speech?.words.map(({ start, end }) => ({ start, end }))
|
|
746
|
+
?? [];
|
|
747
|
+
return [...entry.semantic.usableRanges, ...speechRanges];
|
|
748
|
+
}
|
|
442
749
|
function matchesTag(tags, query) {
|
|
443
750
|
const normalized = query.trim().toLowerCase();
|
|
444
751
|
return tags.some((tag) => tag.value.toLowerCase() === normalized);
|
|
@@ -494,11 +801,12 @@ function validateAudioProvenance(analysis, expected) {
|
|
|
494
801
|
&& !sameMediaSourceIdentity(analysis.sourceIdentity, sourceIdentityOf(expected.media))) {
|
|
495
802
|
throw new Error("TARGET_MISMATCH: audio analysis source identity does not match the requested media");
|
|
496
803
|
}
|
|
497
|
-
if (analysis.requestedRange !== undefined &&
|
|
804
|
+
if (analysis.requestedRange !== undefined && expected.requestedRange !== undefined
|
|
805
|
+
&& !sameRange(analysis.requestedRange, expected.requestedRange)) {
|
|
498
806
|
throw new Error("ANALYSIS_INVALID: audio requested range does not match the runtime request");
|
|
499
807
|
}
|
|
500
808
|
if (analysis.revision !== undefined
|
|
501
|
-
&& (analysis.revision
|
|
809
|
+
&& !sameContextRevision(analysis.revision, expected.project.revision)) {
|
|
502
810
|
throw new Error("ANALYSIS_STALE: audio analysis revision does not match the inspected project");
|
|
503
811
|
}
|
|
504
812
|
if (analysis.provider !== undefined && expected.provider !== undefined
|
|
@@ -507,11 +815,23 @@ function validateAudioProvenance(analysis, expected) {
|
|
|
507
815
|
}
|
|
508
816
|
}
|
|
509
817
|
function validateAudioRange(range, requested, label) {
|
|
510
|
-
|
|
511
|
-
|
|
818
|
+
validateRequestedAudioRange(range, undefined, label);
|
|
819
|
+
if (range.start < requested.start || range.end > requested.end) {
|
|
512
820
|
throw new Error(`ANALYSIS_INVALID: ${label} must fit inside the requested occurrence range`);
|
|
513
821
|
}
|
|
514
822
|
}
|
|
823
|
+
function validateRequestedAudioRange(range, duration, label) {
|
|
824
|
+
if (!range)
|
|
825
|
+
return;
|
|
826
|
+
if (!Number.isFinite(range.start) || !Number.isFinite(range.end)
|
|
827
|
+
|| range.start < 0 || range.end <= range.start
|
|
828
|
+
|| (duration !== undefined && range.end > duration)) {
|
|
829
|
+
throw new Error(`ANALYSIS_INVALID: ${label} must be finite, positive, and inside the source duration`);
|
|
830
|
+
}
|
|
831
|
+
}
|
|
832
|
+
function sameContextRevision(left, right) {
|
|
833
|
+
return left.id === right.id && left.sequence === right.sequence && left.timestamp === right.timestamp;
|
|
834
|
+
}
|
|
515
835
|
function sameRange(left, right) {
|
|
516
836
|
return Math.abs(left.start - right.start) <= 0.000001
|
|
517
837
|
&& Math.abs(left.end - right.end) <= 0.000001;
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { mkdir, readFile, rename, unlink, writeFile } from "node:fs/promises";
|
|
2
|
+
import { dirname } from "node:path";
|
|
3
|
+
import { randomUUID } from "node:crypto";
|
|
4
|
+
import { sameMediaSourceIdentity } from "../domain/media.js";
|
|
5
|
+
export const SEMANTIC_MEDIA_INDEX_SCHEMA_VERSION = 1;
|
|
6
|
+
/** Deterministic store for runtime and integration tests. */
|
|
7
|
+
export class InMemorySemanticMediaIndexStore {
|
|
8
|
+
entries = [];
|
|
9
|
+
async load(source) {
|
|
10
|
+
const entry = this.entries.find((candidate) => sameMediaSourceIdentity(candidate.sourceIdentity, source));
|
|
11
|
+
return entry ? structuredClone(entry) : undefined;
|
|
12
|
+
}
|
|
13
|
+
async list() {
|
|
14
|
+
return structuredClone(this.entries);
|
|
15
|
+
}
|
|
16
|
+
async save(understanding) {
|
|
17
|
+
const retained = this.entries.filter((candidate) => candidate.mediaId !== understanding.mediaId);
|
|
18
|
+
retained.push(structuredClone(understanding));
|
|
19
|
+
retained.sort((left, right) => left.mediaId.localeCompare(right.mediaId));
|
|
20
|
+
this.entries.splice(0, this.entries.length, ...retained);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
/** Atomic JSON persistence for a local semantic media index. */
|
|
24
|
+
export class JsonSemanticMediaIndexStore {
|
|
25
|
+
path;
|
|
26
|
+
constructor(path) {
|
|
27
|
+
this.path = path;
|
|
28
|
+
}
|
|
29
|
+
async load(source) {
|
|
30
|
+
const document = await this.readDocument();
|
|
31
|
+
const entry = document.entries.find((candidate) => sameMediaSourceIdentity(candidate.sourceIdentity, source));
|
|
32
|
+
return entry ? structuredClone(entry) : undefined;
|
|
33
|
+
}
|
|
34
|
+
async list() {
|
|
35
|
+
return structuredClone((await this.readDocument()).entries);
|
|
36
|
+
}
|
|
37
|
+
async save(understanding) {
|
|
38
|
+
const document = await this.readDocument();
|
|
39
|
+
document.entries = [
|
|
40
|
+
...document.entries.filter((candidate) => candidate.mediaId !== understanding.mediaId),
|
|
41
|
+
structuredClone(understanding),
|
|
42
|
+
].sort((left, right) => left.mediaId.localeCompare(right.mediaId));
|
|
43
|
+
await mkdir(dirname(this.path), { recursive: true });
|
|
44
|
+
const temporary = `${this.path}.${randomUUID()}.tmp`;
|
|
45
|
+
try {
|
|
46
|
+
await writeFile(temporary, `${JSON.stringify(document)}\n`, { encoding: "utf8", flag: "wx" });
|
|
47
|
+
await rename(temporary, this.path);
|
|
48
|
+
}
|
|
49
|
+
finally {
|
|
50
|
+
try {
|
|
51
|
+
await unlink(temporary);
|
|
52
|
+
}
|
|
53
|
+
catch (error) {
|
|
54
|
+
if (!isMissingFile(error))
|
|
55
|
+
throw error;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
async readDocument() {
|
|
60
|
+
let raw;
|
|
61
|
+
try {
|
|
62
|
+
raw = await readFile(this.path, "utf8");
|
|
63
|
+
}
|
|
64
|
+
catch (error) {
|
|
65
|
+
if (isMissingFile(error))
|
|
66
|
+
return { schemaVersion: SEMANTIC_MEDIA_INDEX_SCHEMA_VERSION, entries: [] };
|
|
67
|
+
throw error;
|
|
68
|
+
}
|
|
69
|
+
let value;
|
|
70
|
+
try {
|
|
71
|
+
value = JSON.parse(raw);
|
|
72
|
+
}
|
|
73
|
+
catch (error) {
|
|
74
|
+
throw new Error(`SEMANTIC_INDEX_CORRUPT: index JSON is malformed (${String(error)})`);
|
|
75
|
+
}
|
|
76
|
+
if (!value || typeof value !== "object"
|
|
77
|
+
|| value.schemaVersion !== SEMANTIC_MEDIA_INDEX_SCHEMA_VERSION
|
|
78
|
+
|| !Array.isArray(value.entries)) {
|
|
79
|
+
throw new Error("SEMANTIC_INDEX_CORRUPT: unsupported semantic index document");
|
|
80
|
+
}
|
|
81
|
+
return structuredClone(value);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
function isMissingFile(error) {
|
|
85
|
+
return Boolean(error && typeof error === "object" && "code" in error && error.code === "ENOENT");
|
|
86
|
+
}
|
|
@@ -14,6 +14,7 @@ export * from "./timeline/fast-observation.js";
|
|
|
14
14
|
export * from "./application/project-store.js";
|
|
15
15
|
export * from "./application/local-media.js";
|
|
16
16
|
export * from "./application/project-transactions.js";
|
|
17
|
+
export * from "./application/semantic-media-index-store.js";
|
|
17
18
|
export * from "./context/project-catalog.js";
|
|
18
19
|
export * from "./speech/filler-removal.js";
|
|
19
20
|
export * from "./audio/dialogue-normalization.js";
|