@morshoto/framekit 0.1.16 → 0.1.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. package/README.md +10 -4
  2. package/dist-package/adapters/final-cut/typescript/src/canonical.js +19 -7
  3. package/dist-package/adapters/final-cut/typescript/src/connection.js +13 -8
  4. package/dist-package/adapters/final-cut/typescript/src/fcpxml.js +147 -34
  5. package/dist-package/adapters/final-cut/typescript/src/index.js +1 -0
  6. package/dist-package/adapters/final-cut/typescript/src/library-inspection.js +60 -1
  7. package/dist-package/adapters/final-cut/typescript/src/live.js +2 -2
  8. package/dist-package/adapters/final-cut/typescript/src/publish-readback.js +213 -0
  9. package/dist-package/adapters/final-cut/typescript/src/publisher.js +762 -238
  10. package/dist-package/adapters/final-cut/typescript/src/session.js +6 -1
  11. package/dist-package/adapters/final-cut/typescript/src/timeline-ir-fcpxml.js +293 -59
  12. package/dist-package/adapters/headless/typescript/src/ffmpeg-renderer.js +137 -19
  13. package/dist-package/adapters/headless/typescript/src/ffmpeg-verifier.js +2 -2
  14. package/dist-package/apps/cli/src/main.js +34 -19
  15. package/dist-package/apps/mcp-server/src/build-alignment.js +59 -0
  16. package/dist-package/apps/mcp-server/src/build-identity.js +30 -0
  17. package/dist-package/apps/mcp-server/src/build-identity.json +5 -0
  18. package/dist-package/apps/mcp-server/src/headless-project-snapshot.js +101 -0
  19. package/dist-package/apps/mcp-server/src/headless-projects.js +473 -3
  20. package/dist-package/apps/mcp-server/src/main.js +33 -6
  21. package/dist-package/apps/mcp-server/src/server.js +367 -52
  22. package/dist-package/apps/mcp-server/src/version.js +21 -2
  23. package/dist-package/packages/runtime/src/application/media-analysis-service.js +360 -30
  24. package/dist-package/packages/runtime/src/application/semantic-media-index-store.js +86 -0
  25. package/dist-package/packages/runtime/src/index.js +1 -0
  26. package/dist-package/packages/runtime/src/planning/rough-cut.js +103 -7
  27. package/dist-package/packages/runtime/src/render/render-contract.js +3 -0
  28. package/dist-package/packages/runtime/src/runtime.js +11 -2
  29. package/dist-package/packages/runtime/src/timeline/editing-session.js +7 -2
  30. package/package.json +9 -9
@@ -18,12 +18,12 @@ export class MediaAnalysisService {
18
18
  const media = findMedia(project, mediaId);
19
19
  return this.analyzeSpeechForProject(project, media, range);
20
20
  }
21
- async analyzeAudio(mediaId) {
21
+ async analyzeAudio(mediaId, range) {
22
22
  if (!this.options.audioAnalyzer)
23
23
  throw new Error("CAPABILITY_UNAVAILABLE: audio analysis");
24
24
  const project = await this.project.inspectProject();
25
25
  const media = findMedia(project, mediaId);
26
- return this.options.audioAnalyzer.analyze({ project, media });
26
+ return this.analyzeAudioForProject(project, media, range);
27
27
  }
28
28
  async analyzeNoise(mediaId, range) {
29
29
  if (!this.options.noiseAnalyzer)
@@ -45,19 +45,8 @@ export class MediaAnalysisService {
45
45
  const media = findMedia(project, mediaId);
46
46
  const sourceStart = clip.sourceStart ?? 0;
47
47
  const requestedRange = { start: sourceStart, end: sourceStart + clip.duration };
48
- const analysis = await this.options.audioAnalyzer.analyze({ project, media }, requestedRange);
49
- validateAudioProvenance(analysis, {
50
- media,
51
- mediaId,
52
- project,
53
- provider: this.options.audioAnalyzer.descriptor,
54
- requestedRange,
55
- });
56
- const measuredRange = analysis.measuredRange ?? {
57
- start: requestedRange.start,
58
- end: requestedRange.start + (analysis.analyzedDurationSeconds ?? clip.duration),
59
- };
60
- validateAudioRange(measuredRange, requestedRange, "measured audio range");
48
+ const analysis = await this.analyzeAudioForProject(project, media, requestedRange);
49
+ const measuredRange = analysis.measuredRange ?? requestedRange;
61
50
  const analyzedDurationSeconds = analysis.analyzedDurationSeconds ?? measuredRange.end - measuredRange.start;
62
51
  if (Math.abs(analyzedDurationSeconds - (measuredRange.end - measuredRange.start)) > 0.000001) {
63
52
  throw new Error("ANALYSIS_INVALID: audio measured duration does not match its measured range");
@@ -132,11 +121,20 @@ export class MediaAnalysisService {
132
121
  }
133
122
  async understandMedia(mediaId) {
134
123
  const project = await this.project.inspectProject();
124
+ return this.understandMediaInProject(mediaId, project, true);
125
+ }
126
+ async understandMediaInProject(mediaId, project, attachToLiveContext = false) {
135
127
  const media = findMedia(project, mediaId);
128
+ const cached = await this.loadCachedUnderstanding(project, media);
129
+ if (cached) {
130
+ if (attachToLiveContext)
131
+ this.context.attachMediaUnderstanding(cached);
132
+ return structuredClone(cached);
133
+ }
136
134
  const input = { project, media };
137
135
  const [speechResult, audioResult, noiseResult, visualResult, metadataResult] = await Promise.all([
138
136
  settle(() => this.options.speechAnalyzer ? this.analyzeSpeechForProject(project, media) : undefined),
139
- settle(() => this.options.audioAnalyzer?.analyze(input)),
137
+ settle(() => this.options.audioAnalyzer ? this.analyzeAudioForProject(project, media) : undefined),
140
138
  settle(() => this.options.noiseAnalyzer?.analyze(input)),
141
139
  settle(() => this.options.visualAnalyzer?.analyze(input)),
142
140
  settle(() => this.options.metadataAnalyzer?.analyze(input)),
@@ -166,7 +164,9 @@ export class MediaAnalysisService {
166
164
  ],
167
165
  analysisRevision: project.revision,
168
166
  };
169
- this.context.attachMediaUnderstanding(understanding);
167
+ if (attachToLiveContext)
168
+ this.context.attachMediaUnderstanding(understanding);
169
+ await this.options.semanticMediaIndexStore?.save(understanding);
170
170
  return structuredClone(understanding);
171
171
  }
172
172
  async inspectMedia(mediaId) {
@@ -178,12 +178,115 @@ export class MediaAnalysisService {
178
178
  }
179
179
  async indexMedia(query = {}) {
180
180
  const project = await this.project.inspectProject();
181
- return this.indexFromProject(project, query);
181
+ const current = this.indexFromProject(project, query);
182
+ const persisted = await this.persistedIndexEntries(query);
183
+ return mergeIndexEntries(persisted, current);
182
184
  }
183
185
  async planRoughCut(request) {
184
186
  const project = await this.project.inspectProject();
187
+ return this.planRoughCutInProject(request, project);
188
+ }
189
+ async planRoughCutInProject(request, project) {
185
190
  const { maxShots: _maxShots, ...query } = request;
186
- return planRoughCut(this.indexFromProject(project, query), project.revision, request);
191
+ const current = this.indexFromProject(project, query);
192
+ const persisted = await this.persistedIndexEntries(query);
193
+ const projectMedia = project.media.map(sourceIdentityOf);
194
+ const matchingPersisted = persisted.filter((entry) => projectMedia.some((source) => sameMediaSourceIdentity(source, entry.sourceIdentity)));
195
+ return planRoughCut(mergeIndexEntries(matchingPersisted, current), project.revision, request);
196
+ }
197
+ async inspectTimelineSemanticContext(query = {}) {
198
+ const project = await this.project.inspectProject();
199
+ const requestedOccurrenceIds = query.occurrenceIds ? new Set(query.occurrenceIds) : undefined;
200
+ const occurrences = [];
201
+ for (const clip of project.timeline.clips) {
202
+ if (requestedOccurrenceIds && !requestedOccurrenceIds.has(clip.id))
203
+ continue;
204
+ const context = await this.inspectTimelineOccurrence(project, clip, query);
205
+ if (context.status === "available" || requestedOccurrenceIds || !query.query) {
206
+ if (!query.query || context.status === "unavailable" || context.observations.length > 0) {
207
+ occurrences.push(context);
208
+ }
209
+ }
210
+ }
211
+ return {
212
+ projectId: project.projectId,
213
+ timelineId: project.timeline.id,
214
+ revision: structuredClone(project.revision),
215
+ query: structuredClone(query),
216
+ occurrences,
217
+ };
218
+ }
219
+ async inspectTimelineOccurrence(project, clip, query) {
220
+ if (!clip.mediaId)
221
+ return unavailableTimelineOccurrence(clip.id, "timeline occurrence has no media binding");
222
+ const media = project.media.find((candidate) => candidate.mediaId === clip.mediaId);
223
+ if (!media)
224
+ return unavailableTimelineOccurrence(clip.id, `media binding is unavailable: ${clip.mediaId}`, clip.mediaId);
225
+ const sourceStart = clip.sourceStart ?? (clip.sourceStartTime ? rationalSeconds(clip.sourceStartTime) : 0);
226
+ const sourceDuration = clip.durationTime ? rationalSeconds(clip.durationTime) : clip.duration;
227
+ const sourceRange = {
228
+ start: sourceStart,
229
+ end: sourceStart + sourceDuration,
230
+ ...(clip.sourceStartTime ? { startTime: structuredClone(clip.sourceStartTime) } : {}),
231
+ durationTime: structuredClone(clip.durationTime),
232
+ };
233
+ if (!Number.isFinite(sourceRange.start) || !Number.isFinite(sourceRange.end) || sourceRange.start < 0 || sourceRange.end <= sourceRange.start) {
234
+ return unavailableTimelineOccurrence(clip.id, "timeline occurrence source range is invalid", clip.mediaId, sourceRange);
235
+ }
236
+ if (media.duration !== undefined && sourceRange.end > media.duration + 0.000001) {
237
+ return unavailableTimelineOccurrence(clip.id, "timeline occurrence source range exceeds media duration", clip.mediaId, sourceRange);
238
+ }
239
+ const sourceIdentity = sourceIdentityOf(media);
240
+ const understanding = await this.options.semanticMediaIndexStore?.load(sourceIdentity)
241
+ ?? understandingFromMediaContext(media, project.revision);
242
+ if (!understanding) {
243
+ return unavailableTimelineOccurrence(clip.id, "source-bound semantic index entry is unavailable", clip.mediaId, sourceRange, sourceIdentity);
244
+ }
245
+ const observations = timelineSemanticObservations(understanding, sourceRange);
246
+ const filtered = query.query
247
+ ? observations.filter((observation) => timelineObservationMatches(observation, query.query))
248
+ : observations;
249
+ return {
250
+ occurrenceId: clip.id,
251
+ status: "available",
252
+ mediaId: clip.mediaId,
253
+ sourceIdentity: structuredClone(understanding.sourceIdentity),
254
+ sourceRange,
255
+ observations: filtered,
256
+ analysis: structuredClone(understanding.analysis),
257
+ analysisRevision: understanding.analysisRevision.id,
258
+ };
259
+ }
260
+ async loadCachedUnderstanding(project, media) {
261
+ const store = this.options.semanticMediaIndexStore;
262
+ if (!store)
263
+ return undefined;
264
+ const cached = await store.load(sourceIdentityOf(media));
265
+ if (!cached || !cacheMatchesAnalyzers(cached, this.options))
266
+ return undefined;
267
+ const rebound = structuredClone(cached);
268
+ const sourceIdentity = sourceIdentityOf(media);
269
+ rebound.sourceIdentity = sourceIdentity;
270
+ rebound.mediaId = media.mediaId;
271
+ rebound.source = media.source;
272
+ rebound.analysisRevision = structuredClone(project.revision);
273
+ if (rebound.speech) {
274
+ rebound.speech.mediaId = media.mediaId;
275
+ rebound.speech.sourceIdentity = sourceIdentity;
276
+ rebound.speech.revision = structuredClone(project.revision);
277
+ }
278
+ if (rebound.audio) {
279
+ rebound.audio.mediaId = media.mediaId;
280
+ rebound.audio.sourceIdentity = sourceIdentity;
281
+ rebound.audio.revision = structuredClone(project.revision);
282
+ }
283
+ return rebound;
284
+ }
285
+ async persistedIndexEntries(query) {
286
+ const entries = await this.options.semanticMediaIndexStore?.list() ?? [];
287
+ return entries
288
+ .map(understandingToIndexEntry)
289
+ .filter((entry) => matchesMediaIndexQuery(entry, query));
187
290
  }
188
291
  indexFromProject(project, query) {
189
292
  return project.media
@@ -197,6 +300,12 @@ export class MediaAnalysisService {
197
300
  analysisStatus("visual", this.options.visualAnalyzer, sourceIdentityOf(media), false),
198
301
  analysisStatus("metadata", this.options.metadataAnalyzer, sourceIdentityOf(media), false),
199
302
  ],
303
+ ...(media.analysis?.some((record) => record.capability === "speech" && record.status === "analyzed") && media.speech
304
+ ? { speech: structuredClone(media.speech) }
305
+ : {}),
306
+ ...(media.analysis?.some((record) => record.capability === "audio" && record.status === "analyzed") && media.audio
307
+ ? { audio: structuredClone(media.audio) }
308
+ : {}),
200
309
  ...(media.analysisRevision ? { analysisRevision: media.analysisRevision } : {}),
201
310
  }))
202
311
  .filter((entry) => matchesMediaIndexQuery(entry, query));
@@ -239,7 +348,7 @@ export class MediaAnalysisService {
239
348
  }
240
349
  }
241
350
  if (requirements.audio && this.options.audioAnalyzer) {
242
- const analyses = await Promise.all(ranges.map((range) => this.options.audioAnalyzer.analyze(input, range)));
351
+ const analyses = await Promise.all(ranges.map((range) => this.analyzeAudioForProject(next, media, range)));
243
352
  if (analyses[analyses.length - 1])
244
353
  media.audio = analyses[analyses.length - 1];
245
354
  }
@@ -281,6 +390,44 @@ export class MediaAnalysisService {
281
390
  provider: analyzer.descriptor,
282
391
  });
283
392
  }
393
+ async analyzeAudioForProject(project, media, range) {
394
+ const analyzer = this.options.audioAnalyzer;
395
+ if (!analyzer)
396
+ throw new Error("CAPABILITY_UNAVAILABLE: audio analysis");
397
+ const requestedRange = range ?? (media.duration === undefined ? undefined : { start: 0, end: media.duration });
398
+ validateRequestedAudioRange(requestedRange, media.duration, "requested audio range");
399
+ const analysis = await analyzer.analyze({ project, media }, requestedRange);
400
+ validateAudioProvenance(analysis, {
401
+ media,
402
+ mediaId: media.mediaId,
403
+ project,
404
+ provider: analyzer.descriptor,
405
+ requestedRange,
406
+ });
407
+ const measuredRange = analysis.measuredRange ?? (requestedRange ? structuredClone(requestedRange) : undefined);
408
+ if (measuredRange && requestedRange)
409
+ validateAudioRange(measuredRange, requestedRange, "measured audio range");
410
+ if (measuredRange)
411
+ validateRequestedAudioRange(measuredRange, media.duration, "measured audio source range");
412
+ const analyzedDurationSeconds = measuredRange
413
+ ? analysis.analyzedDurationSeconds ?? measuredRange.end - measuredRange.start
414
+ : analysis.analyzedDurationSeconds;
415
+ if (measuredRange && analyzedDurationSeconds !== undefined
416
+ && Math.abs(analyzedDurationSeconds - (measuredRange.end - measuredRange.start)) > 0.000001) {
417
+ throw new Error("ANALYSIS_INVALID: audio measured duration does not match its measured range");
418
+ }
419
+ return {
420
+ ...structuredClone(analysis),
421
+ schemaVersion: 1,
422
+ mediaId: media.mediaId,
423
+ sourceIdentity: sourceIdentityOf(media),
424
+ ...(requestedRange ? { requestedRange: structuredClone(requestedRange) } : {}),
425
+ ...(measuredRange ? { measuredRange: structuredClone(measuredRange) } : {}),
426
+ revision: structuredClone(project.revision),
427
+ provider: analysis.provider ?? analyzer.descriptor ?? { id: "framekit.audio", provider: "unknown" },
428
+ ...(analyzedDurationSeconds !== undefined ? { analyzedDurationSeconds } : {}),
429
+ };
430
+ }
284
431
  }
285
432
  function sourceIdentityOf(media) {
286
433
  return {
@@ -355,10 +502,14 @@ function mergeRanges(ranges) {
355
502
  for (const range of [...ranges].sort((left, right) => left.start - right.start || left.end - right.end)) {
356
503
  const previous = merged[merged.length - 1];
357
504
  if (!previous || range.start > previous.end) {
358
- merged.push({ start: range.start, end: range.end });
505
+ merged.push(structuredClone(range));
359
506
  continue;
360
507
  }
361
- previous.end = Math.max(previous.end, range.end);
508
+ if (range.end > previous.end) {
509
+ previous.end = range.end;
510
+ delete previous.startTime;
511
+ delete previous.durationTime;
512
+ }
362
513
  }
363
514
  return merged;
364
515
  }
@@ -371,6 +522,11 @@ function sameRational(left, right) {
371
522
  return leftParts.value * rightParts.timescale === rightParts.value * leftParts.timescale;
372
523
  }
373
524
  function semanticFromAnalyses(speech, audio, visual, metadata) {
525
+ const usableRanges = mergeRanges([
526
+ ...(metadata?.usableRanges ?? []),
527
+ ...(visual?.scenes ?? []).map((scene) => ({ start: scene.start, end: scene.end })),
528
+ ...(speech?.vadSegments ?? []).filter((segment) => segment.kind === "speech").map((segment) => ({ start: segment.start, end: segment.end })),
529
+ ]);
374
530
  return {
375
531
  subjects: [
376
532
  ...(visual?.subjects ?? []).map((subject) => ({ value: subject.label, confidence: subject.confidence })),
@@ -384,7 +540,7 @@ function semanticFromAnalyses(speech, audio, visual, metadata) {
384
540
  timeOfDay: metadata?.timeOfDay ? structuredClone(metadata.timeOfDay) : [],
385
541
  moods: metadata?.moods ? structuredClone(metadata.moods) : [],
386
542
  ...(visual?.motion ? { motion: structuredClone(visual.motion) } : {}),
387
- usableRanges: metadata?.usableRanges ? structuredClone(metadata.usableRanges) : [],
543
+ usableRanges,
388
544
  ...(speech ? { transcript: speech.words.map((word) => word.text).join(" ") } : {}),
389
545
  ...(audio ? {
390
546
  audio: {
@@ -396,6 +552,159 @@ function semanticFromAnalyses(speech, audio, visual, metadata) {
396
552
  } : {}),
397
553
  };
398
554
  }
555
+ function understandingFromMediaContext(media, revision) {
556
+ if (!media.semantic && !media.analysis && !media.speech && !media.visual && !media.metadata)
557
+ return undefined;
558
+ const sourceIdentity = sourceIdentityOf(media);
559
+ return {
560
+ mediaId: media.mediaId,
561
+ source: media.source,
562
+ sourceIdentity,
563
+ ...(media.metadata ? { metadata: structuredClone(media.metadata) } : {}),
564
+ ...(media.speech ? { speech: structuredClone(media.speech) } : {}),
565
+ ...(media.audio ? { audio: structuredClone(media.audio) } : {}),
566
+ ...(media.noise ? { noise: structuredClone(media.noise) } : {}),
567
+ ...(media.visual ? { visual: structuredClone(media.visual) } : {}),
568
+ semantic: structuredClone(media.semantic ?? emptySemanticDescription()),
569
+ analysis: structuredClone(media.analysis ?? []),
570
+ analysisRevision: {
571
+ id: media.analysisRevision ?? revision.id,
572
+ sequence: revision.sequence,
573
+ timestamp: revision.timestamp,
574
+ },
575
+ };
576
+ }
577
+ function timelineSemanticObservations(understanding, sourceRange) {
578
+ const observations = [];
579
+ const analyzerFor = (capability) => understanding.analysis.find((status) => status.capability === capability)?.provenance?.analyzer;
580
+ for (const word of understanding.speech?.words ?? []) {
581
+ const range = intersectTimeRanges({ start: word.start, end: word.end }, sourceRange);
582
+ if (!range)
583
+ continue;
584
+ observations.push({
585
+ capability: "speech",
586
+ range,
587
+ text: word.text,
588
+ confidence: word.confidence,
589
+ ...(analyzerFor("speech") ? { analyzer: structuredClone(analyzerFor("speech")) } : {}),
590
+ });
591
+ }
592
+ for (const scene of understanding.visual?.scenes ?? []) {
593
+ const range = intersectTimeRanges({ start: scene.start, end: scene.end }, sourceRange);
594
+ if (!range)
595
+ continue;
596
+ observations.push({
597
+ capability: "visual",
598
+ range,
599
+ ...(scene.label ? { label: scene.label } : {}),
600
+ ...(scene.confidence !== undefined ? { confidence: scene.confidence } : {}),
601
+ ...(analyzerFor("visual") ? { analyzer: structuredClone(analyzerFor("visual")) } : {}),
602
+ });
603
+ }
604
+ for (const subject of understanding.visual?.subjects ?? []) {
605
+ const range = intersectTimeRanges({
606
+ start: subject.start ?? sourceRange.start,
607
+ end: subject.end ?? sourceRange.end,
608
+ }, sourceRange);
609
+ if (!range)
610
+ continue;
611
+ observations.push({
612
+ capability: "visual",
613
+ range,
614
+ label: subject.label,
615
+ confidence: subject.confidence,
616
+ ...(analyzerFor("visual") ? { analyzer: structuredClone(analyzerFor("visual")) } : {}),
617
+ });
618
+ }
619
+ for (const range of understanding.metadata?.usableRanges ?? []) {
620
+ const clipped = intersectTimeRanges(range, sourceRange);
621
+ if (!clipped)
622
+ continue;
623
+ observations.push({
624
+ capability: "metadata",
625
+ range: clipped,
626
+ ...(analyzerFor("metadata") ? { analyzer: structuredClone(analyzerFor("metadata")) } : {}),
627
+ });
628
+ }
629
+ return observations.sort((left, right) => left.range.start - right.range.start
630
+ || left.range.end - right.range.end
631
+ || left.capability.localeCompare(right.capability)
632
+ || (left.text ?? left.label ?? "").localeCompare(right.text ?? right.label ?? ""));
633
+ }
634
+ function intersectTimeRanges(left, right) {
635
+ const start = Math.max(left.start, right.start);
636
+ const end = Math.min(left.end, right.end);
637
+ return start < end ? { start, end } : undefined;
638
+ }
639
+ function timelineObservationMatches(observation, query) {
640
+ const normalized = query.trim().toLocaleLowerCase();
641
+ if (!normalized)
642
+ return true;
643
+ return [observation.text, observation.label]
644
+ .filter((value) => Boolean(value))
645
+ .some((value) => value.toLocaleLowerCase().includes(normalized));
646
+ }
647
+ function unavailableTimelineOccurrence(occurrenceId, reason, mediaId, sourceRange, sourceIdentity) {
648
+ return {
649
+ occurrenceId,
650
+ status: "unavailable",
651
+ ...(mediaId ? { mediaId } : {}),
652
+ ...(sourceIdentity ? { sourceIdentity: structuredClone(sourceIdentity) } : {}),
653
+ ...(sourceRange ? { sourceRange: structuredClone(sourceRange) } : {}),
654
+ reason,
655
+ observations: [],
656
+ analysis: [],
657
+ };
658
+ }
659
+ function rationalSeconds(time) {
660
+ const parsed = parseRational(time, "TIMELINE_SEMANTIC_CONTEXT_INVALID");
661
+ const seconds = Number(parsed.value) / Number(parsed.timescale);
662
+ if (!Number.isFinite(seconds))
663
+ throw new Error("TIMELINE_SEMANTIC_CONTEXT_INVALID: rational time is outside supported range");
664
+ return seconds;
665
+ }
666
+ function understandingToIndexEntry(understanding) {
667
+ return {
668
+ sourceIdentity: structuredClone(understanding.sourceIdentity),
669
+ semantic: structuredClone(understanding.semantic),
670
+ analysis: structuredClone(understanding.analysis),
671
+ ...(understanding.speech ? { speech: structuredClone(understanding.speech) } : {}),
672
+ ...(understanding.audio ? { audio: structuredClone(understanding.audio) } : {}),
673
+ analysisRevision: understanding.analysisRevision.id,
674
+ };
675
+ }
676
+ function mergeIndexEntries(persisted, current) {
677
+ const merged = new Map();
678
+ for (const entry of [...persisted, ...current]) {
679
+ const key = JSON.stringify(entry.sourceIdentity);
680
+ const previous = merged.get(key);
681
+ if (!previous || indexEntryScore(entry) >= indexEntryScore(previous))
682
+ merged.set(key, structuredClone(entry));
683
+ }
684
+ return [...merged.values()].sort((left, right) => left.sourceIdentity.mediaId.localeCompare(right.sourceIdentity.mediaId));
685
+ }
686
+ function indexEntryScore(entry) {
687
+ return entry.analysis.filter((record) => record.status === "analyzed").length
688
+ + entry.semantic.usableRanges.length
689
+ + (entry.semantic.transcript?.trim() ? 1 : 0);
690
+ }
691
+ function cacheMatchesAnalyzers(understanding, options) {
692
+ const configured = [
693
+ ["speech", options.speechAnalyzer],
694
+ ["audio", options.audioAnalyzer],
695
+ ["noise", options.noiseAnalyzer],
696
+ ["visual", options.visualAnalyzer],
697
+ ["metadata", options.metadataAnalyzer],
698
+ ];
699
+ return configured.every(([capability, analyzer]) => {
700
+ if (!analyzer)
701
+ return true;
702
+ const record = understanding.analysis.find((candidate) => candidate.capability === capability);
703
+ return record?.status === "analyzed"
704
+ && record.provenance?.analyzer !== undefined
705
+ && sameDescriptor(record.provenance.analyzer, analyzer.descriptor ?? { id: `framekit.${capability}`, provider: "unknown" });
706
+ });
707
+ }
399
708
  function emptySemanticDescription() {
400
709
  return {
401
710
  subjects: [],
@@ -407,7 +716,7 @@ function emptySemanticDescription() {
407
716
  };
408
717
  }
409
718
  function matchesMediaIndexQuery(entry, query) {
410
- const text = query.query?.trim().toLowerCase();
719
+ const terms = query.query?.trim().toLowerCase().split(/\s+/u).filter(Boolean) ?? [];
411
720
  const values = [
412
721
  entry.sourceIdentity.mediaId,
413
722
  entry.sourceIdentity.source,
@@ -418,7 +727,7 @@ function matchesMediaIndexQuery(entry, query) {
418
727
  ...entry.semantic.timeOfDay.map((tag) => tag.value),
419
728
  ...entry.semantic.moods.map((tag) => tag.value),
420
729
  ].map((value) => value.toLowerCase());
421
- if (text && !values.some((value) => value.includes(text)))
730
+ if (terms.length > 0 && !values.some((value) => terms.some((term) => value.includes(term))))
422
731
  return false;
423
732
  if (query.subject && !matchesTag(entry.semantic.subjects, query.subject))
424
733
  return false;
@@ -432,13 +741,21 @@ function matchesMediaIndexQuery(entry, query) {
432
741
  return false;
433
742
  if (query.motion && entry.semantic.motion?.label !== query.motion)
434
743
  return false;
435
- if (query.range && !entry.semantic.usableRanges.some((range) => range.end > query.range.start && range.start < query.range.end))
744
+ if (query.range && !indexEntryRanges(entry).some((range) => range.end > query.range.start && range.start < query.range.end))
436
745
  return false;
437
746
  if (query.capabilities?.some((capability) => !entry.analysis.some((record) => record.capability === capability && record.status !== "unavailable"))) {
438
747
  return false;
439
748
  }
440
749
  return true;
441
750
  }
751
+ function indexEntryRanges(entry) {
752
+ const speechRanges = entry.speech?.vadSegments
753
+ ?.filter((segment) => segment.kind === "speech")
754
+ .map(({ start, end }) => ({ start, end }))
755
+ ?? entry.speech?.words.map(({ start, end }) => ({ start, end }))
756
+ ?? [];
757
+ return [...entry.semantic.usableRanges, ...speechRanges];
758
+ }
442
759
  function matchesTag(tags, query) {
443
760
  const normalized = query.trim().toLowerCase();
444
761
  return tags.some((tag) => tag.value.toLowerCase() === normalized);
@@ -494,11 +811,12 @@ function validateAudioProvenance(analysis, expected) {
494
811
  && !sameMediaSourceIdentity(analysis.sourceIdentity, sourceIdentityOf(expected.media))) {
495
812
  throw new Error("TARGET_MISMATCH: audio analysis source identity does not match the requested media");
496
813
  }
497
- if (analysis.requestedRange !== undefined && !sameRange(analysis.requestedRange, expected.requestedRange)) {
814
+ if (analysis.requestedRange !== undefined && expected.requestedRange !== undefined
815
+ && !sameRange(analysis.requestedRange, expected.requestedRange)) {
498
816
  throw new Error("ANALYSIS_INVALID: audio requested range does not match the runtime request");
499
817
  }
500
818
  if (analysis.revision !== undefined
501
- && (analysis.revision.id !== expected.project.revision.id || analysis.revision.sequence !== expected.project.revision.sequence)) {
819
+ && !sameContextRevision(analysis.revision, expected.project.revision)) {
502
820
  throw new Error("ANALYSIS_STALE: audio analysis revision does not match the inspected project");
503
821
  }
504
822
  if (analysis.provider !== undefined && expected.provider !== undefined
@@ -507,11 +825,23 @@ function validateAudioProvenance(analysis, expected) {
507
825
  }
508
826
  }
509
827
  function validateAudioRange(range, requested, label) {
510
- if (!Number.isFinite(range.start) || !Number.isFinite(range.end)
511
- || range.start < requested.start || range.end > requested.end || range.end <= range.start) {
828
+ validateRequestedAudioRange(range, undefined, label);
829
+ if (range.start < requested.start || range.end > requested.end) {
512
830
  throw new Error(`ANALYSIS_INVALID: ${label} must fit inside the requested occurrence range`);
513
831
  }
514
832
  }
833
+ function validateRequestedAudioRange(range, duration, label) {
834
+ if (!range)
835
+ return;
836
+ if (!Number.isFinite(range.start) || !Number.isFinite(range.end)
837
+ || range.start < 0 || range.end <= range.start
838
+ || (duration !== undefined && range.end > duration)) {
839
+ throw new Error(`ANALYSIS_INVALID: ${label} must be finite, positive, and inside the source duration`);
840
+ }
841
+ }
842
+ function sameContextRevision(left, right) {
843
+ return left.id === right.id && left.sequence === right.sequence && left.timestamp === right.timestamp;
844
+ }
515
845
  function sameRange(left, right) {
516
846
  return Math.abs(left.start - right.start) <= 0.000001
517
847
  && Math.abs(left.end - right.end) <= 0.000001;
@@ -0,0 +1,86 @@
1
+ import { mkdir, readFile, rename, unlink, writeFile } from "node:fs/promises";
2
+ import { dirname } from "node:path";
3
+ import { randomUUID } from "node:crypto";
4
+ import { sameMediaSourceIdentity } from "../domain/media.js";
5
+ export const SEMANTIC_MEDIA_INDEX_SCHEMA_VERSION = 1;
6
+ /** Deterministic store for runtime and integration tests. */
7
+ export class InMemorySemanticMediaIndexStore {
8
+ entries = [];
9
+ async load(source) {
10
+ const entry = this.entries.find((candidate) => sameMediaSourceIdentity(candidate.sourceIdentity, source));
11
+ return entry ? structuredClone(entry) : undefined;
12
+ }
13
+ async list() {
14
+ return structuredClone(this.entries);
15
+ }
16
+ async save(understanding) {
17
+ const retained = this.entries.filter((candidate) => candidate.mediaId !== understanding.mediaId);
18
+ retained.push(structuredClone(understanding));
19
+ retained.sort((left, right) => left.mediaId.localeCompare(right.mediaId));
20
+ this.entries.splice(0, this.entries.length, ...retained);
21
+ }
22
+ }
23
+ /** Atomic JSON persistence for a local semantic media index. */
24
+ export class JsonSemanticMediaIndexStore {
25
+ path;
26
+ constructor(path) {
27
+ this.path = path;
28
+ }
29
+ async load(source) {
30
+ const document = await this.readDocument();
31
+ const entry = document.entries.find((candidate) => sameMediaSourceIdentity(candidate.sourceIdentity, source));
32
+ return entry ? structuredClone(entry) : undefined;
33
+ }
34
+ async list() {
35
+ return structuredClone((await this.readDocument()).entries);
36
+ }
37
+ async save(understanding) {
38
+ const document = await this.readDocument();
39
+ document.entries = [
40
+ ...document.entries.filter((candidate) => candidate.mediaId !== understanding.mediaId),
41
+ structuredClone(understanding),
42
+ ].sort((left, right) => left.mediaId.localeCompare(right.mediaId));
43
+ await mkdir(dirname(this.path), { recursive: true });
44
+ const temporary = `${this.path}.${randomUUID()}.tmp`;
45
+ try {
46
+ await writeFile(temporary, `${JSON.stringify(document)}\n`, { encoding: "utf8", flag: "wx" });
47
+ await rename(temporary, this.path);
48
+ }
49
+ finally {
50
+ try {
51
+ await unlink(temporary);
52
+ }
53
+ catch (error) {
54
+ if (!isMissingFile(error))
55
+ throw error;
56
+ }
57
+ }
58
+ }
59
+ async readDocument() {
60
+ let raw;
61
+ try {
62
+ raw = await readFile(this.path, "utf8");
63
+ }
64
+ catch (error) {
65
+ if (isMissingFile(error))
66
+ return { schemaVersion: SEMANTIC_MEDIA_INDEX_SCHEMA_VERSION, entries: [] };
67
+ throw error;
68
+ }
69
+ let value;
70
+ try {
71
+ value = JSON.parse(raw);
72
+ }
73
+ catch (error) {
74
+ throw new Error(`SEMANTIC_INDEX_CORRUPT: index JSON is malformed (${String(error)})`);
75
+ }
76
+ if (!value || typeof value !== "object"
77
+ || value.schemaVersion !== SEMANTIC_MEDIA_INDEX_SCHEMA_VERSION
78
+ || !Array.isArray(value.entries)) {
79
+ throw new Error("SEMANTIC_INDEX_CORRUPT: unsupported semantic index document");
80
+ }
81
+ return structuredClone(value);
82
+ }
83
+ }
84
+ function isMissingFile(error) {
85
+ return Boolean(error && typeof error === "object" && "code" in error && error.code === "ENOENT");
86
+ }
@@ -14,6 +14,7 @@ export * from "./timeline/fast-observation.js";
14
14
  export * from "./application/project-store.js";
15
15
  export * from "./application/local-media.js";
16
16
  export * from "./application/project-transactions.js";
17
+ export * from "./application/semantic-media-index-store.js";
17
18
  export * from "./context/project-catalog.js";
18
19
  export * from "./speech/filler-removal.js";
19
20
  export * from "./audio/dialogue-normalization.js";