@morshoto/framekit 0.1.14 → 0.1.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/dist-package/adapters/final-cut/typescript/src/background-materialization.js +28 -0
  2. package/dist-package/adapters/final-cut/typescript/src/canonical.js +144 -35
  3. package/dist-package/adapters/final-cut/typescript/src/connection.js +8 -5
  4. package/dist-package/adapters/final-cut/typescript/src/disposable-native.js +36 -7
  5. package/dist-package/adapters/final-cut/typescript/src/fcpxml.js +21 -7
  6. package/dist-package/adapters/final-cut/typescript/src/index.js +2 -0
  7. package/dist-package/adapters/final-cut/typescript/src/live.js +56 -1
  8. package/dist-package/adapters/final-cut/typescript/src/native.js +311 -20
  9. package/dist-package/adapters/final-cut/typescript/src/pasteboard-snapshot.js +312 -0
  10. package/dist-package/adapters/final-cut/typescript/src/publisher.js +68 -8
  11. package/dist-package/adapters/final-cut/typescript/src/sqlite-wal-observation.js +263 -0
  12. package/dist-package/adapters/final-cut/typescript/src/timeline-ir-fcpxml.js +179 -25
  13. package/dist-package/adapters/headless/typescript/src/ffmpeg-media-probe.js +62 -0
  14. package/dist-package/adapters/headless/typescript/src/ffmpeg-renderer.js +551 -0
  15. package/dist-package/adapters/headless/typescript/src/ffmpeg-verifier.js +181 -0
  16. package/dist-package/adapters/headless/typescript/src/index.js +3 -0
  17. package/dist-package/apps/mcp-server/src/headless-projects.js +370 -0
  18. package/dist-package/apps/mcp-server/src/headless-sessions.js +3 -0
  19. package/dist-package/apps/mcp-server/src/main.js +28 -4
  20. package/dist-package/apps/mcp-server/src/materialization-jobs.js +233 -4
  21. package/dist-package/apps/mcp-server/src/readback-policy.js +26 -0
  22. package/dist-package/apps/mcp-server/src/routing.js +134 -5
  23. package/dist-package/apps/mcp-server/src/server.js +311 -34
  24. package/dist-package/packages/runtime/src/application/local-media.js +157 -0
  25. package/dist-package/packages/runtime/src/application/media-analysis-service.js +347 -27
  26. package/dist-package/packages/runtime/src/application/project-store.js +234 -0
  27. package/dist-package/packages/runtime/src/application/project-transactions.js +227 -0
  28. package/dist-package/packages/runtime/src/application/semantic-media-index-store.js +86 -0
  29. package/dist-package/packages/runtime/src/index.js +7 -0
  30. package/dist-package/packages/runtime/src/planning/rough-cut.js +103 -7
  31. package/dist-package/packages/runtime/src/render/render-and-verify.js +126 -0
  32. package/dist-package/packages/runtime/src/render/render-contract.js +146 -0
  33. package/dist-package/packages/runtime/src/runtime.js +5 -2
  34. package/dist-package/packages/runtime/src/timeline/editing-session.js +277 -9
  35. package/dist-package/packages/runtime/src/timeline/fast-observation.js +141 -0
  36. package/package.json +2 -1
@@ -0,0 +1,157 @@
1
+ import { createHash } from "node:crypto";
2
+ import { createReadStream } from "node:fs";
3
+ import { lstat } from "node:fs/promises";
4
+ import { basename, resolve } from "node:path";
5
+ export class LocalMediaError extends Error {
6
+ code;
7
+ details;
8
+ constructor(code, message, details) {
9
+ super(`${code}: ${message}`);
10
+ this.name = "LocalMediaError";
11
+ this.code = code;
12
+ this.details = details;
13
+ }
14
+ }
15
+ export class LocalMediaRegistrar {
16
+ store;
17
+ metadataProbe;
18
+ clock;
19
+ constructor(store, metadataProbe, options = {}) {
20
+ this.store = store;
21
+ this.metadataProbe = metadataProbe;
22
+ this.clock = options.clock ?? (() => new Date().toISOString());
23
+ }
24
+ async register(sourcePath, expectedRevision) {
25
+ const source = await inspectLocalFile(sourcePath);
26
+ const digest = await digestFile(source.path);
27
+ await assertStableSource(source, digest);
28
+ const project = await this.store.load();
29
+ if (expectedRevision && !sameRevision(project.timeline.revision, expectedRevision)) {
30
+ throw new LocalMediaError("MEDIA_CHANGED", "project revision changed before media registration", {
31
+ expectedRevision,
32
+ actualRevision: project.timeline.revision,
33
+ });
34
+ }
35
+ const existing = project.timeline.resources.find((resource) => resource.source === source.path);
36
+ if (existing) {
37
+ if (existing.sourceDigest !== digest) {
38
+ throw new LocalMediaError("MEDIA_CHANGED", `registered source changed: ${source.path}`, {
39
+ id: existing.id,
40
+ source: source.path,
41
+ expectedDigest: existing.sourceDigest,
42
+ actualDigest: digest,
43
+ });
44
+ }
45
+ await assertStableSource(source, digest);
46
+ return { status: "already-registered", project, resource: structuredClone(existing) };
47
+ }
48
+ const metadata = await this.metadataProbe.probe(source.path);
49
+ await assertStableSource(source, digest);
50
+ const resource = {
51
+ id: stableMediaId(source.path, digest),
52
+ name: basename(source.path),
53
+ mediaKind: metadata.streams.some((stream) => stream.kind === "video") ? "video" : "audio",
54
+ source: source.path,
55
+ sourceKind: "local-file",
56
+ sourceDigest: digest,
57
+ metadata: structuredClone(metadata),
58
+ };
59
+ const next = structuredClone(project);
60
+ next.timeline.resources.push(resource);
61
+ const timestamp = this.clock();
62
+ next.timeline.revision = nextRevision(project.timeline.revision, timestamp);
63
+ next.metadata.updatedAt = timestamp;
64
+ const saved = await this.store.save(next, project.timeline.revision);
65
+ return { status: "registered", project: saved, resource: structuredClone(resource) };
66
+ }
67
+ async inspect(project) {
68
+ const loadedProject = project ?? await this.store.load();
69
+ const results = [];
70
+ for (const resource of loadedProject.timeline.resources) {
71
+ if (resource.sourceKind !== "local-file" || !resource.source || !resource.sourceDigest)
72
+ continue;
73
+ try {
74
+ const source = await inspectLocalFile(resource.source);
75
+ const actualDigest = await digestFile(source.path);
76
+ await assertStableSource(source, actualDigest);
77
+ results.push(actualDigest === resource.sourceDigest
78
+ ? { id: resource.id, source: source.path, status: "available", expectedDigest: resource.sourceDigest, actualDigest }
79
+ : { id: resource.id, source: source.path, status: "changed", expectedDigest: resource.sourceDigest, actualDigest });
80
+ }
81
+ catch (error) {
82
+ if (error instanceof LocalMediaError && error.code === "MEDIA_MISSING") {
83
+ results.push({ id: resource.id, source: resource.source, status: "missing", expectedDigest: resource.sourceDigest });
84
+ continue;
85
+ }
86
+ throw error;
87
+ }
88
+ }
89
+ return results;
90
+ }
91
+ async reopen() {
92
+ const project = await this.store.load();
93
+ const statuses = await this.inspect(project);
94
+ const invalid = statuses.find((status) => status.status !== "available");
95
+ if (invalid?.status === "missing") {
96
+ throw new LocalMediaError("MEDIA_MISSING", `registered source is missing: ${invalid.source}`, { media: statuses });
97
+ }
98
+ if (invalid?.status === "changed") {
99
+ throw new LocalMediaError("MEDIA_CHANGED", `registered source changed: ${invalid.source}`, { media: statuses });
100
+ }
101
+ return project;
102
+ }
103
+ }
104
+ async function inspectLocalFile(sourcePath) {
105
+ const path = resolve(sourcePath);
106
+ let details;
107
+ try {
108
+ details = await lstat(path);
109
+ }
110
+ catch (error) {
111
+ if (error.code === "ENOENT") {
112
+ throw new LocalMediaError("MEDIA_MISSING", `local source does not exist: ${path}`, { source: path });
113
+ }
114
+ throw error;
115
+ }
116
+ if (!details.isFile() || details.isSymbolicLink()) {
117
+ throw new LocalMediaError("MEDIA_AMBIGUOUS", `local source is not a regular file: ${path}`, { source: path });
118
+ }
119
+ return { path, device: details.dev, inode: details.ino, size: details.size, mtimeMs: details.mtimeMs, ctimeMs: details.ctimeMs };
120
+ }
121
+ async function assertStableSource(identity, expectedDigest) {
122
+ const current = await inspectLocalFile(identity.path);
123
+ if (current.device !== identity.device
124
+ || current.inode !== identity.inode
125
+ || current.size !== identity.size
126
+ || current.mtimeMs !== identity.mtimeMs
127
+ || current.ctimeMs !== identity.ctimeMs
128
+ || await digestFile(current.path) !== expectedDigest) {
129
+ throw new LocalMediaError("MEDIA_CHANGED", `local source changed during registration: ${identity.path}`, {
130
+ source: identity.path,
131
+ expectedDigest,
132
+ });
133
+ }
134
+ }
135
+ async function digestFile(path) {
136
+ const digest = createHash("sha256");
137
+ await new Promise((resolvePromise, reject) => {
138
+ const stream = createReadStream(path);
139
+ stream.on("data", (chunk) => digest.update(chunk));
140
+ stream.once("error", reject);
141
+ stream.once("end", resolvePromise);
142
+ });
143
+ return digest.digest("hex");
144
+ }
145
+ function stableMediaId(path, digest) {
146
+ return `local-media:${createHash("sha256").update(`${path}\0${digest}`).digest("hex")}`;
147
+ }
148
+ function sameRevision(left, right) {
149
+ return left.id === right.id && left.sequence === right.sequence;
150
+ }
151
+ function nextRevision(previous, timestamp) {
152
+ return {
153
+ id: `revision-${createHash("sha256").update(`${previous.id}\0${previous.sequence + 1}\0${timestamp}`).digest("hex")}`,
154
+ sequence: previous.sequence + 1,
155
+ timestamp,
156
+ };
157
+ }
@@ -18,12 +18,12 @@ export class MediaAnalysisService {
18
18
  const media = findMedia(project, mediaId);
19
19
  return this.analyzeSpeechForProject(project, media, range);
20
20
  }
21
- async analyzeAudio(mediaId) {
21
+ async analyzeAudio(mediaId, range) {
22
22
  if (!this.options.audioAnalyzer)
23
23
  throw new Error("CAPABILITY_UNAVAILABLE: audio analysis");
24
24
  const project = await this.project.inspectProject();
25
25
  const media = findMedia(project, mediaId);
26
- return this.options.audioAnalyzer.analyze({ project, media });
26
+ return this.analyzeAudioForProject(project, media, range);
27
27
  }
28
28
  async analyzeNoise(mediaId, range) {
29
29
  if (!this.options.noiseAnalyzer)
@@ -45,19 +45,8 @@ export class MediaAnalysisService {
45
45
  const media = findMedia(project, mediaId);
46
46
  const sourceStart = clip.sourceStart ?? 0;
47
47
  const requestedRange = { start: sourceStart, end: sourceStart + clip.duration };
48
- const analysis = await this.options.audioAnalyzer.analyze({ project, media }, requestedRange);
49
- validateAudioProvenance(analysis, {
50
- media,
51
- mediaId,
52
- project,
53
- provider: this.options.audioAnalyzer.descriptor,
54
- requestedRange,
55
- });
56
- const measuredRange = analysis.measuredRange ?? {
57
- start: requestedRange.start,
58
- end: requestedRange.start + (analysis.analyzedDurationSeconds ?? clip.duration),
59
- };
60
- validateAudioRange(measuredRange, requestedRange, "measured audio range");
48
+ const analysis = await this.analyzeAudioForProject(project, media, requestedRange);
49
+ const measuredRange = analysis.measuredRange ?? requestedRange;
61
50
  const analyzedDurationSeconds = analysis.analyzedDurationSeconds ?? measuredRange.end - measuredRange.start;
62
51
  if (Math.abs(analyzedDurationSeconds - (measuredRange.end - measuredRange.start)) > 0.000001) {
63
52
  throw new Error("ANALYSIS_INVALID: audio measured duration does not match its measured range");
@@ -133,10 +122,15 @@ export class MediaAnalysisService {
133
122
  async understandMedia(mediaId) {
134
123
  const project = await this.project.inspectProject();
135
124
  const media = findMedia(project, mediaId);
125
+ const cached = await this.loadCachedUnderstanding(project, media);
126
+ if (cached) {
127
+ this.context.attachMediaUnderstanding(cached);
128
+ return structuredClone(cached);
129
+ }
136
130
  const input = { project, media };
137
131
  const [speechResult, audioResult, noiseResult, visualResult, metadataResult] = await Promise.all([
138
132
  settle(() => this.options.speechAnalyzer ? this.analyzeSpeechForProject(project, media) : undefined),
139
- settle(() => this.options.audioAnalyzer?.analyze(input)),
133
+ settle(() => this.options.audioAnalyzer ? this.analyzeAudioForProject(project, media) : undefined),
140
134
  settle(() => this.options.noiseAnalyzer?.analyze(input)),
141
135
  settle(() => this.options.visualAnalyzer?.analyze(input)),
142
136
  settle(() => this.options.metadataAnalyzer?.analyze(input)),
@@ -167,6 +161,7 @@ export class MediaAnalysisService {
167
161
  analysisRevision: project.revision,
168
162
  };
169
163
  this.context.attachMediaUnderstanding(understanding);
164
+ await this.options.semanticMediaIndexStore?.save(understanding);
170
165
  return structuredClone(understanding);
171
166
  }
172
167
  async inspectMedia(mediaId) {
@@ -178,12 +173,110 @@ export class MediaAnalysisService {
178
173
  }
179
174
  async indexMedia(query = {}) {
180
175
  const project = await this.project.inspectProject();
181
- return this.indexFromProject(project, query);
176
+ const current = this.indexFromProject(project, query);
177
+ const persisted = await this.persistedIndexEntries(query);
178
+ return mergeIndexEntries(persisted, current);
182
179
  }
183
180
  async planRoughCut(request) {
184
181
  const project = await this.project.inspectProject();
185
182
  const { maxShots: _maxShots, ...query } = request;
186
- return planRoughCut(this.indexFromProject(project, query), project.revision, request);
183
+ const current = this.indexFromProject(project, query);
184
+ const persisted = await this.persistedIndexEntries(query);
185
+ return planRoughCut(mergeIndexEntries(persisted, current), project.revision, request);
186
+ }
187
+ async inspectTimelineSemanticContext(query = {}) {
188
+ const project = await this.project.inspectProject();
189
+ const requestedOccurrenceIds = query.occurrenceIds ? new Set(query.occurrenceIds) : undefined;
190
+ const occurrences = [];
191
+ for (const clip of project.timeline.clips) {
192
+ if (requestedOccurrenceIds && !requestedOccurrenceIds.has(clip.id))
193
+ continue;
194
+ const context = await this.inspectTimelineOccurrence(project, clip, query);
195
+ if (context.status === "available" || requestedOccurrenceIds || !query.query) {
196
+ if (!query.query || context.status === "unavailable" || context.observations.length > 0) {
197
+ occurrences.push(context);
198
+ }
199
+ }
200
+ }
201
+ return {
202
+ projectId: project.projectId,
203
+ timelineId: project.timeline.id,
204
+ revision: structuredClone(project.revision),
205
+ query: structuredClone(query),
206
+ occurrences,
207
+ };
208
+ }
209
+ async inspectTimelineOccurrence(project, clip, query) {
210
+ if (!clip.mediaId)
211
+ return unavailableTimelineOccurrence(clip.id, "timeline occurrence has no media binding");
212
+ const media = project.media.find((candidate) => candidate.mediaId === clip.mediaId);
213
+ if (!media)
214
+ return unavailableTimelineOccurrence(clip.id, `media binding is unavailable: ${clip.mediaId}`, clip.mediaId);
215
+ const sourceStart = clip.sourceStart ?? (clip.sourceStartTime ? rationalSeconds(clip.sourceStartTime) : 0);
216
+ const sourceDuration = clip.durationTime ? rationalSeconds(clip.durationTime) : clip.duration;
217
+ const sourceRange = {
218
+ start: sourceStart,
219
+ end: sourceStart + sourceDuration,
220
+ ...(clip.sourceStartTime ? { startTime: structuredClone(clip.sourceStartTime) } : {}),
221
+ durationTime: structuredClone(clip.durationTime),
222
+ };
223
+ if (!Number.isFinite(sourceRange.start) || !Number.isFinite(sourceRange.end) || sourceRange.start < 0 || sourceRange.end <= sourceRange.start) {
224
+ return unavailableTimelineOccurrence(clip.id, "timeline occurrence source range is invalid", clip.mediaId, sourceRange);
225
+ }
226
+ if (media.duration !== undefined && sourceRange.end > media.duration + 0.000001) {
227
+ return unavailableTimelineOccurrence(clip.id, "timeline occurrence source range exceeds media duration", clip.mediaId, sourceRange);
228
+ }
229
+ const sourceIdentity = sourceIdentityOf(media);
230
+ const understanding = await this.options.semanticMediaIndexStore?.load(sourceIdentity)
231
+ ?? understandingFromMediaContext(media, project.revision);
232
+ if (!understanding) {
233
+ return unavailableTimelineOccurrence(clip.id, "source-bound semantic index entry is unavailable", clip.mediaId, sourceRange, sourceIdentity);
234
+ }
235
+ const observations = timelineSemanticObservations(understanding, sourceRange);
236
+ const filtered = query.query
237
+ ? observations.filter((observation) => timelineObservationMatches(observation, query.query))
238
+ : observations;
239
+ return {
240
+ occurrenceId: clip.id,
241
+ status: "available",
242
+ mediaId: clip.mediaId,
243
+ sourceIdentity: structuredClone(understanding.sourceIdentity),
244
+ sourceRange,
245
+ observations: filtered,
246
+ analysis: structuredClone(understanding.analysis),
247
+ analysisRevision: understanding.analysisRevision.id,
248
+ };
249
+ }
250
+ async loadCachedUnderstanding(project, media) {
251
+ const store = this.options.semanticMediaIndexStore;
252
+ if (!store)
253
+ return undefined;
254
+ const cached = await store.load(sourceIdentityOf(media));
255
+ if (!cached || !cacheMatchesAnalyzers(cached, this.options))
256
+ return undefined;
257
+ const rebound = structuredClone(cached);
258
+ const sourceIdentity = sourceIdentityOf(media);
259
+ rebound.sourceIdentity = sourceIdentity;
260
+ rebound.mediaId = media.mediaId;
261
+ rebound.source = media.source;
262
+ rebound.analysisRevision = structuredClone(project.revision);
263
+ if (rebound.speech) {
264
+ rebound.speech.mediaId = media.mediaId;
265
+ rebound.speech.sourceIdentity = sourceIdentity;
266
+ rebound.speech.revision = structuredClone(project.revision);
267
+ }
268
+ if (rebound.audio) {
269
+ rebound.audio.mediaId = media.mediaId;
270
+ rebound.audio.sourceIdentity = sourceIdentity;
271
+ rebound.audio.revision = structuredClone(project.revision);
272
+ }
273
+ return rebound;
274
+ }
275
+ async persistedIndexEntries(query) {
276
+ const entries = await this.options.semanticMediaIndexStore?.list() ?? [];
277
+ return entries
278
+ .map(understandingToIndexEntry)
279
+ .filter((entry) => matchesMediaIndexQuery(entry, query));
187
280
  }
188
281
  indexFromProject(project, query) {
189
282
  return project.media
@@ -197,6 +290,12 @@ export class MediaAnalysisService {
197
290
  analysisStatus("visual", this.options.visualAnalyzer, sourceIdentityOf(media), false),
198
291
  analysisStatus("metadata", this.options.metadataAnalyzer, sourceIdentityOf(media), false),
199
292
  ],
293
+ ...(media.analysis?.some((record) => record.capability === "speech" && record.status === "analyzed") && media.speech
294
+ ? { speech: structuredClone(media.speech) }
295
+ : {}),
296
+ ...(media.analysis?.some((record) => record.capability === "audio" && record.status === "analyzed") && media.audio
297
+ ? { audio: structuredClone(media.audio) }
298
+ : {}),
200
299
  ...(media.analysisRevision ? { analysisRevision: media.analysisRevision } : {}),
201
300
  }))
202
301
  .filter((entry) => matchesMediaIndexQuery(entry, query));
@@ -239,7 +338,7 @@ export class MediaAnalysisService {
239
338
  }
240
339
  }
241
340
  if (requirements.audio && this.options.audioAnalyzer) {
242
- const analyses = await Promise.all(ranges.map((range) => this.options.audioAnalyzer.analyze(input, range)));
341
+ const analyses = await Promise.all(ranges.map((range) => this.analyzeAudioForProject(next, media, range)));
243
342
  if (analyses[analyses.length - 1])
244
343
  media.audio = analyses[analyses.length - 1];
245
344
  }
@@ -281,6 +380,44 @@ export class MediaAnalysisService {
281
380
  provider: analyzer.descriptor,
282
381
  });
283
382
  }
383
+ async analyzeAudioForProject(project, media, range) {
384
+ const analyzer = this.options.audioAnalyzer;
385
+ if (!analyzer)
386
+ throw new Error("CAPABILITY_UNAVAILABLE: audio analysis");
387
+ const requestedRange = range ?? (media.duration === undefined ? undefined : { start: 0, end: media.duration });
388
+ validateRequestedAudioRange(requestedRange, media.duration, "requested audio range");
389
+ const analysis = await analyzer.analyze({ project, media }, requestedRange);
390
+ validateAudioProvenance(analysis, {
391
+ media,
392
+ mediaId: media.mediaId,
393
+ project,
394
+ provider: analyzer.descriptor,
395
+ requestedRange,
396
+ });
397
+ const measuredRange = analysis.measuredRange ?? (requestedRange ? structuredClone(requestedRange) : undefined);
398
+ if (measuredRange && requestedRange)
399
+ validateAudioRange(measuredRange, requestedRange, "measured audio range");
400
+ if (measuredRange)
401
+ validateRequestedAudioRange(measuredRange, media.duration, "measured audio source range");
402
+ const analyzedDurationSeconds = measuredRange
403
+ ? analysis.analyzedDurationSeconds ?? measuredRange.end - measuredRange.start
404
+ : analysis.analyzedDurationSeconds;
405
+ if (measuredRange && analyzedDurationSeconds !== undefined
406
+ && Math.abs(analyzedDurationSeconds - (measuredRange.end - measuredRange.start)) > 0.000001) {
407
+ throw new Error("ANALYSIS_INVALID: audio measured duration does not match its measured range");
408
+ }
409
+ return {
410
+ ...structuredClone(analysis),
411
+ schemaVersion: 1,
412
+ mediaId: media.mediaId,
413
+ sourceIdentity: sourceIdentityOf(media),
414
+ ...(requestedRange ? { requestedRange: structuredClone(requestedRange) } : {}),
415
+ ...(measuredRange ? { measuredRange: structuredClone(measuredRange) } : {}),
416
+ revision: structuredClone(project.revision),
417
+ provider: analysis.provider ?? analyzer.descriptor ?? { id: "framekit.audio", provider: "unknown" },
418
+ ...(analyzedDurationSeconds !== undefined ? { analyzedDurationSeconds } : {}),
419
+ };
420
+ }
284
421
  }
285
422
  function sourceIdentityOf(media) {
286
423
  return {
@@ -355,10 +492,14 @@ function mergeRanges(ranges) {
355
492
  for (const range of [...ranges].sort((left, right) => left.start - right.start || left.end - right.end)) {
356
493
  const previous = merged[merged.length - 1];
357
494
  if (!previous || range.start > previous.end) {
358
- merged.push({ start: range.start, end: range.end });
495
+ merged.push(structuredClone(range));
359
496
  continue;
360
497
  }
361
- previous.end = Math.max(previous.end, range.end);
498
+ if (range.end > previous.end) {
499
+ previous.end = range.end;
500
+ delete previous.startTime;
501
+ delete previous.durationTime;
502
+ }
362
503
  }
363
504
  return merged;
364
505
  }
@@ -371,6 +512,11 @@ function sameRational(left, right) {
371
512
  return leftParts.value * rightParts.timescale === rightParts.value * leftParts.timescale;
372
513
  }
373
514
  function semanticFromAnalyses(speech, audio, visual, metadata) {
515
+ const usableRanges = mergeRanges([
516
+ ...(metadata?.usableRanges ?? []),
517
+ ...(visual?.scenes ?? []).map((scene) => ({ start: scene.start, end: scene.end })),
518
+ ...(speech?.vadSegments ?? []).filter((segment) => segment.kind === "speech").map((segment) => ({ start: segment.start, end: segment.end })),
519
+ ]);
374
520
  return {
375
521
  subjects: [
376
522
  ...(visual?.subjects ?? []).map((subject) => ({ value: subject.label, confidence: subject.confidence })),
@@ -384,7 +530,7 @@ function semanticFromAnalyses(speech, audio, visual, metadata) {
384
530
  timeOfDay: metadata?.timeOfDay ? structuredClone(metadata.timeOfDay) : [],
385
531
  moods: metadata?.moods ? structuredClone(metadata.moods) : [],
386
532
  ...(visual?.motion ? { motion: structuredClone(visual.motion) } : {}),
387
- usableRanges: metadata?.usableRanges ? structuredClone(metadata.usableRanges) : [],
533
+ usableRanges,
388
534
  ...(speech ? { transcript: speech.words.map((word) => word.text).join(" ") } : {}),
389
535
  ...(audio ? {
390
536
  audio: {
@@ -396,6 +542,159 @@ function semanticFromAnalyses(speech, audio, visual, metadata) {
396
542
  } : {}),
397
543
  };
398
544
  }
545
+ function understandingFromMediaContext(media, revision) {
546
+ if (!media.semantic && !media.analysis && !media.speech && !media.visual && !media.metadata)
547
+ return undefined;
548
+ const sourceIdentity = sourceIdentityOf(media);
549
+ return {
550
+ mediaId: media.mediaId,
551
+ source: media.source,
552
+ sourceIdentity,
553
+ ...(media.metadata ? { metadata: structuredClone(media.metadata) } : {}),
554
+ ...(media.speech ? { speech: structuredClone(media.speech) } : {}),
555
+ ...(media.audio ? { audio: structuredClone(media.audio) } : {}),
556
+ ...(media.noise ? { noise: structuredClone(media.noise) } : {}),
557
+ ...(media.visual ? { visual: structuredClone(media.visual) } : {}),
558
+ semantic: structuredClone(media.semantic ?? emptySemanticDescription()),
559
+ analysis: structuredClone(media.analysis ?? []),
560
+ analysisRevision: {
561
+ id: media.analysisRevision ?? revision.id,
562
+ sequence: revision.sequence,
563
+ timestamp: revision.timestamp,
564
+ },
565
+ };
566
+ }
567
+ function timelineSemanticObservations(understanding, sourceRange) {
568
+ const observations = [];
569
+ const analyzerFor = (capability) => understanding.analysis.find((status) => status.capability === capability)?.provenance?.analyzer;
570
+ for (const word of understanding.speech?.words ?? []) {
571
+ const range = intersectTimeRanges({ start: word.start, end: word.end }, sourceRange);
572
+ if (!range)
573
+ continue;
574
+ observations.push({
575
+ capability: "speech",
576
+ range,
577
+ text: word.text,
578
+ confidence: word.confidence,
579
+ ...(analyzerFor("speech") ? { analyzer: structuredClone(analyzerFor("speech")) } : {}),
580
+ });
581
+ }
582
+ for (const scene of understanding.visual?.scenes ?? []) {
583
+ const range = intersectTimeRanges({ start: scene.start, end: scene.end }, sourceRange);
584
+ if (!range)
585
+ continue;
586
+ observations.push({
587
+ capability: "visual",
588
+ range,
589
+ ...(scene.label ? { label: scene.label } : {}),
590
+ ...(scene.confidence !== undefined ? { confidence: scene.confidence } : {}),
591
+ ...(analyzerFor("visual") ? { analyzer: structuredClone(analyzerFor("visual")) } : {}),
592
+ });
593
+ }
594
+ for (const subject of understanding.visual?.subjects ?? []) {
595
+ const range = intersectTimeRanges({
596
+ start: subject.start ?? sourceRange.start,
597
+ end: subject.end ?? sourceRange.end,
598
+ }, sourceRange);
599
+ if (!range)
600
+ continue;
601
+ observations.push({
602
+ capability: "visual",
603
+ range,
604
+ label: subject.label,
605
+ confidence: subject.confidence,
606
+ ...(analyzerFor("visual") ? { analyzer: structuredClone(analyzerFor("visual")) } : {}),
607
+ });
608
+ }
609
+ for (const range of understanding.metadata?.usableRanges ?? []) {
610
+ const clipped = intersectTimeRanges(range, sourceRange);
611
+ if (!clipped)
612
+ continue;
613
+ observations.push({
614
+ capability: "metadata",
615
+ range: clipped,
616
+ ...(analyzerFor("metadata") ? { analyzer: structuredClone(analyzerFor("metadata")) } : {}),
617
+ });
618
+ }
619
+ return observations.sort((left, right) => left.range.start - right.range.start
620
+ || left.range.end - right.range.end
621
+ || left.capability.localeCompare(right.capability)
622
+ || (left.text ?? left.label ?? "").localeCompare(right.text ?? right.label ?? ""));
623
+ }
624
+ function intersectTimeRanges(left, right) {
625
+ const start = Math.max(left.start, right.start);
626
+ const end = Math.min(left.end, right.end);
627
+ return start < end ? { start, end } : undefined;
628
+ }
629
+ function timelineObservationMatches(observation, query) {
630
+ const normalized = query.trim().toLocaleLowerCase();
631
+ if (!normalized)
632
+ return true;
633
+ return [observation.text, observation.label]
634
+ .filter((value) => Boolean(value))
635
+ .some((value) => value.toLocaleLowerCase().includes(normalized));
636
+ }
637
+ function unavailableTimelineOccurrence(occurrenceId, reason, mediaId, sourceRange, sourceIdentity) {
638
+ return {
639
+ occurrenceId,
640
+ status: "unavailable",
641
+ ...(mediaId ? { mediaId } : {}),
642
+ ...(sourceIdentity ? { sourceIdentity: structuredClone(sourceIdentity) } : {}),
643
+ ...(sourceRange ? { sourceRange: structuredClone(sourceRange) } : {}),
644
+ reason,
645
+ observations: [],
646
+ analysis: [],
647
+ };
648
+ }
649
+ function rationalSeconds(time) {
650
+ const parsed = parseRational(time, "TIMELINE_SEMANTIC_CONTEXT_INVALID");
651
+ const seconds = Number(parsed.value) / Number(parsed.timescale);
652
+ if (!Number.isFinite(seconds))
653
+ throw new Error("TIMELINE_SEMANTIC_CONTEXT_INVALID: rational time is outside supported range");
654
+ return seconds;
655
+ }
656
+ function understandingToIndexEntry(understanding) {
657
+ return {
658
+ sourceIdentity: structuredClone(understanding.sourceIdentity),
659
+ semantic: structuredClone(understanding.semantic),
660
+ analysis: structuredClone(understanding.analysis),
661
+ ...(understanding.speech ? { speech: structuredClone(understanding.speech) } : {}),
662
+ ...(understanding.audio ? { audio: structuredClone(understanding.audio) } : {}),
663
+ analysisRevision: understanding.analysisRevision.id,
664
+ };
665
+ }
666
+ function mergeIndexEntries(persisted, current) {
667
+ const merged = new Map();
668
+ for (const entry of [...persisted, ...current]) {
669
+ const key = JSON.stringify(entry.sourceIdentity);
670
+ const previous = merged.get(key);
671
+ if (!previous || indexEntryScore(entry) >= indexEntryScore(previous))
672
+ merged.set(key, structuredClone(entry));
673
+ }
674
+ return [...merged.values()].sort((left, right) => left.sourceIdentity.mediaId.localeCompare(right.sourceIdentity.mediaId));
675
+ }
676
+ function indexEntryScore(entry) {
677
+ return entry.analysis.filter((record) => record.status === "analyzed").length
678
+ + entry.semantic.usableRanges.length
679
+ + (entry.semantic.transcript?.trim() ? 1 : 0);
680
+ }
681
+ function cacheMatchesAnalyzers(understanding, options) {
682
+ const configured = [
683
+ ["speech", options.speechAnalyzer],
684
+ ["audio", options.audioAnalyzer],
685
+ ["noise", options.noiseAnalyzer],
686
+ ["visual", options.visualAnalyzer],
687
+ ["metadata", options.metadataAnalyzer],
688
+ ];
689
+ return configured.every(([capability, analyzer]) => {
690
+ if (!analyzer)
691
+ return true;
692
+ const record = understanding.analysis.find((candidate) => candidate.capability === capability);
693
+ return record?.status === "analyzed"
694
+ && record.provenance?.analyzer !== undefined
695
+ && sameDescriptor(record.provenance.analyzer, analyzer.descriptor ?? { id: `framekit.${capability}`, provider: "unknown" });
696
+ });
697
+ }
399
698
  function emptySemanticDescription() {
400
699
  return {
401
700
  subjects: [],
@@ -432,13 +731,21 @@ function matchesMediaIndexQuery(entry, query) {
432
731
  return false;
433
732
  if (query.motion && entry.semantic.motion?.label !== query.motion)
434
733
  return false;
435
- if (query.range && !entry.semantic.usableRanges.some((range) => range.end > query.range.start && range.start < query.range.end))
734
+ if (query.range && !indexEntryRanges(entry).some((range) => range.end > query.range.start && range.start < query.range.end))
436
735
  return false;
437
736
  if (query.capabilities?.some((capability) => !entry.analysis.some((record) => record.capability === capability && record.status !== "unavailable"))) {
438
737
  return false;
439
738
  }
440
739
  return true;
441
740
  }
741
+ function indexEntryRanges(entry) {
742
+ const speechRanges = entry.speech?.vadSegments
743
+ ?.filter((segment) => segment.kind === "speech")
744
+ .map(({ start, end }) => ({ start, end }))
745
+ ?? entry.speech?.words.map(({ start, end }) => ({ start, end }))
746
+ ?? [];
747
+ return [...entry.semantic.usableRanges, ...speechRanges];
748
+ }
442
749
  function matchesTag(tags, query) {
443
750
  const normalized = query.trim().toLowerCase();
444
751
  return tags.some((tag) => tag.value.toLowerCase() === normalized);
@@ -494,11 +801,12 @@ function validateAudioProvenance(analysis, expected) {
494
801
  && !sameMediaSourceIdentity(analysis.sourceIdentity, sourceIdentityOf(expected.media))) {
495
802
  throw new Error("TARGET_MISMATCH: audio analysis source identity does not match the requested media");
496
803
  }
497
- if (analysis.requestedRange !== undefined && !sameRange(analysis.requestedRange, expected.requestedRange)) {
804
+ if (analysis.requestedRange !== undefined && expected.requestedRange !== undefined
805
+ && !sameRange(analysis.requestedRange, expected.requestedRange)) {
498
806
  throw new Error("ANALYSIS_INVALID: audio requested range does not match the runtime request");
499
807
  }
500
808
  if (analysis.revision !== undefined
501
- && (analysis.revision.id !== expected.project.revision.id || analysis.revision.sequence !== expected.project.revision.sequence)) {
809
+ && !sameContextRevision(analysis.revision, expected.project.revision)) {
502
810
  throw new Error("ANALYSIS_STALE: audio analysis revision does not match the inspected project");
503
811
  }
504
812
  if (analysis.provider !== undefined && expected.provider !== undefined
@@ -507,11 +815,23 @@ function validateAudioProvenance(analysis, expected) {
507
815
  }
508
816
  }
509
817
  function validateAudioRange(range, requested, label) {
510
- if (!Number.isFinite(range.start) || !Number.isFinite(range.end)
511
- || range.start < requested.start || range.end > requested.end || range.end <= range.start) {
818
+ validateRequestedAudioRange(range, undefined, label);
819
+ if (range.start < requested.start || range.end > requested.end) {
512
820
  throw new Error(`ANALYSIS_INVALID: ${label} must fit inside the requested occurrence range`);
513
821
  }
514
822
  }
823
+ function validateRequestedAudioRange(range, duration, label) {
824
+ if (!range)
825
+ return;
826
+ if (!Number.isFinite(range.start) || !Number.isFinite(range.end)
827
+ || range.start < 0 || range.end <= range.start
828
+ || (duration !== undefined && range.end > duration)) {
829
+ throw new Error(`ANALYSIS_INVALID: ${label} must be finite, positive, and inside the source duration`);
830
+ }
831
+ }
832
+ function sameContextRevision(left, right) {
833
+ return left.id === right.id && left.sequence === right.sequence && left.timestamp === right.timestamp;
834
+ }
515
835
  function sameRange(left, right) {
516
836
  return Math.abs(left.start - right.start) <= 0.000001
517
837
  && Math.abs(left.end - right.end) <= 0.000001;