@morshoto/framekit 0.1.16 → 0.1.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. package/README.md +10 -4
  2. package/dist-package/adapters/final-cut/typescript/src/canonical.js +19 -7
  3. package/dist-package/adapters/final-cut/typescript/src/connection.js +13 -8
  4. package/dist-package/adapters/final-cut/typescript/src/fcpxml.js +147 -34
  5. package/dist-package/adapters/final-cut/typescript/src/index.js +1 -0
  6. package/dist-package/adapters/final-cut/typescript/src/library-inspection.js +60 -1
  7. package/dist-package/adapters/final-cut/typescript/src/live.js +2 -2
  8. package/dist-package/adapters/final-cut/typescript/src/publish-readback.js +213 -0
  9. package/dist-package/adapters/final-cut/typescript/src/publisher.js +762 -238
  10. package/dist-package/adapters/final-cut/typescript/src/session.js +6 -1
  11. package/dist-package/adapters/final-cut/typescript/src/timeline-ir-fcpxml.js +293 -59
  12. package/dist-package/adapters/headless/typescript/src/ffmpeg-renderer.js +137 -19
  13. package/dist-package/adapters/headless/typescript/src/ffmpeg-verifier.js +2 -2
  14. package/dist-package/apps/cli/src/main.js +34 -19
  15. package/dist-package/apps/mcp-server/src/build-alignment.js +59 -0
  16. package/dist-package/apps/mcp-server/src/build-identity.js +30 -0
  17. package/dist-package/apps/mcp-server/src/build-identity.json +5 -0
  18. package/dist-package/apps/mcp-server/src/headless-project-snapshot.js +101 -0
  19. package/dist-package/apps/mcp-server/src/headless-projects.js +473 -3
  20. package/dist-package/apps/mcp-server/src/main.js +33 -6
  21. package/dist-package/apps/mcp-server/src/server.js +367 -52
  22. package/dist-package/apps/mcp-server/src/version.js +21 -2
  23. package/dist-package/packages/runtime/src/application/media-analysis-service.js +360 -30
  24. package/dist-package/packages/runtime/src/application/semantic-media-index-store.js +86 -0
  25. package/dist-package/packages/runtime/src/index.js +1 -0
  26. package/dist-package/packages/runtime/src/planning/rough-cut.js +103 -7
  27. package/dist-package/packages/runtime/src/render/render-contract.js +3 -0
  28. package/dist-package/packages/runtime/src/runtime.js +11 -2
  29. package/dist-package/packages/runtime/src/timeline/editing-session.js +7 -2
  30. package/package.json +9 -9
@@ -6,8 +6,10 @@ export function planRoughCut(entries, revision, request) {
6
6
  }
7
7
  const candidates = entries
8
8
  .filter((entry) => entry.sourceIdentity.mediaKind !== "audio")
9
- .flatMap((entry) => entry.semantic.usableRanges.map((range) => ({ entry, range })))
9
+ .flatMap((entry) => candidatesForEntry(entry, request))
10
10
  .sort((left, right) => {
11
+ if (left.queryMatches !== right.queryMatches)
12
+ return right.queryMatches - left.queryMatches;
11
13
  const mediaOrder = left.entry.sourceIdentity.mediaId.localeCompare(right.entry.sourceIdentity.mediaId);
12
14
  if (mediaOrder !== 0)
13
15
  return mediaOrder;
@@ -15,14 +17,14 @@ export function planRoughCut(entries, revision, request) {
15
17
  return left.range.start - right.range.start;
16
18
  return left.range.end - right.range.end;
17
19
  });
18
- const shots = candidates.slice(0, maxShots).map(({ entry, range }, index) => shotFor(entry, range, request, index + 1));
20
+ const shots = candidates.slice(0, maxShots).map(({ entry, range, evidence }, index) => shotFor(entry, range, request, index + 1, evidence));
19
21
  const excludedAudioMediaIds = entries
20
22
  .filter((entry) => entry.sourceIdentity.mediaKind === "audio" && entry.semantic.usableRanges.length > 0)
21
23
  .map((entry) => entry.sourceIdentity.mediaId);
22
24
  const warnings = excludedAudioMediaIds.length > 0
23
25
  ? [`Excluded audio-only media from rough-cut shot candidates: ${excludedAudioMediaIds.join(", ")}`]
24
26
  : entries.length > 0 && candidates.length === 0
25
- ? ["No matching media has an explicitly analyzed usable range"]
27
+ ? ["No matching media has an explicitly analyzed usable speech or usable range"]
26
28
  : [];
27
29
  return {
28
30
  planner: { id: "framekit.rough-cut", version: 1 },
@@ -32,7 +34,93 @@ export function planRoughCut(entries, revision, request) {
32
34
  warnings,
33
35
  };
34
36
  }
35
- function shotFor(entry, range, request, order) {
37
+ function candidatesForEntry(entry, request) {
38
+ const speechCandidates = speechCandidatesForEntry(entry, request);
39
+ if (speechCandidates.length > 0)
40
+ return speechCandidates;
41
+ return entry.semantic.usableRanges.map((range) => ({
42
+ entry,
43
+ range: structuredClone(range),
44
+ queryMatches: 0,
45
+ }));
46
+ }
47
+ function speechCandidatesForEntry(entry, request) {
48
+ const speech = entry.speech;
49
+ if (!speech)
50
+ return [];
51
+ const speechSegments = speech.vadSegments?.filter((segment) => segment.kind === "speech") ?? [];
52
+ const ranges = speechSegments.length > 0
53
+ ? speechSegments.map((segment) => ({ range: { start: segment.start, end: segment.end }, vad: segment }))
54
+ : wordRanges(speech.words).map((range) => ({ range }));
55
+ return ranges.flatMap(({ range, vad }) => clipToUsableRanges(entry, range).map((clipped) => {
56
+ const words = speech.words.filter((word) => word.end > clipped.start && word.start < clipped.end);
57
+ const transcript = words.filter((word) => word.filler !== true).map((word) => word.text).join(" ").trim();
58
+ const queryMatches = queryMatchCount(request.query, words);
59
+ const averageWordConfidence = words.length === 0
60
+ ? 0
61
+ : words.reduce((sum, word) => sum + word.confidence, 0) / words.length;
62
+ return {
63
+ entry,
64
+ range: clipped,
65
+ queryMatches,
66
+ evidence: {
67
+ kind: "speech",
68
+ transcript,
69
+ wordCount: words.length,
70
+ averageWordConfidence,
71
+ ...(vad ? {
72
+ vad: {
73
+ kind: vad.kind,
74
+ ...(vad.confidence !== undefined ? { confidence: vad.confidence } : {}),
75
+ },
76
+ } : {}),
77
+ ...(entry.audio ? {
78
+ audio: {
79
+ ...(Number.isFinite(entry.audio.integratedLufs) ? { integratedLufs: entry.audio.integratedLufs } : {}),
80
+ ...(Number.isFinite(entry.audio.truePeakDb) ? { truePeakDb: entry.audio.truePeakDb } : {}),
81
+ ...(Number.isFinite(entry.audio.silenceMs) ? { silenceMs: entry.audio.silenceMs } : {}),
82
+ },
83
+ } : {}),
84
+ },
85
+ };
86
+ }));
87
+ }
88
+ function wordRanges(words) {
89
+ const validWords = words
90
+ .filter((word) => Number.isFinite(word.start) && Number.isFinite(word.end) && word.end > word.start)
91
+ .sort((left, right) => left.start - right.start || left.end - right.end);
92
+ const ranges = [];
93
+ for (const word of validWords) {
94
+ const current = ranges[ranges.length - 1];
95
+ if (current && word.start - current.end <= 0.8) {
96
+ current.end = Math.max(current.end, word.end);
97
+ }
98
+ else {
99
+ ranges.push({ start: word.start, end: word.end });
100
+ }
101
+ }
102
+ return ranges;
103
+ }
104
+ function clipToUsableRanges(entry, range) {
105
+ const bounds = entry.semantic.usableRanges.length > 0 ? entry.semantic.usableRanges : [range];
106
+ const duration = entry.sourceIdentity.duration;
107
+ return bounds.flatMap((bound) => {
108
+ const start = Math.max(range.start, bound.start, 0);
109
+ const end = Math.min(range.end, bound.end, duration ?? Number.POSITIVE_INFINITY);
110
+ return end > start ? [{ start, end }] : [];
111
+ });
112
+ }
113
+ function queryMatchCount(query, words) {
114
+ const terms = tokenize(query);
115
+ if (terms.length === 0)
116
+ return 0;
117
+ const transcript = words.map((word) => word.text.toLowerCase());
118
+ return terms.filter((term) => transcript.some((word) => word.includes(term))).length;
119
+ }
120
+ function tokenize(query) {
121
+ return query?.trim().toLowerCase().split(/\s+/u).filter(Boolean) ?? [];
122
+ }
123
+ function shotFor(entry, range, request, order, evidence) {
36
124
  const matchedProperties = [
37
125
  ...(request.query ? [`query:${request.query}`] : []),
38
126
  ...(request.subject ? [`subject:${request.subject}`] : []),
@@ -54,18 +142,26 @@ function shotFor(entry, range, request, order) {
54
142
  ? activeAnnotations.flatMap(({ query, tags }) => tags.filter((tag) => tag.value.toLowerCase() === query.trim().toLowerCase()))
55
143
  : semanticAnnotations.flatMap(({ tags }) => tags);
56
144
  const confidence = Math.max(0, ...confidenceTags.map((tag) => tag.confidence));
145
+ const speechConfidence = evidence?.averageWordConfidence ?? 0;
146
+ const combinedConfidence = evidence ? Math.max(confidence, speechConfidence) : confidence;
57
147
  const reason = matchedProperties.length > 0
58
148
  ? `matches ${matchedProperties.map((property) => {
59
149
  const [kind, ...value] = property.split(":");
60
150
  return `${kind} "${value.join(":")}"`;
61
151
  }).join(", ")}`
62
- : "has an explicitly analyzed usable range";
152
+ : evidence
153
+ ? `contains speech "${evidence.transcript || "(no transcript text)"}" in a VAD speech interval`
154
+ : "has an explicitly analyzed usable range";
155
+ const speechDetails = evidence
156
+ ? ` Speech evidence covers ${evidence.wordCount} word${evidence.wordCount === 1 ? "" : "s"} ("${evidence.transcript || "no transcript text"}") with average confidence ${evidence.averageWordConfidence.toFixed(2)}.`
157
+ : "";
63
158
  return {
64
159
  order,
65
160
  sourceIdentity: structuredClone(entry.sourceIdentity),
66
161
  range: structuredClone(range),
67
- confidence,
162
+ confidence: combinedConfidence,
68
163
  matchedProperties,
69
- rationale: `Selected ${entry.sourceIdentity.mediaId} because it ${reason}.`,
164
+ rationale: `Selected ${entry.sourceIdentity.mediaId} because it ${reason}.${speechDetails}`,
165
+ ...(evidence ? { evidence } : {}),
70
166
  };
71
167
  }
@@ -43,6 +43,8 @@ export function inferFramekitRenderFeatures(timeline) {
43
43
  const features = ["local-media", "structural-edits"];
44
44
  if (timeline.sequence.occurrences.some(({ gainDb }) => gainDb !== undefined))
45
45
  features.push("audio-gain");
46
+ if (timeline.sequence.occurrences.some(({ role }) => role === "audio" || role === "music"))
47
+ features.push("audio-mixing");
46
48
  if (timeline.sequence.occurrences.some(({ transform }) => transform !== undefined))
47
49
  features.push("transform");
48
50
  if ((timeline.sequence.titles ?? []).length > 0)
@@ -95,6 +97,7 @@ const RENDER_FEATURES = [
95
97
  "local-media",
96
98
  "structural-edits",
97
99
  "audio-gain",
100
+ "audio-mixing",
98
101
  "transform",
99
102
  "titles",
100
103
  "cross-dissolve",
@@ -153,8 +153,8 @@ export class AgentVideoRuntime {
153
153
  async analyzeSpeech(mediaId, range) {
154
154
  return this.media.analyzeSpeech(mediaId, range);
155
155
  }
156
- async analyzeAudio(mediaId) {
157
- return this.media.analyzeAudio(mediaId);
156
+ async analyzeAudio(mediaId, range) {
157
+ return this.media.analyzeAudio(mediaId, range);
158
158
  }
159
159
  async analyzeNoise(mediaId, range) {
160
160
  return this.media.analyzeNoise(mediaId, range);
@@ -165,6 +165,9 @@ export class AgentVideoRuntime {
165
165
  async understandMedia(mediaId) {
166
166
  return this.media.understandMedia(mediaId);
167
167
  }
168
+ async understandMediaInProject(mediaId, project) {
169
+ return this.media.understandMediaInProject(mediaId, project);
170
+ }
168
171
  planDuration(request) {
169
172
  return this.durationPolicy.plan(request);
170
173
  }
@@ -180,6 +183,12 @@ export class AgentVideoRuntime {
180
183
  async planRoughCut(request) {
181
184
  return this.media.planRoughCut(request);
182
185
  }
186
+ async planRoughCutInProject(request, project) {
187
+ return this.media.planRoughCutInProject(request, project);
188
+ }
189
+ async inspectTimelineSemanticContext(query = {}) {
190
+ return this.media.inspectTimelineSemanticContext(query);
191
+ }
183
192
  async listAssets(query) {
184
193
  return this.projects.listAssets(query);
185
194
  }
@@ -386,8 +386,13 @@ function validateMediaMetadata(metadata, field) {
386
386
  if (stream.codec !== undefined)
387
387
  requireText(stream.codec, `${streamField}.codec`);
388
388
  if (stream.kind === "video") {
389
- requirePositiveInteger(stream.width, `${streamField}.width`);
390
- requirePositiveInteger(stream.height, `${streamField}.height`);
389
+ if ((stream.width === undefined) !== (stream.height === undefined)) {
390
+ throw new Error(`TIMELINE_IR_INVALID: ${streamField}.width and height must be provided together`);
391
+ }
392
+ if (stream.width !== undefined && stream.height !== undefined) {
393
+ requirePositiveInteger(stream.width, `${streamField}.width`);
394
+ requirePositiveInteger(stream.height, `${streamField}.height`);
395
+ }
391
396
  if (!stream.frameRate)
392
397
  throw new Error(`TIMELINE_IR_INVALID: ${streamField}.frameRate is required`);
393
398
  validateRational(stream.frameRate, `${streamField}.frameRate`, true);
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@morshoto/framekit",
3
- "version": "0.1.16",
4
- "packageManager": "pnpm@12.5.1",
3
+ "version": "0.1.18",
4
+ "packageManager": "pnpm@12.14.0",
5
5
  "private": false,
6
6
  "type": "module",
7
7
  "description": "Agentic video editing runtime and MCP server for Final Cut Pro",
@@ -21,7 +21,7 @@
21
21
  },
22
22
  "scripts": {
23
23
  "build": "tsc -p tsconfig.json",
24
- "build:package": "tsc -p tsconfig.package.json && tsc-alias -p tsconfig.package.json",
24
+ "build:package": "tsc -p tsconfig.package.json && tsc-alias -p tsconfig.package.json && tsx scripts/write-package-build-identity.ts",
25
25
  "test": "tsx --test tests/integration/*.test.ts tests/evaluation/*.test.ts tests/filler-removal/*.test.ts tests/speech/*.test.ts tests/golden/*.test.ts tests/release-gate/*.test.ts tests/release-report/*.test.ts",
26
26
  "test:coverage": "c8 --reporter=json-summary --reporter=text --reports-dir=\"${C8_REPORTS_DIR:-coverage}\" pnpm test",
27
27
  "test:filler-removal": "tsx --test tests/filler-removal/*.test.ts",
@@ -59,7 +59,7 @@
59
59
  "mcp": "tsx apps/mcp-server/src/main.ts",
60
60
  "framekit": "node ./bin/framekit.mjs",
61
61
  "package:final-cut-release": "bash scripts/package-final-cut-release.sh",
62
- "prepack": "tsc -p tsconfig.package.json && tsc-alias -p tsconfig.package.json",
62
+ "prepack": "pnpm run build:package",
63
63
  "xcode:check": "bash nix/check-xcode.sh",
64
64
  "check:boundaries": "node scripts/check-boundaries.mjs"
65
65
  },
@@ -76,14 +76,14 @@
76
76
  "commitizen": "^4.3.1",
77
77
  "fast-xml-parser": "^5.10.1",
78
78
  "git-cz": "^4.9.0",
79
- "zod": "^3.25.76"
79
+ "zod": "^4.6.7"
80
80
  },
81
81
  "devDependencies": {
82
- "@openai/codex": "0.155.1",
83
- "@types/node": "24.13.6",
82
+ "@openai/codex": "0.163.0",
83
+ "@types/node": "24.19.2",
84
84
  "c8": "12.0.0",
85
- "tsc-alias": "1.9.5",
86
- "tsx": "4.23.13",
85
+ "tsc-alias": "1.9.8",
86
+ "tsx": "4.23.15",
87
87
  "typescript": "7.0.2"
88
88
  }
89
89
  }