@morshoto/framekit 0.1.16 → 0.1.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist-package/adapters/final-cut/typescript/src/connection.js +8 -5
- package/dist-package/adapters/final-cut/typescript/src/fcpxml.js +21 -7
- package/dist-package/adapters/final-cut/typescript/src/timeline-ir-fcpxml.js +110 -28
- package/dist-package/adapters/headless/typescript/src/ffmpeg-renderer.js +120 -12
- package/dist-package/apps/mcp-server/src/headless-projects.js +233 -2
- package/dist-package/apps/mcp-server/src/main.js +6 -3
- package/dist-package/apps/mcp-server/src/server.js +97 -3
- package/dist-package/packages/runtime/src/application/media-analysis-service.js +347 -27
- package/dist-package/packages/runtime/src/application/semantic-media-index-store.js +86 -0
- package/dist-package/packages/runtime/src/index.js +1 -0
- package/dist-package/packages/runtime/src/planning/rough-cut.js +103 -7
- package/dist-package/packages/runtime/src/render/render-contract.js +3 -0
- package/dist-package/packages/runtime/src/runtime.js +5 -2
- package/package.json +1 -1
|
@@ -6,8 +6,10 @@ export function planRoughCut(entries, revision, request) {
|
|
|
6
6
|
}
|
|
7
7
|
const candidates = entries
|
|
8
8
|
.filter((entry) => entry.sourceIdentity.mediaKind !== "audio")
|
|
9
|
-
.flatMap((entry) =>
|
|
9
|
+
.flatMap((entry) => candidatesForEntry(entry, request))
|
|
10
10
|
.sort((left, right) => {
|
|
11
|
+
if (left.queryMatches !== right.queryMatches)
|
|
12
|
+
return right.queryMatches - left.queryMatches;
|
|
11
13
|
const mediaOrder = left.entry.sourceIdentity.mediaId.localeCompare(right.entry.sourceIdentity.mediaId);
|
|
12
14
|
if (mediaOrder !== 0)
|
|
13
15
|
return mediaOrder;
|
|
@@ -15,14 +17,14 @@ export function planRoughCut(entries, revision, request) {
|
|
|
15
17
|
return left.range.start - right.range.start;
|
|
16
18
|
return left.range.end - right.range.end;
|
|
17
19
|
});
|
|
18
|
-
const shots = candidates.slice(0, maxShots).map(({ entry, range }, index) => shotFor(entry, range, request, index + 1));
|
|
20
|
+
const shots = candidates.slice(0, maxShots).map(({ entry, range, evidence }, index) => shotFor(entry, range, request, index + 1, evidence));
|
|
19
21
|
const excludedAudioMediaIds = entries
|
|
20
22
|
.filter((entry) => entry.sourceIdentity.mediaKind === "audio" && entry.semantic.usableRanges.length > 0)
|
|
21
23
|
.map((entry) => entry.sourceIdentity.mediaId);
|
|
22
24
|
const warnings = excludedAudioMediaIds.length > 0
|
|
23
25
|
? [`Excluded audio-only media from rough-cut shot candidates: ${excludedAudioMediaIds.join(", ")}`]
|
|
24
26
|
: entries.length > 0 && candidates.length === 0
|
|
25
|
-
? ["No matching media has an explicitly analyzed usable range"]
|
|
27
|
+
? ["No matching media has an explicitly analyzed usable speech or usable range"]
|
|
26
28
|
: [];
|
|
27
29
|
return {
|
|
28
30
|
planner: { id: "framekit.rough-cut", version: 1 },
|
|
@@ -32,7 +34,93 @@ export function planRoughCut(entries, revision, request) {
|
|
|
32
34
|
warnings,
|
|
33
35
|
};
|
|
34
36
|
}
|
|
35
|
-
function
|
|
37
|
+
function candidatesForEntry(entry, request) {
|
|
38
|
+
const speechCandidates = speechCandidatesForEntry(entry, request);
|
|
39
|
+
if (speechCandidates.length > 0)
|
|
40
|
+
return speechCandidates;
|
|
41
|
+
return entry.semantic.usableRanges.map((range) => ({
|
|
42
|
+
entry,
|
|
43
|
+
range: structuredClone(range),
|
|
44
|
+
queryMatches: 0,
|
|
45
|
+
}));
|
|
46
|
+
}
|
|
47
|
+
function speechCandidatesForEntry(entry, request) {
|
|
48
|
+
const speech = entry.speech;
|
|
49
|
+
if (!speech)
|
|
50
|
+
return [];
|
|
51
|
+
const speechSegments = speech.vadSegments?.filter((segment) => segment.kind === "speech") ?? [];
|
|
52
|
+
const ranges = speechSegments.length > 0
|
|
53
|
+
? speechSegments.map((segment) => ({ range: { start: segment.start, end: segment.end }, vad: segment }))
|
|
54
|
+
: wordRanges(speech.words).map((range) => ({ range }));
|
|
55
|
+
return ranges.flatMap(({ range, vad }) => clipToUsableRanges(entry, range).map((clipped) => {
|
|
56
|
+
const words = speech.words.filter((word) => word.end > clipped.start && word.start < clipped.end);
|
|
57
|
+
const transcript = words.filter((word) => word.filler !== true).map((word) => word.text).join(" ").trim();
|
|
58
|
+
const queryMatches = queryMatchCount(request.query, words);
|
|
59
|
+
const averageWordConfidence = words.length === 0
|
|
60
|
+
? 0
|
|
61
|
+
: words.reduce((sum, word) => sum + word.confidence, 0) / words.length;
|
|
62
|
+
return {
|
|
63
|
+
entry,
|
|
64
|
+
range: clipped,
|
|
65
|
+
queryMatches,
|
|
66
|
+
evidence: {
|
|
67
|
+
kind: "speech",
|
|
68
|
+
transcript,
|
|
69
|
+
wordCount: words.length,
|
|
70
|
+
averageWordConfidence,
|
|
71
|
+
...(vad ? {
|
|
72
|
+
vad: {
|
|
73
|
+
kind: vad.kind,
|
|
74
|
+
...(vad.confidence !== undefined ? { confidence: vad.confidence } : {}),
|
|
75
|
+
},
|
|
76
|
+
} : {}),
|
|
77
|
+
...(entry.audio ? {
|
|
78
|
+
audio: {
|
|
79
|
+
...(Number.isFinite(entry.audio.integratedLufs) ? { integratedLufs: entry.audio.integratedLufs } : {}),
|
|
80
|
+
...(Number.isFinite(entry.audio.truePeakDb) ? { truePeakDb: entry.audio.truePeakDb } : {}),
|
|
81
|
+
...(Number.isFinite(entry.audio.silenceMs) ? { silenceMs: entry.audio.silenceMs } : {}),
|
|
82
|
+
},
|
|
83
|
+
} : {}),
|
|
84
|
+
},
|
|
85
|
+
};
|
|
86
|
+
}));
|
|
87
|
+
}
|
|
88
|
+
function wordRanges(words) {
|
|
89
|
+
const validWords = words
|
|
90
|
+
.filter((word) => Number.isFinite(word.start) && Number.isFinite(word.end) && word.end > word.start)
|
|
91
|
+
.sort((left, right) => left.start - right.start || left.end - right.end);
|
|
92
|
+
const ranges = [];
|
|
93
|
+
for (const word of validWords) {
|
|
94
|
+
const current = ranges[ranges.length - 1];
|
|
95
|
+
if (current && word.start - current.end <= 0.8) {
|
|
96
|
+
current.end = Math.max(current.end, word.end);
|
|
97
|
+
}
|
|
98
|
+
else {
|
|
99
|
+
ranges.push({ start: word.start, end: word.end });
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
return ranges;
|
|
103
|
+
}
|
|
104
|
+
function clipToUsableRanges(entry, range) {
|
|
105
|
+
const bounds = entry.semantic.usableRanges.length > 0 ? entry.semantic.usableRanges : [range];
|
|
106
|
+
const duration = entry.sourceIdentity.duration;
|
|
107
|
+
return bounds.flatMap((bound) => {
|
|
108
|
+
const start = Math.max(range.start, bound.start, 0);
|
|
109
|
+
const end = Math.min(range.end, bound.end, duration ?? Number.POSITIVE_INFINITY);
|
|
110
|
+
return end > start ? [{ start, end }] : [];
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
function queryMatchCount(query, words) {
|
|
114
|
+
const terms = tokenize(query);
|
|
115
|
+
if (terms.length === 0)
|
|
116
|
+
return 0;
|
|
117
|
+
const transcript = words.map((word) => word.text.toLowerCase());
|
|
118
|
+
return terms.filter((term) => transcript.some((word) => word.includes(term))).length;
|
|
119
|
+
}
|
|
120
|
+
function tokenize(query) {
|
|
121
|
+
return query?.trim().toLowerCase().split(/\s+/u).filter(Boolean) ?? [];
|
|
122
|
+
}
|
|
123
|
+
function shotFor(entry, range, request, order, evidence) {
|
|
36
124
|
const matchedProperties = [
|
|
37
125
|
...(request.query ? [`query:${request.query}`] : []),
|
|
38
126
|
...(request.subject ? [`subject:${request.subject}`] : []),
|
|
@@ -54,18 +142,26 @@ function shotFor(entry, range, request, order) {
|
|
|
54
142
|
? activeAnnotations.flatMap(({ query, tags }) => tags.filter((tag) => tag.value.toLowerCase() === query.trim().toLowerCase()))
|
|
55
143
|
: semanticAnnotations.flatMap(({ tags }) => tags);
|
|
56
144
|
const confidence = Math.max(0, ...confidenceTags.map((tag) => tag.confidence));
|
|
145
|
+
const speechConfidence = evidence?.averageWordConfidence ?? 0;
|
|
146
|
+
const combinedConfidence = evidence ? Math.max(confidence, speechConfidence) : confidence;
|
|
57
147
|
const reason = matchedProperties.length > 0
|
|
58
148
|
? `matches ${matchedProperties.map((property) => {
|
|
59
149
|
const [kind, ...value] = property.split(":");
|
|
60
150
|
return `${kind} "${value.join(":")}"`;
|
|
61
151
|
}).join(", ")}`
|
|
62
|
-
:
|
|
152
|
+
: evidence
|
|
153
|
+
? `contains speech "${evidence.transcript || "(no transcript text)"}" in a VAD speech interval`
|
|
154
|
+
: "has an explicitly analyzed usable range";
|
|
155
|
+
const speechDetails = evidence
|
|
156
|
+
? ` Speech evidence covers ${evidence.wordCount} word${evidence.wordCount === 1 ? "" : "s"} ("${evidence.transcript || "no transcript text"}") with average confidence ${evidence.averageWordConfidence.toFixed(2)}.`
|
|
157
|
+
: "";
|
|
63
158
|
return {
|
|
64
159
|
order,
|
|
65
160
|
sourceIdentity: structuredClone(entry.sourceIdentity),
|
|
66
161
|
range: structuredClone(range),
|
|
67
|
-
confidence,
|
|
162
|
+
confidence: combinedConfidence,
|
|
68
163
|
matchedProperties,
|
|
69
|
-
rationale: `Selected ${entry.sourceIdentity.mediaId} because it ${reason}
|
|
164
|
+
rationale: `Selected ${entry.sourceIdentity.mediaId} because it ${reason}.${speechDetails}`,
|
|
165
|
+
...(evidence ? { evidence } : {}),
|
|
70
166
|
};
|
|
71
167
|
}
|
|
@@ -43,6 +43,8 @@ export function inferFramekitRenderFeatures(timeline) {
|
|
|
43
43
|
const features = ["local-media", "structural-edits"];
|
|
44
44
|
if (timeline.sequence.occurrences.some(({ gainDb }) => gainDb !== undefined))
|
|
45
45
|
features.push("audio-gain");
|
|
46
|
+
if (timeline.sequence.occurrences.some(({ role }) => role === "audio" || role === "music"))
|
|
47
|
+
features.push("audio-mixing");
|
|
46
48
|
if (timeline.sequence.occurrences.some(({ transform }) => transform !== undefined))
|
|
47
49
|
features.push("transform");
|
|
48
50
|
if ((timeline.sequence.titles ?? []).length > 0)
|
|
@@ -95,6 +97,7 @@ const RENDER_FEATURES = [
|
|
|
95
97
|
"local-media",
|
|
96
98
|
"structural-edits",
|
|
97
99
|
"audio-gain",
|
|
100
|
+
"audio-mixing",
|
|
98
101
|
"transform",
|
|
99
102
|
"titles",
|
|
100
103
|
"cross-dissolve",
|
|
@@ -153,8 +153,8 @@ export class AgentVideoRuntime {
|
|
|
153
153
|
async analyzeSpeech(mediaId, range) {
|
|
154
154
|
return this.media.analyzeSpeech(mediaId, range);
|
|
155
155
|
}
|
|
156
|
-
async analyzeAudio(mediaId) {
|
|
157
|
-
return this.media.analyzeAudio(mediaId);
|
|
156
|
+
async analyzeAudio(mediaId, range) {
|
|
157
|
+
return this.media.analyzeAudio(mediaId, range);
|
|
158
158
|
}
|
|
159
159
|
async analyzeNoise(mediaId, range) {
|
|
160
160
|
return this.media.analyzeNoise(mediaId, range);
|
|
@@ -180,6 +180,9 @@ export class AgentVideoRuntime {
|
|
|
180
180
|
async planRoughCut(request) {
|
|
181
181
|
return this.media.planRoughCut(request);
|
|
182
182
|
}
|
|
183
|
+
async inspectTimelineSemanticContext(query = {}) {
|
|
184
|
+
return this.media.inspectTimelineSemanticContext(query);
|
|
185
|
+
}
|
|
183
186
|
async listAssets(query) {
|
|
184
187
|
return this.projects.listAssets(query);
|
|
185
188
|
}
|