@morshoto/framekit 0.1.16 → 0.1.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,8 +6,10 @@ export function planRoughCut(entries, revision, request) {
6
6
  }
7
7
  const candidates = entries
8
8
  .filter((entry) => entry.sourceIdentity.mediaKind !== "audio")
9
- .flatMap((entry) => entry.semantic.usableRanges.map((range) => ({ entry, range })))
9
+ .flatMap((entry) => candidatesForEntry(entry, request))
10
10
  .sort((left, right) => {
11
+ if (left.queryMatches !== right.queryMatches)
12
+ return right.queryMatches - left.queryMatches;
11
13
  const mediaOrder = left.entry.sourceIdentity.mediaId.localeCompare(right.entry.sourceIdentity.mediaId);
12
14
  if (mediaOrder !== 0)
13
15
  return mediaOrder;
@@ -15,14 +17,14 @@ export function planRoughCut(entries, revision, request) {
15
17
  return left.range.start - right.range.start;
16
18
  return left.range.end - right.range.end;
17
19
  });
18
- const shots = candidates.slice(0, maxShots).map(({ entry, range }, index) => shotFor(entry, range, request, index + 1));
20
+ const shots = candidates.slice(0, maxShots).map(({ entry, range, evidence }, index) => shotFor(entry, range, request, index + 1, evidence));
19
21
  const excludedAudioMediaIds = entries
20
22
  .filter((entry) => entry.sourceIdentity.mediaKind === "audio" && entry.semantic.usableRanges.length > 0)
21
23
  .map((entry) => entry.sourceIdentity.mediaId);
22
24
  const warnings = excludedAudioMediaIds.length > 0
23
25
  ? [`Excluded audio-only media from rough-cut shot candidates: ${excludedAudioMediaIds.join(", ")}`]
24
26
  : entries.length > 0 && candidates.length === 0
25
- ? ["No matching media has an explicitly analyzed usable range"]
27
+ ? ["No matching media has an explicitly analyzed usable speech or usable range"]
26
28
  : [];
27
29
  return {
28
30
  planner: { id: "framekit.rough-cut", version: 1 },
@@ -32,7 +34,93 @@ export function planRoughCut(entries, revision, request) {
32
34
  warnings,
33
35
  };
34
36
  }
35
- function shotFor(entry, range, request, order) {
37
+ function candidatesForEntry(entry, request) {
38
+ const speechCandidates = speechCandidatesForEntry(entry, request);
39
+ if (speechCandidates.length > 0)
40
+ return speechCandidates;
41
+ return entry.semantic.usableRanges.map((range) => ({
42
+ entry,
43
+ range: structuredClone(range),
44
+ queryMatches: 0,
45
+ }));
46
+ }
47
+ function speechCandidatesForEntry(entry, request) {
48
+ const speech = entry.speech;
49
+ if (!speech)
50
+ return [];
51
+ const speechSegments = speech.vadSegments?.filter((segment) => segment.kind === "speech") ?? [];
52
+ const ranges = speechSegments.length > 0
53
+ ? speechSegments.map((segment) => ({ range: { start: segment.start, end: segment.end }, vad: segment }))
54
+ : wordRanges(speech.words).map((range) => ({ range }));
55
+ return ranges.flatMap(({ range, vad }) => clipToUsableRanges(entry, range).map((clipped) => {
56
+ const words = speech.words.filter((word) => word.end > clipped.start && word.start < clipped.end);
57
+ const transcript = words.filter((word) => word.filler !== true).map((word) => word.text).join(" ").trim();
58
+ const queryMatches = queryMatchCount(request.query, words);
59
+ const averageWordConfidence = words.length === 0
60
+ ? 0
61
+ : words.reduce((sum, word) => sum + word.confidence, 0) / words.length;
62
+ return {
63
+ entry,
64
+ range: clipped,
65
+ queryMatches,
66
+ evidence: {
67
+ kind: "speech",
68
+ transcript,
69
+ wordCount: words.length,
70
+ averageWordConfidence,
71
+ ...(vad ? {
72
+ vad: {
73
+ kind: vad.kind,
74
+ ...(vad.confidence !== undefined ? { confidence: vad.confidence } : {}),
75
+ },
76
+ } : {}),
77
+ ...(entry.audio ? {
78
+ audio: {
79
+ ...(Number.isFinite(entry.audio.integratedLufs) ? { integratedLufs: entry.audio.integratedLufs } : {}),
80
+ ...(Number.isFinite(entry.audio.truePeakDb) ? { truePeakDb: entry.audio.truePeakDb } : {}),
81
+ ...(Number.isFinite(entry.audio.silenceMs) ? { silenceMs: entry.audio.silenceMs } : {}),
82
+ },
83
+ } : {}),
84
+ },
85
+ };
86
+ }));
87
+ }
88
+ function wordRanges(words) {
89
+ const validWords = words
90
+ .filter((word) => Number.isFinite(word.start) && Number.isFinite(word.end) && word.end > word.start)
91
+ .sort((left, right) => left.start - right.start || left.end - right.end);
92
+ const ranges = [];
93
+ for (const word of validWords) {
94
+ const current = ranges[ranges.length - 1];
95
+ if (current && word.start - current.end <= 0.8) {
96
+ current.end = Math.max(current.end, word.end);
97
+ }
98
+ else {
99
+ ranges.push({ start: word.start, end: word.end });
100
+ }
101
+ }
102
+ return ranges;
103
+ }
104
+ function clipToUsableRanges(entry, range) {
105
+ const bounds = entry.semantic.usableRanges.length > 0 ? entry.semantic.usableRanges : [range];
106
+ const duration = entry.sourceIdentity.duration;
107
+ return bounds.flatMap((bound) => {
108
+ const start = Math.max(range.start, bound.start, 0);
109
+ const end = Math.min(range.end, bound.end, duration ?? Number.POSITIVE_INFINITY);
110
+ return end > start ? [{ start, end }] : [];
111
+ });
112
+ }
113
+ function queryMatchCount(query, words) {
114
+ const terms = tokenize(query);
115
+ if (terms.length === 0)
116
+ return 0;
117
+ const transcript = words.map((word) => word.text.toLowerCase());
118
+ return terms.filter((term) => transcript.some((word) => word.includes(term))).length;
119
+ }
120
+ function tokenize(query) {
121
+ return query?.trim().toLowerCase().split(/\s+/u).filter(Boolean) ?? [];
122
+ }
123
+ function shotFor(entry, range, request, order, evidence) {
36
124
  const matchedProperties = [
37
125
  ...(request.query ? [`query:${request.query}`] : []),
38
126
  ...(request.subject ? [`subject:${request.subject}`] : []),
@@ -54,18 +142,26 @@ function shotFor(entry, range, request, order) {
54
142
  ? activeAnnotations.flatMap(({ query, tags }) => tags.filter((tag) => tag.value.toLowerCase() === query.trim().toLowerCase()))
55
143
  : semanticAnnotations.flatMap(({ tags }) => tags);
56
144
  const confidence = Math.max(0, ...confidenceTags.map((tag) => tag.confidence));
145
+ const speechConfidence = evidence?.averageWordConfidence ?? 0;
146
+ const combinedConfidence = evidence ? Math.max(confidence, speechConfidence) : confidence;
57
147
  const reason = matchedProperties.length > 0
58
148
  ? `matches ${matchedProperties.map((property) => {
59
149
  const [kind, ...value] = property.split(":");
60
150
  return `${kind} "${value.join(":")}"`;
61
151
  }).join(", ")}`
62
- : "has an explicitly analyzed usable range";
152
+ : evidence
153
+ ? `contains speech "${evidence.transcript || "(no transcript text)"}" in a VAD speech interval`
154
+ : "has an explicitly analyzed usable range";
155
+ const speechDetails = evidence
156
+ ? ` Speech evidence covers ${evidence.wordCount} word${evidence.wordCount === 1 ? "" : "s"} ("${evidence.transcript || "no transcript text"}") with average confidence ${evidence.averageWordConfidence.toFixed(2)}.`
157
+ : "";
63
158
  return {
64
159
  order,
65
160
  sourceIdentity: structuredClone(entry.sourceIdentity),
66
161
  range: structuredClone(range),
67
- confidence,
162
+ confidence: combinedConfidence,
68
163
  matchedProperties,
69
- rationale: `Selected ${entry.sourceIdentity.mediaId} because it ${reason}.`,
164
+ rationale: `Selected ${entry.sourceIdentity.mediaId} because it ${reason}.${speechDetails}`,
165
+ ...(evidence ? { evidence } : {}),
70
166
  };
71
167
  }
@@ -43,6 +43,8 @@ export function inferFramekitRenderFeatures(timeline) {
43
43
  const features = ["local-media", "structural-edits"];
44
44
  if (timeline.sequence.occurrences.some(({ gainDb }) => gainDb !== undefined))
45
45
  features.push("audio-gain");
46
+ if (timeline.sequence.occurrences.some(({ role }) => role === "audio" || role === "music"))
47
+ features.push("audio-mixing");
46
48
  if (timeline.sequence.occurrences.some(({ transform }) => transform !== undefined))
47
49
  features.push("transform");
48
50
  if ((timeline.sequence.titles ?? []).length > 0)
@@ -95,6 +97,7 @@ const RENDER_FEATURES = [
95
97
  "local-media",
96
98
  "structural-edits",
97
99
  "audio-gain",
100
+ "audio-mixing",
98
101
  "transform",
99
102
  "titles",
100
103
  "cross-dissolve",
@@ -153,8 +153,8 @@ export class AgentVideoRuntime {
153
153
  async analyzeSpeech(mediaId, range) {
154
154
  return this.media.analyzeSpeech(mediaId, range);
155
155
  }
156
- async analyzeAudio(mediaId) {
157
- return this.media.analyzeAudio(mediaId);
156
+ async analyzeAudio(mediaId, range) {
157
+ return this.media.analyzeAudio(mediaId, range);
158
158
  }
159
159
  async analyzeNoise(mediaId, range) {
160
160
  return this.media.analyzeNoise(mediaId, range);
@@ -180,6 +180,9 @@ export class AgentVideoRuntime {
180
180
  async planRoughCut(request) {
181
181
  return this.media.planRoughCut(request);
182
182
  }
183
+ async inspectTimelineSemanticContext(query = {}) {
184
+ return this.media.inspectTimelineSemanticContext(query);
185
+ }
183
186
  async listAssets(query) {
184
187
  return this.projects.listAssets(query);
185
188
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@morshoto/framekit",
3
- "version": "0.1.16",
3
+ "version": "0.1.17",
4
4
  "packageManager": "pnpm@12.5.1",
5
5
  "private": false,
6
6
  "type": "module",