@morshoto/framekit 0.1.4 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (24) hide show
  1. package/README.md +2 -2
  2. package/dist-package/adapters/final-cut/typescript/src/analyzers.js +29 -2
  3. package/dist-package/adapters/final-cut/typescript/src/fcpxml.js +16 -5
  4. package/dist-package/adapters/final-cut/typescript/src/publisher.js +207 -30
  5. package/dist-package/adapters/final-cut/typescript/src/session.js +69 -8
  6. package/dist-package/apps/mcp-server/src/main.js +1 -0
  7. package/dist-package/apps/mcp-server/src/server.js +136 -68
  8. package/dist-package/apps/mcp-server/src/version.js +34 -0
  9. package/dist-package/packages/runtime/src/application/media-analysis-service.js +117 -14
  10. package/dist-package/packages/runtime/src/application/project-service.js +21 -2
  11. package/dist-package/packages/runtime/src/capabilities.js +85 -10
  12. package/dist-package/packages/runtime/src/context/context-engine.js +10 -1
  13. package/dist-package/packages/runtime/src/domain/media.js +1 -0
  14. package/dist-package/packages/runtime/src/editing/edit-service.js +40 -4
  15. package/dist-package/packages/runtime/src/editing/filler-removal-service.js +31 -7
  16. package/dist-package/packages/runtime/src/index.js +2 -0
  17. package/dist-package/packages/runtime/src/planning/rough-cut.js +9 -3
  18. package/dist-package/packages/runtime/src/runtime.js +2 -2
  19. package/dist-package/packages/runtime/src/speech/analysis.js +301 -0
  20. package/dist-package/packages/runtime/src/speech/mapping.js +227 -0
  21. package/dist-package/packages/runtime/src/timeline/rational-time.js +5 -0
  22. package/dist-package/packages/testkit/src/fixture-analyzers.js +1 -0
  23. package/dist-package/packages/testkit/src/in-memory-editor.js +1 -0
  24. package/package.json +3 -2
@@ -1,7 +1,9 @@
1
1
  import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
2
+ import { ListToolsRequestSchema } from "@modelcontextprotocol/sdk/types.js";
2
3
  import { z } from "zod";
3
- import { resolveEditingIntent, withCapabilityFamilies, } from "../../../packages/runtime/src/index.js";
4
+ import { createCapabilityPreflight, resolveEditingIntent, withCapabilityFamilies, } from "../../../packages/runtime/src/index.js";
4
5
  import { EDITOR_FIRST_MCP_INSTRUCTIONS, resolveEditingRoute, } from "./routing.js";
6
+ import { FRAMEKIT_VERSION } from "./version.js";
5
7
  const revisionValueSchema = z.object({
6
8
  id: z.string(),
7
9
  sequence: z.number().int().nonnegative(),
@@ -15,6 +17,9 @@ const rationalTimeSchema = z.object({
15
17
  const rangeSchema = z.object({
16
18
  start: z.number().nonnegative(),
17
19
  end: z.number().positive(),
20
+ }).refine((range) => range.end > range.start, {
21
+ message: "end must be greater than start",
22
+ path: ["end"],
18
23
  });
19
24
  const mediaIndexQuerySchema = z.object({
20
25
  query: z.string().optional(),
@@ -193,31 +198,68 @@ const verificationPolicySchema = z.object({
193
198
  loudnessToleranceDb: z.number().finite().nonnegative().optional(),
194
199
  assertions: z.array(verificationAssertionSchema).optional(),
195
200
  }).strict();
196
- const editToolInputSchema = z.object({
197
- type: z.enum(["rename-clip", "trim-clip", "set-gain", "reduce-noise", "set-color-correction", "ripple-delete", "add-marker"]),
198
- clipId: z.string().min(1).optional(),
199
- name: z.string().min(1).optional(),
200
- duration: z.number().positive().optional(),
201
- durationTime: rationalTimeSchema.optional(),
202
- gainDb: z.number().finite().optional(),
203
- reductionDb: z.number().finite().positive().optional(),
204
- correction: colorCorrectionSchema.shape.correction.optional(),
205
- timelineId: z.string().min(1).optional(),
206
- range: rangeSchema.optional(),
207
- reason: z.string().optional(),
208
- marker: markerSchema.optional(),
209
- baseRevision: revisionSchema,
210
- verification: verificationPolicySchema.optional(),
211
- }).strict();
212
- const artifactEditToolInputSchema = editToolInputSchema.extend({
201
+ function mcpObjectSchema(schema) {
202
+ // The MCP SDK only serializes schemas it recognizes as object-shaped.
203
+ Object.defineProperty(schema, "shape", { value: {}, enumerable: false });
204
+ return schema;
205
+ }
206
+ function mcpDiscriminatedUnion(options) {
207
+ const schema = z.discriminatedUnion("type", options);
208
+ return mcpObjectSchema(schema);
209
+ }
210
+ function mcpUnion(options) {
211
+ const schema = z.union(options);
212
+ return mcpObjectSchema(schema);
213
+ }
214
+ function createEditToolInputSchema(target = {}, baseRevision = revisionSchema) {
215
+ const options = [
216
+ renameClipSchema.extend({ ...target, verification: verificationPolicySchema.optional(), baseRevision }).strict(),
217
+ trimClipSchema.extend({ ...target, verification: verificationPolicySchema.optional(), baseRevision }).strict(),
218
+ setGainSchema.extend({ ...target, verification: verificationPolicySchema.optional(), baseRevision }).strict(),
219
+ reduceNoiseSchema.extend({ ...target, verification: verificationPolicySchema.optional(), baseRevision }).strict(),
220
+ colorCorrectionSchema.extend({ ...target, verification: verificationPolicySchema.optional(), baseRevision }).strict(),
221
+ rippleDeleteSchema.extend({ ...target, verification: verificationPolicySchema.optional(), baseRevision }).strict(),
222
+ addMarkerSchema.extend({ ...target, verification: verificationPolicySchema.optional(), baseRevision }).strict(),
223
+ ];
224
+ return mcpDiscriminatedUnion(options);
225
+ }
226
+ const editToolInputSchema = createEditToolInputSchema();
227
+ const artifactEditToolInputSchema = createEditToolInputSchema({
213
228
  artifactPath: z.string().trim().min(1),
214
- baseRevision: revisionValueSchema,
215
- }).strict();
216
- const editorTimelineEditToolInputSchema = editToolInputSchema.extend({
229
+ }, revisionValueSchema);
230
+ const editorTimelineEditToolInputSchema = createEditToolInputSchema({
217
231
  projectId: z.string().trim().min(1),
218
232
  sequenceId: z.string().trim().min(1),
219
- baseRevision: revisionValueSchema,
220
- }).strict();
233
+ }, revisionValueSchema);
234
+ function exposeObjectRoot(schema) {
235
+ if (!isRecord(schema) || !Array.isArray(schema.anyOf))
236
+ return schema;
237
+ const properties = {};
238
+ for (const branch of schema.anyOf) {
239
+ for (const [key, value] of Object.entries(branch.properties ?? {})) {
240
+ if (key !== "type")
241
+ properties[key] = value;
242
+ }
243
+ }
244
+ return { ...schema, type: "object", properties };
245
+ }
246
+ function installMcpSchemaCompatibility(server) {
247
+ const handlers = server.server._requestHandlers;
248
+ const listToolsHandler = handlers?.get("tools/list");
249
+ if (!listToolsHandler)
250
+ throw new Error("MCP tools/list handler was not initialized");
251
+ server.server.removeRequestHandler("tools/list");
252
+ server.server.setRequestHandler(ListToolsRequestSchema, async (request, extra) => {
253
+ const response = await listToolsHandler(request, extra);
254
+ return {
255
+ ...response,
256
+ tools: response.tools?.map((tool) => ({
257
+ ...tool,
258
+ inputSchema: exposeObjectRoot(tool.inputSchema),
259
+ })) ?? [],
260
+ };
261
+ });
262
+ }
221
263
  const artifactPublishInputSchema = z.object({
222
264
  artifactPath: z.string().trim().min(1),
223
265
  transactionId: z.string().min(1),
@@ -334,13 +376,9 @@ const musicDuckingSchema = z.object({
334
376
  dialogueClipIds: z.array(z.string().min(1)).optional(),
335
377
  reductionDb: z.number().finite().optional(),
336
378
  });
337
- const musicAddInputSchema = {
379
+ const musicAddCommonInputSchema = {
338
380
  baseRevision: revisionValueSchema,
339
381
  occurrenceId: z.string().min(1),
340
- mediaId: z.string().min(1).optional(),
341
- import: musicImportSchema.optional(),
342
- placement: z.enum(["append", "insert"]),
343
- start: z.number().nonnegative().optional(),
344
382
  duration: z.number().positive().optional(),
345
383
  targetLane: z.number().int().refine((lane) => lane !== 0, "music requires a non-primary lane"),
346
384
  gainDb: z.number().finite().optional(),
@@ -349,6 +387,30 @@ const musicAddInputSchema = {
349
387
  ducking: musicDuckingSchema.optional(),
350
388
  verification: verificationPolicySchema.optional(),
351
389
  };
390
+ const musicAddInputSchema = mcpUnion([
391
+ z.object({
392
+ ...musicAddCommonInputSchema,
393
+ mediaId: z.string().min(1),
394
+ placement: z.literal("append"),
395
+ }).strict(),
396
+ z.object({
397
+ ...musicAddCommonInputSchema,
398
+ import: musicImportSchema,
399
+ placement: z.literal("append"),
400
+ }).strict(),
401
+ z.object({
402
+ ...musicAddCommonInputSchema,
403
+ mediaId: z.string().min(1),
404
+ placement: z.literal("insert"),
405
+ start: z.number().nonnegative(),
406
+ }).strict(),
407
+ z.object({
408
+ ...musicAddCommonInputSchema,
409
+ import: musicImportSchema,
410
+ placement: z.literal("insert"),
411
+ start: z.number().nonnegative(),
412
+ }).strict(),
413
+ ]);
352
414
  const fillerRemovalInputSchema = {
353
415
  baseRevision: revisionValueSchema,
354
416
  range: rangeSchema,
@@ -539,7 +601,7 @@ function frameResult(value) {
539
601
  }
540
602
  export function createMcpServer(runtime, options = {}) {
541
603
  runtime.registerBuiltinSkills();
542
- const server = new McpServer({ name: "framekit", version: "0.1.0" }, { instructions: EDITOR_FIRST_MCP_INSTRUCTIONS });
604
+ const server = new McpServer({ name: "framekit", version: FRAMEKIT_VERSION }, { instructions: EDITOR_FIRST_MCP_INSTRUCTIONS });
543
605
  const nativeTransitionAssets = new Map();
544
606
  server.registerTool("connection.status", {
545
607
  description: "Read Framekit's Final Cut connection state before editor-first capability discovery.",
@@ -1184,9 +1246,9 @@ export function createMcpServer(runtime, options = {}) {
1184
1246
  return jsonResult(await runtime.executeEdit(previewToken));
1185
1247
  });
1186
1248
  server.registerTool("speech.analyze", {
1187
- description: "Analyze speech words and filler markers for one media item.",
1188
- inputSchema: { mediaId: z.string().min(1) },
1189
- }, async ({ mediaId }) => jsonResult(await runtime.analyzeSpeech(mediaId)));
1249
+ description: "Analyze speech words, filler markers, and optional VAD for one media item or source range.",
1250
+ inputSchema: { mediaId: z.string().min(1), range: rangeSchema.optional() },
1251
+ }, async ({ mediaId, range }) => jsonResult(await runtime.analyzeSpeech(mediaId, range)));
1190
1252
  server.registerTool("speech.filler.remove.preview", {
1191
1253
  description: "Analyze a selected canonical timeline range and preview removal of high-confidence filler words with safe rational delete ranges.",
1192
1254
  inputSchema: fillerRemovalInputSchema,
@@ -1245,6 +1307,7 @@ export function createMcpServer(runtime, options = {}) {
1245
1307
  description: "Restore the pre-edit snapshot for a completed transaction.",
1246
1308
  inputSchema: { transactionId: z.string().min(1) },
1247
1309
  }, async ({ transactionId }) => jsonResult(await runtime.undo(transactionId)));
1310
+ installMcpSchemaCompatibility(server);
1248
1311
  return server;
1249
1312
  }
1250
1313
  async function resolveNativeTitleAsset(runtime, assetId) {
@@ -1299,44 +1362,49 @@ async function inspectMcpEditor(runtime, options) {
1299
1362
  const native = options.nativeEditor?.capabilities();
1300
1363
  const publishingAvailable = Boolean(options.projectPublisher && (typeof options.projectPublisher.isAvailable !== "function" || options.projectPublisher.isAvailable()));
1301
1364
  const exportAvailable = Boolean(options.videoExporter?.isAvailable());
1365
+ const capabilities = withCapabilityFamilies({
1366
+ ...inspected.capabilities,
1367
+ editor: {
1368
+ ...inspected.capabilities.editor,
1369
+ artifactPublish: publishingAvailable,
1370
+ ...(publishingAvailable ? {} : { timelinePublishNewProject: false }),
1371
+ videoExport: exportAvailable,
1372
+ },
1373
+ }, {
1374
+ backend: inspected.identity.backend,
1375
+ nativeBackend: "final-cut-accessibility",
1376
+ native: {
1377
+ selectionWrite: Boolean(native?.selectionEdit),
1378
+ undo: Boolean(native?.undo),
1379
+ mediaLibrarySearch: Boolean(native?.mediaLibrarySearch),
1380
+ mediaImport: Boolean(native?.mediaImport),
1381
+ mediaSelection: Boolean(native?.mediaSelection),
1382
+ mediaAppendSelected: Boolean(native?.mediaAppendSelected),
1383
+ timelineOccurrenceLocate: Boolean(native?.timelineOccurrenceLocate),
1384
+ bladeAtPlayhead: Boolean(native?.bladeAtPlayhead),
1385
+ deleteRange: Boolean(native?.deleteRange),
1386
+ trimToDuration: Boolean(native?.trimToDuration),
1387
+ mediaAppend: Boolean(native?.mediaAppend),
1388
+ mediaInsert: Boolean(native?.mediaInsert),
1389
+ titlePlacement: Boolean(native?.titlePlacement),
1390
+ transitionDiscovery: Boolean(native?.transitionDiscovery),
1391
+ transitionPlacement: Boolean(native?.transitionPlacement),
1392
+ timelineFocus: Boolean(native?.timelineFocus),
1393
+ projectCreation: false,
1394
+ clipInsertion: false,
1395
+ clipMovement: false,
1396
+ },
1397
+ publishing: publishingAvailable,
1398
+ publishingBackend: "fcpxml-publisher",
1399
+ export: exportAvailable,
1400
+ exportBackend: "final-cut-native-export",
1401
+ });
1302
1402
  return {
1303
1403
  ...inspected,
1304
- capabilities: withCapabilityFamilies({
1305
- ...inspected.capabilities,
1306
- editor: {
1307
- ...inspected.capabilities.editor,
1308
- artifactPublish: publishingAvailable,
1309
- ...(publishingAvailable ? {} : { timelinePublishNewProject: false }),
1310
- videoExport: exportAvailable,
1311
- },
1312
- }, {
1313
- backend: inspected.identity.backend,
1314
- nativeBackend: "final-cut-accessibility",
1315
- native: {
1316
- selectionWrite: Boolean(native?.selectionEdit),
1317
- undo: Boolean(native?.undo),
1318
- mediaLibrarySearch: Boolean(native?.mediaLibrarySearch),
1319
- mediaImport: Boolean(native?.mediaImport),
1320
- mediaSelection: Boolean(native?.mediaSelection),
1321
- mediaAppendSelected: Boolean(native?.mediaAppendSelected),
1322
- timelineOccurrenceLocate: Boolean(native?.timelineOccurrenceLocate),
1323
- bladeAtPlayhead: Boolean(native?.bladeAtPlayhead),
1324
- deleteRange: Boolean(native?.deleteRange),
1325
- trimToDuration: Boolean(native?.trimToDuration),
1326
- mediaAppend: Boolean(native?.mediaAppend),
1327
- mediaInsert: Boolean(native?.mediaInsert),
1328
- titlePlacement: Boolean(native?.titlePlacement),
1329
- transitionDiscovery: Boolean(native?.transitionDiscovery),
1330
- transitionPlacement: Boolean(native?.transitionPlacement),
1331
- timelineFocus: Boolean(native?.timelineFocus),
1332
- projectCreation: false,
1333
- clipInsertion: false,
1334
- clipMovement: false,
1335
- },
1336
- publishing: publishingAvailable,
1337
- publishingBackend: "fcpxml-publisher",
1338
- export: exportAvailable,
1339
- exportBackend: "final-cut-native-export",
1404
+ capabilities,
1405
+ preflight: createCapabilityPreflight(inspected.identity, capabilities, {
1406
+ processMode: options.processMode ?? (native ? "headed" : "headless"),
1407
+ nativeWrite: Boolean(native?.selectionEdit),
1340
1408
  }),
1341
1409
  ...(native ? { native } : {}),
1342
1410
  };
@@ -0,0 +1,34 @@
1
+ import { readFileSync } from "node:fs";
2
+ import { dirname, join } from "node:path";
3
+ import { fileURLToPath } from "node:url";
4
+ const FRAMEKIT_PACKAGE_NAME = "@morshoto/framekit";
5
+ export const FRAMEKIT_VERSION = readFramekitVersion();
6
+ function readFramekitVersion() {
7
+ let directory = dirname(fileURLToPath(import.meta.url));
8
+ while (true) {
9
+ const manifestPath = join(directory, "package.json");
10
+ try {
11
+ const manifest = JSON.parse(readFileSync(manifestPath, "utf8"));
12
+ if (manifest.name === FRAMEKIT_PACKAGE_NAME) {
13
+ if (typeof manifest.version !== "string" || manifest.version.length === 0) {
14
+ throw new Error(`FRAMEKIT_PACKAGE_VERSION_INVALID: ${manifestPath}`);
15
+ }
16
+ return manifest.version;
17
+ }
18
+ }
19
+ catch (error) {
20
+ if (!isMissingFile(error))
21
+ throw error;
22
+ }
23
+ const parent = dirname(directory);
24
+ if (parent === directory)
25
+ break;
26
+ directory = parent;
27
+ }
28
+ throw new Error(`FRAMEKIT_PACKAGE_VERSION_UNAVAILABLE: ${FRAMEKIT_PACKAGE_NAME}`);
29
+ }
30
+ function isMissingFile(error) {
31
+ if (!error || typeof error !== "object" || !("code" in error))
32
+ return false;
33
+ return error.code === "ENOENT";
34
+ }
@@ -1,4 +1,6 @@
1
1
  import { planRoughCut } from "../planning/rough-cut.js";
2
+ import { bindSpeechAnalysis } from "../speech/analysis.js";
3
+ import { parseRational } from "../timeline/rational-time.js";
2
4
  export class MediaAnalysisService {
3
5
  project;
4
6
  context;
@@ -13,7 +15,7 @@ export class MediaAnalysisService {
13
15
  throw new Error("CAPABILITY_UNAVAILABLE: speech analysis");
14
16
  const project = await this.project.inspectProject();
15
17
  const media = findMedia(project, mediaId);
16
- return this.options.speechAnalyzer.analyze({ project, media }, range);
18
+ return this.analyzeSpeechForProject(project, media, range);
17
19
  }
18
20
  async analyzeAudio(mediaId) {
19
21
  if (!this.options.audioAnalyzer)
@@ -119,7 +121,7 @@ export class MediaAnalysisService {
119
121
  const media = findMedia(project, mediaId);
120
122
  const input = { project, media };
121
123
  const [speechResult, audioResult, noiseResult, visualResult, metadataResult] = await Promise.all([
122
- settle(() => this.options.speechAnalyzer?.analyze(input)),
124
+ settle(() => this.options.speechAnalyzer ? this.analyzeSpeechForProject(project, media) : undefined),
123
125
  settle(() => this.options.audioAnalyzer?.analyze(input)),
124
126
  settle(() => this.options.noiseAnalyzer?.analyze(input)),
125
127
  settle(() => this.options.visualAnalyzer?.analyze(input)),
@@ -187,17 +189,18 @@ export class MediaAnalysisService {
187
189
  }))
188
190
  .filter((entry) => matchesMediaIndexQuery(entry, query));
189
191
  }
190
- async reanalyzeAffectedRanges(transaction) {
192
+ async reanalyzeAffectedRanges(transaction, requirements) {
191
193
  const affectedMediaRanges = transaction.diff.affectedRanges.flatMap((range) => transaction.attemptedAfter.timeline.clips.flatMap((clip) => {
192
194
  const intersectionStart = Math.max(range.start, clip.start);
193
195
  const intersectionEnd = Math.min(range.end, clip.start + clip.duration);
194
196
  if (!clip.mediaId || intersectionStart >= intersectionEnd)
195
197
  return [];
198
+ const sourceStart = clip.sourceStart ?? 0;
196
199
  return [{
197
200
  mediaId: clip.mediaId,
198
201
  range: {
199
- start: intersectionStart - clip.start,
200
- end: intersectionEnd - clip.start,
202
+ start: sourceStart + intersectionStart - clip.start,
203
+ end: sourceStart + intersectionEnd - clip.start,
201
204
  },
202
205
  }];
203
206
  }));
@@ -209,25 +212,31 @@ export class MediaAnalysisService {
209
212
  const media = next.media.find((candidate) => candidate.mediaId === mediaId);
210
213
  if (!media)
211
214
  continue;
212
- const ranges = affectedMediaRanges
215
+ const ranges = mergeRanges(affectedMediaRanges
213
216
  .filter((affected) => affected.mediaId === mediaId)
214
- .map((affected) => affected.range);
217
+ .map((affected) => affected.range));
215
218
  const input = { project: next, media };
216
- if (this.options.speechAnalyzer) {
217
- const analyses = await Promise.all(ranges.map((range) => this.options.speechAnalyzer.analyze(input, range)));
218
- media.speech = { words: analyses.flatMap((analysis) => analysis.words) };
219
+ if (requirements.speech && this.options.speechAnalyzer) {
220
+ const analyses = await Promise.all(ranges.map(async (range) => bindSpeechAnalysis(await this.options.speechAnalyzer.analyze(input, range), { input, range, provider: this.options.speechAnalyzer.descriptor })));
221
+ const latest = analyses[analyses.length - 1];
222
+ if (latest) {
223
+ media.speech = mergeSpeechAnalyses(media.speech, analyses, ranges, {
224
+ input,
225
+ provider: this.options.speechAnalyzer.descriptor,
226
+ });
227
+ }
219
228
  }
220
- if (this.options.audioAnalyzer) {
229
+ if (requirements.audio && this.options.audioAnalyzer) {
221
230
  const analyses = await Promise.all(ranges.map((range) => this.options.audioAnalyzer.analyze(input, range)));
222
231
  if (analyses[analyses.length - 1])
223
232
  media.audio = analyses[analyses.length - 1];
224
233
  }
225
- if (this.options.noiseAnalyzer) {
234
+ if (requirements.noise && this.options.noiseAnalyzer) {
226
235
  const analyses = await Promise.all(ranges.map((range) => this.options.noiseAnalyzer.analyze(input, range)));
227
236
  if (analyses[analyses.length - 1])
228
237
  media.noise = analyses[analyses.length - 1];
229
238
  }
230
- if (this.options.visualAnalyzer) {
239
+ if (requirements.visual && this.options.visualAnalyzer) {
231
240
  const analyses = await Promise.all(ranges.map((range) => this.options.visualAnalyzer.analyze(input, range)));
232
241
  media.visual = {
233
242
  scenes: analyses.flatMap((analysis) => analysis.scenes),
@@ -236,7 +245,10 @@ export class MediaAnalysisService {
236
245
  motion: analyses[analyses.length - 1]?.motion,
237
246
  };
238
247
  }
239
- if (this.options.speechAnalyzer || this.options.audioAnalyzer || this.options.noiseAnalyzer || this.options.visualAnalyzer) {
248
+ if ((requirements.speech && this.options.speechAnalyzer)
249
+ || (requirements.audio && this.options.audioAnalyzer)
250
+ || (requirements.noise && this.options.noiseAnalyzer)
251
+ || (requirements.visual && this.options.visualAnalyzer)) {
240
252
  for (const candidate of next.media) {
241
253
  if (candidate.mediaId === mediaId)
242
254
  candidate.analysisRevision = next.revision.id;
@@ -245,6 +257,18 @@ export class MediaAnalysisService {
245
257
  }
246
258
  return next;
247
259
  }
260
+ async analyzeSpeechForProject(project, media, range) {
261
+ const analyzer = this.options.speechAnalyzer;
262
+ if (!analyzer)
263
+ throw new Error("CAPABILITY_UNAVAILABLE: speech analysis");
264
+ const input = { project, media };
265
+ const analysis = await analyzer.analyze(input, range);
266
+ return bindSpeechAnalysis(analysis, {
267
+ input,
268
+ range,
269
+ provider: analyzer.descriptor,
270
+ });
271
+ }
248
272
  }
249
273
  function sourceIdentityOf(media) {
250
274
  return {
@@ -255,6 +279,85 @@ function sourceIdentityOf(media) {
255
279
  ...(media.duration !== undefined ? { duration: media.duration } : {}),
256
280
  };
257
281
  }
282
+ function mergeSpeechAnalyses(previous, analyses, updatedRanges, context) {
283
+ const latest = analyses[analyses.length - 1];
284
+ if (!latest)
285
+ throw new Error("ANALYSIS_INVALID: speech reanalysis returned no results");
286
+ if (previous?.provider && !sameDescriptor(previous.provider, latest.provider)) {
287
+ throw new Error("ANALYSIS_INVALID: speech evidence providers cannot be combined");
288
+ }
289
+ if (previous?.sourceTimebase && !sameRational(previous.sourceTimebase, latest.sourceTimebase)) {
290
+ throw new Error("ANALYSIS_INVALID: speech evidence timebases cannot be combined");
291
+ }
292
+ const requestedRange = encompassingRange([
293
+ ...(previous?.requestedRange ? [previous.requestedRange] : []),
294
+ ...analyses.map((analysis) => analysis.requestedRange),
295
+ ]);
296
+ const observedRange = encompassingRange([
297
+ ...(previous?.observedRange ? [previous.observedRange] : []),
298
+ ...analyses.map((analysis) => analysis.observedRange),
299
+ ]);
300
+ const words = mergeSpeechEvidence(previous?.words, analyses.map((analysis) => analysis.words), updatedRanges);
301
+ const hasVad = previous?.vadSegments !== undefined || analyses.some((analysis) => analysis.vadSegments !== undefined);
302
+ const vadSegments = hasVad
303
+ ? mergeSpeechEvidence(previous?.vadSegments, analyses.map((analysis) => analysis.vadSegments ?? []), updatedRanges)
304
+ : undefined;
305
+ const hasSilence = previous?.silenceSegments !== undefined || analyses.some((analysis) => analysis.silenceSegments !== undefined);
306
+ const silenceSegments = hasSilence
307
+ ? mergeSpeechEvidence(previous?.silenceSegments, analyses.map((analysis) => analysis.silenceSegments ?? []), updatedRanges)
308
+ : undefined;
309
+ const hasProtected = previous?.protectedSegments !== undefined || analyses.some((analysis) => analysis.protectedSegments !== undefined);
310
+ const protectedSegments = hasProtected
311
+ ? mergeSpeechEvidence(previous?.protectedSegments, analyses.map((analysis) => analysis.protectedSegments ?? []), updatedRanges)
312
+ : undefined;
313
+ return bindSpeechAnalysis({
314
+ ...latest,
315
+ requestedRange,
316
+ observedRange,
317
+ capability: vadSegments ? "transcription-plus-vad" : latest.capability,
318
+ words,
319
+ ...(vadSegments ? { vadSegments } : {}),
320
+ ...(silenceSegments ? { silenceSegments } : {}),
321
+ ...(protectedSegments ? { protectedSegments } : {}),
322
+ }, context);
323
+ }
324
+ function mergeSpeechEvidence(previous, fresh, updatedRanges) {
325
+ return [
326
+ ...(previous ?? []).filter((evidence) => !updatedRanges.some((range) => rangesOverlap(evidence, range))),
327
+ ...fresh.flat(),
328
+ ].sort((left, right) => left.start - right.start || left.end - right.end);
329
+ }
330
+ function rangesOverlap(left, right) {
331
+ return left.end > right.start && left.start < right.end;
332
+ }
333
+ function encompassingRange(ranges) {
334
+ if (ranges.length === 0)
335
+ throw new Error("ANALYSIS_INVALID: speech evidence needs a range");
336
+ return {
337
+ start: Math.min(...ranges.map((range) => range.start)),
338
+ end: Math.max(...ranges.map((range) => range.end)),
339
+ };
340
+ }
341
+ function mergeRanges(ranges) {
342
+ const merged = [];
343
+ for (const range of [...ranges].sort((left, right) => left.start - right.start || left.end - right.end)) {
344
+ const previous = merged[merged.length - 1];
345
+ if (!previous || range.start > previous.end) {
346
+ merged.push({ start: range.start, end: range.end });
347
+ continue;
348
+ }
349
+ previous.end = Math.max(previous.end, range.end);
350
+ }
351
+ return merged;
352
+ }
353
+ function sameDescriptor(left, right) {
354
+ return left.id === right.id && left.provider === right.provider && left.version === right.version;
355
+ }
356
+ function sameRational(left, right) {
357
+ const leftParts = parseRational(left, "ANALYSIS_INVALID");
358
+ const rightParts = parseRational(right, "ANALYSIS_INVALID");
359
+ return leftParts.value * rightParts.timescale === rightParts.value * leftParts.timescale;
360
+ }
258
361
  function semanticFromAnalyses(speech, audio, visual, metadata) {
259
362
  return {
260
363
  subjects: [
@@ -88,18 +88,37 @@ export class ProjectService {
88
88
  async inspectEditor() {
89
89
  const identity = await this.adapter.getIdentity();
90
90
  const capabilities = withCapabilityFamilies(await this.adapter.getCapabilities(), { backend: identity.backend });
91
+ const speechAnalyzer = this.options.speechAnalyzer;
92
+ const speechTranscribe = capabilities.analyzers.speechTranscribe
93
+ || (speechAnalyzer ? speechAnalyzer.capabilities?.transcription ?? true : false);
94
+ const speechVad = capabilities.analyzers.speechVad
95
+ || (speechAnalyzer?.capabilities?.vad ?? false);
91
96
  const analyzers = {
92
97
  ...capabilities.analyzers,
93
- speechTranscribe: capabilities.analyzers.speechTranscribe || Boolean(this.options.speechAnalyzer),
98
+ speechTranscribe,
99
+ speechVad,
100
+ speechCapability: speechTranscribe
101
+ ? speechVad ? "transcription-plus-vad" : "transcription-only"
102
+ : "unavailable",
94
103
  audioLoudness: capabilities.analyzers.audioLoudness || Boolean(this.options.audioAnalyzer),
95
104
  audioNoise: capabilities.analyzers.audioNoise || Boolean(this.options.noiseAnalyzer),
96
105
  visualTrack: capabilities.analyzers.visualTrack || Boolean(this.options.visualAnalyzer),
97
106
  metadataDescribe: capabilities.analyzers.metadataDescribe || Boolean(this.options.metadataAnalyzer),
98
107
  };
108
+ const analyzerBackends = {
109
+ speechTranscribe: speechAnalyzer?.descriptor?.provider,
110
+ speechVad: speechAnalyzer?.descriptor?.provider,
111
+ audioLoudness: this.options.audioAnalyzer?.descriptor?.provider,
112
+ audioNoise: this.options.noiseAnalyzer?.descriptor?.provider,
113
+ visualTrack: this.options.visualAnalyzer?.descriptor?.provider,
114
+ };
99
115
  return {
100
116
  identity,
101
117
  capabilities: {
102
- ...withCapabilityFamilies({ ...capabilities, analyzers }, { backend: identity.backend }),
118
+ ...withCapabilityFamilies({ ...capabilities, analyzers }, {
119
+ backend: identity.backend,
120
+ analyzerBackends,
121
+ }),
103
122
  },
104
123
  };
105
124
  }