@morshoto/framekit 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist-package/adapters/final-cut/typescript/src/analyzers.js +29 -2
- package/dist-package/adapters/final-cut/typescript/src/fcpxml.js +16 -5
- package/dist-package/adapters/final-cut/typescript/src/publisher.js +207 -30
- package/dist-package/adapters/final-cut/typescript/src/session.js +69 -8
- package/dist-package/apps/mcp-server/src/main.js +1 -0
- package/dist-package/apps/mcp-server/src/server.js +136 -68
- package/dist-package/apps/mcp-server/src/version.js +34 -0
- package/dist-package/packages/runtime/src/application/media-analysis-service.js +117 -14
- package/dist-package/packages/runtime/src/application/project-service.js +21 -2
- package/dist-package/packages/runtime/src/capabilities.js +85 -10
- package/dist-package/packages/runtime/src/context/context-engine.js +10 -1
- package/dist-package/packages/runtime/src/domain/media.js +1 -0
- package/dist-package/packages/runtime/src/editing/edit-service.js +40 -4
- package/dist-package/packages/runtime/src/editing/filler-removal-service.js +31 -7
- package/dist-package/packages/runtime/src/index.js +2 -0
- package/dist-package/packages/runtime/src/planning/rough-cut.js +9 -3
- package/dist-package/packages/runtime/src/runtime.js +2 -2
- package/dist-package/packages/runtime/src/speech/analysis.js +301 -0
- package/dist-package/packages/runtime/src/speech/mapping.js +227 -0
- package/dist-package/packages/runtime/src/timeline/rational-time.js +5 -0
- package/dist-package/packages/testkit/src/fixture-analyzers.js +1 -0
- package/dist-package/packages/testkit/src/in-memory-editor.js +1 -0
- package/package.json +3 -2
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
2
|
+
import { ListToolsRequestSchema } from "@modelcontextprotocol/sdk/types.js";
|
|
2
3
|
import { z } from "zod";
|
|
3
|
-
import { resolveEditingIntent, withCapabilityFamilies, } from "../../../packages/runtime/src/index.js";
|
|
4
|
+
import { createCapabilityPreflight, resolveEditingIntent, withCapabilityFamilies, } from "../../../packages/runtime/src/index.js";
|
|
4
5
|
import { EDITOR_FIRST_MCP_INSTRUCTIONS, resolveEditingRoute, } from "./routing.js";
|
|
6
|
+
import { FRAMEKIT_VERSION } from "./version.js";
|
|
5
7
|
const revisionValueSchema = z.object({
|
|
6
8
|
id: z.string(),
|
|
7
9
|
sequence: z.number().int().nonnegative(),
|
|
@@ -15,6 +17,9 @@ const rationalTimeSchema = z.object({
|
|
|
15
17
|
const rangeSchema = z.object({
|
|
16
18
|
start: z.number().nonnegative(),
|
|
17
19
|
end: z.number().positive(),
|
|
20
|
+
}).refine((range) => range.end > range.start, {
|
|
21
|
+
message: "end must be greater than start",
|
|
22
|
+
path: ["end"],
|
|
18
23
|
});
|
|
19
24
|
const mediaIndexQuerySchema = z.object({
|
|
20
25
|
query: z.string().optional(),
|
|
@@ -193,31 +198,68 @@ const verificationPolicySchema = z.object({
|
|
|
193
198
|
loudnessToleranceDb: z.number().finite().nonnegative().optional(),
|
|
194
199
|
assertions: z.array(verificationAssertionSchema).optional(),
|
|
195
200
|
}).strict();
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
}).strict()
|
|
212
|
-
|
|
201
|
+
function mcpObjectSchema(schema) {
|
|
202
|
+
// The MCP SDK only serializes schemas it recognizes as object-shaped.
|
|
203
|
+
Object.defineProperty(schema, "shape", { value: {}, enumerable: false });
|
|
204
|
+
return schema;
|
|
205
|
+
}
|
|
206
|
+
function mcpDiscriminatedUnion(options) {
|
|
207
|
+
const schema = z.discriminatedUnion("type", options);
|
|
208
|
+
return mcpObjectSchema(schema);
|
|
209
|
+
}
|
|
210
|
+
function mcpUnion(options) {
|
|
211
|
+
const schema = z.union(options);
|
|
212
|
+
return mcpObjectSchema(schema);
|
|
213
|
+
}
|
|
214
|
+
function createEditToolInputSchema(target = {}, baseRevision = revisionSchema) {
|
|
215
|
+
const options = [
|
|
216
|
+
renameClipSchema.extend({ ...target, verification: verificationPolicySchema.optional(), baseRevision }).strict(),
|
|
217
|
+
trimClipSchema.extend({ ...target, verification: verificationPolicySchema.optional(), baseRevision }).strict(),
|
|
218
|
+
setGainSchema.extend({ ...target, verification: verificationPolicySchema.optional(), baseRevision }).strict(),
|
|
219
|
+
reduceNoiseSchema.extend({ ...target, verification: verificationPolicySchema.optional(), baseRevision }).strict(),
|
|
220
|
+
colorCorrectionSchema.extend({ ...target, verification: verificationPolicySchema.optional(), baseRevision }).strict(),
|
|
221
|
+
rippleDeleteSchema.extend({ ...target, verification: verificationPolicySchema.optional(), baseRevision }).strict(),
|
|
222
|
+
addMarkerSchema.extend({ ...target, verification: verificationPolicySchema.optional(), baseRevision }).strict(),
|
|
223
|
+
];
|
|
224
|
+
return mcpDiscriminatedUnion(options);
|
|
225
|
+
}
|
|
226
|
+
const editToolInputSchema = createEditToolInputSchema();
|
|
227
|
+
const artifactEditToolInputSchema = createEditToolInputSchema({
|
|
213
228
|
artifactPath: z.string().trim().min(1),
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
const editorTimelineEditToolInputSchema = editToolInputSchema.extend({
|
|
229
|
+
}, revisionValueSchema);
|
|
230
|
+
const editorTimelineEditToolInputSchema = createEditToolInputSchema({
|
|
217
231
|
projectId: z.string().trim().min(1),
|
|
218
232
|
sequenceId: z.string().trim().min(1),
|
|
219
|
-
|
|
220
|
-
|
|
233
|
+
}, revisionValueSchema);
|
|
234
|
+
function exposeObjectRoot(schema) {
|
|
235
|
+
if (!isRecord(schema) || !Array.isArray(schema.anyOf))
|
|
236
|
+
return schema;
|
|
237
|
+
const properties = {};
|
|
238
|
+
for (const branch of schema.anyOf) {
|
|
239
|
+
for (const [key, value] of Object.entries(branch.properties ?? {})) {
|
|
240
|
+
if (key !== "type")
|
|
241
|
+
properties[key] = value;
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
return { ...schema, type: "object", properties };
|
|
245
|
+
}
|
|
246
|
+
function installMcpSchemaCompatibility(server) {
|
|
247
|
+
const handlers = server.server._requestHandlers;
|
|
248
|
+
const listToolsHandler = handlers?.get("tools/list");
|
|
249
|
+
if (!listToolsHandler)
|
|
250
|
+
throw new Error("MCP tools/list handler was not initialized");
|
|
251
|
+
server.server.removeRequestHandler("tools/list");
|
|
252
|
+
server.server.setRequestHandler(ListToolsRequestSchema, async (request, extra) => {
|
|
253
|
+
const response = await listToolsHandler(request, extra);
|
|
254
|
+
return {
|
|
255
|
+
...response,
|
|
256
|
+
tools: response.tools?.map((tool) => ({
|
|
257
|
+
...tool,
|
|
258
|
+
inputSchema: exposeObjectRoot(tool.inputSchema),
|
|
259
|
+
})) ?? [],
|
|
260
|
+
};
|
|
261
|
+
});
|
|
262
|
+
}
|
|
221
263
|
const artifactPublishInputSchema = z.object({
|
|
222
264
|
artifactPath: z.string().trim().min(1),
|
|
223
265
|
transactionId: z.string().min(1),
|
|
@@ -334,13 +376,9 @@ const musicDuckingSchema = z.object({
|
|
|
334
376
|
dialogueClipIds: z.array(z.string().min(1)).optional(),
|
|
335
377
|
reductionDb: z.number().finite().optional(),
|
|
336
378
|
});
|
|
337
|
-
const
|
|
379
|
+
const musicAddCommonInputSchema = {
|
|
338
380
|
baseRevision: revisionValueSchema,
|
|
339
381
|
occurrenceId: z.string().min(1),
|
|
340
|
-
mediaId: z.string().min(1).optional(),
|
|
341
|
-
import: musicImportSchema.optional(),
|
|
342
|
-
placement: z.enum(["append", "insert"]),
|
|
343
|
-
start: z.number().nonnegative().optional(),
|
|
344
382
|
duration: z.number().positive().optional(),
|
|
345
383
|
targetLane: z.number().int().refine((lane) => lane !== 0, "music requires a non-primary lane"),
|
|
346
384
|
gainDb: z.number().finite().optional(),
|
|
@@ -349,6 +387,30 @@ const musicAddInputSchema = {
|
|
|
349
387
|
ducking: musicDuckingSchema.optional(),
|
|
350
388
|
verification: verificationPolicySchema.optional(),
|
|
351
389
|
};
|
|
390
|
+
const musicAddInputSchema = mcpUnion([
|
|
391
|
+
z.object({
|
|
392
|
+
...musicAddCommonInputSchema,
|
|
393
|
+
mediaId: z.string().min(1),
|
|
394
|
+
placement: z.literal("append"),
|
|
395
|
+
}).strict(),
|
|
396
|
+
z.object({
|
|
397
|
+
...musicAddCommonInputSchema,
|
|
398
|
+
import: musicImportSchema,
|
|
399
|
+
placement: z.literal("append"),
|
|
400
|
+
}).strict(),
|
|
401
|
+
z.object({
|
|
402
|
+
...musicAddCommonInputSchema,
|
|
403
|
+
mediaId: z.string().min(1),
|
|
404
|
+
placement: z.literal("insert"),
|
|
405
|
+
start: z.number().nonnegative(),
|
|
406
|
+
}).strict(),
|
|
407
|
+
z.object({
|
|
408
|
+
...musicAddCommonInputSchema,
|
|
409
|
+
import: musicImportSchema,
|
|
410
|
+
placement: z.literal("insert"),
|
|
411
|
+
start: z.number().nonnegative(),
|
|
412
|
+
}).strict(),
|
|
413
|
+
]);
|
|
352
414
|
const fillerRemovalInputSchema = {
|
|
353
415
|
baseRevision: revisionValueSchema,
|
|
354
416
|
range: rangeSchema,
|
|
@@ -539,7 +601,7 @@ function frameResult(value) {
|
|
|
539
601
|
}
|
|
540
602
|
export function createMcpServer(runtime, options = {}) {
|
|
541
603
|
runtime.registerBuiltinSkills();
|
|
542
|
-
const server = new McpServer({ name: "framekit", version:
|
|
604
|
+
const server = new McpServer({ name: "framekit", version: FRAMEKIT_VERSION }, { instructions: EDITOR_FIRST_MCP_INSTRUCTIONS });
|
|
543
605
|
const nativeTransitionAssets = new Map();
|
|
544
606
|
server.registerTool("connection.status", {
|
|
545
607
|
description: "Read Framekit's Final Cut connection state before editor-first capability discovery.",
|
|
@@ -1184,9 +1246,9 @@ export function createMcpServer(runtime, options = {}) {
|
|
|
1184
1246
|
return jsonResult(await runtime.executeEdit(previewToken));
|
|
1185
1247
|
});
|
|
1186
1248
|
server.registerTool("speech.analyze", {
|
|
1187
|
-
description: "Analyze speech words
|
|
1188
|
-
inputSchema: { mediaId: z.string().min(1) },
|
|
1189
|
-
}, async ({ mediaId }) => jsonResult(await runtime.analyzeSpeech(mediaId)));
|
|
1249
|
+
description: "Analyze speech words, filler markers, and optional VAD for one media item or source range.",
|
|
1250
|
+
inputSchema: { mediaId: z.string().min(1), range: rangeSchema.optional() },
|
|
1251
|
+
}, async ({ mediaId, range }) => jsonResult(await runtime.analyzeSpeech(mediaId, range)));
|
|
1190
1252
|
server.registerTool("speech.filler.remove.preview", {
|
|
1191
1253
|
description: "Analyze a selected canonical timeline range and preview removal of high-confidence filler words with safe rational delete ranges.",
|
|
1192
1254
|
inputSchema: fillerRemovalInputSchema,
|
|
@@ -1245,6 +1307,7 @@ export function createMcpServer(runtime, options = {}) {
|
|
|
1245
1307
|
description: "Restore the pre-edit snapshot for a completed transaction.",
|
|
1246
1308
|
inputSchema: { transactionId: z.string().min(1) },
|
|
1247
1309
|
}, async ({ transactionId }) => jsonResult(await runtime.undo(transactionId)));
|
|
1310
|
+
installMcpSchemaCompatibility(server);
|
|
1248
1311
|
return server;
|
|
1249
1312
|
}
|
|
1250
1313
|
async function resolveNativeTitleAsset(runtime, assetId) {
|
|
@@ -1299,44 +1362,49 @@ async function inspectMcpEditor(runtime, options) {
|
|
|
1299
1362
|
const native = options.nativeEditor?.capabilities();
|
|
1300
1363
|
const publishingAvailable = Boolean(options.projectPublisher && (typeof options.projectPublisher.isAvailable !== "function" || options.projectPublisher.isAvailable()));
|
|
1301
1364
|
const exportAvailable = Boolean(options.videoExporter?.isAvailable());
|
|
1365
|
+
const capabilities = withCapabilityFamilies({
|
|
1366
|
+
...inspected.capabilities,
|
|
1367
|
+
editor: {
|
|
1368
|
+
...inspected.capabilities.editor,
|
|
1369
|
+
artifactPublish: publishingAvailable,
|
|
1370
|
+
...(publishingAvailable ? {} : { timelinePublishNewProject: false }),
|
|
1371
|
+
videoExport: exportAvailable,
|
|
1372
|
+
},
|
|
1373
|
+
}, {
|
|
1374
|
+
backend: inspected.identity.backend,
|
|
1375
|
+
nativeBackend: "final-cut-accessibility",
|
|
1376
|
+
native: {
|
|
1377
|
+
selectionWrite: Boolean(native?.selectionEdit),
|
|
1378
|
+
undo: Boolean(native?.undo),
|
|
1379
|
+
mediaLibrarySearch: Boolean(native?.mediaLibrarySearch),
|
|
1380
|
+
mediaImport: Boolean(native?.mediaImport),
|
|
1381
|
+
mediaSelection: Boolean(native?.mediaSelection),
|
|
1382
|
+
mediaAppendSelected: Boolean(native?.mediaAppendSelected),
|
|
1383
|
+
timelineOccurrenceLocate: Boolean(native?.timelineOccurrenceLocate),
|
|
1384
|
+
bladeAtPlayhead: Boolean(native?.bladeAtPlayhead),
|
|
1385
|
+
deleteRange: Boolean(native?.deleteRange),
|
|
1386
|
+
trimToDuration: Boolean(native?.trimToDuration),
|
|
1387
|
+
mediaAppend: Boolean(native?.mediaAppend),
|
|
1388
|
+
mediaInsert: Boolean(native?.mediaInsert),
|
|
1389
|
+
titlePlacement: Boolean(native?.titlePlacement),
|
|
1390
|
+
transitionDiscovery: Boolean(native?.transitionDiscovery),
|
|
1391
|
+
transitionPlacement: Boolean(native?.transitionPlacement),
|
|
1392
|
+
timelineFocus: Boolean(native?.timelineFocus),
|
|
1393
|
+
projectCreation: false,
|
|
1394
|
+
clipInsertion: false,
|
|
1395
|
+
clipMovement: false,
|
|
1396
|
+
},
|
|
1397
|
+
publishing: publishingAvailable,
|
|
1398
|
+
publishingBackend: "fcpxml-publisher",
|
|
1399
|
+
export: exportAvailable,
|
|
1400
|
+
exportBackend: "final-cut-native-export",
|
|
1401
|
+
});
|
|
1302
1402
|
return {
|
|
1303
1403
|
...inspected,
|
|
1304
|
-
capabilities
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
artifactPublish: publishingAvailable,
|
|
1309
|
-
...(publishingAvailable ? {} : { timelinePublishNewProject: false }),
|
|
1310
|
-
videoExport: exportAvailable,
|
|
1311
|
-
},
|
|
1312
|
-
}, {
|
|
1313
|
-
backend: inspected.identity.backend,
|
|
1314
|
-
nativeBackend: "final-cut-accessibility",
|
|
1315
|
-
native: {
|
|
1316
|
-
selectionWrite: Boolean(native?.selectionEdit),
|
|
1317
|
-
undo: Boolean(native?.undo),
|
|
1318
|
-
mediaLibrarySearch: Boolean(native?.mediaLibrarySearch),
|
|
1319
|
-
mediaImport: Boolean(native?.mediaImport),
|
|
1320
|
-
mediaSelection: Boolean(native?.mediaSelection),
|
|
1321
|
-
mediaAppendSelected: Boolean(native?.mediaAppendSelected),
|
|
1322
|
-
timelineOccurrenceLocate: Boolean(native?.timelineOccurrenceLocate),
|
|
1323
|
-
bladeAtPlayhead: Boolean(native?.bladeAtPlayhead),
|
|
1324
|
-
deleteRange: Boolean(native?.deleteRange),
|
|
1325
|
-
trimToDuration: Boolean(native?.trimToDuration),
|
|
1326
|
-
mediaAppend: Boolean(native?.mediaAppend),
|
|
1327
|
-
mediaInsert: Boolean(native?.mediaInsert),
|
|
1328
|
-
titlePlacement: Boolean(native?.titlePlacement),
|
|
1329
|
-
transitionDiscovery: Boolean(native?.transitionDiscovery),
|
|
1330
|
-
transitionPlacement: Boolean(native?.transitionPlacement),
|
|
1331
|
-
timelineFocus: Boolean(native?.timelineFocus),
|
|
1332
|
-
projectCreation: false,
|
|
1333
|
-
clipInsertion: false,
|
|
1334
|
-
clipMovement: false,
|
|
1335
|
-
},
|
|
1336
|
-
publishing: publishingAvailable,
|
|
1337
|
-
publishingBackend: "fcpxml-publisher",
|
|
1338
|
-
export: exportAvailable,
|
|
1339
|
-
exportBackend: "final-cut-native-export",
|
|
1404
|
+
capabilities,
|
|
1405
|
+
preflight: createCapabilityPreflight(inspected.identity, capabilities, {
|
|
1406
|
+
processMode: options.processMode ?? (native ? "headed" : "headless"),
|
|
1407
|
+
nativeWrite: Boolean(native?.selectionEdit),
|
|
1340
1408
|
}),
|
|
1341
1409
|
...(native ? { native } : {}),
|
|
1342
1410
|
};
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { dirname, join } from "node:path";
|
|
3
|
+
import { fileURLToPath } from "node:url";
|
|
4
|
+
const FRAMEKIT_PACKAGE_NAME = "@morshoto/framekit";
|
|
5
|
+
export const FRAMEKIT_VERSION = readFramekitVersion();
|
|
6
|
+
function readFramekitVersion() {
|
|
7
|
+
let directory = dirname(fileURLToPath(import.meta.url));
|
|
8
|
+
while (true) {
|
|
9
|
+
const manifestPath = join(directory, "package.json");
|
|
10
|
+
try {
|
|
11
|
+
const manifest = JSON.parse(readFileSync(manifestPath, "utf8"));
|
|
12
|
+
if (manifest.name === FRAMEKIT_PACKAGE_NAME) {
|
|
13
|
+
if (typeof manifest.version !== "string" || manifest.version.length === 0) {
|
|
14
|
+
throw new Error(`FRAMEKIT_PACKAGE_VERSION_INVALID: ${manifestPath}`);
|
|
15
|
+
}
|
|
16
|
+
return manifest.version;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
catch (error) {
|
|
20
|
+
if (!isMissingFile(error))
|
|
21
|
+
throw error;
|
|
22
|
+
}
|
|
23
|
+
const parent = dirname(directory);
|
|
24
|
+
if (parent === directory)
|
|
25
|
+
break;
|
|
26
|
+
directory = parent;
|
|
27
|
+
}
|
|
28
|
+
throw new Error(`FRAMEKIT_PACKAGE_VERSION_UNAVAILABLE: ${FRAMEKIT_PACKAGE_NAME}`);
|
|
29
|
+
}
|
|
30
|
+
function isMissingFile(error) {
|
|
31
|
+
if (!error || typeof error !== "object" || !("code" in error))
|
|
32
|
+
return false;
|
|
33
|
+
return error.code === "ENOENT";
|
|
34
|
+
}
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { planRoughCut } from "../planning/rough-cut.js";
|
|
2
|
+
import { bindSpeechAnalysis } from "../speech/analysis.js";
|
|
3
|
+
import { parseRational } from "../timeline/rational-time.js";
|
|
2
4
|
export class MediaAnalysisService {
|
|
3
5
|
project;
|
|
4
6
|
context;
|
|
@@ -13,7 +15,7 @@ export class MediaAnalysisService {
|
|
|
13
15
|
throw new Error("CAPABILITY_UNAVAILABLE: speech analysis");
|
|
14
16
|
const project = await this.project.inspectProject();
|
|
15
17
|
const media = findMedia(project, mediaId);
|
|
16
|
-
return this.
|
|
18
|
+
return this.analyzeSpeechForProject(project, media, range);
|
|
17
19
|
}
|
|
18
20
|
async analyzeAudio(mediaId) {
|
|
19
21
|
if (!this.options.audioAnalyzer)
|
|
@@ -119,7 +121,7 @@ export class MediaAnalysisService {
|
|
|
119
121
|
const media = findMedia(project, mediaId);
|
|
120
122
|
const input = { project, media };
|
|
121
123
|
const [speechResult, audioResult, noiseResult, visualResult, metadataResult] = await Promise.all([
|
|
122
|
-
settle(() => this.options.speechAnalyzer
|
|
124
|
+
settle(() => this.options.speechAnalyzer ? this.analyzeSpeechForProject(project, media) : undefined),
|
|
123
125
|
settle(() => this.options.audioAnalyzer?.analyze(input)),
|
|
124
126
|
settle(() => this.options.noiseAnalyzer?.analyze(input)),
|
|
125
127
|
settle(() => this.options.visualAnalyzer?.analyze(input)),
|
|
@@ -187,17 +189,18 @@ export class MediaAnalysisService {
|
|
|
187
189
|
}))
|
|
188
190
|
.filter((entry) => matchesMediaIndexQuery(entry, query));
|
|
189
191
|
}
|
|
190
|
-
async reanalyzeAffectedRanges(transaction) {
|
|
192
|
+
async reanalyzeAffectedRanges(transaction, requirements) {
|
|
191
193
|
const affectedMediaRanges = transaction.diff.affectedRanges.flatMap((range) => transaction.attemptedAfter.timeline.clips.flatMap((clip) => {
|
|
192
194
|
const intersectionStart = Math.max(range.start, clip.start);
|
|
193
195
|
const intersectionEnd = Math.min(range.end, clip.start + clip.duration);
|
|
194
196
|
if (!clip.mediaId || intersectionStart >= intersectionEnd)
|
|
195
197
|
return [];
|
|
198
|
+
const sourceStart = clip.sourceStart ?? 0;
|
|
196
199
|
return [{
|
|
197
200
|
mediaId: clip.mediaId,
|
|
198
201
|
range: {
|
|
199
|
-
start: intersectionStart - clip.start,
|
|
200
|
-
end: intersectionEnd - clip.start,
|
|
202
|
+
start: sourceStart + intersectionStart - clip.start,
|
|
203
|
+
end: sourceStart + intersectionEnd - clip.start,
|
|
201
204
|
},
|
|
202
205
|
}];
|
|
203
206
|
}));
|
|
@@ -209,25 +212,31 @@ export class MediaAnalysisService {
|
|
|
209
212
|
const media = next.media.find((candidate) => candidate.mediaId === mediaId);
|
|
210
213
|
if (!media)
|
|
211
214
|
continue;
|
|
212
|
-
const ranges = affectedMediaRanges
|
|
215
|
+
const ranges = mergeRanges(affectedMediaRanges
|
|
213
216
|
.filter((affected) => affected.mediaId === mediaId)
|
|
214
|
-
.map((affected) => affected.range);
|
|
217
|
+
.map((affected) => affected.range));
|
|
215
218
|
const input = { project: next, media };
|
|
216
|
-
if (this.options.speechAnalyzer) {
|
|
217
|
-
const analyses = await Promise.all(ranges.map((range) => this.options.speechAnalyzer.analyze(input, range)));
|
|
218
|
-
|
|
219
|
+
if (requirements.speech && this.options.speechAnalyzer) {
|
|
220
|
+
const analyses = await Promise.all(ranges.map(async (range) => bindSpeechAnalysis(await this.options.speechAnalyzer.analyze(input, range), { input, range, provider: this.options.speechAnalyzer.descriptor })));
|
|
221
|
+
const latest = analyses[analyses.length - 1];
|
|
222
|
+
if (latest) {
|
|
223
|
+
media.speech = mergeSpeechAnalyses(media.speech, analyses, ranges, {
|
|
224
|
+
input,
|
|
225
|
+
provider: this.options.speechAnalyzer.descriptor,
|
|
226
|
+
});
|
|
227
|
+
}
|
|
219
228
|
}
|
|
220
|
-
if (this.options.audioAnalyzer) {
|
|
229
|
+
if (requirements.audio && this.options.audioAnalyzer) {
|
|
221
230
|
const analyses = await Promise.all(ranges.map((range) => this.options.audioAnalyzer.analyze(input, range)));
|
|
222
231
|
if (analyses[analyses.length - 1])
|
|
223
232
|
media.audio = analyses[analyses.length - 1];
|
|
224
233
|
}
|
|
225
|
-
if (this.options.noiseAnalyzer) {
|
|
234
|
+
if (requirements.noise && this.options.noiseAnalyzer) {
|
|
226
235
|
const analyses = await Promise.all(ranges.map((range) => this.options.noiseAnalyzer.analyze(input, range)));
|
|
227
236
|
if (analyses[analyses.length - 1])
|
|
228
237
|
media.noise = analyses[analyses.length - 1];
|
|
229
238
|
}
|
|
230
|
-
if (this.options.visualAnalyzer) {
|
|
239
|
+
if (requirements.visual && this.options.visualAnalyzer) {
|
|
231
240
|
const analyses = await Promise.all(ranges.map((range) => this.options.visualAnalyzer.analyze(input, range)));
|
|
232
241
|
media.visual = {
|
|
233
242
|
scenes: analyses.flatMap((analysis) => analysis.scenes),
|
|
@@ -236,7 +245,10 @@ export class MediaAnalysisService {
|
|
|
236
245
|
motion: analyses[analyses.length - 1]?.motion,
|
|
237
246
|
};
|
|
238
247
|
}
|
|
239
|
-
if (
|
|
248
|
+
if ((requirements.speech && this.options.speechAnalyzer)
|
|
249
|
+
|| (requirements.audio && this.options.audioAnalyzer)
|
|
250
|
+
|| (requirements.noise && this.options.noiseAnalyzer)
|
|
251
|
+
|| (requirements.visual && this.options.visualAnalyzer)) {
|
|
240
252
|
for (const candidate of next.media) {
|
|
241
253
|
if (candidate.mediaId === mediaId)
|
|
242
254
|
candidate.analysisRevision = next.revision.id;
|
|
@@ -245,6 +257,18 @@ export class MediaAnalysisService {
|
|
|
245
257
|
}
|
|
246
258
|
return next;
|
|
247
259
|
}
|
|
260
|
+
async analyzeSpeechForProject(project, media, range) {
|
|
261
|
+
const analyzer = this.options.speechAnalyzer;
|
|
262
|
+
if (!analyzer)
|
|
263
|
+
throw new Error("CAPABILITY_UNAVAILABLE: speech analysis");
|
|
264
|
+
const input = { project, media };
|
|
265
|
+
const analysis = await analyzer.analyze(input, range);
|
|
266
|
+
return bindSpeechAnalysis(analysis, {
|
|
267
|
+
input,
|
|
268
|
+
range,
|
|
269
|
+
provider: analyzer.descriptor,
|
|
270
|
+
});
|
|
271
|
+
}
|
|
248
272
|
}
|
|
249
273
|
function sourceIdentityOf(media) {
|
|
250
274
|
return {
|
|
@@ -255,6 +279,85 @@ function sourceIdentityOf(media) {
|
|
|
255
279
|
...(media.duration !== undefined ? { duration: media.duration } : {}),
|
|
256
280
|
};
|
|
257
281
|
}
|
|
282
|
+
function mergeSpeechAnalyses(previous, analyses, updatedRanges, context) {
|
|
283
|
+
const latest = analyses[analyses.length - 1];
|
|
284
|
+
if (!latest)
|
|
285
|
+
throw new Error("ANALYSIS_INVALID: speech reanalysis returned no results");
|
|
286
|
+
if (previous?.provider && !sameDescriptor(previous.provider, latest.provider)) {
|
|
287
|
+
throw new Error("ANALYSIS_INVALID: speech evidence providers cannot be combined");
|
|
288
|
+
}
|
|
289
|
+
if (previous?.sourceTimebase && !sameRational(previous.sourceTimebase, latest.sourceTimebase)) {
|
|
290
|
+
throw new Error("ANALYSIS_INVALID: speech evidence timebases cannot be combined");
|
|
291
|
+
}
|
|
292
|
+
const requestedRange = encompassingRange([
|
|
293
|
+
...(previous?.requestedRange ? [previous.requestedRange] : []),
|
|
294
|
+
...analyses.map((analysis) => analysis.requestedRange),
|
|
295
|
+
]);
|
|
296
|
+
const observedRange = encompassingRange([
|
|
297
|
+
...(previous?.observedRange ? [previous.observedRange] : []),
|
|
298
|
+
...analyses.map((analysis) => analysis.observedRange),
|
|
299
|
+
]);
|
|
300
|
+
const words = mergeSpeechEvidence(previous?.words, analyses.map((analysis) => analysis.words), updatedRanges);
|
|
301
|
+
const hasVad = previous?.vadSegments !== undefined || analyses.some((analysis) => analysis.vadSegments !== undefined);
|
|
302
|
+
const vadSegments = hasVad
|
|
303
|
+
? mergeSpeechEvidence(previous?.vadSegments, analyses.map((analysis) => analysis.vadSegments ?? []), updatedRanges)
|
|
304
|
+
: undefined;
|
|
305
|
+
const hasSilence = previous?.silenceSegments !== undefined || analyses.some((analysis) => analysis.silenceSegments !== undefined);
|
|
306
|
+
const silenceSegments = hasSilence
|
|
307
|
+
? mergeSpeechEvidence(previous?.silenceSegments, analyses.map((analysis) => analysis.silenceSegments ?? []), updatedRanges)
|
|
308
|
+
: undefined;
|
|
309
|
+
const hasProtected = previous?.protectedSegments !== undefined || analyses.some((analysis) => analysis.protectedSegments !== undefined);
|
|
310
|
+
const protectedSegments = hasProtected
|
|
311
|
+
? mergeSpeechEvidence(previous?.protectedSegments, analyses.map((analysis) => analysis.protectedSegments ?? []), updatedRanges)
|
|
312
|
+
: undefined;
|
|
313
|
+
return bindSpeechAnalysis({
|
|
314
|
+
...latest,
|
|
315
|
+
requestedRange,
|
|
316
|
+
observedRange,
|
|
317
|
+
capability: vadSegments ? "transcription-plus-vad" : latest.capability,
|
|
318
|
+
words,
|
|
319
|
+
...(vadSegments ? { vadSegments } : {}),
|
|
320
|
+
...(silenceSegments ? { silenceSegments } : {}),
|
|
321
|
+
...(protectedSegments ? { protectedSegments } : {}),
|
|
322
|
+
}, context);
|
|
323
|
+
}
|
|
324
|
+
function mergeSpeechEvidence(previous, fresh, updatedRanges) {
|
|
325
|
+
return [
|
|
326
|
+
...(previous ?? []).filter((evidence) => !updatedRanges.some((range) => rangesOverlap(evidence, range))),
|
|
327
|
+
...fresh.flat(),
|
|
328
|
+
].sort((left, right) => left.start - right.start || left.end - right.end);
|
|
329
|
+
}
|
|
330
|
+
function rangesOverlap(left, right) {
|
|
331
|
+
return left.end > right.start && left.start < right.end;
|
|
332
|
+
}
|
|
333
|
+
function encompassingRange(ranges) {
|
|
334
|
+
if (ranges.length === 0)
|
|
335
|
+
throw new Error("ANALYSIS_INVALID: speech evidence needs a range");
|
|
336
|
+
return {
|
|
337
|
+
start: Math.min(...ranges.map((range) => range.start)),
|
|
338
|
+
end: Math.max(...ranges.map((range) => range.end)),
|
|
339
|
+
};
|
|
340
|
+
}
|
|
341
|
+
function mergeRanges(ranges) {
|
|
342
|
+
const merged = [];
|
|
343
|
+
for (const range of [...ranges].sort((left, right) => left.start - right.start || left.end - right.end)) {
|
|
344
|
+
const previous = merged[merged.length - 1];
|
|
345
|
+
if (!previous || range.start > previous.end) {
|
|
346
|
+
merged.push({ start: range.start, end: range.end });
|
|
347
|
+
continue;
|
|
348
|
+
}
|
|
349
|
+
previous.end = Math.max(previous.end, range.end);
|
|
350
|
+
}
|
|
351
|
+
return merged;
|
|
352
|
+
}
|
|
353
|
+
function sameDescriptor(left, right) {
|
|
354
|
+
return left.id === right.id && left.provider === right.provider && left.version === right.version;
|
|
355
|
+
}
|
|
356
|
+
function sameRational(left, right) {
|
|
357
|
+
const leftParts = parseRational(left, "ANALYSIS_INVALID");
|
|
358
|
+
const rightParts = parseRational(right, "ANALYSIS_INVALID");
|
|
359
|
+
return leftParts.value * rightParts.timescale === rightParts.value * leftParts.timescale;
|
|
360
|
+
}
|
|
258
361
|
function semanticFromAnalyses(speech, audio, visual, metadata) {
|
|
259
362
|
return {
|
|
260
363
|
subjects: [
|
|
@@ -88,18 +88,37 @@ export class ProjectService {
|
|
|
88
88
|
async inspectEditor() {
|
|
89
89
|
const identity = await this.adapter.getIdentity();
|
|
90
90
|
const capabilities = withCapabilityFamilies(await this.adapter.getCapabilities(), { backend: identity.backend });
|
|
91
|
+
const speechAnalyzer = this.options.speechAnalyzer;
|
|
92
|
+
const speechTranscribe = capabilities.analyzers.speechTranscribe
|
|
93
|
+
|| (speechAnalyzer ? speechAnalyzer.capabilities?.transcription ?? true : false);
|
|
94
|
+
const speechVad = capabilities.analyzers.speechVad
|
|
95
|
+
|| (speechAnalyzer?.capabilities?.vad ?? false);
|
|
91
96
|
const analyzers = {
|
|
92
97
|
...capabilities.analyzers,
|
|
93
|
-
speechTranscribe
|
|
98
|
+
speechTranscribe,
|
|
99
|
+
speechVad,
|
|
100
|
+
speechCapability: speechTranscribe
|
|
101
|
+
? speechVad ? "transcription-plus-vad" : "transcription-only"
|
|
102
|
+
: "unavailable",
|
|
94
103
|
audioLoudness: capabilities.analyzers.audioLoudness || Boolean(this.options.audioAnalyzer),
|
|
95
104
|
audioNoise: capabilities.analyzers.audioNoise || Boolean(this.options.noiseAnalyzer),
|
|
96
105
|
visualTrack: capabilities.analyzers.visualTrack || Boolean(this.options.visualAnalyzer),
|
|
97
106
|
metadataDescribe: capabilities.analyzers.metadataDescribe || Boolean(this.options.metadataAnalyzer),
|
|
98
107
|
};
|
|
108
|
+
const analyzerBackends = {
|
|
109
|
+
speechTranscribe: speechAnalyzer?.descriptor?.provider,
|
|
110
|
+
speechVad: speechAnalyzer?.descriptor?.provider,
|
|
111
|
+
audioLoudness: this.options.audioAnalyzer?.descriptor?.provider,
|
|
112
|
+
audioNoise: this.options.noiseAnalyzer?.descriptor?.provider,
|
|
113
|
+
visualTrack: this.options.visualAnalyzer?.descriptor?.provider,
|
|
114
|
+
};
|
|
99
115
|
return {
|
|
100
116
|
identity,
|
|
101
117
|
capabilities: {
|
|
102
|
-
...withCapabilityFamilies({ ...capabilities, analyzers }, {
|
|
118
|
+
...withCapabilityFamilies({ ...capabilities, analyzers }, {
|
|
119
|
+
backend: identity.backend,
|
|
120
|
+
analyzerBackends,
|
|
121
|
+
}),
|
|
103
122
|
},
|
|
104
123
|
};
|
|
105
124
|
}
|