@morshoto/framekit 0.1.4 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/README.md +2 -2
  2. package/dist-package/adapters/final-cut/typescript/src/analyzers.js +29 -2
  3. package/dist-package/adapters/final-cut/typescript/src/assets.js +89 -3
  4. package/dist-package/adapters/final-cut/typescript/src/canonical.js +431 -0
  5. package/dist-package/adapters/final-cut/typescript/src/connection.js +19 -4
  6. package/dist-package/adapters/final-cut/typescript/src/fcpxml.js +212 -13
  7. package/dist-package/adapters/final-cut/typescript/src/index.js +1 -0
  8. package/dist-package/adapters/final-cut/typescript/src/native.js +1026 -7
  9. package/dist-package/adapters/final-cut/typescript/src/publisher.js +207 -30
  10. package/dist-package/adapters/final-cut/typescript/src/session.js +69 -8
  11. package/dist-package/apps/mcp-server/src/main.js +49 -11
  12. package/dist-package/apps/mcp-server/src/routing.js +15 -0
  13. package/dist-package/apps/mcp-server/src/server.js +289 -68
  14. package/dist-package/apps/mcp-server/src/version.js +34 -0
  15. package/dist-package/packages/runtime/src/application/media-analysis-service.js +119 -15
  16. package/dist-package/packages/runtime/src/application/project-service.js +21 -2
  17. package/dist-package/packages/runtime/src/audio/dialogue-normalization.js +9 -0
  18. package/dist-package/packages/runtime/src/capabilities.js +90 -10
  19. package/dist-package/packages/runtime/src/context/context-engine.js +10 -1
  20. package/dist-package/packages/runtime/src/domain/editing.js +27 -1
  21. package/dist-package/packages/runtime/src/domain/media.js +1 -0
  22. package/dist-package/packages/runtime/src/domain/skills.js +2 -0
  23. package/dist-package/packages/runtime/src/editing/edit-service.js +82 -4
  24. package/dist-package/packages/runtime/src/editing/filler-removal-service.js +31 -7
  25. package/dist-package/packages/runtime/src/index.js +2 -0
  26. package/dist-package/packages/runtime/src/planning/rough-cut.js +9 -3
  27. package/dist-package/packages/runtime/src/runtime.js +2 -2
  28. package/dist-package/packages/runtime/src/skills/builtin.js +314 -39
  29. package/dist-package/packages/runtime/src/skills/requirements.js +1 -0
  30. package/dist-package/packages/runtime/src/skills/runtime.js +33 -8
  31. package/dist-package/packages/runtime/src/speech/analysis.js +301 -0
  32. package/dist-package/packages/runtime/src/speech/mapping.js +227 -0
  33. package/dist-package/packages/runtime/src/timeline/rational-time.js +5 -0
  34. package/dist-package/packages/runtime/src/verification/verification.js +81 -4
  35. package/dist-package/packages/testkit/src/fixture-analyzers.js +1 -0
  36. package/dist-package/packages/testkit/src/in-memory-editor.js +126 -0
  37. package/package.json +8 -2
@@ -2,7 +2,9 @@ import { randomUUID } from "node:crypto";
2
2
  import { resolve as resolvePath } from "node:path";
3
3
  import { diffSnapshots } from "../timeline/snapshot-diff.js";
4
4
  import { canonicalSnapshotDigest } from "../timeline/snapshot-digest.js";
5
+ import { assertValidMaskConfiguration } from "../domain/editing.js";
5
6
  import { sameRevision } from "../context/revision.js";
7
+ import { parseRational } from "../timeline/rational-time.js";
6
8
  import { assertValidVerificationPolicy } from "../verification/verification.js";
7
9
  export class EditService {
8
10
  adapter;
@@ -47,6 +49,7 @@ export class EditService {
47
49
  if (operation.baseRevision && !sameRevision(operation.baseRevision, before.revision)) {
48
50
  throw new Error("STALE_CONTEXT: operation base revision does not match current editor state");
49
51
  }
52
+ assertValidWorkflowOperation(operation);
50
53
  this.assertSurfaceCapability(target, capabilities.editor);
51
54
  if (!capabilities.editor.timelineSnapshotRead || !capabilities.editor.readAfterWrite || !capabilities.editor.rollback) {
52
55
  throw new Error(`CAPABILITY_UNAVAILABLE: ${target.kind === "artifact" ? "artifact" : "editor timeline"} mutation requires snapshot, read-after-write, and rollback`);
@@ -84,8 +87,7 @@ export class EditService {
84
87
  status: "APPLIED",
85
88
  };
86
89
  try {
87
- transaction.attemptedAfter = await this.analysis.reanalyzeAffectedRanges(transaction);
88
- transaction.after = transaction.attemptedAfter;
90
+ await this.reanalyzeForVerification(transaction, verificationPolicy);
89
91
  }
90
92
  catch (error) {
91
93
  await this.adapter.restore(before, attemptedAfter.revision);
@@ -147,6 +149,7 @@ export class EditService {
147
149
  }
148
150
  if (request.operations.length === 0)
149
151
  throw new Error("INVALID_OPERATION: composite edit requires operations");
152
+ request.operations.forEach(assertValidWorkflowOperation);
150
153
  await this.assertCompositeCapabilities(request.operations, target.kind, capabilities.editor);
151
154
  if (!this.adapter.previewTransaction) {
152
155
  throw new Error("CAPABILITY_UNAVAILABLE: editor composite transaction preview");
@@ -191,6 +194,7 @@ export class EditService {
191
194
  throw new Error("STALE_CONTEXT: preview base revision does not match current editor state");
192
195
  }
193
196
  this.assertTarget(preview.target, before);
197
+ preview.operations.forEach(assertValidWorkflowOperation);
194
198
  await this.assertCompositeCapabilities(preview.operations, preview.target.kind);
195
199
  if (!this.adapter.applyTransaction) {
196
200
  throw new Error("CAPABILITY_UNAVAILABLE: editor composite transaction execution");
@@ -231,9 +235,19 @@ export class EditService {
231
235
  verificationPolicy,
232
236
  status: "APPLIED",
233
237
  };
238
+ const authorizedDiffCheck = sameDiffContent(transaction.diff, preview.expectedDiff)
239
+ ? undefined
240
+ : {
241
+ name: "authorized-diff",
242
+ passed: false,
243
+ status: "failed",
244
+ reason: "UNAUTHORIZED_DIFF",
245
+ expected: structuredClone(preview.expectedDiff),
246
+ observed: structuredClone(transaction.diff),
247
+ detail: "canonical diff contains changes outside the preview-authorized operations",
248
+ };
234
249
  try {
235
- transaction.attemptedAfter = await this.analysis.reanalyzeAffectedRanges(transaction);
236
- transaction.after = transaction.attemptedAfter;
250
+ await this.reanalyzeForVerification(transaction, verificationPolicy);
237
251
  }
238
252
  catch (error) {
239
253
  await this.adapter.restore(before, attemptedAfter.revision);
@@ -256,6 +270,13 @@ export class EditService {
256
270
  }
257
271
  throw new Error(`VERIFICATION_FAILED: canonical state was restored (${String(verificationError)})`);
258
272
  }
273
+ if (authorizedDiffCheck && transaction.verification) {
274
+ transaction.verification = {
275
+ ...transaction.verification,
276
+ passed: false,
277
+ checks: [...transaction.verification.checks, authorizedDiffCheck],
278
+ };
279
+ }
259
280
  if (transaction.verification.passed) {
260
281
  transaction.status = "VERIFIED";
261
282
  }
@@ -277,12 +298,22 @@ export class EditService {
277
298
  async verifyTransaction(transactionId) {
278
299
  return this.getTransaction(transactionId).verification;
279
300
  }
301
+ async reanalyzeForVerification(transaction, policy) {
302
+ const requirements = postWriteAnalysisRequirements(policy);
303
+ if (!Object.values(requirements).some(Boolean))
304
+ return;
305
+ transaction.attemptedAfter = await this.analysis.reanalyzeAffectedRanges(transaction, requirements);
306
+ transaction.after = transaction.attemptedAfter;
307
+ }
280
308
  async undo(transactionId) {
281
309
  const transaction = this.getTransaction(transactionId);
282
310
  const current = await this.project.inspectProject();
283
311
  if (current.projectId !== transaction.before.projectId || current.timeline.id !== transaction.before.timeline.id) {
284
312
  throw new Error(`TARGET_MISMATCH: cannot undo ${transaction.before.projectId}/${transaction.before.timeline.id} while ${current.projectId}/${current.timeline.id} is active`);
285
313
  }
314
+ if (!sameRevision(current.revision, transaction.after.revision)) {
315
+ throw new Error(`STALE_CONTEXT: transaction ${transactionId} changed after edit`);
316
+ }
286
317
  await this.adapter.restore(transaction.before, current.revision);
287
318
  const restored = await this.project.inspectProject();
288
319
  this.assertRestored(transaction.before, restored);
@@ -304,6 +335,9 @@ export class EditService {
304
335
  if (operations.some((operation) => operation.type === "timeline.media.add") && !capabilities.mediaPlacement) {
305
336
  throw new Error("CAPABILITY_UNAVAILABLE: timeline media placement");
306
337
  }
338
+ if (operations.some((operation) => operation.type === "timeline.picture-in-picture.add") && !capabilities.pictureInPicture) {
339
+ throw new Error("CAPABILITY_UNAVAILABLE: picture-in-picture placement");
340
+ }
307
341
  if (operations.some((operation) => operation.type === "timeline.title.add")
308
342
  && (!capabilities.titlePlacement || !capabilities.assetDiscovery)) {
309
343
  throw new Error("CAPABILITY_UNAVAILABLE: timeline title placement");
@@ -332,6 +366,13 @@ export class EditService {
332
366
  && (!capabilities.transitionPlacement || !capabilities.assetDiscovery)) {
333
367
  throw new Error("CAPABILITY_UNAVAILABLE: timeline transition placement");
334
368
  }
369
+ const maskOperations = operations.filter((operation) => operation.type === "timeline.mask.add");
370
+ if (maskOperations.some((operation) => operation.mask.mode !== "person-cutout") && !capabilities.masking) {
371
+ throw new Error("CAPABILITY_UNAVAILABLE: masking");
372
+ }
373
+ if (maskOperations.some((operation) => operation.mask.mode === "person-cutout") && !capabilities.personCutout) {
374
+ throw new Error("CAPABILITY_UNAVAILABLE: person cutout");
375
+ }
335
376
  if (operations.some((operation) => operation.type === "timeline.audio.attach") && !capabilities.audioAttachment) {
336
377
  throw new Error("CAPABILITY_UNAVAILABLE: timeline audio attachment");
337
378
  }
@@ -411,3 +452,40 @@ export class EditService {
411
452
  }
412
453
  }
413
454
  }
455
+ function postWriteAnalysisRequirements(policy) {
456
+ const assertions = policy.assertions ?? [];
457
+ return {
458
+ speech: policy.requireSpeechContinuity === true,
459
+ audio: policy.maxTruePeakDb !== undefined
460
+ || policy.targetLufs !== undefined
461
+ || assertions.some((assertion) => assertion.type === "audio-audibility" || assertion.type === "audio-loudness"),
462
+ noise: assertions.some((assertion) => assertion.type === "audio-noise"),
463
+ visual: assertions.some((assertion) => assertion.type === "visual-content"),
464
+ };
465
+ }
466
+ function sameDiffContent(left, right) {
467
+ const comparable = (diff) => {
468
+ const { from: _from, to: _to, ...content } = diff;
469
+ return content;
470
+ };
471
+ return JSON.stringify(comparable(left)) === JSON.stringify(comparable(right));
472
+ }
473
+ function assertValidWorkflowOperation(operation) {
474
+ if (operation.type === "timeline.mask.add") {
475
+ if (!operation.occurrenceId.trim())
476
+ throw new Error("INVALID_OPERATION: mask occurrenceId is required");
477
+ assertValidMaskConfiguration(operation.mask);
478
+ return;
479
+ }
480
+ if (operation.type !== "trim-clip")
481
+ return;
482
+ if (!Number.isFinite(operation.duration) || operation.duration <= 0) {
483
+ throw new Error("INVALID_OPERATION: clip duration must be positive");
484
+ }
485
+ if (!operation.durationTime)
486
+ return;
487
+ const duration = parseRational(operation.durationTime, "INVALID_OPERATION");
488
+ if (duration.value <= 0n) {
489
+ throw new Error("INVALID_OPERATION: clip duration must be positive");
490
+ }
491
+ }
@@ -1,9 +1,10 @@
1
1
  import { randomUUID } from "node:crypto";
2
2
  import { diffSnapshots } from "../timeline/snapshot-diff.js";
3
3
  import { sameRevision } from "../context/revision.js";
4
- import { translateRationalRange } from "../timeline/rational-time.js";
5
4
  import { canonicalSnapshotDigest } from "../timeline/snapshot-digest.js";
6
5
  import { planFillerRemoval } from "../speech/filler-removal.js";
6
+ import { bindSpeechAnalysis } from "../speech/analysis.js";
7
+ import { mapSourceRangeToSequenceRange } from "../speech/mapping.js";
7
8
  export class FillerRemovalService {
8
9
  adapter;
9
10
  project;
@@ -48,15 +49,20 @@ export class FillerRemovalService {
48
49
  };
49
50
  if (localRange.end <= localRange.start)
50
51
  continue;
51
- const speech = await this.options.speechAnalyzer.analyze({ project: before, media }, localRange);
52
- analysisRangesByClip.set(clip.id, structuredClone(localRange));
53
- for (const candidate of planFillerRemoval(speech.words, localRange, request)) {
52
+ const sourceStart = clip.sourceStart ?? 0;
53
+ const sourceRange = {
54
+ start: sourceStart + localRange.start,
55
+ end: sourceStart + localRange.end,
56
+ };
57
+ const speech = bindSpeechAnalysis(await this.options.speechAnalyzer.analyze({ project: before, media }, sourceRange), { input: { project: before, media }, range: sourceRange, provider: this.options.speechAnalyzer.descriptor });
58
+ analysisRangesByClip.set(clip.id, structuredClone(sourceRange));
59
+ for (const candidate of planFillerRemoval(speech.words, sourceRange, request)) {
54
60
  candidates.push({
55
61
  ...candidate,
56
62
  clipId: clip.id,
57
63
  mediaId: clip.mediaId,
58
64
  sourceRange: structuredClone(candidate.range),
59
- range: translateRationalRange(clip.startTime, clip.start, candidate.range),
65
+ range: translateSourceRangeToSequence(clip, candidate.range),
60
66
  });
61
67
  }
62
68
  }
@@ -205,7 +211,7 @@ export class FillerRemovalService {
205
211
  const fillerCandidates = record.preview.candidates.filter((candidate) => candidate.clipId === clipId);
206
212
  const deletes = fillerCandidates.map((candidate) => candidate.sourceRange);
207
213
  const postEditRange = translateRangeAfterDeletes(analysisRange, deletes);
208
- const speech = await this.options.speechAnalyzer.analyze({ project: next, media }, postEditRange);
214
+ const speech = bindSpeechAnalysis(await this.options.speechAnalyzer.analyze({ project: next, media }, postEditRange), { input: { project: next, media }, range: postEditRange, provider: this.options.speechAnalyzer.descriptor });
209
215
  const retainedWords = transaction.before.media
210
216
  .find((candidate) => candidate.mediaId === beforeClip.mediaId)
211
217
  ?.speech?.words ?? [];
@@ -213,6 +219,7 @@ export class FillerRemovalService {
213
219
  .filter((word) => !fillerCandidates.some((candidate) => sameSpeechWord(word, candidate.word)))
214
220
  .map((word) => translateSpeechWordAfterDeletes(word, deletes));
215
221
  media.speech = {
222
+ ...speech,
216
223
  words: [
217
224
  ...translatedRetainedWords.filter((word) => !rangesOverlap(word.start, word.end, postEditRange.start, postEditRange.end)),
218
225
  ...speech.words.filter((word) => word.start >= postEditRange.start && word.end <= postEditRange.end),
@@ -317,7 +324,7 @@ function verifyFillerSpeechContinuity(transaction, record) {
317
324
  if (expected.text.trim().toLowerCase() !== actual.text.trim().toLowerCase()
318
325
  || Math.abs(expected.start - actual.start) > 0.02
319
326
  || Math.abs(expected.end - actual.end) > 0.02
320
- || actual.end > afterClip.duration + 0.02) {
327
+ || actual.end > (afterClip.sourceStart ?? 0) + afterClip.duration + 0.02) {
321
328
  return {
322
329
  name: "filler-speech-continuity",
323
330
  passed: false,
@@ -335,3 +342,20 @@ function verifyFillerSpeechContinuity(transaction, record) {
335
342
  function sameSpeechWord(left, right) {
336
343
  return left.text === right.text && left.start === right.start && left.end === right.end;
337
344
  }
345
+ function translateSourceRangeToSequence(clip, sourceRange) {
346
+ const sourceStart = clip.sourceStart ?? 0;
347
+ return mapSourceRangeToSequenceRange(sourceRange, {
348
+ sourceRange: {
349
+ start: sourceStart,
350
+ end: sourceStart + clip.duration,
351
+ startTime: clip.sourceStartTime,
352
+ durationTime: clip.durationTime,
353
+ },
354
+ sequenceRange: {
355
+ start: clip.start,
356
+ end: clip.start + clip.duration,
357
+ startTime: clip.startTime,
358
+ durationTime: clip.durationTime,
359
+ },
360
+ });
361
+ }
@@ -15,3 +15,5 @@ export * from "./skills/requirements.js";
15
15
  export * from "./skills/index.js";
16
16
  export * from "./speech/filler-detector.js";
17
17
  export * from "./speech/safe-cut-resolver.js";
18
+ export * from "./speech/analysis.js";
19
+ export * from "./speech/mapping.js";
@@ -5,6 +5,7 @@ export function planRoughCut(entries, revision, request) {
5
5
  throw new Error("INVALID_ROUGH_CUT_REQUEST: maxShots must be a positive integer");
6
6
  }
7
7
  const candidates = entries
8
+ .filter((entry) => entry.sourceIdentity.mediaKind !== "audio")
8
9
  .flatMap((entry) => entry.semantic.usableRanges.map((range) => ({ entry, range })))
9
10
  .sort((left, right) => {
10
11
  const mediaOrder = left.entry.sourceIdentity.mediaId.localeCompare(right.entry.sourceIdentity.mediaId);
@@ -15,9 +16,14 @@ export function planRoughCut(entries, revision, request) {
15
16
  return left.range.end - right.range.end;
16
17
  });
17
18
  const shots = candidates.slice(0, maxShots).map(({ entry, range }, index) => shotFor(entry, range, request, index + 1));
18
- const warnings = entries.length > 0 && candidates.length === 0
19
- ? ["No matching media has an explicitly analyzed usable range"]
20
- : [];
19
+ const excludedAudioMediaIds = entries
20
+ .filter((entry) => entry.sourceIdentity.mediaKind === "audio" && entry.semantic.usableRanges.length > 0)
21
+ .map((entry) => entry.sourceIdentity.mediaId);
22
+ const warnings = excludedAudioMediaIds.length > 0
23
+ ? [`Excluded audio-only media from rough-cut shot candidates: ${excludedAudioMediaIds.join(", ")}`]
24
+ : entries.length > 0 && candidates.length === 0
25
+ ? ["No matching media has an explicitly analyzed usable range"]
26
+ : [];
21
27
  return {
22
28
  planner: { id: "framekit.rough-cut", version: 1 },
23
29
  revision: structuredClone(revision),
@@ -147,8 +147,8 @@ export class AgentVideoRuntime {
147
147
  async contextChangesSince(revision, waitMs = 0) {
148
148
  return this.contexts.contextChangesSince(revision, waitMs);
149
149
  }
150
- async analyzeSpeech(mediaId) {
151
- return this.media.analyzeSpeech(mediaId);
150
+ async analyzeSpeech(mediaId, range) {
151
+ return this.media.analyzeSpeech(mediaId, range);
152
152
  }
153
153
  async analyzeAudio(mediaId) {
154
154
  return this.media.analyzeAudio(mediaId);