@nodetool-ai/agents 0.8.1 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.nodetool-build-stamp +1 -1
- package/dist/analysis/audio-dsp.d.ts +3 -2
- package/dist/analysis/audio-dsp.d.ts.map +1 -1
- package/dist/analysis/audio-dsp.js +11 -8
- package/dist/analysis/audio-dsp.js.map +1 -1
- package/dist/analysis/media-decode.d.ts +13 -0
- package/dist/analysis/media-decode.d.ts.map +1 -1
- package/dist/analysis/media-decode.js +33 -6
- package/dist/analysis/media-decode.js.map +1 -1
- package/dist/apify/assets.d.ts.map +1 -1
- package/dist/apify/assets.js +4 -0
- package/dist/apify/assets.js.map +1 -1
- package/dist/author-graph.d.ts +2 -3
- package/dist/author-graph.d.ts.map +1 -1
- package/dist/author-graph.js +3 -5
- package/dist/author-graph.js.map +1 -1
- package/dist/browser-gate.d.ts +2 -2
- package/dist/browser-gate.js +2 -2
- package/dist/capabilities/GamePublishRequest.d.ts +14 -0
- package/dist/capabilities/GamePublishRequest.d.ts.map +1 -0
- package/dist/capabilities/GamePublishRequest.js +2 -0
- package/dist/capabilities/GamePublishRequest.js.map +1 -0
- package/dist/capabilities/adapters.d.ts +4 -13
- package/dist/capabilities/adapters.d.ts.map +1 -1
- package/dist/capabilities/adapters.js +9 -14
- package/dist/capabilities/adapters.js.map +1 -1
- package/dist/capabilities/agent-activity.d.ts +27 -0
- package/dist/capabilities/agent-activity.d.ts.map +1 -0
- package/dist/capabilities/agent-activity.js +103 -0
- package/dist/capabilities/agent-activity.js.map +1 -0
- package/dist/capabilities/agentic-storyboard-finish.d.ts +67 -0
- package/dist/capabilities/agentic-storyboard-finish.d.ts.map +1 -0
- package/dist/capabilities/agentic-storyboard-finish.js +1321 -0
- package/dist/capabilities/agentic-storyboard-finish.js.map +1 -0
- package/dist/capabilities/agents.d.ts +5 -2
- package/dist/capabilities/agents.d.ts.map +1 -1
- package/dist/capabilities/agents.js +14 -4
- package/dist/capabilities/agents.js.map +1 -1
- package/dist/capabilities/agents.specs.d.ts +3 -0
- package/dist/capabilities/agents.specs.d.ts.map +1 -1
- package/dist/capabilities/agents.specs.js +71 -1
- package/dist/capabilities/agents.specs.js.map +1 -1
- package/dist/capabilities/analysis.d.ts.map +1 -1
- package/dist/capabilities/analysis.js +33 -22
- package/dist/capabilities/analysis.js.map +1 -1
- package/dist/capabilities/analysis.specs.js +2 -2
- package/dist/capabilities/analysis.specs.js.map +1 -1
- package/dist/capabilities/apps.d.ts.map +1 -1
- package/dist/capabilities/apps.js +5 -3
- package/dist/capabilities/apps.js.map +1 -1
- package/dist/capabilities/args.d.ts +2 -5
- package/dist/capabilities/args.d.ts.map +1 -1
- package/dist/capabilities/args.js +2 -5
- package/dist/capabilities/args.js.map +1 -1
- package/dist/capabilities/assets.d.ts.map +1 -1
- package/dist/capabilities/assets.js +40 -10
- package/dist/capabilities/assets.js.map +1 -1
- package/dist/capabilities/browser.d.ts +1 -1
- package/dist/capabilities/browser.js +1 -1
- package/dist/capabilities/browser.specs.d.ts +1 -1
- package/dist/capabilities/browser.specs.js +1 -1
- package/dist/capabilities/code.d.ts +5 -0
- package/dist/capabilities/code.d.ts.map +1 -1
- package/dist/capabilities/code.js +128 -89
- package/dist/capabilities/code.js.map +1 -1
- package/dist/capabilities/collections.d.ts +4 -6
- package/dist/capabilities/collections.d.ts.map +1 -1
- package/dist/capabilities/collections.js +16 -11
- package/dist/capabilities/collections.js.map +1 -1
- package/dist/capabilities/derive-shot.d.ts +45 -0
- package/dist/capabilities/derive-shot.d.ts.map +1 -0
- package/dist/capabilities/derive-shot.js +82 -0
- package/dist/capabilities/derive-shot.js.map +1 -0
- package/dist/capabilities/documents.d.ts +2 -1
- package/dist/capabilities/documents.d.ts.map +1 -1
- package/dist/capabilities/documents.js +12 -6
- package/dist/capabilities/documents.js.map +1 -1
- package/dist/capabilities/email.d.ts +2 -1
- package/dist/capabilities/email.d.ts.map +1 -1
- package/dist/capabilities/email.js +5 -3
- package/dist/capabilities/email.js.map +1 -1
- package/dist/capabilities/errors.d.ts +19 -0
- package/dist/capabilities/errors.d.ts.map +1 -0
- package/dist/capabilities/errors.js +109 -0
- package/dist/capabilities/errors.js.map +1 -0
- package/dist/capabilities/errors.specs.d.ts +11 -0
- package/dist/capabilities/errors.specs.d.ts.map +1 -0
- package/dist/capabilities/errors.specs.js +100 -0
- package/dist/capabilities/errors.specs.js.map +1 -0
- package/dist/capabilities/example-timelines.d.ts.map +1 -1
- package/dist/capabilities/example-timelines.js +52 -10
- package/dist/capabilities/example-timelines.js.map +1 -1
- package/dist/capabilities/example-timelines.specs.d.ts +1 -0
- package/dist/capabilities/example-timelines.specs.d.ts.map +1 -1
- package/dist/capabilities/example-timelines.specs.js +21 -3
- package/dist/capabilities/example-timelines.specs.js.map +1 -1
- package/dist/capabilities/finish-storyboard.d.ts +5 -0
- package/dist/capabilities/finish-storyboard.d.ts.map +1 -0
- package/dist/capabilities/finish-storyboard.js +278 -0
- package/dist/capabilities/finish-storyboard.js.map +1 -0
- package/dist/capabilities/game-asset-browser.d.ts +11 -0
- package/dist/capabilities/game-asset-browser.d.ts.map +1 -0
- package/dist/capabilities/game-asset-browser.js +152 -0
- package/dist/capabilities/game-asset-browser.js.map +1 -0
- package/dist/capabilities/game-asset-source.js +1 -1
- package/dist/capabilities/game-asset-source.js.map +1 -1
- package/dist/capabilities/game.d.ts +44 -2
- package/dist/capabilities/game.d.ts.map +1 -1
- package/dist/capabilities/game.js +408 -160
- package/dist/capabilities/game.js.map +1 -1
- package/dist/capabilities/game.specs.d.ts +4 -0
- package/dist/capabilities/game.specs.d.ts.map +1 -1
- package/dist/capabilities/game.specs.js +42 -13
- package/dist/capabilities/game.specs.js.map +1 -1
- package/dist/capabilities/game3d.d.ts +17 -0
- package/dist/capabilities/game3d.d.ts.map +1 -0
- package/dist/capabilities/game3d.js +434 -0
- package/dist/capabilities/game3d.js.map +1 -0
- package/dist/capabilities/gate-from-context.d.ts +1 -1
- package/dist/capabilities/gate-from-context.js +1 -1
- package/dist/capabilities/generations.d.ts +2 -0
- package/dist/capabilities/generations.d.ts.map +1 -1
- package/dist/capabilities/generations.js +14 -23
- package/dist/capabilities/generations.js.map +1 -1
- package/dist/capabilities/index.d.ts +2 -2
- package/dist/capabilities/index.d.ts.map +1 -1
- package/dist/capabilities/index.js +2 -2
- package/dist/capabilities/index.js.map +1 -1
- package/dist/capabilities/invoke.d.ts +14 -25
- package/dist/capabilities/invoke.d.ts.map +1 -1
- package/dist/capabilities/invoke.js +72 -35
- package/dist/capabilities/invoke.js.map +1 -1
- package/dist/capabilities/jobs.d.ts.map +1 -1
- package/dist/capabilities/jobs.js +47 -3
- package/dist/capabilities/jobs.js.map +1 -1
- package/dist/capabilities/jobs.specs.d.ts.map +1 -1
- package/dist/capabilities/jobs.specs.js +3 -1
- package/dist/capabilities/jobs.specs.js.map +1 -1
- package/dist/capabilities/js-scripts.d.ts +19 -1
- package/dist/capabilities/js-scripts.d.ts.map +1 -1
- package/dist/capabilities/js-scripts.js +109 -2
- package/dist/capabilities/js-scripts.js.map +1 -1
- package/dist/capabilities/js-scripts.specs.d.ts +2 -0
- package/dist/capabilities/js-scripts.specs.d.ts.map +1 -1
- package/dist/capabilities/js-scripts.specs.js +50 -0
- package/dist/capabilities/js-scripts.specs.js.map +1 -1
- package/dist/capabilities/lazy-tool-core.d.ts +23 -0
- package/dist/capabilities/lazy-tool-core.d.ts.map +1 -0
- package/dist/capabilities/lazy-tool-core.js +56 -0
- package/dist/capabilities/lazy-tool-core.js.map +1 -0
- package/dist/capabilities/lazy-tool.d.ts +3 -23
- package/dist/capabilities/lazy-tool.d.ts.map +1 -1
- package/dist/capabilities/lazy-tool.js +6 -64
- package/dist/capabilities/lazy-tool.js.map +1 -1
- package/dist/capabilities/legacy-tools.d.ts +15 -0
- package/dist/capabilities/legacy-tools.d.ts.map +1 -0
- package/dist/capabilities/legacy-tools.js +98 -0
- package/dist/capabilities/legacy-tools.js.map +1 -0
- package/dist/capabilities/media.d.ts +0 -9
- package/dist/capabilities/media.d.ts.map +1 -1
- package/dist/capabilities/media.js +314 -261
- package/dist/capabilities/media.js.map +1 -1
- package/dist/capabilities/media.specs.d.ts.map +1 -1
- package/dist/capabilities/media.specs.js +17 -0
- package/dist/capabilities/media.specs.js.map +1 -1
- package/dist/capabilities/metadata.d.ts +62 -0
- package/dist/capabilities/metadata.d.ts.map +1 -0
- package/dist/capabilities/metadata.js +114 -0
- package/dist/capabilities/metadata.js.map +1 -0
- package/dist/capabilities/model3d.d.ts.map +1 -1
- package/dist/capabilities/model3d.js +4 -0
- package/dist/capabilities/model3d.js.map +1 -1
- package/dist/capabilities/models.d.ts +2 -2
- package/dist/capabilities/models.d.ts.map +1 -1
- package/dist/capabilities/models.js +5 -3
- package/dist/capabilities/models.js.map +1 -1
- package/dist/capabilities/registry.d.ts +6 -19
- package/dist/capabilities/registry.d.ts.map +1 -1
- package/dist/capabilities/registry.js +72 -113
- package/dist/capabilities/registry.js.map +1 -1
- package/dist/capabilities/run-agent.d.ts +19 -0
- package/dist/capabilities/run-agent.d.ts.map +1 -0
- package/dist/capabilities/run-agent.js +183 -0
- package/dist/capabilities/run-agent.js.map +1 -0
- package/dist/capabilities/runs.d.ts +9 -0
- package/dist/capabilities/runs.d.ts.map +1 -0
- package/dist/capabilities/runs.js +46 -0
- package/dist/capabilities/runs.js.map +1 -0
- package/dist/capabilities/runs.specs.d.ts +39 -0
- package/dist/capabilities/runs.specs.d.ts.map +1 -0
- package/dist/capabilities/runs.specs.js +52 -0
- package/dist/capabilities/runs.specs.js.map +1 -0
- package/dist/capabilities/scripts.d.ts.map +1 -1
- package/dist/capabilities/scripts.js +11 -5
- package/dist/capabilities/scripts.js.map +1 -1
- package/dist/capabilities/settings.d.ts.map +1 -1
- package/dist/capabilities/settings.js +4 -2
- package/dist/capabilities/settings.js.map +1 -1
- package/dist/capabilities/sketches.d.ts.map +1 -1
- package/dist/capabilities/sketches.js +4 -2
- package/dist/capabilities/sketches.js.map +1 -1
- package/dist/capabilities/skills.d.ts.map +1 -1
- package/dist/capabilities/skills.js +22 -2
- package/dist/capabilities/skills.js.map +1 -1
- package/dist/capabilities/storyboard-layout-check.d.ts +33 -0
- package/dist/capabilities/storyboard-layout-check.d.ts.map +1 -0
- package/dist/capabilities/storyboard-layout-check.js +229 -0
- package/dist/capabilities/storyboard-layout-check.js.map +1 -0
- package/dist/capabilities/storyboard-measured-sources.d.ts +10 -0
- package/dist/capabilities/storyboard-measured-sources.d.ts.map +1 -0
- package/dist/capabilities/storyboard-measured-sources.js +67 -0
- package/dist/capabilities/storyboard-measured-sources.js.map +1 -0
- package/dist/capabilities/storyboard-review-times.d.ts +4 -0
- package/dist/capabilities/storyboard-review-times.d.ts.map +1 -0
- package/dist/capabilities/storyboard-review-times.js +39 -0
- package/dist/capabilities/storyboard-review-times.js.map +1 -0
- package/dist/capabilities/storyboards.d.ts +12 -18
- package/dist/capabilities/storyboards.d.ts.map +1 -1
- package/dist/capabilities/storyboards.js +779 -96
- package/dist/capabilities/storyboards.js.map +1 -1
- package/dist/capabilities/storyboards.specs.d.ts +9 -0
- package/dist/capabilities/storyboards.specs.d.ts.map +1 -1
- package/dist/capabilities/storyboards.specs.js +173 -5
- package/dist/capabilities/storyboards.specs.js.map +1 -1
- package/dist/capabilities/threads.js +1 -1
- package/dist/capabilities/threads.js.map +1 -1
- package/dist/capabilities/timeline-bridge.d.ts +368 -0
- package/dist/capabilities/timeline-bridge.d.ts.map +1 -0
- package/dist/capabilities/timeline-bridge.js +1008 -0
- package/dist/capabilities/timeline-bridge.js.map +1 -0
- package/dist/capabilities/timeline-clip-frames.d.ts +17 -0
- package/dist/capabilities/timeline-clip-frames.d.ts.map +1 -0
- package/dist/capabilities/timeline-clip-frames.js +84 -0
- package/dist/capabilities/timeline-clip-frames.js.map +1 -0
- package/dist/capabilities/timeline-media-edit.d.ts +8 -0
- package/dist/capabilities/timeline-media-edit.d.ts.map +1 -0
- package/dist/capabilities/timeline-media-edit.js +94 -0
- package/dist/capabilities/timeline-media-edit.js.map +1 -0
- package/dist/capabilities/timeline-native-media.d.ts.map +1 -1
- package/dist/capabilities/timeline-native-media.js +40 -0
- package/dist/capabilities/timeline-native-media.js.map +1 -1
- package/dist/capabilities/timeline-operations.d.ts +52 -0
- package/dist/capabilities/timeline-operations.d.ts.map +1 -0
- package/dist/capabilities/timeline-operations.js +413 -0
- package/dist/capabilities/timeline-operations.js.map +1 -0
- package/dist/capabilities/timelines.d.ts +28 -3
- package/dist/capabilities/timelines.d.ts.map +1 -1
- package/dist/capabilities/timelines.js +410 -294
- package/dist/capabilities/timelines.js.map +1 -1
- package/dist/capabilities/timelines.specs.d.ts +8 -0
- package/dist/capabilities/timelines.specs.d.ts.map +1 -1
- package/dist/capabilities/timelines.specs.js +178 -9
- package/dist/capabilities/timelines.specs.js.map +1 -1
- package/dist/capabilities/types.d.ts +3 -1
- package/dist/capabilities/types.d.ts.map +1 -1
- package/dist/capabilities/types.js.map +1 -1
- package/dist/capabilities/ui.d.ts.map +1 -1
- package/dist/capabilities/ui.js +24 -2
- package/dist/capabilities/ui.js.map +1 -1
- package/dist/capabilities/web.d.ts.map +1 -1
- package/dist/capabilities/web.js +4 -0
- package/dist/capabilities/web.js.map +1 -1
- package/dist/capabilities/workflows.d.ts +3 -1
- package/dist/capabilities/workflows.d.ts.map +1 -1
- package/dist/capabilities/workflows.js +244 -39
- package/dist/capabilities/workflows.js.map +1 -1
- package/dist/capabilities/workflows.specs.d.ts +2 -0
- package/dist/capabilities/workflows.specs.d.ts.map +1 -1
- package/dist/capabilities/workflows.specs.js +72 -2
- package/dist/capabilities/workflows.specs.js.map +1 -1
- package/dist/codeact/action-diagnostics.d.ts.map +1 -1
- package/dist/codeact/action-diagnostics.js +51 -1
- package/dist/codeact/action-diagnostics.js.map +1 -1
- package/dist/codeact/codeact-executor.d.ts.map +1 -1
- package/dist/codeact/codeact-executor.js +16 -8
- package/dist/codeact/codeact-executor.js.map +1 -1
- package/dist/codeact/compact-ids.d.ts.map +1 -1
- package/dist/codeact/compact-ids.js +5 -1
- package/dist/codeact/compact-ids.js.map +1 -1
- package/dist/codeact/graph-model.d.ts +1 -1
- package/dist/codeact/graph-model.d.ts.map +1 -1
- package/dist/codeact/graph-model.js +2 -3
- package/dist/codeact/graph-model.js.map +1 -1
- package/dist/codeact/nodetool-api.d.ts +1 -1
- package/dist/codeact/nodetool-api.d.ts.map +1 -1
- package/dist/codeact/nodetool-api.js +202 -284
- package/dist/codeact/nodetool-api.js.map +1 -1
- package/dist/codeact/sandbox-packages.d.ts +0 -7
- package/dist/codeact/sandbox-packages.d.ts.map +1 -1
- package/dist/codeact/sandbox-packages.js +1 -10
- package/dist/codeact/sandbox-packages.js.map +1 -1
- package/dist/codeact/tool-api.d.ts +6 -3
- package/dist/codeact/tool-api.d.ts.map +1 -1
- package/dist/codeact/tool-api.js +22 -15
- package/dist/codeact/tool-api.js.map +1 -1
- package/dist/evals/codeact-api-cases.d.ts.map +1 -1
- package/dist/evals/codeact-api-cases.js +2 -0
- package/dist/evals/codeact-api-cases.js.map +1 -1
- package/dist/evals/codeact-api-runs.d.ts +3 -0
- package/dist/evals/codeact-api-runs.d.ts.map +1 -0
- package/dist/evals/codeact-api-runs.js +33 -0
- package/dist/evals/codeact-api-runs.js.map +1 -0
- package/dist/evals/surfaces/game.d.ts +8 -2
- package/dist/evals/surfaces/game.d.ts.map +1 -1
- package/dist/evals/surfaces/game.js +460 -2
- package/dist/evals/surfaces/game.js.map +1 -1
- package/dist/evals/surfaces/sketch.d.ts +2 -0
- package/dist/evals/surfaces/sketch.d.ts.map +1 -1
- package/dist/evals/surfaces/sketch.js +11 -3
- package/dist/evals/surfaces/sketch.js.map +1 -1
- package/dist/evals/surfaces/timeline.d.ts +4 -357
- package/dist/evals/surfaces/timeline.d.ts.map +1 -1
- package/dist/evals/surfaces/timeline.js +9 -2621
- package/dist/evals/surfaces/timeline.js.map +1 -1
- package/dist/evals/tool-loop-bridge.d.ts +4 -0
- package/dist/evals/tool-loop-bridge.d.ts.map +1 -1
- package/dist/evals/tool-loop-bridge.js.map +1 -1
- package/dist/evals/tool-loop-eval.d.ts +2 -0
- package/dist/evals/tool-loop-eval.d.ts.map +1 -1
- package/dist/evals/tool-loop-eval.js +6 -1
- package/dist/evals/tool-loop-eval.js.map +1 -1
- package/dist/evals/tool-loop-permission.d.ts +1 -15
- package/dist/evals/tool-loop-permission.d.ts.map +1 -1
- package/dist/evals/tool-loop-permission.js +23 -41
- package/dist/evals/tool-loop-permission.js.map +1 -1
- package/dist/game-authoring.d.ts +9573 -0
- package/dist/game-authoring.d.ts.map +1 -0
- package/dist/game-authoring.js +146 -0
- package/dist/game-authoring.js.map +1 -0
- package/dist/game-build.d.ts +3 -0
- package/dist/game-build.d.ts.map +1 -0
- package/dist/game-build.js +3 -0
- package/dist/game-build.js.map +1 -0
- package/dist/game-code-bake.d.ts +26 -0
- package/dist/game-code-bake.d.ts.map +1 -0
- package/dist/game-code-bake.js +98 -0
- package/dist/game-code-bake.js.map +1 -0
- package/dist/game-examples.d.ts +2227 -18
- package/dist/game-examples.d.ts.map +1 -1
- package/dist/game-examples.js +5 -4
- package/dist/game-examples.js.map +1 -1
- package/dist/generation-cancel.d.ts +31 -0
- package/dist/generation-cancel.d.ts.map +1 -0
- package/dist/generation-cancel.js +28 -0
- package/dist/generation-cancel.js.map +1 -0
- package/dist/host-access-gate.d.ts +29 -0
- package/dist/host-access-gate.d.ts.map +1 -0
- package/dist/host-access-gate.js +39 -0
- package/dist/host-access-gate.js.map +1 -0
- package/dist/host-modules/dispatcher.d.ts +7 -1
- package/dist/host-modules/dispatcher.d.ts.map +1 -1
- package/dist/host-modules/dispatcher.js +13 -2
- package/dist/host-modules/dispatcher.js.map +1 -1
- package/dist/host-modules/docx.d.ts.map +1 -1
- package/dist/host-modules/docx.js +2 -2
- package/dist/host-modules/docx.js.map +1 -1
- package/dist/host-modules/epub.d.ts.map +1 -1
- package/dist/host-modules/epub.js +2 -2
- package/dist/host-modules/epub.js.map +1 -1
- package/dist/host-modules/fabric.d.ts.map +1 -1
- package/dist/host-modules/fabric.js +1 -1
- package/dist/host-modules/fabric.js.map +1 -1
- package/dist/host-modules/limits.d.ts +10 -8
- package/dist/host-modules/limits.d.ts.map +1 -1
- package/dist/host-modules/limits.js +14 -9
- package/dist/host-modules/limits.js.map +1 -1
- package/dist/host-modules/mammoth.d.ts.map +1 -1
- package/dist/host-modules/mammoth.js +2 -2
- package/dist/host-modules/mammoth.js.map +1 -1
- package/dist/host-modules/ocr.d.ts.map +1 -1
- package/dist/host-modules/ocr.js +1 -1
- package/dist/host-modules/ocr.js.map +1 -1
- package/dist/host-modules/pdf.d.ts.map +1 -1
- package/dist/host-modules/pdf.js +2 -2
- package/dist/host-modules/pdf.js.map +1 -1
- package/dist/host-modules/pdflib.d.ts.map +1 -1
- package/dist/host-modules/pdflib.js +2 -2
- package/dist/host-modules/pdflib.js.map +1 -1
- package/dist/host-modules/pptx.d.ts.map +1 -1
- package/dist/host-modules/pptx.js +2 -2
- package/dist/host-modules/pptx.js.map +1 -1
- package/dist/host-modules/pptxgen.d.ts.map +1 -1
- package/dist/host-modules/pptxgen.js +2 -2
- package/dist/host-modules/pptxgen.js.map +1 -1
- package/dist/host-modules/tfjs.d.ts.map +1 -1
- package/dist/host-modules/tfjs.js +4 -4
- package/dist/host-modules/tfjs.js.map +1 -1
- package/dist/host-modules/xlsx.d.ts.map +1 -1
- package/dist/host-modules/xlsx.js +2 -2
- package/dist/host-modules/xlsx.js.map +1 -1
- package/dist/index.d.ts +18 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +17 -3
- package/dist/index.js.map +1 -1
- package/dist/js-sandbox-worker/engine.d.ts +4 -0
- package/dist/js-sandbox-worker/engine.d.ts.map +1 -0
- package/dist/js-sandbox-worker/engine.js +45 -0
- package/dist/js-sandbox-worker/engine.js.map +1 -0
- package/dist/js-sandbox-worker/host.js +1 -1
- package/dist/js-sandbox-worker/host.js.map +1 -1
- package/dist/js-sandbox-worker/interpreter.d.ts +1 -1
- package/dist/js-sandbox-worker/interpreter.d.ts.map +1 -1
- package/dist/js-sandbox-worker/interpreter.js +94 -2
- package/dist/js-sandbox-worker/interpreter.js.map +1 -1
- package/dist/js-sandbox-worker/worker-entry.d.ts.map +1 -1
- package/dist/js-sandbox-worker/worker-entry.js +3 -2
- package/dist/js-sandbox-worker/worker-entry.js.map +1 -1
- package/dist/js-sandbox.d.ts +9 -1
- package/dist/js-sandbox.d.ts.map +1 -1
- package/dist/js-sandbox.js +76 -15
- package/dist/js-sandbox.js.map +1 -1
- package/dist/js-script-app-runner.d.ts +2 -0
- package/dist/js-script-app-runner.d.ts.map +1 -1
- package/dist/js-script-app-runner.js +8 -3
- package/dist/js-script-app-runner.js.map +1 -1
- package/dist/js-script-sandbox.d.ts +12 -18
- package/dist/js-script-sandbox.d.ts.map +1 -1
- package/dist/js-script-sandbox.js +3 -49
- package/dist/js-script-sandbox.js.map +1 -1
- package/dist/prompts/product-knowledge.d.ts +1 -1
- package/dist/prompts/product-knowledge.d.ts.map +1 -1
- package/dist/prompts/product-knowledge.js +82 -41
- package/dist/prompts/product-knowledge.js.map +1 -1
- package/dist/sandbox-pack-resolution.d.ts +21 -0
- package/dist/sandbox-pack-resolution.d.ts.map +1 -0
- package/dist/sandbox-pack-resolution.js +42 -0
- package/dist/sandbox-pack-resolution.js.map +1 -0
- package/dist/sandbox-toolbelt.d.ts +3 -1
- package/dist/sandbox-toolbelt.d.ts.map +1 -1
- package/dist/sandbox-toolbelt.js +8 -2
- package/dist/sandbox-toolbelt.js.map +1 -1
- package/dist/skill-prompt.d.ts +11 -1
- package/dist/skill-prompt.d.ts.map +1 -1
- package/dist/skill-prompt.js +17 -6
- package/dist/skill-prompt.js.map +1 -1
- package/dist/system-skills.d.ts +7 -1
- package/dist/system-skills.d.ts.map +1 -1
- package/dist/system-skills.js +2 -2
- package/dist/system-skills.js.map +1 -1
- package/dist/timeline-capture.d.ts +21 -0
- package/dist/timeline-capture.d.ts.map +1 -0
- package/dist/timeline-capture.js +564 -0
- package/dist/timeline-capture.js.map +1 -0
- package/dist/timeline-code-bake.d.ts +75 -0
- package/dist/timeline-code-bake.d.ts.map +1 -0
- package/dist/timeline-code-bake.js +327 -0
- package/dist/timeline-code-bake.js.map +1 -0
- package/dist/timeline-preview/frames.d.ts +26 -0
- package/dist/timeline-preview/frames.d.ts.map +1 -1
- package/dist/timeline-preview/frames.js +155 -70
- package/dist/timeline-preview/frames.js.map +1 -1
- package/dist/tools/base-tool.d.ts +1 -0
- package/dist/tools/base-tool.d.ts.map +1 -1
- package/dist/tools/base-tool.js +1 -1
- package/dist/tools/base-tool.js.map +1 -1
- package/dist/tools/builtin-tools.d.ts +4 -3
- package/dist/tools/builtin-tools.d.ts.map +1 -1
- package/dist/tools/builtin-tools.js +24 -3
- package/dist/tools/builtin-tools.js.map +1 -1
- package/dist/tools/external-capability-tools.d.ts +1 -1
- package/dist/tools/external-capability-tools.js +1 -1
- package/dist/tools/mcp-tool-support.d.ts +10 -9
- package/dist/tools/mcp-tool-support.d.ts.map +1 -1
- package/dist/tools/mcp-tool-support.js +14 -13
- package/dist/tools/mcp-tool-support.js.map +1 -1
- package/dist/tools/mcp-tools.d.ts +6 -4
- package/dist/tools/mcp-tools.d.ts.map +1 -1
- package/dist/tools/mcp-tools.js +65 -62
- package/dist/tools/mcp-tools.js.map +1 -1
- package/dist/tools/tool-permissions.d.ts +1 -1
- package/dist/tools/tool-permissions.js +1 -1
- package/package.json +124 -41
- package/dist/capabilities/gate-tools.d.ts +0 -18
- package/dist/capabilities/gate-tools.d.ts.map +0 -1
- package/dist/capabilities/gate-tools.js +0 -66
- package/dist/capabilities/gate-tools.js.map +0 -1
- package/dist/tsconfig.tests.tmp.tsbuildinfo +0 -1
|
@@ -0,0 +1,1321 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { loadImage } from "@napi-rs/canvas";
|
|
3
|
+
import { z } from "zod";
|
|
4
|
+
import { zodToJsonSchema, budgetFromContext, isProviderStop } from "@nodetool-ai/runtime";
|
|
5
|
+
import { isRecord, resolveEffectiveProductionRequirement } from "@nodetool-ai/protocol";
|
|
6
|
+
import { buildTimelineToolContracts } from "@nodetool-ai/protocol/api-schemas/timeline-tool-contracts.js";
|
|
7
|
+
import { uiToolParams } from "@nodetool-ai/protocol/api-schemas/ui-tool-contract.js";
|
|
8
|
+
import { validateTimelineSequence } from "@nodetool-ai/execution/timeline-debug";
|
|
9
|
+
import { activeStoryboardGraphics, ANIMATED_PROPERTIES, STAGGER_UNITS, DEFAULT_BEAT_TOLERANCE_MS, buildStoryboardDesignFrame, makeClip, makeTrack, getAnimationPreset, resolvePresetParams, validateProducedTimeline, stampStoryboardMaterializationBaseline } from "@nodetool-ai/timeline";
|
|
10
|
+
import { applyOps, parseOps } from "./timelines.js";
|
|
11
|
+
import { editTimelineSpec } from "./timelines.specs.js";
|
|
12
|
+
import { storyboardReviewTimes } from "./storyboard-review-times.js";
|
|
13
|
+
import { findLayoutDefects, shotHoldTimes } from "./storyboard-layout-check.js";
|
|
14
|
+
import { renderTimelineFrames } from "../timeline-preview/frames.js";
|
|
15
|
+
import { agentActivityReporter } from "./agent-activity.js";
|
|
16
|
+
const frameReviewSchema = z.object({
|
|
17
|
+
timeMs: z.number().finite(),
|
|
18
|
+
passed: z.boolean(),
|
|
19
|
+
findings: z.array(z.string().min(1))
|
|
20
|
+
});
|
|
21
|
+
const reviewSchema = z.object({
|
|
22
|
+
passed: z.boolean(),
|
|
23
|
+
findings: z.array(z.string().min(1)),
|
|
24
|
+
summary: z.string().min(1),
|
|
25
|
+
frameReviews: z.array(frameReviewSchema).min(1)
|
|
26
|
+
});
|
|
27
|
+
/** The words of a review finding that name the defect, without numbers or short words. */
|
|
28
|
+
const defectWords = (finding) => new Set(finding.toLowerCase().match(/[a-z]{4,}/g) ?? []);
|
|
29
|
+
/**
|
|
30
|
+
* Whether two rounds report the same defects. A reviewer rewords a finding
|
|
31
|
+
* each round, so findings match when they share at least half their words.
|
|
32
|
+
*/
|
|
33
|
+
function sameDefects(previous, current) {
|
|
34
|
+
const before = previous.map(defectWords);
|
|
35
|
+
return current.length === previous.length && current.map(defectWords).every((words) => before.some((other) => {
|
|
36
|
+
const shared = [...words].filter((word) => other.has(word)).length;
|
|
37
|
+
const union = new Set([...words, ...other]).size;
|
|
38
|
+
return union > 0 && shared / union >= 0.5;
|
|
39
|
+
}));
|
|
40
|
+
}
|
|
41
|
+
/** Generated decorations one layout pass may add, across the whole cut. */
|
|
42
|
+
const MAX_DECORATIONS = 8;
|
|
43
|
+
function json(value) {
|
|
44
|
+
return JSON.stringify(value);
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* JSON with object keys sorted, the form a materialization baseline is stored
|
|
48
|
+
* in. A saved clip keeps its own key order, so comparing it with a baseline
|
|
49
|
+
* value through plain JSON reports every finished layer as manually edited.
|
|
50
|
+
*/
|
|
51
|
+
function canonical(value) {
|
|
52
|
+
return JSON.stringify(value, (_key, entry) => isRecord(entry)
|
|
53
|
+
? Object.fromEntries(Object.keys(entry)
|
|
54
|
+
.sort()
|
|
55
|
+
.map((key) => [key, entry[key]]))
|
|
56
|
+
: entry);
|
|
57
|
+
}
|
|
58
|
+
/** Placement fields a person may own on a finished layer. */
|
|
59
|
+
const MANUAL_FIELDS = [
|
|
60
|
+
"transform",
|
|
61
|
+
"startMs",
|
|
62
|
+
"durationMs",
|
|
63
|
+
"layout",
|
|
64
|
+
"flexItem",
|
|
65
|
+
"mask",
|
|
66
|
+
"transitionIn"
|
|
67
|
+
];
|
|
68
|
+
/** The placement fields a person changed after the last accepted finish. */
|
|
69
|
+
function manualFields(clip) {
|
|
70
|
+
if (!clip.storyboardMaterializationBaseline) {
|
|
71
|
+
return [];
|
|
72
|
+
}
|
|
73
|
+
const baseline = JSON.parse(clip.storyboardMaterializationBaseline);
|
|
74
|
+
if (!isRecord(baseline)) {
|
|
75
|
+
return [];
|
|
76
|
+
}
|
|
77
|
+
return MANUAL_FIELDS.filter((field) => canonical(clip[field]) !== canonical(baseline[field]));
|
|
78
|
+
}
|
|
79
|
+
/** Compare authored composition, excluding ownership baseline and animation IDs. */
|
|
80
|
+
function composition(document) {
|
|
81
|
+
const trackIndices = new Map(document.tracks.map((track) => [track.id, track.index]));
|
|
82
|
+
return json(document.clips
|
|
83
|
+
.map((clip) => ({
|
|
84
|
+
id: clip.id,
|
|
85
|
+
trackIndex: trackIndices.get(clip.trackId),
|
|
86
|
+
mediaType: clip.mediaType,
|
|
87
|
+
transform: clip.transform,
|
|
88
|
+
layout: clip.layout,
|
|
89
|
+
flexItem: clip.flexItem,
|
|
90
|
+
mask: clip.mask,
|
|
91
|
+
transitionIn: clip.transitionIn,
|
|
92
|
+
textStyle: clip.textStyle,
|
|
93
|
+
shapeStyle: clip.shapeStyle,
|
|
94
|
+
opacity: clip.opacity ?? 1,
|
|
95
|
+
hidden: clip.hidden ?? false,
|
|
96
|
+
blendMode: clip.blendMode ?? "normal",
|
|
97
|
+
parentId: clip.parentId,
|
|
98
|
+
matte: clip.matte,
|
|
99
|
+
crop: clip.crop,
|
|
100
|
+
effects: clip.effects,
|
|
101
|
+
animations: (clip.animations ?? []).map(({ id: _id, ...animation }) => animation)
|
|
102
|
+
}))
|
|
103
|
+
.sort((a, b) => a.id.localeCompare(b.id)));
|
|
104
|
+
}
|
|
105
|
+
function sourceWindowConflict(before, after, boardId) {
|
|
106
|
+
if (before?.storyboardBoardId === boardId &&
|
|
107
|
+
(before.startMs !== after.startMs || before.durationMs !== after.durationMs)) {
|
|
108
|
+
return `Finishing must preserve the deterministic shot window on ${before.id}. Keep startMs=${before.startMs} and durationMs=${before.durationMs}; use animation delay/duration for motion within that window.`;
|
|
109
|
+
}
|
|
110
|
+
return undefined;
|
|
111
|
+
}
|
|
112
|
+
function authoredPlacement(element) {
|
|
113
|
+
if (!isRecord(element)) {
|
|
114
|
+
return undefined;
|
|
115
|
+
}
|
|
116
|
+
const lock = Array.isArray(element["lock"])
|
|
117
|
+
? element["lock"].filter((value) => typeof value === "string")
|
|
118
|
+
: [];
|
|
119
|
+
const limits = {};
|
|
120
|
+
const rawLimits = element["limits"];
|
|
121
|
+
if (isRecord(rawLimits)) {
|
|
122
|
+
for (const key of ["x", "y", "scale"]) {
|
|
123
|
+
const value = rawLimits[key];
|
|
124
|
+
if (typeof value === "number" && Number.isFinite(value)) {
|
|
125
|
+
limits[key] = value;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
const { frame, typography } = element;
|
|
130
|
+
if (frame === undefined &&
|
|
131
|
+
typography === undefined &&
|
|
132
|
+
!lock.length &&
|
|
133
|
+
!Object.keys(limits).length) {
|
|
134
|
+
return undefined;
|
|
135
|
+
}
|
|
136
|
+
return { frame, typography, lock, limits };
|
|
137
|
+
}
|
|
138
|
+
function frameBox(placement) {
|
|
139
|
+
const frame = placement.frame;
|
|
140
|
+
const box = isRecord(frame) ? frame["box"] : undefined;
|
|
141
|
+
return Array.isArray(box) && box.every((value) => typeof value === "number")
|
|
142
|
+
? box
|
|
143
|
+
: undefined;
|
|
144
|
+
}
|
|
145
|
+
/** Properties a moving animation drives, by the placement property it breaks. */
|
|
146
|
+
const POSITION_PROPERTIES = new Set(["offsetX", "offsetY", "positionX", "positionY", "anchorX", "anchorY"]);
|
|
147
|
+
const SCALE_PROPERTIES = new Set(["scale", "scaleX", "scaleY"]);
|
|
148
|
+
/** The properties one animation drives, or undefined when they cannot be known. */
|
|
149
|
+
function animatedProperties(animation, canvas) {
|
|
150
|
+
const properties = new Set();
|
|
151
|
+
for (const track of animation.styleTracks ?? []) {
|
|
152
|
+
properties.add(track.target);
|
|
153
|
+
}
|
|
154
|
+
if (animation.preset === "custom") {
|
|
155
|
+
const curves = animation.custom?.curves;
|
|
156
|
+
if (!Array.isArray(curves)) {
|
|
157
|
+
return undefined;
|
|
158
|
+
}
|
|
159
|
+
for (const curve of curves) {
|
|
160
|
+
properties.add(curve.property);
|
|
161
|
+
}
|
|
162
|
+
return properties;
|
|
163
|
+
}
|
|
164
|
+
const preset = getAnimationPreset(animation.preset);
|
|
165
|
+
if (!preset) {
|
|
166
|
+
return undefined;
|
|
167
|
+
}
|
|
168
|
+
try {
|
|
169
|
+
for (const curve of preset.curves(resolvePresetParams(preset, animation.params), canvas, animation.role, animation.durationMs)) {
|
|
170
|
+
properties.add(curve.property);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
catch {
|
|
174
|
+
return undefined;
|
|
175
|
+
}
|
|
176
|
+
return properties;
|
|
177
|
+
}
|
|
178
|
+
const POSITION_EPSILON_PX = 0.5;
|
|
179
|
+
const RATIO_EPSILON = 0.001;
|
|
180
|
+
/**
|
|
181
|
+
* A clip's position and scale in frame space: the parent chain composed into
|
|
182
|
+
* its own transform. Rotation and anchor of a parent are not applied.
|
|
183
|
+
*/
|
|
184
|
+
function worldTransform(clips, clip) {
|
|
185
|
+
const own = clip.transform;
|
|
186
|
+
const local = {
|
|
187
|
+
x: own?.position.x ?? 0,
|
|
188
|
+
y: own?.position.y ?? 0,
|
|
189
|
+
sx: own?.scale.x ?? 1,
|
|
190
|
+
sy: own?.scale.y ?? 1
|
|
191
|
+
};
|
|
192
|
+
const seen = new Set([clip.id]);
|
|
193
|
+
let parent = clip.parentId ? clips.get(clip.parentId) : undefined;
|
|
194
|
+
while (parent && !seen.has(parent.id)) {
|
|
195
|
+
seen.add(parent.id);
|
|
196
|
+
const group = parent.transform;
|
|
197
|
+
const gsx = group?.scale.x ?? 1;
|
|
198
|
+
const gsy = group?.scale.y ?? 1;
|
|
199
|
+
local.x = (group?.position.x ?? 0) + gsx * local.x;
|
|
200
|
+
local.y = (group?.position.y ?? 0) + gsy * local.y;
|
|
201
|
+
local.sx *= gsx;
|
|
202
|
+
local.sy *= gsy;
|
|
203
|
+
parent = parent.parentId ? clips.get(parent.parentId) : undefined;
|
|
204
|
+
}
|
|
205
|
+
return local;
|
|
206
|
+
}
|
|
207
|
+
const cropInsets = (clip) => [
|
|
208
|
+
clip.crop?.left ?? 0,
|
|
209
|
+
clip.crop?.top ?? 0,
|
|
210
|
+
clip.crop?.right ?? 0,
|
|
211
|
+
clip.crop?.bottom ?? 0
|
|
212
|
+
];
|
|
213
|
+
/** Existing authored presentation survives while semantic source truth is refreshed. */
|
|
214
|
+
function reuseAcceptedPresentation(input, scaffold) {
|
|
215
|
+
const document = structuredClone(scaffold);
|
|
216
|
+
const accepted = new Map(input.current?.clips.map((clip) => [clip.id, clip]));
|
|
217
|
+
const protectedColors = new Set();
|
|
218
|
+
for (const shot of input.shots) {
|
|
219
|
+
const protections = new Map((resolveEffectiveProductionRequirement(input.production, shot.production)
|
|
220
|
+
?.protected_inputs ?? []).map((protection) => [protection.id, protection]));
|
|
221
|
+
for (const element of activeStoryboardGraphics(shot)) {
|
|
222
|
+
if (element.protected_input_id &&
|
|
223
|
+
protections.get(element.protected_input_id)?.kind === "brand_color") {
|
|
224
|
+
protectedColors.add(`${shot.id}/${element.id}`);
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
for (const clip of document.clips) {
|
|
229
|
+
const prior = accepted.get(clip.id);
|
|
230
|
+
if (!prior ||
|
|
231
|
+
clip.storyboardBoardId !== input.boardId ||
|
|
232
|
+
prior.mediaType !== clip.mediaType ||
|
|
233
|
+
prior.storyboardShotId !== clip.storyboardShotId ||
|
|
234
|
+
prior.storyboardElementId !== clip.storyboardElementId) {
|
|
235
|
+
continue;
|
|
236
|
+
}
|
|
237
|
+
const protectedColor = protectedColors.has(`${clip.storyboardShotId}/${clip.storyboardElementId}`);
|
|
238
|
+
Object.assign(clip, structuredClone({
|
|
239
|
+
name: prior.name,
|
|
240
|
+
transform: prior.transform,
|
|
241
|
+
layout: prior.layout,
|
|
242
|
+
flexItem: prior.flexItem,
|
|
243
|
+
opacity: prior.opacity,
|
|
244
|
+
hidden: prior.hidden,
|
|
245
|
+
blendMode: prior.blendMode,
|
|
246
|
+
parentId: prior.parentId,
|
|
247
|
+
mask: prior.mask,
|
|
248
|
+
matte: prior.matte,
|
|
249
|
+
crop: prior.crop,
|
|
250
|
+
effects: prior.effects,
|
|
251
|
+
animations: prior.animations,
|
|
252
|
+
transitionIn: prior.transitionIn
|
|
253
|
+
}));
|
|
254
|
+
if (clip.textStyle && prior.textStyle) {
|
|
255
|
+
clip.textStyle = {
|
|
256
|
+
...prior.textStyle,
|
|
257
|
+
text: clip.textStyle.text,
|
|
258
|
+
...(protectedColor && {
|
|
259
|
+
color: clip.textStyle.color
|
|
260
|
+
})
|
|
261
|
+
};
|
|
262
|
+
}
|
|
263
|
+
if (clip.shapeStyle && prior.shapeStyle) {
|
|
264
|
+
clip.shapeStyle = {
|
|
265
|
+
...prior.shapeStyle,
|
|
266
|
+
kind: clip.shapeStyle.kind,
|
|
267
|
+
...(protectedColor && {
|
|
268
|
+
fill: clip.shapeStyle.fill
|
|
269
|
+
})
|
|
270
|
+
};
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
return document;
|
|
274
|
+
}
|
|
275
|
+
/** A child authors only an in-memory draft. Existing CAS is the sole write. */
|
|
276
|
+
export async function finishStoryboardAgentically(run, input, board, sequence, scaffold, runtime, options = {}) {
|
|
277
|
+
const initialCost = runtime.provider.getTotalCost();
|
|
278
|
+
const layoutPhase = options.phase === "layout";
|
|
279
|
+
const priorStage = input.current?.storyboardMaterializations?.find((entry) => entry.boardId === input.boardId)?.stage;
|
|
280
|
+
// A cut that holds only the planned layout still needs its motion, so the
|
|
281
|
+
// finish must author it rather than accept the cut as it is.
|
|
282
|
+
const motionPending = !layoutPhase && priorStage === "layout";
|
|
283
|
+
const needsInitialAuthoring = !input.current || motionPending;
|
|
284
|
+
const manualEdits = (input.current?.clips ?? [])
|
|
285
|
+
.map((clip) => ({ clipId: clip.id, name: clip.name, fields: manualFields(clip) }))
|
|
286
|
+
.filter((edit) => edit.fields.length > 0);
|
|
287
|
+
if (input.shots.length === 0 || input.shots.length > 24) {
|
|
288
|
+
throw new Error("Agentic finishing supports 1–24 shots per reviewed cut. Split a larger board before finishing.");
|
|
289
|
+
}
|
|
290
|
+
const signal = run.signal ?? run.context.signal;
|
|
291
|
+
let document = reuseAcceptedPresentation(input, scaffold);
|
|
292
|
+
// What the agent starts from: the scaffold, or the accepted layout it keeps.
|
|
293
|
+
const startComposition = composition(document);
|
|
294
|
+
const preflightPolicy = validateProducedTimeline(input, document);
|
|
295
|
+
const preflightStructure = validateTimelineSequence(document, {
|
|
296
|
+
fps: sequence.fps,
|
|
297
|
+
width: input.width,
|
|
298
|
+
height: input.height
|
|
299
|
+
});
|
|
300
|
+
if (preflightPolicy.length || !preflightStructure.ok) {
|
|
301
|
+
throw new Error(`Accepted presentation cannot honor the current production requirements: ${json({ policy: preflightPolicy, structural: preflightStructure })}`);
|
|
302
|
+
}
|
|
303
|
+
const reviews = [];
|
|
304
|
+
let feedback = [];
|
|
305
|
+
const initial = new Map(scaffold.clips.map((clip) => [clip.id, clip]));
|
|
306
|
+
// Image layers this pass generated, by clip id. They are new source media,
|
|
307
|
+
// so the board gains a matching decorative element when the cut is saved.
|
|
308
|
+
const generated = new Map();
|
|
309
|
+
const windows = input.shots.map((shot) => {
|
|
310
|
+
const clips = scaffold.clips.filter((clip) => clip.storyboardShotId === shot.id);
|
|
311
|
+
const start = Math.min(...clips.map((clip) => clip.startMs));
|
|
312
|
+
const end = Math.max(...clips.map((clip) => clip.startMs + clip.durationMs));
|
|
313
|
+
return { shotId: shot.id, start, duration: end - start };
|
|
314
|
+
});
|
|
315
|
+
// Authored placements by `shot/element`. The scaffold is the baseline for
|
|
316
|
+
// every lock and limit, never the previous candidate.
|
|
317
|
+
const placements = new Map();
|
|
318
|
+
for (const shot of input.shots) {
|
|
319
|
+
for (const element of activeStoryboardGraphics(shot)) {
|
|
320
|
+
const placement = authoredPlacement(element);
|
|
321
|
+
if (placement && (placement.lock.length || Object.keys(placement.limits).length)) {
|
|
322
|
+
placements.set(`${shot.id}/${element.id}`, placement);
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
const startAnimations = new Map(document.clips.map((clip) => [
|
|
327
|
+
clip.id,
|
|
328
|
+
new Set((clip.animations ?? []).map(({ id: _id, ...animation }) => json(animation)))
|
|
329
|
+
]));
|
|
330
|
+
const assertAuthoredPlacements = (candidate) => {
|
|
331
|
+
if (!placements.size) {
|
|
332
|
+
return;
|
|
333
|
+
}
|
|
334
|
+
const baseline = new Map(scaffold.clips.map((clip) => [clip.id, clip]));
|
|
335
|
+
const after = new Map(candidate.clips.map((clip) => [clip.id, clip]));
|
|
336
|
+
const fixed = (value) => String(Math.round(value * 1000) / 1000);
|
|
337
|
+
for (const clip of candidate.clips) {
|
|
338
|
+
const placement = placements.get(`${clip.storyboardShotId}/${clip.storyboardElementId}`);
|
|
339
|
+
const base = baseline.get(clip.id);
|
|
340
|
+
if (!placement || !base) {
|
|
341
|
+
continue;
|
|
342
|
+
}
|
|
343
|
+
const name = `${clip.storyboardShotId}/${clip.storyboardElementId}`;
|
|
344
|
+
const was = worldTransform(baseline, base);
|
|
345
|
+
const now = worldTransform(after, clip);
|
|
346
|
+
const dx = now.x - was.x;
|
|
347
|
+
const dy = now.y - was.y;
|
|
348
|
+
const ratioX = was.sx ? now.sx / was.sx : 1;
|
|
349
|
+
const ratioY = was.sy ? now.sy / was.sy : 1;
|
|
350
|
+
const locked = new Set(placement.lock);
|
|
351
|
+
const fail = (property, detail) => {
|
|
352
|
+
throw new Error(`Authored placement violated on ${name}: ${property} ${detail} Restore the authored value and edit only what the placement allows.`);
|
|
353
|
+
};
|
|
354
|
+
if (locked.has("position")) {
|
|
355
|
+
for (const [axis, delta, authored] of [["x", dx, was.x], ["y", dy, was.y]]) {
|
|
356
|
+
if (Math.abs(delta) > POSITION_EPSILON_PX) {
|
|
357
|
+
fail(`position.${axis}`, `is locked at the authored value ${fixed(authored)}px, but the candidate has ${fixed(authored + delta)}px (limit: no change).`);
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
if (locked.has("scale")) {
|
|
362
|
+
for (const [axis, ratio, authored] of [["x", ratioX, was.sx], ["y", ratioY, was.sy]]) {
|
|
363
|
+
if (Math.abs(ratio - 1) > RATIO_EPSILON) {
|
|
364
|
+
fail(`scale.${axis}`, `is locked at the authored value ${fixed(authored)}, but the candidate has ${fixed(authored * ratio)} (limit: no change).`);
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
if (locked.has("crop")) {
|
|
369
|
+
const authored = cropInsets(base);
|
|
370
|
+
const value = cropInsets(clip);
|
|
371
|
+
const names = ["left", "top", "right", "bottom"];
|
|
372
|
+
value.forEach((inset, index) => {
|
|
373
|
+
if (Math.abs(inset - authored[index]) > RATIO_EPSILON) {
|
|
374
|
+
fail(`crop.${names[index]}`, `is locked at the authored value ${fixed(authored[index])}, but the candidate has ${fixed(inset)} (limit: no change).`);
|
|
375
|
+
}
|
|
376
|
+
});
|
|
377
|
+
}
|
|
378
|
+
const { x: limitX, y: limitY, scale: limitScale } = placement.limits;
|
|
379
|
+
if (limitX !== undefined && Math.abs(dx) / input.width > limitX + 1e-6) {
|
|
380
|
+
fail("position.x", `moved ${fixed(dx)}px from the authored ${fixed(was.x)}px, which is ${fixed(Math.abs(dx) / input.width)} of the frame width. The limit is ${limitX}.`);
|
|
381
|
+
}
|
|
382
|
+
if (limitY !== undefined && Math.abs(dy) / input.height > limitY + 1e-6) {
|
|
383
|
+
fail("position.y", `moved ${fixed(dy)}px from the authored ${fixed(was.y)}px, which is ${fixed(Math.abs(dy) / input.height)} of the frame height. The limit is ${limitY}.`);
|
|
384
|
+
}
|
|
385
|
+
if (limitScale !== undefined) {
|
|
386
|
+
for (const [axis, ratio, authored] of [["x", ratioX, was.sx], ["y", ratioY, was.sy]]) {
|
|
387
|
+
if (Math.abs(ratio - 1) > limitScale + 1e-6) {
|
|
388
|
+
fail(`scale.${axis}`, `changed from the authored ${fixed(authored)} to ${fixed(authored * ratio)}, a ratio change of ${fixed(Math.abs(ratio - 1))}. The limit is ${limitScale}.`);
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
if (!layoutPhase && (locked.has("position") || locked.has("scale"))) {
|
|
393
|
+
for (const animation of clip.animations ?? []) {
|
|
394
|
+
const { id: _id, ...rest } = animation;
|
|
395
|
+
if (startAnimations.get(clip.id)?.has(json(rest))) {
|
|
396
|
+
continue;
|
|
397
|
+
}
|
|
398
|
+
const properties = animatedProperties(animation, { width: input.width, height: input.height });
|
|
399
|
+
const moves = properties
|
|
400
|
+
? [...properties].filter((property) => (locked.has("position") && POSITION_PROPERTIES.has(property)) ||
|
|
401
|
+
(locked.has("scale") && SCALE_PROPERTIES.has(property)))
|
|
402
|
+
: undefined;
|
|
403
|
+
if (!moves || moves.length) {
|
|
404
|
+
fail(`animation ${animation.preset}`, `would ${moves ? `drive ${moves.join(", ")}` : "drive properties that cannot be determined"}, but this placement is locked. A locked layer takes only opacity animations such as fade.`);
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
};
|
|
410
|
+
const assertCandidateOwnership = (candidate) => {
|
|
411
|
+
for (const clip of candidate.clips) {
|
|
412
|
+
const before = initial.get(clip.id);
|
|
413
|
+
if (before) {
|
|
414
|
+
if (clip.storyboardBoardId !== before.storyboardBoardId ||
|
|
415
|
+
clip.storyboardShotId !== before.storyboardShotId ||
|
|
416
|
+
clip.storyboardElementId !== before.storyboardElementId) {
|
|
417
|
+
throw new Error(`Finishing cannot replace semantic ownership on ${clip.id}.`);
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
else {
|
|
421
|
+
const window = windows.find((value) => clip.startMs >= value.start &&
|
|
422
|
+
clip.startMs + clip.durationMs <= value.start + value.duration);
|
|
423
|
+
const decoration = generated.get(clip.id);
|
|
424
|
+
if (decoration) {
|
|
425
|
+
if (window?.shotId !== decoration.shotId ||
|
|
426
|
+
clip.storyboardElementId !== decoration.element.id ||
|
|
427
|
+
clip.currentAssetId !== decoration.element.asset_id) {
|
|
428
|
+
throw new Error(`Finishing cannot move or replace the generated decoration ${clip.id}.`);
|
|
429
|
+
}
|
|
430
|
+
continue;
|
|
431
|
+
}
|
|
432
|
+
if (!window || !["text", "shape", "group"].includes(clip.mediaType)) {
|
|
433
|
+
throw new Error("New finishing layers must be editable text/shapes/groups inside an existing shot window. Add source media to the Storyboard first.");
|
|
434
|
+
}
|
|
435
|
+
if (clip.mediaType === "text") {
|
|
436
|
+
const shot = input.shots.find((value) => value.id === window.shotId);
|
|
437
|
+
const allowedCopy = (shot ? activeStoryboardGraphics(shot) : [])
|
|
438
|
+
.filter((element) => element.kind === "text" && !element.protected_input_id)
|
|
439
|
+
.map((element) => element.text);
|
|
440
|
+
if (!allowedCopy.includes(clip.textStyle?.text)) {
|
|
441
|
+
throw new Error("Additional text must use approved unprotected Storyboard copy. Protected copy already has its exact semantic layer.");
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
const current = input.current?.clips.find((value) => value.id === clip.id);
|
|
446
|
+
if (current && clip.name !== current.name) {
|
|
447
|
+
throw new Error(`Finishing cannot rename existing Timeline layer ${clip.id}. Preserve its current user-owned name.`);
|
|
448
|
+
}
|
|
449
|
+
if (current) {
|
|
450
|
+
for (const field of manualFields(current)) {
|
|
451
|
+
if (canonical(clip[field]) !== canonical(current[field])) {
|
|
452
|
+
throw new Error(`Manual ${field} edit on ${clip.id} conflicts with finishing. Keep the manual value or use a separate Timeline.`);
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
if (json(candidate.markers) !== json(scaffold.markers)) {
|
|
458
|
+
throw new Error("Finishing cannot modify manually owned Timeline markers.");
|
|
459
|
+
}
|
|
460
|
+
for (const track of scaffold.tracks) {
|
|
461
|
+
if (input.current?.tracks.some((value) => value.id === track.id) &&
|
|
462
|
+
json(candidate.tracks.find((value) => value.id === track.id)) !==
|
|
463
|
+
json(track)) {
|
|
464
|
+
throw new Error(`Finishing cannot change an existing track's manually owned configuration: ${track.id}.`);
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
for (const before of scaffold.clips) {
|
|
468
|
+
const after = candidate.clips.find((value) => value.id === before.id);
|
|
469
|
+
if (!after) {
|
|
470
|
+
throw new Error(`Finishing cannot delete source layer ${before.id}.`);
|
|
471
|
+
}
|
|
472
|
+
if (before.storyboardBoardId !== input.boardId &&
|
|
473
|
+
json(before) !== json(after)) {
|
|
474
|
+
throw new Error(`Finishing cannot change manually owned layer ${before.id}.`);
|
|
475
|
+
}
|
|
476
|
+
if (before.storyboardBoardId === input.boardId &&
|
|
477
|
+
(before.mediaType !== after.mediaType ||
|
|
478
|
+
before.currentAssetId !== after.currentAssetId ||
|
|
479
|
+
before.sourceType !== after.sourceType ||
|
|
480
|
+
json(before.versions) !== json(after.versions))) {
|
|
481
|
+
throw new Error(`Finishing cannot replace accepted source media on ${before.id}.`);
|
|
482
|
+
}
|
|
483
|
+
const windowError = sourceWindowConflict(before, after, input.boardId);
|
|
484
|
+
if (windowError) {
|
|
485
|
+
throw new Error(windowError);
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
assertAuthoredPlacements(candidate);
|
|
489
|
+
};
|
|
490
|
+
const permittedOpNames = [
|
|
491
|
+
"get_state",
|
|
492
|
+
"list_animation_presets",
|
|
493
|
+
"add_track",
|
|
494
|
+
"move_track",
|
|
495
|
+
"delete_track",
|
|
496
|
+
"add_text_clip",
|
|
497
|
+
"add_shape_clip",
|
|
498
|
+
"set_clip_params",
|
|
499
|
+
"animate_clip",
|
|
500
|
+
"clear_animations",
|
|
501
|
+
"set_transition",
|
|
502
|
+
"set_parent",
|
|
503
|
+
"set_matte",
|
|
504
|
+
"add_group"
|
|
505
|
+
];
|
|
506
|
+
// A layout pass composes still frames. Motion is the finish pass's work.
|
|
507
|
+
const MOTION_OPS = new Set(["list_animation_presets", "animate_clip", "clear_animations", "set_transition"]);
|
|
508
|
+
const phaseOpNames = permittedOpNames.filter((op) => !layoutPhase || !MOTION_OPS.has(op));
|
|
509
|
+
const permittedOps = new Set(phaseOpNames);
|
|
510
|
+
const contracts = buildTimelineToolContracts({
|
|
511
|
+
staggerUnits: STAGGER_UNITS,
|
|
512
|
+
animatedProperties: ANIMATED_PROPERTIES,
|
|
513
|
+
beatToleranceMs: DEFAULT_BEAT_TOLERANCE_MS
|
|
514
|
+
});
|
|
515
|
+
const operationContracts = Object.fromEntries(phaseOpNames.map((op) => {
|
|
516
|
+
const contract = contracts[`ui_timeline_${op}`];
|
|
517
|
+
return [
|
|
518
|
+
op,
|
|
519
|
+
{
|
|
520
|
+
description: contract.description,
|
|
521
|
+
parameters: zodToJsonSchema(uiToolParams(contract))
|
|
522
|
+
}
|
|
523
|
+
];
|
|
524
|
+
}));
|
|
525
|
+
const scopeInstructions = (layoutPhase
|
|
526
|
+
? "This is the layout pass: compose the settled frame of every shot and author no motion. The finish pass adds motion later and keeps your placement. "
|
|
527
|
+
: "") +
|
|
528
|
+
"Only these operations are available: " +
|
|
529
|
+
[...permittedOps].join(", ") +
|
|
530
|
+
". Each op is {op, ...flat arguments} from operationContracts. There is no saved timeline_id requirement for this isolated draft. " +
|
|
531
|
+
"A layer with an authored placement in authoredPlacements keeps it: edit it only within its limits, never change a locked property (position, scale or crop), and expect a rejected edit otherwise. " +
|
|
532
|
+
(layoutPhase ? "" : "A locked layer takes only opacity animations such as fade, never a preset that moves or scales it. ") +
|
|
533
|
+
"Keep existing source layer startMs/durationMs, assets, exact protected text/colors and semantic ownership unchanged. " +
|
|
534
|
+
"set_clip_params uses fontSizePx (not fontSize), textStyle and transform.position.x/y in sequence pixels relative to frame center. " +
|
|
535
|
+
"Protected layers may use only fade (requires allowed opacity), slide (opacity+position) or pop (opacity+scale), each with role in or out. No emphasis or loop animation is available on a protected layer; express emphasis through size, weight, color, spacing or entrance timing. " +
|
|
536
|
+
"Copy must keep clear space from images, other copy and the frame edge at every shot's hold frame. A mechanical layout check measures this before visual review; fix every layoutDefects entry by moving or resizing the named clips. " +
|
|
537
|
+
"Other protected presets/custom curves, group inheritance, masks/effects fail policy validation. set_transition is forbidden on every protected layer, including crossfade; use permitted fade animations instead. Unprotected decorative layers can use existing animation presets/custom curves. " +
|
|
538
|
+
"New editable decorative layers require stable name/startMs/durationMs inside one shot window. New text may only use approved unprotected Storyboard copy; protected copy already has its exact semantic layer, so use set_clip_params on that existing clip to author its presentation. Read every edit result and fix failures before submitting.";
|
|
539
|
+
let previousCandidateImages = [];
|
|
540
|
+
const inspect = {
|
|
541
|
+
name: "get_timeline",
|
|
542
|
+
description: "Read the entire current isolated editable draft, with source and semantic provenance.",
|
|
543
|
+
inputSchema: { type: "object", properties: {}, additionalProperties: false }
|
|
544
|
+
};
|
|
545
|
+
const submit = {
|
|
546
|
+
name: "submit_finished_cut",
|
|
547
|
+
description: "Submit an authored draft for mandatory policy, structure, render and visual review. A new cut requires actual editable composition or motion changes from its deterministic scaffold. Existing reviewed cuts may remain unchanged. No document is saved yet.",
|
|
548
|
+
inputSchema: { type: "object", properties: {}, additionalProperties: false }
|
|
549
|
+
};
|
|
550
|
+
const edit = {
|
|
551
|
+
...editTimelineSpec,
|
|
552
|
+
inputSchema: {
|
|
553
|
+
type: "object",
|
|
554
|
+
properties: {
|
|
555
|
+
ops: {
|
|
556
|
+
type: "array",
|
|
557
|
+
items: { type: "object" },
|
|
558
|
+
description: scopeInstructions
|
|
559
|
+
}
|
|
560
|
+
},
|
|
561
|
+
required: ["ops"],
|
|
562
|
+
additionalProperties: false
|
|
563
|
+
},
|
|
564
|
+
description: "Apply existing Timeline operations to this isolated draft. No save, generation or external write is available. Use exact clip IDs from get_timeline. " +
|
|
565
|
+
scopeInstructions
|
|
566
|
+
};
|
|
567
|
+
const review = {
|
|
568
|
+
name: "review_finished_cut",
|
|
569
|
+
description: "Inspect EVERY actual candidate frame and report one frameReviews entry per supplied candidate timeMs. References are separate, never candidate frames. findings contain only current unresolved material defects, never positive observations, withdrawn suspicions or resolved issues. Each frame passed must equal whether its findings are empty. Overall passed must equal every frame passed; overall findings is the unique union of frame findings. Submit an internally consistent explicit verdict and visual summary. Cannot waive source/policy/structure validation.",
|
|
570
|
+
inputSchema: zodToJsonSchema(reviewSchema)
|
|
571
|
+
};
|
|
572
|
+
const decorate = layoutPhase && options.imageModel
|
|
573
|
+
? {
|
|
574
|
+
name: "generate_decoration",
|
|
575
|
+
description: `Generate one image with the selected image model and add it to one shot as an editable decorative layer. Use it for a background that fills the frame or for a decorative object, never for a product, logo, person, text or a replacement of a source image. layer "background" sits directly above the shot's color background and is scaled to cover the frame. layer "overlay" sits above every layer at half size, centered; place and size it with set_clip_params. Describe the image completely in prompt, with no text, letters or logos. At most ${MAX_DECORATIONS} per cut. Reuse a name to read back the layer it already made.`,
|
|
576
|
+
inputSchema: {
|
|
577
|
+
type: "object",
|
|
578
|
+
properties: {
|
|
579
|
+
shotId: { type: "string", description: "The shot the layer belongs to." },
|
|
580
|
+
name: { type: "string", description: "A short stable name, such as warm-gradient." },
|
|
581
|
+
prompt: { type: "string", description: "The complete image description." },
|
|
582
|
+
layer: { type: "string", enum: ["background", "overlay"] }
|
|
583
|
+
},
|
|
584
|
+
required: ["shotId", "name", "prompt", "layer"],
|
|
585
|
+
additionalProperties: false
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
: undefined;
|
|
589
|
+
const drive = async (label, messages, tools, execute) => {
|
|
590
|
+
signal?.throwIfAborted();
|
|
591
|
+
// The loop runs for minutes. Its text and tool calls go on the run's
|
|
592
|
+
// context, so a host streaming it (a mini app) shows the work.
|
|
593
|
+
const reporter = agentActivityReporter(run.context, label);
|
|
594
|
+
// SDK-native loops can issue parallel MCP callbacks despite sequentialTools.
|
|
595
|
+
// Serialize access to this one isolated draft, including reads and submission.
|
|
596
|
+
let executionTail = Promise.resolve();
|
|
597
|
+
for await (const event of runtime.provider.generateLoop({
|
|
598
|
+
messages,
|
|
599
|
+
model: runtime.model,
|
|
600
|
+
effort: "medium",
|
|
601
|
+
thinking: { type: "disabled" },
|
|
602
|
+
tools,
|
|
603
|
+
providedToolsOnly: true,
|
|
604
|
+
executeTool: (call) => {
|
|
605
|
+
const pending = executionTail.then(async () => {
|
|
606
|
+
signal?.throwIfAborted();
|
|
607
|
+
try {
|
|
608
|
+
const result = await execute(call);
|
|
609
|
+
reporter.toolResult(call, result, false);
|
|
610
|
+
return result;
|
|
611
|
+
}
|
|
612
|
+
catch (error) {
|
|
613
|
+
reporter.toolResult(call, error instanceof Error ? error.message : String(error), true);
|
|
614
|
+
throw error;
|
|
615
|
+
}
|
|
616
|
+
});
|
|
617
|
+
executionTail = pending.then(() => undefined, () => undefined);
|
|
618
|
+
return pending;
|
|
619
|
+
},
|
|
620
|
+
maxIterations: 12,
|
|
621
|
+
sequentialTools: true,
|
|
622
|
+
signal,
|
|
623
|
+
turnBudget: run.budget ?? runtime.budget ?? budgetFromContext(run.context)
|
|
624
|
+
})) {
|
|
625
|
+
signal?.throwIfAborted();
|
|
626
|
+
if (isProviderStop(event)) {
|
|
627
|
+
throw new Error(`Finishing stopped: ${event.reason}`);
|
|
628
|
+
}
|
|
629
|
+
reporter.event(event);
|
|
630
|
+
}
|
|
631
|
+
await executionTail;
|
|
632
|
+
};
|
|
633
|
+
const { current: _current, ...referenceInput } = input;
|
|
634
|
+
const references = input.shots.map((shot) => ({
|
|
635
|
+
shotId: shot.id,
|
|
636
|
+
...buildStoryboardDesignFrame(referenceInput, shot.id, {
|
|
637
|
+
style: board.style,
|
|
638
|
+
context: board.creative_context
|
|
639
|
+
})
|
|
640
|
+
}));
|
|
641
|
+
if (references.some((reference) => reference.validation.length)) {
|
|
642
|
+
throw new Error("The semantic Storyboard cannot produce faithful design reference frames.");
|
|
643
|
+
}
|
|
644
|
+
const { loadMediaRefBytes } = await import("@nodetool-ai/runtime");
|
|
645
|
+
signal?.throwIfAborted();
|
|
646
|
+
const referenceRender = await renderTimelineFrames({
|
|
647
|
+
sequence: { ...sequence.toTimelineSequence(), ...references[0].document },
|
|
648
|
+
timesMs: references.map((reference) => reference.timeMs),
|
|
649
|
+
width: 540,
|
|
650
|
+
loadAsset: (assetId) => {
|
|
651
|
+
signal?.throwIfAborted();
|
|
652
|
+
return loadMediaRefBytes({ asset_id: assetId }, run.context);
|
|
653
|
+
}
|
|
654
|
+
});
|
|
655
|
+
signal?.throwIfAborted();
|
|
656
|
+
if (!referenceRender.complete ||
|
|
657
|
+
referenceRender.effectsNotApplied.length ||
|
|
658
|
+
referenceRender.fontsUnavailable.length) {
|
|
659
|
+
const failures = [
|
|
660
|
+
...referenceRender.fontsUnavailable.map((font) => `Unavailable font: ${font}`),
|
|
661
|
+
...referenceRender.effectsNotApplied.map((effect) => `Unsupported effect: ${effect}`),
|
|
662
|
+
...referenceRender.frames.flatMap((frame) => frame.failures.map((failure) => `${failure.clip_name ?? failure.clip_id ?? "Layer"}: ${failure.reason}`))
|
|
663
|
+
];
|
|
664
|
+
throw new Error(`Derived Storyboard design references could not render completely. ${[...new Set(failures)].join(" ")}`);
|
|
665
|
+
}
|
|
666
|
+
const referenceEvidence = references.map((reference, index) => ({
|
|
667
|
+
shotId: reference.shotId,
|
|
668
|
+
fingerprint: createHash("sha256")
|
|
669
|
+
.update(reference.fingerprint)
|
|
670
|
+
.digest("hex"),
|
|
671
|
+
timeMs: reference.timeMs,
|
|
672
|
+
sha256: createHash("sha256")
|
|
673
|
+
.update(referenceRender.frames[index].png)
|
|
674
|
+
.digest("hex")
|
|
675
|
+
}));
|
|
676
|
+
const loadAsset = (assetId) => {
|
|
677
|
+
signal?.throwIfAborted();
|
|
678
|
+
return loadMediaRefBytes({ asset_id: assetId }, run.context);
|
|
679
|
+
};
|
|
680
|
+
// The reference is derived from the same materializer, so it can carry the
|
|
681
|
+
// same layout defect. Name it so neither agent copies it as approved.
|
|
682
|
+
const referenceLayoutDefects = await findLayoutDefects({
|
|
683
|
+
sequence: { ...sequence.toTimelineSequence(), ...references[0].document },
|
|
684
|
+
windows,
|
|
685
|
+
holds: shotHoldTimes(references[0].document.clips, windows, input.width, input.height),
|
|
686
|
+
loadAsset,
|
|
687
|
+
signal
|
|
688
|
+
});
|
|
689
|
+
const referenceImages = referenceRender.frames.flatMap((frame, index) => [
|
|
690
|
+
{
|
|
691
|
+
type: "text",
|
|
692
|
+
text: `Derived design reference for shot ${references[index].shotId} at ${frame.time_ms}ms, from the expected Storyboard revision's semantic intent. Fingerprint ${referenceEvidence[index].fingerprint}. This is not a separate historical pixel approval.`
|
|
693
|
+
},
|
|
694
|
+
{
|
|
695
|
+
type: "image_url",
|
|
696
|
+
image: {
|
|
697
|
+
uri: `data:image/png;base64,${Buffer.from(frame.png).toString("base64")}`
|
|
698
|
+
}
|
|
699
|
+
}
|
|
700
|
+
]);
|
|
701
|
+
const authoringPrompt = layoutPhase
|
|
702
|
+
? "Lay out the approved WHOLE CUT as still frames: one settled frame per shot, consistent across the cut. The deterministic scaffold stacks every layer in fixed slots by role. Treat it only as the list of what each shot shows, never as a layout to keep. Inspect the design references and the semantic graphic direction, then position and scale every image and copy layer with set_clip_params so each hold frame reads as one deliberate composition: one clear focal image, a readable type hierarchy, balanced empty space, and a grid and margins that repeat from shot to shot. Set copy size, weight, alignment and line width. Scale each image to its role. An image may bleed off a frame edge. authoredPlacements lists the elements that have an authored placement: an element with a frame keeps that placement, you adjust it only within its limits, you never change a property listed in its lock, and you compose freely only the elements without a frame. A placement that breaks a lock or a limit is rejected. You may add editable shapes for panels, rules or frames that hold copy. " +
|
|
703
|
+
(decorate
|
|
704
|
+
? "An image model is selected, so you may use generate_decoration for a background or a decorative object when the direction of a shot calls for one. A generated background carries the theme of its shot with texture, light or a scene, not a flat gradient, and keeps a calm area behind the copy. Keep every product and source image the subject, and keep copy readable on what you generate. "
|
|
705
|
+
: "") +
|
|
706
|
+
"Author no motion: entrances, exits and transitions belong to the finish pass, which keeps your placement. Use the supplied scaffold and operationContracts directly; do not request get_timeline when its data is already present. Preserve all source assets, exact copy/colors, semantic IDs, shot windows and manual edits. manualEdits lists the layer fields a person changed: never edit those fields, and compose around them. On a cut that already has a layout, keep what works and change only what the review or the new content needs. referenceLayoutDefects lists collisions the scaffold itself contains, and the layout must not repeat them. Read every edit result, correct rejected edits, then submit_finished_cut when ready. At a revision, fix every reported defect, including every previousReview.layoutDefects entry."
|
|
707
|
+
: (motionPending
|
|
708
|
+
? "The current cut carries the approved static layout from planning. Keep every layer's position, scale, crop and type style, and author the motion: entrances, emphasis, exits and transitions that follow the motion direction and the shot timing. "
|
|
709
|
+
: "") +
|
|
710
|
+
"Finish the approved WHOLE CUT, not an isolated shot. The deterministic scaffold establishes faithful source layers and timing; it is your starting point, not evidence of agentic finishing. Inspect all derived design references and semantic graphic/motion direction, then deliberately author an editable composition: readable type hierarchy, balanced product/copy layout, graphic rhythm and continuity across the cut. Use the supplied full scaffold and operationContracts directly; do not request get_timeline or list_animation_presets redundantly when their data is already present. Author a focused batch of deliberate edits rather than redesigning every layer. Use existing Timeline operations and their exact operationContracts. Make meaningful layout, typography, decorative or animation choices that serve this board. Do not make arbitrary nudges, metadata changes or add empty layers merely to satisfy authoring. On a new cut, author actual layout or motion before submitting; on an existing finished cut, preserve good prior work and submit unchanged if no revision is needed. Preserve all source assets, exact copy/colors, semantic IDs, shot windows and manual edits. manualEdits lists the layer fields a person changed: never edit those fields, and compose around them. Do not invent or generate replacement media. The design references show content, hierarchy and intent, not approved pixel layout: referenceLayoutDefects lists collisions the references themselves contain, and the cut must not repeat them. Read every edit result, correct rejected edits, then submit_finished_cut when ready. At a revision, fix every reported defect, including every previousReview.layoutDefects entry. Storyboard contains semantic intent, never an animation implementation. The Timeline is execution truth.";
|
|
711
|
+
const decorations = [];
|
|
712
|
+
const authoredPlacements = input.shots.flatMap((shot) => activeStoryboardGraphics(shot).flatMap((element) => {
|
|
713
|
+
const placement = authoredPlacement(element);
|
|
714
|
+
return placement
|
|
715
|
+
? [{ shotId: shot.id, elementId: element.id, frame: placement.frame, typography: placement.typography, lock: placement.lock, limits: placement.limits }]
|
|
716
|
+
: [];
|
|
717
|
+
}));
|
|
718
|
+
const shotReviewRules = input.shots.flatMap((shot) => {
|
|
719
|
+
const rules = isRecord(shot.graphics) ? shot.graphics["review_rules"] : undefined;
|
|
720
|
+
const list = Array.isArray(rules) ? rules.filter((rule) => typeof rule === "string" && !!rule.trim()) : [];
|
|
721
|
+
return list.length ? [{ shotId: shot.id, rules: list }] : [];
|
|
722
|
+
});
|
|
723
|
+
// What the reviewer must not call a defect: placements a template locked.
|
|
724
|
+
const lockedSummary = input.shots.flatMap((shot) => activeStoryboardGraphics(shot).flatMap((element) => {
|
|
725
|
+
const placement = authoredPlacement(element);
|
|
726
|
+
if (!placement?.lock.length) {
|
|
727
|
+
return [];
|
|
728
|
+
}
|
|
729
|
+
const box = frameBox(placement);
|
|
730
|
+
return [`${shot.id}/${element.id}${box ? ` box [${box.join(", ")}]` : ""} locked: ${placement.lock.join(", ")}`];
|
|
731
|
+
}));
|
|
732
|
+
const lockedPlacementNote = lockedSummary.length
|
|
733
|
+
? `These placements are authored and locked: ${lockedSummary.join("; ")}. Do not report their position or size as a defect.`
|
|
734
|
+
: undefined;
|
|
735
|
+
const addDecoration = async (args) => {
|
|
736
|
+
const imageModel = options.imageModel;
|
|
737
|
+
const { shotId, name, prompt, layer } = args;
|
|
738
|
+
if (!imageModel) {
|
|
739
|
+
return json({ ok: false, error: "No image model is selected for this cut." });
|
|
740
|
+
}
|
|
741
|
+
if (typeof shotId !== "string" ||
|
|
742
|
+
typeof name !== "string" ||
|
|
743
|
+
typeof prompt !== "string" ||
|
|
744
|
+
!prompt.trim() ||
|
|
745
|
+
(layer !== "background" && layer !== "overlay")) {
|
|
746
|
+
return json({ ok: false, error: "Give shotId, name, prompt and layer (background or overlay)." });
|
|
747
|
+
}
|
|
748
|
+
const window = windows.find((value) => value.shotId === shotId);
|
|
749
|
+
const shot = input.shots.find((value) => value.id === shotId);
|
|
750
|
+
const slug = name.trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
|
|
751
|
+
if (!window || !shot || !slug) {
|
|
752
|
+
return json({ ok: false, error: `Use a shot id from the storyboard and a name with letters or digits.` });
|
|
753
|
+
}
|
|
754
|
+
const elementId = `decor-${slug}`;
|
|
755
|
+
const existing = document.clips.find((clip) => clip.storyboardShotId === shotId && clip.storyboardElementId === elementId);
|
|
756
|
+
if (existing) {
|
|
757
|
+
return json({ ok: true, alreadyExists: true, clipId: existing.id });
|
|
758
|
+
}
|
|
759
|
+
if (decorations.length >= MAX_DECORATIONS) {
|
|
760
|
+
return json({ ok: false, error: `This cut already has ${MAX_DECORATIONS} generated decorations. Place or restyle those.` });
|
|
761
|
+
}
|
|
762
|
+
// Ask for the frame's own shape, so a background covers it with little crop.
|
|
763
|
+
const portrait = input.height >= input.width;
|
|
764
|
+
const shortSide = Math.max(512, Math.round((1536 * Math.min(input.width, input.height)) / Math.max(input.width, input.height) / 64) * 64);
|
|
765
|
+
const requested = layer === "overlay"
|
|
766
|
+
? { width: 1024, height: 1024 }
|
|
767
|
+
: portrait
|
|
768
|
+
? { width: shortSide, height: 1536 }
|
|
769
|
+
: { width: 1536, height: shortSide };
|
|
770
|
+
let result;
|
|
771
|
+
try {
|
|
772
|
+
result = await run.invoke("generate_image", {
|
|
773
|
+
model: { provider: imageModel.provider, id: imageModel.id },
|
|
774
|
+
prompt: prompt.trim(),
|
|
775
|
+
...requested
|
|
776
|
+
});
|
|
777
|
+
}
|
|
778
|
+
catch (error) {
|
|
779
|
+
result = { error: error instanceof Error ? error.message : String(error) };
|
|
780
|
+
}
|
|
781
|
+
signal?.throwIfAborted();
|
|
782
|
+
const assetId = isRecord(result) && typeof result["asset_id"] === "string" ? result["asset_id"] : undefined;
|
|
783
|
+
if (!assetId) {
|
|
784
|
+
const reason = isRecord(result) && typeof result["error"] === "string" ? result["error"] : "The image model returned no image.";
|
|
785
|
+
return json({ ok: false, error: reason });
|
|
786
|
+
}
|
|
787
|
+
let size = requested;
|
|
788
|
+
const bytes = await loadAsset(assetId);
|
|
789
|
+
if (bytes) {
|
|
790
|
+
try {
|
|
791
|
+
const image = await loadImage(Buffer.from(bytes));
|
|
792
|
+
size = { width: image.width, height: image.height };
|
|
793
|
+
}
|
|
794
|
+
catch {
|
|
795
|
+
// Keep the requested size. Only the cover scale depends on it.
|
|
796
|
+
}
|
|
797
|
+
}
|
|
798
|
+
// A layer draws contained in the frame at scale 1. Covering the frame takes
|
|
799
|
+
// the ratio of the two fits.
|
|
800
|
+
const fitX = input.width / size.width;
|
|
801
|
+
const fitY = input.height / size.height;
|
|
802
|
+
const scale = layer === "background" ? Math.max(fitX, fitY) / Math.min(fitX, fitY) : 0.5;
|
|
803
|
+
const tracks = document.tracks.map((track) => ({ ...track }));
|
|
804
|
+
let index = Math.max(-1, ...tracks.map((track) => track.index)) + 1;
|
|
805
|
+
if (layer === "background") {
|
|
806
|
+
const backdrop = document.clips.find((clip) => clip.storyboardShotId === shotId && clip.storyboardElementId === "background");
|
|
807
|
+
const above = tracks.find((track) => track.id === backdrop?.trackId)?.index;
|
|
808
|
+
index = above === undefined ? Math.min(0, ...tracks.map((track) => track.index)) : above + 1;
|
|
809
|
+
for (const track of tracks) {
|
|
810
|
+
if (track.index >= index) {
|
|
811
|
+
track.index += 1;
|
|
812
|
+
}
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
const track = makeTrack({ type: "overlay", name: `${shot.slug ?? shot.id}: ${elementId}`, index });
|
|
816
|
+
const clip = makeClip({
|
|
817
|
+
trackId: track.id,
|
|
818
|
+
name: elementId,
|
|
819
|
+
startMs: window.start,
|
|
820
|
+
durationMs: window.duration,
|
|
821
|
+
mediaType: "image",
|
|
822
|
+
sourceType: "imported",
|
|
823
|
+
status: "generated",
|
|
824
|
+
versions: [],
|
|
825
|
+
storyboardBoardId: input.boardId,
|
|
826
|
+
storyboardShotId: shotId,
|
|
827
|
+
storyboardElementId: elementId,
|
|
828
|
+
storyboardElementRole: "decorative",
|
|
829
|
+
currentAssetId: assetId,
|
|
830
|
+
transform: { position: { x: 0, y: 0 }, scale: { x: scale, y: scale }, rotation: 0, anchor: { x: 0.5, y: 0.5 } },
|
|
831
|
+
animations: []
|
|
832
|
+
});
|
|
833
|
+
const decoration = {
|
|
834
|
+
shotId,
|
|
835
|
+
element: { id: elementId, kind: "asset", role: "decorative", asset_id: assetId, direction: prompt.trim(), origin: "layout_agent" }
|
|
836
|
+
};
|
|
837
|
+
const candidate = { ...document, tracks: [...tracks, track], clips: [...document.clips, clip] };
|
|
838
|
+
const graphics = (shot.graphics ??= { mode: "graphics_first", elements: [] });
|
|
839
|
+
const elements = (graphics.elements ??= []);
|
|
840
|
+
elements.push(decoration.element);
|
|
841
|
+
generated.set(clip.id, decoration);
|
|
842
|
+
try {
|
|
843
|
+
assertCandidateOwnership(candidate);
|
|
844
|
+
const policy = validateProducedTimeline(input, candidate);
|
|
845
|
+
if (policy.length) {
|
|
846
|
+
throw new Error(json(policy));
|
|
847
|
+
}
|
|
848
|
+
}
|
|
849
|
+
catch (error) {
|
|
850
|
+
elements.pop();
|
|
851
|
+
generated.delete(clip.id);
|
|
852
|
+
return json({ ok: false, error: error instanceof Error ? error.message : String(error) });
|
|
853
|
+
}
|
|
854
|
+
document = candidate;
|
|
855
|
+
decorations.push(decoration);
|
|
856
|
+
return json({ ok: true, clipId: clip.id, trackId: track.id, assetId, imageSize: size, scale });
|
|
857
|
+
};
|
|
858
|
+
const commit = (document) => {
|
|
859
|
+
for (const clip of document.clips) {
|
|
860
|
+
if (clip.storyboardBoardId !== input.boardId) {
|
|
861
|
+
continue;
|
|
862
|
+
}
|
|
863
|
+
stampStoryboardMaterializationBaseline(clip);
|
|
864
|
+
const previous = input.current?.clips.find((value) => value.id === clip.id);
|
|
865
|
+
if (previous?.storyboardMaterializationBaseline &&
|
|
866
|
+
!clip.storyboardElementId?.startsWith("$agent:")) {
|
|
867
|
+
const prior = JSON.parse(previous.storyboardMaterializationBaseline);
|
|
868
|
+
const accepted = JSON.parse(clip.storyboardMaterializationBaseline ?? "null");
|
|
869
|
+
if (isRecord(prior) && isRecord(accepted)) {
|
|
870
|
+
for (const field of ["transform", "startMs", "durationMs"]) {
|
|
871
|
+
if (canonical(previous[field]) !== canonical(prior[field])) {
|
|
872
|
+
accepted[field] = prior[field];
|
|
873
|
+
}
|
|
874
|
+
}
|
|
875
|
+
clip.storyboardMaterializationBaseline = canonical(accepted);
|
|
876
|
+
}
|
|
877
|
+
}
|
|
878
|
+
}
|
|
879
|
+
document.storyboardMaterializations = [
|
|
880
|
+
...(document.storyboardMaterializations ?? []).filter((entry) => entry.boardId !== input.boardId),
|
|
881
|
+
{
|
|
882
|
+
boardId: input.boardId,
|
|
883
|
+
elementKeys: document.clips
|
|
884
|
+
.filter((clip) => clip.storyboardBoardId === input.boardId)
|
|
885
|
+
.map((clip) => `${clip.storyboardShotId}/${clip.storyboardElementId}`),
|
|
886
|
+
// A new layout waits for its motion. A finished cut laid out again
|
|
887
|
+
// keeps its motion, so it stays finished.
|
|
888
|
+
stage: layoutPhase && priorStage !== "finished" ? "layout" : "finished"
|
|
889
|
+
}
|
|
890
|
+
];
|
|
891
|
+
};
|
|
892
|
+
// The best reviewed candidate so far, and the failing frame times (as JSON)
|
|
893
|
+
// and findings of each reviewed round, for the early stop on a repeated failure.
|
|
894
|
+
let best;
|
|
895
|
+
const failingByRound = new Map();
|
|
896
|
+
for (let round = 0; round < 3; round += 1) {
|
|
897
|
+
const beforeRevision = json(document);
|
|
898
|
+
let submitted = false;
|
|
899
|
+
let rejectedEdit;
|
|
900
|
+
const editResults = [];
|
|
901
|
+
const recordEditResult = (result) => {
|
|
902
|
+
editResults.push(result.slice(0, 1600));
|
|
903
|
+
return result;
|
|
904
|
+
};
|
|
905
|
+
const diagnostics = () => json({
|
|
906
|
+
previousReview: feedback,
|
|
907
|
+
lastEditResults: editResults.slice(-4)
|
|
908
|
+
}).slice(0, 8000);
|
|
909
|
+
const authoringMessages = [
|
|
910
|
+
{
|
|
911
|
+
role: "system",
|
|
912
|
+
content: authoringPrompt
|
|
913
|
+
},
|
|
914
|
+
{
|
|
915
|
+
role: "user",
|
|
916
|
+
content: [
|
|
917
|
+
{
|
|
918
|
+
type: "text",
|
|
919
|
+
text: json({
|
|
920
|
+
storyboard: board,
|
|
921
|
+
resolvedProduction: input,
|
|
922
|
+
scaffold: document,
|
|
923
|
+
target: {
|
|
924
|
+
width: input.width,
|
|
925
|
+
height: input.height,
|
|
926
|
+
fps: sequence.fps
|
|
927
|
+
},
|
|
928
|
+
previousReview: feedback,
|
|
929
|
+
referenceLayoutDefects,
|
|
930
|
+
needsInitialAuthoring,
|
|
931
|
+
manualEdits,
|
|
932
|
+
authoredPlacements,
|
|
933
|
+
...(shotReviewRules.length && { reviewRules: shotReviewRules }),
|
|
934
|
+
operationContracts,
|
|
935
|
+
finishingConstraints: scopeInstructions
|
|
936
|
+
})
|
|
937
|
+
},
|
|
938
|
+
...referenceImages,
|
|
939
|
+
...previousCandidateImages
|
|
940
|
+
]
|
|
941
|
+
}
|
|
942
|
+
];
|
|
943
|
+
const authoringTools = decorate ? [inspect, edit, decorate, submit] : [inspect, edit, submit];
|
|
944
|
+
const author = async (call) => {
|
|
945
|
+
signal?.throwIfAborted();
|
|
946
|
+
if (submitted) {
|
|
947
|
+
return "The candidate is submitted. End this authoring turn.";
|
|
948
|
+
}
|
|
949
|
+
if (call.name === inspect.name) {
|
|
950
|
+
return json(document);
|
|
951
|
+
}
|
|
952
|
+
if (decorate && call.name === decorate.name) {
|
|
953
|
+
return recordEditResult(await addDecoration(call.args));
|
|
954
|
+
}
|
|
955
|
+
if (call.name === submit.name) {
|
|
956
|
+
if (rejectedEdit) {
|
|
957
|
+
return recordEditResult(`Correct the rejected edit before submitting: ${rejectedEdit}`);
|
|
958
|
+
}
|
|
959
|
+
if (needsInitialAuthoring &&
|
|
960
|
+
composition(document) === startComposition) {
|
|
961
|
+
return recordEditResult(motionPending
|
|
962
|
+
? "This cut still holds only the planned static layout. Author its motion with animate_clip and set_transition from the motion direction, read the results, then submit again. Keep the planned placement and sizes."
|
|
963
|
+
: "This new cut is still the unchanged deterministic scaffold. Author meaningful editable layout or motion from the full-board direction with edit_timeline, read its result, then submit again. Read-only, no-op and metadata-only edits do not finish a new agentic cut.");
|
|
964
|
+
}
|
|
965
|
+
submitted = true;
|
|
966
|
+
return "Candidate submitted for mandatory review.";
|
|
967
|
+
}
|
|
968
|
+
if (call.name !== edit.name) {
|
|
969
|
+
return "Only isolated Timeline editing is available.";
|
|
970
|
+
}
|
|
971
|
+
const ops = parseOps(call.args["ops"]);
|
|
972
|
+
if (!Array.isArray(ops)) {
|
|
973
|
+
return recordEditResult(json(ops));
|
|
974
|
+
}
|
|
975
|
+
if (ops.some((operation) => !permittedOps.has(operation.op.replace("ui_timeline_", "")))) {
|
|
976
|
+
return recordEditResult("This finishing operation is unavailable. " + scopeInstructions);
|
|
977
|
+
}
|
|
978
|
+
const pending = [];
|
|
979
|
+
const existingResults = [];
|
|
980
|
+
const identities = new Set();
|
|
981
|
+
for (const operation of ops) {
|
|
982
|
+
const kind = operation.op === "ui_timeline_add_shape_clip"
|
|
983
|
+
? "shape"
|
|
984
|
+
: operation.op === "ui_timeline_add_text_clip"
|
|
985
|
+
? "text"
|
|
986
|
+
: operation.op === "ui_timeline_add_group"
|
|
987
|
+
? "group"
|
|
988
|
+
: undefined;
|
|
989
|
+
if (!kind) {
|
|
990
|
+
pending.push(operation);
|
|
991
|
+
continue;
|
|
992
|
+
}
|
|
993
|
+
const name = operation.input["name"];
|
|
994
|
+
const start = operation.input["startMs"];
|
|
995
|
+
const duration = operation.input["durationMs"];
|
|
996
|
+
const window = windows.find((value) => typeof start === "number" &&
|
|
997
|
+
typeof duration === "number" &&
|
|
998
|
+
start >= value.start &&
|
|
999
|
+
start + duration <= value.start + value.duration);
|
|
1000
|
+
if (typeof name !== "string" || !name.trim() || !window) {
|
|
1001
|
+
return recordEditResult("New layers require an explicit stable semantic name, startMs and durationMs inside one shot window. Reuse that name on reruns; edit an existing layer with set_clip_params.");
|
|
1002
|
+
}
|
|
1003
|
+
const key = `${window.shotId}/$agent:${kind}:${encodeURIComponent(name.trim())}`;
|
|
1004
|
+
if (identities.has(key)) {
|
|
1005
|
+
return recordEditResult(`Duplicate semantic addition ${key} in one batch.`);
|
|
1006
|
+
}
|
|
1007
|
+
identities.add(key);
|
|
1008
|
+
const existing = document.clips.find((clip) => clip.storyboardBoardId === input.boardId &&
|
|
1009
|
+
clip.storyboardShotId === window.shotId &&
|
|
1010
|
+
clip.storyboardElementId === key.slice(window.shotId.length + 1));
|
|
1011
|
+
if (existing) {
|
|
1012
|
+
existingResults.push({
|
|
1013
|
+
op: operation.op,
|
|
1014
|
+
ok: true,
|
|
1015
|
+
result: {
|
|
1016
|
+
alreadyExists: true,
|
|
1017
|
+
clipId: existing.id,
|
|
1018
|
+
semanticIdentity: key
|
|
1019
|
+
}
|
|
1020
|
+
});
|
|
1021
|
+
}
|
|
1022
|
+
else {
|
|
1023
|
+
pending.push(operation);
|
|
1024
|
+
}
|
|
1025
|
+
}
|
|
1026
|
+
if (!pending.length) {
|
|
1027
|
+
return recordEditResult(json(existingResults));
|
|
1028
|
+
}
|
|
1029
|
+
// Hermetic mode removes generation, Blender rendering and DB-writing hooks.
|
|
1030
|
+
const outcome = await applyOps(run, sequence, document, pending, {
|
|
1031
|
+
hermetic: true
|
|
1032
|
+
});
|
|
1033
|
+
if (!outcome.records.some((record) => record.ok &&
|
|
1034
|
+
!["get_state", "list_animation_presets"].includes(record.op.replace("ui_timeline_", "")))) {
|
|
1035
|
+
return recordEditResult(json([...existingResults, ...outcome.records]));
|
|
1036
|
+
}
|
|
1037
|
+
const candidate = {
|
|
1038
|
+
...document,
|
|
1039
|
+
tracks: outcome.state.documentTracks,
|
|
1040
|
+
clips: outcome.state.documentClips,
|
|
1041
|
+
markers: outcome.state.markers,
|
|
1042
|
+
camera2d: outcome.state.camera2d,
|
|
1043
|
+
mediaTracks: outcome.state.mediaTracks,
|
|
1044
|
+
tempo: outcome.state.tempo
|
|
1045
|
+
};
|
|
1046
|
+
for (const clip of candidate.clips) {
|
|
1047
|
+
if (initial.has(clip.id) || clip.storyboardBoardId) {
|
|
1048
|
+
continue;
|
|
1049
|
+
}
|
|
1050
|
+
const window = windows.find((value) => clip.startMs >= value.start &&
|
|
1051
|
+
clip.startMs + clip.durationMs <= value.start + value.duration);
|
|
1052
|
+
if (window && ["text", "shape", "group"].includes(clip.mediaType)) {
|
|
1053
|
+
clip.storyboardBoardId = input.boardId;
|
|
1054
|
+
clip.storyboardShotId = window.shotId;
|
|
1055
|
+
clip.storyboardElementId = `$agent:${clip.mediaType}:${encodeURIComponent(clip.name.trim())}`;
|
|
1056
|
+
}
|
|
1057
|
+
}
|
|
1058
|
+
try {
|
|
1059
|
+
assertCandidateOwnership(candidate);
|
|
1060
|
+
const policy = validateProducedTimeline(input, candidate);
|
|
1061
|
+
if (policy.length) {
|
|
1062
|
+
throw new Error(`Produced Timeline violates production requirements: ${json(policy)}`);
|
|
1063
|
+
}
|
|
1064
|
+
}
|
|
1065
|
+
catch (error) {
|
|
1066
|
+
rejectedEdit = error instanceof Error ? error.message : String(error);
|
|
1067
|
+
return recordEditResult(json({
|
|
1068
|
+
ok: false,
|
|
1069
|
+
rolledBack: true,
|
|
1070
|
+
error: rejectedEdit,
|
|
1071
|
+
resolution: "No changes from this batch were applied. Preserve source timing, ownership and manual edits. Edit existing protected copy with set_clip_params for presentation only. New text must use approved unprotected Storyboard copy. Retry the corrected batch."
|
|
1072
|
+
}));
|
|
1073
|
+
}
|
|
1074
|
+
document = candidate;
|
|
1075
|
+
rejectedEdit = undefined;
|
|
1076
|
+
return recordEditResult(json([...existingResults, ...outcome.records]));
|
|
1077
|
+
};
|
|
1078
|
+
const phaseLabel = layoutPhase ? "layout" : "finish";
|
|
1079
|
+
await drive(phaseLabel, authoringMessages, authoringTools, author);
|
|
1080
|
+
// A model that ends its turn after accepted edits has authored a
|
|
1081
|
+
// candidate. The checks and the visual review below still gate it.
|
|
1082
|
+
const authored = () => !rejectedEdit &&
|
|
1083
|
+
beforeRevision !== json(document) &&
|
|
1084
|
+
(!needsInitialAuthoring || composition(document) !== startComposition);
|
|
1085
|
+
if (!submitted && !authored()) {
|
|
1086
|
+
// A model sometimes ends its turn after it reads a failed review. One
|
|
1087
|
+
// reminder turn names the work that is still due.
|
|
1088
|
+
await drive(phaseLabel, [
|
|
1089
|
+
...authoringMessages,
|
|
1090
|
+
{
|
|
1091
|
+
role: "user",
|
|
1092
|
+
content: `Your turn ended without ${rejectedEdit ? "a corrected edit" : "an edit"} or a submission. ${round > 0 ? "Fix every finding in previousReview" : "Author the composition"} with edit_timeline, read each result, then call submit_finished_cut.`
|
|
1093
|
+
}
|
|
1094
|
+
], authoringTools, author);
|
|
1095
|
+
}
|
|
1096
|
+
if (!submitted && !authored()) {
|
|
1097
|
+
throw new Error(`Finished-cut agent did not submit a candidate: ${diagnostics()}`);
|
|
1098
|
+
}
|
|
1099
|
+
if (round > 0 && beforeRevision === json(document)) {
|
|
1100
|
+
throw new Error(`Finishing did not revise the draft after reported defects: ${diagnostics()}`);
|
|
1101
|
+
}
|
|
1102
|
+
assertCandidateOwnership(document);
|
|
1103
|
+
const policy = validateProducedTimeline(input, document);
|
|
1104
|
+
const structural = validateTimelineSequence(document, {
|
|
1105
|
+
fps: sequence.fps,
|
|
1106
|
+
width: input.width,
|
|
1107
|
+
height: input.height
|
|
1108
|
+
});
|
|
1109
|
+
if (policy.length || !structural.ok) {
|
|
1110
|
+
feedback = { policy, structural };
|
|
1111
|
+
continue;
|
|
1112
|
+
}
|
|
1113
|
+
const candidateSequence = { ...sequence.toTimelineSequence(), ...document };
|
|
1114
|
+
const holds = shotHoldTimes(document.clips, windows, input.width, input.height);
|
|
1115
|
+
signal?.throwIfAborted();
|
|
1116
|
+
const layoutDefects = await findLayoutDefects({
|
|
1117
|
+
sequence: candidateSequence,
|
|
1118
|
+
windows,
|
|
1119
|
+
holds,
|
|
1120
|
+
loadAsset,
|
|
1121
|
+
signal
|
|
1122
|
+
});
|
|
1123
|
+
if (layoutDefects.length) {
|
|
1124
|
+
feedback = {
|
|
1125
|
+
layoutDefects,
|
|
1126
|
+
resolution: "A pixel measurement of each layer at its shot's hold frame found these collisions. Move or resize the named clips with set_clip_params, then submit again."
|
|
1127
|
+
};
|
|
1128
|
+
continue;
|
|
1129
|
+
}
|
|
1130
|
+
// The hold frame is where a viewer reads each shot, so the review always sees it.
|
|
1131
|
+
const holdShots = new Map([...holds].map(([shotId, timeMs]) => [timeMs, shotId]));
|
|
1132
|
+
const timesMs = [
|
|
1133
|
+
...new Set([
|
|
1134
|
+
// A layout has no motion to judge, so its review sees the hold frames.
|
|
1135
|
+
...(layoutPhase ? [] : storyboardReviewTimes(document.clips, input.width, input.height, sequence.fps)),
|
|
1136
|
+
...holdShots.keys()
|
|
1137
|
+
])
|
|
1138
|
+
].sort((left, right) => left - right);
|
|
1139
|
+
signal?.throwIfAborted();
|
|
1140
|
+
const frames = await renderTimelineFrames({
|
|
1141
|
+
sequence: candidateSequence,
|
|
1142
|
+
timesMs,
|
|
1143
|
+
width: 540,
|
|
1144
|
+
loadAsset
|
|
1145
|
+
});
|
|
1146
|
+
signal?.throwIfAborted();
|
|
1147
|
+
if (!frames.complete ||
|
|
1148
|
+
!frames.frames.length ||
|
|
1149
|
+
frames.effectsNotApplied.length ||
|
|
1150
|
+
frames.fontsUnavailable.length) {
|
|
1151
|
+
feedback = {
|
|
1152
|
+
renderDefects: frames.frames.map((frame) => ({
|
|
1153
|
+
timeMs: frame.time_ms,
|
|
1154
|
+
dropped: frame.dropped,
|
|
1155
|
+
degraded: frame.degraded,
|
|
1156
|
+
failures: frame.failures
|
|
1157
|
+
})),
|
|
1158
|
+
effectsNotApplied: frames.effectsNotApplied,
|
|
1159
|
+
fontsUnavailable: frames.fontsUnavailable
|
|
1160
|
+
};
|
|
1161
|
+
continue;
|
|
1162
|
+
}
|
|
1163
|
+
previousCandidateImages = frames.frames.flatMap((frame) => [
|
|
1164
|
+
{
|
|
1165
|
+
type: "text",
|
|
1166
|
+
text: `Previous candidate cut frame at ${frame.time_ms}ms. Revise defects identified in previousReview. This is the candidate, not the derived design reference.`
|
|
1167
|
+
},
|
|
1168
|
+
{
|
|
1169
|
+
type: "image_url",
|
|
1170
|
+
image: {
|
|
1171
|
+
uri: `data:image/png;base64,${Buffer.from(frame.png).toString("base64")}`
|
|
1172
|
+
}
|
|
1173
|
+
}
|
|
1174
|
+
]);
|
|
1175
|
+
let verdict;
|
|
1176
|
+
const content = [
|
|
1177
|
+
{
|
|
1178
|
+
type: "text",
|
|
1179
|
+
text: json({
|
|
1180
|
+
storyboard: board,
|
|
1181
|
+
resolvedProduction: input,
|
|
1182
|
+
timeline: document,
|
|
1183
|
+
frameTimesMs: frames.frames.map((frame) => frame.time_ms),
|
|
1184
|
+
candidateFrameCount: frames.frames.length,
|
|
1185
|
+
designReferenceFrameCount: referenceEvidence.length,
|
|
1186
|
+
instruction: layoutPhase
|
|
1187
|
+
? "Visually review the settled HOLD frame of every shot as a still layout against the approved semantic graphics and the full-cut direction. The first images are labeled design references derived from the expected Storyboard revision; they show content and intent, not an approved layout. The subsequent images are the actual candidate frames. Exact identity and copy clearance were checked mechanically before this review. In each frame, check every pair of neighbouring elements: copy that touches or crosses an image edge, copy crowded against other copy, copy near the frame edge, an image too small or too large for its role, an asset drawn as an opaque box where the design wants a cut-out, a generated background that fights the copy or the product, unbalanced empty areas, a grid or margins that change from shot to shot, and wrong hierarchy or contrast are all material defects. Motion is not part of this review. Fail on any material defect."
|
|
1188
|
+
: "Visually review all rendered frames against approved semantic graphics and full-cut direction. The first images are labeled design references derived from the expected Storyboard revision. The subsequent images are the actual candidate cut. Use the references for content, hierarchy and intent only. They are not approved pixel layout and can contain the same defects: referenceLayoutDefects lists collisions measured in them, and a candidate that repeats a reference defect fails. Exact identity and copy clearance at hold frames are checked mechanically before this review. At each HOLD frame, check every pair of neighbouring elements: copy that touches or crosses an image edge, copy crowded against other copy, copy near the frame edge, an asset drawn as an opaque box where the design wants a cut-out, unbalanced empty areas and wrong hierarchy or contrast are all material defects. Across motion frames, find awkward motion endpoints and continuity failures. Fail on any material defect.",
|
|
1189
|
+
referenceLayoutDefects,
|
|
1190
|
+
...(shotReviewRules.length && { reviewRules: shotReviewRules }),
|
|
1191
|
+
...(lockedPlacementNote && { lockedPlacements: lockedPlacementNote })
|
|
1192
|
+
})
|
|
1193
|
+
},
|
|
1194
|
+
...referenceImages,
|
|
1195
|
+
...frames.frames.flatMap((frame) => [
|
|
1196
|
+
{
|
|
1197
|
+
type: "text",
|
|
1198
|
+
text: holdShots.has(frame.time_ms)
|
|
1199
|
+
? `Actual candidate HOLD frame for shot ${holdShots.get(frame.time_ms)} at ${frame.time_ms}ms: every entrance has finished and no exit has started. Judge the settled composition, spacing and hierarchy here. Include exactly one frameReviews entry for this timeMs. This is not a derived design reference.`
|
|
1200
|
+
: `Actual candidate motion frame at ${frame.time_ms}ms. Include exactly one frameReviews entry for this timeMs. This is not a derived design reference.`
|
|
1201
|
+
},
|
|
1202
|
+
{
|
|
1203
|
+
type: "image_url",
|
|
1204
|
+
image: {
|
|
1205
|
+
uri: `data:image/png;base64,${Buffer.from(frame.png).toString("base64")}`
|
|
1206
|
+
}
|
|
1207
|
+
}
|
|
1208
|
+
])
|
|
1209
|
+
];
|
|
1210
|
+
await drive("review", [
|
|
1211
|
+
{
|
|
1212
|
+
role: "system",
|
|
1213
|
+
content: "You are the visual finishing reviewer. Inspect the actual supplied composited frame pixels. Design references and actual candidate frames are labeled separately. Review each actual candidate timeMs exactly once. The schedule deliberately includes entrance and exit boundaries where intended fades can be transparent. Judge layout, spacing and legibility at the labeled HOLD frames, and motion continuity across neighboring motion frames, not constant visibility at intentionally transparent boundaries. A design reference never excuses a defect: if the candidate shares a collision or crowding with its reference, report it. Call review_finished_cut with an internally consistent overall and per-frame verdict. Findings are ONLY current unresolved actionable material defects. Put positive observations and withdrawn suspicions in summary, not findings. If no material defect remains, explicitly set passed true with empty findings for every frame and overall. If any frame has a material defect, set that frame and overall passed false with the exact defect in both findings lists. When the payload lists reviewRules, judge each shot against its rules as material defects. When it lists lockedPlacements, never report the position or size of those layers as a defect. Never approve without inspecting the whole cut. Never reject merely to report positive observations or previously resolved issues."
|
|
1214
|
+
},
|
|
1215
|
+
{ role: "user", content }
|
|
1216
|
+
], [review], async (call) => {
|
|
1217
|
+
if (call.name !== review.name) {
|
|
1218
|
+
return "Call review_finished_cut.";
|
|
1219
|
+
}
|
|
1220
|
+
if (verdict) {
|
|
1221
|
+
return "Review already recorded. End this review turn.";
|
|
1222
|
+
}
|
|
1223
|
+
const parsed = reviewSchema.safeParse(call.args);
|
|
1224
|
+
if (!parsed.success) {
|
|
1225
|
+
return json(parsed.error.issues);
|
|
1226
|
+
}
|
|
1227
|
+
const expectedTimes = frames.frames.map((frame) => frame.time_ms);
|
|
1228
|
+
const reportedTimes = parsed.data.frameReviews.map((frame) => frame.timeMs);
|
|
1229
|
+
const frameFindings = [
|
|
1230
|
+
...new Set(parsed.data.frameReviews.flatMap((frame) => frame.findings))
|
|
1231
|
+
].sort();
|
|
1232
|
+
const consistent = reportedTimes.length === expectedTimes.length &&
|
|
1233
|
+
new Set(reportedTimes).size === expectedTimes.length &&
|
|
1234
|
+
reportedTimes.every((time) => expectedTimes.includes(time)) &&
|
|
1235
|
+
parsed.data.frameReviews.every((frame) => frame.passed === (frame.findings.length === 0)) &&
|
|
1236
|
+
parsed.data.passed ===
|
|
1237
|
+
parsed.data.frameReviews.every((frame) => frame.passed) &&
|
|
1238
|
+
json([...parsed.data.findings].sort()) === json(frameFindings);
|
|
1239
|
+
if (!consistent) {
|
|
1240
|
+
return json({
|
|
1241
|
+
ok: false,
|
|
1242
|
+
error: "Review not recorded. Submit consistent overall and per-frame verdicts.",
|
|
1243
|
+
expectedFrameTimesMs: expectedTimes,
|
|
1244
|
+
resolution: "Review each actual candidate timeMs exactly once, without reference frames. Each frame passed must equal whether its unresolved defect findings are empty. Overall passed must equal every frame passed. Overall findings must be the unique union of frame findings. Correct the explicit report and call review_finished_cut again. No verdict will be inferred from summary."
|
|
1245
|
+
});
|
|
1246
|
+
}
|
|
1247
|
+
verdict = parsed.data;
|
|
1248
|
+
return "Review recorded.";
|
|
1249
|
+
});
|
|
1250
|
+
if (!verdict) {
|
|
1251
|
+
throw new Error("Finished cut has no explicit visual review.");
|
|
1252
|
+
}
|
|
1253
|
+
const evidence = {
|
|
1254
|
+
round,
|
|
1255
|
+
...verdict,
|
|
1256
|
+
referenceFrames: referenceEvidence,
|
|
1257
|
+
frames: frames.frames.map((frame) => ({
|
|
1258
|
+
timeMs: frame.time_ms,
|
|
1259
|
+
sha256: createHash("sha256").update(frame.png).digest("hex"),
|
|
1260
|
+
width: frame.width,
|
|
1261
|
+
height: frame.height
|
|
1262
|
+
}))
|
|
1263
|
+
};
|
|
1264
|
+
reviews.push(evidence);
|
|
1265
|
+
if (!verdict.passed) {
|
|
1266
|
+
feedback = evidence;
|
|
1267
|
+
const failing = verdict.frameReviews
|
|
1268
|
+
.filter((frame) => !frame.passed)
|
|
1269
|
+
.map((frame) => frame.timeMs)
|
|
1270
|
+
.sort((left, right) => left - right);
|
|
1271
|
+
failingByRound.set(round, { frames: json(failing), findings: verdict.findings });
|
|
1272
|
+
// Every reviewed candidate passed the hard checks. The best has the
|
|
1273
|
+
// fewest distinct defects, then the fewest failing frames, and a later
|
|
1274
|
+
// candidate wins a tie. One defect repeated on every world shot is not
|
|
1275
|
+
// worse than a missing logo on one frame.
|
|
1276
|
+
const defects = new Set(verdict.findings).size;
|
|
1277
|
+
if (!best ||
|
|
1278
|
+
defects < best.defects ||
|
|
1279
|
+
(defects === best.defects && failing.length <= best.failing)) {
|
|
1280
|
+
best = {
|
|
1281
|
+
document: structuredClone(document),
|
|
1282
|
+
decorations: [...decorations],
|
|
1283
|
+
defects,
|
|
1284
|
+
failing: failing.length,
|
|
1285
|
+
findings: [...verdict.findings]
|
|
1286
|
+
};
|
|
1287
|
+
}
|
|
1288
|
+
// A round repeats the last one when the same frames fail for the same
|
|
1289
|
+
// defects. New defects on the same frames still earn the next round.
|
|
1290
|
+
const previous = failingByRound.get(round - 1);
|
|
1291
|
+
const current = failingByRound.get(round);
|
|
1292
|
+
const repeated = previous !== undefined &&
|
|
1293
|
+
previous.frames === current.frames &&
|
|
1294
|
+
sameDefects(previous.findings, current.findings);
|
|
1295
|
+
if (repeated || round === 2) {
|
|
1296
|
+
break;
|
|
1297
|
+
}
|
|
1298
|
+
continue;
|
|
1299
|
+
}
|
|
1300
|
+
commit(document);
|
|
1301
|
+
return {
|
|
1302
|
+
document,
|
|
1303
|
+
reviews,
|
|
1304
|
+
costUsd: Math.max(0, runtime.provider.getTotalCost() - initialCost),
|
|
1305
|
+
decorations
|
|
1306
|
+
};
|
|
1307
|
+
}
|
|
1308
|
+
if (layoutPhase && best) {
|
|
1309
|
+
commit(best.document);
|
|
1310
|
+
return {
|
|
1311
|
+
document: best.document,
|
|
1312
|
+
reviews,
|
|
1313
|
+
costUsd: Math.max(0, runtime.provider.getTotalCost() - initialCost),
|
|
1314
|
+
decorations: best.decorations,
|
|
1315
|
+
needsReview: true,
|
|
1316
|
+
findings: best.findings
|
|
1317
|
+
};
|
|
1318
|
+
}
|
|
1319
|
+
throw new Error(`Finished cut still has unresolved defects after three candidates: ${json(feedback)}`);
|
|
1320
|
+
}
|
|
1321
|
+
//# sourceMappingURL=agentic-storyboard-finish.js.map
|