@nodetool-ai/agents 0.8.1 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.nodetool-build-stamp +1 -1
- package/dist/analysis/audio-dsp.d.ts +3 -2
- package/dist/analysis/audio-dsp.d.ts.map +1 -1
- package/dist/analysis/audio-dsp.js +11 -8
- package/dist/analysis/audio-dsp.js.map +1 -1
- package/dist/analysis/media-decode.d.ts +13 -0
- package/dist/analysis/media-decode.d.ts.map +1 -1
- package/dist/analysis/media-decode.js +33 -6
- package/dist/analysis/media-decode.js.map +1 -1
- package/dist/apify/assets.d.ts.map +1 -1
- package/dist/apify/assets.js +4 -0
- package/dist/apify/assets.js.map +1 -1
- package/dist/author-graph.d.ts +2 -3
- package/dist/author-graph.d.ts.map +1 -1
- package/dist/author-graph.js +3 -5
- package/dist/author-graph.js.map +1 -1
- package/dist/browser-gate.d.ts +2 -2
- package/dist/browser-gate.js +2 -2
- package/dist/capabilities/GamePublishRequest.d.ts +14 -0
- package/dist/capabilities/GamePublishRequest.d.ts.map +1 -0
- package/dist/capabilities/GamePublishRequest.js +2 -0
- package/dist/capabilities/GamePublishRequest.js.map +1 -0
- package/dist/capabilities/adapters.d.ts +4 -13
- package/dist/capabilities/adapters.d.ts.map +1 -1
- package/dist/capabilities/adapters.js +9 -14
- package/dist/capabilities/adapters.js.map +1 -1
- package/dist/capabilities/agent-activity.d.ts +27 -0
- package/dist/capabilities/agent-activity.d.ts.map +1 -0
- package/dist/capabilities/agent-activity.js +103 -0
- package/dist/capabilities/agent-activity.js.map +1 -0
- package/dist/capabilities/agentic-storyboard-finish.d.ts +67 -0
- package/dist/capabilities/agentic-storyboard-finish.d.ts.map +1 -0
- package/dist/capabilities/agentic-storyboard-finish.js +1321 -0
- package/dist/capabilities/agentic-storyboard-finish.js.map +1 -0
- package/dist/capabilities/agents.d.ts +5 -2
- package/dist/capabilities/agents.d.ts.map +1 -1
- package/dist/capabilities/agents.js +14 -4
- package/dist/capabilities/agents.js.map +1 -1
- package/dist/capabilities/agents.specs.d.ts +3 -0
- package/dist/capabilities/agents.specs.d.ts.map +1 -1
- package/dist/capabilities/agents.specs.js +71 -1
- package/dist/capabilities/agents.specs.js.map +1 -1
- package/dist/capabilities/analysis.d.ts.map +1 -1
- package/dist/capabilities/analysis.js +33 -22
- package/dist/capabilities/analysis.js.map +1 -1
- package/dist/capabilities/analysis.specs.js +2 -2
- package/dist/capabilities/analysis.specs.js.map +1 -1
- package/dist/capabilities/apps.d.ts.map +1 -1
- package/dist/capabilities/apps.js +5 -3
- package/dist/capabilities/apps.js.map +1 -1
- package/dist/capabilities/args.d.ts +2 -5
- package/dist/capabilities/args.d.ts.map +1 -1
- package/dist/capabilities/args.js +2 -5
- package/dist/capabilities/args.js.map +1 -1
- package/dist/capabilities/assets.d.ts.map +1 -1
- package/dist/capabilities/assets.js +40 -10
- package/dist/capabilities/assets.js.map +1 -1
- package/dist/capabilities/browser.d.ts +1 -1
- package/dist/capabilities/browser.js +1 -1
- package/dist/capabilities/browser.specs.d.ts +1 -1
- package/dist/capabilities/browser.specs.js +1 -1
- package/dist/capabilities/code.d.ts +5 -0
- package/dist/capabilities/code.d.ts.map +1 -1
- package/dist/capabilities/code.js +128 -89
- package/dist/capabilities/code.js.map +1 -1
- package/dist/capabilities/collections.d.ts +4 -6
- package/dist/capabilities/collections.d.ts.map +1 -1
- package/dist/capabilities/collections.js +16 -11
- package/dist/capabilities/collections.js.map +1 -1
- package/dist/capabilities/derive-shot.d.ts +45 -0
- package/dist/capabilities/derive-shot.d.ts.map +1 -0
- package/dist/capabilities/derive-shot.js +82 -0
- package/dist/capabilities/derive-shot.js.map +1 -0
- package/dist/capabilities/documents.d.ts +2 -1
- package/dist/capabilities/documents.d.ts.map +1 -1
- package/dist/capabilities/documents.js +12 -6
- package/dist/capabilities/documents.js.map +1 -1
- package/dist/capabilities/email.d.ts +2 -1
- package/dist/capabilities/email.d.ts.map +1 -1
- package/dist/capabilities/email.js +5 -3
- package/dist/capabilities/email.js.map +1 -1
- package/dist/capabilities/errors.d.ts +19 -0
- package/dist/capabilities/errors.d.ts.map +1 -0
- package/dist/capabilities/errors.js +109 -0
- package/dist/capabilities/errors.js.map +1 -0
- package/dist/capabilities/errors.specs.d.ts +11 -0
- package/dist/capabilities/errors.specs.d.ts.map +1 -0
- package/dist/capabilities/errors.specs.js +100 -0
- package/dist/capabilities/errors.specs.js.map +1 -0
- package/dist/capabilities/example-timelines.d.ts.map +1 -1
- package/dist/capabilities/example-timelines.js +52 -10
- package/dist/capabilities/example-timelines.js.map +1 -1
- package/dist/capabilities/example-timelines.specs.d.ts +1 -0
- package/dist/capabilities/example-timelines.specs.d.ts.map +1 -1
- package/dist/capabilities/example-timelines.specs.js +21 -3
- package/dist/capabilities/example-timelines.specs.js.map +1 -1
- package/dist/capabilities/finish-storyboard.d.ts +5 -0
- package/dist/capabilities/finish-storyboard.d.ts.map +1 -0
- package/dist/capabilities/finish-storyboard.js +278 -0
- package/dist/capabilities/finish-storyboard.js.map +1 -0
- package/dist/capabilities/game-asset-browser.d.ts +11 -0
- package/dist/capabilities/game-asset-browser.d.ts.map +1 -0
- package/dist/capabilities/game-asset-browser.js +152 -0
- package/dist/capabilities/game-asset-browser.js.map +1 -0
- package/dist/capabilities/game-asset-source.js +1 -1
- package/dist/capabilities/game-asset-source.js.map +1 -1
- package/dist/capabilities/game.d.ts +44 -2
- package/dist/capabilities/game.d.ts.map +1 -1
- package/dist/capabilities/game.js +408 -160
- package/dist/capabilities/game.js.map +1 -1
- package/dist/capabilities/game.specs.d.ts +4 -0
- package/dist/capabilities/game.specs.d.ts.map +1 -1
- package/dist/capabilities/game.specs.js +42 -13
- package/dist/capabilities/game.specs.js.map +1 -1
- package/dist/capabilities/game3d.d.ts +17 -0
- package/dist/capabilities/game3d.d.ts.map +1 -0
- package/dist/capabilities/game3d.js +434 -0
- package/dist/capabilities/game3d.js.map +1 -0
- package/dist/capabilities/gate-from-context.d.ts +1 -1
- package/dist/capabilities/gate-from-context.js +1 -1
- package/dist/capabilities/generations.d.ts +2 -0
- package/dist/capabilities/generations.d.ts.map +1 -1
- package/dist/capabilities/generations.js +14 -23
- package/dist/capabilities/generations.js.map +1 -1
- package/dist/capabilities/index.d.ts +2 -2
- package/dist/capabilities/index.d.ts.map +1 -1
- package/dist/capabilities/index.js +2 -2
- package/dist/capabilities/index.js.map +1 -1
- package/dist/capabilities/invoke.d.ts +14 -25
- package/dist/capabilities/invoke.d.ts.map +1 -1
- package/dist/capabilities/invoke.js +72 -35
- package/dist/capabilities/invoke.js.map +1 -1
- package/dist/capabilities/jobs.d.ts.map +1 -1
- package/dist/capabilities/jobs.js +47 -3
- package/dist/capabilities/jobs.js.map +1 -1
- package/dist/capabilities/jobs.specs.d.ts.map +1 -1
- package/dist/capabilities/jobs.specs.js +3 -1
- package/dist/capabilities/jobs.specs.js.map +1 -1
- package/dist/capabilities/js-scripts.d.ts +19 -1
- package/dist/capabilities/js-scripts.d.ts.map +1 -1
- package/dist/capabilities/js-scripts.js +109 -2
- package/dist/capabilities/js-scripts.js.map +1 -1
- package/dist/capabilities/js-scripts.specs.d.ts +2 -0
- package/dist/capabilities/js-scripts.specs.d.ts.map +1 -1
- package/dist/capabilities/js-scripts.specs.js +50 -0
- package/dist/capabilities/js-scripts.specs.js.map +1 -1
- package/dist/capabilities/lazy-tool-core.d.ts +23 -0
- package/dist/capabilities/lazy-tool-core.d.ts.map +1 -0
- package/dist/capabilities/lazy-tool-core.js +56 -0
- package/dist/capabilities/lazy-tool-core.js.map +1 -0
- package/dist/capabilities/lazy-tool.d.ts +3 -23
- package/dist/capabilities/lazy-tool.d.ts.map +1 -1
- package/dist/capabilities/lazy-tool.js +6 -64
- package/dist/capabilities/lazy-tool.js.map +1 -1
- package/dist/capabilities/legacy-tools.d.ts +15 -0
- package/dist/capabilities/legacy-tools.d.ts.map +1 -0
- package/dist/capabilities/legacy-tools.js +98 -0
- package/dist/capabilities/legacy-tools.js.map +1 -0
- package/dist/capabilities/media.d.ts +0 -9
- package/dist/capabilities/media.d.ts.map +1 -1
- package/dist/capabilities/media.js +314 -261
- package/dist/capabilities/media.js.map +1 -1
- package/dist/capabilities/media.specs.d.ts.map +1 -1
- package/dist/capabilities/media.specs.js +17 -0
- package/dist/capabilities/media.specs.js.map +1 -1
- package/dist/capabilities/metadata.d.ts +62 -0
- package/dist/capabilities/metadata.d.ts.map +1 -0
- package/dist/capabilities/metadata.js +114 -0
- package/dist/capabilities/metadata.js.map +1 -0
- package/dist/capabilities/model3d.d.ts.map +1 -1
- package/dist/capabilities/model3d.js +4 -0
- package/dist/capabilities/model3d.js.map +1 -1
- package/dist/capabilities/models.d.ts +2 -2
- package/dist/capabilities/models.d.ts.map +1 -1
- package/dist/capabilities/models.js +5 -3
- package/dist/capabilities/models.js.map +1 -1
- package/dist/capabilities/registry.d.ts +6 -19
- package/dist/capabilities/registry.d.ts.map +1 -1
- package/dist/capabilities/registry.js +72 -113
- package/dist/capabilities/registry.js.map +1 -1
- package/dist/capabilities/run-agent.d.ts +19 -0
- package/dist/capabilities/run-agent.d.ts.map +1 -0
- package/dist/capabilities/run-agent.js +183 -0
- package/dist/capabilities/run-agent.js.map +1 -0
- package/dist/capabilities/runs.d.ts +9 -0
- package/dist/capabilities/runs.d.ts.map +1 -0
- package/dist/capabilities/runs.js +46 -0
- package/dist/capabilities/runs.js.map +1 -0
- package/dist/capabilities/runs.specs.d.ts +39 -0
- package/dist/capabilities/runs.specs.d.ts.map +1 -0
- package/dist/capabilities/runs.specs.js +52 -0
- package/dist/capabilities/runs.specs.js.map +1 -0
- package/dist/capabilities/scripts.d.ts.map +1 -1
- package/dist/capabilities/scripts.js +11 -5
- package/dist/capabilities/scripts.js.map +1 -1
- package/dist/capabilities/settings.d.ts.map +1 -1
- package/dist/capabilities/settings.js +4 -2
- package/dist/capabilities/settings.js.map +1 -1
- package/dist/capabilities/sketches.d.ts.map +1 -1
- package/dist/capabilities/sketches.js +4 -2
- package/dist/capabilities/sketches.js.map +1 -1
- package/dist/capabilities/skills.d.ts.map +1 -1
- package/dist/capabilities/skills.js +22 -2
- package/dist/capabilities/skills.js.map +1 -1
- package/dist/capabilities/storyboard-layout-check.d.ts +33 -0
- package/dist/capabilities/storyboard-layout-check.d.ts.map +1 -0
- package/dist/capabilities/storyboard-layout-check.js +229 -0
- package/dist/capabilities/storyboard-layout-check.js.map +1 -0
- package/dist/capabilities/storyboard-measured-sources.d.ts +10 -0
- package/dist/capabilities/storyboard-measured-sources.d.ts.map +1 -0
- package/dist/capabilities/storyboard-measured-sources.js +67 -0
- package/dist/capabilities/storyboard-measured-sources.js.map +1 -0
- package/dist/capabilities/storyboard-review-times.d.ts +4 -0
- package/dist/capabilities/storyboard-review-times.d.ts.map +1 -0
- package/dist/capabilities/storyboard-review-times.js +39 -0
- package/dist/capabilities/storyboard-review-times.js.map +1 -0
- package/dist/capabilities/storyboards.d.ts +12 -18
- package/dist/capabilities/storyboards.d.ts.map +1 -1
- package/dist/capabilities/storyboards.js +779 -96
- package/dist/capabilities/storyboards.js.map +1 -1
- package/dist/capabilities/storyboards.specs.d.ts +9 -0
- package/dist/capabilities/storyboards.specs.d.ts.map +1 -1
- package/dist/capabilities/storyboards.specs.js +173 -5
- package/dist/capabilities/storyboards.specs.js.map +1 -1
- package/dist/capabilities/threads.js +1 -1
- package/dist/capabilities/threads.js.map +1 -1
- package/dist/capabilities/timeline-bridge.d.ts +368 -0
- package/dist/capabilities/timeline-bridge.d.ts.map +1 -0
- package/dist/capabilities/timeline-bridge.js +1008 -0
- package/dist/capabilities/timeline-bridge.js.map +1 -0
- package/dist/capabilities/timeline-clip-frames.d.ts +17 -0
- package/dist/capabilities/timeline-clip-frames.d.ts.map +1 -0
- package/dist/capabilities/timeline-clip-frames.js +84 -0
- package/dist/capabilities/timeline-clip-frames.js.map +1 -0
- package/dist/capabilities/timeline-media-edit.d.ts +8 -0
- package/dist/capabilities/timeline-media-edit.d.ts.map +1 -0
- package/dist/capabilities/timeline-media-edit.js +94 -0
- package/dist/capabilities/timeline-media-edit.js.map +1 -0
- package/dist/capabilities/timeline-native-media.d.ts.map +1 -1
- package/dist/capabilities/timeline-native-media.js +40 -0
- package/dist/capabilities/timeline-native-media.js.map +1 -1
- package/dist/capabilities/timeline-operations.d.ts +52 -0
- package/dist/capabilities/timeline-operations.d.ts.map +1 -0
- package/dist/capabilities/timeline-operations.js +413 -0
- package/dist/capabilities/timeline-operations.js.map +1 -0
- package/dist/capabilities/timelines.d.ts +28 -3
- package/dist/capabilities/timelines.d.ts.map +1 -1
- package/dist/capabilities/timelines.js +410 -294
- package/dist/capabilities/timelines.js.map +1 -1
- package/dist/capabilities/timelines.specs.d.ts +8 -0
- package/dist/capabilities/timelines.specs.d.ts.map +1 -1
- package/dist/capabilities/timelines.specs.js +178 -9
- package/dist/capabilities/timelines.specs.js.map +1 -1
- package/dist/capabilities/types.d.ts +3 -1
- package/dist/capabilities/types.d.ts.map +1 -1
- package/dist/capabilities/types.js.map +1 -1
- package/dist/capabilities/ui.d.ts.map +1 -1
- package/dist/capabilities/ui.js +24 -2
- package/dist/capabilities/ui.js.map +1 -1
- package/dist/capabilities/web.d.ts.map +1 -1
- package/dist/capabilities/web.js +4 -0
- package/dist/capabilities/web.js.map +1 -1
- package/dist/capabilities/workflows.d.ts +3 -1
- package/dist/capabilities/workflows.d.ts.map +1 -1
- package/dist/capabilities/workflows.js +244 -39
- package/dist/capabilities/workflows.js.map +1 -1
- package/dist/capabilities/workflows.specs.d.ts +2 -0
- package/dist/capabilities/workflows.specs.d.ts.map +1 -1
- package/dist/capabilities/workflows.specs.js +72 -2
- package/dist/capabilities/workflows.specs.js.map +1 -1
- package/dist/codeact/action-diagnostics.d.ts.map +1 -1
- package/dist/codeact/action-diagnostics.js +51 -1
- package/dist/codeact/action-diagnostics.js.map +1 -1
- package/dist/codeact/codeact-executor.d.ts.map +1 -1
- package/dist/codeact/codeact-executor.js +16 -8
- package/dist/codeact/codeact-executor.js.map +1 -1
- package/dist/codeact/compact-ids.d.ts.map +1 -1
- package/dist/codeact/compact-ids.js +5 -1
- package/dist/codeact/compact-ids.js.map +1 -1
- package/dist/codeact/graph-model.d.ts +1 -1
- package/dist/codeact/graph-model.d.ts.map +1 -1
- package/dist/codeact/graph-model.js +2 -3
- package/dist/codeact/graph-model.js.map +1 -1
- package/dist/codeact/nodetool-api.d.ts +1 -1
- package/dist/codeact/nodetool-api.d.ts.map +1 -1
- package/dist/codeact/nodetool-api.js +202 -284
- package/dist/codeact/nodetool-api.js.map +1 -1
- package/dist/codeact/sandbox-packages.d.ts +0 -7
- package/dist/codeact/sandbox-packages.d.ts.map +1 -1
- package/dist/codeact/sandbox-packages.js +1 -10
- package/dist/codeact/sandbox-packages.js.map +1 -1
- package/dist/codeact/tool-api.d.ts +6 -3
- package/dist/codeact/tool-api.d.ts.map +1 -1
- package/dist/codeact/tool-api.js +22 -15
- package/dist/codeact/tool-api.js.map +1 -1
- package/dist/evals/codeact-api-cases.d.ts.map +1 -1
- package/dist/evals/codeact-api-cases.js +2 -0
- package/dist/evals/codeact-api-cases.js.map +1 -1
- package/dist/evals/codeact-api-runs.d.ts +3 -0
- package/dist/evals/codeact-api-runs.d.ts.map +1 -0
- package/dist/evals/codeact-api-runs.js +33 -0
- package/dist/evals/codeact-api-runs.js.map +1 -0
- package/dist/evals/surfaces/game.d.ts +8 -2
- package/dist/evals/surfaces/game.d.ts.map +1 -1
- package/dist/evals/surfaces/game.js +460 -2
- package/dist/evals/surfaces/game.js.map +1 -1
- package/dist/evals/surfaces/sketch.d.ts +2 -0
- package/dist/evals/surfaces/sketch.d.ts.map +1 -1
- package/dist/evals/surfaces/sketch.js +11 -3
- package/dist/evals/surfaces/sketch.js.map +1 -1
- package/dist/evals/surfaces/timeline.d.ts +4 -357
- package/dist/evals/surfaces/timeline.d.ts.map +1 -1
- package/dist/evals/surfaces/timeline.js +9 -2621
- package/dist/evals/surfaces/timeline.js.map +1 -1
- package/dist/evals/tool-loop-bridge.d.ts +4 -0
- package/dist/evals/tool-loop-bridge.d.ts.map +1 -1
- package/dist/evals/tool-loop-bridge.js.map +1 -1
- package/dist/evals/tool-loop-eval.d.ts +2 -0
- package/dist/evals/tool-loop-eval.d.ts.map +1 -1
- package/dist/evals/tool-loop-eval.js +6 -1
- package/dist/evals/tool-loop-eval.js.map +1 -1
- package/dist/evals/tool-loop-permission.d.ts +1 -15
- package/dist/evals/tool-loop-permission.d.ts.map +1 -1
- package/dist/evals/tool-loop-permission.js +23 -41
- package/dist/evals/tool-loop-permission.js.map +1 -1
- package/dist/game-authoring.d.ts +9573 -0
- package/dist/game-authoring.d.ts.map +1 -0
- package/dist/game-authoring.js +146 -0
- package/dist/game-authoring.js.map +1 -0
- package/dist/game-build.d.ts +3 -0
- package/dist/game-build.d.ts.map +1 -0
- package/dist/game-build.js +3 -0
- package/dist/game-build.js.map +1 -0
- package/dist/game-code-bake.d.ts +26 -0
- package/dist/game-code-bake.d.ts.map +1 -0
- package/dist/game-code-bake.js +98 -0
- package/dist/game-code-bake.js.map +1 -0
- package/dist/game-examples.d.ts +2227 -18
- package/dist/game-examples.d.ts.map +1 -1
- package/dist/game-examples.js +5 -4
- package/dist/game-examples.js.map +1 -1
- package/dist/generation-cancel.d.ts +31 -0
- package/dist/generation-cancel.d.ts.map +1 -0
- package/dist/generation-cancel.js +28 -0
- package/dist/generation-cancel.js.map +1 -0
- package/dist/host-access-gate.d.ts +29 -0
- package/dist/host-access-gate.d.ts.map +1 -0
- package/dist/host-access-gate.js +39 -0
- package/dist/host-access-gate.js.map +1 -0
- package/dist/host-modules/dispatcher.d.ts +7 -1
- package/dist/host-modules/dispatcher.d.ts.map +1 -1
- package/dist/host-modules/dispatcher.js +13 -2
- package/dist/host-modules/dispatcher.js.map +1 -1
- package/dist/host-modules/docx.d.ts.map +1 -1
- package/dist/host-modules/docx.js +2 -2
- package/dist/host-modules/docx.js.map +1 -1
- package/dist/host-modules/epub.d.ts.map +1 -1
- package/dist/host-modules/epub.js +2 -2
- package/dist/host-modules/epub.js.map +1 -1
- package/dist/host-modules/fabric.d.ts.map +1 -1
- package/dist/host-modules/fabric.js +1 -1
- package/dist/host-modules/fabric.js.map +1 -1
- package/dist/host-modules/limits.d.ts +10 -8
- package/dist/host-modules/limits.d.ts.map +1 -1
- package/dist/host-modules/limits.js +14 -9
- package/dist/host-modules/limits.js.map +1 -1
- package/dist/host-modules/mammoth.d.ts.map +1 -1
- package/dist/host-modules/mammoth.js +2 -2
- package/dist/host-modules/mammoth.js.map +1 -1
- package/dist/host-modules/ocr.d.ts.map +1 -1
- package/dist/host-modules/ocr.js +1 -1
- package/dist/host-modules/ocr.js.map +1 -1
- package/dist/host-modules/pdf.d.ts.map +1 -1
- package/dist/host-modules/pdf.js +2 -2
- package/dist/host-modules/pdf.js.map +1 -1
- package/dist/host-modules/pdflib.d.ts.map +1 -1
- package/dist/host-modules/pdflib.js +2 -2
- package/dist/host-modules/pdflib.js.map +1 -1
- package/dist/host-modules/pptx.d.ts.map +1 -1
- package/dist/host-modules/pptx.js +2 -2
- package/dist/host-modules/pptx.js.map +1 -1
- package/dist/host-modules/pptxgen.d.ts.map +1 -1
- package/dist/host-modules/pptxgen.js +2 -2
- package/dist/host-modules/pptxgen.js.map +1 -1
- package/dist/host-modules/tfjs.d.ts.map +1 -1
- package/dist/host-modules/tfjs.js +4 -4
- package/dist/host-modules/tfjs.js.map +1 -1
- package/dist/host-modules/xlsx.d.ts.map +1 -1
- package/dist/host-modules/xlsx.js +2 -2
- package/dist/host-modules/xlsx.js.map +1 -1
- package/dist/index.d.ts +18 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +17 -3
- package/dist/index.js.map +1 -1
- package/dist/js-sandbox-worker/engine.d.ts +4 -0
- package/dist/js-sandbox-worker/engine.d.ts.map +1 -0
- package/dist/js-sandbox-worker/engine.js +45 -0
- package/dist/js-sandbox-worker/engine.js.map +1 -0
- package/dist/js-sandbox-worker/host.js +1 -1
- package/dist/js-sandbox-worker/host.js.map +1 -1
- package/dist/js-sandbox-worker/interpreter.d.ts +1 -1
- package/dist/js-sandbox-worker/interpreter.d.ts.map +1 -1
- package/dist/js-sandbox-worker/interpreter.js +94 -2
- package/dist/js-sandbox-worker/interpreter.js.map +1 -1
- package/dist/js-sandbox-worker/worker-entry.d.ts.map +1 -1
- package/dist/js-sandbox-worker/worker-entry.js +3 -2
- package/dist/js-sandbox-worker/worker-entry.js.map +1 -1
- package/dist/js-sandbox.d.ts +9 -1
- package/dist/js-sandbox.d.ts.map +1 -1
- package/dist/js-sandbox.js +76 -15
- package/dist/js-sandbox.js.map +1 -1
- package/dist/js-script-app-runner.d.ts +2 -0
- package/dist/js-script-app-runner.d.ts.map +1 -1
- package/dist/js-script-app-runner.js +8 -3
- package/dist/js-script-app-runner.js.map +1 -1
- package/dist/js-script-sandbox.d.ts +12 -18
- package/dist/js-script-sandbox.d.ts.map +1 -1
- package/dist/js-script-sandbox.js +3 -49
- package/dist/js-script-sandbox.js.map +1 -1
- package/dist/prompts/product-knowledge.d.ts +1 -1
- package/dist/prompts/product-knowledge.d.ts.map +1 -1
- package/dist/prompts/product-knowledge.js +82 -41
- package/dist/prompts/product-knowledge.js.map +1 -1
- package/dist/sandbox-pack-resolution.d.ts +21 -0
- package/dist/sandbox-pack-resolution.d.ts.map +1 -0
- package/dist/sandbox-pack-resolution.js +42 -0
- package/dist/sandbox-pack-resolution.js.map +1 -0
- package/dist/sandbox-toolbelt.d.ts +3 -1
- package/dist/sandbox-toolbelt.d.ts.map +1 -1
- package/dist/sandbox-toolbelt.js +8 -2
- package/dist/sandbox-toolbelt.js.map +1 -1
- package/dist/skill-prompt.d.ts +11 -1
- package/dist/skill-prompt.d.ts.map +1 -1
- package/dist/skill-prompt.js +17 -6
- package/dist/skill-prompt.js.map +1 -1
- package/dist/system-skills.d.ts +7 -1
- package/dist/system-skills.d.ts.map +1 -1
- package/dist/system-skills.js +2 -2
- package/dist/system-skills.js.map +1 -1
- package/dist/timeline-capture.d.ts +21 -0
- package/dist/timeline-capture.d.ts.map +1 -0
- package/dist/timeline-capture.js +564 -0
- package/dist/timeline-capture.js.map +1 -0
- package/dist/timeline-code-bake.d.ts +75 -0
- package/dist/timeline-code-bake.d.ts.map +1 -0
- package/dist/timeline-code-bake.js +327 -0
- package/dist/timeline-code-bake.js.map +1 -0
- package/dist/timeline-preview/frames.d.ts +26 -0
- package/dist/timeline-preview/frames.d.ts.map +1 -1
- package/dist/timeline-preview/frames.js +155 -70
- package/dist/timeline-preview/frames.js.map +1 -1
- package/dist/tools/base-tool.d.ts +1 -0
- package/dist/tools/base-tool.d.ts.map +1 -1
- package/dist/tools/base-tool.js +1 -1
- package/dist/tools/base-tool.js.map +1 -1
- package/dist/tools/builtin-tools.d.ts +4 -3
- package/dist/tools/builtin-tools.d.ts.map +1 -1
- package/dist/tools/builtin-tools.js +24 -3
- package/dist/tools/builtin-tools.js.map +1 -1
- package/dist/tools/external-capability-tools.d.ts +1 -1
- package/dist/tools/external-capability-tools.js +1 -1
- package/dist/tools/mcp-tool-support.d.ts +10 -9
- package/dist/tools/mcp-tool-support.d.ts.map +1 -1
- package/dist/tools/mcp-tool-support.js +14 -13
- package/dist/tools/mcp-tool-support.js.map +1 -1
- package/dist/tools/mcp-tools.d.ts +6 -4
- package/dist/tools/mcp-tools.d.ts.map +1 -1
- package/dist/tools/mcp-tools.js +65 -62
- package/dist/tools/mcp-tools.js.map +1 -1
- package/dist/tools/tool-permissions.d.ts +1 -1
- package/dist/tools/tool-permissions.js +1 -1
- package/package.json +124 -41
- package/dist/capabilities/gate-tools.d.ts +0 -18
- package/dist/capabilities/gate-tools.d.ts.map +0 -1
- package/dist/capabilities/gate-tools.js +0 -66
- package/dist/capabilities/gate-tools.js.map +0 -1
- package/dist/tsconfig.tests.tmp.tsbuildinfo +0 -1
|
@@ -22,7 +22,7 @@ import { Buffer } from "node:buffer";
|
|
|
22
22
|
import { randomInt, randomUUID } from "node:crypto";
|
|
23
23
|
import { mkdir, readFile } from "node:fs/promises";
|
|
24
24
|
import path from "node:path";
|
|
25
|
-
import { loadMediaRefBytes } from "@nodetool-ai/runtime";
|
|
25
|
+
import { currentGenerationReceipt, loadMediaRefBytes } from "@nodetool-ai/runtime";
|
|
26
26
|
import { getMaxLocalUploadBytes } from "@nodetool-ai/storage";
|
|
27
27
|
import { backgroundGenerationLimitError, startBackgroundGeneration } from "./background-generation.js";
|
|
28
28
|
import { HostBinaryMissingError, clampTimeoutSeconds, mimeFromFilename, runHostBinary } from "../host-binaries.js";
|
|
@@ -31,7 +31,7 @@ import { assertFetchUrlAllowed, assertResolvedHostAllowed } from "../network-gua
|
|
|
31
31
|
import { encodeBase64 as encodeMediaBase64 } from "../sandbox-bytes.js";
|
|
32
32
|
import { imagePixelSize, sniffImageFormat } from "../sandbox-media.js";
|
|
33
33
|
import { DEFAULT_MIME, MAX_MEDIA_REF_BYTES, filesystemPathForUri, mimeForRef } from "../sandbox-media-ref.js";
|
|
34
|
-
import { MIME_TO_EXT, inferImageMime, persistOutput } from "../tools/asset-persist.js";
|
|
34
|
+
import { MIME_TO_EXT, inferImageMime, persistOutput, timestampedName } from "../tools/asset-persist.js";
|
|
35
35
|
import { persistBinaryOutput } from "../tools/binary-output.js";
|
|
36
36
|
import { extractJSON, extractJSONAllowingTruncation } from "../utils/json-parser.js";
|
|
37
37
|
import { isYtDlpEnabled } from "../yt-dlp-gate.js";
|
|
@@ -267,16 +267,6 @@ async function finishOutput(context, bytes, assets, opts) {
|
|
|
267
267
|
}
|
|
268
268
|
return result;
|
|
269
269
|
}
|
|
270
|
-
/**
|
|
271
|
-
* The asset the seam saved for a generation, read off the registry's settled
|
|
272
|
-
* outcome. The encoded-audio paths return the provider's result rather than
|
|
273
|
-
* the seam's, so this is how a capability learns what was persisted.
|
|
274
|
-
*/
|
|
275
|
-
async function seamAssetFor(id) {
|
|
276
|
-
const { generationRegistry } = await import("@nodetool-ai/runtime");
|
|
277
|
-
const outcome = generationRegistry.outcome(id);
|
|
278
|
-
return outcome?.asset_ids[0] ?? null;
|
|
279
|
-
}
|
|
280
270
|
/** What a `background: true` call answers with. */
|
|
281
271
|
function backgroundReceipt(id, spec) {
|
|
282
272
|
return {
|
|
@@ -375,6 +365,8 @@ const generateImage = {
|
|
|
375
365
|
negative_prompt: params["negative_prompt"],
|
|
376
366
|
width: params["width"],
|
|
377
367
|
height: params["height"],
|
|
368
|
+
aspect_ratio: aliased(params, "aspect_ratio", "aspectRatio", "aspect"),
|
|
369
|
+
resolution: aliased(params, "resolution"),
|
|
378
370
|
quality: params["quality"]
|
|
379
371
|
}
|
|
380
372
|
});
|
|
@@ -418,6 +410,9 @@ const editImage = {
|
|
|
418
410
|
negative_prompt: params["negative_prompt"],
|
|
419
411
|
target_width: params["target_width"],
|
|
420
412
|
target_height: params["target_height"],
|
|
413
|
+
aspect_ratio: aliased(params, "aspect_ratio", "aspectRatio", "aspect"),
|
|
414
|
+
resolution: aliased(params, "resolution"),
|
|
415
|
+
quality: params["quality"],
|
|
421
416
|
strength: params["strength"]
|
|
422
417
|
}
|
|
423
418
|
});
|
|
@@ -742,6 +737,9 @@ const videoFromReferences = {
|
|
|
742
737
|
});
|
|
743
738
|
}
|
|
744
739
|
};
|
|
740
|
+
// ---------------------------------------------------------------------------
|
|
741
|
+
// generate_speech
|
|
742
|
+
// ---------------------------------------------------------------------------
|
|
745
743
|
function int16ToUint8(samples) {
|
|
746
744
|
const bytes = new Uint8Array(samples.byteLength);
|
|
747
745
|
const view = new DataView(bytes.buffer);
|
|
@@ -840,228 +838,244 @@ async function encodedSpeech(context, m, params, generation) {
|
|
|
840
838
|
return await provider.textToSpeechEncoded({
|
|
841
839
|
text: params.text,
|
|
842
840
|
model: m.model,
|
|
843
|
-
voice: params.voice,
|
|
844
|
-
speed: params.speed,
|
|
841
|
+
voice: isString(params.voice) ? params.voice : undefined,
|
|
842
|
+
speed: typeof params.speed === "number" ? params.speed : undefined,
|
|
845
843
|
audioFormat: params.audioFormat
|
|
846
844
|
});
|
|
847
845
|
}
|
|
848
846
|
throw new Error("this runtime's context has no encoded text_to_speech path");
|
|
849
847
|
}
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
text,
|
|
872
|
-
voice: params["voice"],
|
|
873
|
-
speed: params["speed"],
|
|
874
|
-
audioFormat: desiredFormat
|
|
875
|
-
};
|
|
876
|
-
if (params["background"] === true) {
|
|
877
|
-
const spec = {
|
|
878
|
-
capability: "text_to_speech",
|
|
879
|
-
m,
|
|
880
|
-
type: "audio",
|
|
881
|
-
namePrefix: "generated-speech",
|
|
882
|
-
params: speechParams
|
|
848
|
+
async function speechAudio(context, provider, m, params, signal, outputFile) {
|
|
849
|
+
signal.throwIfAborted();
|
|
850
|
+
const providerParams = {
|
|
851
|
+
text: params.text,
|
|
852
|
+
model: m.model,
|
|
853
|
+
voice: isString(params.voice) ? params.voice : undefined,
|
|
854
|
+
speed: typeof params.speed === "number" ? params.speed : undefined,
|
|
855
|
+
audioFormat: params.audioFormat,
|
|
856
|
+
signal
|
|
857
|
+
};
|
|
858
|
+
let encodedError;
|
|
859
|
+
try {
|
|
860
|
+
const encoded = provider
|
|
861
|
+
? await provider.textToSpeechEncoded(providerParams)
|
|
862
|
+
: await encodedSpeech(context, m, params);
|
|
863
|
+
signal.throwIfAborted();
|
|
864
|
+
if (encoded?.data && encoded.data.length > 0) {
|
|
865
|
+
return {
|
|
866
|
+
data: encoded.data,
|
|
867
|
+
mimeType: encoded.mimeType ?? "audio/mpeg",
|
|
868
|
+
outputFile
|
|
883
869
|
};
|
|
884
|
-
const started = startBackgroundGeneration(context, () => encodedSpeech(context, m, speechParams, generation));
|
|
885
|
-
if (!started) {
|
|
886
|
-
return backgroundGenerationLimitError();
|
|
887
|
-
}
|
|
888
|
-
return backgroundReceipt(generationId, spec);
|
|
889
870
|
}
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
871
|
+
}
|
|
872
|
+
catch (error) {
|
|
873
|
+
if (signal.aborted || currentGenerationReceipt()?.provider_request_id) {
|
|
874
|
+
throw error;
|
|
875
|
+
}
|
|
876
|
+
encodedError = error;
|
|
877
|
+
}
|
|
878
|
+
if (currentGenerationReceipt()?.provider_request_id) {
|
|
879
|
+
throw new Error("The accepted speech request returned no audio; recover that request before submitting another.");
|
|
880
|
+
}
|
|
881
|
+
const parts = [];
|
|
882
|
+
let pcmOnly = true;
|
|
883
|
+
let mimeType;
|
|
884
|
+
let sampleRate;
|
|
885
|
+
try {
|
|
886
|
+
const stream = provider
|
|
887
|
+
? provider.textToSpeech(providerParams)
|
|
888
|
+
: context.streamProviderPrediction({
|
|
889
|
+
provider: m.provider,
|
|
890
|
+
capability: "text_to_speech",
|
|
891
|
+
model: m.model,
|
|
892
|
+
params: { ...providerParams, signal }
|
|
893
|
+
});
|
|
894
|
+
for await (const item of stream) {
|
|
895
|
+
signal.throwIfAborted();
|
|
896
|
+
if (!isRecord(item)) {
|
|
897
|
+
continue;
|
|
913
898
|
}
|
|
914
|
-
|
|
915
|
-
|
|
899
|
+
const chunk = item;
|
|
900
|
+
if (chunk.data instanceof Uint8Array || isString(chunk.data)) {
|
|
901
|
+
const data = chunk.data instanceof Uint8Array
|
|
902
|
+
? chunk.data
|
|
903
|
+
: Buffer.from(chunk.data, "base64");
|
|
904
|
+
if (data.length > 0) {
|
|
905
|
+
parts.push(data);
|
|
906
|
+
}
|
|
907
|
+
if (isString(chunk.mimeType)) {
|
|
908
|
+
mimeType = chunk.mimeType;
|
|
909
|
+
}
|
|
910
|
+
pcmOnly = false;
|
|
916
911
|
}
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
// PCM (MiniMax 32000, Together varies). Hardcoding 24000 in the WAV
|
|
925
|
-
// header makes those play back at the wrong speed/pitch.
|
|
926
|
-
let pcmSampleRate;
|
|
927
|
-
try {
|
|
928
|
-
for await (const item of context.streamProviderPrediction({
|
|
929
|
-
id: generationId,
|
|
930
|
-
origin: generation.origin,
|
|
931
|
-
provider: m.provider,
|
|
932
|
-
capability: "text_to_speech",
|
|
933
|
-
model: m.model,
|
|
934
|
-
params: {
|
|
935
|
-
text,
|
|
936
|
-
voice: params["voice"],
|
|
937
|
-
speed: params["speed"]
|
|
938
|
-
}
|
|
939
|
-
})) {
|
|
940
|
-
const chunk = item;
|
|
941
|
-
if (chunk.data instanceof Uint8Array) {
|
|
942
|
-
parts.push(chunk.data);
|
|
943
|
-
if (chunk.mimeType)
|
|
944
|
-
mimeType = chunk.mimeType;
|
|
945
|
-
pcmOnly = false;
|
|
946
|
-
}
|
|
947
|
-
else if (isString(chunk.data)) {
|
|
948
|
-
parts.push(Buffer.from(chunk.data, "base64"));
|
|
949
|
-
if (chunk.mimeType)
|
|
950
|
-
mimeType = chunk.mimeType;
|
|
951
|
-
pcmOnly = false;
|
|
952
|
-
}
|
|
953
|
-
else if (chunk.samples) {
|
|
954
|
-
pcmSampleRate ??= chunk.sampleRate;
|
|
955
|
-
parts.push(int16ToUint8(chunk.samples));
|
|
956
|
-
}
|
|
912
|
+
else {
|
|
913
|
+
if (chunk.sampleRate !== undefined) {
|
|
914
|
+
if (typeof chunk.sampleRate !== "number" ||
|
|
915
|
+
!Number.isInteger(chunk.sampleRate) ||
|
|
916
|
+
chunk.sampleRate <= 0 ||
|
|
917
|
+
chunk.sampleRate > 0xffffffff / 2) {
|
|
918
|
+
throw new Error("TTS returned an invalid PCM sample rate");
|
|
957
919
|
}
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
// Both paths failed. Report the encoded one first: on a provider
|
|
961
|
-
// that only does encoded TTS the streaming error is the base class's
|
|
962
|
-
// "does not support textToSpeech", which is true of the method and
|
|
963
|
-
// false of the provider.
|
|
964
|
-
if (encodedError) {
|
|
965
|
-
throw new Error(`${encodedError} (the streaming fallback then failed too: ${e instanceof Error ? e.message : String(e)})`);
|
|
920
|
+
if (sampleRate !== undefined && sampleRate !== chunk.sampleRate) {
|
|
921
|
+
throw new Error("TTS changed PCM sample rate during the stream");
|
|
966
922
|
}
|
|
967
|
-
|
|
923
|
+
sampleRate = chunk.sampleRate;
|
|
968
924
|
}
|
|
969
|
-
if (
|
|
970
|
-
|
|
971
|
-
error: encodedError
|
|
972
|
-
? `Provider returned no audio data: ${encodedError}`
|
|
973
|
-
: "Provider returned no audio data"
|
|
974
|
-
};
|
|
925
|
+
if (chunk.samples instanceof Int16Array && chunk.samples.length > 0) {
|
|
926
|
+
parts.push(int16ToUint8(chunk.samples));
|
|
975
927
|
}
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
928
|
+
}
|
|
929
|
+
}
|
|
930
|
+
}
|
|
931
|
+
catch (error) {
|
|
932
|
+
if (encodedError) {
|
|
933
|
+
throw new Error(`${encodedError instanceof Error ? encodedError.message : String(encodedError)} (the streaming fallback then failed too: ${error instanceof Error ? error.message : String(error)})`);
|
|
934
|
+
}
|
|
935
|
+
throw error;
|
|
936
|
+
}
|
|
937
|
+
signal.throwIfAborted();
|
|
938
|
+
const merged = concatUint8(parts);
|
|
939
|
+
if (merged.length === 0) {
|
|
940
|
+
throw new Error("Provider returned no audio data");
|
|
941
|
+
}
|
|
942
|
+
if (pcmOnly && !mimeType) {
|
|
943
|
+
if (merged.length < 2 || merged.length % 2 !== 0) {
|
|
944
|
+
throw new Error("TTS produced no complete audio samples");
|
|
945
|
+
}
|
|
946
|
+
const finalPath = outputFile
|
|
947
|
+
? path.join(path.dirname(outputFile), `${path.basename(outputFile, path.extname(outputFile))}.wav`)
|
|
948
|
+
: undefined;
|
|
949
|
+
return {
|
|
950
|
+
data: wrapPcmAsWav(merged, sampleRate ?? 24000),
|
|
951
|
+
mimeType: "audio/wav",
|
|
952
|
+
outputFile: finalPath
|
|
953
|
+
};
|
|
954
|
+
}
|
|
955
|
+
return { data: merged, mimeType: mimeType ?? "audio/mpeg", outputFile };
|
|
956
|
+
}
|
|
957
|
+
function runAudioGeneration(context, req, spec, params, call) {
|
|
958
|
+
const work = async () => {
|
|
959
|
+
let saved;
|
|
960
|
+
const result = await context.runGenerationWith(req, async (provider, signal) => {
|
|
961
|
+
const audio = await call(provider, signal);
|
|
962
|
+
if (audio.outputFile && req.persist) {
|
|
963
|
+
req.persist.name = path.basename(audio.outputFile);
|
|
964
|
+
}
|
|
965
|
+
return audio;
|
|
966
|
+
}, {
|
|
967
|
+
finalizeOutput: async (output, assets) => {
|
|
968
|
+
if (!isRecord(output) ||
|
|
969
|
+
!(output.data instanceof Uint8Array) ||
|
|
970
|
+
!isString(output.mimeType)) {
|
|
971
|
+
throw new Error("Provider returned invalid audio output");
|
|
988
972
|
}
|
|
989
|
-
|
|
990
|
-
|
|
973
|
+
const mime = output.mimeType;
|
|
974
|
+
const first = assets[0];
|
|
975
|
+
saved = { bytes: output.data.length, mime_type: mime };
|
|
976
|
+
if (first?.asset_id) {
|
|
977
|
+
saved.asset_id = first.asset_id;
|
|
978
|
+
saved.asset_uri = first.uri;
|
|
979
|
+
saved.url = first.uri;
|
|
991
980
|
}
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
981
|
+
const outputFile = isString(output.outputFile)
|
|
982
|
+
? output.outputFile
|
|
983
|
+
: undefined;
|
|
984
|
+
if (outputFile || !saved.asset_id) {
|
|
985
|
+
const destination = outputFile ??
|
|
986
|
+
timestampedName(spec.namePrefix, MIME_TO_EXT[mime] ?? "bin");
|
|
987
|
+
if (!context.workspace) {
|
|
988
|
+
return {
|
|
989
|
+
status: "failed",
|
|
990
|
+
path: destination,
|
|
991
|
+
error: `Could not deliver audio to "${destination}": no workspace is available.`
|
|
992
|
+
};
|
|
999
993
|
}
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
994
|
+
try {
|
|
995
|
+
await context.workspace.write(destination, output.data, mime);
|
|
996
|
+
}
|
|
997
|
+
catch (error) {
|
|
998
|
+
return {
|
|
999
|
+
status: "failed",
|
|
1000
|
+
path: destination,
|
|
1001
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1002
|
+
};
|
|
1003
|
+
}
|
|
1004
|
+
saved.path = destination;
|
|
1005
|
+
return { status: "completed", path: destination };
|
|
1006
|
+
}
|
|
1007
|
+
return { status: "completed" };
|
|
1008
|
+
}
|
|
1009
|
+
});
|
|
1010
|
+
const response = {
|
|
1011
|
+
type: "audio",
|
|
1012
|
+
provider: spec.m.provider,
|
|
1013
|
+
model: spec.m.model,
|
|
1014
|
+
generation_id: result.id,
|
|
1015
|
+
...saved
|
|
1016
|
+
};
|
|
1017
|
+
if (result.delivery) {
|
|
1018
|
+
response.delivery = result.delivery;
|
|
1013
1019
|
}
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1020
|
+
return response;
|
|
1021
|
+
};
|
|
1022
|
+
if (params.background === true) {
|
|
1023
|
+
if (!startBackgroundGeneration(context, work)) {
|
|
1024
|
+
return backgroundGenerationLimitError();
|
|
1019
1025
|
}
|
|
1026
|
+
return backgroundReceipt(req.id ?? "", spec);
|
|
1027
|
+
}
|
|
1028
|
+
return work().catch((error) => ({
|
|
1029
|
+
...predictionError(spec.capability, spec.m, error),
|
|
1030
|
+
generation_id: req.id
|
|
1031
|
+
}));
|
|
1032
|
+
}
|
|
1033
|
+
const generateSpeech = {
|
|
1034
|
+
spec: generateSpeechSpec,
|
|
1035
|
+
impl: async (run, params) => {
|
|
1036
|
+
const m = parseModelArgs(params);
|
|
1037
|
+
if ("error" in m) {
|
|
1038
|
+
return m;
|
|
1039
|
+
}
|
|
1040
|
+
const text = params.text;
|
|
1041
|
+
if (!isNonEmptyString(text)) {
|
|
1042
|
+
return { error: "text is required" };
|
|
1043
|
+
}
|
|
1044
|
+
const outputFile = outputFileOf(params);
|
|
1045
|
+
const speechParams = {
|
|
1046
|
+
text,
|
|
1047
|
+
voice: params.voice,
|
|
1048
|
+
speed: params.speed,
|
|
1049
|
+
audioFormat: audioFormatFromOutputFile(outputFile) ?? "mp3"
|
|
1050
|
+
};
|
|
1051
|
+
const spec = {
|
|
1052
|
+
capability: "text_to_speech",
|
|
1053
|
+
m,
|
|
1054
|
+
type: "audio",
|
|
1055
|
+
namePrefix: "generated-speech",
|
|
1056
|
+
params: { ...speechParams }
|
|
1057
|
+
};
|
|
1058
|
+
const req = generationRequest(randomUUID(), params, spec);
|
|
1059
|
+
return runAudioGeneration(run.context, req, spec, params, (provider, signal) => speechAudio(run.context, provider, m, speechParams, signal, outputFile));
|
|
1020
1060
|
}
|
|
1021
1061
|
};
|
|
1022
|
-
/**
|
|
1023
|
-
* Music from a prompt — the counterpart to {@link generateSpeech}, and the
|
|
1024
|
-
* hole every other generation capability made conspicuous.
|
|
1025
|
-
*
|
|
1026
|
-
* Without it, scoring a cut meant running `nodetool.audio.TextToMusic` through
|
|
1027
|
-
* `run_node`: a one-node graph, and therefore the graph validator, a typed
|
|
1028
|
-
* `music_model` property and a model ref to satisfy before a single second of
|
|
1029
|
-
* audio existed. A live session spent five rounds there and shipped silent.
|
|
1030
|
-
*/
|
|
1031
1062
|
const generateMusic = {
|
|
1032
1063
|
spec: generateMusicSpec,
|
|
1033
1064
|
impl: async (run, params) => {
|
|
1034
|
-
const context = run.context;
|
|
1035
1065
|
const m = parseModelArgs(params);
|
|
1036
|
-
if ("error" in m)
|
|
1066
|
+
if ("error" in m) {
|
|
1037
1067
|
return m;
|
|
1038
|
-
|
|
1039
|
-
|
|
1068
|
+
}
|
|
1069
|
+
const prompt = params.prompt;
|
|
1070
|
+
if (!isNonEmptyString(prompt)) {
|
|
1040
1071
|
return { error: "prompt is required" };
|
|
1041
|
-
if (!isFunction(context.textToMusic)) {
|
|
1042
|
-
return {
|
|
1043
|
-
error: "this runtime's context has no text_to_music path"
|
|
1044
|
-
};
|
|
1045
1072
|
}
|
|
1046
|
-
const outputFile =
|
|
1047
|
-
? params["output_file"]
|
|
1048
|
-
: undefined;
|
|
1049
|
-
const desiredFormat = audioFormatFromOutputFile(outputFile) ?? "mp3";
|
|
1050
|
-
const generationId = randomUUID();
|
|
1073
|
+
const outputFile = outputFileOf(params);
|
|
1051
1074
|
const musicParams = {
|
|
1052
1075
|
prompt,
|
|
1053
|
-
lyrics: params
|
|
1076
|
+
lyrics: params.lyrics,
|
|
1054
1077
|
duration_seconds: aliased(params, "duration_seconds", "durationSeconds", "duration"),
|
|
1055
|
-
audioFormat:
|
|
1056
|
-
};
|
|
1057
|
-
const req = {
|
|
1058
|
-
id: generationId,
|
|
1059
|
-
provider: m.provider,
|
|
1060
|
-
capability: "text_to_music",
|
|
1061
|
-
model: m.model,
|
|
1062
|
-
params: musicParams,
|
|
1063
|
-
origin: { surface: "capability", tool_call_id: toolCallIdOf(params) },
|
|
1064
|
-
persist: outputFile ? { name: path.basename(outputFile) } : {}
|
|
1078
|
+
audioFormat: audioFormatFromOutputFile(outputFile) ?? "mp3"
|
|
1065
1079
|
};
|
|
1066
1080
|
const spec = {
|
|
1067
1081
|
capability: "text_to_music",
|
|
@@ -1070,45 +1084,31 @@ const generateMusic = {
|
|
|
1070
1084
|
namePrefix: "generated-music",
|
|
1071
1085
|
params: musicParams
|
|
1072
1086
|
};
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1087
|
+
const req = generationRequest(randomUUID(), params, spec);
|
|
1088
|
+
return runAudioGeneration(run.context, req, spec, params, async (provider, signal) => {
|
|
1089
|
+
signal.throwIfAborted();
|
|
1090
|
+
const encoded = provider
|
|
1091
|
+
? await provider.textToMusic({
|
|
1092
|
+
prompt,
|
|
1093
|
+
model: { id: m.model, name: m.model, provider: m.provider },
|
|
1094
|
+
lyrics: isString(params.lyrics) ? params.lyrics : undefined,
|
|
1095
|
+
durationSeconds: typeof musicParams.duration_seconds === "number"
|
|
1096
|
+
? musicParams.duration_seconds
|
|
1097
|
+
: undefined,
|
|
1098
|
+
audioFormat: musicParams.audioFormat,
|
|
1099
|
+
signal
|
|
1100
|
+
})
|
|
1101
|
+
: await run.context.textToMusic(req);
|
|
1102
|
+
signal.throwIfAborted();
|
|
1082
1103
|
if (!encoded?.data || encoded.data.length === 0) {
|
|
1083
|
-
|
|
1084
|
-
error: "Provider returned no audio data",
|
|
1085
|
-
generation_id: generationId
|
|
1086
|
-
};
|
|
1104
|
+
throw new Error("Provider returned no audio data");
|
|
1087
1105
|
}
|
|
1088
|
-
const mime = encoded.mimeType ?? "audio/mpeg";
|
|
1089
|
-
const seamAssetId = await seamAssetFor(generationId);
|
|
1090
|
-
const persisted = await finishOutput(context, encoded.data, seamAssetId
|
|
1091
|
-
? [
|
|
1092
|
-
{
|
|
1093
|
-
asset_id: seamAssetId,
|
|
1094
|
-
uri: `asset://${seamAssetId}.${MIME_TO_EXT[mime] ?? "bin"}`
|
|
1095
|
-
}
|
|
1096
|
-
]
|
|
1097
|
-
: [], { namePrefix: "generated-music", mime, outputFile });
|
|
1098
1106
|
return {
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
generation_id: generationId,
|
|
1103
|
-
...persisted
|
|
1107
|
+
data: encoded.data,
|
|
1108
|
+
mimeType: encoded.mimeType ?? "audio/mpeg",
|
|
1109
|
+
outputFile
|
|
1104
1110
|
};
|
|
1105
|
-
}
|
|
1106
|
-
catch (e) {
|
|
1107
|
-
return {
|
|
1108
|
-
...predictionError("text_to_music", m, e),
|
|
1109
|
-
generation_id: generationId
|
|
1110
|
-
};
|
|
1111
|
-
}
|
|
1111
|
+
});
|
|
1112
1112
|
}
|
|
1113
1113
|
};
|
|
1114
1114
|
const transcribeAudio = {
|
|
@@ -1244,7 +1244,7 @@ function messageText(message) {
|
|
|
1244
1244
|
}
|
|
1245
1245
|
return "";
|
|
1246
1246
|
}
|
|
1247
|
-
async function judgeCall(context, m, content, maxTokens = JUDGE_MAX_TOKENS) {
|
|
1247
|
+
async function judgeCall(context, m, content, maxTokens = JUDGE_MAX_TOKENS, requiredImages) {
|
|
1248
1248
|
const result = (await context.runProviderPrediction({
|
|
1249
1249
|
provider: m.provider,
|
|
1250
1250
|
capability: "generate_message",
|
|
@@ -1252,7 +1252,10 @@ async function judgeCall(context, m, content, maxTokens = JUDGE_MAX_TOKENS) {
|
|
|
1252
1252
|
params: {
|
|
1253
1253
|
messages: [{ role: "user", content }],
|
|
1254
1254
|
max_tokens: maxTokens,
|
|
1255
|
-
temperature: 0
|
|
1255
|
+
temperature: 0,
|
|
1256
|
+
strict_image_evidence: content.some((part) => part.type === "image_url") ||
|
|
1257
|
+
Boolean(requiredImages?.length),
|
|
1258
|
+
required_images: requiredImages?.map(normalizeMediaSource)
|
|
1256
1259
|
}
|
|
1257
1260
|
}));
|
|
1258
1261
|
return messageText(result);
|
|
@@ -1403,11 +1406,7 @@ const compareImages = {
|
|
|
1403
1406
|
`\n\nRespond with JSON only: {"winner": 1 | 2, "reason": "one sentence"}`;
|
|
1404
1407
|
const judgePair = async (a, b) => {
|
|
1405
1408
|
const call = async (first, second) => {
|
|
1406
|
-
const text = await judgeCall(context, m, [
|
|
1407
|
-
textPart(prompt),
|
|
1408
|
-
imagePart(first),
|
|
1409
|
-
imagePart(second)
|
|
1410
|
-
]);
|
|
1409
|
+
const text = await judgeCall(context, m, [textPart(prompt), imagePart(first), imagePart(second)], JUDGE_MAX_TOKENS, images.map(String));
|
|
1411
1410
|
const verdict = parsePairVerdict(text);
|
|
1412
1411
|
if (!verdict) {
|
|
1413
1412
|
throw new Error(`Judge did not return parseable JSON: ${text.slice(0, 200)}`);
|
|
@@ -1502,7 +1501,7 @@ const scoreImageAdherence = {
|
|
|
1502
1501
|
` visually checkable.` +
|
|
1503
1502
|
`\n\nBrief:\n${brief}` +
|
|
1504
1503
|
`\n\nRespond with JSON only: {"questions": ["..."]}`)
|
|
1505
|
-
]);
|
|
1504
|
+
], JUDGE_MAX_TOKENS, [image]);
|
|
1506
1505
|
const parsed = extractJSON(decomposeText);
|
|
1507
1506
|
const list = isRecord(parsed) ? parsed["questions"] : parsed;
|
|
1508
1507
|
questions = Array.isArray(list) ? list.map(String).filter(Boolean) : [];
|
|
@@ -1513,36 +1512,82 @@ const scoreImageAdherence = {
|
|
|
1513
1512
|
}
|
|
1514
1513
|
}
|
|
1515
1514
|
questions = questions.slice(0, MAX_ADHERENCE_QUESTIONS);
|
|
1515
|
+
const checks = questions.map((question, i) => ({
|
|
1516
|
+
id: `q${i + 1}`,
|
|
1517
|
+
question
|
|
1518
|
+
}));
|
|
1516
1519
|
const answerText = await judgeCall(context, m, [
|
|
1517
1520
|
textPart(`Answer each question about the image with a strict yes or no. "yes" only if` +
|
|
1518
1521
|
` the image clearly satisfies it; when unsure, answer "no" and say why in the note.` +
|
|
1519
|
-
`\n\nQuestions:\n${
|
|
1522
|
+
`\n\nQuestions:\n${checks.map((q) => `${q.id}: ${q.question}`).join("\n")}` +
|
|
1520
1523
|
`\n\nRespond with JSON only:` +
|
|
1521
|
-
`\n{"answers": [{"question": "...", "answer": "yes" | "no", "note": "..."}]}`),
|
|
1524
|
+
`\n{"answers": [{"id": "q1", "question": "...", "answer": "yes" | "no", "note": "..."}]}`),
|
|
1522
1525
|
imagePart(image)
|
|
1523
1526
|
]);
|
|
1524
1527
|
const parsed = extractJSON(answerText);
|
|
1525
1528
|
const rawAnswers = isRecord(parsed) ? parsed["answers"] : null;
|
|
1526
1529
|
if (!Array.isArray(rawAnswers) || rawAnswers.length === 0) {
|
|
1527
1530
|
return {
|
|
1528
|
-
error: `Judge did not return parseable answers: ${answerText.slice(0, 300)}
|
|
1531
|
+
error: `Judge did not return parseable answers: ${answerText.slice(0, 300)}`,
|
|
1532
|
+
evidence_status: "invalid",
|
|
1533
|
+
total: checks.length,
|
|
1534
|
+
missing_ids: checks.map((check) => check.id)
|
|
1529
1535
|
};
|
|
1530
1536
|
}
|
|
1531
|
-
const
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1537
|
+
const byId = new Map();
|
|
1538
|
+
const invalid = [];
|
|
1539
|
+
for (const raw of rawAnswers) {
|
|
1540
|
+
if (!isRecord(raw)) {
|
|
1541
|
+
invalid.push("answer must be an object");
|
|
1542
|
+
continue;
|
|
1543
|
+
}
|
|
1544
|
+
const id = String(raw["id"] ?? "");
|
|
1545
|
+
const check = checks.find((c) => c.id === id);
|
|
1546
|
+
if (!check || raw["question"] !== check.question) {
|
|
1547
|
+
invalid.push(`unrequested or substituted check: ${id}`);
|
|
1548
|
+
continue;
|
|
1549
|
+
}
|
|
1550
|
+
if (byId.has(id)) {
|
|
1551
|
+
invalid.push(`duplicate check: ${id}`);
|
|
1552
|
+
continue;
|
|
1553
|
+
}
|
|
1554
|
+
if (raw["answer"] !== "yes" && raw["answer"] !== "no") {
|
|
1555
|
+
invalid.push(`invalid answer: ${id}`);
|
|
1556
|
+
continue;
|
|
1557
|
+
}
|
|
1558
|
+
byId.set(id, {
|
|
1559
|
+
id,
|
|
1560
|
+
question: check.question,
|
|
1561
|
+
answer: raw["answer"],
|
|
1562
|
+
note: String(raw["note"] ?? "")
|
|
1563
|
+
});
|
|
1564
|
+
}
|
|
1565
|
+
const missingIds = checks.filter((c) => !byId.has(c.id)).map((c) => c.id);
|
|
1566
|
+
if (invalid.length || missingIds.length) {
|
|
1567
|
+
return {
|
|
1568
|
+
error: "Judge returned invalid or incomplete adherence evidence",
|
|
1569
|
+
evidence_status: "invalid",
|
|
1570
|
+
total: checks.length,
|
|
1571
|
+
missing_ids: missingIds,
|
|
1572
|
+
invalid
|
|
1573
|
+
};
|
|
1574
|
+
}
|
|
1575
|
+
const answers = [];
|
|
1576
|
+
for (const check of checks) {
|
|
1577
|
+
const answer = byId.get(check.id);
|
|
1578
|
+
if (answer) {
|
|
1579
|
+
answers.push(answer);
|
|
1580
|
+
}
|
|
1581
|
+
}
|
|
1538
1582
|
const passed = answers.filter((a) => a.answer === "yes").length;
|
|
1539
1583
|
return {
|
|
1540
1584
|
type: "adherence",
|
|
1585
|
+
evidence_status: "complete",
|
|
1541
1586
|
provider: m.provider,
|
|
1542
1587
|
model: m.model,
|
|
1543
|
-
score:
|
|
1588
|
+
score: passed / checks.length,
|
|
1544
1589
|
passed,
|
|
1545
|
-
total:
|
|
1590
|
+
total: checks.length,
|
|
1546
1591
|
failed: answers.filter((a) => a.answer === "no"),
|
|
1547
1592
|
answers
|
|
1548
1593
|
};
|
|
@@ -1810,7 +1855,8 @@ async function stageInputs(context, workspace, cwd, raw) {
|
|
|
1810
1855
|
const localAssetLimit = workspace.localDir
|
|
1811
1856
|
? getMaxLocalUploadBytes()
|
|
1812
1857
|
: undefined;
|
|
1813
|
-
const hasLocalAsset = localAssetLimit !== undefined &&
|
|
1858
|
+
const hasLocalAsset = localAssetLimit !== undefined &&
|
|
1859
|
+
entries.some(([, ref]) => isNonBlankString(ref) && ref.trim().startsWith("asset://"));
|
|
1814
1860
|
const totalLimit = hasLocalAsset
|
|
1815
1861
|
? Math.max(MAX_STAGED_TOTAL_BYTES, localAssetLimit ?? 0)
|
|
1816
1862
|
: MAX_STAGED_TOTAL_BYTES;
|
|
@@ -1826,7 +1872,14 @@ async function stageInputs(context, workspace, cwd, raw) {
|
|
|
1826
1872
|
let bytes = null;
|
|
1827
1873
|
let readError;
|
|
1828
1874
|
try {
|
|
1829
|
-
|
|
1875
|
+
// `loadMediaRefBytes` reads `file://` URIs and absolute paths straight
|
|
1876
|
+
// off the host disk. A filesystem-shaped ref resolves through the
|
|
1877
|
+
// storage adapter alone, which keeps it inside the storage root.
|
|
1878
|
+
const uri = ref.trim();
|
|
1879
|
+
bytes =
|
|
1880
|
+
filesystemPathForUri(uri) !== null
|
|
1881
|
+
? ((await context.storage?.retrieve(uri)) ?? null)
|
|
1882
|
+
: await loadMediaRefBytes({ uri }, context);
|
|
1830
1883
|
}
|
|
1831
1884
|
catch (e) {
|
|
1832
1885
|
// Keep why. Naming only the accepted forms is useless to a caller who
|