@hraness/slopcamera 3.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/NOTICE.md +60 -0
- package/PRIVACY.md +46 -0
- package/README.md +311 -0
- package/SECURITY.md +21 -0
- package/apps/desktop/README.md +966 -0
- package/apps/desktop/analysis/protocol.ts +223 -0
- package/apps/desktop/application/capability-binding.ts +594 -0
- package/apps/desktop/application/context.ts +97 -0
- package/apps/desktop/application/creative-iteration.ts +1385 -0
- package/apps/desktop/application/default-registry.ts +146 -0
- package/apps/desktop/application/directing-contract.ts +218 -0
- package/apps/desktop/application/directing-plan.ts +178 -0
- package/apps/desktop/application/errors.ts +18 -0
- package/apps/desktop/application/gateway-port.ts +867 -0
- package/apps/desktop/application/html-overlay-browser-runtime.ts +687 -0
- package/apps/desktop/application/html-overlay-integrity.ts +337 -0
- package/apps/desktop/application/html-overlay-renderer.ts +45 -0
- package/apps/desktop/application/index.ts +13 -0
- package/apps/desktop/application/operation-completion-checkpoint.ts +333 -0
- package/apps/desktop/application/operation-effects.ts +94 -0
- package/apps/desktop/application/operation.ts +224 -0
- package/apps/desktop/application/operations/analysis/faces.ts +244 -0
- package/apps/desktop/application/operations/analysis/music.ts +248 -0
- package/apps/desktop/application/operations/analysis/project-auto-zooms.ts +302 -0
- package/apps/desktop/application/operations/analysis/project-inactivity.ts +381 -0
- package/apps/desktop/application/operations/analysis/scenes.ts +272 -0
- package/apps/desktop/application/operations/analysis/shared.ts +202 -0
- package/apps/desktop/application/operations/derive/edit-batch.ts +967 -0
- package/apps/desktop/application/operations/derive/follow-faces.ts +454 -0
- package/apps/desktop/application/operations/edit/bind-candidate-revision.ts +91 -0
- package/apps/desktop/application/operations/edit/create-candidate-revision.ts +377 -0
- package/apps/desktop/application/operations/edit/create-revision.ts +255 -0
- package/apps/desktop/application/operations/edit/freeze-revision.ts +182 -0
- package/apps/desktop/application/operations/gateway/image.ts +41 -0
- package/apps/desktop/application/operations/gateway/index.ts +17 -0
- package/apps/desktop/application/operations/gateway/shared.ts +122 -0
- package/apps/desktop/application/operations/gateway/speech.ts +41 -0
- package/apps/desktop/application/operations/gateway/transcription.ts +42 -0
- package/apps/desktop/application/operations/gateway/video.ts +41 -0
- package/apps/desktop/application/operations/index.ts +31 -0
- package/apps/desktop/application/operations/iteration/create-candidate.ts +240 -0
- package/apps/desktop/application/operations/iteration/create-matrix.ts +117 -0
- package/apps/desktop/application/operations/iteration/select.ts +155 -0
- package/apps/desktop/application/operations/iteration/shared.ts +250 -0
- package/apps/desktop/application/operations/media/audio-effects.ts +376 -0
- package/apps/desktop/application/operations/media/binding.ts +79 -0
- package/apps/desktop/application/operations/media/capabilities.ts +112 -0
- package/apps/desktop/application/operations/media/color-grade.ts +343 -0
- package/apps/desktop/application/operations/media/html-overlay.ts +802 -0
- package/apps/desktop/application/operations/media/index.ts +8 -0
- package/apps/desktop/application/operations/media/ingest.ts +303 -0
- package/apps/desktop/application/operations/media/overlay.ts +729 -0
- package/apps/desktop/application/operations/media/shared.ts +845 -0
- package/apps/desktop/application/operations/media/transform-platform.ts +120 -0
- package/apps/desktop/application/operations/project/commit-edits.ts +2054 -0
- package/apps/desktop/application/operations/project/promote-selection.ts +495 -0
- package/apps/desktop/application/operations/project/snapshot.ts +95 -0
- package/apps/desktop/application/operations/recording/index.ts +5 -0
- package/apps/desktop/application/operations/recording/pause.ts +34 -0
- package/apps/desktop/application/operations/recording/resume.ts +34 -0
- package/apps/desktop/application/operations/recording/shared.ts +199 -0
- package/apps/desktop/application/operations/recording/start.ts +169 -0
- package/apps/desktop/application/operations/recording/stop.ts +34 -0
- package/apps/desktop/application/operations/render/bind-candidate-output.ts +324 -0
- package/apps/desktop/application/operations/render/materialize-selection.ts +322 -0
- package/apps/desktop/application/operations/render/project-caption-plan.ts +287 -0
- package/apps/desktop/application/operations/render/project-plan.ts +939 -0
- package/apps/desktop/application/operations/render/project-spatial-receipt.ts +70 -0
- package/apps/desktop/application/operations/render/project.ts +1922 -0
- package/apps/desktop/application/operations/shared.ts +67 -0
- package/apps/desktop/application/operations/slopcamera-portable.ts +1188 -0
- package/apps/desktop/application/operations/slopcamera-visuals.ts +784 -0
- package/apps/desktop/application/operations/spatial-project.ts +209 -0
- package/apps/desktop/application/operations/spatial-render.ts +415 -0
- package/apps/desktop/application/operations/spatial-scene.ts +97 -0
- package/apps/desktop/application/operations/studio.ts +143 -0
- package/apps/desktop/application/output-publication-lease.ts +163 -0
- package/apps/desktop/application/project-publication-lease.ts +233 -0
- package/apps/desktop/application/project-store.ts +196 -0
- package/apps/desktop/application/receipts.ts +533 -0
- package/apps/desktop/application/registry.ts +182 -0
- package/apps/desktop/application/spatial-assets.ts +675 -0
- package/apps/desktop/application/spatial-compositor-cadence.ts +23 -0
- package/apps/desktop/application/spatial-project-authority.ts +35 -0
- package/apps/desktop/application/spatial-project-lease.ts +126 -0
- package/apps/desktop/application/spatial-project-render-preparation.ts +225 -0
- package/apps/desktop/application/spatial-project-store.ts +286 -0
- package/apps/desktop/application/spatial-render-projection.ts +201 -0
- package/apps/desktop/application/spatial-render.ts +677 -0
- package/apps/desktop/application/spatial-spz.ts +44 -0
- package/apps/desktop/application/spatial-world-import.ts +135 -0
- package/apps/desktop/application/spatial-world-provenance.ts +46 -0
- package/apps/desktop/application/studio-port.ts +58 -0
- package/apps/desktop/application/verified-receipt-reconciliation.ts +1574 -0
- package/apps/desktop/capture/protocol.ts +689 -0
- package/apps/desktop/cli/analyzer.ts +406 -0
- package/apps/desktop/cli/args.ts +3181 -0
- package/apps/desktop/cli/asset-ingest.ts +747 -0
- package/apps/desktop/cli/atomic-render-effects.ts +210 -0
- package/apps/desktop/cli/atomic-render-platform.ts +205 -0
- package/apps/desktop/cli/atomic-render.ts +50 -0
- package/apps/desktop/cli/audio-analysis.ts +252 -0
- package/apps/desktop/cli/bounded-map.ts +39 -0
- package/apps/desktop/cli/bundle-service.ts +256 -0
- package/apps/desktop/cli/capabilities.ts +212 -0
- package/apps/desktop/cli/capture-bundle.ts +1117 -0
- package/apps/desktop/cli/command-host-resources.ts +251 -0
- package/apps/desktop/cli/commands.ts +6836 -0
- package/apps/desktop/cli/directing-blob.ts +291 -0
- package/apps/desktop/cli/directing-media.ts +517 -0
- package/apps/desktop/cli/directing-quote.ts +89 -0
- package/apps/desktop/cli/directing-service.ts +288 -0
- package/apps/desktop/cli/emoji-assets.ts +538 -0
- package/apps/desktop/cli/errors.ts +29 -0
- package/apps/desktop/cli/face-analysis-service.ts +940 -0
- package/apps/desktop/cli/filter-script.ts +76 -0
- package/apps/desktop/cli/gateway-application-port.ts +1142 -0
- package/apps/desktop/cli/gateway-credential.ts +81 -0
- package/apps/desktop/cli/gateway-media-artifacts.ts +723 -0
- package/apps/desktop/cli/gateway-media-catalog.ts +922 -0
- package/apps/desktop/cli/gateway-media-service.ts +3106 -0
- package/apps/desktop/cli/gateway-media-signature.ts +122 -0
- package/apps/desktop/cli/gateway-provider-options.ts +562 -0
- package/apps/desktop/cli/gateway-scene-provider.ts +391 -0
- package/apps/desktop/cli/help.ts +501 -0
- package/apps/desktop/cli/html-overlay-gpu.ts +97 -0
- package/apps/desktop/cli/html-overlay-renderer.ts +2478 -0
- package/apps/desktop/cli/io.ts +290 -0
- package/apps/desktop/cli/main.ts +123 -0
- package/apps/desktop/cli/media-effects-service.ts +885 -0
- package/apps/desktop/cli/media-ingest-model.ts +364 -0
- package/apps/desktop/cli/media-ingest-platform.ts +227 -0
- package/apps/desktop/cli/media-ingest-program.ts +200 -0
- package/apps/desktop/cli/media-ingest.ts +30 -0
- package/apps/desktop/cli/music-analysis-service.ts +434 -0
- package/apps/desktop/cli/mutation-lock.ts +530 -0
- package/apps/desktop/cli/native-assets.d.ts +12 -0
- package/apps/desktop/cli/native-media-runtime.macos.ts +7 -0
- package/apps/desktop/cli/overlay-playback.ts +177 -0
- package/apps/desktop/cli/paths.ts +265 -0
- package/apps/desktop/cli/portable-surface.ts +242 -0
- package/apps/desktop/cli/project-camera-receipt.ts +112 -0
- package/apps/desktop/cli/project-face-camera.ts +394 -0
- package/apps/desktop/cli/project-inactivity-service.ts +552 -0
- package/apps/desktop/cli/project-media-integrity.ts +121 -0
- package/apps/desktop/cli/project-renderer.ts +1526 -0
- package/apps/desktop/cli/project-service.ts +519 -0
- package/apps/desktop/cli/project-state-transaction.ts +520 -0
- package/apps/desktop/cli/query-limits.ts +1 -0
- package/apps/desktop/cli/recording-controller.ts +1567 -0
- package/apps/desktop/cli/recording-daemon.ts +931 -0
- package/apps/desktop/cli/recording-ref.ts +56 -0
- package/apps/desktop/cli/renamed-environment.ts +6 -0
- package/apps/desktop/cli/renderer.ts +1439 -0
- package/apps/desktop/cli/run-cli-test-helper.ts +60 -0
- package/apps/desktop/cli/scene-analysis-service.ts +879 -0
- package/apps/desktop/cli/spatial-compositor-verifier.ts +213 -0
- package/apps/desktop/cli/spatial-project-service.ts +90 -0
- package/apps/desktop/cli/spatial-scene-service.ts +105 -0
- package/apps/desktop/cli/spatial-world-service.ts +39 -0
- package/apps/desktop/cli/speech-analysis-service.ts +682 -0
- package/apps/desktop/cli/studio-assemble.ts +58 -0
- package/apps/desktop/cli/studio-command.ts +60 -0
- package/apps/desktop/cli/studio-custody.ts +60 -0
- package/apps/desktop/cli/studio-encode.ts +216 -0
- package/apps/desktop/cli/studio-exr.ts +52 -0
- package/apps/desktop/cli/studio-files.ts +179 -0
- package/apps/desktop/cli/studio-output-validation.ts +162 -0
- package/apps/desktop/cli/studio-process.ts +145 -0
- package/apps/desktop/cli/studio-runtime.ts +77 -0
- package/apps/desktop/cli/studio-scaffold.ts +70 -0
- package/apps/desktop/cli/studio-service.ts +301 -0
- package/apps/desktop/cli/studio-spatial-asset.ts +248 -0
- package/apps/desktop/cli/studio-template-names.ts +3 -0
- package/apps/desktop/cli/studio-workflow.ts +19 -0
- package/apps/desktop/cli/time.ts +25 -0
- package/apps/desktop/cli/workflow-code.ts +584 -0
- package/apps/desktop/cli/workflow-runs.ts +576 -0
- package/apps/desktop/code/advanced.ts +139 -0
- package/apps/desktop/code/application-node-planner.ts +1288 -0
- package/apps/desktop/code/compiler.ts +329 -0
- package/apps/desktop/code/contracts.ts +13 -0
- package/apps/desktop/code/creative-recipes.ts +94 -0
- package/apps/desktop/code/define-workflow.ts +115 -0
- package/apps/desktop/code/editing.ts +284 -0
- package/apps/desktop/code/file-candidate-provenance.ts +420 -0
- package/apps/desktop/code/file-candidate.ts +69 -0
- package/apps/desktop/code/graph-builder.ts +10 -0
- package/apps/desktop/code/host-resource-policy.ts +55 -0
- package/apps/desktop/code/plan-contracts.ts +159 -0
- package/apps/desktop/code/planning.ts +361 -0
- package/apps/desktop/code/public.ts +134 -0
- package/apps/desktop/code/reference-projection.ts +62 -0
- package/apps/desktop/code/run-contracts.ts +466 -0
- package/apps/desktop/code/run-store.ts +2478 -0
- package/apps/desktop/code/runtime-identity.ts +328 -0
- package/apps/desktop/code/scheduler.ts +3067 -0
- package/apps/desktop/code/semantic-builder.ts +1436 -0
- package/apps/desktop/code/source-bundle.ts +1641 -0
- package/apps/desktop/code/source-typecheck.ts +211 -0
- package/apps/desktop/code/testing.ts +79 -0
- package/apps/desktop/code/worker-client.ts +1675 -0
- package/apps/desktop/code/worker-effects.ts +141 -0
- package/apps/desktop/code/worker-entry.ts +435 -0
- package/apps/desktop/code/worker-lease-guardian.ts +179 -0
- package/apps/desktop/code/worker-process-identity.ts +484 -0
- package/apps/desktop/code/worker-protocol.ts +211 -0
- package/apps/desktop/code/workflow-effects.ts +132 -0
- package/apps/desktop/contracts/analysis.ts +949 -0
- package/apps/desktop/contracts/edit.ts +576 -0
- package/apps/desktop/contracts/index.ts +9 -0
- package/apps/desktop/contracts/media-effects.ts +139 -0
- package/apps/desktop/contracts/project-render.ts +644 -0
- package/apps/desktop/contracts/project.ts +740 -0
- package/apps/desktop/contracts/recording.ts +1434 -0
- package/apps/desktop/contracts/render.ts +334 -0
- package/apps/desktop/contracts/runtime.ts +155 -0
- package/apps/desktop/contracts/spatial-compositor.ts +58 -0
- package/apps/desktop/contracts/spatial-project.ts +159 -0
- package/apps/desktop/contracts/spatial-world.ts +48 -0
- package/apps/desktop/contracts/video-effects.ts +120 -0
- package/apps/desktop/core/alignment-apply.ts +400 -0
- package/apps/desktop/core/audio-alignment.ts +466 -0
- package/apps/desktop/core/auto-zoom.ts +209 -0
- package/apps/desktop/core/canonical-json.ts +14 -0
- package/apps/desktop/core/emoji.ts +210 -0
- package/apps/desktop/core/events.ts +409 -0
- package/apps/desktop/core/face-framing.ts +533 -0
- package/apps/desktop/core/face-tracks.ts +326 -0
- package/apps/desktop/core/inactivity.ts +137 -0
- package/apps/desktop/core/index.ts +30 -0
- package/apps/desktop/core/intervals.ts +123 -0
- package/apps/desktop/core/montage.ts +236 -0
- package/apps/desktop/core/music-analysis.ts +518 -0
- package/apps/desktop/core/plan.ts +244 -0
- package/apps/desktop/core/project-auto-zoom.ts +75 -0
- package/apps/desktop/core/project-camera-spatial.ts +676 -0
- package/apps/desktop/core/project-camera.ts +403 -0
- package/apps/desktop/core/project-captions.ts +888 -0
- package/apps/desktop/core/project-inactivity.ts +125 -0
- package/apps/desktop/core/project-metadata.ts +927 -0
- package/apps/desktop/core/project-plan.ts +375 -0
- package/apps/desktop/core/project-render-plan.ts +303 -0
- package/apps/desktop/core/project-time.ts +469 -0
- package/apps/desktop/core/render-plan.ts +442 -0
- package/apps/desktop/core/scene-cache.ts +174 -0
- package/apps/desktop/core/scene-sampling.ts +448 -0
- package/apps/desktop/core/spatial-compositor.ts +22 -0
- package/apps/desktop/core/spatial-durability.ts +99 -0
- package/apps/desktop/core/spatial-project.ts +169 -0
- package/apps/desktop/core/spatial-shot-clock.ts +32 -0
- package/apps/desktop/core/speech-analysis.ts +333 -0
- package/apps/desktop/core/storage.ts +970 -0
- package/apps/desktop/core/time-map.ts +91 -0
- package/apps/desktop/core/time.ts +61 -0
- package/apps/desktop/core/video-effects.ts +660 -0
- package/apps/desktop/dist/cli/NebulaSans-Bold-26se8aek.otf +0 -0
- package/apps/desktop/dist/cli/NebulaSans-Bold-bcz7y08t.woff2 +0 -0
- package/apps/desktop/dist/cli/NebulaSans-Book-5ax05zvn.woff2 +0 -0
- package/apps/desktop/dist/cli/NebulaSans-Book-8cenzchw.otf +0 -0
- package/apps/desktop/dist/cli/main.js +5086 -0
- package/apps/desktop/html-overlay/catalog.ts +171 -0
- package/apps/desktop/html-overlay/contracts.ts +372 -0
- package/apps/desktop/html-overlay/execution-profile.ts +77 -0
- package/apps/desktop/html-overlay/index.ts +98 -0
- package/apps/desktop/html-overlay/libraries.ts +263 -0
- package/apps/desktop/html-overlay/random.ts +49 -0
- package/apps/desktop/html-overlay/runtime.ts +813 -0
- package/apps/desktop/html-overlay/scaffolds.ts +792 -0
- package/apps/desktop/html-overlay/spark-runtime.ts +88 -0
- package/apps/desktop/html-overlay/spatial-splat-runtime.ts +60 -0
- package/apps/desktop/html-overlay/spatial.ts +720 -0
- package/apps/desktop/studio/assets/README.md +39 -0
- package/apps/desktop/studio/assets/contracts.ts +129 -0
- package/apps/desktop/studio/assets/network.ts +64 -0
- package/apps/desktop/studio/assets/poly-haven.ts +148 -0
- package/apps/desktop/studio/drivers/blender_driver.py +466 -0
- package/apps/desktop/studio/drivers/cadquery_driver.py +176 -0
- package/apps/desktop/studio/education/driver.py +308 -0
- package/apps/desktop/studio/education/lesson.py +180 -0
- package/apps/desktop/studio/education/toolkit.py +249 -0
- package/apps/desktop/workflows/chaptered-demo.ts +48 -0
- package/apps/desktop/workflows/creative-iteration.ts +107 -0
- package/apps/desktop/workflows/creative-selection.ts +92 -0
- package/apps/desktop/workflows/fragments.ts +486 -0
- package/apps/desktop/workflows/index.ts +93 -0
- package/apps/desktop/workflows/polished-screen-demo.ts +128 -0
- package/apps/desktop/workflows/scene-direction.ts +12 -0
- package/apps/desktop/workflows/social-variants.ts +75 -0
- package/apps/desktop/workflows/talking-head-cleanup.ts +53 -0
- package/dist/NebulaSans-Bold-26se8aek.otf +0 -0
- package/dist/NebulaSans-Bold-bcz7y08t.woff2 +0 -0
- package/dist/NebulaSans-Book-5ax05zvn.woff2 +0 -0
- package/dist/NebulaSans-Book-8cenzchw.otf +0 -0
- package/dist/cli.js +479 -0
- package/dist/code/advanced.js +270 -0
- package/dist/code/index.js +2240 -0
- package/dist/generate.js +24 -0
- package/dist/host-resources.js +26 -0
- package/dist/index-42zsesc1.js +2257 -0
- package/dist/index-7308egqr.js +48 -0
- package/dist/index-8txs6fkn.js +712 -0
- package/dist/index-mcy8z0br.js +192 -0
- package/dist/index-p63wavx0.js +2029 -0
- package/dist/index-r7gdhmsp.js +424 -0
- package/dist/index-sh6xbav6.js +1219 -0
- package/dist/index-z1w83f81.js +4 -0
- package/dist/index-z7239b4h.js +2363 -0
- package/dist/index-zfeg9hmw.js +2013 -0
- package/dist/index.js +241 -0
- package/dist/operations.js +29 -0
- package/dist/skill-install-dh0ntqba.js +10 -0
- package/dist/vectorize/worker.js +142 -0
- package/dist/workflow.js +16 -0
- package/docs/README.md +38 -0
- package/docs/studio.md +215 -0
- package/examples/studio/blender/README.md +26 -0
- package/examples/studio/blender/character.py +152 -0
- package/examples/studio/blender/cloth.py +101 -0
- package/examples/studio/blender/color_chart.py +29 -0
- package/examples/studio/blender/fluid.py +78 -0
- package/examples/studio/blender/import_model.py +74 -0
- package/examples/studio/blender/product.py +53 -0
- package/examples/studio/blender/shaded_street.py +481 -0
- package/examples/studio/blender/studio_scene.py +147 -0
- package/examples/studio/cadquery/README.md +13 -0
- package/examples/studio/cadquery/bracket.py +36 -0
- package/examples/studio/cadquery/import_step.py +12 -0
- package/examples/studio/education/README.md +46 -0
- package/examples/studio/education/lesson.json +99 -0
- package/examples/studio/education/scene.py +7 -0
- package/examples/studio/hybrid-scene.ts +96 -0
- package/examples/studio/native-workflow.ts +14 -0
- package/examples/studio/vgpu/README.md +64 -0
- package/examples/studio/vgpu/heat-field.mjs +290 -0
- package/package.json +304 -0
- package/packages/scene/src/index.ts +1 -0
- package/packages/scene/src/scene.ts +215 -0
- package/schema/diagram.schema.json +334 -0
- package/skills/slopcamera/SKILL.md +45 -0
- package/skills/slopcamera/agents/openai.yaml +4 -0
- package/skills/slopcamera/references/customization.md +77 -0
- package/skills/slopcamera/references/diagrams.md +153 -0
- package/skills/slopcamera/references/directed-scenes.md +42 -0
- package/skills/slopcamera/references/directing-video.md +18 -0
- package/skills/slopcamera/references/educational-video.md +26 -0
- package/skills/slopcamera/references/gateway-media.md +134 -0
- package/skills/slopcamera/references/install.md +62 -0
- package/skills/slopcamera/references/native-studio.md +31 -0
- package/skills/slopcamera/references/reference-led-3d.md +140 -0
- package/skills/slopcamera/references/rubber-stamp-examples/poster-example-1.jpg +0 -0
- package/skills/slopcamera/references/rubber-stamp-examples/poster-example-2.jpg +0 -0
- package/skills/slopcamera/references/rubber-stamp-examples/stamp-style-1.png +0 -0
- package/skills/slopcamera/references/rubber-stamp-examples/stamp-style-2.png +0 -0
- package/skills/slopcamera/references/rubber-stamp-field-notes.md +144 -0
- package/skills/slopcamera/references/vectorization.md +29 -0
- package/skills/slopcamera/references/video-projects.md +187 -0
- package/skills/slopcamera/references/visual-communication.md +120 -0
- package/skills/slopcamera/references/workflows-sdk.md +45 -0
- package/skills/slopcamera/scripts/compose-rubber-stamp-field-note.ts +229 -0
- package/src/artifacts.ts +106 -0
- package/src/assets/fonts/nebula-sans/LICENSE.txt +96 -0
- package/src/assets/fonts/nebula-sans/NebulaSans-Bold.otf +0 -0
- package/src/assets/fonts/nebula-sans/NebulaSans-Bold.woff2 +0 -0
- package/src/assets/fonts/nebula-sans/NebulaSans-Book.otf +0 -0
- package/src/assets/fonts/nebula-sans/NebulaSans-Book.woff2 +0 -0
- package/src/assets/fonts/nebula-sans/PROVENANCE.md +21 -0
- package/src/assets.d.ts +9 -0
- package/src/cli.ts +601 -0
- package/src/cloud-errors.ts +26 -0
- package/src/code/advanced.ts +10 -0
- package/src/code/boundary.ts +21 -0
- package/src/code/canonical-json.ts +129 -0
- package/src/code/compiler.ts +953 -0
- package/src/code/contracts.ts +576 -0
- package/src/code/define-workflow.ts +334 -0
- package/src/code/errors.ts +41 -0
- package/src/code/graph-builder.ts +945 -0
- package/src/code/index.ts +121 -0
- package/src/code/json-snapshot.ts +899 -0
- package/src/code/json-utf8.ts +88 -0
- package/src/code/portable-builder.ts +136 -0
- package/src/code/projection.ts +378 -0
- package/src/code/public-operations.ts +450 -0
- package/src/code/runtime.ts +580 -0
- package/src/code/sha256.ts +36 -0
- package/src/config.ts +180 -0
- package/src/desktop.ts +253 -0
- package/src/fs.ts +10 -0
- package/src/generate.ts +704 -0
- package/src/host-resource-posix.ts +161 -0
- package/src/host-resources.ts +1749 -0
- package/src/icons.ts +58 -0
- package/src/index.ts +200 -0
- package/src/label-layout.ts +130 -0
- package/src/layout.ts +413 -0
- package/src/lint.ts +127 -0
- package/src/mcp/boundary.ts +354 -0
- package/src/mcp/index.ts +33 -0
- package/src/mcp/server.ts +272 -0
- package/src/mcp/tools.ts +963 -0
- package/src/mcp/types.ts +71 -0
- package/src/operations.ts +775 -0
- package/src/parse.ts +861 -0
- package/src/process-environment.ts +17 -0
- package/src/render.ts +437 -0
- package/src/skill-install.ts +49 -0
- package/src/spatial-scene/authoring.ts +27 -0
- package/src/spatial-scene/camera-track.ts +83 -0
- package/src/spatial-scene/contracts.ts +277 -0
- package/src/spatial-scene/evaluate.ts +145 -0
- package/src/spatial-scene/gltf.ts +558 -0
- package/src/spatial-scene/identity.ts +260 -0
- package/src/spatial-scene/index.ts +10 -0
- package/src/spatial-scene/inspect.ts +68 -0
- package/src/spatial-scene/math.ts +280 -0
- package/src/spatial-scene/patch.ts +131 -0
- package/src/spatial-scene/time.ts +53 -0
- package/src/studio/contracts.ts +129 -0
- package/src/studio/index.ts +2 -0
- package/src/studio/plan.ts +156 -0
- package/src/studio/shared.ts +41 -0
- package/src/theme.ts +68 -0
- package/src/tldr.ts +474 -0
- package/src/types.ts +212 -0
- package/src/vectorize/archive.ts +101 -0
- package/src/vectorize/command.ts +797 -0
- package/src/vectorize/index.ts +27 -0
- package/src/vectorize/limits.ts +61 -0
- package/src/vectorize/metrics.ts +351 -0
- package/src/vectorize/pixels.ts +323 -0
- package/src/vectorize/supervisor.ts +339 -0
- package/src/vectorize/svg.ts +188 -0
- package/src/vectorize/tool.ts +542 -0
- package/src/vectorize/types.ts +113 -0
- package/src/vectorize/vectorize.ts +634 -0
- package/src/vectorize/worker-protocol.ts +40 -0
- package/src/vectorize/worker.ts +209 -0
- package/src/version.ts +1 -0
- package/src/workflow.ts +409 -0
|
@@ -0,0 +1,3106 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { Buffer } from "node:buffer";
|
|
3
|
+
import { lookup } from "node:dns/promises";
|
|
4
|
+
import { request as httpsRequest } from "node:https";
|
|
5
|
+
import { BlockList, isIP, type LookupFunction } from "node:net";
|
|
6
|
+
import { Readable } from "node:stream";
|
|
7
|
+
|
|
8
|
+
import { canonicalJsonSha256 } from "../core/canonical-json";
|
|
9
|
+
import type { ActiveGatewayCredential } from "./gateway-credential";
|
|
10
|
+
import {
|
|
11
|
+
type GatewayGeneratedFile,
|
|
12
|
+
type GatewayMediaArtifactBundle,
|
|
13
|
+
type GatewayMediaArtifactStore,
|
|
14
|
+
type GatewayMediaInputDigest,
|
|
15
|
+
type GatewayMediaRoutingReceipt,
|
|
16
|
+
} from "./gateway-media-artifacts";
|
|
17
|
+
import {
|
|
18
|
+
GATEWAY_MEDIA_CATALOG_URL,
|
|
19
|
+
type GatewayJsonValue,
|
|
20
|
+
type GatewayMediaCatalogView,
|
|
21
|
+
type GatewayMediaKind,
|
|
22
|
+
type GatewayMediaModel,
|
|
23
|
+
} from "./gateway-media-catalog";
|
|
24
|
+
import {
|
|
25
|
+
gatewayProviderOptionsSummary,
|
|
26
|
+
parseGatewayProviderOptions,
|
|
27
|
+
type GatewayProviderOptions,
|
|
28
|
+
} from "./gateway-provider-options";
|
|
29
|
+
import { gatewayMediaBytesMatchType } from "./gateway-media-signature";
|
|
30
|
+
|
|
31
|
+
export const GATEWAY_MEDIA_API_BASE_URL = "https://ai-gateway.vercel.sh/v4/ai";
|
|
32
|
+
export const GATEWAY_MEDIA_UPLOAD_POLICY = "slopcamera.gateway-media-cloud-upload.v1";
|
|
33
|
+
export const LEGACY_GATEWAY_MEDIA_UPLOAD_POLICY = "studio.gateway-media-cloud-upload.v1";
|
|
34
|
+
|
|
35
|
+
const MAXIMUM_IMAGE_INPUT_BYTES = 50 * 1024 * 1024;
|
|
36
|
+
const MAXIMUM_IMAGE_INPUT_TOTAL_BYTES = 200 * 1024 * 1024;
|
|
37
|
+
const MAXIMUM_REFERENCE_INPUT_BYTES = 256 * 1024 * 1024;
|
|
38
|
+
const MAXIMUM_REFERENCE_INPUT_TOTAL_BYTES = 512 * 1024 * 1024;
|
|
39
|
+
const MAXIMUM_TRANSCRIPTION_INPUT_BYTES = 256 * 1024 * 1024;
|
|
40
|
+
const MAXIMUM_DOWNLOAD_BYTES = 512 * 1024 * 1024;
|
|
41
|
+
const MAXIMUM_VIDEO_OUTPUT_TOTAL_BYTES = 1024 * 1024 * 1024;
|
|
42
|
+
const DEFAULT_GATEWAY_RESPONSE_MAX_BYTES = 512 * 1024 * 1024;
|
|
43
|
+
const MAXIMUM_PROMPT_CHARACTERS = 100_000;
|
|
44
|
+
const MAXIMUM_TRANSCRIPT_CHARACTERS = 10_000_000;
|
|
45
|
+
const MAXIMUM_TRANSCRIPT_SEGMENTS = 100_000;
|
|
46
|
+
const MAXIMUM_TRANSCRIPT_TEXT_BYTES = 32 * 1024 * 1024;
|
|
47
|
+
const MAXIMUM_TRANSCRIPT_SEGMENT_TEXT_BYTES = 32 * 1024 * 1024;
|
|
48
|
+
const MAXIMUM_TRANSCRIPT_ARTIFACT_BYTES = 256 * 1024 * 1024;
|
|
49
|
+
const MAXIMUM_INPUTS = 32;
|
|
50
|
+
const DEFAULT_IMAGE_TIMEOUT_MS = 5 * 60_000;
|
|
51
|
+
const DEFAULT_SPEECH_TIMEOUT_MS = 5 * 60_000;
|
|
52
|
+
const DEFAULT_TRANSCRIPTION_TIMEOUT_MS = 10 * 60_000;
|
|
53
|
+
const DEFAULT_VIDEO_TIMEOUT_MS = 20 * 60_000;
|
|
54
|
+
const MAXIMUM_DOWNLOAD_REDIRECTS = 5;
|
|
55
|
+
|
|
56
|
+
export type GatewayMediaExecutionErrorCode =
|
|
57
|
+
| "cloud-consent-required"
|
|
58
|
+
| "download-failed"
|
|
59
|
+
| "input-too-large"
|
|
60
|
+
| "invalid-request"
|
|
61
|
+
| "invalid-response"
|
|
62
|
+
| "model-kind-mismatch"
|
|
63
|
+
| "model-not-found"
|
|
64
|
+
| "model-operation-unsupported"
|
|
65
|
+
| "provider-failed";
|
|
66
|
+
|
|
67
|
+
const ERROR_MESSAGES: Readonly<Record<GatewayMediaExecutionErrorCode, string>> = {
|
|
68
|
+
"cloud-consent-required": "This Gateway media request requires explicit cloud-upload consent.",
|
|
69
|
+
"download-failed": "The generated Gateway media output could not be downloaded safely.",
|
|
70
|
+
"input-too-large": "The Gateway media input exceeds its upload limit.",
|
|
71
|
+
"invalid-request": "The Gateway media request is invalid.",
|
|
72
|
+
"invalid-response": "The Gateway media provider returned an invalid response.",
|
|
73
|
+
"model-kind-mismatch": "The selected Gateway model has the wrong media kind.",
|
|
74
|
+
"model-not-found": "The selected Gateway media model is not in the current catalog.",
|
|
75
|
+
"model-operation-unsupported": "The selected Gateway model does not support this request shape or setting.",
|
|
76
|
+
"provider-failed": "The Gateway media provider request failed.",
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
export class GatewayMediaExecutionError extends Error {
|
|
80
|
+
readonly code: GatewayMediaExecutionErrorCode;
|
|
81
|
+
readonly reconciliation: Readonly<{
|
|
82
|
+
readonly failureSha256: string;
|
|
83
|
+
readonly routing: GatewayMediaRoutingReceipt;
|
|
84
|
+
readonly statusCode?: number;
|
|
85
|
+
}> | undefined;
|
|
86
|
+
|
|
87
|
+
constructor(
|
|
88
|
+
code: GatewayMediaExecutionErrorCode,
|
|
89
|
+
reconciliation?: Readonly<{
|
|
90
|
+
readonly failureSha256: string;
|
|
91
|
+
readonly routing: GatewayMediaRoutingReceipt;
|
|
92
|
+
readonly statusCode?: number;
|
|
93
|
+
}>,
|
|
94
|
+
) {
|
|
95
|
+
super(ERROR_MESSAGES[code]);
|
|
96
|
+
this.code = code;
|
|
97
|
+
this.name = "GatewayMediaExecutionError";
|
|
98
|
+
this.reconciliation = reconciliation;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export interface GatewayUploadConsent {
|
|
103
|
+
readonly acknowledgedAt: string;
|
|
104
|
+
readonly allowCloudUpload: true;
|
|
105
|
+
readonly policy:
|
|
106
|
+
| typeof GATEWAY_MEDIA_UPLOAD_POLICY
|
|
107
|
+
| typeof LEGACY_GATEWAY_MEDIA_UPLOAD_POLICY;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
interface GatewayMediaInputFacts {
|
|
111
|
+
readonly durationSeconds?: number;
|
|
112
|
+
readonly height?: number;
|
|
113
|
+
readonly width?: number;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export type GatewayMediaInput =
|
|
117
|
+
| Readonly<{
|
|
118
|
+
readonly data: Uint8Array;
|
|
119
|
+
readonly facts?: GatewayMediaInputFacts;
|
|
120
|
+
readonly mediaType: string;
|
|
121
|
+
readonly url?: never;
|
|
122
|
+
}>
|
|
123
|
+
| Readonly<{
|
|
124
|
+
readonly data?: never;
|
|
125
|
+
readonly facts?: GatewayMediaInputFacts;
|
|
126
|
+
readonly mediaType: string;
|
|
127
|
+
readonly url: string;
|
|
128
|
+
}>;
|
|
129
|
+
|
|
130
|
+
type GatewayInlineMediaInput = Extract<
|
|
131
|
+
GatewayMediaInput,
|
|
132
|
+
Readonly<{ readonly data: Uint8Array }>
|
|
133
|
+
>;
|
|
134
|
+
|
|
135
|
+
type GatewaySdkMediaInput = Uint8Array | string;
|
|
136
|
+
|
|
137
|
+
export interface GatewaySdkImageRequest {
|
|
138
|
+
readonly abortSignal: AbortSignal;
|
|
139
|
+
readonly aspectRatio?: string;
|
|
140
|
+
readonly maxImagesPerCall?: number;
|
|
141
|
+
readonly maxRetries: 0;
|
|
142
|
+
readonly modelId: string;
|
|
143
|
+
readonly n?: number;
|
|
144
|
+
readonly prompt:
|
|
145
|
+
| string
|
|
146
|
+
| Readonly<{
|
|
147
|
+
images: readonly GatewaySdkMediaInput[];
|
|
148
|
+
mask?: GatewaySdkMediaInput;
|
|
149
|
+
text?: string;
|
|
150
|
+
}>;
|
|
151
|
+
readonly providerOptions?: GatewayProviderOptions;
|
|
152
|
+
readonly seed?: number;
|
|
153
|
+
readonly size?: string;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
export interface GatewaySdkLanguageImageRequest {
|
|
157
|
+
readonly abortSignal: AbortSignal;
|
|
158
|
+
readonly maxOutputTokens?: number;
|
|
159
|
+
readonly maxRetries: 0;
|
|
160
|
+
readonly messages?: readonly Readonly<{
|
|
161
|
+
content: readonly (
|
|
162
|
+
| Readonly<{ text: string; type: "text" }>
|
|
163
|
+
| Readonly<{
|
|
164
|
+
image: GatewaySdkMediaInput;
|
|
165
|
+
mediaType: string;
|
|
166
|
+
type: "image";
|
|
167
|
+
}>
|
|
168
|
+
)[];
|
|
169
|
+
role: "user";
|
|
170
|
+
}>[];
|
|
171
|
+
readonly modelId: string;
|
|
172
|
+
readonly prompt?: string;
|
|
173
|
+
readonly providerOptions?: GatewayProviderOptions;
|
|
174
|
+
readonly stopSequences?: readonly string[];
|
|
175
|
+
readonly temperature?: number;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
export interface GatewaySdkVideoRequest {
|
|
179
|
+
readonly abortSignal: AbortSignal;
|
|
180
|
+
readonly aspectRatio?: string;
|
|
181
|
+
readonly download: GatewayMediaDownload;
|
|
182
|
+
readonly duration?: number;
|
|
183
|
+
readonly fps?: number;
|
|
184
|
+
readonly frameImages?: readonly Readonly<{
|
|
185
|
+
frameType: string;
|
|
186
|
+
image: GatewaySdkMediaInput;
|
|
187
|
+
}>[];
|
|
188
|
+
readonly generateAudio?: boolean;
|
|
189
|
+
readonly inputReferences?: readonly Readonly<{
|
|
190
|
+
data: GatewaySdkMediaInput;
|
|
191
|
+
mediaType: string;
|
|
192
|
+
}>[];
|
|
193
|
+
readonly maxRetries: 0;
|
|
194
|
+
readonly maxVideosPerCall?: number;
|
|
195
|
+
readonly modelId: string;
|
|
196
|
+
readonly n?: number;
|
|
197
|
+
readonly prompt:
|
|
198
|
+
| string
|
|
199
|
+
| Readonly<{
|
|
200
|
+
image: GatewaySdkMediaInput;
|
|
201
|
+
text?: string;
|
|
202
|
+
}>;
|
|
203
|
+
readonly providerOptions?: GatewayProviderOptions;
|
|
204
|
+
readonly resolution?: string;
|
|
205
|
+
readonly seed?: number;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
export interface GatewaySdkSpeechRequest {
|
|
209
|
+
readonly abortSignal: AbortSignal;
|
|
210
|
+
readonly instructions?: string;
|
|
211
|
+
readonly language?: string;
|
|
212
|
+
readonly maxRetries: 0;
|
|
213
|
+
readonly modelId: string;
|
|
214
|
+
readonly outputFormat?: string;
|
|
215
|
+
readonly providerOptions?: GatewayProviderOptions;
|
|
216
|
+
readonly speed?: number;
|
|
217
|
+
readonly text: string;
|
|
218
|
+
readonly voice?: string;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
export interface GatewaySdkTranscriptionRequest {
|
|
222
|
+
readonly abortSignal: AbortSignal;
|
|
223
|
+
readonly audio: Uint8Array;
|
|
224
|
+
readonly maxRetries: 0;
|
|
225
|
+
readonly modelId: string;
|
|
226
|
+
readonly providerOptions?: GatewayProviderOptions;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
export interface GatewayMediaSdk {
|
|
230
|
+
generateImage(apiKey: string, request: GatewaySdkImageRequest): Promise<unknown>;
|
|
231
|
+
generateLanguageImage(
|
|
232
|
+
apiKey: string,
|
|
233
|
+
request: GatewaySdkLanguageImageRequest,
|
|
234
|
+
): Promise<unknown>;
|
|
235
|
+
generateSpeech(apiKey: string, request: GatewaySdkSpeechRequest): Promise<unknown>;
|
|
236
|
+
generateVideo(apiKey: string, request: GatewaySdkVideoRequest): Promise<unknown>;
|
|
237
|
+
transcribe(apiKey: string, request: GatewaySdkTranscriptionRequest): Promise<unknown>;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
export type GatewayMediaDownload = (input: Readonly<{
|
|
241
|
+
abortSignal?: AbortSignal;
|
|
242
|
+
maximumBytes?: number;
|
|
243
|
+
url: URL;
|
|
244
|
+
}>) => Promise<Readonly<{
|
|
245
|
+
data: Uint8Array;
|
|
246
|
+
mediaType: string | undefined;
|
|
247
|
+
}>>;
|
|
248
|
+
|
|
249
|
+
export type GatewayMediaFetch = (
|
|
250
|
+
input: RequestInfo | URL,
|
|
251
|
+
init?: RequestInit,
|
|
252
|
+
) => Promise<Response>;
|
|
253
|
+
|
|
254
|
+
interface GatewayAiProvider {
|
|
255
|
+
imageModel(modelId: string): unknown;
|
|
256
|
+
languageModel(modelId: string): unknown;
|
|
257
|
+
speech(modelId: string): unknown;
|
|
258
|
+
transcription(modelId: string): unknown;
|
|
259
|
+
video(modelId: string): unknown;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
export interface GatewayAiSdkRuntime {
|
|
263
|
+
createGateway(settings: Readonly<{
|
|
264
|
+
apiKey: string;
|
|
265
|
+
baseURL: typeof GATEWAY_MEDIA_API_BASE_URL;
|
|
266
|
+
fetch: GatewayMediaFetch;
|
|
267
|
+
}>): GatewayAiProvider;
|
|
268
|
+
generateImage(input: Readonly<Record<string, unknown>>): Promise<unknown>;
|
|
269
|
+
generateSpeech(input: Readonly<Record<string, unknown>>): Promise<unknown>;
|
|
270
|
+
generateText(input: Readonly<Record<string, unknown>>): Promise<unknown>;
|
|
271
|
+
generateVideo(input: Readonly<Record<string, unknown>>): Promise<unknown>;
|
|
272
|
+
transcribe(input: Readonly<Record<string, unknown>>): Promise<unknown>;
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
type DynamicImport = (specifier: string) => Promise<unknown>;
|
|
276
|
+
type AsyncRecordFunction = (
|
|
277
|
+
input: Readonly<Record<string, unknown>>,
|
|
278
|
+
) => Promise<unknown>;
|
|
279
|
+
type CreateGatewayFunction = (
|
|
280
|
+
settings: Readonly<Record<string, unknown>>,
|
|
281
|
+
) => unknown;
|
|
282
|
+
type ModelFactory = (modelId: string) => unknown;
|
|
283
|
+
|
|
284
|
+
function disableAiSdkWarningLogging(): void {
|
|
285
|
+
(
|
|
286
|
+
globalThis as typeof globalThis & {
|
|
287
|
+
AI_SDK_LOG_WARNINGS?: false;
|
|
288
|
+
}
|
|
289
|
+
).AI_SDK_LOG_WARNINGS = false;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
function isObject(value: unknown): value is Record<string, unknown> {
|
|
293
|
+
return (typeof value === "object" && value !== null) || typeof value === "function";
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
297
|
+
if (!isObject(value) || Array.isArray(value)) return false;
|
|
298
|
+
const prototype: unknown = Object.getPrototypeOf(value);
|
|
299
|
+
return prototype === Object.prototype || prototype === null;
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
function parseFunction(
|
|
303
|
+
value: unknown,
|
|
304
|
+
): AsyncRecordFunction {
|
|
305
|
+
if (typeof value !== "function") {
|
|
306
|
+
throw new GatewayMediaExecutionError("provider-failed");
|
|
307
|
+
}
|
|
308
|
+
return value as AsyncRecordFunction;
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
async function defaultGatewayAiSdkRuntime(
|
|
312
|
+
importModule: DynamicImport | undefined,
|
|
313
|
+
): Promise<GatewayAiSdkRuntime> {
|
|
314
|
+
let aiModule: unknown;
|
|
315
|
+
let gatewayModule: unknown;
|
|
316
|
+
try {
|
|
317
|
+
[aiModule, gatewayModule] = await Promise.all([
|
|
318
|
+
importModule === undefined ? import("ai-v7") : importModule("ai-v7"),
|
|
319
|
+
importModule === undefined
|
|
320
|
+
? import("@ai-sdk/gateway-v4")
|
|
321
|
+
: importModule("@ai-sdk/gateway-v4"),
|
|
322
|
+
]);
|
|
323
|
+
} catch {
|
|
324
|
+
throw new GatewayMediaExecutionError("provider-failed");
|
|
325
|
+
}
|
|
326
|
+
if (!isObject(aiModule) || !isObject(gatewayModule)) {
|
|
327
|
+
throw new GatewayMediaExecutionError("provider-failed");
|
|
328
|
+
}
|
|
329
|
+
const generateImage = parseFunction(aiModule.generateImage);
|
|
330
|
+
const generateSpeech = parseFunction(aiModule.generateSpeech);
|
|
331
|
+
const generateText = parseFunction(aiModule.generateText);
|
|
332
|
+
const generateVideo = parseFunction(aiModule.experimental_generateVideo);
|
|
333
|
+
const transcribe = parseFunction(aiModule.transcribe);
|
|
334
|
+
if (typeof gatewayModule.createGateway !== "function") {
|
|
335
|
+
throw new GatewayMediaExecutionError("provider-failed");
|
|
336
|
+
}
|
|
337
|
+
const createGateway = gatewayModule.createGateway as CreateGatewayFunction;
|
|
338
|
+
return {
|
|
339
|
+
createGateway: (settings) => {
|
|
340
|
+
const provider = createGateway(settings);
|
|
341
|
+
if (
|
|
342
|
+
!isObject(provider)
|
|
343
|
+
|| typeof provider.imageModel !== "function"
|
|
344
|
+
|| typeof provider.languageModel !== "function"
|
|
345
|
+
|| typeof provider.speech !== "function"
|
|
346
|
+
|| typeof provider.transcription !== "function"
|
|
347
|
+
|| typeof provider.video !== "function"
|
|
348
|
+
) {
|
|
349
|
+
throw new GatewayMediaExecutionError("provider-failed");
|
|
350
|
+
}
|
|
351
|
+
const imageModel = provider.imageModel as ModelFactory;
|
|
352
|
+
const languageModel = provider.languageModel as ModelFactory;
|
|
353
|
+
const speech = provider.speech as ModelFactory;
|
|
354
|
+
const transcription = provider.transcription as ModelFactory;
|
|
355
|
+
const video = provider.video as ModelFactory;
|
|
356
|
+
return {
|
|
357
|
+
imageModel: modelId => imageModel.call(provider, modelId),
|
|
358
|
+
languageModel: modelId => languageModel.call(provider, modelId),
|
|
359
|
+
speech: modelId => speech.call(provider, modelId),
|
|
360
|
+
transcription: modelId => transcription.call(provider, modelId),
|
|
361
|
+
video: modelId => video.call(provider, modelId),
|
|
362
|
+
};
|
|
363
|
+
},
|
|
364
|
+
generateImage,
|
|
365
|
+
generateSpeech,
|
|
366
|
+
generateText,
|
|
367
|
+
generateVideo,
|
|
368
|
+
transcribe,
|
|
369
|
+
};
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
/**
|
|
373
|
+
* Loads AI SDK 7 lazily so catalog and local-only commands do not initialize a
|
|
374
|
+
* provider client. The string-valued import seam also keeps tests independent
|
|
375
|
+
* of the installed SDK while the desktop package's dependency upgrade lands.
|
|
376
|
+
*/
|
|
377
|
+
export function createAiSdkGatewayMediaSdk(options: Readonly<{
|
|
378
|
+
fetch?: GatewayMediaFetch;
|
|
379
|
+
importModule?: DynamicImport;
|
|
380
|
+
loadRuntime?: () => Promise<GatewayAiSdkRuntime>;
|
|
381
|
+
maximumResponseBytes?: number;
|
|
382
|
+
}> = {}): GatewayMediaSdk {
|
|
383
|
+
let runtimePromise: Promise<GatewayAiSdkRuntime> | undefined;
|
|
384
|
+
const loadRuntime = async (): Promise<GatewayAiSdkRuntime> => {
|
|
385
|
+
runtimePromise ??= options.loadRuntime === undefined
|
|
386
|
+
? defaultGatewayAiSdkRuntime(options.importModule)
|
|
387
|
+
: options.loadRuntime();
|
|
388
|
+
return await runtimePromise;
|
|
389
|
+
};
|
|
390
|
+
const gatewayFetch = createFixedGatewayMediaApiFetch({
|
|
391
|
+
...(options.fetch === undefined ? {} : { fetch: options.fetch }),
|
|
392
|
+
...(options.maximumResponseBytes === undefined
|
|
393
|
+
? {}
|
|
394
|
+
: { maximumResponseBytes: options.maximumResponseBytes }),
|
|
395
|
+
});
|
|
396
|
+
const provider = async (
|
|
397
|
+
apiKey: string,
|
|
398
|
+
signal: AbortSignal,
|
|
399
|
+
): Promise<Readonly<{ provider: GatewayAiProvider; runtime: GatewayAiSdkRuntime }>> => {
|
|
400
|
+
// AI SDK logs raw provider warnings by default. Slopcamera persists only
|
|
401
|
+
// allowlisted warning types and message hashes, so disable the SDK logger
|
|
402
|
+
// before loading or invoking its runtime.
|
|
403
|
+
disableAiSdkWarningLogging();
|
|
404
|
+
const runtime = await loadRuntime();
|
|
405
|
+
assertProviderDispatchActive(signal);
|
|
406
|
+
return {
|
|
407
|
+
provider: runtime.createGateway({
|
|
408
|
+
apiKey,
|
|
409
|
+
baseURL: GATEWAY_MEDIA_API_BASE_URL,
|
|
410
|
+
fetch: gatewayFetch,
|
|
411
|
+
}),
|
|
412
|
+
runtime,
|
|
413
|
+
};
|
|
414
|
+
};
|
|
415
|
+
return {
|
|
416
|
+
generateImage: async (apiKey, request) => {
|
|
417
|
+
const loaded = await provider(apiKey, request.abortSignal);
|
|
418
|
+
assertProviderDispatchActive(request.abortSignal);
|
|
419
|
+
const { modelId, ...settings } = request;
|
|
420
|
+
return await loaded.runtime.generateImage({
|
|
421
|
+
...settings,
|
|
422
|
+
model: loaded.provider.imageModel(modelId),
|
|
423
|
+
});
|
|
424
|
+
},
|
|
425
|
+
generateLanguageImage: async (apiKey, request) => {
|
|
426
|
+
const loaded = await provider(apiKey, request.abortSignal);
|
|
427
|
+
assertProviderDispatchActive(request.abortSignal);
|
|
428
|
+
const { modelId, ...settings } = request;
|
|
429
|
+
return await loaded.runtime.generateText({
|
|
430
|
+
...settings,
|
|
431
|
+
model: loaded.provider.languageModel(modelId),
|
|
432
|
+
});
|
|
433
|
+
},
|
|
434
|
+
generateSpeech: async (apiKey, request) => {
|
|
435
|
+
const loaded = await provider(apiKey, request.abortSignal);
|
|
436
|
+
assertProviderDispatchActive(request.abortSignal);
|
|
437
|
+
const { modelId, ...settings } = request;
|
|
438
|
+
return await loaded.runtime.generateSpeech({
|
|
439
|
+
...settings,
|
|
440
|
+
model: loaded.provider.speech(modelId),
|
|
441
|
+
});
|
|
442
|
+
},
|
|
443
|
+
generateVideo: async (apiKey, request) => {
|
|
444
|
+
const loaded = await provider(apiKey, request.abortSignal);
|
|
445
|
+
assertProviderDispatchActive(request.abortSignal);
|
|
446
|
+
const { modelId, ...settings } = request;
|
|
447
|
+
return await loaded.runtime.generateVideo({
|
|
448
|
+
...settings,
|
|
449
|
+
model: loaded.provider.video(modelId),
|
|
450
|
+
});
|
|
451
|
+
},
|
|
452
|
+
transcribe: async (apiKey, request) => {
|
|
453
|
+
const loaded = await provider(apiKey, request.abortSignal);
|
|
454
|
+
assertProviderDispatchActive(request.abortSignal);
|
|
455
|
+
const { modelId, ...settings } = request;
|
|
456
|
+
return await loaded.runtime.transcribe({
|
|
457
|
+
...settings,
|
|
458
|
+
model: loaded.provider.transcription(modelId),
|
|
459
|
+
});
|
|
460
|
+
},
|
|
461
|
+
};
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
function gatewayRequestUrl(input: RequestInfo | URL): URL {
|
|
465
|
+
try {
|
|
466
|
+
if (input instanceof Request) return new URL(input.url);
|
|
467
|
+
if (input instanceof URL) return new URL(input.href);
|
|
468
|
+
return new URL(input, GATEWAY_MEDIA_API_BASE_URL);
|
|
469
|
+
} catch {
|
|
470
|
+
throw new GatewayMediaExecutionError("provider-failed");
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
async function boundedGatewayResponse(
|
|
475
|
+
response: Response,
|
|
476
|
+
maximumResponseBytes: number,
|
|
477
|
+
): Promise<Response> {
|
|
478
|
+
const declared = response.headers.get("content-length");
|
|
479
|
+
if (declared !== null) {
|
|
480
|
+
const bytes = Number(declared);
|
|
481
|
+
if (Number.isFinite(bytes) && bytes > maximumResponseBytes) {
|
|
482
|
+
await response.body?.cancel().catch(() => undefined);
|
|
483
|
+
throw new GatewayMediaExecutionError("invalid-response");
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
if (response.body === null) return response;
|
|
487
|
+
const reader = response.body.getReader();
|
|
488
|
+
let bytesRead = 0;
|
|
489
|
+
const body = new ReadableStream<Uint8Array>({
|
|
490
|
+
cancel: async reason => await reader.cancel(reason),
|
|
491
|
+
pull: async (controller) => {
|
|
492
|
+
try {
|
|
493
|
+
const next = await reader.read();
|
|
494
|
+
if (next.done) {
|
|
495
|
+
controller.close();
|
|
496
|
+
reader.releaseLock();
|
|
497
|
+
return;
|
|
498
|
+
}
|
|
499
|
+
bytesRead += next.value.byteLength;
|
|
500
|
+
if (bytesRead > maximumResponseBytes) {
|
|
501
|
+
await reader.cancel().catch(() => undefined);
|
|
502
|
+
controller.error(new GatewayMediaExecutionError("invalid-response"));
|
|
503
|
+
return;
|
|
504
|
+
}
|
|
505
|
+
controller.enqueue(next.value);
|
|
506
|
+
} catch {
|
|
507
|
+
controller.error(new GatewayMediaExecutionError("invalid-response"));
|
|
508
|
+
}
|
|
509
|
+
},
|
|
510
|
+
});
|
|
511
|
+
return new Response(body, {
|
|
512
|
+
headers: response.headers,
|
|
513
|
+
status: response.status,
|
|
514
|
+
statusText: response.statusText,
|
|
515
|
+
});
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
/**
|
|
519
|
+
* A credential-bearing SDK request may reach only the pinned Gateway v4 media
|
|
520
|
+
* prefix. Redirects fail closed, and a counting stream bounds every response
|
|
521
|
+
* without buffering long-running video SSE responses in memory.
|
|
522
|
+
*/
|
|
523
|
+
export function createFixedGatewayMediaApiFetch(options: Readonly<{
|
|
524
|
+
fetch?: GatewayMediaFetch;
|
|
525
|
+
maximumResponseBytes?: number;
|
|
526
|
+
}> = {}): GatewayMediaFetch {
|
|
527
|
+
const fetchImplementation = options.fetch ?? globalThis.fetch;
|
|
528
|
+
const maximumResponseBytes = options.maximumResponseBytes
|
|
529
|
+
?? DEFAULT_GATEWAY_RESPONSE_MAX_BYTES;
|
|
530
|
+
if (
|
|
531
|
+
!Number.isSafeInteger(maximumResponseBytes)
|
|
532
|
+
|| maximumResponseBytes < 1
|
|
533
|
+
|| maximumResponseBytes > 1024 * 1024 * 1024
|
|
534
|
+
) {
|
|
535
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
536
|
+
}
|
|
537
|
+
const expected = new URL(GATEWAY_MEDIA_API_BASE_URL);
|
|
538
|
+
return async (input, init) => {
|
|
539
|
+
const url = gatewayRequestUrl(input);
|
|
540
|
+
if (
|
|
541
|
+
url.origin !== expected.origin
|
|
542
|
+
|| (
|
|
543
|
+
url.pathname !== expected.pathname
|
|
544
|
+
&& !url.pathname.startsWith(`${expected.pathname}/`)
|
|
545
|
+
)
|
|
546
|
+
|| url.username !== ""
|
|
547
|
+
|| url.password !== ""
|
|
548
|
+
) {
|
|
549
|
+
throw new GatewayMediaExecutionError("provider-failed");
|
|
550
|
+
}
|
|
551
|
+
let response: Response;
|
|
552
|
+
try {
|
|
553
|
+
const base = input instanceof Request
|
|
554
|
+
? new Request(url.href, input)
|
|
555
|
+
: new Request(url.href);
|
|
556
|
+
const canonicalRequest = new Request(base, { ...init, redirect: "error" });
|
|
557
|
+
response = await fetchImplementation(canonicalRequest);
|
|
558
|
+
} catch (error) {
|
|
559
|
+
if (error instanceof GatewayMediaExecutionError) throw error;
|
|
560
|
+
throw new GatewayMediaExecutionError("provider-failed");
|
|
561
|
+
}
|
|
562
|
+
if (response.redirected || (response.status >= 300 && response.status < 400)) {
|
|
563
|
+
await response.body?.cancel().catch(() => undefined);
|
|
564
|
+
throw new GatewayMediaExecutionError("provider-failed");
|
|
565
|
+
}
|
|
566
|
+
return await boundedGatewayResponse(response, maximumResponseBytes);
|
|
567
|
+
};
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
function combineAbortSignals(
|
|
571
|
+
callerSignal: AbortSignal | undefined,
|
|
572
|
+
timeoutMs: number,
|
|
573
|
+
): Readonly<{ dispose(): void; signal: AbortSignal }> {
|
|
574
|
+
const controller = new AbortController();
|
|
575
|
+
const abortFromCaller = (): void => controller.abort(callerSignal?.reason);
|
|
576
|
+
if (callerSignal?.aborted === true) abortFromCaller();
|
|
577
|
+
callerSignal?.addEventListener("abort", abortFromCaller, { once: true });
|
|
578
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
579
|
+
return {
|
|
580
|
+
dispose: () => {
|
|
581
|
+
clearTimeout(timer);
|
|
582
|
+
callerSignal?.removeEventListener("abort", abortFromCaller);
|
|
583
|
+
},
|
|
584
|
+
signal: controller.signal,
|
|
585
|
+
};
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
function settleBeforeAbort<Result>(
|
|
589
|
+
operation: Promise<Result>,
|
|
590
|
+
signal: AbortSignal,
|
|
591
|
+
errorCode: GatewayMediaExecutionErrorCode,
|
|
592
|
+
): Promise<Result> {
|
|
593
|
+
if (signal.aborted) return Promise.reject(new GatewayMediaExecutionError(errorCode));
|
|
594
|
+
return new Promise<Result>((resolveResult, rejectResult) => {
|
|
595
|
+
let settled = false;
|
|
596
|
+
const finish = (settle: () => void): void => {
|
|
597
|
+
if (settled) return;
|
|
598
|
+
settled = true;
|
|
599
|
+
signal.removeEventListener("abort", abort);
|
|
600
|
+
settle();
|
|
601
|
+
};
|
|
602
|
+
const abort = (): void => finish(() => rejectResult(new GatewayMediaExecutionError(errorCode)));
|
|
603
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
604
|
+
if (signal.aborted) abort();
|
|
605
|
+
void operation.then(
|
|
606
|
+
value => finish(() => resolveResult(value)),
|
|
607
|
+
error => finish(() => rejectResult(error)),
|
|
608
|
+
);
|
|
609
|
+
});
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
function assertProviderDispatchActive(signal: AbortSignal): void {
|
|
613
|
+
if (signal.aborted) {
|
|
614
|
+
throw new GatewayMediaExecutionError("provider-failed");
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
async function readBoundedDownload(
|
|
619
|
+
response: Response,
|
|
620
|
+
maximumBytes: number,
|
|
621
|
+
): Promise<Uint8Array> {
|
|
622
|
+
const declared = response.headers.get("content-length");
|
|
623
|
+
if (declared !== null) {
|
|
624
|
+
const bytes = Number(declared);
|
|
625
|
+
if (Number.isFinite(bytes) && bytes > maximumBytes) {
|
|
626
|
+
await response.body?.cancel().catch(() => undefined);
|
|
627
|
+
throw new GatewayMediaExecutionError("download-failed");
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
if (response.body === null) {
|
|
631
|
+
throw new GatewayMediaExecutionError("download-failed");
|
|
632
|
+
}
|
|
633
|
+
const reader = response.body.getReader();
|
|
634
|
+
const chunks: Uint8Array[] = [];
|
|
635
|
+
let length = 0;
|
|
636
|
+
try {
|
|
637
|
+
while (true) {
|
|
638
|
+
const next = await reader.read();
|
|
639
|
+
if (next.done) break;
|
|
640
|
+
length += next.value.byteLength;
|
|
641
|
+
if (length > maximumBytes) {
|
|
642
|
+
await reader.cancel().catch(() => undefined);
|
|
643
|
+
throw new GatewayMediaExecutionError("download-failed");
|
|
644
|
+
}
|
|
645
|
+
chunks.push(next.value);
|
|
646
|
+
}
|
|
647
|
+
} finally {
|
|
648
|
+
reader.releaseLock();
|
|
649
|
+
}
|
|
650
|
+
const body = new Uint8Array(length);
|
|
651
|
+
let offset = 0;
|
|
652
|
+
for (const chunk of chunks) {
|
|
653
|
+
body.set(chunk, offset);
|
|
654
|
+
offset += chunk.byteLength;
|
|
655
|
+
}
|
|
656
|
+
return body;
|
|
657
|
+
}
|
|
658
|
+
|
|
659
|
+
interface GatewayResolvedAddress {
|
|
660
|
+
readonly address: string;
|
|
661
|
+
readonly family: 4 | 6;
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
type GatewayHostnameResolver = (
|
|
665
|
+
hostname: string,
|
|
666
|
+
) => Promise<readonly GatewayResolvedAddress[]>;
|
|
667
|
+
|
|
668
|
+
type GatewayPinnedHttpsRequest = (
|
|
669
|
+
url: URL,
|
|
670
|
+
input: Readonly<{
|
|
671
|
+
abortSignal: AbortSignal;
|
|
672
|
+
addresses: readonly GatewayResolvedAddress[];
|
|
673
|
+
}>,
|
|
674
|
+
) => Promise<Response>;
|
|
675
|
+
|
|
676
|
+
const BLOCKED_DOWNLOAD_ADDRESSES = (() => {
|
|
677
|
+
const ipv4 = new BlockList();
|
|
678
|
+
const ipv6 = new BlockList();
|
|
679
|
+
for (const [address, prefix] of [
|
|
680
|
+
["0.0.0.0", 8],
|
|
681
|
+
["10.0.0.0", 8],
|
|
682
|
+
["100.64.0.0", 10],
|
|
683
|
+
["127.0.0.0", 8],
|
|
684
|
+
["169.254.0.0", 16],
|
|
685
|
+
["172.16.0.0", 12],
|
|
686
|
+
["192.0.0.0", 24],
|
|
687
|
+
["192.0.2.0", 24],
|
|
688
|
+
["192.168.0.0", 16],
|
|
689
|
+
["198.18.0.0", 15],
|
|
690
|
+
["198.51.100.0", 24],
|
|
691
|
+
["203.0.113.0", 24],
|
|
692
|
+
["224.0.0.0", 4],
|
|
693
|
+
["240.0.0.0", 4],
|
|
694
|
+
] as const) {
|
|
695
|
+
ipv4.addSubnet(address, prefix, "ipv4");
|
|
696
|
+
}
|
|
697
|
+
for (const [address, prefix] of [
|
|
698
|
+
["::", 128],
|
|
699
|
+
["::1", 128],
|
|
700
|
+
["::ffff:0:0", 96],
|
|
701
|
+
["64:ff9b::", 96],
|
|
702
|
+
["64:ff9b:1::", 48],
|
|
703
|
+
["100::", 64],
|
|
704
|
+
["2001:db8::", 32],
|
|
705
|
+
["2001::", 23],
|
|
706
|
+
["2002::", 16],
|
|
707
|
+
["fc00::", 7],
|
|
708
|
+
["fe80::", 10],
|
|
709
|
+
["fec0::", 10],
|
|
710
|
+
["ff00::", 8],
|
|
711
|
+
] as const) {
|
|
712
|
+
ipv6.addSubnet(address, prefix, "ipv6");
|
|
713
|
+
}
|
|
714
|
+
return { ipv4, ipv6 };
|
|
715
|
+
})();
|
|
716
|
+
|
|
717
|
+
function normalizedDownloadHostname(url: URL): string {
|
|
718
|
+
const raw = url.hostname.toLocaleLowerCase("en-US").replace(/\.+$/u, "");
|
|
719
|
+
const hostname = raw.startsWith("[") && raw.endsWith("]")
|
|
720
|
+
? raw.slice(1, -1)
|
|
721
|
+
: raw;
|
|
722
|
+
if (
|
|
723
|
+
hostname.length === 0
|
|
724
|
+
|| hostname.includes("%")
|
|
725
|
+
|| hostname === "localhost"
|
|
726
|
+
|| hostname === "localhost.localdomain"
|
|
727
|
+
|| [".localhost", ".local", ".internal", ".home", ".lan"]
|
|
728
|
+
.some(suffix => hostname.endsWith(suffix))
|
|
729
|
+
) {
|
|
730
|
+
throw new GatewayMediaExecutionError("download-failed");
|
|
731
|
+
}
|
|
732
|
+
return hostname;
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
function assertPublicDownloadAddress(address: GatewayResolvedAddress): void {
|
|
736
|
+
const family = isIP(address.address);
|
|
737
|
+
if (
|
|
738
|
+
family !== address.family
|
|
739
|
+
|| (
|
|
740
|
+
family === 4
|
|
741
|
+
? BLOCKED_DOWNLOAD_ADDRESSES.ipv4.check(address.address, "ipv4")
|
|
742
|
+
: BLOCKED_DOWNLOAD_ADDRESSES.ipv6.check(address.address, "ipv6")
|
|
743
|
+
)
|
|
744
|
+
) {
|
|
745
|
+
throw new GatewayMediaExecutionError("download-failed");
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
async function assertSafeDownloadTarget(
|
|
750
|
+
url: URL,
|
|
751
|
+
resolveHostname: GatewayHostnameResolver,
|
|
752
|
+
signal: AbortSignal,
|
|
753
|
+
): Promise<readonly GatewayResolvedAddress[]> {
|
|
754
|
+
if (
|
|
755
|
+
url.protocol !== "https:"
|
|
756
|
+
|| url.username !== ""
|
|
757
|
+
|| url.password !== ""
|
|
758
|
+
) {
|
|
759
|
+
throw new GatewayMediaExecutionError("download-failed");
|
|
760
|
+
}
|
|
761
|
+
const hostname = normalizedDownloadHostname(url);
|
|
762
|
+
const literalFamily = isIP(hostname);
|
|
763
|
+
if (literalFamily === 4 || literalFamily === 6) {
|
|
764
|
+
const address = {
|
|
765
|
+
address: hostname,
|
|
766
|
+
family: literalFamily,
|
|
767
|
+
} as const;
|
|
768
|
+
assertPublicDownloadAddress(address);
|
|
769
|
+
return [address];
|
|
770
|
+
}
|
|
771
|
+
let addresses: readonly GatewayResolvedAddress[];
|
|
772
|
+
try {
|
|
773
|
+
const resolution = resolveHostname(hostname);
|
|
774
|
+
addresses = await new Promise<readonly GatewayResolvedAddress[]>(
|
|
775
|
+
(resolvePromise, reject) => {
|
|
776
|
+
const abort = (): void => reject(
|
|
777
|
+
new GatewayMediaExecutionError("download-failed"),
|
|
778
|
+
);
|
|
779
|
+
if (signal.aborted) {
|
|
780
|
+
abort();
|
|
781
|
+
return;
|
|
782
|
+
}
|
|
783
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
784
|
+
void resolution.then(
|
|
785
|
+
value => {
|
|
786
|
+
signal.removeEventListener("abort", abort);
|
|
787
|
+
resolvePromise(value);
|
|
788
|
+
},
|
|
789
|
+
error => {
|
|
790
|
+
signal.removeEventListener("abort", abort);
|
|
791
|
+
reject(
|
|
792
|
+
error instanceof Error
|
|
793
|
+
? error
|
|
794
|
+
: new Error("DNS lookup failed."),
|
|
795
|
+
);
|
|
796
|
+
},
|
|
797
|
+
);
|
|
798
|
+
},
|
|
799
|
+
);
|
|
800
|
+
} catch {
|
|
801
|
+
throw new GatewayMediaExecutionError("download-failed");
|
|
802
|
+
}
|
|
803
|
+
if (addresses.length === 0 || addresses.length > 64) {
|
|
804
|
+
throw new GatewayMediaExecutionError("download-failed");
|
|
805
|
+
}
|
|
806
|
+
for (const address of addresses) assertPublicDownloadAddress(address);
|
|
807
|
+
return addresses;
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
function pinnedLookup(
|
|
811
|
+
addresses: readonly GatewayResolvedAddress[],
|
|
812
|
+
): LookupFunction {
|
|
813
|
+
return (_hostname, options, callback) => {
|
|
814
|
+
const requestedFamily = options.family;
|
|
815
|
+
const compatible = addresses.filter(address => (
|
|
816
|
+
requestedFamily === undefined
|
|
817
|
+
|| requestedFamily === 0
|
|
818
|
+
|| requestedFamily === address.family
|
|
819
|
+
));
|
|
820
|
+
if (compatible.length === 0) {
|
|
821
|
+
const error: NodeJS.ErrnoException =
|
|
822
|
+
new Error("No validated address matches the requested family.");
|
|
823
|
+
error.code = "ENOTFOUND";
|
|
824
|
+
callback(error, "", 0);
|
|
825
|
+
return;
|
|
826
|
+
}
|
|
827
|
+
if (options.all) {
|
|
828
|
+
callback(
|
|
829
|
+
null,
|
|
830
|
+
compatible.map(address => ({
|
|
831
|
+
address: address.address,
|
|
832
|
+
family: address.family,
|
|
833
|
+
})),
|
|
834
|
+
);
|
|
835
|
+
return;
|
|
836
|
+
}
|
|
837
|
+
const selected = compatible[0]!;
|
|
838
|
+
callback(null, selected.address, selected.family);
|
|
839
|
+
};
|
|
840
|
+
}
|
|
841
|
+
|
|
842
|
+
function pinnedHttpsRequest(
|
|
843
|
+
url: URL,
|
|
844
|
+
input: Readonly<{
|
|
845
|
+
abortSignal: AbortSignal;
|
|
846
|
+
addresses: readonly GatewayResolvedAddress[];
|
|
847
|
+
}>,
|
|
848
|
+
): Promise<Response> {
|
|
849
|
+
return new Promise<Response>((resolvePromise, reject) => {
|
|
850
|
+
const request = httpsRequest(url, {
|
|
851
|
+
agent: false,
|
|
852
|
+
headers: {
|
|
853
|
+
accept: "video/*,audio/*,image/*,application/octet-stream",
|
|
854
|
+
},
|
|
855
|
+
lookup: pinnedLookup(input.addresses),
|
|
856
|
+
method: "GET",
|
|
857
|
+
signal: input.abortSignal,
|
|
858
|
+
}, response => {
|
|
859
|
+
const status = response.statusCode ?? 0;
|
|
860
|
+
if (status < 200 || status > 599) {
|
|
861
|
+
response.destroy();
|
|
862
|
+
reject(new GatewayMediaExecutionError("download-failed"));
|
|
863
|
+
return;
|
|
864
|
+
}
|
|
865
|
+
try {
|
|
866
|
+
const headers = new Headers();
|
|
867
|
+
for (let index = 0; index < response.rawHeaders.length; index += 2) {
|
|
868
|
+
const name = response.rawHeaders[index];
|
|
869
|
+
const value = response.rawHeaders[index + 1];
|
|
870
|
+
if (name !== undefined && value !== undefined) headers.append(name, value);
|
|
871
|
+
}
|
|
872
|
+
const forbidsBody = status === 204
|
|
873
|
+
|| status === 304
|
|
874
|
+
|| (status >= 100 && status < 200);
|
|
875
|
+
if (forbidsBody) response.resume();
|
|
876
|
+
resolvePromise(new Response(
|
|
877
|
+
forbidsBody
|
|
878
|
+
? null
|
|
879
|
+
: Readable.toWeb(response) as unknown as ReadableStream<Uint8Array>,
|
|
880
|
+
{
|
|
881
|
+
headers,
|
|
882
|
+
status,
|
|
883
|
+
...(response.statusMessage === undefined
|
|
884
|
+
? {}
|
|
885
|
+
: { statusText: response.statusMessage }),
|
|
886
|
+
},
|
|
887
|
+
));
|
|
888
|
+
} catch {
|
|
889
|
+
response.destroy();
|
|
890
|
+
reject(new GatewayMediaExecutionError("download-failed"));
|
|
891
|
+
}
|
|
892
|
+
});
|
|
893
|
+
request.once("error", () => {
|
|
894
|
+
reject(new GatewayMediaExecutionError("download-failed"));
|
|
895
|
+
});
|
|
896
|
+
request.end();
|
|
897
|
+
});
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
async function fetchSafeDownload(
|
|
901
|
+
initialUrl: URL,
|
|
902
|
+
input: Readonly<{
|
|
903
|
+
abortSignal: AbortSignal;
|
|
904
|
+
request: GatewayPinnedHttpsRequest;
|
|
905
|
+
resolveHostname: GatewayHostnameResolver;
|
|
906
|
+
}>,
|
|
907
|
+
): Promise<Response> {
|
|
908
|
+
let currentUrl = initialUrl;
|
|
909
|
+
const redirectStatuses = new Set([301, 302, 303, 307, 308]);
|
|
910
|
+
for (
|
|
911
|
+
let redirectCount = 0;
|
|
912
|
+
redirectCount <= MAXIMUM_DOWNLOAD_REDIRECTS;
|
|
913
|
+
redirectCount += 1
|
|
914
|
+
) {
|
|
915
|
+
const addresses = await assertSafeDownloadTarget(
|
|
916
|
+
currentUrl,
|
|
917
|
+
input.resolveHostname,
|
|
918
|
+
input.abortSignal,
|
|
919
|
+
);
|
|
920
|
+
let response: Response;
|
|
921
|
+
try {
|
|
922
|
+
response = await input.request(currentUrl, {
|
|
923
|
+
abortSignal: input.abortSignal,
|
|
924
|
+
addresses,
|
|
925
|
+
});
|
|
926
|
+
} catch (error) {
|
|
927
|
+
if (error instanceof GatewayMediaExecutionError) throw error;
|
|
928
|
+
throw new GatewayMediaExecutionError("download-failed");
|
|
929
|
+
}
|
|
930
|
+
const location = response.headers.get("location");
|
|
931
|
+
if (redirectStatuses.has(response.status) && location !== null) {
|
|
932
|
+
await response.body?.cancel().catch(() => undefined);
|
|
933
|
+
if (redirectCount === MAXIMUM_DOWNLOAD_REDIRECTS) {
|
|
934
|
+
throw new GatewayMediaExecutionError("download-failed");
|
|
935
|
+
}
|
|
936
|
+
try {
|
|
937
|
+
currentUrl = new URL(location, currentUrl);
|
|
938
|
+
} catch {
|
|
939
|
+
throw new GatewayMediaExecutionError("download-failed");
|
|
940
|
+
}
|
|
941
|
+
continue;
|
|
942
|
+
}
|
|
943
|
+
if (
|
|
944
|
+
!response.ok
|
|
945
|
+
|| response.redirected
|
|
946
|
+
|| (response.status >= 300 && response.status < 400)
|
|
947
|
+
) {
|
|
948
|
+
await response.body?.cancel().catch(() => undefined);
|
|
949
|
+
throw new GatewayMediaExecutionError("download-failed");
|
|
950
|
+
}
|
|
951
|
+
return response;
|
|
952
|
+
}
|
|
953
|
+
throw new GatewayMediaExecutionError("download-failed");
|
|
954
|
+
}
|
|
955
|
+
|
|
956
|
+
export function createBoundedGatewayMediaDownload(options: Readonly<{
|
|
957
|
+
maximumBytes?: number;
|
|
958
|
+
request?: GatewayPinnedHttpsRequest;
|
|
959
|
+
resolveHostname?: GatewayHostnameResolver;
|
|
960
|
+
timeoutMs?: number;
|
|
961
|
+
}> = {}): GatewayMediaDownload {
|
|
962
|
+
const request = options.request ?? pinnedHttpsRequest;
|
|
963
|
+
const resolveHostname = options.resolveHostname
|
|
964
|
+
?? (async hostname => (await lookup(hostname, {
|
|
965
|
+
all: true,
|
|
966
|
+
verbatim: true,
|
|
967
|
+
})).flatMap(answer => (
|
|
968
|
+
answer.family === 4 || answer.family === 6
|
|
969
|
+
? [{ address: answer.address, family: answer.family }]
|
|
970
|
+
: []
|
|
971
|
+
)));
|
|
972
|
+
const maximumBytes = options.maximumBytes ?? MAXIMUM_DOWNLOAD_BYTES;
|
|
973
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_VIDEO_TIMEOUT_MS;
|
|
974
|
+
if (
|
|
975
|
+
!Number.isSafeInteger(maximumBytes)
|
|
976
|
+
|| maximumBytes < 1
|
|
977
|
+
|| maximumBytes > 1024 * 1024 * 1024
|
|
978
|
+
|| !Number.isSafeInteger(timeoutMs)
|
|
979
|
+
|| timeoutMs < 1
|
|
980
|
+
|| timeoutMs > 30 * 60_000
|
|
981
|
+
) {
|
|
982
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
983
|
+
}
|
|
984
|
+
return async ({ abortSignal, maximumBytes: requestMaximumBytes, url }) => {
|
|
985
|
+
const effectiveMaximumBytes = requestMaximumBytes === undefined
|
|
986
|
+
? maximumBytes
|
|
987
|
+
: Math.min(maximumBytes, requestMaximumBytes);
|
|
988
|
+
if (
|
|
989
|
+
!Number.isSafeInteger(effectiveMaximumBytes)
|
|
990
|
+
|| effectiveMaximumBytes < 1
|
|
991
|
+
) {
|
|
992
|
+
throw new GatewayMediaExecutionError("download-failed");
|
|
993
|
+
}
|
|
994
|
+
const combined = combineAbortSignals(abortSignal, timeoutMs);
|
|
995
|
+
try {
|
|
996
|
+
const response = await settleBeforeAbort(fetchSafeDownload(url, {
|
|
997
|
+
abortSignal: combined.signal,
|
|
998
|
+
request,
|
|
999
|
+
resolveHostname,
|
|
1000
|
+
}), combined.signal, "download-failed");
|
|
1001
|
+
return {
|
|
1002
|
+
data: await settleBeforeAbort(
|
|
1003
|
+
readBoundedDownload(response, effectiveMaximumBytes),
|
|
1004
|
+
combined.signal,
|
|
1005
|
+
"download-failed",
|
|
1006
|
+
),
|
|
1007
|
+
mediaType: response.headers.get("content-type")?.split(";", 1)[0]?.trim(),
|
|
1008
|
+
};
|
|
1009
|
+
} catch (error) {
|
|
1010
|
+
if (error instanceof GatewayMediaExecutionError) throw error;
|
|
1011
|
+
throw new GatewayMediaExecutionError("download-failed");
|
|
1012
|
+
} finally {
|
|
1013
|
+
combined.dispose();
|
|
1014
|
+
}
|
|
1015
|
+
};
|
|
1016
|
+
}
|
|
1017
|
+
|
|
1018
|
+
export function createAggregateGatewayMediaDownload(
|
|
1019
|
+
download: GatewayMediaDownload,
|
|
1020
|
+
options: Readonly<{
|
|
1021
|
+
maximumFileBytes?: number;
|
|
1022
|
+
maximumTotalBytes?: number;
|
|
1023
|
+
}> = {},
|
|
1024
|
+
): GatewayMediaDownload {
|
|
1025
|
+
const maximumFileBytes = options.maximumFileBytes ?? MAXIMUM_DOWNLOAD_BYTES;
|
|
1026
|
+
const maximumTotalBytes = options.maximumTotalBytes
|
|
1027
|
+
?? MAXIMUM_VIDEO_OUTPUT_TOTAL_BYTES;
|
|
1028
|
+
if (
|
|
1029
|
+
!Number.isSafeInteger(maximumFileBytes)
|
|
1030
|
+
|| maximumFileBytes < 1
|
|
1031
|
+
|| maximumFileBytes > MAXIMUM_DOWNLOAD_BYTES
|
|
1032
|
+
|| !Number.isSafeInteger(maximumTotalBytes)
|
|
1033
|
+
|| maximumTotalBytes < 1
|
|
1034
|
+
|| maximumTotalBytes > MAXIMUM_VIDEO_OUTPUT_TOTAL_BYTES
|
|
1035
|
+
) {
|
|
1036
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
1037
|
+
}
|
|
1038
|
+
let remainingBytes = maximumTotalBytes;
|
|
1039
|
+
let previous = Promise.resolve();
|
|
1040
|
+
return input => {
|
|
1041
|
+
const current = previous.then(async () => {
|
|
1042
|
+
if (input.abortSignal?.aborted === true || remainingBytes < 1) {
|
|
1043
|
+
throw new GatewayMediaExecutionError("download-failed");
|
|
1044
|
+
}
|
|
1045
|
+
const result = await download({
|
|
1046
|
+
...input,
|
|
1047
|
+
maximumBytes: Math.min(maximumFileBytes, remainingBytes),
|
|
1048
|
+
});
|
|
1049
|
+
if (result.data.byteLength < 1 || result.data.byteLength > remainingBytes) {
|
|
1050
|
+
throw new GatewayMediaExecutionError("download-failed");
|
|
1051
|
+
}
|
|
1052
|
+
remainingBytes -= result.data.byteLength;
|
|
1053
|
+
return result;
|
|
1054
|
+
});
|
|
1055
|
+
previous = current.then(
|
|
1056
|
+
() => undefined,
|
|
1057
|
+
() => undefined,
|
|
1058
|
+
);
|
|
1059
|
+
return current;
|
|
1060
|
+
};
|
|
1061
|
+
}
|
|
1062
|
+
|
|
1063
|
+
function assertAllowedKeys(
|
|
1064
|
+
value: Record<string, unknown>,
|
|
1065
|
+
allowed: ReadonlySet<string>,
|
|
1066
|
+
): void {
|
|
1067
|
+
if (Object.keys(value).some(key => !allowed.has(key))) {
|
|
1068
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
1069
|
+
}
|
|
1070
|
+
}
|
|
1071
|
+
|
|
1072
|
+
function parseString(
|
|
1073
|
+
value: unknown,
|
|
1074
|
+
maximumLength: number,
|
|
1075
|
+
allowEmpty = false,
|
|
1076
|
+
): string {
|
|
1077
|
+
if (
|
|
1078
|
+
typeof value !== "string"
|
|
1079
|
+
|| value.length > maximumLength
|
|
1080
|
+
|| (!allowEmpty && value.trim().length === 0)
|
|
1081
|
+
|| [...value].some((character) => {
|
|
1082
|
+
const codePoint = character.codePointAt(0);
|
|
1083
|
+
return codePoint !== undefined && (
|
|
1084
|
+
codePoint <= 8
|
|
1085
|
+
|| codePoint === 11
|
|
1086
|
+
|| codePoint === 12
|
|
1087
|
+
|| (codePoint >= 14 && codePoint <= 31)
|
|
1088
|
+
|| codePoint === 127
|
|
1089
|
+
);
|
|
1090
|
+
})
|
|
1091
|
+
) {
|
|
1092
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
1093
|
+
}
|
|
1094
|
+
return value;
|
|
1095
|
+
}
|
|
1096
|
+
|
|
1097
|
+
function parseOptionalString(
|
|
1098
|
+
value: unknown,
|
|
1099
|
+
maximumLength: number,
|
|
1100
|
+
): string | undefined {
|
|
1101
|
+
return value === undefined ? undefined : parseString(value, maximumLength);
|
|
1102
|
+
}
|
|
1103
|
+
|
|
1104
|
+
function parseSafeInteger(
|
|
1105
|
+
value: unknown,
|
|
1106
|
+
minimum: number,
|
|
1107
|
+
maximum: number,
|
|
1108
|
+
): number {
|
|
1109
|
+
if (!Number.isSafeInteger(value) || (value as number) < minimum || (value as number) > maximum) {
|
|
1110
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
1111
|
+
}
|
|
1112
|
+
return value as number;
|
|
1113
|
+
}
|
|
1114
|
+
|
|
1115
|
+
function parseOptionalSafeInteger(
|
|
1116
|
+
value: unknown,
|
|
1117
|
+
minimum: number,
|
|
1118
|
+
maximum: number,
|
|
1119
|
+
): number | undefined {
|
|
1120
|
+
return value === undefined ? undefined : parseSafeInteger(value, minimum, maximum);
|
|
1121
|
+
}
|
|
1122
|
+
|
|
1123
|
+
function parseOptionalNumber(
|
|
1124
|
+
value: unknown,
|
|
1125
|
+
minimum: number,
|
|
1126
|
+
maximum: number,
|
|
1127
|
+
): number | undefined {
|
|
1128
|
+
if (value === undefined) return undefined;
|
|
1129
|
+
if (
|
|
1130
|
+
typeof value !== "number"
|
|
1131
|
+
|| !Number.isFinite(value)
|
|
1132
|
+
|| value < minimum
|
|
1133
|
+
|| value > maximum
|
|
1134
|
+
) {
|
|
1135
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
1136
|
+
}
|
|
1137
|
+
return value;
|
|
1138
|
+
}
|
|
1139
|
+
|
|
1140
|
+
function parseOptionalBoolean(value: unknown): boolean | undefined {
|
|
1141
|
+
if (value === undefined) return undefined;
|
|
1142
|
+
if (typeof value !== "boolean") {
|
|
1143
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
1144
|
+
}
|
|
1145
|
+
return value;
|
|
1146
|
+
}
|
|
1147
|
+
|
|
1148
|
+
function parseModelId(value: unknown): string {
|
|
1149
|
+
const id = parseString(value, 256);
|
|
1150
|
+
if (!id.includes("/") || /\s/u.test(id) || id.startsWith("/") || id.endsWith("/")) {
|
|
1151
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
1152
|
+
}
|
|
1153
|
+
return id;
|
|
1154
|
+
}
|
|
1155
|
+
|
|
1156
|
+
function parseMediaType(value: unknown): string {
|
|
1157
|
+
const mediaType = parseString(value, 128).toLocaleLowerCase("en-US");
|
|
1158
|
+
if (!/^[a-z0-9][a-z0-9!#$&^_.+-]*\/[a-z0-9][a-z0-9!#$&^_.+-]*$/iu.test(mediaType)) {
|
|
1159
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
1160
|
+
}
|
|
1161
|
+
return mediaType;
|
|
1162
|
+
}
|
|
1163
|
+
|
|
1164
|
+
export function validateGatewayMediaSourceUrl(value: unknown): string {
|
|
1165
|
+
if (typeof value !== "string" || value.length < 1 || value.length > 8_192) {
|
|
1166
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
1167
|
+
}
|
|
1168
|
+
let url: URL;
|
|
1169
|
+
try {
|
|
1170
|
+
url = new URL(value);
|
|
1171
|
+
} catch {
|
|
1172
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
1173
|
+
}
|
|
1174
|
+
if (
|
|
1175
|
+
url.protocol !== "https:"
|
|
1176
|
+
|| url.username !== ""
|
|
1177
|
+
|| url.password !== ""
|
|
1178
|
+
|| url.hash !== ""
|
|
1179
|
+
) {
|
|
1180
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
1181
|
+
}
|
|
1182
|
+
try {
|
|
1183
|
+
const hostname = normalizedDownloadHostname(url);
|
|
1184
|
+
const family = isIP(hostname);
|
|
1185
|
+
if (family === 4 || family === 6) {
|
|
1186
|
+
assertPublicDownloadAddress({ address: hostname, family });
|
|
1187
|
+
}
|
|
1188
|
+
} catch {
|
|
1189
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
1190
|
+
}
|
|
1191
|
+
return url.href;
|
|
1192
|
+
}
|
|
1193
|
+
|
|
1194
|
+
function gatewayInputByteLength(input: GatewayMediaInput): number {
|
|
1195
|
+
return input.data?.byteLength ?? 0;
|
|
1196
|
+
}
|
|
1197
|
+
|
|
1198
|
+
function gatewaySdkInput(input: GatewayMediaInput): GatewaySdkMediaInput {
|
|
1199
|
+
return input.data ?? input.url;
|
|
1200
|
+
}
|
|
1201
|
+
|
|
1202
|
+
function gatewayInputSource(input: GatewayMediaInput): "base64" | "url" {
|
|
1203
|
+
return input.url === undefined ? "base64" : "url";
|
|
1204
|
+
}
|
|
1205
|
+
|
|
1206
|
+
function parseInput(
|
|
1207
|
+
value: unknown,
|
|
1208
|
+
maximumBytes: number,
|
|
1209
|
+
allowedPrefixes: readonly string[],
|
|
1210
|
+
): GatewayMediaInput {
|
|
1211
|
+
if (!isRecord(value)) throw new GatewayMediaExecutionError("invalid-request");
|
|
1212
|
+
assertAllowedKeys(value, new Set(["data", "facts", "mediaType", "url"]));
|
|
1213
|
+
const data = value.data instanceof Uint8Array ? value.data : undefined;
|
|
1214
|
+
const hasData = data !== undefined;
|
|
1215
|
+
const hasUrl = value.url !== undefined;
|
|
1216
|
+
if (hasData === hasUrl) {
|
|
1217
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
1218
|
+
}
|
|
1219
|
+
if (data !== undefined && data.byteLength < 1) {
|
|
1220
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
1221
|
+
}
|
|
1222
|
+
if (data !== undefined && data.byteLength > maximumBytes) {
|
|
1223
|
+
throw new GatewayMediaExecutionError("input-too-large");
|
|
1224
|
+
}
|
|
1225
|
+
const mediaType = parseMediaType(value.mediaType);
|
|
1226
|
+
if (!allowedPrefixes.some(prefix => mediaType.startsWith(prefix))) {
|
|
1227
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
1228
|
+
}
|
|
1229
|
+
if (data !== undefined && !gatewayMediaBytesMatchType(data, mediaType)) {
|
|
1230
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
1231
|
+
}
|
|
1232
|
+
let facts: GatewayMediaInput["facts"];
|
|
1233
|
+
if (value.facts !== undefined) {
|
|
1234
|
+
if (!isRecord(value.facts)) {
|
|
1235
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
1236
|
+
}
|
|
1237
|
+
assertAllowedKeys(value.facts, new Set([
|
|
1238
|
+
"durationSeconds",
|
|
1239
|
+
"height",
|
|
1240
|
+
"width",
|
|
1241
|
+
]));
|
|
1242
|
+
const durationSeconds = parseOptionalNumber(
|
|
1243
|
+
value.facts.durationSeconds,
|
|
1244
|
+
0.000_001,
|
|
1245
|
+
1_000_000_000,
|
|
1246
|
+
);
|
|
1247
|
+
const height = parseOptionalSafeInteger(value.facts.height, 1, 1_000_000);
|
|
1248
|
+
const width = parseOptionalSafeInteger(value.facts.width, 1, 1_000_000);
|
|
1249
|
+
facts = {
|
|
1250
|
+
...(durationSeconds === undefined ? {} : { durationSeconds }),
|
|
1251
|
+
...(height === undefined ? {} : { height }),
|
|
1252
|
+
...(width === undefined ? {} : { width }),
|
|
1253
|
+
};
|
|
1254
|
+
}
|
|
1255
|
+
return data !== undefined
|
|
1256
|
+
? {
|
|
1257
|
+
data,
|
|
1258
|
+
...(facts === undefined ? {} : { facts }),
|
|
1259
|
+
mediaType,
|
|
1260
|
+
}
|
|
1261
|
+
: {
|
|
1262
|
+
...(facts === undefined ? {} : { facts }),
|
|
1263
|
+
mediaType,
|
|
1264
|
+
url: validateGatewayMediaSourceUrl(value.url),
|
|
1265
|
+
};
|
|
1266
|
+
}
|
|
1267
|
+
|
|
1268
|
+
function parseInputArray(
|
|
1269
|
+
value: unknown,
|
|
1270
|
+
options: Readonly<{
|
|
1271
|
+
allowedPrefixes: readonly string[];
|
|
1272
|
+
maximumBytes: number;
|
|
1273
|
+
maximumTotalBytes: number;
|
|
1274
|
+
}>,
|
|
1275
|
+
): readonly GatewayMediaInput[] {
|
|
1276
|
+
if (value === undefined) return [];
|
|
1277
|
+
if (!Array.isArray(value) || value.length > MAXIMUM_INPUTS) {
|
|
1278
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
1279
|
+
}
|
|
1280
|
+
const inputs = value.map(item => parseInput(
|
|
1281
|
+
item,
|
|
1282
|
+
options.maximumBytes,
|
|
1283
|
+
options.allowedPrefixes,
|
|
1284
|
+
));
|
|
1285
|
+
if (
|
|
1286
|
+
inputs.reduce((sum, input) => sum + gatewayInputByteLength(input), 0)
|
|
1287
|
+
> options.maximumTotalBytes
|
|
1288
|
+
) {
|
|
1289
|
+
throw new GatewayMediaExecutionError("input-too-large");
|
|
1290
|
+
}
|
|
1291
|
+
return inputs;
|
|
1292
|
+
}
|
|
1293
|
+
|
|
1294
|
+
function parseConsent(
|
|
1295
|
+
value: unknown,
|
|
1296
|
+
now: Date,
|
|
1297
|
+
): GatewayUploadConsent {
|
|
1298
|
+
if (!isRecord(value)) {
|
|
1299
|
+
throw new GatewayMediaExecutionError("cloud-consent-required");
|
|
1300
|
+
}
|
|
1301
|
+
assertAllowedKeys(value, new Set([
|
|
1302
|
+
"acknowledgedAt",
|
|
1303
|
+
"allowCloudUpload",
|
|
1304
|
+
"policy",
|
|
1305
|
+
]));
|
|
1306
|
+
if (
|
|
1307
|
+
value.allowCloudUpload !== true
|
|
1308
|
+
|| (
|
|
1309
|
+
value.policy !== GATEWAY_MEDIA_UPLOAD_POLICY
|
|
1310
|
+
&& value.policy !== LEGACY_GATEWAY_MEDIA_UPLOAD_POLICY
|
|
1311
|
+
)
|
|
1312
|
+
|| typeof value.acknowledgedAt !== "string"
|
|
1313
|
+
) {
|
|
1314
|
+
throw new GatewayMediaExecutionError("cloud-consent-required");
|
|
1315
|
+
}
|
|
1316
|
+
const acknowledgedMs = Date.parse(value.acknowledgedAt);
|
|
1317
|
+
const nowMs = now.getTime();
|
|
1318
|
+
if (
|
|
1319
|
+
!Number.isFinite(acknowledgedMs)
|
|
1320
|
+
|| new Date(acknowledgedMs).toISOString() !== value.acknowledgedAt
|
|
1321
|
+
|| acknowledgedMs < nowMs - 15 * 60_000
|
|
1322
|
+
|| acknowledgedMs > nowMs + 60_000
|
|
1323
|
+
) {
|
|
1324
|
+
throw new GatewayMediaExecutionError("cloud-consent-required");
|
|
1325
|
+
}
|
|
1326
|
+
return {
|
|
1327
|
+
acknowledgedAt: value.acknowledgedAt,
|
|
1328
|
+
allowCloudUpload: true,
|
|
1329
|
+
policy: value.policy,
|
|
1330
|
+
};
|
|
1331
|
+
}
|
|
1332
|
+
|
|
1333
|
+
function parseProviderOptions(
|
|
1334
|
+
value: unknown,
|
|
1335
|
+
): GatewayProviderOptions | undefined {
|
|
1336
|
+
if (value === undefined) return undefined;
|
|
1337
|
+
try {
|
|
1338
|
+
return parseGatewayProviderOptions(value);
|
|
1339
|
+
} catch {
|
|
1340
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
1341
|
+
}
|
|
1342
|
+
}
|
|
1343
|
+
|
|
1344
|
+
function inputDigest(
|
|
1345
|
+
input: GatewayMediaInput,
|
|
1346
|
+
role: string,
|
|
1347
|
+
): GatewayMediaInputDigest {
|
|
1348
|
+
const content = input.data ?? new TextEncoder().encode(input.url);
|
|
1349
|
+
return {
|
|
1350
|
+
bytes: gatewayInputByteLength(input),
|
|
1351
|
+
mediaType: input.mediaType,
|
|
1352
|
+
role,
|
|
1353
|
+
sha256: createHash("sha256").update(content).digest("hex"),
|
|
1354
|
+
source: input.url === undefined ? "inline" : "url",
|
|
1355
|
+
};
|
|
1356
|
+
}
|
|
1357
|
+
|
|
1358
|
+
function requestSummary(
|
|
1359
|
+
fields: Readonly<Record<string, GatewayJsonValue>>,
|
|
1360
|
+
prompt: string | undefined,
|
|
1361
|
+
providerOptions: GatewayProviderOptions | undefined,
|
|
1362
|
+
): Readonly<Record<string, GatewayJsonValue>> {
|
|
1363
|
+
const provider = providerOptions === undefined
|
|
1364
|
+
? undefined
|
|
1365
|
+
: gatewayProviderOptionsSummary(providerOptions);
|
|
1366
|
+
return {
|
|
1367
|
+
...fields,
|
|
1368
|
+
...(prompt === undefined
|
|
1369
|
+
? {}
|
|
1370
|
+
: {
|
|
1371
|
+
promptCharacters: prompt.length,
|
|
1372
|
+
promptSha256: createHash("sha256").update(prompt, "utf8").digest("hex"),
|
|
1373
|
+
}),
|
|
1374
|
+
...(provider === undefined
|
|
1375
|
+
? {}
|
|
1376
|
+
: {
|
|
1377
|
+
providerNamespaces: provider.namespaces,
|
|
1378
|
+
providerOptionsSha256: provider.sha256,
|
|
1379
|
+
}),
|
|
1380
|
+
};
|
|
1381
|
+
}
|
|
1382
|
+
|
|
1383
|
+
function optionalSummaryValue(
|
|
1384
|
+
value: boolean | number | string | undefined,
|
|
1385
|
+
): GatewayJsonValue | undefined {
|
|
1386
|
+
return value;
|
|
1387
|
+
}
|
|
1388
|
+
|
|
1389
|
+
function cleanSummary(
|
|
1390
|
+
value: Readonly<Record<string, GatewayJsonValue | undefined>>,
|
|
1391
|
+
): Readonly<Record<string, GatewayJsonValue>> {
|
|
1392
|
+
return Object.fromEntries(
|
|
1393
|
+
Object.entries(value).filter(
|
|
1394
|
+
(entry): entry is [string, GatewayJsonValue] => entry[1] !== undefined,
|
|
1395
|
+
),
|
|
1396
|
+
);
|
|
1397
|
+
}
|
|
1398
|
+
|
|
1399
|
+
function parseWarning(value: unknown): string {
|
|
1400
|
+
let message: string;
|
|
1401
|
+
if (typeof value === "string") {
|
|
1402
|
+
message = value;
|
|
1403
|
+
} else if (isObject(value) && typeof value.message === "string") {
|
|
1404
|
+
message = value.message;
|
|
1405
|
+
} else if (isObject(value) && typeof value.details === "string") {
|
|
1406
|
+
message = value.details;
|
|
1407
|
+
} else {
|
|
1408
|
+
message = "Provider warning";
|
|
1409
|
+
}
|
|
1410
|
+
const warningTypes = new Set([
|
|
1411
|
+
"other",
|
|
1412
|
+
"unsupported-setting",
|
|
1413
|
+
"unsupported-tool",
|
|
1414
|
+
]);
|
|
1415
|
+
const type = isObject(value)
|
|
1416
|
+
&& typeof value.type === "string"
|
|
1417
|
+
&& warningTypes.has(value.type)
|
|
1418
|
+
? value.type
|
|
1419
|
+
: "provider-warning";
|
|
1420
|
+
return `${type} sha256:${createHash("sha256").update(message).digest("hex")}`;
|
|
1421
|
+
}
|
|
1422
|
+
|
|
1423
|
+
function parseWarnings(value: unknown): readonly string[] {
|
|
1424
|
+
if (value === undefined) return [];
|
|
1425
|
+
if (!Array.isArray(value) || value.length > 100) {
|
|
1426
|
+
throw new GatewayMediaExecutionError("invalid-response");
|
|
1427
|
+
}
|
|
1428
|
+
return value.map(parseWarning);
|
|
1429
|
+
}
|
|
1430
|
+
|
|
1431
|
+
function boundedMetadataString(value: unknown): string | undefined {
|
|
1432
|
+
return typeof value === "string" && value.length > 0
|
|
1433
|
+
? value.slice(0, 256)
|
|
1434
|
+
: undefined;
|
|
1435
|
+
}
|
|
1436
|
+
|
|
1437
|
+
function opaqueMetadataDigest(value: unknown): string | undefined {
|
|
1438
|
+
const bounded = boundedMetadataString(value);
|
|
1439
|
+
return bounded === undefined
|
|
1440
|
+
? undefined
|
|
1441
|
+
: `sha256:${createHash("sha256").update(bounded, "utf8").digest("hex")}`;
|
|
1442
|
+
}
|
|
1443
|
+
|
|
1444
|
+
function knownCredentialType(value: unknown): string | undefined {
|
|
1445
|
+
const bounded = boundedMetadataString(value);
|
|
1446
|
+
return bounded !== undefined && new Set([
|
|
1447
|
+
"api-key",
|
|
1448
|
+
"oauth",
|
|
1449
|
+
"oidc",
|
|
1450
|
+
"service-account",
|
|
1451
|
+
"unknown",
|
|
1452
|
+
]).has(bounded)
|
|
1453
|
+
? bounded
|
|
1454
|
+
: undefined;
|
|
1455
|
+
}
|
|
1456
|
+
|
|
1457
|
+
function parseGatewayRoutingMetadata(
|
|
1458
|
+
providerMetadata: unknown,
|
|
1459
|
+
): GatewayMediaRoutingReceipt {
|
|
1460
|
+
const gateway = isObject(providerMetadata) && isObject(providerMetadata.gateway)
|
|
1461
|
+
? providerMetadata.gateway
|
|
1462
|
+
: undefined;
|
|
1463
|
+
const allAttempts = gateway !== undefined && Array.isArray(gateway.attempts)
|
|
1464
|
+
? gateway.attempts
|
|
1465
|
+
: [];
|
|
1466
|
+
const rawAttempts = allAttempts.slice(0, 32);
|
|
1467
|
+
const attempts = rawAttempts.flatMap((value) => {
|
|
1468
|
+
if (!isObject(value)) return [];
|
|
1469
|
+
const configuredTimeoutMs = typeof value.configuredTimeoutMs === "number"
|
|
1470
|
+
&& Number.isSafeInteger(value.configuredTimeoutMs)
|
|
1471
|
+
&& value.configuredTimeoutMs >= 0
|
|
1472
|
+
? value.configuredTimeoutMs
|
|
1473
|
+
: undefined;
|
|
1474
|
+
const statusCode = typeof value.statusCode === "number"
|
|
1475
|
+
&& Number.isSafeInteger(value.statusCode)
|
|
1476
|
+
&& value.statusCode >= 0
|
|
1477
|
+
&& value.statusCode <= 999
|
|
1478
|
+
? value.statusCode
|
|
1479
|
+
: undefined;
|
|
1480
|
+
const provider = opaqueMetadataDigest(value.provider);
|
|
1481
|
+
const error = boundedMetadataString(value.error);
|
|
1482
|
+
const credentialType = knownCredentialType(value.credentialType);
|
|
1483
|
+
const model = opaqueMetadataDigest(value.model);
|
|
1484
|
+
return [{
|
|
1485
|
+
...(configuredTimeoutMs === undefined ? {} : { configuredTimeoutMs }),
|
|
1486
|
+
...(credentialType === undefined ? {} : { credentialType }),
|
|
1487
|
+
...(error === undefined
|
|
1488
|
+
? {}
|
|
1489
|
+
: {
|
|
1490
|
+
error: `sha256:${createHash("sha256")
|
|
1491
|
+
.update(error, "utf8")
|
|
1492
|
+
.digest("hex")}`,
|
|
1493
|
+
}),
|
|
1494
|
+
...(model === undefined ? {} : { model }),
|
|
1495
|
+
...(provider === undefined ? {} : { provider }),
|
|
1496
|
+
...(typeof value.providerTimeout === "boolean"
|
|
1497
|
+
? { providerTimeout: value.providerTimeout }
|
|
1498
|
+
: {}),
|
|
1499
|
+
...(statusCode === undefined ? {} : { statusCode }),
|
|
1500
|
+
...(typeof value.success === "boolean" ? { success: value.success } : {}),
|
|
1501
|
+
}];
|
|
1502
|
+
});
|
|
1503
|
+
const directProvider = opaqueMetadataDigest(gateway?.provider);
|
|
1504
|
+
const providers = [
|
|
1505
|
+
...(directProvider === undefined ? [] : [directProvider]),
|
|
1506
|
+
...attempts.flatMap(attempt => (
|
|
1507
|
+
attempt.provider === undefined ? [] : [attempt.provider]
|
|
1508
|
+
)),
|
|
1509
|
+
];
|
|
1510
|
+
const uniqueProviders = [...new Set(providers)];
|
|
1511
|
+
const generationId = opaqueMetadataDigest(gateway?.generationId);
|
|
1512
|
+
return {
|
|
1513
|
+
attemptCount: allAttempts.length,
|
|
1514
|
+
attempts,
|
|
1515
|
+
attemptsTruncated: allAttempts.length !== attempts.length,
|
|
1516
|
+
clientMaxRetries: 0,
|
|
1517
|
+
gatewayProviderFailover: "may-attempt-multiple-providers",
|
|
1518
|
+
...(generationId === undefined ? {} : { generationId }),
|
|
1519
|
+
providerCount: uniqueProviders.length,
|
|
1520
|
+
providers: uniqueProviders.slice(0, 32),
|
|
1521
|
+
providersTruncated: uniqueProviders.length > 32,
|
|
1522
|
+
};
|
|
1523
|
+
}
|
|
1524
|
+
|
|
1525
|
+
function parseGeneratedFile(
|
|
1526
|
+
value: unknown,
|
|
1527
|
+
allowedPrefixes: readonly string[],
|
|
1528
|
+
): GatewayGeneratedFile {
|
|
1529
|
+
if (
|
|
1530
|
+
!isObject(value)
|
|
1531
|
+
|| typeof value.mediaType !== "string"
|
|
1532
|
+
|| !(value.uint8Array instanceof Uint8Array)
|
|
1533
|
+
|| value.uint8Array.byteLength < 1
|
|
1534
|
+
) {
|
|
1535
|
+
throw new GatewayMediaExecutionError("invalid-response");
|
|
1536
|
+
}
|
|
1537
|
+
const mediaType = parseMediaType(value.mediaType);
|
|
1538
|
+
if (!allowedPrefixes.some(prefix => mediaType.startsWith(prefix))) {
|
|
1539
|
+
throw new GatewayMediaExecutionError("invalid-response");
|
|
1540
|
+
}
|
|
1541
|
+
if (!gatewayMediaBytesMatchType(value.uint8Array, mediaType)) {
|
|
1542
|
+
throw new GatewayMediaExecutionError("invalid-response");
|
|
1543
|
+
}
|
|
1544
|
+
return {
|
|
1545
|
+
mediaType,
|
|
1546
|
+
uint8Array: value.uint8Array,
|
|
1547
|
+
};
|
|
1548
|
+
}
|
|
1549
|
+
|
|
1550
|
+
function parseGeneratedFiles(
|
|
1551
|
+
result: unknown,
|
|
1552
|
+
pluralKey: "images" | "videos",
|
|
1553
|
+
allowedPrefixes: readonly string[],
|
|
1554
|
+
): Readonly<{
|
|
1555
|
+
files: readonly GatewayGeneratedFile[];
|
|
1556
|
+
routing: GatewayMediaRoutingReceipt;
|
|
1557
|
+
warnings: readonly string[];
|
|
1558
|
+
}> {
|
|
1559
|
+
if (!isObject(result) || !Array.isArray(result[pluralKey])) {
|
|
1560
|
+
throw new GatewayMediaExecutionError("invalid-response");
|
|
1561
|
+
}
|
|
1562
|
+
const values = result[pluralKey];
|
|
1563
|
+
if (values.length < 1 || values.length > 32) {
|
|
1564
|
+
throw new GatewayMediaExecutionError("invalid-response");
|
|
1565
|
+
}
|
|
1566
|
+
return {
|
|
1567
|
+
files: values.map(value => parseGeneratedFile(value, allowedPrefixes)),
|
|
1568
|
+
routing: parseGatewayRoutingMetadata(result.providerMetadata),
|
|
1569
|
+
warnings: parseWarnings(result.warnings),
|
|
1570
|
+
};
|
|
1571
|
+
}
|
|
1572
|
+
|
|
1573
|
+
function validateTimeout(value: number | undefined, fallback: number): number {
|
|
1574
|
+
const timeout = value ?? fallback;
|
|
1575
|
+
if (
|
|
1576
|
+
!Number.isSafeInteger(timeout)
|
|
1577
|
+
|| timeout < 1_000
|
|
1578
|
+
|| timeout > 30 * 60_000
|
|
1579
|
+
) {
|
|
1580
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
1581
|
+
}
|
|
1582
|
+
return timeout;
|
|
1583
|
+
}
|
|
1584
|
+
|
|
1585
|
+
interface CommonRequest {
|
|
1586
|
+
readonly consent: GatewayUploadConsent;
|
|
1587
|
+
readonly model: string;
|
|
1588
|
+
readonly providerOptions?: GatewayProviderOptions;
|
|
1589
|
+
}
|
|
1590
|
+
|
|
1591
|
+
interface ParsedImageRequest extends CommonRequest {
|
|
1592
|
+
readonly aspectRatio?: string;
|
|
1593
|
+
readonly images: readonly GatewayMediaInput[];
|
|
1594
|
+
readonly mask?: GatewayMediaInput;
|
|
1595
|
+
readonly maxImagesPerCall?: number;
|
|
1596
|
+
readonly maxOutputTokens?: number;
|
|
1597
|
+
readonly n?: number;
|
|
1598
|
+
readonly prompt: string;
|
|
1599
|
+
readonly seed?: number;
|
|
1600
|
+
readonly size?: string;
|
|
1601
|
+
readonly stopSequences?: readonly string[];
|
|
1602
|
+
readonly temperature?: number;
|
|
1603
|
+
}
|
|
1604
|
+
|
|
1605
|
+
function parseOptionalStringArray(
|
|
1606
|
+
value: unknown,
|
|
1607
|
+
maximumItems: number,
|
|
1608
|
+
maximumItemLength: number,
|
|
1609
|
+
): readonly string[] | undefined {
|
|
1610
|
+
if (value === undefined) return undefined;
|
|
1611
|
+
if (!Array.isArray(value) || value.length > maximumItems) {
|
|
1612
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
1613
|
+
}
|
|
1614
|
+
return value.map(item => parseString(item, maximumItemLength));
|
|
1615
|
+
}
|
|
1616
|
+
|
|
1617
|
+
function parseImageRequest(value: unknown, now: Date): ParsedImageRequest {
|
|
1618
|
+
if (!isRecord(value)) throw new GatewayMediaExecutionError("invalid-request");
|
|
1619
|
+
assertAllowedKeys(value, new Set([
|
|
1620
|
+
"aspectRatio",
|
|
1621
|
+
"consent",
|
|
1622
|
+
"images",
|
|
1623
|
+
"mask",
|
|
1624
|
+
"maxImagesPerCall",
|
|
1625
|
+
"maxOutputTokens",
|
|
1626
|
+
"model",
|
|
1627
|
+
"n",
|
|
1628
|
+
"prompt",
|
|
1629
|
+
"providerOptions",
|
|
1630
|
+
"seed",
|
|
1631
|
+
"size",
|
|
1632
|
+
"stopSequences",
|
|
1633
|
+
"temperature",
|
|
1634
|
+
]));
|
|
1635
|
+
const size = parseOptionalString(value.size, 32);
|
|
1636
|
+
const aspectRatio = parseOptionalString(value.aspectRatio, 32);
|
|
1637
|
+
if (
|
|
1638
|
+
(size !== undefined && !/^[1-9]\d{0,5}x[1-9]\d{0,5}$/u.test(size))
|
|
1639
|
+
|| (aspectRatio !== undefined && !/^[1-9]\d{0,5}:[1-9]\d{0,5}$/u.test(aspectRatio))
|
|
1640
|
+
|| (size !== undefined && aspectRatio !== undefined)
|
|
1641
|
+
) {
|
|
1642
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
1643
|
+
}
|
|
1644
|
+
const images = parseInputArray(value.images, {
|
|
1645
|
+
allowedPrefixes: ["image/"],
|
|
1646
|
+
maximumBytes: MAXIMUM_IMAGE_INPUT_BYTES,
|
|
1647
|
+
maximumTotalBytes: MAXIMUM_IMAGE_INPUT_TOTAL_BYTES,
|
|
1648
|
+
});
|
|
1649
|
+
const mask = value.mask === undefined
|
|
1650
|
+
? undefined
|
|
1651
|
+
: parseInput(value.mask, MAXIMUM_IMAGE_INPUT_BYTES, ["image/"]);
|
|
1652
|
+
if (
|
|
1653
|
+
mask !== undefined
|
|
1654
|
+
&& images.reduce(
|
|
1655
|
+
(sum, input) => sum + gatewayInputByteLength(input),
|
|
1656
|
+
gatewayInputByteLength(mask),
|
|
1657
|
+
)
|
|
1658
|
+
> MAXIMUM_IMAGE_INPUT_TOTAL_BYTES
|
|
1659
|
+
) {
|
|
1660
|
+
throw new GatewayMediaExecutionError("input-too-large");
|
|
1661
|
+
}
|
|
1662
|
+
if (mask !== undefined && images.length === 0) {
|
|
1663
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
1664
|
+
}
|
|
1665
|
+
const maxImagesPerCall = parseOptionalSafeInteger(value.maxImagesPerCall, 1, 32);
|
|
1666
|
+
const maxOutputTokens = parseOptionalSafeInteger(value.maxOutputTokens, 1, 1_000_000);
|
|
1667
|
+
const n = parseOptionalSafeInteger(value.n, 1, 32);
|
|
1668
|
+
if (
|
|
1669
|
+
(n ?? 1) > 1
|
|
1670
|
+
&& (
|
|
1671
|
+
maxImagesPerCall === undefined
|
|
1672
|
+
|| maxImagesPerCall < (n ?? 1)
|
|
1673
|
+
)
|
|
1674
|
+
) {
|
|
1675
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
1676
|
+
}
|
|
1677
|
+
const providerOptions = parseProviderOptions(value.providerOptions);
|
|
1678
|
+
const seed = parseOptionalSafeInteger(value.seed, 0, 0xffff_ffff);
|
|
1679
|
+
const stopSequences = parseOptionalStringArray(value.stopSequences, 64, 1_024);
|
|
1680
|
+
const temperature = parseOptionalNumber(value.temperature, 0, 100);
|
|
1681
|
+
return {
|
|
1682
|
+
...(aspectRatio === undefined ? {} : { aspectRatio }),
|
|
1683
|
+
consent: parseConsent(value.consent, now),
|
|
1684
|
+
images,
|
|
1685
|
+
...(mask === undefined ? {} : { mask }),
|
|
1686
|
+
...(maxImagesPerCall === undefined ? {} : { maxImagesPerCall }),
|
|
1687
|
+
...(maxOutputTokens === undefined ? {} : { maxOutputTokens }),
|
|
1688
|
+
model: parseModelId(value.model),
|
|
1689
|
+
...(n === undefined ? {} : { n }),
|
|
1690
|
+
prompt: parseString(
|
|
1691
|
+
value.prompt,
|
|
1692
|
+
MAXIMUM_PROMPT_CHARACTERS,
|
|
1693
|
+
images.length > 0,
|
|
1694
|
+
),
|
|
1695
|
+
...(providerOptions === undefined ? {} : { providerOptions }),
|
|
1696
|
+
...(seed === undefined ? {} : { seed }),
|
|
1697
|
+
...(size === undefined ? {} : { size }),
|
|
1698
|
+
...(stopSequences === undefined ? {} : { stopSequences }),
|
|
1699
|
+
...(temperature === undefined ? {} : { temperature }),
|
|
1700
|
+
};
|
|
1701
|
+
}
|
|
1702
|
+
|
|
1703
|
+
interface ParsedVideoFrame {
|
|
1704
|
+
readonly frameType: string;
|
|
1705
|
+
readonly image: GatewayMediaInput;
|
|
1706
|
+
}
|
|
1707
|
+
|
|
1708
|
+
interface ParsedVideoRequest extends CommonRequest {
|
|
1709
|
+
readonly aspectRatio?: string;
|
|
1710
|
+
readonly duration?: number;
|
|
1711
|
+
readonly fps?: number;
|
|
1712
|
+
readonly frameImages: readonly ParsedVideoFrame[];
|
|
1713
|
+
readonly generateAudio?: boolean;
|
|
1714
|
+
readonly inputReferences: readonly GatewayMediaInput[];
|
|
1715
|
+
readonly maxVideosPerCall?: number;
|
|
1716
|
+
readonly n?: number;
|
|
1717
|
+
readonly prompt: string;
|
|
1718
|
+
readonly promptImage?: GatewayMediaInput;
|
|
1719
|
+
readonly resolution?: string;
|
|
1720
|
+
readonly seed?: number;
|
|
1721
|
+
}
|
|
1722
|
+
|
|
1723
|
+
function parseVideoFrames(value: unknown): readonly ParsedVideoFrame[] {
|
|
1724
|
+
if (value === undefined) return [];
|
|
1725
|
+
if (!Array.isArray(value) || value.length > MAXIMUM_INPUTS) {
|
|
1726
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
1727
|
+
}
|
|
1728
|
+
const frames = value.map((frame): ParsedVideoFrame => {
|
|
1729
|
+
if (!isRecord(frame)) throw new GatewayMediaExecutionError("invalid-request");
|
|
1730
|
+
assertAllowedKeys(frame, new Set(["frameType", "image"]));
|
|
1731
|
+
const rawFrameType = parseString(frame.frameType, 64);
|
|
1732
|
+
const frameType = rawFrameType === "first"
|
|
1733
|
+
? "first_frame"
|
|
1734
|
+
: rawFrameType === "last"
|
|
1735
|
+
? "last_frame"
|
|
1736
|
+
: rawFrameType;
|
|
1737
|
+
if (frameType !== "first_frame" && frameType !== "last_frame") {
|
|
1738
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
1739
|
+
}
|
|
1740
|
+
return {
|
|
1741
|
+
frameType,
|
|
1742
|
+
image: parseInput(frame.image, MAXIMUM_IMAGE_INPUT_BYTES, ["image/"]),
|
|
1743
|
+
};
|
|
1744
|
+
});
|
|
1745
|
+
if (
|
|
1746
|
+
frames.reduce(
|
|
1747
|
+
(sum, frame) => sum + gatewayInputByteLength(frame.image),
|
|
1748
|
+
0,
|
|
1749
|
+
)
|
|
1750
|
+
> MAXIMUM_IMAGE_INPUT_TOTAL_BYTES
|
|
1751
|
+
|| new Set(frames.map(frame => frame.frameType)).size !== frames.length
|
|
1752
|
+
) {
|
|
1753
|
+
throw new GatewayMediaExecutionError(
|
|
1754
|
+
frames.reduce(
|
|
1755
|
+
(sum, frame) => sum + gatewayInputByteLength(frame.image),
|
|
1756
|
+
0,
|
|
1757
|
+
)
|
|
1758
|
+
> MAXIMUM_IMAGE_INPUT_TOTAL_BYTES
|
|
1759
|
+
? "input-too-large"
|
|
1760
|
+
: "invalid-request",
|
|
1761
|
+
);
|
|
1762
|
+
}
|
|
1763
|
+
return frames;
|
|
1764
|
+
}
|
|
1765
|
+
|
|
1766
|
+
function parseVideoRequest(value: unknown, now: Date): ParsedVideoRequest {
|
|
1767
|
+
if (!isRecord(value)) throw new GatewayMediaExecutionError("invalid-request");
|
|
1768
|
+
assertAllowedKeys(value, new Set([
|
|
1769
|
+
"aspectRatio",
|
|
1770
|
+
"consent",
|
|
1771
|
+
"duration",
|
|
1772
|
+
"fps",
|
|
1773
|
+
"frameImages",
|
|
1774
|
+
"generateAudio",
|
|
1775
|
+
"inputReferences",
|
|
1776
|
+
"maxVideosPerCall",
|
|
1777
|
+
"model",
|
|
1778
|
+
"n",
|
|
1779
|
+
"prompt",
|
|
1780
|
+
"promptImage",
|
|
1781
|
+
"providerOptions",
|
|
1782
|
+
"resolution",
|
|
1783
|
+
"seed",
|
|
1784
|
+
]));
|
|
1785
|
+
const aspectRatio = parseOptionalString(value.aspectRatio, 32);
|
|
1786
|
+
const resolution = parseOptionalString(value.resolution, 32);
|
|
1787
|
+
if (
|
|
1788
|
+
(aspectRatio !== undefined && !/^[1-9]\d{0,5}:[1-9]\d{0,5}$/u.test(aspectRatio))
|
|
1789
|
+
|| (resolution !== undefined && !/^(?:[1-9]\d{0,5}x[1-9]\d{0,5}|[1-9]\d{2,4}p|[1-9]\d{0,2}k)$/iu.test(resolution))
|
|
1790
|
+
) {
|
|
1791
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
1792
|
+
}
|
|
1793
|
+
const promptImage = value.promptImage === undefined
|
|
1794
|
+
? undefined
|
|
1795
|
+
: parseInput(value.promptImage, MAXIMUM_IMAGE_INPUT_BYTES, ["image/"]);
|
|
1796
|
+
const frameImages = parseVideoFrames(value.frameImages);
|
|
1797
|
+
const inputReferences = parseInputArray(value.inputReferences, {
|
|
1798
|
+
allowedPrefixes: ["audio/", "image/", "video/"],
|
|
1799
|
+
maximumBytes: MAXIMUM_REFERENCE_INPUT_BYTES,
|
|
1800
|
+
maximumTotalBytes: MAXIMUM_REFERENCE_INPUT_TOTAL_BYTES,
|
|
1801
|
+
});
|
|
1802
|
+
const providerOptions = parseProviderOptions(value.providerOptions);
|
|
1803
|
+
if (
|
|
1804
|
+
(frameImages.length > 0 && inputReferences.length > 0)
|
|
1805
|
+
|| (
|
|
1806
|
+
promptImage !== undefined
|
|
1807
|
+
&& frameImages.some(frame => frame.frameType === "first_frame")
|
|
1808
|
+
)
|
|
1809
|
+
|| (
|
|
1810
|
+
frameImages.some(frame => frame.frameType === "last_frame")
|
|
1811
|
+
&& promptImage === undefined
|
|
1812
|
+
&& !frameImages.some(frame => frame.frameType === "first_frame")
|
|
1813
|
+
)
|
|
1814
|
+
) {
|
|
1815
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
1816
|
+
}
|
|
1817
|
+
const duration = parseOptionalNumber(value.duration, 0.01, 3_600);
|
|
1818
|
+
const fps = parseOptionalNumber(value.fps, 0.01, 1_000);
|
|
1819
|
+
const generateAudio = parseOptionalBoolean(value.generateAudio);
|
|
1820
|
+
const maxVideosPerCall = parseOptionalSafeInteger(value.maxVideosPerCall, 1, 32);
|
|
1821
|
+
const n = parseOptionalSafeInteger(value.n, 1, 32);
|
|
1822
|
+
if (
|
|
1823
|
+
(n ?? 1) > 1
|
|
1824
|
+
&& (
|
|
1825
|
+
maxVideosPerCall === undefined
|
|
1826
|
+
|| maxVideosPerCall < (n ?? 1)
|
|
1827
|
+
)
|
|
1828
|
+
) {
|
|
1829
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
1830
|
+
}
|
|
1831
|
+
const seed = parseOptionalSafeInteger(value.seed, 0, 0xffff_ffff);
|
|
1832
|
+
return {
|
|
1833
|
+
...(aspectRatio === undefined ? {} : { aspectRatio }),
|
|
1834
|
+
consent: parseConsent(value.consent, now),
|
|
1835
|
+
...(duration === undefined ? {} : { duration }),
|
|
1836
|
+
...(fps === undefined ? {} : { fps }),
|
|
1837
|
+
frameImages,
|
|
1838
|
+
...(generateAudio === undefined ? {} : { generateAudio }),
|
|
1839
|
+
inputReferences,
|
|
1840
|
+
...(maxVideosPerCall === undefined ? {} : { maxVideosPerCall }),
|
|
1841
|
+
model: parseModelId(value.model),
|
|
1842
|
+
...(n === undefined ? {} : { n }),
|
|
1843
|
+
prompt: parseString(
|
|
1844
|
+
value.prompt,
|
|
1845
|
+
MAXIMUM_PROMPT_CHARACTERS,
|
|
1846
|
+
promptImage !== undefined
|
|
1847
|
+
|| frameImages.length > 0
|
|
1848
|
+
|| inputReferences.length > 0,
|
|
1849
|
+
),
|
|
1850
|
+
...(promptImage === undefined ? {} : { promptImage }),
|
|
1851
|
+
...(providerOptions === undefined ? {} : { providerOptions }),
|
|
1852
|
+
...(resolution === undefined ? {} : { resolution }),
|
|
1853
|
+
...(seed === undefined ? {} : { seed }),
|
|
1854
|
+
};
|
|
1855
|
+
}
|
|
1856
|
+
|
|
1857
|
+
interface ParsedSpeechRequest extends CommonRequest {
|
|
1858
|
+
readonly instructions?: string;
|
|
1859
|
+
readonly language?: string;
|
|
1860
|
+
readonly outputFormat?: string;
|
|
1861
|
+
readonly speed?: number;
|
|
1862
|
+
readonly text: string;
|
|
1863
|
+
readonly voice?: string;
|
|
1864
|
+
}
|
|
1865
|
+
|
|
1866
|
+
function parseSpeechRequest(value: unknown, now: Date): ParsedSpeechRequest {
|
|
1867
|
+
if (!isRecord(value)) throw new GatewayMediaExecutionError("invalid-request");
|
|
1868
|
+
assertAllowedKeys(value, new Set([
|
|
1869
|
+
"consent",
|
|
1870
|
+
"instructions",
|
|
1871
|
+
"language",
|
|
1872
|
+
"model",
|
|
1873
|
+
"outputFormat",
|
|
1874
|
+
"providerOptions",
|
|
1875
|
+
"speed",
|
|
1876
|
+
"text",
|
|
1877
|
+
"voice",
|
|
1878
|
+
]));
|
|
1879
|
+
const outputFormat = parseOptionalString(value.outputFormat, 64);
|
|
1880
|
+
if (outputFormat !== undefined && !/^[a-z0-9][a-z0-9._+-]*$/iu.test(outputFormat)) {
|
|
1881
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
1882
|
+
}
|
|
1883
|
+
const instructions = parseOptionalString(value.instructions, MAXIMUM_PROMPT_CHARACTERS);
|
|
1884
|
+
const language = parseOptionalString(value.language, 64);
|
|
1885
|
+
const providerOptions = parseProviderOptions(value.providerOptions);
|
|
1886
|
+
const speed = parseOptionalNumber(value.speed, 0.01, 100);
|
|
1887
|
+
const voice = parseOptionalString(value.voice, 256);
|
|
1888
|
+
return {
|
|
1889
|
+
consent: parseConsent(value.consent, now),
|
|
1890
|
+
...(instructions === undefined ? {} : { instructions }),
|
|
1891
|
+
...(language === undefined ? {} : { language }),
|
|
1892
|
+
model: parseModelId(value.model),
|
|
1893
|
+
...(outputFormat === undefined ? {} : { outputFormat }),
|
|
1894
|
+
...(providerOptions === undefined ? {} : { providerOptions }),
|
|
1895
|
+
...(speed === undefined ? {} : { speed }),
|
|
1896
|
+
text: parseString(value.text, MAXIMUM_PROMPT_CHARACTERS),
|
|
1897
|
+
...(voice === undefined ? {} : { voice }),
|
|
1898
|
+
};
|
|
1899
|
+
}
|
|
1900
|
+
|
|
1901
|
+
interface ParsedTranscriptionRequest extends CommonRequest {
|
|
1902
|
+
readonly audio: GatewayInlineMediaInput;
|
|
1903
|
+
}
|
|
1904
|
+
|
|
1905
|
+
function parseTranscriptionRequest(
|
|
1906
|
+
value: unknown,
|
|
1907
|
+
now: Date,
|
|
1908
|
+
): ParsedTranscriptionRequest {
|
|
1909
|
+
if (!isRecord(value)) throw new GatewayMediaExecutionError("invalid-request");
|
|
1910
|
+
assertAllowedKeys(value, new Set([
|
|
1911
|
+
"audio",
|
|
1912
|
+
"consent",
|
|
1913
|
+
"model",
|
|
1914
|
+
"providerOptions",
|
|
1915
|
+
]));
|
|
1916
|
+
const providerOptions = parseProviderOptions(value.providerOptions);
|
|
1917
|
+
const audio = parseInput(
|
|
1918
|
+
value.audio,
|
|
1919
|
+
MAXIMUM_TRANSCRIPTION_INPUT_BYTES,
|
|
1920
|
+
["audio/"],
|
|
1921
|
+
);
|
|
1922
|
+
if (audio.url !== undefined) {
|
|
1923
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
1924
|
+
}
|
|
1925
|
+
return {
|
|
1926
|
+
audio,
|
|
1927
|
+
consent: parseConsent(value.consent, now),
|
|
1928
|
+
model: parseModelId(value.model),
|
|
1929
|
+
...(providerOptions === undefined ? {} : { providerOptions }),
|
|
1930
|
+
};
|
|
1931
|
+
}
|
|
1932
|
+
|
|
1933
|
+
interface TranscriptionSegment {
|
|
1934
|
+
readonly endSecond: number;
|
|
1935
|
+
readonly startSecond: number;
|
|
1936
|
+
readonly text: string;
|
|
1937
|
+
}
|
|
1938
|
+
|
|
1939
|
+
interface ParsedTranscriptionResult {
|
|
1940
|
+
readonly durationInSeconds?: number;
|
|
1941
|
+
readonly language?: string;
|
|
1942
|
+
readonly routing: GatewayMediaRoutingReceipt;
|
|
1943
|
+
readonly segments: readonly TranscriptionSegment[];
|
|
1944
|
+
readonly text: string;
|
|
1945
|
+
readonly warnings: readonly string[];
|
|
1946
|
+
}
|
|
1947
|
+
|
|
1948
|
+
function parseTranscriptionResult(
|
|
1949
|
+
value: unknown,
|
|
1950
|
+
): ParsedTranscriptionResult {
|
|
1951
|
+
if (
|
|
1952
|
+
!isObject(value)
|
|
1953
|
+
|| typeof value.text !== "string"
|
|
1954
|
+
|| value.text.length > MAXIMUM_TRANSCRIPT_CHARACTERS
|
|
1955
|
+
|| !Array.isArray(value.segments)
|
|
1956
|
+
|| value.segments.length > MAXIMUM_TRANSCRIPT_SEGMENTS
|
|
1957
|
+
) {
|
|
1958
|
+
throw new GatewayMediaExecutionError("invalid-response");
|
|
1959
|
+
}
|
|
1960
|
+
if (Buffer.byteLength(value.text, "utf8") > MAXIMUM_TRANSCRIPT_TEXT_BYTES) {
|
|
1961
|
+
throw new GatewayMediaExecutionError("invalid-response");
|
|
1962
|
+
}
|
|
1963
|
+
let segmentTextBytes = 0;
|
|
1964
|
+
const segments = value.segments.map((segment): TranscriptionSegment => {
|
|
1965
|
+
if (
|
|
1966
|
+
!isObject(segment)
|
|
1967
|
+
|| typeof segment.text !== "string"
|
|
1968
|
+
|| segment.text.length > 100_000
|
|
1969
|
+
|| typeof segment.startSecond !== "number"
|
|
1970
|
+
|| typeof segment.endSecond !== "number"
|
|
1971
|
+
|| !Number.isFinite(segment.startSecond)
|
|
1972
|
+
|| !Number.isFinite(segment.endSecond)
|
|
1973
|
+
|| segment.startSecond < 0
|
|
1974
|
+
|| segment.endSecond < segment.startSecond
|
|
1975
|
+
|| segment.endSecond > 1_000_000_000
|
|
1976
|
+
) {
|
|
1977
|
+
throw new GatewayMediaExecutionError("invalid-response");
|
|
1978
|
+
}
|
|
1979
|
+
segmentTextBytes += Buffer.byteLength(segment.text, "utf8");
|
|
1980
|
+
if (segmentTextBytes > MAXIMUM_TRANSCRIPT_SEGMENT_TEXT_BYTES) {
|
|
1981
|
+
throw new GatewayMediaExecutionError("invalid-response");
|
|
1982
|
+
}
|
|
1983
|
+
return {
|
|
1984
|
+
endSecond: segment.endSecond,
|
|
1985
|
+
startSecond: segment.startSecond,
|
|
1986
|
+
text: segment.text,
|
|
1987
|
+
};
|
|
1988
|
+
});
|
|
1989
|
+
const language = value.language === undefined
|
|
1990
|
+
? undefined
|
|
1991
|
+
: parseString(value.language, 64);
|
|
1992
|
+
const durationInSeconds = value.durationInSeconds === undefined
|
|
1993
|
+
? undefined
|
|
1994
|
+
: parseOptionalNumber(value.durationInSeconds, 0, 1_000_000_000);
|
|
1995
|
+
if (
|
|
1996
|
+
segments.some((segment, index) => (
|
|
1997
|
+
(
|
|
1998
|
+
index > 0
|
|
1999
|
+
&& segment.startSecond < segments[index - 1]!.startSecond
|
|
2000
|
+
)
|
|
2001
|
+
|| (
|
|
2002
|
+
durationInSeconds !== undefined
|
|
2003
|
+
&& segment.endSecond > durationInSeconds + 0.001
|
|
2004
|
+
)
|
|
2005
|
+
))
|
|
2006
|
+
) {
|
|
2007
|
+
throw new GatewayMediaExecutionError("invalid-response");
|
|
2008
|
+
}
|
|
2009
|
+
return {
|
|
2010
|
+
...(durationInSeconds === undefined ? {} : { durationInSeconds }),
|
|
2011
|
+
...(language === undefined ? {} : { language }),
|
|
2012
|
+
routing: parseGatewayRoutingMetadata(value.providerMetadata),
|
|
2013
|
+
segments,
|
|
2014
|
+
text: value.text,
|
|
2015
|
+
warnings: parseWarnings(value.warnings),
|
|
2016
|
+
};
|
|
2017
|
+
}
|
|
2018
|
+
|
|
2019
|
+
function subtitleTimestamp(
|
|
2020
|
+
seconds: number,
|
|
2021
|
+
separator: "," | ".",
|
|
2022
|
+
): string {
|
|
2023
|
+
const totalMilliseconds = Math.round(seconds * 1_000);
|
|
2024
|
+
const hours = Math.floor(totalMilliseconds / 3_600_000);
|
|
2025
|
+
const minutes = Math.floor(totalMilliseconds / 60_000) % 60;
|
|
2026
|
+
const wholeSeconds = Math.floor(totalMilliseconds / 1_000) % 60;
|
|
2027
|
+
const milliseconds = totalMilliseconds % 1_000;
|
|
2028
|
+
return `${String(hours).padStart(2, "0")}:${String(minutes).padStart(2, "0")}:${String(wholeSeconds).padStart(2, "0")}${separator}${String(milliseconds).padStart(3, "0")}`;
|
|
2029
|
+
}
|
|
2030
|
+
|
|
2031
|
+
function safeSubtitleText(value: string): string {
|
|
2032
|
+
let printableValue = "";
|
|
2033
|
+
for (const character of value) {
|
|
2034
|
+
const codePoint = character.codePointAt(0) ?? 0;
|
|
2035
|
+
printableValue += (
|
|
2036
|
+
codePoint <= 8
|
|
2037
|
+
|| codePoint === 11
|
|
2038
|
+
|| codePoint === 12
|
|
2039
|
+
|| (codePoint >= 14 && codePoint <= 31)
|
|
2040
|
+
|| codePoint === 127
|
|
2041
|
+
)
|
|
2042
|
+
? " "
|
|
2043
|
+
: character;
|
|
2044
|
+
}
|
|
2045
|
+
return printableValue
|
|
2046
|
+
.replaceAll("&", "&")
|
|
2047
|
+
.replaceAll("<", "<")
|
|
2048
|
+
.replaceAll(">", ">")
|
|
2049
|
+
.replaceAll(/\r\n?|\n/gu, " ");
|
|
2050
|
+
}
|
|
2051
|
+
|
|
2052
|
+
function subtitleBody(
|
|
2053
|
+
result: ParsedTranscriptionResult,
|
|
2054
|
+
format: "srt" | "vtt",
|
|
2055
|
+
): string {
|
|
2056
|
+
const separator = format === "srt" ? "," : ".";
|
|
2057
|
+
const cues = result.segments.map((segment, index) => [
|
|
2058
|
+
...(format === "srt" ? [String(index + 1)] : []),
|
|
2059
|
+
`${subtitleTimestamp(segment.startSecond, separator)} --> ${subtitleTimestamp(segment.endSecond, separator)}`,
|
|
2060
|
+
safeSubtitleText(segment.text),
|
|
2061
|
+
].join("\n")).join("\n\n");
|
|
2062
|
+
const body = cues.length === 0 ? "" : `${cues}\n`;
|
|
2063
|
+
return format === "vtt" ? `WEBVTT\n\n${body}` : body.length === 0 ? "\n" : body;
|
|
2064
|
+
}
|
|
2065
|
+
|
|
2066
|
+
function transcriptFiles(
|
|
2067
|
+
result: ParsedTranscriptionResult,
|
|
2068
|
+
): readonly GatewayGeneratedFile[] {
|
|
2069
|
+
const body = {
|
|
2070
|
+
...(result.durationInSeconds === undefined
|
|
2071
|
+
? {}
|
|
2072
|
+
: { durationInSeconds: result.durationInSeconds }),
|
|
2073
|
+
...(result.language === undefined ? {} : { language: result.language }),
|
|
2074
|
+
segments: result.segments,
|
|
2075
|
+
text: result.text,
|
|
2076
|
+
};
|
|
2077
|
+
const encoder = new TextEncoder();
|
|
2078
|
+
let totalBytes = 0;
|
|
2079
|
+
const files: GatewayGeneratedFile[] = [];
|
|
2080
|
+
const append = (mediaType: string, contents: string): void => {
|
|
2081
|
+
const uint8Array = encoder.encode(contents);
|
|
2082
|
+
totalBytes += uint8Array.byteLength;
|
|
2083
|
+
if (totalBytes > MAXIMUM_TRANSCRIPT_ARTIFACT_BYTES) {
|
|
2084
|
+
throw new GatewayMediaExecutionError("invalid-response");
|
|
2085
|
+
}
|
|
2086
|
+
files.push({ mediaType, uint8Array });
|
|
2087
|
+
};
|
|
2088
|
+
append("application/json", `${JSON.stringify(body)}\n`);
|
|
2089
|
+
append(
|
|
2090
|
+
"text/plain",
|
|
2091
|
+
result.text.endsWith("\n") ? result.text : `${result.text}\n`,
|
|
2092
|
+
);
|
|
2093
|
+
append("application/x-subrip", subtitleBody(result, "srt"));
|
|
2094
|
+
append("text/vtt", subtitleBody(result, "vtt"));
|
|
2095
|
+
return files;
|
|
2096
|
+
}
|
|
2097
|
+
|
|
2098
|
+
interface CatalogCache {
|
|
2099
|
+
get(input: Readonly<{
|
|
2100
|
+
freshness: "allow-stale" | "require-fresh";
|
|
2101
|
+
signal?: AbortSignal;
|
|
2102
|
+
}>): Promise<GatewayMediaCatalogView>;
|
|
2103
|
+
}
|
|
2104
|
+
|
|
2105
|
+
async function modelCatalog(
|
|
2106
|
+
cache: CatalogCache,
|
|
2107
|
+
modelId: string,
|
|
2108
|
+
expectedKind: GatewayMediaKind,
|
|
2109
|
+
signal: AbortSignal | undefined,
|
|
2110
|
+
): Promise<Readonly<{
|
|
2111
|
+
catalog: GatewayMediaCatalogView;
|
|
2112
|
+
model: GatewayMediaModel;
|
|
2113
|
+
}>> {
|
|
2114
|
+
const catalog = await cache.get({
|
|
2115
|
+
freshness: "require-fresh",
|
|
2116
|
+
...(signal === undefined ? {} : { signal }),
|
|
2117
|
+
});
|
|
2118
|
+
const model = catalog.snapshot.models.find(item => item.id === modelId);
|
|
2119
|
+
if (model === undefined) throw new GatewayMediaExecutionError("model-not-found");
|
|
2120
|
+
if (model.kind !== expectedKind) {
|
|
2121
|
+
throw new GatewayMediaExecutionError("model-kind-mismatch");
|
|
2122
|
+
}
|
|
2123
|
+
return { catalog, model };
|
|
2124
|
+
}
|
|
2125
|
+
|
|
2126
|
+
function unsupportedModelSetting(): never {
|
|
2127
|
+
throw new GatewayMediaExecutionError("model-operation-unsupported");
|
|
2128
|
+
}
|
|
2129
|
+
|
|
2130
|
+
function capabilityStringList(
|
|
2131
|
+
capabilities: GatewayMediaModel["capabilities"],
|
|
2132
|
+
name: string,
|
|
2133
|
+
): readonly string[] | undefined {
|
|
2134
|
+
const value = capabilities?.[name];
|
|
2135
|
+
if (value === undefined || value === null) return undefined;
|
|
2136
|
+
if (!Array.isArray(value) || value.some(item => typeof item !== "string")) {
|
|
2137
|
+
return unsupportedModelSetting();
|
|
2138
|
+
}
|
|
2139
|
+
return value.filter((item): item is string => typeof item === "string");
|
|
2140
|
+
}
|
|
2141
|
+
|
|
2142
|
+
function capabilityNumberList(
|
|
2143
|
+
capabilities: GatewayMediaModel["capabilities"],
|
|
2144
|
+
name: string,
|
|
2145
|
+
): readonly number[] | undefined {
|
|
2146
|
+
const value = capabilities?.[name];
|
|
2147
|
+
if (value === undefined || value === null) return undefined;
|
|
2148
|
+
if (
|
|
2149
|
+
!Array.isArray(value)
|
|
2150
|
+
|| value.some(item => typeof item !== "number" || !Number.isFinite(item))
|
|
2151
|
+
) {
|
|
2152
|
+
return unsupportedModelSetting();
|
|
2153
|
+
}
|
|
2154
|
+
return value.filter(
|
|
2155
|
+
(item): item is number => typeof item === "number" && Number.isFinite(item),
|
|
2156
|
+
);
|
|
2157
|
+
}
|
|
2158
|
+
|
|
2159
|
+
function capabilityNumber(
|
|
2160
|
+
value: Readonly<Record<string, unknown>>,
|
|
2161
|
+
name: string,
|
|
2162
|
+
): number | undefined {
|
|
2163
|
+
const candidate = value[name];
|
|
2164
|
+
if (candidate === undefined || candidate === null) return undefined;
|
|
2165
|
+
if (
|
|
2166
|
+
typeof candidate !== "number"
|
|
2167
|
+
|| !Number.isFinite(candidate)
|
|
2168
|
+
|| candidate < 0
|
|
2169
|
+
) {
|
|
2170
|
+
return unsupportedModelSetting();
|
|
2171
|
+
}
|
|
2172
|
+
return candidate;
|
|
2173
|
+
}
|
|
2174
|
+
|
|
2175
|
+
function capabilityInteger(
|
|
2176
|
+
value: Readonly<Record<string, unknown>>,
|
|
2177
|
+
name: string,
|
|
2178
|
+
): number | undefined {
|
|
2179
|
+
const candidate = capabilityNumber(value, name);
|
|
2180
|
+
if (candidate !== undefined && !Number.isSafeInteger(candidate)) {
|
|
2181
|
+
return unsupportedModelSetting();
|
|
2182
|
+
}
|
|
2183
|
+
return candidate;
|
|
2184
|
+
}
|
|
2185
|
+
|
|
2186
|
+
function assertSupportedOperation(
|
|
2187
|
+
model: GatewayMediaModel,
|
|
2188
|
+
candidates: readonly string[],
|
|
2189
|
+
): void {
|
|
2190
|
+
const supported = capabilityStringList(
|
|
2191
|
+
model.capabilities,
|
|
2192
|
+
"supported_operations",
|
|
2193
|
+
);
|
|
2194
|
+
if (
|
|
2195
|
+
supported !== undefined
|
|
2196
|
+
&& !candidates.some(candidate => supported.includes(candidate))
|
|
2197
|
+
) {
|
|
2198
|
+
unsupportedModelSetting();
|
|
2199
|
+
}
|
|
2200
|
+
}
|
|
2201
|
+
|
|
2202
|
+
function assertListedString(
|
|
2203
|
+
capabilities: GatewayMediaModel["capabilities"],
|
|
2204
|
+
name: string,
|
|
2205
|
+
value: string | undefined,
|
|
2206
|
+
): void {
|
|
2207
|
+
const supported = capabilityStringList(capabilities, name);
|
|
2208
|
+
if (
|
|
2209
|
+
value !== undefined
|
|
2210
|
+
&& supported !== undefined
|
|
2211
|
+
&& !supported.some(candidate => (
|
|
2212
|
+
candidate.toLocaleLowerCase("en-US")
|
|
2213
|
+
=== value.toLocaleLowerCase("en-US")
|
|
2214
|
+
))
|
|
2215
|
+
) {
|
|
2216
|
+
unsupportedModelSetting();
|
|
2217
|
+
}
|
|
2218
|
+
}
|
|
2219
|
+
|
|
2220
|
+
function assertListedNumber(
|
|
2221
|
+
capabilities: GatewayMediaModel["capabilities"],
|
|
2222
|
+
name: string,
|
|
2223
|
+
value: number | undefined,
|
|
2224
|
+
): void {
|
|
2225
|
+
const supported = capabilityNumberList(capabilities, name);
|
|
2226
|
+
if (
|
|
2227
|
+
value !== undefined
|
|
2228
|
+
&& supported !== undefined
|
|
2229
|
+
&& !supported.includes(value)
|
|
2230
|
+
) {
|
|
2231
|
+
unsupportedModelSetting();
|
|
2232
|
+
}
|
|
2233
|
+
}
|
|
2234
|
+
|
|
2235
|
+
function formatAliases(mediaType: string): readonly string[] {
|
|
2236
|
+
const aliases: Readonly<Record<string, readonly string[]>> = {
|
|
2237
|
+
"audio/aac": ["aac"],
|
|
2238
|
+
"audio/flac": ["flac"],
|
|
2239
|
+
"audio/mp4": ["m4a", "mp4"],
|
|
2240
|
+
"audio/mpeg": ["mp3", "mpeg"],
|
|
2241
|
+
"audio/ogg": ["ogg"],
|
|
2242
|
+
"audio/opus": ["ogg", "opus"],
|
|
2243
|
+
"audio/wav": ["wav"],
|
|
2244
|
+
"audio/webm": ["webm"],
|
|
2245
|
+
"image/avif": ["avif"],
|
|
2246
|
+
"image/bmp": ["bmp"],
|
|
2247
|
+
"image/gif": ["gif"],
|
|
2248
|
+
"image/jpeg": ["jpeg", "jpg"],
|
|
2249
|
+
"image/png": ["png"],
|
|
2250
|
+
"image/tiff": ["tif", "tiff"],
|
|
2251
|
+
"image/webp": ["webp"],
|
|
2252
|
+
"video/mp4": ["mp4"],
|
|
2253
|
+
"video/quicktime": ["mov", "quicktime"],
|
|
2254
|
+
"video/webm": ["webm"],
|
|
2255
|
+
"video/x-matroska": ["mkv", "matroska"],
|
|
2256
|
+
};
|
|
2257
|
+
return aliases[mediaType] ?? [mediaType.split("/").at(-1) ?? mediaType];
|
|
2258
|
+
}
|
|
2259
|
+
|
|
2260
|
+
function capabilityAspectRatio(value: unknown): number | undefined {
|
|
2261
|
+
if (value === undefined || value === null) return undefined;
|
|
2262
|
+
if (typeof value !== "string") unsupportedModelSetting();
|
|
2263
|
+
const match = /^([1-9]\d*):([1-9]\d*)$/u.exec(value);
|
|
2264
|
+
if (match === null) unsupportedModelSetting();
|
|
2265
|
+
const ratio = Number(match[1]) / Number(match[2]);
|
|
2266
|
+
return Number.isFinite(ratio) && ratio > 0
|
|
2267
|
+
? ratio
|
|
2268
|
+
: unsupportedModelSetting();
|
|
2269
|
+
}
|
|
2270
|
+
|
|
2271
|
+
function assertInputLimits(
|
|
2272
|
+
model: GatewayMediaModel,
|
|
2273
|
+
prompt: string,
|
|
2274
|
+
inputs: readonly GatewayMediaInput[],
|
|
2275
|
+
): void {
|
|
2276
|
+
const rawLimits = model.capabilities?.input_limits;
|
|
2277
|
+
if (rawLimits === undefined || rawLimits === null) return;
|
|
2278
|
+
if (!isRecord(rawLimits)) unsupportedModelSetting();
|
|
2279
|
+
const limits = rawLimits as Readonly<Record<string, unknown>>;
|
|
2280
|
+
const maximumTotal = capabilityInteger(limits, "max_total_inputs");
|
|
2281
|
+
if (maximumTotal !== undefined && inputs.length > maximumTotal) {
|
|
2282
|
+
unsupportedModelSetting();
|
|
2283
|
+
}
|
|
2284
|
+
const textLimits = limits.text;
|
|
2285
|
+
if (textLimits !== undefined && textLimits !== null) {
|
|
2286
|
+
if (!isRecord(textLimits)) unsupportedModelSetting();
|
|
2287
|
+
const maximumCharacters = capabilityInteger(textLimits, "max_chars");
|
|
2288
|
+
if (
|
|
2289
|
+
maximumCharacters !== undefined
|
|
2290
|
+
&& prompt.length > maximumCharacters
|
|
2291
|
+
) {
|
|
2292
|
+
unsupportedModelSetting();
|
|
2293
|
+
}
|
|
2294
|
+
}
|
|
2295
|
+
for (const kind of ["audio", "image", "video"] as const) {
|
|
2296
|
+
const matching = inputs.filter(input => input.mediaType.startsWith(`${kind}/`));
|
|
2297
|
+
if (matching.length === 0) continue;
|
|
2298
|
+
const inlineInputs = matching.filter(input => input.url === undefined);
|
|
2299
|
+
const rawKindLimits = limits[kind];
|
|
2300
|
+
if (rawKindLimits === undefined || rawKindLimits === null) {
|
|
2301
|
+
unsupportedModelSetting();
|
|
2302
|
+
}
|
|
2303
|
+
if (!isRecord(rawKindLimits)) unsupportedModelSetting();
|
|
2304
|
+
const kindLimits = rawKindLimits as Readonly<Record<string, unknown>>;
|
|
2305
|
+
if (
|
|
2306
|
+
inlineInputs.some(input => (
|
|
2307
|
+
(kind === "image" || kind === "video")
|
|
2308
|
+
? input.facts?.width === undefined || input.facts.height === undefined
|
|
2309
|
+
: input.facts?.durationSeconds === undefined
|
|
2310
|
+
))
|
|
2311
|
+
|| (
|
|
2312
|
+
kind === "video"
|
|
2313
|
+
&& inlineInputs.some(input => input.facts?.durationSeconds === undefined)
|
|
2314
|
+
)
|
|
2315
|
+
) {
|
|
2316
|
+
unsupportedModelSetting();
|
|
2317
|
+
}
|
|
2318
|
+
const maximumCount = capabilityInteger(kindLimits, "max_count");
|
|
2319
|
+
if (maximumCount !== undefined && matching.length > maximumCount) {
|
|
2320
|
+
unsupportedModelSetting();
|
|
2321
|
+
}
|
|
2322
|
+
const maximumFileSizeMb = capabilityNumber(
|
|
2323
|
+
kindLimits,
|
|
2324
|
+
"max_file_size_mb",
|
|
2325
|
+
);
|
|
2326
|
+
if (
|
|
2327
|
+
maximumFileSizeMb !== undefined
|
|
2328
|
+
&& inlineInputs.some(
|
|
2329
|
+
input => gatewayInputByteLength(input) > maximumFileSizeMb * 1024 * 1024,
|
|
2330
|
+
)
|
|
2331
|
+
) {
|
|
2332
|
+
unsupportedModelSetting();
|
|
2333
|
+
}
|
|
2334
|
+
const minimumDimension = capabilityInteger(
|
|
2335
|
+
kindLimits,
|
|
2336
|
+
"min_dimension_pixels",
|
|
2337
|
+
);
|
|
2338
|
+
const maximumDimension = capabilityInteger(
|
|
2339
|
+
kindLimits,
|
|
2340
|
+
"max_dimension_pixels",
|
|
2341
|
+
);
|
|
2342
|
+
const minimumAspectRatio = capabilityAspectRatio(
|
|
2343
|
+
kindLimits.min_aspect_ratio,
|
|
2344
|
+
);
|
|
2345
|
+
const maximumAspectRatio = capabilityAspectRatio(
|
|
2346
|
+
kindLimits.max_aspect_ratio,
|
|
2347
|
+
);
|
|
2348
|
+
if (
|
|
2349
|
+
(kind === "image" || kind === "video")
|
|
2350
|
+
&& inlineInputs.some((input) => {
|
|
2351
|
+
const width = input.facts!.width!;
|
|
2352
|
+
const height = input.facts!.height!;
|
|
2353
|
+
const aspectRatio = width / height;
|
|
2354
|
+
return (
|
|
2355
|
+
(minimumDimension !== undefined
|
|
2356
|
+
&& (width < minimumDimension || height < minimumDimension))
|
|
2357
|
+
|| (maximumDimension !== undefined
|
|
2358
|
+
&& (width > maximumDimension || height > maximumDimension))
|
|
2359
|
+
|| (minimumAspectRatio !== undefined
|
|
2360
|
+
&& aspectRatio < minimumAspectRatio)
|
|
2361
|
+
|| (maximumAspectRatio !== undefined
|
|
2362
|
+
&& aspectRatio > maximumAspectRatio)
|
|
2363
|
+
);
|
|
2364
|
+
})
|
|
2365
|
+
) {
|
|
2366
|
+
unsupportedModelSetting();
|
|
2367
|
+
}
|
|
2368
|
+
const minimumDuration = capabilityNumber(
|
|
2369
|
+
kindLimits,
|
|
2370
|
+
"min_duration_seconds",
|
|
2371
|
+
);
|
|
2372
|
+
const maximumDuration = capabilityNumber(
|
|
2373
|
+
kindLimits,
|
|
2374
|
+
"max_duration_seconds",
|
|
2375
|
+
);
|
|
2376
|
+
if (
|
|
2377
|
+
(kind === "audio" || kind === "video")
|
|
2378
|
+
&& inlineInputs.some((input) => {
|
|
2379
|
+
const duration = input.facts!.durationSeconds!;
|
|
2380
|
+
return (
|
|
2381
|
+
(minimumDuration !== undefined && duration < minimumDuration)
|
|
2382
|
+
|| (maximumDuration !== undefined && duration > maximumDuration)
|
|
2383
|
+
);
|
|
2384
|
+
})
|
|
2385
|
+
) {
|
|
2386
|
+
unsupportedModelSetting();
|
|
2387
|
+
}
|
|
2388
|
+
const supportedFormats = kindLimits.supported_formats;
|
|
2389
|
+
if (supportedFormats !== undefined && supportedFormats !== null) {
|
|
2390
|
+
if (
|
|
2391
|
+
!Array.isArray(supportedFormats)
|
|
2392
|
+
|| supportedFormats.some(item => typeof item !== "string")
|
|
2393
|
+
) {
|
|
2394
|
+
unsupportedModelSetting();
|
|
2395
|
+
}
|
|
2396
|
+
const normalized = supportedFormats
|
|
2397
|
+
.filter((item): item is string => typeof item === "string")
|
|
2398
|
+
.map(item => item.toLocaleLowerCase("en-US"));
|
|
2399
|
+
if (
|
|
2400
|
+
matching.some(input => !formatAliases(input.mediaType).some(
|
|
2401
|
+
format => normalized.includes(format),
|
|
2402
|
+
))
|
|
2403
|
+
) {
|
|
2404
|
+
unsupportedModelSetting();
|
|
2405
|
+
}
|
|
2406
|
+
}
|
|
2407
|
+
const supportedSources = kindLimits.supported_sources;
|
|
2408
|
+
if (supportedSources !== undefined && supportedSources !== null) {
|
|
2409
|
+
if (
|
|
2410
|
+
!Array.isArray(supportedSources)
|
|
2411
|
+
|| supportedSources.some(item => typeof item !== "string")
|
|
2412
|
+
) {
|
|
2413
|
+
unsupportedModelSetting();
|
|
2414
|
+
}
|
|
2415
|
+
const normalizedSources = supportedSources
|
|
2416
|
+
.filter((item): item is string => typeof item === "string")
|
|
2417
|
+
.map(item => item.toLocaleLowerCase("en-US"));
|
|
2418
|
+
if (
|
|
2419
|
+
matching.some(input => (
|
|
2420
|
+
gatewayInputSource(input) === "url"
|
|
2421
|
+
? !normalizedSources.includes("url")
|
|
2422
|
+
: !normalizedSources.includes("base64")
|
|
2423
|
+
&& !normalizedSources.includes("buffer")
|
|
2424
|
+
))
|
|
2425
|
+
) {
|
|
2426
|
+
unsupportedModelSetting();
|
|
2427
|
+
}
|
|
2428
|
+
}
|
|
2429
|
+
}
|
|
2430
|
+
}
|
|
2431
|
+
|
|
2432
|
+
function assertMaximumSamples(
|
|
2433
|
+
model: GatewayMediaModel,
|
|
2434
|
+
count: number | undefined,
|
|
2435
|
+
): void {
|
|
2436
|
+
if (count === undefined || model.capabilities === null) return;
|
|
2437
|
+
const maximum = capabilityInteger(
|
|
2438
|
+
model.capabilities,
|
|
2439
|
+
"max_sample_count",
|
|
2440
|
+
);
|
|
2441
|
+
if (maximum !== undefined && count > maximum) unsupportedModelSetting();
|
|
2442
|
+
}
|
|
2443
|
+
|
|
2444
|
+
function assertImageCapabilities(
|
|
2445
|
+
model: GatewayMediaModel,
|
|
2446
|
+
request: ParsedImageRequest,
|
|
2447
|
+
): void {
|
|
2448
|
+
const operationCandidates = request.mask !== undefined
|
|
2449
|
+
? ["inpainting", "image-editing", "image-to-image"]
|
|
2450
|
+
: request.images.length > 0
|
|
2451
|
+
? ["image-to-image", "image-editing"]
|
|
2452
|
+
: ["text-to-image"];
|
|
2453
|
+
assertSupportedOperation(model, operationCandidates);
|
|
2454
|
+
assertListedString(
|
|
2455
|
+
model.capabilities,
|
|
2456
|
+
"supported_aspect_ratios",
|
|
2457
|
+
request.aspectRatio,
|
|
2458
|
+
);
|
|
2459
|
+
const supportedSizes = capabilityStringList(
|
|
2460
|
+
model.capabilities,
|
|
2461
|
+
"supported_sizes",
|
|
2462
|
+
) ?? capabilityStringList(model.capabilities, "supported_resolutions");
|
|
2463
|
+
if (
|
|
2464
|
+
request.size !== undefined
|
|
2465
|
+
&& supportedSizes !== undefined
|
|
2466
|
+
&& !supportedSizes.some(candidate => (
|
|
2467
|
+
candidate.toLocaleLowerCase("en-US")
|
|
2468
|
+
=== request.size!.toLocaleLowerCase("en-US")
|
|
2469
|
+
))
|
|
2470
|
+
) {
|
|
2471
|
+
unsupportedModelSetting();
|
|
2472
|
+
}
|
|
2473
|
+
assertMaximumSamples(model, request.n ?? 1);
|
|
2474
|
+
assertInputLimits(
|
|
2475
|
+
model,
|
|
2476
|
+
request.prompt,
|
|
2477
|
+
[
|
|
2478
|
+
...request.images,
|
|
2479
|
+
...(request.mask === undefined ? [] : [request.mask]),
|
|
2480
|
+
],
|
|
2481
|
+
);
|
|
2482
|
+
}
|
|
2483
|
+
|
|
2484
|
+
function assertVideoCapabilities(
|
|
2485
|
+
model: GatewayMediaModel,
|
|
2486
|
+
request: ParsedVideoRequest,
|
|
2487
|
+
): void {
|
|
2488
|
+
const hasLastFrame = request.frameImages.some(
|
|
2489
|
+
frame => frame.frameType === "last_frame",
|
|
2490
|
+
);
|
|
2491
|
+
const visualReferences = request.inputReferences.filter(
|
|
2492
|
+
reference => !reference.mediaType.startsWith("audio/"),
|
|
2493
|
+
);
|
|
2494
|
+
const hasVideoReference = visualReferences.some(
|
|
2495
|
+
reference => reference.mediaType.startsWith("video/"),
|
|
2496
|
+
);
|
|
2497
|
+
const operationCandidates = visualReferences.length > 0
|
|
2498
|
+
? [
|
|
2499
|
+
"reference-to-video",
|
|
2500
|
+
...(hasVideoReference
|
|
2501
|
+
? ["motion-control", "video-editing", "extend-video"]
|
|
2502
|
+
: []),
|
|
2503
|
+
]
|
|
2504
|
+
: hasLastFrame
|
|
2505
|
+
? ["first-last-frame", "first-last-frame-to-video"]
|
|
2506
|
+
: request.promptImage !== undefined || request.frameImages.length > 0
|
|
2507
|
+
? ["image-to-video"]
|
|
2508
|
+
: ["text-to-video"];
|
|
2509
|
+
assertSupportedOperation(model, operationCandidates);
|
|
2510
|
+
assertListedString(
|
|
2511
|
+
model.capabilities,
|
|
2512
|
+
"supported_aspect_ratios",
|
|
2513
|
+
request.aspectRatio,
|
|
2514
|
+
);
|
|
2515
|
+
assertListedString(
|
|
2516
|
+
model.capabilities,
|
|
2517
|
+
"supported_resolutions",
|
|
2518
|
+
request.resolution?.toLocaleLowerCase("en-US"),
|
|
2519
|
+
);
|
|
2520
|
+
assertListedNumber(
|
|
2521
|
+
model.capabilities,
|
|
2522
|
+
"supported_durations_seconds",
|
|
2523
|
+
request.duration,
|
|
2524
|
+
);
|
|
2525
|
+
assertListedNumber(model.capabilities, "supported_fps", request.fps);
|
|
2526
|
+
if (
|
|
2527
|
+
request.generateAudio === true
|
|
2528
|
+
&& model.capabilities?.generate_audio === false
|
|
2529
|
+
) {
|
|
2530
|
+
unsupportedModelSetting();
|
|
2531
|
+
}
|
|
2532
|
+
assertMaximumSamples(model, request.n ?? 1);
|
|
2533
|
+
assertInputLimits(
|
|
2534
|
+
model,
|
|
2535
|
+
request.prompt,
|
|
2536
|
+
[
|
|
2537
|
+
...(request.promptImage === undefined ? [] : [request.promptImage]),
|
|
2538
|
+
...request.frameImages.map(frame => frame.image),
|
|
2539
|
+
...request.inputReferences,
|
|
2540
|
+
],
|
|
2541
|
+
);
|
|
2542
|
+
}
|
|
2543
|
+
|
|
2544
|
+
function catalogReceipt(
|
|
2545
|
+
catalog: GatewayMediaCatalogView,
|
|
2546
|
+
): Readonly<{ snapshotId: string; status: "fresh" | "stale" }> {
|
|
2547
|
+
return {
|
|
2548
|
+
snapshotId: catalog.snapshot.snapshotId,
|
|
2549
|
+
status: catalog.status,
|
|
2550
|
+
};
|
|
2551
|
+
}
|
|
2552
|
+
|
|
2553
|
+
function providerFailureReconciliation(
|
|
2554
|
+
error: unknown,
|
|
2555
|
+
): NonNullable<GatewayMediaExecutionError["reconciliation"]> {
|
|
2556
|
+
let candidate: unknown = error;
|
|
2557
|
+
let generationId: string | undefined;
|
|
2558
|
+
let providerMetadata: unknown;
|
|
2559
|
+
let statusCode: number | undefined;
|
|
2560
|
+
for (let depth = 0; depth < 4 && candidate !== undefined; depth += 1) {
|
|
2561
|
+
if (!isObject(candidate)) break;
|
|
2562
|
+
try {
|
|
2563
|
+
generationId ??= opaqueMetadataDigest(candidate.generationId);
|
|
2564
|
+
if (
|
|
2565
|
+
statusCode === undefined
|
|
2566
|
+
&& typeof candidate.statusCode === "number"
|
|
2567
|
+
&& Number.isSafeInteger(candidate.statusCode)
|
|
2568
|
+
&& candidate.statusCode >= 0
|
|
2569
|
+
&& candidate.statusCode <= 999
|
|
2570
|
+
) {
|
|
2571
|
+
statusCode = candidate.statusCode;
|
|
2572
|
+
}
|
|
2573
|
+
if (providerMetadata === undefined) {
|
|
2574
|
+
if (isObject(candidate.providerMetadata)) {
|
|
2575
|
+
providerMetadata = candidate.providerMetadata;
|
|
2576
|
+
} else if (isObject(candidate.data)) {
|
|
2577
|
+
providerMetadata = isObject(candidate.data.providerMetadata)
|
|
2578
|
+
? candidate.data.providerMetadata
|
|
2579
|
+
: isObject(candidate.data.gateway)
|
|
2580
|
+
? { gateway: candidate.data.gateway }
|
|
2581
|
+
: undefined;
|
|
2582
|
+
}
|
|
2583
|
+
}
|
|
2584
|
+
candidate = candidate.cause;
|
|
2585
|
+
} catch {
|
|
2586
|
+
break;
|
|
2587
|
+
}
|
|
2588
|
+
}
|
|
2589
|
+
const parsedRouting = parseGatewayRoutingMetadata(providerMetadata);
|
|
2590
|
+
const routing = generationId === undefined || parsedRouting.generationId !== undefined
|
|
2591
|
+
? parsedRouting
|
|
2592
|
+
: { ...parsedRouting, generationId };
|
|
2593
|
+
const failureSource = error instanceof Error
|
|
2594
|
+
? `${error.name}\u0000${error.message}`
|
|
2595
|
+
: typeof error === "string"
|
|
2596
|
+
? error
|
|
2597
|
+
: typeof error;
|
|
2598
|
+
return {
|
|
2599
|
+
failureSha256: createHash("sha256")
|
|
2600
|
+
.update(failureSource, "utf8")
|
|
2601
|
+
.digest("hex"),
|
|
2602
|
+
routing,
|
|
2603
|
+
...(statusCode === undefined ? {} : { statusCode }),
|
|
2604
|
+
};
|
|
2605
|
+
}
|
|
2606
|
+
|
|
2607
|
+
async function providerCall<Result>(
|
|
2608
|
+
credential: ActiveGatewayCredential,
|
|
2609
|
+
operation: (apiKey: string) => Promise<Result>,
|
|
2610
|
+
): Promise<Result> {
|
|
2611
|
+
try {
|
|
2612
|
+
return await credential.withApiKey(operation);
|
|
2613
|
+
} catch (error) {
|
|
2614
|
+
if (
|
|
2615
|
+
error instanceof GatewayMediaExecutionError
|
|
2616
|
+
&& (error.code === "invalid-response" || error.code === "download-failed")
|
|
2617
|
+
) {
|
|
2618
|
+
throw error;
|
|
2619
|
+
}
|
|
2620
|
+
throw new GatewayMediaExecutionError(
|
|
2621
|
+
"provider-failed",
|
|
2622
|
+
providerFailureReconciliation(error),
|
|
2623
|
+
);
|
|
2624
|
+
}
|
|
2625
|
+
}
|
|
2626
|
+
|
|
2627
|
+
export interface GatewayMediaService {
|
|
2628
|
+
generateImage(
|
|
2629
|
+
request: unknown,
|
|
2630
|
+
options?: Readonly<{ signal?: AbortSignal; timeoutMs?: number }>,
|
|
2631
|
+
): Promise<GatewayMediaArtifactBundle>;
|
|
2632
|
+
generateSpeech(
|
|
2633
|
+
request: unknown,
|
|
2634
|
+
options?: Readonly<{ signal?: AbortSignal; timeoutMs?: number }>,
|
|
2635
|
+
): Promise<GatewayMediaArtifactBundle>;
|
|
2636
|
+
generateVideo(
|
|
2637
|
+
request: unknown,
|
|
2638
|
+
options?: Readonly<{ signal?: AbortSignal; timeoutMs?: number }>,
|
|
2639
|
+
): Promise<GatewayMediaArtifactBundle>;
|
|
2640
|
+
transcribe(
|
|
2641
|
+
request: unknown,
|
|
2642
|
+
options?: Readonly<{ signal?: AbortSignal; timeoutMs?: number }>,
|
|
2643
|
+
): Promise<Readonly<ParsedTranscriptionResult & {
|
|
2644
|
+
artifact: GatewayMediaArtifactBundle;
|
|
2645
|
+
}>>;
|
|
2646
|
+
}
|
|
2647
|
+
|
|
2648
|
+
export interface GatewayMediaDispatchEvent {
|
|
2649
|
+
readonly model: string;
|
|
2650
|
+
readonly operation:
|
|
2651
|
+
| "image.generate"
|
|
2652
|
+
| "speech.generate"
|
|
2653
|
+
| "transcription.create"
|
|
2654
|
+
| "video.generate";
|
|
2655
|
+
readonly startedAt: string;
|
|
2656
|
+
}
|
|
2657
|
+
|
|
2658
|
+
export function createGatewayMediaService(options: Readonly<{
|
|
2659
|
+
artifactStore: GatewayMediaArtifactStore;
|
|
2660
|
+
catalog: CatalogCache;
|
|
2661
|
+
download?: GatewayMediaDownload;
|
|
2662
|
+
loadCredential(now: Date): Promise<ActiveGatewayCredential>;
|
|
2663
|
+
now?: () => Date;
|
|
2664
|
+
onDispatch?: (event: GatewayMediaDispatchEvent) => Promise<void> | void;
|
|
2665
|
+
sdk: GatewayMediaSdk;
|
|
2666
|
+
}>): GatewayMediaService {
|
|
2667
|
+
const now = options.now ?? (() => new Date());
|
|
2668
|
+
const download = options.download ?? createBoundedGatewayMediaDownload();
|
|
2669
|
+
return {
|
|
2670
|
+
generateImage: async (requestValue, execution = {}) => {
|
|
2671
|
+
const startedAt = now();
|
|
2672
|
+
const request = parseImageRequest(requestValue, startedAt);
|
|
2673
|
+
const catalog = await modelCatalog(
|
|
2674
|
+
options.catalog,
|
|
2675
|
+
request.model,
|
|
2676
|
+
"image",
|
|
2677
|
+
execution.signal,
|
|
2678
|
+
);
|
|
2679
|
+
const isLanguageImage = catalog.model.executionMode === "language-image";
|
|
2680
|
+
if (
|
|
2681
|
+
isLanguageImage
|
|
2682
|
+
&& (
|
|
2683
|
+
request.aspectRatio !== undefined
|
|
2684
|
+
|| request.mask !== undefined
|
|
2685
|
+
|| request.maxImagesPerCall !== undefined
|
|
2686
|
+
|| request.n !== undefined
|
|
2687
|
+
|| request.seed !== undefined
|
|
2688
|
+
|| request.size !== undefined
|
|
2689
|
+
)
|
|
2690
|
+
) {
|
|
2691
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
2692
|
+
}
|
|
2693
|
+
if (
|
|
2694
|
+
!isLanguageImage
|
|
2695
|
+
&& (
|
|
2696
|
+
request.maxOutputTokens !== undefined
|
|
2697
|
+
|| request.stopSequences !== undefined
|
|
2698
|
+
|| request.temperature !== undefined
|
|
2699
|
+
)
|
|
2700
|
+
) {
|
|
2701
|
+
throw new GatewayMediaExecutionError("invalid-request");
|
|
2702
|
+
}
|
|
2703
|
+
if (
|
|
2704
|
+
isLanguageImage
|
|
2705
|
+
&& request.images.length > 0
|
|
2706
|
+
&& !catalog.model.modalities.input.includes("image")
|
|
2707
|
+
) {
|
|
2708
|
+
throw new GatewayMediaExecutionError("model-operation-unsupported");
|
|
2709
|
+
}
|
|
2710
|
+
assertImageCapabilities(catalog.model, request);
|
|
2711
|
+
const dispatchAt = now();
|
|
2712
|
+
const credential = await options.loadCredential(dispatchAt);
|
|
2713
|
+
const timeout = combineAbortSignals(
|
|
2714
|
+
execution.signal,
|
|
2715
|
+
validateTimeout(execution.timeoutMs, DEFAULT_IMAGE_TIMEOUT_MS),
|
|
2716
|
+
);
|
|
2717
|
+
try {
|
|
2718
|
+
assertProviderDispatchActive(timeout.signal);
|
|
2719
|
+
await options.onDispatch?.({
|
|
2720
|
+
model: request.model,
|
|
2721
|
+
operation: "image.generate",
|
|
2722
|
+
startedAt: dispatchAt.toISOString(),
|
|
2723
|
+
});
|
|
2724
|
+
assertProviderDispatchActive(timeout.signal);
|
|
2725
|
+
const result = await settleBeforeAbort(providerCall(credential, async apiKey => {
|
|
2726
|
+
assertProviderDispatchActive(timeout.signal);
|
|
2727
|
+
if (isLanguageImage) {
|
|
2728
|
+
const sdkResult = await options.sdk.generateLanguageImage(apiKey, {
|
|
2729
|
+
abortSignal: timeout.signal,
|
|
2730
|
+
...(request.maxOutputTokens === undefined
|
|
2731
|
+
? {}
|
|
2732
|
+
: { maxOutputTokens: request.maxOutputTokens }),
|
|
2733
|
+
maxRetries: 0,
|
|
2734
|
+
...(request.images.length === 0
|
|
2735
|
+
? { prompt: request.prompt }
|
|
2736
|
+
: {
|
|
2737
|
+
messages: [{
|
|
2738
|
+
content: [
|
|
2739
|
+
{ text: request.prompt, type: "text" },
|
|
2740
|
+
...request.images.map(image => ({
|
|
2741
|
+
image: gatewaySdkInput(image),
|
|
2742
|
+
mediaType: image.mediaType,
|
|
2743
|
+
type: "image" as const,
|
|
2744
|
+
})),
|
|
2745
|
+
],
|
|
2746
|
+
role: "user" as const,
|
|
2747
|
+
}],
|
|
2748
|
+
}),
|
|
2749
|
+
modelId: request.model,
|
|
2750
|
+
...(request.providerOptions === undefined
|
|
2751
|
+
? {}
|
|
2752
|
+
: { providerOptions: request.providerOptions }),
|
|
2753
|
+
...(request.stopSequences === undefined
|
|
2754
|
+
? {}
|
|
2755
|
+
: { stopSequences: request.stopSequences }),
|
|
2756
|
+
...(request.temperature === undefined
|
|
2757
|
+
? {}
|
|
2758
|
+
: { temperature: request.temperature }),
|
|
2759
|
+
});
|
|
2760
|
+
if (!isObject(sdkResult) || !Array.isArray(sdkResult.files)) {
|
|
2761
|
+
throw new GatewayMediaExecutionError("invalid-response");
|
|
2762
|
+
}
|
|
2763
|
+
if (sdkResult.files.length < 1 || sdkResult.files.length > 32) {
|
|
2764
|
+
throw new GatewayMediaExecutionError("invalid-response");
|
|
2765
|
+
}
|
|
2766
|
+
return {
|
|
2767
|
+
files: sdkResult.files.map(
|
|
2768
|
+
value => parseGeneratedFile(value, ["image/"]),
|
|
2769
|
+
),
|
|
2770
|
+
routing: parseGatewayRoutingMetadata(sdkResult.providerMetadata),
|
|
2771
|
+
warnings: parseWarnings(sdkResult.warnings),
|
|
2772
|
+
};
|
|
2773
|
+
}
|
|
2774
|
+
const sdkResult = await options.sdk.generateImage(apiKey, {
|
|
2775
|
+
abortSignal: timeout.signal,
|
|
2776
|
+
...(request.aspectRatio === undefined
|
|
2777
|
+
? {}
|
|
2778
|
+
: { aspectRatio: request.aspectRatio }),
|
|
2779
|
+
...(request.maxImagesPerCall === undefined
|
|
2780
|
+
? {}
|
|
2781
|
+
: { maxImagesPerCall: request.maxImagesPerCall }),
|
|
2782
|
+
maxRetries: 0,
|
|
2783
|
+
modelId: request.model,
|
|
2784
|
+
...(request.n === undefined ? {} : { n: request.n }),
|
|
2785
|
+
prompt: request.images.length === 0 && request.mask === undefined
|
|
2786
|
+
? request.prompt
|
|
2787
|
+
: {
|
|
2788
|
+
images: request.images.map(gatewaySdkInput),
|
|
2789
|
+
...(request.mask === undefined
|
|
2790
|
+
? {}
|
|
2791
|
+
: { mask: gatewaySdkInput(request.mask) }),
|
|
2792
|
+
text: request.prompt,
|
|
2793
|
+
},
|
|
2794
|
+
...(request.providerOptions === undefined
|
|
2795
|
+
? {}
|
|
2796
|
+
: { providerOptions: request.providerOptions }),
|
|
2797
|
+
...(request.seed === undefined ? {} : { seed: request.seed }),
|
|
2798
|
+
...(request.size === undefined ? {} : { size: request.size }),
|
|
2799
|
+
});
|
|
2800
|
+
return parseGeneratedFiles(sdkResult, "images", ["image/"]);
|
|
2801
|
+
}), timeout.signal, "provider-failed");
|
|
2802
|
+
const inputs = [
|
|
2803
|
+
...request.images.map((input, index) => inputDigest(input, `image.${index + 1}`)),
|
|
2804
|
+
...(request.mask === undefined ? [] : [inputDigest(request.mask, "mask")]),
|
|
2805
|
+
];
|
|
2806
|
+
return await options.artifactStore.commit({
|
|
2807
|
+
catalog: catalogReceipt(catalog.catalog),
|
|
2808
|
+
createdAt: startedAt.toISOString(),
|
|
2809
|
+
files: result.files,
|
|
2810
|
+
inputs,
|
|
2811
|
+
mediaKind: "image",
|
|
2812
|
+
model: request.model,
|
|
2813
|
+
operation: "image.generate",
|
|
2814
|
+
request: requestSummary(cleanSummary({
|
|
2815
|
+
aspectRatio: optionalSummaryValue(request.aspectRatio),
|
|
2816
|
+
maxImagesPerCall: optionalSummaryValue(request.maxImagesPerCall),
|
|
2817
|
+
maxOutputTokens: optionalSummaryValue(request.maxOutputTokens),
|
|
2818
|
+
n: optionalSummaryValue(request.n),
|
|
2819
|
+
seed: optionalSummaryValue(request.seed),
|
|
2820
|
+
size: optionalSummaryValue(request.size),
|
|
2821
|
+
stopSequencesCount: request.stopSequences?.length,
|
|
2822
|
+
stopSequencesSha256: request.stopSequences === undefined
|
|
2823
|
+
? undefined
|
|
2824
|
+
: canonicalJsonSha256(request.stopSequences),
|
|
2825
|
+
temperature: optionalSummaryValue(request.temperature),
|
|
2826
|
+
}), request.prompt, request.providerOptions),
|
|
2827
|
+
routing: result.routing,
|
|
2828
|
+
signal: timeout.signal,
|
|
2829
|
+
sampleFulfillment: {
|
|
2830
|
+
produced: result.files.length,
|
|
2831
|
+
requested: request.n ?? 1,
|
|
2832
|
+
},
|
|
2833
|
+
warnings: result.warnings,
|
|
2834
|
+
});
|
|
2835
|
+
} finally {
|
|
2836
|
+
timeout.dispose();
|
|
2837
|
+
}
|
|
2838
|
+
},
|
|
2839
|
+
generateSpeech: async (requestValue, execution = {}) => {
|
|
2840
|
+
const startedAt = now();
|
|
2841
|
+
const request = parseSpeechRequest(requestValue, startedAt);
|
|
2842
|
+
const catalog = await modelCatalog(
|
|
2843
|
+
options.catalog,
|
|
2844
|
+
request.model,
|
|
2845
|
+
"speech",
|
|
2846
|
+
execution.signal,
|
|
2847
|
+
);
|
|
2848
|
+
const dispatchAt = now();
|
|
2849
|
+
const credential = await options.loadCredential(dispatchAt);
|
|
2850
|
+
const timeout = combineAbortSignals(
|
|
2851
|
+
execution.signal,
|
|
2852
|
+
validateTimeout(execution.timeoutMs, DEFAULT_SPEECH_TIMEOUT_MS),
|
|
2853
|
+
);
|
|
2854
|
+
try {
|
|
2855
|
+
assertProviderDispatchActive(timeout.signal);
|
|
2856
|
+
await options.onDispatch?.({
|
|
2857
|
+
model: request.model,
|
|
2858
|
+
operation: "speech.generate",
|
|
2859
|
+
startedAt: dispatchAt.toISOString(),
|
|
2860
|
+
});
|
|
2861
|
+
assertProviderDispatchActive(timeout.signal);
|
|
2862
|
+
const result = await settleBeforeAbort(providerCall(credential, async apiKey => {
|
|
2863
|
+
assertProviderDispatchActive(timeout.signal);
|
|
2864
|
+
const sdkResult = await options.sdk.generateSpeech(apiKey, {
|
|
2865
|
+
abortSignal: timeout.signal,
|
|
2866
|
+
...(request.instructions === undefined
|
|
2867
|
+
? {}
|
|
2868
|
+
: { instructions: request.instructions }),
|
|
2869
|
+
...(request.language === undefined ? {} : { language: request.language }),
|
|
2870
|
+
maxRetries: 0,
|
|
2871
|
+
modelId: request.model,
|
|
2872
|
+
...(request.outputFormat === undefined
|
|
2873
|
+
? {}
|
|
2874
|
+
: { outputFormat: request.outputFormat }),
|
|
2875
|
+
...(request.providerOptions === undefined
|
|
2876
|
+
? {}
|
|
2877
|
+
: { providerOptions: request.providerOptions }),
|
|
2878
|
+
...(request.speed === undefined ? {} : { speed: request.speed }),
|
|
2879
|
+
text: request.text,
|
|
2880
|
+
...(request.voice === undefined ? {} : { voice: request.voice }),
|
|
2881
|
+
});
|
|
2882
|
+
if (!isObject(sdkResult)) {
|
|
2883
|
+
throw new GatewayMediaExecutionError("invalid-response");
|
|
2884
|
+
}
|
|
2885
|
+
return {
|
|
2886
|
+
files: [parseGeneratedFile(sdkResult.audio, ["audio/"])],
|
|
2887
|
+
routing: parseGatewayRoutingMetadata(sdkResult.providerMetadata),
|
|
2888
|
+
warnings: parseWarnings(sdkResult.warnings),
|
|
2889
|
+
};
|
|
2890
|
+
}), timeout.signal, "provider-failed");
|
|
2891
|
+
return await options.artifactStore.commit({
|
|
2892
|
+
catalog: catalogReceipt(catalog.catalog),
|
|
2893
|
+
createdAt: startedAt.toISOString(),
|
|
2894
|
+
files: result.files,
|
|
2895
|
+
inputs: [],
|
|
2896
|
+
mediaKind: "speech",
|
|
2897
|
+
model: request.model,
|
|
2898
|
+
operation: "speech.generate",
|
|
2899
|
+
request: requestSummary(cleanSummary({
|
|
2900
|
+
instructionsCharacters: request.instructions?.length,
|
|
2901
|
+
instructionsSha256: request.instructions === undefined
|
|
2902
|
+
? undefined
|
|
2903
|
+
: createHash("sha256")
|
|
2904
|
+
.update(request.instructions, "utf8")
|
|
2905
|
+
.digest("hex"),
|
|
2906
|
+
language: optionalSummaryValue(request.language),
|
|
2907
|
+
outputFormat: optionalSummaryValue(request.outputFormat),
|
|
2908
|
+
speed: optionalSummaryValue(request.speed),
|
|
2909
|
+
textCharacters: request.text.length,
|
|
2910
|
+
textSha256: createHash("sha256")
|
|
2911
|
+
.update(request.text, "utf8")
|
|
2912
|
+
.digest("hex"),
|
|
2913
|
+
voice: optionalSummaryValue(request.voice),
|
|
2914
|
+
}), undefined, request.providerOptions),
|
|
2915
|
+
routing: result.routing,
|
|
2916
|
+
signal: timeout.signal,
|
|
2917
|
+
warnings: result.warnings,
|
|
2918
|
+
});
|
|
2919
|
+
} finally {
|
|
2920
|
+
timeout.dispose();
|
|
2921
|
+
}
|
|
2922
|
+
},
|
|
2923
|
+
generateVideo: async (requestValue, execution = {}) => {
|
|
2924
|
+
const startedAt = now();
|
|
2925
|
+
const request = parseVideoRequest(requestValue, startedAt);
|
|
2926
|
+
const catalog = await modelCatalog(
|
|
2927
|
+
options.catalog,
|
|
2928
|
+
request.model,
|
|
2929
|
+
"video",
|
|
2930
|
+
execution.signal,
|
|
2931
|
+
);
|
|
2932
|
+
assertVideoCapabilities(catalog.model, request);
|
|
2933
|
+
const dispatchAt = now();
|
|
2934
|
+
const credential = await options.loadCredential(dispatchAt);
|
|
2935
|
+
const timeout = combineAbortSignals(
|
|
2936
|
+
execution.signal,
|
|
2937
|
+
validateTimeout(execution.timeoutMs, DEFAULT_VIDEO_TIMEOUT_MS),
|
|
2938
|
+
);
|
|
2939
|
+
const requestDownload = createAggregateGatewayMediaDownload(download);
|
|
2940
|
+
try {
|
|
2941
|
+
assertProviderDispatchActive(timeout.signal);
|
|
2942
|
+
await options.onDispatch?.({
|
|
2943
|
+
model: request.model,
|
|
2944
|
+
operation: "video.generate",
|
|
2945
|
+
startedAt: dispatchAt.toISOString(),
|
|
2946
|
+
});
|
|
2947
|
+
assertProviderDispatchActive(timeout.signal);
|
|
2948
|
+
const result = await settleBeforeAbort(providerCall(credential, async apiKey => {
|
|
2949
|
+
assertProviderDispatchActive(timeout.signal);
|
|
2950
|
+
const sdkResult = await options.sdk.generateVideo(apiKey, {
|
|
2951
|
+
abortSignal: timeout.signal,
|
|
2952
|
+
...(request.aspectRatio === undefined
|
|
2953
|
+
? {}
|
|
2954
|
+
: { aspectRatio: request.aspectRatio }),
|
|
2955
|
+
download: requestDownload,
|
|
2956
|
+
...(request.duration === undefined ? {} : { duration: request.duration }),
|
|
2957
|
+
...(request.fps === undefined ? {} : { fps: request.fps }),
|
|
2958
|
+
...(request.frameImages.length === 0
|
|
2959
|
+
? {}
|
|
2960
|
+
: {
|
|
2961
|
+
frameImages: request.frameImages.map(frame => ({
|
|
2962
|
+
frameType: frame.frameType,
|
|
2963
|
+
image: gatewaySdkInput(frame.image),
|
|
2964
|
+
})),
|
|
2965
|
+
}),
|
|
2966
|
+
...(request.generateAudio === undefined
|
|
2967
|
+
? {}
|
|
2968
|
+
: { generateAudio: request.generateAudio }),
|
|
2969
|
+
...(request.inputReferences.length === 0
|
|
2970
|
+
? {}
|
|
2971
|
+
: {
|
|
2972
|
+
inputReferences: request.inputReferences.map(reference => ({
|
|
2973
|
+
data: gatewaySdkInput(reference),
|
|
2974
|
+
mediaType: reference.mediaType,
|
|
2975
|
+
})),
|
|
2976
|
+
}),
|
|
2977
|
+
maxRetries: 0,
|
|
2978
|
+
...(request.maxVideosPerCall === undefined
|
|
2979
|
+
? {}
|
|
2980
|
+
: { maxVideosPerCall: request.maxVideosPerCall }),
|
|
2981
|
+
modelId: request.model,
|
|
2982
|
+
...(request.n === undefined ? {} : { n: request.n }),
|
|
2983
|
+
prompt: request.promptImage === undefined
|
|
2984
|
+
? request.prompt
|
|
2985
|
+
: {
|
|
2986
|
+
image: gatewaySdkInput(request.promptImage),
|
|
2987
|
+
...(request.prompt.length === 0 ? {} : { text: request.prompt }),
|
|
2988
|
+
},
|
|
2989
|
+
...(request.providerOptions === undefined
|
|
2990
|
+
? {}
|
|
2991
|
+
: { providerOptions: request.providerOptions }),
|
|
2992
|
+
...(request.resolution === undefined
|
|
2993
|
+
? {}
|
|
2994
|
+
: { resolution: request.resolution }),
|
|
2995
|
+
...(request.seed === undefined ? {} : { seed: request.seed }),
|
|
2996
|
+
});
|
|
2997
|
+
return parseGeneratedFiles(sdkResult, "videos", ["video/"]);
|
|
2998
|
+
}), timeout.signal, "provider-failed");
|
|
2999
|
+
const inputs = [
|
|
3000
|
+
...(request.promptImage === undefined
|
|
3001
|
+
? []
|
|
3002
|
+
: [inputDigest(request.promptImage, "prompt-image")]),
|
|
3003
|
+
...request.frameImages.map((frame, index) => inputDigest(
|
|
3004
|
+
frame.image,
|
|
3005
|
+
`frame.${index + 1}.${frame.frameType}`,
|
|
3006
|
+
)),
|
|
3007
|
+
...request.inputReferences.map((reference, index) => inputDigest(
|
|
3008
|
+
reference,
|
|
3009
|
+
`reference.${index + 1}`,
|
|
3010
|
+
)),
|
|
3011
|
+
];
|
|
3012
|
+
return await options.artifactStore.commit({
|
|
3013
|
+
catalog: catalogReceipt(catalog.catalog),
|
|
3014
|
+
createdAt: startedAt.toISOString(),
|
|
3015
|
+
files: result.files,
|
|
3016
|
+
inputs,
|
|
3017
|
+
mediaKind: "video",
|
|
3018
|
+
model: request.model,
|
|
3019
|
+
operation: "video.generate",
|
|
3020
|
+
request: requestSummary(cleanSummary({
|
|
3021
|
+
aspectRatio: optionalSummaryValue(request.aspectRatio),
|
|
3022
|
+
duration: optionalSummaryValue(request.duration),
|
|
3023
|
+
fps: optionalSummaryValue(request.fps),
|
|
3024
|
+
frameTypes: request.frameImages.map(frame => frame.frameType),
|
|
3025
|
+
generateAudio: optionalSummaryValue(request.generateAudio),
|
|
3026
|
+
maxVideosPerCall: optionalSummaryValue(request.maxVideosPerCall),
|
|
3027
|
+
n: optionalSummaryValue(request.n),
|
|
3028
|
+
resolution: optionalSummaryValue(request.resolution),
|
|
3029
|
+
seed: optionalSummaryValue(request.seed),
|
|
3030
|
+
}), request.prompt, request.providerOptions),
|
|
3031
|
+
routing: result.routing,
|
|
3032
|
+
signal: timeout.signal,
|
|
3033
|
+
sampleFulfillment: {
|
|
3034
|
+
produced: result.files.length,
|
|
3035
|
+
requested: request.n ?? 1,
|
|
3036
|
+
},
|
|
3037
|
+
warnings: result.warnings,
|
|
3038
|
+
});
|
|
3039
|
+
} finally {
|
|
3040
|
+
timeout.dispose();
|
|
3041
|
+
}
|
|
3042
|
+
},
|
|
3043
|
+
transcribe: async (requestValue, execution = {}) => {
|
|
3044
|
+
const startedAt = now();
|
|
3045
|
+
const request = parseTranscriptionRequest(requestValue, startedAt);
|
|
3046
|
+
const catalog = await modelCatalog(
|
|
3047
|
+
options.catalog,
|
|
3048
|
+
request.model,
|
|
3049
|
+
"transcription",
|
|
3050
|
+
execution.signal,
|
|
3051
|
+
);
|
|
3052
|
+
if (catalog.model.tags.includes("websocket-realtime")) {
|
|
3053
|
+
throw new GatewayMediaExecutionError("model-operation-unsupported");
|
|
3054
|
+
}
|
|
3055
|
+
const dispatchAt = now();
|
|
3056
|
+
const credential = await options.loadCredential(dispatchAt);
|
|
3057
|
+
const timeout = combineAbortSignals(
|
|
3058
|
+
execution.signal,
|
|
3059
|
+
validateTimeout(execution.timeoutMs, DEFAULT_TRANSCRIPTION_TIMEOUT_MS),
|
|
3060
|
+
);
|
|
3061
|
+
try {
|
|
3062
|
+
assertProviderDispatchActive(timeout.signal);
|
|
3063
|
+
await options.onDispatch?.({
|
|
3064
|
+
model: request.model,
|
|
3065
|
+
operation: "transcription.create",
|
|
3066
|
+
startedAt: dispatchAt.toISOString(),
|
|
3067
|
+
});
|
|
3068
|
+
assertProviderDispatchActive(timeout.signal);
|
|
3069
|
+
const result = await settleBeforeAbort(providerCall(credential, async apiKey => {
|
|
3070
|
+
assertProviderDispatchActive(timeout.signal);
|
|
3071
|
+
const sdkResult = await options.sdk.transcribe(apiKey, {
|
|
3072
|
+
abortSignal: timeout.signal,
|
|
3073
|
+
audio: request.audio.data,
|
|
3074
|
+
maxRetries: 0,
|
|
3075
|
+
modelId: request.model,
|
|
3076
|
+
...(request.providerOptions === undefined
|
|
3077
|
+
? {}
|
|
3078
|
+
: { providerOptions: request.providerOptions }),
|
|
3079
|
+
});
|
|
3080
|
+
return parseTranscriptionResult(sdkResult);
|
|
3081
|
+
}), timeout.signal, "provider-failed");
|
|
3082
|
+
const artifact = await options.artifactStore.commit({
|
|
3083
|
+
catalog: catalogReceipt(catalog.catalog),
|
|
3084
|
+
createdAt: startedAt.toISOString(),
|
|
3085
|
+
files: transcriptFiles(result),
|
|
3086
|
+
inputs: [inputDigest(request.audio, "audio")],
|
|
3087
|
+
mediaKind: "transcription",
|
|
3088
|
+
model: request.model,
|
|
3089
|
+
operation: "transcription.create",
|
|
3090
|
+
request: requestSummary({}, undefined, request.providerOptions),
|
|
3091
|
+
routing: result.routing,
|
|
3092
|
+
signal: timeout.signal,
|
|
3093
|
+
warnings: result.warnings,
|
|
3094
|
+
});
|
|
3095
|
+
return { ...result, artifact };
|
|
3096
|
+
} finally {
|
|
3097
|
+
timeout.dispose();
|
|
3098
|
+
}
|
|
3099
|
+
},
|
|
3100
|
+
};
|
|
3101
|
+
}
|
|
3102
|
+
|
|
3103
|
+
export const GATEWAY_MEDIA_ENDPOINTS = {
|
|
3104
|
+
catalog: GATEWAY_MEDIA_CATALOG_URL,
|
|
3105
|
+
media: GATEWAY_MEDIA_API_BASE_URL,
|
|
3106
|
+
} as const;
|