@dreamlake/dreamlake-cli 0.88.1 → 0.89.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/access.d.ts +83 -0
- package/dist/api/access.js +77 -0
- package/dist/api/agent.d.ts +156 -0
- package/dist/api/agent.js +111 -0
- package/dist/api/browser.d.ts +90 -0
- package/dist/api/browser.js +102 -0
- package/dist/api/client.d.ts +53 -0
- package/dist/api/client.js +102 -0
- package/dist/api/common.d.ts +28 -0
- package/dist/api/common.js +16 -0
- package/dist/api/content.d.ts +195 -0
- package/dist/api/content.js +147 -0
- package/dist/api/execution.d.ts +84 -0
- package/dist/api/execution.js +73 -0
- package/dist/api/http.d.ts +19 -0
- package/dist/api/http.js +66 -0
- package/dist/api/index.d.ts +2330 -0
- package/dist/api/index.js +114 -0
- package/dist/api/infra.d.ts +2 -0
- package/dist/api/infra.js +2 -0
- package/dist/api/jobs.d.ts +2 -0
- package/dist/api/jobs.js +2 -0
- package/dist/api/layout.d.ts +2 -0
- package/dist/api/layout.js +2 -0
- package/dist/api/library-edit.d.ts +3 -0
- package/dist/api/library-edit.js +3 -0
- package/dist/api/library-manifest.d.ts +2 -0
- package/dist/api/library-manifest.js +2 -0
- package/dist/api/note-character-plan.d.ts +2 -0
- package/dist/api/note-character-plan.js +2 -0
- package/dist/api/note-editing.d.ts +2 -0
- package/dist/api/note-editing.js +2 -0
- package/dist/api/note-edits.d.ts +21 -0
- package/dist/api/note-edits.js +10 -0
- package/dist/api/note-events.d.ts +5 -0
- package/dist/api/note-events.js +11 -0
- package/dist/api/note-files.d.ts +75 -0
- package/dist/api/note-files.js +65 -0
- package/dist/api/note-html-read.d.ts +2 -0
- package/dist/api/note-html-read.js +2 -0
- package/dist/api/note-html.d.ts +2 -0
- package/dist/api/note-html.js +2 -0
- package/dist/api/note-toc.d.ts +2 -0
- package/dist/api/note-toc.js +2 -0
- package/dist/api/notes.d.ts +304 -0
- package/dist/api/notes.js +181 -0
- package/dist/api/open-resources.d.ts +27 -0
- package/dist/api/open-resources.js +102 -0
- package/dist/api/ownership.d.ts +12 -0
- package/dist/api/ownership.js +24 -0
- package/dist/api/platform.d.ts +83 -0
- package/dist/api/platform.js +56 -0
- package/dist/api/presigned-transfer.d.ts +2 -0
- package/dist/api/presigned-transfer.js +2 -0
- package/dist/api/registered-runs.d.ts +48 -0
- package/dist/api/registered-runs.js +33 -0
- package/dist/api/sessions.d.ts +299 -0
- package/dist/api/sessions.js +105 -0
- package/dist/api/storage.d.ts +46 -0
- package/dist/api/storage.js +34 -0
- package/dist/api/streams.d.ts +18 -0
- package/dist/api/streams.js +70 -0
- package/dist/api/vault.d.ts +93 -0
- package/dist/api/vault.js +79 -0
- package/dist/api/workspace.d.ts +180 -0
- package/dist/api/workspace.js +146 -0
- package/dist/cli/agent-identity.d.ts +4 -0
- package/dist/cli/agent-identity.js +21 -0
- package/dist/cli/agents/identity.d.ts +50 -0
- package/dist/cli/agents/identity.js +105 -0
- package/dist/cli/agents/index.d.ts +31 -0
- package/dist/cli/agents/index.js +580 -0
- package/dist/cli/agents/manage.d.ts +43 -0
- package/dist/cli/agents/manage.js +376 -0
- package/dist/cli/agents/spec.d.ts +104 -0
- package/dist/cli/agents/spec.js +280 -0
- package/dist/cli/agents/web.d.ts +7 -0
- package/dist/cli/agents/web.js +109 -0
- package/dist/cli/artifact/incremental.d.ts +21 -0
- package/dist/cli/artifact/incremental.js +80 -0
- package/dist/cli/artifact/index.d.ts +52 -0
- package/dist/cli/artifact/index.js +405 -0
- package/dist/cli/artifact/meta.d.ts +69 -0
- package/dist/cli/artifact/meta.js +189 -0
- package/dist/cli/auth/commands.d.ts +29 -0
- package/dist/cli/auth/commands.js +327 -0
- package/dist/cli/auth/constants.d.ts +13 -0
- package/dist/cli/auth/constants.js +18 -0
- package/dist/cli/auth/credentials.d.ts +37 -0
- package/dist/cli/auth/credentials.js +155 -0
- package/dist/cli/auth/device-flow.d.ts +32 -0
- package/dist/cli/auth/device-flow.js +154 -0
- package/dist/cli/auth/device-secret.d.ts +6 -0
- package/dist/cli/auth/device-secret.js +26 -0
- package/dist/cli/auth/machine.d.ts +15 -0
- package/dist/cli/auth/machine.js +126 -0
- package/dist/cli/auth/materialized.d.ts +32 -0
- package/dist/cli/auth/materialized.js +102 -0
- package/dist/cli/auth/migrate.d.ts +10 -0
- package/dist/cli/auth/migrate.js +75 -0
- package/dist/cli/client.d.ts +102 -0
- package/dist/cli/client.js +206 -0
- package/dist/cli/collection-list.d.ts +3 -0
- package/dist/cli/collection-list.js +25 -0
- package/dist/cli/compat/ctx.d.ts +26 -0
- package/dist/cli/compat/ctx.js +76 -0
- package/dist/cli/compat/dreamdb-wasm-embedded.d.ts +2 -0
- package/dist/cli/compat/dreamdb-wasm-embedded.js +8 -0
- package/dist/cli/compat/dreamdb-wasm.d.ts +1 -0
- package/dist/cli/compat/dreamdb-wasm.js +26 -0
- package/dist/cli/compat/dreamdb.d.ts +112 -0
- package/dist/cli/compat/dreamdb.js +219 -0
- package/dist/cli/compat/flags.d.ts +2 -0
- package/dist/cli/compat/flags.js +13 -0
- package/dist/cli/compat/http.d.ts +23 -0
- package/dist/cli/compat/http.js +137 -0
- package/dist/cli/compat/presignedTransfer.d.ts +139 -0
- package/dist/cli/compat/presignedTransfer.js +580 -0
- package/dist/cli/compat/s3-backend.d.ts +73 -0
- package/dist/cli/compat/s3-backend.js +132 -0
- package/dist/cli/config.d.ts +22 -0
- package/dist/cli/config.js +96 -0
- package/dist/cli/connection/dto.d.ts +159 -0
- package/dist/cli/connection/dto.js +497 -0
- package/dist/cli/connection/index.d.ts +50 -0
- package/dist/cli/connection/index.js +413 -0
- package/dist/cli/connection/oauth.d.ts +23 -0
- package/dist/cli/connection/oauth.js +290 -0
- package/dist/cli/connection/write.d.ts +47 -0
- package/dist/cli/connection/write.js +573 -0
- package/dist/cli/create/index.d.ts +19 -0
- package/dist/cli/create/index.js +209 -0
- package/dist/cli/declare/index.d.ts +29 -0
- package/dist/cli/declare/index.js +304 -0
- package/dist/cli/delete/index.d.ts +18 -0
- package/dist/cli/delete/index.js +232 -0
- package/dist/cli/download/index.d.ts +9 -0
- package/dist/cli/download/index.js +128 -0
- package/dist/cli/env/compose.d.ts +88 -0
- package/dist/cli/env/compose.js +402 -0
- package/dist/cli/env/index.d.ts +176 -0
- package/dist/cli/env/index.js +1011 -0
- package/dist/cli/experiments/index.d.ts +11 -0
- package/dist/cli/experiments/index.js +168 -0
- package/dist/cli/glob.d.ts +1 -0
- package/dist/cli/glob.js +45 -0
- package/dist/cli/graphql-helpers.d.ts +23 -0
- package/dist/cli/graphql-helpers.js +42 -0
- package/dist/cli/graphql.d.ts +10 -0
- package/dist/cli/graphql.js +47 -0
- package/dist/cli/help-examples.generated.d.ts +1 -0
- package/dist/cli/help-examples.generated.js +192 -0
- package/dist/cli/helpers.d.ts +8 -0
- package/dist/cli/helpers.js +106 -0
- package/dist/cli/hosts/api.d.ts +29 -0
- package/dist/cli/hosts/api.js +54 -0
- package/dist/cli/hosts/bootstrap-source.d.ts +1 -0
- package/dist/cli/hosts/bootstrap-source.js +2 -0
- package/dist/cli/hosts/catalog.d.ts +26 -0
- package/dist/cli/hosts/catalog.js +55 -0
- package/dist/cli/hosts/config.d.ts +45 -0
- package/dist/cli/hosts/config.js +179 -0
- package/dist/cli/hosts/credentials.d.ts +19 -0
- package/dist/cli/hosts/credentials.js +173 -0
- package/dist/cli/hosts/enroll.d.ts +17 -0
- package/dist/cli/hosts/enroll.js +109 -0
- package/dist/cli/hosts/enrollments.d.ts +25 -0
- package/dist/cli/hosts/enrollments.js +54 -0
- package/dist/cli/hosts/index.d.ts +6 -0
- package/dist/cli/hosts/index.js +279 -0
- package/dist/cli/hosts/inventory.d.ts +40 -0
- package/dist/cli/hosts/inventory.js +91 -0
- package/dist/cli/hosts/lifecycle.d.ts +15 -0
- package/dist/cli/hosts/lifecycle.js +108 -0
- package/dist/cli/hosts/metrics-source.d.ts +12 -0
- package/dist/cli/hosts/metrics-source.js +89 -0
- package/dist/cli/hosts/observations.d.ts +39 -0
- package/dist/cli/hosts/observations.js +102 -0
- package/dist/cli/hosts/remote.d.ts +5 -0
- package/dist/cli/hosts/remote.js +66 -0
- package/dist/cli/hosts/ssh.d.ts +63 -0
- package/dist/cli/hosts/ssh.js +276 -0
- package/dist/cli/index.d.ts +1 -0
- package/dist/cli/index.js +288 -0
- package/dist/cli/infra/api.d.ts +95 -0
- package/dist/cli/infra/api.js +291 -0
- package/dist/cli/infra/index.d.ts +5 -0
- package/dist/cli/infra/index.js +102 -0
- package/dist/cli/infra/move.d.ts +15 -0
- package/dist/cli/infra/move.js +123 -0
- package/dist/cli/infra/transfer.d.ts +15 -0
- package/dist/cli/infra/transfer.js +280 -0
- package/dist/cli/jobs/api.d.ts +69 -0
- package/dist/cli/jobs/api.js +121 -0
- package/dist/cli/jobs/index.d.ts +5 -0
- package/dist/cli/jobs/index.js +140 -0
- package/dist/cli/lakeshore/index.d.ts +98 -0
- package/dist/cli/lakeshore/index.js +502 -0
- package/dist/cli/launch/index.d.ts +43 -0
- package/dist/cli/launch/index.js +270 -0
- package/dist/cli/layout/index.d.ts +2 -0
- package/dist/cli/layout/index.js +44 -0
- package/dist/cli/layout/transport.d.ts +12 -0
- package/dist/cli/layout/transport.js +70 -0
- package/dist/cli/library/add.d.ts +60 -0
- package/dist/cli/library/add.js +348 -0
- package/dist/cli/library/cache.d.ts +26 -0
- package/dist/cli/library/cache.js +88 -0
- package/dist/cli/library/config.d.ts +75 -0
- package/dist/cli/library/config.js +198 -0
- package/dist/cli/library/digest.d.ts +5 -0
- package/dist/cli/library/digest.js +19 -0
- package/dist/cli/library/discover.d.ts +87 -0
- package/dist/cli/library/discover.js +534 -0
- package/dist/cli/library/freshness.d.ts +35 -0
- package/dist/cli/library/freshness.js +109 -0
- package/dist/cli/library/generation.d.ts +35 -0
- package/dist/cli/library/generation.js +211 -0
- package/dist/cli/library/index.d.ts +2 -0
- package/dist/cli/library/index.js +36 -0
- package/dist/cli/library/info.d.ts +69 -0
- package/dist/cli/library/info.js +311 -0
- package/dist/cli/library/list.d.ts +33 -0
- package/dist/cli/library/list.js +86 -0
- package/dist/cli/library/manifest.d.ts +81 -0
- package/dist/cli/library/manifest.js +200 -0
- package/dist/cli/library/manifestCache.d.ts +45 -0
- package/dist/cli/library/manifestCache.js +169 -0
- package/dist/cli/library/pull.d.ts +30 -0
- package/dist/cli/library/pull.js +438 -0
- package/dist/cli/library/push.d.ts +72 -0
- package/dist/cli/library/push.js +642 -0
- package/dist/cli/library/rm.d.ts +31 -0
- package/dist/cli/library/rm.js +200 -0
- package/dist/cli/library/search.d.ts +18 -0
- package/dist/cli/library/search.js +181 -0
- package/dist/cli/library/stat.d.ts +15 -0
- package/dist/cli/library/stat.js +127 -0
- package/dist/cli/library/tools.d.ts +104 -0
- package/dist/cli/library/tools.js +163 -0
- package/dist/cli/library/wire.d.ts +47 -0
- package/dist/cli/library/wire.js +120 -0
- package/dist/cli/list/index.d.ts +26 -0
- package/dist/cli/list/index.js +256 -0
- package/dist/cli/notes/character-plan.d.ts +22 -0
- package/dist/cli/notes/character-plan.js +232 -0
- package/dist/cli/notes/document.d.ts +73 -0
- package/dist/cli/notes/document.js +306 -0
- package/dist/cli/notes/edit-commands.d.ts +106 -0
- package/dist/cli/notes/edit-commands.js +623 -0
- package/dist/cli/notes/editing.d.ts +81 -0
- package/dist/cli/notes/editing.js +305 -0
- package/dist/cli/notes/events.d.ts +36 -0
- package/dist/cli/notes/events.js +54 -0
- package/dist/cli/notes/files-commands.d.ts +120 -0
- package/dist/cli/notes/files-commands.js +554 -0
- package/dist/cli/notes/html-read.d.ts +9 -0
- package/dist/cli/notes/html-read.js +90 -0
- package/dist/cli/notes/html.d.ts +73 -0
- package/dist/cli/notes/html.js +381 -0
- package/dist/cli/notes/index.d.ts +123 -0
- package/dist/cli/notes/index.js +1303 -0
- package/dist/cli/notes/linger.d.ts +59 -0
- package/dist/cli/notes/linger.js +339 -0
- package/dist/cli/notes/media-commands.d.ts +5 -0
- package/dist/cli/notes/media-commands.js +51 -0
- package/dist/cli/notes/move-command.d.ts +83 -0
- package/dist/cli/notes/move-command.js +259 -0
- package/dist/cli/notes/rm-command.d.ts +22 -0
- package/dist/cli/notes/rm-command.js +110 -0
- package/dist/cli/notes/select-command.d.ts +12 -0
- package/dist/cli/notes/select-command.js +84 -0
- package/dist/cli/notes/suggest-diff.d.ts +56 -0
- package/dist/cli/notes/suggest-diff.js +311 -0
- package/dist/cli/notes/toc.d.ts +38 -0
- package/dist/cli/notes/toc.js +144 -0
- package/dist/cli/notes/transfer-command.d.ts +6 -0
- package/dist/cli/notes/transfer-command.js +9 -0
- package/dist/cli/notes/version-commands.d.ts +19 -0
- package/dist/cli/notes/version-commands.js +178 -0
- package/dist/cli/nymph/index.d.ts +74 -0
- package/dist/cli/nymph/index.js +349 -0
- package/dist/cli/org/index.d.ts +34 -0
- package/dist/cli/org/index.js +341 -0
- package/dist/cli/ownership/index.d.ts +12 -0
- package/dist/cli/ownership/index.js +122 -0
- package/dist/cli/ownership/metadata-transfer.d.ts +20 -0
- package/dist/cli/ownership/metadata-transfer.js +159 -0
- package/dist/cli/page/index.d.ts +58 -0
- package/dist/cli/page/index.js +387 -0
- package/dist/cli/pipeline/index.d.ts +69 -0
- package/dist/cli/pipeline/index.js +604 -0
- package/dist/cli/progress.d.ts +60 -0
- package/dist/cli/progress.js +276 -0
- package/dist/cli/project/attachments.d.ts +10 -0
- package/dist/cli/project/attachments.js +39 -0
- package/dist/cli/project/grants.d.ts +10 -0
- package/dist/cli/project/grants.js +39 -0
- package/dist/cli/project/index.d.ts +8 -0
- package/dist/cli/project/index.js +92 -0
- package/dist/cli/project/move.d.ts +57 -0
- package/dist/cli/project/move.js +242 -0
- package/dist/cli/project/output.d.ts +2 -0
- package/dist/cli/project/output.js +29 -0
- package/dist/cli/project/stream.d.ts +55 -0
- package/dist/cli/project/stream.js +207 -0
- package/dist/cli/project/transfer-preview.d.ts +8 -0
- package/dist/cli/project/transfer-preview.js +10 -0
- package/dist/cli/prompt.d.ts +1 -0
- package/dist/cli/prompt.js +17 -0
- package/dist/cli/provider/dreamdir.d.ts +52 -0
- package/dist/cli/provider/dreamdir.js +138 -0
- package/dist/cli/provider/index.d.ts +4 -0
- package/dist/cli/provider/index.js +419 -0
- package/dist/cli/provider/jump-worker.d.ts +9 -0
- package/dist/cli/provider/jump-worker.js +141 -0
- package/dist/cli/provider/lifecycle.d.ts +149 -0
- package/dist/cli/provider/lifecycle.js +1390 -0
- package/dist/cli/provider/machine.d.ts +17 -0
- package/dist/cli/provider/machine.js +116 -0
- package/dist/cli/provider/ssh.d.ts +105 -0
- package/dist/cli/provider/ssh.js +305 -0
- package/dist/cli/provider/terraform.d.ts +140 -0
- package/dist/cli/provider/terraform.js +288 -0
- package/dist/cli/providers/index.d.ts +18 -0
- package/dist/cli/providers/index.js +422 -0
- package/dist/cli/python.d.ts +61 -0
- package/dist/cli/python.js +255 -0
- package/dist/cli/queues/index.d.ts +33 -0
- package/dist/cli/queues/index.js +248 -0
- package/dist/cli/queues/slurm.d.ts +10 -0
- package/dist/cli/queues/slurm.js +134 -0
- package/dist/cli/resources.d.ts +72 -0
- package/dist/cli/resources.js +137 -0
- package/dist/cli/run/index.d.ts +15 -0
- package/dist/cli/run/index.js +163 -0
- package/dist/cli/run/input.d.ts +34 -0
- package/dist/cli/run/input.js +135 -0
- package/dist/cli/run/recovery.d.ts +2 -0
- package/dist/cli/run/recovery.js +18 -0
- package/dist/cli/run/setup.d.ts +3 -0
- package/dist/cli/run/setup.js +62 -0
- package/dist/cli/runconfig/index.d.ts +12 -0
- package/dist/cli/runconfig/index.js +130 -0
- package/dist/cli/runnable/index.d.ts +35 -0
- package/dist/cli/runnable/index.js +575 -0
- package/dist/cli/runnable/path.d.ts +115 -0
- package/dist/cli/runnable/path.js +288 -0
- package/dist/cli/runnable/run.d.ts +91 -0
- package/dist/cli/runnable/run.js +239 -0
- package/dist/cli/runnable/upload.d.ts +43 -0
- package/dist/cli/runnable/upload.js +342 -0
- package/dist/cli/runnable/worker.d.ts +1 -0
- package/dist/cli/runnable/worker.js +44 -0
- package/dist/cli/schemas/workflow-spec.schema.json +335 -0
- package/dist/cli/self-update/index.d.ts +5 -0
- package/dist/cli/self-update/index.js +107 -0
- package/dist/cli/self-update/install.d.ts +60 -0
- package/dist/cli/self-update/install.js +428 -0
- package/dist/cli/self-update/paths.d.ts +40 -0
- package/dist/cli/self-update/paths.js +87 -0
- package/dist/cli/self-update/releases.d.ts +41 -0
- package/dist/cli/self-update/releases.js +153 -0
- package/dist/cli/self-update/settings.d.ts +16 -0
- package/dist/cli/self-update/settings.js +68 -0
- package/dist/cli/self-update/update.d.ts +18 -0
- package/dist/cli/self-update/update.js +241 -0
- package/dist/cli/service-accounts/index.d.ts +3 -0
- package/dist/cli/service-accounts/index.js +149 -0
- package/dist/cli/session-affiliation.d.ts +26 -0
- package/dist/cli/session-affiliation.js +135 -0
- package/dist/cli/sessions/affiliation.d.ts +8 -0
- package/dist/cli/sessions/affiliation.js +66 -0
- package/dist/cli/sessions/api.d.ts +55 -0
- package/dist/cli/sessions/api.js +88 -0
- package/dist/cli/sessions/control.d.ts +6 -0
- package/dist/cli/sessions/control.js +159 -0
- package/dist/cli/sessions/conversation.d.ts +15 -0
- package/dist/cli/sessions/conversation.js +118 -0
- package/dist/cli/sessions/index.d.ts +2 -0
- package/dist/cli/sessions/index.js +209 -0
- package/dist/cli/sessions/managed.d.ts +39 -0
- package/dist/cli/sessions/managed.js +63 -0
- package/dist/cli/sessions/redaction.d.ts +2 -0
- package/dist/cli/sessions/redaction.js +11 -0
- package/dist/cli/sessions/runner/exec.d.ts +108 -0
- package/dist/cli/sessions/runner/exec.js +335 -0
- package/dist/cli/sessions/runner/journal.d.ts +26 -0
- package/dist/cli/sessions/runner/journal.js +49 -0
- package/dist/cli/sessions/runner/native.d.ts +48 -0
- package/dist/cli/sessions/runner/native.js +170 -0
- package/dist/cli/sessions/runner/pod.d.ts +30 -0
- package/dist/cli/sessions/runner/pod.js +129 -0
- package/dist/cli/sessions/runner/serve.d.ts +85 -0
- package/dist/cli/sessions/runner/serve.js +558 -0
- package/dist/cli/sessions/sync.d.ts +47 -0
- package/dist/cli/sessions/sync.js +80 -0
- package/dist/cli/sessions/transport.d.ts +14 -0
- package/dist/cli/sessions/transport.js +116 -0
- package/dist/cli/share.d.ts +15 -0
- package/dist/cli/share.js +139 -0
- package/dist/cli/skill/catalog.d.ts +141 -0
- package/dist/cli/skill/catalog.js +442 -0
- package/dist/cli/skill/files.d.ts +89 -0
- package/dist/cli/skill/files.js +264 -0
- package/dist/cli/skill/index.d.ts +268 -0
- package/dist/cli/skill/index.js +1216 -0
- package/dist/cli/skill/manifest.d.ts +30 -0
- package/dist/cli/skill/manifest.js +101 -0
- package/dist/cli/skill/prefs.d.ts +9 -0
- package/dist/cli/skill/prefs.js +32 -0
- package/dist/cli/skill/reminders.d.ts +15 -0
- package/dist/cli/skill/reminders.js +105 -0
- package/dist/cli/skill/tar.d.ts +17 -0
- package/dist/cli/skill/tar.js +195 -0
- package/dist/cli/snapshot/index.d.ts +24 -0
- package/dist/cli/snapshot/index.js +146 -0
- package/dist/cli/source/access.d.ts +45 -0
- package/dist/cli/source/access.js +563 -0
- package/dist/cli/source/control.d.ts +103 -0
- package/dist/cli/source/control.js +546 -0
- package/dist/cli/source/delete.d.ts +89 -0
- package/dist/cli/source/delete.js +463 -0
- package/dist/cli/source/index.d.ts +288 -0
- package/dist/cli/source/index.js +1348 -0
- package/dist/cli/source/share.d.ts +65 -0
- package/dist/cli/source/share.js +530 -0
- package/dist/cli/source/upload.d.ts +62 -0
- package/dist/cli/source/upload.js +258 -0
- package/dist/cli/source/write.d.ts +77 -0
- package/dist/cli/source/write.js +335 -0
- package/dist/cli/spawn.d.ts +28 -0
- package/dist/cli/spawn.js +81 -0
- package/dist/cli/target.d.ts +22 -0
- package/dist/cli/target.js +85 -0
- package/dist/cli/tasks/index.d.ts +38 -0
- package/dist/cli/tasks/index.js +365 -0
- package/dist/cli/team/index.d.ts +37 -0
- package/dist/cli/team/index.js +407 -0
- package/dist/cli/telemetry.d.ts +23 -0
- package/dist/cli/telemetry.js +262 -0
- package/dist/cli/telemetryJournal.d.ts +26 -0
- package/dist/cli/telemetryJournal.js +104 -0
- package/dist/cli/update/index.d.ts +21 -0
- package/dist/cli/update/index.js +284 -0
- package/dist/cli/upload/index.d.ts +11 -0
- package/dist/cli/upload/index.js +261 -0
- package/dist/cli/upload/kinds.d.ts +10 -0
- package/dist/cli/upload/kinds.js +85 -0
- package/dist/cli/upload/multipart.d.ts +21 -0
- package/dist/cli/upload/multipart.js +248 -0
- package/dist/cli/vault/affected-preview.d.ts +15 -0
- package/dist/cli/vault/affected-preview.js +38 -0
- package/dist/cli/vault/cloud.d.ts +55 -0
- package/dist/cli/vault/cloud.js +287 -0
- package/dist/cli/vault/format.d.ts +18 -0
- package/dist/cli/vault/format.js +104 -0
- package/dist/cli/vault/hostCredentials.d.ts +21 -0
- package/dist/cli/vault/hostCredentials.js +71 -0
- package/dist/cli/vault/hostSupersessions.d.ts +18 -0
- package/dist/cli/vault/hostSupersessions.js +68 -0
- package/dist/cli/vault/import.d.ts +3 -0
- package/dist/cli/vault/import.js +79 -0
- package/dist/cli/vault/index.d.ts +14 -0
- package/dist/cli/vault/index.js +225 -0
- package/dist/cli/vault/keys.d.ts +5 -0
- package/dist/cli/vault/keys.js +165 -0
- package/dist/cli/vault/kms.d.ts +6 -0
- package/dist/cli/vault/kms.js +169 -0
- package/dist/cli/vault/list.d.ts +13 -0
- package/dist/cli/vault/list.js +60 -0
- package/dist/cli/vault/listView.d.ts +21 -0
- package/dist/cli/vault/listView.js +123 -0
- package/dist/cli/vault/managementMetadata.d.ts +2 -0
- package/dist/cli/vault/managementMetadata.js +23 -0
- package/dist/cli/vault/mutations.d.ts +8 -0
- package/dist/cli/vault/mutations.js +151 -0
- package/dist/cli/vault/otp.d.ts +6 -0
- package/dist/cli/vault/otp.js +62 -0
- package/dist/cli/vault/otpIntent.d.ts +23 -0
- package/dist/cli/vault/otpIntent.js +76 -0
- package/dist/cli/vault/passImport.d.ts +25 -0
- package/dist/cli/vault/passImport.js +129 -0
- package/dist/cli/vault/passOtp.d.ts +51 -0
- package/dist/cli/vault/passOtp.js +101 -0
- package/dist/cli/vault/passStore.d.ts +26 -0
- package/dist/cli/vault/passStore.js +134 -0
- package/dist/cli/vault/passwordCommand.d.ts +39 -0
- package/dist/cli/vault/passwordCommand.js +101 -0
- package/dist/cli/vault/passwordProbe.d.ts +27 -0
- package/dist/cli/vault/passwordProbe.js +282 -0
- package/dist/cli/vault/passwordRotation.d.ts +9 -0
- package/dist/cli/vault/passwordRotation.js +273 -0
- package/dist/cli/vault/rotationApi.d.ts +13 -0
- package/dist/cli/vault/rotationApi.js +58 -0
- package/dist/cli/vault/rotationCommand.d.ts +2 -0
- package/dist/cli/vault/rotationCommand.js +75 -0
- package/dist/cli/vault/rotationEngine.d.ts +19 -0
- package/dist/cli/vault/rotationEngine.js +65 -0
- package/dist/cli/vault/rotationHelperSource.d.ts +2 -0
- package/dist/cli/vault/rotationHelperSource.js +3 -0
- package/dist/cli/vault/rotationInput.d.ts +11 -0
- package/dist/cli/vault/rotationInput.js +48 -0
- package/dist/cli/vault/rotationJournal.d.ts +16 -0
- package/dist/cli/vault/rotationJournal.js +183 -0
- package/dist/cli/vault/rotationKeys.d.ts +4 -0
- package/dist/cli/vault/rotationKeys.js +48 -0
- package/dist/cli/vault/rotationPrepare.d.ts +17 -0
- package/dist/cli/vault/rotationPrepare.js +108 -0
- package/dist/cli/vault/rotationRemote.d.ts +33 -0
- package/dist/cli/vault/rotationRemote.js +156 -0
- package/dist/cli/vault/rotationState.d.ts +42 -0
- package/dist/cli/vault/rotationState.js +115 -0
- package/dist/cli/vault/rotationTransport.d.ts +18 -0
- package/dist/cli/vault/rotationTransport.js +71 -0
- package/dist/cli/vault/sshSource.d.ts +18 -0
- package/dist/cli/vault/sshSource.js +196 -0
- package/dist/cli/vault/sshSync.d.ts +17 -0
- package/dist/cli/vault/sshSync.js +224 -0
- package/dist/cli/vault/sshTransfer.d.ts +22 -0
- package/dist/cli/vault/sshTransfer.js +140 -0
- package/dist/cli/vault/terraformState.d.ts +66 -0
- package/dist/cli/vault/terraformState.js +252 -0
- package/dist/cli/vault/terraformStateCommand.d.ts +2 -0
- package/dist/cli/vault/terraformStateCommand.js +61 -0
- package/dist/cli/vault/tree.d.ts +43 -0
- package/dist/cli/vault/tree.js +292 -0
- package/dist/cli/version.d.ts +1 -0
- package/dist/cli/version.js +9 -0
- package/dist/cli/workflow/index.d.ts +76 -0
- package/dist/cli/workflow/index.js +826 -0
- package/dist/cli/workflow/validate.d.ts +19 -0
- package/dist/cli/workflow/validate.js +353 -0
- package/dist/package.json +1 -0
- package/package.json +28 -10
|
@@ -0,0 +1,580 @@
|
|
|
1
|
+
// The presigned-batch transfer executor, shared by `dreamlake source upload`
|
|
2
|
+
// and `dreamlake library push`.
|
|
3
|
+
//
|
|
4
|
+
// Both surfaces answer an upload-authorizations call with the same transfer
|
|
5
|
+
// shape per file — `{kind:'single-put'}` or `{kind:'s3-multipart'}` — and this
|
|
6
|
+
// module owns everything that happens after an authorization is in hand:
|
|
7
|
+
// validating each transfer plan, streaming the bytes, completing multiparts
|
|
8
|
+
// via the XML body, and abandoning every multipart the batch opened when any
|
|
9
|
+
// file fails. It is deliberately server-shape-agnostic: input is the
|
|
10
|
+
// `{relativePath, size, transfer}` triple plus a local-path resolver, and the
|
|
11
|
+
// caller keeps whatever envelope its own API wraps around that.
|
|
12
|
+
//
|
|
13
|
+
// Two properties are carried over from the legacy BSS uploader rather than
|
|
14
|
+
// rewritten, because they were the parts that were right:
|
|
15
|
+
//
|
|
16
|
+
// - the file is never buffered whole. Every PUT streams off one open handle,
|
|
17
|
+
// which is what lets a file past Buffer's 2 GiB cap upload at all;
|
|
18
|
+
// - fewer bytes than promised is an error, not something to pad around. It
|
|
19
|
+
// means the file shrank underneath us, and what was already sent describes
|
|
20
|
+
// a file that no longer exists.
|
|
21
|
+
//
|
|
22
|
+
// There is no checksum handshake here: neither contract publishes one, and an
|
|
23
|
+
// ETag is an opaque token, passed to Complete and never interpreted.
|
|
24
|
+
import { open } from "node:fs/promises";
|
|
25
|
+
import { Readable, Transform } from "node:stream";
|
|
26
|
+
export const COMPLETE_BODY_FORMAT = "s3-complete-multipart-xml";
|
|
27
|
+
const COMPLETE_ROOT = "CompleteMultipartUploadResult";
|
|
28
|
+
export function isRecord(v) {
|
|
29
|
+
return typeof v === "object" && v !== null && !Array.isArray(v);
|
|
30
|
+
}
|
|
31
|
+
/** The default refusal, when a caller has no protocol-specific one. */
|
|
32
|
+
export function transferBad(what) {
|
|
33
|
+
throw new Error(`incompatible upload authorization: ${what}`);
|
|
34
|
+
}
|
|
35
|
+
export function assertSignedOperation(v, where, bad = transferBad) {
|
|
36
|
+
if (!isRecord(v))
|
|
37
|
+
bad(`${where}: expected a signed operation`);
|
|
38
|
+
const op = v;
|
|
39
|
+
if (typeof op.url !== "string" || op.url === "")
|
|
40
|
+
bad(`${where}: missing url`);
|
|
41
|
+
if (!isRecord(op.headers))
|
|
42
|
+
bad(`${where}: missing headers`);
|
|
43
|
+
for (const [k, value] of Object.entries(op.headers)) {
|
|
44
|
+
// A non-string header value throws inside `fetch` rather than failing the
|
|
45
|
+
// transfer, and the throw escapes as something the caller cannot act on.
|
|
46
|
+
if (typeof value !== "string")
|
|
47
|
+
bad(`${where}: header ${k} is not a string`);
|
|
48
|
+
}
|
|
49
|
+
return v;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Check one file's transfer plan against the size it was requested for,
|
|
53
|
+
* before a byte moves.
|
|
54
|
+
*
|
|
55
|
+
* The part plan is the dangerous half. Each range is derived from `partNumber`
|
|
56
|
+
* and `partSize` alone, so a gap sends the wrong bytes under a signature minted
|
|
57
|
+
* for another, a duplicate sends one range twice and omits another, and too few
|
|
58
|
+
* parts leave the tail unsent — and in every case each PUT succeeds, Complete
|
|
59
|
+
* succeeds, and the object exists holding bytes nobody checked.
|
|
60
|
+
*/
|
|
61
|
+
export function assertPresignedTransfer(v, size, where, bad = transferBad,
|
|
62
|
+
/**
|
|
63
|
+
* Part numbers storage already holds, when `v` came from a RESUME.
|
|
64
|
+
*
|
|
65
|
+
* Supplying this switches the part-plan check from "every part, in order" to
|
|
66
|
+
* "together with these, every part exactly once" — the same property, over
|
|
67
|
+
* the union. It is NOT a relaxation: a resume legitimately omits the parts
|
|
68
|
+
* that already landed, and the danger the check exists for is unchanged, so
|
|
69
|
+
* the check has to cover the same ground by a different route. Omit it and
|
|
70
|
+
* the original dense check runs byte for byte as before.
|
|
71
|
+
*/
|
|
72
|
+
completed) {
|
|
73
|
+
if (!isRecord(v))
|
|
74
|
+
bad(`${where}.transfer: expected an object`);
|
|
75
|
+
const t = v;
|
|
76
|
+
if (t.kind === "single-put") {
|
|
77
|
+
assertSignedOperation(t.put, `${where}.transfer.put`, bad);
|
|
78
|
+
return v;
|
|
79
|
+
}
|
|
80
|
+
if (t.kind !== "s3-multipart")
|
|
81
|
+
bad(`${where}.transfer: unknown kind`);
|
|
82
|
+
if (typeof t.uploadId !== "string" || t.uploadId === "")
|
|
83
|
+
bad(`${where}: missing uploadId`);
|
|
84
|
+
if (typeof t.partSize !== "number" || !Number.isSafeInteger(t.partSize) || t.partSize <= 0) {
|
|
85
|
+
bad(`${where}: partSize must be a positive integer`);
|
|
86
|
+
}
|
|
87
|
+
const partSize = t.partSize;
|
|
88
|
+
const partCount = Math.ceil(size / partSize);
|
|
89
|
+
// A resume whose every part already landed sends none, and has only the
|
|
90
|
+
// completion left to issue — so "no parts" is a refusal only on the fresh
|
|
91
|
+
// path, where it would mean the file is not covered at all.
|
|
92
|
+
if (!Array.isArray(t.parts) || (t.parts.length === 0 && completed === undefined)) {
|
|
93
|
+
bad(`${where}: no parts`);
|
|
94
|
+
}
|
|
95
|
+
const parts = t.parts;
|
|
96
|
+
if (completed === undefined) {
|
|
97
|
+
if (parts.length !== partCount) {
|
|
98
|
+
bad(`${where}: the parts do not cover the file`);
|
|
99
|
+
}
|
|
100
|
+
parts.forEach((part, p) => {
|
|
101
|
+
if (!isRecord(part) || part.partNumber !== p + 1) {
|
|
102
|
+
bad(`${where}.parts[${p}]: expected partNumber ${p + 1}`);
|
|
103
|
+
}
|
|
104
|
+
assertSignedOperation(part.put, `${where}.parts[${p}].put`, bad);
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
else {
|
|
108
|
+
/*
|
|
109
|
+
* The union has to be the whole file, exactly once over.
|
|
110
|
+
*
|
|
111
|
+
* Each gap in it is a range nobody sends, each overlap a range sent twice
|
|
112
|
+
* under one of two signatures — and both finish with every PUT and the
|
|
113
|
+
* Complete succeeding, leaving an object whose bytes no one checked. The
|
|
114
|
+
* stored side is as dangerous as the signed side: trusting a `completed`
|
|
115
|
+
* entry that is not really there drops that range just as silently.
|
|
116
|
+
*/
|
|
117
|
+
const have = new Set();
|
|
118
|
+
for (const partNumber of completed) {
|
|
119
|
+
if (!Number.isSafeInteger(partNumber) || partNumber < 1 || partNumber > partCount) {
|
|
120
|
+
bad(`${where}: stored part ${partNumber} is outside the ${partCount} this file has`);
|
|
121
|
+
}
|
|
122
|
+
if (have.has(partNumber))
|
|
123
|
+
bad(`${where}: stored part ${partNumber} is listed twice`);
|
|
124
|
+
have.add(partNumber);
|
|
125
|
+
}
|
|
126
|
+
parts.forEach((part, p) => {
|
|
127
|
+
if (!isRecord(part))
|
|
128
|
+
bad(`${where}.parts[${p}]: expected an object`);
|
|
129
|
+
const partNumber = part.partNumber;
|
|
130
|
+
if (typeof partNumber !== "number" || !Number.isSafeInteger(partNumber)) {
|
|
131
|
+
bad(`${where}.parts[${p}]: partNumber must be an integer`);
|
|
132
|
+
}
|
|
133
|
+
if (partNumber < 1 || partNumber > partCount) {
|
|
134
|
+
bad(`${where}.parts[${p}]: partNumber ${partNumber} is outside the ${partCount} this file has`);
|
|
135
|
+
}
|
|
136
|
+
if (have.has(partNumber)) {
|
|
137
|
+
bad(`${where}.parts[${p}]: part ${partNumber} is already stored`);
|
|
138
|
+
}
|
|
139
|
+
have.add(partNumber);
|
|
140
|
+
assertSignedOperation(part.put, `${where}.parts[${p}].put`, bad);
|
|
141
|
+
});
|
|
142
|
+
if (have.size !== partCount) {
|
|
143
|
+
bad(`${where}: the stored and signed parts together do not cover the file`);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
const complete = assertSignedOperation(t.complete, `${where}.transfer.complete`, bad);
|
|
147
|
+
if (complete.method !== "POST")
|
|
148
|
+
bad(`${where}: complete is not a POST`);
|
|
149
|
+
// The literal the contract fixes. It decides what document this client
|
|
150
|
+
// assembles, so a different value is a body nobody promised to accept.
|
|
151
|
+
if (complete.bodyFormat !== COMPLETE_BODY_FORMAT) {
|
|
152
|
+
bad(`${where}: complete.bodyFormat is not ${COMPLETE_BODY_FORMAT}`);
|
|
153
|
+
}
|
|
154
|
+
const abort = assertSignedOperation(t.abort, `${where}.transfer.abort`, bad);
|
|
155
|
+
// Required because this executor USES it: a multipart that fails partway is
|
|
156
|
+
// aborted so storage does not keep the parts it already accepted.
|
|
157
|
+
if (abort.method !== "DELETE")
|
|
158
|
+
bad(`${where}: abort is not a DELETE`);
|
|
159
|
+
return v;
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* The root element of an XML document, or null if it is not one document.
|
|
163
|
+
*
|
|
164
|
+
* A scanner with a tag stack rather than a substring search. Two cases motivate
|
|
165
|
+
* it, and both are shapes storage really produces:
|
|
166
|
+
*
|
|
167
|
+
* - truncation. A response whose open tag arrived and whose rest did not
|
|
168
|
+
* contains the success element's name, and `includes()` reads it as
|
|
169
|
+
* success — exactly when the object is least likely to exist;
|
|
170
|
+
* - a second top-level element, or text beside the root. `<Error/>` next to a
|
|
171
|
+
* success document is not a success document, and neither is one with a
|
|
172
|
+
* proxy's bytes glued to it.
|
|
173
|
+
*
|
|
174
|
+
* So: exactly one root, self-closing elements counted as roots too, and nothing
|
|
175
|
+
* outside it but whitespace, declarations and comments.
|
|
176
|
+
*
|
|
177
|
+
* Deliberately not a general XML parser. It resolves one question and returns
|
|
178
|
+
* null rather than guessing when it cannot.
|
|
179
|
+
*/
|
|
180
|
+
export function xmlRootElement(text) {
|
|
181
|
+
const body = text.trim();
|
|
182
|
+
if (body === "")
|
|
183
|
+
return null;
|
|
184
|
+
const stack = [];
|
|
185
|
+
let root = null;
|
|
186
|
+
let rootClosed = false;
|
|
187
|
+
let i = 0;
|
|
188
|
+
/** Text outside the root is content this document does not account for. */
|
|
189
|
+
const outsideText = (from, to) => body.slice(from, to).trim() !== "";
|
|
190
|
+
/**
|
|
191
|
+
* The `>` that ends a tag, skipping any inside an attribute value.
|
|
192
|
+
*
|
|
193
|
+
* `<Tag bad="><Other/></Tag>` has its first `>` inside an unterminated
|
|
194
|
+
* attribute. Reading that as the end of the tag turns one malformed document
|
|
195
|
+
* into what looks like a well-formed pair — and the quote is never closed, so
|
|
196
|
+
* this is not XML at all. Returns -1 for both "no `>`" and "a quote that
|
|
197
|
+
* never closed", which are the same answer here: not one document.
|
|
198
|
+
*/
|
|
199
|
+
const tagEnd = (from) => {
|
|
200
|
+
let quote = null;
|
|
201
|
+
for (let at = from; at < body.length; at += 1) {
|
|
202
|
+
const ch = body[at];
|
|
203
|
+
if (quote) {
|
|
204
|
+
if (ch === quote)
|
|
205
|
+
quote = null;
|
|
206
|
+
continue;
|
|
207
|
+
}
|
|
208
|
+
if (ch === '"' || ch === "'")
|
|
209
|
+
quote = ch;
|
|
210
|
+
else if (ch === ">")
|
|
211
|
+
return at;
|
|
212
|
+
}
|
|
213
|
+
return -1;
|
|
214
|
+
};
|
|
215
|
+
while (i < body.length) {
|
|
216
|
+
const lt = body.indexOf("<", i);
|
|
217
|
+
if (lt === -1) {
|
|
218
|
+
// Trailing bytes after the last tag.
|
|
219
|
+
return outsideText(i, body.length) ? null : rootClosed ? root : null;
|
|
220
|
+
}
|
|
221
|
+
if (stack.length === 0 && outsideText(i, lt))
|
|
222
|
+
return null;
|
|
223
|
+
if (body.startsWith("<?", lt) || body.startsWith("<!--", lt) || body.startsWith("<![CDATA[", lt)) {
|
|
224
|
+
const [open, close] = body.startsWith("<?", lt)
|
|
225
|
+
? ["<?", "?>"]
|
|
226
|
+
: body.startsWith("<!--", lt)
|
|
227
|
+
? ["<!--", "-->"]
|
|
228
|
+
: ["<![CDATA[", "]]>"];
|
|
229
|
+
// CDATA outside the root is content, not decoration.
|
|
230
|
+
if (open === "<![CDATA[" && stack.length === 0)
|
|
231
|
+
return null;
|
|
232
|
+
const at = body.indexOf(close, lt + open.length);
|
|
233
|
+
if (at === -1)
|
|
234
|
+
return null;
|
|
235
|
+
i = at + close.length;
|
|
236
|
+
continue;
|
|
237
|
+
}
|
|
238
|
+
const gt = tagEnd(lt + 1);
|
|
239
|
+
// An unterminated tag — or an unclosed attribute quote — is the truncation
|
|
240
|
+
// case.
|
|
241
|
+
if (gt === -1)
|
|
242
|
+
return null;
|
|
243
|
+
const tag = body.slice(lt + 1, gt).trim();
|
|
244
|
+
if (tag.startsWith("/")) {
|
|
245
|
+
const name = tag.slice(1).trim();
|
|
246
|
+
if (stack.pop() !== name)
|
|
247
|
+
return null;
|
|
248
|
+
if (stack.length === 0)
|
|
249
|
+
rootClosed = true;
|
|
250
|
+
}
|
|
251
|
+
else {
|
|
252
|
+
const selfClosing = tag.endsWith("/");
|
|
253
|
+
const name = tag.replace(/\/$/, "").trim().split(/[\s]/)[0] ?? "";
|
|
254
|
+
if (name === "")
|
|
255
|
+
return null;
|
|
256
|
+
/*
|
|
257
|
+
* A second root, whether paired or self-closing.
|
|
258
|
+
*
|
|
259
|
+
* Self-closing was the hole: `<Error/>` beside a success document never
|
|
260
|
+
* touched the stack, so it neither became the root nor disqualified the
|
|
261
|
+
* one that followed.
|
|
262
|
+
*/
|
|
263
|
+
if (stack.length === 0) {
|
|
264
|
+
if (rootClosed || root !== null)
|
|
265
|
+
return null;
|
|
266
|
+
root = name;
|
|
267
|
+
if (selfClosing)
|
|
268
|
+
rootClosed = true;
|
|
269
|
+
}
|
|
270
|
+
if (!selfClosing)
|
|
271
|
+
stack.push(name);
|
|
272
|
+
}
|
|
273
|
+
i = gt + 1;
|
|
274
|
+
}
|
|
275
|
+
return stack.length === 0 && rootClosed ? root : null;
|
|
276
|
+
}
|
|
277
|
+
/**
|
|
278
|
+
* The signed headers plus a content length, reconciled case-insensitively.
|
|
279
|
+
*
|
|
280
|
+
* HTTP header names are case-insensitive, but object spread is not: a plan that
|
|
281
|
+
* already declares `Content-Length` and a lowercase one added beside it become
|
|
282
|
+
* two values, which Node joins into `content-length: 10, 10`. That is an
|
|
283
|
+
* invalid request, and on a signed operation it may also break the signature.
|
|
284
|
+
*
|
|
285
|
+
* A value that is already there and agrees is reused. One that disagrees is a
|
|
286
|
+
* plan that cannot be executed as given, and is refused before any byte moves
|
|
287
|
+
* rather than sent and puzzled over.
|
|
288
|
+
*/
|
|
289
|
+
function withContentLength(signed, length, what) {
|
|
290
|
+
const existing = Object.entries(signed).filter(([name]) => name.toLowerCase() === "content-length");
|
|
291
|
+
if (existing.length === 0)
|
|
292
|
+
return { ...signed, "content-length": String(length) };
|
|
293
|
+
/*
|
|
294
|
+
* More than one spelling is already broken.
|
|
295
|
+
*
|
|
296
|
+
* `Content-Length` and `content-length` together are two headers to this
|
|
297
|
+
* object and one header to HTTP: Node joins them into `10, 10` whatever their
|
|
298
|
+
* values are, so reusing them unchanged sends an invalid request. Finding
|
|
299
|
+
* only the first match would have passed this through.
|
|
300
|
+
*/
|
|
301
|
+
if (existing.length > 1) {
|
|
302
|
+
throw new Error(`${what}: the authorization declares Content-Length ${existing.length} times`);
|
|
303
|
+
}
|
|
304
|
+
const [, value] = existing[0];
|
|
305
|
+
if (value.trim() !== String(length)) {
|
|
306
|
+
throw new Error(`${what}: the authorization declares Content-Length ${value} for a ${length}-byte range`);
|
|
307
|
+
}
|
|
308
|
+
return { ...signed };
|
|
309
|
+
}
|
|
310
|
+
/** Counts what passes through, so a short file cannot be sent as a whole one. */
|
|
311
|
+
function counting() {
|
|
312
|
+
let total = 0;
|
|
313
|
+
const t = new Transform({
|
|
314
|
+
transform(chunk, _enc, cb) {
|
|
315
|
+
total += chunk.length;
|
|
316
|
+
cb(null, chunk);
|
|
317
|
+
},
|
|
318
|
+
});
|
|
319
|
+
return Object.assign(t, { total: () => total });
|
|
320
|
+
}
|
|
321
|
+
/**
|
|
322
|
+
* Stream one byte range off the open handle and PUT it.
|
|
323
|
+
*
|
|
324
|
+
* `createReadStream` on the HANDLE, never a fresh open by path: between the
|
|
325
|
+
* request and this moment the file may have been replaced, and a second lookup
|
|
326
|
+
* would send whatever is there now under a signature minted for what was
|
|
327
|
+
* measured.
|
|
328
|
+
*/
|
|
329
|
+
async function putRange(fh, op, range, what, fetchFn) {
|
|
330
|
+
const counter = counting();
|
|
331
|
+
/*
|
|
332
|
+
* An empty range has no stream.
|
|
333
|
+
*
|
|
334
|
+
* `end: start - 1` is what a zero-length range computes to, and Node rejects
|
|
335
|
+
* it outright — so an empty file could not be uploaded at all. An empty body
|
|
336
|
+
* with `content-length: 0` is what an empty object actually is.
|
|
337
|
+
*/
|
|
338
|
+
const source = range.length === 0
|
|
339
|
+
? null
|
|
340
|
+
: fh.createReadStream({
|
|
341
|
+
start: range.start,
|
|
342
|
+
end: range.start + range.length - 1,
|
|
343
|
+
autoClose: false,
|
|
344
|
+
});
|
|
345
|
+
const short = () => new Error(`${what}: sent ${counter.total()} of ${range.length} bytes (file changed mid-upload?)`);
|
|
346
|
+
const headers = withContentLength(op.headers, range.length, what);
|
|
347
|
+
let res;
|
|
348
|
+
try {
|
|
349
|
+
res = await fetchFn(op.url, {
|
|
350
|
+
method: "PUT",
|
|
351
|
+
headers,
|
|
352
|
+
// Node's fetch takes a web ReadableStream body; the DOM's `BodyInit` is
|
|
353
|
+
// not in scope here, so the conversion is asserted rather than named.
|
|
354
|
+
body: source ? Readable.toWeb(source.pipe(counter)) : undefined,
|
|
355
|
+
// Node requires this for a streaming request body.
|
|
356
|
+
duplex: "half",
|
|
357
|
+
});
|
|
358
|
+
}
|
|
359
|
+
catch (err) {
|
|
360
|
+
/*
|
|
361
|
+
* A stream that ends early breaks the request, and `fetch` reports that as
|
|
362
|
+
* a bare "fetch failed" — true, and useless. The counter knows what
|
|
363
|
+
* actually happened, so the diagnosis is put back: the file shrank while it
|
|
364
|
+
* was being sent, which is the one cause worth naming.
|
|
365
|
+
*/
|
|
366
|
+
if (counter.total() < range.length)
|
|
367
|
+
throw short();
|
|
368
|
+
throw err;
|
|
369
|
+
}
|
|
370
|
+
if (!res.ok)
|
|
371
|
+
throw new Error(`${what}: storage answered ${res.status}`);
|
|
372
|
+
/*
|
|
373
|
+
* A second line of defence, and honestly labelled as one.
|
|
374
|
+
*
|
|
375
|
+
* Node's fetch enforces `content-length` against what the body produced, so a
|
|
376
|
+
* short stream fails above rather than here — no test reaches this branch,
|
|
377
|
+
* and pretending otherwise would be a comment claiming a guard that never
|
|
378
|
+
* runs. It stays because the enforcement belongs to the runtime rather than
|
|
379
|
+
* to this contract, and a runtime that stopped doing it would otherwise
|
|
380
|
+
* report a truncated object as uploaded.
|
|
381
|
+
*/
|
|
382
|
+
if (counter.total() !== range.length)
|
|
383
|
+
throw short();
|
|
384
|
+
// Opaque. Neither contract publishes a checksum algorithm, and an ETag is
|
|
385
|
+
// only an MD5 under conditions this client cannot observe — under SSE it is
|
|
386
|
+
// not one, and reading a legitimate value as corruption would fail uploads
|
|
387
|
+
// that are fine.
|
|
388
|
+
return { etag: res.headers.get("etag"), bytes: counter.total() };
|
|
389
|
+
}
|
|
390
|
+
function completeBody(parts) {
|
|
391
|
+
const escape = (v) => v.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");
|
|
392
|
+
const entries = parts
|
|
393
|
+
.slice()
|
|
394
|
+
.sort((a, b) => a.partNumber - b.partNumber)
|
|
395
|
+
.map((p) => `<Part><PartNumber>${p.partNumber}</PartNumber><ETag>${escape(p.etag)}</ETag></Part>`)
|
|
396
|
+
.join("");
|
|
397
|
+
return `<CompleteMultipartUpload>${entries}</CompleteMultipartUpload>`;
|
|
398
|
+
}
|
|
399
|
+
/**
|
|
400
|
+
* Best-effort abandonment of a multipart upload that will not finish.
|
|
401
|
+
*
|
|
402
|
+
* Storage lifecycle policy cleans unfinished uploads up eventually, so a failed
|
|
403
|
+
* abort costs nothing and must never replace the error the caller actually
|
|
404
|
+
* needs to see.
|
|
405
|
+
*/
|
|
406
|
+
async function abortMultipart(transfer, fetchFn) {
|
|
407
|
+
if (transfer.kind !== "s3-multipart")
|
|
408
|
+
return;
|
|
409
|
+
try {
|
|
410
|
+
await fetchFn(transfer.abort.url, {
|
|
411
|
+
method: transfer.abort.method,
|
|
412
|
+
headers: transfer.abort.headers,
|
|
413
|
+
});
|
|
414
|
+
}
|
|
415
|
+
catch {
|
|
416
|
+
// Deliberately swallowed. See above.
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
async function sendOne(file, localPath, fetchFn, partConcurrency, onPartDone) {
|
|
420
|
+
const fh = await open(localPath, "r");
|
|
421
|
+
try {
|
|
422
|
+
/*
|
|
423
|
+
* The file as it is now, against the size the authorization was minted for.
|
|
424
|
+
*
|
|
425
|
+
* Measured through the OPEN handle rather than by path, for the same reason
|
|
426
|
+
* the reads are. The byte counting below catches a file that shrinks while
|
|
427
|
+
* it is being sent; it cannot catch one that GREW before the first read —
|
|
428
|
+
* every range would stream cleanly and the tail would simply never have
|
|
429
|
+
* been authorized — which is what this catches.
|
|
430
|
+
*/
|
|
431
|
+
const actual = (await fh.stat()).size;
|
|
432
|
+
if (actual !== file.size) {
|
|
433
|
+
throw new Error(`${file.relativePath}: ${actual} bytes on disk but ${file.size} were authorized ` +
|
|
434
|
+
`(file changed after it was measured?)`);
|
|
435
|
+
}
|
|
436
|
+
if (file.transfer.kind === "single-put") {
|
|
437
|
+
const { bytes } = await putRange(fh, file.transfer.put, { start: 0, length: file.size }, file.relativePath, fetchFn);
|
|
438
|
+
return bytes;
|
|
439
|
+
}
|
|
440
|
+
const { partSize, parts, complete } = file.transfer;
|
|
441
|
+
/*
|
|
442
|
+
* Seeded with the parts a previous run already stored.
|
|
443
|
+
*
|
|
444
|
+
* Complete names EVERY part or the object is assembled from a subset, so
|
|
445
|
+
* the ones this run does not send still have to appear. They are carried
|
|
446
|
+
* on the file because only the caller knows whether its authorization was
|
|
447
|
+
* a fresh one or a resume.
|
|
448
|
+
*/
|
|
449
|
+
const etags = (file.completedParts ?? []).map((p) => ({ partNumber: p.partNumber, etag: p.etag }));
|
|
450
|
+
let sent = 0;
|
|
451
|
+
/*
|
|
452
|
+
* Parts in flight at once.
|
|
453
|
+
*
|
|
454
|
+
* Ranges are read off the one handle by absolute offset, so they do not
|
|
455
|
+
* contend; `putRange` already derives everything it needs from the part
|
|
456
|
+
* number. Defaults to 1 — strictly sequential, the shape this executor has
|
|
457
|
+
* always had — so a caller that does not ask for parallelism gets exactly
|
|
458
|
+
* the previous behaviour.
|
|
459
|
+
*/
|
|
460
|
+
const lanes = Math.max(1, Math.min(Math.floor(partConcurrency), parts.length || 1));
|
|
461
|
+
let cursor = 0;
|
|
462
|
+
let partFailure;
|
|
463
|
+
await Promise.all(Array.from({ length: lanes }, async () => {
|
|
464
|
+
for (;;) {
|
|
465
|
+
if (partFailure !== undefined)
|
|
466
|
+
return;
|
|
467
|
+
const i = cursor++;
|
|
468
|
+
if (i >= parts.length)
|
|
469
|
+
return;
|
|
470
|
+
const part = parts[i];
|
|
471
|
+
const start = (part.partNumber - 1) * partSize;
|
|
472
|
+
try {
|
|
473
|
+
const { etag, bytes } = await putRange(fh, part.put, { start, length: Math.min(partSize, file.size - start) }, `${file.relativePath} part ${part.partNumber}`, fetchFn);
|
|
474
|
+
if (!etag) {
|
|
475
|
+
// The PUT succeeded, so retrying returns the same missing header —
|
|
476
|
+
// on real storage a configuration that does not expose it.
|
|
477
|
+
throw new Error(`${file.relativePath}: storage returned no ETag for part ${part.partNumber}`);
|
|
478
|
+
}
|
|
479
|
+
etags.push({ partNumber: part.partNumber, etag });
|
|
480
|
+
sent += bytes;
|
|
481
|
+
onPartDone?.(part.partNumber, etag);
|
|
482
|
+
}
|
|
483
|
+
catch (err) {
|
|
484
|
+
if (partFailure === undefined)
|
|
485
|
+
partFailure = err;
|
|
486
|
+
return;
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
}));
|
|
490
|
+
if (partFailure !== undefined)
|
|
491
|
+
throw partFailure;
|
|
492
|
+
const res = await fetchFn(complete.url, {
|
|
493
|
+
method: complete.method,
|
|
494
|
+
headers: complete.headers,
|
|
495
|
+
body: completeBody(etags),
|
|
496
|
+
});
|
|
497
|
+
if (!res.ok)
|
|
498
|
+
throw new Error(`${file.relativePath}: complete answered ${res.status}`);
|
|
499
|
+
/*
|
|
500
|
+
* A 2xx from Complete is where it starts, not where it ends.
|
|
501
|
+
*
|
|
502
|
+
* S3 holds the connection open with whitespace while it assembles the
|
|
503
|
+
* parts, so the status line is written before the outcome is known and
|
|
504
|
+
* the body may be an `<Error>`.
|
|
505
|
+
* https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html
|
|
506
|
+
*/
|
|
507
|
+
if (xmlRootElement(await res.text()) !== COMPLETE_ROOT) {
|
|
508
|
+
throw new Error(`${file.relativePath}: storage refused to complete the upload`);
|
|
509
|
+
}
|
|
510
|
+
return sent;
|
|
511
|
+
}
|
|
512
|
+
finally {
|
|
513
|
+
await fh.close();
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
/**
|
|
517
|
+
* Execute one authorized batch: every file, in order, and — on any failure —
|
|
518
|
+
* every multipart the AUTHORIZATION opened is abandoned, including ones the
|
|
519
|
+
* executor never reached. The authorization may start a multipart upload for
|
|
520
|
+
* the whole batch, so a failure anywhere leaves parts held for files that were
|
|
521
|
+
* never started, and storage keeps them until its lifecycle policy notices.
|
|
522
|
+
*
|
|
523
|
+
* No new file starts after a failure; in-flight files are allowed to settle
|
|
524
|
+
* before the abandonment so an abort never races its own parts. The original
|
|
525
|
+
* error is rethrown whatever the aborts do.
|
|
526
|
+
*
|
|
527
|
+
* `concurrency` defaults to 1 — strictly sequential, the shape
|
|
528
|
+
* `source upload` has always had.
|
|
529
|
+
*/
|
|
530
|
+
export async function executePresignedBatch(files, opts) {
|
|
531
|
+
const fetchFn = opts.fetchFn ?? ((input, init) => fetch(input, init));
|
|
532
|
+
const concurrency = Math.max(1, Math.floor(opts.concurrency ?? 1));
|
|
533
|
+
const partConcurrency = Math.max(1, Math.floor(opts.partConcurrency ?? 1));
|
|
534
|
+
const unfinished = new Set(files.filter((f) => f.transfer.kind === "s3-multipart"));
|
|
535
|
+
const results = new Array(files.length);
|
|
536
|
+
let next = 0;
|
|
537
|
+
let failed = false;
|
|
538
|
+
let firstError;
|
|
539
|
+
const workers = Array.from({ length: Math.min(concurrency, files.length) }, async () => {
|
|
540
|
+
for (;;) {
|
|
541
|
+
if (failed)
|
|
542
|
+
return;
|
|
543
|
+
const i = next++;
|
|
544
|
+
if (i >= files.length)
|
|
545
|
+
return;
|
|
546
|
+
const file = files[i];
|
|
547
|
+
try {
|
|
548
|
+
const bytes = await sendOne(file, opts.localPath(file, i), fetchFn, partConcurrency, opts.onPartDone && ((pn, etag) => opts.onPartDone(file, pn, etag, i)));
|
|
549
|
+
unfinished.delete(file);
|
|
550
|
+
results[i] = { file, bytes };
|
|
551
|
+
opts.onFileDone?.(file, bytes, i);
|
|
552
|
+
}
|
|
553
|
+
catch (err) {
|
|
554
|
+
if (!failed) {
|
|
555
|
+
failed = true;
|
|
556
|
+
firstError = err;
|
|
557
|
+
}
|
|
558
|
+
return;
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
});
|
|
562
|
+
await Promise.all(workers);
|
|
563
|
+
if (failed) {
|
|
564
|
+
/*
|
|
565
|
+
* Best-effort, and after the error is already in hand.
|
|
566
|
+
*
|
|
567
|
+
* Abandoning the parts must not replace the failure the caller needs to
|
|
568
|
+
* see — `abortMultipart` swallows its own trouble, and the original error
|
|
569
|
+
* is rethrown whatever happens here.
|
|
570
|
+
*
|
|
571
|
+
* Skipped entirely when the caller asked to keep them: for a resumable
|
|
572
|
+
* upload the stored parts are the whole point of failing cleanly.
|
|
573
|
+
*/
|
|
574
|
+
if ((opts.onFailure ?? "abort") === "abort") {
|
|
575
|
+
await Promise.all([...unfinished].map((f) => abortMultipart(f.transfer, fetchFn)));
|
|
576
|
+
}
|
|
577
|
+
throw firstError;
|
|
578
|
+
}
|
|
579
|
+
return results;
|
|
580
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/** Result of a conditional PUT, as DreamDB's connector expects it. */
|
|
2
|
+
export type PutStatus = "created" | "exists" | "casFailed";
|
|
3
|
+
/**
|
|
4
|
+
* The object-store surface DreamDB drives. Declared structurally so the
|
|
5
|
+
* write path can be exercised against a filesystem stand-in without an S3
|
|
6
|
+
* (or a live bucket) in the loop.
|
|
7
|
+
*/
|
|
8
|
+
export interface DreamdbBackend {
|
|
9
|
+
get(key: string, range?: {
|
|
10
|
+
start: number;
|
|
11
|
+
end: number;
|
|
12
|
+
}): Promise<{
|
|
13
|
+
bytes: Uint8Array;
|
|
14
|
+
etag?: string;
|
|
15
|
+
}>;
|
|
16
|
+
head(key: string): Promise<{
|
|
17
|
+
exists?: boolean;
|
|
18
|
+
etag?: string;
|
|
19
|
+
size?: number;
|
|
20
|
+
}>;
|
|
21
|
+
put(key: string, bytes: Uint8Array, opts?: {
|
|
22
|
+
ifMatch?: string;
|
|
23
|
+
ifNoneMatchStar?: boolean;
|
|
24
|
+
}): Promise<{
|
|
25
|
+
status: PutStatus;
|
|
26
|
+
etag?: string;
|
|
27
|
+
}>;
|
|
28
|
+
list(prefix: string): Promise<string[]>;
|
|
29
|
+
}
|
|
30
|
+
export interface S3Credentials {
|
|
31
|
+
accessKeyId: string;
|
|
32
|
+
secretAccessKey: string;
|
|
33
|
+
/** Empty for static-endpoint (MinIO) brokers — must then be omitted
|
|
34
|
+
* entirely, since an empty token breaks SigV4. */
|
|
35
|
+
sessionToken?: string;
|
|
36
|
+
}
|
|
37
|
+
export declare class S3Backend {
|
|
38
|
+
private readonly client;
|
|
39
|
+
private readonly base;
|
|
40
|
+
/**
|
|
41
|
+
* @param baseUrl path-style dataset base, e.g.
|
|
42
|
+
* `https://s3.us-east-1.amazonaws.com/<bucket>/users/<ns>/<id>`
|
|
43
|
+
*/
|
|
44
|
+
constructor(baseUrl: string, credentials: S3Credentials, region: string);
|
|
45
|
+
private url;
|
|
46
|
+
get(key: string, range?: {
|
|
47
|
+
start: number;
|
|
48
|
+
end: number;
|
|
49
|
+
}): Promise<{
|
|
50
|
+
bytes: Uint8Array;
|
|
51
|
+
etag?: string;
|
|
52
|
+
}>;
|
|
53
|
+
head(key: string): Promise<{
|
|
54
|
+
exists?: boolean;
|
|
55
|
+
etag?: string;
|
|
56
|
+
size?: number;
|
|
57
|
+
}>;
|
|
58
|
+
/**
|
|
59
|
+
* Conditional PUT. DreamDB publishes refs with a compare-and-swap, so the
|
|
60
|
+
* precondition failures are expected control flow, not errors:
|
|
61
|
+
* 412 → someone else advanced the ref → casFailed
|
|
62
|
+
* 412/409 under If-None-Match: * → the object already exists
|
|
63
|
+
*/
|
|
64
|
+
put(key: string, bytes: Uint8Array, opts?: {
|
|
65
|
+
ifMatch?: string;
|
|
66
|
+
ifNoneMatchStar?: boolean;
|
|
67
|
+
}): Promise<{
|
|
68
|
+
status: PutStatus;
|
|
69
|
+
etag?: string;
|
|
70
|
+
}>;
|
|
71
|
+
/** Keys under `prefix`, relative to the dataset base (ListObjectsV2). */
|
|
72
|
+
list(prefix: string): Promise<string[]>;
|
|
73
|
+
}
|