@dreamlake/dreamlake-cli 0.88.0 → 0.89.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/access.d.ts +83 -0
- package/dist/api/access.js +77 -0
- package/dist/api/agent.d.ts +156 -0
- package/dist/api/agent.js +111 -0
- package/dist/api/browser.d.ts +90 -0
- package/dist/api/browser.js +102 -0
- package/dist/api/client.d.ts +53 -0
- package/dist/api/client.js +102 -0
- package/dist/api/common.d.ts +28 -0
- package/dist/api/common.js +16 -0
- package/dist/api/content.d.ts +195 -0
- package/dist/api/content.js +147 -0
- package/dist/api/execution.d.ts +84 -0
- package/dist/api/execution.js +73 -0
- package/dist/api/http.d.ts +19 -0
- package/dist/api/http.js +66 -0
- package/dist/api/index.d.ts +2330 -0
- package/dist/api/index.js +114 -0
- package/dist/api/infra.d.ts +2 -0
- package/dist/api/infra.js +2 -0
- package/dist/api/jobs.d.ts +2 -0
- package/dist/api/jobs.js +2 -0
- package/dist/api/layout.d.ts +2 -0
- package/dist/api/layout.js +2 -0
- package/dist/api/library-edit.d.ts +3 -0
- package/dist/api/library-edit.js +3 -0
- package/dist/api/library-manifest.d.ts +2 -0
- package/dist/api/library-manifest.js +2 -0
- package/dist/api/note-character-plan.d.ts +2 -0
- package/dist/api/note-character-plan.js +2 -0
- package/dist/api/note-editing.d.ts +2 -0
- package/dist/api/note-editing.js +2 -0
- package/dist/api/note-edits.d.ts +21 -0
- package/dist/api/note-edits.js +10 -0
- package/dist/api/note-events.d.ts +5 -0
- package/dist/api/note-events.js +11 -0
- package/dist/api/note-files.d.ts +75 -0
- package/dist/api/note-files.js +65 -0
- package/dist/api/note-html-read.d.ts +2 -0
- package/dist/api/note-html-read.js +2 -0
- package/dist/api/note-html.d.ts +2 -0
- package/dist/api/note-html.js +2 -0
- package/dist/api/note-toc.d.ts +2 -0
- package/dist/api/note-toc.js +2 -0
- package/dist/api/notes.d.ts +304 -0
- package/dist/api/notes.js +181 -0
- package/dist/api/open-resources.d.ts +27 -0
- package/dist/api/open-resources.js +102 -0
- package/dist/api/ownership.d.ts +12 -0
- package/dist/api/ownership.js +24 -0
- package/dist/api/platform.d.ts +83 -0
- package/dist/api/platform.js +56 -0
- package/dist/api/presigned-transfer.d.ts +2 -0
- package/dist/api/presigned-transfer.js +2 -0
- package/dist/api/registered-runs.d.ts +48 -0
- package/dist/api/registered-runs.js +33 -0
- package/dist/api/sessions.d.ts +299 -0
- package/dist/api/sessions.js +105 -0
- package/dist/api/storage.d.ts +46 -0
- package/dist/api/storage.js +34 -0
- package/dist/api/streams.d.ts +18 -0
- package/dist/api/streams.js +70 -0
- package/dist/api/vault.d.ts +93 -0
- package/dist/api/vault.js +79 -0
- package/dist/api/workspace.d.ts +180 -0
- package/dist/api/workspace.js +146 -0
- package/dist/cli/agent-identity.d.ts +4 -0
- package/dist/cli/agent-identity.js +21 -0
- package/dist/cli/agents/identity.d.ts +50 -0
- package/dist/cli/agents/identity.js +105 -0
- package/dist/cli/agents/index.d.ts +31 -0
- package/dist/cli/agents/index.js +580 -0
- package/dist/cli/agents/manage.d.ts +43 -0
- package/dist/cli/agents/manage.js +376 -0
- package/dist/cli/agents/spec.d.ts +104 -0
- package/dist/cli/agents/spec.js +280 -0
- package/dist/cli/agents/web.d.ts +7 -0
- package/dist/cli/agents/web.js +109 -0
- package/dist/cli/artifact/incremental.d.ts +21 -0
- package/dist/cli/artifact/incremental.js +80 -0
- package/dist/cli/artifact/index.d.ts +52 -0
- package/dist/cli/artifact/index.js +405 -0
- package/dist/cli/artifact/meta.d.ts +69 -0
- package/dist/cli/artifact/meta.js +189 -0
- package/dist/cli/auth/commands.d.ts +29 -0
- package/dist/cli/auth/commands.js +327 -0
- package/dist/cli/auth/constants.d.ts +13 -0
- package/dist/cli/auth/constants.js +18 -0
- package/dist/cli/auth/credentials.d.ts +37 -0
- package/dist/cli/auth/credentials.js +155 -0
- package/dist/cli/auth/device-flow.d.ts +32 -0
- package/dist/cli/auth/device-flow.js +154 -0
- package/dist/cli/auth/device-secret.d.ts +6 -0
- package/dist/cli/auth/device-secret.js +26 -0
- package/dist/cli/auth/machine.d.ts +15 -0
- package/dist/cli/auth/machine.js +126 -0
- package/dist/cli/auth/materialized.d.ts +32 -0
- package/dist/cli/auth/materialized.js +102 -0
- package/dist/cli/auth/migrate.d.ts +10 -0
- package/dist/cli/auth/migrate.js +75 -0
- package/dist/cli/client.d.ts +102 -0
- package/dist/cli/client.js +206 -0
- package/dist/cli/collection-list.d.ts +3 -0
- package/dist/cli/collection-list.js +25 -0
- package/dist/cli/compat/ctx.d.ts +26 -0
- package/dist/cli/compat/ctx.js +76 -0
- package/dist/cli/compat/dreamdb-wasm-embedded.d.ts +2 -0
- package/dist/cli/compat/dreamdb-wasm-embedded.js +8 -0
- package/dist/cli/compat/dreamdb-wasm.d.ts +1 -0
- package/dist/cli/compat/dreamdb-wasm.js +26 -0
- package/dist/cli/compat/dreamdb.d.ts +112 -0
- package/dist/cli/compat/dreamdb.js +219 -0
- package/dist/cli/compat/flags.d.ts +2 -0
- package/dist/cli/compat/flags.js +13 -0
- package/dist/cli/compat/http.d.ts +23 -0
- package/dist/cli/compat/http.js +137 -0
- package/dist/cli/compat/presignedTransfer.d.ts +139 -0
- package/dist/cli/compat/presignedTransfer.js +580 -0
- package/dist/cli/compat/s3-backend.d.ts +73 -0
- package/dist/cli/compat/s3-backend.js +132 -0
- package/dist/cli/config.d.ts +22 -0
- package/dist/cli/config.js +96 -0
- package/dist/cli/connection/dto.d.ts +159 -0
- package/dist/cli/connection/dto.js +497 -0
- package/dist/cli/connection/index.d.ts +50 -0
- package/dist/cli/connection/index.js +413 -0
- package/dist/cli/connection/oauth.d.ts +23 -0
- package/dist/cli/connection/oauth.js +290 -0
- package/dist/cli/connection/write.d.ts +47 -0
- package/dist/cli/connection/write.js +573 -0
- package/dist/cli/create/index.d.ts +19 -0
- package/dist/cli/create/index.js +209 -0
- package/dist/cli/declare/index.d.ts +29 -0
- package/dist/cli/declare/index.js +304 -0
- package/dist/cli/delete/index.d.ts +18 -0
- package/dist/cli/delete/index.js +232 -0
- package/dist/cli/download/index.d.ts +9 -0
- package/dist/cli/download/index.js +128 -0
- package/dist/cli/env/compose.d.ts +88 -0
- package/dist/cli/env/compose.js +402 -0
- package/dist/cli/env/index.d.ts +176 -0
- package/dist/cli/env/index.js +1011 -0
- package/dist/cli/experiments/index.d.ts +11 -0
- package/dist/cli/experiments/index.js +168 -0
- package/dist/cli/glob.d.ts +1 -0
- package/dist/cli/glob.js +45 -0
- package/dist/cli/graphql-helpers.d.ts +23 -0
- package/dist/cli/graphql-helpers.js +42 -0
- package/dist/cli/graphql.d.ts +10 -0
- package/dist/cli/graphql.js +47 -0
- package/dist/cli/help-examples.generated.d.ts +1 -0
- package/dist/cli/help-examples.generated.js +192 -0
- package/dist/cli/helpers.d.ts +8 -0
- package/dist/cli/helpers.js +106 -0
- package/dist/cli/hosts/api.d.ts +29 -0
- package/dist/cli/hosts/api.js +54 -0
- package/dist/cli/hosts/bootstrap-source.d.ts +1 -0
- package/dist/cli/hosts/bootstrap-source.js +2 -0
- package/dist/cli/hosts/catalog.d.ts +26 -0
- package/dist/cli/hosts/catalog.js +55 -0
- package/dist/cli/hosts/config.d.ts +45 -0
- package/dist/cli/hosts/config.js +179 -0
- package/dist/cli/hosts/credentials.d.ts +19 -0
- package/dist/cli/hosts/credentials.js +173 -0
- package/dist/cli/hosts/enroll.d.ts +17 -0
- package/dist/cli/hosts/enroll.js +109 -0
- package/dist/cli/hosts/enrollments.d.ts +25 -0
- package/dist/cli/hosts/enrollments.js +54 -0
- package/dist/cli/hosts/index.d.ts +6 -0
- package/dist/cli/hosts/index.js +279 -0
- package/dist/cli/hosts/inventory.d.ts +40 -0
- package/dist/cli/hosts/inventory.js +91 -0
- package/dist/cli/hosts/lifecycle.d.ts +15 -0
- package/dist/cli/hosts/lifecycle.js +108 -0
- package/dist/cli/hosts/metrics-source.d.ts +12 -0
- package/dist/cli/hosts/metrics-source.js +89 -0
- package/dist/cli/hosts/observations.d.ts +39 -0
- package/dist/cli/hosts/observations.js +102 -0
- package/dist/cli/hosts/remote.d.ts +5 -0
- package/dist/cli/hosts/remote.js +66 -0
- package/dist/cli/hosts/ssh.d.ts +63 -0
- package/dist/cli/hosts/ssh.js +276 -0
- package/dist/cli/index.d.ts +1 -0
- package/dist/cli/index.js +288 -0
- package/dist/cli/infra/api.d.ts +95 -0
- package/dist/cli/infra/api.js +291 -0
- package/dist/cli/infra/index.d.ts +5 -0
- package/dist/cli/infra/index.js +102 -0
- package/dist/cli/infra/move.d.ts +15 -0
- package/dist/cli/infra/move.js +123 -0
- package/dist/cli/infra/transfer.d.ts +15 -0
- package/dist/cli/infra/transfer.js +280 -0
- package/dist/cli/jobs/api.d.ts +69 -0
- package/dist/cli/jobs/api.js +121 -0
- package/dist/cli/jobs/index.d.ts +5 -0
- package/dist/cli/jobs/index.js +140 -0
- package/dist/cli/lakeshore/index.d.ts +98 -0
- package/dist/cli/lakeshore/index.js +502 -0
- package/dist/cli/launch/index.d.ts +43 -0
- package/dist/cli/launch/index.js +270 -0
- package/dist/cli/layout/index.d.ts +2 -0
- package/dist/cli/layout/index.js +44 -0
- package/dist/cli/layout/transport.d.ts +12 -0
- package/dist/cli/layout/transport.js +70 -0
- package/dist/cli/library/add.d.ts +60 -0
- package/dist/cli/library/add.js +348 -0
- package/dist/cli/library/cache.d.ts +26 -0
- package/dist/cli/library/cache.js +88 -0
- package/dist/cli/library/config.d.ts +75 -0
- package/dist/cli/library/config.js +198 -0
- package/dist/cli/library/digest.d.ts +5 -0
- package/dist/cli/library/digest.js +19 -0
- package/dist/cli/library/discover.d.ts +87 -0
- package/dist/cli/library/discover.js +534 -0
- package/dist/cli/library/freshness.d.ts +35 -0
- package/dist/cli/library/freshness.js +109 -0
- package/dist/cli/library/generation.d.ts +35 -0
- package/dist/cli/library/generation.js +211 -0
- package/dist/cli/library/index.d.ts +2 -0
- package/dist/cli/library/index.js +36 -0
- package/dist/cli/library/info.d.ts +69 -0
- package/dist/cli/library/info.js +311 -0
- package/dist/cli/library/list.d.ts +33 -0
- package/dist/cli/library/list.js +86 -0
- package/dist/cli/library/manifest.d.ts +81 -0
- package/dist/cli/library/manifest.js +200 -0
- package/dist/cli/library/manifestCache.d.ts +45 -0
- package/dist/cli/library/manifestCache.js +169 -0
- package/dist/cli/library/pull.d.ts +30 -0
- package/dist/cli/library/pull.js +438 -0
- package/dist/cli/library/push.d.ts +72 -0
- package/dist/cli/library/push.js +642 -0
- package/dist/cli/library/rm.d.ts +31 -0
- package/dist/cli/library/rm.js +200 -0
- package/dist/cli/library/search.d.ts +18 -0
- package/dist/cli/library/search.js +181 -0
- package/dist/cli/library/stat.d.ts +15 -0
- package/dist/cli/library/stat.js +127 -0
- package/dist/cli/library/tools.d.ts +104 -0
- package/dist/cli/library/tools.js +163 -0
- package/dist/cli/library/wire.d.ts +47 -0
- package/dist/cli/library/wire.js +120 -0
- package/dist/cli/list/index.d.ts +26 -0
- package/dist/cli/list/index.js +256 -0
- package/dist/cli/notes/character-plan.d.ts +22 -0
- package/dist/cli/notes/character-plan.js +232 -0
- package/dist/cli/notes/document.d.ts +73 -0
- package/dist/cli/notes/document.js +306 -0
- package/dist/cli/notes/edit-commands.d.ts +106 -0
- package/dist/cli/notes/edit-commands.js +623 -0
- package/dist/cli/notes/editing.d.ts +81 -0
- package/dist/cli/notes/editing.js +305 -0
- package/dist/cli/notes/events.d.ts +36 -0
- package/dist/cli/notes/events.js +54 -0
- package/dist/cli/notes/files-commands.d.ts +120 -0
- package/dist/cli/notes/files-commands.js +554 -0
- package/dist/cli/notes/html-read.d.ts +9 -0
- package/dist/cli/notes/html-read.js +90 -0
- package/dist/cli/notes/html.d.ts +73 -0
- package/dist/cli/notes/html.js +381 -0
- package/dist/cli/notes/index.d.ts +123 -0
- package/dist/cli/notes/index.js +1303 -0
- package/dist/cli/notes/linger.d.ts +59 -0
- package/dist/cli/notes/linger.js +339 -0
- package/dist/cli/notes/media-commands.d.ts +5 -0
- package/dist/cli/notes/media-commands.js +51 -0
- package/dist/cli/notes/move-command.d.ts +83 -0
- package/dist/cli/notes/move-command.js +259 -0
- package/dist/cli/notes/rm-command.d.ts +22 -0
- package/dist/cli/notes/rm-command.js +110 -0
- package/dist/cli/notes/select-command.d.ts +12 -0
- package/dist/cli/notes/select-command.js +84 -0
- package/dist/cli/notes/suggest-diff.d.ts +56 -0
- package/dist/cli/notes/suggest-diff.js +311 -0
- package/dist/cli/notes/toc.d.ts +38 -0
- package/dist/cli/notes/toc.js +144 -0
- package/dist/cli/notes/transfer-command.d.ts +6 -0
- package/dist/cli/notes/transfer-command.js +9 -0
- package/dist/cli/notes/version-commands.d.ts +19 -0
- package/dist/cli/notes/version-commands.js +178 -0
- package/dist/cli/nymph/index.d.ts +74 -0
- package/dist/cli/nymph/index.js +349 -0
- package/dist/cli/org/index.d.ts +34 -0
- package/dist/cli/org/index.js +341 -0
- package/dist/cli/ownership/index.d.ts +12 -0
- package/dist/cli/ownership/index.js +122 -0
- package/dist/cli/ownership/metadata-transfer.d.ts +20 -0
- package/dist/cli/ownership/metadata-transfer.js +159 -0
- package/dist/cli/page/index.d.ts +58 -0
- package/dist/cli/page/index.js +387 -0
- package/dist/cli/pipeline/index.d.ts +69 -0
- package/dist/cli/pipeline/index.js +604 -0
- package/dist/cli/progress.d.ts +60 -0
- package/dist/cli/progress.js +276 -0
- package/dist/cli/project/attachments.d.ts +10 -0
- package/dist/cli/project/attachments.js +39 -0
- package/dist/cli/project/grants.d.ts +10 -0
- package/dist/cli/project/grants.js +39 -0
- package/dist/cli/project/index.d.ts +8 -0
- package/dist/cli/project/index.js +92 -0
- package/dist/cli/project/move.d.ts +57 -0
- package/dist/cli/project/move.js +242 -0
- package/dist/cli/project/output.d.ts +2 -0
- package/dist/cli/project/output.js +29 -0
- package/dist/cli/project/stream.d.ts +55 -0
- package/dist/cli/project/stream.js +207 -0
- package/dist/cli/project/transfer-preview.d.ts +8 -0
- package/dist/cli/project/transfer-preview.js +10 -0
- package/dist/cli/prompt.d.ts +1 -0
- package/dist/cli/prompt.js +17 -0
- package/dist/cli/provider/dreamdir.d.ts +52 -0
- package/dist/cli/provider/dreamdir.js +138 -0
- package/dist/cli/provider/index.d.ts +4 -0
- package/dist/cli/provider/index.js +419 -0
- package/dist/cli/provider/jump-worker.d.ts +9 -0
- package/dist/cli/provider/jump-worker.js +141 -0
- package/dist/cli/provider/lifecycle.d.ts +149 -0
- package/dist/cli/provider/lifecycle.js +1390 -0
- package/dist/cli/provider/machine.d.ts +17 -0
- package/dist/cli/provider/machine.js +116 -0
- package/dist/cli/provider/ssh.d.ts +105 -0
- package/dist/cli/provider/ssh.js +305 -0
- package/dist/cli/provider/terraform.d.ts +140 -0
- package/dist/cli/provider/terraform.js +288 -0
- package/dist/cli/providers/index.d.ts +18 -0
- package/dist/cli/providers/index.js +422 -0
- package/dist/cli/python.d.ts +61 -0
- package/dist/cli/python.js +255 -0
- package/dist/cli/queues/index.d.ts +33 -0
- package/dist/cli/queues/index.js +248 -0
- package/dist/cli/queues/slurm.d.ts +10 -0
- package/dist/cli/queues/slurm.js +134 -0
- package/dist/cli/resources.d.ts +72 -0
- package/dist/cli/resources.js +137 -0
- package/dist/cli/run/index.d.ts +15 -0
- package/dist/cli/run/index.js +163 -0
- package/dist/cli/run/input.d.ts +34 -0
- package/dist/cli/run/input.js +135 -0
- package/dist/cli/run/recovery.d.ts +2 -0
- package/dist/cli/run/recovery.js +18 -0
- package/dist/cli/run/setup.d.ts +3 -0
- package/dist/cli/run/setup.js +62 -0
- package/dist/cli/runconfig/index.d.ts +12 -0
- package/dist/cli/runconfig/index.js +130 -0
- package/dist/cli/runnable/index.d.ts +35 -0
- package/dist/cli/runnable/index.js +575 -0
- package/dist/cli/runnable/path.d.ts +115 -0
- package/dist/cli/runnable/path.js +288 -0
- package/dist/cli/runnable/run.d.ts +91 -0
- package/dist/cli/runnable/run.js +239 -0
- package/dist/cli/runnable/upload.d.ts +43 -0
- package/dist/cli/runnable/upload.js +342 -0
- package/dist/cli/runnable/worker.d.ts +1 -0
- package/dist/cli/runnable/worker.js +44 -0
- package/dist/cli/schemas/workflow-spec.schema.json +335 -0
- package/dist/cli/self-update/index.d.ts +5 -0
- package/dist/cli/self-update/index.js +107 -0
- package/dist/cli/self-update/install.d.ts +60 -0
- package/dist/cli/self-update/install.js +428 -0
- package/dist/cli/self-update/paths.d.ts +40 -0
- package/dist/cli/self-update/paths.js +87 -0
- package/dist/cli/self-update/releases.d.ts +41 -0
- package/dist/cli/self-update/releases.js +153 -0
- package/dist/cli/self-update/settings.d.ts +16 -0
- package/dist/cli/self-update/settings.js +68 -0
- package/dist/cli/self-update/update.d.ts +18 -0
- package/dist/cli/self-update/update.js +241 -0
- package/dist/cli/service-accounts/index.d.ts +3 -0
- package/dist/cli/service-accounts/index.js +149 -0
- package/dist/cli/session-affiliation.d.ts +26 -0
- package/dist/cli/session-affiliation.js +135 -0
- package/dist/cli/sessions/affiliation.d.ts +8 -0
- package/dist/cli/sessions/affiliation.js +66 -0
- package/dist/cli/sessions/api.d.ts +55 -0
- package/dist/cli/sessions/api.js +88 -0
- package/dist/cli/sessions/control.d.ts +6 -0
- package/dist/cli/sessions/control.js +159 -0
- package/dist/cli/sessions/conversation.d.ts +15 -0
- package/dist/cli/sessions/conversation.js +118 -0
- package/dist/cli/sessions/index.d.ts +2 -0
- package/dist/cli/sessions/index.js +209 -0
- package/dist/cli/sessions/managed.d.ts +39 -0
- package/dist/cli/sessions/managed.js +63 -0
- package/dist/cli/sessions/redaction.d.ts +2 -0
- package/dist/cli/sessions/redaction.js +11 -0
- package/dist/cli/sessions/runner/exec.d.ts +108 -0
- package/dist/cli/sessions/runner/exec.js +335 -0
- package/dist/cli/sessions/runner/journal.d.ts +26 -0
- package/dist/cli/sessions/runner/journal.js +49 -0
- package/dist/cli/sessions/runner/native.d.ts +48 -0
- package/dist/cli/sessions/runner/native.js +170 -0
- package/dist/cli/sessions/runner/pod.d.ts +30 -0
- package/dist/cli/sessions/runner/pod.js +129 -0
- package/dist/cli/sessions/runner/serve.d.ts +85 -0
- package/dist/cli/sessions/runner/serve.js +558 -0
- package/dist/cli/sessions/sync.d.ts +47 -0
- package/dist/cli/sessions/sync.js +80 -0
- package/dist/cli/sessions/transport.d.ts +14 -0
- package/dist/cli/sessions/transport.js +116 -0
- package/dist/cli/share.d.ts +15 -0
- package/dist/cli/share.js +139 -0
- package/dist/cli/skill/catalog.d.ts +141 -0
- package/dist/cli/skill/catalog.js +442 -0
- package/dist/cli/skill/files.d.ts +89 -0
- package/dist/cli/skill/files.js +264 -0
- package/dist/cli/skill/index.d.ts +268 -0
- package/dist/cli/skill/index.js +1216 -0
- package/dist/cli/skill/manifest.d.ts +30 -0
- package/dist/cli/skill/manifest.js +101 -0
- package/dist/cli/skill/prefs.d.ts +9 -0
- package/dist/cli/skill/prefs.js +32 -0
- package/dist/cli/skill/reminders.d.ts +15 -0
- package/dist/cli/skill/reminders.js +105 -0
- package/dist/cli/skill/tar.d.ts +17 -0
- package/dist/cli/skill/tar.js +195 -0
- package/dist/cli/snapshot/index.d.ts +24 -0
- package/dist/cli/snapshot/index.js +146 -0
- package/dist/cli/source/access.d.ts +45 -0
- package/dist/cli/source/access.js +563 -0
- package/dist/cli/source/control.d.ts +103 -0
- package/dist/cli/source/control.js +546 -0
- package/dist/cli/source/delete.d.ts +89 -0
- package/dist/cli/source/delete.js +463 -0
- package/dist/cli/source/index.d.ts +288 -0
- package/dist/cli/source/index.js +1348 -0
- package/dist/cli/source/share.d.ts +65 -0
- package/dist/cli/source/share.js +530 -0
- package/dist/cli/source/upload.d.ts +62 -0
- package/dist/cli/source/upload.js +258 -0
- package/dist/cli/source/write.d.ts +77 -0
- package/dist/cli/source/write.js +335 -0
- package/dist/cli/spawn.d.ts +28 -0
- package/dist/cli/spawn.js +81 -0
- package/dist/cli/target.d.ts +22 -0
- package/dist/cli/target.js +85 -0
- package/dist/cli/tasks/index.d.ts +38 -0
- package/dist/cli/tasks/index.js +365 -0
- package/dist/cli/team/index.d.ts +37 -0
- package/dist/cli/team/index.js +407 -0
- package/dist/cli/telemetry.d.ts +23 -0
- package/dist/cli/telemetry.js +262 -0
- package/dist/cli/telemetryJournal.d.ts +26 -0
- package/dist/cli/telemetryJournal.js +104 -0
- package/dist/cli/update/index.d.ts +21 -0
- package/dist/cli/update/index.js +284 -0
- package/dist/cli/upload/index.d.ts +11 -0
- package/dist/cli/upload/index.js +261 -0
- package/dist/cli/upload/kinds.d.ts +10 -0
- package/dist/cli/upload/kinds.js +85 -0
- package/dist/cli/upload/multipart.d.ts +21 -0
- package/dist/cli/upload/multipart.js +248 -0
- package/dist/cli/vault/affected-preview.d.ts +15 -0
- package/dist/cli/vault/affected-preview.js +38 -0
- package/dist/cli/vault/cloud.d.ts +55 -0
- package/dist/cli/vault/cloud.js +287 -0
- package/dist/cli/vault/format.d.ts +18 -0
- package/dist/cli/vault/format.js +104 -0
- package/dist/cli/vault/hostCredentials.d.ts +21 -0
- package/dist/cli/vault/hostCredentials.js +71 -0
- package/dist/cli/vault/hostSupersessions.d.ts +18 -0
- package/dist/cli/vault/hostSupersessions.js +68 -0
- package/dist/cli/vault/import.d.ts +3 -0
- package/dist/cli/vault/import.js +79 -0
- package/dist/cli/vault/index.d.ts +14 -0
- package/dist/cli/vault/index.js +225 -0
- package/dist/cli/vault/keys.d.ts +5 -0
- package/dist/cli/vault/keys.js +165 -0
- package/dist/cli/vault/kms.d.ts +6 -0
- package/dist/cli/vault/kms.js +169 -0
- package/dist/cli/vault/list.d.ts +13 -0
- package/dist/cli/vault/list.js +60 -0
- package/dist/cli/vault/listView.d.ts +21 -0
- package/dist/cli/vault/listView.js +123 -0
- package/dist/cli/vault/managementMetadata.d.ts +2 -0
- package/dist/cli/vault/managementMetadata.js +23 -0
- package/dist/cli/vault/mutations.d.ts +8 -0
- package/dist/cli/vault/mutations.js +151 -0
- package/dist/cli/vault/otp.d.ts +6 -0
- package/dist/cli/vault/otp.js +62 -0
- package/dist/cli/vault/otpIntent.d.ts +23 -0
- package/dist/cli/vault/otpIntent.js +76 -0
- package/dist/cli/vault/passImport.d.ts +25 -0
- package/dist/cli/vault/passImport.js +129 -0
- package/dist/cli/vault/passOtp.d.ts +51 -0
- package/dist/cli/vault/passOtp.js +101 -0
- package/dist/cli/vault/passStore.d.ts +26 -0
- package/dist/cli/vault/passStore.js +134 -0
- package/dist/cli/vault/passwordCommand.d.ts +39 -0
- package/dist/cli/vault/passwordCommand.js +101 -0
- package/dist/cli/vault/passwordProbe.d.ts +27 -0
- package/dist/cli/vault/passwordProbe.js +282 -0
- package/dist/cli/vault/passwordRotation.d.ts +9 -0
- package/dist/cli/vault/passwordRotation.js +273 -0
- package/dist/cli/vault/rotationApi.d.ts +13 -0
- package/dist/cli/vault/rotationApi.js +58 -0
- package/dist/cli/vault/rotationCommand.d.ts +2 -0
- package/dist/cli/vault/rotationCommand.js +75 -0
- package/dist/cli/vault/rotationEngine.d.ts +19 -0
- package/dist/cli/vault/rotationEngine.js +65 -0
- package/dist/cli/vault/rotationHelperSource.d.ts +2 -0
- package/dist/cli/vault/rotationHelperSource.js +3 -0
- package/dist/cli/vault/rotationInput.d.ts +11 -0
- package/dist/cli/vault/rotationInput.js +48 -0
- package/dist/cli/vault/rotationJournal.d.ts +16 -0
- package/dist/cli/vault/rotationJournal.js +183 -0
- package/dist/cli/vault/rotationKeys.d.ts +4 -0
- package/dist/cli/vault/rotationKeys.js +48 -0
- package/dist/cli/vault/rotationPrepare.d.ts +17 -0
- package/dist/cli/vault/rotationPrepare.js +108 -0
- package/dist/cli/vault/rotationRemote.d.ts +33 -0
- package/dist/cli/vault/rotationRemote.js +156 -0
- package/dist/cli/vault/rotationState.d.ts +42 -0
- package/dist/cli/vault/rotationState.js +115 -0
- package/dist/cli/vault/rotationTransport.d.ts +18 -0
- package/dist/cli/vault/rotationTransport.js +71 -0
- package/dist/cli/vault/sshSource.d.ts +18 -0
- package/dist/cli/vault/sshSource.js +196 -0
- package/dist/cli/vault/sshSync.d.ts +17 -0
- package/dist/cli/vault/sshSync.js +224 -0
- package/dist/cli/vault/sshTransfer.d.ts +22 -0
- package/dist/cli/vault/sshTransfer.js +140 -0
- package/dist/cli/vault/terraformState.d.ts +66 -0
- package/dist/cli/vault/terraformState.js +252 -0
- package/dist/cli/vault/terraformStateCommand.d.ts +2 -0
- package/dist/cli/vault/terraformStateCommand.js +61 -0
- package/dist/cli/vault/tree.d.ts +43 -0
- package/dist/cli/vault/tree.js +292 -0
- package/dist/cli/version.d.ts +1 -0
- package/dist/cli/version.js +9 -0
- package/dist/cli/workflow/index.d.ts +76 -0
- package/dist/cli/workflow/index.js +826 -0
- package/dist/cli/workflow/validate.d.ts +19 -0
- package/dist/cli/workflow/validate.js +353 -0
- package/dist/package.json +1 -0
- package/package.json +28 -10
|
@@ -0,0 +1,1348 @@
|
|
|
1
|
+
import { collectionList } from "../collection-list.js";
|
|
2
|
+
// `dreamlake source ...` — read data out of connected sources (S3 and Hugging
|
|
3
|
+
// Face; those are the two connectors Source v1 has).
|
|
4
|
+
//
|
|
5
|
+
// A Source is a *named*, credentialed connection stored server-side; the CLI
|
|
6
|
+
// never holds bucket keys. Every command except `list` addresses one by name,
|
|
7
|
+
// because a namespace routinely holds several sources from the same provider —
|
|
8
|
+
// two S3 buckets, a staging and a prod — so the provider alone cannot identify
|
|
9
|
+
// one. `list` is how you discover the names.
|
|
10
|
+
//
|
|
11
|
+
// source list → discover source names
|
|
12
|
+
// source browse --source <name> → list paths inside one
|
|
13
|
+
// source fetch --source <name> → mint a short-lived read capability
|
|
14
|
+
// source download --source <name> → stream bytes to disk
|
|
15
|
+
//
|
|
16
|
+
// `fetch` prints a URL, but a capability is not always JUST a URL: it may
|
|
17
|
+
// carry headers that are part of what was signed. When it does, the bare-URL
|
|
18
|
+
// output is refused and `--json` carries the whole thing.
|
|
19
|
+
//
|
|
20
|
+
// Server endpoints used — Source v1 (all read-only, all gated on Source USE):
|
|
21
|
+
// GET /namespaces/:ns/sources → {items, nextCursor, …}
|
|
22
|
+
// GET /namespaces/:ns/sources/:id/browse → {entries, nextCursor, …}
|
|
23
|
+
// POST /namespaces/:ns/sources/:id/read-capabilities → a read capability
|
|
24
|
+
//
|
|
25
|
+
// The v1 surface replaces the legacy one wholesale; the two are mutually
|
|
26
|
+
// exclusive server-side, so there is no dual-path fallback here. The legacy
|
|
27
|
+
// `GET /presign` this used to call does not exist under v1 and answered 404.
|
|
28
|
+
//
|
|
29
|
+
// A capability is SHORT-LIVED and is a bearer secret: its `url` and
|
|
30
|
+
// `allowedHeaders` together are the credential. They are printed only when the
|
|
31
|
+
// user asked for them, and never logged.
|
|
32
|
+
//
|
|
33
|
+
// Capabilities are minted one object at a time through a small concurrency
|
|
34
|
+
// pool — each is a signing round trip that re-resolves the current winner.
|
|
35
|
+
import { createWriteStream, mkdirSync, renameSync, rmSync, statSync } from "node:fs";
|
|
36
|
+
import path from "node:path";
|
|
37
|
+
import { Readable, Transform } from "node:stream";
|
|
38
|
+
import { pipeline } from "node:stream/promises";
|
|
39
|
+
import { HttpError, requestJson, requestRaw } from "../client.js";
|
|
40
|
+
import { resolveNamespace, resolveRemote, resolveToken } from "../config.js";
|
|
41
|
+
import { emitJson, fail, renderTable } from "../helpers.js";
|
|
42
|
+
import { registerSourceWriteCommands } from "./write.js";
|
|
43
|
+
import { registerSourceUploadCommand } from "./upload.js";
|
|
44
|
+
import { registerSourceDeleteCommands } from "./delete.js";
|
|
45
|
+
import { registerSourceControlCommands } from "./control.js";
|
|
46
|
+
import { registerSourceAccessCommands } from "./access.js";
|
|
47
|
+
import { registerSourceShareCommands } from "./share.js";
|
|
48
|
+
import { BarProgress } from "../progress.js";
|
|
49
|
+
/** Stop runaway recursion into a whole bucket. Surfaced, never silent. */
|
|
50
|
+
const MAX_ENTRIES = 10000;
|
|
51
|
+
/**
|
|
52
|
+
* Pages one cursor walk will follow before it declares the peer broken.
|
|
53
|
+
*
|
|
54
|
+
* Not a limit on how much data is supported — every real listing terminates
|
|
55
|
+
* long before this. It exists because an opaque cursor, unlike a page number
|
|
56
|
+
* against a known total, gives a client no way to tell "more results" from
|
|
57
|
+
* "this server will never stop".
|
|
58
|
+
*/
|
|
59
|
+
const MAX_PAGES = 1000;
|
|
60
|
+
/** Parallel mint / download requests. Modest on purpose — every one of these
|
|
61
|
+
* is a signing round trip that re-resolves the winner on the server, and
|
|
62
|
+
* Hugging Face rate limits the upstream API behind it. */
|
|
63
|
+
const CONCURRENCY = 6;
|
|
64
|
+
// ─── shared context resolution ────────────────────────────────────────────────
|
|
65
|
+
export async function ctx(nsFlag) {
|
|
66
|
+
const token = resolveToken();
|
|
67
|
+
if (!token) {
|
|
68
|
+
fail("not authenticated — run 'dreamlake login' first");
|
|
69
|
+
return null;
|
|
70
|
+
}
|
|
71
|
+
const remote = resolveRemote();
|
|
72
|
+
const ns = await resolveNamespace(nsFlag, { token, remote });
|
|
73
|
+
if (!ns) {
|
|
74
|
+
fail("could not resolve namespace — run 'dreamlake login' or pass --namespace");
|
|
75
|
+
return null;
|
|
76
|
+
}
|
|
77
|
+
return { token, remote, ns };
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* One URL path segment, encoded as one.
|
|
81
|
+
*
|
|
82
|
+
* Namespace slugs and Source ids reach this module from a command line — the
|
|
83
|
+
* slug directly from `--namespace`, the id from a name the caller typed — and a
|
|
84
|
+
* `/` or a `?` in either is not a bad argument, it is a DIFFERENT request.
|
|
85
|
+
* `--namespace 'a/../../b'` interpolated raw addresses another namespace's
|
|
86
|
+
* route while looking like a typo. Encoding keeps a malformed argument a 404.
|
|
87
|
+
*
|
|
88
|
+
* This applies to PATH parameters only. An object's canonical path is not one:
|
|
89
|
+
* §7 puts it in the query string or the body, where it travels as a value and
|
|
90
|
+
* is encoded by the query builder. Nothing here normalizes, trims or re-splits
|
|
91
|
+
* a canonical path — the server owns what a valid one is, and a client that
|
|
92
|
+
* tidied it would change which object was addressed.
|
|
93
|
+
*/
|
|
94
|
+
export function seg(v) {
|
|
95
|
+
return encodeURIComponent(v);
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Turn a transport error into something a user can act on. The server's own
|
|
99
|
+
* message is usually good; these three statuses are the ones where it isn't
|
|
100
|
+
* specific enough to tell you what to do next.
|
|
101
|
+
*/
|
|
102
|
+
export function explain(err) {
|
|
103
|
+
if (err instanceof HttpError) {
|
|
104
|
+
if (err.status === 401)
|
|
105
|
+
return "not authenticated — run 'dreamlake login' first";
|
|
106
|
+
if (err.status === 403)
|
|
107
|
+
return "permission denied — you need read access to this namespace";
|
|
108
|
+
}
|
|
109
|
+
return err.message;
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Shell-quote a path for display, but only when stdout is a terminal — the
|
|
113
|
+
* same rule GNU ls follows.
|
|
114
|
+
*
|
|
115
|
+
* A path printed here is usually one the user is about to copy straight into
|
|
116
|
+
* `source fetch`, and source paths carry spaces constantly (screen recordings,
|
|
117
|
+
* anything named in a GUI). Printed bare, it pastes as two arguments and the
|
|
118
|
+
* server answers with a 400 about a path they never typed. Printed quoted, it
|
|
119
|
+
* pastes correctly. When stdout is a pipe the consumer wants the raw bytes, so
|
|
120
|
+
* nothing is added — use --print0 for paths that need to survive xargs.
|
|
121
|
+
*/
|
|
122
|
+
function displayPath(p) {
|
|
123
|
+
if (!process.stdout.isTTY)
|
|
124
|
+
return p;
|
|
125
|
+
if (!/[\s'"\\$`|&;<>()*?[\]#~!]/.test(p))
|
|
126
|
+
return p;
|
|
127
|
+
return `'${p.replace(/'/g, `'\\''`)}'`;
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Reject anything that is not already a Source v1 canonical path.
|
|
131
|
+
*
|
|
132
|
+
* REJECTS rather than repairs, which is the whole point. The server states the
|
|
133
|
+
* rule and the reason (paths.ts:30): "Deliberately does NOT repair: no
|
|
134
|
+
* collapsing of `//`, no stripping of a leading slash, no resolving of `..`."
|
|
135
|
+
* A client that quietly rewrote `/dbx/x.png` into `dbx/x.png` would address a
|
|
136
|
+
* DIFFERENT object and report success for it.
|
|
137
|
+
*
|
|
138
|
+
* Canonical form (core contract §5.1): the root is `""`, and every other path
|
|
139
|
+
* is a relative path with no leading or trailing slash, no empty segment and
|
|
140
|
+
* no `.` or `..`.
|
|
141
|
+
*
|
|
142
|
+
* This is a client-side pre-check for a better message, never the authority —
|
|
143
|
+
* the server validates every path again and is the only authority on it.
|
|
144
|
+
*
|
|
145
|
+
* The absolute-path affordance this replaces existed for Dropbox, whose API
|
|
146
|
+
* needs a leading slash. Source v1 has no Dropbox connector; the capability
|
|
147
|
+
* left with the migration. If one is ever added, mapping a canonical relative
|
|
148
|
+
* path onto the provider's absolute path is the connector's job — `/` must not
|
|
149
|
+
* leak back into the public Source path contract.
|
|
150
|
+
*/
|
|
151
|
+
function checkPath(p) {
|
|
152
|
+
if (p.startsWith("/") || p.endsWith("/")) {
|
|
153
|
+
return `Source v1 paths must be relative and must not start or end with '/': ${p}`;
|
|
154
|
+
}
|
|
155
|
+
const segments = p.split("/");
|
|
156
|
+
if (segments.some((seg) => seg === "")) {
|
|
157
|
+
return `path may not contain an empty segment: ${p}`;
|
|
158
|
+
}
|
|
159
|
+
if (segments.some((seg) => seg === ".." || seg === ".")) {
|
|
160
|
+
return `path may not contain '.' or '..' segments: ${p}`;
|
|
161
|
+
}
|
|
162
|
+
return null;
|
|
163
|
+
}
|
|
164
|
+
// ─── wire shape guards ────────────────────────────────────────────────────────
|
|
165
|
+
/**
|
|
166
|
+
* Refuse a response that is not a Source v1 response.
|
|
167
|
+
*
|
|
168
|
+
* Without this the failure is silent and worse than an error. A legacy server
|
|
169
|
+
* answers list with `{sources: [...]}`, and `res.items ?? []` turned that into
|
|
170
|
+
* an empty array — so the CLI reported "no sources in namespace 'x' — connect
|
|
171
|
+
* one in the DreamLake UI first" to a namespace full of Sources. The user is
|
|
172
|
+
* told a fact about their data that is false, and the actual cause, a server
|
|
173
|
+
* speaking the older protocol, is nowhere on screen.
|
|
174
|
+
*
|
|
175
|
+
* Browse has the same shape of bug with worse consequences: a legacy entry
|
|
176
|
+
* carries `type: "container" | "leaf"` where v1 carries `kind`, so every entry
|
|
177
|
+
* would classify as a file, and `-r` would download nothing while reporting
|
|
178
|
+
* success.
|
|
179
|
+
*
|
|
180
|
+
* So the required fields are checked, and a missing one is named. The old
|
|
181
|
+
* protocol is called out by name where its marker is recognisable, because
|
|
182
|
+
* "upgrade the server" is a different action from "report a bug".
|
|
183
|
+
*/
|
|
184
|
+
export function bad(what) {
|
|
185
|
+
throw new Error(`incompatible Source API response: ${what}`);
|
|
186
|
+
}
|
|
187
|
+
export function isObject(v) {
|
|
188
|
+
return typeof v === "object" && v !== null && !Array.isArray(v);
|
|
189
|
+
}
|
|
190
|
+
/** `nextCursor` is REQUIRED and nullable — absent is not the same as null. */
|
|
191
|
+
function checkCursor(res, where) {
|
|
192
|
+
if (!("nextCursor" in res))
|
|
193
|
+
bad(`${where} is missing 'nextCursor'`);
|
|
194
|
+
const c = res.nextCursor;
|
|
195
|
+
if (c !== null && typeof c !== "string")
|
|
196
|
+
bad(`${where} has a non-string 'nextCursor'`);
|
|
197
|
+
}
|
|
198
|
+
/**
|
|
199
|
+
* `lifecycleCounts` — required, and either three counts or an explicit null.
|
|
200
|
+
*
|
|
201
|
+
* Absent is not `null`. `null` is the server saying the collection is past what
|
|
202
|
+
* it will count within; absent would leave this client deciding what to print,
|
|
203
|
+
* and "no numbers" and "an older server" are not the same fact.
|
|
204
|
+
*/
|
|
205
|
+
function checkCounts(res, where) {
|
|
206
|
+
if (!("lifecycleCounts" in res))
|
|
207
|
+
bad(`${where} is missing 'lifecycleCounts'`);
|
|
208
|
+
const v = res.lifecycleCounts;
|
|
209
|
+
if (v === null)
|
|
210
|
+
return;
|
|
211
|
+
if (!isObject(v))
|
|
212
|
+
bad(`${where} has a 'lifecycleCounts' that is neither counts nor null`);
|
|
213
|
+
for (const k of ["all", "active", "disabled"]) {
|
|
214
|
+
const n = v[k];
|
|
215
|
+
if (typeof n !== "number" || !Number.isSafeInteger(n) || n < 0) {
|
|
216
|
+
bad(`${where} has a non-count '${k}' in 'lifecycleCounts'`);
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
/*
|
|
220
|
+
* `all === active + disabled`. §4 states it as an identity the server must
|
|
221
|
+
* satisfy, not as a hint: `all` is derived from the other two rather than
|
|
222
|
+
* queried separately, so a set that breaks it means the counting itself went
|
|
223
|
+
* wrong. Refused rather than rendered — three numbers that contradict each
|
|
224
|
+
* other cannot be acted on, and deciding which two to believe is not this
|
|
225
|
+
* client's call.
|
|
226
|
+
*/
|
|
227
|
+
const all = v.all;
|
|
228
|
+
const active = v.active;
|
|
229
|
+
const disabled = v.disabled;
|
|
230
|
+
if (all !== active + disabled) {
|
|
231
|
+
bad(`${where} has counts that do not add up: all=${all}, active=${active}, disabled=${disabled}`);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
/** A field the contract declares required AND nullable. */
|
|
235
|
+
export function nullableString(v) {
|
|
236
|
+
return v === null || typeof v === "string";
|
|
237
|
+
}
|
|
238
|
+
function nullableInt(v) {
|
|
239
|
+
return v === null || (typeof v === "number" && Number.isInteger(v));
|
|
240
|
+
}
|
|
241
|
+
export function checkBooleans(v, keys, where) {
|
|
242
|
+
if (!isObject(v))
|
|
243
|
+
bad(`${where} is missing 'capabilities'`);
|
|
244
|
+
for (const k of keys) {
|
|
245
|
+
if (typeof v[k] !== "boolean")
|
|
246
|
+
bad(`${where} capabilities are missing '${k}'`);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
/**
|
|
250
|
+
* One wire spelling of a provider, as the canonical identity.
|
|
251
|
+
*
|
|
252
|
+
* Current servers name the product as `{id, label}`; servers from before the
|
|
253
|
+
* product-identity change published a bare display label. A label alone
|
|
254
|
+
* becomes both halves — the old wire carried no stable id, and the label is
|
|
255
|
+
* the only key it ever published, so `id` degrades to "the label" rather than
|
|
256
|
+
* to a value this client invented.
|
|
257
|
+
*
|
|
258
|
+
* Returns null for anything else: an empty string, a pair with either half
|
|
259
|
+
* missing or empty, a number. Those are not degraded providers — rendered,
|
|
260
|
+
* they would be a blank chip on a Source that claims an external layer.
|
|
261
|
+
*/
|
|
262
|
+
export function asProviderIdentity(v) {
|
|
263
|
+
if (typeof v === "string" && v !== "")
|
|
264
|
+
return { id: v, label: v };
|
|
265
|
+
if (isObject(v) &&
|
|
266
|
+
typeof v.id === "string" && v.id !== "" &&
|
|
267
|
+
typeof v.label === "string" && v.label !== "") {
|
|
268
|
+
// Projected, not forwarded: only the two halves this client consumes.
|
|
269
|
+
return { id: v.id, label: v.label };
|
|
270
|
+
}
|
|
271
|
+
return null;
|
|
272
|
+
}
|
|
273
|
+
/**
|
|
274
|
+
* The provider of a Source summary or detail, across both generations of the
|
|
275
|
+
* wire.
|
|
276
|
+
*
|
|
277
|
+
* Required and nullable in each generation: current servers carry
|
|
278
|
+
* `provider: {id, label} | null` (dreamlake-server src/sources/v1/dto.ts),
|
|
279
|
+
* older ones `externalProvider: string | null`, and in both `null` means "no
|
|
280
|
+
* external layer". So a projection carrying NEITHER key is refused — absent
|
|
281
|
+
* would be rendered as a blank provider for a Source that has one — and so is
|
|
282
|
+
* a value that fits neither spelling. `provider` wins when both are present:
|
|
283
|
+
* it is the current contract, and the legacy field is the one a transitional
|
|
284
|
+
* server would keep emitting for old clients.
|
|
285
|
+
*/
|
|
286
|
+
export function readSourceProvider(res, where) {
|
|
287
|
+
const key = "provider" in res ? "provider" : "externalProvider" in res ? "externalProvider" : null;
|
|
288
|
+
if (key === null)
|
|
289
|
+
bad(`${where} carries neither 'provider' nor 'externalProvider'`);
|
|
290
|
+
const raw = res[key];
|
|
291
|
+
if (raw === null)
|
|
292
|
+
return null;
|
|
293
|
+
const identity = asProviderIdentity(raw);
|
|
294
|
+
if (identity === null)
|
|
295
|
+
bad(`${where} has no usable '${key}'`);
|
|
296
|
+
return identity;
|
|
297
|
+
}
|
|
298
|
+
function assertSourceListPage(res) {
|
|
299
|
+
if (!isObject(res))
|
|
300
|
+
bad("the source list was not an object");
|
|
301
|
+
if (!Array.isArray(res.items)) {
|
|
302
|
+
if (Array.isArray(res.sources)) {
|
|
303
|
+
bad("the server answered the legacy source list — it predates Source v1");
|
|
304
|
+
}
|
|
305
|
+
bad("the source list is missing 'items'");
|
|
306
|
+
}
|
|
307
|
+
checkCursor(res, "the source list");
|
|
308
|
+
checkBooleans(res.capabilities, ["createSource"], "the source list");
|
|
309
|
+
checkCounts(res, "the source list");
|
|
310
|
+
// The items themselves, not just the envelope. A partial item renders as
|
|
311
|
+
// `undefined` in the table and as a missing key in --json, which reads as a
|
|
312
|
+
// fact about the Source rather than as a broken response.
|
|
313
|
+
for (const item of res.items) {
|
|
314
|
+
if (!isObject(item))
|
|
315
|
+
bad("a source list item was not an object");
|
|
316
|
+
for (const k of ["id", "name", "lifecycle"]) {
|
|
317
|
+
if (typeof item[k] !== "string")
|
|
318
|
+
bad(`a source list item is missing '${k}'`);
|
|
319
|
+
}
|
|
320
|
+
// Both generations of the wire, one canonical shape — see
|
|
321
|
+
// readSourceProvider. Written back onto the item so the cast below hands
|
|
322
|
+
// every consumer `provider: ProviderIdentity | null`.
|
|
323
|
+
item.provider = readSourceProvider(item, "a source list item");
|
|
324
|
+
checkBooleans(item.capabilities, ["use"], `source '${String(item.name)}'`);
|
|
325
|
+
}
|
|
326
|
+
return res;
|
|
327
|
+
}
|
|
328
|
+
/**
|
|
329
|
+
* Refuse a capability that cannot be used as one.
|
|
330
|
+
*
|
|
331
|
+
* The generic on `requestJson` is a TypeScript claim about a value that
|
|
332
|
+
* arrived over a wire; it validates nothing. A capability missing its `url` is
|
|
333
|
+
* not a degraded capability — it is a 200 that turns into `fetch(undefined)`
|
|
334
|
+
* three frames later, reported as a download failure rather than as a server
|
|
335
|
+
* that answered wrongly.
|
|
336
|
+
*
|
|
337
|
+
* `allowedHeaders` is checked as a string map because it is fed directly to
|
|
338
|
+
* `fetch` and a non-string value there throws from inside the transport.
|
|
339
|
+
*/
|
|
340
|
+
function assertReadCapability(res, filePath) {
|
|
341
|
+
const at = `the read capability for '${filePath}'`;
|
|
342
|
+
if (!isObject(res))
|
|
343
|
+
bad(`${at} was not an object`);
|
|
344
|
+
if (typeof res.url !== "string" || res.url === "")
|
|
345
|
+
bad(`${at} is missing 'url'`);
|
|
346
|
+
// The verb is covered by the signature, and it is also the only thing this
|
|
347
|
+
// client knows how to issue. Anything else is refused rather than coerced.
|
|
348
|
+
if (res.method !== "GET")
|
|
349
|
+
bad(`${at} names a method this client cannot issue`);
|
|
350
|
+
if (typeof res.expiresAt !== "string")
|
|
351
|
+
bad(`${at} is missing 'expiresAt'`);
|
|
352
|
+
if (res.delivery !== "attachment" && res.delivery !== "inline") {
|
|
353
|
+
bad(`${at} is missing 'delivery'`);
|
|
354
|
+
}
|
|
355
|
+
if (!nullableInt(res.observedSizeBytes))
|
|
356
|
+
bad(`${at} is missing 'observedSizeBytes'`);
|
|
357
|
+
// Absent is legal here, unlike the fields above — so the check is on the
|
|
358
|
+
// TYPE when present, not on presence.
|
|
359
|
+
if (res.mediaType !== undefined && !nullableString(res.mediaType)) {
|
|
360
|
+
bad(`${at} has no usable 'mediaType'`);
|
|
361
|
+
}
|
|
362
|
+
if (typeof res.supportsRange !== "boolean")
|
|
363
|
+
bad(`${at} is missing 'supportsRange'`);
|
|
364
|
+
if (res.allowedHeaders !== undefined) {
|
|
365
|
+
if (!isObject(res.allowedHeaders))
|
|
366
|
+
bad(`${at} has a malformed 'allowedHeaders'`);
|
|
367
|
+
for (const [k, v] of Object.entries(res.allowedHeaders)) {
|
|
368
|
+
if (typeof v !== "string")
|
|
369
|
+
bad(`${at} has a non-string value for header '${k}'`);
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
return res;
|
|
373
|
+
}
|
|
374
|
+
function assertBrowsePage(res, dirPath) {
|
|
375
|
+
const at = dirPath === "" ? "the source root" : `'${dirPath}'`;
|
|
376
|
+
if (!isObject(res))
|
|
377
|
+
bad(`the browse of ${at} was not an object`);
|
|
378
|
+
if (!Array.isArray(res.entries))
|
|
379
|
+
bad(`the browse of ${at} is missing 'entries'`);
|
|
380
|
+
if ("nextToken" in res && !("nextCursor" in res)) {
|
|
381
|
+
bad(`the server answered a legacy browse page for ${at} — it predates Source v1`);
|
|
382
|
+
}
|
|
383
|
+
checkCursor(res, `the browse of ${at}`);
|
|
384
|
+
checkBooleans(res.capabilities, ["use", "editManagedObjects"], `the browse of ${at}`);
|
|
385
|
+
for (const e of res.entries) {
|
|
386
|
+
if (!isObject(e))
|
|
387
|
+
bad(`an entry under ${at} was not an object`);
|
|
388
|
+
/*
|
|
389
|
+
* The legacy discriminator, checked FIRST. An older server's entry is
|
|
390
|
+
* missing several v1 fields, so any other order diagnoses it as one absent
|
|
391
|
+
* field and hides the actual cause — the wrong protocol.
|
|
392
|
+
*/
|
|
393
|
+
if (e.kind !== "file" && e.kind !== "directory") {
|
|
394
|
+
bad("type" in e
|
|
395
|
+
? `entries under ${at} carry the legacy 'type' instead of 'kind'`
|
|
396
|
+
: `an entry under ${at} has no usable 'kind'`);
|
|
397
|
+
}
|
|
398
|
+
if (typeof e.path !== "string")
|
|
399
|
+
bad(`an entry under ${at} is missing 'path'`);
|
|
400
|
+
if (typeof e.name !== "string")
|
|
401
|
+
bad(`an entry under ${at} is missing 'name'`);
|
|
402
|
+
// Every field this CLI actually consumes, not only the ones that would
|
|
403
|
+
// crash. `size` feeds the progress total, the other two are printed.
|
|
404
|
+
if (!nullableInt(e.size))
|
|
405
|
+
bad(`an entry under ${at} has no usable 'size'`);
|
|
406
|
+
if (!nullableString(e.mediaType))
|
|
407
|
+
bad(`an entry under ${at} has no usable 'mediaType'`);
|
|
408
|
+
if (!nullableString(e.modifiedAt))
|
|
409
|
+
bad(`an entry under ${at} has no usable 'modifiedAt'`);
|
|
410
|
+
/*
|
|
411
|
+
* `canDownload` in particular. It is consumed as `!== false`, so a MISSING
|
|
412
|
+
* one reads as permission granted — the exact direction a missing field
|
|
413
|
+
* must not fail in. Checking it here is what makes that read safe.
|
|
414
|
+
*/
|
|
415
|
+
for (const k of ["canPreview", "canDownload", "canDelete"]) {
|
|
416
|
+
if (typeof e[k] !== "boolean")
|
|
417
|
+
bad(`an entry under ${at} is missing '${k}'`);
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
return res;
|
|
421
|
+
}
|
|
422
|
+
/** One page, with its filters. */
|
|
423
|
+
async function listPage(c, query) {
|
|
424
|
+
return assertSourceListPage(await requestJson(c.remote, `/namespaces/${seg(c.ns)}/sources`, {
|
|
425
|
+
token: c.token,
|
|
426
|
+
// Spelled out rather than spread: these five names ARE the querystring
|
|
427
|
+
// contract, and the schema is closed, so a field that arrived here by
|
|
428
|
+
// accident would be a 400 rather than something the server ignores.
|
|
429
|
+
query: {
|
|
430
|
+
cursor: query.cursor,
|
|
431
|
+
limit: query.limit,
|
|
432
|
+
nameContains: query.nameContains,
|
|
433
|
+
provider: query.provider,
|
|
434
|
+
hasExternal: query.hasExternal,
|
|
435
|
+
lifecycle: query.lifecycle,
|
|
436
|
+
},
|
|
437
|
+
}));
|
|
438
|
+
}
|
|
439
|
+
/**
|
|
440
|
+
* Pages of the namespace's source list, under a filter.
|
|
441
|
+
*
|
|
442
|
+
* `MAX_PAGES` replaces a terminator the old shape had for free. Page counting
|
|
443
|
+
* ended on `page >= totalPages`; an opaque cursor publishes no total, so a
|
|
444
|
+
* server that kept handing back a cursor would spin here forever. It fails
|
|
445
|
+
* loudly instead — the bound is not a limit on how many Sources are supported,
|
|
446
|
+
* it is a bound on a malfunctioning peer.
|
|
447
|
+
*/
|
|
448
|
+
async function walkSources(c, filters) {
|
|
449
|
+
const all = [];
|
|
450
|
+
let cursor;
|
|
451
|
+
for (let page = 0;; page++) {
|
|
452
|
+
if (page >= MAX_PAGES) {
|
|
453
|
+
throw new Error(`source list did not terminate after ${MAX_PAGES} pages`);
|
|
454
|
+
}
|
|
455
|
+
const res = await listPage(c, { ...filters, cursor, limit: 100 });
|
|
456
|
+
all.push(...res.items);
|
|
457
|
+
if (!res.nextCursor)
|
|
458
|
+
break;
|
|
459
|
+
cursor = res.nextCursor;
|
|
460
|
+
}
|
|
461
|
+
return all;
|
|
462
|
+
}
|
|
463
|
+
/**
|
|
464
|
+
* Resolve `--source <name>` to a source record.
|
|
465
|
+
*
|
|
466
|
+
* Narrowed server-side with `nameContains` and then matched EXACTLY here, which
|
|
467
|
+
* is two different jobs and needs both halves. `nameContains` is a substring
|
|
468
|
+
* search, so it cannot decide which Source was meant — `prod` matches
|
|
469
|
+
* `prod-eu` — and the exact comparison is what does. What the filter buys is
|
|
470
|
+
* that the walk is over the handful of Sources whose names contain the term,
|
|
471
|
+
* not over the whole Namespace: resolving one name used to pull every Source in
|
|
472
|
+
* pages of a hundred, so a Namespace with ten thousand of them cost a hundred
|
|
473
|
+
* round trips before the first byte of the actual command.
|
|
474
|
+
*
|
|
475
|
+
* A Source name is at most 64 characters and the search term cap is 64 code
|
|
476
|
+
* points, so any name that could match is a legal term. There is no case where
|
|
477
|
+
* this has to fall back to an unfiltered walk.
|
|
478
|
+
*
|
|
479
|
+
* On a miss, name what IS available — under the same filter, so the suggestions
|
|
480
|
+
* are near misses rather than the first hundred Sources in the Namespace.
|
|
481
|
+
*/
|
|
482
|
+
export async function resolveSource(c, name) {
|
|
483
|
+
let near;
|
|
484
|
+
try {
|
|
485
|
+
near = await walkSources(c, { nameContains: name });
|
|
486
|
+
}
|
|
487
|
+
catch (error) {
|
|
488
|
+
if (!(error instanceof HttpError) || error.code !== 'SOURCE_FORBIDDEN')
|
|
489
|
+
throw error;
|
|
490
|
+
const source = await requestJson(c.remote, `/namespaces/${seg(c.ns)}/sources/by-name/${seg(name)}`, { token: c.token });
|
|
491
|
+
if (!isObject(source) || typeof source.id !== 'string' || source.name !== name || !isObject(source.capabilities))
|
|
492
|
+
bad('invalid source by-name response');
|
|
493
|
+
return source;
|
|
494
|
+
}
|
|
495
|
+
const hit = near.find((s) => s.name === name);
|
|
496
|
+
if (hit)
|
|
497
|
+
return hit;
|
|
498
|
+
if (near.length === 0) {
|
|
499
|
+
throw new Error(`source '${name}' not found in namespace '${c.ns}' — no source's name contains it. ` +
|
|
500
|
+
`See what exists: dreamlake source list --namespace ${c.ns}`);
|
|
501
|
+
}
|
|
502
|
+
throw new Error(`source '${name}' not found in namespace '${c.ns}'. did you mean: ${near
|
|
503
|
+
.map((s) => s.name)
|
|
504
|
+
.join(", ")}`);
|
|
505
|
+
}
|
|
506
|
+
// ─── browsing ─────────────────────────────────────────────────────────────────
|
|
507
|
+
/** One directory level, following `nextCursor` until the backend runs dry. */
|
|
508
|
+
async function browseLevel(c, sourceId, dirPath) {
|
|
509
|
+
const entries = [];
|
|
510
|
+
let cursor;
|
|
511
|
+
for (let page = 0;; page++) {
|
|
512
|
+
if (page >= MAX_PAGES) {
|
|
513
|
+
throw new Error(`browse of '${dirPath}' did not terminate after ${MAX_PAGES} pages`);
|
|
514
|
+
}
|
|
515
|
+
const res = assertBrowsePage(await requestJson(c.remote, `/namespaces/${seg(c.ns)}/sources/${seg(sourceId)}/browse`, {
|
|
516
|
+
token: c.token,
|
|
517
|
+
query: { path: dirPath, cursor, limit: 1000 },
|
|
518
|
+
}), dirPath);
|
|
519
|
+
entries.push(...res.entries);
|
|
520
|
+
if (!res.nextCursor)
|
|
521
|
+
break;
|
|
522
|
+
cursor = res.nextCursor;
|
|
523
|
+
}
|
|
524
|
+
return entries;
|
|
525
|
+
}
|
|
526
|
+
/**
|
|
527
|
+
* Collect entries under `root`.
|
|
528
|
+
*
|
|
529
|
+
* Depth: one level by default. `browse` with no path starts at the source
|
|
530
|
+
* root, and recursing there automatically would walk an entire bucket on the
|
|
531
|
+
* user's first command. `-r` opts into the full walk, capped at MAX_ENTRIES so
|
|
532
|
+
* a bucket with a million objects fails loudly instead of hanging.
|
|
533
|
+
*
|
|
534
|
+
* Contents: everything by default. Hiding directories made an ordinary folder
|
|
535
|
+
* of folders report "0 files", which reads as "nothing here" — directories are
|
|
536
|
+
* marked with a trailing slash, so a mixed listing is unambiguous.
|
|
537
|
+
*/
|
|
538
|
+
async function collect(c, sourceId, root, opts) {
|
|
539
|
+
const out = [];
|
|
540
|
+
const queue = [root];
|
|
541
|
+
let truncated = false;
|
|
542
|
+
// Counted whether or not they were included, so `--files` can still explain
|
|
543
|
+
// an empty result by pointing at the subdirectories it skipped.
|
|
544
|
+
let containers = 0;
|
|
545
|
+
let leaves = 0;
|
|
546
|
+
while (queue.length > 0 && !truncated) {
|
|
547
|
+
const dir = queue.shift();
|
|
548
|
+
const level = await browseLevel(c, sourceId, dir);
|
|
549
|
+
// Directories first within each level — a mixed listing is easier to scan
|
|
550
|
+
// when the branches are grouped, and it matches --group-directories-first.
|
|
551
|
+
const ordered = [
|
|
552
|
+
...level.filter((e) => e.kind === "directory"),
|
|
553
|
+
...level.filter((e) => e.kind !== "directory"),
|
|
554
|
+
];
|
|
555
|
+
for (const e of ordered) {
|
|
556
|
+
if (e.kind === "directory") {
|
|
557
|
+
containers += 1;
|
|
558
|
+
if (opts.include !== "files")
|
|
559
|
+
out.push(e);
|
|
560
|
+
if (opts.recursive)
|
|
561
|
+
queue.push(e.path);
|
|
562
|
+
}
|
|
563
|
+
else {
|
|
564
|
+
leaves += 1;
|
|
565
|
+
if (opts.include !== "dirs")
|
|
566
|
+
out.push(e);
|
|
567
|
+
}
|
|
568
|
+
if (out.length >= MAX_ENTRIES) {
|
|
569
|
+
truncated = true;
|
|
570
|
+
break;
|
|
571
|
+
}
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
return { entries: out, truncated, containers, leaves };
|
|
575
|
+
}
|
|
576
|
+
// ─── read capabilities ────────────────────────────────────────────────────────
|
|
577
|
+
/**
|
|
578
|
+
* Mint a capability for one exact object.
|
|
579
|
+
*
|
|
580
|
+
* No `stat` first, deliberately. This call IS the decision: it re-resolves the
|
|
581
|
+
* current winner, refuses a path that does not exist
|
|
582
|
+
* (`SOURCE_PATH_NOT_FOUND`) and refuses a directory
|
|
583
|
+
* (`SOURCE_PATH_KIND_CONFLICT`) — `resolvedSourceService.ts:563`. A stat
|
|
584
|
+
* beforehand would repeat the provider resolution to answer a question this
|
|
585
|
+
* call answers anyway, and still could not close the window between the two.
|
|
586
|
+
*/
|
|
587
|
+
async function readCapability(c, sourceId, filePath, intent) {
|
|
588
|
+
return assertReadCapability(await requestJson(c.remote, `/namespaces/${seg(c.ns)}/sources/${seg(sourceId)}/read-capabilities`, { token: c.token, method: "POST", json: { path: filePath, intent } }), filePath);
|
|
589
|
+
}
|
|
590
|
+
/** Whether a capability can only be used by a caller that sends headers. */
|
|
591
|
+
function requiresHeaders(cap) {
|
|
592
|
+
return cap.allowedHeaders !== undefined && Object.keys(cap.allowedHeaders).length > 0;
|
|
593
|
+
}
|
|
594
|
+
/**
|
|
595
|
+
* How long this capability still has, from the server's own `expiresAt`.
|
|
596
|
+
*
|
|
597
|
+
* The per-backend guesswork this replaces was a lie by construction: it printed
|
|
598
|
+
* "~4h (Dropbox sets its own expiry)" from a hard-coded table because the
|
|
599
|
+
* legacy `expiresIn` echoed what was REQUESTED rather than what was issued.
|
|
600
|
+
* `expiresAt` is the issued instant, so there is nothing left to guess.
|
|
601
|
+
*/
|
|
602
|
+
function expiryNote(expiresAt) {
|
|
603
|
+
const ms = Date.parse(expiresAt) - Date.now();
|
|
604
|
+
if (!Number.isFinite(ms))
|
|
605
|
+
return `until ${expiresAt}`;
|
|
606
|
+
if (ms <= 0)
|
|
607
|
+
return "already expired";
|
|
608
|
+
const mins = Math.round(ms / 60000);
|
|
609
|
+
return mins >= 60 ? `~${Math.round(mins / 60)}h (until ${expiresAt})` : `~${mins}m (until ${expiresAt})`;
|
|
610
|
+
}
|
|
611
|
+
// ─── bounded parallelism ──────────────────────────────────────────────────────
|
|
612
|
+
/**
|
|
613
|
+
* Run `worker` over `items` with at most `limit` in flight. Never rejects —
|
|
614
|
+
* a failing item yields its error so one bad object in a thousand-file
|
|
615
|
+
* download doesn't discard the 999 that succeeded.
|
|
616
|
+
*/
|
|
617
|
+
async function pool(items, limit, worker) {
|
|
618
|
+
const results = new Array(items.length);
|
|
619
|
+
let next = 0;
|
|
620
|
+
const runners = Array.from({ length: Math.min(limit, items.length) }, async () => {
|
|
621
|
+
for (;;) {
|
|
622
|
+
const i = next++;
|
|
623
|
+
if (i >= items.length)
|
|
624
|
+
return;
|
|
625
|
+
try {
|
|
626
|
+
results[i] = { ok: true, value: await worker(items[i], i) };
|
|
627
|
+
}
|
|
628
|
+
catch (err) {
|
|
629
|
+
results[i] = { ok: false, error: err };
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
});
|
|
633
|
+
await Promise.all(runners);
|
|
634
|
+
return results;
|
|
635
|
+
}
|
|
636
|
+
// ─── source list ──────────────────────────────────────────────────────────────
|
|
637
|
+
/**
|
|
638
|
+
* The server's page bound, restated so a bad `--limit` costs nothing.
|
|
639
|
+
*
|
|
640
|
+
* The querystring is closed (`additionalProperties: false`), so a filter this
|
|
641
|
+
* client invented is a 400 rather than a field quietly ignored — which is the
|
|
642
|
+
* failure the closed schema exists to produce, and the reason `namePrefix` is
|
|
643
|
+
* NOT accepted here: it was removed from the contract in favour of
|
|
644
|
+
* `nameContains`, and forwarding it would ask for prefix semantics the endpoint
|
|
645
|
+
* no longer has.
|
|
646
|
+
*/
|
|
647
|
+
const LIST_MAX_LIMIT = 200;
|
|
648
|
+
function parseListLimit(raw) {
|
|
649
|
+
if (raw === undefined)
|
|
650
|
+
return undefined;
|
|
651
|
+
const limit = Number(raw);
|
|
652
|
+
if (!Number.isSafeInteger(limit) || limit < 1 || limit > LIST_MAX_LIMIT) {
|
|
653
|
+
fail(`--limit must be an integer between 1 and ${LIST_MAX_LIMIT}`);
|
|
654
|
+
return null;
|
|
655
|
+
}
|
|
656
|
+
return limit;
|
|
657
|
+
}
|
|
658
|
+
/** Quote a pasted term only when a shell would otherwise re-split it. */
|
|
659
|
+
function shellQuote(v) {
|
|
660
|
+
return /^[A-Za-z0-9._@%+:,/-]+$/.test(v) ? v : `'${v.replace(/'/g, `'\\''`)}'`;
|
|
661
|
+
}
|
|
662
|
+
/**
|
|
663
|
+
* The continuation command, carrying the namespace and the filters.
|
|
664
|
+
*
|
|
665
|
+
* A cursor is scoped to the query that issued it — both to the Namespace, which
|
|
666
|
+
* is not necessarily the active login, and to the filter set, because the
|
|
667
|
+
* server binds the filters into the cursor and rejects it under a different
|
|
668
|
+
* one. A bare `list --cursor …` pasted after a filtered read is not the next
|
|
669
|
+
* page of what was on screen.
|
|
670
|
+
*/
|
|
671
|
+
function listContinuation(ns, opts, cursor) {
|
|
672
|
+
const parts = ["dreamlake source list", `--namespace ${ns}`];
|
|
673
|
+
if (opts.nameContains !== undefined)
|
|
674
|
+
parts.push(`--name-contains ${shellQuote(opts.nameContains)}`);
|
|
675
|
+
if (opts.provider !== undefined)
|
|
676
|
+
parts.push(`--provider ${shellQuote(opts.provider)}`);
|
|
677
|
+
if (opts.hasExternal !== undefined)
|
|
678
|
+
parts.push(opts.hasExternal ? "--external" : "--no-external");
|
|
679
|
+
if (opts.lifecycle !== undefined)
|
|
680
|
+
parts.push(`--lifecycle ${opts.lifecycle}`);
|
|
681
|
+
return `${parts.join(" ")} --cursor ${cursor}`;
|
|
682
|
+
}
|
|
683
|
+
export async function runSourceList(opts) {
|
|
684
|
+
const limit = parseListLimit(opts.limit);
|
|
685
|
+
if (limit === null)
|
|
686
|
+
return 1;
|
|
687
|
+
/*
|
|
688
|
+
* Two words, fixed by the contract and enumerated in the server's
|
|
689
|
+
* querystring. Checking here is not this client having a view on the schema:
|
|
690
|
+
* it names the two accepted values instead of spending a round trip to be
|
|
691
|
+
* told the request was malformed.
|
|
692
|
+
*/
|
|
693
|
+
if (opts.lifecycle !== undefined && opts.lifecycle !== "active" && opts.lifecycle !== "disabled") {
|
|
694
|
+
fail("--lifecycle must be 'active' or 'disabled'");
|
|
695
|
+
return 1;
|
|
696
|
+
}
|
|
697
|
+
const c = await ctx(opts.namespace);
|
|
698
|
+
if (!c)
|
|
699
|
+
return 1;
|
|
700
|
+
try {
|
|
701
|
+
/*
|
|
702
|
+
* ONE page.
|
|
703
|
+
*
|
|
704
|
+
* This used to follow every cursor to the end before printing anything, so
|
|
705
|
+
* a Namespace with ten thousand Sources meant a hundred round trips and a
|
|
706
|
+
* table nobody could read. Bounded output is also the only honest shape
|
|
707
|
+
* once filters exist: a partial answer that says which page it is beats a
|
|
708
|
+
* complete one that arrives too late to be used.
|
|
709
|
+
*/
|
|
710
|
+
const page = await listPage(c, {
|
|
711
|
+
cursor: opts.cursor,
|
|
712
|
+
limit,
|
|
713
|
+
nameContains: opts.nameContains,
|
|
714
|
+
provider: opts.provider,
|
|
715
|
+
hasExternal: opts.hasExternal,
|
|
716
|
+
lifecycle: opts.lifecycle,
|
|
717
|
+
});
|
|
718
|
+
// The v1 projection, under v1's own field names. The legacy trio
|
|
719
|
+
// (kind/status/access) is NOT reconstructed: `access` in particular would
|
|
720
|
+
// have to flatten four independent capability booleans into one word, and a
|
|
721
|
+
// familiar column name carrying a value the server never published is worse
|
|
722
|
+
// than a changed column name.
|
|
723
|
+
// `provider` stays the display label this output has always carried;
|
|
724
|
+
// `providerId` is the stable id beside it — the value `--provider`
|
|
725
|
+
// filters on against current servers, and the reason the id exists.
|
|
726
|
+
const project = (x) => ({
|
|
727
|
+
name: x.name,
|
|
728
|
+
provider: x.provider?.label ?? null,
|
|
729
|
+
providerId: x.provider?.id ?? null,
|
|
730
|
+
lifecycle: x.lifecycle,
|
|
731
|
+
use: x.capabilities.use,
|
|
732
|
+
});
|
|
733
|
+
if (opts.json) {
|
|
734
|
+
/*
|
|
735
|
+
* An object, not a bare array. The array was the whole namespace, and it
|
|
736
|
+
* could be: nothing was paged. A page has to publish where it stops and
|
|
737
|
+
* what the totals are, or a script reads a hundred Sources as all of
|
|
738
|
+
* them — and `capabilities.createSource` is the answer to "may I create
|
|
739
|
+
* one", which no count implies.
|
|
740
|
+
*/
|
|
741
|
+
emitJson({
|
|
742
|
+
items: page.items.map(project),
|
|
743
|
+
nextCursor: page.nextCursor,
|
|
744
|
+
capabilities: page.capabilities,
|
|
745
|
+
lifecycleCounts: page.lifecycleCounts,
|
|
746
|
+
});
|
|
747
|
+
return 0;
|
|
748
|
+
}
|
|
749
|
+
if (page.items.length === 0) {
|
|
750
|
+
/*
|
|
751
|
+
* An empty page is not always an empty Namespace. It can be a filter that
|
|
752
|
+
* matched nothing, and — with a cursor still attached — an interior page.
|
|
753
|
+
* Saying "there are none" in either case answers a question nobody asked.
|
|
754
|
+
*/
|
|
755
|
+
if (page.nextCursor !== null) {
|
|
756
|
+
process.stdout.write(" Nothing on this page — but this is not the last one.\n");
|
|
757
|
+
process.stdout.write(`\n More to come. Next page:\n ${listContinuation(c.ns, opts, page.nextCursor)}\n`);
|
|
758
|
+
return 0;
|
|
759
|
+
}
|
|
760
|
+
process.stdout.write(hasFilters(opts)
|
|
761
|
+
? `No sources in namespace '${c.ns}' match those filters.\n`
|
|
762
|
+
: `No sources in namespace '${c.ns}' — connect one in the DreamLake UI first.\n`);
|
|
763
|
+
printSourceCounts(page.lifecycleCounts, c.ns, opts);
|
|
764
|
+
return 0;
|
|
765
|
+
}
|
|
766
|
+
const rows = page.items.map((x) => ({
|
|
767
|
+
name: x.name,
|
|
768
|
+
// A Source with no external layer has no provider — "-" says that, where
|
|
769
|
+
// an invented word like "dreamlake" would read as a provider name.
|
|
770
|
+
provider: x.provider?.label ?? "-",
|
|
771
|
+
lifecycle: x.lifecycle,
|
|
772
|
+
use: x.capabilities.use ? "yes" : "no",
|
|
773
|
+
}));
|
|
774
|
+
process.stdout.write(renderTable(rows, ["name", "provider", "lifecycle", "use"]));
|
|
775
|
+
process.stdout.write(`\n ${page.items.length} source(s) on this page\n`);
|
|
776
|
+
printSourceCounts(page.lifecycleCounts, c.ns, opts);
|
|
777
|
+
if (!page.capabilities.createSource) {
|
|
778
|
+
process.stdout.write(" (you cannot create a Source in this namespace)\n");
|
|
779
|
+
}
|
|
780
|
+
if (page.nextCursor !== null) {
|
|
781
|
+
process.stdout.write(`\n More to come. Next page:\n ${listContinuation(c.ns, opts, page.nextCursor)}\n`);
|
|
782
|
+
}
|
|
783
|
+
return 0;
|
|
784
|
+
}
|
|
785
|
+
catch (err) {
|
|
786
|
+
fail(explain(err));
|
|
787
|
+
return 1;
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
/**
|
|
791
|
+
* The search term as the SERVER will read it: trim → NFKC → lowercase, and
|
|
792
|
+
* empty after that means no search.
|
|
793
|
+
*
|
|
794
|
+
* §4 canonicalizes the term and treats one that canonicalizes to nothing as
|
|
795
|
+
* absent — deliberately, so a whitespace-only search does not match nothing and
|
|
796
|
+
* get reported as an empty collection. Judging `!== undefined` made this client
|
|
797
|
+
* say exactly that: `--name-contains ' '` returns the unfiltered Namespace
|
|
798
|
+
* while the output called it a filtered result.
|
|
799
|
+
*
|
|
800
|
+
* The RAW term is still what gets sent: the server canonicalizes it itself and
|
|
801
|
+
* measures its length cap on what arrived, so normalizing before transmission
|
|
802
|
+
* would bound a different string than the caller typed. This decides what to
|
|
803
|
+
* SAY, not what to send.
|
|
804
|
+
*/
|
|
805
|
+
function effectiveTerm(raw) {
|
|
806
|
+
if (raw === undefined)
|
|
807
|
+
return undefined;
|
|
808
|
+
const canonical = raw.trim().normalize("NFKC").toLowerCase();
|
|
809
|
+
return canonical === "" ? undefined : canonical;
|
|
810
|
+
}
|
|
811
|
+
/**
|
|
812
|
+
* `provider` is NOT a search term, and is not normalized like one.
|
|
813
|
+
*
|
|
814
|
+
* The server maps only the exact empty string to "no filter" — no trim. So
|
|
815
|
+
* `--provider ' '` is a real filter naming a provider nothing matches, and
|
|
816
|
+
* treating it as absent here would claim an unfiltered read of a filtered one.
|
|
817
|
+
*/
|
|
818
|
+
function effectiveProvider(raw) {
|
|
819
|
+
return raw === undefined || raw === "" ? undefined : raw;
|
|
820
|
+
}
|
|
821
|
+
function hasFilters(o) {
|
|
822
|
+
return (effectiveTerm(o.nameContains) !== undefined ||
|
|
823
|
+
effectiveProvider(o.provider) !== undefined ||
|
|
824
|
+
o.hasExternal !== undefined ||
|
|
825
|
+
o.lifecycle !== undefined);
|
|
826
|
+
}
|
|
827
|
+
/**
|
|
828
|
+
* The counts, and what they are actually counting.
|
|
829
|
+
*
|
|
830
|
+
* They are NOT the namespace unless the read was unfiltered. §4's aggregate
|
|
831
|
+
* applies `nameContains`, `provider` and `hasExternal`, and deliberately omits
|
|
832
|
+
* `lifecycle` — which is the point of the three numbers: under
|
|
833
|
+
* `--lifecycle disabled` they still show how many active ones exist, so the set
|
|
834
|
+
* you are not looking at stays visible. Under any of the other filters they
|
|
835
|
+
* describe the matching set, and calling that "in ns" would present a search
|
|
836
|
+
* result as the size of the Namespace.
|
|
837
|
+
*
|
|
838
|
+
* `null` says only that the counts are unavailable. §4 returns it for a pointer
|
|
839
|
+
* inconsistency in the counted set, for an exhausted or cancelled budget, and
|
|
840
|
+
* for a result shape the server cannot read — so "too many to count" was one
|
|
841
|
+
* cause of several, presented as the cause.
|
|
842
|
+
*/
|
|
843
|
+
function printSourceCounts(counts, ns, opts) {
|
|
844
|
+
if (counts === null) {
|
|
845
|
+
process.stdout.write(" (counts unavailable)\n");
|
|
846
|
+
return;
|
|
847
|
+
}
|
|
848
|
+
// `lifecycle` is excluded from the aggregate, so it never narrows what these
|
|
849
|
+
// describe. Each of the other three does — when it is a filter the server
|
|
850
|
+
// will actually apply.
|
|
851
|
+
const narrowed = effectiveTerm(opts.nameContains) !== undefined ||
|
|
852
|
+
effectiveProvider(opts.provider) !== undefined ||
|
|
853
|
+
opts.hasExternal !== undefined;
|
|
854
|
+
process.stdout.write(` ${counts.all} ${narrowed ? "matching those filters" : `in ${ns}`}` +
|
|
855
|
+
` — ${counts.active} active, ${counts.disabled} disabled\n`);
|
|
856
|
+
}
|
|
857
|
+
function assertConnectors(res) {
|
|
858
|
+
if (!isObject(res))
|
|
859
|
+
bad("the connector list was not an object");
|
|
860
|
+
if (!Array.isArray(res.connectors))
|
|
861
|
+
bad("the connector list is missing 'connectors'");
|
|
862
|
+
const out = [];
|
|
863
|
+
for (const raw of res.connectors) {
|
|
864
|
+
if (!isObject(raw))
|
|
865
|
+
bad("a connector entry was not an object");
|
|
866
|
+
if (typeof raw.kind !== "string" || raw.kind === "")
|
|
867
|
+
bad("a connector entry is missing 'kind'");
|
|
868
|
+
const mode = raw.connectionMode;
|
|
869
|
+
if (mode !== "required" && mode !== "optional" && mode !== "none") {
|
|
870
|
+
bad(`connector '${raw.kind}' has no usable 'connectionMode'`);
|
|
871
|
+
}
|
|
872
|
+
if (!Array.isArray(raw.compatibleKinds) || raw.compatibleKinds.some((k) => typeof k !== "string")) {
|
|
873
|
+
bad(`connector '${raw.kind}' has no usable 'compatibleKinds'`);
|
|
874
|
+
}
|
|
875
|
+
out.push({
|
|
876
|
+
kind: raw.kind,
|
|
877
|
+
connectionMode: mode,
|
|
878
|
+
compatibleKinds: raw.compatibleKinds,
|
|
879
|
+
});
|
|
880
|
+
}
|
|
881
|
+
return out;
|
|
882
|
+
}
|
|
883
|
+
export async function runSourceConnectors(opts) {
|
|
884
|
+
const c = await ctx(opts.namespace);
|
|
885
|
+
if (!c)
|
|
886
|
+
return 1;
|
|
887
|
+
try {
|
|
888
|
+
const connectors = assertConnectors(await requestJson(c.remote, `/namespaces/${seg(c.ns)}/source-connectors`, {
|
|
889
|
+
token: c.token,
|
|
890
|
+
}));
|
|
891
|
+
if (opts.json) {
|
|
892
|
+
emitJson({ connectors });
|
|
893
|
+
return 0;
|
|
894
|
+
}
|
|
895
|
+
if (connectors.length === 0) {
|
|
896
|
+
/*
|
|
897
|
+
* An empty list is an answer, not a blank. It means this deployment has
|
|
898
|
+
* no Source connectors registered — a Source cannot be imported at all —
|
|
899
|
+
* which is a different sentence from "nothing is compatible".
|
|
900
|
+
*/
|
|
901
|
+
process.stdout.write("No source connectors are registered on this server.\n");
|
|
902
|
+
return 0;
|
|
903
|
+
}
|
|
904
|
+
process.stdout.write(renderTable(connectors.map((x) => ({
|
|
905
|
+
connector: x.kind,
|
|
906
|
+
connection: x.connectionMode,
|
|
907
|
+
// "-" where a connector takes no Connection at all. An empty cell
|
|
908
|
+
// would read as "none are compatible", which is a different fact.
|
|
909
|
+
accepts: x.connectionMode === "none" ? "-" : x.compatibleKinds.join(", ") || "(none)",
|
|
910
|
+
})), ["connector", "connection", "accepts"]));
|
|
911
|
+
process.stdout.write("\n connection: required — import must name one with --connection\n" +
|
|
912
|
+
" optional — it may, and works without\n" +
|
|
913
|
+
" none — it takes none\n" +
|
|
914
|
+
` See what you have: dreamlake connection list --namespace ${c.ns}\n`);
|
|
915
|
+
return 0;
|
|
916
|
+
}
|
|
917
|
+
catch (err) {
|
|
918
|
+
if (err instanceof HttpError && err.status === 404 && err.code === undefined) {
|
|
919
|
+
/*
|
|
920
|
+
* Restated rather than shared with `connection/`, deliberately, for the
|
|
921
|
+
* same reason the idempotency helper is: this module does not depend on
|
|
922
|
+
* that one. A bare 404 here is the Connection scope not being mounted —
|
|
923
|
+
* this route lives inside it — and reporting that as a missing Source or
|
|
924
|
+
* a permission problem would send someone to fix neither.
|
|
925
|
+
*/
|
|
926
|
+
fail("this server does not publish connector compatibility.\n" +
|
|
927
|
+
" The route is not mounted, which is what a deployment with the\n" +
|
|
928
|
+
" Connection plane switched off looks like from here.\n" +
|
|
929
|
+
` (${err.message})`);
|
|
930
|
+
return 1;
|
|
931
|
+
}
|
|
932
|
+
fail(explain(err));
|
|
933
|
+
return 1;
|
|
934
|
+
}
|
|
935
|
+
}
|
|
936
|
+
// ─── source browse ────────────────────────────────────────────────────────────
|
|
937
|
+
export async function runSourceBrowse(pathArg, opts) {
|
|
938
|
+
const root = pathArg ?? "";
|
|
939
|
+
const bad = root ? checkPath(root) : null;
|
|
940
|
+
if (bad) {
|
|
941
|
+
fail(bad);
|
|
942
|
+
return 1;
|
|
943
|
+
}
|
|
944
|
+
if (opts.dirs && opts.files) {
|
|
945
|
+
fail("--dirs and --files are mutually exclusive (omit both to list everything)");
|
|
946
|
+
return 1;
|
|
947
|
+
}
|
|
948
|
+
const include = opts.dirs ? "dirs" : opts.files ? "files" : "all";
|
|
949
|
+
const c = await ctx(opts.namespace);
|
|
950
|
+
if (!c)
|
|
951
|
+
return 1;
|
|
952
|
+
try {
|
|
953
|
+
const src = await resolveSource(c, opts.source);
|
|
954
|
+
const { entries, truncated, containers } = await collect(c, src.id, root, {
|
|
955
|
+
recursive: opts.recursive === true,
|
|
956
|
+
include,
|
|
957
|
+
});
|
|
958
|
+
if (opts.json) {
|
|
959
|
+
emitJson(entries.map((e) => ({
|
|
960
|
+
path: e.path,
|
|
961
|
+
kind: e.kind,
|
|
962
|
+
size: e.size,
|
|
963
|
+
mediaType: e.mediaType,
|
|
964
|
+
modifiedAt: e.modifiedAt,
|
|
965
|
+
canDownload: e.canDownload,
|
|
966
|
+
})));
|
|
967
|
+
}
|
|
968
|
+
else if (opts.print0) {
|
|
969
|
+
// NUL-separated for `xargs -0`: the only separator that can't appear in
|
|
970
|
+
// a path, and therefore the only one safe for names with spaces.
|
|
971
|
+
for (const e of entries) {
|
|
972
|
+
process.stdout.write(`${e.path}${e.kind === "directory" ? "/" : ""}\0`);
|
|
973
|
+
}
|
|
974
|
+
}
|
|
975
|
+
else {
|
|
976
|
+
// One path per line, nothing else on stdout — this is meant to be piped
|
|
977
|
+
// into xargs. Counts and warnings go to stderr.
|
|
978
|
+
for (const e of entries) {
|
|
979
|
+
process.stdout.write(`${displayPath(`${e.path}${e.kind === "directory" ? "/" : ""}`)}\n`);
|
|
980
|
+
}
|
|
981
|
+
const dirCount = entries.filter((e) => e.kind === "directory").length;
|
|
982
|
+
const fileCount = entries.length - dirCount;
|
|
983
|
+
const dirs = `${dirCount} director${dirCount === 1 ? "y" : "ies"}`;
|
|
984
|
+
const files = `${fileCount} file${fileCount === 1 ? "" : "s"}`;
|
|
985
|
+
const counted = include === "dirs" ? dirs : include === "files" ? files : `${dirs}, ${files}`;
|
|
986
|
+
// Only --files can now report an empty listing while the level actually
|
|
987
|
+
// holds something; the default shows the directories itself.
|
|
988
|
+
const hint = include === "files" && !opts.recursive && entries.length === 0 && containers > 0
|
|
989
|
+
? ` (${containers} subdirector${containers === 1 ? "y" : "ies"} here — use -r to recurse)`
|
|
990
|
+
: "";
|
|
991
|
+
process.stderr.write(`→ ${counted}${hint}\n`);
|
|
992
|
+
}
|
|
993
|
+
if (truncated) {
|
|
994
|
+
fail(`listing truncated at ${MAX_ENTRIES} entries — narrow the path or drop --recursive`);
|
|
995
|
+
return 1;
|
|
996
|
+
}
|
|
997
|
+
return 0;
|
|
998
|
+
}
|
|
999
|
+
catch (err) {
|
|
1000
|
+
fail(explain(err));
|
|
1001
|
+
return 1;
|
|
1002
|
+
}
|
|
1003
|
+
}
|
|
1004
|
+
// ─── source fetch ─────────────────────────────────────────────────────────────
|
|
1005
|
+
export async function runSourceFetch(pathArg, opts) {
|
|
1006
|
+
const filePath = pathArg;
|
|
1007
|
+
const bad = checkPath(filePath);
|
|
1008
|
+
if (bad) {
|
|
1009
|
+
fail(bad);
|
|
1010
|
+
return 1;
|
|
1011
|
+
}
|
|
1012
|
+
const c = await ctx(opts.namespace);
|
|
1013
|
+
if (!c)
|
|
1014
|
+
return 1;
|
|
1015
|
+
try {
|
|
1016
|
+
const src = await resolveSource(c, opts.source);
|
|
1017
|
+
const cap = await readCapability(c, src.id, filePath, "download");
|
|
1018
|
+
if (opts.json) {
|
|
1019
|
+
// The WHOLE capability. A consumer that gets only the url cannot issue
|
|
1020
|
+
// the request that was signed, and trimming the object here is how a
|
|
1021
|
+
// caller silently loses the headers that make it valid.
|
|
1022
|
+
emitJson({
|
|
1023
|
+
path: filePath,
|
|
1024
|
+
url: cap.url,
|
|
1025
|
+
method: cap.method,
|
|
1026
|
+
expiresAt: cap.expiresAt,
|
|
1027
|
+
allowedHeaders: cap.allowedHeaders ?? {},
|
|
1028
|
+
supportsRange: cap.supportsRange,
|
|
1029
|
+
filename: cap.filename ?? null,
|
|
1030
|
+
// `?? null` rather than the raw value: an absent field would make
|
|
1031
|
+
// JSON.stringify drop the key entirely, so a legal response would
|
|
1032
|
+
// change the SHAPE of our output. A consumer reading this should see
|
|
1033
|
+
// the same keys every time and read the absence in the value.
|
|
1034
|
+
mediaType: cap.mediaType ?? null,
|
|
1035
|
+
delivery: cap.delivery,
|
|
1036
|
+
observedSizeBytes: cap.observedSizeBytes,
|
|
1037
|
+
});
|
|
1038
|
+
return 0;
|
|
1039
|
+
}
|
|
1040
|
+
/*
|
|
1041
|
+
* Fail closed rather than print half a credential.
|
|
1042
|
+
*
|
|
1043
|
+
* `allowedHeaders` is part of what was signed, so a URL printed without it
|
|
1044
|
+
* is not a weaker version of the capability — it is a request the server
|
|
1045
|
+
* did not authorize, and it fails at the provider with an opaque signature
|
|
1046
|
+
* error that reads like a permissions problem. Printing it would hand the
|
|
1047
|
+
* user something that looks usable and is not.
|
|
1048
|
+
*
|
|
1049
|
+
* No live connector emits headers today. This exists so the day one does,
|
|
1050
|
+
* it is a refusal here rather than a mystery three layers down.
|
|
1051
|
+
*/
|
|
1052
|
+
if (requiresHeaders(cap)) {
|
|
1053
|
+
fail("this capability requires request headers, so a bare URL cannot be used — " +
|
|
1054
|
+
"run with --json to get the full capability, or use 'dreamlake source download'");
|
|
1055
|
+
return 1;
|
|
1056
|
+
}
|
|
1057
|
+
// URL alone on stdout so `$(dreamlake source fetch …)` is directly usable.
|
|
1058
|
+
process.stdout.write(`${cap.url}\n`);
|
|
1059
|
+
process.stderr.write(`→ valid ${expiryNote(cap.expiresAt)}\n`);
|
|
1060
|
+
return 0;
|
|
1061
|
+
}
|
|
1062
|
+
catch (err) {
|
|
1063
|
+
fail(explain(err));
|
|
1064
|
+
return 1;
|
|
1065
|
+
}
|
|
1066
|
+
}
|
|
1067
|
+
// ─── source download ──────────────────────────────────────────────────────────
|
|
1068
|
+
/**
|
|
1069
|
+
* Abort a transfer that has gone quiet, rather than one that is merely long.
|
|
1070
|
+
*
|
|
1071
|
+
* A fixed deadline is the wrong shape for a download: a healthy 300 MB file
|
|
1072
|
+
* over a slow link legitimately outruns any constant, while a dead socket can
|
|
1073
|
+
* hang forever inside one. This fires only when no bytes have arrived for
|
|
1074
|
+
* STALL_TIMEOUT_MS.
|
|
1075
|
+
*/
|
|
1076
|
+
const STALL_TIMEOUT_MS = 60_000;
|
|
1077
|
+
/** Stream one capability to disk, issuing EXACTLY the request that was signed.
|
|
1078
|
+
*
|
|
1079
|
+
* `capability.method` and `capability.allowedHeaders` are sent verbatim, and
|
|
1080
|
+
* nothing else is added — no Authorization in particular. `allowedHeaders` is
|
|
1081
|
+
* exhaustive by contract, so a call site that reaches for the url and writes
|
|
1082
|
+
* its own request is sending a different one from the one the server signed.
|
|
1083
|
+
*
|
|
1084
|
+
* Byte counting goes through a Transform rather than a `data` listener,
|
|
1085
|
+
* because attaching one would flip the stream into flowing mode and race
|
|
1086
|
+
* `pipeline` for the first chunks.
|
|
1087
|
+
*
|
|
1088
|
+
* Bytes land in a sibling .part file and are renamed into place only after
|
|
1089
|
+
* the stream closes cleanly. An interrupted download previously left a
|
|
1090
|
+
* truncated file at the destination — indistinguishable from a real one, so
|
|
1091
|
+
* a 226 MB fragment of a 286 MB video sat there looking downloaded. */
|
|
1092
|
+
async function streamTo(cap, dest, hooks) {
|
|
1093
|
+
const resolved = path.resolve(dest);
|
|
1094
|
+
mkdirSync(path.dirname(resolved), { recursive: true });
|
|
1095
|
+
const tmp = `${resolved}.${process.pid}.part`;
|
|
1096
|
+
const abort = new AbortController();
|
|
1097
|
+
let watchdog;
|
|
1098
|
+
const kick = () => {
|
|
1099
|
+
clearTimeout(watchdog);
|
|
1100
|
+
watchdog = setTimeout(() => abort.abort(new Error(`stalled — no data for ${STALL_TIMEOUT_MS / 1000}s`)), STALL_TIMEOUT_MS);
|
|
1101
|
+
};
|
|
1102
|
+
try {
|
|
1103
|
+
kick();
|
|
1104
|
+
const res = await requestRaw(cap.url, {
|
|
1105
|
+
method: cap.method,
|
|
1106
|
+
headers: cap.allowedHeaders ?? {},
|
|
1107
|
+
signal: abort.signal,
|
|
1108
|
+
// A capability's headers are a bearer secret, and Node forwards custom
|
|
1109
|
+
// headers across an origin change. Following a redirect would hand them
|
|
1110
|
+
// to whatever host it names, so a capability that needs headers does not
|
|
1111
|
+
// get to be redirected. Header-free capabilities are unaffected — there
|
|
1112
|
+
// is nothing to leak, and providers legitimately redirect.
|
|
1113
|
+
followRedirects: !requiresHeaders(cap),
|
|
1114
|
+
});
|
|
1115
|
+
if (!res.body)
|
|
1116
|
+
throw new Error("empty response body");
|
|
1117
|
+
// Content-Length is what THIS response promised, so it is the only figure
|
|
1118
|
+
// the completeness check below may use. `observedSizeBytes` is what the
|
|
1119
|
+
// server saw while signing — good enough to give the bar a total when the
|
|
1120
|
+
// response carries no length, and it costs no extra request.
|
|
1121
|
+
const len = Number(res.headers.get("content-length"));
|
|
1122
|
+
if (Number.isFinite(len) && len > 0)
|
|
1123
|
+
hooks?.onTotal?.(len);
|
|
1124
|
+
else if (cap.observedSizeBytes !== null && cap.observedSizeBytes > 0) {
|
|
1125
|
+
hooks?.onTotal?.(cap.observedSizeBytes);
|
|
1126
|
+
}
|
|
1127
|
+
const nodeStream = Readable.fromWeb(res.body);
|
|
1128
|
+
const counter = new Transform({
|
|
1129
|
+
transform(chunk, _enc, cb) {
|
|
1130
|
+
kick();
|
|
1131
|
+
hooks?.onChunk?.(chunk.length);
|
|
1132
|
+
cb(null, chunk);
|
|
1133
|
+
},
|
|
1134
|
+
});
|
|
1135
|
+
await pipeline(nodeStream, counter, createWriteStream(tmp));
|
|
1136
|
+
// Short read with a known length is a silent corruption otherwise: the
|
|
1137
|
+
// stream can end early without erroring.
|
|
1138
|
+
if (Number.isFinite(len) && len > 0) {
|
|
1139
|
+
const got = statSync(tmp).size;
|
|
1140
|
+
if (got !== len) {
|
|
1141
|
+
throw new Error(`incomplete download — got ${got} of ${len} bytes`);
|
|
1142
|
+
}
|
|
1143
|
+
}
|
|
1144
|
+
renameSync(tmp, resolved);
|
|
1145
|
+
}
|
|
1146
|
+
catch (err) {
|
|
1147
|
+
rmSync(tmp, { force: true });
|
|
1148
|
+
throw err;
|
|
1149
|
+
}
|
|
1150
|
+
finally {
|
|
1151
|
+
clearTimeout(watchdog);
|
|
1152
|
+
}
|
|
1153
|
+
}
|
|
1154
|
+
export async function runSourceDownload(pathArg, opts) {
|
|
1155
|
+
const root = pathArg;
|
|
1156
|
+
const bad = checkPath(root);
|
|
1157
|
+
if (bad) {
|
|
1158
|
+
fail(bad);
|
|
1159
|
+
return 1;
|
|
1160
|
+
}
|
|
1161
|
+
const c = await ctx(opts.namespace);
|
|
1162
|
+
if (!c)
|
|
1163
|
+
return 1;
|
|
1164
|
+
// Default to the working directory — the same place wget and curl -O put
|
|
1165
|
+
// things, and the place people look first.
|
|
1166
|
+
const outDir = path.resolve(opts.output ?? process.cwd());
|
|
1167
|
+
try {
|
|
1168
|
+
const src = await resolveSource(c, opts.source);
|
|
1169
|
+
// Build the (remote path → local path) plan first, so a bad root fails
|
|
1170
|
+
// before anything is written.
|
|
1171
|
+
let plan;
|
|
1172
|
+
if (opts.recursive) {
|
|
1173
|
+
const { entries, truncated } = await collect(c, src.id, root, {
|
|
1174
|
+
recursive: true,
|
|
1175
|
+
include: "files",
|
|
1176
|
+
});
|
|
1177
|
+
if (truncated) {
|
|
1178
|
+
fail(`listing truncated at ${MAX_ENTRIES} entries — narrow the path before downloading`);
|
|
1179
|
+
return 1;
|
|
1180
|
+
}
|
|
1181
|
+
if (entries.length === 0) {
|
|
1182
|
+
fail(`no files under '${root}' in source '${src.name}'`);
|
|
1183
|
+
return 1;
|
|
1184
|
+
}
|
|
1185
|
+
/*
|
|
1186
|
+
* Skip what the server already said it will not sign.
|
|
1187
|
+
*
|
|
1188
|
+
* A UI hint, never authorization — the mint re-checks everything from
|
|
1189
|
+
* scratch (adapters/types.ts:49). Used here only to avoid firing a
|
|
1190
|
+
* request per object that is already known to fail, and reported rather
|
|
1191
|
+
* than dropped so a short listing explains itself.
|
|
1192
|
+
*
|
|
1193
|
+
* Single-file downloads deliberately do NOT consult it: they never
|
|
1194
|
+
* browse, and fetching the hint would cost a round trip to pre-empt an
|
|
1195
|
+
* answer the mint gives anyway.
|
|
1196
|
+
*/
|
|
1197
|
+
const refused = entries.filter((e) => e.canDownload === false);
|
|
1198
|
+
if (refused.length > 0) {
|
|
1199
|
+
process.stderr.write(`→ skipping ${refused.length} file(s) this source does not offer for download\n`);
|
|
1200
|
+
}
|
|
1201
|
+
const downloadable = entries.filter((e) => e.canDownload !== false);
|
|
1202
|
+
if (downloadable.length === 0) {
|
|
1203
|
+
fail(`no downloadable files under '${root}' in source '${src.name}'`);
|
|
1204
|
+
return 1;
|
|
1205
|
+
}
|
|
1206
|
+
plan = downloadable.map((e) => ({
|
|
1207
|
+
remote: e.path,
|
|
1208
|
+
// Preserve the tree *below* the requested root, matching
|
|
1209
|
+
// `aws s3 cp --recursive` and the existing `dreamlake download`.
|
|
1210
|
+
local: path.join(outDir, path.relative(root, e.path) || path.basename(e.path)),
|
|
1211
|
+
size: e.size,
|
|
1212
|
+
}));
|
|
1213
|
+
}
|
|
1214
|
+
else {
|
|
1215
|
+
plan = [{ remote: root, local: path.join(outDir, path.basename(root)) }];
|
|
1216
|
+
}
|
|
1217
|
+
// Sizes come free from `browse` on most backends. When they don't, the
|
|
1218
|
+
// bar falls back to counting files, and a single-file transfer learns its
|
|
1219
|
+
// total from Content-Length — or from the capability's `observedSizeBytes`
|
|
1220
|
+
// when the response carries no length.
|
|
1221
|
+
const knownBytes = plan.reduce((sum, item) => sum + (item.size ?? 0), 0);
|
|
1222
|
+
const bar = new BarProgress(knownBytes, plan.length);
|
|
1223
|
+
const results = await pool(plan, CONCURRENCY, async (item) => {
|
|
1224
|
+
const cap = await readCapability(c, src.id, item.remote, "download");
|
|
1225
|
+
await streamTo(cap, item.local, {
|
|
1226
|
+
onTotal: (n) => bar.setTotal(n),
|
|
1227
|
+
onChunk: (n) => bar.addBytes(n),
|
|
1228
|
+
});
|
|
1229
|
+
bar.fileDone();
|
|
1230
|
+
return item;
|
|
1231
|
+
});
|
|
1232
|
+
const failed = results.filter((r) => !r.ok).length;
|
|
1233
|
+
if (failed === plan.length)
|
|
1234
|
+
bar.abort();
|
|
1235
|
+
else
|
|
1236
|
+
bar.finish();
|
|
1237
|
+
// Paths land on stdout only after the bar has released the line, so a
|
|
1238
|
+
// repainting bar never interleaves with the piped output.
|
|
1239
|
+
for (const r of results) {
|
|
1240
|
+
if (r.ok)
|
|
1241
|
+
process.stdout.write(`${displayPath(r.value.local)}\n`);
|
|
1242
|
+
}
|
|
1243
|
+
for (let i = 0; i < results.length; i++) {
|
|
1244
|
+
const r = results[i];
|
|
1245
|
+
if (!r.ok)
|
|
1246
|
+
fail(`${plan[i].remote}: ${explain(r.error)}`);
|
|
1247
|
+
}
|
|
1248
|
+
const done = plan.length - failed;
|
|
1249
|
+
process.stderr.write(`→ ${done}/${plan.length} file(s) downloaded to ${outDir}\n`);
|
|
1250
|
+
return failed > 0 ? 1 : 0;
|
|
1251
|
+
}
|
|
1252
|
+
catch (err) {
|
|
1253
|
+
fail(explain(err));
|
|
1254
|
+
return 1;
|
|
1255
|
+
}
|
|
1256
|
+
}
|
|
1257
|
+
// ─── registration ─────────────────────────────────────────────────────────────
|
|
1258
|
+
/**
|
|
1259
|
+
* Reject extra positional arguments, and say why they usually happen.
|
|
1260
|
+
*
|
|
1261
|
+
* Source paths carry spaces far more often than repo paths do — screen
|
|
1262
|
+
* recordings, exported assets, anything a person named in a GUI. Unquoted,
|
|
1263
|
+
* the shell splits one of those into several argv entries, Commander's default
|
|
1264
|
+
* is to ignore the extras, and the user gets a typed path error from the
|
|
1265
|
+
* server about a path they never typed.
|
|
1266
|
+
*/
|
|
1267
|
+
export function strictArgs(cmd) {
|
|
1268
|
+
return cmd.allowExcessArguments(false).configureOutput({
|
|
1269
|
+
writeErr: (str) => {
|
|
1270
|
+
process.stderr.write(str);
|
|
1271
|
+
if (str.includes("too many arguments")) {
|
|
1272
|
+
process.stderr.write("\nA path with spaces has to be quoted:\n" +
|
|
1273
|
+
' dreamlake source fetch --source my-src "folder/my file.mov"\n');
|
|
1274
|
+
}
|
|
1275
|
+
},
|
|
1276
|
+
});
|
|
1277
|
+
}
|
|
1278
|
+
export function registerSourceCommand(program) {
|
|
1279
|
+
const source = program
|
|
1280
|
+
.command("source")
|
|
1281
|
+
.description("create, import and read sources (S3, Hugging Face)");
|
|
1282
|
+
// ── source list ──
|
|
1283
|
+
collectionList(source, "list")
|
|
1284
|
+
.description("list connected sources in a namespace (one page)")
|
|
1285
|
+
.option("--limit <n>", `page size (1-${LIST_MAX_LIMIT})`)
|
|
1286
|
+
.option("--cursor <cursor>", "continue from the cursor printed by the previous page")
|
|
1287
|
+
.option("--name-contains <text>", "only sources whose name contains this")
|
|
1288
|
+
.option("--provider <id>", "only sources pointing at this provider (the product id, e.g. 'huggingface')")
|
|
1289
|
+
.option("--external", "only sources that have an external layer")
|
|
1290
|
+
.option("--no-external", "only sources with no external layer")
|
|
1291
|
+
.option("--lifecycle <state>", "only 'active' or only 'disabled'")
|
|
1292
|
+
.option("--namespace <slug>", "namespace slug (default: active login)")
|
|
1293
|
+
.option("--json", "emit JSON")
|
|
1294
|
+
.action(async (opts) =>
|
|
1295
|
+
/*
|
|
1296
|
+
* Commander folds `--external` and `--no-external` onto one `external`
|
|
1297
|
+
* boolean. Declaring the positive form FIRST is what keeps the default
|
|
1298
|
+
* absent — measured: no flag yields `{}`, `--external` yields `true`,
|
|
1299
|
+
* `--no-external` yields `false`. Declaring only the negated form would
|
|
1300
|
+
* default it to `true`, which would send `hasExternal=true` on every
|
|
1301
|
+
* unfiltered list and silently hide every managed-only Source. Absent has
|
|
1302
|
+
* to stay absent here: it means "either".
|
|
1303
|
+
*/
|
|
1304
|
+
process.exit(await runSourceList({ ...opts, hasExternal: opts.external })));
|
|
1305
|
+
// ── source connectors ──
|
|
1306
|
+
strictArgs(source.command("connectors"))
|
|
1307
|
+
.description("which connection kinds each source connector accepts")
|
|
1308
|
+
.option("--namespace <slug>", "namespace slug (default: active login)")
|
|
1309
|
+
.option("--json", "emit JSON")
|
|
1310
|
+
.action(async (opts) => process.exit(await runSourceConnectors(opts)));
|
|
1311
|
+
// ── source browse ──
|
|
1312
|
+
strictArgs(source.command("browse"))
|
|
1313
|
+
.description("list directories and files inside a source (one level; -r to recurse)")
|
|
1314
|
+
.argument("[path]", "path within the source (default: root)")
|
|
1315
|
+
.requiredOption("--source <name>", "source name (see 'dreamlake source list')")
|
|
1316
|
+
.option("--namespace <slug>", "namespace slug (default: active login)")
|
|
1317
|
+
.option("--dirs", "list only directories")
|
|
1318
|
+
.option("--files", "list only files (for piping into fetch/download)")
|
|
1319
|
+
.option("-r, --recursive", "descend into subdirectories")
|
|
1320
|
+
.option("-0, --print0", "NUL-separate paths (for `xargs -0`; safe with spaces)")
|
|
1321
|
+
.option("--json", "emit JSON")
|
|
1322
|
+
.action(async (pathArg, opts) => process.exit(await runSourceBrowse(pathArg, opts)));
|
|
1323
|
+
// ── source fetch ──
|
|
1324
|
+
strictArgs(source.command("fetch"))
|
|
1325
|
+
.description("mint a short-lived read capability for one file")
|
|
1326
|
+
.argument("<path>", "file path within the source (quote it if it has spaces)")
|
|
1327
|
+
.requiredOption("--source <name>", "source name (see 'dreamlake source list')")
|
|
1328
|
+
.option("--namespace <slug>", "namespace slug (default: active login)")
|
|
1329
|
+
.option("--json", "emit JSON")
|
|
1330
|
+
.action(async (pathArg, opts) => process.exit(await runSourceFetch(pathArg, opts)));
|
|
1331
|
+
// ── source download ──
|
|
1332
|
+
strictArgs(source.command("download"))
|
|
1333
|
+
.description("download a file (or a folder with -r) to local disk")
|
|
1334
|
+
.argument("<path>", "file or folder path within the source (quote it if it has spaces)")
|
|
1335
|
+
.requiredOption("--source <name>", "source name (see 'dreamlake source list')")
|
|
1336
|
+
.option("--namespace <slug>", "namespace slug (default: active login)")
|
|
1337
|
+
.option("-o, --output <dir>", "output directory (default: current directory)")
|
|
1338
|
+
.option("-r, --recursive", "download every file under <path>")
|
|
1339
|
+
.action(async (pathArg, opts) => process.exit(await runSourceDownload(pathArg, opts)));
|
|
1340
|
+
registerSourceWriteCommands(source);
|
|
1341
|
+
registerSourceUploadCommand(source);
|
|
1342
|
+
registerSourceDeleteCommands(source);
|
|
1343
|
+
registerSourceControlCommands(source);
|
|
1344
|
+
registerSourceAccessCommands(source);
|
|
1345
|
+
registerSourceShareCommands(source);
|
|
1346
|
+
}
|
|
1347
|
+
// Exported for tests.
|
|
1348
|
+
export { collect, checkPath, displayPath, pool, MAX_ENTRIES, };
|