@dreamlake/dreamlake-cli 0.88.0 → 0.89.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/access.d.ts +83 -0
- package/dist/api/access.js +77 -0
- package/dist/api/agent.d.ts +156 -0
- package/dist/api/agent.js +111 -0
- package/dist/api/browser.d.ts +90 -0
- package/dist/api/browser.js +102 -0
- package/dist/api/client.d.ts +53 -0
- package/dist/api/client.js +102 -0
- package/dist/api/common.d.ts +28 -0
- package/dist/api/common.js +16 -0
- package/dist/api/content.d.ts +195 -0
- package/dist/api/content.js +147 -0
- package/dist/api/execution.d.ts +84 -0
- package/dist/api/execution.js +73 -0
- package/dist/api/http.d.ts +19 -0
- package/dist/api/http.js +66 -0
- package/dist/api/index.d.ts +2330 -0
- package/dist/api/index.js +114 -0
- package/dist/api/infra.d.ts +2 -0
- package/dist/api/infra.js +2 -0
- package/dist/api/jobs.d.ts +2 -0
- package/dist/api/jobs.js +2 -0
- package/dist/api/layout.d.ts +2 -0
- package/dist/api/layout.js +2 -0
- package/dist/api/library-edit.d.ts +3 -0
- package/dist/api/library-edit.js +3 -0
- package/dist/api/library-manifest.d.ts +2 -0
- package/dist/api/library-manifest.js +2 -0
- package/dist/api/note-character-plan.d.ts +2 -0
- package/dist/api/note-character-plan.js +2 -0
- package/dist/api/note-editing.d.ts +2 -0
- package/dist/api/note-editing.js +2 -0
- package/dist/api/note-edits.d.ts +21 -0
- package/dist/api/note-edits.js +10 -0
- package/dist/api/note-events.d.ts +5 -0
- package/dist/api/note-events.js +11 -0
- package/dist/api/note-files.d.ts +75 -0
- package/dist/api/note-files.js +65 -0
- package/dist/api/note-html-read.d.ts +2 -0
- package/dist/api/note-html-read.js +2 -0
- package/dist/api/note-html.d.ts +2 -0
- package/dist/api/note-html.js +2 -0
- package/dist/api/note-toc.d.ts +2 -0
- package/dist/api/note-toc.js +2 -0
- package/dist/api/notes.d.ts +304 -0
- package/dist/api/notes.js +181 -0
- package/dist/api/open-resources.d.ts +27 -0
- package/dist/api/open-resources.js +102 -0
- package/dist/api/ownership.d.ts +12 -0
- package/dist/api/ownership.js +24 -0
- package/dist/api/platform.d.ts +83 -0
- package/dist/api/platform.js +56 -0
- package/dist/api/presigned-transfer.d.ts +2 -0
- package/dist/api/presigned-transfer.js +2 -0
- package/dist/api/registered-runs.d.ts +48 -0
- package/dist/api/registered-runs.js +33 -0
- package/dist/api/sessions.d.ts +299 -0
- package/dist/api/sessions.js +105 -0
- package/dist/api/storage.d.ts +46 -0
- package/dist/api/storage.js +34 -0
- package/dist/api/streams.d.ts +18 -0
- package/dist/api/streams.js +70 -0
- package/dist/api/vault.d.ts +93 -0
- package/dist/api/vault.js +79 -0
- package/dist/api/workspace.d.ts +180 -0
- package/dist/api/workspace.js +146 -0
- package/dist/cli/agent-identity.d.ts +4 -0
- package/dist/cli/agent-identity.js +21 -0
- package/dist/cli/agents/identity.d.ts +50 -0
- package/dist/cli/agents/identity.js +105 -0
- package/dist/cli/agents/index.d.ts +31 -0
- package/dist/cli/agents/index.js +580 -0
- package/dist/cli/agents/manage.d.ts +43 -0
- package/dist/cli/agents/manage.js +376 -0
- package/dist/cli/agents/spec.d.ts +104 -0
- package/dist/cli/agents/spec.js +280 -0
- package/dist/cli/agents/web.d.ts +7 -0
- package/dist/cli/agents/web.js +109 -0
- package/dist/cli/artifact/incremental.d.ts +21 -0
- package/dist/cli/artifact/incremental.js +80 -0
- package/dist/cli/artifact/index.d.ts +52 -0
- package/dist/cli/artifact/index.js +405 -0
- package/dist/cli/artifact/meta.d.ts +69 -0
- package/dist/cli/artifact/meta.js +189 -0
- package/dist/cli/auth/commands.d.ts +29 -0
- package/dist/cli/auth/commands.js +327 -0
- package/dist/cli/auth/constants.d.ts +13 -0
- package/dist/cli/auth/constants.js +18 -0
- package/dist/cli/auth/credentials.d.ts +37 -0
- package/dist/cli/auth/credentials.js +155 -0
- package/dist/cli/auth/device-flow.d.ts +32 -0
- package/dist/cli/auth/device-flow.js +154 -0
- package/dist/cli/auth/device-secret.d.ts +6 -0
- package/dist/cli/auth/device-secret.js +26 -0
- package/dist/cli/auth/machine.d.ts +15 -0
- package/dist/cli/auth/machine.js +126 -0
- package/dist/cli/auth/materialized.d.ts +32 -0
- package/dist/cli/auth/materialized.js +102 -0
- package/dist/cli/auth/migrate.d.ts +10 -0
- package/dist/cli/auth/migrate.js +75 -0
- package/dist/cli/client.d.ts +102 -0
- package/dist/cli/client.js +206 -0
- package/dist/cli/collection-list.d.ts +3 -0
- package/dist/cli/collection-list.js +25 -0
- package/dist/cli/compat/ctx.d.ts +26 -0
- package/dist/cli/compat/ctx.js +76 -0
- package/dist/cli/compat/dreamdb-wasm-embedded.d.ts +2 -0
- package/dist/cli/compat/dreamdb-wasm-embedded.js +8 -0
- package/dist/cli/compat/dreamdb-wasm.d.ts +1 -0
- package/dist/cli/compat/dreamdb-wasm.js +26 -0
- package/dist/cli/compat/dreamdb.d.ts +112 -0
- package/dist/cli/compat/dreamdb.js +219 -0
- package/dist/cli/compat/flags.d.ts +2 -0
- package/dist/cli/compat/flags.js +13 -0
- package/dist/cli/compat/http.d.ts +23 -0
- package/dist/cli/compat/http.js +137 -0
- package/dist/cli/compat/presignedTransfer.d.ts +139 -0
- package/dist/cli/compat/presignedTransfer.js +580 -0
- package/dist/cli/compat/s3-backend.d.ts +73 -0
- package/dist/cli/compat/s3-backend.js +132 -0
- package/dist/cli/config.d.ts +22 -0
- package/dist/cli/config.js +96 -0
- package/dist/cli/connection/dto.d.ts +159 -0
- package/dist/cli/connection/dto.js +497 -0
- package/dist/cli/connection/index.d.ts +50 -0
- package/dist/cli/connection/index.js +413 -0
- package/dist/cli/connection/oauth.d.ts +23 -0
- package/dist/cli/connection/oauth.js +290 -0
- package/dist/cli/connection/write.d.ts +47 -0
- package/dist/cli/connection/write.js +573 -0
- package/dist/cli/create/index.d.ts +19 -0
- package/dist/cli/create/index.js +209 -0
- package/dist/cli/declare/index.d.ts +29 -0
- package/dist/cli/declare/index.js +304 -0
- package/dist/cli/delete/index.d.ts +18 -0
- package/dist/cli/delete/index.js +232 -0
- package/dist/cli/download/index.d.ts +9 -0
- package/dist/cli/download/index.js +128 -0
- package/dist/cli/env/compose.d.ts +88 -0
- package/dist/cli/env/compose.js +402 -0
- package/dist/cli/env/index.d.ts +176 -0
- package/dist/cli/env/index.js +1011 -0
- package/dist/cli/experiments/index.d.ts +11 -0
- package/dist/cli/experiments/index.js +168 -0
- package/dist/cli/glob.d.ts +1 -0
- package/dist/cli/glob.js +45 -0
- package/dist/cli/graphql-helpers.d.ts +23 -0
- package/dist/cli/graphql-helpers.js +42 -0
- package/dist/cli/graphql.d.ts +10 -0
- package/dist/cli/graphql.js +47 -0
- package/dist/cli/help-examples.generated.d.ts +1 -0
- package/dist/cli/help-examples.generated.js +192 -0
- package/dist/cli/helpers.d.ts +8 -0
- package/dist/cli/helpers.js +106 -0
- package/dist/cli/hosts/api.d.ts +29 -0
- package/dist/cli/hosts/api.js +54 -0
- package/dist/cli/hosts/bootstrap-source.d.ts +1 -0
- package/dist/cli/hosts/bootstrap-source.js +2 -0
- package/dist/cli/hosts/catalog.d.ts +26 -0
- package/dist/cli/hosts/catalog.js +55 -0
- package/dist/cli/hosts/config.d.ts +45 -0
- package/dist/cli/hosts/config.js +179 -0
- package/dist/cli/hosts/credentials.d.ts +19 -0
- package/dist/cli/hosts/credentials.js +173 -0
- package/dist/cli/hosts/enroll.d.ts +17 -0
- package/dist/cli/hosts/enroll.js +109 -0
- package/dist/cli/hosts/enrollments.d.ts +25 -0
- package/dist/cli/hosts/enrollments.js +54 -0
- package/dist/cli/hosts/index.d.ts +6 -0
- package/dist/cli/hosts/index.js +279 -0
- package/dist/cli/hosts/inventory.d.ts +40 -0
- package/dist/cli/hosts/inventory.js +91 -0
- package/dist/cli/hosts/lifecycle.d.ts +15 -0
- package/dist/cli/hosts/lifecycle.js +108 -0
- package/dist/cli/hosts/metrics-source.d.ts +12 -0
- package/dist/cli/hosts/metrics-source.js +89 -0
- package/dist/cli/hosts/observations.d.ts +39 -0
- package/dist/cli/hosts/observations.js +102 -0
- package/dist/cli/hosts/remote.d.ts +5 -0
- package/dist/cli/hosts/remote.js +66 -0
- package/dist/cli/hosts/ssh.d.ts +63 -0
- package/dist/cli/hosts/ssh.js +276 -0
- package/dist/cli/index.d.ts +1 -0
- package/dist/cli/index.js +288 -0
- package/dist/cli/infra/api.d.ts +95 -0
- package/dist/cli/infra/api.js +291 -0
- package/dist/cli/infra/index.d.ts +5 -0
- package/dist/cli/infra/index.js +102 -0
- package/dist/cli/infra/move.d.ts +15 -0
- package/dist/cli/infra/move.js +123 -0
- package/dist/cli/infra/transfer.d.ts +15 -0
- package/dist/cli/infra/transfer.js +280 -0
- package/dist/cli/jobs/api.d.ts +69 -0
- package/dist/cli/jobs/api.js +121 -0
- package/dist/cli/jobs/index.d.ts +5 -0
- package/dist/cli/jobs/index.js +140 -0
- package/dist/cli/lakeshore/index.d.ts +98 -0
- package/dist/cli/lakeshore/index.js +502 -0
- package/dist/cli/launch/index.d.ts +43 -0
- package/dist/cli/launch/index.js +270 -0
- package/dist/cli/layout/index.d.ts +2 -0
- package/dist/cli/layout/index.js +44 -0
- package/dist/cli/layout/transport.d.ts +12 -0
- package/dist/cli/layout/transport.js +70 -0
- package/dist/cli/library/add.d.ts +60 -0
- package/dist/cli/library/add.js +348 -0
- package/dist/cli/library/cache.d.ts +26 -0
- package/dist/cli/library/cache.js +88 -0
- package/dist/cli/library/config.d.ts +75 -0
- package/dist/cli/library/config.js +198 -0
- package/dist/cli/library/digest.d.ts +5 -0
- package/dist/cli/library/digest.js +19 -0
- package/dist/cli/library/discover.d.ts +87 -0
- package/dist/cli/library/discover.js +534 -0
- package/dist/cli/library/freshness.d.ts +35 -0
- package/dist/cli/library/freshness.js +109 -0
- package/dist/cli/library/generation.d.ts +35 -0
- package/dist/cli/library/generation.js +211 -0
- package/dist/cli/library/index.d.ts +2 -0
- package/dist/cli/library/index.js +36 -0
- package/dist/cli/library/info.d.ts +69 -0
- package/dist/cli/library/info.js +311 -0
- package/dist/cli/library/list.d.ts +33 -0
- package/dist/cli/library/list.js +86 -0
- package/dist/cli/library/manifest.d.ts +81 -0
- package/dist/cli/library/manifest.js +200 -0
- package/dist/cli/library/manifestCache.d.ts +45 -0
- package/dist/cli/library/manifestCache.js +169 -0
- package/dist/cli/library/pull.d.ts +30 -0
- package/dist/cli/library/pull.js +438 -0
- package/dist/cli/library/push.d.ts +72 -0
- package/dist/cli/library/push.js +642 -0
- package/dist/cli/library/rm.d.ts +31 -0
- package/dist/cli/library/rm.js +200 -0
- package/dist/cli/library/search.d.ts +18 -0
- package/dist/cli/library/search.js +181 -0
- package/dist/cli/library/stat.d.ts +15 -0
- package/dist/cli/library/stat.js +127 -0
- package/dist/cli/library/tools.d.ts +104 -0
- package/dist/cli/library/tools.js +163 -0
- package/dist/cli/library/wire.d.ts +47 -0
- package/dist/cli/library/wire.js +120 -0
- package/dist/cli/list/index.d.ts +26 -0
- package/dist/cli/list/index.js +256 -0
- package/dist/cli/notes/character-plan.d.ts +22 -0
- package/dist/cli/notes/character-plan.js +232 -0
- package/dist/cli/notes/document.d.ts +73 -0
- package/dist/cli/notes/document.js +306 -0
- package/dist/cli/notes/edit-commands.d.ts +106 -0
- package/dist/cli/notes/edit-commands.js +623 -0
- package/dist/cli/notes/editing.d.ts +81 -0
- package/dist/cli/notes/editing.js +305 -0
- package/dist/cli/notes/events.d.ts +36 -0
- package/dist/cli/notes/events.js +54 -0
- package/dist/cli/notes/files-commands.d.ts +120 -0
- package/dist/cli/notes/files-commands.js +554 -0
- package/dist/cli/notes/html-read.d.ts +9 -0
- package/dist/cli/notes/html-read.js +90 -0
- package/dist/cli/notes/html.d.ts +73 -0
- package/dist/cli/notes/html.js +381 -0
- package/dist/cli/notes/index.d.ts +123 -0
- package/dist/cli/notes/index.js +1303 -0
- package/dist/cli/notes/linger.d.ts +59 -0
- package/dist/cli/notes/linger.js +339 -0
- package/dist/cli/notes/media-commands.d.ts +5 -0
- package/dist/cli/notes/media-commands.js +51 -0
- package/dist/cli/notes/move-command.d.ts +83 -0
- package/dist/cli/notes/move-command.js +259 -0
- package/dist/cli/notes/rm-command.d.ts +22 -0
- package/dist/cli/notes/rm-command.js +110 -0
- package/dist/cli/notes/select-command.d.ts +12 -0
- package/dist/cli/notes/select-command.js +84 -0
- package/dist/cli/notes/suggest-diff.d.ts +56 -0
- package/dist/cli/notes/suggest-diff.js +311 -0
- package/dist/cli/notes/toc.d.ts +38 -0
- package/dist/cli/notes/toc.js +144 -0
- package/dist/cli/notes/transfer-command.d.ts +6 -0
- package/dist/cli/notes/transfer-command.js +9 -0
- package/dist/cli/notes/version-commands.d.ts +19 -0
- package/dist/cli/notes/version-commands.js +178 -0
- package/dist/cli/nymph/index.d.ts +74 -0
- package/dist/cli/nymph/index.js +349 -0
- package/dist/cli/org/index.d.ts +34 -0
- package/dist/cli/org/index.js +341 -0
- package/dist/cli/ownership/index.d.ts +12 -0
- package/dist/cli/ownership/index.js +122 -0
- package/dist/cli/ownership/metadata-transfer.d.ts +20 -0
- package/dist/cli/ownership/metadata-transfer.js +159 -0
- package/dist/cli/page/index.d.ts +58 -0
- package/dist/cli/page/index.js +387 -0
- package/dist/cli/pipeline/index.d.ts +69 -0
- package/dist/cli/pipeline/index.js +604 -0
- package/dist/cli/progress.d.ts +60 -0
- package/dist/cli/progress.js +276 -0
- package/dist/cli/project/attachments.d.ts +10 -0
- package/dist/cli/project/attachments.js +39 -0
- package/dist/cli/project/grants.d.ts +10 -0
- package/dist/cli/project/grants.js +39 -0
- package/dist/cli/project/index.d.ts +8 -0
- package/dist/cli/project/index.js +92 -0
- package/dist/cli/project/move.d.ts +57 -0
- package/dist/cli/project/move.js +242 -0
- package/dist/cli/project/output.d.ts +2 -0
- package/dist/cli/project/output.js +29 -0
- package/dist/cli/project/stream.d.ts +55 -0
- package/dist/cli/project/stream.js +207 -0
- package/dist/cli/project/transfer-preview.d.ts +8 -0
- package/dist/cli/project/transfer-preview.js +10 -0
- package/dist/cli/prompt.d.ts +1 -0
- package/dist/cli/prompt.js +17 -0
- package/dist/cli/provider/dreamdir.d.ts +52 -0
- package/dist/cli/provider/dreamdir.js +138 -0
- package/dist/cli/provider/index.d.ts +4 -0
- package/dist/cli/provider/index.js +419 -0
- package/dist/cli/provider/jump-worker.d.ts +9 -0
- package/dist/cli/provider/jump-worker.js +141 -0
- package/dist/cli/provider/lifecycle.d.ts +149 -0
- package/dist/cli/provider/lifecycle.js +1390 -0
- package/dist/cli/provider/machine.d.ts +17 -0
- package/dist/cli/provider/machine.js +116 -0
- package/dist/cli/provider/ssh.d.ts +105 -0
- package/dist/cli/provider/ssh.js +305 -0
- package/dist/cli/provider/terraform.d.ts +140 -0
- package/dist/cli/provider/terraform.js +288 -0
- package/dist/cli/providers/index.d.ts +18 -0
- package/dist/cli/providers/index.js +422 -0
- package/dist/cli/python.d.ts +61 -0
- package/dist/cli/python.js +255 -0
- package/dist/cli/queues/index.d.ts +33 -0
- package/dist/cli/queues/index.js +248 -0
- package/dist/cli/queues/slurm.d.ts +10 -0
- package/dist/cli/queues/slurm.js +134 -0
- package/dist/cli/resources.d.ts +72 -0
- package/dist/cli/resources.js +137 -0
- package/dist/cli/run/index.d.ts +15 -0
- package/dist/cli/run/index.js +163 -0
- package/dist/cli/run/input.d.ts +34 -0
- package/dist/cli/run/input.js +135 -0
- package/dist/cli/run/recovery.d.ts +2 -0
- package/dist/cli/run/recovery.js +18 -0
- package/dist/cli/run/setup.d.ts +3 -0
- package/dist/cli/run/setup.js +62 -0
- package/dist/cli/runconfig/index.d.ts +12 -0
- package/dist/cli/runconfig/index.js +130 -0
- package/dist/cli/runnable/index.d.ts +35 -0
- package/dist/cli/runnable/index.js +575 -0
- package/dist/cli/runnable/path.d.ts +115 -0
- package/dist/cli/runnable/path.js +288 -0
- package/dist/cli/runnable/run.d.ts +91 -0
- package/dist/cli/runnable/run.js +239 -0
- package/dist/cli/runnable/upload.d.ts +43 -0
- package/dist/cli/runnable/upload.js +342 -0
- package/dist/cli/runnable/worker.d.ts +1 -0
- package/dist/cli/runnable/worker.js +44 -0
- package/dist/cli/schemas/workflow-spec.schema.json +335 -0
- package/dist/cli/self-update/index.d.ts +5 -0
- package/dist/cli/self-update/index.js +107 -0
- package/dist/cli/self-update/install.d.ts +60 -0
- package/dist/cli/self-update/install.js +428 -0
- package/dist/cli/self-update/paths.d.ts +40 -0
- package/dist/cli/self-update/paths.js +87 -0
- package/dist/cli/self-update/releases.d.ts +41 -0
- package/dist/cli/self-update/releases.js +153 -0
- package/dist/cli/self-update/settings.d.ts +16 -0
- package/dist/cli/self-update/settings.js +68 -0
- package/dist/cli/self-update/update.d.ts +18 -0
- package/dist/cli/self-update/update.js +241 -0
- package/dist/cli/service-accounts/index.d.ts +3 -0
- package/dist/cli/service-accounts/index.js +149 -0
- package/dist/cli/session-affiliation.d.ts +26 -0
- package/dist/cli/session-affiliation.js +135 -0
- package/dist/cli/sessions/affiliation.d.ts +8 -0
- package/dist/cli/sessions/affiliation.js +66 -0
- package/dist/cli/sessions/api.d.ts +55 -0
- package/dist/cli/sessions/api.js +88 -0
- package/dist/cli/sessions/control.d.ts +6 -0
- package/dist/cli/sessions/control.js +159 -0
- package/dist/cli/sessions/conversation.d.ts +15 -0
- package/dist/cli/sessions/conversation.js +118 -0
- package/dist/cli/sessions/index.d.ts +2 -0
- package/dist/cli/sessions/index.js +209 -0
- package/dist/cli/sessions/managed.d.ts +39 -0
- package/dist/cli/sessions/managed.js +63 -0
- package/dist/cli/sessions/redaction.d.ts +2 -0
- package/dist/cli/sessions/redaction.js +11 -0
- package/dist/cli/sessions/runner/exec.d.ts +108 -0
- package/dist/cli/sessions/runner/exec.js +335 -0
- package/dist/cli/sessions/runner/journal.d.ts +26 -0
- package/dist/cli/sessions/runner/journal.js +49 -0
- package/dist/cli/sessions/runner/native.d.ts +48 -0
- package/dist/cli/sessions/runner/native.js +170 -0
- package/dist/cli/sessions/runner/pod.d.ts +30 -0
- package/dist/cli/sessions/runner/pod.js +129 -0
- package/dist/cli/sessions/runner/serve.d.ts +85 -0
- package/dist/cli/sessions/runner/serve.js +558 -0
- package/dist/cli/sessions/sync.d.ts +47 -0
- package/dist/cli/sessions/sync.js +80 -0
- package/dist/cli/sessions/transport.d.ts +14 -0
- package/dist/cli/sessions/transport.js +116 -0
- package/dist/cli/share.d.ts +15 -0
- package/dist/cli/share.js +139 -0
- package/dist/cli/skill/catalog.d.ts +141 -0
- package/dist/cli/skill/catalog.js +442 -0
- package/dist/cli/skill/files.d.ts +89 -0
- package/dist/cli/skill/files.js +264 -0
- package/dist/cli/skill/index.d.ts +268 -0
- package/dist/cli/skill/index.js +1216 -0
- package/dist/cli/skill/manifest.d.ts +30 -0
- package/dist/cli/skill/manifest.js +101 -0
- package/dist/cli/skill/prefs.d.ts +9 -0
- package/dist/cli/skill/prefs.js +32 -0
- package/dist/cli/skill/reminders.d.ts +15 -0
- package/dist/cli/skill/reminders.js +105 -0
- package/dist/cli/skill/tar.d.ts +17 -0
- package/dist/cli/skill/tar.js +195 -0
- package/dist/cli/snapshot/index.d.ts +24 -0
- package/dist/cli/snapshot/index.js +146 -0
- package/dist/cli/source/access.d.ts +45 -0
- package/dist/cli/source/access.js +563 -0
- package/dist/cli/source/control.d.ts +103 -0
- package/dist/cli/source/control.js +546 -0
- package/dist/cli/source/delete.d.ts +89 -0
- package/dist/cli/source/delete.js +463 -0
- package/dist/cli/source/index.d.ts +288 -0
- package/dist/cli/source/index.js +1348 -0
- package/dist/cli/source/share.d.ts +65 -0
- package/dist/cli/source/share.js +530 -0
- package/dist/cli/source/upload.d.ts +62 -0
- package/dist/cli/source/upload.js +258 -0
- package/dist/cli/source/write.d.ts +77 -0
- package/dist/cli/source/write.js +335 -0
- package/dist/cli/spawn.d.ts +28 -0
- package/dist/cli/spawn.js +81 -0
- package/dist/cli/target.d.ts +22 -0
- package/dist/cli/target.js +85 -0
- package/dist/cli/tasks/index.d.ts +38 -0
- package/dist/cli/tasks/index.js +365 -0
- package/dist/cli/team/index.d.ts +37 -0
- package/dist/cli/team/index.js +407 -0
- package/dist/cli/telemetry.d.ts +23 -0
- package/dist/cli/telemetry.js +262 -0
- package/dist/cli/telemetryJournal.d.ts +26 -0
- package/dist/cli/telemetryJournal.js +104 -0
- package/dist/cli/update/index.d.ts +21 -0
- package/dist/cli/update/index.js +284 -0
- package/dist/cli/upload/index.d.ts +11 -0
- package/dist/cli/upload/index.js +261 -0
- package/dist/cli/upload/kinds.d.ts +10 -0
- package/dist/cli/upload/kinds.js +85 -0
- package/dist/cli/upload/multipart.d.ts +21 -0
- package/dist/cli/upload/multipart.js +248 -0
- package/dist/cli/vault/affected-preview.d.ts +15 -0
- package/dist/cli/vault/affected-preview.js +38 -0
- package/dist/cli/vault/cloud.d.ts +55 -0
- package/dist/cli/vault/cloud.js +287 -0
- package/dist/cli/vault/format.d.ts +18 -0
- package/dist/cli/vault/format.js +104 -0
- package/dist/cli/vault/hostCredentials.d.ts +21 -0
- package/dist/cli/vault/hostCredentials.js +71 -0
- package/dist/cli/vault/hostSupersessions.d.ts +18 -0
- package/dist/cli/vault/hostSupersessions.js +68 -0
- package/dist/cli/vault/import.d.ts +3 -0
- package/dist/cli/vault/import.js +79 -0
- package/dist/cli/vault/index.d.ts +14 -0
- package/dist/cli/vault/index.js +225 -0
- package/dist/cli/vault/keys.d.ts +5 -0
- package/dist/cli/vault/keys.js +165 -0
- package/dist/cli/vault/kms.d.ts +6 -0
- package/dist/cli/vault/kms.js +169 -0
- package/dist/cli/vault/list.d.ts +13 -0
- package/dist/cli/vault/list.js +60 -0
- package/dist/cli/vault/listView.d.ts +21 -0
- package/dist/cli/vault/listView.js +123 -0
- package/dist/cli/vault/managementMetadata.d.ts +2 -0
- package/dist/cli/vault/managementMetadata.js +23 -0
- package/dist/cli/vault/mutations.d.ts +8 -0
- package/dist/cli/vault/mutations.js +151 -0
- package/dist/cli/vault/otp.d.ts +6 -0
- package/dist/cli/vault/otp.js +62 -0
- package/dist/cli/vault/otpIntent.d.ts +23 -0
- package/dist/cli/vault/otpIntent.js +76 -0
- package/dist/cli/vault/passImport.d.ts +25 -0
- package/dist/cli/vault/passImport.js +129 -0
- package/dist/cli/vault/passOtp.d.ts +51 -0
- package/dist/cli/vault/passOtp.js +101 -0
- package/dist/cli/vault/passStore.d.ts +26 -0
- package/dist/cli/vault/passStore.js +134 -0
- package/dist/cli/vault/passwordCommand.d.ts +39 -0
- package/dist/cli/vault/passwordCommand.js +101 -0
- package/dist/cli/vault/passwordProbe.d.ts +27 -0
- package/dist/cli/vault/passwordProbe.js +282 -0
- package/dist/cli/vault/passwordRotation.d.ts +9 -0
- package/dist/cli/vault/passwordRotation.js +273 -0
- package/dist/cli/vault/rotationApi.d.ts +13 -0
- package/dist/cli/vault/rotationApi.js +58 -0
- package/dist/cli/vault/rotationCommand.d.ts +2 -0
- package/dist/cli/vault/rotationCommand.js +75 -0
- package/dist/cli/vault/rotationEngine.d.ts +19 -0
- package/dist/cli/vault/rotationEngine.js +65 -0
- package/dist/cli/vault/rotationHelperSource.d.ts +2 -0
- package/dist/cli/vault/rotationHelperSource.js +3 -0
- package/dist/cli/vault/rotationInput.d.ts +11 -0
- package/dist/cli/vault/rotationInput.js +48 -0
- package/dist/cli/vault/rotationJournal.d.ts +16 -0
- package/dist/cli/vault/rotationJournal.js +183 -0
- package/dist/cli/vault/rotationKeys.d.ts +4 -0
- package/dist/cli/vault/rotationKeys.js +48 -0
- package/dist/cli/vault/rotationPrepare.d.ts +17 -0
- package/dist/cli/vault/rotationPrepare.js +108 -0
- package/dist/cli/vault/rotationRemote.d.ts +33 -0
- package/dist/cli/vault/rotationRemote.js +156 -0
- package/dist/cli/vault/rotationState.d.ts +42 -0
- package/dist/cli/vault/rotationState.js +115 -0
- package/dist/cli/vault/rotationTransport.d.ts +18 -0
- package/dist/cli/vault/rotationTransport.js +71 -0
- package/dist/cli/vault/sshSource.d.ts +18 -0
- package/dist/cli/vault/sshSource.js +196 -0
- package/dist/cli/vault/sshSync.d.ts +17 -0
- package/dist/cli/vault/sshSync.js +224 -0
- package/dist/cli/vault/sshTransfer.d.ts +22 -0
- package/dist/cli/vault/sshTransfer.js +140 -0
- package/dist/cli/vault/terraformState.d.ts +66 -0
- package/dist/cli/vault/terraformState.js +252 -0
- package/dist/cli/vault/terraformStateCommand.d.ts +2 -0
- package/dist/cli/vault/terraformStateCommand.js +61 -0
- package/dist/cli/vault/tree.d.ts +43 -0
- package/dist/cli/vault/tree.js +292 -0
- package/dist/cli/version.d.ts +1 -0
- package/dist/cli/version.js +9 -0
- package/dist/cli/workflow/index.d.ts +76 -0
- package/dist/cli/workflow/index.js +826 -0
- package/dist/cli/workflow/validate.d.ts +19 -0
- package/dist/cli/workflow/validate.js +353 -0
- package/dist/package.json +1 -0
- package/package.json +28 -10
|
@@ -0,0 +1,1011 @@
|
|
|
1
|
+
import { collectionList } from "../collection-list.js";
|
|
2
|
+
// `dreamlake scene push|create|list|pull|delete|restore` — simulation
|
|
3
|
+
// environments (one entry file — MuJoCo MJCF or URDF — plus asset files)
|
|
4
|
+
// under a namespace.
|
|
5
|
+
//
|
|
6
|
+
// push writes the version itself: broker credentials from dreamlake-server,
|
|
7
|
+
// then append straight to the env's two DreamDB datasets in S3
|
|
8
|
+
// (refs/blobs — one row per unique file, refs/manifests — one row per
|
|
9
|
+
// version), then register the catalog row. Content never transits the API
|
|
10
|
+
// server, and unchanged files are never re-uploaded: the blobs dataset's
|
|
11
|
+
// `hash` column is the dedup set (one track read, no per-file probes).
|
|
12
|
+
//
|
|
13
|
+
// pull is PURE REST — no DreamDB involved: the server resolves the manifest
|
|
14
|
+
// (GET .../versions/:v) and hands back presigned S3 URLs; this side just
|
|
15
|
+
// downloads and hash-verifies. Any HTTP client could do the same.
|
|
16
|
+
//
|
|
17
|
+
// (`dreamlake env` used to be the login-environment switcher; that moved to
|
|
18
|
+
// `dreamlake auth env` when MuJoCo envs took the name.)
|
|
19
|
+
import { mkdirSync, readdirSync, readFileSync, lstatSync, openSync, readSync, closeSync, writeFileSync, existsSync } from "node:fs";
|
|
20
|
+
import path from "node:path";
|
|
21
|
+
import { confirm } from "../prompt.js";
|
|
22
|
+
import { emitJson, fail, ok, warn, humanSize } from "../helpers.js";
|
|
23
|
+
import { resolveCtx } from "../compat/ctx.js";
|
|
24
|
+
import { errorMessage, jsonRequest, requireNamespace, requireToken } from "../compat/http.js";
|
|
25
|
+
import { withConnFlags } from "../compat/flags.js";
|
|
26
|
+
import { webBase } from "../workflow/index.js";
|
|
27
|
+
import { VERSION } from "../version.js";
|
|
28
|
+
import { ENV_BLOB_SCHEMA, ENV_MANIFEST_SCHEMA, backendFrom, isPublishConflict, multihashBase32, realEnvDreamdbIO, } from "../compat/dreamdb.js";
|
|
29
|
+
import { runEnvCompose, srcIsLocal, STACK_BASENAME } from "./compose.js";
|
|
30
|
+
// ─── limits & names ──────────────────────────────────────────────────
|
|
31
|
+
//
|
|
32
|
+
// Server-side mirrors: dreamlake-server/src/services/envStorage.ts
|
|
33
|
+
// (ENV_NAME_RE / reserved names) — keep byte-identical.
|
|
34
|
+
export const ENV_NAME_RE = /^[a-z0-9][a-z0-9._-]{0,63}$/;
|
|
35
|
+
export const ENV_RESERVED_NAMES = new Set(["upload-credentials", "versions"]);
|
|
36
|
+
export const MAX_FILES = 1000;
|
|
37
|
+
export const MAX_FILE_BYTES = 100 * 1024 * 1024; // 100 MiB
|
|
38
|
+
export const MAX_TOTAL_BYTES = 1024 * 1024 * 1024; // 1 GiB
|
|
39
|
+
const APPEND_BATCH_BYTES = 64 * 1024 * 1024; // bounds wasm memory per commit
|
|
40
|
+
const CONFLICT_RETRIES = 3;
|
|
41
|
+
/** Server-side mirror: dreamlake-server/src/services/envResolve.ts
|
|
42
|
+
* ENV_THUMB_MAX_BYTES — decoded PNG cap for the /thumbnail endpoint. */
|
|
43
|
+
export const THUMB_MAX_BYTES = 512 * 1024;
|
|
44
|
+
const BLOBS_REF = "blobs";
|
|
45
|
+
const MANIFESTS_REF = "manifests";
|
|
46
|
+
/** Simulator family — free string; 'mujoco' and 'urdf' have web viewers.
|
|
47
|
+
* Server-side mirror: dreamlake-server/src/routes/envsShared.ts ENV_TYPE_RE. */
|
|
48
|
+
export const DEFAULT_ENV_TYPE = "mujoco";
|
|
49
|
+
export const URDF_ENV_TYPE = "urdf";
|
|
50
|
+
export const ENV_TYPE_RE = /^[a-z0-9][a-z0-9._-]{0,31}$/;
|
|
51
|
+
export function invalidEnvName(name) {
|
|
52
|
+
if (!ENV_NAME_RE.test(name))
|
|
53
|
+
return "scene name must match ^[a-z0-9][a-z0-9._-]{0,63}$";
|
|
54
|
+
if (ENV_RESERVED_NAMES.has(name))
|
|
55
|
+
return `'${name}' is a reserved name`;
|
|
56
|
+
return null;
|
|
57
|
+
}
|
|
58
|
+
/** Directory basename → a valid scene name ('env' when nothing survives). */
|
|
59
|
+
export function envSlug(text) {
|
|
60
|
+
let out = text
|
|
61
|
+
.toLowerCase()
|
|
62
|
+
.replace(/[^a-z0-9._-]/g, "-")
|
|
63
|
+
.replace(/^[^a-z0-9]+/, "")
|
|
64
|
+
.replace(/-+$/, "");
|
|
65
|
+
while (out.includes("--"))
|
|
66
|
+
out = out.replaceAll("--", "-");
|
|
67
|
+
out = out.slice(0, 64);
|
|
68
|
+
return ENV_NAME_RE.test(out) ? out : "env";
|
|
69
|
+
}
|
|
70
|
+
const SKIP_DIRS = new Set(["node_modules", "__pycache__"]);
|
|
71
|
+
/** Every regular file under `dir`, relative POSIX paths, sorted. Dot
|
|
72
|
+
* entries, node_modules/__pycache__ and symlinks are skipped (a symlink
|
|
73
|
+
* could smuggle content from outside the directory into the env). */
|
|
74
|
+
export function walkEnvDir(dir) {
|
|
75
|
+
const out = [];
|
|
76
|
+
const visit = (rel) => {
|
|
77
|
+
const abs = rel ? path.join(dir, rel) : dir;
|
|
78
|
+
for (const entry of readdirSync(abs, { withFileTypes: true })) {
|
|
79
|
+
if (entry.name.startsWith("."))
|
|
80
|
+
continue;
|
|
81
|
+
if (entry.isDirectory() && SKIP_DIRS.has(entry.name))
|
|
82
|
+
continue;
|
|
83
|
+
const childRel = rel ? `${rel}/${entry.name}` : entry.name;
|
|
84
|
+
const childAbs = path.join(dir, childRel);
|
|
85
|
+
// Dirent.isSymbolicLink covers the entry itself; lstat again for safety
|
|
86
|
+
// on platforms where readdir types are unavailable.
|
|
87
|
+
const st = lstatSync(childAbs);
|
|
88
|
+
if (st.isSymbolicLink())
|
|
89
|
+
continue;
|
|
90
|
+
if (st.isDirectory())
|
|
91
|
+
visit(childRel);
|
|
92
|
+
else if (st.isFile())
|
|
93
|
+
out.push({ rel: childRel, abs: childAbs, size: st.size });
|
|
94
|
+
}
|
|
95
|
+
};
|
|
96
|
+
visit("");
|
|
97
|
+
out.sort((a, b) => (a.rel < b.rel ? -1 : a.rel > b.rel ? 1 : 0));
|
|
98
|
+
return out;
|
|
99
|
+
}
|
|
100
|
+
/** Reject a manifest/pull path that could escape its directory. Server-side
|
|
101
|
+
* mirror: envResolve.invalidManifestPath. */
|
|
102
|
+
export function invalidRelPath(p) {
|
|
103
|
+
if (!p)
|
|
104
|
+
return "empty path";
|
|
105
|
+
if (p.length > 1024)
|
|
106
|
+
return "path too long";
|
|
107
|
+
if (p.startsWith("/") || p.includes("\\") || p.includes("://"))
|
|
108
|
+
return `invalid path '${p}'`;
|
|
109
|
+
if (p.split("/").some((seg) => seg === "" || seg === "." || seg === "..")) {
|
|
110
|
+
return `invalid path '${p}'`;
|
|
111
|
+
}
|
|
112
|
+
return null;
|
|
113
|
+
}
|
|
114
|
+
// ─── progress ────────────────────────────────────────────────────────
|
|
115
|
+
/** A \r-refreshed progress line on a TTY; silent otherwise so CI logs and
|
|
116
|
+
* the test harness stay clean. `done()` clears the line before the next
|
|
117
|
+
* ordinary print. */
|
|
118
|
+
function makeProgressLine() {
|
|
119
|
+
const tty = process.stdout.isTTY === true;
|
|
120
|
+
return {
|
|
121
|
+
update(text) {
|
|
122
|
+
if (tty)
|
|
123
|
+
process.stdout.write(`\r\x1b[2K ${text}`);
|
|
124
|
+
},
|
|
125
|
+
done() {
|
|
126
|
+
if (tty)
|
|
127
|
+
process.stdout.write("\r\x1b[2K");
|
|
128
|
+
},
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
// ─── entry detection ─────────────────────────────────────────────────
|
|
132
|
+
/** First 4 KiB of a file, as latin1 text (enough to spot `<mujoco`). */
|
|
133
|
+
function headText(abs) {
|
|
134
|
+
const fd = openSync(abs, "r");
|
|
135
|
+
try {
|
|
136
|
+
const buf = Buffer.alloc(4096);
|
|
137
|
+
const n = readSync(fd, buf, 0, 4096, 0);
|
|
138
|
+
return buf.subarray(0, n).toString("latin1");
|
|
139
|
+
}
|
|
140
|
+
finally {
|
|
141
|
+
closeSync(fd);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* The env's entry file plus the simulator family it implies.
|
|
146
|
+
*
|
|
147
|
+
* `--entry` wins verbatim (must be in the walk; a `.urdf` extension infers
|
|
148
|
+
* type urdf unless `--type` says otherwise). Without it: the single
|
|
149
|
+
* root-level *.xml/*.mjcf whose head contains `<mujoco` → mujoco; when no
|
|
150
|
+
* MJCF exists (or `--type urdf` was passed), the single root-level URDF —
|
|
151
|
+
* *.urdf whose head contains `<robot` (plain *.xml counts too under an
|
|
152
|
+
* explicit `--type urdf`; a mujoco directory may carry unrelated <robot>
|
|
153
|
+
* xmls, so auto-detection sticks to the .urdf extension). Zero or several
|
|
154
|
+
* candidates → an error listing them, demanding --entry.
|
|
155
|
+
*/
|
|
156
|
+
export function detectEntry(files, explicit, type) {
|
|
157
|
+
if (explicit) {
|
|
158
|
+
const rel = explicit.replaceAll("\\", "/").replace(/^\.\//, "");
|
|
159
|
+
if (!files.some((f) => f.rel === rel)) {
|
|
160
|
+
return { error: `--entry '${explicit}' is not a file inside the directory` };
|
|
161
|
+
}
|
|
162
|
+
return {
|
|
163
|
+
entry: rel,
|
|
164
|
+
envType: type ?? (/\.urdf$/i.test(rel) ? URDF_ENV_TYPE : DEFAULT_ENV_TYPE),
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
const rootLevel = files.filter((f) => !f.rel.includes("/"));
|
|
168
|
+
const urdfCandidates = (xmlToo) => rootLevel.filter((f) => (/\.urdf$/i.test(f.rel) || (xmlToo && /\.xml$/i.test(f.rel))) &&
|
|
169
|
+
/<robot[\s/>]/i.test(headText(f.abs)));
|
|
170
|
+
const listed = (candidates, family) => `several root-level ${family} candidates — pass --entry:\n` +
|
|
171
|
+
candidates.map((c) => ` ${c.rel}`).join("\n");
|
|
172
|
+
if (type === URDF_ENV_TYPE) {
|
|
173
|
+
const candidates = urdfCandidates(true);
|
|
174
|
+
if (candidates.length === 1)
|
|
175
|
+
return { entry: candidates[0].rel, envType: URDF_ENV_TYPE };
|
|
176
|
+
if (candidates.length === 0) {
|
|
177
|
+
return { error: "no root-level URDF (*.urdf or *.xml with <robot>) found — pass --entry <relpath>" };
|
|
178
|
+
}
|
|
179
|
+
return { error: listed(candidates, "URDF") };
|
|
180
|
+
}
|
|
181
|
+
const mjcf = rootLevel.filter((f) => /\.(xml|mjcf)$/i.test(f.rel) && /<mujoco[\s>]/i.test(headText(f.abs)));
|
|
182
|
+
if (mjcf.length === 1)
|
|
183
|
+
return { entry: mjcf[0].rel, envType: type ?? DEFAULT_ENV_TYPE };
|
|
184
|
+
if (mjcf.length > 1)
|
|
185
|
+
return { error: listed(mjcf, "MJCF") };
|
|
186
|
+
// No MJCF anywhere. Without an explicit --type, a single root-level *.urdf
|
|
187
|
+
// makes this a urdf env — `dreamlake scene push ./robot` just works.
|
|
188
|
+
if (type == null) {
|
|
189
|
+
const urdf = urdfCandidates(false);
|
|
190
|
+
if (urdf.length === 1)
|
|
191
|
+
return { entry: urdf[0].rel, envType: URDF_ENV_TYPE };
|
|
192
|
+
if (urdf.length > 1)
|
|
193
|
+
return { error: listed(urdf, "URDF") };
|
|
194
|
+
return {
|
|
195
|
+
error: "no root-level MJCF (*.xml with <mujoco>) or URDF (*.urdf with <robot>) found — pass --entry <relpath>",
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
return { error: "no root-level MJCF (*.xml with <mujoco>) found — pass --entry <relpath>" };
|
|
199
|
+
}
|
|
200
|
+
/** Order-independent identity of a version's file list — path + content
|
|
201
|
+
* hash per file. Two manifests with equal keys hold identical trees. */
|
|
202
|
+
export function manifestFilesKey(files) {
|
|
203
|
+
return files.map((f) => `${f.path}\u0000${f.hash}`).sort().join("\n");
|
|
204
|
+
}
|
|
205
|
+
export function buildManifest(entry, files, envType = DEFAULT_ENV_TYPE) {
|
|
206
|
+
return {
|
|
207
|
+
schema: 1,
|
|
208
|
+
entry,
|
|
209
|
+
envType,
|
|
210
|
+
files: [...files].sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)),
|
|
211
|
+
createdAt: new Date().toISOString(),
|
|
212
|
+
tool: `dreamlake-cli/${VERSION}`,
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
// ─── thumbnail (--thumbnail) ─────────────────────────────────────────
|
|
216
|
+
const PNG_MAGIC = [0x89, 0x50, 0x4e, 0x47];
|
|
217
|
+
/** Read + validate the --thumbnail file locally, BEFORE any upload work —
|
|
218
|
+
* a bad cover must not surface only after gigabytes have moved. */
|
|
219
|
+
function readThumbnailFile(p) {
|
|
220
|
+
let bytes;
|
|
221
|
+
try {
|
|
222
|
+
bytes = readFileSync(p);
|
|
223
|
+
}
|
|
224
|
+
catch {
|
|
225
|
+
return { error: `--thumbnail: could not read ${p}` };
|
|
226
|
+
}
|
|
227
|
+
if (bytes.length < PNG_MAGIC.length || !PNG_MAGIC.every((b, i) => bytes[i] === b)) {
|
|
228
|
+
return { error: `--thumbnail must be a PNG file (got ${p})` };
|
|
229
|
+
}
|
|
230
|
+
if (bytes.length > THUMB_MAX_BYTES) {
|
|
231
|
+
return {
|
|
232
|
+
error: `--thumbnail too large: ${humanSize(bytes.length)} > ${humanSize(THUMB_MAX_BYTES)} — downscale it (the app shows it at ~960px wide)`,
|
|
233
|
+
};
|
|
234
|
+
}
|
|
235
|
+
return new Uint8Array(bytes);
|
|
236
|
+
}
|
|
237
|
+
/** POST the cover to the catalog's thumbnail endpoint as a MANUAL capture —
|
|
238
|
+
* the app then shows it as-is and won't auto-overwrite it for this version.
|
|
239
|
+
* Cosmetic: a failure warns but never fails a push whose content landed. */
|
|
240
|
+
async function postThumbnail(remote, token, namespace, name, version, bytes) {
|
|
241
|
+
const r = await jsonRequest("POST", `${remote}/namespaces/${encodeURIComponent(namespace)}/scenes/${encodeURIComponent(name)}/thumbnail`, token, { imageBase64: Buffer.from(bytes).toString("base64"), version, source: "manual" });
|
|
242
|
+
if (r.status >= 200 && r.status < 300) {
|
|
243
|
+
process.stdout.write(` thumbnail: set (v${version})\n`);
|
|
244
|
+
}
|
|
245
|
+
else {
|
|
246
|
+
warn(`could not store the thumbnail → ${r.status}: ${errorMessage(r)}`);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
/**
|
|
250
|
+
* The unpinned/local layer srcs of the pushed dir's ROOT-LEVEL
|
|
251
|
+
* `dreamlake.layers.json`, or null when there is no parseable stack (no
|
|
252
|
+
* such file, invalid JSON, no `layers` array — none of push's business)
|
|
253
|
+
* or nothing is local. Cheap: one small file read, no network.
|
|
254
|
+
*/
|
|
255
|
+
export function unpinnedStackSources(dir, files) {
|
|
256
|
+
const stackFile = files.find((f) => f.rel === "dreamlake.layers.json");
|
|
257
|
+
if (!stackFile)
|
|
258
|
+
return null;
|
|
259
|
+
let stack;
|
|
260
|
+
try {
|
|
261
|
+
stack = JSON.parse(readFileSync(stackFile.abs, "utf8"));
|
|
262
|
+
}
|
|
263
|
+
catch {
|
|
264
|
+
return null;
|
|
265
|
+
}
|
|
266
|
+
const layers = stack?.layers;
|
|
267
|
+
if (!Array.isArray(layers))
|
|
268
|
+
return null;
|
|
269
|
+
const out = [];
|
|
270
|
+
layers.forEach((layer, index) => {
|
|
271
|
+
const c = layer;
|
|
272
|
+
if (!c || typeof c !== "object" || Array.isArray(c))
|
|
273
|
+
return;
|
|
274
|
+
const hasLocalSrc = typeof c.src === "string" && c.src !== "" && srcIsLocal(c.src) && c.pin == null;
|
|
275
|
+
if (!hasLocalSrc && c.unpinned !== true)
|
|
276
|
+
return;
|
|
277
|
+
const raw = typeof c.src === "string" && srcIsLocal(c.src) ? c.src : "";
|
|
278
|
+
const abs = raw === "" ? "" : path.isAbsolute(raw) ? raw : path.resolve(dir, raw);
|
|
279
|
+
out.push({ index, raw, abs });
|
|
280
|
+
});
|
|
281
|
+
return out.length > 0 ? out : null;
|
|
282
|
+
}
|
|
283
|
+
/**
|
|
284
|
+
* `--push-layers`: push each local layer directory as its own env, named by
|
|
285
|
+
* its basename — which must therefore BE a valid scene name as-is (an
|
|
286
|
+
* "obvious" name; anything that needs slugging needs a human and a manual
|
|
287
|
+
* `env push --name`). The composed env itself is NOT pushed: its stack
|
|
288
|
+
* still references local paths, and rewriting pins + recomposing is the
|
|
289
|
+
* author's move (edit dreamlake.layers.json, `env compose`, push again).
|
|
290
|
+
*/
|
|
291
|
+
async function pushLocalLayers(args, sources, deps) {
|
|
292
|
+
const pathless = sources.filter((s) => s.abs === "");
|
|
293
|
+
if (pathless.length > 0) {
|
|
294
|
+
fail(`--push-layers cannot push ${pathless.map((s) => `layers[${s.index}]`).join(", ")} — ` +
|
|
295
|
+
`marked "unpinned" but carrying no local path`);
|
|
296
|
+
return 1;
|
|
297
|
+
}
|
|
298
|
+
const dirs = new Map(); // unique, first index wins
|
|
299
|
+
for (const s of sources)
|
|
300
|
+
if (!dirs.has(s.abs))
|
|
301
|
+
dirs.set(s.abs, s);
|
|
302
|
+
for (const [abs, s] of dirs) {
|
|
303
|
+
let isDir = false;
|
|
304
|
+
try {
|
|
305
|
+
isDir = lstatSync(abs).isDirectory();
|
|
306
|
+
}
|
|
307
|
+
catch {
|
|
308
|
+
/* not found */
|
|
309
|
+
}
|
|
310
|
+
if (!isDir) {
|
|
311
|
+
fail(`--push-layers: layers[${s.index}] source directory not found: ${abs}`);
|
|
312
|
+
return 1;
|
|
313
|
+
}
|
|
314
|
+
const base = path.basename(abs);
|
|
315
|
+
const bad = invalidEnvName(base);
|
|
316
|
+
if (bad) {
|
|
317
|
+
fail(`--push-layers: layers[${s.index}] directory '${base}' is not a usable scene name (${bad})`);
|
|
318
|
+
process.stderr.write(` --push-layers names each scene after its directory. Rename the directory, or push it\n` +
|
|
319
|
+
` yourself: dreamlake scene push ${abs} --name <name>\n`);
|
|
320
|
+
return 1;
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
const listing = [...dirs.keys()].map((abs) => ` ${path.basename(abs)} ← ${abs}`).join("\n");
|
|
324
|
+
process.stdout.write(`--push-layers will push ${dirs.size} local layer dir(s) as scenes:\n${listing}\n`);
|
|
325
|
+
const confirmFn = deps.confirmFn ?? ((m) => confirm(m, false));
|
|
326
|
+
if (!(await confirmFn(`push ${dirs.size} layer(s) now?`))) {
|
|
327
|
+
process.stdout.write("aborted.\n");
|
|
328
|
+
return 0;
|
|
329
|
+
}
|
|
330
|
+
for (const [abs] of dirs) {
|
|
331
|
+
// Connection opts only — the composed env's own metadata (title,
|
|
332
|
+
// description, visibility, …) must not leak onto its layers. The
|
|
333
|
+
// recursive push runs the discipline gate too, so a layer that is
|
|
334
|
+
// itself an unpinned composition refuses rather than recursing.
|
|
335
|
+
const rc = await runEnvPush({
|
|
336
|
+
remote: args.remote,
|
|
337
|
+
token: args.token,
|
|
338
|
+
debug: args.debug,
|
|
339
|
+
namespace: args.namespace,
|
|
340
|
+
dir: abs,
|
|
341
|
+
name: path.basename(abs),
|
|
342
|
+
}, deps);
|
|
343
|
+
if (rc !== 0) {
|
|
344
|
+
fail(`--push-layers: pushing ${abs} failed — stopping`);
|
|
345
|
+
return rc;
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
ok(`pushed ${dirs.size} local layer(s)`);
|
|
349
|
+
process.stdout.write(` now pin them in dreamlake.layers.json ("src": "<namespace>/<name>@<version>" as\n` +
|
|
350
|
+
` printed above), recompose (dreamlake scene compose), and push again.\n`);
|
|
351
|
+
fail(`composed scene NOT pushed — its stack still references local paths`);
|
|
352
|
+
return 1;
|
|
353
|
+
}
|
|
354
|
+
export async function runEnvPush(args, deps = {}) {
|
|
355
|
+
// 1) local: walk, caps, entry.
|
|
356
|
+
let files;
|
|
357
|
+
try {
|
|
358
|
+
if (!lstatSync(args.dir).isDirectory())
|
|
359
|
+
throw new Error("not a directory");
|
|
360
|
+
files = walkEnvDir(args.dir);
|
|
361
|
+
}
|
|
362
|
+
catch {
|
|
363
|
+
fail(`directory not found: ${args.dir}`);
|
|
364
|
+
return 1;
|
|
365
|
+
}
|
|
366
|
+
if (files.length === 0) {
|
|
367
|
+
fail(`no files under ${args.dir} (dot-files, symlinks and node_modules are skipped)`);
|
|
368
|
+
return 1;
|
|
369
|
+
}
|
|
370
|
+
if (files.length > MAX_FILES) {
|
|
371
|
+
fail(`too many files: ${files.length} > ${MAX_FILES}`);
|
|
372
|
+
return 1;
|
|
373
|
+
}
|
|
374
|
+
const totalBytes = files.reduce((s, f) => s + f.size, 0);
|
|
375
|
+
const oversize = files.find((f) => f.size > MAX_FILE_BYTES);
|
|
376
|
+
if (oversize) {
|
|
377
|
+
fail(`file too large: ${oversize.rel} (${humanSize(oversize.size)} > ${humanSize(MAX_FILE_BYTES)})`);
|
|
378
|
+
return 1;
|
|
379
|
+
}
|
|
380
|
+
if (totalBytes > MAX_TOTAL_BYTES) {
|
|
381
|
+
fail(`version too large: ${humanSize(totalBytes)} > ${humanSize(MAX_TOTAL_BYTES)}`);
|
|
382
|
+
return 1;
|
|
383
|
+
}
|
|
384
|
+
for (const f of files) {
|
|
385
|
+
const bad = invalidRelPath(f.rel);
|
|
386
|
+
if (bad) {
|
|
387
|
+
fail(bad);
|
|
388
|
+
return 1;
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
const name = args.name ?? envSlug(path.basename(path.resolve(args.dir)));
|
|
392
|
+
const badName = invalidEnvName(name);
|
|
393
|
+
if (badName) {
|
|
394
|
+
fail(badName);
|
|
395
|
+
return 1;
|
|
396
|
+
}
|
|
397
|
+
if (args.visibility && args.visibility !== "public" && args.visibility !== "private") {
|
|
398
|
+
fail(`--visibility must be public or private (got '${args.visibility}')`);
|
|
399
|
+
return 1;
|
|
400
|
+
}
|
|
401
|
+
if (args.type != null && !ENV_TYPE_RE.test(args.type)) {
|
|
402
|
+
fail(`--type must match ^[a-z0-9][a-z0-9._-]{0,31}$ (got '${args.type}')`);
|
|
403
|
+
return 1;
|
|
404
|
+
}
|
|
405
|
+
let thumbBytes = null;
|
|
406
|
+
if (args.thumbnail) {
|
|
407
|
+
const t = readThumbnailFile(args.thumbnail);
|
|
408
|
+
if ("error" in t) {
|
|
409
|
+
fail(t.error);
|
|
410
|
+
return 1;
|
|
411
|
+
}
|
|
412
|
+
thumbBytes = t;
|
|
413
|
+
}
|
|
414
|
+
const detected = detectEntry(files, args.entry, args.type);
|
|
415
|
+
if ("error" in detected) {
|
|
416
|
+
fail(detected.error);
|
|
417
|
+
return 1;
|
|
418
|
+
}
|
|
419
|
+
const entry = detected.entry;
|
|
420
|
+
const envType = detected.envType;
|
|
421
|
+
// Only an explicit --type or a non-default detection reaches the server —
|
|
422
|
+
// a plain re-push must never clobber the type a scene was registered with.
|
|
423
|
+
const sendType = args.type != null || envType !== DEFAULT_ENV_TYPE;
|
|
424
|
+
// Push discipline: a stack with local/unpinned sources is not re-openable
|
|
425
|
+
// once pushed. Refuse by default, before any network I/O.
|
|
426
|
+
const localSources = unpinnedStackSources(args.dir, files);
|
|
427
|
+
if (localSources) {
|
|
428
|
+
if (args.pushLayers) {
|
|
429
|
+
return pushLocalLayers(args, localSources, deps);
|
|
430
|
+
}
|
|
431
|
+
if (args.allowLocal) {
|
|
432
|
+
warn(`dreamlake.layers.json references ${localSources.length} local/unpinned layer(s) — ` +
|
|
433
|
+
`the pushed provenance will not be re-openable`);
|
|
434
|
+
}
|
|
435
|
+
else {
|
|
436
|
+
fail(`dreamlake.layers.json references local, unpinned layers: ` +
|
|
437
|
+
localSources.map((s) => `layers[${s.index}]`).join(", "));
|
|
438
|
+
process.stderr.write(` a pushed scene should carry a fully pinned stack, or nobody can re-open its composition.\n` +
|
|
439
|
+
` --push-layers push those local layer dirs as scenes first (named by their basename)\n` +
|
|
440
|
+
` --allow-local push anyway; the provenance stays marked non-resolvable\n` +
|
|
441
|
+
` or pin the layers in dreamlake.layers.json and \`dreamlake scene compose\` again.\n`);
|
|
442
|
+
return 1;
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
const ctx = resolveCtx(args);
|
|
446
|
+
const token = requireToken(ctx);
|
|
447
|
+
if (!token)
|
|
448
|
+
return 1;
|
|
449
|
+
const namespace = await requireNamespace(ctx, args.namespace);
|
|
450
|
+
if (!namespace)
|
|
451
|
+
return 1;
|
|
452
|
+
// 2) hash every file locally (bytes are re-read at append time so at most
|
|
453
|
+
// one batch is ever in memory).
|
|
454
|
+
const manifestFiles = [];
|
|
455
|
+
const byHash = new Map(); // one representative per hash
|
|
456
|
+
const hashProgress = makeProgressLine();
|
|
457
|
+
for (const [i, f] of files.entries()) {
|
|
458
|
+
hashProgress.update(`hashing ${i + 1}/${files.length} ${f.rel}`);
|
|
459
|
+
const bytes = readFileSync(f.abs);
|
|
460
|
+
const hash = multihashBase32(new Uint8Array(bytes));
|
|
461
|
+
manifestFiles.push({ path: f.rel, hash, size: f.size });
|
|
462
|
+
if (!byHash.has(hash))
|
|
463
|
+
byHash.set(hash, f);
|
|
464
|
+
}
|
|
465
|
+
hashProgress.done();
|
|
466
|
+
// 3) nothing changed? Then there is nothing to version. Compare against the
|
|
467
|
+
// latest manifest via the plain read API — entry plus every (path, hash)
|
|
468
|
+
// pair. Metadata flags still go through as a patch; file-identical pushes
|
|
469
|
+
// never mint a pointless new version. This probe sees only what the
|
|
470
|
+
// CATALOG can resolve — a missing or soft-deleted row 404s here even
|
|
471
|
+
// though pushed versions exist, which is why guard 6) below re-checks
|
|
472
|
+
// against the datasets themselves.
|
|
473
|
+
const latest = await jsonRequest("GET", `${ctx.remote}/namespaces/${encodeURIComponent(namespace)}/scenes/${encodeURIComponent(name)}/versions/latest`, token);
|
|
474
|
+
if (latest.status === 200) {
|
|
475
|
+
const prev = latest.body;
|
|
476
|
+
if (Array.isArray(prev.files) && typeof prev.entry === "string") {
|
|
477
|
+
if (prev.entry === entry && manifestFilesKey(prev.files) === manifestFilesKey(manifestFiles)) {
|
|
478
|
+
const meta = {
|
|
479
|
+
...(args.title ? { title: args.title } : {}),
|
|
480
|
+
...(args.description ? { description: args.description } : {}),
|
|
481
|
+
...(args.type ? { envType } : {}),
|
|
482
|
+
// (auto-detected urdf is NOT re-sent here: with zero file changes
|
|
483
|
+
// the registered type is already right, or deliberately different)
|
|
484
|
+
...(args.visibility ? { visibility: args.visibility } : {}),
|
|
485
|
+
...(args.share ? { share: true } : {}),
|
|
486
|
+
};
|
|
487
|
+
if (Object.keys(meta).length === 0 && !thumbBytes) {
|
|
488
|
+
ok(`no changes since v${prev.version} — nothing to push`);
|
|
489
|
+
return 0;
|
|
490
|
+
}
|
|
491
|
+
if (Object.keys(meta).length > 0) {
|
|
492
|
+
const patch = await jsonRequest("POST", `${ctx.remote}/namespaces/${encodeURIComponent(namespace)}/scenes/${encodeURIComponent(name)}`, token, meta);
|
|
493
|
+
if (patch.status < 200 || patch.status >= 300) {
|
|
494
|
+
fail(`no file changes, but the metadata update failed → ${patch.status}: ${errorMessage(patch)}`);
|
|
495
|
+
return 1;
|
|
496
|
+
}
|
|
497
|
+
const patched = (patch.body ?? {});
|
|
498
|
+
ok(`no changes since v${prev.version} — updated metadata only`);
|
|
499
|
+
if (patched.visibility)
|
|
500
|
+
process.stdout.write(` visibility: ${patched.visibility}\n`);
|
|
501
|
+
let metaUrl = `${webBase(ctx.remote)}/${namespace}/scenes/${name}`;
|
|
502
|
+
if (args.share && patched.shareToken)
|
|
503
|
+
metaUrl += `?share=${patched.shareToken}`;
|
|
504
|
+
process.stdout.write(` open: ${metaUrl}\n`);
|
|
505
|
+
}
|
|
506
|
+
else {
|
|
507
|
+
ok(`no changes since v${prev.version} — updating the thumbnail only`);
|
|
508
|
+
}
|
|
509
|
+
// A thumbnail-only (or thumbnail-and-metadata) push must still land
|
|
510
|
+
// the cover — the early return above only fires with neither.
|
|
511
|
+
if (thumbBytes && typeof prev.version === "number") {
|
|
512
|
+
await postThumbnail(ctx.remote, token, namespace, name, prev.version, thumbBytes);
|
|
513
|
+
}
|
|
514
|
+
return 0;
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
// 4) broker: short-lived, prefix-scoped credentials for THIS env's prefix.
|
|
519
|
+
const cred = await jsonRequest("POST", `${ctx.remote}/namespaces/${encodeURIComponent(namespace)}/scenes/upload-credentials`, token, { name });
|
|
520
|
+
if (cred.status < 200 || cred.status >= 300) {
|
|
521
|
+
fail(`could not get upload credentials → ${cred.status}: ${errorMessage(cred)}`);
|
|
522
|
+
return 1;
|
|
523
|
+
}
|
|
524
|
+
const broker = cred.body;
|
|
525
|
+
if (!broker?.credentials?.accessKeyId || !broker.backendUrl) {
|
|
526
|
+
fail("unexpected upload-credentials response (missing credentials or backendUrl)");
|
|
527
|
+
return 1;
|
|
528
|
+
}
|
|
529
|
+
const backend = backendFrom(broker);
|
|
530
|
+
const io = deps.envDreamdb ?? realEnvDreamdbIO;
|
|
531
|
+
// 5) remote state, ONE read each: the dedup set, and the manifests head —
|
|
532
|
+
// the next version number plus the latest manifest for the second
|
|
533
|
+
// no-change guard below. Read failures abort — silently disabled dedup
|
|
534
|
+
// would re-upload everything, and a guessed version would clobber
|
|
535
|
+
// history.
|
|
536
|
+
let existingHashes;
|
|
537
|
+
let head;
|
|
538
|
+
try {
|
|
539
|
+
existingHashes = await io.readStringColumn(broker.backendUrl, BLOBS_REF, backend, "hash");
|
|
540
|
+
head = await io.readManifestsHead(broker.backendUrl, MANIFESTS_REF, backend);
|
|
541
|
+
}
|
|
542
|
+
catch (e) {
|
|
543
|
+
fail(`could not read the scene's current state: ${e instanceof Error ? e.message : String(e)}`);
|
|
544
|
+
return 1;
|
|
545
|
+
}
|
|
546
|
+
let version = (head?.version ?? 0) + 1;
|
|
547
|
+
const matchesLocal = (h) => h?.manifest != null &&
|
|
548
|
+
h.manifest.entry === entry &&
|
|
549
|
+
manifestFilesKey(h.manifest.files) === manifestFilesKey(manifestFiles);
|
|
550
|
+
// The catalog registration, shared by the guard below (re-registering an
|
|
551
|
+
// existing version) and the normal end-of-push path.
|
|
552
|
+
const registerInCatalog = (v) => jsonRequest("POST", `${ctx.remote}/namespaces/${encodeURIComponent(namespace)}/scenes/${encodeURIComponent(name)}`, token, {
|
|
553
|
+
version: v,
|
|
554
|
+
entryFile: entry,
|
|
555
|
+
fileCount: manifestFiles.length,
|
|
556
|
+
totalBytes,
|
|
557
|
+
...(sendType ? { envType } : {}),
|
|
558
|
+
...(args.title ? { title: args.title } : {}),
|
|
559
|
+
...(args.description ? { description: args.description } : {}),
|
|
560
|
+
...(args.visibility ? { visibility: args.visibility } : {}),
|
|
561
|
+
...(args.share ? { share: true } : {}),
|
|
562
|
+
});
|
|
563
|
+
const printCatalogRow = (body) => {
|
|
564
|
+
const cat = (body ?? {});
|
|
565
|
+
if (cat.visibility)
|
|
566
|
+
process.stdout.write(` visibility: ${cat.visibility}\n`);
|
|
567
|
+
let url = `${webBase(ctx.remote)}/${namespace}/scenes/${name}`;
|
|
568
|
+
if (args.share && cat.shareToken)
|
|
569
|
+
url += `?share=${cat.shareToken}`;
|
|
570
|
+
process.stdout.write(` open: ${url}\n`);
|
|
571
|
+
};
|
|
572
|
+
// 6) second no-change guard, against the datasets themselves. The REST
|
|
573
|
+
// probe in 3) answers 404 whenever the CATALOG row is missing or
|
|
574
|
+
// soft-deleted — the states a failed registration or an `env delete`
|
|
575
|
+
// leave behind — while the datasets still hold every pushed version,
|
|
576
|
+
// so a probe miss must not mean "mint a new version". Content
|
|
577
|
+
// identical → re-register the version that already exists (which also
|
|
578
|
+
// revives or re-creates the catalog row) and stop.
|
|
579
|
+
if (head && matchesLocal(head)) {
|
|
580
|
+
const reg = await registerInCatalog(head.version);
|
|
581
|
+
if (reg.status < 200 || reg.status >= 300) {
|
|
582
|
+
fail(`no file changes since v${head.version}, but re-registering it in the catalog failed → ${reg.status}: ${errorMessage(reg)}`);
|
|
583
|
+
return 1;
|
|
584
|
+
}
|
|
585
|
+
ok(`no changes since v${head.version} — nothing to push`);
|
|
586
|
+
if (thumbBytes)
|
|
587
|
+
await postThumbnail(ctx.remote, token, namespace, name, head.version, thumbBytes);
|
|
588
|
+
printCatalogRow(reg.body);
|
|
589
|
+
return 0;
|
|
590
|
+
}
|
|
591
|
+
let toUpload = [...byHash.entries()].filter(([hash]) => !existingHashes.has(hash));
|
|
592
|
+
const reused = manifestFiles.length - toUpload.length;
|
|
593
|
+
process.stdout.write(`Pushing ${name} v${version} from ${args.dir}\n`);
|
|
594
|
+
process.stdout.write(` namespace: ${namespace}\n`);
|
|
595
|
+
process.stdout.write(` entry: ${entry}${envType === DEFAULT_ENV_TYPE ? "" : ` (type: ${envType})`}\n`);
|
|
596
|
+
process.stdout.write(` files: ${manifestFiles.length} (${toUpload.length} to upload, ${reused} reused, ${humanSize(totalBytes)})\n`);
|
|
597
|
+
// 7) append the missing blobs in ≤64 MiB batches. A PublishConflict means a
|
|
598
|
+
// concurrent push advanced refs/blobs — its content PUTs are idempotent
|
|
599
|
+
// by hash, so refresh the dedup set, drop what it already wrote, retry.
|
|
600
|
+
const uploadBytes = toUpload.reduce((s, [, f]) => s + f.size, 0);
|
|
601
|
+
// Per-file network progress: every blob lands as one content-object PUT
|
|
602
|
+
// through OUR backend, so counting `created` results there is exact — a
|
|
603
|
+
// retried batch re-PUTs idempotently ('exists') and is not double-counted.
|
|
604
|
+
const uploadProgress = makeProgressLine();
|
|
605
|
+
let sentFiles = 0;
|
|
606
|
+
let sentBytes = 0;
|
|
607
|
+
const rawPut = backend.put.bind(backend);
|
|
608
|
+
backend.put = async (key, bytes, opts) => {
|
|
609
|
+
const result = await rawPut(key, bytes, opts);
|
|
610
|
+
// Fragment objects only — `<timeline>/image.envfile/<bucket16>/<hash>`.
|
|
611
|
+
// The modality also appears in track/items keys, which are bookkeeping,
|
|
612
|
+
// not file uploads.
|
|
613
|
+
if (result.status === "created" && /\/image\.envfile\/[0-9a-f]{16}\//.test(key)) {
|
|
614
|
+
sentFiles += 1;
|
|
615
|
+
sentBytes += bytes.length;
|
|
616
|
+
uploadProgress.update(`uploading ${sentFiles}/${toUpload.length} files · ${humanSize(sentBytes)} / ${humanSize(uploadBytes)}`);
|
|
617
|
+
}
|
|
618
|
+
return result;
|
|
619
|
+
};
|
|
620
|
+
try {
|
|
621
|
+
let batch = [];
|
|
622
|
+
let batchBytes = 0;
|
|
623
|
+
const flush = async () => {
|
|
624
|
+
if (batch.length === 0)
|
|
625
|
+
return;
|
|
626
|
+
const rows = batch.map(([hash, f]) => ({
|
|
627
|
+
content: { kind: "image", bytes: new Uint8Array(readFileSync(f.abs)) },
|
|
628
|
+
hash: { kind: "string", value: hash },
|
|
629
|
+
size: { kind: "int", value: f.size },
|
|
630
|
+
}));
|
|
631
|
+
for (let attempt = 0;; attempt++) {
|
|
632
|
+
try {
|
|
633
|
+
await io.appendRows(broker.backendUrl, BLOBS_REF, ENV_BLOB_SCHEMA, backend, rows);
|
|
634
|
+
break;
|
|
635
|
+
}
|
|
636
|
+
catch (e) {
|
|
637
|
+
if (!isPublishConflict(e) || attempt >= CONFLICT_RETRIES)
|
|
638
|
+
throw e;
|
|
639
|
+
const fresh = await io.readStringColumn(broker.backendUrl, BLOBS_REF, backend, "hash");
|
|
640
|
+
for (let i = rows.length - 1; i >= 0; i--) {
|
|
641
|
+
const h = rows[i].hash.value;
|
|
642
|
+
if (fresh.has(h))
|
|
643
|
+
rows.splice(i, 1);
|
|
644
|
+
}
|
|
645
|
+
if (rows.length === 0)
|
|
646
|
+
break;
|
|
647
|
+
}
|
|
648
|
+
}
|
|
649
|
+
batch = [];
|
|
650
|
+
batchBytes = 0;
|
|
651
|
+
};
|
|
652
|
+
for (const item of toUpload) {
|
|
653
|
+
if (batch.length > 0 && batchBytes + item[1].size > APPEND_BATCH_BYTES)
|
|
654
|
+
await flush();
|
|
655
|
+
batch.push(item);
|
|
656
|
+
batchBytes += item[1].size;
|
|
657
|
+
}
|
|
658
|
+
await flush();
|
|
659
|
+
}
|
|
660
|
+
catch (e) {
|
|
661
|
+
uploadProgress.done();
|
|
662
|
+
fail(`upload failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
663
|
+
return 1;
|
|
664
|
+
}
|
|
665
|
+
uploadProgress.done();
|
|
666
|
+
// 8) append ONE manifests row. On conflict the version number may be taken —
|
|
667
|
+
// recompute, rebuild, retry. If the concurrent winner pushed THIS exact
|
|
668
|
+
// content (two terminals, same directory), adopt its version instead of
|
|
669
|
+
// minting a duplicate.
|
|
670
|
+
try {
|
|
671
|
+
for (let attempt = 0;; attempt++) {
|
|
672
|
+
const manifest = buildManifest(entry, manifestFiles, envType);
|
|
673
|
+
try {
|
|
674
|
+
await io.appendRows(broker.backendUrl, MANIFESTS_REF, ENV_MANIFEST_SCHEMA, backend, [
|
|
675
|
+
{
|
|
676
|
+
content: {
|
|
677
|
+
kind: "image",
|
|
678
|
+
bytes: new TextEncoder().encode(JSON.stringify(manifest, null, 2)),
|
|
679
|
+
},
|
|
680
|
+
version: { kind: "int", value: version },
|
|
681
|
+
entry: { kind: "string", value: entry },
|
|
682
|
+
file_count: { kind: "int", value: manifestFiles.length },
|
|
683
|
+
total_bytes: { kind: "int", value: totalBytes },
|
|
684
|
+
},
|
|
685
|
+
]);
|
|
686
|
+
break;
|
|
687
|
+
}
|
|
688
|
+
catch (e) {
|
|
689
|
+
if (!isPublishConflict(e) || attempt >= CONFLICT_RETRIES)
|
|
690
|
+
throw e;
|
|
691
|
+
const fresh = await io.readManifestsHead(broker.backendUrl, MANIFESTS_REF, backend);
|
|
692
|
+
if (fresh && matchesLocal(fresh)) {
|
|
693
|
+
version = fresh.version;
|
|
694
|
+
break;
|
|
695
|
+
}
|
|
696
|
+
version = (fresh?.version ?? 0) + 1;
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
}
|
|
700
|
+
catch (e) {
|
|
701
|
+
fail(`could not record the version manifest: ${e instanceof Error ? e.message : String(e)}`);
|
|
702
|
+
return 1;
|
|
703
|
+
}
|
|
704
|
+
ok(`pushed ${namespace}/${name} v${version} — ${manifestFiles.length} files (${toUpload.length} uploaded ${humanSize(uploadBytes)}, ${reused} reused)`);
|
|
705
|
+
// 9) register in the server catalog. The content is written either way;
|
|
706
|
+
// without the row it won't list — report failure so CI notices.
|
|
707
|
+
const reg = await registerInCatalog(version);
|
|
708
|
+
if (reg.status < 200 || reg.status >= 300) {
|
|
709
|
+
fail(`uploaded the content but failed to register it in the catalog → ${reg.status}: ${errorMessage(reg)}`);
|
|
710
|
+
process.stderr.write(` it won't show in the list until you re-push '${name}'.\n`);
|
|
711
|
+
return 1;
|
|
712
|
+
}
|
|
713
|
+
if (thumbBytes)
|
|
714
|
+
await postThumbnail(ctx.remote, token, namespace, name, version, thumbBytes);
|
|
715
|
+
printCatalogRow(reg.body);
|
|
716
|
+
return 0;
|
|
717
|
+
}
|
|
718
|
+
// ─── create (push, but refuses an existing env) ──────────────────────
|
|
719
|
+
export async function runEnvCreate(args, deps = {}) {
|
|
720
|
+
const name = args.name ?? envSlug(path.basename(path.resolve(args.dir)));
|
|
721
|
+
const ctx = resolveCtx(args);
|
|
722
|
+
const namespace = await requireNamespace(ctx, args.namespace);
|
|
723
|
+
if (!namespace)
|
|
724
|
+
return 1;
|
|
725
|
+
const probe = await jsonRequest("GET", `${ctx.remote}/namespaces/${encodeURIComponent(namespace)}/scenes/${encodeURIComponent(name)}`, ctx.token);
|
|
726
|
+
if (probe.status === 200) {
|
|
727
|
+
fail(`scene '${name}' already exists in '${namespace}' — use \`dreamlake scene push\` to add a version`);
|
|
728
|
+
return 1;
|
|
729
|
+
}
|
|
730
|
+
if (probe.status !== 404) {
|
|
731
|
+
fail(`could not check for an existing scene → ${probe.status}: ${errorMessage(probe)}`);
|
|
732
|
+
return 1;
|
|
733
|
+
}
|
|
734
|
+
return runEnvPush({ ...args, name }, deps);
|
|
735
|
+
}
|
|
736
|
+
export async function runEnvList(args) {
|
|
737
|
+
const ctx = resolveCtx(args);
|
|
738
|
+
const namespace = await requireNamespace(ctx, args.namespace);
|
|
739
|
+
if (!namespace)
|
|
740
|
+
return 1;
|
|
741
|
+
const r = await jsonRequest("GET", `${ctx.remote}/namespaces/${encodeURIComponent(namespace)}/scenes`, ctx.token);
|
|
742
|
+
if (r.status !== 200) {
|
|
743
|
+
fail(`could not list scenes → ${r.status}: ${errorMessage(r)}`);
|
|
744
|
+
return 1;
|
|
745
|
+
}
|
|
746
|
+
const body = r.body;
|
|
747
|
+
const envs = body.scenes ?? body.envs ?? [];
|
|
748
|
+
if (args.json) {
|
|
749
|
+
emitJson(envs);
|
|
750
|
+
return 0;
|
|
751
|
+
}
|
|
752
|
+
if (envs.length === 0) {
|
|
753
|
+
process.stdout.write(`no scenes in namespace '${namespace}'.\n`);
|
|
754
|
+
return 0;
|
|
755
|
+
}
|
|
756
|
+
process.stdout.write(`Scenes in ${namespace}:\n`);
|
|
757
|
+
for (const e of envs) {
|
|
758
|
+
const vis = typeof e.visibility === "string" ? e.visibility : "private";
|
|
759
|
+
const vtag = vis === "private" ? "" : ` [${vis}]`;
|
|
760
|
+
const title = e.title ? ` ${String(e.title)}` : "";
|
|
761
|
+
const type = typeof e.envType === "string" && e.envType ? e.envType : "mujoco";
|
|
762
|
+
process.stdout.write(` ${String(e.name)} v${String(e.latestVersion)} · ${type} · ${String(e.fileCount ?? "?")} files · ${String(e.entryFile)}${title}${vtag}\n`);
|
|
763
|
+
}
|
|
764
|
+
return 0;
|
|
765
|
+
}
|
|
766
|
+
export function parseEnvSpec(spec) {
|
|
767
|
+
const at = spec.lastIndexOf("@");
|
|
768
|
+
if (at < 0)
|
|
769
|
+
return { name: spec };
|
|
770
|
+
const name = spec.slice(0, at);
|
|
771
|
+
const v = Number(spec.slice(at + 1));
|
|
772
|
+
if (!Number.isInteger(v) || v < 1)
|
|
773
|
+
return { error: `bad version in '${spec}' — use <name>@<n>` };
|
|
774
|
+
return { name, version: v };
|
|
775
|
+
}
|
|
776
|
+
const PULL_CONCURRENCY = 4;
|
|
777
|
+
/** Resolve `name@version` (or `@latest`) to its presigned manifest.
|
|
778
|
+
* Shared by `env pull` and the compose cache fill — pull is PURE REST,
|
|
779
|
+
* and so is this. Throws with the friendly message on any non-200. */
|
|
780
|
+
export async function resolveEnvVersion(ctx, namespace, name, version) {
|
|
781
|
+
const r = await jsonRequest("GET", `${ctx.remote}/namespaces/${encodeURIComponent(namespace)}/scenes/${encodeURIComponent(name)}/versions/${version ?? "latest"}`, ctx.token);
|
|
782
|
+
if (r.status !== 200) {
|
|
783
|
+
const extra = r.body?.availableVersions;
|
|
784
|
+
const hint = extra?.length ? ` (available: ${extra.join(", ")})` : "";
|
|
785
|
+
throw new Error(`could not resolve ${name}@${version ?? "latest"} → ${r.status}: ${errorMessage(r)}${hint}`);
|
|
786
|
+
}
|
|
787
|
+
return r.body;
|
|
788
|
+
}
|
|
789
|
+
/**
|
|
790
|
+
* Download every file of a resolved manifest into `outDir`, verifying each
|
|
791
|
+
* against its manifest hash (the hash IS the content address, so this is
|
|
792
|
+
* end-to-end integrity for free). Small pool; ONE presign refresh via
|
|
793
|
+
* `reResolve` on an expired batch. Unsafe manifest paths are refused before
|
|
794
|
+
* any download. Shared by `env pull` and the compose cache fill; throws on
|
|
795
|
+
* any failure.
|
|
796
|
+
*/
|
|
797
|
+
export async function downloadEnvVersion(resolved, outDir, opts = {}) {
|
|
798
|
+
const fetchFn = opts.fetchFn ?? fetch;
|
|
799
|
+
for (const f of resolved.files) {
|
|
800
|
+
const bad = invalidRelPath(f.path);
|
|
801
|
+
if (bad)
|
|
802
|
+
throw new Error(`manifest contains an unsafe path: ${bad}`);
|
|
803
|
+
}
|
|
804
|
+
let refreshed = false;
|
|
805
|
+
let downloaded = 0;
|
|
806
|
+
let downloadedBytes = 0;
|
|
807
|
+
const pullTotalBytes = resolved.files.reduce((s, f) => s + f.size, 0);
|
|
808
|
+
const pullProgress = makeProgressLine();
|
|
809
|
+
const queue = [...resolved.files];
|
|
810
|
+
const urlByPath = new Map(resolved.files.map((f) => [f.path, f.url]));
|
|
811
|
+
const fetchOne = async (file) => {
|
|
812
|
+
for (let attempt = 0;; attempt++) {
|
|
813
|
+
const res = await fetchFn(urlByPath.get(file.path));
|
|
814
|
+
if (!res.ok) {
|
|
815
|
+
if ((res.status === 403 || res.status === 400) && !refreshed && attempt === 0 && opts.reResolve) {
|
|
816
|
+
// Presigned batch likely expired mid-pull — re-resolve once.
|
|
817
|
+
refreshed = true;
|
|
818
|
+
const again = await opts.reResolve();
|
|
819
|
+
for (const f of again.files)
|
|
820
|
+
urlByPath.set(f.path, f.url);
|
|
821
|
+
continue;
|
|
822
|
+
}
|
|
823
|
+
throw new Error(`download failed for ${file.path} → ${res.status}`);
|
|
824
|
+
}
|
|
825
|
+
const bytes = new Uint8Array(await res.arrayBuffer());
|
|
826
|
+
const got = multihashBase32(bytes);
|
|
827
|
+
if (got !== file.hash) {
|
|
828
|
+
throw new Error(`hash mismatch for ${file.path}: expected ${file.hash}, got ${got}`);
|
|
829
|
+
}
|
|
830
|
+
const target = path.join(outDir, file.path);
|
|
831
|
+
mkdirSync(path.dirname(target), { recursive: true });
|
|
832
|
+
writeFileSync(target, bytes);
|
|
833
|
+
downloaded++;
|
|
834
|
+
downloadedBytes += bytes.length;
|
|
835
|
+
pullProgress.update(`downloading ${downloaded}/${resolved.files.length} files · ${humanSize(downloadedBytes)} / ${humanSize(pullTotalBytes)}`);
|
|
836
|
+
return;
|
|
837
|
+
}
|
|
838
|
+
};
|
|
839
|
+
try {
|
|
840
|
+
const workers = Array.from({ length: Math.min(PULL_CONCURRENCY, queue.length) }, async () => {
|
|
841
|
+
for (;;) {
|
|
842
|
+
const file = queue.shift();
|
|
843
|
+
if (!file)
|
|
844
|
+
return;
|
|
845
|
+
await fetchOne(file);
|
|
846
|
+
}
|
|
847
|
+
});
|
|
848
|
+
await Promise.all(workers);
|
|
849
|
+
}
|
|
850
|
+
finally {
|
|
851
|
+
pullProgress.done();
|
|
852
|
+
}
|
|
853
|
+
return { files: downloaded, bytes: downloadedBytes };
|
|
854
|
+
}
|
|
855
|
+
export async function runEnvPull(args, deps = {}) {
|
|
856
|
+
const parsed = parseEnvSpec(args.spec);
|
|
857
|
+
if ("error" in parsed) {
|
|
858
|
+
fail(parsed.error);
|
|
859
|
+
return 1;
|
|
860
|
+
}
|
|
861
|
+
const { name, version } = parsed;
|
|
862
|
+
const ctx = resolveCtx(args);
|
|
863
|
+
const namespace = await requireNamespace(ctx, args.namespace);
|
|
864
|
+
if (!namespace)
|
|
865
|
+
return 1;
|
|
866
|
+
let resolved;
|
|
867
|
+
try {
|
|
868
|
+
resolved = await resolveEnvVersion(ctx, namespace, name, version);
|
|
869
|
+
}
|
|
870
|
+
catch (e) {
|
|
871
|
+
fail(e instanceof Error ? e.message : String(e));
|
|
872
|
+
return 1;
|
|
873
|
+
}
|
|
874
|
+
const outDir = args.output ?? `./${name}`;
|
|
875
|
+
if (existsSync(outDir)) {
|
|
876
|
+
const entries = readdirSync(outDir);
|
|
877
|
+
if (entries.length > 0 && !args.force) {
|
|
878
|
+
fail(`output directory ${outDir} is not empty — pass --force to write into it`);
|
|
879
|
+
return 1;
|
|
880
|
+
}
|
|
881
|
+
}
|
|
882
|
+
process.stdout.write(`Pulling ${namespace}/${name} v${resolved.version} → ${outDir}\n`);
|
|
883
|
+
process.stdout.write(` entry: ${resolved.entry}\n`);
|
|
884
|
+
let got;
|
|
885
|
+
try {
|
|
886
|
+
got = await downloadEnvVersion(resolved, outDir, {
|
|
887
|
+
fetchFn: deps.fetchFn,
|
|
888
|
+
reResolve: () => resolveEnvVersion(ctx, namespace, name, version),
|
|
889
|
+
});
|
|
890
|
+
}
|
|
891
|
+
catch (e) {
|
|
892
|
+
fail(e instanceof Error ? e.message : String(e));
|
|
893
|
+
return 1;
|
|
894
|
+
}
|
|
895
|
+
ok(`pulled ${got.files} files → ${outDir} (entry: ${path.join(outDir, resolved.entry)})`);
|
|
896
|
+
return 0;
|
|
897
|
+
}
|
|
898
|
+
export async function runEnvDelete(args, deps = {}) {
|
|
899
|
+
const ctx = resolveCtx(args);
|
|
900
|
+
const namespace = await requireNamespace(ctx, args.namespace);
|
|
901
|
+
if (!namespace)
|
|
902
|
+
return 1;
|
|
903
|
+
const token = requireToken(ctx);
|
|
904
|
+
if (!token)
|
|
905
|
+
return 1;
|
|
906
|
+
if (!args.yes) {
|
|
907
|
+
const message = args.permanent
|
|
908
|
+
? `PERMANENTLY delete scene '${args.name}' from '${namespace}'? This erases all versions and cannot be undone.`
|
|
909
|
+
: `Delete scene '${args.name}' from '${namespace}'?`;
|
|
910
|
+
const confirmFn = deps.confirmFn ?? ((m) => confirm(m, false));
|
|
911
|
+
if (!(await confirmFn(message))) {
|
|
912
|
+
process.stdout.write("aborted.\n");
|
|
913
|
+
return 0;
|
|
914
|
+
}
|
|
915
|
+
}
|
|
916
|
+
const base = `${ctx.remote}/namespaces/${encodeURIComponent(namespace)}/scenes/${encodeURIComponent(args.name)}`;
|
|
917
|
+
const r = await jsonRequest("DELETE", args.permanent ? `${base}/purge` : base, token);
|
|
918
|
+
if (r.status < 200 || r.status >= 300) {
|
|
919
|
+
fail(`could not delete scene → ${r.status}: ${errorMessage(r)}`);
|
|
920
|
+
return 1;
|
|
921
|
+
}
|
|
922
|
+
if (args.permanent) {
|
|
923
|
+
const n = r.body?.objectsDeleted;
|
|
924
|
+
ok(`permanently deleted ${args.name}${typeof n === "number" ? ` (${n} objects removed)` : ""}`);
|
|
925
|
+
}
|
|
926
|
+
else {
|
|
927
|
+
ok(`deleted ${args.name} (restore with \`dreamlake scene restore ${args.name}\`)`);
|
|
928
|
+
}
|
|
929
|
+
return 0;
|
|
930
|
+
}
|
|
931
|
+
export async function runEnvRestore(args) {
|
|
932
|
+
const ctx = resolveCtx(args);
|
|
933
|
+
const namespace = await requireNamespace(ctx, args.namespace);
|
|
934
|
+
if (!namespace)
|
|
935
|
+
return 1;
|
|
936
|
+
const token = requireToken(ctx);
|
|
937
|
+
if (!token)
|
|
938
|
+
return 1;
|
|
939
|
+
const r = await jsonRequest("POST", `${ctx.remote}/namespaces/${encodeURIComponent(namespace)}/scenes/${encodeURIComponent(args.name)}/restore`, token);
|
|
940
|
+
if (r.status < 200 || r.status >= 300) {
|
|
941
|
+
fail(`could not restore scene → ${r.status}: ${errorMessage(r)}`);
|
|
942
|
+
return 1;
|
|
943
|
+
}
|
|
944
|
+
ok(`restored ${args.name}`);
|
|
945
|
+
return 0;
|
|
946
|
+
}
|
|
947
|
+
// ─── registration ────────────────────────────────────────────────────
|
|
948
|
+
export function registerEnvCommand(program) {
|
|
949
|
+
const env = program
|
|
950
|
+
.command("scene")
|
|
951
|
+
.alias("env")
|
|
952
|
+
.description("versioned scenes (MuJoCo MJCF or URDF + assets) — push, pull, visualize on the web");
|
|
953
|
+
const pushOpts = (cmd) => withConnFlags(cmd
|
|
954
|
+
.argument("<dir>", "directory containing the entry file (MJCF or URDF) + assets")
|
|
955
|
+
.option("--name <name>", "scene name (default: slug of the directory name)")
|
|
956
|
+
.option("--entry <relpath>", "entry file (default: the single root-level *.xml with <mujoco>, else the single *.urdf with <robot>)")
|
|
957
|
+
.option("--type <envType>", "simulator family: mujoco | urdf | isaaclab | … (default: auto-detected from the entry; mujoco and urdf have web viewers)")
|
|
958
|
+
.option("--title <title>", "human title")
|
|
959
|
+
.option("--description <text>", "description shown on the scene page")
|
|
960
|
+
.option("--namespace <slug>", "namespace (default: active login)")
|
|
961
|
+
.option("--visibility <visibility>", "read visibility: public | private (default: private)")
|
|
962
|
+
.option("--share", "mint a ?share= link readable by anyone who has it")
|
|
963
|
+
.option("--thumbnail <path>", "PNG (≤512 KiB) stored as the scene's cover image in the app — marks the cover 'manual' so the viewer won't auto-overwrite it for this version")
|
|
964
|
+
.option("--push-layers", "when dreamlake.layers.json references local {path} layers: push each layer dir as its own scene first (named by the dir basename), then stop so you can pin + recompose")
|
|
965
|
+
.option("--allow-local", "push even though dreamlake.layers.json references local, unpinned layers (the provenance stays marked non-resolvable)"));
|
|
966
|
+
pushOpts(env.command("push").description("push a new version of a scene (unchanged files are not re-uploaded)")).action(async (dir, opts) => {
|
|
967
|
+
process.exit(await runEnvPush({ ...opts, dir }));
|
|
968
|
+
});
|
|
969
|
+
pushOpts(env.command("create").description("push the FIRST version of a scene — fails if the name already exists")).action(async (dir, opts) => {
|
|
970
|
+
process.exit(await runEnvCreate({ ...opts, dir }));
|
|
971
|
+
});
|
|
972
|
+
withConnFlags(collectionList(env, "list")
|
|
973
|
+
.description("list scenes in a namespace")
|
|
974
|
+
.option("--namespace <slug>", "namespace (default: active login)")
|
|
975
|
+
.option("--json", "emit raw JSON")).action(async (opts) => {
|
|
976
|
+
process.exit(await runEnvList(opts));
|
|
977
|
+
});
|
|
978
|
+
withConnFlags(env
|
|
979
|
+
.command("pull")
|
|
980
|
+
.description("download a scene version as a directory (hash-verified)")
|
|
981
|
+
.argument("<spec>", "<name> or <name>@<version> (default: latest)")
|
|
982
|
+
.option("-o, --output <dir>", "destination directory (default: ./<name>)")
|
|
983
|
+
.option("--force", "write into a non-empty destination")
|
|
984
|
+
.option("--namespace <slug>", "namespace (default: active login)")).action(async (spec, opts) => {
|
|
985
|
+
process.exit(await runEnvPull({ ...opts, spec }));
|
|
986
|
+
});
|
|
987
|
+
withConnFlags(env
|
|
988
|
+
.command("compose")
|
|
989
|
+
.description("materialize a layered scene stack (dreamlake.layers.json) into a scene directory — resolves registry refs through the version cache, then delegates composition to the local Python engine (dreamlake.envlayer)")
|
|
990
|
+
.argument("[stack]", `stack file (default: ./${STACK_BASENAME})`)
|
|
991
|
+
.option("-o, --out <dir>", 'output directory (default: ./<stack "name", else the stack directory\'s basename>)')
|
|
992
|
+
.option("--force", "write into a non-empty output directory")).action(async (stack, opts) => {
|
|
993
|
+
process.exit(await runEnvCompose({ ...opts, stackPath: stack }));
|
|
994
|
+
});
|
|
995
|
+
withConnFlags(env
|
|
996
|
+
.command("delete")
|
|
997
|
+
.description("soft-delete a scene (restorable), or purge with --permanent")
|
|
998
|
+
.argument("<name>", "scene name")
|
|
999
|
+
.option("--namespace <slug>", "namespace (default: active login)")
|
|
1000
|
+
.option("-y, --yes", "skip the confirmation prompt")
|
|
1001
|
+
.option("-p, --permanent", "purge storage + catalog — irreversible")).action(async (name, opts) => {
|
|
1002
|
+
process.exit(await runEnvDelete({ ...opts, name }));
|
|
1003
|
+
});
|
|
1004
|
+
withConnFlags(env
|
|
1005
|
+
.command("restore")
|
|
1006
|
+
.description("restore a soft-deleted scene")
|
|
1007
|
+
.argument("<name>", "scene name")
|
|
1008
|
+
.option("--namespace <slug>", "namespace (default: active login)")).action(async (name, opts) => {
|
|
1009
|
+
process.exit(await runEnvRestore({ ...opts, name }));
|
|
1010
|
+
});
|
|
1011
|
+
}
|