@effect-kit/storage 0.0.0-stage → 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +204 -0
- package/README.md +369 -2
- package/dist/_virtual/_rolldown/runtime.js +17 -0
- package/dist/azure/account-admin.d.ts +116 -0
- package/dist/azure/account-admin.js +90 -0
- package/dist/azure/account-admin.js.map +1 -0
- package/dist/azure/admin.d.ts +68 -0
- package/dist/azure/admin.js +85 -0
- package/dist/azure/admin.js.map +1 -0
- package/dist/azure/archive.d.ts +95 -0
- package/dist/azure/archive.js +52 -0
- package/dist/azure/archive.js.map +1 -0
- package/dist/azure/events.d.ts +40 -0
- package/dist/azure/events.js +62 -0
- package/dist/azure/events.js.map +1 -0
- package/dist/azure/governance.d.ts +139 -0
- package/dist/azure/governance.js +59 -0
- package/dist/azure/governance.js.map +1 -0
- package/dist/azure/identity.d.ts +4 -0
- package/dist/azure/identity.js +8 -0
- package/dist/azure/identity.js.map +1 -0
- package/dist/azure/lifecycle.d.ts +21 -0
- package/dist/azure/lifecycle.js +193 -0
- package/dist/azure/lifecycle.js.map +1 -0
- package/dist/azure/signing.d.ts +17 -0
- package/dist/azure/signing.js +71 -0
- package/dist/azure/signing.js.map +1 -0
- package/dist/azure/uploads.d.ts +11 -0
- package/dist/azure/uploads.js +83 -0
- package/dist/azure/uploads.js.map +1 -0
- package/dist/azure.d.ts +427 -0
- package/dist/azure.js +77 -0
- package/dist/azure.js.map +1 -0
- package/dist/b2/admin.d.ts +149 -0
- package/dist/b2/admin.js +157 -0
- package/dist/b2/admin.js.map +1 -0
- package/dist/b2.d.ts +15 -0
- package/dist/b2.js +15 -0
- package/dist/b2.js.map +1 -0
- package/dist/browser.d.ts +7 -0
- package/dist/browser.js +11 -0
- package/dist/browser.js.map +1 -0
- package/dist/bun.d.ts +14 -0
- package/dist/bun.js +17 -0
- package/dist/bun.js.map +1 -0
- package/dist/cache.d.ts +18 -0
- package/dist/cache.js +116 -0
- package/dist/cache.js.map +1 -0
- package/dist/client/atoms/controller.d.ts +10 -0
- package/dist/client/atoms/controller.js +267 -0
- package/dist/client/atoms/controller.js.map +1 -0
- package/dist/client/atoms/state.d.ts +17 -0
- package/dist/client/atoms/state.js +11 -0
- package/dist/client/atoms/state.js.map +1 -0
- package/dist/client/atoms.d.ts +59 -0
- package/dist/client/atoms.js +39 -0
- package/dist/client/atoms.js.map +1 -0
- package/dist/client/fingerprint.js +28 -0
- package/dist/client/fingerprint.js.map +1 -0
- package/dist/client/store.d.ts +31 -0
- package/dist/client/store.js +56 -0
- package/dist/client/store.js.map +1 -0
- package/dist/client/transport.d.ts +11 -0
- package/dist/client/transport.js +78 -0
- package/dist/client/transport.js.map +1 -0
- package/dist/client.d.ts +259 -0
- package/dist/client.js +350 -0
- package/dist/client.js.map +1 -0
- package/dist/composition/layer.js +22 -0
- package/dist/composition/layer.js.map +1 -0
- package/dist/composition/representation/staging.js +46 -0
- package/dist/composition/representation/staging.js.map +1 -0
- package/dist/composition/representation/storage.js +188 -0
- package/dist/composition/representation/storage.js.map +1 -0
- package/dist/composition.d.ts +20 -0
- package/dist/composition.js +124 -0
- package/dist/composition.js.map +1 -0
- package/dist/compression.d.ts +16 -0
- package/dist/compression.js +38 -0
- package/dist/compression.js.map +1 -0
- package/dist/content.d.ts +590 -0
- package/dist/content.js +168 -0
- package/dist/content.js.map +1 -0
- package/dist/core/capabilities.d.ts +53 -0
- package/dist/core/capabilities.js +9 -0
- package/dist/core/capabilities.js.map +1 -0
- package/dist/core/contracts/multipart.d.ts +56 -0
- package/dist/core/contracts/resumable.d.ts +70 -0
- package/dist/core/errors.d.ts +55 -0
- package/dist/core/errors.js +55 -0
- package/dist/core/errors.js.map +1 -0
- package/dist/core/object.d.ts +97 -0
- package/dist/core/object.js +55 -0
- package/dist/core/object.js.map +1 -0
- package/dist/core/ranges.d.ts +17 -0
- package/dist/core/ranges.js +27 -0
- package/dist/core/ranges.js.map +1 -0
- package/dist/core/requests.d.ts +185 -0
- package/dist/core/requests.js +42 -0
- package/dist/core/requests.js.map +1 -0
- package/dist/core/storage.d.ts +249 -0
- package/dist/core/storage.js +173 -0
- package/dist/core/storage.js.map +1 -0
- package/dist/deno.d.ts +14 -0
- package/dist/deno.js +17 -0
- package/dist/deno.js.map +1 -0
- package/dist/encryption/age.d.ts +14 -0
- package/dist/encryption/age.js +46 -0
- package/dist/encryption/age.js.map +1 -0
- package/dist/encryption.d.ts +16 -0
- package/dist/encryption.js +16 -0
- package/dist/encryption.js.map +1 -0
- package/dist/events.d.ts +57 -0
- package/dist/events.js +106 -0
- package/dist/events.js.map +1 -0
- package/dist/filesystem.d.ts +23 -0
- package/dist/filesystem.js +75 -0
- package/dist/filesystem.js.map +1 -0
- package/dist/gcs/admin.d.ts +470 -0
- package/dist/gcs/admin.js +206 -0
- package/dist/gcs/admin.js.map +1 -0
- package/dist/gcs/events.d.ts +41 -0
- package/dist/gcs/events.js +45 -0
- package/dist/gcs/events.js.map +1 -0
- package/dist/gcs/governance.d.ts +158 -0
- package/dist/gcs/governance.js +100 -0
- package/dist/gcs/governance.js.map +1 -0
- package/dist/gcs/lifecycle.d.ts +161 -0
- package/dist/gcs/lifecycle.js +174 -0
- package/dist/gcs/lifecycle.js.map +1 -0
- package/dist/gcs/post.d.ts +31 -0
- package/dist/gcs/post.js +70 -0
- package/dist/gcs/post.js.map +1 -0
- package/dist/gcs/signing.d.ts +10 -0
- package/dist/gcs/signing.js +61 -0
- package/dist/gcs/signing.js.map +1 -0
- package/dist/gcs/uploads.d.ts +10 -0
- package/dist/gcs/uploads.js +76 -0
- package/dist/gcs/uploads.js.map +1 -0
- package/dist/gcs.d.ts +888 -0
- package/dist/gcs.js +91 -0
- package/dist/gcs.js.map +1 -0
- package/dist/hetzner/admin.d.ts +112 -0
- package/dist/hetzner/admin.js +19 -0
- package/dist/hetzner/admin.js.map +1 -0
- package/dist/hetzner.d.ts +15 -0
- package/dist/hetzner.js +15 -0
- package/dist/hetzner.js.map +1 -0
- package/dist/http/api.d.ts +236 -0
- package/dist/http/api.js +54 -0
- package/dist/http/api.js.map +1 -0
- package/dist/http/download.js +99 -0
- package/dist/http/download.js.map +1 -0
- package/dist/http/schema.d.ts +3 -0
- package/dist/http/schema.js +4 -0
- package/dist/http.d.ts +35 -0
- package/dist/http.js +178 -0
- package/dist/http.js.map +1 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.js +9 -0
- package/dist/internal/http/control-plane.js +71 -0
- package/dist/internal/http/control-plane.js.map +1 -0
- package/dist/internal/listing-cursor.js +44 -0
- package/dist/internal/listing-cursor.js.map +1 -0
- package/dist/internal/node/provider-errors.js +16 -0
- package/dist/internal/node/provider-errors.js.map +1 -0
- package/dist/internal/node/upload-body.js +111 -0
- package/dist/internal/node/upload-body.js.map +1 -0
- package/dist/internal/provider-response.js +14 -0
- package/dist/internal/provider-response.js.map +1 -0
- package/dist/internal/public-urls.js +30 -0
- package/dist/internal/public-urls.js.map +1 -0
- package/dist/internal/read-condition.js +11 -0
- package/dist/internal/read-condition.js.map +1 -0
- package/dist/internal/request-validation.js +24 -0
- package/dist/internal/request-validation.js.map +1 -0
- package/dist/internal/signing-request.js +22 -0
- package/dist/internal/signing-request.js.map +1 -0
- package/dist/internal/web-upload-body.js +71 -0
- package/dist/internal/web-upload-body.js.map +1 -0
- package/dist/lifecycle/configuration-state.js +26 -0
- package/dist/lifecycle/configuration-state.js.map +1 -0
- package/dist/lifecycle/metadata.js +28 -0
- package/dist/lifecycle/metadata.js.map +1 -0
- package/dist/lifecycle.d.ts +269 -0
- package/dist/lifecycle.js +72 -0
- package/dist/lifecycle.js.map +1 -0
- package/dist/migrations.d.ts +537 -0
- package/dist/migrations.js +296 -0
- package/dist/migrations.js.map +1 -0
- package/dist/minio/admin.d.ts +112 -0
- package/dist/minio/admin.js +19 -0
- package/dist/minio/admin.js.map +1 -0
- package/dist/minio.d.ts +15 -0
- package/dist/minio.js +15 -0
- package/dist/minio.js.map +1 -0
- package/dist/mirror.d.ts +830 -0
- package/dist/mirror.js +290 -0
- package/dist/mirror.js.map +1 -0
- package/dist/node.d.ts +14 -0
- package/dist/node.js +17 -0
- package/dist/node.js.map +1 -0
- package/dist/providers/azure/blocks.js +94 -0
- package/dist/providers/azure/blocks.js.map +1 -0
- package/dist/providers/azure/config.d.ts +41 -0
- package/dist/providers/azure/config.js +62 -0
- package/dist/providers/azure/config.js.map +1 -0
- package/dist/providers/azure/mapping.js +83 -0
- package/dist/providers/azure/mapping.js.map +1 -0
- package/dist/providers/azure/native.js +88 -0
- package/dist/providers/azure/native.js.map +1 -0
- package/dist/providers/azure/primitives.js +203 -0
- package/dist/providers/azure/primitives.js.map +1 -0
- package/dist/providers/azure/properties.d.ts +180 -0
- package/dist/providers/azure/properties.js +55 -0
- package/dist/providers/azure/properties.js.map +1 -0
- package/dist/providers/azure/transport.js +14 -0
- package/dist/providers/azure/transport.js.map +1 -0
- package/dist/providers/azure/validation.js +11 -0
- package/dist/providers/azure/validation.js.map +1 -0
- package/dist/providers/filesystem/envelope.js +56 -0
- package/dist/providers/filesystem/envelope.js.map +1 -0
- package/dist/providers/filesystem/identity.js +68 -0
- package/dist/providers/filesystem/identity.js.map +1 -0
- package/dist/providers/filesystem/layout.d.ts +15 -0
- package/dist/providers/filesystem/layout.js +112 -0
- package/dist/providers/filesystem/layout.js.map +1 -0
- package/dist/providers/filesystem/list.js +82 -0
- package/dist/providers/filesystem/list.js.map +1 -0
- package/dist/providers/filesystem/operations.js +134 -0
- package/dist/providers/filesystem/operations.js.map +1 -0
- package/dist/providers/gcs/config.d.ts +34 -0
- package/dist/providers/gcs/config.js +60 -0
- package/dist/providers/gcs/config.js.map +1 -0
- package/dist/providers/gcs/iam.js +45 -0
- package/dist/providers/gcs/iam.js.map +1 -0
- package/dist/providers/gcs/mapping.js +118 -0
- package/dist/providers/gcs/mapping.js.map +1 -0
- package/dist/providers/gcs/native.d.ts +11 -0
- package/dist/providers/gcs/native.js +122 -0
- package/dist/providers/gcs/native.js.map +1 -0
- package/dist/providers/gcs/primitives.js +194 -0
- package/dist/providers/gcs/primitives.js.map +1 -0
- package/dist/providers/gcs/resumable.js +145 -0
- package/dist/providers/gcs/resumable.js.map +1 -0
- package/dist/providers/gcs/tier.d.ts +176 -0
- package/dist/providers/gcs/tier.js +34 -0
- package/dist/providers/gcs/tier.js.map +1 -0
- package/dist/providers/gcs/transport.js +123 -0
- package/dist/providers/gcs/transport.js.map +1 -0
- package/dist/providers/gcs/upload.js +28 -0
- package/dist/providers/gcs/upload.js.map +1 -0
- package/dist/providers/gcs/validation.js +10 -0
- package/dist/providers/gcs/validation.js.map +1 -0
- package/dist/providers/r2-workers/binding.d.ts +102 -0
- package/dist/providers/r2-workers/body.js +82 -0
- package/dist/providers/r2-workers/body.js.map +1 -0
- package/dist/providers/r2-workers/list.js +62 -0
- package/dist/providers/r2-workers/list.js.map +1 -0
- package/dist/providers/r2-workers/mapping.js +140 -0
- package/dist/providers/r2-workers/mapping.js.map +1 -0
- package/dist/providers/r2-workers/multipart.js +121 -0
- package/dist/providers/r2-workers/multipart.js.map +1 -0
- package/dist/providers/r2-workers/primitives.js +80 -0
- package/dist/providers/r2-workers/primitives.js.map +1 -0
- package/dist/providers/s3/admin-configuration.d.ts +19 -0
- package/dist/providers/s3/admin-configuration.js +171 -0
- package/dist/providers/s3/admin-configuration.js.map +1 -0
- package/dist/providers/s3/bulk-delete.d.ts +43 -0
- package/dist/providers/s3/bulk-delete.js +58 -0
- package/dist/providers/s3/bulk-delete.js.map +1 -0
- package/dist/providers/s3/config.d.ts +43 -0
- package/dist/providers/s3/config.js +89 -0
- package/dist/providers/s3/config.js.map +1 -0
- package/dist/providers/s3/copy.d.ts +115 -0
- package/dist/providers/s3/copy.js +122 -0
- package/dist/providers/s3/copy.js.map +1 -0
- package/dist/providers/s3/lifecycle-validation.js +33 -0
- package/dist/providers/s3/lifecycle-validation.js.map +1 -0
- package/dist/providers/s3/listing.js +56 -0
- package/dist/providers/s3/listing.js.map +1 -0
- package/dist/providers/s3/metadata.d.ts +10 -0
- package/dist/providers/s3/metadata.js +90 -0
- package/dist/providers/s3/metadata.js.map +1 -0
- package/dist/providers/s3/multipart.d.ts +41 -0
- package/dist/providers/s3/multipart.js +299 -0
- package/dist/providers/s3/multipart.js.map +1 -0
- package/dist/providers/s3/native-models.d.ts +82 -0
- package/dist/providers/s3/native-models.js +91 -0
- package/dist/providers/s3/native-models.js.map +1 -0
- package/dist/providers/s3/object-mapping.js +114 -0
- package/dist/providers/s3/object-mapping.js.map +1 -0
- package/dist/providers/s3/primitives.d.ts +8 -0
- package/dist/providers/s3/primitives.js +235 -0
- package/dist/providers/s3/primitives.js.map +1 -0
- package/dist/providers/s3/profile.d.ts +25 -0
- package/dist/providers/s3/profile.js +46 -0
- package/dist/providers/s3/profile.js.map +1 -0
- package/dist/providers/s3/request-extension.js +29 -0
- package/dist/providers/s3/request-extension.js.map +1 -0
- package/dist/providers/s3/service.js +55 -0
- package/dist/providers/s3/service.js.map +1 -0
- package/dist/providers/s3/signing-credentials.js +32 -0
- package/dist/providers/s3/signing-credentials.js.map +1 -0
- package/dist/providers/s3/tags.d.ts +10 -0
- package/dist/providers/s3/tags.js +65 -0
- package/dist/providers/s3/tags.js.map +1 -0
- package/dist/providers/s3/tier.d.ts +171 -0
- package/dist/providers/s3/tier.js +100 -0
- package/dist/providers/s3/tier.js.map +1 -0
- package/dist/providers/s3/transfer-driver.js +72 -0
- package/dist/providers/s3/transfer-driver.js.map +1 -0
- package/dist/providers/s3/transport.js +94 -0
- package/dist/providers/s3/transport.js.map +1 -0
- package/dist/providers/s3/upload-body.js +10 -0
- package/dist/providers/s3/upload-body.js.map +1 -0
- package/dist/providers/s3/validation.js +31 -0
- package/dist/providers/s3/validation.js.map +1 -0
- package/dist/providers/s3/versions.d.ts +10 -0
- package/dist/providers/s3/versions.js +143 -0
- package/dist/providers/s3/versions.js.map +1 -0
- package/dist/providers/s3-compatible/config.d.ts +36 -0
- package/dist/providers/s3-compatible/config.js +174 -0
- package/dist/providers/s3-compatible/config.js.map +1 -0
- package/dist/providers/s3-compatible/service.js +48 -0
- package/dist/providers/s3-compatible/service.js.map +1 -0
- package/dist/providers/supabase/config.d.ts +20 -0
- package/dist/providers/supabase/config.js +41 -0
- package/dist/providers/supabase/config.js.map +1 -0
- package/dist/providers/supabase/mapping.js +60 -0
- package/dist/providers/supabase/mapping.js.map +1 -0
- package/dist/providers/supabase/primitives.js +205 -0
- package/dist/providers/supabase/primitives.js.map +1 -0
- package/dist/providers/supabase/transport.js +64 -0
- package/dist/providers/supabase/transport.js.map +1 -0
- package/dist/providers/supabase/tus.js +163 -0
- package/dist/providers/supabase/tus.js.map +1 -0
- package/dist/providers/tigris/configuration.js +118 -0
- package/dist/providers/tigris/configuration.js.map +1 -0
- package/dist/providers/tigris/snapshots.js +87 -0
- package/dist/providers/tigris/snapshots.js.map +1 -0
- package/dist/providers/tigris/transport.js +18 -0
- package/dist/providers/tigris/transport.js.map +1 -0
- package/dist/providers/vercel/config.d.ts +16 -0
- package/dist/providers/vercel/config.js +48 -0
- package/dist/providers/vercel/config.js.map +1 -0
- package/dist/providers/vercel/mapping.js +79 -0
- package/dist/providers/vercel/mapping.js.map +1 -0
- package/dist/providers/vercel/multipart.js +113 -0
- package/dist/providers/vercel/multipart.js.map +1 -0
- package/dist/providers/vercel/primitives.js +168 -0
- package/dist/providers/vercel/primitives.js.map +1 -0
- package/dist/providers/vercel/transport.js +32 -0
- package/dist/providers/vercel/transport.js.map +1 -0
- package/dist/providers/wasabi/compliance.d.ts +52 -0
- package/dist/providers/wasabi/compliance.js +89 -0
- package/dist/providers/wasabi/compliance.js.map +1 -0
- package/dist/providers/wasabi/xml.js +55 -0
- package/dist/providers/wasabi/xml.js.map +1 -0
- package/dist/r2/admin.d.ts +126 -0
- package/dist/r2/admin.js +138 -0
- package/dist/r2/admin.js.map +1 -0
- package/dist/r2/events.d.ts +41 -0
- package/dist/r2/events.js +37 -0
- package/dist/r2/events.js.map +1 -0
- package/dist/r2-workers.d.ts +24 -0
- package/dist/r2-workers.js +115 -0
- package/dist/r2-workers.js.map +1 -0
- package/dist/r2.d.ts +15 -0
- package/dist/r2.js +15 -0
- package/dist/r2.js.map +1 -0
- package/dist/react.d.ts +73 -0
- package/dist/react.js +49 -0
- package/dist/react.js.map +1 -0
- package/dist/representation.d.ts +24 -0
- package/dist/representation.js +7 -0
- package/dist/representation.js.map +1 -0
- package/dist/runtime/bun-filesystem.js +136 -0
- package/dist/runtime/bun-filesystem.js.map +1 -0
- package/dist/runtime/file-target.d.ts +10 -0
- package/dist/runtime/file-target.js +66 -0
- package/dist/runtime/file-target.js.map +1 -0
- package/dist/runtime/filesystem.d.ts +6 -0
- package/dist/runtime/filesystem.js +68 -0
- package/dist/runtime/filesystem.js.map +1 -0
- package/dist/runtime/node-hash.d.ts +4 -0
- package/dist/runtime/node-hash.js +31 -0
- package/dist/runtime/node-hash.js.map +1 -0
- package/dist/runtime/node-protection.d.ts +4 -0
- package/dist/runtime/node-protection.js +46 -0
- package/dist/runtime/node-protection.js.map +1 -0
- package/dist/runtime/portable-hash.d.ts +3 -0
- package/dist/runtime/portable-hash.js +31 -0
- package/dist/runtime/portable-hash.js.map +1 -0
- package/dist/runtime/session-store.d.ts +12 -0
- package/dist/runtime/session-store.js +19 -0
- package/dist/runtime/session-store.js.map +1 -0
- package/dist/runtime/web-protection.d.ts +7 -0
- package/dist/runtime/web-protection.js +54 -0
- package/dist/runtime/web-protection.js.map +1 -0
- package/dist/s3/admin.d.ts +183 -0
- package/dist/s3/admin.js +197 -0
- package/dist/s3/admin.js.map +1 -0
- package/dist/s3/archive.d.ts +25 -0
- package/dist/s3/archive.js +113 -0
- package/dist/s3/archive.js.map +1 -0
- package/dist/s3/events.d.ts +42 -0
- package/dist/s3/events.js +69 -0
- package/dist/s3/events.js.map +1 -0
- package/dist/s3/governance.d.ts +319 -0
- package/dist/s3/governance.js +108 -0
- package/dist/s3/governance.js.map +1 -0
- package/dist/s3/lifecycle.d.ts +4 -0
- package/dist/s3/lifecycle.js +5 -0
- package/dist/s3/post.d.ts +31 -0
- package/dist/s3/post.js +93 -0
- package/dist/s3/post.js.map +1 -0
- package/dist/s3/signing.d.ts +105 -0
- package/dist/s3/signing.js +198 -0
- package/dist/s3/signing.js.map +1 -0
- package/dist/s3/uploads.d.ts +11 -0
- package/dist/s3/uploads.js +173 -0
- package/dist/s3/uploads.js.map +1 -0
- package/dist/s3.d.ts +37 -0
- package/dist/s3.js +114 -0
- package/dist/s3.js.map +1 -0
- package/dist/scaleway/admin.d.ts +112 -0
- package/dist/scaleway/admin.js +19 -0
- package/dist/scaleway/admin.js.map +1 -0
- package/dist/scaleway.d.ts +15 -0
- package/dist/scaleway.js +15 -0
- package/dist/scaleway.js.map +1 -0
- package/dist/session-events.d.ts +22 -0
- package/dist/session-events.js +22 -0
- package/dist/session-events.js.map +1 -0
- package/dist/sessions/checkpoint.d.ts +423 -0
- package/dist/sessions/checkpoint.js +102 -0
- package/dist/sessions/checkpoint.js.map +1 -0
- package/dist/sessions/download.d.ts +17 -0
- package/dist/sessions/download.js +111 -0
- package/dist/sessions/download.js.map +1 -0
- package/dist/sessions/native-upload.js +168 -0
- package/dist/sessions/native-upload.js.map +1 -0
- package/dist/sessions/store.d.ts +40 -0
- package/dist/sessions/store.js +77 -0
- package/dist/sessions/store.js.map +1 -0
- package/dist/sessions/upload.d.ts +10 -0
- package/dist/sessions/upload.js +329 -0
- package/dist/sessions/upload.js.map +1 -0
- package/dist/sessions.d.ts +98 -0
- package/dist/sessions.js +268 -0
- package/dist/sessions.js.map +1 -0
- package/dist/spaces/admin.d.ts +112 -0
- package/dist/spaces/admin.js +19 -0
- package/dist/spaces/admin.js.map +1 -0
- package/dist/spaces.d.ts +15 -0
- package/dist/spaces.js +15 -0
- package/dist/spaces.js.map +1 -0
- package/dist/supabase/admin.d.ts +77 -0
- package/dist/supabase/admin.js +74 -0
- package/dist/supabase/admin.js.map +1 -0
- package/dist/supabase/s3.d.ts +311 -0
- package/dist/supabase/s3.js +153 -0
- package/dist/supabase/s3.js.map +1 -0
- package/dist/supabase/signing.d.ts +23 -0
- package/dist/supabase/signing.js +93 -0
- package/dist/supabase/signing.js.map +1 -0
- package/dist/supabase/tus-uploads.d.ts +13 -0
- package/dist/supabase/tus-uploads.js +109 -0
- package/dist/supabase/tus-uploads.js.map +1 -0
- package/dist/supabase/uploads.d.ts +5 -0
- package/dist/supabase/uploads.js +32 -0
- package/dist/supabase/uploads.js.map +1 -0
- package/dist/supabase.d.ts +100 -0
- package/dist/supabase.js +50 -0
- package/dist/supabase.js.map +1 -0
- package/dist/testing/body-failure.js +18 -0
- package/dist/testing/body-failure.js.map +1 -0
- package/dist/testing/conformance/assertions.d.ts +10 -0
- package/dist/testing/conformance/assertions.js +21 -0
- package/dist/testing/conformance/assertions.js.map +1 -0
- package/dist/testing/conformance/core-cases.js +107 -0
- package/dist/testing/conformance/core-cases.js.map +1 -0
- package/dist/testing/conformance/list-cases.js +72 -0
- package/dist/testing/conformance/list-cases.js.map +1 -0
- package/dist/testing/conformance/read-cases.js +105 -0
- package/dist/testing/conformance/read-cases.js.map +1 -0
- package/dist/testing/conformance.d.ts +23 -0
- package/dist/testing/conformance.js +37 -0
- package/dist/testing/conformance.js.map +1 -0
- package/dist/testing/faults.d.ts +258 -0
- package/dist/testing/faults.js +111 -0
- package/dist/testing/faults.js.map +1 -0
- package/dist/testing/memory-list.js +79 -0
- package/dist/testing/memory-list.js.map +1 -0
- package/dist/testing/memory-state.d.ts +18 -0
- package/dist/testing/memory-state.js +57 -0
- package/dist/testing/memory-state.js.map +1 -0
- package/dist/testing/memory-upload.js +123 -0
- package/dist/testing/memory-upload.js.map +1 -0
- package/dist/testing/memory.d.ts +23 -0
- package/dist/testing/memory.js +121 -0
- package/dist/testing/memory.js.map +1 -0
- package/dist/testing.d.ts +6 -0
- package/dist/testing.js +6 -0
- package/dist/tigris/events.d.ts +42 -0
- package/dist/tigris/events.js +42 -0
- package/dist/tigris/events.js.map +1 -0
- package/dist/tigris/native.d.ts +438 -0
- package/dist/tigris/native.js +166 -0
- package/dist/tigris/native.js.map +1 -0
- package/dist/tigris.d.ts +15 -0
- package/dist/tigris.js +15 -0
- package/dist/tigris.js.map +1 -0
- package/dist/transfers/budget.js +50 -0
- package/dist/transfers/budget.js.map +1 -0
- package/dist/transfers/copy.js +250 -0
- package/dist/transfers/copy.js.map +1 -0
- package/dist/transfers/dependencies.d.ts +10 -0
- package/dist/transfers/dependencies.js +11 -0
- package/dist/transfers/dependencies.js.map +1 -0
- package/dist/transfers/download.js +132 -0
- package/dist/transfers/download.js.map +1 -0
- package/dist/transfers/managed-runtime.d.ts +34 -0
- package/dist/transfers/managed-runtime.js +30 -0
- package/dist/transfers/managed-runtime.js.map +1 -0
- package/dist/transfers/models.d.ts +160 -0
- package/dist/transfers/native-upload.js +195 -0
- package/dist/transfers/native-upload.js.map +1 -0
- package/dist/transfers/part-receipts.js +32 -0
- package/dist/transfers/part-receipts.js.map +1 -0
- package/dist/transfers/planner.d.ts +19 -0
- package/dist/transfers/planner.js +153 -0
- package/dist/transfers/planner.js.map +1 -0
- package/dist/transfers/reader.js +81 -0
- package/dist/transfers/reader.js.map +1 -0
- package/dist/transfers/runtime.d.ts +19 -0
- package/dist/transfers/runtime.js +132 -0
- package/dist/transfers/runtime.js.map +1 -0
- package/dist/transfers/sources.d.ts +22 -0
- package/dist/transfers/sources.js +83 -0
- package/dist/transfers/sources.js.map +1 -0
- package/dist/transfers/upload.js +235 -0
- package/dist/transfers/upload.js.map +1 -0
- package/dist/transfers.d.ts +31 -0
- package/dist/transfers.js +82 -0
- package/dist/transfers.js.map +1 -0
- package/dist/uploads/driver.d.ts +22 -0
- package/dist/uploads/driver.js +18 -0
- package/dist/uploads/driver.js.map +1 -0
- package/dist/uploads/policy.d.ts +39 -0
- package/dist/uploads/schema.d.ts +278 -0
- package/dist/uploads/schema.js +167 -0
- package/dist/uploads/schema.js.map +1 -0
- package/dist/uploads/signed-driver.d.ts +8 -0
- package/dist/uploads/signed-driver.js +36 -0
- package/dist/uploads/signed-driver.js.map +1 -0
- package/dist/uploads/store.d.ts +26 -0
- package/dist/uploads/store.js +51 -0
- package/dist/uploads/store.js.map +1 -0
- package/dist/uploads/verification.js +86 -0
- package/dist/uploads/verification.js.map +1 -0
- package/dist/uploads.d.ts +518 -0
- package/dist/uploads.js +330 -0
- package/dist/uploads.js.map +1 -0
- package/dist/vercel/authorized-uploads.d.ts +7 -0
- package/dist/vercel/authorized-uploads.js +15 -0
- package/dist/vercel/authorized-uploads.js.map +1 -0
- package/dist/vercel/client.d.ts +16 -0
- package/dist/vercel/client.js +28 -0
- package/dist/vercel/client.js.map +1 -0
- package/dist/vercel/signing.d.ts +19 -0
- package/dist/vercel/signing.js +120 -0
- package/dist/vercel/signing.js.map +1 -0
- package/dist/vercel/uploads.d.ts +36 -0
- package/dist/vercel/uploads.js +161 -0
- package/dist/vercel/uploads.js.map +1 -0
- package/dist/vercel.d.ts +86 -0
- package/dist/vercel.js +46 -0
- package/dist/vercel.js.map +1 -0
- package/dist/wasabi/admin.d.ts +112 -0
- package/dist/wasabi/admin.js +19 -0
- package/dist/wasabi/admin.js.map +1 -0
- package/dist/wasabi/native.d.ts +200 -0
- package/dist/wasabi/native.js +132 -0
- package/dist/wasabi/native.js.map +1 -0
- package/dist/wasabi.d.ts +15 -0
- package/dist/wasabi.js +15 -0
- package/dist/wasabi.js.map +1 -0
- package/dist/workflows/content/collection.js +122 -0
- package/dist/workflows/content/collection.js.map +1 -0
- package/dist/workflows/content/references.js +113 -0
- package/dist/workflows/content/references.js.map +1 -0
- package/dist/workflows/content/schema.d.ts +58 -0
- package/dist/workflows/content/schema.js +64 -0
- package/dist/workflows/content/schema.js.map +1 -0
- package/dist/workflows/endpoint.js +16 -0
- package/dist/workflows/endpoint.js.map +1 -0
- package/dist/workflows/migrations/schema.d.ts +149 -0
- package/dist/workflows/migrations/schema.js +58 -0
- package/dist/workflows/migrations/schema.js.map +1 -0
- package/dist/workflows/mirror/operations.js +175 -0
- package/dist/workflows/mirror/operations.js.map +1 -0
- package/dist/workflows/mirror/schema.d.ts +68 -0
- package/dist/workflows/mirror/schema.js +57 -0
- package/dist/workflows/mirror/schema.js.map +1 -0
- package/dist/workflows/mirror/view.js +103 -0
- package/dist/workflows/mirror/view.js.map +1 -0
- package/dist/workflows/objects.js +54 -0
- package/dist/workflows/objects.js.map +1 -0
- package/dist/workflows/store.d.ts +13 -0
- package/dist/workflows/store.js +58 -0
- package/dist/workflows/store.js.map +1 -0
- package/package.json +513 -3
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { Checksum, ObjectSize } from "../core/object.js";
|
|
2
|
+
import { storageError } from "../core/errors.js";
|
|
3
|
+
import { Effect, Schema } from "effect";
|
|
4
|
+
|
|
5
|
+
// src/transfers/part-receipts.ts
|
|
6
|
+
const Receipt = Schema.Struct({
|
|
7
|
+
partNumber: Schema.Int.check(Schema.isGreaterThan(0)),
|
|
8
|
+
etag: Schema.NonEmptyString.check(Schema.isMaxLength(256)),
|
|
9
|
+
size: Schema.optional(ObjectSize),
|
|
10
|
+
checksums: Schema.optional(Schema.Array(Schema.Struct({
|
|
11
|
+
...Checksum.fields,
|
|
12
|
+
algorithm: Schema.String.check(Schema.isMaxLength(32)),
|
|
13
|
+
value: Schema.String.check(Schema.isMaxLength(128))
|
|
14
|
+
})).check(Schema.isMaxLength(8)))
|
|
15
|
+
});
|
|
16
|
+
/** Bounds identities retained until completion and the subsequent protocol serialization. */
|
|
17
|
+
const partReceipt = Effect.fnUntraced(function* (input, partNumber, size, provider) {
|
|
18
|
+
const receipt = yield* Schema.decodeUnknownEffect(Receipt)(input).pipe(Effect.mapError(() => storageError("upload.part", provider, { _tag: "IntegrityMismatch" }, "not-started")));
|
|
19
|
+
if (receipt.partNumber !== partNumber) return yield* storageError("upload.part", provider, { _tag: "IntegrityMismatch" }, "not-started");
|
|
20
|
+
if (receipt.size !== void 0 && receipt.size !== size) return yield* storageError("upload.part", provider, {
|
|
21
|
+
_tag: "LengthMismatch",
|
|
22
|
+
expected: size,
|
|
23
|
+
actual: receipt.size ?? 0
|
|
24
|
+
}, "not-started");
|
|
25
|
+
return {
|
|
26
|
+
...receipt,
|
|
27
|
+
size
|
|
28
|
+
};
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
export { partReceipt };
|
|
32
|
+
//# sourceMappingURL=part-receipts.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"part-receipts.js","names":[],"sources":["../../src/transfers/part-receipts.ts"],"sourcesContent":["import { Effect, Schema } from \"effect\";\n\nimport type { UploadedPart } from \"../core/contracts/multipart.js\";\nimport { storageError } from \"../core/errors.js\";\nimport { Checksum, ObjectSize } from \"../core/object.js\";\n\nconst Receipt = Schema.Struct({\n partNumber: Schema.Int.check(Schema.isGreaterThan(0)),\n etag: Schema.NonEmptyString.check(Schema.isMaxLength(256)),\n size: Schema.optional(ObjectSize),\n checksums: Schema.optional(\n Schema.Array(\n Schema.Struct({\n ...Checksum.fields,\n algorithm: Schema.String.check(Schema.isMaxLength(32)),\n value: Schema.String.check(Schema.isMaxLength(128)),\n }),\n ).check(Schema.isMaxLength(8)),\n ),\n});\n\n/** Bounds identities retained until completion and the subsequent protocol serialization. */\nexport const partReceipt = Effect.fnUntraced(function* (\n input: UploadedPart,\n partNumber: number,\n size: number,\n provider: string,\n) {\n const receipt = yield* Schema.decodeUnknownEffect(Receipt)(input).pipe(\n Effect.mapError(() =>\n storageError(\"upload.part\", provider, { _tag: \"IntegrityMismatch\" }, \"not-started\"),\n ),\n );\n if (receipt.partNumber !== partNumber)\n return yield* storageError(\n \"upload.part\",\n provider,\n { _tag: \"IntegrityMismatch\" },\n \"not-started\",\n );\n if (receipt.size !== undefined && receipt.size !== size)\n return yield* storageError(\n \"upload.part\",\n provider,\n { _tag: \"LengthMismatch\", expected: size, actual: receipt.size ?? 0 },\n \"not-started\",\n );\n return { ...receipt, size };\n});\n"],"mappings":";;;;;AAMA,MAAM,UAAU,OAAO,OAAO;CAC5B,YAAY,OAAO,IAAI,MAAM,OAAO,cAAc,CAAC,CAAC;CACpD,MAAM,OAAO,eAAe,MAAM,OAAO,YAAY,GAAG,CAAC;CACzD,MAAM,OAAO,SAAS,UAAU;CAChC,WAAW,OAAO,SAChB,OAAO,MACL,OAAO,OAAO;EACZ,GAAG,SAAS;EACZ,WAAW,OAAO,OAAO,MAAM,OAAO,YAAY,EAAE,CAAC;EACrD,OAAO,OAAO,OAAO,MAAM,OAAO,YAAY,GAAG,CAAC;CACpD,CAAC,CACH,CAAC,CAAC,MAAM,OAAO,YAAY,CAAC,CAAC,CAC/B;AACF,CAAC;;AAGD,MAAa,cAAc,OAAO,WAAW,WAC3C,OACA,YACA,MACA,UACA;CACA,MAAM,UAAU,OAAO,OAAO,oBAAoB,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,KAChE,OAAO,eACL,aAAa,eAAe,UAAU,EAAE,MAAM,oBAAoB,GAAG,aAAa,CACpF,CACF;CACA,IAAI,QAAQ,eAAe,YACzB,OAAO,OAAO,aACZ,eACA,UACA,EAAE,MAAM,oBAAoB,GAC5B,aACF;CACF,IAAI,QAAQ,SAAS,UAAa,QAAQ,SAAS,MACjD,OAAO,OAAO,aACZ,eACA,UACA;EAAE,MAAM;EAAkB,UAAU;EAAM,QAAQ,QAAQ,QAAQ;CAAE,GACpE,aACF;CACF,OAAO;EAAE,GAAG;EAAS;CAAK;AAC5B,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import "../core/requests.js";
|
|
2
|
+
import "../index.js";
|
|
3
|
+
import "./models.js";
|
|
4
|
+
import { Effect } from "effect";
|
|
5
|
+
export interface ResolvedPolicy {
|
|
6
|
+
readonly memoryBudgetBytes: number;
|
|
7
|
+
readonly partBytes: number;
|
|
8
|
+
readonly partConcurrency: number;
|
|
9
|
+
readonly requestConcurrency: number;
|
|
10
|
+
readonly maxSourceChunkBytes: number;
|
|
11
|
+
readonly maxBytes: number;
|
|
12
|
+
readonly deadlineMillis: number;
|
|
13
|
+
readonly maxAttempts: number;
|
|
14
|
+
readonly retryBaseMillis: number;
|
|
15
|
+
readonly retryMaxMillis: number;
|
|
16
|
+
readonly cleanupTimeoutMillis: number;
|
|
17
|
+
readonly observerTimeoutMillis: number;
|
|
18
|
+
}
|
|
19
|
+
//# sourceMappingURL=planner.d.ts.map
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
import { storageError } from "../core/errors.js";
|
|
2
|
+
import { Effect } from "effect";
|
|
3
|
+
|
|
4
|
+
// src/transfers/planner.ts
|
|
5
|
+
const MiB = 1048576;
|
|
6
|
+
function resolvePolicy(input, endpoint) {
|
|
7
|
+
return Effect.gen(function* () {
|
|
8
|
+
const policy = {
|
|
9
|
+
memoryBudgetBytes: input.memoryBudgetBytes ?? 64 * MiB,
|
|
10
|
+
partBytes: input.partBytes ?? 8 * MiB,
|
|
11
|
+
partConcurrency: input.partConcurrency ?? 3,
|
|
12
|
+
requestConcurrency: input.requestConcurrency ?? 6,
|
|
13
|
+
maxSourceChunkBytes: input.maxSourceChunkBytes ?? MiB,
|
|
14
|
+
maxBytes: input.maxBytes ?? Math.min(endpoint.maximumObjectBytes, 1024 * MiB),
|
|
15
|
+
deadlineMillis: input.deadlineMillis ?? 9e5,
|
|
16
|
+
maxAttempts: input.maxAttempts ?? 1e3,
|
|
17
|
+
retryBaseMillis: input.retryBaseMillis ?? 100,
|
|
18
|
+
retryMaxMillis: input.retryMaxMillis ?? 1e4,
|
|
19
|
+
cleanupTimeoutMillis: input.cleanupTimeoutMillis ?? 5e3,
|
|
20
|
+
observerTimeoutMillis: input.observerTimeoutMillis ?? 100
|
|
21
|
+
};
|
|
22
|
+
for (const [field, value] of Object.entries(policy)) if (!Number.isSafeInteger(value) || value < (field === "maxBytes" || field === "retryBaseMillis" ? 0 : 1)) return yield* storageError("transfers.make", endpoint.storage.provider, {
|
|
23
|
+
_tag: "InvalidConfiguration",
|
|
24
|
+
field
|
|
25
|
+
});
|
|
26
|
+
if ([
|
|
27
|
+
endpoint.maximumObjectBytes,
|
|
28
|
+
endpoint.maximumSingleUploadBytes,
|
|
29
|
+
endpoint.transportOverheadBytes,
|
|
30
|
+
input.digest?.workspaceBytes
|
|
31
|
+
].some((value) => !Number.isSafeInteger(value) || value < 0) || !input.digest?.algorithm || typeof input.digest.create !== "function") return yield* storageError("transfers.make", endpoint.storage.provider, {
|
|
32
|
+
_tag: "InvalidConfiguration",
|
|
33
|
+
field: "endpoint-or-digest"
|
|
34
|
+
});
|
|
35
|
+
if (endpoint.multipart) {
|
|
36
|
+
const { minimumPartBytes, maximumPartBytes, maximumParts } = endpoint.multipart.limits;
|
|
37
|
+
if (![
|
|
38
|
+
minimumPartBytes,
|
|
39
|
+
maximumPartBytes,
|
|
40
|
+
maximumParts
|
|
41
|
+
].every((value) => Number.isSafeInteger(value) && value > 0) || minimumPartBytes > maximumPartBytes) return yield* storageError("transfers.make", endpoint.storage.provider, {
|
|
42
|
+
_tag: "InvalidConfiguration",
|
|
43
|
+
field: "multipart.limits"
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
return policy;
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
function planUpload(endpoint, policy, length, inputDigestWorkspace, strategy = "auto", serverCopy = false, condition) {
|
|
50
|
+
return Effect.gen(function* () {
|
|
51
|
+
const provider = endpoint.storage.provider;
|
|
52
|
+
const maximumBytes = Math.min(policy.maxBytes, endpoint.maximumObjectBytes);
|
|
53
|
+
if (length !== void 0 && (!Number.isSafeInteger(length) || length < 0)) return yield* storageError("upload.plan", provider, {
|
|
54
|
+
_tag: "InvalidInput",
|
|
55
|
+
field: "source.length"
|
|
56
|
+
}, "not-started");
|
|
57
|
+
if (length !== void 0 && length > maximumBytes) return yield* storageError("upload.plan", provider, {
|
|
58
|
+
_tag: "LimitExceeded",
|
|
59
|
+
limit: maximumBytes
|
|
60
|
+
}, "not-started");
|
|
61
|
+
if (![
|
|
62
|
+
"auto",
|
|
63
|
+
"single",
|
|
64
|
+
"multipart",
|
|
65
|
+
"offset",
|
|
66
|
+
"blocks"
|
|
67
|
+
].includes(strategy)) return yield* storageError("upload.plan", provider, {
|
|
68
|
+
_tag: "InvalidInput",
|
|
69
|
+
field: "strategy"
|
|
70
|
+
}, "not-started");
|
|
71
|
+
const native = strategy === "offset" || strategy === "blocks" ? strategy : strategy === "auto" && !endpoint.multipart ? endpoint.offsets ? "offset" : endpoint.blocks ? "blocks" : void 0 : void 0;
|
|
72
|
+
if (native && (native === "offset" ? !endpoint.offsets || length === void 0 : !endpoint.blocks)) return yield* storageError("upload.plan", provider, {
|
|
73
|
+
_tag: "Unsupported",
|
|
74
|
+
capability: `${native}-known-source`
|
|
75
|
+
}, "not-started");
|
|
76
|
+
const single = strategy === "single" || strategy === "auto" && length !== void 0 && length <= Math.min(policy.partBytes, endpoint.maximumSingleUploadBytes) || !endpoint.multipart && !native;
|
|
77
|
+
let partBytes;
|
|
78
|
+
let concurrency;
|
|
79
|
+
if (single) {
|
|
80
|
+
partBytes = length ?? maximumBytes;
|
|
81
|
+
concurrency = 1;
|
|
82
|
+
if (strategy === "multipart" || partBytes > endpoint.maximumSingleUploadBytes) return yield* storageError("upload.plan", provider, {
|
|
83
|
+
_tag: "Unsupported",
|
|
84
|
+
capability: "bounded-single-upload"
|
|
85
|
+
}, "not-started");
|
|
86
|
+
} else if (native) {
|
|
87
|
+
const alignment = native === "offset" ? endpoint.offsets?.limits.alignmentBytes ?? 0 : 1;
|
|
88
|
+
const maxPart = native === "offset" ? endpoint.offsets?.limits.maximumChunkBytes ?? 0 : endpoint.blocks?.limits.maximumBlockBytes ?? 0;
|
|
89
|
+
const count = native === "blocks" ? endpoint.blocks?.limits.maximumBlocks ?? 0 : Number.MAX_SAFE_INTEGER;
|
|
90
|
+
if (![
|
|
91
|
+
alignment,
|
|
92
|
+
maxPart,
|
|
93
|
+
count
|
|
94
|
+
].every((value) => Number.isSafeInteger(value) && value > 0)) return yield* storageError("upload.plan", provider, {
|
|
95
|
+
_tag: "InvalidConfiguration",
|
|
96
|
+
field: "native.limits"
|
|
97
|
+
}, "not-started");
|
|
98
|
+
const fixed = native === "offset" ? endpoint.offsets?.limits.fixedChunkBytes : void 0;
|
|
99
|
+
if (fixed !== void 0 && (!Number.isSafeInteger(fixed) || fixed <= 0 || fixed > maxPart || fixed % alignment !== 0)) return yield* storageError("upload.plan", provider, {
|
|
100
|
+
_tag: "InvalidConfiguration",
|
|
101
|
+
field: "native.fixedChunkBytes"
|
|
102
|
+
}, "not-started");
|
|
103
|
+
partBytes = fixed ?? Math.ceil(Math.max(policy.partBytes, Math.ceil((length ?? maximumBytes) / count)) / alignment) * alignment;
|
|
104
|
+
concurrency = 1;
|
|
105
|
+
if (partBytes > maxPart) return yield* storageError("upload.plan", provider, {
|
|
106
|
+
_tag: "LimitExceeded",
|
|
107
|
+
limit: maxPart
|
|
108
|
+
}, "not-started");
|
|
109
|
+
if (length !== void 0 && Math.max(1, Math.ceil(length / partBytes)) + 3 > policy.maxAttempts) return yield* storageError("upload.plan", provider, {
|
|
110
|
+
_tag: "LimitExceeded",
|
|
111
|
+
limit: policy.maxAttempts
|
|
112
|
+
}, "not-started");
|
|
113
|
+
} else {
|
|
114
|
+
const multipart = endpoint.multipart;
|
|
115
|
+
if (!multipart) return yield* storageError("upload.plan", provider, {
|
|
116
|
+
_tag: "Unsupported",
|
|
117
|
+
capability: "multipart"
|
|
118
|
+
}, "not-started");
|
|
119
|
+
const limits = multipart.limits;
|
|
120
|
+
if (condition && limits.completeConditions && !limits.completeConditions.includes(condition._tag === "CreateOnly" ? "create-only" : "if-match")) return yield* storageError("upload.plan", provider, {
|
|
121
|
+
_tag: "Unsupported",
|
|
122
|
+
capability: "multipart-conditional-completion"
|
|
123
|
+
}, "not-started");
|
|
124
|
+
partBytes = Math.max(policy.partBytes, limits.minimumPartBytes, Math.ceil((length ?? maximumBytes) / limits.maximumParts));
|
|
125
|
+
concurrency = policy.partConcurrency;
|
|
126
|
+
if (partBytes > limits.maximumPartBytes) return yield* storageError("upload.plan", provider, {
|
|
127
|
+
_tag: "LimitExceeded",
|
|
128
|
+
limit: limits.maximumPartBytes * limits.maximumParts
|
|
129
|
+
}, "not-started");
|
|
130
|
+
if (length !== void 0 && Math.max(1, Math.ceil(length / partBytes)) + 2 > policy.maxAttempts) return yield* storageError("upload.plan", provider, {
|
|
131
|
+
_tag: "LimitExceeded",
|
|
132
|
+
limit: policy.maxAttempts
|
|
133
|
+
}, "not-started");
|
|
134
|
+
}
|
|
135
|
+
const receiptBytes = single ? 0 : Math.max(1, Math.ceil((length ?? maximumBytes) / partBytes)) * 8192;
|
|
136
|
+
const uploadWorkspaceBytes = (serverCopy ? 0 : (concurrency + 1) * Math.max(1, partBytes) + policy.maxSourceChunkBytes) + concurrency * endpoint.transportOverheadBytes + inputDigestWorkspace + receiptBytes;
|
|
137
|
+
const workspaceBytes = Math.max(uploadWorkspaceBytes, native ? policy.maxSourceChunkBytes * 2 + endpoint.transportOverheadBytes + inputDigestWorkspace : 0);
|
|
138
|
+
if (!Number.isSafeInteger(workspaceBytes) || workspaceBytes > policy.memoryBudgetBytes) return yield* storageError("upload.plan", provider, {
|
|
139
|
+
_tag: "QuotaExceeded",
|
|
140
|
+
limit: policy.memoryBudgetBytes
|
|
141
|
+
}, "not-started");
|
|
142
|
+
return {
|
|
143
|
+
strategy: single ? "single" : native ?? "multipart",
|
|
144
|
+
partBytes,
|
|
145
|
+
concurrency,
|
|
146
|
+
maximumBytes,
|
|
147
|
+
workspaceBytes
|
|
148
|
+
};
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
export { planUpload, resolvePolicy };
|
|
153
|
+
//# sourceMappingURL=planner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"planner.js","names":[],"sources":["../../src/transfers/planner.ts"],"sourcesContent":["import { Effect } from \"effect\";\n\nimport { storageError } from \"../core/errors.js\";\nimport type { WriteCondition } from \"../core/requests.js\";\nimport type { TransferEndpoint, TransferConfiguration } from \"./models.js\";\n\nconst MiB = 1024 * 1024;\n\nexport interface ResolvedPolicy {\n readonly memoryBudgetBytes: number;\n readonly partBytes: number;\n readonly partConcurrency: number;\n readonly requestConcurrency: number;\n readonly maxSourceChunkBytes: number;\n readonly maxBytes: number;\n readonly deadlineMillis: number;\n readonly maxAttempts: number;\n readonly retryBaseMillis: number;\n readonly retryMaxMillis: number;\n readonly cleanupTimeoutMillis: number;\n readonly observerTimeoutMillis: number;\n}\n\nexport function resolvePolicy(input: TransferConfiguration, endpoint: TransferEndpoint) {\n return Effect.gen(function* () {\n const policy: ResolvedPolicy = {\n memoryBudgetBytes: input.memoryBudgetBytes ?? 64 * MiB,\n partBytes: input.partBytes ?? 8 * MiB,\n partConcurrency: input.partConcurrency ?? 3,\n requestConcurrency: input.requestConcurrency ?? 6,\n maxSourceChunkBytes: input.maxSourceChunkBytes ?? MiB,\n maxBytes: input.maxBytes ?? Math.min(endpoint.maximumObjectBytes, 1024 * MiB),\n deadlineMillis: input.deadlineMillis ?? 15 * 60 * 1000,\n maxAttempts: input.maxAttempts ?? 1000,\n retryBaseMillis: input.retryBaseMillis ?? 100,\n retryMaxMillis: input.retryMaxMillis ?? 10000,\n cleanupTimeoutMillis: input.cleanupTimeoutMillis ?? 5000,\n observerTimeoutMillis: input.observerTimeoutMillis ?? 100,\n };\n for (const [field, value] of Object.entries(policy)) {\n if (\n !Number.isSafeInteger(value) ||\n value < (field === \"maxBytes\" || field === \"retryBaseMillis\" ? 0 : 1)\n )\n return yield* storageError(\"transfers.make\", endpoint.storage.provider, {\n _tag: \"InvalidConfiguration\",\n field,\n });\n }\n const limits = [\n endpoint.maximumObjectBytes,\n endpoint.maximumSingleUploadBytes,\n endpoint.transportOverheadBytes,\n input.digest?.workspaceBytes,\n ];\n if (\n limits.some((value) => !Number.isSafeInteger(value) || value < 0) ||\n !input.digest?.algorithm ||\n typeof input.digest.create !== \"function\"\n )\n return yield* storageError(\"transfers.make\", endpoint.storage.provider, {\n _tag: \"InvalidConfiguration\",\n field: \"endpoint-or-digest\",\n });\n if (endpoint.multipart) {\n const { minimumPartBytes, maximumPartBytes, maximumParts } = endpoint.multipart.limits;\n if (\n ![minimumPartBytes, maximumPartBytes, maximumParts].every(\n (value) => Number.isSafeInteger(value) && value > 0,\n ) ||\n minimumPartBytes > maximumPartBytes\n )\n return yield* storageError(\"transfers.make\", endpoint.storage.provider, {\n _tag: \"InvalidConfiguration\",\n field: \"multipart.limits\",\n });\n }\n return policy;\n });\n}\n\nexport interface UploadPlan {\n readonly strategy: \"single\" | \"multipart\" | \"offset\" | \"blocks\";\n readonly partBytes: number;\n readonly concurrency: number;\n readonly maximumBytes: number;\n readonly workspaceBytes: number;\n}\n\nexport function planUpload(\n endpoint: TransferEndpoint,\n policy: ResolvedPolicy,\n length: number | undefined,\n inputDigestWorkspace: number,\n strategy: \"auto\" | \"single\" | \"multipart\" | \"offset\" | \"blocks\" = \"auto\",\n serverCopy = false,\n condition?: WriteCondition,\n) {\n return Effect.gen(function* () {\n const provider = endpoint.storage.provider;\n const maximumBytes = Math.min(policy.maxBytes, endpoint.maximumObjectBytes);\n if (length !== undefined && (!Number.isSafeInteger(length) || length < 0))\n return yield* storageError(\n \"upload.plan\",\n provider,\n { _tag: \"InvalidInput\", field: \"source.length\" },\n \"not-started\",\n );\n if (length !== undefined && length > maximumBytes)\n return yield* storageError(\n \"upload.plan\",\n provider,\n { _tag: \"LimitExceeded\", limit: maximumBytes },\n \"not-started\",\n );\n if (![\"auto\", \"single\", \"multipart\", \"offset\", \"blocks\"].includes(strategy))\n return yield* storageError(\n \"upload.plan\",\n provider,\n { _tag: \"InvalidInput\", field: \"strategy\" },\n \"not-started\",\n );\n\n const native =\n strategy === \"offset\" || strategy === \"blocks\"\n ? strategy\n : strategy === \"auto\" && !endpoint.multipart\n ? endpoint.offsets\n ? \"offset\"\n : endpoint.blocks\n ? \"blocks\"\n : undefined\n : undefined;\n if (\n native &&\n (native === \"offset\" ? !endpoint.offsets || length === undefined : !endpoint.blocks)\n )\n return yield* storageError(\n \"upload.plan\",\n provider,\n { _tag: \"Unsupported\", capability: `${native}-known-source` },\n \"not-started\",\n );\n const single =\n strategy === \"single\" ||\n (strategy === \"auto\" &&\n length !== undefined &&\n length <= Math.min(policy.partBytes, endpoint.maximumSingleUploadBytes)) ||\n (!endpoint.multipart && !native);\n let partBytes: number;\n let concurrency: number;\n if (single) {\n partBytes = length ?? maximumBytes;\n concurrency = 1;\n if (strategy === \"multipart\" || partBytes > endpoint.maximumSingleUploadBytes)\n return yield* storageError(\n \"upload.plan\",\n provider,\n { _tag: \"Unsupported\", capability: \"bounded-single-upload\" },\n \"not-started\",\n );\n } else if (native) {\n const alignment = native === \"offset\" ? (endpoint.offsets?.limits.alignmentBytes ?? 0) : 1;\n const maxPart =\n native === \"offset\"\n ? (endpoint.offsets?.limits.maximumChunkBytes ?? 0)\n : (endpoint.blocks?.limits.maximumBlockBytes ?? 0);\n const count =\n native === \"blocks\"\n ? (endpoint.blocks?.limits.maximumBlocks ?? 0)\n : Number.MAX_SAFE_INTEGER;\n if (![alignment, maxPart, count].every((value) => Number.isSafeInteger(value) && value > 0))\n return yield* storageError(\n \"upload.plan\",\n provider,\n { _tag: \"InvalidConfiguration\", field: \"native.limits\" },\n \"not-started\",\n );\n const fixed = native === \"offset\" ? endpoint.offsets?.limits.fixedChunkBytes : undefined;\n if (\n fixed !== undefined &&\n (!Number.isSafeInteger(fixed) || fixed <= 0 || fixed > maxPart || fixed % alignment !== 0)\n )\n return yield* storageError(\n \"upload.plan\",\n provider,\n { _tag: \"InvalidConfiguration\", field: \"native.fixedChunkBytes\" },\n \"not-started\",\n );\n partBytes =\n fixed ??\n Math.ceil(\n Math.max(policy.partBytes, Math.ceil((length ?? maximumBytes) / count)) / alignment,\n ) * alignment;\n concurrency = 1;\n if (partBytes > maxPart)\n return yield* storageError(\n \"upload.plan\",\n provider,\n { _tag: \"LimitExceeded\", limit: maxPart },\n \"not-started\",\n );\n if (\n length !== undefined &&\n Math.max(1, Math.ceil(length / partBytes)) + 3 > policy.maxAttempts\n )\n return yield* storageError(\n \"upload.plan\",\n provider,\n { _tag: \"LimitExceeded\", limit: policy.maxAttempts },\n \"not-started\",\n );\n } else {\n const multipart = endpoint.multipart;\n if (!multipart)\n return yield* storageError(\n \"upload.plan\",\n provider,\n { _tag: \"Unsupported\", capability: \"multipart\" },\n \"not-started\",\n );\n const limits = multipart.limits;\n if (\n condition &&\n limits.completeConditions &&\n !limits.completeConditions.includes(\n condition._tag === \"CreateOnly\" ? \"create-only\" : \"if-match\",\n )\n )\n return yield* storageError(\n \"upload.plan\",\n provider,\n { _tag: \"Unsupported\", capability: \"multipart-conditional-completion\" },\n \"not-started\",\n );\n partBytes = Math.max(\n policy.partBytes,\n limits.minimumPartBytes,\n Math.ceil((length ?? maximumBytes) / limits.maximumParts),\n );\n concurrency = policy.partConcurrency;\n if (partBytes > limits.maximumPartBytes)\n return yield* storageError(\n \"upload.plan\",\n provider,\n { _tag: \"LimitExceeded\", limit: limits.maximumPartBytes * limits.maximumParts },\n \"not-started\",\n );\n if (\n length !== undefined &&\n Math.max(1, Math.ceil(length / partBytes)) + 2 > policy.maxAttempts\n )\n return yield* storageError(\n \"upload.plan\",\n provider,\n { _tag: \"LimitExceeded\", limit: policy.maxAttempts },\n \"not-started\",\n );\n }\n\n // One producer buffer, retained worker buffers, one upstream chunk, and explicit transport allowance.\n const receiptBytes = single\n ? 0\n : Math.max(1, Math.ceil((length ?? maximumBytes) / partBytes)) * 8192;\n const uploadWorkspaceBytes =\n (serverCopy ? 0 : (concurrency + 1) * Math.max(1, partBytes) + policy.maxSourceChunkBytes) +\n concurrency * endpoint.transportOverheadBytes +\n inputDigestWorkspace +\n receiptBytes;\n const workspaceBytes = Math.max(\n uploadWorkspaceBytes,\n native\n ? policy.maxSourceChunkBytes * 2 + endpoint.transportOverheadBytes + inputDigestWorkspace\n : 0,\n );\n if (!Number.isSafeInteger(workspaceBytes) || workspaceBytes > policy.memoryBudgetBytes)\n return yield* storageError(\n \"upload.plan\",\n provider,\n { _tag: \"QuotaExceeded\", limit: policy.memoryBudgetBytes },\n \"not-started\",\n );\n return {\n strategy: single ? (\"single\" as const) : (native ?? (\"multipart\" as const)),\n partBytes,\n concurrency,\n maximumBytes,\n workspaceBytes,\n };\n });\n}\n"],"mappings":";;;;AAMA,MAAM,MAAM;AAiBZ,SAAgB,cAAc,OAA8B,UAA4B;CACtF,OAAO,OAAO,IAAI,aAAa;EAC7B,MAAM,SAAyB;GAC7B,mBAAmB,MAAM,qBAAqB,KAAK;GACnD,WAAW,MAAM,aAAa,IAAI;GAClC,iBAAiB,MAAM,mBAAmB;GAC1C,oBAAoB,MAAM,sBAAsB;GAChD,qBAAqB,MAAM,uBAAuB;GAClD,UAAU,MAAM,YAAY,KAAK,IAAI,SAAS,oBAAoB,OAAO,GAAG;GAC5E,gBAAgB,MAAM,kBAAkB;GACxC,aAAa,MAAM,eAAe;GAClC,iBAAiB,MAAM,mBAAmB;GAC1C,gBAAgB,MAAM,kBAAkB;GACxC,sBAAsB,MAAM,wBAAwB;GACpD,uBAAuB,MAAM,yBAAyB;EACxD;EACA,KAAK,MAAM,CAAC,OAAO,UAAU,OAAO,QAAQ,MAAM,GAChD,IACE,CAAC,OAAO,cAAc,KAAK,KAC3B,SAAS,UAAU,cAAc,UAAU,oBAAoB,IAAI,IAEnE,OAAO,OAAO,aAAa,kBAAkB,SAAS,QAAQ,UAAU;GACtE,MAAM;GACN;EACF,CAAC;EAQL,IACE;GANA,SAAS;GACT,SAAS;GACT,SAAS;GACT,MAAM,QAAQ;EAGT,CAAC,CAAC,MAAM,UAAU,CAAC,OAAO,cAAc,KAAK,KAAK,QAAQ,CAAC,KAChE,CAAC,MAAM,QAAQ,aACf,OAAO,MAAM,OAAO,WAAW,YAE/B,OAAO,OAAO,aAAa,kBAAkB,SAAS,QAAQ,UAAU;GACtE,MAAM;GACN,OAAO;EACT,CAAC;EACH,IAAI,SAAS,WAAW;GACtB,MAAM,EAAE,kBAAkB,kBAAkB,iBAAiB,SAAS,UAAU;GAChF,IACE,CAAC;IAAC;IAAkB;IAAkB;GAAY,CAAC,CAAC,OACjD,UAAU,OAAO,cAAc,KAAK,KAAK,QAAQ,CACpD,KACA,mBAAmB,kBAEnB,OAAO,OAAO,aAAa,kBAAkB,SAAS,QAAQ,UAAU;IACtE,MAAM;IACN,OAAO;GACT,CAAC;EACL;EACA,OAAO;CACT,CAAC;AACH;AAUA,SAAgB,WACd,UACA,QACA,QACA,sBACA,WAAkE,QAClE,aAAa,OACb,WACA;CACA,OAAO,OAAO,IAAI,aAAa;EAC7B,MAAM,WAAW,SAAS,QAAQ;EAClC,MAAM,eAAe,KAAK,IAAI,OAAO,UAAU,SAAS,kBAAkB;EAC1E,IAAI,WAAW,WAAc,CAAC,OAAO,cAAc,MAAM,KAAK,SAAS,IACrE,OAAO,OAAO,aACZ,eACA,UACA;GAAE,MAAM;GAAgB,OAAO;EAAgB,GAC/C,aACF;EACF,IAAI,WAAW,UAAa,SAAS,cACnC,OAAO,OAAO,aACZ,eACA,UACA;GAAE,MAAM;GAAiB,OAAO;EAAa,GAC7C,aACF;EACF,IAAI,CAAC;GAAC;GAAQ;GAAU;GAAa;GAAU;EAAQ,CAAC,CAAC,SAAS,QAAQ,GACxE,OAAO,OAAO,aACZ,eACA,UACA;GAAE,MAAM;GAAgB,OAAO;EAAW,GAC1C,aACF;EAEF,MAAM,SACJ,aAAa,YAAY,aAAa,WAClC,WACA,aAAa,UAAU,CAAC,SAAS,YAC/B,SAAS,UACP,WACA,SAAS,SACP,WACA,SACJ;EACR,IACE,WACC,WAAW,WAAW,CAAC,SAAS,WAAW,WAAW,SAAY,CAAC,SAAS,SAE7E,OAAO,OAAO,aACZ,eACA,UACA;GAAE,MAAM;GAAe,YAAY,GAAG,OAAO;EAAe,GAC5D,aACF;EACF,MAAM,SACJ,aAAa,YACZ,aAAa,UACZ,WAAW,UACX,UAAU,KAAK,IAAI,OAAO,WAAW,SAAS,wBAAwB,KACvE,CAAC,SAAS,aAAa,CAAC;EAC3B,IAAI;EACJ,IAAI;EACJ,IAAI,QAAQ;GACV,YAAY,UAAU;GACtB,cAAc;GACd,IAAI,aAAa,eAAe,YAAY,SAAS,0BACnD,OAAO,OAAO,aACZ,eACA,UACA;IAAE,MAAM;IAAe,YAAY;GAAwB,GAC3D,aACF;EACJ,OAAO,IAAI,QAAQ;GACjB,MAAM,YAAY,WAAW,WAAY,SAAS,SAAS,OAAO,kBAAkB,IAAK;GACzF,MAAM,UACJ,WAAW,WACN,SAAS,SAAS,OAAO,qBAAqB,IAC9C,SAAS,QAAQ,OAAO,qBAAqB;GACpD,MAAM,QACJ,WAAW,WACN,SAAS,QAAQ,OAAO,iBAAiB,IAC1C,OAAO;GACb,IAAI,CAAC;IAAC;IAAW;IAAS;GAAK,CAAC,CAAC,OAAO,UAAU,OAAO,cAAc,KAAK,KAAK,QAAQ,CAAC,GACxF,OAAO,OAAO,aACZ,eACA,UACA;IAAE,MAAM;IAAwB,OAAO;GAAgB,GACvD,aACF;GACF,MAAM,QAAQ,WAAW,WAAW,SAAS,SAAS,OAAO,kBAAkB;GAC/E,IACE,UAAU,WACT,CAAC,OAAO,cAAc,KAAK,KAAK,SAAS,KAAK,QAAQ,WAAW,QAAQ,cAAc,IAExF,OAAO,OAAO,aACZ,eACA,UACA;IAAE,MAAM;IAAwB,OAAO;GAAyB,GAChE,aACF;GACF,YACE,SACA,KAAK,KACH,KAAK,IAAI,OAAO,WAAW,KAAK,MAAM,UAAU,gBAAgB,KAAK,CAAC,IAAI,SAC5E,IAAI;GACN,cAAc;GACd,IAAI,YAAY,SACd,OAAO,OAAO,aACZ,eACA,UACA;IAAE,MAAM;IAAiB,OAAO;GAAQ,GACxC,aACF;GACF,IACE,WAAW,UACX,KAAK,IAAI,GAAG,KAAK,KAAK,SAAS,SAAS,CAAC,IAAI,IAAI,OAAO,aAExD,OAAO,OAAO,aACZ,eACA,UACA;IAAE,MAAM;IAAiB,OAAO,OAAO;GAAY,GACnD,aACF;EACJ,OAAO;GACL,MAAM,YAAY,SAAS;GAC3B,IAAI,CAAC,WACH,OAAO,OAAO,aACZ,eACA,UACA;IAAE,MAAM;IAAe,YAAY;GAAY,GAC/C,aACF;GACF,MAAM,SAAS,UAAU;GACzB,IACE,aACA,OAAO,sBACP,CAAC,OAAO,mBAAmB,SACzB,UAAU,SAAS,eAAe,gBAAgB,UACpD,GAEA,OAAO,OAAO,aACZ,eACA,UACA;IAAE,MAAM;IAAe,YAAY;GAAmC,GACtE,aACF;GACF,YAAY,KAAK,IACf,OAAO,WACP,OAAO,kBACP,KAAK,MAAM,UAAU,gBAAgB,OAAO,YAAY,CAC1D;GACA,cAAc,OAAO;GACrB,IAAI,YAAY,OAAO,kBACrB,OAAO,OAAO,aACZ,eACA,UACA;IAAE,MAAM;IAAiB,OAAO,OAAO,mBAAmB,OAAO;GAAa,GAC9E,aACF;GACF,IACE,WAAW,UACX,KAAK,IAAI,GAAG,KAAK,KAAK,SAAS,SAAS,CAAC,IAAI,IAAI,OAAO,aAExD,OAAO,OAAO,aACZ,eACA,UACA;IAAE,MAAM;IAAiB,OAAO,OAAO;GAAY,GACnD,aACF;EACJ;EAGA,MAAM,eAAe,SACjB,IACA,KAAK,IAAI,GAAG,KAAK,MAAM,UAAU,gBAAgB,SAAS,CAAC,IAAI;EACnE,MAAM,wBACH,aAAa,KAAK,cAAc,KAAK,KAAK,IAAI,GAAG,SAAS,IAAI,OAAO,uBACtE,cAAc,SAAS,yBACvB,uBACA;EACF,MAAM,iBAAiB,KAAK,IAC1B,sBACA,SACI,OAAO,sBAAsB,IAAI,SAAS,yBAAyB,uBACnE,CACN;EACA,IAAI,CAAC,OAAO,cAAc,cAAc,KAAK,iBAAiB,OAAO,mBACnE,OAAO,OAAO,aACZ,eACA,UACA;GAAE,MAAM;GAAiB,OAAO,OAAO;EAAkB,GACzD,aACF;EACF,OAAO;GACL,UAAU,SAAU,WAAsB,UAAW;GACrD;GACA;GACA;GACA;EACF;CACF,CAAC;AACH"}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { storageError } from "../core/errors.js";
|
|
2
|
+
import { Effect, Pull, Stream } from "effect";
|
|
3
|
+
import { Base64, Hex } from "effect/encoding";
|
|
4
|
+
|
|
5
|
+
// src/transfers/reader.ts
|
|
6
|
+
const failure = (reason) => storageError("source.read", "transfer", reason, "not-started");
|
|
7
|
+
const makeReader = Effect.fnUntraced(function* (source, options) {
|
|
8
|
+
const pull = yield* Stream.toPull(source.open().pipe(Stream.rechunk(1)));
|
|
9
|
+
const hash = options.digest.create();
|
|
10
|
+
let pending;
|
|
11
|
+
let offset = 0;
|
|
12
|
+
let total = 0;
|
|
13
|
+
let ended = false;
|
|
14
|
+
let checksum;
|
|
15
|
+
return {
|
|
16
|
+
next: Effect.fnUntraced(function* (capacity) {
|
|
17
|
+
if (ended && !pending) return void 0;
|
|
18
|
+
const buffer = new Uint8Array(Math.max(1, capacity));
|
|
19
|
+
let used = 0;
|
|
20
|
+
while (used < buffer.length) {
|
|
21
|
+
if (!pending) {
|
|
22
|
+
const chunks = yield* pull.pipe(Pull.catchDone(() => Effect.succeed(void 0)));
|
|
23
|
+
if (!chunks) {
|
|
24
|
+
ended = true;
|
|
25
|
+
if (source.length !== void 0 && total !== source.length) return yield* failure({
|
|
26
|
+
_tag: "LengthMismatch",
|
|
27
|
+
expected: source.length,
|
|
28
|
+
actual: total
|
|
29
|
+
});
|
|
30
|
+
checksum = hash.finish();
|
|
31
|
+
break;
|
|
32
|
+
}
|
|
33
|
+
pending = chunks[0];
|
|
34
|
+
offset = 0;
|
|
35
|
+
if (!(pending instanceof Uint8Array)) return yield* failure({
|
|
36
|
+
_tag: "InvalidInput",
|
|
37
|
+
field: "source.chunk"
|
|
38
|
+
});
|
|
39
|
+
if (pending.buffer.byteLength > options.maximumChunkBytes) return yield* failure({
|
|
40
|
+
_tag: "LimitExceeded",
|
|
41
|
+
limit: options.maximumChunkBytes
|
|
42
|
+
});
|
|
43
|
+
if (pending.byteLength === 0) {
|
|
44
|
+
pending = void 0;
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
const count = Math.min(buffer.length - used, pending.length - offset);
|
|
49
|
+
if (total + count > options.maximumBytes) return yield* failure({
|
|
50
|
+
_tag: "LimitExceeded",
|
|
51
|
+
limit: options.maximumBytes
|
|
52
|
+
});
|
|
53
|
+
if (source.length !== void 0 && total + count > source.length) return yield* failure({
|
|
54
|
+
_tag: "LengthMismatch",
|
|
55
|
+
expected: source.length,
|
|
56
|
+
actual: total + count
|
|
57
|
+
});
|
|
58
|
+
const slice = pending.subarray(offset, offset + count);
|
|
59
|
+
buffer.set(slice, used);
|
|
60
|
+
hash.update(slice);
|
|
61
|
+
used += count;
|
|
62
|
+
total += count;
|
|
63
|
+
offset += count;
|
|
64
|
+
options.onRead(count);
|
|
65
|
+
if (offset === pending.length) pending = void 0;
|
|
66
|
+
}
|
|
67
|
+
return used === 0 ? void 0 : buffer.subarray(0, used);
|
|
68
|
+
}),
|
|
69
|
+
count: () => total,
|
|
70
|
+
checksum: () => checksum
|
|
71
|
+
};
|
|
72
|
+
});
|
|
73
|
+
function sameChecksum(actual, expected) {
|
|
74
|
+
if (expected.scope !== "full-object" || actual.scope !== "full-object" || actual.algorithm !== expected.algorithm) return false;
|
|
75
|
+
const left = actual.encoding === "hex" ? Hex.decode(actual.value) : Base64.decode(actual.value);
|
|
76
|
+
const right = expected.encoding === "hex" ? Hex.decode(expected.value) : Base64.decode(expected.value);
|
|
77
|
+
return left._tag === "Success" && right._tag === "Success" && left.success.length === right.success.length && left.success.every((byte, index) => byte === right.success[index]);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export { makeReader, sameChecksum };
|
|
81
|
+
//# sourceMappingURL=reader.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reader.js","names":[],"sources":["../../src/transfers/reader.ts"],"sourcesContent":["import { Effect, Pull, Stream } from \"effect\";\nimport { Base64, Hex } from \"effect/encoding\";\n\nimport type { StorageError } from \"../core/errors.js\";\nimport { storageError } from \"../core/errors.js\";\nimport type { Checksum } from \"../core/object.js\";\nimport type { TransferDigest } from \"./models.js\";\nimport type { UploadSource } from \"./sources.js\";\n\nconst failure = (reason: StorageError[\"reason\"]) =>\n storageError(\"source.read\", \"transfer\", reason, \"not-started\");\n\nexport const makeReader = Effect.fnUntraced(function* <E, R>(\n source: UploadSource<E, R>,\n options: {\n readonly maximumBytes: number;\n readonly maximumChunkBytes: number;\n readonly digest: TransferDigest;\n readonly onRead: (count: number) => void;\n },\n) {\n const pull = yield* Stream.toPull(source.open().pipe(Stream.rechunk(1)));\n const hash = options.digest.create();\n let pending: Uint8Array | undefined;\n let offset = 0;\n let total = 0;\n let ended = false;\n let checksum: Checksum | undefined;\n\n const next = Effect.fnUntraced(function* (\n capacity: number,\n ): Effect.fn.Return<Uint8Array | undefined, E | StorageError, R> {\n if (ended && !pending) return undefined;\n const buffer = new Uint8Array(Math.max(1, capacity));\n let used = 0;\n while (used < buffer.length) {\n if (!pending) {\n const chunks = yield* pull.pipe(Pull.catchDone(() => Effect.succeed(undefined)));\n if (!chunks) {\n ended = true;\n if (source.length !== undefined && total !== source.length)\n return yield* failure({\n _tag: \"LengthMismatch\",\n expected: source.length,\n actual: total,\n });\n checksum = hash.finish();\n break;\n }\n pending = chunks[0];\n offset = 0;\n if (!(pending instanceof Uint8Array))\n return yield* failure({ _tag: \"InvalidInput\", field: \"source.chunk\" });\n // A tiny subarray can keep a huge backing allocation alive through a retry.\n if (pending.buffer.byteLength > options.maximumChunkBytes)\n return yield* failure({ _tag: \"LimitExceeded\", limit: options.maximumChunkBytes });\n if (pending.byteLength === 0) {\n pending = undefined;\n continue;\n }\n }\n\n const count = Math.min(buffer.length - used, pending.length - offset);\n if (total + count > options.maximumBytes)\n return yield* failure({ _tag: \"LimitExceeded\", limit: options.maximumBytes });\n if (source.length !== undefined && total + count > source.length)\n return yield* failure({\n _tag: \"LengthMismatch\",\n expected: source.length,\n actual: total + count,\n });\n const slice = pending.subarray(offset, offset + count);\n buffer.set(slice, used);\n hash.update(slice);\n used += count;\n total += count;\n offset += count;\n options.onRead(count);\n if (offset === pending.length) pending = undefined;\n }\n return used === 0 ? undefined : buffer.subarray(0, used);\n });\n\n return {\n next,\n count: () => total,\n checksum: () => checksum,\n };\n});\n\nexport function sameChecksum(actual: Checksum, expected: Checksum): boolean {\n if (\n expected.scope !== \"full-object\" ||\n actual.scope !== \"full-object\" ||\n actual.algorithm !== expected.algorithm\n )\n return false;\n const left = actual.encoding === \"hex\" ? Hex.decode(actual.value) : Base64.decode(actual.value);\n const right =\n expected.encoding === \"hex\" ? Hex.decode(expected.value) : Base64.decode(expected.value);\n return (\n left._tag === \"Success\" &&\n right._tag === \"Success\" &&\n left.success.length === right.success.length &&\n left.success.every((byte, index) => byte === right.success[index])\n );\n}\n"],"mappings":";;;;;AASA,MAAM,WAAW,WACf,aAAa,eAAe,YAAY,QAAQ,aAAa;AAE/D,MAAa,aAAa,OAAO,WAAW,WAC1C,QACA,SAMA;CACA,MAAM,OAAO,OAAO,OAAO,OAAO,OAAO,KAAK,CAAC,CAAC,KAAK,OAAO,QAAQ,CAAC,CAAC,CAAC;CACvE,MAAM,OAAO,QAAQ,OAAO,OAAO;CACnC,IAAI;CACJ,IAAI,SAAS;CACb,IAAI,QAAQ;CACZ,IAAI,QAAQ;CACZ,IAAI;CAwDJ,OAAO;EACL,MAvDW,OAAO,WAAW,WAC7B,UAC+D;GAC/D,IAAI,SAAS,CAAC,SAAS,OAAO;GAC9B,MAAM,SAAS,IAAI,WAAW,KAAK,IAAI,GAAG,QAAQ,CAAC;GACnD,IAAI,OAAO;GACX,OAAO,OAAO,OAAO,QAAQ;IAC3B,IAAI,CAAC,SAAS;KACZ,MAAM,SAAS,OAAO,KAAK,KAAK,KAAK,gBAAgB,OAAO,QAAQ,MAAS,CAAC,CAAC;KAC/E,IAAI,CAAC,QAAQ;MACX,QAAQ;MACR,IAAI,OAAO,WAAW,UAAa,UAAU,OAAO,QAClD,OAAO,OAAO,QAAQ;OACpB,MAAM;OACN,UAAU,OAAO;OACjB,QAAQ;MACV,CAAC;MACH,WAAW,KAAK,OAAO;MACvB;KACF;KACA,UAAU,OAAO;KACjB,SAAS;KACT,IAAI,EAAE,mBAAmB,aACvB,OAAO,OAAO,QAAQ;MAAE,MAAM;MAAgB,OAAO;KAAe,CAAC;KAEvE,IAAI,QAAQ,OAAO,aAAa,QAAQ,mBACtC,OAAO,OAAO,QAAQ;MAAE,MAAM;MAAiB,OAAO,QAAQ;KAAkB,CAAC;KACnF,IAAI,QAAQ,eAAe,GAAG;MAC5B,UAAU;MACV;KACF;IACF;IAEA,MAAM,QAAQ,KAAK,IAAI,OAAO,SAAS,MAAM,QAAQ,SAAS,MAAM;IACpE,IAAI,QAAQ,QAAQ,QAAQ,cAC1B,OAAO,OAAO,QAAQ;KAAE,MAAM;KAAiB,OAAO,QAAQ;IAAa,CAAC;IAC9E,IAAI,OAAO,WAAW,UAAa,QAAQ,QAAQ,OAAO,QACxD,OAAO,OAAO,QAAQ;KACpB,MAAM;KACN,UAAU,OAAO;KACjB,QAAQ,QAAQ;IAClB,CAAC;IACH,MAAM,QAAQ,QAAQ,SAAS,QAAQ,SAAS,KAAK;IACrD,OAAO,IAAI,OAAO,IAAI;IACtB,KAAK,OAAO,KAAK;IACjB,QAAQ;IACR,SAAS;IACT,UAAU;IACV,QAAQ,OAAO,KAAK;IACpB,IAAI,WAAW,QAAQ,QAAQ,UAAU;GAC3C;GACA,OAAO,SAAS,IAAI,SAAY,OAAO,SAAS,GAAG,IAAI;EACzD,CAGK;EACH,aAAa;EACb,gBAAgB;CAClB;AACF,CAAC;AAED,SAAgB,aAAa,QAAkB,UAA6B;CAC1E,IACE,SAAS,UAAU,iBACnB,OAAO,UAAU,iBACjB,OAAO,cAAc,SAAS,WAE9B,OAAO;CACT,MAAM,OAAO,OAAO,aAAa,QAAQ,IAAI,OAAO,OAAO,KAAK,IAAI,OAAO,OAAO,OAAO,KAAK;CAC9F,MAAM,QACJ,SAAS,aAAa,QAAQ,IAAI,OAAO,SAAS,KAAK,IAAI,OAAO,OAAO,SAAS,KAAK;CACzF,OACE,KAAK,SAAS,aACd,MAAM,SAAS,aACf,KAAK,QAAQ,WAAW,MAAM,QAAQ,UACtC,KAAK,QAAQ,OAAO,MAAM,UAAU,SAAS,MAAM,QAAQ,MAAM;AAErE"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { StorageError } from "../core/errors.js";
|
|
2
|
+
import { TransferProgress } from "./models.js";
|
|
3
|
+
import "./planner.js";
|
|
4
|
+
import { Effect } from "effect";
|
|
5
|
+
import { SecureRandom } from "effect/crypto";
|
|
6
|
+
export interface Operation {
|
|
7
|
+
readonly id: string;
|
|
8
|
+
readonly progress: { -readonly [K in keyof TransferProgress]: TransferProgress[K]; };
|
|
9
|
+
readonly startedAt: number;
|
|
10
|
+
mutationStarted: boolean;
|
|
11
|
+
committed: boolean;
|
|
12
|
+
readonly request: <A, E = never, R = never>(effect: Effect.Effect<A, StorageError | E, R>, options?: {
|
|
13
|
+
readonly retry?: boolean;
|
|
14
|
+
readonly retryUnknown?: boolean;
|
|
15
|
+
readonly canRetry?: () => boolean;
|
|
16
|
+
}) => Effect.Effect<A, StorageError | E, R>;
|
|
17
|
+
readonly observe: (phase?: TransferProgress["phase"]) => Effect.Effect<void>;
|
|
18
|
+
}
|
|
19
|
+
//# sourceMappingURL=runtime.d.ts.map
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
import { StorageError, storageError } from "../core/errors.js";
|
|
2
|
+
import { makeBudget } from "./budget.js";
|
|
3
|
+
import { Cause, Clock, Effect, Exit, Metric, Random, Semaphore } from "effect";
|
|
4
|
+
|
|
5
|
+
// src/transfers/runtime.ts
|
|
6
|
+
const operations = Metric.counter("storage.transfer.operations");
|
|
7
|
+
const attempts = Metric.counter("storage.transfer.attempts");
|
|
8
|
+
const retries = Metric.counter("storage.transfer.retries");
|
|
9
|
+
const cleanupDebt = Metric.counter("storage.transfer.cleanup_debt");
|
|
10
|
+
const inFlight = Metric.gauge("storage.transfer.reserved_bytes");
|
|
11
|
+
function makeRuntime(endpoint, input, policy, random) {
|
|
12
|
+
const budget = makeBudget(policy.memoryBudgetBytes);
|
|
13
|
+
const requests = Semaphore.makeUnsafe(policy.requestConcurrency);
|
|
14
|
+
let activeRequests = 0;
|
|
15
|
+
let peakActiveRequests = 0;
|
|
16
|
+
let dropped = 0;
|
|
17
|
+
const diagnostics = [];
|
|
18
|
+
const diagnostic = (entry) => Effect.gen(function* () {
|
|
19
|
+
if (diagnostics.length < 128) diagnostics.push(entry);
|
|
20
|
+
else dropped++;
|
|
21
|
+
if (entry.kind === "cleanup" || entry.kind === "unknown-outcome") yield* Metric.update(cleanupDebt, 1);
|
|
22
|
+
});
|
|
23
|
+
return {
|
|
24
|
+
endpoint,
|
|
25
|
+
newId: () => random.uuidV4(),
|
|
26
|
+
input,
|
|
27
|
+
policy,
|
|
28
|
+
begin: Effect.fnUntraced(function* (operation, totalBytes) {
|
|
29
|
+
const id = yield* random.uuidV4();
|
|
30
|
+
const startedAt = yield* Clock.currentTimeMillis;
|
|
31
|
+
const progress = {
|
|
32
|
+
operation,
|
|
33
|
+
phase: "running",
|
|
34
|
+
totalBytes,
|
|
35
|
+
bytesRead: 0,
|
|
36
|
+
bytesSent: 0,
|
|
37
|
+
bytesAcknowledged: 0,
|
|
38
|
+
bytesVerified: 0,
|
|
39
|
+
attempts: 0
|
|
40
|
+
};
|
|
41
|
+
yield* Metric.update(operations, 1);
|
|
42
|
+
const state = {
|
|
43
|
+
id,
|
|
44
|
+
progress,
|
|
45
|
+
startedAt,
|
|
46
|
+
mutationStarted: false,
|
|
47
|
+
committed: false,
|
|
48
|
+
request: (effect, options = {}) => Effect.gen(function* () {
|
|
49
|
+
let retry = 0;
|
|
50
|
+
while (true) {
|
|
51
|
+
if (progress.attempts >= policy.maxAttempts) return yield* storageError(operation, endpoint.storage.provider, {
|
|
52
|
+
_tag: "LimitExceeded",
|
|
53
|
+
limit: policy.maxAttempts
|
|
54
|
+
}, state.committed ? "succeeded" : state.mutationStarted ? "unknown" : "not-started");
|
|
55
|
+
const result = yield* requests.withPermit(Effect.suspend(() => {
|
|
56
|
+
if (progress.attempts >= policy.maxAttempts) return Effect.fail(storageError(operation, endpoint.storage.provider, {
|
|
57
|
+
_tag: "LimitExceeded",
|
|
58
|
+
limit: policy.maxAttempts
|
|
59
|
+
}, state.committed ? "succeeded" : state.mutationStarted ? "unknown" : "not-started"));
|
|
60
|
+
progress.attempts++;
|
|
61
|
+
activeRequests++;
|
|
62
|
+
peakActiveRequests = Math.max(peakActiveRequests, activeRequests);
|
|
63
|
+
return Metric.update(attempts, 1).pipe(Effect.andThen(effect), Effect.ensuring(Effect.sync(() => {
|
|
64
|
+
activeRequests--;
|
|
65
|
+
})));
|
|
66
|
+
})).pipe(Effect.result);
|
|
67
|
+
if (result._tag === "Success") return result.success;
|
|
68
|
+
const error = result.failure;
|
|
69
|
+
if (!(error instanceof StorageError)) return yield* Effect.fail(error);
|
|
70
|
+
const reason = error.reason;
|
|
71
|
+
const transient = reason._tag === "TransportFailure" || reason._tag === "Throttled" || reason._tag === "ProviderFailure" && [
|
|
72
|
+
"InternalError",
|
|
73
|
+
"ServiceUnavailable",
|
|
74
|
+
"SlowDown",
|
|
75
|
+
"RequestTimeout"
|
|
76
|
+
].includes(reason.code ?? "");
|
|
77
|
+
if (!options.retry || options.canRetry?.() === false || !transient || error.mutationOutcome === "unknown" && !options.retryUnknown || progress.attempts >= policy.maxAttempts) return yield* error;
|
|
78
|
+
const cap = Math.min(policy.retryMaxMillis, policy.retryBaseMillis * 2 ** Math.min(retry++, 20));
|
|
79
|
+
const jitter = yield* Random.next;
|
|
80
|
+
const requested = reason._tag === "Throttled" ? reason.retryAfterMillis ?? 0 : 0;
|
|
81
|
+
const delay = Math.max(requested, Math.floor(cap * (.5 + jitter * .5)));
|
|
82
|
+
if ((yield* Clock.currentTimeMillis) + delay >= startedAt + policy.deadlineMillis) return yield* error;
|
|
83
|
+
yield* Metric.update(retries, 1);
|
|
84
|
+
yield* Effect.sleep(delay);
|
|
85
|
+
}
|
|
86
|
+
}),
|
|
87
|
+
observe: (phase = progress.phase) => Effect.gen(function* () {
|
|
88
|
+
progress.phase = phase;
|
|
89
|
+
const observer = input.observer;
|
|
90
|
+
if (!observer) return;
|
|
91
|
+
const result = yield* Effect.exit(Effect.suspend(() => observer({ ...progress })).pipe(Effect.timeout(policy.observerTimeoutMillis)));
|
|
92
|
+
if (Exit.isFailure(result)) yield* diagnostic({
|
|
93
|
+
kind: "observer",
|
|
94
|
+
operationId: id
|
|
95
|
+
});
|
|
96
|
+
})
|
|
97
|
+
};
|
|
98
|
+
return state;
|
|
99
|
+
}),
|
|
100
|
+
diagnostic,
|
|
101
|
+
diagnostics: Effect.sync(() => ({
|
|
102
|
+
entries: [...diagnostics],
|
|
103
|
+
dropped
|
|
104
|
+
})),
|
|
105
|
+
resources: Effect.sync(() => ({
|
|
106
|
+
...budget.snapshot(),
|
|
107
|
+
activeRequests,
|
|
108
|
+
peakActiveRequests
|
|
109
|
+
})),
|
|
110
|
+
cleanupRequest: (effect) => requests.withPermit(Effect.suspend(() => {
|
|
111
|
+
activeRequests++;
|
|
112
|
+
peakActiveRequests = Math.max(peakActiveRequests, activeRequests);
|
|
113
|
+
return Metric.update(attempts, 1).pipe(Effect.andThen(effect), Effect.ensuring(Effect.sync(() => {
|
|
114
|
+
activeRequests--;
|
|
115
|
+
})));
|
|
116
|
+
})),
|
|
117
|
+
lease: (bytes, effect) => budget.withLease(bytes, Effect.acquireUseRelease(Metric.modify(inFlight, bytes), () => effect, () => Metric.modify(inFlight, -bytes))),
|
|
118
|
+
deadline: (operation, effect) => effect.pipe(Effect.onExit((exit) => Exit.isFailure(exit) && Cause.hasInterrupts(exit.cause) && operation.mutationStarted && !operation.committed ? diagnostic({
|
|
119
|
+
kind: "unknown-outcome",
|
|
120
|
+
operationId: operation.id
|
|
121
|
+
}) : Effect.void), Effect.timeoutOrElse({
|
|
122
|
+
duration: policy.deadlineMillis,
|
|
123
|
+
orElse: () => Effect.fail(storageError(operation.progress.operation, endpoint.storage.provider, {
|
|
124
|
+
_tag: "TransportFailure",
|
|
125
|
+
code: "TransferDeadlineExceeded"
|
|
126
|
+
}, operation.committed ? "succeeded" : operation.mutationStarted ? "unknown" : "not-started"))
|
|
127
|
+
}))
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
export { makeRuntime };
|
|
132
|
+
//# sourceMappingURL=runtime.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runtime.js","names":[],"sources":["../../src/transfers/runtime.ts"],"sourcesContent":["import { Cause, Clock, Effect, Exit, Metric, Random, Semaphore } from \"effect\";\nimport type { SecureRandom } from \"effect/crypto\";\n\nimport { StorageError, storageError } from \"../core/errors.js\";\nimport { makeBudget } from \"./budget.js\";\nimport type {\n TransferDiagnostic,\n TransferEndpoint,\n TransferConfiguration,\n TransferProgress,\n} from \"./models.js\";\nimport type { ResolvedPolicy } from \"./planner.js\";\n\nconst operations = Metric.counter(\"storage.transfer.operations\");\nconst attempts = Metric.counter(\"storage.transfer.attempts\");\nconst retries = Metric.counter(\"storage.transfer.retries\");\nconst cleanupDebt = Metric.counter(\"storage.transfer.cleanup_debt\");\nconst inFlight = Metric.gauge(\"storage.transfer.reserved_bytes\");\n\nexport interface Operation {\n readonly id: string;\n readonly progress: { -readonly [K in keyof TransferProgress]: TransferProgress[K] };\n readonly startedAt: number;\n mutationStarted: boolean;\n committed: boolean;\n readonly request: <A, E = never, R = never>(\n effect: Effect.Effect<A, StorageError | E, R>,\n options?: {\n readonly retry?: boolean;\n readonly retryUnknown?: boolean;\n readonly canRetry?: () => boolean;\n },\n ) => Effect.Effect<A, StorageError | E, R>;\n readonly observe: (phase?: TransferProgress[\"phase\"]) => Effect.Effect<void>;\n}\n\nexport function makeRuntime(\n endpoint: TransferEndpoint,\n input: TransferConfiguration,\n policy: ResolvedPolicy,\n random: SecureRandom.SecureRandom[\"Service\"],\n) {\n const budget = makeBudget(policy.memoryBudgetBytes);\n const requests = Semaphore.makeUnsafe(policy.requestConcurrency);\n let activeRequests = 0;\n let peakActiveRequests = 0;\n let dropped = 0;\n const diagnostics: Array<TransferDiagnostic> = [];\n\n const diagnostic = (entry: TransferDiagnostic) =>\n Effect.gen(function* () {\n if (diagnostics.length < 128) diagnostics.push(entry);\n else dropped++;\n if (entry.kind === \"cleanup\" || entry.kind === \"unknown-outcome\")\n yield* Metric.update(cleanupDebt, 1);\n });\n\n const begin = Effect.fnUntraced(function* (\n operation: TransferProgress[\"operation\"],\n totalBytes?: number,\n ): Effect.fn.Return<Operation> {\n const id = yield* random.uuidV4();\n const startedAt = yield* Clock.currentTimeMillis;\n const progress = {\n operation,\n phase: \"running\" as TransferProgress[\"phase\"],\n totalBytes,\n bytesRead: 0,\n bytesSent: 0,\n bytesAcknowledged: 0,\n bytesVerified: 0,\n attempts: 0,\n };\n yield* Metric.update(operations, 1);\n\n const state: Operation = {\n id,\n progress,\n startedAt,\n mutationStarted: false,\n committed: false,\n request: <A, E = never, R = never>(\n effect: Effect.Effect<A, StorageError | E, R>,\n options: {\n readonly retry?: boolean;\n readonly retryUnknown?: boolean;\n readonly canRetry?: () => boolean;\n } = {},\n ) =>\n Effect.gen(function* () {\n let retry = 0;\n while (true) {\n if (progress.attempts >= policy.maxAttempts)\n return yield* storageError(\n operation,\n endpoint.storage.provider,\n { _tag: \"LimitExceeded\", limit: policy.maxAttempts },\n state.committed ? \"succeeded\" : state.mutationStarted ? \"unknown\" : \"not-started\",\n );\n const result = yield* requests\n .withPermit(\n Effect.suspend(() => {\n if (progress.attempts >= policy.maxAttempts)\n return Effect.fail(\n storageError(\n operation,\n endpoint.storage.provider,\n { _tag: \"LimitExceeded\", limit: policy.maxAttempts },\n state.committed\n ? \"succeeded\"\n : state.mutationStarted\n ? \"unknown\"\n : \"not-started\",\n ),\n );\n progress.attempts++;\n activeRequests++;\n peakActiveRequests = Math.max(peakActiveRequests, activeRequests);\n return Metric.update(attempts, 1).pipe(\n Effect.andThen(effect),\n Effect.ensuring(\n Effect.sync(() => {\n activeRequests--;\n }),\n ),\n );\n }),\n )\n .pipe(Effect.result);\n if (result._tag === \"Success\") return result.success;\n\n const error = result.failure;\n if (!(error instanceof StorageError)) return yield* Effect.fail(error);\n const reason = error.reason;\n const transient =\n reason._tag === \"TransportFailure\" ||\n reason._tag === \"Throttled\" ||\n (reason._tag === \"ProviderFailure\" &&\n [\"InternalError\", \"ServiceUnavailable\", \"SlowDown\", \"RequestTimeout\"].includes(\n reason.code ?? \"\",\n ));\n if (\n !options.retry ||\n options.canRetry?.() === false ||\n !transient ||\n (error.mutationOutcome === \"unknown\" && !options.retryUnknown) ||\n progress.attempts >= policy.maxAttempts\n )\n return yield* error;\n\n const cap = Math.min(\n policy.retryMaxMillis,\n policy.retryBaseMillis * 2 ** Math.min(retry++, 20),\n );\n const jitter = yield* Random.next;\n const requested = reason._tag === \"Throttled\" ? (reason.retryAfterMillis ?? 0) : 0;\n // Never retry sooner than Retry-After. If it exceeds the operation deadline, fail here.\n const delay = Math.max(requested, Math.floor(cap * (0.5 + jitter * 0.5)));\n const now = yield* Clock.currentTimeMillis;\n if (now + delay >= startedAt + policy.deadlineMillis) return yield* error;\n yield* Metric.update(retries, 1);\n yield* Effect.sleep(delay);\n }\n }),\n observe: (phase = progress.phase) =>\n Effect.gen(function* () {\n progress.phase = phase;\n const observer = input.observer;\n if (!observer) return;\n const result = yield* Effect.exit(\n Effect.suspend(() => observer({ ...progress })).pipe(\n Effect.timeout(policy.observerTimeoutMillis),\n ),\n );\n if (Exit.isFailure(result)) yield* diagnostic({ kind: \"observer\", operationId: id });\n }),\n };\n return state;\n });\n\n return {\n endpoint,\n newId: () => random.uuidV4(),\n input,\n policy,\n begin,\n diagnostic,\n diagnostics: Effect.sync(() => ({ entries: [...diagnostics], dropped })),\n resources: Effect.sync(() => ({ ...budget.snapshot(), activeRequests, peakActiveRequests })),\n cleanupRequest: <A, E, R>(effect: Effect.Effect<A, E, R>) =>\n requests.withPermit(\n Effect.suspend(() => {\n activeRequests++;\n peakActiveRequests = Math.max(peakActiveRequests, activeRequests);\n return Metric.update(attempts, 1).pipe(\n Effect.andThen(effect),\n Effect.ensuring(\n Effect.sync(() => {\n activeRequests--;\n }),\n ),\n );\n }),\n ),\n lease: <A, E, R>(bytes: number, effect: Effect.Effect<A, E, R>) =>\n budget.withLease(\n bytes,\n Effect.acquireUseRelease(\n Metric.modify(inFlight, bytes),\n () => effect,\n () => Metric.modify(inFlight, -bytes),\n ),\n ),\n deadline: <A, E, R>(operation: Operation, effect: Effect.Effect<A, E, R>) =>\n effect.pipe(\n Effect.onExit((exit) =>\n Exit.isFailure(exit) &&\n Cause.hasInterrupts(exit.cause) &&\n operation.mutationStarted &&\n !operation.committed\n ? diagnostic({ kind: \"unknown-outcome\", operationId: operation.id })\n : Effect.void,\n ),\n Effect.timeoutOrElse({\n duration: policy.deadlineMillis,\n orElse: () =>\n Effect.fail(\n storageError(\n operation.progress.operation,\n endpoint.storage.provider,\n { _tag: \"TransportFailure\", code: \"TransferDeadlineExceeded\" },\n operation.committed\n ? \"succeeded\"\n : operation.mutationStarted\n ? \"unknown\"\n : \"not-started\",\n ),\n ),\n }),\n ),\n };\n}\n\nexport type TransferRuntime = ReturnType<typeof makeRuntime>;\n"],"mappings":";;;;;AAaA,MAAM,aAAa,OAAO,QAAQ,6BAA6B;AAC/D,MAAM,WAAW,OAAO,QAAQ,2BAA2B;AAC3D,MAAM,UAAU,OAAO,QAAQ,0BAA0B;AACzD,MAAM,cAAc,OAAO,QAAQ,+BAA+B;AAClE,MAAM,WAAW,OAAO,MAAM,iCAAiC;AAmB/D,SAAgB,YACd,UACA,OACA,QACA,QACA;CACA,MAAM,SAAS,WAAW,OAAO,iBAAiB;CAClD,MAAM,WAAW,UAAU,WAAW,OAAO,kBAAkB;CAC/D,IAAI,iBAAiB;CACrB,IAAI,qBAAqB;CACzB,IAAI,UAAU;CACd,MAAM,cAAyC,CAAC;CAEhD,MAAM,cAAc,UAClB,OAAO,IAAI,aAAa;EACtB,IAAI,YAAY,SAAS,KAAK,YAAY,KAAK,KAAK;OAC/C;EACL,IAAI,MAAM,SAAS,aAAa,MAAM,SAAS,mBAC7C,OAAO,OAAO,OAAO,aAAa,CAAC;CACvC,CAAC;CA6HH,OAAO;EACL;EACA,aAAa,OAAO,OAAO;EAC3B;EACA;EACA,OAhIY,OAAO,WAAW,WAC9B,WACA,YAC6B;GAC7B,MAAM,KAAK,OAAO,OAAO,OAAO;GAChC,MAAM,YAAY,OAAO,MAAM;GAC/B,MAAM,WAAW;IACf;IACA,OAAO;IACP;IACA,WAAW;IACX,WAAW;IACX,mBAAmB;IACnB,eAAe;IACf,UAAU;GACZ;GACA,OAAO,OAAO,OAAO,YAAY,CAAC;GAElC,MAAM,QAAmB;IACvB;IACA;IACA;IACA,iBAAiB;IACjB,WAAW;IACX,UACE,QACA,UAII,CAAC,MAEL,OAAO,IAAI,aAAa;KACtB,IAAI,QAAQ;KACZ,OAAO,MAAM;MACX,IAAI,SAAS,YAAY,OAAO,aAC9B,OAAO,OAAO,aACZ,WACA,SAAS,QAAQ,UACjB;OAAE,MAAM;OAAiB,OAAO,OAAO;MAAY,GACnD,MAAM,YAAY,cAAc,MAAM,kBAAkB,YAAY,aACtE;MACF,MAAM,SAAS,OAAO,SACnB,WACC,OAAO,cAAc;OACnB,IAAI,SAAS,YAAY,OAAO,aAC9B,OAAO,OAAO,KACZ,aACE,WACA,SAAS,QAAQ,UACjB;QAAE,MAAM;QAAiB,OAAO,OAAO;OAAY,GACnD,MAAM,YACF,cACA,MAAM,kBACJ,YACA,aACR,CACF;OACF,SAAS;OACT;OACA,qBAAqB,KAAK,IAAI,oBAAoB,cAAc;OAChE,OAAO,OAAO,OAAO,UAAU,CAAC,CAAC,CAAC,KAChC,OAAO,QAAQ,MAAM,GACrB,OAAO,SACL,OAAO,WAAW;QAChB;OACF,CAAC,CACH,CACF;MACF,CAAC,CACH,CAAC,CACA,KAAK,OAAO,MAAM;MACrB,IAAI,OAAO,SAAS,WAAW,OAAO,OAAO;MAE7C,MAAM,QAAQ,OAAO;MACrB,IAAI,EAAE,iBAAiB,eAAe,OAAO,OAAO,OAAO,KAAK,KAAK;MACrE,MAAM,SAAS,MAAM;MACrB,MAAM,YACJ,OAAO,SAAS,sBAChB,OAAO,SAAS,eACf,OAAO,SAAS,qBACf;OAAC;OAAiB;OAAsB;OAAY;MAAgB,CAAC,CAAC,SACpE,OAAO,QAAQ,EACjB;MACJ,IACE,CAAC,QAAQ,SACT,QAAQ,WAAW,MAAM,SACzB,CAAC,aACA,MAAM,oBAAoB,aAAa,CAAC,QAAQ,gBACjD,SAAS,YAAY,OAAO,aAE5B,OAAO,OAAO;MAEhB,MAAM,MAAM,KAAK,IACf,OAAO,gBACP,OAAO,kBAAkB,KAAK,KAAK,IAAI,SAAS,EAAE,CACpD;MACA,MAAM,SAAS,OAAO,OAAO;MAC7B,MAAM,YAAY,OAAO,SAAS,cAAe,OAAO,oBAAoB,IAAK;MAEjF,MAAM,QAAQ,KAAK,IAAI,WAAW,KAAK,MAAM,OAAO,KAAM,SAAS,GAAI,CAAC;MAExE,KAAI,OADe,MAAM,qBACf,SAAS,YAAY,OAAO,gBAAgB,OAAO,OAAO;MACpE,OAAO,OAAO,OAAO,SAAS,CAAC;MAC/B,OAAO,OAAO,MAAM,KAAK;KAC3B;IACF,CAAC;IACH,UAAU,QAAQ,SAAS,UACzB,OAAO,IAAI,aAAa;KACtB,SAAS,QAAQ;KACjB,MAAM,WAAW,MAAM;KACvB,IAAI,CAAC,UAAU;KACf,MAAM,SAAS,OAAO,OAAO,KAC3B,OAAO,cAAc,SAAS,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,KAC9C,OAAO,QAAQ,OAAO,qBAAqB,CAC7C,CACF;KACA,IAAI,KAAK,UAAU,MAAM,GAAG,OAAO,WAAW;MAAE,MAAM;MAAY,aAAa;KAAG,CAAC;IACrF,CAAC;GACL;GACA,OAAO;EACT,CAOM;EACJ;EACA,aAAa,OAAO,YAAY;GAAE,SAAS,CAAC,GAAG,WAAW;GAAG;EAAQ,EAAE;EACvE,WAAW,OAAO,YAAY;GAAE,GAAG,OAAO,SAAS;GAAG;GAAgB;EAAmB,EAAE;EAC3F,iBAA0B,WACxB,SAAS,WACP,OAAO,cAAc;GACnB;GACA,qBAAqB,KAAK,IAAI,oBAAoB,cAAc;GAChE,OAAO,OAAO,OAAO,UAAU,CAAC,CAAC,CAAC,KAChC,OAAO,QAAQ,MAAM,GACrB,OAAO,SACL,OAAO,WAAW;IAChB;GACF,CAAC,CACH,CACF;EACF,CAAC,CACH;EACF,QAAiB,OAAe,WAC9B,OAAO,UACL,OACA,OAAO,kBACL,OAAO,OAAO,UAAU,KAAK,SACvB,cACA,OAAO,OAAO,UAAU,CAAC,KAAK,CACtC,CACF;EACF,WAAoB,WAAsB,WACxC,OAAO,KACL,OAAO,QAAQ,SACb,KAAK,UAAU,IAAI,KACnB,MAAM,cAAc,KAAK,KAAK,KAC9B,UAAU,mBACV,CAAC,UAAU,YACP,WAAW;GAAE,MAAM;GAAmB,aAAa,UAAU;EAAG,CAAC,IACjE,OAAO,IACb,GACA,OAAO,cAAc;GACnB,UAAU,OAAO;GACjB,cACE,OAAO,KACL,aACE,UAAU,SAAS,WACnB,SAAS,QAAQ,UACjB;IAAE,MAAM;IAAoB,MAAM;GAA2B,GAC7D,UAAU,YACN,cACA,UAAU,kBACR,YACA,aACR,CACF;EACJ,CAAC,CACH;CACJ;AACF"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { StorageError } from "../core/errors.js";
|
|
2
|
+
import { TransferHash } from "./dependencies.js";
|
|
3
|
+
import { Effect, FileSystem, Stream } from "effect";
|
|
4
|
+
import { PlatformError } from "effect/PlatformError";
|
|
5
|
+
declare namespace sources_d_exports {
|
|
6
|
+
export { UploadSource, bytes, file, replayable, stream };
|
|
7
|
+
}
|
|
8
|
+
export interface UploadSource<E = never, R = never> {
|
|
9
|
+
readonly length?: number | undefined;
|
|
10
|
+
readonly replayable: boolean;
|
|
11
|
+
readonly workspaceBytes?: number | undefined;
|
|
12
|
+
readonly open: () => Stream.Stream<Uint8Array, E | StorageError, R>;
|
|
13
|
+
}
|
|
14
|
+
/** Copies the caller's bytes once. This immutable snapshot is outside the engine's workspace. */
|
|
15
|
+
export declare function bytes(input: Uint8Array): UploadSource;
|
|
16
|
+
export declare function stream<E, R>(body: Stream.Stream<Uint8Array, E, R>, length?: number): UploadSource<E, R>;
|
|
17
|
+
/** The factory promises the same representation on every opening. */
|
|
18
|
+
export declare function replayable<E, R>(open: () => Stream.Stream<Uint8Array, E, R>, length?: number): UploadSource<E, R>;
|
|
19
|
+
/** Captures FileSystem once. Every opening checks identity and a full bounded-memory fingerprint. */
|
|
20
|
+
export declare const file: (path: string) => Effect.Effect<UploadSource<PlatformError | StorageError, never>, PlatformError | StorageError, FileSystem.FileSystem | TransferHash>;
|
|
21
|
+
export { sources_d_exports };
|
|
22
|
+
//# sourceMappingURL=sources.d.ts.map
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { __exportAll } from "../_virtual/_rolldown/runtime.js";
|
|
2
|
+
import { storageError } from "../core/errors.js";
|
|
3
|
+
import { TransferHash } from "./dependencies.js";
|
|
4
|
+
import { ByteSize, Cause, Effect, FileSystem, Option, Stream } from "effect";
|
|
5
|
+
|
|
6
|
+
// src/transfers/sources.ts
|
|
7
|
+
var sources_exports = /* @__PURE__ */ __exportAll({
|
|
8
|
+
bytes: () => bytes,
|
|
9
|
+
file: () => file,
|
|
10
|
+
replayable: () => replayable,
|
|
11
|
+
stream: () => stream
|
|
12
|
+
});
|
|
13
|
+
const sourceError = (field) => storageError("source", "transfer", {
|
|
14
|
+
_tag: "InvalidInput",
|
|
15
|
+
field
|
|
16
|
+
}, "not-started");
|
|
17
|
+
/** Copies the caller's bytes once. This immutable snapshot is outside the engine's workspace. */
|
|
18
|
+
function bytes(input) {
|
|
19
|
+
const snapshot = Uint8Array.from(input);
|
|
20
|
+
return replayable(() => Stream.fromIterable((function* () {
|
|
21
|
+
for (let start = 0; start < snapshot.length; start += 65536) yield snapshot.slice(start, start + 65536);
|
|
22
|
+
})(), { chunkSize: 1 }), snapshot.length);
|
|
23
|
+
}
|
|
24
|
+
function stream(body, length) {
|
|
25
|
+
let consumed = false;
|
|
26
|
+
return {
|
|
27
|
+
length,
|
|
28
|
+
replayable: false,
|
|
29
|
+
open: () => Stream.unwrap(Effect.suspend(() => {
|
|
30
|
+
if (consumed) return Effect.fail(storageError("source", "transfer", { _tag: "BodyConsumed" }, "not-started"));
|
|
31
|
+
consumed = true;
|
|
32
|
+
return Effect.succeed(body);
|
|
33
|
+
}))
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
/** The factory promises the same representation on every opening. */
|
|
37
|
+
function replayable(open, length) {
|
|
38
|
+
return {
|
|
39
|
+
length,
|
|
40
|
+
replayable: true,
|
|
41
|
+
open: () => Stream.suspend(open)
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
const identity = (info) => `${String(info.dev)}:${Option.getOrUndefined(info.ino)}:${String(ByteSize.toBigInt(info.size))}:${Option.getOrUndefined(info.mtime)?.getTime()}`;
|
|
45
|
+
/** Captures FileSystem once. Every opening checks identity and a full bounded-memory fingerprint. */
|
|
46
|
+
const file = Effect.fnUntraced(function* (path) {
|
|
47
|
+
const fs = yield* FileSystem.FileSystem;
|
|
48
|
+
const digest = yield* TransferHash;
|
|
49
|
+
const original = yield* fs.stat(path);
|
|
50
|
+
const length = Number(ByteSize.toBigInt(original.size));
|
|
51
|
+
if (original.type !== "File" || !Number.isSafeInteger(length)) return yield* sourceError("file");
|
|
52
|
+
const fingerprint = Effect.scoped(Effect.gen(function* () {
|
|
53
|
+
const handle = yield* fs.open(path);
|
|
54
|
+
const state = digest.create();
|
|
55
|
+
while (true) {
|
|
56
|
+
const chunk = yield* handle.readAlloc(65536);
|
|
57
|
+
if (Option.isNone(chunk)) break;
|
|
58
|
+
state.update(chunk.value);
|
|
59
|
+
}
|
|
60
|
+
return state.finish().value;
|
|
61
|
+
}));
|
|
62
|
+
const expected = yield* fingerprint;
|
|
63
|
+
return {
|
|
64
|
+
...replayable(() => Stream.unwrap(Effect.gen(function* () {
|
|
65
|
+
const current = yield* fs.stat(path);
|
|
66
|
+
if (identity(current) !== identity(original) || (yield* fingerprint) !== expected) return yield* sourceError("file.changed");
|
|
67
|
+
const state = digest.create();
|
|
68
|
+
const handle = yield* fs.open(path);
|
|
69
|
+
return Stream.fromPull(Effect.succeed(Effect.gen(function* () {
|
|
70
|
+
const chunk = yield* handle.readAlloc(65536);
|
|
71
|
+
if (Option.isNone(chunk)) return yield* Cause.done();
|
|
72
|
+
return [chunk.value];
|
|
73
|
+
}))).pipe(Stream.tap((chunk) => Effect.sync(() => state.update(chunk))), Stream.concat(Stream.drain(Stream.fromEffect(Effect.gen(function* () {
|
|
74
|
+
const after = yield* fs.stat(path);
|
|
75
|
+
if (identity(after) !== identity(original) || state.finish().value !== expected) return yield* sourceError("file.changed");
|
|
76
|
+
})))));
|
|
77
|
+
})), length),
|
|
78
|
+
workspaceBytes: 65536 + digest.workspaceBytes
|
|
79
|
+
};
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
export { bytes, file, replayable, sources_exports, stream };
|
|
83
|
+
//# sourceMappingURL=sources.js.map
|