@owlmeans/file-streams 0.0.0-stage → 0.2.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +71 -2
- package/agent-meta/manifest.json +64 -0
- package/agent-meta/skills/file-streams/SKILL.md +183 -0
- package/agent-meta/skills/file-streams/parts/network.md +47 -0
- package/agent-meta/skills/file-streams/parts/workspaces.md +129 -0
- package/build/archive/consts.d.ts +16 -0
- package/build/archive/consts.d.ts.map +1 -0
- package/build/archive/consts.js +24 -0
- package/build/archive/consts.js.map +1 -0
- package/build/archive/formats.d.ts +4 -0
- package/build/archive/formats.d.ts.map +1 -0
- package/build/archive/formats.js +53 -0
- package/build/archive/formats.js.map +1 -0
- package/build/archive/guard.d.ts +5 -0
- package/build/archive/guard.d.ts.map +1 -0
- package/build/archive/guard.js +62 -0
- package/build/archive/guard.js.map +1 -0
- package/build/archive/ratio.d.ts +5 -0
- package/build/archive/ratio.d.ts.map +1 -0
- package/build/archive/ratio.js +30 -0
- package/build/archive/ratio.js.map +1 -0
- package/build/archive/tar.d.ts +4 -0
- package/build/archive/tar.d.ts.map +1 -0
- package/build/archive/tar.js +142 -0
- package/build/archive/tar.js.map +1 -0
- package/build/archive/types.d.ts +95 -0
- package/build/archive/types.d.ts.map +1 -0
- package/build/archive/types.js +2 -0
- package/build/archive/types.js.map +1 -0
- package/build/archive/types.local.d.ts +64 -0
- package/build/archive/types.local.d.ts.map +1 -0
- package/build/archive/types.local.js +2 -0
- package/build/archive/types.local.js.map +1 -0
- package/build/archive/zip.d.ts +4 -0
- package/build/archive/zip.d.ts.map +1 -0
- package/build/archive/zip.js +135 -0
- package/build/archive/zip.js.map +1 -0
- package/build/archive.d.ts +4 -0
- package/build/archive.d.ts.map +1 -0
- package/build/archive.js +265 -0
- package/build/archive.js.map +1 -0
- package/build/backed-workspace/types.d.ts +67 -0
- package/build/backed-workspace/types.d.ts.map +1 -0
- package/build/backed-workspace/types.js +2 -0
- package/build/backed-workspace/types.js.map +1 -0
- package/build/backed-workspace.d.ts +4 -0
- package/build/backed-workspace.d.ts.map +1 -0
- package/build/backed-workspace.js +63 -0
- package/build/backed-workspace.js.map +1 -0
- package/build/consts.d.ts +3 -0
- package/build/consts.d.ts.map +1 -0
- package/build/consts.js +3 -0
- package/build/consts.js.map +1 -0
- package/build/consts.local.d.ts +3 -0
- package/build/consts.local.d.ts.map +1 -0
- package/build/consts.local.js +3 -0
- package/build/consts.local.js.map +1 -0
- package/build/errors.d.ts +59 -0
- package/build/errors.d.ts.map +1 -0
- package/build/errors.js +105 -0
- package/build/errors.js.map +1 -0
- package/build/http/connector.d.ts +11 -0
- package/build/http/connector.d.ts.map +1 -0
- package/build/http/connector.js +124 -0
- package/build/http/connector.js.map +1 -0
- package/build/http/consts.d.ts +9 -0
- package/build/http/consts.d.ts.map +1 -0
- package/build/http/consts.js +8 -0
- package/build/http/consts.js.map +1 -0
- package/build/http/source.d.ts +5 -0
- package/build/http/source.d.ts.map +1 -0
- package/build/http/source.js +76 -0
- package/build/http/source.js.map +1 -0
- package/build/http/types.d.ts +67 -0
- package/build/http/types.d.ts.map +1 -0
- package/build/http/types.js +2 -0
- package/build/http/types.js.map +1 -0
- package/build/http/types.local.d.ts +21 -0
- package/build/http/types.local.d.ts.map +1 -0
- package/build/http/types.local.js +2 -0
- package/build/http/types.local.js.map +1 -0
- package/build/http.d.ts +4 -0
- package/build/http.d.ts.map +1 -0
- package/build/http.js +248 -0
- package/build/http.js.map +1 -0
- package/build/index.d.ts +53 -0
- package/build/index.d.ts.map +1 -0
- package/build/index.js +33 -0
- package/build/index.js.map +1 -0
- package/build/network/consts.d.ts +12 -0
- package/build/network/consts.d.ts.map +1 -0
- package/build/network/consts.js +20 -0
- package/build/network/consts.js.map +1 -0
- package/build/network/types.d.ts +43 -0
- package/build/network/types.d.ts.map +1 -0
- package/build/network/types.js +2 -0
- package/build/network/types.js.map +1 -0
- package/build/network.d.ts +4 -0
- package/build/network.d.ts.map +1 -0
- package/build/network.js +165 -0
- package/build/network.js.map +1 -0
- package/build/proxy/consts.d.ts +7 -0
- package/build/proxy/consts.d.ts.map +1 -0
- package/build/proxy/consts.js +9 -0
- package/build/proxy/consts.js.map +1 -0
- package/build/proxy/endpoint.d.ts +4 -0
- package/build/proxy/endpoint.d.ts.map +1 -0
- package/build/proxy/endpoint.js +27 -0
- package/build/proxy/endpoint.js.map +1 -0
- package/build/proxy/http.d.ts +5 -0
- package/build/proxy/http.d.ts.map +1 -0
- package/build/proxy/http.js +95 -0
- package/build/proxy/http.js.map +1 -0
- package/build/proxy/router.d.ts +4 -0
- package/build/proxy/router.d.ts.map +1 -0
- package/build/proxy/router.js +85 -0
- package/build/proxy/router.js.map +1 -0
- package/build/proxy/socks.d.ts +5 -0
- package/build/proxy/socks.d.ts.map +1 -0
- package/build/proxy/socks.js +44 -0
- package/build/proxy/socks.js.map +1 -0
- package/build/proxy/types.d.ts +99 -0
- package/build/proxy/types.d.ts.map +1 -0
- package/build/proxy/types.js +2 -0
- package/build/proxy/types.js.map +1 -0
- package/build/resolvers/google-drive/consts.d.ts +10 -0
- package/build/resolvers/google-drive/consts.d.ts.map +1 -0
- package/build/resolvers/google-drive/consts.js +16 -0
- package/build/resolvers/google-drive/consts.js.map +1 -0
- package/build/resolvers/google-drive/types.d.ts +21 -0
- package/build/resolvers/google-drive/types.d.ts.map +1 -0
- package/build/resolvers/google-drive/types.js +2 -0
- package/build/resolvers/google-drive/types.js.map +1 -0
- package/build/resolvers/google-drive.d.ts +10 -0
- package/build/resolvers/google-drive.d.ts.map +1 -0
- package/build/resolvers/google-drive.js +130 -0
- package/build/resolvers/google-drive.js.map +1 -0
- package/build/resolvers/types.d.ts +39 -0
- package/build/resolvers/types.d.ts.map +1 -0
- package/build/resolvers/types.js +2 -0
- package/build/resolvers/types.js.map +1 -0
- package/build/s3/consts.d.ts +9 -0
- package/build/s3/consts.d.ts.map +1 -0
- package/build/s3/consts.js +9 -0
- package/build/s3/consts.js.map +1 -0
- package/build/s3/prefix.d.ts +4 -0
- package/build/s3/prefix.d.ts.map +1 -0
- package/build/s3/prefix.js +87 -0
- package/build/s3/prefix.js.map +1 -0
- package/build/s3/sink.d.ts +6 -0
- package/build/s3/sink.d.ts.map +1 -0
- package/build/s3/sink.js +30 -0
- package/build/s3/sink.js.map +1 -0
- package/build/s3/source.d.ts +5 -0
- package/build/s3/source.d.ts.map +1 -0
- package/build/s3/source.js +81 -0
- package/build/s3/source.js.map +1 -0
- package/build/s3/types.d.ts +66 -0
- package/build/s3/types.d.ts.map +1 -0
- package/build/s3/types.js +2 -0
- package/build/s3/types.js.map +1 -0
- package/build/s3/types.local.d.ts +43 -0
- package/build/s3/types.local.d.ts.map +1 -0
- package/build/s3/types.local.js +2 -0
- package/build/s3/types.local.js.map +1 -0
- package/build/s3/upload.d.ts +4 -0
- package/build/s3/upload.d.ts.map +1 -0
- package/build/s3/upload.js +118 -0
- package/build/s3/upload.js.map +1 -0
- package/build/s3.d.ts +5 -0
- package/build/s3.d.ts.map +1 -0
- package/build/s3.js +68 -0
- package/build/s3.js.map +1 -0
- package/build/service/types.d.ts +66 -0
- package/build/service/types.d.ts.map +1 -0
- package/build/service/types.js +2 -0
- package/build/service/types.js.map +1 -0
- package/build/service/types.local.d.ts +11 -0
- package/build/service/types.local.d.ts.map +1 -0
- package/build/service/types.local.js +2 -0
- package/build/service/types.local.js.map +1 -0
- package/build/service.d.ts +6 -0
- package/build/service.d.ts.map +1 -0
- package/build/service.js +73 -0
- package/build/service.js.map +1 -0
- package/build/sinks/buffer/consts.d.ts +2 -0
- package/build/sinks/buffer/consts.d.ts.map +1 -0
- package/build/sinks/buffer/consts.js +2 -0
- package/build/sinks/buffer/consts.js.map +1 -0
- package/build/sinks/buffer/types.d.ts +11 -0
- package/build/sinks/buffer/types.d.ts.map +1 -0
- package/build/sinks/buffer/types.js +2 -0
- package/build/sinks/buffer/types.js.map +1 -0
- package/build/sinks/buffer.d.ts +4 -0
- package/build/sinks/buffer.d.ts.map +1 -0
- package/build/sinks/buffer.js +13 -0
- package/build/sinks/buffer.js.map +1 -0
- package/build/sinks/directory/types.d.ts +7 -0
- package/build/sinks/directory/types.d.ts.map +1 -0
- package/build/sinks/directory/types.js +2 -0
- package/build/sinks/directory/types.js.map +1 -0
- package/build/sinks/directory.d.ts +8 -0
- package/build/sinks/directory.d.ts.map +1 -0
- package/build/sinks/directory.js +43 -0
- package/build/sinks/directory.js.map +1 -0
- package/build/sinks/file.d.ts +5 -0
- package/build/sinks/file.d.ts.map +1 -0
- package/build/sinks/file.js +46 -0
- package/build/sinks/file.js.map +1 -0
- package/build/sinks/types.d.ts +11 -0
- package/build/sinks/types.d.ts.map +1 -0
- package/build/sinks/types.js +2 -0
- package/build/sinks/types.js.map +1 -0
- package/build/sinks/workload/consts.d.ts +5 -0
- package/build/sinks/workload/consts.d.ts.map +1 -0
- package/build/sinks/workload/consts.js +5 -0
- package/build/sinks/workload/consts.js.map +1 -0
- package/build/sinks/workload/types.d.ts +17 -0
- package/build/sinks/workload/types.d.ts.map +1 -0
- package/build/sinks/workload/types.js +2 -0
- package/build/sinks/workload/types.js.map +1 -0
- package/build/sinks/workload.d.ts +8 -0
- package/build/sinks/workload.d.ts.map +1 -0
- package/build/sinks/workload.js +51 -0
- package/build/sinks/workload.js.map +1 -0
- package/build/sources/buffer.d.ts +5 -0
- package/build/sources/buffer.d.ts.map +1 -0
- package/build/sources/buffer.js +20 -0
- package/build/sources/buffer.js.map +1 -0
- package/build/sources/file.d.ts +5 -0
- package/build/sources/file.d.ts.map +1 -0
- package/build/sources/file.js +50 -0
- package/build/sources/file.js.map +1 -0
- package/build/sources/stream.d.ts +5 -0
- package/build/sources/stream.d.ts.map +1 -0
- package/build/sources/stream.js +19 -0
- package/build/sources/stream.js.map +1 -0
- package/build/sources/types.d.ts +10 -0
- package/build/sources/types.d.ts.map +1 -0
- package/build/sources/types.js +2 -0
- package/build/sources/types.js.map +1 -0
- package/build/sources/workload/types.d.ts +29 -0
- package/build/sources/workload/types.d.ts.map +1 -0
- package/build/sources/workload/types.js +2 -0
- package/build/sources/workload/types.js.map +1 -0
- package/build/sources/workload.d.ts +7 -0
- package/build/sources/workload.d.ts.map +1 -0
- package/build/sources/workload.js +67 -0
- package/build/sources/workload.js.map +1 -0
- package/build/transfer/consts.d.ts +5 -0
- package/build/transfer/consts.d.ts.map +1 -0
- package/build/transfer/consts.js +8 -0
- package/build/transfer/consts.js.map +1 -0
- package/build/transfer/types.d.ts +49 -0
- package/build/transfer/types.d.ts.map +1 -0
- package/build/transfer/types.js +2 -0
- package/build/transfer/types.js.map +1 -0
- package/build/transfer.d.ts +4 -0
- package/build/transfer.d.ts.map +1 -0
- package/build/transfer.js +75 -0
- package/build/transfer.js.map +1 -0
- package/build/types.d.ts +99 -0
- package/build/types.d.ts.map +1 -0
- package/build/types.js +2 -0
- package/build/types.js.map +1 -0
- package/build/utils/names/consts.d.ts +5 -0
- package/build/utils/names/consts.d.ts.map +1 -0
- package/build/utils/names/consts.js +5 -0
- package/build/utils/names/consts.js.map +1 -0
- package/build/utils/names/types.d.ts +13 -0
- package/build/utils/names/types.d.ts.map +1 -0
- package/build/utils/names/types.js +2 -0
- package/build/utils/names/types.js.map +1 -0
- package/build/utils/names.d.ts +4 -0
- package/build/utils/names.d.ts.map +1 -0
- package/build/utils/names.js +60 -0
- package/build/utils/names.js.map +1 -0
- package/build/utils/paths/consts.d.ts +3 -0
- package/build/utils/paths/consts.d.ts.map +1 -0
- package/build/utils/paths/consts.js +2 -0
- package/build/utils/paths/consts.js.map +1 -0
- package/build/utils/paths/types.d.ts +20 -0
- package/build/utils/paths/types.d.ts.map +1 -0
- package/build/utils/paths/types.js +2 -0
- package/build/utils/paths/types.js.map +1 -0
- package/build/utils/paths.d.ts +4 -0
- package/build/utils/paths.d.ts.map +1 -0
- package/build/utils/paths.js +67 -0
- package/build/utils/paths.js.map +1 -0
- package/build/utils/range/types.d.ts +11 -0
- package/build/utils/range/types.d.ts.map +1 -0
- package/build/utils/range/types.js +2 -0
- package/build/utils/range/types.js.map +1 -0
- package/build/utils/range.d.ts +4 -0
- package/build/utils/range.d.ts.map +1 -0
- package/build/utils/range.js +64 -0
- package/build/utils/range.js.map +1 -0
- package/build/utils/redact/types.d.ts +16 -0
- package/build/utils/redact/types.d.ts.map +1 -0
- package/build/utils/redact/types.js +2 -0
- package/build/utils/redact/types.js.map +1 -0
- package/build/utils/redact.d.ts +4 -0
- package/build/utils/redact.d.ts.map +1 -0
- package/build/utils/redact.js +18 -0
- package/build/utils/redact.js.map +1 -0
- package/build/utils/signals/types.d.ts +17 -0
- package/build/utils/signals/types.d.ts.map +1 -0
- package/build/utils/signals/types.js +2 -0
- package/build/utils/signals/types.js.map +1 -0
- package/build/utils/signals.d.ts +4 -0
- package/build/utils/signals.d.ts.map +1 -0
- package/build/utils/signals.js +52 -0
- package/build/utils/signals.js.map +1 -0
- package/build/utils/streams/consts.d.ts +3 -0
- package/build/utils/streams/consts.d.ts.map +1 -0
- package/build/utils/streams/consts.js +3 -0
- package/build/utils/streams/consts.js.map +1 -0
- package/build/utils/streams/types.d.ts +59 -0
- package/build/utils/streams/types.d.ts.map +1 -0
- package/build/utils/streams/types.js +2 -0
- package/build/utils/streams/types.js.map +1 -0
- package/build/utils/streams.d.ts +4 -0
- package/build/utils/streams.d.ts.map +1 -0
- package/build/utils/streams.js +167 -0
- package/build/utils/streams.js.map +1 -0
- package/build/utils/tree/types.d.ts +6 -0
- package/build/utils/tree/types.d.ts.map +1 -0
- package/build/utils/tree/types.js +2 -0
- package/build/utils/tree/types.js.map +1 -0
- package/build/utils/tree.d.ts +4 -0
- package/build/utils/tree.d.ts.map +1 -0
- package/build/utils/tree.js +14 -0
- package/build/utils/tree.js.map +1 -0
- package/build/workspace/consts.d.ts +12 -0
- package/build/workspace/consts.d.ts.map +1 -0
- package/build/workspace/consts.js +12 -0
- package/build/workspace/consts.js.map +1 -0
- package/build/workspace/consts.local.d.ts +3 -0
- package/build/workspace/consts.local.d.ts.map +1 -0
- package/build/workspace/consts.local.js +3 -0
- package/build/workspace/consts.local.js.map +1 -0
- package/build/workspace/fs.d.ts +4 -0
- package/build/workspace/fs.d.ts.map +1 -0
- package/build/workspace/fs.js +45 -0
- package/build/workspace/fs.js.map +1 -0
- package/build/workspace/scope.d.ts +6 -0
- package/build/workspace/scope.d.ts.map +1 -0
- package/build/workspace/scope.js +180 -0
- package/build/workspace/scope.js.map +1 -0
- package/build/workspace/types.d.ts +66 -0
- package/build/workspace/types.d.ts.map +1 -0
- package/build/workspace/types.js +2 -0
- package/build/workspace/types.js.map +1 -0
- package/build/workspace/types.local.d.ts +20 -0
- package/build/workspace/types.local.d.ts.map +1 -0
- package/build/workspace/types.local.js +2 -0
- package/build/workspace/types.local.js.map +1 -0
- package/build/workspace.d.ts +7 -0
- package/build/workspace.d.ts.map +1 -0
- package/build/workspace.js +87 -0
- package/build/workspace.js.map +1 -0
- package/package.json +61 -4
- package/src/archive/consts.ts +28 -0
- package/src/archive/formats.ts +60 -0
- package/src/archive/guard.ts +68 -0
- package/src/archive/ratio.ts +35 -0
- package/src/archive/tar.ts +147 -0
- package/src/archive/types.local.ts +72 -0
- package/src/archive/types.ts +107 -0
- package/src/archive/zip.ts +142 -0
- package/src/archive.ts +298 -0
- package/src/backed-workspace/types.ts +72 -0
- package/src/backed-workspace.ts +85 -0
- package/src/consts.local.ts +2 -0
- package/src/consts.ts +2 -0
- package/src/errors.ts +128 -0
- package/src/http/connector.ts +136 -0
- package/src/http/consts.ts +13 -0
- package/src/http/source.ts +89 -0
- package/src/http/types.local.ts +23 -0
- package/src/http/types.ts +74 -0
- package/src/http.ts +276 -0
- package/src/index.ts +65 -0
- package/src/network/consts.ts +22 -0
- package/src/network/types.ts +45 -0
- package/src/network.ts +179 -0
- package/src/proxy/consts.ts +11 -0
- package/src/proxy/endpoint.ts +28 -0
- package/src/proxy/http.ts +108 -0
- package/src/proxy/router.ts +93 -0
- package/src/proxy/socks.ts +48 -0
- package/src/proxy/types.ts +114 -0
- package/src/resolvers/google-drive/consts.ts +22 -0
- package/src/resolvers/google-drive/types.ts +22 -0
- package/src/resolvers/google-drive.ts +153 -0
- package/src/resolvers/types.ts +45 -0
- package/src/s3/consts.ts +12 -0
- package/src/s3/prefix.ts +100 -0
- package/src/s3/sink.ts +36 -0
- package/src/s3/source.ts +90 -0
- package/src/s3/types.local.ts +49 -0
- package/src/s3/types.ts +70 -0
- package/src/s3/upload.ts +130 -0
- package/src/s3.ts +83 -0
- package/src/service/types.local.ts +9 -0
- package/src/service/types.ts +71 -0
- package/src/service.ts +97 -0
- package/src/sinks/buffer/consts.ts +1 -0
- package/src/sinks/buffer/types.ts +12 -0
- package/src/sinks/buffer.ts +18 -0
- package/src/sinks/directory/types.ts +7 -0
- package/src/sinks/directory.ts +49 -0
- package/src/sinks/file.ts +52 -0
- package/src/sinks/types.ts +12 -0
- package/src/sinks/workload/consts.ts +5 -0
- package/src/sinks/workload/types.ts +17 -0
- package/src/sinks/workload.ts +59 -0
- package/src/sources/buffer.ts +27 -0
- package/src/sources/file.ts +57 -0
- package/src/sources/stream.ts +26 -0
- package/src/sources/types.ts +12 -0
- package/src/sources/workload/types.ts +31 -0
- package/src/sources/workload.ts +79 -0
- package/src/transfer/consts.ts +10 -0
- package/src/transfer/types.ts +55 -0
- package/src/transfer.ts +89 -0
- package/src/types.ts +106 -0
- package/src/utils/names/consts.ts +5 -0
- package/src/utils/names/types.ts +12 -0
- package/src/utils/names.ts +70 -0
- package/src/utils/paths/consts.ts +3 -0
- package/src/utils/paths/types.ts +20 -0
- package/src/utils/paths.ts +75 -0
- package/src/utils/range/types.ts +11 -0
- package/src/utils/range.ts +80 -0
- package/src/utils/redact/types.ts +16 -0
- package/src/utils/redact.ts +23 -0
- package/src/utils/signals/types.ts +17 -0
- package/src/utils/signals.ts +63 -0
- package/src/utils/streams/consts.ts +2 -0
- package/src/utils/streams/types.ts +65 -0
- package/src/utils/streams.ts +188 -0
- package/src/utils/tree/types.ts +6 -0
- package/src/utils/tree.ts +18 -0
- package/src/workspace/consts.local.ts +2 -0
- package/src/workspace/consts.ts +14 -0
- package/src/workspace/fs.ts +48 -0
- package/src/workspace/scope.ts +196 -0
- package/src/workspace/types.local.ts +21 -0
- package/src/workspace/types.ts +71 -0
- package/src/workspace.ts +102 -0
- package/tests/archive.spec.ts +134 -0
- package/tests/backed.spec.ts +147 -0
- package/tests/context.ts +114 -0
- package/tests/fixtures.ts +190 -0
- package/tests/google-drive.spec.ts +76 -0
- package/tests/harness/drive.ts +92 -0
- package/tests/harness/http-proxy.ts +96 -0
- package/tests/harness/origin.ts +134 -0
- package/tests/harness/socks5.ts +135 -0
- package/tests/harness/workload.ts +118 -0
- package/tests/http.spec.ts +101 -0
- package/tests/inspector.spec.ts +40 -0
- package/tests/live.spec.ts +27 -0
- package/tests/network.spec.ts +79 -0
- package/tests/proxy.spec.ts +149 -0
- package/tests/s3.spec.ts +133 -0
- package/tests/service.spec.ts +77 -0
- package/tests/sources-sinks.spec.ts +96 -0
- package/tests/transfer.spec.ts +99 -0
- package/tests/tsconfig.json +12 -0
- package/tests/workload.spec.ts +140 -0
- package/tests/workspace.spec.ts +158 -0
- package/tsconfig.json +16 -0
package/src/s3/upload.ts
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import {
|
|
2
|
+
AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, PutObjectCommand, UploadPartCommand,
|
|
3
|
+
} from '@aws-sdk/client-s3'
|
|
4
|
+
import type { CompletedPart, ObjectCannedACL } from '@aws-sdk/client-s3'
|
|
5
|
+
import type { Readable } from 'node:stream'
|
|
6
|
+
import { logger } from '@owlmeans/log'
|
|
7
|
+
import { LimitError, S3TransferError } from '../errors.js'
|
|
8
|
+
import { MAX_PARTS } from './consts.js'
|
|
9
|
+
import type { S3Core, S3Uploader, S3UploadRequest, S3UploadResult } from './types.local.js'
|
|
10
|
+
|
|
11
|
+
const log = logger('file-streams:s3')
|
|
12
|
+
|
|
13
|
+
/** Uploads of unknown length: one PutObject when the body fits a part, a multipart upload otherwise. */
|
|
14
|
+
export const makeS3Uploader = (core: S3Core): S3Uploader => {
|
|
15
|
+
const parts = async function * (stream: Readable, size: number): AsyncGenerator<Buffer> {
|
|
16
|
+
let pending: Buffer[] = []
|
|
17
|
+
let length = 0
|
|
18
|
+
for await (const chunk of stream) {
|
|
19
|
+
let buffer = chunk as Buffer
|
|
20
|
+
while (buffer.length > 0) {
|
|
21
|
+
const take = Math.min(size - length, buffer.length)
|
|
22
|
+
pending.push(buffer.subarray(0, take))
|
|
23
|
+
length += take
|
|
24
|
+
buffer = buffer.subarray(take)
|
|
25
|
+
if (length === size) {
|
|
26
|
+
yield Buffer.concat(pending, length)
|
|
27
|
+
pending = []
|
|
28
|
+
length = 0
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
if (length > 0) {
|
|
33
|
+
yield Buffer.concat(pending, length)
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const acl = (): { ACL?: ObjectCannedACL } => core.model.acl != null ? { ACL: core.model.acl as ObjectCannedACL } : {}
|
|
38
|
+
|
|
39
|
+
const upload = async (request: S3UploadRequest): Promise<S3UploadResult> => {
|
|
40
|
+
const client = core.client()
|
|
41
|
+
const base = {
|
|
42
|
+
Bucket: core.model.bucket,
|
|
43
|
+
Key: request.key,
|
|
44
|
+
ContentType: request.contentType,
|
|
45
|
+
ContentDisposition: request.disposition,
|
|
46
|
+
...(request.cacheControl != null ? { CacheControl: request.cacheControl } : {}),
|
|
47
|
+
...(request.metadata != null ? { Metadata: request.metadata } : {}),
|
|
48
|
+
...acl(),
|
|
49
|
+
}
|
|
50
|
+
const iterator = parts(request.stream, core.partSize)
|
|
51
|
+
const first = await iterator.next()
|
|
52
|
+
const firstPart = first.done === true ? Buffer.alloc(0) : first.value
|
|
53
|
+
const second = first.done === true ? first : await iterator.next()
|
|
54
|
+
|
|
55
|
+
if (second.done === true) {
|
|
56
|
+
try {
|
|
57
|
+
const result = await client.send(new PutObjectCommand({ ...base, Body: firstPart, ContentLength: firstPart.length }), { abortSignal: request.signal })
|
|
58
|
+
return { bytes: firstPart.length, etag: result.ETag }
|
|
59
|
+
} catch (error) {
|
|
60
|
+
if (request.signal?.aborted === true) {
|
|
61
|
+
throw request.signal.reason
|
|
62
|
+
}
|
|
63
|
+
log.error('S3 upload failed', error as Error)
|
|
64
|
+
throw new S3TransferError('put')
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
let uploadId: string | undefined
|
|
69
|
+
try {
|
|
70
|
+
const created = await client.send(new CreateMultipartUploadCommand(base), { abortSignal: request.signal })
|
|
71
|
+
uploadId = created.UploadId
|
|
72
|
+
const completed: CompletedPart[] = []
|
|
73
|
+
const inFlight = new Set<Promise<void>>()
|
|
74
|
+
let bytes = 0
|
|
75
|
+
let number = 0
|
|
76
|
+
|
|
77
|
+
const send = (body: Buffer): void => {
|
|
78
|
+
const partNumber = ++number
|
|
79
|
+
if (partNumber > MAX_PARTS) {
|
|
80
|
+
throw new LimitError('s3-parts')
|
|
81
|
+
}
|
|
82
|
+
bytes += body.length
|
|
83
|
+
const pending = client.send(new UploadPartCommand({
|
|
84
|
+
Bucket: core.model.bucket, Key: request.key, UploadId: uploadId, PartNumber: partNumber, Body: body, ContentLength: body.length,
|
|
85
|
+
}), { abortSignal: request.signal }).then(result => {
|
|
86
|
+
completed.push({ ETag: result.ETag, PartNumber: partNumber })
|
|
87
|
+
})
|
|
88
|
+
const tracked = pending.finally(() => inFlight.delete(tracked))
|
|
89
|
+
inFlight.add(tracked)
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
send(firstPart)
|
|
93
|
+
send(second.value)
|
|
94
|
+
for (;;) {
|
|
95
|
+
while (inFlight.size >= core.concurrency) {
|
|
96
|
+
await Promise.race(inFlight)
|
|
97
|
+
}
|
|
98
|
+
const next = await iterator.next()
|
|
99
|
+
if (next.done === true) {
|
|
100
|
+
break
|
|
101
|
+
}
|
|
102
|
+
send(next.value)
|
|
103
|
+
}
|
|
104
|
+
await Promise.all(inFlight)
|
|
105
|
+
completed.sort((a, b) => (a.PartNumber ?? 0) - (b.PartNumber ?? 0))
|
|
106
|
+
const result = await client.send(new CompleteMultipartUploadCommand({
|
|
107
|
+
Bucket: core.model.bucket, Key: request.key, UploadId: uploadId, MultipartUpload: { Parts: completed },
|
|
108
|
+
}), { abortSignal: request.signal })
|
|
109
|
+
log.debug('Multipart upload completed', { parts: completed.length, bytes })
|
|
110
|
+
|
|
111
|
+
return { bytes, etag: result.ETag }
|
|
112
|
+
} catch (error) {
|
|
113
|
+
if (uploadId != null) {
|
|
114
|
+
await client.send(new AbortMultipartUploadCommand({ Bucket: core.model.bucket, Key: request.key, UploadId: uploadId }))
|
|
115
|
+
.catch(abortError => log.warn('Multipart abort failed', abortError as Error))
|
|
116
|
+
}
|
|
117
|
+
await iterator.return(undefined).catch(() => undefined)
|
|
118
|
+
if (request.signal?.aborted === true) {
|
|
119
|
+
throw request.signal.reason
|
|
120
|
+
}
|
|
121
|
+
if (error instanceof LimitError || error instanceof S3TransferError) {
|
|
122
|
+
throw error
|
|
123
|
+
}
|
|
124
|
+
log.error('S3 multipart upload failed', error as Error)
|
|
125
|
+
throw new S3TransferError('multipart')
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
return { upload }
|
|
130
|
+
}
|
package/src/s3.ts
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { DeleteObjectCommand, HeadObjectCommand } from '@aws-sdk/client-s3'
|
|
2
|
+
import type { S3Client } from '@aws-sdk/client-s3'
|
|
3
|
+
import { makeStorageBucketModel } from '@owlmeans/storage-resource'
|
|
4
|
+
import type { StorageBucketModel, StorageConfig } from '@owlmeans/storage-resource'
|
|
5
|
+
import type { StoredFileInstance } from '@owlmeans/storage-common'
|
|
6
|
+
import { S3TransferError } from './errors.js'
|
|
7
|
+
import { DEFAULT_PART_SIZE, DEFAULT_UPLOAD_CONCURRENCY, MIN_PART_SIZE } from './s3/consts.js'
|
|
8
|
+
import { makeS3PrefixOps } from './s3/prefix.js'
|
|
9
|
+
import { makeS3Sink } from './s3/sink.js'
|
|
10
|
+
import { makeS3Source } from './s3/source.js'
|
|
11
|
+
import type { S3Bucket, S3BucketOptions, S3ObjectInfo, S3SinkOptions } from './s3/types.js'
|
|
12
|
+
import type { S3Core } from './s3/types.local.js'
|
|
13
|
+
import { makeS3Uploader } from './s3/upload.js'
|
|
14
|
+
import type { FileSink, FileSource, SignalOptions, SinkResult, TreeSink } from './types.js'
|
|
15
|
+
import { pathUtils } from './utils/paths.js'
|
|
16
|
+
|
|
17
|
+
/** A storage bucket — a `cfg.storageBuckets` entry or its model — as sources and sinks. */
|
|
18
|
+
export const createS3Bucket = (config: StorageBucketModel | StorageConfig, options: S3BucketOptions = {}): S3Bucket => {
|
|
19
|
+
const model = 'createClient' in config ? config : makeStorageBucketModel(config)
|
|
20
|
+
let created: S3Client | undefined
|
|
21
|
+
const client = (): S3Client => {
|
|
22
|
+
created ??= model.createClient()
|
|
23
|
+
return created
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const keyOf = (prefix: string): string => model.keyOf(pathUtils.entry(prefix))
|
|
27
|
+
|
|
28
|
+
const core: S3Core = {
|
|
29
|
+
model,
|
|
30
|
+
client,
|
|
31
|
+
keyOf,
|
|
32
|
+
location: key => `s3://${model.bucket}/${key}`,
|
|
33
|
+
partSize: Math.max(options.partSize ?? DEFAULT_PART_SIZE, MIN_PART_SIZE),
|
|
34
|
+
concurrency: Math.max(1, options.concurrency ?? DEFAULT_UPLOAD_CONCURRENCY),
|
|
35
|
+
}
|
|
36
|
+
const uploader = makeS3Uploader(core)
|
|
37
|
+
const { list, removePrefix } = makeS3PrefixOps(core)
|
|
38
|
+
|
|
39
|
+
const urlOf = (prefix: string): string => model.urlOf(pathUtils.entry(prefix))
|
|
40
|
+
|
|
41
|
+
const source = (prefix: string): FileSource => makeS3Source(core, keyOf(prefix))
|
|
42
|
+
|
|
43
|
+
const sink = (prefix: string, sinkOptions?: S3SinkOptions): FileSink => makeS3Sink(core, uploader, pathUtils.entry(prefix), sinkOptions)
|
|
44
|
+
|
|
45
|
+
const tree = (prefix: string, sinkOptions?: S3SinkOptions): TreeSink => {
|
|
46
|
+
const base = pathUtils.entry(prefix)
|
|
47
|
+
return {
|
|
48
|
+
kind: 's3',
|
|
49
|
+
entry: path => sink(`${base}/${pathUtils.entry(path)}`, sinkOptions),
|
|
50
|
+
directory: async () => undefined,
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const head = async (prefix: string, headOptions: SignalOptions = {}): Promise<S3ObjectInfo | null> => {
|
|
55
|
+
const key = keyOf(prefix)
|
|
56
|
+
try {
|
|
57
|
+
const result = await client().send(new HeadObjectCommand({ Bucket: model.bucket, Key: key }), { abortSignal: headOptions.signal })
|
|
58
|
+
return { key, size: result.ContentLength ?? 0, etag: result.ETag, mime: result.ContentType, modifiedAt: result.LastModified }
|
|
59
|
+
} catch (error) {
|
|
60
|
+
if ((error as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode === 404 || (error as Error).name === 'NotFound') {
|
|
61
|
+
return null
|
|
62
|
+
}
|
|
63
|
+
throw new S3TransferError('head')
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const remove = async (prefix: string, removeOptions: SignalOptions = {}): Promise<void> => {
|
|
68
|
+
try {
|
|
69
|
+
await client().send(new DeleteObjectCommand({ Bucket: model.bucket, Key: keyOf(prefix) }), { abortSignal: removeOptions.signal })
|
|
70
|
+
} catch {
|
|
71
|
+
throw new S3TransferError('delete')
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const instanceOf = (result: SinkResult, alias: string): StoredFileInstance => ({ size: result.bytes, alias, url: result.url ?? '' })
|
|
76
|
+
|
|
77
|
+
const close = (): void => {
|
|
78
|
+
created?.destroy()
|
|
79
|
+
created = undefined
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
return { model, keyOf, urlOf, source, sink, tree, head, remove, list, removePrefix, instanceOf, close }
|
|
83
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { BasicConfig, BasicContext } from '@owlmeans/context'
|
|
2
|
+
import type { StorageConfig } from '@owlmeans/storage-resource'
|
|
3
|
+
import type { FileStreamsConfigAppend } from './types.js'
|
|
4
|
+
|
|
5
|
+
export interface Config extends BasicConfig, FileStreamsConfigAppend {
|
|
6
|
+
storageBuckets?: { [key: string]: StorageConfig }
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export interface Context<C extends Config = Config> extends BasicContext<C> {}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import type { InitializedService } from '@owlmeans/context'
|
|
2
|
+
import type { FileInspector, FileInspectorOptions } from '@owlmeans/file-inspector'
|
|
3
|
+
import type { ArchiveHelper, ArchiveLimits } from '../archive/types.js'
|
|
4
|
+
import type { BackedWorkspace } from '../backed-workspace/types.js'
|
|
5
|
+
import type { HttpClient } from '../http/types.js'
|
|
6
|
+
import type { AddressResolver, NetworkPolicyOptions } from '../network/types.js'
|
|
7
|
+
import type { ProxyCredentialsProvider, ProxyPlugin, ProxyRouterOptions, ProxyStrategy } from '../proxy/types.js'
|
|
8
|
+
import type { UrlResolver } from '../resolvers/types.js'
|
|
9
|
+
import type { S3Bucket, S3BucketOptions } from '../s3/types.js'
|
|
10
|
+
import type { TransferHelper, TransferLimits } from '../transfer/types.js'
|
|
11
|
+
import type { Workspace, WorkspaceOptions } from '../workspace/types.js'
|
|
12
|
+
|
|
13
|
+
/** A proxy in config; string leaves may be secret files (`/etc/…`), which server contexts resolve. */
|
|
14
|
+
export interface FileStreamsProxyConfig {
|
|
15
|
+
/** `http://host:3128`, `socks5h://host:1080`. */
|
|
16
|
+
url: string
|
|
17
|
+
username?: string
|
|
18
|
+
password?: string
|
|
19
|
+
id?: string
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** `cfg.fileStreams`. */
|
|
23
|
+
export interface FileStreamsConfig {
|
|
24
|
+
limits?: Partial<TransferLimits>
|
|
25
|
+
network?: Omit<NetworkPolicyOptions, 'resolver'>
|
|
26
|
+
proxies?: FileStreamsProxyConfig[]
|
|
27
|
+
proxyStrategy?: ProxyStrategy
|
|
28
|
+
userAgent?: string
|
|
29
|
+
/**
|
|
30
|
+
* The temporary workspace: `root`, folder and file modes. A server context reads every string leaf
|
|
31
|
+
* starting with `/` as a secret file, so give `root` relative to `os.tmpdir()` (default
|
|
32
|
+
* `owlmeans-workspace`), or as a file that holds an absolute path.
|
|
33
|
+
*/
|
|
34
|
+
workspace?: WorkspaceOptions
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export interface FileStreamsBackedOptions {
|
|
38
|
+
/** Objects land under `<prefix>/` inside the bucket's base prefix. */
|
|
39
|
+
prefix?: string
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export interface FileStreamsConfigAppend {
|
|
43
|
+
fileStreams?: FileStreamsConfig
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** What config cannot carry: callbacks and plugins. */
|
|
47
|
+
export interface FileStreamsServiceOptions {
|
|
48
|
+
/** Replaces `cfg.fileStreams.proxies` — a list or a provider by key and URL. */
|
|
49
|
+
proxies?: ProxyRouterOptions['proxies']
|
|
50
|
+
proxyCredentials?: ProxyCredentialsProvider
|
|
51
|
+
proxyPlugins?: ProxyPlugin[]
|
|
52
|
+
resolvers?: UrlResolver[]
|
|
53
|
+
resolver?: AddressResolver
|
|
54
|
+
bucketOptions?: S3BucketOptions
|
|
55
|
+
inspector?: FileInspectorOptions
|
|
56
|
+
archiveLimits?: Partial<ArchiveLimits>
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** The package's objects configured from the context. */
|
|
60
|
+
export interface FileStreamsService extends InitializedService {
|
|
61
|
+
http: () => HttpClient
|
|
62
|
+
/** The bucket of a `cfg.storageBuckets` entry, addressed as storage-resource addresses it. */
|
|
63
|
+
bucket: (configKey: string) => S3Bucket
|
|
64
|
+
transfer: () => TransferHelper
|
|
65
|
+
inspector: () => FileInspector
|
|
66
|
+
archives: () => ArchiveHelper
|
|
67
|
+
/** The workspace configured by `cfg.fileStreams.workspace`. */
|
|
68
|
+
workspace: () => Workspace
|
|
69
|
+
/** The workspace mirrored to the bucket of a `cfg.storageBuckets` entry (one object per key and prefix). */
|
|
70
|
+
backed: (bucketConfigKey: string, options?: FileStreamsBackedOptions) => BackedWorkspace
|
|
71
|
+
}
|
package/src/service.ts
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import { createService, memoHelper } from '@owlmeans/context'
|
|
2
|
+
import type { BasicConfig, BasicContext } from '@owlmeans/context'
|
|
3
|
+
import { FilePropertyError } from '@owlmeans/storage-common'
|
|
4
|
+
import { createFileInspector } from '@owlmeans/file-inspector'
|
|
5
|
+
import { createArchiveHelper } from './archive.js'
|
|
6
|
+
import { createBackedWorkspace } from './backed-workspace.js'
|
|
7
|
+
import type { BackedWorkspace } from './backed-workspace/types.js'
|
|
8
|
+
import { FILE_STREAMS_SERVICE } from './consts.js'
|
|
9
|
+
import { createHttpClient } from './http.js'
|
|
10
|
+
import { parseProxyEndpoint } from './proxy/endpoint.js'
|
|
11
|
+
import type { ProxyEndpoint } from './proxy/types.js'
|
|
12
|
+
import { createS3Bucket } from './s3.js'
|
|
13
|
+
import type { S3Bucket } from './s3/types.js'
|
|
14
|
+
import type {
|
|
15
|
+
FileStreamsBackedOptions, FileStreamsProxyConfig, FileStreamsService, FileStreamsServiceOptions,
|
|
16
|
+
} from './service/types.js'
|
|
17
|
+
import type { Config, Context } from './service/types.local.js'
|
|
18
|
+
import { createTransferHelper } from './transfer.js'
|
|
19
|
+
import { createWorkspace } from './workspace.js'
|
|
20
|
+
|
|
21
|
+
export const createFileStreamsService = (
|
|
22
|
+
alias: string = FILE_STREAMS_SERVICE, options: FileStreamsServiceOptions = {}
|
|
23
|
+
): FileStreamsService => {
|
|
24
|
+
const buckets = new Map<string, S3Bucket>()
|
|
25
|
+
const backedWorkspaces = new Map<string, BackedWorkspace>()
|
|
26
|
+
|
|
27
|
+
const configOf = (): Config => service.assertCtx<Config, Context>(alias).cfg
|
|
28
|
+
|
|
29
|
+
const endpointOf = (proxy: FileStreamsProxyConfig): ProxyEndpoint => {
|
|
30
|
+
const endpoint = parseProxyEndpoint(proxy.url)
|
|
31
|
+
return {
|
|
32
|
+
...endpoint,
|
|
33
|
+
...(proxy.username != null ? { username: proxy.username, password: proxy.password ?? '' } : {}),
|
|
34
|
+
...(proxy.id != null ? { id: proxy.id } : {}),
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const http = memoHelper.once(() => {
|
|
39
|
+
const config = configOf().fileStreams ?? {}
|
|
40
|
+
return createHttpClient({
|
|
41
|
+
network: { ...config.network, resolver: options.resolver },
|
|
42
|
+
proxies: {
|
|
43
|
+
proxies: options.proxies ?? config.proxies?.map(endpointOf),
|
|
44
|
+
strategy: config.proxyStrategy,
|
|
45
|
+
credentials: options.proxyCredentials,
|
|
46
|
+
plugins: options.proxyPlugins,
|
|
47
|
+
},
|
|
48
|
+
resolvers: options.resolvers,
|
|
49
|
+
userAgent: config.userAgent,
|
|
50
|
+
})
|
|
51
|
+
})
|
|
52
|
+
|
|
53
|
+
const inspector = memoHelper.once(() => createFileInspector(options.inspector))
|
|
54
|
+
|
|
55
|
+
const transfer = memoHelper.once(() => createTransferHelper({ inspector: inspector(), limits: configOf().fileStreams?.limits }))
|
|
56
|
+
|
|
57
|
+
const archives = memoHelper.once(() => createArchiveHelper({ inspector: inspector(), limits: options.archiveLimits }))
|
|
58
|
+
|
|
59
|
+
const bucket = (configKey: string): S3Bucket => {
|
|
60
|
+
let known = buckets.get(configKey)
|
|
61
|
+
if (known == null) {
|
|
62
|
+
const entry = configOf().storageBuckets?.[configKey]
|
|
63
|
+
if (entry == null) {
|
|
64
|
+
throw new FilePropertyError(`storage-config:${configKey}`)
|
|
65
|
+
}
|
|
66
|
+
known = createS3Bucket(entry, options.bucketOptions)
|
|
67
|
+
buckets.set(configKey, known)
|
|
68
|
+
}
|
|
69
|
+
return known
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const workspace = memoHelper.once(() => createWorkspace(configOf().fileStreams?.workspace))
|
|
73
|
+
|
|
74
|
+
const backed = (bucketConfigKey: string, backedOptions: FileStreamsBackedOptions = {}): BackedWorkspace => {
|
|
75
|
+
const key = `${bucketConfigKey}\u0000${backedOptions.prefix ?? ''}`
|
|
76
|
+
let known = backedWorkspaces.get(key)
|
|
77
|
+
if (known == null) {
|
|
78
|
+
known = createBackedWorkspace({ workspace: workspace(), bucket: bucket(bucketConfigKey), prefix: backedOptions.prefix, transfer: transfer() })
|
|
79
|
+
backedWorkspaces.set(key, known)
|
|
80
|
+
}
|
|
81
|
+
return known
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const service: FileStreamsService = createService<FileStreamsService>(alias, {
|
|
85
|
+
http, bucket, transfer, inspector, archives, workspace, backed,
|
|
86
|
+
})
|
|
87
|
+
|
|
88
|
+
return service
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** Registers the file-streams service on a context. */
|
|
92
|
+
export const appendFileStreamsService = <C extends BasicConfig, T extends BasicContext<C>>(
|
|
93
|
+
ctx: T, alias: string = FILE_STREAMS_SERVICE, options?: FileStreamsServiceOptions
|
|
94
|
+
): T => {
|
|
95
|
+
ctx.registerService(createFileStreamsService(alias, options))
|
|
96
|
+
return ctx
|
|
97
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export const DEFAULT_BUFFER_SINK_BYTES = 64 * 1024 * 1024
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { FileSink } from '../../types.js'
|
|
2
|
+
|
|
3
|
+
export interface BufferSinkOptions {
|
|
4
|
+
/** Ceiling on collected bytes. Default 64 MiB. */
|
|
5
|
+
maxBytes?: number
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
/** A sink that keeps what it receives in memory. */
|
|
9
|
+
export interface BufferSink extends FileSink {
|
|
10
|
+
/** The bytes of the last write. */
|
|
11
|
+
bytes: () => Uint8Array
|
|
12
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { SinkInput, SinkResult, SinkWriteOptions } from '../types.js'
|
|
2
|
+
import { streamUtils } from '../utils/streams.js'
|
|
3
|
+
import { DEFAULT_BUFFER_SINK_BYTES } from './buffer/consts.js'
|
|
4
|
+
import type { BufferSink, BufferSinkOptions } from './buffer/types.js'
|
|
5
|
+
|
|
6
|
+
/** Collects a file in memory, refusing past its ceiling. */
|
|
7
|
+
export const createBufferSink = (options: BufferSinkOptions = {}): BufferSink => {
|
|
8
|
+
let collected: Uint8Array = new Uint8Array(0)
|
|
9
|
+
|
|
10
|
+
const write = async (input: SinkInput, writeOptions: SinkWriteOptions = {}): Promise<SinkResult> => {
|
|
11
|
+
const ceiling = Math.min(options.maxBytes ?? DEFAULT_BUFFER_SINK_BYTES, writeOptions.maxBytes ?? Infinity)
|
|
12
|
+
collected = await streamUtils.collect(input.stream, ceiling)
|
|
13
|
+
|
|
14
|
+
return { kind: 'buffer', location: 'buffer', bytes: collected.length, mime: input.verdict?.detected?.mime ?? input.info.mime }
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
return { kind: 'buffer', write, bytes: () => collected }
|
|
18
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { FileSinkOptions } from '../types.js'
|
|
2
|
+
import type { PathLimits } from '../../utils/paths/types.js'
|
|
3
|
+
|
|
4
|
+
export interface DirectorySinkOptions extends FileSinkOptions, Partial<PathLimits> {
|
|
5
|
+
/** Mode of created directories. Default `0o755`. */
|
|
6
|
+
dirMode?: number
|
|
7
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { mkdir } from 'node:fs/promises'
|
|
2
|
+
import { resolve } from 'node:path'
|
|
3
|
+
import { PathError } from '../errors.js'
|
|
4
|
+
import type { FileSink, SignalOptions, TreeSink } from '../types.js'
|
|
5
|
+
import { pathUtils } from '../utils/paths.js'
|
|
6
|
+
import { createFileSink } from './file.js'
|
|
7
|
+
import type { DirectorySinkOptions } from './directory/types.js'
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* A directory every entry is confined to: normalized relative paths only, never through an
|
|
11
|
+
* existing symlink, never the same entry (or a case-fold twin) twice.
|
|
12
|
+
*/
|
|
13
|
+
export const createDirectorySink = (root: string, options: DirectorySinkOptions = {}): TreeSink => {
|
|
14
|
+
const base = resolve(root)
|
|
15
|
+
const seen = new Set<string>()
|
|
16
|
+
|
|
17
|
+
const entry = (path: string): FileSink => {
|
|
18
|
+
const relative = pathUtils.entry(path, options)
|
|
19
|
+
const target = pathUtils.join(base, relative)
|
|
20
|
+
const folded = relative.toLowerCase()
|
|
21
|
+
if (seen.has(folded)) {
|
|
22
|
+
throw new PathError('duplicate')
|
|
23
|
+
}
|
|
24
|
+
const sink = createFileSink(target, options)
|
|
25
|
+
|
|
26
|
+
return {
|
|
27
|
+
kind: 'file',
|
|
28
|
+
write: async (input, writeOptions) => {
|
|
29
|
+
if (seen.has(folded)) {
|
|
30
|
+
throw new PathError('duplicate')
|
|
31
|
+
}
|
|
32
|
+
await mkdir(base, { recursive: true, mode: options.dirMode ?? 0o755 })
|
|
33
|
+
await pathUtils.assertNoLinks(base, target)
|
|
34
|
+
const result = await sink.write(input, writeOptions)
|
|
35
|
+
seen.add(folded)
|
|
36
|
+
return result
|
|
37
|
+
},
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const directory = async (path: string, _options?: SignalOptions): Promise<void> => {
|
|
42
|
+
const target = pathUtils.join(base, pathUtils.entry(path, options))
|
|
43
|
+
await mkdir(base, { recursive: true, mode: options.dirMode ?? 0o755 })
|
|
44
|
+
await pathUtils.assertNoLinks(base, target)
|
|
45
|
+
await mkdir(target, { recursive: true, mode: options.dirMode ?? 0o755 })
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
return { kind: 'directory', entry, directory }
|
|
49
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { randomBytes } from 'node:crypto'
|
|
2
|
+
import { createWriteStream } from 'node:fs'
|
|
3
|
+
import { chmod, link, mkdir, rename, rm } from 'node:fs/promises'
|
|
4
|
+
import { basename, dirname, join } from 'node:path'
|
|
5
|
+
import { pipeline } from 'node:stream/promises'
|
|
6
|
+
import { logger } from '@owlmeans/log'
|
|
7
|
+
import { PathError } from '../errors.js'
|
|
8
|
+
import type { FileSink, SinkInput, SinkResult, SinkWriteOptions } from '../types.js'
|
|
9
|
+
import { streamUtils } from '../utils/streams.js'
|
|
10
|
+
import type { FileSinkOptions } from './types.js'
|
|
11
|
+
|
|
12
|
+
const log = logger('file-streams:file')
|
|
13
|
+
|
|
14
|
+
/** One file on disk, written to a private temporary file and committed atomically. */
|
|
15
|
+
export const createFileSink = (path: string, options: FileSinkOptions = {}): FileSink => {
|
|
16
|
+
const write = async (input: SinkInput, writeOptions: SinkWriteOptions = {}): Promise<SinkResult> => {
|
|
17
|
+
const dir = dirname(path)
|
|
18
|
+
if (options.mkdirs !== false) {
|
|
19
|
+
await mkdir(dir, { recursive: true, mode: 0o755 })
|
|
20
|
+
}
|
|
21
|
+
const temporary = join(dir, `.${basename(path)}.${randomBytes(6).toString('hex')}.tmp`)
|
|
22
|
+
const meter = streamUtils.meter({ maxBytes: writeOptions.maxBytes })
|
|
23
|
+
|
|
24
|
+
try {
|
|
25
|
+
await pipeline(
|
|
26
|
+
input.stream, meter.stream, createWriteStream(temporary, { mode: 0o600, flags: 'wx', flush: true }),
|
|
27
|
+
{ signal: writeOptions.signal }
|
|
28
|
+
)
|
|
29
|
+
await chmod(temporary, options.mode ?? 0o644)
|
|
30
|
+
if (options.onConflict === 'overwrite') {
|
|
31
|
+
await rename(temporary, path)
|
|
32
|
+
} else {
|
|
33
|
+
try {
|
|
34
|
+
await link(temporary, path)
|
|
35
|
+
} catch (error) {
|
|
36
|
+
throw (error as NodeJS.ErrnoException).code === 'EEXIST' ? new PathError('exists') : error
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
} catch (error) {
|
|
40
|
+
streamUtils.discard(input.stream)
|
|
41
|
+
throw error
|
|
42
|
+
} finally {
|
|
43
|
+
await rm(temporary, { force: true })
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
log.debug('File written', { bytes: meter.bytes() })
|
|
47
|
+
|
|
48
|
+
return { kind: 'file', location: path, bytes: meter.bytes(), mime: input.verdict?.detected?.mime ?? input.info.mime }
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
return { kind: 'file', write }
|
|
52
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/** What a file sink does when its target already exists. */
|
|
2
|
+
// Two behaviours, nothing else to say about them — a union.
|
|
3
|
+
export type ConflictPolicy = 'error' | 'overwrite'
|
|
4
|
+
|
|
5
|
+
export interface FileSinkOptions {
|
|
6
|
+
/** Default `error` (`PathError('exists')`). */
|
|
7
|
+
onConflict?: ConflictPolicy
|
|
8
|
+
/** File mode of the committed file. Default `0o644`. */
|
|
9
|
+
mode?: number
|
|
10
|
+
/** Create missing parent directories. Default `true`. */
|
|
11
|
+
mkdirs?: boolean
|
|
12
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
/** The publisher refuses a `writeBinaryFile` payload longer than this many base64 characters. */
|
|
2
|
+
export const WORKLOAD_MAX_BASE64_CHARS = 16 * 1024 * 1024
|
|
3
|
+
|
|
4
|
+
/** The largest file whose base64 fits the publisher's ceiling (12 MiB). */
|
|
5
|
+
export const WORKLOAD_MAX_FILE_BYTES = WORKLOAD_MAX_BASE64_CHARS / 4 * 3
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The one binary write a Viable ephemeral workload accepts (its publisher's `writeBinaryFile`
|
|
3
|
+
* command). viable-agent's `FileHelper` satisfies it as is — pass the helper itself, never a
|
|
4
|
+
* destructured method.
|
|
5
|
+
*/
|
|
6
|
+
export interface WorkloadWriter {
|
|
7
|
+
writeBinaryFile: (filePath: string, base64: string) => Promise<void>
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export interface WorkloadSinkOptions {
|
|
11
|
+
/** Folder inside the workload's project, e.g. `sources/web/public/media`. */
|
|
12
|
+
root?: string
|
|
13
|
+
/** Per-file ceiling; never above (and by default) `WORKLOAD_MAX_FILE_BYTES`. */
|
|
14
|
+
maxFileBytes?: number
|
|
15
|
+
/** Retries of a failed write — the write is atomic on the workload, so a retry is safe. Default 1. */
|
|
16
|
+
retries?: number
|
|
17
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { logger } from '@owlmeans/log'
|
|
2
|
+
import { LimitError, WorkloadError } from '../errors.js'
|
|
3
|
+
import type { FileSink, SinkInput, SinkResult, SinkWriteOptions, TreeSink } from '../types.js'
|
|
4
|
+
import { pathUtils } from '../utils/paths.js'
|
|
5
|
+
import { streamUtils } from '../utils/streams.js'
|
|
6
|
+
import { WORKLOAD_MAX_FILE_BYTES } from './workload/consts.js'
|
|
7
|
+
import type { WorkloadSinkOptions, WorkloadWriter } from './workload/types.js'
|
|
8
|
+
|
|
9
|
+
const log = logger('file-streams:workload')
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Files written into a Viable ephemeral workload through its writer: one atomic write per file,
|
|
13
|
+
* each at most 12 MiB (the publisher's base64 ceiling) or a lower `maxFileBytes`.
|
|
14
|
+
*/
|
|
15
|
+
export const makeWorkloadSink = (writer: WorkloadWriter, options: WorkloadSinkOptions = {}): TreeSink => {
|
|
16
|
+
const ceiling = Math.min(options.maxFileBytes ?? WORKLOAD_MAX_FILE_BYTES, WORKLOAD_MAX_FILE_BYTES)
|
|
17
|
+
const root = options.root != null && options.root.replace(/^\/+|\/+$/g, '') !== '' ? pathUtils.entry(options.root) : ''
|
|
18
|
+
const retries = options.retries ?? 1
|
|
19
|
+
|
|
20
|
+
const entry = (path: string): FileSink => {
|
|
21
|
+
const filePath = root === '' ? pathUtils.entry(path) : `${root}/${pathUtils.entry(path)}`
|
|
22
|
+
|
|
23
|
+
const write = async (input: SinkInput, writeOptions: SinkWriteOptions = {}): Promise<SinkResult> => {
|
|
24
|
+
const limit = Math.min(ceiling, writeOptions.maxBytes ?? Infinity)
|
|
25
|
+
if (input.info.size != null && input.info.size > limit) {
|
|
26
|
+
streamUtils.discard(input.stream)
|
|
27
|
+
throw new WorkloadError('too-large')
|
|
28
|
+
}
|
|
29
|
+
let bytes: Buffer
|
|
30
|
+
try {
|
|
31
|
+
bytes = await streamUtils.collect(input.stream, limit)
|
|
32
|
+
} catch (error) {
|
|
33
|
+
throw error instanceof LimitError ? new WorkloadError('too-large') : error
|
|
34
|
+
}
|
|
35
|
+
const base64 = bytes.toString('base64')
|
|
36
|
+
|
|
37
|
+
for (let attempt = 0; ; attempt++) {
|
|
38
|
+
writeOptions.signal?.throwIfAborted()
|
|
39
|
+
try {
|
|
40
|
+
await writer.writeBinaryFile(filePath, base64)
|
|
41
|
+
break
|
|
42
|
+
} catch (error) {
|
|
43
|
+
if (attempt >= retries) {
|
|
44
|
+
log.warn('Workload write failed', { bytes: bytes.length, attempts: attempt + 1 })
|
|
45
|
+
throw new WorkloadError('write-failed')
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
return { kind: 'workload', location: filePath, bytes: bytes.length, mime: input.verdict?.detected?.mime ?? input.info.mime }
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
return { kind: 'workload', write }
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const directory = async (): Promise<void> => undefined
|
|
57
|
+
|
|
58
|
+
return { kind: 'workload', entry, directory }
|
|
59
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { Readable } from 'node:stream'
|
|
2
|
+
import type { FileInfo, FileSource, OpenedFile, OpenOptions, RangeReader } from '../types.js'
|
|
3
|
+
import { nameUtils } from '../utils/names.js'
|
|
4
|
+
import { rangeUtils } from '../utils/range.js'
|
|
5
|
+
import type { SourceClaims } from './types.js'
|
|
6
|
+
|
|
7
|
+
/** Bytes in memory — a screenshot, a generated file: re-openable, with random access. */
|
|
8
|
+
export const createBufferSource = (bytes: Uint8Array, claims: SourceClaims = {}): FileSource => {
|
|
9
|
+
const known: FileInfo = { origin: 'buffer', name: nameUtils.sanitize(claims.name), size: bytes.length, mime: claims.mime }
|
|
10
|
+
|
|
11
|
+
const info = async (): Promise<FileInfo> => known
|
|
12
|
+
|
|
13
|
+
const open = async (options: OpenOptions = {}): Promise<OpenedFile> => {
|
|
14
|
+
if (options.range == null) {
|
|
15
|
+
return { stream: Readable.from([Buffer.from(bytes)], { objectMode: false }), info: known }
|
|
16
|
+
}
|
|
17
|
+
const start = Math.min(options.range.start, bytes.length)
|
|
18
|
+
const end = Math.min(options.range.end ?? bytes.length - 1, bytes.length - 1)
|
|
19
|
+
const slice = Buffer.from(bytes.subarray(start, end + 1))
|
|
20
|
+
|
|
21
|
+
return { stream: Readable.from(slice.length > 0 ? [slice] : [], { objectMode: false }), info: known, range: { start, end, total: bytes.length } }
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const ranges = async (): Promise<RangeReader> => rangeUtils.ofBuffer(bytes)
|
|
25
|
+
|
|
26
|
+
return { kind: 'buffer', info, open, ranges }
|
|
27
|
+
}
|