@owlmeans/file-streams 0.0.0-stage → 0.2.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +71 -2
- package/agent-meta/manifest.json +64 -0
- package/agent-meta/skills/file-streams/SKILL.md +183 -0
- package/agent-meta/skills/file-streams/parts/network.md +47 -0
- package/agent-meta/skills/file-streams/parts/workspaces.md +129 -0
- package/build/archive/consts.d.ts +16 -0
- package/build/archive/consts.d.ts.map +1 -0
- package/build/archive/consts.js +24 -0
- package/build/archive/consts.js.map +1 -0
- package/build/archive/formats.d.ts +4 -0
- package/build/archive/formats.d.ts.map +1 -0
- package/build/archive/formats.js +53 -0
- package/build/archive/formats.js.map +1 -0
- package/build/archive/guard.d.ts +5 -0
- package/build/archive/guard.d.ts.map +1 -0
- package/build/archive/guard.js +62 -0
- package/build/archive/guard.js.map +1 -0
- package/build/archive/ratio.d.ts +5 -0
- package/build/archive/ratio.d.ts.map +1 -0
- package/build/archive/ratio.js +30 -0
- package/build/archive/ratio.js.map +1 -0
- package/build/archive/tar.d.ts +4 -0
- package/build/archive/tar.d.ts.map +1 -0
- package/build/archive/tar.js +142 -0
- package/build/archive/tar.js.map +1 -0
- package/build/archive/types.d.ts +95 -0
- package/build/archive/types.d.ts.map +1 -0
- package/build/archive/types.js +2 -0
- package/build/archive/types.js.map +1 -0
- package/build/archive/types.local.d.ts +64 -0
- package/build/archive/types.local.d.ts.map +1 -0
- package/build/archive/types.local.js +2 -0
- package/build/archive/types.local.js.map +1 -0
- package/build/archive/zip.d.ts +4 -0
- package/build/archive/zip.d.ts.map +1 -0
- package/build/archive/zip.js +135 -0
- package/build/archive/zip.js.map +1 -0
- package/build/archive.d.ts +4 -0
- package/build/archive.d.ts.map +1 -0
- package/build/archive.js +265 -0
- package/build/archive.js.map +1 -0
- package/build/backed-workspace/types.d.ts +67 -0
- package/build/backed-workspace/types.d.ts.map +1 -0
- package/build/backed-workspace/types.js +2 -0
- package/build/backed-workspace/types.js.map +1 -0
- package/build/backed-workspace.d.ts +4 -0
- package/build/backed-workspace.d.ts.map +1 -0
- package/build/backed-workspace.js +63 -0
- package/build/backed-workspace.js.map +1 -0
- package/build/consts.d.ts +3 -0
- package/build/consts.d.ts.map +1 -0
- package/build/consts.js +3 -0
- package/build/consts.js.map +1 -0
- package/build/consts.local.d.ts +3 -0
- package/build/consts.local.d.ts.map +1 -0
- package/build/consts.local.js +3 -0
- package/build/consts.local.js.map +1 -0
- package/build/errors.d.ts +59 -0
- package/build/errors.d.ts.map +1 -0
- package/build/errors.js +105 -0
- package/build/errors.js.map +1 -0
- package/build/http/connector.d.ts +11 -0
- package/build/http/connector.d.ts.map +1 -0
- package/build/http/connector.js +124 -0
- package/build/http/connector.js.map +1 -0
- package/build/http/consts.d.ts +9 -0
- package/build/http/consts.d.ts.map +1 -0
- package/build/http/consts.js +8 -0
- package/build/http/consts.js.map +1 -0
- package/build/http/source.d.ts +5 -0
- package/build/http/source.d.ts.map +1 -0
- package/build/http/source.js +76 -0
- package/build/http/source.js.map +1 -0
- package/build/http/types.d.ts +67 -0
- package/build/http/types.d.ts.map +1 -0
- package/build/http/types.js +2 -0
- package/build/http/types.js.map +1 -0
- package/build/http/types.local.d.ts +21 -0
- package/build/http/types.local.d.ts.map +1 -0
- package/build/http/types.local.js +2 -0
- package/build/http/types.local.js.map +1 -0
- package/build/http.d.ts +4 -0
- package/build/http.d.ts.map +1 -0
- package/build/http.js +248 -0
- package/build/http.js.map +1 -0
- package/build/index.d.ts +53 -0
- package/build/index.d.ts.map +1 -0
- package/build/index.js +33 -0
- package/build/index.js.map +1 -0
- package/build/network/consts.d.ts +12 -0
- package/build/network/consts.d.ts.map +1 -0
- package/build/network/consts.js +20 -0
- package/build/network/consts.js.map +1 -0
- package/build/network/types.d.ts +43 -0
- package/build/network/types.d.ts.map +1 -0
- package/build/network/types.js +2 -0
- package/build/network/types.js.map +1 -0
- package/build/network.d.ts +4 -0
- package/build/network.d.ts.map +1 -0
- package/build/network.js +165 -0
- package/build/network.js.map +1 -0
- package/build/proxy/consts.d.ts +7 -0
- package/build/proxy/consts.d.ts.map +1 -0
- package/build/proxy/consts.js +9 -0
- package/build/proxy/consts.js.map +1 -0
- package/build/proxy/endpoint.d.ts +4 -0
- package/build/proxy/endpoint.d.ts.map +1 -0
- package/build/proxy/endpoint.js +27 -0
- package/build/proxy/endpoint.js.map +1 -0
- package/build/proxy/http.d.ts +5 -0
- package/build/proxy/http.d.ts.map +1 -0
- package/build/proxy/http.js +95 -0
- package/build/proxy/http.js.map +1 -0
- package/build/proxy/router.d.ts +4 -0
- package/build/proxy/router.d.ts.map +1 -0
- package/build/proxy/router.js +85 -0
- package/build/proxy/router.js.map +1 -0
- package/build/proxy/socks.d.ts +5 -0
- package/build/proxy/socks.d.ts.map +1 -0
- package/build/proxy/socks.js +44 -0
- package/build/proxy/socks.js.map +1 -0
- package/build/proxy/types.d.ts +99 -0
- package/build/proxy/types.d.ts.map +1 -0
- package/build/proxy/types.js +2 -0
- package/build/proxy/types.js.map +1 -0
- package/build/resolvers/google-drive/consts.d.ts +10 -0
- package/build/resolvers/google-drive/consts.d.ts.map +1 -0
- package/build/resolvers/google-drive/consts.js +16 -0
- package/build/resolvers/google-drive/consts.js.map +1 -0
- package/build/resolvers/google-drive/types.d.ts +21 -0
- package/build/resolvers/google-drive/types.d.ts.map +1 -0
- package/build/resolvers/google-drive/types.js +2 -0
- package/build/resolvers/google-drive/types.js.map +1 -0
- package/build/resolvers/google-drive.d.ts +10 -0
- package/build/resolvers/google-drive.d.ts.map +1 -0
- package/build/resolvers/google-drive.js +130 -0
- package/build/resolvers/google-drive.js.map +1 -0
- package/build/resolvers/types.d.ts +39 -0
- package/build/resolvers/types.d.ts.map +1 -0
- package/build/resolvers/types.js +2 -0
- package/build/resolvers/types.js.map +1 -0
- package/build/s3/consts.d.ts +9 -0
- package/build/s3/consts.d.ts.map +1 -0
- package/build/s3/consts.js +9 -0
- package/build/s3/consts.js.map +1 -0
- package/build/s3/prefix.d.ts +4 -0
- package/build/s3/prefix.d.ts.map +1 -0
- package/build/s3/prefix.js +87 -0
- package/build/s3/prefix.js.map +1 -0
- package/build/s3/sink.d.ts +6 -0
- package/build/s3/sink.d.ts.map +1 -0
- package/build/s3/sink.js +30 -0
- package/build/s3/sink.js.map +1 -0
- package/build/s3/source.d.ts +5 -0
- package/build/s3/source.d.ts.map +1 -0
- package/build/s3/source.js +81 -0
- package/build/s3/source.js.map +1 -0
- package/build/s3/types.d.ts +66 -0
- package/build/s3/types.d.ts.map +1 -0
- package/build/s3/types.js +2 -0
- package/build/s3/types.js.map +1 -0
- package/build/s3/types.local.d.ts +43 -0
- package/build/s3/types.local.d.ts.map +1 -0
- package/build/s3/types.local.js +2 -0
- package/build/s3/types.local.js.map +1 -0
- package/build/s3/upload.d.ts +4 -0
- package/build/s3/upload.d.ts.map +1 -0
- package/build/s3/upload.js +118 -0
- package/build/s3/upload.js.map +1 -0
- package/build/s3.d.ts +5 -0
- package/build/s3.d.ts.map +1 -0
- package/build/s3.js +68 -0
- package/build/s3.js.map +1 -0
- package/build/service/types.d.ts +66 -0
- package/build/service/types.d.ts.map +1 -0
- package/build/service/types.js +2 -0
- package/build/service/types.js.map +1 -0
- package/build/service/types.local.d.ts +11 -0
- package/build/service/types.local.d.ts.map +1 -0
- package/build/service/types.local.js +2 -0
- package/build/service/types.local.js.map +1 -0
- package/build/service.d.ts +6 -0
- package/build/service.d.ts.map +1 -0
- package/build/service.js +73 -0
- package/build/service.js.map +1 -0
- package/build/sinks/buffer/consts.d.ts +2 -0
- package/build/sinks/buffer/consts.d.ts.map +1 -0
- package/build/sinks/buffer/consts.js +2 -0
- package/build/sinks/buffer/consts.js.map +1 -0
- package/build/sinks/buffer/types.d.ts +11 -0
- package/build/sinks/buffer/types.d.ts.map +1 -0
- package/build/sinks/buffer/types.js +2 -0
- package/build/sinks/buffer/types.js.map +1 -0
- package/build/sinks/buffer.d.ts +4 -0
- package/build/sinks/buffer.d.ts.map +1 -0
- package/build/sinks/buffer.js +13 -0
- package/build/sinks/buffer.js.map +1 -0
- package/build/sinks/directory/types.d.ts +7 -0
- package/build/sinks/directory/types.d.ts.map +1 -0
- package/build/sinks/directory/types.js +2 -0
- package/build/sinks/directory/types.js.map +1 -0
- package/build/sinks/directory.d.ts +8 -0
- package/build/sinks/directory.d.ts.map +1 -0
- package/build/sinks/directory.js +43 -0
- package/build/sinks/directory.js.map +1 -0
- package/build/sinks/file.d.ts +5 -0
- package/build/sinks/file.d.ts.map +1 -0
- package/build/sinks/file.js +46 -0
- package/build/sinks/file.js.map +1 -0
- package/build/sinks/types.d.ts +11 -0
- package/build/sinks/types.d.ts.map +1 -0
- package/build/sinks/types.js +2 -0
- package/build/sinks/types.js.map +1 -0
- package/build/sinks/workload/consts.d.ts +5 -0
- package/build/sinks/workload/consts.d.ts.map +1 -0
- package/build/sinks/workload/consts.js +5 -0
- package/build/sinks/workload/consts.js.map +1 -0
- package/build/sinks/workload/types.d.ts +17 -0
- package/build/sinks/workload/types.d.ts.map +1 -0
- package/build/sinks/workload/types.js +2 -0
- package/build/sinks/workload/types.js.map +1 -0
- package/build/sinks/workload.d.ts +8 -0
- package/build/sinks/workload.d.ts.map +1 -0
- package/build/sinks/workload.js +51 -0
- package/build/sinks/workload.js.map +1 -0
- package/build/sources/buffer.d.ts +5 -0
- package/build/sources/buffer.d.ts.map +1 -0
- package/build/sources/buffer.js +20 -0
- package/build/sources/buffer.js.map +1 -0
- package/build/sources/file.d.ts +5 -0
- package/build/sources/file.d.ts.map +1 -0
- package/build/sources/file.js +50 -0
- package/build/sources/file.js.map +1 -0
- package/build/sources/stream.d.ts +5 -0
- package/build/sources/stream.d.ts.map +1 -0
- package/build/sources/stream.js +19 -0
- package/build/sources/stream.js.map +1 -0
- package/build/sources/types.d.ts +10 -0
- package/build/sources/types.d.ts.map +1 -0
- package/build/sources/types.js +2 -0
- package/build/sources/types.js.map +1 -0
- package/build/sources/workload/types.d.ts +29 -0
- package/build/sources/workload/types.d.ts.map +1 -0
- package/build/sources/workload/types.js +2 -0
- package/build/sources/workload/types.js.map +1 -0
- package/build/sources/workload.d.ts +7 -0
- package/build/sources/workload.d.ts.map +1 -0
- package/build/sources/workload.js +67 -0
- package/build/sources/workload.js.map +1 -0
- package/build/transfer/consts.d.ts +5 -0
- package/build/transfer/consts.d.ts.map +1 -0
- package/build/transfer/consts.js +8 -0
- package/build/transfer/consts.js.map +1 -0
- package/build/transfer/types.d.ts +49 -0
- package/build/transfer/types.d.ts.map +1 -0
- package/build/transfer/types.js +2 -0
- package/build/transfer/types.js.map +1 -0
- package/build/transfer.d.ts +4 -0
- package/build/transfer.d.ts.map +1 -0
- package/build/transfer.js +75 -0
- package/build/transfer.js.map +1 -0
- package/build/types.d.ts +99 -0
- package/build/types.d.ts.map +1 -0
- package/build/types.js +2 -0
- package/build/types.js.map +1 -0
- package/build/utils/names/consts.d.ts +5 -0
- package/build/utils/names/consts.d.ts.map +1 -0
- package/build/utils/names/consts.js +5 -0
- package/build/utils/names/consts.js.map +1 -0
- package/build/utils/names/types.d.ts +13 -0
- package/build/utils/names/types.d.ts.map +1 -0
- package/build/utils/names/types.js +2 -0
- package/build/utils/names/types.js.map +1 -0
- package/build/utils/names.d.ts +4 -0
- package/build/utils/names.d.ts.map +1 -0
- package/build/utils/names.js +60 -0
- package/build/utils/names.js.map +1 -0
- package/build/utils/paths/consts.d.ts +3 -0
- package/build/utils/paths/consts.d.ts.map +1 -0
- package/build/utils/paths/consts.js +2 -0
- package/build/utils/paths/consts.js.map +1 -0
- package/build/utils/paths/types.d.ts +20 -0
- package/build/utils/paths/types.d.ts.map +1 -0
- package/build/utils/paths/types.js +2 -0
- package/build/utils/paths/types.js.map +1 -0
- package/build/utils/paths.d.ts +4 -0
- package/build/utils/paths.d.ts.map +1 -0
- package/build/utils/paths.js +67 -0
- package/build/utils/paths.js.map +1 -0
- package/build/utils/range/types.d.ts +11 -0
- package/build/utils/range/types.d.ts.map +1 -0
- package/build/utils/range/types.js +2 -0
- package/build/utils/range/types.js.map +1 -0
- package/build/utils/range.d.ts +4 -0
- package/build/utils/range.d.ts.map +1 -0
- package/build/utils/range.js +64 -0
- package/build/utils/range.js.map +1 -0
- package/build/utils/redact/types.d.ts +16 -0
- package/build/utils/redact/types.d.ts.map +1 -0
- package/build/utils/redact/types.js +2 -0
- package/build/utils/redact/types.js.map +1 -0
- package/build/utils/redact.d.ts +4 -0
- package/build/utils/redact.d.ts.map +1 -0
- package/build/utils/redact.js +18 -0
- package/build/utils/redact.js.map +1 -0
- package/build/utils/signals/types.d.ts +17 -0
- package/build/utils/signals/types.d.ts.map +1 -0
- package/build/utils/signals/types.js +2 -0
- package/build/utils/signals/types.js.map +1 -0
- package/build/utils/signals.d.ts +4 -0
- package/build/utils/signals.d.ts.map +1 -0
- package/build/utils/signals.js +52 -0
- package/build/utils/signals.js.map +1 -0
- package/build/utils/streams/consts.d.ts +3 -0
- package/build/utils/streams/consts.d.ts.map +1 -0
- package/build/utils/streams/consts.js +3 -0
- package/build/utils/streams/consts.js.map +1 -0
- package/build/utils/streams/types.d.ts +59 -0
- package/build/utils/streams/types.d.ts.map +1 -0
- package/build/utils/streams/types.js +2 -0
- package/build/utils/streams/types.js.map +1 -0
- package/build/utils/streams.d.ts +4 -0
- package/build/utils/streams.d.ts.map +1 -0
- package/build/utils/streams.js +167 -0
- package/build/utils/streams.js.map +1 -0
- package/build/utils/tree/types.d.ts +6 -0
- package/build/utils/tree/types.d.ts.map +1 -0
- package/build/utils/tree/types.js +2 -0
- package/build/utils/tree/types.js.map +1 -0
- package/build/utils/tree.d.ts +4 -0
- package/build/utils/tree.d.ts.map +1 -0
- package/build/utils/tree.js +14 -0
- package/build/utils/tree.js.map +1 -0
- package/build/workspace/consts.d.ts +12 -0
- package/build/workspace/consts.d.ts.map +1 -0
- package/build/workspace/consts.js +12 -0
- package/build/workspace/consts.js.map +1 -0
- package/build/workspace/consts.local.d.ts +3 -0
- package/build/workspace/consts.local.d.ts.map +1 -0
- package/build/workspace/consts.local.js +3 -0
- package/build/workspace/consts.local.js.map +1 -0
- package/build/workspace/fs.d.ts +4 -0
- package/build/workspace/fs.d.ts.map +1 -0
- package/build/workspace/fs.js +45 -0
- package/build/workspace/fs.js.map +1 -0
- package/build/workspace/scope.d.ts +6 -0
- package/build/workspace/scope.d.ts.map +1 -0
- package/build/workspace/scope.js +180 -0
- package/build/workspace/scope.js.map +1 -0
- package/build/workspace/types.d.ts +66 -0
- package/build/workspace/types.d.ts.map +1 -0
- package/build/workspace/types.js +2 -0
- package/build/workspace/types.js.map +1 -0
- package/build/workspace/types.local.d.ts +20 -0
- package/build/workspace/types.local.d.ts.map +1 -0
- package/build/workspace/types.local.js +2 -0
- package/build/workspace/types.local.js.map +1 -0
- package/build/workspace.d.ts +7 -0
- package/build/workspace.d.ts.map +1 -0
- package/build/workspace.js +87 -0
- package/build/workspace.js.map +1 -0
- package/package.json +61 -4
- package/src/archive/consts.ts +28 -0
- package/src/archive/formats.ts +60 -0
- package/src/archive/guard.ts +68 -0
- package/src/archive/ratio.ts +35 -0
- package/src/archive/tar.ts +147 -0
- package/src/archive/types.local.ts +72 -0
- package/src/archive/types.ts +107 -0
- package/src/archive/zip.ts +142 -0
- package/src/archive.ts +298 -0
- package/src/backed-workspace/types.ts +72 -0
- package/src/backed-workspace.ts +85 -0
- package/src/consts.local.ts +2 -0
- package/src/consts.ts +2 -0
- package/src/errors.ts +128 -0
- package/src/http/connector.ts +136 -0
- package/src/http/consts.ts +13 -0
- package/src/http/source.ts +89 -0
- package/src/http/types.local.ts +23 -0
- package/src/http/types.ts +74 -0
- package/src/http.ts +276 -0
- package/src/index.ts +65 -0
- package/src/network/consts.ts +22 -0
- package/src/network/types.ts +45 -0
- package/src/network.ts +179 -0
- package/src/proxy/consts.ts +11 -0
- package/src/proxy/endpoint.ts +28 -0
- package/src/proxy/http.ts +108 -0
- package/src/proxy/router.ts +93 -0
- package/src/proxy/socks.ts +48 -0
- package/src/proxy/types.ts +114 -0
- package/src/resolvers/google-drive/consts.ts +22 -0
- package/src/resolvers/google-drive/types.ts +22 -0
- package/src/resolvers/google-drive.ts +153 -0
- package/src/resolvers/types.ts +45 -0
- package/src/s3/consts.ts +12 -0
- package/src/s3/prefix.ts +100 -0
- package/src/s3/sink.ts +36 -0
- package/src/s3/source.ts +90 -0
- package/src/s3/types.local.ts +49 -0
- package/src/s3/types.ts +70 -0
- package/src/s3/upload.ts +130 -0
- package/src/s3.ts +83 -0
- package/src/service/types.local.ts +9 -0
- package/src/service/types.ts +71 -0
- package/src/service.ts +97 -0
- package/src/sinks/buffer/consts.ts +1 -0
- package/src/sinks/buffer/types.ts +12 -0
- package/src/sinks/buffer.ts +18 -0
- package/src/sinks/directory/types.ts +7 -0
- package/src/sinks/directory.ts +49 -0
- package/src/sinks/file.ts +52 -0
- package/src/sinks/types.ts +12 -0
- package/src/sinks/workload/consts.ts +5 -0
- package/src/sinks/workload/types.ts +17 -0
- package/src/sinks/workload.ts +59 -0
- package/src/sources/buffer.ts +27 -0
- package/src/sources/file.ts +57 -0
- package/src/sources/stream.ts +26 -0
- package/src/sources/types.ts +12 -0
- package/src/sources/workload/types.ts +31 -0
- package/src/sources/workload.ts +79 -0
- package/src/transfer/consts.ts +10 -0
- package/src/transfer/types.ts +55 -0
- package/src/transfer.ts +89 -0
- package/src/types.ts +106 -0
- package/src/utils/names/consts.ts +5 -0
- package/src/utils/names/types.ts +12 -0
- package/src/utils/names.ts +70 -0
- package/src/utils/paths/consts.ts +3 -0
- package/src/utils/paths/types.ts +20 -0
- package/src/utils/paths.ts +75 -0
- package/src/utils/range/types.ts +11 -0
- package/src/utils/range.ts +80 -0
- package/src/utils/redact/types.ts +16 -0
- package/src/utils/redact.ts +23 -0
- package/src/utils/signals/types.ts +17 -0
- package/src/utils/signals.ts +63 -0
- package/src/utils/streams/consts.ts +2 -0
- package/src/utils/streams/types.ts +65 -0
- package/src/utils/streams.ts +188 -0
- package/src/utils/tree/types.ts +6 -0
- package/src/utils/tree.ts +18 -0
- package/src/workspace/consts.local.ts +2 -0
- package/src/workspace/consts.ts +14 -0
- package/src/workspace/fs.ts +48 -0
- package/src/workspace/scope.ts +196 -0
- package/src/workspace/types.local.ts +21 -0
- package/src/workspace/types.ts +71 -0
- package/src/workspace.ts +102 -0
- package/tests/archive.spec.ts +134 -0
- package/tests/backed.spec.ts +147 -0
- package/tests/context.ts +114 -0
- package/tests/fixtures.ts +190 -0
- package/tests/google-drive.spec.ts +76 -0
- package/tests/harness/drive.ts +92 -0
- package/tests/harness/http-proxy.ts +96 -0
- package/tests/harness/origin.ts +134 -0
- package/tests/harness/socks5.ts +135 -0
- package/tests/harness/workload.ts +118 -0
- package/tests/http.spec.ts +101 -0
- package/tests/inspector.spec.ts +40 -0
- package/tests/live.spec.ts +27 -0
- package/tests/network.spec.ts +79 -0
- package/tests/proxy.spec.ts +149 -0
- package/tests/s3.spec.ts +133 -0
- package/tests/service.spec.ts +77 -0
- package/tests/sources-sinks.spec.ts +96 -0
- package/tests/transfer.spec.ts +99 -0
- package/tests/tsconfig.json +12 -0
- package/tests/workload.spec.ts +140 -0
- package/tests/workspace.spec.ts +158 -0
- package/tsconfig.json +16 -0
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { createReadStream } from 'node:fs'
|
|
2
|
+
import { stat } from 'node:fs/promises'
|
|
3
|
+
import { basename } from 'node:path'
|
|
4
|
+
import { SourceError } from '../errors.js'
|
|
5
|
+
import type { FileInfo, FileSource, OpenedFile, OpenOptions, RangeReader } from '../types.js'
|
|
6
|
+
import { nameUtils } from '../utils/names.js'
|
|
7
|
+
import { rangeUtils } from '../utils/range.js'
|
|
8
|
+
import type { SourceClaims } from './types.js'
|
|
9
|
+
|
|
10
|
+
/** A regular file on disk: re-openable, with random access. */
|
|
11
|
+
export const createFileSource = (path: string, claims: SourceClaims = {}): FileSource => {
|
|
12
|
+
let known: FileInfo | undefined
|
|
13
|
+
|
|
14
|
+
const info = async (): Promise<FileInfo> => {
|
|
15
|
+
if (known != null) {
|
|
16
|
+
return known
|
|
17
|
+
}
|
|
18
|
+
let stats
|
|
19
|
+
try {
|
|
20
|
+
stats = await stat(path)
|
|
21
|
+
} catch (error) {
|
|
22
|
+
throw (error as NodeJS.ErrnoException).code === 'ENOENT' ? new SourceError('not-found') : error
|
|
23
|
+
}
|
|
24
|
+
if (!stats.isFile()) {
|
|
25
|
+
throw new SourceError('not-a-file')
|
|
26
|
+
}
|
|
27
|
+
known = {
|
|
28
|
+
origin: path,
|
|
29
|
+
name: nameUtils.sanitize(claims.name ?? basename(path)),
|
|
30
|
+
size: stats.size,
|
|
31
|
+
mime: claims.mime,
|
|
32
|
+
etag: `${Math.trunc(stats.mtimeMs)}-${stats.size}`,
|
|
33
|
+
modifiedAt: stats.mtime,
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
return known
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const open = async (options: OpenOptions = {}): Promise<OpenedFile> => {
|
|
40
|
+
const current = await info()
|
|
41
|
+
const size = current.size ?? 0
|
|
42
|
+
if (options.range == null) {
|
|
43
|
+
return { stream: createReadStream(path, { signal: options.signal }), info: current }
|
|
44
|
+
}
|
|
45
|
+
const start = Math.min(options.range.start, size)
|
|
46
|
+
const end = Math.min(options.range.end ?? size - 1, size - 1)
|
|
47
|
+
const stream = end < start
|
|
48
|
+
? createReadStream(path, { start: 0, end: -1, signal: options.signal })
|
|
49
|
+
: createReadStream(path, { start, end, signal: options.signal })
|
|
50
|
+
|
|
51
|
+
return { stream, info: current, range: { start, end, total: size } }
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const ranges = async (): Promise<RangeReader> => rangeUtils.ofFile(path, (await info()).size ?? 0)
|
|
55
|
+
|
|
56
|
+
return { kind: 'file', info, open, ranges }
|
|
57
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { SourceError } from '../errors.js'
|
|
2
|
+
import type { FileInfo, FileSource, OpenedFile } from '../types.js'
|
|
3
|
+
import { nameUtils } from '../utils/names.js'
|
|
4
|
+
import { streamUtils } from '../utils/streams.js'
|
|
5
|
+
import type { SourceClaims, StreamInput } from './types.js'
|
|
6
|
+
|
|
7
|
+
/** A stream someone else opened: one-shot, without random access. */
|
|
8
|
+
export const createStreamSource = (input: StreamInput, claims: SourceClaims = {}): FileSource => {
|
|
9
|
+
const known: FileInfo = { origin: 'stream', name: nameUtils.sanitize(claims.name), size: claims.size, mime: claims.mime }
|
|
10
|
+
let consumed = false
|
|
11
|
+
|
|
12
|
+
const info = async (): Promise<FileInfo> => known
|
|
13
|
+
|
|
14
|
+
const open = async (): Promise<OpenedFile> => {
|
|
15
|
+
if (consumed) {
|
|
16
|
+
throw new SourceError('consumed')
|
|
17
|
+
}
|
|
18
|
+
consumed = true
|
|
19
|
+
|
|
20
|
+
return { stream: streamUtils.toReadable(input), info: known }
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const ranges = async (): Promise<null> => null
|
|
24
|
+
|
|
25
|
+
return { kind: 'stream', info, open, ranges }
|
|
26
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { Readable } from 'node:stream'
|
|
2
|
+
|
|
3
|
+
/** What a caller knows about bytes it hands over — claims a verdict checks, never facts. */
|
|
4
|
+
export interface SourceClaims {
|
|
5
|
+
name?: string
|
|
6
|
+
mime?: string
|
|
7
|
+
size?: number
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
/** A stream in any shape the package accepts: Node, web, or any async iterable of bytes. */
|
|
11
|
+
// A union of unrelated stream shapes — an interface cannot express it.
|
|
12
|
+
export type StreamInput = Readable | ReadableStream<Uint8Array> | AsyncIterable<Uint8Array>
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { FileSource } from '../../types.js'
|
|
2
|
+
import type { SourceClaims } from '../types.js'
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* The one binary read a Viable ephemeral workload answers (its publisher's `readBinaryFile`
|
|
6
|
+
* command, `{ filePath } → { base64 }`). viable-agent's `FileHelper` satisfies it as is — pass the
|
|
7
|
+
* helper itself, never a destructured method.
|
|
8
|
+
*/
|
|
9
|
+
export interface WorkloadReader {
|
|
10
|
+
readBinaryFile: (filePath: string) => Promise<string>
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export interface WorkloadSourceOptions {
|
|
14
|
+
/** Folder inside the workload's project, e.g. `docs/materials`. */
|
|
15
|
+
root?: string
|
|
16
|
+
/** Per-file ceiling; never above (and by default) `WORKLOAD_MAX_FILE_BYTES`. */
|
|
17
|
+
maxFileBytes?: number
|
|
18
|
+
/** Retries of a failed read. Default 1. */
|
|
19
|
+
retries?: number
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** Files of a Viable ephemeral workload, read through its reader. */
|
|
23
|
+
export interface WorkloadSource {
|
|
24
|
+
readonly kind: string
|
|
25
|
+
/**
|
|
26
|
+
* One file as a source: re-openable (each open is one read of the whole file), its size known
|
|
27
|
+
* after the first read, random access over the bytes of one read.
|
|
28
|
+
* @throws {PathError} when the path is not a confined relative path.
|
|
29
|
+
*/
|
|
30
|
+
entry: (path: string, claims?: SourceClaims) => FileSource
|
|
31
|
+
}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { Readable } from 'node:stream'
|
|
2
|
+
import { logger } from '@owlmeans/log'
|
|
3
|
+
import { WorkloadError } from '../errors.js'
|
|
4
|
+
import { WORKLOAD_MAX_FILE_BYTES } from '../sinks/workload/consts.js'
|
|
5
|
+
import type { FileInfo, FileSource, OpenedFile, OpenOptions, RangeReader, SignalOptions } from '../types.js'
|
|
6
|
+
import { nameUtils } from '../utils/names.js'
|
|
7
|
+
import { pathUtils } from '../utils/paths.js'
|
|
8
|
+
import { rangeUtils } from '../utils/range.js'
|
|
9
|
+
import type { SourceClaims } from './types.js'
|
|
10
|
+
import type { WorkloadReader, WorkloadSource, WorkloadSourceOptions } from './workload/types.js'
|
|
11
|
+
|
|
12
|
+
const log = logger('file-streams:workload')
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Files read from a Viable ephemeral workload through its reader — the read twin of
|
|
16
|
+
* `makeWorkloadSink`: one read per open, each file at most 12 MiB or a lower `maxFileBytes`.
|
|
17
|
+
*/
|
|
18
|
+
export const makeWorkloadSource = (reader: WorkloadReader, options: WorkloadSourceOptions = {}): WorkloadSource => {
|
|
19
|
+
const ceiling = Math.min(options.maxFileBytes ?? WORKLOAD_MAX_FILE_BYTES, WORKLOAD_MAX_FILE_BYTES)
|
|
20
|
+
const root = options.root != null && options.root.replace(/^\/+|\/+$/g, '') !== '' ? pathUtils.entry(options.root) : ''
|
|
21
|
+
const retries = options.retries ?? 1
|
|
22
|
+
|
|
23
|
+
const entry = (path: string, claims: SourceClaims = {}): FileSource => {
|
|
24
|
+
const filePath = root === '' ? pathUtils.entry(path) : `${root}/${pathUtils.entry(path)}`
|
|
25
|
+
let known: FileInfo = { origin: filePath, name: nameUtils.sanitize(claims.name ?? filePath.split('/').at(-1)), mime: claims.mime }
|
|
26
|
+
|
|
27
|
+
const read = async (signal?: AbortSignal): Promise<Buffer> => {
|
|
28
|
+
let base64: unknown
|
|
29
|
+
for (let attempt = 0; ; attempt++) {
|
|
30
|
+
signal?.throwIfAborted()
|
|
31
|
+
try {
|
|
32
|
+
base64 = await reader.readBinaryFile(filePath)
|
|
33
|
+
break
|
|
34
|
+
} catch {
|
|
35
|
+
if (attempt >= retries) {
|
|
36
|
+
log.warn('Workload read failed', { attempts: attempt + 1 })
|
|
37
|
+
throw new WorkloadError('read-failed')
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
signal?.throwIfAborted()
|
|
42
|
+
if (typeof base64 !== 'string') {
|
|
43
|
+
throw new WorkloadError('read-failed')
|
|
44
|
+
}
|
|
45
|
+
// Four characters carry three bytes: refuse an oversized payload before decoding it.
|
|
46
|
+
if (base64.length > Math.ceil(ceiling / 3) * 4) {
|
|
47
|
+
throw new WorkloadError('too-large')
|
|
48
|
+
}
|
|
49
|
+
const bytes = Buffer.from(base64, 'base64')
|
|
50
|
+
if (bytes.length > ceiling) {
|
|
51
|
+
throw new WorkloadError('too-large')
|
|
52
|
+
}
|
|
53
|
+
known = { ...known, size: bytes.length }
|
|
54
|
+
|
|
55
|
+
return bytes
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// No read of its own: the size is known once a read has happened.
|
|
59
|
+
const info = async (): Promise<FileInfo> => known
|
|
60
|
+
|
|
61
|
+
const open = async (openOptions: OpenOptions = {}): Promise<OpenedFile> => {
|
|
62
|
+
const bytes = await read(openOptions.signal)
|
|
63
|
+
if (openOptions.range == null) {
|
|
64
|
+
return { stream: Readable.from(bytes.length > 0 ? [bytes] : [], { objectMode: false }), info: known }
|
|
65
|
+
}
|
|
66
|
+
const start = Math.min(openOptions.range.start, bytes.length)
|
|
67
|
+
const end = Math.min(openOptions.range.end ?? bytes.length - 1, bytes.length - 1)
|
|
68
|
+
const slice = bytes.subarray(start, end + 1)
|
|
69
|
+
|
|
70
|
+
return { stream: Readable.from(slice.length > 0 ? [slice] : [], { objectMode: false }), info: known, range: { start, end, total: bytes.length } }
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const ranges = async (rangeOptions: SignalOptions = {}): Promise<RangeReader> => rangeUtils.ofBuffer(await read(rangeOptions.signal))
|
|
74
|
+
|
|
75
|
+
return { kind: 'workload', info, open, ranges }
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
return { kind: 'workload', entry }
|
|
79
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { TransferLimits } from './types.js'
|
|
2
|
+
|
|
3
|
+
export const DEFAULT_TRANSFER_LIMITS: Readonly<TransferLimits> = Object.freeze({
|
|
4
|
+
maxBytes: 512 * 1024 * 1024,
|
|
5
|
+
timeoutMs: 600_000,
|
|
6
|
+
idleTimeoutMs: 30_000,
|
|
7
|
+
})
|
|
8
|
+
|
|
9
|
+
/** The name a file is written under when nothing names it. */
|
|
10
|
+
export const FALLBACK_FILE_NAME = 'download'
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import type { FileInspector, FileVerdict, VerifyPolicy } from '@owlmeans/file-inspector'
|
|
2
|
+
import type { FileInfo, FileSink, FileSource, SignalOptions, SinkResult, TreeSink } from '../types.js'
|
|
3
|
+
|
|
4
|
+
export interface TransferLimits {
|
|
5
|
+
/** Bytes accepted, counted after content decoding. Default 512 MiB. */
|
|
6
|
+
maxBytes: number
|
|
7
|
+
/** Whole-transfer deadline. Default 600 000 ms. */
|
|
8
|
+
timeoutMs: number
|
|
9
|
+
/** Longest silence between chunks. Default 30 000 ms. */
|
|
10
|
+
idleTimeoutMs: number
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export interface TransferProgress {
|
|
14
|
+
bytes: number
|
|
15
|
+
total?: number
|
|
16
|
+
elapsedMs: number
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export interface TransferProgressListener {
|
|
20
|
+
(progress: TransferProgress): void
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export interface TransferOptions extends SignalOptions {
|
|
24
|
+
limits?: Partial<TransferLimits>
|
|
25
|
+
/** Head verification before the first byte reaches the sink. Default `{}` (on); `false` skips it. */
|
|
26
|
+
verify?: VerifyPolicy | false
|
|
27
|
+
onProgress?: TransferProgressListener
|
|
28
|
+
/** Default 250 ms; a final call always fires. */
|
|
29
|
+
progressIntervalMs?: number
|
|
30
|
+
hash?: 'sha256'
|
|
31
|
+
/** Overrides of the source's claims (a target name, a known media type). */
|
|
32
|
+
info?: Partial<Omit<FileInfo, 'origin'>>
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export interface TransferResult {
|
|
36
|
+
bytes: number
|
|
37
|
+
durationMs: number
|
|
38
|
+
info: FileInfo
|
|
39
|
+
verdict?: FileVerdict
|
|
40
|
+
sha256?: string
|
|
41
|
+
sink: SinkResult
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export interface TransferHelperOptions {
|
|
45
|
+
inspector?: FileInspector
|
|
46
|
+
limits?: Partial<TransferLimits>
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** Moves one file from a source to a sink under limits, verifying it on the way. */
|
|
50
|
+
export interface TransferHelper {
|
|
51
|
+
/** Source → sink; the head is verified before the sink sees a byte unless `verify: false`. */
|
|
52
|
+
copy: (source: FileSource, sink: FileSink, options?: TransferOptions) => Promise<TransferResult>
|
|
53
|
+
/** The same into a tree, under the source's own sanitized name (or `options.info.name`). */
|
|
54
|
+
copyInto: (source: FileSource, tree: TreeSink, options?: TransferOptions) => Promise<TransferResult>
|
|
55
|
+
}
|
package/src/transfer.ts
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { pipeline } from 'node:stream'
|
|
2
|
+
import type { Readable } from 'node:stream'
|
|
3
|
+
import { logger } from '@owlmeans/log'
|
|
4
|
+
import { DEFAULT_SAMPLE_BYTES, fileInspector } from '@owlmeans/file-inspector'
|
|
5
|
+
import type { FileVerdict } from '@owlmeans/file-inspector'
|
|
6
|
+
import { LimitError } from './errors.js'
|
|
7
|
+
import { DEFAULT_TRANSFER_LIMITS, FALLBACK_FILE_NAME } from './transfer/consts.js'
|
|
8
|
+
import type { TransferHelper, TransferHelperOptions, TransferOptions, TransferResult } from './transfer/types.js'
|
|
9
|
+
import type { FileInfo, FileSink, FileSource, TreeSink } from './types.js'
|
|
10
|
+
import { nameUtils } from './utils/names.js'
|
|
11
|
+
import { signalUtils } from './utils/signals.js'
|
|
12
|
+
import { streamUtils } from './utils/streams.js'
|
|
13
|
+
|
|
14
|
+
const log = logger('file-streams:transfer')
|
|
15
|
+
|
|
16
|
+
export const createTransferHelper = (options: TransferHelperOptions = {}): TransferHelper => {
|
|
17
|
+
const inspector = options.inspector ?? fileInspector
|
|
18
|
+
|
|
19
|
+
const run = async (source: FileSource, sinkOf: (info: FileInfo) => FileSink, transferOptions: TransferOptions = {}): Promise<TransferResult> => {
|
|
20
|
+
const limits = { ...DEFAULT_TRANSFER_LIMITS, ...options.limits, ...transferOptions.limits }
|
|
21
|
+
const composed = signalUtils.compose([transferOptions.signal], limits.timeoutMs)
|
|
22
|
+
const signal = composed.signal
|
|
23
|
+
const started = Date.now()
|
|
24
|
+
let stream: Readable | undefined
|
|
25
|
+
|
|
26
|
+
try {
|
|
27
|
+
const opened = await source.open({ signal })
|
|
28
|
+
const info: FileInfo = { ...opened.info, ...transferOptions.info }
|
|
29
|
+
if (transferOptions.info?.name != null) {
|
|
30
|
+
info.name = nameUtils.sanitize(transferOptions.info.name)
|
|
31
|
+
}
|
|
32
|
+
if (info.size != null && info.size > limits.maxBytes) {
|
|
33
|
+
streamUtils.discard(opened.stream)
|
|
34
|
+
throw new LimitError('max-bytes')
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const meter = streamUtils.meter({
|
|
38
|
+
maxBytes: limits.maxBytes,
|
|
39
|
+
idleTimeoutMs: limits.idleTimeoutMs,
|
|
40
|
+
total: info.size,
|
|
41
|
+
onProgress: transferOptions.onProgress,
|
|
42
|
+
progressIntervalMs: transferOptions.progressIntervalMs,
|
|
43
|
+
hash: transferOptions.hash,
|
|
44
|
+
})
|
|
45
|
+
stream = (pipeline as unknown as (...parts: unknown[]) => Readable)(opened.stream, meter.stream, () => undefined)
|
|
46
|
+
const current = stream
|
|
47
|
+
const release = signalUtils.onAbort(signal, reason => current.destroy(reason as Error))
|
|
48
|
+
|
|
49
|
+
try {
|
|
50
|
+
let verdict: FileVerdict | undefined
|
|
51
|
+
if (transferOptions.verify !== false) {
|
|
52
|
+
const policy = transferOptions.verify ?? {}
|
|
53
|
+
const peeked = await streamUtils.peek(stream, policy.sampleBytes ?? DEFAULT_SAMPLE_BYTES)
|
|
54
|
+
stream = peeked.stream
|
|
55
|
+
verdict = inspector.enforce(
|
|
56
|
+
await inspector.inspect(peeked.head, { name: info.name, mime: info.mime }, { complete: peeked.ended }), policy
|
|
57
|
+
)
|
|
58
|
+
}
|
|
59
|
+
const sink = await sinkOf(info).write({ stream, info, verdict }, { signal, maxBytes: limits.maxBytes })
|
|
60
|
+
const result: TransferResult = { bytes: meter.bytes(), durationMs: Date.now() - started, info, verdict, sink }
|
|
61
|
+
const digest = meter.sha256()
|
|
62
|
+
if (digest != null) {
|
|
63
|
+
result.sha256 = digest
|
|
64
|
+
}
|
|
65
|
+
log.debug('Transferred', { origin: info.origin, bytes: result.bytes, sink: sink.kind })
|
|
66
|
+
return result
|
|
67
|
+
} finally {
|
|
68
|
+
release()
|
|
69
|
+
}
|
|
70
|
+
} catch (error) {
|
|
71
|
+
if (stream != null) {
|
|
72
|
+
streamUtils.discard(stream)
|
|
73
|
+
}
|
|
74
|
+
throw signal.aborted && signal.reason != null ? signal.reason : error
|
|
75
|
+
} finally {
|
|
76
|
+
composed.dispose()
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
const copy = async (source: FileSource, sink: FileSink, transferOptions?: TransferOptions): Promise<TransferResult> =>
|
|
81
|
+
await run(source, () => sink, transferOptions)
|
|
82
|
+
|
|
83
|
+
const copyInto = async (source: FileSource, tree: TreeSink, transferOptions?: TransferOptions): Promise<TransferResult> =>
|
|
84
|
+
await run(source, info => tree.entry(info.name ?? FALLBACK_FILE_NAME), transferOptions)
|
|
85
|
+
|
|
86
|
+
return { copy, copyInto }
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export const transferHelper = createTransferHelper()
|
package/src/types.ts
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import type { Readable } from 'node:stream'
|
|
2
|
+
import type { FileVerdict } from '@owlmeans/file-inspector'
|
|
3
|
+
|
|
4
|
+
/** What is known about a file. Every field is a claim until a verdict confirms it. */
|
|
5
|
+
export interface FileInfo {
|
|
6
|
+
/**
|
|
7
|
+
* Redacted origin for logs and results: `https://host/path` (no userinfo, query or fragment;
|
|
8
|
+
* long opaque segments masked), `s3://bucket/key`, a filesystem path, or `buffer`.
|
|
9
|
+
*/
|
|
10
|
+
origin: string
|
|
11
|
+
/** A safe base name — from Content-Disposition, the URL's last segment, the key, or the caller. */
|
|
12
|
+
name?: string
|
|
13
|
+
/** Declared bytes. Never trusted as a limit: counted bytes are. */
|
|
14
|
+
size?: number
|
|
15
|
+
/** Declared media type, lower-cased, without parameters; uninformative ones are dropped. */
|
|
16
|
+
mime?: string
|
|
17
|
+
/** A strong validator (ETag, `<mtimeMs>-<size>`); range reads refuse an origin that changed. */
|
|
18
|
+
etag?: string
|
|
19
|
+
modifiedAt?: Date
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** A byte span, `end` inclusive like HTTP; no `end` means to the last byte. */
|
|
23
|
+
export interface ByteRange {
|
|
24
|
+
start: number
|
|
25
|
+
end?: number
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export interface SignalOptions {
|
|
29
|
+
/** Aborts the operation; the rejection is the signal's reason. */
|
|
30
|
+
signal?: AbortSignal
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export interface OpenOptions extends SignalOptions {
|
|
34
|
+
/** Open only this span — honoured when the origin supports ranges. */
|
|
35
|
+
range?: ByteRange
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface OpenedFile {
|
|
39
|
+
stream: Readable
|
|
40
|
+
info: FileInfo
|
|
41
|
+
/** The span actually served, when a range was asked for and honoured. */
|
|
42
|
+
range?: { start: number, end: number, total: number }
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** Random access over ONE immutable version of a file. */
|
|
46
|
+
export interface RangeReader {
|
|
47
|
+
/** Total bytes of the file. */
|
|
48
|
+
readonly size: number
|
|
49
|
+
/** Up to `length` bytes at `offset`; fewer only at the end of the file. */
|
|
50
|
+
read: (offset: number, length: number, signal?: AbortSignal) => Promise<Uint8Array>
|
|
51
|
+
/** A stream over `[start, end)` — for spans too large to buffer. */
|
|
52
|
+
stream: (start: number, end: number, signal?: AbortSignal) => Promise<Readable>
|
|
53
|
+
close: () => Promise<void>
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Where a file's bytes come from. */
|
|
57
|
+
export interface FileSource {
|
|
58
|
+
/** `http`, `s3`, `file`, `buffer`, `stream`, `archive`, `gzip`. */
|
|
59
|
+
readonly kind: string
|
|
60
|
+
/** Metadata without the body (HEAD, HeadObject, stat); cached. */
|
|
61
|
+
info: (options?: SignalOptions) => Promise<FileInfo>
|
|
62
|
+
/** A fresh body stream. A one-shot source refuses a second open with `SourceError('consumed')`. */
|
|
63
|
+
open: (options?: OpenOptions) => Promise<OpenedFile>
|
|
64
|
+
/** Random access when the origin offers it, otherwise `null` (probing then reads the head only). */
|
|
65
|
+
ranges: (options?: SignalOptions) => Promise<RangeReader | null>
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export interface SinkInput {
|
|
69
|
+
stream: Readable
|
|
70
|
+
info: FileInfo
|
|
71
|
+
/** The transfer's verdict, when one ran — sinks take the Content-Type from it. */
|
|
72
|
+
verdict?: FileVerdict
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export interface SinkWriteOptions extends SignalOptions {
|
|
76
|
+
/** Ceiling on accepted bytes; a sink may lower it. */
|
|
77
|
+
maxBytes?: number
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export interface SinkResult {
|
|
81
|
+
/** `file`, `s3`, `workload`, `buffer`. */
|
|
82
|
+
kind: string
|
|
83
|
+
/** Path, `s3://bucket/key` or workload file path. */
|
|
84
|
+
location: string
|
|
85
|
+
bytes: number
|
|
86
|
+
/** Public URL when the destination has one — for S3 the one storage-resource answers. */
|
|
87
|
+
url?: string
|
|
88
|
+
etag?: string
|
|
89
|
+
mime?: string
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** Where one file's bytes go. */
|
|
93
|
+
export interface FileSink {
|
|
94
|
+
readonly kind: string
|
|
95
|
+
/** Writes the whole stream and commits atomically; on failure nothing is left behind. */
|
|
96
|
+
write: (input: SinkInput, options?: SinkWriteOptions) => Promise<SinkResult>
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** Many files under one root — a directory, an S3 prefix, a workload folder. */
|
|
100
|
+
export interface TreeSink {
|
|
101
|
+
readonly kind: string
|
|
102
|
+
/** The sink of one entry; `path` is normalized and confined to the root, otherwise `PathError`. */
|
|
103
|
+
entry: (path: string) => FileSink
|
|
104
|
+
/** An empty directory where the store has directories; a no-op elsewhere. */
|
|
105
|
+
directory: (path: string, options?: SignalOptions) => Promise<void>
|
|
106
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export interface NameUtils {
|
|
2
|
+
/** The file name a `Content-Disposition` header carries (`filename*` per RFC 5987 first), sanitized. */
|
|
3
|
+
fromDisposition: (header: string | undefined) => string | undefined
|
|
4
|
+
/** The last path segment of a URL, decoded and sanitized. */
|
|
5
|
+
fromUrl: (url: URL) => string | undefined
|
|
6
|
+
/** The base name of an object key or a path, sanitized. */
|
|
7
|
+
fromKey: (key: string) => string | undefined
|
|
8
|
+
/** A base name safe to write: no directories, control characters, `..` or leading dots. */
|
|
9
|
+
sanitize: (name: string | undefined) => string | undefined
|
|
10
|
+
/** The lower-cased extension, keeping the compound ones (`tar.gz`). */
|
|
11
|
+
extOf: (name: string | undefined) => string | undefined
|
|
12
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { COMPOUND_EXTENSIONS, MAX_NAME_BYTES } from './names/consts.js'
|
|
2
|
+
import type { NameUtils } from './names/types.js'
|
|
3
|
+
|
|
4
|
+
export const createNameUtils = (): NameUtils => {
|
|
5
|
+
const sanitize = (name: string | undefined): string | undefined => {
|
|
6
|
+
if (name == null) {
|
|
7
|
+
return undefined
|
|
8
|
+
}
|
|
9
|
+
let base = name.replace(/\\/g, '/').split('/').pop()?.replace(/[\u0000-\u001f\u007f\u202a-\u202e\u2066-\u2069]/g, '') ?? ''
|
|
10
|
+
base = base.normalize('NFC').trim().replace(/^\.+/, '').replace(/[. ]+$/, '')
|
|
11
|
+
if (base === '' || base === '..') {
|
|
12
|
+
return undefined
|
|
13
|
+
}
|
|
14
|
+
while (Buffer.byteLength(base) > MAX_NAME_BYTES) {
|
|
15
|
+
base = base.slice(0, -1)
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
return base
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
const decode = (value: string): string => {
|
|
22
|
+
try {
|
|
23
|
+
return decodeURIComponent(value)
|
|
24
|
+
} catch {
|
|
25
|
+
return value
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
const fromDisposition = (header: string | undefined): string | undefined => {
|
|
30
|
+
if (header == null || header === '') {
|
|
31
|
+
return undefined
|
|
32
|
+
}
|
|
33
|
+
const extended = /filename\*\s*=\s*([^']*)'[^']*'([^;]+)/i.exec(header)
|
|
34
|
+
if (extended != null) {
|
|
35
|
+
return sanitize(decode(extended[2].trim().replace(/^"|"$/g, '')))
|
|
36
|
+
}
|
|
37
|
+
const plain = /filename\s*=\s*("(?:[^"\\]|\\.)*"|[^;]+)/i.exec(header)
|
|
38
|
+
if (plain != null) {
|
|
39
|
+
const value = plain[1].trim()
|
|
40
|
+
return sanitize(value.startsWith('"') ? value.slice(1, -1).replace(/\\(.)/g, '$1') : value)
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
return undefined
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const fromUrl = (url: URL): string | undefined => {
|
|
47
|
+
const segment = url.pathname.split('/').filter(part => part !== '').pop()
|
|
48
|
+
return segment != null ? sanitize(decode(segment)) : undefined
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const fromKey = (key: string): string | undefined => sanitize(key.split('/').filter(part => part !== '').pop())
|
|
52
|
+
|
|
53
|
+
const extOf = (name: string | undefined): string | undefined => {
|
|
54
|
+
if (name == null) {
|
|
55
|
+
return undefined
|
|
56
|
+
}
|
|
57
|
+
const lower = name.toLowerCase()
|
|
58
|
+
const compound = COMPOUND_EXTENSIONS.find(ext => lower.endsWith(`.${ext}`))
|
|
59
|
+
if (compound != null) {
|
|
60
|
+
return compound
|
|
61
|
+
}
|
|
62
|
+
const dot = lower.lastIndexOf('.')
|
|
63
|
+
|
|
64
|
+
return dot > 0 && dot < lower.length - 1 ? lower.slice(dot + 1) : undefined
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
return { fromDisposition, fromUrl, fromKey, sanitize, extOf }
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export const nameUtils = createNameUtils()
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
export interface PathLimits {
|
|
2
|
+
/** Longest relative path, in UTF-8 bytes. */
|
|
3
|
+
maxPathBytes: number
|
|
4
|
+
/** Deepest relative path, in segments. */
|
|
5
|
+
maxPathSegments: number
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
export interface PathUtils {
|
|
9
|
+
/**
|
|
10
|
+
* A relative entry path normalized (`\` → `/`, NFC, `.` and empty segments dropped), or a
|
|
11
|
+
* `PathError`: `invalid-char`, `absolute`, `traversal`, `empty`, `too-long`, `too-deep`.
|
|
12
|
+
*/
|
|
13
|
+
entry: (raw: string, limits?: Partial<PathLimits>) => string
|
|
14
|
+
/** The absolute path of a normalized relative path under `root`; refuses an escape. */
|
|
15
|
+
join: (root: string, relative: string) => string
|
|
16
|
+
/** Refuses (`PathError('symlink')`) when any existing ancestor of `target` below `root` is a link. */
|
|
17
|
+
assertNoLinks: (root: string, target: string) => Promise<void>
|
|
18
|
+
/** An object key: `prefix` and a normalized relative path joined by `/`. */
|
|
19
|
+
key: (prefix: string, relative: string) => string
|
|
20
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { lstat } from 'node:fs/promises'
|
|
2
|
+
import { isAbsolute, relative as relativeOf, resolve, sep } from 'node:path'
|
|
3
|
+
import { PathError } from '../errors.js'
|
|
4
|
+
import { DEFAULT_PATH_LIMITS } from './paths/consts.js'
|
|
5
|
+
import type { PathLimits, PathUtils } from './paths/types.js'
|
|
6
|
+
|
|
7
|
+
export const createPathUtils = (): PathUtils => {
|
|
8
|
+
const forbidden = /[\u0000-\u001f\u007f\u202a-\u202e\u2066-\u2069]/
|
|
9
|
+
|
|
10
|
+
const entry = (raw: string, limits: Partial<PathLimits> = {}): string => {
|
|
11
|
+
const { maxPathBytes, maxPathSegments } = { ...DEFAULT_PATH_LIMITS, ...limits }
|
|
12
|
+
const path = raw.normalize('NFC').replace(/\\/g, '/')
|
|
13
|
+
if (forbidden.test(path)) {
|
|
14
|
+
throw new PathError('invalid-char')
|
|
15
|
+
}
|
|
16
|
+
if (path.startsWith('/') || /^[A-Za-z]:/.test(path)) {
|
|
17
|
+
throw new PathError('absolute')
|
|
18
|
+
}
|
|
19
|
+
const segments = path.split('/').filter(segment => segment !== '' && segment !== '.')
|
|
20
|
+
if (segments.includes('..')) {
|
|
21
|
+
throw new PathError('traversal')
|
|
22
|
+
}
|
|
23
|
+
if (segments.length === 0) {
|
|
24
|
+
throw new PathError('empty')
|
|
25
|
+
}
|
|
26
|
+
const normalized = segments.join('/')
|
|
27
|
+
if (Buffer.byteLength(normalized) > maxPathBytes) {
|
|
28
|
+
throw new PathError('too-long')
|
|
29
|
+
}
|
|
30
|
+
if (segments.length > maxPathSegments) {
|
|
31
|
+
throw new PathError('too-deep')
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
return normalized
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const join = (root: string, relative: string): string => {
|
|
38
|
+
const base = resolve(root)
|
|
39
|
+
const target = resolve(base, relative)
|
|
40
|
+
const inside = relativeOf(base, target)
|
|
41
|
+
if (inside === '' || inside.startsWith('..') || isAbsolute(inside)) {
|
|
42
|
+
throw new PathError('traversal')
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
return target
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const assertNoLinks = async (root: string, target: string): Promise<void> => {
|
|
49
|
+
const base = resolve(root)
|
|
50
|
+
const parts = relativeOf(base, resolve(target)).split(sep).filter(part => part !== '')
|
|
51
|
+
let current = base
|
|
52
|
+
for (const part of parts) {
|
|
53
|
+
current = `${current}${sep}${part}`
|
|
54
|
+
try {
|
|
55
|
+
if ((await lstat(current)).isSymbolicLink()) {
|
|
56
|
+
throw new PathError('symlink')
|
|
57
|
+
}
|
|
58
|
+
} catch (error) {
|
|
59
|
+
if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
|
|
60
|
+
return
|
|
61
|
+
}
|
|
62
|
+
throw error
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const key = (prefix: string, relative: string): string => {
|
|
68
|
+
const clean = prefix.replace(/\/+$/, '')
|
|
69
|
+
return clean === '' ? entry(relative) : `${clean}/${entry(relative)}`
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
return { entry, join, assertNoLinks, key }
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export const pathUtils = createPathUtils()
|