@owlmeans/file-streams 0.0.0-stage → 0.2.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +71 -2
- package/agent-meta/manifest.json +64 -0
- package/agent-meta/skills/file-streams/SKILL.md +183 -0
- package/agent-meta/skills/file-streams/parts/network.md +47 -0
- package/agent-meta/skills/file-streams/parts/workspaces.md +129 -0
- package/build/archive/consts.d.ts +16 -0
- package/build/archive/consts.d.ts.map +1 -0
- package/build/archive/consts.js +24 -0
- package/build/archive/consts.js.map +1 -0
- package/build/archive/formats.d.ts +4 -0
- package/build/archive/formats.d.ts.map +1 -0
- package/build/archive/formats.js +53 -0
- package/build/archive/formats.js.map +1 -0
- package/build/archive/guard.d.ts +5 -0
- package/build/archive/guard.d.ts.map +1 -0
- package/build/archive/guard.js +62 -0
- package/build/archive/guard.js.map +1 -0
- package/build/archive/ratio.d.ts +5 -0
- package/build/archive/ratio.d.ts.map +1 -0
- package/build/archive/ratio.js +30 -0
- package/build/archive/ratio.js.map +1 -0
- package/build/archive/tar.d.ts +4 -0
- package/build/archive/tar.d.ts.map +1 -0
- package/build/archive/tar.js +142 -0
- package/build/archive/tar.js.map +1 -0
- package/build/archive/types.d.ts +95 -0
- package/build/archive/types.d.ts.map +1 -0
- package/build/archive/types.js +2 -0
- package/build/archive/types.js.map +1 -0
- package/build/archive/types.local.d.ts +64 -0
- package/build/archive/types.local.d.ts.map +1 -0
- package/build/archive/types.local.js +2 -0
- package/build/archive/types.local.js.map +1 -0
- package/build/archive/zip.d.ts +4 -0
- package/build/archive/zip.d.ts.map +1 -0
- package/build/archive/zip.js +135 -0
- package/build/archive/zip.js.map +1 -0
- package/build/archive.d.ts +4 -0
- package/build/archive.d.ts.map +1 -0
- package/build/archive.js +265 -0
- package/build/archive.js.map +1 -0
- package/build/backed-workspace/types.d.ts +67 -0
- package/build/backed-workspace/types.d.ts.map +1 -0
- package/build/backed-workspace/types.js +2 -0
- package/build/backed-workspace/types.js.map +1 -0
- package/build/backed-workspace.d.ts +4 -0
- package/build/backed-workspace.d.ts.map +1 -0
- package/build/backed-workspace.js +63 -0
- package/build/backed-workspace.js.map +1 -0
- package/build/consts.d.ts +3 -0
- package/build/consts.d.ts.map +1 -0
- package/build/consts.js +3 -0
- package/build/consts.js.map +1 -0
- package/build/consts.local.d.ts +3 -0
- package/build/consts.local.d.ts.map +1 -0
- package/build/consts.local.js +3 -0
- package/build/consts.local.js.map +1 -0
- package/build/errors.d.ts +59 -0
- package/build/errors.d.ts.map +1 -0
- package/build/errors.js +105 -0
- package/build/errors.js.map +1 -0
- package/build/http/connector.d.ts +11 -0
- package/build/http/connector.d.ts.map +1 -0
- package/build/http/connector.js +124 -0
- package/build/http/connector.js.map +1 -0
- package/build/http/consts.d.ts +9 -0
- package/build/http/consts.d.ts.map +1 -0
- package/build/http/consts.js +8 -0
- package/build/http/consts.js.map +1 -0
- package/build/http/source.d.ts +5 -0
- package/build/http/source.d.ts.map +1 -0
- package/build/http/source.js +76 -0
- package/build/http/source.js.map +1 -0
- package/build/http/types.d.ts +67 -0
- package/build/http/types.d.ts.map +1 -0
- package/build/http/types.js +2 -0
- package/build/http/types.js.map +1 -0
- package/build/http/types.local.d.ts +21 -0
- package/build/http/types.local.d.ts.map +1 -0
- package/build/http/types.local.js +2 -0
- package/build/http/types.local.js.map +1 -0
- package/build/http.d.ts +4 -0
- package/build/http.d.ts.map +1 -0
- package/build/http.js +248 -0
- package/build/http.js.map +1 -0
- package/build/index.d.ts +53 -0
- package/build/index.d.ts.map +1 -0
- package/build/index.js +33 -0
- package/build/index.js.map +1 -0
- package/build/network/consts.d.ts +12 -0
- package/build/network/consts.d.ts.map +1 -0
- package/build/network/consts.js +20 -0
- package/build/network/consts.js.map +1 -0
- package/build/network/types.d.ts +43 -0
- package/build/network/types.d.ts.map +1 -0
- package/build/network/types.js +2 -0
- package/build/network/types.js.map +1 -0
- package/build/network.d.ts +4 -0
- package/build/network.d.ts.map +1 -0
- package/build/network.js +165 -0
- package/build/network.js.map +1 -0
- package/build/proxy/consts.d.ts +7 -0
- package/build/proxy/consts.d.ts.map +1 -0
- package/build/proxy/consts.js +9 -0
- package/build/proxy/consts.js.map +1 -0
- package/build/proxy/endpoint.d.ts +4 -0
- package/build/proxy/endpoint.d.ts.map +1 -0
- package/build/proxy/endpoint.js +27 -0
- package/build/proxy/endpoint.js.map +1 -0
- package/build/proxy/http.d.ts +5 -0
- package/build/proxy/http.d.ts.map +1 -0
- package/build/proxy/http.js +95 -0
- package/build/proxy/http.js.map +1 -0
- package/build/proxy/router.d.ts +4 -0
- package/build/proxy/router.d.ts.map +1 -0
- package/build/proxy/router.js +85 -0
- package/build/proxy/router.js.map +1 -0
- package/build/proxy/socks.d.ts +5 -0
- package/build/proxy/socks.d.ts.map +1 -0
- package/build/proxy/socks.js +44 -0
- package/build/proxy/socks.js.map +1 -0
- package/build/proxy/types.d.ts +99 -0
- package/build/proxy/types.d.ts.map +1 -0
- package/build/proxy/types.js +2 -0
- package/build/proxy/types.js.map +1 -0
- package/build/resolvers/google-drive/consts.d.ts +10 -0
- package/build/resolvers/google-drive/consts.d.ts.map +1 -0
- package/build/resolvers/google-drive/consts.js +16 -0
- package/build/resolvers/google-drive/consts.js.map +1 -0
- package/build/resolvers/google-drive/types.d.ts +21 -0
- package/build/resolvers/google-drive/types.d.ts.map +1 -0
- package/build/resolvers/google-drive/types.js +2 -0
- package/build/resolvers/google-drive/types.js.map +1 -0
- package/build/resolvers/google-drive.d.ts +10 -0
- package/build/resolvers/google-drive.d.ts.map +1 -0
- package/build/resolvers/google-drive.js +130 -0
- package/build/resolvers/google-drive.js.map +1 -0
- package/build/resolvers/types.d.ts +39 -0
- package/build/resolvers/types.d.ts.map +1 -0
- package/build/resolvers/types.js +2 -0
- package/build/resolvers/types.js.map +1 -0
- package/build/s3/consts.d.ts +9 -0
- package/build/s3/consts.d.ts.map +1 -0
- package/build/s3/consts.js +9 -0
- package/build/s3/consts.js.map +1 -0
- package/build/s3/prefix.d.ts +4 -0
- package/build/s3/prefix.d.ts.map +1 -0
- package/build/s3/prefix.js +87 -0
- package/build/s3/prefix.js.map +1 -0
- package/build/s3/sink.d.ts +6 -0
- package/build/s3/sink.d.ts.map +1 -0
- package/build/s3/sink.js +30 -0
- package/build/s3/sink.js.map +1 -0
- package/build/s3/source.d.ts +5 -0
- package/build/s3/source.d.ts.map +1 -0
- package/build/s3/source.js +81 -0
- package/build/s3/source.js.map +1 -0
- package/build/s3/types.d.ts +66 -0
- package/build/s3/types.d.ts.map +1 -0
- package/build/s3/types.js +2 -0
- package/build/s3/types.js.map +1 -0
- package/build/s3/types.local.d.ts +43 -0
- package/build/s3/types.local.d.ts.map +1 -0
- package/build/s3/types.local.js +2 -0
- package/build/s3/types.local.js.map +1 -0
- package/build/s3/upload.d.ts +4 -0
- package/build/s3/upload.d.ts.map +1 -0
- package/build/s3/upload.js +118 -0
- package/build/s3/upload.js.map +1 -0
- package/build/s3.d.ts +5 -0
- package/build/s3.d.ts.map +1 -0
- package/build/s3.js +68 -0
- package/build/s3.js.map +1 -0
- package/build/service/types.d.ts +66 -0
- package/build/service/types.d.ts.map +1 -0
- package/build/service/types.js +2 -0
- package/build/service/types.js.map +1 -0
- package/build/service/types.local.d.ts +11 -0
- package/build/service/types.local.d.ts.map +1 -0
- package/build/service/types.local.js +2 -0
- package/build/service/types.local.js.map +1 -0
- package/build/service.d.ts +6 -0
- package/build/service.d.ts.map +1 -0
- package/build/service.js +73 -0
- package/build/service.js.map +1 -0
- package/build/sinks/buffer/consts.d.ts +2 -0
- package/build/sinks/buffer/consts.d.ts.map +1 -0
- package/build/sinks/buffer/consts.js +2 -0
- package/build/sinks/buffer/consts.js.map +1 -0
- package/build/sinks/buffer/types.d.ts +11 -0
- package/build/sinks/buffer/types.d.ts.map +1 -0
- package/build/sinks/buffer/types.js +2 -0
- package/build/sinks/buffer/types.js.map +1 -0
- package/build/sinks/buffer.d.ts +4 -0
- package/build/sinks/buffer.d.ts.map +1 -0
- package/build/sinks/buffer.js +13 -0
- package/build/sinks/buffer.js.map +1 -0
- package/build/sinks/directory/types.d.ts +7 -0
- package/build/sinks/directory/types.d.ts.map +1 -0
- package/build/sinks/directory/types.js +2 -0
- package/build/sinks/directory/types.js.map +1 -0
- package/build/sinks/directory.d.ts +8 -0
- package/build/sinks/directory.d.ts.map +1 -0
- package/build/sinks/directory.js +43 -0
- package/build/sinks/directory.js.map +1 -0
- package/build/sinks/file.d.ts +5 -0
- package/build/sinks/file.d.ts.map +1 -0
- package/build/sinks/file.js +46 -0
- package/build/sinks/file.js.map +1 -0
- package/build/sinks/types.d.ts +11 -0
- package/build/sinks/types.d.ts.map +1 -0
- package/build/sinks/types.js +2 -0
- package/build/sinks/types.js.map +1 -0
- package/build/sinks/workload/consts.d.ts +5 -0
- package/build/sinks/workload/consts.d.ts.map +1 -0
- package/build/sinks/workload/consts.js +5 -0
- package/build/sinks/workload/consts.js.map +1 -0
- package/build/sinks/workload/types.d.ts +17 -0
- package/build/sinks/workload/types.d.ts.map +1 -0
- package/build/sinks/workload/types.js +2 -0
- package/build/sinks/workload/types.js.map +1 -0
- package/build/sinks/workload.d.ts +8 -0
- package/build/sinks/workload.d.ts.map +1 -0
- package/build/sinks/workload.js +51 -0
- package/build/sinks/workload.js.map +1 -0
- package/build/sources/buffer.d.ts +5 -0
- package/build/sources/buffer.d.ts.map +1 -0
- package/build/sources/buffer.js +20 -0
- package/build/sources/buffer.js.map +1 -0
- package/build/sources/file.d.ts +5 -0
- package/build/sources/file.d.ts.map +1 -0
- package/build/sources/file.js +50 -0
- package/build/sources/file.js.map +1 -0
- package/build/sources/stream.d.ts +5 -0
- package/build/sources/stream.d.ts.map +1 -0
- package/build/sources/stream.js +19 -0
- package/build/sources/stream.js.map +1 -0
- package/build/sources/types.d.ts +10 -0
- package/build/sources/types.d.ts.map +1 -0
- package/build/sources/types.js +2 -0
- package/build/sources/types.js.map +1 -0
- package/build/sources/workload/types.d.ts +29 -0
- package/build/sources/workload/types.d.ts.map +1 -0
- package/build/sources/workload/types.js +2 -0
- package/build/sources/workload/types.js.map +1 -0
- package/build/sources/workload.d.ts +7 -0
- package/build/sources/workload.d.ts.map +1 -0
- package/build/sources/workload.js +67 -0
- package/build/sources/workload.js.map +1 -0
- package/build/transfer/consts.d.ts +5 -0
- package/build/transfer/consts.d.ts.map +1 -0
- package/build/transfer/consts.js +8 -0
- package/build/transfer/consts.js.map +1 -0
- package/build/transfer/types.d.ts +49 -0
- package/build/transfer/types.d.ts.map +1 -0
- package/build/transfer/types.js +2 -0
- package/build/transfer/types.js.map +1 -0
- package/build/transfer.d.ts +4 -0
- package/build/transfer.d.ts.map +1 -0
- package/build/transfer.js +75 -0
- package/build/transfer.js.map +1 -0
- package/build/types.d.ts +99 -0
- package/build/types.d.ts.map +1 -0
- package/build/types.js +2 -0
- package/build/types.js.map +1 -0
- package/build/utils/names/consts.d.ts +5 -0
- package/build/utils/names/consts.d.ts.map +1 -0
- package/build/utils/names/consts.js +5 -0
- package/build/utils/names/consts.js.map +1 -0
- package/build/utils/names/types.d.ts +13 -0
- package/build/utils/names/types.d.ts.map +1 -0
- package/build/utils/names/types.js +2 -0
- package/build/utils/names/types.js.map +1 -0
- package/build/utils/names.d.ts +4 -0
- package/build/utils/names.d.ts.map +1 -0
- package/build/utils/names.js +60 -0
- package/build/utils/names.js.map +1 -0
- package/build/utils/paths/consts.d.ts +3 -0
- package/build/utils/paths/consts.d.ts.map +1 -0
- package/build/utils/paths/consts.js +2 -0
- package/build/utils/paths/consts.js.map +1 -0
- package/build/utils/paths/types.d.ts +20 -0
- package/build/utils/paths/types.d.ts.map +1 -0
- package/build/utils/paths/types.js +2 -0
- package/build/utils/paths/types.js.map +1 -0
- package/build/utils/paths.d.ts +4 -0
- package/build/utils/paths.d.ts.map +1 -0
- package/build/utils/paths.js +67 -0
- package/build/utils/paths.js.map +1 -0
- package/build/utils/range/types.d.ts +11 -0
- package/build/utils/range/types.d.ts.map +1 -0
- package/build/utils/range/types.js +2 -0
- package/build/utils/range/types.js.map +1 -0
- package/build/utils/range.d.ts +4 -0
- package/build/utils/range.d.ts.map +1 -0
- package/build/utils/range.js +64 -0
- package/build/utils/range.js.map +1 -0
- package/build/utils/redact/types.d.ts +16 -0
- package/build/utils/redact/types.d.ts.map +1 -0
- package/build/utils/redact/types.js +2 -0
- package/build/utils/redact/types.js.map +1 -0
- package/build/utils/redact.d.ts +4 -0
- package/build/utils/redact.d.ts.map +1 -0
- package/build/utils/redact.js +18 -0
- package/build/utils/redact.js.map +1 -0
- package/build/utils/signals/types.d.ts +17 -0
- package/build/utils/signals/types.d.ts.map +1 -0
- package/build/utils/signals/types.js +2 -0
- package/build/utils/signals/types.js.map +1 -0
- package/build/utils/signals.d.ts +4 -0
- package/build/utils/signals.d.ts.map +1 -0
- package/build/utils/signals.js +52 -0
- package/build/utils/signals.js.map +1 -0
- package/build/utils/streams/consts.d.ts +3 -0
- package/build/utils/streams/consts.d.ts.map +1 -0
- package/build/utils/streams/consts.js +3 -0
- package/build/utils/streams/consts.js.map +1 -0
- package/build/utils/streams/types.d.ts +59 -0
- package/build/utils/streams/types.d.ts.map +1 -0
- package/build/utils/streams/types.js +2 -0
- package/build/utils/streams/types.js.map +1 -0
- package/build/utils/streams.d.ts +4 -0
- package/build/utils/streams.d.ts.map +1 -0
- package/build/utils/streams.js +167 -0
- package/build/utils/streams.js.map +1 -0
- package/build/utils/tree/types.d.ts +6 -0
- package/build/utils/tree/types.d.ts.map +1 -0
- package/build/utils/tree/types.js +2 -0
- package/build/utils/tree/types.js.map +1 -0
- package/build/utils/tree.d.ts +4 -0
- package/build/utils/tree.d.ts.map +1 -0
- package/build/utils/tree.js +14 -0
- package/build/utils/tree.js.map +1 -0
- package/build/workspace/consts.d.ts +12 -0
- package/build/workspace/consts.d.ts.map +1 -0
- package/build/workspace/consts.js +12 -0
- package/build/workspace/consts.js.map +1 -0
- package/build/workspace/consts.local.d.ts +3 -0
- package/build/workspace/consts.local.d.ts.map +1 -0
- package/build/workspace/consts.local.js +3 -0
- package/build/workspace/consts.local.js.map +1 -0
- package/build/workspace/fs.d.ts +4 -0
- package/build/workspace/fs.d.ts.map +1 -0
- package/build/workspace/fs.js +45 -0
- package/build/workspace/fs.js.map +1 -0
- package/build/workspace/scope.d.ts +6 -0
- package/build/workspace/scope.d.ts.map +1 -0
- package/build/workspace/scope.js +180 -0
- package/build/workspace/scope.js.map +1 -0
- package/build/workspace/types.d.ts +66 -0
- package/build/workspace/types.d.ts.map +1 -0
- package/build/workspace/types.js +2 -0
- package/build/workspace/types.js.map +1 -0
- package/build/workspace/types.local.d.ts +20 -0
- package/build/workspace/types.local.d.ts.map +1 -0
- package/build/workspace/types.local.js +2 -0
- package/build/workspace/types.local.js.map +1 -0
- package/build/workspace.d.ts +7 -0
- package/build/workspace.d.ts.map +1 -0
- package/build/workspace.js +87 -0
- package/build/workspace.js.map +1 -0
- package/package.json +61 -4
- package/src/archive/consts.ts +28 -0
- package/src/archive/formats.ts +60 -0
- package/src/archive/guard.ts +68 -0
- package/src/archive/ratio.ts +35 -0
- package/src/archive/tar.ts +147 -0
- package/src/archive/types.local.ts +72 -0
- package/src/archive/types.ts +107 -0
- package/src/archive/zip.ts +142 -0
- package/src/archive.ts +298 -0
- package/src/backed-workspace/types.ts +72 -0
- package/src/backed-workspace.ts +85 -0
- package/src/consts.local.ts +2 -0
- package/src/consts.ts +2 -0
- package/src/errors.ts +128 -0
- package/src/http/connector.ts +136 -0
- package/src/http/consts.ts +13 -0
- package/src/http/source.ts +89 -0
- package/src/http/types.local.ts +23 -0
- package/src/http/types.ts +74 -0
- package/src/http.ts +276 -0
- package/src/index.ts +65 -0
- package/src/network/consts.ts +22 -0
- package/src/network/types.ts +45 -0
- package/src/network.ts +179 -0
- package/src/proxy/consts.ts +11 -0
- package/src/proxy/endpoint.ts +28 -0
- package/src/proxy/http.ts +108 -0
- package/src/proxy/router.ts +93 -0
- package/src/proxy/socks.ts +48 -0
- package/src/proxy/types.ts +114 -0
- package/src/resolvers/google-drive/consts.ts +22 -0
- package/src/resolvers/google-drive/types.ts +22 -0
- package/src/resolvers/google-drive.ts +153 -0
- package/src/resolvers/types.ts +45 -0
- package/src/s3/consts.ts +12 -0
- package/src/s3/prefix.ts +100 -0
- package/src/s3/sink.ts +36 -0
- package/src/s3/source.ts +90 -0
- package/src/s3/types.local.ts +49 -0
- package/src/s3/types.ts +70 -0
- package/src/s3/upload.ts +130 -0
- package/src/s3.ts +83 -0
- package/src/service/types.local.ts +9 -0
- package/src/service/types.ts +71 -0
- package/src/service.ts +97 -0
- package/src/sinks/buffer/consts.ts +1 -0
- package/src/sinks/buffer/types.ts +12 -0
- package/src/sinks/buffer.ts +18 -0
- package/src/sinks/directory/types.ts +7 -0
- package/src/sinks/directory.ts +49 -0
- package/src/sinks/file.ts +52 -0
- package/src/sinks/types.ts +12 -0
- package/src/sinks/workload/consts.ts +5 -0
- package/src/sinks/workload/types.ts +17 -0
- package/src/sinks/workload.ts +59 -0
- package/src/sources/buffer.ts +27 -0
- package/src/sources/file.ts +57 -0
- package/src/sources/stream.ts +26 -0
- package/src/sources/types.ts +12 -0
- package/src/sources/workload/types.ts +31 -0
- package/src/sources/workload.ts +79 -0
- package/src/transfer/consts.ts +10 -0
- package/src/transfer/types.ts +55 -0
- package/src/transfer.ts +89 -0
- package/src/types.ts +106 -0
- package/src/utils/names/consts.ts +5 -0
- package/src/utils/names/types.ts +12 -0
- package/src/utils/names.ts +70 -0
- package/src/utils/paths/consts.ts +3 -0
- package/src/utils/paths/types.ts +20 -0
- package/src/utils/paths.ts +75 -0
- package/src/utils/range/types.ts +11 -0
- package/src/utils/range.ts +80 -0
- package/src/utils/redact/types.ts +16 -0
- package/src/utils/redact.ts +23 -0
- package/src/utils/signals/types.ts +17 -0
- package/src/utils/signals.ts +63 -0
- package/src/utils/streams/consts.ts +2 -0
- package/src/utils/streams/types.ts +65 -0
- package/src/utils/streams.ts +188 -0
- package/src/utils/tree/types.ts +6 -0
- package/src/utils/tree.ts +18 -0
- package/src/workspace/consts.local.ts +2 -0
- package/src/workspace/consts.ts +14 -0
- package/src/workspace/fs.ts +48 -0
- package/src/workspace/scope.ts +196 -0
- package/src/workspace/types.local.ts +21 -0
- package/src/workspace/types.ts +71 -0
- package/src/workspace.ts +102 -0
- package/tests/archive.spec.ts +134 -0
- package/tests/backed.spec.ts +147 -0
- package/tests/context.ts +114 -0
- package/tests/fixtures.ts +190 -0
- package/tests/google-drive.spec.ts +76 -0
- package/tests/harness/drive.ts +92 -0
- package/tests/harness/http-proxy.ts +96 -0
- package/tests/harness/origin.ts +134 -0
- package/tests/harness/socks5.ts +135 -0
- package/tests/harness/workload.ts +118 -0
- package/tests/http.spec.ts +101 -0
- package/tests/inspector.spec.ts +40 -0
- package/tests/live.spec.ts +27 -0
- package/tests/network.spec.ts +79 -0
- package/tests/proxy.spec.ts +149 -0
- package/tests/s3.spec.ts +133 -0
- package/tests/service.spec.ts +77 -0
- package/tests/sources-sinks.spec.ts +96 -0
- package/tests/transfer.spec.ts +99 -0
- package/tests/tsconfig.json +12 -0
- package/tests/workload.spec.ts +140 -0
- package/tests/workspace.spec.ts +158 -0
- package/tsconfig.json +16 -0
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
import { lstat, mkdir, readdir, rm, writeFile } from 'node:fs/promises'
|
|
2
|
+
import { dirname, join } from 'node:path'
|
|
3
|
+
import { fileScopeHelper } from '@owlmeans/storage-common'
|
|
4
|
+
import type { FileScope } from '@owlmeans/storage-common'
|
|
5
|
+
import { PathError } from '../errors.js'
|
|
6
|
+
import { createDirectorySink } from '../sinks/directory.js'
|
|
7
|
+
import { createFileSource } from '../sources/file.js'
|
|
8
|
+
import type { SourceClaims } from '../sources/types.js'
|
|
9
|
+
import type { FileSink, FileSource, SinkInput, TreeSink } from '../types.js'
|
|
10
|
+
import { pathUtils } from '../utils/paths.js'
|
|
11
|
+
import { streamUtils } from '../utils/streams.js'
|
|
12
|
+
import { IN_FLIGHT_FILE } from './consts.local.js'
|
|
13
|
+
import { WORKSPACE_SCOPE_MARKER } from './consts.js'
|
|
14
|
+
import { workspaceFs } from './fs.js'
|
|
15
|
+
import type { WorkspaceFile, WorkspaceScope, WorkspaceWriteOptions } from './types.js'
|
|
16
|
+
import type { WorkspaceCore, WorkspaceTarget } from './types.local.js'
|
|
17
|
+
|
|
18
|
+
/** The folder of one scope; the marker file is written with its first write. */
|
|
19
|
+
export const makeWorkspaceScope = (core: WorkspaceCore, scope: FileScope): WorkspaceScope => {
|
|
20
|
+
const dir = join(core.root, fileScopeHelper.pathOf(scope))
|
|
21
|
+
const marker = join(dir, WORKSPACE_SCOPE_MARKER)
|
|
22
|
+
const frozen: FileScope = Object.freeze({ ...scope })
|
|
23
|
+
|
|
24
|
+
const target = (path: string): WorkspaceTarget => {
|
|
25
|
+
const relative = pathUtils.entry(path)
|
|
26
|
+
if (relative.split('/').includes(WORKSPACE_SCOPE_MARKER)) {
|
|
27
|
+
throw new PathError('reserved')
|
|
28
|
+
}
|
|
29
|
+
return { relative, absolute: pathUtils.join(dir, relative) }
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
// `<organization>/<purpose>` would contain `<organization>/<project>/…` when a project segment
|
|
33
|
+
// equals a purpose; whichever scope is on disk first keeps the folder.
|
|
34
|
+
const assertNoOverlap = async (): Promise<void> => {
|
|
35
|
+
const parent = dirname(dir)
|
|
36
|
+
if (scope.project != null && await workspaceFs.isFile(join(parent, WORKSPACE_SCOPE_MARKER))) {
|
|
37
|
+
throw new PathError('scope-overlap')
|
|
38
|
+
}
|
|
39
|
+
if (scope.project == null) {
|
|
40
|
+
for (const child of await workspaceFs.dirs(dir)) {
|
|
41
|
+
if (await workspaceFs.isFile(join(dir, child, WORKSPACE_SCOPE_MARKER))) {
|
|
42
|
+
throw new PathError('scope-overlap')
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const ensure = async (): Promise<void> => {
|
|
49
|
+
await pathUtils.assertNoLinks(core.root, dir)
|
|
50
|
+
if (await workspaceFs.isFile(marker)) {
|
|
51
|
+
return
|
|
52
|
+
}
|
|
53
|
+
await assertNoOverlap()
|
|
54
|
+
// A concurrent drop may remove an empty parent between the two calls: retry once.
|
|
55
|
+
for (let attempt = 0; ; attempt++) {
|
|
56
|
+
try {
|
|
57
|
+
await mkdir(dir, { recursive: true, mode: core.dirMode })
|
|
58
|
+
await pathUtils.assertNoLinks(core.root, dir)
|
|
59
|
+
await writeFile(marker, `${JSON.stringify(frozen)}\n`, { flag: 'wx', mode: core.fileMode })
|
|
60
|
+
return
|
|
61
|
+
} catch (error) {
|
|
62
|
+
const code = (error as NodeJS.ErrnoException).code
|
|
63
|
+
if (code === 'EEXIST') {
|
|
64
|
+
return
|
|
65
|
+
}
|
|
66
|
+
if (code !== 'ENOENT' || attempt > 0) {
|
|
67
|
+
throw error
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const prepare = async ({ absolute }: WorkspaceTarget, input?: SinkInput): Promise<void> => {
|
|
74
|
+
try {
|
|
75
|
+
await ensure()
|
|
76
|
+
await pathUtils.assertNoLinks(core.root, absolute)
|
|
77
|
+
await mkdir(dirname(absolute), { recursive: true, mode: core.dirMode })
|
|
78
|
+
await pathUtils.assertNoLinks(core.root, absolute)
|
|
79
|
+
} catch (error) {
|
|
80
|
+
if (input != null) {
|
|
81
|
+
streamUtils.discard(input.stream)
|
|
82
|
+
}
|
|
83
|
+
throw error
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const sinkOver = (tree: TreeSink, path: string): FileSink => {
|
|
88
|
+
const resolved = target(path)
|
|
89
|
+
const sink = tree.entry(resolved.relative)
|
|
90
|
+
return {
|
|
91
|
+
kind: sink.kind,
|
|
92
|
+
write: async (input, writeOptions) => {
|
|
93
|
+
await prepare(resolved, input)
|
|
94
|
+
return await sink.write(input, writeOptions)
|
|
95
|
+
},
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const directoryOf = (options: WorkspaceWriteOptions): TreeSink =>
|
|
100
|
+
createDirectorySink(dir, { onConflict: options.onConflict, dirMode: core.dirMode, mode: core.fileMode })
|
|
101
|
+
|
|
102
|
+
const tree = (options: WorkspaceWriteOptions = {}): TreeSink => {
|
|
103
|
+
const inner = directoryOf(options)
|
|
104
|
+
return {
|
|
105
|
+
kind: inner.kind,
|
|
106
|
+
entry: path => sinkOver(inner, path),
|
|
107
|
+
directory: async (path, signalOptions) => {
|
|
108
|
+
const resolved = target(path)
|
|
109
|
+
await prepare(resolved)
|
|
110
|
+
await inner.directory(resolved.relative, signalOptions)
|
|
111
|
+
},
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
const entry = (path: string, options: WorkspaceWriteOptions = {}): FileSink => sinkOver(directoryOf(options), path)
|
|
116
|
+
|
|
117
|
+
const source = (path: string, claims: SourceClaims = {}): FileSource => {
|
|
118
|
+
const resolved = target(path)
|
|
119
|
+
const file = createFileSource(resolved.absolute, claims)
|
|
120
|
+
const guard = async (): Promise<void> => { await pathUtils.assertNoLinks(core.root, resolved.absolute) }
|
|
121
|
+
return {
|
|
122
|
+
kind: file.kind,
|
|
123
|
+
info: async options => { await guard(); return await file.info(options) },
|
|
124
|
+
open: async options => { await guard(); return await file.open(options) },
|
|
125
|
+
ranges: async options => { await guard(); return await file.ranges(options) },
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
const list = async (): Promise<WorkspaceFile[]> => {
|
|
130
|
+
await pathUtils.assertNoLinks(core.root, dir)
|
|
131
|
+
const files: WorkspaceFile[] = []
|
|
132
|
+
const walk = async (absolute: string, relative: string): Promise<void> => {
|
|
133
|
+
let items
|
|
134
|
+
try {
|
|
135
|
+
items = await readdir(absolute, { withFileTypes: true })
|
|
136
|
+
} catch (error) {
|
|
137
|
+
if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
|
|
138
|
+
return
|
|
139
|
+
}
|
|
140
|
+
throw error
|
|
141
|
+
}
|
|
142
|
+
for (const item of items) {
|
|
143
|
+
const path = relative === '' ? item.name : `${relative}/${item.name}`
|
|
144
|
+
if (item.name === WORKSPACE_SCOPE_MARKER || item.isSymbolicLink()) {
|
|
145
|
+
continue
|
|
146
|
+
}
|
|
147
|
+
if (item.isDirectory()) {
|
|
148
|
+
await walk(join(absolute, item.name), path)
|
|
149
|
+
} else if (item.isFile() && !IN_FLIGHT_FILE.test(item.name)) {
|
|
150
|
+
try {
|
|
151
|
+
const stats = await lstat(join(absolute, item.name))
|
|
152
|
+
files.push({ path, size: stats.size, modifiedAt: stats.mtime })
|
|
153
|
+
} catch (error) {
|
|
154
|
+
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
|
|
155
|
+
throw error
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
await walk(dir, '')
|
|
162
|
+
|
|
163
|
+
return files.sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0)
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
const has = async (path: string): Promise<boolean> => {
|
|
167
|
+
const { absolute } = target(path)
|
|
168
|
+
await pathUtils.assertNoLinks(core.root, absolute)
|
|
169
|
+
return await workspaceFs.isFile(absolute)
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
const remove = async (path: string): Promise<boolean> => {
|
|
173
|
+
const { absolute } = target(path)
|
|
174
|
+
await pathUtils.assertNoLinks(core.root, absolute)
|
|
175
|
+
try {
|
|
176
|
+
await lstat(absolute)
|
|
177
|
+
} catch (error) {
|
|
178
|
+
if (['ENOENT', 'ENOTDIR'].includes((error as NodeJS.ErrnoException).code ?? '')) {
|
|
179
|
+
return false
|
|
180
|
+
}
|
|
181
|
+
throw error
|
|
182
|
+
}
|
|
183
|
+
await rm(absolute, { recursive: true, force: true })
|
|
184
|
+
await workspaceFs.removeEmptyParents(dirname(absolute), dir)
|
|
185
|
+
|
|
186
|
+
return true
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
const drop = async (): Promise<void> => {
|
|
190
|
+
await pathUtils.assertNoLinks(core.root, dir)
|
|
191
|
+
await rm(dir, { recursive: true, force: true })
|
|
192
|
+
await workspaceFs.removeEmptyParents(dirname(dir), core.root)
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
return { scope: frozen, dir, tree, entry, source, list, has, remove, drop }
|
|
196
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/** What a workspace's scopes share. */
|
|
2
|
+
export interface WorkspaceCore {
|
|
3
|
+
readonly root: string
|
|
4
|
+
readonly dirMode: number
|
|
5
|
+
readonly fileMode: number
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
export interface WorkspaceFs {
|
|
9
|
+
/** Whether `path` is a regular file (a link is not). */
|
|
10
|
+
isFile: (path: string) => Promise<boolean>
|
|
11
|
+
/** The names of the real folders (no links) directly inside `path`; none when it is absent. */
|
|
12
|
+
dirs: (path: string) => Promise<string[]>
|
|
13
|
+
/** Removes `from` and its parents while they are empty, stopping below `root`. */
|
|
14
|
+
removeEmptyParents: (from: string, root: string) => Promise<void>
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/** An entry path of a scope, normalized, and where it lands. */
|
|
18
|
+
export interface WorkspaceTarget {
|
|
19
|
+
relative: string
|
|
20
|
+
absolute: string
|
|
21
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import type { FileScope } from '@owlmeans/storage-common'
|
|
2
|
+
import type { ConflictPolicy } from '../sinks/types.js'
|
|
3
|
+
import type { SourceClaims } from '../sources/types.js'
|
|
4
|
+
import type { FileSink, FileSource, TreeSink } from '../types.js'
|
|
5
|
+
|
|
6
|
+
export interface WorkspaceOptions {
|
|
7
|
+
/**
|
|
8
|
+
* An absolute folder, or one relative to `os.tmpdir()` (which `TMPDIR` moves) — never to the cwd.
|
|
9
|
+
* Default `<os.tmpdir()>/owlmeans-workspace`. Not persistent across pod restarts.
|
|
10
|
+
*/
|
|
11
|
+
root?: string
|
|
12
|
+
/** Mode of created folders. Default `0o700`. */
|
|
13
|
+
dirMode?: number
|
|
14
|
+
/** Mode of written files. Default `0o600`. */
|
|
15
|
+
fileMode?: number
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** One file of a scope. */
|
|
19
|
+
export interface WorkspaceFile {
|
|
20
|
+
/** Relative to the scope folder, `/`-separated. */
|
|
21
|
+
path: string
|
|
22
|
+
size: number
|
|
23
|
+
modifiedAt: Date
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export interface WorkspaceWriteOptions {
|
|
27
|
+
/** What a write does when the entry exists. Default `error` (`PathError('exists')`). */
|
|
28
|
+
onConflict?: ConflictPolicy
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export interface WorkspacePruneOptions {
|
|
32
|
+
/** Scopes whose newest file, folder or marker is older than this are dropped. */
|
|
33
|
+
olderThanMs: number
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* The folder of one {@link FileScope}: `<root>/<organization>/<project?>/<purpose>`. Every path is
|
|
38
|
+
* relative, normalized and confined — `..`, absolute paths, the scope marker's name and any existing
|
|
39
|
+
* symlink on the way from the workspace root are refused (`PathError`).
|
|
40
|
+
*/
|
|
41
|
+
export interface WorkspaceScope {
|
|
42
|
+
readonly scope: FileScope
|
|
43
|
+
/** The absolute scope folder. */
|
|
44
|
+
readonly dir: string
|
|
45
|
+
/** A confined directory sink over the scope (an archive extracts into it); case-fold twins refused. */
|
|
46
|
+
tree: (options?: WorkspaceWriteOptions) => TreeSink
|
|
47
|
+
/** The sink of one file of the scope. */
|
|
48
|
+
entry: (path: string, options?: WorkspaceWriteOptions) => FileSink
|
|
49
|
+
/** One file of the scope as a re-openable, random-access source. */
|
|
50
|
+
source: (path: string, claims?: SourceClaims) => FileSource
|
|
51
|
+
/** Every regular file, recursively, sorted by path; links are neither followed nor listed. */
|
|
52
|
+
list: () => Promise<WorkspaceFile[]>
|
|
53
|
+
/** Whether the path is a regular file of the scope. */
|
|
54
|
+
has: (path: string) => Promise<boolean>
|
|
55
|
+
/** Removes a file or a folder of the scope (and parents it leaves empty); `false` when absent. */
|
|
56
|
+
remove: (path: string) => Promise<boolean>
|
|
57
|
+
/** Removes the scope folder and every parent it leaves empty, up to the workspace root. */
|
|
58
|
+
drop: () => Promise<void>
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** A service's temporary files, one folder per {@link FileScope}. */
|
|
62
|
+
export interface Workspace {
|
|
63
|
+
/** The absolute root. */
|
|
64
|
+
readonly root: string
|
|
65
|
+
/** The scope's folder object; no I/O until it is used. @throws {FileScopeError} for an invalid scope. */
|
|
66
|
+
scope: (scope: FileScope) => WorkspaceScope
|
|
67
|
+
/** The scopes present on disk — of one organization, or all — sorted by path. */
|
|
68
|
+
scopes: (organization?: string) => Promise<FileScope[]>
|
|
69
|
+
/** Drops every scope untouched for `olderThanMs`; answers how many were dropped. */
|
|
70
|
+
prune: (options: WorkspacePruneOptions) => Promise<number>
|
|
71
|
+
}
|
package/src/workspace.ts
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { lstat } from 'node:fs/promises'
|
|
2
|
+
import { tmpdir } from 'node:os'
|
|
3
|
+
import { join, resolve } from 'node:path'
|
|
4
|
+
import { logger } from '@owlmeans/log'
|
|
5
|
+
import { FileScopeError, fileScopeHelper } from '@owlmeans/storage-common'
|
|
6
|
+
import type { FileScope } from '@owlmeans/storage-common'
|
|
7
|
+
import {
|
|
8
|
+
DEFAULT_WORKSPACE_DIR, DEFAULT_WORKSPACE_DIR_MODE, DEFAULT_WORKSPACE_FILE_MODE, WORKSPACE_SCOPE_MARKER,
|
|
9
|
+
} from './workspace/consts.js'
|
|
10
|
+
import { workspaceFs } from './workspace/fs.js'
|
|
11
|
+
import { makeWorkspaceScope } from './workspace/scope.js'
|
|
12
|
+
import type { Workspace, WorkspaceOptions, WorkspacePruneOptions, WorkspaceScope } from './workspace/types.js'
|
|
13
|
+
import type { WorkspaceCore } from './workspace/types.local.js'
|
|
14
|
+
|
|
15
|
+
const log = logger('file-streams:workspace')
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Temporary files of a service, nested by {@link FileScope}: `<root>/<organization>/<project?>/<purpose>`.
|
|
19
|
+
* Nothing here survives a pod restart — a backed workspace restores a scope from its bucket.
|
|
20
|
+
*/
|
|
21
|
+
export const createWorkspace = (options: WorkspaceOptions = {}): Workspace => {
|
|
22
|
+
const core: WorkspaceCore = {
|
|
23
|
+
// A relative root lives under the temporary folder (which TMPDIR moves), never under the cwd.
|
|
24
|
+
root: resolve(tmpdir(), options.root ?? DEFAULT_WORKSPACE_DIR),
|
|
25
|
+
dirMode: options.dirMode ?? DEFAULT_WORKSPACE_DIR_MODE,
|
|
26
|
+
fileMode: options.fileMode ?? DEFAULT_WORKSPACE_FILE_MODE,
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
const scope = (value: FileScope): WorkspaceScope => makeWorkspaceScope(core, value)
|
|
30
|
+
|
|
31
|
+
const isMarked = async (...segments: string[]): Promise<boolean> =>
|
|
32
|
+
await workspaceFs.isFile(join(core.root, ...segments, WORKSPACE_SCOPE_MARKER))
|
|
33
|
+
|
|
34
|
+
const segmentsIn = async (...segments: string[]): Promise<string[]> =>
|
|
35
|
+
(await workspaceFs.dirs(join(core.root, ...segments))).filter(fileScopeHelper.isSegment)
|
|
36
|
+
|
|
37
|
+
const scopes = async (organization?: string): Promise<FileScope[]> => {
|
|
38
|
+
let organizations: string[]
|
|
39
|
+
if (organization != null) {
|
|
40
|
+
if (!fileScopeHelper.isSegment(organization)) {
|
|
41
|
+
throw new FileScopeError('organization')
|
|
42
|
+
}
|
|
43
|
+
organizations = (await segmentsIn()).includes(organization) ? [organization] : []
|
|
44
|
+
} else {
|
|
45
|
+
organizations = await segmentsIn()
|
|
46
|
+
}
|
|
47
|
+
const found: FileScope[] = []
|
|
48
|
+
for (const org of organizations) {
|
|
49
|
+
for (const second of await segmentsIn(org)) {
|
|
50
|
+
if (await isMarked(org, second)) {
|
|
51
|
+
found.push({ organization: org, purpose: second })
|
|
52
|
+
}
|
|
53
|
+
for (const third of await segmentsIn(org, second)) {
|
|
54
|
+
if (await isMarked(org, second, third)) {
|
|
55
|
+
found.push({ organization: org, project: second, purpose: third })
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
return found
|
|
62
|
+
.map(value => ({ value, path: fileScopeHelper.pathOf(value) }))
|
|
63
|
+
.sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0)
|
|
64
|
+
.map(({ value }) => value)
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const lastTouched = async (local: WorkspaceScope): Promise<number> => {
|
|
68
|
+
let latest = 0
|
|
69
|
+
for (const path of [local.dir, join(local.dir, WORKSPACE_SCOPE_MARKER)]) {
|
|
70
|
+
try {
|
|
71
|
+
latest = Math.max(latest, (await lstat(path)).mtimeMs)
|
|
72
|
+
} catch (error) {
|
|
73
|
+
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
|
|
74
|
+
throw error
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
for (const file of await local.list()) {
|
|
79
|
+
latest = Math.max(latest, file.modifiedAt.getTime())
|
|
80
|
+
}
|
|
81
|
+
return latest
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const prune = async ({ olderThanMs }: WorkspacePruneOptions): Promise<number> => {
|
|
85
|
+
const threshold = Date.now() - olderThanMs
|
|
86
|
+
let dropped = 0
|
|
87
|
+
for (const found of await scopes()) {
|
|
88
|
+
const local = scope(found)
|
|
89
|
+
if (await lastTouched(local) < threshold) {
|
|
90
|
+
await local.drop()
|
|
91
|
+
dropped++
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
if (dropped > 0) {
|
|
95
|
+
log.info('Stale workspace scopes dropped', { dropped })
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
return dropped
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
return { root: core.root, scope, scopes, prune }
|
|
102
|
+
}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from 'bun:test'
|
|
2
|
+
import { readdir, readFile, writeFile } from 'node:fs/promises'
|
|
3
|
+
import { join } from 'node:path'
|
|
4
|
+
import { Readable } from 'node:stream'
|
|
5
|
+
import { gzipSync } from 'node:zlib'
|
|
6
|
+
import {
|
|
7
|
+
ArchiveError, ArchiveFormat, archiveHelper, createArchiveHelper, createBufferSink, createBufferSource, createDirectorySink, createFileSource,
|
|
8
|
+
createStreamSource, LimitError, PathError,
|
|
9
|
+
} from '../src/index.js'
|
|
10
|
+
import type { FileSource } from '../src/index.js'
|
|
11
|
+
import { centralRecords, noise, patch, tarOf, tgzOf, zipOf } from './fixtures.js'
|
|
12
|
+
import { startOrigin } from './harness/origin.js'
|
|
13
|
+
import type { Origin } from './harness/origin.js'
|
|
14
|
+
import { localClient, removeDir, tempDir } from './context.js'
|
|
15
|
+
|
|
16
|
+
const bytesOf = async (source: FileSource): Promise<Buffer> => {
|
|
17
|
+
const sink = createBufferSink({ maxBytes: 64 * 1024 * 1024 })
|
|
18
|
+
await sink.write({ stream: (await source.open()).stream, info: await source.info() })
|
|
19
|
+
return Buffer.from(sink.bytes())
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
const listAll = async (dir: string, prefix = ''): Promise<string[]> => {
|
|
23
|
+
const out: string[] = []
|
|
24
|
+
for (const entry of await readdir(join(dir, prefix), { withFileTypes: true })) {
|
|
25
|
+
const path = prefix === '' ? entry.name : `${prefix}/${entry.name}`
|
|
26
|
+
if (entry.isDirectory()) out.push(...await listAll(dir, path))
|
|
27
|
+
else out.push(path)
|
|
28
|
+
}
|
|
29
|
+
return out.sort()
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
describe('@owlmeans/file-streams — archives', () => {
|
|
33
|
+
let dir: string
|
|
34
|
+
let origin: Origin
|
|
35
|
+
beforeAll(() => { origin = startOrigin() })
|
|
36
|
+
afterAll(() => origin.stop())
|
|
37
|
+
beforeEach(async () => { dir = await tempDir('archive') })
|
|
38
|
+
afterEach(async () => { await removeDir(dir) })
|
|
39
|
+
|
|
40
|
+
test('packs and extracts zip, tar, tar.gz and gz, detecting the format from bytes', async () => {
|
|
41
|
+
const onDisk = join(dir, 'source.bin')
|
|
42
|
+
await writeFile(onDisk, noise(50_000))
|
|
43
|
+
for (const format of [ArchiveFormat.Zip, ArchiveFormat.Tar, ArchiveFormat.TarGz]) {
|
|
44
|
+
const packed = await bytesOf(archiveHelper.pack(format, [
|
|
45
|
+
{ path: 'docs/читать.txt', source: Buffer.from('unicode name') },
|
|
46
|
+
{ path: 'bin/noise.bin', source: createFileSource(onDisk) },
|
|
47
|
+
{ path: 'empty.txt', source: new Uint8Array(0) },
|
|
48
|
+
]))
|
|
49
|
+
expect(await archiveHelper.detect(createBufferSource(packed))).toBe(format)
|
|
50
|
+
const out = join(dir, `out-${format}`)
|
|
51
|
+
const result = await archiveHelper.extract(createBufferSource(packed, { name: `bundle.${format}` }), createDirectorySink(out))
|
|
52
|
+
expect(result.format).toBe(format)
|
|
53
|
+
expect(await listAll(out)).toEqual(['bin/noise.bin', 'docs/читать.txt', 'empty.txt'])
|
|
54
|
+
expect((await readFile(join(out, 'bin/noise.bin'))).equals(noise(50_000))).toBe(true)
|
|
55
|
+
expect(await readFile(join(out, 'docs/читать.txt'), 'utf8')).toBe('unicode name')
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const gz = archiveHelper.gzip(createBufferSource(Buffer.from('compressed words'), { name: 'notes.txt' }))
|
|
59
|
+
expect((await gz.info()).name).toBe('notes.txt.gz')
|
|
60
|
+
const result = await archiveHelper.extract(gz, createDirectorySink(join(dir, 'gz')))
|
|
61
|
+
expect(result.format).toBe(ArchiveFormat.Gzip)
|
|
62
|
+
expect(await readFile(join(dir, 'gz', 'notes.txt'), 'utf8')).toBe('compressed words')
|
|
63
|
+
const plain = await bytesOf(archiveHelper.gunzip(createBufferSource(gzipSync(Buffer.from('round trip')), { name: 'x.txt.gz' })))
|
|
64
|
+
expect(plain.toString()).toBe('round trip')
|
|
65
|
+
})
|
|
66
|
+
|
|
67
|
+
test('lists a remote zip from its central directory, and spools a zip that has no random access', async () => {
|
|
68
|
+
origin.files.set('media.zip', { bytes: await zipOf([{ name: 'video.bin', data: noise(8 * 1024 * 1024), compress: false }, { name: 'cover.png', data: 'png' }]) })
|
|
69
|
+
const entries = await archiveHelper.list(localClient().source(origin.url('/files/media.zip')))
|
|
70
|
+
expect(entries.map(entry => [entry.path, entry.size])).toEqual([['video.bin', 8 * 1024 * 1024], ['cover.png', 3]])
|
|
71
|
+
expect(origin.served.get('media.zip') ?? 0).toBeLessThan(512 * 1024)
|
|
72
|
+
|
|
73
|
+
const zip = await zipOf([{ name: 'a/b.txt', data: 'streamed' }])
|
|
74
|
+
const result = await archiveHelper.extract(createStreamSource(Readable.from([zip])), createDirectorySink(join(dir, 'spooled')))
|
|
75
|
+
expect(result.extracted.map(entry => entry.path)).toEqual(['a/b.txt'])
|
|
76
|
+
expect(await readFile(join(dir, 'spooled', 'a', 'b.txt'), 'utf8')).toBe('streamed')
|
|
77
|
+
})
|
|
78
|
+
|
|
79
|
+
test('refuses traversal, links, duplicates, encryption, overlaps and lying sizes; nothing escapes', async () => {
|
|
80
|
+
const out = join(dir, 'hostile')
|
|
81
|
+
const tree = (): ReturnType<typeof createDirectorySink> => createDirectorySink(out)
|
|
82
|
+
const extract = async (bytes: Buffer, options = {}): Promise<unknown> => await archiveHelper.extract(createBufferSource(bytes), tree(), options)
|
|
83
|
+
|
|
84
|
+
await expect(extract(patch(await zipOf([{ name: 'aa/evil.txt', data: 'x' }]), 'aa/evil.txt', '../evil.txt'))).rejects.toThrow('path:traversal')
|
|
85
|
+
await expect(extract(patch(await zipOf([{ name: 'xetc/passwd', data: 'x' }]), 'xetc/passwd', '/etc/passwd'))).rejects.toBeInstanceOf(PathError)
|
|
86
|
+
await expect(extract(await tarOf([{ name: '../../evil.txt', data: 'x' }]))).rejects.toThrow('path:traversal')
|
|
87
|
+
await expect(extract(await tarOf([{ name: 'bad\u0001name.txt', data: 'x' }]))).rejects.toThrow('path:invalid-char')
|
|
88
|
+
await expect(extract(await tarOf([{ name: 'same.txt', data: '1' }, { name: 'SAME.txt', data: '2' }]))).rejects.toThrow('path:duplicate')
|
|
89
|
+
|
|
90
|
+
const links = await tarOf([{ name: 'link', type: 'symlink', linkname: '/etc/passwd' }, { name: 'hard', type: 'link', linkname: 'x' }, { name: 'ok.txt', data: 'ok' }])
|
|
91
|
+
const skipped = await archiveHelper.extract(createBufferSource(links), createDirectorySink(join(dir, 'links')))
|
|
92
|
+
expect(skipped.skipped.map(entry => [entry.entry.path, entry.reason])).toEqual([['link', 'link'], ['hard', 'link']])
|
|
93
|
+
expect(await listAll(join(dir, 'links'))).toEqual(['ok.txt'])
|
|
94
|
+
await expect(extract(links, { links: 'refuse' })).rejects.toBeInstanceOf(ArchiveError)
|
|
95
|
+
const zipLink = await archiveHelper.extract(createBufferSource(await zipOf([{ name: 'link', data: '/etc/passwd', mode: 0o120777 }])), createDirectorySink(join(dir, 'zip-link')))
|
|
96
|
+
expect(zipLink.skipped.map(entry => entry.reason)).toEqual(['link'])
|
|
97
|
+
|
|
98
|
+
const encrypted = await zipOf([{ name: 'secret.txt', data: 'x' }])
|
|
99
|
+
encrypted.writeUInt16LE(encrypted.readUInt16LE(6) | 1, 6)
|
|
100
|
+
for (const record of centralRecords(encrypted)) encrypted.writeUInt16LE(encrypted.readUInt16LE(record + 8) | 1, record + 8)
|
|
101
|
+
await expect(extract(encrypted)).rejects.toThrow('archive:encrypted')
|
|
102
|
+
|
|
103
|
+
const overlapping = await zipOf([{ name: 'one.txt', data: 'one' }, { name: 'two.txt', data: 'two' }])
|
|
104
|
+
overlapping.writeUInt32LE(0, centralRecords(overlapping)[1] + 42)
|
|
105
|
+
await expect(extract(overlapping)).rejects.toThrow('archive:overlap')
|
|
106
|
+
|
|
107
|
+
const lying = await zipOf([{ name: 'lie.txt', data: 'much more than one byte', compress: false }])
|
|
108
|
+
lying.writeUInt32LE(1, centralRecords(lying)[0] + 24)
|
|
109
|
+
await expect(extract(lying)).rejects.toThrow()
|
|
110
|
+
|
|
111
|
+
// Each entry commits atomically: what was admitted before a refusal stays, nothing escapes, no temp file remains.
|
|
112
|
+
expect(await listAll(out)).toEqual(['same.txt'])
|
|
113
|
+
expect((await listAll(dir)).filter(path => /evil|passwd|\.tmp$/.test(path))).toEqual([])
|
|
114
|
+
})
|
|
115
|
+
|
|
116
|
+
test('stops bombs and entry floods, and follows nested archives only as deep as allowed', async () => {
|
|
117
|
+
const out = createDirectorySink(join(dir, 'bombs'))
|
|
118
|
+
const zeros = Buffer.alloc(20 * 1024 * 1024)
|
|
119
|
+
await expect(archiveHelper.extract(createBufferSource(await zipOf([{ name: 'zeros.bin', data: zeros }])), out)).rejects.toThrow('limit:ratio')
|
|
120
|
+
await expect(archiveHelper.extract(createBufferSource(await tgzOf([{ name: 'zeros.bin', data: zeros }])), out)).rejects.toThrow('limit:ratio')
|
|
121
|
+
await expect(archiveHelper.extract(createBufferSource(gzipSync(zeros)), out)).rejects.toBeInstanceOf(LimitError)
|
|
122
|
+
const flood = await zipOf([{ name: 'a', data: '1' }, { name: 'b', data: '2' }, { name: 'c', data: '3' }])
|
|
123
|
+
await expect(createArchiveHelper({ limits: { maxEntries: 2 } }).extract(createBufferSource(flood), out)).rejects.toThrow('limit:entries')
|
|
124
|
+
await expect(archiveHelper.extract(createBufferSource(await tarOf([{ name: 'big.bin', data: noise(4096) }])), out, { limits: { maxEntryBytes: 1000 } }))
|
|
125
|
+
.rejects.toThrow('limit:entry-bytes')
|
|
126
|
+
|
|
127
|
+
const inner = await zipOf([{ name: 'deep.txt', data: 'deep' }])
|
|
128
|
+
const outer = await zipOf([{ name: 'media/inner.zip', data: inner }, { name: 'top.txt', data: 'top' }])
|
|
129
|
+
await archiveHelper.extract(createBufferSource(outer), createDirectorySink(join(dir, 'depth-1')))
|
|
130
|
+
expect(await listAll(join(dir, 'depth-1'))).toEqual(['media/inner.zip', 'top.txt'])
|
|
131
|
+
await archiveHelper.extract(createBufferSource(outer), createDirectorySink(join(dir, 'depth-2')), { limits: { maxDepth: 2 } })
|
|
132
|
+
expect(await listAll(join(dir, 'depth-2'))).toEqual(['media/inner/deep.txt', 'top.txt'])
|
|
133
|
+
}, 60_000)
|
|
134
|
+
})
|