@owlmeans/file-streams 0.0.0-stage → 0.2.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +71 -2
- package/agent-meta/manifest.json +64 -0
- package/agent-meta/skills/file-streams/SKILL.md +183 -0
- package/agent-meta/skills/file-streams/parts/network.md +47 -0
- package/agent-meta/skills/file-streams/parts/workspaces.md +129 -0
- package/build/archive/consts.d.ts +16 -0
- package/build/archive/consts.d.ts.map +1 -0
- package/build/archive/consts.js +24 -0
- package/build/archive/consts.js.map +1 -0
- package/build/archive/formats.d.ts +4 -0
- package/build/archive/formats.d.ts.map +1 -0
- package/build/archive/formats.js +53 -0
- package/build/archive/formats.js.map +1 -0
- package/build/archive/guard.d.ts +5 -0
- package/build/archive/guard.d.ts.map +1 -0
- package/build/archive/guard.js +62 -0
- package/build/archive/guard.js.map +1 -0
- package/build/archive/ratio.d.ts +5 -0
- package/build/archive/ratio.d.ts.map +1 -0
- package/build/archive/ratio.js +30 -0
- package/build/archive/ratio.js.map +1 -0
- package/build/archive/tar.d.ts +4 -0
- package/build/archive/tar.d.ts.map +1 -0
- package/build/archive/tar.js +142 -0
- package/build/archive/tar.js.map +1 -0
- package/build/archive/types.d.ts +95 -0
- package/build/archive/types.d.ts.map +1 -0
- package/build/archive/types.js +2 -0
- package/build/archive/types.js.map +1 -0
- package/build/archive/types.local.d.ts +64 -0
- package/build/archive/types.local.d.ts.map +1 -0
- package/build/archive/types.local.js +2 -0
- package/build/archive/types.local.js.map +1 -0
- package/build/archive/zip.d.ts +4 -0
- package/build/archive/zip.d.ts.map +1 -0
- package/build/archive/zip.js +135 -0
- package/build/archive/zip.js.map +1 -0
- package/build/archive.d.ts +4 -0
- package/build/archive.d.ts.map +1 -0
- package/build/archive.js +265 -0
- package/build/archive.js.map +1 -0
- package/build/backed-workspace/types.d.ts +67 -0
- package/build/backed-workspace/types.d.ts.map +1 -0
- package/build/backed-workspace/types.js +2 -0
- package/build/backed-workspace/types.js.map +1 -0
- package/build/backed-workspace.d.ts +4 -0
- package/build/backed-workspace.d.ts.map +1 -0
- package/build/backed-workspace.js +63 -0
- package/build/backed-workspace.js.map +1 -0
- package/build/consts.d.ts +3 -0
- package/build/consts.d.ts.map +1 -0
- package/build/consts.js +3 -0
- package/build/consts.js.map +1 -0
- package/build/consts.local.d.ts +3 -0
- package/build/consts.local.d.ts.map +1 -0
- package/build/consts.local.js +3 -0
- package/build/consts.local.js.map +1 -0
- package/build/errors.d.ts +59 -0
- package/build/errors.d.ts.map +1 -0
- package/build/errors.js +105 -0
- package/build/errors.js.map +1 -0
- package/build/http/connector.d.ts +11 -0
- package/build/http/connector.d.ts.map +1 -0
- package/build/http/connector.js +124 -0
- package/build/http/connector.js.map +1 -0
- package/build/http/consts.d.ts +9 -0
- package/build/http/consts.d.ts.map +1 -0
- package/build/http/consts.js +8 -0
- package/build/http/consts.js.map +1 -0
- package/build/http/source.d.ts +5 -0
- package/build/http/source.d.ts.map +1 -0
- package/build/http/source.js +76 -0
- package/build/http/source.js.map +1 -0
- package/build/http/types.d.ts +67 -0
- package/build/http/types.d.ts.map +1 -0
- package/build/http/types.js +2 -0
- package/build/http/types.js.map +1 -0
- package/build/http/types.local.d.ts +21 -0
- package/build/http/types.local.d.ts.map +1 -0
- package/build/http/types.local.js +2 -0
- package/build/http/types.local.js.map +1 -0
- package/build/http.d.ts +4 -0
- package/build/http.d.ts.map +1 -0
- package/build/http.js +248 -0
- package/build/http.js.map +1 -0
- package/build/index.d.ts +53 -0
- package/build/index.d.ts.map +1 -0
- package/build/index.js +33 -0
- package/build/index.js.map +1 -0
- package/build/network/consts.d.ts +12 -0
- package/build/network/consts.d.ts.map +1 -0
- package/build/network/consts.js +20 -0
- package/build/network/consts.js.map +1 -0
- package/build/network/types.d.ts +43 -0
- package/build/network/types.d.ts.map +1 -0
- package/build/network/types.js +2 -0
- package/build/network/types.js.map +1 -0
- package/build/network.d.ts +4 -0
- package/build/network.d.ts.map +1 -0
- package/build/network.js +165 -0
- package/build/network.js.map +1 -0
- package/build/proxy/consts.d.ts +7 -0
- package/build/proxy/consts.d.ts.map +1 -0
- package/build/proxy/consts.js +9 -0
- package/build/proxy/consts.js.map +1 -0
- package/build/proxy/endpoint.d.ts +4 -0
- package/build/proxy/endpoint.d.ts.map +1 -0
- package/build/proxy/endpoint.js +27 -0
- package/build/proxy/endpoint.js.map +1 -0
- package/build/proxy/http.d.ts +5 -0
- package/build/proxy/http.d.ts.map +1 -0
- package/build/proxy/http.js +95 -0
- package/build/proxy/http.js.map +1 -0
- package/build/proxy/router.d.ts +4 -0
- package/build/proxy/router.d.ts.map +1 -0
- package/build/proxy/router.js +85 -0
- package/build/proxy/router.js.map +1 -0
- package/build/proxy/socks.d.ts +5 -0
- package/build/proxy/socks.d.ts.map +1 -0
- package/build/proxy/socks.js +44 -0
- package/build/proxy/socks.js.map +1 -0
- package/build/proxy/types.d.ts +99 -0
- package/build/proxy/types.d.ts.map +1 -0
- package/build/proxy/types.js +2 -0
- package/build/proxy/types.js.map +1 -0
- package/build/resolvers/google-drive/consts.d.ts +10 -0
- package/build/resolvers/google-drive/consts.d.ts.map +1 -0
- package/build/resolvers/google-drive/consts.js +16 -0
- package/build/resolvers/google-drive/consts.js.map +1 -0
- package/build/resolvers/google-drive/types.d.ts +21 -0
- package/build/resolvers/google-drive/types.d.ts.map +1 -0
- package/build/resolvers/google-drive/types.js +2 -0
- package/build/resolvers/google-drive/types.js.map +1 -0
- package/build/resolvers/google-drive.d.ts +10 -0
- package/build/resolvers/google-drive.d.ts.map +1 -0
- package/build/resolvers/google-drive.js +130 -0
- package/build/resolvers/google-drive.js.map +1 -0
- package/build/resolvers/types.d.ts +39 -0
- package/build/resolvers/types.d.ts.map +1 -0
- package/build/resolvers/types.js +2 -0
- package/build/resolvers/types.js.map +1 -0
- package/build/s3/consts.d.ts +9 -0
- package/build/s3/consts.d.ts.map +1 -0
- package/build/s3/consts.js +9 -0
- package/build/s3/consts.js.map +1 -0
- package/build/s3/prefix.d.ts +4 -0
- package/build/s3/prefix.d.ts.map +1 -0
- package/build/s3/prefix.js +87 -0
- package/build/s3/prefix.js.map +1 -0
- package/build/s3/sink.d.ts +6 -0
- package/build/s3/sink.d.ts.map +1 -0
- package/build/s3/sink.js +30 -0
- package/build/s3/sink.js.map +1 -0
- package/build/s3/source.d.ts +5 -0
- package/build/s3/source.d.ts.map +1 -0
- package/build/s3/source.js +81 -0
- package/build/s3/source.js.map +1 -0
- package/build/s3/types.d.ts +66 -0
- package/build/s3/types.d.ts.map +1 -0
- package/build/s3/types.js +2 -0
- package/build/s3/types.js.map +1 -0
- package/build/s3/types.local.d.ts +43 -0
- package/build/s3/types.local.d.ts.map +1 -0
- package/build/s3/types.local.js +2 -0
- package/build/s3/types.local.js.map +1 -0
- package/build/s3/upload.d.ts +4 -0
- package/build/s3/upload.d.ts.map +1 -0
- package/build/s3/upload.js +118 -0
- package/build/s3/upload.js.map +1 -0
- package/build/s3.d.ts +5 -0
- package/build/s3.d.ts.map +1 -0
- package/build/s3.js +68 -0
- package/build/s3.js.map +1 -0
- package/build/service/types.d.ts +66 -0
- package/build/service/types.d.ts.map +1 -0
- package/build/service/types.js +2 -0
- package/build/service/types.js.map +1 -0
- package/build/service/types.local.d.ts +11 -0
- package/build/service/types.local.d.ts.map +1 -0
- package/build/service/types.local.js +2 -0
- package/build/service/types.local.js.map +1 -0
- package/build/service.d.ts +6 -0
- package/build/service.d.ts.map +1 -0
- package/build/service.js +73 -0
- package/build/service.js.map +1 -0
- package/build/sinks/buffer/consts.d.ts +2 -0
- package/build/sinks/buffer/consts.d.ts.map +1 -0
- package/build/sinks/buffer/consts.js +2 -0
- package/build/sinks/buffer/consts.js.map +1 -0
- package/build/sinks/buffer/types.d.ts +11 -0
- package/build/sinks/buffer/types.d.ts.map +1 -0
- package/build/sinks/buffer/types.js +2 -0
- package/build/sinks/buffer/types.js.map +1 -0
- package/build/sinks/buffer.d.ts +4 -0
- package/build/sinks/buffer.d.ts.map +1 -0
- package/build/sinks/buffer.js +13 -0
- package/build/sinks/buffer.js.map +1 -0
- package/build/sinks/directory/types.d.ts +7 -0
- package/build/sinks/directory/types.d.ts.map +1 -0
- package/build/sinks/directory/types.js +2 -0
- package/build/sinks/directory/types.js.map +1 -0
- package/build/sinks/directory.d.ts +8 -0
- package/build/sinks/directory.d.ts.map +1 -0
- package/build/sinks/directory.js +43 -0
- package/build/sinks/directory.js.map +1 -0
- package/build/sinks/file.d.ts +5 -0
- package/build/sinks/file.d.ts.map +1 -0
- package/build/sinks/file.js +46 -0
- package/build/sinks/file.js.map +1 -0
- package/build/sinks/types.d.ts +11 -0
- package/build/sinks/types.d.ts.map +1 -0
- package/build/sinks/types.js +2 -0
- package/build/sinks/types.js.map +1 -0
- package/build/sinks/workload/consts.d.ts +5 -0
- package/build/sinks/workload/consts.d.ts.map +1 -0
- package/build/sinks/workload/consts.js +5 -0
- package/build/sinks/workload/consts.js.map +1 -0
- package/build/sinks/workload/types.d.ts +17 -0
- package/build/sinks/workload/types.d.ts.map +1 -0
- package/build/sinks/workload/types.js +2 -0
- package/build/sinks/workload/types.js.map +1 -0
- package/build/sinks/workload.d.ts +8 -0
- package/build/sinks/workload.d.ts.map +1 -0
- package/build/sinks/workload.js +51 -0
- package/build/sinks/workload.js.map +1 -0
- package/build/sources/buffer.d.ts +5 -0
- package/build/sources/buffer.d.ts.map +1 -0
- package/build/sources/buffer.js +20 -0
- package/build/sources/buffer.js.map +1 -0
- package/build/sources/file.d.ts +5 -0
- package/build/sources/file.d.ts.map +1 -0
- package/build/sources/file.js +50 -0
- package/build/sources/file.js.map +1 -0
- package/build/sources/stream.d.ts +5 -0
- package/build/sources/stream.d.ts.map +1 -0
- package/build/sources/stream.js +19 -0
- package/build/sources/stream.js.map +1 -0
- package/build/sources/types.d.ts +10 -0
- package/build/sources/types.d.ts.map +1 -0
- package/build/sources/types.js +2 -0
- package/build/sources/types.js.map +1 -0
- package/build/sources/workload/types.d.ts +29 -0
- package/build/sources/workload/types.d.ts.map +1 -0
- package/build/sources/workload/types.js +2 -0
- package/build/sources/workload/types.js.map +1 -0
- package/build/sources/workload.d.ts +7 -0
- package/build/sources/workload.d.ts.map +1 -0
- package/build/sources/workload.js +67 -0
- package/build/sources/workload.js.map +1 -0
- package/build/transfer/consts.d.ts +5 -0
- package/build/transfer/consts.d.ts.map +1 -0
- package/build/transfer/consts.js +8 -0
- package/build/transfer/consts.js.map +1 -0
- package/build/transfer/types.d.ts +49 -0
- package/build/transfer/types.d.ts.map +1 -0
- package/build/transfer/types.js +2 -0
- package/build/transfer/types.js.map +1 -0
- package/build/transfer.d.ts +4 -0
- package/build/transfer.d.ts.map +1 -0
- package/build/transfer.js +75 -0
- package/build/transfer.js.map +1 -0
- package/build/types.d.ts +99 -0
- package/build/types.d.ts.map +1 -0
- package/build/types.js +2 -0
- package/build/types.js.map +1 -0
- package/build/utils/names/consts.d.ts +5 -0
- package/build/utils/names/consts.d.ts.map +1 -0
- package/build/utils/names/consts.js +5 -0
- package/build/utils/names/consts.js.map +1 -0
- package/build/utils/names/types.d.ts +13 -0
- package/build/utils/names/types.d.ts.map +1 -0
- package/build/utils/names/types.js +2 -0
- package/build/utils/names/types.js.map +1 -0
- package/build/utils/names.d.ts +4 -0
- package/build/utils/names.d.ts.map +1 -0
- package/build/utils/names.js +60 -0
- package/build/utils/names.js.map +1 -0
- package/build/utils/paths/consts.d.ts +3 -0
- package/build/utils/paths/consts.d.ts.map +1 -0
- package/build/utils/paths/consts.js +2 -0
- package/build/utils/paths/consts.js.map +1 -0
- package/build/utils/paths/types.d.ts +20 -0
- package/build/utils/paths/types.d.ts.map +1 -0
- package/build/utils/paths/types.js +2 -0
- package/build/utils/paths/types.js.map +1 -0
- package/build/utils/paths.d.ts +4 -0
- package/build/utils/paths.d.ts.map +1 -0
- package/build/utils/paths.js +67 -0
- package/build/utils/paths.js.map +1 -0
- package/build/utils/range/types.d.ts +11 -0
- package/build/utils/range/types.d.ts.map +1 -0
- package/build/utils/range/types.js +2 -0
- package/build/utils/range/types.js.map +1 -0
- package/build/utils/range.d.ts +4 -0
- package/build/utils/range.d.ts.map +1 -0
- package/build/utils/range.js +64 -0
- package/build/utils/range.js.map +1 -0
- package/build/utils/redact/types.d.ts +16 -0
- package/build/utils/redact/types.d.ts.map +1 -0
- package/build/utils/redact/types.js +2 -0
- package/build/utils/redact/types.js.map +1 -0
- package/build/utils/redact.d.ts +4 -0
- package/build/utils/redact.d.ts.map +1 -0
- package/build/utils/redact.js +18 -0
- package/build/utils/redact.js.map +1 -0
- package/build/utils/signals/types.d.ts +17 -0
- package/build/utils/signals/types.d.ts.map +1 -0
- package/build/utils/signals/types.js +2 -0
- package/build/utils/signals/types.js.map +1 -0
- package/build/utils/signals.d.ts +4 -0
- package/build/utils/signals.d.ts.map +1 -0
- package/build/utils/signals.js +52 -0
- package/build/utils/signals.js.map +1 -0
- package/build/utils/streams/consts.d.ts +3 -0
- package/build/utils/streams/consts.d.ts.map +1 -0
- package/build/utils/streams/consts.js +3 -0
- package/build/utils/streams/consts.js.map +1 -0
- package/build/utils/streams/types.d.ts +59 -0
- package/build/utils/streams/types.d.ts.map +1 -0
- package/build/utils/streams/types.js +2 -0
- package/build/utils/streams/types.js.map +1 -0
- package/build/utils/streams.d.ts +4 -0
- package/build/utils/streams.d.ts.map +1 -0
- package/build/utils/streams.js +167 -0
- package/build/utils/streams.js.map +1 -0
- package/build/utils/tree/types.d.ts +6 -0
- package/build/utils/tree/types.d.ts.map +1 -0
- package/build/utils/tree/types.js +2 -0
- package/build/utils/tree/types.js.map +1 -0
- package/build/utils/tree.d.ts +4 -0
- package/build/utils/tree.d.ts.map +1 -0
- package/build/utils/tree.js +14 -0
- package/build/utils/tree.js.map +1 -0
- package/build/workspace/consts.d.ts +12 -0
- package/build/workspace/consts.d.ts.map +1 -0
- package/build/workspace/consts.js +12 -0
- package/build/workspace/consts.js.map +1 -0
- package/build/workspace/consts.local.d.ts +3 -0
- package/build/workspace/consts.local.d.ts.map +1 -0
- package/build/workspace/consts.local.js +3 -0
- package/build/workspace/consts.local.js.map +1 -0
- package/build/workspace/fs.d.ts +4 -0
- package/build/workspace/fs.d.ts.map +1 -0
- package/build/workspace/fs.js +45 -0
- package/build/workspace/fs.js.map +1 -0
- package/build/workspace/scope.d.ts +6 -0
- package/build/workspace/scope.d.ts.map +1 -0
- package/build/workspace/scope.js +180 -0
- package/build/workspace/scope.js.map +1 -0
- package/build/workspace/types.d.ts +66 -0
- package/build/workspace/types.d.ts.map +1 -0
- package/build/workspace/types.js +2 -0
- package/build/workspace/types.js.map +1 -0
- package/build/workspace/types.local.d.ts +20 -0
- package/build/workspace/types.local.d.ts.map +1 -0
- package/build/workspace/types.local.js +2 -0
- package/build/workspace/types.local.js.map +1 -0
- package/build/workspace.d.ts +7 -0
- package/build/workspace.d.ts.map +1 -0
- package/build/workspace.js +87 -0
- package/build/workspace.js.map +1 -0
- package/package.json +61 -4
- package/src/archive/consts.ts +28 -0
- package/src/archive/formats.ts +60 -0
- package/src/archive/guard.ts +68 -0
- package/src/archive/ratio.ts +35 -0
- package/src/archive/tar.ts +147 -0
- package/src/archive/types.local.ts +72 -0
- package/src/archive/types.ts +107 -0
- package/src/archive/zip.ts +142 -0
- package/src/archive.ts +298 -0
- package/src/backed-workspace/types.ts +72 -0
- package/src/backed-workspace.ts +85 -0
- package/src/consts.local.ts +2 -0
- package/src/consts.ts +2 -0
- package/src/errors.ts +128 -0
- package/src/http/connector.ts +136 -0
- package/src/http/consts.ts +13 -0
- package/src/http/source.ts +89 -0
- package/src/http/types.local.ts +23 -0
- package/src/http/types.ts +74 -0
- package/src/http.ts +276 -0
- package/src/index.ts +65 -0
- package/src/network/consts.ts +22 -0
- package/src/network/types.ts +45 -0
- package/src/network.ts +179 -0
- package/src/proxy/consts.ts +11 -0
- package/src/proxy/endpoint.ts +28 -0
- package/src/proxy/http.ts +108 -0
- package/src/proxy/router.ts +93 -0
- package/src/proxy/socks.ts +48 -0
- package/src/proxy/types.ts +114 -0
- package/src/resolvers/google-drive/consts.ts +22 -0
- package/src/resolvers/google-drive/types.ts +22 -0
- package/src/resolvers/google-drive.ts +153 -0
- package/src/resolvers/types.ts +45 -0
- package/src/s3/consts.ts +12 -0
- package/src/s3/prefix.ts +100 -0
- package/src/s3/sink.ts +36 -0
- package/src/s3/source.ts +90 -0
- package/src/s3/types.local.ts +49 -0
- package/src/s3/types.ts +70 -0
- package/src/s3/upload.ts +130 -0
- package/src/s3.ts +83 -0
- package/src/service/types.local.ts +9 -0
- package/src/service/types.ts +71 -0
- package/src/service.ts +97 -0
- package/src/sinks/buffer/consts.ts +1 -0
- package/src/sinks/buffer/types.ts +12 -0
- package/src/sinks/buffer.ts +18 -0
- package/src/sinks/directory/types.ts +7 -0
- package/src/sinks/directory.ts +49 -0
- package/src/sinks/file.ts +52 -0
- package/src/sinks/types.ts +12 -0
- package/src/sinks/workload/consts.ts +5 -0
- package/src/sinks/workload/types.ts +17 -0
- package/src/sinks/workload.ts +59 -0
- package/src/sources/buffer.ts +27 -0
- package/src/sources/file.ts +57 -0
- package/src/sources/stream.ts +26 -0
- package/src/sources/types.ts +12 -0
- package/src/sources/workload/types.ts +31 -0
- package/src/sources/workload.ts +79 -0
- package/src/transfer/consts.ts +10 -0
- package/src/transfer/types.ts +55 -0
- package/src/transfer.ts +89 -0
- package/src/types.ts +106 -0
- package/src/utils/names/consts.ts +5 -0
- package/src/utils/names/types.ts +12 -0
- package/src/utils/names.ts +70 -0
- package/src/utils/paths/consts.ts +3 -0
- package/src/utils/paths/types.ts +20 -0
- package/src/utils/paths.ts +75 -0
- package/src/utils/range/types.ts +11 -0
- package/src/utils/range.ts +80 -0
- package/src/utils/redact/types.ts +16 -0
- package/src/utils/redact.ts +23 -0
- package/src/utils/signals/types.ts +17 -0
- package/src/utils/signals.ts +63 -0
- package/src/utils/streams/consts.ts +2 -0
- package/src/utils/streams/types.ts +65 -0
- package/src/utils/streams.ts +188 -0
- package/src/utils/tree/types.ts +6 -0
- package/src/utils/tree.ts +18 -0
- package/src/workspace/consts.local.ts +2 -0
- package/src/workspace/consts.ts +14 -0
- package/src/workspace/fs.ts +48 -0
- package/src/workspace/scope.ts +196 -0
- package/src/workspace/types.local.ts +21 -0
- package/src/workspace/types.ts +71 -0
- package/src/workspace.ts +102 -0
- package/tests/archive.spec.ts +134 -0
- package/tests/backed.spec.ts +147 -0
- package/tests/context.ts +114 -0
- package/tests/fixtures.ts +190 -0
- package/tests/google-drive.spec.ts +76 -0
- package/tests/harness/drive.ts +92 -0
- package/tests/harness/http-proxy.ts +96 -0
- package/tests/harness/origin.ts +134 -0
- package/tests/harness/socks5.ts +135 -0
- package/tests/harness/workload.ts +118 -0
- package/tests/http.spec.ts +101 -0
- package/tests/inspector.spec.ts +40 -0
- package/tests/live.spec.ts +27 -0
- package/tests/network.spec.ts +79 -0
- package/tests/proxy.spec.ts +149 -0
- package/tests/s3.spec.ts +133 -0
- package/tests/service.spec.ts +77 -0
- package/tests/sources-sinks.spec.ts +96 -0
- package/tests/transfer.spec.ts +99 -0
- package/tests/tsconfig.json +12 -0
- package/tests/workload.spec.ts +140 -0
- package/tests/workspace.spec.ts +158 -0
- package/tsconfig.json +16 -0
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { afterAll, beforeAll, describe, expect, test } from 'bun:test'
|
|
2
|
+
import { createHttpClient, createNetworkPolicy, NetworkPolicyError } from '../src/index.js'
|
|
3
|
+
import { streamUtils } from '../src/utils/streams.js'
|
|
4
|
+
import { startOrigin } from './harness/origin.js'
|
|
5
|
+
import type { Origin } from './harness/origin.js'
|
|
6
|
+
import { LOCAL_NETWORK, localClient } from './context.js'
|
|
7
|
+
|
|
8
|
+
describe('@owlmeans/file-streams — network policy', () => {
|
|
9
|
+
let origin: Origin
|
|
10
|
+
beforeAll(() => { origin = startOrigin() })
|
|
11
|
+
afterAll(() => origin.stop())
|
|
12
|
+
|
|
13
|
+
test('refuses internal, special and embedded addresses; allows public ones', () => {
|
|
14
|
+
const policy = createNetworkPolicy()
|
|
15
|
+
const refused = [
|
|
16
|
+
'127.0.0.1', '10.1.2.3', '172.20.0.1', '192.168.1.4', '169.254.169.254', '100.64.0.1', '0.0.0.0', '224.0.0.1',
|
|
17
|
+
'240.0.0.1', '255.255.255.255', '198.18.0.1', '::1', '::', 'fd00::1', 'fe80::1', 'ff02::1', '::ffff:10.0.0.1',
|
|
18
|
+
'::ffff:7f00:1', '64:ff9b::a00:1', '2002:c0a8:0101::1', '2001:0:0:0:0:0:f5ff:fffe', 'not-an-ip',
|
|
19
|
+
]
|
|
20
|
+
for (const address of refused) {
|
|
21
|
+
expect(policy.allowsAddress(address), address).toBe(false)
|
|
22
|
+
}
|
|
23
|
+
for (const address of ['8.8.8.8', '1.1.1.1', '2606:4700:4700::1111', '64:ff9b::808:808']) {
|
|
24
|
+
expect(policy.allowsAddress(address), address).toBe(true)
|
|
25
|
+
}
|
|
26
|
+
expect(createNetworkPolicy({ allowCidrs: ['127.0.0.1/32'] }).allowsAddress('127.0.0.1')).toBe(true)
|
|
27
|
+
expect(createNetworkPolicy({ denyCidrs: ['8.8.8.0/24'] }).allowsAddress('8.8.8.8')).toBe(false)
|
|
28
|
+
})
|
|
29
|
+
|
|
30
|
+
test('checks schemes, credentials, hosts, ports and downgrades before any DNS', () => {
|
|
31
|
+
const policy = createNetworkPolicy({ ports: [80, 443], denyHosts: ['*.evil.test'] })
|
|
32
|
+
const cases: Array<[string, string, string?]> = [
|
|
33
|
+
['ftp://example.com/x', 'scheme'],
|
|
34
|
+
['file:///etc/passwd', 'scheme'],
|
|
35
|
+
['https://user:pass@example.com/', 'credentials'],
|
|
36
|
+
['https://cdn.evil.test/', 'blocked-host'],
|
|
37
|
+
['https://example.com:8443/', 'port'],
|
|
38
|
+
['http://example.com/', 'downgrade', 'https://example.com/'],
|
|
39
|
+
]
|
|
40
|
+
for (const [url, reason, from] of cases) {
|
|
41
|
+
expect(() => policy.assertUrl(new URL(url), from != null ? new URL(from) : undefined), url).toThrow(`policy:${reason}`)
|
|
42
|
+
}
|
|
43
|
+
expect(() => policy.assertUrl(new URL('https://example.com/a'))).not.toThrow()
|
|
44
|
+
expect(() => createNetworkPolicy({ allowHosts: ['*.example.com'] }).assertUrl(new URL('https://other.test/'))).toThrow('policy:blocked-host')
|
|
45
|
+
})
|
|
46
|
+
|
|
47
|
+
test('the default client refuses a loopback origin before sending a byte', async () => {
|
|
48
|
+
origin.files.set('a.txt', { bytes: Buffer.from('a') })
|
|
49
|
+
const before = origin.requests.length
|
|
50
|
+
await expect(createHttpClient().request(origin.url('/files/a.txt'))).rejects.toThrow('policy:blocked-address')
|
|
51
|
+
await expect(createHttpClient().request(`http://2130706433:${new URL(origin.base).port}/files/a.txt`)).rejects.toThrow('policy:blocked-address')
|
|
52
|
+
await expect(createHttpClient().request('http://localhost/')).rejects.toThrow('policy:blocked-host')
|
|
53
|
+
await expect(createHttpClient().request('http://metadata.google.internal/')).rejects.toThrow('policy:blocked-host')
|
|
54
|
+
expect(origin.requests.length).toBe(before)
|
|
55
|
+
})
|
|
56
|
+
|
|
57
|
+
test('connects to the address it checked and refuses a mixed answer or a refused redirect', async () => {
|
|
58
|
+
origin.files.set('pinned.txt', { bytes: Buffer.from('pinned') })
|
|
59
|
+
const port = new URL(origin.base).port
|
|
60
|
+
const resolved: string[] = []
|
|
61
|
+
const pinned = localClient({
|
|
62
|
+
network: { ...LOCAL_NETWORK, resolver: async host => { resolved.push(host); return [{ address: '127.0.0.1', family: 4 }] } },
|
|
63
|
+
})
|
|
64
|
+
const response = await pinned.request(`http://files.example:${port}/headers`)
|
|
65
|
+
const echoed = JSON.parse((await streamUtils.collect(response.body, 10_000)).toString()) as Record<string, string>
|
|
66
|
+
expect(echoed.host).toBe(`files.example:${port}`)
|
|
67
|
+
expect(resolved).toEqual(['files.example'])
|
|
68
|
+
|
|
69
|
+
const mixed = localClient({
|
|
70
|
+
network: { ...LOCAL_NETWORK, resolver: async () => [{ address: '127.0.0.1', family: 4 }, { address: '10.0.0.7', family: 4 }] },
|
|
71
|
+
})
|
|
72
|
+
await expect(mixed.request(`http://files.example:${port}/files/pinned.txt`)).rejects.toBeInstanceOf(NetworkPolicyError)
|
|
73
|
+
|
|
74
|
+
const before = origin.requests.length
|
|
75
|
+
await expect(localClient().request(origin.url(`/go?to=${encodeURIComponent('http://169.254.169.254/latest/meta-data')}`)))
|
|
76
|
+
.rejects.toThrow('policy:blocked-address')
|
|
77
|
+
expect(origin.requests.length).toBe(before + 1)
|
|
78
|
+
})
|
|
79
|
+
})
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
import { afterAll, beforeAll, describe, expect, test } from 'bun:test'
|
|
2
|
+
import { addLogPlugin, configureLog, logConfig, memoryPlugin } from '@owlmeans/log'
|
|
3
|
+
import { createHttpProxyPlugin, ProxyError } from '../src/index.js'
|
|
4
|
+
import type { ProxyRequest } from '../src/index.js'
|
|
5
|
+
import { streamUtils } from '../src/utils/streams.js'
|
|
6
|
+
import { startHttpProxy } from './harness/http-proxy.js'
|
|
7
|
+
import type { LocalHttpProxy } from './harness/http-proxy.js'
|
|
8
|
+
import { startOrigin } from './harness/origin.js'
|
|
9
|
+
import type { Origin } from './harness/origin.js'
|
|
10
|
+
import { startSocksProxy } from './harness/socks5.js'
|
|
11
|
+
import type { LocalSocksProxy } from './harness/socks5.js'
|
|
12
|
+
import { LOCAL_NETWORK, localClient, tlsMaterial } from './context.js'
|
|
13
|
+
import type { TlsMaterial } from './context.js'
|
|
14
|
+
|
|
15
|
+
const body = async (promise: Promise<{ body: import('node:stream').Readable }>): Promise<string> =>
|
|
16
|
+
(await streamUtils.collect((await promise).body, 100_000)).toString()
|
|
17
|
+
|
|
18
|
+
describe('@owlmeans/file-streams — proxy plugins', () => {
|
|
19
|
+
let origin: Origin
|
|
20
|
+
let secure: Origin | undefined
|
|
21
|
+
let tls: TlsMaterial | undefined
|
|
22
|
+
let httpProxy: LocalHttpProxy
|
|
23
|
+
let plainProxy: LocalHttpProxy
|
|
24
|
+
let socks: LocalSocksProxy
|
|
25
|
+
|
|
26
|
+
beforeAll(async () => {
|
|
27
|
+
origin = startOrigin()
|
|
28
|
+
origin.files.set('a.txt', { bytes: Buffer.from('through the proxy'), type: 'text/plain' })
|
|
29
|
+
tls = await tlsMaterial()
|
|
30
|
+
if (tls != null) {
|
|
31
|
+
secure = startOrigin({ tls })
|
|
32
|
+
secure.files.set('s.txt', { bytes: Buffer.from('tls inside the tunnel'), type: 'text/plain' })
|
|
33
|
+
}
|
|
34
|
+
httpProxy = await startHttpProxy({ username: 'proxy-user', password: 'proxy-secret' })
|
|
35
|
+
plainProxy = await startHttpProxy()
|
|
36
|
+
socks = await startSocksProxy({ username: 'socks-user', password: 'socks-secret' })
|
|
37
|
+
})
|
|
38
|
+
afterAll(() => {
|
|
39
|
+
origin.stop()
|
|
40
|
+
secure?.stop()
|
|
41
|
+
httpProxy.stop()
|
|
42
|
+
plainProxy.stop()
|
|
43
|
+
socks.stop()
|
|
44
|
+
})
|
|
45
|
+
|
|
46
|
+
test('an HTTP proxy forwards plain-http targets and tunnels the checked IP with CONNECT', async () => {
|
|
47
|
+
const forward = localClient({ proxies: { proxies: [`http://proxy-user:proxy-secret@127.0.0.1:${httpProxy.port}`] } })
|
|
48
|
+
expect(await body(forward.request(origin.url('/files/a.txt')))).toBe('through the proxy')
|
|
49
|
+
expect(httpProxy.records.at(-1)).toMatchObject({ method: 'GET', target: origin.url('/files/a.txt') })
|
|
50
|
+
|
|
51
|
+
const port = new URL(origin.base).port
|
|
52
|
+
const tunnel = localClient({
|
|
53
|
+
network: { ...LOCAL_NETWORK, resolver: async () => [{ address: '127.0.0.1', family: 4 }] },
|
|
54
|
+
proxies: {
|
|
55
|
+
proxies: [`http://127.0.0.1:${httpProxy.port}`],
|
|
56
|
+
credentials: () => ({ username: 'proxy-user', password: 'proxy-secret' }),
|
|
57
|
+
plugins: [createHttpProxyPlugin({ httpTargets: 'connect' })],
|
|
58
|
+
},
|
|
59
|
+
})
|
|
60
|
+
expect(await body(tunnel.request(`http://files.example:${port}/files/a.txt`))).toBe('through the proxy')
|
|
61
|
+
expect(httpProxy.records.at(-1)).toMatchObject({ method: 'CONNECT', target: `127.0.0.1:${port}` })
|
|
62
|
+
|
|
63
|
+
const denied = localClient({ proxies: { proxies: [`http://wrong:creds@127.0.0.1:${httpProxy.port}`], retries: 0 } })
|
|
64
|
+
await expect(denied.request(origin.url('/files/a.txt'))).rejects.toBeInstanceOf(ProxyError)
|
|
65
|
+
await expect(denied.request(origin.url('/files/a.txt'))).rejects.toThrow('proxy:')
|
|
66
|
+
})
|
|
67
|
+
|
|
68
|
+
test('SOCKS5 carries the checked IP; SOCKS5h hands the name to the proxy', async () => {
|
|
69
|
+
const pinned = localClient({ proxies: { proxies: [`socks5://socks-user:socks-secret@127.0.0.1:${socks.port}`] } })
|
|
70
|
+
expect(await body(pinned.request(origin.url('/files/a.txt')))).toBe('through the proxy')
|
|
71
|
+
expect(socks.records.at(-1)).toMatchObject({ atyp: 1, address: '127.0.0.1', username: 'socks-user' })
|
|
72
|
+
|
|
73
|
+
const remote = localClient({
|
|
74
|
+
network: { allowCidrs: ['127.0.0.1/32', '::1/128'], allowHosts: ['localhost'] },
|
|
75
|
+
proxies: { proxies: [`socks5h://socks-user:socks-secret@127.0.0.1:${socks.port}`] },
|
|
76
|
+
})
|
|
77
|
+
const port = new URL(origin.base).port
|
|
78
|
+
expect(await body(remote.request(`http://localhost:${port}/files/a.txt`))).toBe('through the proxy')
|
|
79
|
+
expect(socks.records.at(-1)).toMatchObject({ atyp: 3, address: 'localhost' })
|
|
80
|
+
|
|
81
|
+
const wrong = localClient({ proxies: { proxies: [`socks5://socks-user:nope@127.0.0.1:${socks.port}`], retries: 0 } })
|
|
82
|
+
await expect(wrong.request(origin.url('/files/a.txt'))).rejects.toBeInstanceOf(ProxyError)
|
|
83
|
+
})
|
|
84
|
+
|
|
85
|
+
test('pools rotate, fail over, stick by key; providers decide by key and URL; bypass goes direct', async () => {
|
|
86
|
+
const dead = 'http://127.0.0.1:1'
|
|
87
|
+
const failover = localClient({ proxies: { proxies: [dead, `http://127.0.0.1:${plainProxy.port}`], strategy: 'failover', retries: 1 } })
|
|
88
|
+
expect(await body(failover.request(origin.url('/files/a.txt')))).toBe('through the proxy')
|
|
89
|
+
|
|
90
|
+
const before = plainProxy.records.length + httpProxy.records.length
|
|
91
|
+
const sticky = localClient({
|
|
92
|
+
proxies: {
|
|
93
|
+
proxies: [`http://127.0.0.1:${plainProxy.port}`, `http://proxy-user:proxy-secret@127.0.0.1:${httpProxy.port}`], strategy: 'by-key',
|
|
94
|
+
},
|
|
95
|
+
})
|
|
96
|
+
const plainBefore = plainProxy.records.length
|
|
97
|
+
for (let index = 0; index < 4; index++) {
|
|
98
|
+
await body(sticky.request(origin.url('/files/a.txt'), { proxyKey: 'tenant-7' }))
|
|
99
|
+
}
|
|
100
|
+
const plainUsed = plainProxy.records.length - plainBefore
|
|
101
|
+
expect(plainUsed === 0 || plainUsed === 4).toBe(true)
|
|
102
|
+
expect(plainProxy.records.length + httpProxy.records.length - before).toBe(4)
|
|
103
|
+
|
|
104
|
+
const asked: ProxyRequest[] = []
|
|
105
|
+
const provided = localClient({
|
|
106
|
+
proxies: {
|
|
107
|
+
proxies: request => {
|
|
108
|
+
asked.push(request)
|
|
109
|
+
return request.key === 'direct' ? null : { type: 'socks5', host: '127.0.0.1', port: socks.port, username: 'socks-user', password: 'socks-secret' }
|
|
110
|
+
},
|
|
111
|
+
},
|
|
112
|
+
})
|
|
113
|
+
const socksBefore = socks.records.length
|
|
114
|
+
await body(provided.request(origin.url('/files/a.txt'), { proxyKey: 'via-socks' }))
|
|
115
|
+
await body(provided.request(origin.url('/files/a.txt'), { proxyKey: 'direct' }))
|
|
116
|
+
expect(socks.records.length - socksBefore).toBe(1)
|
|
117
|
+
expect(asked.map(request => [request.key, request.url.pathname])).toEqual([['via-socks', '/files/a.txt'], ['direct', '/files/a.txt']])
|
|
118
|
+
|
|
119
|
+
const bypassed = localClient({ proxies: { proxies: [dead], bypass: ['127.0.0.1'] } })
|
|
120
|
+
expect(await body(bypassed.request(origin.url('/files/a.txt')))).toBe('through the proxy')
|
|
121
|
+
})
|
|
122
|
+
|
|
123
|
+
test('TLS rides inside both tunnels, and no proxy secret reaches the log', async () => {
|
|
124
|
+
const previous = logConfig().level
|
|
125
|
+
const memory = memoryPlugin('proxy-secrets')
|
|
126
|
+
const remove = addLogPlugin(memory)
|
|
127
|
+
configureLog({ level: 'debug' })
|
|
128
|
+
try {
|
|
129
|
+
const failing = localClient({ proxies: { proxies: ['http://user:leaked-secret@127.0.0.1:1', `socks5://u:other-secret@127.0.0.1:1`], retries: 1 } })
|
|
130
|
+
const error = await failing.request(origin.url('/files/a.txt')).catch((failure: Error) => failure)
|
|
131
|
+
expect(error).toBeInstanceOf(ProxyError)
|
|
132
|
+
expect(String((error as Error).message)).not.toContain('secret')
|
|
133
|
+
expect(JSON.stringify(memory.records)).not.toContain('secret')
|
|
134
|
+
} finally {
|
|
135
|
+
configureLog({ level: previous })
|
|
136
|
+
remove()
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
if (secure == null || tls == null) {
|
|
140
|
+
console.warn('openssl unavailable — HTTPS-through-proxy cases skipped')
|
|
141
|
+
return
|
|
142
|
+
}
|
|
143
|
+
const viaConnect = localClient({ ca: tls.cert, proxies: { proxies: [`http://proxy-user:proxy-secret@127.0.0.1:${httpProxy.port}`] } })
|
|
144
|
+
expect(await body(viaConnect.request(secure.url('/files/s.txt')))).toBe('tls inside the tunnel')
|
|
145
|
+
expect(httpProxy.records.at(-1)?.method).toBe('CONNECT')
|
|
146
|
+
const viaSocks = localClient({ ca: tls.cert, proxies: { proxies: [`socks5://socks-user:socks-secret@127.0.0.1:${socks.port}`] } })
|
|
147
|
+
expect(await body(viaSocks.request(secure.url('/files/s.txt')))).toBe('tls inside the tunnel')
|
|
148
|
+
})
|
|
149
|
+
})
|
package/tests/s3.spec.ts
ADDED
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import { afterAll, beforeAll, describe, expect, test } from 'bun:test'
|
|
2
|
+
import { readFile } from 'node:fs/promises'
|
|
3
|
+
import { join } from 'node:path'
|
|
4
|
+
import { Readable } from 'node:stream'
|
|
5
|
+
import { DeleteObjectCommand, ListMultipartUploadsCommand, ListObjectsV2Command } from '@aws-sdk/client-s3'
|
|
6
|
+
import { config, makeServerContext } from '@owlmeans/server-context'
|
|
7
|
+
import { appendStorageResource } from '@owlmeans/storage-resource'
|
|
8
|
+
import type { StorageResource } from '@owlmeans/storage-resource'
|
|
9
|
+
import { randomNamespace } from '@owlmeans/test-integration'
|
|
10
|
+
import {
|
|
11
|
+
appendFileStreamsService, archiveHelper, createBufferSource, createFileSink, createS3Bucket, createStreamSource, FILE_STREAMS_SERVICE,
|
|
12
|
+
fileInspector, FileKind, transferHelper,
|
|
13
|
+
} from '../src/index.js'
|
|
14
|
+
import type { FileStreamsService, S3Bucket } from '../src/index.js'
|
|
15
|
+
import { streamUtils } from '../src/utils/streams.js'
|
|
16
|
+
import { docxOf, noise, PNG, tgzOf } from './fixtures.js'
|
|
17
|
+
import { startOrigin } from './harness/origin.js'
|
|
18
|
+
import type { Origin } from './harness/origin.js'
|
|
19
|
+
import { bucketConfig, localClient, removeDir, s3, tempDir } from './context.js'
|
|
20
|
+
import type { TestConfig } from './context.js'
|
|
21
|
+
|
|
22
|
+
describe('@owlmeans/file-streams — S3', () => {
|
|
23
|
+
if (s3.skip) {
|
|
24
|
+
test.skip(s3.reason ?? 's3 gate closed', () => { })
|
|
25
|
+
return
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const basePrefix = randomNamespace('omt_file_streams')
|
|
29
|
+
let bucket: S3Bucket
|
|
30
|
+
let origin: Origin
|
|
31
|
+
let dir: string
|
|
32
|
+
|
|
33
|
+
beforeAll(async () => {
|
|
34
|
+
bucket = createS3Bucket(bucketConfig(basePrefix), { partSize: 5 * 1024 * 1024 })
|
|
35
|
+
origin = startOrigin()
|
|
36
|
+
origin.files.set('photo.png', { bytes: PNG, type: 'image/png' })
|
|
37
|
+
dir = await tempDir('s3')
|
|
38
|
+
})
|
|
39
|
+
|
|
40
|
+
afterAll(async () => {
|
|
41
|
+
const client = bucket.model.createClient()
|
|
42
|
+
try {
|
|
43
|
+
let token: string | undefined
|
|
44
|
+
do {
|
|
45
|
+
const page = await client.send(new ListObjectsV2Command({ Bucket: bucket.model.bucket, Prefix: `${basePrefix}/`, ContinuationToken: token }))
|
|
46
|
+
for (const object of page.Contents ?? []) {
|
|
47
|
+
await client.send(new DeleteObjectCommand({ Bucket: bucket.model.bucket, Key: object.Key }))
|
|
48
|
+
}
|
|
49
|
+
token = page.NextContinuationToken
|
|
50
|
+
} while (token != null)
|
|
51
|
+
} finally {
|
|
52
|
+
client.destroy()
|
|
53
|
+
bucket.close()
|
|
54
|
+
origin.stop()
|
|
55
|
+
await removeDir(dir)
|
|
56
|
+
}
|
|
57
|
+
})
|
|
58
|
+
|
|
59
|
+
test('puts an object with its verified type and reads it whole, ranged and by stat', async () => {
|
|
60
|
+
const result = await transferHelper.copy(createBufferSource(PNG, { name: 'shot.png' }), bucket.sink('screens/shot.png'), {
|
|
61
|
+
verify: { allow: [FileKind.Image] },
|
|
62
|
+
})
|
|
63
|
+
expect(result.sink).toMatchObject({ kind: 's3', bytes: PNG.length, mime: 'image/png', url: bucket.urlOf('screens/shot.png') })
|
|
64
|
+
expect(bucket.instanceOf(result.sink, 'original')).toEqual({ size: PNG.length, alias: 'original', url: bucket.urlOf('screens/shot.png') })
|
|
65
|
+
|
|
66
|
+
expect(await bucket.head('screens/shot.png')).toMatchObject({ key: `${basePrefix}/screens/shot.png`, size: PNG.length, mime: 'image/png' })
|
|
67
|
+
expect(await bucket.head('screens/missing.png')).toBeNull()
|
|
68
|
+
const source = bucket.source('screens/shot.png')
|
|
69
|
+
expect((await source.info()).size).toBe(PNG.length)
|
|
70
|
+
expect(await streamUtils.collect((await source.open()).stream, 10_000)).toEqual(PNG)
|
|
71
|
+
const reader = await source.ranges()
|
|
72
|
+
expect(Buffer.from(await reader!.read(1, 3))).toEqual(PNG.subarray(1, 4))
|
|
73
|
+
}, 60_000)
|
|
74
|
+
|
|
75
|
+
test('uploads a stream of unknown length in parts, and aborts a failed upload', async () => {
|
|
76
|
+
const payload = noise(12 * 1024 * 1024)
|
|
77
|
+
const chunks = Array.from({ length: 12 }, (_, index) => payload.subarray(index * 1024 * 1024, (index + 1) * 1024 * 1024))
|
|
78
|
+
await transferHelper.copy(createStreamSource(Readable.from(chunks), { name: 'noise.bin' }), bucket.sink('big/noise.bin'), { verify: false })
|
|
79
|
+
const head = await bucket.head('big/noise.bin')
|
|
80
|
+
expect(head?.size).toBe(payload.length)
|
|
81
|
+
expect(head?.etag).toContain('-')
|
|
82
|
+
const back = await streamUtils.collect((await bucket.source('big/noise.bin').open()).stream, 20 * 1024 * 1024)
|
|
83
|
+
expect(back.equals(payload)).toBe(true)
|
|
84
|
+
|
|
85
|
+
const failing = Readable.from((async function * () {
|
|
86
|
+
for (let index = 0; index < 7; index++) yield noise(1024 * 1024, index)
|
|
87
|
+
throw new Error('source broke')
|
|
88
|
+
})())
|
|
89
|
+
await expect(transferHelper.copy(createStreamSource(failing), bucket.sink('big/broken.bin'), { verify: false })).rejects.toThrow()
|
|
90
|
+
expect(await bucket.head('big/broken.bin')).toBeNull()
|
|
91
|
+
const client = bucket.model.createClient()
|
|
92
|
+
try {
|
|
93
|
+
const uploads = await client.send(new ListMultipartUploadsCommand({ Bucket: bucket.model.bucket, Prefix: `${basePrefix}/big/broken.bin` }))
|
|
94
|
+
expect(uploads.Uploads ?? []).toEqual([])
|
|
95
|
+
} finally {
|
|
96
|
+
client.destroy()
|
|
97
|
+
}
|
|
98
|
+
}, 120_000)
|
|
99
|
+
|
|
100
|
+
test('probes by range and moves files between S3, HTTP, folders and archives', async () => {
|
|
101
|
+
await transferHelper.copy(createBufferSource(await docxOf(), { name: 'letter.docx' }), bucket.sink('docs/letter.docx'))
|
|
102
|
+
const verdict = await fileInspector.probe(bucket.source('docs/letter.docx'))
|
|
103
|
+
expect(verdict).toMatchObject({ status: 'match', detected: { ext: 'docx' }, probe: { randomAccess: true } })
|
|
104
|
+
|
|
105
|
+
await transferHelper.copy(localClient().source(origin.url('/files/photo.png')), bucket.sink('from-http/photo.png'))
|
|
106
|
+
await transferHelper.copy(bucket.source('from-http/photo.png'), createFileSink(join(dir, 'photo.png')))
|
|
107
|
+
expect(await readFile(join(dir, 'photo.png'))).toEqual(PNG)
|
|
108
|
+
|
|
109
|
+
const archive = await tgzOf([{ name: 'media/a.png', data: PNG }, { name: 'media/b.txt', data: 'b' }])
|
|
110
|
+
const extracted = await archiveHelper.extract(createBufferSource(archive), bucket.tree('extracted'))
|
|
111
|
+
expect(extracted.extracted.map(entry => entry.result.location)).toEqual([
|
|
112
|
+
`s3://${bucket.model.bucket}/${basePrefix}/extracted/media/a.png`, `s3://${bucket.model.bucket}/${basePrefix}/extracted/media/b.txt`,
|
|
113
|
+
])
|
|
114
|
+
expect((await bucket.head('extracted/media/a.png'))?.mime).toBe('image/png')
|
|
115
|
+
}, 60_000)
|
|
116
|
+
|
|
117
|
+
test('reads what storage-resource uploaded, by the same config entry, key and URL', async () => {
|
|
118
|
+
const cfg = config('file-streams-s3-tests') as TestConfig
|
|
119
|
+
cfg.storageBuckets = { interop: bucketConfig(basePrefix) }
|
|
120
|
+
const context = makeServerContext(cfg)
|
|
121
|
+
appendStorageResource(context as unknown as Parameters<typeof appendStorageResource>[0], 'interop')
|
|
122
|
+
appendFileStreamsService(context)
|
|
123
|
+
context.configure()
|
|
124
|
+
await context.init()
|
|
125
|
+
|
|
126
|
+
const stored = await context.resource<StorageResource>('interop').create({ prefix: 'interop/dot.png', stream: Readable.from([PNG]), size: PNG.length, type: 'image/png' })
|
|
127
|
+
const files = context.service<FileStreamsService>(FILE_STREAMS_SERVICE)
|
|
128
|
+
const same = files.bucket('interop')
|
|
129
|
+
expect(stored.url).toBe(same.urlOf('interop/dot.png'))
|
|
130
|
+
expect(await streamUtils.collect((await same.source('interop/dot.png').open()).stream, 10_000)).toEqual(PNG)
|
|
131
|
+
same.close()
|
|
132
|
+
}, 60_000)
|
|
133
|
+
})
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { afterAll, afterEach, beforeAll, describe, expect, test } from 'bun:test'
|
|
2
|
+
import { writeFile } from 'node:fs/promises'
|
|
3
|
+
import { tmpdir } from 'node:os'
|
|
4
|
+
import { join } from 'node:path'
|
|
5
|
+
import { FilePropertyError } from '@owlmeans/storage-common'
|
|
6
|
+
import { makeStorageBucketModel } from '@owlmeans/storage-resource'
|
|
7
|
+
import { createBufferSource, FileKind } from '../src/index.js'
|
|
8
|
+
import { streamUtils } from '../src/utils/streams.js'
|
|
9
|
+
import { PNG } from './fixtures.js'
|
|
10
|
+
import { startOrigin } from './harness/origin.js'
|
|
11
|
+
import type { Origin } from './harness/origin.js'
|
|
12
|
+
import { startSocksProxy } from './harness/socks5.js'
|
|
13
|
+
import type { LocalSocksProxy } from './harness/socks5.js'
|
|
14
|
+
import { boot, removeDir, tempDir } from './context.js'
|
|
15
|
+
|
|
16
|
+
describe('@owlmeans/file-streams — context service', () => {
|
|
17
|
+
let origin: Origin
|
|
18
|
+
let socks: LocalSocksProxy
|
|
19
|
+
let dir: string
|
|
20
|
+
beforeAll(async () => {
|
|
21
|
+
origin = startOrigin()
|
|
22
|
+
origin.files.set('a.txt', { bytes: Buffer.from('configured'), type: 'text/plain' })
|
|
23
|
+
socks = await startSocksProxy()
|
|
24
|
+
dir = await tempDir('service')
|
|
25
|
+
})
|
|
26
|
+
afterAll(async () => {
|
|
27
|
+
origin.stop()
|
|
28
|
+
socks.stop()
|
|
29
|
+
await removeDir(dir)
|
|
30
|
+
})
|
|
31
|
+
afterEach(() => undefined)
|
|
32
|
+
|
|
33
|
+
test('builds its HTTP client from config, reading a proxy URL from a secret file', async () => {
|
|
34
|
+
const secret = join(dir, 'proxy-url')
|
|
35
|
+
await writeFile(secret, `socks5://127.0.0.1:${socks.port}\n`)
|
|
36
|
+
const { files } = await boot({ fileStreams: { network: { allowCidrs: ['127.0.0.1/32'] }, proxies: [{ url: secret }] } })
|
|
37
|
+
|
|
38
|
+
const response = await files.http().request(origin.url('/files/a.txt'))
|
|
39
|
+
expect((await streamUtils.collect(response.body, 100)).toString()).toBe('configured')
|
|
40
|
+
expect(socks.records.at(-1)).toMatchObject({ atyp: 1, address: '127.0.0.1' })
|
|
41
|
+
expect(files.http()).toBe(files.http())
|
|
42
|
+
})
|
|
43
|
+
|
|
44
|
+
test('addresses storage buckets exactly as storage-resource does, without I/O until used', async () => {
|
|
45
|
+
const entry = { url: 'media.s3.eu-central-1.amazonaws.com', apiKey: 'key:secret', basePrefix: 'tenant' }
|
|
46
|
+
const { files } = await boot({ storageBuckets: { media: entry } })
|
|
47
|
+
const bucket = files.bucket('media')
|
|
48
|
+
expect(bucket).toBe(files.bucket('media'))
|
|
49
|
+
expect(bucket.urlOf('a/b.png')).toBe(makeStorageBucketModel(entry).urlOf('a/b.png'))
|
|
50
|
+
expect(bucket.keyOf('a/b.png')).toBe('tenant/a/b.png')
|
|
51
|
+
expect(() => bucket.keyOf('../escape')).toThrow('path:traversal')
|
|
52
|
+
expect(() => files.bucket('unknown')).toThrow(FilePropertyError)
|
|
53
|
+
})
|
|
54
|
+
|
|
55
|
+
test('hands out working inspector, transfer and archive objects', async () => {
|
|
56
|
+
const { files } = await boot()
|
|
57
|
+
const verdict = await files.inspector().probe(createBufferSource(PNG, { name: 'dot.png' }))
|
|
58
|
+
expect(verdict.detected?.kind).toBe(FileKind.Image)
|
|
59
|
+
expect(files.transfer()).toBe(files.transfer())
|
|
60
|
+
expect(await files.archives().detect(createBufferSource(PNG))).toBeNull()
|
|
61
|
+
})
|
|
62
|
+
|
|
63
|
+
test('builds the workspace from config and backed workspaces per bucket entry and prefix, without I/O until used', async () => {
|
|
64
|
+
const entry = { url: 'media.s3.eu-central-1.amazonaws.com', apiKey: 'key:secret', basePrefix: 'tenant' }
|
|
65
|
+
const { files } = await boot({ storageBuckets: { media: entry }, fileStreams: { workspace: { root: 'owlmeans-fs-service-workspace' } } })
|
|
66
|
+
expect(files.workspace().root).toBe(join(tmpdir(), 'owlmeans-fs-service-workspace'))
|
|
67
|
+
expect(files.workspace()).toBe(files.workspace())
|
|
68
|
+
|
|
69
|
+
const backed = files.backed('media', { prefix: 'materials' })
|
|
70
|
+
expect(files.backed('media', { prefix: 'materials' })).toBe(backed)
|
|
71
|
+
expect(files.backed('media')).not.toBe(backed)
|
|
72
|
+
expect(backed.bucket).toBe(files.bucket('media'))
|
|
73
|
+
expect(backed.scope({ organization: 'acme', project: 'slot-1', purpose: 'new-project' }).key('a.png'))
|
|
74
|
+
.toBe('tenant/materials/acme/slot-1/new-project/a.png')
|
|
75
|
+
expect(() => files.backed('unknown')).toThrow(FilePropertyError)
|
|
76
|
+
})
|
|
77
|
+
})
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, test } from 'bun:test'
|
|
2
|
+
import { mkdir, readdir, readFile, symlink, writeFile } from 'node:fs/promises'
|
|
3
|
+
import { join } from 'node:path'
|
|
4
|
+
import { Readable } from 'node:stream'
|
|
5
|
+
import {
|
|
6
|
+
createBufferSink, createBufferSource, createDirectorySink, createFileSink, createFileSource, createStreamSource, LimitError, PathError,
|
|
7
|
+
SourceError,
|
|
8
|
+
} from '../src/index.js'
|
|
9
|
+
import { PNG } from './fixtures.js'
|
|
10
|
+
import { removeDir, tempDir } from './context.js'
|
|
11
|
+
|
|
12
|
+
const text = async (stream: Readable): Promise<string> => {
|
|
13
|
+
const chunks: Buffer[] = []
|
|
14
|
+
for await (const chunk of stream) chunks.push(chunk as Buffer)
|
|
15
|
+
return Buffer.concat(chunks).toString()
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
describe('@owlmeans/file-streams — sources and sinks', () => {
|
|
19
|
+
let dir: string
|
|
20
|
+
beforeEach(async () => { dir = await tempDir('sinks') })
|
|
21
|
+
afterEach(async () => { await removeDir(dir) })
|
|
22
|
+
|
|
23
|
+
test('a file source answers info, whole and ranged opens and random access', async () => {
|
|
24
|
+
const path = join(dir, 'data.txt')
|
|
25
|
+
await writeFile(path, '0123456789')
|
|
26
|
+
const source = createFileSource(path)
|
|
27
|
+
|
|
28
|
+
const info = await source.info()
|
|
29
|
+
expect(info).toMatchObject({ name: 'data.txt', size: 10, origin: path })
|
|
30
|
+
expect(info.etag).toBeString()
|
|
31
|
+
expect(await text((await source.open()).stream)).toBe('0123456789')
|
|
32
|
+
const ranged = await source.open({ range: { start: 2, end: 5 } })
|
|
33
|
+
expect(await text(ranged.stream)).toBe('2345')
|
|
34
|
+
expect(ranged.range).toEqual({ start: 2, end: 5, total: 10 })
|
|
35
|
+
const reader = await source.ranges()
|
|
36
|
+
expect(Buffer.from(await reader!.read(7, 10)).toString()).toBe('789')
|
|
37
|
+
await reader!.close()
|
|
38
|
+
|
|
39
|
+
await expect(createFileSource(join(dir, 'missing')).info()).rejects.toBeInstanceOf(SourceError)
|
|
40
|
+
await expect(createFileSource(dir).info()).rejects.toThrow('source:not-a-file')
|
|
41
|
+
})
|
|
42
|
+
|
|
43
|
+
test('buffer sources reopen; stream sources are one-shot', async () => {
|
|
44
|
+
const buffer = createBufferSource(PNG, { name: 'dot.png' })
|
|
45
|
+
expect((await buffer.info()).size).toBe(PNG.length)
|
|
46
|
+
expect((await (await buffer.ranges())!.read(1, 3))).toEqual(new Uint8Array(PNG.subarray(1, 4)))
|
|
47
|
+
|
|
48
|
+
const stream = createStreamSource(Readable.from([Buffer.from('once')]), { name: 'x.txt' })
|
|
49
|
+
expect(await stream.ranges()).toBeNull()
|
|
50
|
+
expect(await text((await stream.open()).stream)).toBe('once')
|
|
51
|
+
await expect(stream.open()).rejects.toThrow('source:consumed')
|
|
52
|
+
})
|
|
53
|
+
|
|
54
|
+
test('a file sink commits atomically, refuses to clobber, and leaves nothing on failure', async () => {
|
|
55
|
+
const path = join(dir, 'out', 'file.txt')
|
|
56
|
+
const result = await createFileSink(path).write({ stream: Readable.from([Buffer.from('first')]), info: { origin: 'test' } })
|
|
57
|
+
expect(result).toMatchObject({ kind: 'file', bytes: 5, location: path })
|
|
58
|
+
expect(await readFile(path, 'utf8')).toBe('first')
|
|
59
|
+
|
|
60
|
+
await expect(createFileSink(path).write({ stream: Readable.from([Buffer.from('second')]), info: { origin: 'test' } }))
|
|
61
|
+
.rejects.toThrow('path:exists')
|
|
62
|
+
await createFileSink(path, { onConflict: 'overwrite' }).write({ stream: Readable.from([Buffer.from('second')]), info: { origin: 'test' } })
|
|
63
|
+
expect(await readFile(path, 'utf8')).toBe('second')
|
|
64
|
+
|
|
65
|
+
const failing = new Readable({ read () { this.destroy(new Error('boom')) } })
|
|
66
|
+
await expect(createFileSink(join(dir, 'out', 'broken.txt')).write({ stream: failing, info: { origin: 'test' } })).rejects.toThrow('boom')
|
|
67
|
+
expect((await readdir(join(dir, 'out'))).sort()).toEqual(['file.txt'])
|
|
68
|
+
|
|
69
|
+
const limited = createFileSink(join(dir, 'out', 'big.txt'))
|
|
70
|
+
await expect(limited.write({ stream: Readable.from([Buffer.alloc(100)]), info: { origin: 'test' } }, { maxBytes: 10 }))
|
|
71
|
+
.rejects.toBeInstanceOf(LimitError)
|
|
72
|
+
expect((await readdir(join(dir, 'out'))).sort()).toEqual(['file.txt'])
|
|
73
|
+
})
|
|
74
|
+
|
|
75
|
+
test('a directory sink confines entries, refuses duplicates and symlinked parents', async () => {
|
|
76
|
+
const tree = createDirectorySink(join(dir, 'tree'))
|
|
77
|
+
await tree.entry('a/b/c.txt').write({ stream: Readable.from([Buffer.from('c')]), info: { origin: 'test' } })
|
|
78
|
+
expect(await readFile(join(dir, 'tree', 'a', 'b', 'c.txt'), 'utf8')).toBe('c')
|
|
79
|
+
|
|
80
|
+
for (const bad of ['../escape.txt', '/etc/passwd', 'a/../../x', 'C:\\x', 'nul\u0000.txt']) {
|
|
81
|
+
expect(() => tree.entry(bad)).toThrow(PathError)
|
|
82
|
+
}
|
|
83
|
+
expect(() => tree.entry('A/B/C.TXT')).toThrow('path:duplicate')
|
|
84
|
+
|
|
85
|
+
await mkdir(join(dir, 'outside'))
|
|
86
|
+
await mkdir(join(dir, 'linked'))
|
|
87
|
+
await symlink(join(dir, 'outside'), join(dir, 'linked', 'link'))
|
|
88
|
+
const linked = createDirectorySink(join(dir, 'linked'))
|
|
89
|
+
await expect(linked.entry('link/x.txt').write({ stream: Readable.from([Buffer.from('x')]), info: { origin: 'test' } }))
|
|
90
|
+
.rejects.toThrow('path:symlink')
|
|
91
|
+
expect(await readdir(join(dir, 'outside'))).toEqual([])
|
|
92
|
+
|
|
93
|
+
const sink = createBufferSink({ maxBytes: 4 })
|
|
94
|
+
await expect(sink.write({ stream: Readable.from([Buffer.from('too long')]), info: { origin: 'test' } })).rejects.toBeInstanceOf(LimitError)
|
|
95
|
+
})
|
|
96
|
+
})
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from 'bun:test'
|
|
2
|
+
import { createHash } from 'node:crypto'
|
|
3
|
+
import { readdir, readFile } from 'node:fs/promises'
|
|
4
|
+
import { join } from 'node:path'
|
|
5
|
+
import { Readable } from 'node:stream'
|
|
6
|
+
import { FileTypeError } from '@owlmeans/storage-common'
|
|
7
|
+
import { createBufferSink, createDirectorySink, createFileSink, createStreamSource, FileKind, LimitError, transferHelper } from '../src/index.js'
|
|
8
|
+
import type { FileSink, SinkInput, TransferProgress } from '../src/index.js'
|
|
9
|
+
import { HTML_ERROR, noise, PNG } from './fixtures.js'
|
|
10
|
+
import { startOrigin } from './harness/origin.js'
|
|
11
|
+
import type { Origin } from './harness/origin.js'
|
|
12
|
+
import { localClient, removeDir, tempDir } from './context.js'
|
|
13
|
+
|
|
14
|
+
/** A sink that only records being called. */
|
|
15
|
+
const spySink = (): FileSink & { calls: number } => {
|
|
16
|
+
const spy = {
|
|
17
|
+
kind: 'spy',
|
|
18
|
+
calls: 0,
|
|
19
|
+
write: async (input: SinkInput) => {
|
|
20
|
+
spy.calls++
|
|
21
|
+
input.stream.resume()
|
|
22
|
+
return { kind: 'spy', location: 'spy', bytes: 0 }
|
|
23
|
+
},
|
|
24
|
+
}
|
|
25
|
+
return spy
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
describe('@owlmeans/file-streams — transfers', () => {
|
|
29
|
+
let origin: Origin
|
|
30
|
+
let dir: string
|
|
31
|
+
beforeAll(() => {
|
|
32
|
+
origin = startOrigin()
|
|
33
|
+
origin.files.set('photo.png', { bytes: PNG, type: 'image/png' })
|
|
34
|
+
origin.files.set('named.bin', { bytes: Buffer.from('named content'), disposition: 'attachment; filename*=UTF-8\'\'..%2F..%2Fsecret%20plan.txt' })
|
|
35
|
+
origin.files.set('invoice.pdf', { bytes: HTML_ERROR, type: 'text/html' })
|
|
36
|
+
origin.files.set('large.bin', { bytes: noise(2 * 1024 * 1024) })
|
|
37
|
+
})
|
|
38
|
+
afterAll(() => origin.stop())
|
|
39
|
+
beforeEach(async () => { dir = await tempDir('transfer') })
|
|
40
|
+
afterEach(async () => { await removeDir(dir) })
|
|
41
|
+
|
|
42
|
+
test('downloads into a folder under the sanitized name the server gives', async () => {
|
|
43
|
+
const client = localClient()
|
|
44
|
+
const named = await transferHelper.copyInto(client.source(origin.url('/files/named.bin')), createDirectorySink(dir))
|
|
45
|
+
expect(named.sink.location).toBe(join(dir, 'secret plan.txt'))
|
|
46
|
+
expect(await readFile(join(dir, 'secret plan.txt'), 'utf8')).toBe('named content')
|
|
47
|
+
|
|
48
|
+
const image = await transferHelper.copyInto(client.source(origin.url('/files/photo.png')), createDirectorySink(dir), { verify: { allow: [FileKind.Image] } })
|
|
49
|
+
expect(image.verdict).toMatchObject({ status: 'match', detected: { ext: 'png', kind: FileKind.Image } })
|
|
50
|
+
expect(await readFile(join(dir, 'photo.png'))).toEqual(PNG)
|
|
51
|
+
})
|
|
52
|
+
|
|
53
|
+
test('refuses an oversized body before the sink, and mid-stream without leaving a file', async () => {
|
|
54
|
+
const client = localClient()
|
|
55
|
+
const spy = spySink()
|
|
56
|
+
await expect(transferHelper.copy(client.source(origin.url('/files/large.bin')), spy, { limits: { maxBytes: 1024 } })).rejects.toThrow('limit:max-bytes')
|
|
57
|
+
expect(spy.calls).toBe(0)
|
|
58
|
+
|
|
59
|
+
const unknown = createStreamSource(Readable.from([noise(4096), noise(4096)]), { name: 'stream.bin' })
|
|
60
|
+
await expect(transferHelper.copy(unknown, createFileSink(join(dir, 'stream.bin')), { limits: { maxBytes: 5000 }, verify: false }))
|
|
61
|
+
.rejects.toBeInstanceOf(LimitError)
|
|
62
|
+
expect(await readdir(dir)).toEqual([])
|
|
63
|
+
})
|
|
64
|
+
|
|
65
|
+
test('times out, aborts, reports progress and hashes', async () => {
|
|
66
|
+
const client = localClient({ timeouts: { idleMs: 10_000 } })
|
|
67
|
+
await expect(transferHelper.copy(client.source(origin.url('/stall-body')), createBufferSink(), { limits: { idleTimeoutMs: 200 }, verify: false }))
|
|
68
|
+
.rejects.toThrow('limit:idle-timeout')
|
|
69
|
+
await expect(transferHelper.copy(client.source(origin.url('/stall-body')), createBufferSink(), { limits: { timeoutMs: 200, idleTimeoutMs: 5000 }, verify: false }))
|
|
70
|
+
.rejects.toThrow('limit:timeout')
|
|
71
|
+
const controller = new AbortController()
|
|
72
|
+
setTimeout(() => controller.abort(new Error('stop now')), 100)
|
|
73
|
+
await expect(transferHelper.copy(client.source(origin.url('/stall-body')), createBufferSink(), { signal: controller.signal, verify: false }))
|
|
74
|
+
.rejects.toThrow('stop now')
|
|
75
|
+
|
|
76
|
+
const progress: TransferProgress[] = []
|
|
77
|
+
const result = await transferHelper.copy(client.source(origin.url('/files/large.bin')), createFileSink(join(dir, 'large.bin')), {
|
|
78
|
+
onProgress: event => progress.push(event), progressIntervalMs: 0, hash: 'sha256', verify: false,
|
|
79
|
+
})
|
|
80
|
+
expect(result.bytes).toBe(2 * 1024 * 1024)
|
|
81
|
+
expect(result.sha256).toBe(createHash('sha256').update(noise(2 * 1024 * 1024)).digest('hex'))
|
|
82
|
+
expect(progress.at(-1)?.bytes).toBe(2 * 1024 * 1024)
|
|
83
|
+
expect(progress.every((event, index) => index === 0 || event.bytes >= progress[index - 1].bytes)).toBe(true)
|
|
84
|
+
})
|
|
85
|
+
|
|
86
|
+
test('verifies the head before a byte reaches the sink', async () => {
|
|
87
|
+
const client = localClient()
|
|
88
|
+
const spy = spySink()
|
|
89
|
+
await expect(transferHelper.copy(client.source(origin.url('/files/invoice.pdf')), spy)).rejects.toBeInstanceOf(FileTypeError)
|
|
90
|
+
expect(spy.calls).toBe(0)
|
|
91
|
+
await expect(transferHelper.copy(client.source(origin.url('/files/photo.png')), spy, { verify: { allow: [FileKind.Pdf] } }))
|
|
92
|
+
.rejects.toBeInstanceOf(FileTypeError)
|
|
93
|
+
expect(spy.calls).toBe(0)
|
|
94
|
+
|
|
95
|
+
const unchecked = await transferHelper.copy(client.source(origin.url('/files/invoice.pdf')), createFileSink(join(dir, 'page.html')), { verify: false })
|
|
96
|
+
expect(unchecked.verdict).toBeUndefined()
|
|
97
|
+
expect(await readFile(join(dir, 'page.html'))).toEqual(HTML_ERROR)
|
|
98
|
+
})
|
|
99
|
+
})
|