@owlmeans/file-streams 0.0.0-stage → 0.2.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +71 -2
- package/agent-meta/manifest.json +64 -0
- package/agent-meta/skills/file-streams/SKILL.md +183 -0
- package/agent-meta/skills/file-streams/parts/network.md +47 -0
- package/agent-meta/skills/file-streams/parts/workspaces.md +129 -0
- package/build/archive/consts.d.ts +16 -0
- package/build/archive/consts.d.ts.map +1 -0
- package/build/archive/consts.js +24 -0
- package/build/archive/consts.js.map +1 -0
- package/build/archive/formats.d.ts +4 -0
- package/build/archive/formats.d.ts.map +1 -0
- package/build/archive/formats.js +53 -0
- package/build/archive/formats.js.map +1 -0
- package/build/archive/guard.d.ts +5 -0
- package/build/archive/guard.d.ts.map +1 -0
- package/build/archive/guard.js +62 -0
- package/build/archive/guard.js.map +1 -0
- package/build/archive/ratio.d.ts +5 -0
- package/build/archive/ratio.d.ts.map +1 -0
- package/build/archive/ratio.js +30 -0
- package/build/archive/ratio.js.map +1 -0
- package/build/archive/tar.d.ts +4 -0
- package/build/archive/tar.d.ts.map +1 -0
- package/build/archive/tar.js +142 -0
- package/build/archive/tar.js.map +1 -0
- package/build/archive/types.d.ts +95 -0
- package/build/archive/types.d.ts.map +1 -0
- package/build/archive/types.js +2 -0
- package/build/archive/types.js.map +1 -0
- package/build/archive/types.local.d.ts +64 -0
- package/build/archive/types.local.d.ts.map +1 -0
- package/build/archive/types.local.js +2 -0
- package/build/archive/types.local.js.map +1 -0
- package/build/archive/zip.d.ts +4 -0
- package/build/archive/zip.d.ts.map +1 -0
- package/build/archive/zip.js +135 -0
- package/build/archive/zip.js.map +1 -0
- package/build/archive.d.ts +4 -0
- package/build/archive.d.ts.map +1 -0
- package/build/archive.js +265 -0
- package/build/archive.js.map +1 -0
- package/build/backed-workspace/types.d.ts +67 -0
- package/build/backed-workspace/types.d.ts.map +1 -0
- package/build/backed-workspace/types.js +2 -0
- package/build/backed-workspace/types.js.map +1 -0
- package/build/backed-workspace.d.ts +4 -0
- package/build/backed-workspace.d.ts.map +1 -0
- package/build/backed-workspace.js +63 -0
- package/build/backed-workspace.js.map +1 -0
- package/build/consts.d.ts +3 -0
- package/build/consts.d.ts.map +1 -0
- package/build/consts.js +3 -0
- package/build/consts.js.map +1 -0
- package/build/consts.local.d.ts +3 -0
- package/build/consts.local.d.ts.map +1 -0
- package/build/consts.local.js +3 -0
- package/build/consts.local.js.map +1 -0
- package/build/errors.d.ts +59 -0
- package/build/errors.d.ts.map +1 -0
- package/build/errors.js +105 -0
- package/build/errors.js.map +1 -0
- package/build/http/connector.d.ts +11 -0
- package/build/http/connector.d.ts.map +1 -0
- package/build/http/connector.js +124 -0
- package/build/http/connector.js.map +1 -0
- package/build/http/consts.d.ts +9 -0
- package/build/http/consts.d.ts.map +1 -0
- package/build/http/consts.js +8 -0
- package/build/http/consts.js.map +1 -0
- package/build/http/source.d.ts +5 -0
- package/build/http/source.d.ts.map +1 -0
- package/build/http/source.js +76 -0
- package/build/http/source.js.map +1 -0
- package/build/http/types.d.ts +67 -0
- package/build/http/types.d.ts.map +1 -0
- package/build/http/types.js +2 -0
- package/build/http/types.js.map +1 -0
- package/build/http/types.local.d.ts +21 -0
- package/build/http/types.local.d.ts.map +1 -0
- package/build/http/types.local.js +2 -0
- package/build/http/types.local.js.map +1 -0
- package/build/http.d.ts +4 -0
- package/build/http.d.ts.map +1 -0
- package/build/http.js +248 -0
- package/build/http.js.map +1 -0
- package/build/index.d.ts +53 -0
- package/build/index.d.ts.map +1 -0
- package/build/index.js +33 -0
- package/build/index.js.map +1 -0
- package/build/network/consts.d.ts +12 -0
- package/build/network/consts.d.ts.map +1 -0
- package/build/network/consts.js +20 -0
- package/build/network/consts.js.map +1 -0
- package/build/network/types.d.ts +43 -0
- package/build/network/types.d.ts.map +1 -0
- package/build/network/types.js +2 -0
- package/build/network/types.js.map +1 -0
- package/build/network.d.ts +4 -0
- package/build/network.d.ts.map +1 -0
- package/build/network.js +165 -0
- package/build/network.js.map +1 -0
- package/build/proxy/consts.d.ts +7 -0
- package/build/proxy/consts.d.ts.map +1 -0
- package/build/proxy/consts.js +9 -0
- package/build/proxy/consts.js.map +1 -0
- package/build/proxy/endpoint.d.ts +4 -0
- package/build/proxy/endpoint.d.ts.map +1 -0
- package/build/proxy/endpoint.js +27 -0
- package/build/proxy/endpoint.js.map +1 -0
- package/build/proxy/http.d.ts +5 -0
- package/build/proxy/http.d.ts.map +1 -0
- package/build/proxy/http.js +95 -0
- package/build/proxy/http.js.map +1 -0
- package/build/proxy/router.d.ts +4 -0
- package/build/proxy/router.d.ts.map +1 -0
- package/build/proxy/router.js +85 -0
- package/build/proxy/router.js.map +1 -0
- package/build/proxy/socks.d.ts +5 -0
- package/build/proxy/socks.d.ts.map +1 -0
- package/build/proxy/socks.js +44 -0
- package/build/proxy/socks.js.map +1 -0
- package/build/proxy/types.d.ts +99 -0
- package/build/proxy/types.d.ts.map +1 -0
- package/build/proxy/types.js +2 -0
- package/build/proxy/types.js.map +1 -0
- package/build/resolvers/google-drive/consts.d.ts +10 -0
- package/build/resolvers/google-drive/consts.d.ts.map +1 -0
- package/build/resolvers/google-drive/consts.js +16 -0
- package/build/resolvers/google-drive/consts.js.map +1 -0
- package/build/resolvers/google-drive/types.d.ts +21 -0
- package/build/resolvers/google-drive/types.d.ts.map +1 -0
- package/build/resolvers/google-drive/types.js +2 -0
- package/build/resolvers/google-drive/types.js.map +1 -0
- package/build/resolvers/google-drive.d.ts +10 -0
- package/build/resolvers/google-drive.d.ts.map +1 -0
- package/build/resolvers/google-drive.js +130 -0
- package/build/resolvers/google-drive.js.map +1 -0
- package/build/resolvers/types.d.ts +39 -0
- package/build/resolvers/types.d.ts.map +1 -0
- package/build/resolvers/types.js +2 -0
- package/build/resolvers/types.js.map +1 -0
- package/build/s3/consts.d.ts +9 -0
- package/build/s3/consts.d.ts.map +1 -0
- package/build/s3/consts.js +9 -0
- package/build/s3/consts.js.map +1 -0
- package/build/s3/prefix.d.ts +4 -0
- package/build/s3/prefix.d.ts.map +1 -0
- package/build/s3/prefix.js +87 -0
- package/build/s3/prefix.js.map +1 -0
- package/build/s3/sink.d.ts +6 -0
- package/build/s3/sink.d.ts.map +1 -0
- package/build/s3/sink.js +30 -0
- package/build/s3/sink.js.map +1 -0
- package/build/s3/source.d.ts +5 -0
- package/build/s3/source.d.ts.map +1 -0
- package/build/s3/source.js +81 -0
- package/build/s3/source.js.map +1 -0
- package/build/s3/types.d.ts +66 -0
- package/build/s3/types.d.ts.map +1 -0
- package/build/s3/types.js +2 -0
- package/build/s3/types.js.map +1 -0
- package/build/s3/types.local.d.ts +43 -0
- package/build/s3/types.local.d.ts.map +1 -0
- package/build/s3/types.local.js +2 -0
- package/build/s3/types.local.js.map +1 -0
- package/build/s3/upload.d.ts +4 -0
- package/build/s3/upload.d.ts.map +1 -0
- package/build/s3/upload.js +118 -0
- package/build/s3/upload.js.map +1 -0
- package/build/s3.d.ts +5 -0
- package/build/s3.d.ts.map +1 -0
- package/build/s3.js +68 -0
- package/build/s3.js.map +1 -0
- package/build/service/types.d.ts +66 -0
- package/build/service/types.d.ts.map +1 -0
- package/build/service/types.js +2 -0
- package/build/service/types.js.map +1 -0
- package/build/service/types.local.d.ts +11 -0
- package/build/service/types.local.d.ts.map +1 -0
- package/build/service/types.local.js +2 -0
- package/build/service/types.local.js.map +1 -0
- package/build/service.d.ts +6 -0
- package/build/service.d.ts.map +1 -0
- package/build/service.js +73 -0
- package/build/service.js.map +1 -0
- package/build/sinks/buffer/consts.d.ts +2 -0
- package/build/sinks/buffer/consts.d.ts.map +1 -0
- package/build/sinks/buffer/consts.js +2 -0
- package/build/sinks/buffer/consts.js.map +1 -0
- package/build/sinks/buffer/types.d.ts +11 -0
- package/build/sinks/buffer/types.d.ts.map +1 -0
- package/build/sinks/buffer/types.js +2 -0
- package/build/sinks/buffer/types.js.map +1 -0
- package/build/sinks/buffer.d.ts +4 -0
- package/build/sinks/buffer.d.ts.map +1 -0
- package/build/sinks/buffer.js +13 -0
- package/build/sinks/buffer.js.map +1 -0
- package/build/sinks/directory/types.d.ts +7 -0
- package/build/sinks/directory/types.d.ts.map +1 -0
- package/build/sinks/directory/types.js +2 -0
- package/build/sinks/directory/types.js.map +1 -0
- package/build/sinks/directory.d.ts +8 -0
- package/build/sinks/directory.d.ts.map +1 -0
- package/build/sinks/directory.js +43 -0
- package/build/sinks/directory.js.map +1 -0
- package/build/sinks/file.d.ts +5 -0
- package/build/sinks/file.d.ts.map +1 -0
- package/build/sinks/file.js +46 -0
- package/build/sinks/file.js.map +1 -0
- package/build/sinks/types.d.ts +11 -0
- package/build/sinks/types.d.ts.map +1 -0
- package/build/sinks/types.js +2 -0
- package/build/sinks/types.js.map +1 -0
- package/build/sinks/workload/consts.d.ts +5 -0
- package/build/sinks/workload/consts.d.ts.map +1 -0
- package/build/sinks/workload/consts.js +5 -0
- package/build/sinks/workload/consts.js.map +1 -0
- package/build/sinks/workload/types.d.ts +17 -0
- package/build/sinks/workload/types.d.ts.map +1 -0
- package/build/sinks/workload/types.js +2 -0
- package/build/sinks/workload/types.js.map +1 -0
- package/build/sinks/workload.d.ts +8 -0
- package/build/sinks/workload.d.ts.map +1 -0
- package/build/sinks/workload.js +51 -0
- package/build/sinks/workload.js.map +1 -0
- package/build/sources/buffer.d.ts +5 -0
- package/build/sources/buffer.d.ts.map +1 -0
- package/build/sources/buffer.js +20 -0
- package/build/sources/buffer.js.map +1 -0
- package/build/sources/file.d.ts +5 -0
- package/build/sources/file.d.ts.map +1 -0
- package/build/sources/file.js +50 -0
- package/build/sources/file.js.map +1 -0
- package/build/sources/stream.d.ts +5 -0
- package/build/sources/stream.d.ts.map +1 -0
- package/build/sources/stream.js +19 -0
- package/build/sources/stream.js.map +1 -0
- package/build/sources/types.d.ts +10 -0
- package/build/sources/types.d.ts.map +1 -0
- package/build/sources/types.js +2 -0
- package/build/sources/types.js.map +1 -0
- package/build/sources/workload/types.d.ts +29 -0
- package/build/sources/workload/types.d.ts.map +1 -0
- package/build/sources/workload/types.js +2 -0
- package/build/sources/workload/types.js.map +1 -0
- package/build/sources/workload.d.ts +7 -0
- package/build/sources/workload.d.ts.map +1 -0
- package/build/sources/workload.js +67 -0
- package/build/sources/workload.js.map +1 -0
- package/build/transfer/consts.d.ts +5 -0
- package/build/transfer/consts.d.ts.map +1 -0
- package/build/transfer/consts.js +8 -0
- package/build/transfer/consts.js.map +1 -0
- package/build/transfer/types.d.ts +49 -0
- package/build/transfer/types.d.ts.map +1 -0
- package/build/transfer/types.js +2 -0
- package/build/transfer/types.js.map +1 -0
- package/build/transfer.d.ts +4 -0
- package/build/transfer.d.ts.map +1 -0
- package/build/transfer.js +75 -0
- package/build/transfer.js.map +1 -0
- package/build/types.d.ts +99 -0
- package/build/types.d.ts.map +1 -0
- package/build/types.js +2 -0
- package/build/types.js.map +1 -0
- package/build/utils/names/consts.d.ts +5 -0
- package/build/utils/names/consts.d.ts.map +1 -0
- package/build/utils/names/consts.js +5 -0
- package/build/utils/names/consts.js.map +1 -0
- package/build/utils/names/types.d.ts +13 -0
- package/build/utils/names/types.d.ts.map +1 -0
- package/build/utils/names/types.js +2 -0
- package/build/utils/names/types.js.map +1 -0
- package/build/utils/names.d.ts +4 -0
- package/build/utils/names.d.ts.map +1 -0
- package/build/utils/names.js +60 -0
- package/build/utils/names.js.map +1 -0
- package/build/utils/paths/consts.d.ts +3 -0
- package/build/utils/paths/consts.d.ts.map +1 -0
- package/build/utils/paths/consts.js +2 -0
- package/build/utils/paths/consts.js.map +1 -0
- package/build/utils/paths/types.d.ts +20 -0
- package/build/utils/paths/types.d.ts.map +1 -0
- package/build/utils/paths/types.js +2 -0
- package/build/utils/paths/types.js.map +1 -0
- package/build/utils/paths.d.ts +4 -0
- package/build/utils/paths.d.ts.map +1 -0
- package/build/utils/paths.js +67 -0
- package/build/utils/paths.js.map +1 -0
- package/build/utils/range/types.d.ts +11 -0
- package/build/utils/range/types.d.ts.map +1 -0
- package/build/utils/range/types.js +2 -0
- package/build/utils/range/types.js.map +1 -0
- package/build/utils/range.d.ts +4 -0
- package/build/utils/range.d.ts.map +1 -0
- package/build/utils/range.js +64 -0
- package/build/utils/range.js.map +1 -0
- package/build/utils/redact/types.d.ts +16 -0
- package/build/utils/redact/types.d.ts.map +1 -0
- package/build/utils/redact/types.js +2 -0
- package/build/utils/redact/types.js.map +1 -0
- package/build/utils/redact.d.ts +4 -0
- package/build/utils/redact.d.ts.map +1 -0
- package/build/utils/redact.js +18 -0
- package/build/utils/redact.js.map +1 -0
- package/build/utils/signals/types.d.ts +17 -0
- package/build/utils/signals/types.d.ts.map +1 -0
- package/build/utils/signals/types.js +2 -0
- package/build/utils/signals/types.js.map +1 -0
- package/build/utils/signals.d.ts +4 -0
- package/build/utils/signals.d.ts.map +1 -0
- package/build/utils/signals.js +52 -0
- package/build/utils/signals.js.map +1 -0
- package/build/utils/streams/consts.d.ts +3 -0
- package/build/utils/streams/consts.d.ts.map +1 -0
- package/build/utils/streams/consts.js +3 -0
- package/build/utils/streams/consts.js.map +1 -0
- package/build/utils/streams/types.d.ts +59 -0
- package/build/utils/streams/types.d.ts.map +1 -0
- package/build/utils/streams/types.js +2 -0
- package/build/utils/streams/types.js.map +1 -0
- package/build/utils/streams.d.ts +4 -0
- package/build/utils/streams.d.ts.map +1 -0
- package/build/utils/streams.js +167 -0
- package/build/utils/streams.js.map +1 -0
- package/build/utils/tree/types.d.ts +6 -0
- package/build/utils/tree/types.d.ts.map +1 -0
- package/build/utils/tree/types.js +2 -0
- package/build/utils/tree/types.js.map +1 -0
- package/build/utils/tree.d.ts +4 -0
- package/build/utils/tree.d.ts.map +1 -0
- package/build/utils/tree.js +14 -0
- package/build/utils/tree.js.map +1 -0
- package/build/workspace/consts.d.ts +12 -0
- package/build/workspace/consts.d.ts.map +1 -0
- package/build/workspace/consts.js +12 -0
- package/build/workspace/consts.js.map +1 -0
- package/build/workspace/consts.local.d.ts +3 -0
- package/build/workspace/consts.local.d.ts.map +1 -0
- package/build/workspace/consts.local.js +3 -0
- package/build/workspace/consts.local.js.map +1 -0
- package/build/workspace/fs.d.ts +4 -0
- package/build/workspace/fs.d.ts.map +1 -0
- package/build/workspace/fs.js +45 -0
- package/build/workspace/fs.js.map +1 -0
- package/build/workspace/scope.d.ts +6 -0
- package/build/workspace/scope.d.ts.map +1 -0
- package/build/workspace/scope.js +180 -0
- package/build/workspace/scope.js.map +1 -0
- package/build/workspace/types.d.ts +66 -0
- package/build/workspace/types.d.ts.map +1 -0
- package/build/workspace/types.js +2 -0
- package/build/workspace/types.js.map +1 -0
- package/build/workspace/types.local.d.ts +20 -0
- package/build/workspace/types.local.d.ts.map +1 -0
- package/build/workspace/types.local.js +2 -0
- package/build/workspace/types.local.js.map +1 -0
- package/build/workspace.d.ts +7 -0
- package/build/workspace.d.ts.map +1 -0
- package/build/workspace.js +87 -0
- package/build/workspace.js.map +1 -0
- package/package.json +61 -4
- package/src/archive/consts.ts +28 -0
- package/src/archive/formats.ts +60 -0
- package/src/archive/guard.ts +68 -0
- package/src/archive/ratio.ts +35 -0
- package/src/archive/tar.ts +147 -0
- package/src/archive/types.local.ts +72 -0
- package/src/archive/types.ts +107 -0
- package/src/archive/zip.ts +142 -0
- package/src/archive.ts +298 -0
- package/src/backed-workspace/types.ts +72 -0
- package/src/backed-workspace.ts +85 -0
- package/src/consts.local.ts +2 -0
- package/src/consts.ts +2 -0
- package/src/errors.ts +128 -0
- package/src/http/connector.ts +136 -0
- package/src/http/consts.ts +13 -0
- package/src/http/source.ts +89 -0
- package/src/http/types.local.ts +23 -0
- package/src/http/types.ts +74 -0
- package/src/http.ts +276 -0
- package/src/index.ts +65 -0
- package/src/network/consts.ts +22 -0
- package/src/network/types.ts +45 -0
- package/src/network.ts +179 -0
- package/src/proxy/consts.ts +11 -0
- package/src/proxy/endpoint.ts +28 -0
- package/src/proxy/http.ts +108 -0
- package/src/proxy/router.ts +93 -0
- package/src/proxy/socks.ts +48 -0
- package/src/proxy/types.ts +114 -0
- package/src/resolvers/google-drive/consts.ts +22 -0
- package/src/resolvers/google-drive/types.ts +22 -0
- package/src/resolvers/google-drive.ts +153 -0
- package/src/resolvers/types.ts +45 -0
- package/src/s3/consts.ts +12 -0
- package/src/s3/prefix.ts +100 -0
- package/src/s3/sink.ts +36 -0
- package/src/s3/source.ts +90 -0
- package/src/s3/types.local.ts +49 -0
- package/src/s3/types.ts +70 -0
- package/src/s3/upload.ts +130 -0
- package/src/s3.ts +83 -0
- package/src/service/types.local.ts +9 -0
- package/src/service/types.ts +71 -0
- package/src/service.ts +97 -0
- package/src/sinks/buffer/consts.ts +1 -0
- package/src/sinks/buffer/types.ts +12 -0
- package/src/sinks/buffer.ts +18 -0
- package/src/sinks/directory/types.ts +7 -0
- package/src/sinks/directory.ts +49 -0
- package/src/sinks/file.ts +52 -0
- package/src/sinks/types.ts +12 -0
- package/src/sinks/workload/consts.ts +5 -0
- package/src/sinks/workload/types.ts +17 -0
- package/src/sinks/workload.ts +59 -0
- package/src/sources/buffer.ts +27 -0
- package/src/sources/file.ts +57 -0
- package/src/sources/stream.ts +26 -0
- package/src/sources/types.ts +12 -0
- package/src/sources/workload/types.ts +31 -0
- package/src/sources/workload.ts +79 -0
- package/src/transfer/consts.ts +10 -0
- package/src/transfer/types.ts +55 -0
- package/src/transfer.ts +89 -0
- package/src/types.ts +106 -0
- package/src/utils/names/consts.ts +5 -0
- package/src/utils/names/types.ts +12 -0
- package/src/utils/names.ts +70 -0
- package/src/utils/paths/consts.ts +3 -0
- package/src/utils/paths/types.ts +20 -0
- package/src/utils/paths.ts +75 -0
- package/src/utils/range/types.ts +11 -0
- package/src/utils/range.ts +80 -0
- package/src/utils/redact/types.ts +16 -0
- package/src/utils/redact.ts +23 -0
- package/src/utils/signals/types.ts +17 -0
- package/src/utils/signals.ts +63 -0
- package/src/utils/streams/consts.ts +2 -0
- package/src/utils/streams/types.ts +65 -0
- package/src/utils/streams.ts +188 -0
- package/src/utils/tree/types.ts +6 -0
- package/src/utils/tree.ts +18 -0
- package/src/workspace/consts.local.ts +2 -0
- package/src/workspace/consts.ts +14 -0
- package/src/workspace/fs.ts +48 -0
- package/src/workspace/scope.ts +196 -0
- package/src/workspace/types.local.ts +21 -0
- package/src/workspace/types.ts +71 -0
- package/src/workspace.ts +102 -0
- package/tests/archive.spec.ts +134 -0
- package/tests/backed.spec.ts +147 -0
- package/tests/context.ts +114 -0
- package/tests/fixtures.ts +190 -0
- package/tests/google-drive.spec.ts +76 -0
- package/tests/harness/drive.ts +92 -0
- package/tests/harness/http-proxy.ts +96 -0
- package/tests/harness/origin.ts +134 -0
- package/tests/harness/socks5.ts +135 -0
- package/tests/harness/workload.ts +118 -0
- package/tests/http.spec.ts +101 -0
- package/tests/inspector.spec.ts +40 -0
- package/tests/live.spec.ts +27 -0
- package/tests/network.spec.ts +79 -0
- package/tests/proxy.spec.ts +149 -0
- package/tests/s3.spec.ts +133 -0
- package/tests/service.spec.ts +77 -0
- package/tests/sources-sinks.spec.ts +96 -0
- package/tests/transfer.spec.ts +99 -0
- package/tests/tsconfig.json +12 -0
- package/tests/workload.spec.ts +140 -0
- package/tests/workspace.spec.ts +158 -0
- package/tsconfig.json +16 -0
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import { afterAll, beforeAll, describe, expect, test } from 'bun:test'
|
|
2
|
+
import { mkdir, readFile, writeFile } from 'node:fs/promises'
|
|
3
|
+
import { join } from 'node:path'
|
|
4
|
+
import { Readable } from 'node:stream'
|
|
5
|
+
import {
|
|
6
|
+
archiveHelper, createBufferSource, createDirectorySink, createStreamSource, createWorkspace, fileInspector, FileKind, makeWorkloadSink,
|
|
7
|
+
makeWorkloadSource, transferHelper, WORKLOAD_MAX_FILE_BYTES, WorkloadError,
|
|
8
|
+
} from '../src/index.js'
|
|
9
|
+
import { streamUtils } from '../src/utils/streams.js'
|
|
10
|
+
import { docxOf, noise, PNG, tgzOf, zipOf } from './fixtures.js'
|
|
11
|
+
import { startOrigin } from './harness/origin.js'
|
|
12
|
+
import type { Origin } from './harness/origin.js'
|
|
13
|
+
import { startWorkload, workloadReader, workloadWriter } from './harness/workload.js'
|
|
14
|
+
import type { LocalWorkload } from './harness/workload.js'
|
|
15
|
+
import { localClient, removeDir, tempDir } from './context.js'
|
|
16
|
+
|
|
17
|
+
describe('@owlmeans/file-streams — Viable workload destination', () => {
|
|
18
|
+
let origin: Origin
|
|
19
|
+
let sandbox: string
|
|
20
|
+
let workload: LocalWorkload
|
|
21
|
+
beforeAll(async () => {
|
|
22
|
+
origin = startOrigin()
|
|
23
|
+
origin.files.set('hero.png', { bytes: PNG, type: 'image/png' })
|
|
24
|
+
sandbox = await tempDir('sandbox')
|
|
25
|
+
workload = startWorkload(sandbox)
|
|
26
|
+
})
|
|
27
|
+
afterAll(async () => {
|
|
28
|
+
origin.stop()
|
|
29
|
+
workload.stop()
|
|
30
|
+
await removeDir(sandbox)
|
|
31
|
+
})
|
|
32
|
+
|
|
33
|
+
test('downloads an image into the target project through the writeBinaryFile contract', async () => {
|
|
34
|
+
const sink = makeWorkloadSink(workloadWriter(workload.url), { root: 'sources/web/public/media' })
|
|
35
|
+
const result = await transferHelper.copy(localClient().source(origin.url('/files/hero.png')), sink.entry('hero.png'), {
|
|
36
|
+
verify: { allow: [FileKind.Image] },
|
|
37
|
+
})
|
|
38
|
+
expect(result.sink).toMatchObject({ kind: 'workload', location: 'sources/web/public/media/hero.png', bytes: PNG.length })
|
|
39
|
+
expect(await readFile(join(sandbox, 'sources/web/public/media/hero.png'))).toEqual(PNG)
|
|
40
|
+
})
|
|
41
|
+
|
|
42
|
+
test('takes exactly its ceiling and refuses one byte more, before or during the read', async () => {
|
|
43
|
+
const lowered = makeWorkloadSink(workloadWriter(workload.url), { root: 'limits', maxFileBytes: 1024 })
|
|
44
|
+
await expect(lowered.entry('lowered.bin').write({ stream: Readable.from([noise(1025)]), info: { origin: 'test' } }))
|
|
45
|
+
.rejects.toThrow('workload:too-large')
|
|
46
|
+
const sink = makeWorkloadSink(workloadWriter(workload.url), { root: 'limits' })
|
|
47
|
+
const exact = noise(WORKLOAD_MAX_FILE_BYTES)
|
|
48
|
+
await sink.entry('exact.bin').write({ stream: Readable.from([exact]), info: { origin: 'test', size: exact.length } })
|
|
49
|
+
expect((await readFile(join(sandbox, 'limits/exact.bin'))).length).toBe(WORKLOAD_MAX_FILE_BYTES)
|
|
50
|
+
|
|
51
|
+
const before = workload.commands
|
|
52
|
+
const over = noise(WORKLOAD_MAX_FILE_BYTES + 1)
|
|
53
|
+
await expect(transferHelper.copy(createBufferSource(over, { name: 'over.bin' }), sink.entry('over.bin'), { verify: false }))
|
|
54
|
+
.rejects.toBeInstanceOf(WorkloadError)
|
|
55
|
+
await expect(transferHelper.copy(createStreamSource(Readable.from([over.subarray(0, 8_000_000), over.subarray(8_000_000)])), sink.entry('over-stream.bin'), { verify: false }))
|
|
56
|
+
.rejects.toThrow('workload:too-large')
|
|
57
|
+
expect(workload.commands).toBe(before)
|
|
58
|
+
}, 60_000)
|
|
59
|
+
|
|
60
|
+
test('fans an archive out into one atomic write per file, skipping what does not fit', async () => {
|
|
61
|
+
const archive = await tgzOf([
|
|
62
|
+
{ name: 'media/cover.png', data: PNG },
|
|
63
|
+
{ name: 'media/clip.bin', data: noise(13 * 1024 * 1024) },
|
|
64
|
+
{ name: 'media/notes.txt', data: 'notes' },
|
|
65
|
+
])
|
|
66
|
+
const result = await archiveHelper.extract(createBufferSource(archive, { name: 'media.tar.gz' }), makeWorkloadSink(workloadWriter(workload.url), { root: 'assets' }), {
|
|
67
|
+
onEntryError: 'skip',
|
|
68
|
+
})
|
|
69
|
+
expect(result.extracted.map(entry => entry.path)).toEqual(['media/cover.png', 'media/notes.txt'])
|
|
70
|
+
expect(result.skipped.map(entry => [entry.entry.path, entry.reason])).toEqual([['media/clip.bin', 'file-streams:workload:too-large']])
|
|
71
|
+
expect(await readFile(join(sandbox, 'assets/media/notes.txt'), 'utf8')).toBe('notes')
|
|
72
|
+
|
|
73
|
+
const sink = makeWorkloadSink(workloadWriter(workload.url))
|
|
74
|
+
expect(() => sink.entry('../outside.txt')).toThrow('path:traversal')
|
|
75
|
+
}, 60_000)
|
|
76
|
+
|
|
77
|
+
test('retries a failed write once', async () => {
|
|
78
|
+
const before = workload.commands
|
|
79
|
+
workload.failNext(1)
|
|
80
|
+
const sink = makeWorkloadSink(workloadWriter(workload.url), { root: 'retry' })
|
|
81
|
+
await sink.entry('again.txt').write({ stream: Readable.from([Buffer.from('second time lucky')]), info: { origin: 'test' } })
|
|
82
|
+
expect(workload.commands - before).toBe(2)
|
|
83
|
+
workload.failNext(2)
|
|
84
|
+
await expect(sink.entry('never.txt').write({ stream: Readable.from([Buffer.from('x')]), info: { origin: 'test' } })).rejects.toThrow('workload:write-failed')
|
|
85
|
+
})
|
|
86
|
+
|
|
87
|
+
test('reads a placed file back through the readBinaryFile contract: re-openable, sized after the first read, ranged', async () => {
|
|
88
|
+
const payload = noise(300_000, 7)
|
|
89
|
+
await makeWorkloadSink(workloadWriter(workload.url), { root: 'docs/materials' }).entry('notes/payload.bin')
|
|
90
|
+
.write({ stream: Readable.from([payload]), info: { origin: 'test' } })
|
|
91
|
+
const materials = makeWorkloadSource(workloadReader(workload.url), { root: 'docs/materials' })
|
|
92
|
+
const source = materials.entry('notes/payload.bin')
|
|
93
|
+
expect(await source.info()).toMatchObject({ origin: 'docs/materials/notes/payload.bin', name: 'payload.bin' })
|
|
94
|
+
expect((await source.info()).size).toBeUndefined()
|
|
95
|
+
|
|
96
|
+
const before = workload.commands
|
|
97
|
+
expect((await streamUtils.collect((await source.open()).stream, 1_000_000)).equals(payload)).toBe(true)
|
|
98
|
+
expect((await source.info()).size).toBe(payload.length)
|
|
99
|
+
const ranged = await source.open({ range: { start: 10, end: 19 } })
|
|
100
|
+
expect(ranged.range).toEqual({ start: 10, end: 19, total: payload.length })
|
|
101
|
+
expect((await streamUtils.collect(ranged.stream, 100)).equals(payload.subarray(10, 20))).toBe(true)
|
|
102
|
+
expect(workload.commands - before).toBe(2)
|
|
103
|
+
})
|
|
104
|
+
|
|
105
|
+
test('copies a workload file into agent tmp, verified, and reads an archive from the workload by random access', async () => {
|
|
106
|
+
await mkdir(join(sandbox, 'assets'), { recursive: true })
|
|
107
|
+
await writeFile(join(sandbox, 'assets/logo.png'), PNG)
|
|
108
|
+
await writeFile(join(sandbox, 'assets/brief.docx'), await docxOf())
|
|
109
|
+
await writeFile(join(sandbox, 'assets/kit.zip'), await zipOf([{ name: 'kit/a.png', data: PNG }, { name: 'kit/b.txt', data: 'b' }]))
|
|
110
|
+
const assets = makeWorkloadSource(workloadReader(workload.url), { root: 'assets' })
|
|
111
|
+
|
|
112
|
+
const root = await tempDir('workload-tmp')
|
|
113
|
+
try {
|
|
114
|
+
const local = createWorkspace({ root }).scope({ organization: 'acme', project: 'slot-1', purpose: 'materials' })
|
|
115
|
+
const copied = await transferHelper.copy(assets.entry('logo.png'), local.entry('logo.png'), { verify: { allow: [FileKind.Image] } })
|
|
116
|
+
expect(copied.verdict?.detected?.kind).toBe(FileKind.Image)
|
|
117
|
+
expect(await readFile(join(local.dir, 'logo.png'))).toEqual(PNG)
|
|
118
|
+
|
|
119
|
+
expect(await fileInspector.probe(assets.entry('brief.docx'))).toMatchObject({ status: 'match', detected: { ext: 'docx' } })
|
|
120
|
+
const extracted = await archiveHelper.extract(assets.entry('kit.zip'), createDirectorySink(join(root, 'kit')))
|
|
121
|
+
expect(extracted.extracted.map(entry => entry.path)).toEqual(['kit/a.png', 'kit/b.txt'])
|
|
122
|
+
} finally {
|
|
123
|
+
await removeDir(root)
|
|
124
|
+
}
|
|
125
|
+
})
|
|
126
|
+
|
|
127
|
+
test('refuses a file above its ceiling, a missing file after one retry, and an escaping path', async () => {
|
|
128
|
+
await mkdir(join(sandbox, 'big'), { recursive: true })
|
|
129
|
+
await writeFile(join(sandbox, 'big/one-k.bin'), noise(1025))
|
|
130
|
+
const lowered = makeWorkloadSource(workloadReader(workload.url), { root: 'big', maxFileBytes: 1024 })
|
|
131
|
+
await expect(lowered.entry('one-k.bin').open()).rejects.toThrow('workload:too-large')
|
|
132
|
+
|
|
133
|
+
const before = workload.commands
|
|
134
|
+
const source = makeWorkloadSource(workloadReader(workload.url))
|
|
135
|
+
await expect(source.entry('missing/nothing.bin').open()).rejects.toThrow('workload:read-failed')
|
|
136
|
+
expect(workload.commands - before).toBe(2)
|
|
137
|
+
await expect(source.entry('missing/nothing.bin').open()).rejects.toBeInstanceOf(WorkloadError)
|
|
138
|
+
expect(() => source.entry('../outside.txt')).toThrow('path:traversal')
|
|
139
|
+
})
|
|
140
|
+
})
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, test } from 'bun:test'
|
|
2
|
+
import { mkdir, readFile, stat, symlink, utimes, writeFile } from 'node:fs/promises'
|
|
3
|
+
import { tmpdir } from 'node:os'
|
|
4
|
+
import { join } from 'node:path'
|
|
5
|
+
import { Readable } from 'node:stream'
|
|
6
|
+
import {
|
|
7
|
+
archiveHelper, createBufferSource, createWorkspace, FileKind, FileScopeError, PathError, transferHelper, WORKSPACE_SCOPE_MARKER,
|
|
8
|
+
} from '../src/index.js'
|
|
9
|
+
import type { FileScope, Workspace, WorkspaceScope } from '../src/index.js'
|
|
10
|
+
import { streamUtils } from '../src/utils/streams.js'
|
|
11
|
+
import { PNG, tgzOf } from './fixtures.js'
|
|
12
|
+
import { removeDir, tempDir } from './context.js'
|
|
13
|
+
|
|
14
|
+
const write = async (local: WorkspaceScope, path: string, data: string | Buffer): Promise<void> => {
|
|
15
|
+
await local.entry(path).write({ stream: Readable.from([Buffer.from(data)]), info: { origin: 'test' } })
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
const exists = async (path: string): Promise<boolean> => await stat(path).then(() => true, () => false)
|
|
19
|
+
|
|
20
|
+
describe('@owlmeans/file-streams — workspace', () => {
|
|
21
|
+
let root: string
|
|
22
|
+
let workspace: Workspace
|
|
23
|
+
const project: FileScope = { organization: 'acme', project: 'slot-1', purpose: 'new-project' }
|
|
24
|
+
const upload: FileScope = { organization: 'acme', purpose: 'upload' }
|
|
25
|
+
|
|
26
|
+
beforeEach(async () => {
|
|
27
|
+
root = await tempDir('workspace')
|
|
28
|
+
workspace = createWorkspace({ root })
|
|
29
|
+
})
|
|
30
|
+
afterEach(async () => { await removeDir(root) })
|
|
31
|
+
|
|
32
|
+
test('nests scope folders by organization, project and purpose, and finds them again', async () => {
|
|
33
|
+
expect(createWorkspace().root).toBe(join(tmpdir(), 'owlmeans-workspace'))
|
|
34
|
+
expect(createWorkspace({ root: 'service/files' }).root).toBe(join(tmpdir(), 'service/files'))
|
|
35
|
+
const local = workspace.scope(project)
|
|
36
|
+
expect(local.dir).toBe(join(root, 'acme/slot-1/new-project'))
|
|
37
|
+
expect(workspace.scope(upload).dir).toBe(join(root, 'acme/upload'))
|
|
38
|
+
expect(await local.list()).toEqual([])
|
|
39
|
+
|
|
40
|
+
const copied = await transferHelper.copy(createBufferSource(PNG, { name: 'logo.png' }), local.entry('brand/logo.png'), {
|
|
41
|
+
verify: { allow: [FileKind.Image] },
|
|
42
|
+
})
|
|
43
|
+
expect(copied.sink).toMatchObject({ kind: 'file', location: join(local.dir, 'brand/logo.png'), bytes: PNG.length })
|
|
44
|
+
const archive = await tgzOf([{ name: 'kit/a.txt', data: 'a' }, { name: 'kit/deep/b.txt', data: 'bb' }])
|
|
45
|
+
await archiveHelper.extract(createBufferSource(archive, { name: 'kit.tar.gz' }), local.tree())
|
|
46
|
+
await write(workspace.scope(upload), 'note.md', '# note')
|
|
47
|
+
|
|
48
|
+
expect((await local.list()).map(file => [file.path, file.size])).toEqual([['brand/logo.png', PNG.length], ['kit/a.txt', 1], ['kit/deep/b.txt', 2]])
|
|
49
|
+
expect((await local.list())[0].modifiedAt).toBeInstanceOf(Date)
|
|
50
|
+
expect(await local.has('kit/deep/b.txt')).toBe(true)
|
|
51
|
+
expect(await local.has('kit/deep')).toBe(false)
|
|
52
|
+
expect(await streamUtils.collect((await local.source('brand/logo.png').open()).stream, 10_000)).toEqual(PNG)
|
|
53
|
+
expect(await exists(join(local.dir, WORKSPACE_SCOPE_MARKER))).toBe(true)
|
|
54
|
+
|
|
55
|
+
expect(await workspace.scopes()).toEqual([project, upload])
|
|
56
|
+
expect(await workspace.scopes('acme')).toEqual([project, upload])
|
|
57
|
+
expect(await workspace.scopes('other')).toEqual([])
|
|
58
|
+
expect(() => workspace.scope({ organization: '..', purpose: 'x' })).toThrow(FileScopeError)
|
|
59
|
+
await expect(workspace.scopes('../etc')).rejects.toThrow('stored-file:scope:organization')
|
|
60
|
+
})
|
|
61
|
+
|
|
62
|
+
test('confines every path: traversal, absolute paths, the marker name and symlinks are refused', async () => {
|
|
63
|
+
const local = workspace.scope(project)
|
|
64
|
+
await write(local, 'inside.txt', 'in')
|
|
65
|
+
expect(() => local.entry('../escape.txt')).toThrow('path:traversal')
|
|
66
|
+
expect(() => local.entry('/etc/passwd')).toThrow('path:absolute')
|
|
67
|
+
expect(() => local.source('a/../../b')).toThrow(PathError)
|
|
68
|
+
expect(() => local.entry(WORKSPACE_SCOPE_MARKER)).toThrow('path:reserved')
|
|
69
|
+
expect(() => local.entry(`media/${WORKSPACE_SCOPE_MARKER}`)).toThrow('path:reserved')
|
|
70
|
+
|
|
71
|
+
const outside = await tempDir('outside')
|
|
72
|
+
try {
|
|
73
|
+
await writeFile(join(outside, 'secret.txt'), 'secret')
|
|
74
|
+
await symlink(outside, join(local.dir, 'linked'))
|
|
75
|
+
await symlink(join(outside, 'secret.txt'), join(local.dir, 'secret.txt'))
|
|
76
|
+
await expect(write(local, 'linked/planted.txt', 'x')).rejects.toThrow('path:symlink')
|
|
77
|
+
expect(await exists(join(outside, 'planted.txt'))).toBe(false)
|
|
78
|
+
await expect(local.source('secret.txt').open()).rejects.toThrow('path:symlink')
|
|
79
|
+
await expect(local.has('linked/secret.txt')).rejects.toThrow('path:symlink')
|
|
80
|
+
await expect(local.remove('linked/secret.txt')).rejects.toThrow('path:symlink')
|
|
81
|
+
expect((await local.list()).map(file => file.path)).toEqual(['inside.txt'])
|
|
82
|
+
|
|
83
|
+
await mkdir(join(root, 'evil'))
|
|
84
|
+
await symlink(outside, join(root, 'evil/slot-2'))
|
|
85
|
+
await expect(write(workspace.scope({ organization: 'evil', project: 'slot-2', purpose: 'p' }), 'a.txt', 'a')).rejects.toThrow('path:symlink')
|
|
86
|
+
expect(await exists(join(outside, 'p'))).toBe(false)
|
|
87
|
+
} finally {
|
|
88
|
+
await removeDir(outside)
|
|
89
|
+
}
|
|
90
|
+
})
|
|
91
|
+
|
|
92
|
+
test('removes files and folders, drops a scope with the parents it empties, and keeps scopes from overlapping', async () => {
|
|
93
|
+
const local = workspace.scope(project)
|
|
94
|
+
await write(local, 'a/b/c.txt', 'c')
|
|
95
|
+
await write(local, 'd.txt', 'd')
|
|
96
|
+
await expect(write(local, 'd.txt', 'again')).rejects.toThrow('path:exists')
|
|
97
|
+
await local.tree({ onConflict: 'overwrite' }).entry('d.txt').write({ stream: Readable.from([Buffer.from('dd')]), info: { origin: 'test' } })
|
|
98
|
+
expect(await readFile(join(local.dir, 'd.txt'), 'utf8')).toBe('dd')
|
|
99
|
+
|
|
100
|
+
expect(await local.remove('a/b/c.txt')).toBe(true)
|
|
101
|
+
expect(await exists(join(local.dir, 'a'))).toBe(false)
|
|
102
|
+
expect(await local.remove('a/b/c.txt')).toBe(false)
|
|
103
|
+
expect((await local.list()).map(file => file.path)).toEqual(['d.txt'])
|
|
104
|
+
|
|
105
|
+
const sibling = workspace.scope({ organization: 'acme', project: 'slot-1', purpose: 'other' })
|
|
106
|
+
await write(sibling, 'x.txt', 'x')
|
|
107
|
+
await local.drop()
|
|
108
|
+
expect(await exists(local.dir)).toBe(false)
|
|
109
|
+
expect(await exists(join(root, 'acme/slot-1'))).toBe(true)
|
|
110
|
+
await sibling.drop()
|
|
111
|
+
expect(await exists(join(root, 'acme'))).toBe(false)
|
|
112
|
+
expect(await exists(root)).toBe(true)
|
|
113
|
+
await sibling.drop()
|
|
114
|
+
|
|
115
|
+
// `acme/slot-1` cannot be both a purpose and a project folder.
|
|
116
|
+
await write(local, 'mine.txt', 'mine')
|
|
117
|
+
await expect(write(workspace.scope({ organization: 'acme', purpose: 'slot-1' }), 'x.txt', 'x')).rejects.toThrow('path:scope-overlap')
|
|
118
|
+
await local.drop()
|
|
119
|
+
await write(workspace.scope({ organization: 'acme', purpose: 'slot-1' }), 'x.txt', 'x')
|
|
120
|
+
await expect(write(local, 'mine.txt', 'mine')).rejects.toThrow('path:scope-overlap')
|
|
121
|
+
})
|
|
122
|
+
|
|
123
|
+
test('prunes the scopes untouched for longer than the threshold', async () => {
|
|
124
|
+
const stale = workspace.scope(project)
|
|
125
|
+
const fresh = workspace.scope(upload)
|
|
126
|
+
await write(stale, 'old/a.txt', 'a')
|
|
127
|
+
await write(fresh, 'b.txt', 'b')
|
|
128
|
+
const past = new Date(Date.now() - 2 * 3600_000)
|
|
129
|
+
for (const path of ['old/a.txt', 'old', WORKSPACE_SCOPE_MARKER, '']) {
|
|
130
|
+
await utimes(join(stale.dir, path), past, past)
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
expect(await workspace.prune({ olderThanMs: 3 * 3600_000 })).toBe(0)
|
|
134
|
+
expect(await workspace.prune({ olderThanMs: 3600_000 })).toBe(1)
|
|
135
|
+
expect(await workspace.scopes()).toEqual([upload])
|
|
136
|
+
expect(await exists(join(root, 'acme/slot-1'))).toBe(false)
|
|
137
|
+
expect((await fresh.list()).map(file => file.path)).toEqual(['b.txt'])
|
|
138
|
+
})
|
|
139
|
+
|
|
140
|
+
test('writes to two scopes concurrently while a third scope of the organization is dropped', async () => {
|
|
141
|
+
const first = workspace.scope(project)
|
|
142
|
+
const second = workspace.scope(upload)
|
|
143
|
+
const third = workspace.scope({ organization: 'acme', purpose: 'scratch' })
|
|
144
|
+
await write(third, 'tmp.txt', 'tmp')
|
|
145
|
+
|
|
146
|
+
const names = Array.from({ length: 20 }, (_, index) => `part-${String(index).padStart(2, '0')}.txt`)
|
|
147
|
+
await Promise.all([
|
|
148
|
+
...names.map(async name => { await write(first, `batch/${name}`, `first ${name}`) }),
|
|
149
|
+
...names.map(async name => { await write(second, name, `second ${name}`) }),
|
|
150
|
+
third.drop(),
|
|
151
|
+
])
|
|
152
|
+
|
|
153
|
+
expect((await first.list()).map(file => file.path)).toEqual(names.map(name => `batch/${name}`))
|
|
154
|
+
expect((await second.list()).map(file => file.path)).toEqual(names)
|
|
155
|
+
expect(await readFile(join(second.dir, names[7]), 'utf8')).toBe(`second ${names[7]}`)
|
|
156
|
+
expect(await workspace.scopes()).toEqual([project, upload])
|
|
157
|
+
})
|
|
158
|
+
})
|
package/tsconfig.json
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
"extends": [
|
|
3
|
+
"@owlmeans/dep-config/tsconfig.base.json",
|
|
4
|
+
"@owlmeans/dep-config/tsconfig.node.json"
|
|
5
|
+
],
|
|
6
|
+
"compilerOptions": {
|
|
7
|
+
"rootDir": "./src/",
|
|
8
|
+
"outDir": "./build/"
|
|
9
|
+
},
|
|
10
|
+
"exclude": [
|
|
11
|
+
"./dist/**/*",
|
|
12
|
+
"./build/**/*",
|
|
13
|
+
"./tests/**/*",
|
|
14
|
+
"./*.ts"
|
|
15
|
+
]
|
|
16
|
+
}
|