@owlmeans/file-streams 0.0.0-stage → 0.2.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +71 -2
- package/agent-meta/manifest.json +64 -0
- package/agent-meta/skills/file-streams/SKILL.md +183 -0
- package/agent-meta/skills/file-streams/parts/network.md +47 -0
- package/agent-meta/skills/file-streams/parts/workspaces.md +129 -0
- package/build/archive/consts.d.ts +16 -0
- package/build/archive/consts.d.ts.map +1 -0
- package/build/archive/consts.js +24 -0
- package/build/archive/consts.js.map +1 -0
- package/build/archive/formats.d.ts +4 -0
- package/build/archive/formats.d.ts.map +1 -0
- package/build/archive/formats.js +53 -0
- package/build/archive/formats.js.map +1 -0
- package/build/archive/guard.d.ts +5 -0
- package/build/archive/guard.d.ts.map +1 -0
- package/build/archive/guard.js +62 -0
- package/build/archive/guard.js.map +1 -0
- package/build/archive/ratio.d.ts +5 -0
- package/build/archive/ratio.d.ts.map +1 -0
- package/build/archive/ratio.js +30 -0
- package/build/archive/ratio.js.map +1 -0
- package/build/archive/tar.d.ts +4 -0
- package/build/archive/tar.d.ts.map +1 -0
- package/build/archive/tar.js +142 -0
- package/build/archive/tar.js.map +1 -0
- package/build/archive/types.d.ts +95 -0
- package/build/archive/types.d.ts.map +1 -0
- package/build/archive/types.js +2 -0
- package/build/archive/types.js.map +1 -0
- package/build/archive/types.local.d.ts +64 -0
- package/build/archive/types.local.d.ts.map +1 -0
- package/build/archive/types.local.js +2 -0
- package/build/archive/types.local.js.map +1 -0
- package/build/archive/zip.d.ts +4 -0
- package/build/archive/zip.d.ts.map +1 -0
- package/build/archive/zip.js +135 -0
- package/build/archive/zip.js.map +1 -0
- package/build/archive.d.ts +4 -0
- package/build/archive.d.ts.map +1 -0
- package/build/archive.js +265 -0
- package/build/archive.js.map +1 -0
- package/build/backed-workspace/types.d.ts +67 -0
- package/build/backed-workspace/types.d.ts.map +1 -0
- package/build/backed-workspace/types.js +2 -0
- package/build/backed-workspace/types.js.map +1 -0
- package/build/backed-workspace.d.ts +4 -0
- package/build/backed-workspace.d.ts.map +1 -0
- package/build/backed-workspace.js +63 -0
- package/build/backed-workspace.js.map +1 -0
- package/build/consts.d.ts +3 -0
- package/build/consts.d.ts.map +1 -0
- package/build/consts.js +3 -0
- package/build/consts.js.map +1 -0
- package/build/consts.local.d.ts +3 -0
- package/build/consts.local.d.ts.map +1 -0
- package/build/consts.local.js +3 -0
- package/build/consts.local.js.map +1 -0
- package/build/errors.d.ts +59 -0
- package/build/errors.d.ts.map +1 -0
- package/build/errors.js +105 -0
- package/build/errors.js.map +1 -0
- package/build/http/connector.d.ts +11 -0
- package/build/http/connector.d.ts.map +1 -0
- package/build/http/connector.js +124 -0
- package/build/http/connector.js.map +1 -0
- package/build/http/consts.d.ts +9 -0
- package/build/http/consts.d.ts.map +1 -0
- package/build/http/consts.js +8 -0
- package/build/http/consts.js.map +1 -0
- package/build/http/source.d.ts +5 -0
- package/build/http/source.d.ts.map +1 -0
- package/build/http/source.js +76 -0
- package/build/http/source.js.map +1 -0
- package/build/http/types.d.ts +67 -0
- package/build/http/types.d.ts.map +1 -0
- package/build/http/types.js +2 -0
- package/build/http/types.js.map +1 -0
- package/build/http/types.local.d.ts +21 -0
- package/build/http/types.local.d.ts.map +1 -0
- package/build/http/types.local.js +2 -0
- package/build/http/types.local.js.map +1 -0
- package/build/http.d.ts +4 -0
- package/build/http.d.ts.map +1 -0
- package/build/http.js +248 -0
- package/build/http.js.map +1 -0
- package/build/index.d.ts +53 -0
- package/build/index.d.ts.map +1 -0
- package/build/index.js +33 -0
- package/build/index.js.map +1 -0
- package/build/network/consts.d.ts +12 -0
- package/build/network/consts.d.ts.map +1 -0
- package/build/network/consts.js +20 -0
- package/build/network/consts.js.map +1 -0
- package/build/network/types.d.ts +43 -0
- package/build/network/types.d.ts.map +1 -0
- package/build/network/types.js +2 -0
- package/build/network/types.js.map +1 -0
- package/build/network.d.ts +4 -0
- package/build/network.d.ts.map +1 -0
- package/build/network.js +165 -0
- package/build/network.js.map +1 -0
- package/build/proxy/consts.d.ts +7 -0
- package/build/proxy/consts.d.ts.map +1 -0
- package/build/proxy/consts.js +9 -0
- package/build/proxy/consts.js.map +1 -0
- package/build/proxy/endpoint.d.ts +4 -0
- package/build/proxy/endpoint.d.ts.map +1 -0
- package/build/proxy/endpoint.js +27 -0
- package/build/proxy/endpoint.js.map +1 -0
- package/build/proxy/http.d.ts +5 -0
- package/build/proxy/http.d.ts.map +1 -0
- package/build/proxy/http.js +95 -0
- package/build/proxy/http.js.map +1 -0
- package/build/proxy/router.d.ts +4 -0
- package/build/proxy/router.d.ts.map +1 -0
- package/build/proxy/router.js +85 -0
- package/build/proxy/router.js.map +1 -0
- package/build/proxy/socks.d.ts +5 -0
- package/build/proxy/socks.d.ts.map +1 -0
- package/build/proxy/socks.js +44 -0
- package/build/proxy/socks.js.map +1 -0
- package/build/proxy/types.d.ts +99 -0
- package/build/proxy/types.d.ts.map +1 -0
- package/build/proxy/types.js +2 -0
- package/build/proxy/types.js.map +1 -0
- package/build/resolvers/google-drive/consts.d.ts +10 -0
- package/build/resolvers/google-drive/consts.d.ts.map +1 -0
- package/build/resolvers/google-drive/consts.js +16 -0
- package/build/resolvers/google-drive/consts.js.map +1 -0
- package/build/resolvers/google-drive/types.d.ts +21 -0
- package/build/resolvers/google-drive/types.d.ts.map +1 -0
- package/build/resolvers/google-drive/types.js +2 -0
- package/build/resolvers/google-drive/types.js.map +1 -0
- package/build/resolvers/google-drive.d.ts +10 -0
- package/build/resolvers/google-drive.d.ts.map +1 -0
- package/build/resolvers/google-drive.js +130 -0
- package/build/resolvers/google-drive.js.map +1 -0
- package/build/resolvers/types.d.ts +39 -0
- package/build/resolvers/types.d.ts.map +1 -0
- package/build/resolvers/types.js +2 -0
- package/build/resolvers/types.js.map +1 -0
- package/build/s3/consts.d.ts +9 -0
- package/build/s3/consts.d.ts.map +1 -0
- package/build/s3/consts.js +9 -0
- package/build/s3/consts.js.map +1 -0
- package/build/s3/prefix.d.ts +4 -0
- package/build/s3/prefix.d.ts.map +1 -0
- package/build/s3/prefix.js +87 -0
- package/build/s3/prefix.js.map +1 -0
- package/build/s3/sink.d.ts +6 -0
- package/build/s3/sink.d.ts.map +1 -0
- package/build/s3/sink.js +30 -0
- package/build/s3/sink.js.map +1 -0
- package/build/s3/source.d.ts +5 -0
- package/build/s3/source.d.ts.map +1 -0
- package/build/s3/source.js +81 -0
- package/build/s3/source.js.map +1 -0
- package/build/s3/types.d.ts +66 -0
- package/build/s3/types.d.ts.map +1 -0
- package/build/s3/types.js +2 -0
- package/build/s3/types.js.map +1 -0
- package/build/s3/types.local.d.ts +43 -0
- package/build/s3/types.local.d.ts.map +1 -0
- package/build/s3/types.local.js +2 -0
- package/build/s3/types.local.js.map +1 -0
- package/build/s3/upload.d.ts +4 -0
- package/build/s3/upload.d.ts.map +1 -0
- package/build/s3/upload.js +118 -0
- package/build/s3/upload.js.map +1 -0
- package/build/s3.d.ts +5 -0
- package/build/s3.d.ts.map +1 -0
- package/build/s3.js +68 -0
- package/build/s3.js.map +1 -0
- package/build/service/types.d.ts +66 -0
- package/build/service/types.d.ts.map +1 -0
- package/build/service/types.js +2 -0
- package/build/service/types.js.map +1 -0
- package/build/service/types.local.d.ts +11 -0
- package/build/service/types.local.d.ts.map +1 -0
- package/build/service/types.local.js +2 -0
- package/build/service/types.local.js.map +1 -0
- package/build/service.d.ts +6 -0
- package/build/service.d.ts.map +1 -0
- package/build/service.js +73 -0
- package/build/service.js.map +1 -0
- package/build/sinks/buffer/consts.d.ts +2 -0
- package/build/sinks/buffer/consts.d.ts.map +1 -0
- package/build/sinks/buffer/consts.js +2 -0
- package/build/sinks/buffer/consts.js.map +1 -0
- package/build/sinks/buffer/types.d.ts +11 -0
- package/build/sinks/buffer/types.d.ts.map +1 -0
- package/build/sinks/buffer/types.js +2 -0
- package/build/sinks/buffer/types.js.map +1 -0
- package/build/sinks/buffer.d.ts +4 -0
- package/build/sinks/buffer.d.ts.map +1 -0
- package/build/sinks/buffer.js +13 -0
- package/build/sinks/buffer.js.map +1 -0
- package/build/sinks/directory/types.d.ts +7 -0
- package/build/sinks/directory/types.d.ts.map +1 -0
- package/build/sinks/directory/types.js +2 -0
- package/build/sinks/directory/types.js.map +1 -0
- package/build/sinks/directory.d.ts +8 -0
- package/build/sinks/directory.d.ts.map +1 -0
- package/build/sinks/directory.js +43 -0
- package/build/sinks/directory.js.map +1 -0
- package/build/sinks/file.d.ts +5 -0
- package/build/sinks/file.d.ts.map +1 -0
- package/build/sinks/file.js +46 -0
- package/build/sinks/file.js.map +1 -0
- package/build/sinks/types.d.ts +11 -0
- package/build/sinks/types.d.ts.map +1 -0
- package/build/sinks/types.js +2 -0
- package/build/sinks/types.js.map +1 -0
- package/build/sinks/workload/consts.d.ts +5 -0
- package/build/sinks/workload/consts.d.ts.map +1 -0
- package/build/sinks/workload/consts.js +5 -0
- package/build/sinks/workload/consts.js.map +1 -0
- package/build/sinks/workload/types.d.ts +17 -0
- package/build/sinks/workload/types.d.ts.map +1 -0
- package/build/sinks/workload/types.js +2 -0
- package/build/sinks/workload/types.js.map +1 -0
- package/build/sinks/workload.d.ts +8 -0
- package/build/sinks/workload.d.ts.map +1 -0
- package/build/sinks/workload.js +51 -0
- package/build/sinks/workload.js.map +1 -0
- package/build/sources/buffer.d.ts +5 -0
- package/build/sources/buffer.d.ts.map +1 -0
- package/build/sources/buffer.js +20 -0
- package/build/sources/buffer.js.map +1 -0
- package/build/sources/file.d.ts +5 -0
- package/build/sources/file.d.ts.map +1 -0
- package/build/sources/file.js +50 -0
- package/build/sources/file.js.map +1 -0
- package/build/sources/stream.d.ts +5 -0
- package/build/sources/stream.d.ts.map +1 -0
- package/build/sources/stream.js +19 -0
- package/build/sources/stream.js.map +1 -0
- package/build/sources/types.d.ts +10 -0
- package/build/sources/types.d.ts.map +1 -0
- package/build/sources/types.js +2 -0
- package/build/sources/types.js.map +1 -0
- package/build/sources/workload/types.d.ts +29 -0
- package/build/sources/workload/types.d.ts.map +1 -0
- package/build/sources/workload/types.js +2 -0
- package/build/sources/workload/types.js.map +1 -0
- package/build/sources/workload.d.ts +7 -0
- package/build/sources/workload.d.ts.map +1 -0
- package/build/sources/workload.js +67 -0
- package/build/sources/workload.js.map +1 -0
- package/build/transfer/consts.d.ts +5 -0
- package/build/transfer/consts.d.ts.map +1 -0
- package/build/transfer/consts.js +8 -0
- package/build/transfer/consts.js.map +1 -0
- package/build/transfer/types.d.ts +49 -0
- package/build/transfer/types.d.ts.map +1 -0
- package/build/transfer/types.js +2 -0
- package/build/transfer/types.js.map +1 -0
- package/build/transfer.d.ts +4 -0
- package/build/transfer.d.ts.map +1 -0
- package/build/transfer.js +75 -0
- package/build/transfer.js.map +1 -0
- package/build/types.d.ts +99 -0
- package/build/types.d.ts.map +1 -0
- package/build/types.js +2 -0
- package/build/types.js.map +1 -0
- package/build/utils/names/consts.d.ts +5 -0
- package/build/utils/names/consts.d.ts.map +1 -0
- package/build/utils/names/consts.js +5 -0
- package/build/utils/names/consts.js.map +1 -0
- package/build/utils/names/types.d.ts +13 -0
- package/build/utils/names/types.d.ts.map +1 -0
- package/build/utils/names/types.js +2 -0
- package/build/utils/names/types.js.map +1 -0
- package/build/utils/names.d.ts +4 -0
- package/build/utils/names.d.ts.map +1 -0
- package/build/utils/names.js +60 -0
- package/build/utils/names.js.map +1 -0
- package/build/utils/paths/consts.d.ts +3 -0
- package/build/utils/paths/consts.d.ts.map +1 -0
- package/build/utils/paths/consts.js +2 -0
- package/build/utils/paths/consts.js.map +1 -0
- package/build/utils/paths/types.d.ts +20 -0
- package/build/utils/paths/types.d.ts.map +1 -0
- package/build/utils/paths/types.js +2 -0
- package/build/utils/paths/types.js.map +1 -0
- package/build/utils/paths.d.ts +4 -0
- package/build/utils/paths.d.ts.map +1 -0
- package/build/utils/paths.js +67 -0
- package/build/utils/paths.js.map +1 -0
- package/build/utils/range/types.d.ts +11 -0
- package/build/utils/range/types.d.ts.map +1 -0
- package/build/utils/range/types.js +2 -0
- package/build/utils/range/types.js.map +1 -0
- package/build/utils/range.d.ts +4 -0
- package/build/utils/range.d.ts.map +1 -0
- package/build/utils/range.js +64 -0
- package/build/utils/range.js.map +1 -0
- package/build/utils/redact/types.d.ts +16 -0
- package/build/utils/redact/types.d.ts.map +1 -0
- package/build/utils/redact/types.js +2 -0
- package/build/utils/redact/types.js.map +1 -0
- package/build/utils/redact.d.ts +4 -0
- package/build/utils/redact.d.ts.map +1 -0
- package/build/utils/redact.js +18 -0
- package/build/utils/redact.js.map +1 -0
- package/build/utils/signals/types.d.ts +17 -0
- package/build/utils/signals/types.d.ts.map +1 -0
- package/build/utils/signals/types.js +2 -0
- package/build/utils/signals/types.js.map +1 -0
- package/build/utils/signals.d.ts +4 -0
- package/build/utils/signals.d.ts.map +1 -0
- package/build/utils/signals.js +52 -0
- package/build/utils/signals.js.map +1 -0
- package/build/utils/streams/consts.d.ts +3 -0
- package/build/utils/streams/consts.d.ts.map +1 -0
- package/build/utils/streams/consts.js +3 -0
- package/build/utils/streams/consts.js.map +1 -0
- package/build/utils/streams/types.d.ts +59 -0
- package/build/utils/streams/types.d.ts.map +1 -0
- package/build/utils/streams/types.js +2 -0
- package/build/utils/streams/types.js.map +1 -0
- package/build/utils/streams.d.ts +4 -0
- package/build/utils/streams.d.ts.map +1 -0
- package/build/utils/streams.js +167 -0
- package/build/utils/streams.js.map +1 -0
- package/build/utils/tree/types.d.ts +6 -0
- package/build/utils/tree/types.d.ts.map +1 -0
- package/build/utils/tree/types.js +2 -0
- package/build/utils/tree/types.js.map +1 -0
- package/build/utils/tree.d.ts +4 -0
- package/build/utils/tree.d.ts.map +1 -0
- package/build/utils/tree.js +14 -0
- package/build/utils/tree.js.map +1 -0
- package/build/workspace/consts.d.ts +12 -0
- package/build/workspace/consts.d.ts.map +1 -0
- package/build/workspace/consts.js +12 -0
- package/build/workspace/consts.js.map +1 -0
- package/build/workspace/consts.local.d.ts +3 -0
- package/build/workspace/consts.local.d.ts.map +1 -0
- package/build/workspace/consts.local.js +3 -0
- package/build/workspace/consts.local.js.map +1 -0
- package/build/workspace/fs.d.ts +4 -0
- package/build/workspace/fs.d.ts.map +1 -0
- package/build/workspace/fs.js +45 -0
- package/build/workspace/fs.js.map +1 -0
- package/build/workspace/scope.d.ts +6 -0
- package/build/workspace/scope.d.ts.map +1 -0
- package/build/workspace/scope.js +180 -0
- package/build/workspace/scope.js.map +1 -0
- package/build/workspace/types.d.ts +66 -0
- package/build/workspace/types.d.ts.map +1 -0
- package/build/workspace/types.js +2 -0
- package/build/workspace/types.js.map +1 -0
- package/build/workspace/types.local.d.ts +20 -0
- package/build/workspace/types.local.d.ts.map +1 -0
- package/build/workspace/types.local.js +2 -0
- package/build/workspace/types.local.js.map +1 -0
- package/build/workspace.d.ts +7 -0
- package/build/workspace.d.ts.map +1 -0
- package/build/workspace.js +87 -0
- package/build/workspace.js.map +1 -0
- package/package.json +61 -4
- package/src/archive/consts.ts +28 -0
- package/src/archive/formats.ts +60 -0
- package/src/archive/guard.ts +68 -0
- package/src/archive/ratio.ts +35 -0
- package/src/archive/tar.ts +147 -0
- package/src/archive/types.local.ts +72 -0
- package/src/archive/types.ts +107 -0
- package/src/archive/zip.ts +142 -0
- package/src/archive.ts +298 -0
- package/src/backed-workspace/types.ts +72 -0
- package/src/backed-workspace.ts +85 -0
- package/src/consts.local.ts +2 -0
- package/src/consts.ts +2 -0
- package/src/errors.ts +128 -0
- package/src/http/connector.ts +136 -0
- package/src/http/consts.ts +13 -0
- package/src/http/source.ts +89 -0
- package/src/http/types.local.ts +23 -0
- package/src/http/types.ts +74 -0
- package/src/http.ts +276 -0
- package/src/index.ts +65 -0
- package/src/network/consts.ts +22 -0
- package/src/network/types.ts +45 -0
- package/src/network.ts +179 -0
- package/src/proxy/consts.ts +11 -0
- package/src/proxy/endpoint.ts +28 -0
- package/src/proxy/http.ts +108 -0
- package/src/proxy/router.ts +93 -0
- package/src/proxy/socks.ts +48 -0
- package/src/proxy/types.ts +114 -0
- package/src/resolvers/google-drive/consts.ts +22 -0
- package/src/resolvers/google-drive/types.ts +22 -0
- package/src/resolvers/google-drive.ts +153 -0
- package/src/resolvers/types.ts +45 -0
- package/src/s3/consts.ts +12 -0
- package/src/s3/prefix.ts +100 -0
- package/src/s3/sink.ts +36 -0
- package/src/s3/source.ts +90 -0
- package/src/s3/types.local.ts +49 -0
- package/src/s3/types.ts +70 -0
- package/src/s3/upload.ts +130 -0
- package/src/s3.ts +83 -0
- package/src/service/types.local.ts +9 -0
- package/src/service/types.ts +71 -0
- package/src/service.ts +97 -0
- package/src/sinks/buffer/consts.ts +1 -0
- package/src/sinks/buffer/types.ts +12 -0
- package/src/sinks/buffer.ts +18 -0
- package/src/sinks/directory/types.ts +7 -0
- package/src/sinks/directory.ts +49 -0
- package/src/sinks/file.ts +52 -0
- package/src/sinks/types.ts +12 -0
- package/src/sinks/workload/consts.ts +5 -0
- package/src/sinks/workload/types.ts +17 -0
- package/src/sinks/workload.ts +59 -0
- package/src/sources/buffer.ts +27 -0
- package/src/sources/file.ts +57 -0
- package/src/sources/stream.ts +26 -0
- package/src/sources/types.ts +12 -0
- package/src/sources/workload/types.ts +31 -0
- package/src/sources/workload.ts +79 -0
- package/src/transfer/consts.ts +10 -0
- package/src/transfer/types.ts +55 -0
- package/src/transfer.ts +89 -0
- package/src/types.ts +106 -0
- package/src/utils/names/consts.ts +5 -0
- package/src/utils/names/types.ts +12 -0
- package/src/utils/names.ts +70 -0
- package/src/utils/paths/consts.ts +3 -0
- package/src/utils/paths/types.ts +20 -0
- package/src/utils/paths.ts +75 -0
- package/src/utils/range/types.ts +11 -0
- package/src/utils/range.ts +80 -0
- package/src/utils/redact/types.ts +16 -0
- package/src/utils/redact.ts +23 -0
- package/src/utils/signals/types.ts +17 -0
- package/src/utils/signals.ts +63 -0
- package/src/utils/streams/consts.ts +2 -0
- package/src/utils/streams/types.ts +65 -0
- package/src/utils/streams.ts +188 -0
- package/src/utils/tree/types.ts +6 -0
- package/src/utils/tree.ts +18 -0
- package/src/workspace/consts.local.ts +2 -0
- package/src/workspace/consts.ts +14 -0
- package/src/workspace/fs.ts +48 -0
- package/src/workspace/scope.ts +196 -0
- package/src/workspace/types.local.ts +21 -0
- package/src/workspace/types.ts +71 -0
- package/src/workspace.ts +102 -0
- package/tests/archive.spec.ts +134 -0
- package/tests/backed.spec.ts +147 -0
- package/tests/context.ts +114 -0
- package/tests/fixtures.ts +190 -0
- package/tests/google-drive.spec.ts +76 -0
- package/tests/harness/drive.ts +92 -0
- package/tests/harness/http-proxy.ts +96 -0
- package/tests/harness/origin.ts +134 -0
- package/tests/harness/socks5.ts +135 -0
- package/tests/harness/workload.ts +118 -0
- package/tests/http.spec.ts +101 -0
- package/tests/inspector.spec.ts +40 -0
- package/tests/live.spec.ts +27 -0
- package/tests/network.spec.ts +79 -0
- package/tests/proxy.spec.ts +149 -0
- package/tests/s3.spec.ts +133 -0
- package/tests/service.spec.ts +77 -0
- package/tests/sources-sinks.spec.ts +96 -0
- package/tests/transfer.spec.ts +99 -0
- package/tests/tsconfig.json +12 -0
- package/tests/workload.spec.ts +140 -0
- package/tests/workspace.spec.ts +158 -0
- package/tsconfig.json +16 -0
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
import { afterAll, beforeAll, describe, expect, test } from 'bun:test'
|
|
2
|
+
import { readFile, stat, writeFile } from 'node:fs/promises'
|
|
3
|
+
import { tmpdir } from 'node:os'
|
|
4
|
+
import { join, relative } from 'node:path'
|
|
5
|
+
import { randomNamespace } from '@owlmeans/test-integration'
|
|
6
|
+
import {
|
|
7
|
+
createBackedWorkspace, createBufferSource, createS3Bucket, createWorkspace, FileKind, FileMimicryError, PathError, S3_DELETE_BATCH_SIZE,
|
|
8
|
+
transferHelper,
|
|
9
|
+
} from '../src/index.js'
|
|
10
|
+
import type { BackedWorkspace, FileScope, S3Bucket } from '../src/index.js'
|
|
11
|
+
import { streamUtils } from '../src/utils/streams.js'
|
|
12
|
+
import { ELF, MARKDOWN, noise, PNG } from './fixtures.js'
|
|
13
|
+
import { boot, bucketConfig, purgeBasePrefix, removeDir, s3, tempDir } from './context.js'
|
|
14
|
+
|
|
15
|
+
const exists = async (path: string): Promise<boolean> => await stat(path).then(() => true, () => false)
|
|
16
|
+
|
|
17
|
+
describe('@owlmeans/file-streams — backed workspace and S3 prefixes', () => {
|
|
18
|
+
if (s3.skip) {
|
|
19
|
+
test.skip(s3.reason ?? 's3 gate closed', () => { })
|
|
20
|
+
return
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const basePrefix = randomNamespace('omt_file_streams_backed')
|
|
24
|
+
let bucket: S3Bucket
|
|
25
|
+
let root: string
|
|
26
|
+
let backed: BackedWorkspace
|
|
27
|
+
const scope: FileScope = { organization: 'acme', project: 'slot-1', purpose: 'new-project' }
|
|
28
|
+
const big = noise(6 * 1024 * 1024, 3)
|
|
29
|
+
|
|
30
|
+
beforeAll(async () => {
|
|
31
|
+
bucket = createS3Bucket(bucketConfig(basePrefix))
|
|
32
|
+
root = await tempDir('backed')
|
|
33
|
+
backed = createBackedWorkspace({ workspace: createWorkspace({ root }), bucket, prefix: 'materials' })
|
|
34
|
+
})
|
|
35
|
+
|
|
36
|
+
afterAll(async () => {
|
|
37
|
+
try {
|
|
38
|
+
await purgeBasePrefix(bucket)
|
|
39
|
+
} finally {
|
|
40
|
+
bucket.close()
|
|
41
|
+
await removeDir(root)
|
|
42
|
+
}
|
|
43
|
+
})
|
|
44
|
+
|
|
45
|
+
test('lists a prefix with full keys and relative paths, and removes it past one DeleteObjects batch', async () => {
|
|
46
|
+
const names = Array.from({ length: S3_DELETE_BATCH_SIZE + 105 }, (_, index) => `item-${String(index).padStart(4, '0')}.txt`)
|
|
47
|
+
for (let index = 0; index < names.length; index += 50) {
|
|
48
|
+
await Promise.all(names.slice(index, index + 50).map(async name => {
|
|
49
|
+
await transferHelper.copy(createBufferSource(Buffer.from(name), { name }), bucket.sink(`bulk/many/${name}`), { verify: false })
|
|
50
|
+
}))
|
|
51
|
+
}
|
|
52
|
+
await transferHelper.copy(createBufferSource(Buffer.from('kept')), bucket.sink('bulk/manyfold.txt'), { verify: false })
|
|
53
|
+
|
|
54
|
+
const listed = await bucket.list('bulk/many')
|
|
55
|
+
expect(listed.length).toBe(names.length)
|
|
56
|
+
expect(listed[0]).toMatchObject({ key: `${basePrefix}/bulk/many/${names[0]}`, path: names[0], size: names[0].length })
|
|
57
|
+
expect(listed.map(item => item.path).sort()).toEqual(names)
|
|
58
|
+
await expect(bucket.list('')).rejects.toBeInstanceOf(PathError)
|
|
59
|
+
expect(() => bucket.removePrefix('../outside')).toThrow('path:traversal')
|
|
60
|
+
|
|
61
|
+
expect(await bucket.removePrefix('bulk/many')).toBe(names.length)
|
|
62
|
+
expect(await bucket.list('bulk/many')).toEqual([])
|
|
63
|
+
expect((await bucket.list('bulk')).map(item => item.path)).toEqual(['manyfold.txt'])
|
|
64
|
+
expect(await bucket.removePrefix('bulk/many')).toBe(0)
|
|
65
|
+
}, 180_000)
|
|
66
|
+
|
|
67
|
+
test('puts files locally and remotely; evict then restore brings them back byte-identical', async () => {
|
|
68
|
+
const files = backed.scope(scope)
|
|
69
|
+
expect(files.key('brand/logo.png')).toBe(`${basePrefix}/materials/acme/slot-1/new-project/brand/logo.png`)
|
|
70
|
+
expect(files.prefixOf('brand/logo.png')).toBe('materials/acme/slot-1/new-project/brand/logo.png')
|
|
71
|
+
|
|
72
|
+
const logo = await files.put('brand/logo.png', createBufferSource(PNG, { name: 'logo.png' }), { verify: { allow: [FileKind.Image] }, hash: 'sha256' })
|
|
73
|
+
expect(logo.local).toMatchObject({ kind: 'file', location: join(files.local.dir, 'brand/logo.png'), bytes: PNG.length })
|
|
74
|
+
expect(logo.remote).toMatchObject({ kind: 's3', location: `s3://${bucket.model.bucket}/${files.key('brand/logo.png')}`, bytes: PNG.length, mime: 'image/png' })
|
|
75
|
+
expect(logo.transfer.verdict?.detected?.kind).toBe(FileKind.Image)
|
|
76
|
+
expect(logo.transfer.sha256).toHaveLength(64)
|
|
77
|
+
await files.put('docs/brief.md', createBufferSource(MARKDOWN, { name: 'brief.md' }))
|
|
78
|
+
await files.put('media/clip.bin', createBufferSource(big, { name: 'clip.bin' }), { verify: false })
|
|
79
|
+
|
|
80
|
+
expect((await files.list()).map(item => [item.path, item.size])).toEqual([
|
|
81
|
+
['brand/logo.png', PNG.length], ['docs/brief.md', MARKDOWN.length], ['media/clip.bin', big.length],
|
|
82
|
+
])
|
|
83
|
+
expect((await bucket.head(files.prefixOf('brand/logo.png')))?.mime).toBe('image/png')
|
|
84
|
+
|
|
85
|
+
await files.evict()
|
|
86
|
+
expect(await exists(files.local.dir)).toBe(false)
|
|
87
|
+
expect(await files.restore()).toEqual({ restored: ['brand/logo.png', 'docs/brief.md', 'media/clip.bin'], kept: [] })
|
|
88
|
+
expect(await readFile(join(files.local.dir, 'brand/logo.png'))).toEqual(PNG)
|
|
89
|
+
expect((await readFile(join(files.local.dir, 'media/clip.bin'))).equals(big)).toBe(true)
|
|
90
|
+
expect(await files.restore()).toEqual({ restored: [], kept: ['brand/logo.png', 'docs/brief.md', 'media/clip.bin'] })
|
|
91
|
+
}, 120_000)
|
|
92
|
+
|
|
93
|
+
test('downloads a size-changed file again, and a refused put leaves nothing on either side', async () => {
|
|
94
|
+
const files = backed.scope(scope)
|
|
95
|
+
await writeFile(join(files.local.dir, 'docs/brief.md'), 'tampered, and longer than the original brief')
|
|
96
|
+
expect((await files.restore()).restored).toEqual(['docs/brief.md'])
|
|
97
|
+
expect(await readFile(join(files.local.dir, 'docs/brief.md'))).toEqual(MARKDOWN)
|
|
98
|
+
|
|
99
|
+
await expect(files.put('brand/fake.png', createBufferSource(ELF, { name: 'fake.png' }))).rejects.toBeInstanceOf(FileMimicryError)
|
|
100
|
+
expect(await files.local.has('brand/fake.png')).toBe(false)
|
|
101
|
+
expect(await bucket.head(files.prefixOf('brand/fake.png'))).toBeNull()
|
|
102
|
+
expect(() => files.prefixOf('../../escape.png')).toThrow('path:traversal')
|
|
103
|
+
}, 60_000)
|
|
104
|
+
|
|
105
|
+
test('removes a file on both sides and drops the whole scope prefix', async () => {
|
|
106
|
+
const files = backed.scope(scope)
|
|
107
|
+
const other = backed.scope({ organization: 'acme', project: 'slot-2', purpose: 'new-project' })
|
|
108
|
+
await other.put('keep.txt', createBufferSource(Buffer.from('keep'), { name: 'keep.txt' }))
|
|
109
|
+
|
|
110
|
+
await files.remove('docs/brief.md')
|
|
111
|
+
expect(await files.local.has('docs/brief.md')).toBe(false)
|
|
112
|
+
expect(await bucket.head(files.prefixOf('docs/brief.md'))).toBeNull()
|
|
113
|
+
expect((await files.list()).map(item => item.path)).toEqual(['brand/logo.png', 'media/clip.bin'])
|
|
114
|
+
|
|
115
|
+
await files.drop()
|
|
116
|
+
expect(await files.list()).toEqual([])
|
|
117
|
+
expect(await exists(files.local.dir)).toBe(false)
|
|
118
|
+
expect(await files.restore()).toEqual({ restored: [], kept: [] })
|
|
119
|
+
expect((await other.list()).map(item => item.path)).toEqual(['keep.txt'])
|
|
120
|
+
expect(await other.local.has('keep.txt')).toBe(true)
|
|
121
|
+
}, 60_000)
|
|
122
|
+
|
|
123
|
+
test('the service hands out the configured workspace and backed workspaces of a storage entry', async () => {
|
|
124
|
+
const serviceRoot = await tempDir('backed-service')
|
|
125
|
+
try {
|
|
126
|
+
// A server context reads a string leaf starting with `/` as a secret file: the root is relative to the tmp folder.
|
|
127
|
+
const { files } = await boot({
|
|
128
|
+
storageBuckets: { projects: bucketConfig(basePrefix) }, fileStreams: { workspace: { root: relative(tmpdir(), serviceRoot) } },
|
|
129
|
+
})
|
|
130
|
+
expect(files.workspace().root).toBe(serviceRoot)
|
|
131
|
+
const projects = files.backed('projects', { prefix: 'service' })
|
|
132
|
+
expect(files.backed('projects', { prefix: 'service' })).toBe(projects)
|
|
133
|
+
expect(projects.workspace).toBe(files.workspace())
|
|
134
|
+
|
|
135
|
+
const upload = projects.scope({ organization: 'acme', purpose: 'upload' })
|
|
136
|
+
await upload.put('dot.png', createBufferSource(PNG, { name: 'dot.png' }))
|
|
137
|
+
expect(upload.key('dot.png')).toBe(`${basePrefix}/service/acme/upload/dot.png`)
|
|
138
|
+
await upload.evict()
|
|
139
|
+
expect((await upload.restore()).restored).toEqual(['dot.png'])
|
|
140
|
+
expect(await streamUtils.collect((await upload.local.source('dot.png').open()).stream, 10_000)).toEqual(PNG)
|
|
141
|
+
await upload.drop()
|
|
142
|
+
expect(await files.workspace().scopes()).toEqual([])
|
|
143
|
+
} finally {
|
|
144
|
+
await removeDir(serviceRoot)
|
|
145
|
+
}
|
|
146
|
+
}, 60_000)
|
|
147
|
+
})
|
package/tests/context.ts
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import { spawnSync } from 'node:child_process'
|
|
2
|
+
import { DeleteObjectsCommand, ListObjectsV2Command } from '@aws-sdk/client-s3'
|
|
3
|
+
import { mkdtemp, readFile, rm } from 'node:fs/promises'
|
|
4
|
+
import { tmpdir } from 'node:os'
|
|
5
|
+
import { join } from 'node:path'
|
|
6
|
+
import { config, makeServerContext } from '@owlmeans/server-context'
|
|
7
|
+
import type { ServerConfig, ServerContext } from '@owlmeans/server-context'
|
|
8
|
+
import { makeGates } from '@owlmeans/test'
|
|
9
|
+
import { gateHelper } from '@owlmeans/test-integration'
|
|
10
|
+
import type { IntegrationGate, S3Env } from '@owlmeans/test-integration'
|
|
11
|
+
import type { StorageConfig } from '@owlmeans/storage-resource'
|
|
12
|
+
import { appendFileStreamsService, createHttpClient, FILE_STREAMS_SERVICE } from '../src/index.js'
|
|
13
|
+
import type {
|
|
14
|
+
FileStreamsConfig, FileStreamsService, FileStreamsServiceOptions, HttpClient, HttpClientOptions, S3Bucket,
|
|
15
|
+
} from '../src/index.js'
|
|
16
|
+
|
|
17
|
+
/** S3 gate: empty `S3_*` variables print a skip, never a failure. */
|
|
18
|
+
export const s3: IntegrationGate<S3Env> = gateHelper.s3Gate()
|
|
19
|
+
|
|
20
|
+
/** Live-internet gates: opt-in only. */
|
|
21
|
+
export const live = makeGates({
|
|
22
|
+
internet: ['FILE_STREAMS_LIVE'],
|
|
23
|
+
drive: ['FILE_STREAMS_LIVE', 'FILE_STREAMS_LIVE_DRIVE_URL'],
|
|
24
|
+
})
|
|
25
|
+
|
|
26
|
+
/** Stand-ins listen on loopback, which the default policy refuses. */
|
|
27
|
+
export const LOCAL_NETWORK = { allowCidrs: ['127.0.0.1/32'] }
|
|
28
|
+
|
|
29
|
+
export const localClient = (options: HttpClientOptions = {}): HttpClient => createHttpClient({
|
|
30
|
+
...options,
|
|
31
|
+
network: options.network ?? LOCAL_NETWORK,
|
|
32
|
+
resolvers: options.resolvers ?? [],
|
|
33
|
+
timeouts: { connectMs: 3_000, headersMs: 3_000, idleMs: 3_000, ...options.timeouts },
|
|
34
|
+
})
|
|
35
|
+
|
|
36
|
+
export const tempDir = async (label: string): Promise<string> => await mkdtemp(join(tmpdir(), `owlmeans-fs-${label}-`))
|
|
37
|
+
|
|
38
|
+
export const removeDir = async (dir: string): Promise<void> => { await rm(dir, { recursive: true, force: true }) }
|
|
39
|
+
|
|
40
|
+
export interface TlsMaterial {
|
|
41
|
+
key: string
|
|
42
|
+
cert: string
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** A one-day self-signed certificate for 127.0.0.1, or `undefined` when openssl is missing. */
|
|
46
|
+
export const tlsMaterial = async (): Promise<TlsMaterial | undefined> => {
|
|
47
|
+
const dir = await tempDir('tls')
|
|
48
|
+
try {
|
|
49
|
+
const result = spawnSync('openssl', [
|
|
50
|
+
'req', '-x509', '-newkey', 'ec', '-pkeyopt', 'ec_paramgen_curve:P-256', '-nodes', '-days', '1', '-subj', '/CN=localhost',
|
|
51
|
+
'-addext', 'subjectAltName=IP:127.0.0.1,DNS:localhost', '-keyout', join(dir, 'key.pem'), '-out', join(dir, 'cert.pem'),
|
|
52
|
+
], { stdio: 'ignore' })
|
|
53
|
+
if (result.status !== 0) {
|
|
54
|
+
return undefined
|
|
55
|
+
}
|
|
56
|
+
return { key: await readFile(join(dir, 'key.pem'), 'utf8'), cert: await readFile(join(dir, 'cert.pem'), 'utf8') }
|
|
57
|
+
} catch {
|
|
58
|
+
return undefined
|
|
59
|
+
} finally {
|
|
60
|
+
await removeDir(dir)
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** The S3 test bucket as a storage entry: explicit endpoint, path-style for anything but AWS, no ACL. */
|
|
65
|
+
export const bucketConfig = (basePrefix: string): StorageConfig => {
|
|
66
|
+
const endpoint = s3.env.S3_ENDPOINT as string
|
|
67
|
+
return {
|
|
68
|
+
endpoint,
|
|
69
|
+
bucket: s3.env.S3_TEST_BUCKET as string,
|
|
70
|
+
region: s3.env.S3_REGION ?? 'us-east-1',
|
|
71
|
+
forcePathStyle: !/\.amazonaws\.com/.test(endpoint),
|
|
72
|
+
apiKey: `${s3.env.S3_KEY as string}:${s3.env.S3_SECRET as string}`,
|
|
73
|
+
basePrefix,
|
|
74
|
+
acl: false,
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** Deletes every object under the bucket's random base prefix — a spec's own cleanup, outside the API under test. */
|
|
79
|
+
export const purgeBasePrefix = async (bucket: S3Bucket): Promise<void> => {
|
|
80
|
+
const client = bucket.model.createClient()
|
|
81
|
+
try {
|
|
82
|
+
for (;;) {
|
|
83
|
+
const page = await client.send(new ListObjectsV2Command({ Bucket: bucket.model.bucket, Prefix: `${bucket.model.basePrefix}/` }))
|
|
84
|
+
const keys = (page.Contents ?? []).map(object => ({ Key: object.Key as string }))
|
|
85
|
+
if (keys.length === 0) {
|
|
86
|
+
return
|
|
87
|
+
}
|
|
88
|
+
await client.send(new DeleteObjectsCommand({ Bucket: bucket.model.bucket, Delete: { Objects: keys, Quiet: true } }))
|
|
89
|
+
}
|
|
90
|
+
} finally {
|
|
91
|
+
client.destroy()
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export interface TestConfig extends ServerConfig {
|
|
96
|
+
fileStreams?: FileStreamsConfig
|
|
97
|
+
storageBuckets?: { [key: string]: StorageConfig }
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export interface Booted {
|
|
101
|
+
context: ServerContext<TestConfig>
|
|
102
|
+
files: FileStreamsService
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** A real server context with the service registered — the path a server app takes. */
|
|
106
|
+
export const boot = async (setup: Partial<TestConfig> = {}, options?: FileStreamsServiceOptions): Promise<Booted> => {
|
|
107
|
+
const cfg = { ...config('file-streams-tests'), ...setup } as TestConfig
|
|
108
|
+
const context = makeServerContext(cfg) as ServerContext<TestConfig>
|
|
109
|
+
appendFileStreamsService(context, FILE_STREAMS_SERVICE, options)
|
|
110
|
+
context.configure()
|
|
111
|
+
await context.init()
|
|
112
|
+
|
|
113
|
+
return { context, files: context.service<FileStreamsService>(FILE_STREAMS_SERVICE) }
|
|
114
|
+
}
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
import { Readable } from 'node:stream'
|
|
2
|
+
import { gzipSync } from 'node:zlib'
|
|
3
|
+
import tarStream from 'tar-stream'
|
|
4
|
+
import yazl from 'yazl'
|
|
5
|
+
|
|
6
|
+
const bytes = (...parts: Array<number[] | string | Uint8Array>): Buffer => Buffer.concat(parts.map(part =>
|
|
7
|
+
typeof part === 'string' ? Buffer.from(part, 'latin1') : Buffer.from(part)))
|
|
8
|
+
|
|
9
|
+
export const PNG = Buffer.from(
|
|
10
|
+
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==', 'base64'
|
|
11
|
+
)
|
|
12
|
+
export const JPEG = bytes([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10], 'JFIF', [0, 1, 1, 0, 0, 1, 0, 1, 0, 0], new Uint8Array(64), [0xff, 0xd9])
|
|
13
|
+
export const GIF = bytes('GIF89a', [1, 0, 1, 0, 0x80, 0, 0], new Uint8Array(32), [0x3b])
|
|
14
|
+
export const WEBP = bytes('RIFF', [26, 0, 0, 0], 'WEBPVP8 ', [14, 0, 0, 0], new Uint8Array(14))
|
|
15
|
+
export const PDF = bytes('%PDF-1.4\n1 0 obj << /Type /Catalog >> endobj\ntrailer << /Root 1 0 R >>\n%%EOF\n')
|
|
16
|
+
export const MP3 = bytes('ID3', [3, 0, 0, 0, 0, 0, 0], [0xff, 0xfb, 0x90, 0x64], new Uint8Array(413), [0xff, 0xfb, 0x90, 0x64], new Uint8Array(413))
|
|
17
|
+
export const WAV = bytes('RIFF', [36, 0, 0, 0], 'WAVEfmt ', [16, 0, 0, 0, 1, 0, 1, 0, 0x44, 0xac, 0, 0, 0x88, 0x58, 1, 0, 2, 0, 16, 0], 'data', [0, 0, 0, 0])
|
|
18
|
+
export const OGG = bytes('OggS', [0, 2], new Uint8Array(8), [1, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [1, 30], [1], 'vorbis', new Uint8Array(23))
|
|
19
|
+
export const FLAC = bytes('fLaC', [0x80, 0, 0, 34], new Uint8Array(34))
|
|
20
|
+
export const MP4 = bytes([0, 0, 0, 0x18], 'ftypisom', [0, 0, 2, 0], 'isomiso2', [0, 0, 0, 8], 'free')
|
|
21
|
+
export const WEBM = bytes([0x1a, 0x45, 0xdf, 0xa3, 0x9f, 0x42, 0x86, 0x81, 1, 0x42, 0xf7, 0x81, 1, 0x42, 0xf2, 0x81, 4, 0x42, 0xf3, 0x81, 8,
|
|
22
|
+
0x42, 0x82, 0x84], 'webm', [0x42, 0x87, 0x81, 4, 0x42, 0x85, 0x81, 2])
|
|
23
|
+
export const ELF = bytes([0x7f, 0x45, 0x4c, 0x46, 2, 1, 1, 0], new Uint8Array(56))
|
|
24
|
+
export const PE = bytes('MZ', [0x90, 0], new Uint8Array(56), [0x40, 0, 0, 0], 'PE', [0, 0], new Uint8Array(64))
|
|
25
|
+
|
|
26
|
+
export const CSV = Buffer.from('name,age,city\nAnn,31,Oslo\nBob,42,Rome\nCid,27,Lima\n')
|
|
27
|
+
export const TSV = Buffer.from('name\tage\nAnn\t31\nBob\t42\n')
|
|
28
|
+
export const JSON_TEXT = Buffer.from('{"name":"Ann","tags":["a","b"]}')
|
|
29
|
+
export const HTML_ERROR = Buffer.from('<!DOCTYPE html><html><head><title>Error</title></head><body><h1>404</h1></body></html>')
|
|
30
|
+
export const SVG = Buffer.from('<?xml version="1.0"?><svg xmlns="http://www.w3.org/2000/svg" width="1" height="1"><rect width="1" height="1"/></svg>')
|
|
31
|
+
export const ACTIVE_SVG = Buffer.from('<svg xmlns="http://www.w3.org/2000/svg" onload="alert(1)"><script>alert(2)</script></svg>')
|
|
32
|
+
export const TS_CODE = Buffer.from('import { readFile } from "node:fs/promises"\nexport const answer: number = 42\n')
|
|
33
|
+
export const PY_CODE = Buffer.from('def main():\n print("hi")\n\nif __name__ == "__main__":\n main()\n')
|
|
34
|
+
export const SHEBANG = Buffer.from('#!/usr/bin/env python3\nprint("hi")\n')
|
|
35
|
+
export const MARKDOWN = Buffer.from('# Title\n\nSome *text* here.\n')
|
|
36
|
+
export const PLAIN = Buffer.from('Just a plain sentence of text.\nAnd a second line.\n')
|
|
37
|
+
export const UTF16 = Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from('Hello, wide world', 'utf16le')])
|
|
38
|
+
|
|
39
|
+
export interface ZipEntrySpec {
|
|
40
|
+
name: string
|
|
41
|
+
data: Uint8Array | string
|
|
42
|
+
compress?: boolean
|
|
43
|
+
mode?: number
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** A zip built in memory with yazl. */
|
|
47
|
+
export const zipOf = async (entries: ZipEntrySpec[]): Promise<Buffer> => {
|
|
48
|
+
const zip = new yazl.ZipFile()
|
|
49
|
+
for (const entry of entries) {
|
|
50
|
+
zip.addBuffer(Buffer.from(entry.data), entry.name, {
|
|
51
|
+
compress: entry.compress ?? true, mtime: new Date('2026-01-01T00:00:00Z'), ...(entry.mode != null ? { mode: entry.mode } : {}),
|
|
52
|
+
})
|
|
53
|
+
}
|
|
54
|
+
zip.end()
|
|
55
|
+
const chunks: Buffer[] = []
|
|
56
|
+
for await (const chunk of zip.outputStream as Readable) {
|
|
57
|
+
chunks.push(chunk as Buffer)
|
|
58
|
+
}
|
|
59
|
+
return Buffer.concat(chunks)
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const contentTypes = (part: string, type: string): string =>
|
|
63
|
+
`<?xml version="1.0" encoding="UTF-8"?><Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">`
|
|
64
|
+
+ `<Default Extension="xml" ContentType="application/xml"/><Override PartName="${part}" ContentType="${type}"/></Types>`
|
|
65
|
+
|
|
66
|
+
export const docxOf = async (): Promise<Buffer> => await zipOf([
|
|
67
|
+
{ name: '[Content_Types].xml', data: contentTypes('/word/document.xml', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml') },
|
|
68
|
+
{ name: 'word/document.xml', data: '<w:document/>' },
|
|
69
|
+
])
|
|
70
|
+
export const xlsxOf = async (): Promise<Buffer> => await zipOf([
|
|
71
|
+
{ name: '[Content_Types].xml', data: contentTypes('/xl/workbook.xml', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml') },
|
|
72
|
+
{ name: 'xl/workbook.xml', data: '<workbook/>' },
|
|
73
|
+
])
|
|
74
|
+
export const pptxOf = async (): Promise<Buffer> => await zipOf([
|
|
75
|
+
{ name: '[Content_Types].xml', data: contentTypes('/ppt/presentation.xml', 'application/vnd.openxmlformats-officedocument.presentationml.presentation.main+xml') },
|
|
76
|
+
{ name: 'ppt/presentation.xml', data: '<presentation/>' },
|
|
77
|
+
])
|
|
78
|
+
export const odtOf = async (): Promise<Buffer> => await zipOf([
|
|
79
|
+
{ name: 'mimetype', data: 'application/vnd.oasis.opendocument.text', compress: false },
|
|
80
|
+
{ name: 'content.xml', data: '<office:document-content/>' },
|
|
81
|
+
])
|
|
82
|
+
|
|
83
|
+
/** A docx whose markers sit behind a large stored entry — detectable only by seeking. */
|
|
84
|
+
export const bigDocxOf = async (padding: number): Promise<Buffer> => await zipOf([
|
|
85
|
+
{ name: 'padding.bin', data: Buffer.alloc(padding, 7), compress: false },
|
|
86
|
+
{ name: '[Content_Types].xml', data: contentTypes('/word/document.xml', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml') },
|
|
87
|
+
{ name: 'word/document.xml', data: '<w:document/>' },
|
|
88
|
+
])
|
|
89
|
+
|
|
90
|
+
const guid = (text: string): Buffer => {
|
|
91
|
+
const hex = text.replace(/-/g, '')
|
|
92
|
+
const raw = Buffer.from(hex, 'hex')
|
|
93
|
+
return Buffer.concat([Buffer.from(raw.subarray(0, 4)).reverse(), Buffer.from(raw.subarray(4, 6)).reverse(), Buffer.from(raw.subarray(6, 8)).reverse(), raw.subarray(8)])
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export const CLSID = {
|
|
97
|
+
doc: '00020906-0000-0000-c000-000000000046',
|
|
98
|
+
xls: '00020820-0000-0000-c000-000000000046',
|
|
99
|
+
ppt: '64818d10-4f9b-11cf-86ea-00aa00b929e8',
|
|
100
|
+
} as const
|
|
101
|
+
|
|
102
|
+
/** A minimal compound file: header, FAT sector, a directory whose root entry carries `clsid`. */
|
|
103
|
+
export const cfbOf = (clsid: string, directorySector = 1, sectors = 3): Buffer => {
|
|
104
|
+
const file = Buffer.alloc(512 * (sectors + 1))
|
|
105
|
+
Buffer.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]).copy(file, 0)
|
|
106
|
+
file.writeUInt16LE(0x3e, 24)
|
|
107
|
+
file.writeUInt16LE(3, 26)
|
|
108
|
+
file.writeUInt16LE(0xfffe, 28)
|
|
109
|
+
file.writeUInt16LE(9, 30)
|
|
110
|
+
file.writeUInt16LE(6, 32)
|
|
111
|
+
file.writeUInt32LE(1, 44)
|
|
112
|
+
file.writeUInt32LE(directorySector, 48)
|
|
113
|
+
file.writeUInt32LE(4096, 56)
|
|
114
|
+
file.writeUInt32LE(0xfffffffe, 60)
|
|
115
|
+
file.writeUInt32LE(0xfffffffe, 68)
|
|
116
|
+
file.writeUInt32LE(0, 76)
|
|
117
|
+
for (let index = 1; index < 109; index++) file.writeUInt32LE(0xffffffff, 76 + index * 4)
|
|
118
|
+
const fat = 512
|
|
119
|
+
file.writeUInt32LE(0xfffffffd, fat)
|
|
120
|
+
for (let index = 1; index < 128; index++) file.writeUInt32LE(0xffffffff, fat + index * 4)
|
|
121
|
+
file.writeUInt32LE(0xfffffffe, fat + directorySector * 4)
|
|
122
|
+
const directory = 512 + directorySector * 512
|
|
123
|
+
Buffer.from('Root Entry', 'utf16le').copy(file, directory)
|
|
124
|
+
file.writeUInt16LE(22, directory + 64)
|
|
125
|
+
file[directory + 66] = 5
|
|
126
|
+
guid(clsid).copy(file, directory + 80)
|
|
127
|
+
return file
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
export interface TarEntrySpec {
|
|
131
|
+
name: string
|
|
132
|
+
data?: Uint8Array | string
|
|
133
|
+
type?: 'file' | 'directory' | 'symlink' | 'link'
|
|
134
|
+
linkname?: string
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** A tar built in memory with tar-stream. */
|
|
138
|
+
export const tarOf = async (entries: TarEntrySpec[]): Promise<Buffer> => {
|
|
139
|
+
const pack = tarStream.pack()
|
|
140
|
+
const done = (async () => {
|
|
141
|
+
const chunks: Buffer[] = []
|
|
142
|
+
for await (const chunk of pack) chunks.push(Buffer.from(chunk as Uint8Array))
|
|
143
|
+
return Buffer.concat(chunks)
|
|
144
|
+
})()
|
|
145
|
+
for (const entry of entries) {
|
|
146
|
+
const data = entry.data != null ? Buffer.from(entry.data) : Buffer.alloc(0)
|
|
147
|
+
await new Promise<void>((resolve, reject) => {
|
|
148
|
+
pack.entry({ name: entry.name, type: entry.type ?? 'file', linkname: entry.linkname, size: entry.type == null || entry.type === 'file' ? data.length : 0 },
|
|
149
|
+
entry.type == null || entry.type === 'file' ? data : undefined as unknown as Buffer, error => error != null ? reject(error) : resolve())
|
|
150
|
+
})
|
|
151
|
+
}
|
|
152
|
+
pack.finalize()
|
|
153
|
+
return await done
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
export const tgzOf = async (entries: TarEntrySpec[]): Promise<Buffer> => gzipSync(await tarOf(entries))
|
|
157
|
+
|
|
158
|
+
/** Replaces every occurrence of one same-length byte string — to plant names a writer would refuse. */
|
|
159
|
+
export const patch = (buffer: Buffer, from: string, to: string): Buffer => {
|
|
160
|
+
if (from.length !== to.length) throw new Error('patch needs equal lengths')
|
|
161
|
+
const out = Buffer.from(buffer)
|
|
162
|
+
let index = out.indexOf(from)
|
|
163
|
+
while (index !== -1) {
|
|
164
|
+
out.write(to, index, 'latin1')
|
|
165
|
+
index = out.indexOf(from, index + 1)
|
|
166
|
+
}
|
|
167
|
+
return out
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/** Offsets of zip central-directory records. */
|
|
171
|
+
export const centralRecords = (zip: Buffer): number[] => {
|
|
172
|
+
const offsets: number[] = []
|
|
173
|
+
let index = zip.indexOf(Buffer.from([0x50, 0x4b, 0x01, 0x02]))
|
|
174
|
+
while (index !== -1) {
|
|
175
|
+
offsets.push(index)
|
|
176
|
+
index = zip.indexOf(Buffer.from([0x50, 0x4b, 0x01, 0x02]), index + 4)
|
|
177
|
+
}
|
|
178
|
+
return offsets
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/** Deterministic pseudo-random bytes. */
|
|
182
|
+
export const noise = (length: number, seed = 1): Buffer => {
|
|
183
|
+
const out = Buffer.alloc(length)
|
|
184
|
+
let state = seed >>> 0
|
|
185
|
+
for (let index = 0; index < length; index++) {
|
|
186
|
+
state = (state * 1664525 + 1013904223) >>> 0
|
|
187
|
+
out[index] = state >>> 24
|
|
188
|
+
}
|
|
189
|
+
return out
|
|
190
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { afterAll, beforeAll, describe, expect, test } from 'bun:test'
|
|
2
|
+
import { createGoogleDriveResolver, fileInspector, FileKind, googleDriveResolver, ResolverError } from '../src/index.js'
|
|
3
|
+
import { streamUtils } from '../src/utils/streams.js'
|
|
4
|
+
import { docxOf, noise, PNG, pptxOf, xlsxOf } from './fixtures.js'
|
|
5
|
+
import { startFakeDrive } from './harness/drive.js'
|
|
6
|
+
import type { FakeDrive } from './harness/drive.js'
|
|
7
|
+
import { startOrigin } from './harness/origin.js'
|
|
8
|
+
import type { Origin } from './harness/origin.js'
|
|
9
|
+
import { localClient } from './context.js'
|
|
10
|
+
|
|
11
|
+
describe('@owlmeans/file-streams — Google Drive resolver', () => {
|
|
12
|
+
let drive: FakeDrive
|
|
13
|
+
let foreign: Origin
|
|
14
|
+
beforeAll(async () => {
|
|
15
|
+
foreign = startOrigin()
|
|
16
|
+
drive = startFakeDrive({ small: PNG, large: noise(300_000), docx: await docxOf(), xlsx: await xlsxOf(), pptx: await pptxOf() }, foreign)
|
|
17
|
+
})
|
|
18
|
+
afterAll(() => {
|
|
19
|
+
drive.stop()
|
|
20
|
+
foreign.stop()
|
|
21
|
+
})
|
|
22
|
+
|
|
23
|
+
const client = () => localClient({
|
|
24
|
+
resolvers: [createGoogleDriveResolver({ origins: { drive: drive.base, usercontent: drive.base, docs: drive.base, accounts: drive.accounts } })],
|
|
25
|
+
})
|
|
26
|
+
|
|
27
|
+
test('rewrites share and editor links to the URLs that serve bytes', () => {
|
|
28
|
+
const id = '1AbCdEfGhIjKlMnOpQrStUvWxYz'
|
|
29
|
+
const cases: Array<[string, string]> = [
|
|
30
|
+
[`https://drive.google.com/file/d/${id}/view?usp=sharing`, `https://drive.usercontent.google.com/download?id=${id}&export=download`],
|
|
31
|
+
[`https://drive.google.com/open?id=${id}`, `https://drive.usercontent.google.com/download?id=${id}&export=download`],
|
|
32
|
+
[`https://drive.google.com/uc?id=${id}&export=download`, `https://drive.usercontent.google.com/download?id=${id}&export=download`],
|
|
33
|
+
[`https://drive.google.com/file/d/${id}/view?resourcekey=0-key`, `https://drive.usercontent.google.com/download?id=${id}&export=download&resourcekey=0-key`],
|
|
34
|
+
[`https://docs.google.com/document/d/${id}/edit`, `https://docs.google.com/document/d/${id}/export?format=docx`],
|
|
35
|
+
[`https://docs.google.com/spreadsheets/d/${id}/edit#gid=0`, `https://docs.google.com/spreadsheets/d/${id}/export?format=xlsx`],
|
|
36
|
+
[`https://docs.google.com/presentation/d/${id}/edit`, `https://docs.google.com/presentation/d/${id}/export/pptx`],
|
|
37
|
+
[`https://docs.google.com/drawings/d/${id}/edit`, `https://docs.google.com/drawings/d/${id}/export/png`],
|
|
38
|
+
]
|
|
39
|
+
for (const [input, expected] of cases) {
|
|
40
|
+
expect(googleDriveResolver.rewrite(new URL(input)).href).toBe(expected)
|
|
41
|
+
}
|
|
42
|
+
const csv = createGoogleDriveResolver({ exports: { spreadsheet: 'csv' } })
|
|
43
|
+
expect(csv.rewrite(new URL(`https://docs.google.com/spreadsheets/d/${id}/edit#gid=42`)).href)
|
|
44
|
+
.toBe(`https://docs.google.com/spreadsheets/d/${id}/export?format=csv&gid=42`)
|
|
45
|
+
expect(() => googleDriveResolver.rewrite(new URL('https://drive.google.com/drive/folders/abcdefghijk'))).toThrow(ResolverError)
|
|
46
|
+
expect(googleDriveResolver.matches(new URL('https://example.com/file/d/x'))).toBe(false)
|
|
47
|
+
})
|
|
48
|
+
|
|
49
|
+
test('downloads a small file at once and passes the virus-scan page of a large one', async () => {
|
|
50
|
+
const small = await client().request(`${drive.base}/file/d/SMALLFILE0001/view?usp=sharing`)
|
|
51
|
+
expect(small.info.name).toBe('small.png')
|
|
52
|
+
expect(await streamUtils.collect(small.body, 1_000_000)).toEqual(PNG)
|
|
53
|
+
|
|
54
|
+
const large = await client().request(`${drive.base}/file/d/LARGEFILE0001/view`)
|
|
55
|
+
expect(await streamUtils.collect(large.body, 1_000_000)).toEqual(noise(300_000))
|
|
56
|
+
expect(drive.paths.some(path => path.includes('confirm=t') && path.includes('uuid=u-123'))).toBe(true)
|
|
57
|
+
})
|
|
58
|
+
|
|
59
|
+
test('exports editor files in the configured formats', async () => {
|
|
60
|
+
const expected: Array<[string, FileKind]> = [
|
|
61
|
+
[`${drive.base}/document/d/DOCUMENTID01/edit`, FileKind.Document],
|
|
62
|
+
[`${drive.base}/spreadsheets/d/SHEETID00001/edit`, FileKind.Spreadsheet],
|
|
63
|
+
[`${drive.base}/presentation/d/SLIDESID0001/edit`, FileKind.Presentation],
|
|
64
|
+
]
|
|
65
|
+
for (const [url, kind] of expected) {
|
|
66
|
+
const verdict = await fileInspector.probe(client().source(url))
|
|
67
|
+
expect({ url, status: verdict.status, kind: verdict.detected?.kind }).toEqual({ url, status: 'match', kind })
|
|
68
|
+
}
|
|
69
|
+
})
|
|
70
|
+
|
|
71
|
+
test('fails on quotas, private files and forms that post elsewhere', async () => {
|
|
72
|
+
await expect(client().request(`${drive.base}/uc?id=QUOTAFILE0001`)).rejects.toThrow('resolver:google-drive:quota')
|
|
73
|
+
await expect(client().request(`${drive.base}/file/d/PRIVATEFILE01/view`)).rejects.toThrow('resolver:google-drive:not-public')
|
|
74
|
+
await expect(client().request(`${drive.base}/file/d/FOREIGNFORM01/view`)).rejects.toThrow('resolver:google-drive:foreign-form')
|
|
75
|
+
})
|
|
76
|
+
})
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import type { Origin } from './origin.js'
|
|
2
|
+
|
|
3
|
+
export interface FakeDrive {
|
|
4
|
+
readonly base: string
|
|
5
|
+
readonly accounts: string
|
|
6
|
+
/** Requests seen, as path + query. */
|
|
7
|
+
paths: string[]
|
|
8
|
+
stop: () => void
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export interface FakeDriveFiles {
|
|
12
|
+
small: Uint8Array
|
|
13
|
+
large: Uint8Array
|
|
14
|
+
docx: Uint8Array
|
|
15
|
+
xlsx: Uint8Array
|
|
16
|
+
pptx: Uint8Array
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Google Drive's download flow in miniature: a small file downloads at once, a large one answers
|
|
21
|
+
* the "can't scan for viruses" page whose form carries `confirm` and `uuid`, editor files export,
|
|
22
|
+
* a quota page, a private file bouncing to the sign-in origin, a form posting elsewhere.
|
|
23
|
+
*/
|
|
24
|
+
export const startFakeDrive = (files: FakeDriveFiles, foreign: Origin): FakeDrive => {
|
|
25
|
+
const paths: string[] = []
|
|
26
|
+
const accounts = Bun.serve({
|
|
27
|
+
port: 0,
|
|
28
|
+
hostname: '127.0.0.1',
|
|
29
|
+
fetch: () => new Response('<html><body>Sign in</body></html>', { headers: { 'content-type': 'text/html' } }),
|
|
30
|
+
})
|
|
31
|
+
const accountsBase = `http://127.0.0.1:${accounts.port}`
|
|
32
|
+
|
|
33
|
+
const attachment = (bytes: Uint8Array, name: string, type = 'application/octet-stream'): Response =>
|
|
34
|
+
new Response(bytes, { headers: { 'content-type': type, 'content-disposition': `attachment; filename="${name}"` } })
|
|
35
|
+
|
|
36
|
+
const html = (body: string): Response => new Response(`<!DOCTYPE html><html><body>${body}</body></html>`, {
|
|
37
|
+
headers: { 'content-type': 'text/html; charset=utf-8' },
|
|
38
|
+
})
|
|
39
|
+
|
|
40
|
+
let base = ''
|
|
41
|
+
const server = Bun.serve({
|
|
42
|
+
port: 0,
|
|
43
|
+
hostname: '127.0.0.1',
|
|
44
|
+
fetch: (request: Request): Response => {
|
|
45
|
+
const url = new URL(request.url)
|
|
46
|
+
paths.push(`${url.pathname}${url.search}`)
|
|
47
|
+
const id = url.searchParams.get('id')
|
|
48
|
+
if (url.pathname === '/download') {
|
|
49
|
+
switch (id) {
|
|
50
|
+
case 'SMALLFILE0001':
|
|
51
|
+
return attachment(files.small, 'small.png', 'image/png')
|
|
52
|
+
case 'LARGEFILE0001':
|
|
53
|
+
if (url.searchParams.get('confirm') === 't' && url.searchParams.get('uuid') === 'u-123') {
|
|
54
|
+
return attachment(files.large, 'large.bin')
|
|
55
|
+
}
|
|
56
|
+
return html(`<p>Google Drive can't scan this file for viruses.</p>`
|
|
57
|
+
+ `<form id="download-form" action="${base}/download" method="get">`
|
|
58
|
+
+ `<input type="hidden" name="id" value="LARGEFILE0001"><input type="hidden" name="export" value="download">`
|
|
59
|
+
+ `<input type="hidden" name="confirm" value="t"><input type="hidden" name="uuid" value="u-123">`
|
|
60
|
+
+ `<input type="submit" value="Download anyway"></form>`)
|
|
61
|
+
case 'QUOTAFILE0001':
|
|
62
|
+
return html('<p>Sorry, you can't view or download this file at this time. Too many users have viewed or downloaded this file recently.</p>')
|
|
63
|
+
case 'PRIVATEFILE01':
|
|
64
|
+
return new Response(null, { status: 302, headers: { location: `${accountsBase}/ServiceLogin?continue=x` } })
|
|
65
|
+
case 'FOREIGNFORM01':
|
|
66
|
+
return html(`<form id="download-form" action="${foreign.base}/files/x" method="get"><input type="hidden" name="id" value="FOREIGNFORM01"></form>`)
|
|
67
|
+
default:
|
|
68
|
+
return new Response('no such file', { status: 404 })
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
const exported = /^\/(document|spreadsheets|presentation)\/d\/([^/]+)\/export(?:\/([a-z]+))?$/.exec(url.pathname)
|
|
72
|
+
if (exported != null) {
|
|
73
|
+
const format = exported[3] ?? url.searchParams.get('format')
|
|
74
|
+
if (exported[1] === 'document' && format === 'docx') return attachment(files.docx, 'doc.docx', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document')
|
|
75
|
+
if (exported[1] === 'spreadsheets' && format === 'xlsx') return attachment(files.xlsx, 'sheet.xlsx', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')
|
|
76
|
+
if (exported[1] === 'presentation' && format === 'pptx') return attachment(files.pptx, 'deck.pptx', 'application/vnd.openxmlformats-officedocument.presentationml.presentation')
|
|
77
|
+
}
|
|
78
|
+
return new Response('not found', { status: 404 })
|
|
79
|
+
},
|
|
80
|
+
})
|
|
81
|
+
base = `http://127.0.0.1:${server.port}`
|
|
82
|
+
|
|
83
|
+
return {
|
|
84
|
+
base,
|
|
85
|
+
accounts: accountsBase,
|
|
86
|
+
paths,
|
|
87
|
+
stop: () => {
|
|
88
|
+
server.stop(true)
|
|
89
|
+
accounts.stop(true)
|
|
90
|
+
},
|
|
91
|
+
}
|
|
92
|
+
}
|