@owlmeans/file-streams 0.0.0-stage → 0.2.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +71 -2
- package/agent-meta/manifest.json +64 -0
- package/agent-meta/skills/file-streams/SKILL.md +183 -0
- package/agent-meta/skills/file-streams/parts/network.md +47 -0
- package/agent-meta/skills/file-streams/parts/workspaces.md +129 -0
- package/build/archive/consts.d.ts +16 -0
- package/build/archive/consts.d.ts.map +1 -0
- package/build/archive/consts.js +24 -0
- package/build/archive/consts.js.map +1 -0
- package/build/archive/formats.d.ts +4 -0
- package/build/archive/formats.d.ts.map +1 -0
- package/build/archive/formats.js +53 -0
- package/build/archive/formats.js.map +1 -0
- package/build/archive/guard.d.ts +5 -0
- package/build/archive/guard.d.ts.map +1 -0
- package/build/archive/guard.js +62 -0
- package/build/archive/guard.js.map +1 -0
- package/build/archive/ratio.d.ts +5 -0
- package/build/archive/ratio.d.ts.map +1 -0
- package/build/archive/ratio.js +30 -0
- package/build/archive/ratio.js.map +1 -0
- package/build/archive/tar.d.ts +4 -0
- package/build/archive/tar.d.ts.map +1 -0
- package/build/archive/tar.js +142 -0
- package/build/archive/tar.js.map +1 -0
- package/build/archive/types.d.ts +95 -0
- package/build/archive/types.d.ts.map +1 -0
- package/build/archive/types.js +2 -0
- package/build/archive/types.js.map +1 -0
- package/build/archive/types.local.d.ts +64 -0
- package/build/archive/types.local.d.ts.map +1 -0
- package/build/archive/types.local.js +2 -0
- package/build/archive/types.local.js.map +1 -0
- package/build/archive/zip.d.ts +4 -0
- package/build/archive/zip.d.ts.map +1 -0
- package/build/archive/zip.js +135 -0
- package/build/archive/zip.js.map +1 -0
- package/build/archive.d.ts +4 -0
- package/build/archive.d.ts.map +1 -0
- package/build/archive.js +265 -0
- package/build/archive.js.map +1 -0
- package/build/backed-workspace/types.d.ts +67 -0
- package/build/backed-workspace/types.d.ts.map +1 -0
- package/build/backed-workspace/types.js +2 -0
- package/build/backed-workspace/types.js.map +1 -0
- package/build/backed-workspace.d.ts +4 -0
- package/build/backed-workspace.d.ts.map +1 -0
- package/build/backed-workspace.js +63 -0
- package/build/backed-workspace.js.map +1 -0
- package/build/consts.d.ts +3 -0
- package/build/consts.d.ts.map +1 -0
- package/build/consts.js +3 -0
- package/build/consts.js.map +1 -0
- package/build/consts.local.d.ts +3 -0
- package/build/consts.local.d.ts.map +1 -0
- package/build/consts.local.js +3 -0
- package/build/consts.local.js.map +1 -0
- package/build/errors.d.ts +59 -0
- package/build/errors.d.ts.map +1 -0
- package/build/errors.js +105 -0
- package/build/errors.js.map +1 -0
- package/build/http/connector.d.ts +11 -0
- package/build/http/connector.d.ts.map +1 -0
- package/build/http/connector.js +124 -0
- package/build/http/connector.js.map +1 -0
- package/build/http/consts.d.ts +9 -0
- package/build/http/consts.d.ts.map +1 -0
- package/build/http/consts.js +8 -0
- package/build/http/consts.js.map +1 -0
- package/build/http/source.d.ts +5 -0
- package/build/http/source.d.ts.map +1 -0
- package/build/http/source.js +76 -0
- package/build/http/source.js.map +1 -0
- package/build/http/types.d.ts +67 -0
- package/build/http/types.d.ts.map +1 -0
- package/build/http/types.js +2 -0
- package/build/http/types.js.map +1 -0
- package/build/http/types.local.d.ts +21 -0
- package/build/http/types.local.d.ts.map +1 -0
- package/build/http/types.local.js +2 -0
- package/build/http/types.local.js.map +1 -0
- package/build/http.d.ts +4 -0
- package/build/http.d.ts.map +1 -0
- package/build/http.js +248 -0
- package/build/http.js.map +1 -0
- package/build/index.d.ts +53 -0
- package/build/index.d.ts.map +1 -0
- package/build/index.js +33 -0
- package/build/index.js.map +1 -0
- package/build/network/consts.d.ts +12 -0
- package/build/network/consts.d.ts.map +1 -0
- package/build/network/consts.js +20 -0
- package/build/network/consts.js.map +1 -0
- package/build/network/types.d.ts +43 -0
- package/build/network/types.d.ts.map +1 -0
- package/build/network/types.js +2 -0
- package/build/network/types.js.map +1 -0
- package/build/network.d.ts +4 -0
- package/build/network.d.ts.map +1 -0
- package/build/network.js +165 -0
- package/build/network.js.map +1 -0
- package/build/proxy/consts.d.ts +7 -0
- package/build/proxy/consts.d.ts.map +1 -0
- package/build/proxy/consts.js +9 -0
- package/build/proxy/consts.js.map +1 -0
- package/build/proxy/endpoint.d.ts +4 -0
- package/build/proxy/endpoint.d.ts.map +1 -0
- package/build/proxy/endpoint.js +27 -0
- package/build/proxy/endpoint.js.map +1 -0
- package/build/proxy/http.d.ts +5 -0
- package/build/proxy/http.d.ts.map +1 -0
- package/build/proxy/http.js +95 -0
- package/build/proxy/http.js.map +1 -0
- package/build/proxy/router.d.ts +4 -0
- package/build/proxy/router.d.ts.map +1 -0
- package/build/proxy/router.js +85 -0
- package/build/proxy/router.js.map +1 -0
- package/build/proxy/socks.d.ts +5 -0
- package/build/proxy/socks.d.ts.map +1 -0
- package/build/proxy/socks.js +44 -0
- package/build/proxy/socks.js.map +1 -0
- package/build/proxy/types.d.ts +99 -0
- package/build/proxy/types.d.ts.map +1 -0
- package/build/proxy/types.js +2 -0
- package/build/proxy/types.js.map +1 -0
- package/build/resolvers/google-drive/consts.d.ts +10 -0
- package/build/resolvers/google-drive/consts.d.ts.map +1 -0
- package/build/resolvers/google-drive/consts.js +16 -0
- package/build/resolvers/google-drive/consts.js.map +1 -0
- package/build/resolvers/google-drive/types.d.ts +21 -0
- package/build/resolvers/google-drive/types.d.ts.map +1 -0
- package/build/resolvers/google-drive/types.js +2 -0
- package/build/resolvers/google-drive/types.js.map +1 -0
- package/build/resolvers/google-drive.d.ts +10 -0
- package/build/resolvers/google-drive.d.ts.map +1 -0
- package/build/resolvers/google-drive.js +130 -0
- package/build/resolvers/google-drive.js.map +1 -0
- package/build/resolvers/types.d.ts +39 -0
- package/build/resolvers/types.d.ts.map +1 -0
- package/build/resolvers/types.js +2 -0
- package/build/resolvers/types.js.map +1 -0
- package/build/s3/consts.d.ts +9 -0
- package/build/s3/consts.d.ts.map +1 -0
- package/build/s3/consts.js +9 -0
- package/build/s3/consts.js.map +1 -0
- package/build/s3/prefix.d.ts +4 -0
- package/build/s3/prefix.d.ts.map +1 -0
- package/build/s3/prefix.js +87 -0
- package/build/s3/prefix.js.map +1 -0
- package/build/s3/sink.d.ts +6 -0
- package/build/s3/sink.d.ts.map +1 -0
- package/build/s3/sink.js +30 -0
- package/build/s3/sink.js.map +1 -0
- package/build/s3/source.d.ts +5 -0
- package/build/s3/source.d.ts.map +1 -0
- package/build/s3/source.js +81 -0
- package/build/s3/source.js.map +1 -0
- package/build/s3/types.d.ts +66 -0
- package/build/s3/types.d.ts.map +1 -0
- package/build/s3/types.js +2 -0
- package/build/s3/types.js.map +1 -0
- package/build/s3/types.local.d.ts +43 -0
- package/build/s3/types.local.d.ts.map +1 -0
- package/build/s3/types.local.js +2 -0
- package/build/s3/types.local.js.map +1 -0
- package/build/s3/upload.d.ts +4 -0
- package/build/s3/upload.d.ts.map +1 -0
- package/build/s3/upload.js +118 -0
- package/build/s3/upload.js.map +1 -0
- package/build/s3.d.ts +5 -0
- package/build/s3.d.ts.map +1 -0
- package/build/s3.js +68 -0
- package/build/s3.js.map +1 -0
- package/build/service/types.d.ts +66 -0
- package/build/service/types.d.ts.map +1 -0
- package/build/service/types.js +2 -0
- package/build/service/types.js.map +1 -0
- package/build/service/types.local.d.ts +11 -0
- package/build/service/types.local.d.ts.map +1 -0
- package/build/service/types.local.js +2 -0
- package/build/service/types.local.js.map +1 -0
- package/build/service.d.ts +6 -0
- package/build/service.d.ts.map +1 -0
- package/build/service.js +73 -0
- package/build/service.js.map +1 -0
- package/build/sinks/buffer/consts.d.ts +2 -0
- package/build/sinks/buffer/consts.d.ts.map +1 -0
- package/build/sinks/buffer/consts.js +2 -0
- package/build/sinks/buffer/consts.js.map +1 -0
- package/build/sinks/buffer/types.d.ts +11 -0
- package/build/sinks/buffer/types.d.ts.map +1 -0
- package/build/sinks/buffer/types.js +2 -0
- package/build/sinks/buffer/types.js.map +1 -0
- package/build/sinks/buffer.d.ts +4 -0
- package/build/sinks/buffer.d.ts.map +1 -0
- package/build/sinks/buffer.js +13 -0
- package/build/sinks/buffer.js.map +1 -0
- package/build/sinks/directory/types.d.ts +7 -0
- package/build/sinks/directory/types.d.ts.map +1 -0
- package/build/sinks/directory/types.js +2 -0
- package/build/sinks/directory/types.js.map +1 -0
- package/build/sinks/directory.d.ts +8 -0
- package/build/sinks/directory.d.ts.map +1 -0
- package/build/sinks/directory.js +43 -0
- package/build/sinks/directory.js.map +1 -0
- package/build/sinks/file.d.ts +5 -0
- package/build/sinks/file.d.ts.map +1 -0
- package/build/sinks/file.js +46 -0
- package/build/sinks/file.js.map +1 -0
- package/build/sinks/types.d.ts +11 -0
- package/build/sinks/types.d.ts.map +1 -0
- package/build/sinks/types.js +2 -0
- package/build/sinks/types.js.map +1 -0
- package/build/sinks/workload/consts.d.ts +5 -0
- package/build/sinks/workload/consts.d.ts.map +1 -0
- package/build/sinks/workload/consts.js +5 -0
- package/build/sinks/workload/consts.js.map +1 -0
- package/build/sinks/workload/types.d.ts +17 -0
- package/build/sinks/workload/types.d.ts.map +1 -0
- package/build/sinks/workload/types.js +2 -0
- package/build/sinks/workload/types.js.map +1 -0
- package/build/sinks/workload.d.ts +8 -0
- package/build/sinks/workload.d.ts.map +1 -0
- package/build/sinks/workload.js +51 -0
- package/build/sinks/workload.js.map +1 -0
- package/build/sources/buffer.d.ts +5 -0
- package/build/sources/buffer.d.ts.map +1 -0
- package/build/sources/buffer.js +20 -0
- package/build/sources/buffer.js.map +1 -0
- package/build/sources/file.d.ts +5 -0
- package/build/sources/file.d.ts.map +1 -0
- package/build/sources/file.js +50 -0
- package/build/sources/file.js.map +1 -0
- package/build/sources/stream.d.ts +5 -0
- package/build/sources/stream.d.ts.map +1 -0
- package/build/sources/stream.js +19 -0
- package/build/sources/stream.js.map +1 -0
- package/build/sources/types.d.ts +10 -0
- package/build/sources/types.d.ts.map +1 -0
- package/build/sources/types.js +2 -0
- package/build/sources/types.js.map +1 -0
- package/build/sources/workload/types.d.ts +29 -0
- package/build/sources/workload/types.d.ts.map +1 -0
- package/build/sources/workload/types.js +2 -0
- package/build/sources/workload/types.js.map +1 -0
- package/build/sources/workload.d.ts +7 -0
- package/build/sources/workload.d.ts.map +1 -0
- package/build/sources/workload.js +67 -0
- package/build/sources/workload.js.map +1 -0
- package/build/transfer/consts.d.ts +5 -0
- package/build/transfer/consts.d.ts.map +1 -0
- package/build/transfer/consts.js +8 -0
- package/build/transfer/consts.js.map +1 -0
- package/build/transfer/types.d.ts +49 -0
- package/build/transfer/types.d.ts.map +1 -0
- package/build/transfer/types.js +2 -0
- package/build/transfer/types.js.map +1 -0
- package/build/transfer.d.ts +4 -0
- package/build/transfer.d.ts.map +1 -0
- package/build/transfer.js +75 -0
- package/build/transfer.js.map +1 -0
- package/build/types.d.ts +99 -0
- package/build/types.d.ts.map +1 -0
- package/build/types.js +2 -0
- package/build/types.js.map +1 -0
- package/build/utils/names/consts.d.ts +5 -0
- package/build/utils/names/consts.d.ts.map +1 -0
- package/build/utils/names/consts.js +5 -0
- package/build/utils/names/consts.js.map +1 -0
- package/build/utils/names/types.d.ts +13 -0
- package/build/utils/names/types.d.ts.map +1 -0
- package/build/utils/names/types.js +2 -0
- package/build/utils/names/types.js.map +1 -0
- package/build/utils/names.d.ts +4 -0
- package/build/utils/names.d.ts.map +1 -0
- package/build/utils/names.js +60 -0
- package/build/utils/names.js.map +1 -0
- package/build/utils/paths/consts.d.ts +3 -0
- package/build/utils/paths/consts.d.ts.map +1 -0
- package/build/utils/paths/consts.js +2 -0
- package/build/utils/paths/consts.js.map +1 -0
- package/build/utils/paths/types.d.ts +20 -0
- package/build/utils/paths/types.d.ts.map +1 -0
- package/build/utils/paths/types.js +2 -0
- package/build/utils/paths/types.js.map +1 -0
- package/build/utils/paths.d.ts +4 -0
- package/build/utils/paths.d.ts.map +1 -0
- package/build/utils/paths.js +67 -0
- package/build/utils/paths.js.map +1 -0
- package/build/utils/range/types.d.ts +11 -0
- package/build/utils/range/types.d.ts.map +1 -0
- package/build/utils/range/types.js +2 -0
- package/build/utils/range/types.js.map +1 -0
- package/build/utils/range.d.ts +4 -0
- package/build/utils/range.d.ts.map +1 -0
- package/build/utils/range.js +64 -0
- package/build/utils/range.js.map +1 -0
- package/build/utils/redact/types.d.ts +16 -0
- package/build/utils/redact/types.d.ts.map +1 -0
- package/build/utils/redact/types.js +2 -0
- package/build/utils/redact/types.js.map +1 -0
- package/build/utils/redact.d.ts +4 -0
- package/build/utils/redact.d.ts.map +1 -0
- package/build/utils/redact.js +18 -0
- package/build/utils/redact.js.map +1 -0
- package/build/utils/signals/types.d.ts +17 -0
- package/build/utils/signals/types.d.ts.map +1 -0
- package/build/utils/signals/types.js +2 -0
- package/build/utils/signals/types.js.map +1 -0
- package/build/utils/signals.d.ts +4 -0
- package/build/utils/signals.d.ts.map +1 -0
- package/build/utils/signals.js +52 -0
- package/build/utils/signals.js.map +1 -0
- package/build/utils/streams/consts.d.ts +3 -0
- package/build/utils/streams/consts.d.ts.map +1 -0
- package/build/utils/streams/consts.js +3 -0
- package/build/utils/streams/consts.js.map +1 -0
- package/build/utils/streams/types.d.ts +59 -0
- package/build/utils/streams/types.d.ts.map +1 -0
- package/build/utils/streams/types.js +2 -0
- package/build/utils/streams/types.js.map +1 -0
- package/build/utils/streams.d.ts +4 -0
- package/build/utils/streams.d.ts.map +1 -0
- package/build/utils/streams.js +167 -0
- package/build/utils/streams.js.map +1 -0
- package/build/utils/tree/types.d.ts +6 -0
- package/build/utils/tree/types.d.ts.map +1 -0
- package/build/utils/tree/types.js +2 -0
- package/build/utils/tree/types.js.map +1 -0
- package/build/utils/tree.d.ts +4 -0
- package/build/utils/tree.d.ts.map +1 -0
- package/build/utils/tree.js +14 -0
- package/build/utils/tree.js.map +1 -0
- package/build/workspace/consts.d.ts +12 -0
- package/build/workspace/consts.d.ts.map +1 -0
- package/build/workspace/consts.js +12 -0
- package/build/workspace/consts.js.map +1 -0
- package/build/workspace/consts.local.d.ts +3 -0
- package/build/workspace/consts.local.d.ts.map +1 -0
- package/build/workspace/consts.local.js +3 -0
- package/build/workspace/consts.local.js.map +1 -0
- package/build/workspace/fs.d.ts +4 -0
- package/build/workspace/fs.d.ts.map +1 -0
- package/build/workspace/fs.js +45 -0
- package/build/workspace/fs.js.map +1 -0
- package/build/workspace/scope.d.ts +6 -0
- package/build/workspace/scope.d.ts.map +1 -0
- package/build/workspace/scope.js +180 -0
- package/build/workspace/scope.js.map +1 -0
- package/build/workspace/types.d.ts +66 -0
- package/build/workspace/types.d.ts.map +1 -0
- package/build/workspace/types.js +2 -0
- package/build/workspace/types.js.map +1 -0
- package/build/workspace/types.local.d.ts +20 -0
- package/build/workspace/types.local.d.ts.map +1 -0
- package/build/workspace/types.local.js +2 -0
- package/build/workspace/types.local.js.map +1 -0
- package/build/workspace.d.ts +7 -0
- package/build/workspace.d.ts.map +1 -0
- package/build/workspace.js +87 -0
- package/build/workspace.js.map +1 -0
- package/package.json +61 -4
- package/src/archive/consts.ts +28 -0
- package/src/archive/formats.ts +60 -0
- package/src/archive/guard.ts +68 -0
- package/src/archive/ratio.ts +35 -0
- package/src/archive/tar.ts +147 -0
- package/src/archive/types.local.ts +72 -0
- package/src/archive/types.ts +107 -0
- package/src/archive/zip.ts +142 -0
- package/src/archive.ts +298 -0
- package/src/backed-workspace/types.ts +72 -0
- package/src/backed-workspace.ts +85 -0
- package/src/consts.local.ts +2 -0
- package/src/consts.ts +2 -0
- package/src/errors.ts +128 -0
- package/src/http/connector.ts +136 -0
- package/src/http/consts.ts +13 -0
- package/src/http/source.ts +89 -0
- package/src/http/types.local.ts +23 -0
- package/src/http/types.ts +74 -0
- package/src/http.ts +276 -0
- package/src/index.ts +65 -0
- package/src/network/consts.ts +22 -0
- package/src/network/types.ts +45 -0
- package/src/network.ts +179 -0
- package/src/proxy/consts.ts +11 -0
- package/src/proxy/endpoint.ts +28 -0
- package/src/proxy/http.ts +108 -0
- package/src/proxy/router.ts +93 -0
- package/src/proxy/socks.ts +48 -0
- package/src/proxy/types.ts +114 -0
- package/src/resolvers/google-drive/consts.ts +22 -0
- package/src/resolvers/google-drive/types.ts +22 -0
- package/src/resolvers/google-drive.ts +153 -0
- package/src/resolvers/types.ts +45 -0
- package/src/s3/consts.ts +12 -0
- package/src/s3/prefix.ts +100 -0
- package/src/s3/sink.ts +36 -0
- package/src/s3/source.ts +90 -0
- package/src/s3/types.local.ts +49 -0
- package/src/s3/types.ts +70 -0
- package/src/s3/upload.ts +130 -0
- package/src/s3.ts +83 -0
- package/src/service/types.local.ts +9 -0
- package/src/service/types.ts +71 -0
- package/src/service.ts +97 -0
- package/src/sinks/buffer/consts.ts +1 -0
- package/src/sinks/buffer/types.ts +12 -0
- package/src/sinks/buffer.ts +18 -0
- package/src/sinks/directory/types.ts +7 -0
- package/src/sinks/directory.ts +49 -0
- package/src/sinks/file.ts +52 -0
- package/src/sinks/types.ts +12 -0
- package/src/sinks/workload/consts.ts +5 -0
- package/src/sinks/workload/types.ts +17 -0
- package/src/sinks/workload.ts +59 -0
- package/src/sources/buffer.ts +27 -0
- package/src/sources/file.ts +57 -0
- package/src/sources/stream.ts +26 -0
- package/src/sources/types.ts +12 -0
- package/src/sources/workload/types.ts +31 -0
- package/src/sources/workload.ts +79 -0
- package/src/transfer/consts.ts +10 -0
- package/src/transfer/types.ts +55 -0
- package/src/transfer.ts +89 -0
- package/src/types.ts +106 -0
- package/src/utils/names/consts.ts +5 -0
- package/src/utils/names/types.ts +12 -0
- package/src/utils/names.ts +70 -0
- package/src/utils/paths/consts.ts +3 -0
- package/src/utils/paths/types.ts +20 -0
- package/src/utils/paths.ts +75 -0
- package/src/utils/range/types.ts +11 -0
- package/src/utils/range.ts +80 -0
- package/src/utils/redact/types.ts +16 -0
- package/src/utils/redact.ts +23 -0
- package/src/utils/signals/types.ts +17 -0
- package/src/utils/signals.ts +63 -0
- package/src/utils/streams/consts.ts +2 -0
- package/src/utils/streams/types.ts +65 -0
- package/src/utils/streams.ts +188 -0
- package/src/utils/tree/types.ts +6 -0
- package/src/utils/tree.ts +18 -0
- package/src/workspace/consts.local.ts +2 -0
- package/src/workspace/consts.ts +14 -0
- package/src/workspace/fs.ts +48 -0
- package/src/workspace/scope.ts +196 -0
- package/src/workspace/types.local.ts +21 -0
- package/src/workspace/types.ts +71 -0
- package/src/workspace.ts +102 -0
- package/tests/archive.spec.ts +134 -0
- package/tests/backed.spec.ts +147 -0
- package/tests/context.ts +114 -0
- package/tests/fixtures.ts +190 -0
- package/tests/google-drive.spec.ts +76 -0
- package/tests/harness/drive.ts +92 -0
- package/tests/harness/http-proxy.ts +96 -0
- package/tests/harness/origin.ts +134 -0
- package/tests/harness/socks5.ts +135 -0
- package/tests/harness/workload.ts +118 -0
- package/tests/http.spec.ts +101 -0
- package/tests/inspector.spec.ts +40 -0
- package/tests/live.spec.ts +27 -0
- package/tests/network.spec.ts +79 -0
- package/tests/proxy.spec.ts +149 -0
- package/tests/s3.spec.ts +133 -0
- package/tests/service.spec.ts +77 -0
- package/tests/sources-sinks.spec.ts +96 -0
- package/tests/transfer.spec.ts +99 -0
- package/tests/tsconfig.json +12 -0
- package/tests/workload.spec.ts +140 -0
- package/tests/workspace.spec.ts +158 -0
- package/tsconfig.json +16 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 OwlMeans Common — Fullstack typescript framework
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,72 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @owlmeans/file-streams
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Online files as Node streams, for OwlMeans servers and agents (Node and Bun):
|
|
4
|
+
|
|
5
|
+
- **Sources** — a URL (under a connect-time SSRF policy, through HTTP/SOCKS proxy plugins, Google
|
|
6
|
+
Drive links resolved), an S3 object, a file, bytes in memory, a foreign stream.
|
|
7
|
+
- **Sinks** — an atomic file, a confined directory tree, an S3 object or prefix (addressed exactly
|
|
8
|
+
like `@owlmeans/storage-resource`), a buffer, a Viable ephemeral workload (which is also a source).
|
|
9
|
+
- **Workspaces** — a service's temporary files, one confined folder per organization / project /
|
|
10
|
+
purpose, optionally mirrored to an S3 prefix under the same nesting so a restarted pod restores
|
|
11
|
+
them; whole S3 prefixes listed and deleted.
|
|
12
|
+
- **Archives** — zip, tar, tar.gz and gz: detect, list, extract, pack — with traversal, link, bomb and
|
|
13
|
+
overlap guards.
|
|
14
|
+
- **Verification** — what a file really is (images, audio, video, pdf, office documents including
|
|
15
|
+
legacy doc/xls/ppt, csv, text, source code), compared with what its name and media type claim,
|
|
16
|
+
from its head or a few range reads — never a full download.
|
|
17
|
+
|
|
18
|
+
## Installation
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
bun add @owlmeans/file-streams@^0.2.0-alpha.0
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
## Usage
|
|
25
|
+
|
|
26
|
+
```typescript
|
|
27
|
+
import { createDirectorySink, createS3Bucket, FileKind, fileInspector, httpClient, transferHelper } from '@owlmeans/file-streams'
|
|
28
|
+
|
|
29
|
+
// Download an image into a folder — refused before a byte is written unless it is an image
|
|
30
|
+
await transferHelper.copyInto(httpClient.source('https://example.com/hero.png'), createDirectorySink('/work/assets'), {
|
|
31
|
+
verify: { allow: [FileKind.Image] },
|
|
32
|
+
})
|
|
33
|
+
|
|
34
|
+
// Check a remote file without downloading it
|
|
35
|
+
const verdict = await fileInspector.probe(httpClient.source('https://example.com/report.pdf'))
|
|
36
|
+
verdict.status // 'match' | 'mismatch' | 'inconclusive'
|
|
37
|
+
verdict.detected // { ext: 'pdf', mime: 'application/pdf', kind: 'pdf', … }
|
|
38
|
+
|
|
39
|
+
// Store it in the bucket a cfg.storageBuckets entry names
|
|
40
|
+
const bucket = createS3Bucket(cfg.storageBuckets.media)
|
|
41
|
+
await transferHelper.copy(httpClient.source('https://example.com/report.pdf'), bucket.sink('reports/q3.pdf'))
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
```typescript
|
|
45
|
+
import { createBackedWorkspace, createBufferSource, createS3Bucket, createWorkspace } from '@owlmeans/file-streams'
|
|
46
|
+
|
|
47
|
+
// Keep an upload in the service's tmp folder and in the bucket, under <org>/<project>/<purpose>
|
|
48
|
+
const backed = createBackedWorkspace({ workspace: createWorkspace(), bucket: createS3Bucket(cfg.storageBuckets.projects) })
|
|
49
|
+
const files = backed.scope({ organization: 'acme', project: 'slot-12', purpose: 'new-project' })
|
|
50
|
+
await files.put('brand/logo.png', createBufferSource(png, { name: 'logo.png' }))
|
|
51
|
+
await files.restore() // after a restart: download what the tmp folder lost
|
|
52
|
+
await files.drop() // the folder and every object under the scope's prefix
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Proxies, the Viable workload destination and source, workspaces, archive limits, the context service
|
|
56
|
+
and every default are described in the package skill (`file-streams`).
|
|
57
|
+
|
|
58
|
+
<!-- owlmeans:agent-guidance:start -->
|
|
59
|
+
## Agent guidance
|
|
60
|
+
|
|
61
|
+
This package ships embedded agent skills under `agent-meta/`. After installing your
|
|
62
|
+
`@owlmeans/*` packages, run the OwlMeans agent-skills installer to place them into
|
|
63
|
+
your project's skill store (`.agents/skills/`):
|
|
64
|
+
|
|
65
|
+
```sh
|
|
66
|
+
npx @owlmeans/agent-skills@^0.2.0-alpha.2
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
The embedded files are version-matched to this package release. Do not edit them
|
|
70
|
+
directly — they are regenerated on each publish. To contribute guidance edits,
|
|
71
|
+
open a PR against the source monorepo.
|
|
72
|
+
<!-- owlmeans:agent-guidance:end -->
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 3,
|
|
3
|
+
"package": "@owlmeans/file-streams",
|
|
4
|
+
"version": "0.2.0-alpha.0",
|
|
5
|
+
"generatedAt": "2026-11-11T15:44:02.969Z",
|
|
6
|
+
"canonicalRepo": "https://github.com/owlmeans/common",
|
|
7
|
+
"entries": [
|
|
8
|
+
{
|
|
9
|
+
"kind": "skill",
|
|
10
|
+
"name": "file-streams",
|
|
11
|
+
"category": "package-specific",
|
|
12
|
+
"file": "skills/file-streams/SKILL.md",
|
|
13
|
+
"canonicalPath": ".agents/skills/file-streams/SKILL.md",
|
|
14
|
+
"parts": [
|
|
15
|
+
{
|
|
16
|
+
"name": "network",
|
|
17
|
+
"file": "skills/file-streams/parts/network.md",
|
|
18
|
+
"description": "What the HTTP client may reach — the connect-time network policy, redirects and sensitive headers, the HTTP and SOCKS proxy plugins and their pinning, why the client is node:http over its own socket under Bun, and the Google Drive and Docs resolver.",
|
|
19
|
+
"purposes": [
|
|
20
|
+
"config",
|
|
21
|
+
"ops"
|
|
22
|
+
],
|
|
23
|
+
"mentions": [
|
|
24
|
+
"createNetworkPolicy",
|
|
25
|
+
"allowCidrs",
|
|
26
|
+
"allowPrivate",
|
|
27
|
+
"createProxyRouter",
|
|
28
|
+
"createHttpProxyPlugin",
|
|
29
|
+
"createSocksProxyPlugin",
|
|
30
|
+
"trustProxyDns",
|
|
31
|
+
"createGoogleDriveResolver",
|
|
32
|
+
"googleDriveResolver",
|
|
33
|
+
"ResolverError",
|
|
34
|
+
"NetworkPolicyError"
|
|
35
|
+
]
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
"name": "workspaces",
|
|
39
|
+
"file": "skills/file-streams/parts/workspaces.md",
|
|
40
|
+
"description": "Scoped temporary workspaces nested by organization/project/purpose, workspaces mirrored to a bucket (put, restore, evict, drop), bucket prefix listing and removal, reading files back from a Viable workload, and the service's workspace config.",
|
|
41
|
+
"purposes": [
|
|
42
|
+
"data",
|
|
43
|
+
"config",
|
|
44
|
+
"ops"
|
|
45
|
+
],
|
|
46
|
+
"mentions": [
|
|
47
|
+
"createWorkspace",
|
|
48
|
+
"createBackedWorkspace",
|
|
49
|
+
"makeWorkloadSource",
|
|
50
|
+
"WorkloadReader",
|
|
51
|
+
"readBinaryFile",
|
|
52
|
+
"removePrefix",
|
|
53
|
+
"fileScopeHelper",
|
|
54
|
+
"FileScope",
|
|
55
|
+
"WorkspaceScope",
|
|
56
|
+
"BackedScope",
|
|
57
|
+
"BackedWorkspace",
|
|
58
|
+
"WORKSPACE_SCOPE_MARKER"
|
|
59
|
+
]
|
|
60
|
+
}
|
|
61
|
+
]
|
|
62
|
+
}
|
|
63
|
+
]
|
|
64
|
+
}
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: file-streams
|
|
3
|
+
description: "How to use @owlmeans/file-streams — online files as Node streams: URL downloads under a connect-time SSRF policy, HTTP and SOCKS proxy plugins (a list or a callback by key and URL), the Google Drive resolver, S3 buckets addressed exactly like storage-resource (with prefix listing and removal), folders, buffers and Viable ephemeral workloads (writes and reads); scoped temporary workspaces nested by organization/project/purpose and mirrored to a bucket for restore after a restart; zip/tar/tar.gz/gz with traversal and bomb guards; file-type verification (office, pdf, csv, media, images, source code) from a head or a few range reads. Auto-invoked when downloading a file, moving bytes between HTTP, S3, disk and a workload, keeping uploaded files in a service tmp folder backed by S3, extracting or packing an archive, checking that a file is what it claims, or configuring outbound proxies."
|
|
4
|
+
user-invocable: false
|
|
5
|
+
---
|
|
6
|
+
<!-- AUTO-GENERATED — do not edit. Regenerate via sync-agent-meta. -->
|
|
7
|
+
|
|
8
|
+
# @owlmeans/file-streams
|
|
9
|
+
|
|
10
|
+
**Layer:** Infra
|
|
11
|
+
**Install:** `"@owlmeans/file-streams": "^0.2.0-alpha.0"` in `dependencies`
|
|
12
|
+
|
|
13
|
+
Server-side (Node and Bun). Every byte moves as a Node `Readable` — the shape storage-resource's
|
|
14
|
+
`StoredRecord.stream`, the AWS SDK, zlib and the archive codecs share. A web stream, an async
|
|
15
|
+
iterable or a `Uint8Array` is accepted where a source is built.
|
|
16
|
+
|
|
17
|
+
## Key Exports
|
|
18
|
+
|
|
19
|
+
| Export | What it is |
|
|
20
|
+
|---|---|
|
|
21
|
+
| `FileSource` / `FileSink` / `TreeSink` | Where bytes come from (`info`, `open`, `ranges`), where one file goes, where many go |
|
|
22
|
+
| `httpClient` / `createHttpClient(opts)` | Requests under the network policy, through proxies, redirects and resolvers; `.source(url)` is a re-openable, Range-capable source |
|
|
23
|
+
| `createFileSource`, `createBufferSource`, `createStreamSource` | A file on disk, bytes in memory (a screenshot), a stream opened elsewhere (one-shot) |
|
|
24
|
+
| `createFileSink`, `createDirectorySink`, `createBufferSink` | Atomic file write, a confined directory tree, an in-memory collector |
|
|
25
|
+
| `createS3Bucket(entry or model, opts)` | A `cfg.storageBuckets` entry as `source`, `sink`, `tree`, `head`, `remove`, `list`, `removePrefix`, `instanceOf` |
|
|
26
|
+
| `makeWorkloadSink(writer, { root })` / `makeWorkloadSource(reader, { root })` | A Viable ephemeral workload as a tree, through its `writeBinaryFile` port; its files as sources, through `readBinaryFile` |
|
|
27
|
+
| `createWorkspace({ root?, dirMode?, fileMode? })` | A service's temporary files, one confined folder per `FileScope` (`scope`, `scopes`, `prune`) |
|
|
28
|
+
| `createBackedWorkspace({ workspace, bucket, prefix? })` | Scopes mirrored to a bucket under the same nesting: `put`, `restore`, `list`, `remove`, `evict`, `drop` |
|
|
29
|
+
| `fileScopeHelper`, `FileScope`, `FileScopeSchema`, `FileScopeError` | Re-exported from `@owlmeans/storage-common`: the `<organization>/<project?>/<purpose>` nesting |
|
|
30
|
+
| `transferHelper.copy(source, sink, opts)` / `.copyInto(source, tree, opts)` | One file moved under limits, its head verified before the sink sees a byte |
|
|
31
|
+
| `fileInspector.probe(source)` / `.inspect(bytes, claims)` / `.enforce(verdict, policy)` | What a file really is, against what it claims — re-exported from `@owlmeans/file-inspector` |
|
|
32
|
+
| `archiveHelper.extract / list / pack / gzip / gunzip / detect` | zip, tar, tar.gz and gz, detected from bytes |
|
|
33
|
+
| `createProxyRouter`, `createHttpProxyPlugin`, `createSocksProxyPlugin` | Proxy selection and the built-in protocol plugins |
|
|
34
|
+
| `createGoogleDriveResolver(opts)` / `googleDriveResolver` | Drive and Docs links to the bytes behind them (on by default in `createHttpClient`) |
|
|
35
|
+
| `createNetworkPolicy(opts)` | What may be reached; every connection goes through it |
|
|
36
|
+
| `appendFileStreamsService(ctx, alias?, opts?)` | The objects configured from `cfg.fileStreams` and `cfg.storageBuckets` |
|
|
37
|
+
| `FileKind`, `ArchiveFormat`, `WORKLOAD_MAX_FILE_BYTES`, `S3_DELETE_BATCH_SIZE`, `WORKSPACE_SCOPE_MARKER` | Enums, ceilings, the reserved marker name |
|
|
38
|
+
| `NetworkPolicyError`, `HttpError` (`.status`), `ProxyError`, `LimitError`, `SourceError`, `PathError`, `ArchiveError`, `ResolverError`, `WorkloadError` | Failures, by message code (`file-streams:<area>:<reason>`) |
|
|
39
|
+
| `FileMimicryError`, `FileKindError` (storage `FileTypeError`, from file-inspector), `S3TransferError` (storage `StorageApiError`) | Type refusals and S3 failures a storage-resource catch site already handles |
|
|
40
|
+
|
|
41
|
+
## Usage
|
|
42
|
+
|
|
43
|
+
```typescript
|
|
44
|
+
import {
|
|
45
|
+
archiveHelper, createBufferSource, createDirectorySink, createHttpClient, createS3Bucket, FileKind,
|
|
46
|
+
fileInspector, httpClient, makeWorkloadSink, transferHelper,
|
|
47
|
+
} from '@owlmeans/file-streams'
|
|
48
|
+
|
|
49
|
+
// An image from the internet into a folder, refused unless it is an image
|
|
50
|
+
await transferHelper.copyInto(httpClient.source(url), createDirectorySink(workDir), { verify: { allow: [FileKind.Image] } })
|
|
51
|
+
|
|
52
|
+
// A screenshot (a Buffer) into the bucket a storage entry names — same key and URL storage-resource uses
|
|
53
|
+
const bucket = createS3Bucket(cfg.storageBuckets.media)
|
|
54
|
+
const { sink } = await transferHelper.copy(createBufferSource(png, { name: 'shot.png' }), bucket.sink(`${entityId}/shots/${id}.png`))
|
|
55
|
+
sink.url // === storage-resource's URL for that prefix
|
|
56
|
+
bucket.instanceOf(sink, 'original') // a storage-common StoredFileInstance
|
|
57
|
+
|
|
58
|
+
// A media archive into a folder or a workload; entries that cannot be written are skipped
|
|
59
|
+
await archiveHelper.extract(httpClient.source(archiveUrl), createDirectorySink(dir), { onEntryError: 'skip' })
|
|
60
|
+
|
|
61
|
+
// What a remote file is, without downloading it (a few range reads, at most 1 MiB)
|
|
62
|
+
const verdict = await fileInspector.probe(httpClient.source(url))
|
|
63
|
+
|
|
64
|
+
// Proxies: a list (round-robin, random, failover, by-key) or a callback by key and URL
|
|
65
|
+
const pooled = createHttpClient({ proxies: { proxies: ['socks5h://u:p@egress-1:1080', 'http://egress-2:3128'], strategy: 'by-key' } })
|
|
66
|
+
const vaulted = createHttpClient({ proxies: { proxies: async ({ key, url }) => vault.proxyFor(key, url.hostname) } })
|
|
67
|
+
await pooled.request(url, { proxyKey: entityId })
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
`copy` reads the head of the very stream it moves (no second download), verifies it, and replays
|
|
71
|
+
it into the sink. Verification is on by default: a kind mismatch is refused, dangerous content (an
|
|
72
|
+
executable under another claim, a script under a passive one, a PDF with active content in the head
|
|
73
|
+
sample) is always refused, `verify.refuse` refuses kinds outright, an inconclusive verdict passes
|
|
74
|
+
unless `refuseInconclusive`; `verify: false` turns it off.
|
|
75
|
+
|
|
76
|
+
## The Viable workload
|
|
77
|
+
|
|
78
|
+
A workload accepts bytes only as the publisher's signed `writeBinaryFile { filePath, base64 }`
|
|
79
|
+
command; the agent reaches it through viable-agent's `FileHelper`, which satisfies
|
|
80
|
+
`WorkloadWriter` as is (pass the helper, never a destructured method):
|
|
81
|
+
|
|
82
|
+
```typescript
|
|
83
|
+
const media = makeWorkloadSink(fileHelper, { root: 'sources/web/public/media' })
|
|
84
|
+
await transferHelper.copy(httpClient.source(imageUrl), media.entry('hero.png'), { verify: { allow: [FileKind.Image] } })
|
|
85
|
+
await archiveHelper.extract(httpClient.source(zipUrl), media, { onEntryError: 'skip' })
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
Each file is one atomic write of at most 12 MiB (`WORKLOAD_MAX_FILE_BYTES`, the publisher's 16 MiB
|
|
89
|
+
base64 limit); `maxFileBytes` lowers it, and an oversized file is refused before any request
|
|
90
|
+
(`WorkloadError('too-large')`). Larger files need a product route (chunked writes, or the workload
|
|
91
|
+
pulling a URL itself — it has public egress and can run this package).
|
|
92
|
+
|
|
93
|
+
Reads go the other way through `makeWorkloadSource(reader, { root })` (`readBinaryFile`, same
|
|
94
|
+
ceiling): the [workspaces](parts/workspaces.md) part.
|
|
95
|
+
|
|
96
|
+
## Workspaces
|
|
97
|
+
|
|
98
|
+
`createWorkspace().scope({ organization, project?, purpose })` is the folder
|
|
99
|
+
`<root>/<organization>/<project?>/<purpose>` (segments by `fileScopeHelper`); a backed workspace
|
|
100
|
+
mirrors every scope to `<basePrefix>/<prefix?>/<scope path>/<path>` in a bucket, and the bucket is the
|
|
101
|
+
authority — a pod restart loses the tmp folder, `restore()` brings the scope back. Paths are confined
|
|
102
|
+
(no `..`, no absolute paths, no existing symlink from the workspace root, the marker name reserved);
|
|
103
|
+
a relative `root` lives under `os.tmpdir()`. Full rules, overlap and prune semantics, prefix
|
|
104
|
+
operations: the [workspaces](parts/workspaces.md) part.
|
|
105
|
+
|
|
106
|
+
## Detection
|
|
107
|
+
|
|
108
|
+
Verification is `@owlmeans/file-inspector` (its `file-inspector` skill: verdicts, policies, kinds,
|
|
109
|
+
active PDF, probes). This package re-exports its inspector, kinds, tables, verdict types and the two
|
|
110
|
+
refusals unchanged, and every source here (`FileSource`) is a file-inspector `ProbeSource` as is: a
|
|
111
|
+
probe seeks through `ranges()` when the origin has them and reads the head of `open()` otherwise. The
|
|
112
|
+
S3 sink takes the Content-Type from the verdict and writes `attachment` for active svg/html.
|
|
113
|
+
|
|
114
|
+
## Archives
|
|
115
|
+
|
|
116
|
+
zip is read from its central directory (yauzl) — through range reads when the source has them, else
|
|
117
|
+
from a private temporary spool — and written as a stream (yazl); tar and tar.gz stream through
|
|
118
|
+
tar-stream (an entry is drained, never destroyed: destroying one destroys the archive); gz is zlib.
|
|
119
|
+
The format comes from the bytes. Refused: absolute paths, `..`, drive letters, control and bidi
|
|
120
|
+
characters, overlong or deep paths, duplicate and case-fold-twin entries, encrypted entries,
|
|
121
|
+
overlapping zip entries, an expansion ratio above 200 (past 1 MiB), more entries or bytes than the
|
|
122
|
+
limits. Links and devices are skipped (or refused with `links: 'refuse'`), never created. Nested
|
|
123
|
+
archives stay files unless `limits.maxDepth` > 1. Each entry commits atomically; a refusal leaves the
|
|
124
|
+
entries already written.
|
|
125
|
+
|
|
126
|
+
## Defaults
|
|
127
|
+
|
|
128
|
+
| Area | Default |
|
|
129
|
+
|---|---|
|
|
130
|
+
| Transfer | 512 MiB · 10 min total · 30 s idle |
|
|
131
|
+
| HTTP | connect 10 s · headers 30 s · idle 30 s · 5 redirects · `Accept-Encoding: identity` |
|
|
132
|
+
| Probe | 1 MiB budget · 64 KiB head sample |
|
|
133
|
+
| Archives | 10 000 entries · 512 MiB per entry · 2 GiB total · ratio 200 · depth 1 |
|
|
134
|
+
| S3 | 8 MiB parts (≥ 5 MiB), 2 in flight; one PutObject when the body fits a part; aborted on failure |
|
|
135
|
+
| Workload | 12 MiB per file (`WORKLOAD_MAX_FILE_BYTES`), one retry, writes and reads |
|
|
136
|
+
| Workspace | `<os.tmpdir()>/owlmeans-workspace`, folders `0o700`, files `0o600` |
|
|
137
|
+
| S3 prefixes | ListObjectsV2 all pages; DeleteObjects ≤ 1000 keys, checksummed by Content-MD5 |
|
|
138
|
+
|
|
139
|
+
## Service
|
|
140
|
+
|
|
141
|
+
`appendFileStreamsService(ctx)` registers `file-streams`: `http()`, `transfer()`, `inspector()`,
|
|
142
|
+
`archives()`, `bucket(configKey)`, `workspace()` and `backed(configKey, { prefix? })` (one object per
|
|
143
|
+
key and prefix, over the configured workspace and transfer helper), built lazily from
|
|
144
|
+
`cfg.fileStreams` (`limits`, `network`, `proxies: [{ url, username?, password? }]`, `proxyStrategy`,
|
|
145
|
+
`userAgent`, `workspace: { root?, dirMode?, fileMode? }`) and `cfg.storageBuckets`. Proxy entries are
|
|
146
|
+
objects so a server context resolves their secret-file leaves; callbacks and plugins come through the
|
|
147
|
+
options argument. A server context reads EVERY string leaf starting with `/` as a file, so
|
|
148
|
+
`workspace.root` is given relative to `os.tmpdir()` (or as a file holding an absolute path).
|
|
149
|
+
|
|
150
|
+
## External docs
|
|
151
|
+
|
|
152
|
+
- https://github.com/thejoshwolfe/yauzl — central-directory zip reading, `fromRandomAccessReader` (override `read` and `_readStreamForRange`), `validateEntrySizes`, automatic `validateFileName` (v3)
|
|
153
|
+
- https://github.com/mafintosh/tar-stream — v3 ships streamx typings; destroying an entry destroys the extract, so entries are drained
|
|
154
|
+
|
|
155
|
+
## Tests
|
|
156
|
+
|
|
157
|
+
Category C. Offline specs run against local stand-ins (`tests/harness/`: an origin with Range,
|
|
158
|
+
redirects and stalls, a CONNECT/forward proxy, a SOCKS5 server, a fake Drive, a workload
|
|
159
|
+
`/command/` receiver answering `writeBinaryFile` and `readBinaryFile` with the real limits) and
|
|
160
|
+
code-built fixtures; `workspace.spec.ts` runs offline over a temporary root; `s3.spec.ts` and
|
|
161
|
+
`backed.spec.ts` need the S3 gate (path-style for a non-AWS endpoint, objects under a random base
|
|
162
|
+
prefix, deleted after); `live.spec.ts` needs `FILE_STREAMS_LIVE=1` (and `FILE_STREAMS_LIVE_DRIVE_URL`).
|
|
163
|
+
|
|
164
|
+
## Depends On
|
|
165
|
+
|
|
166
|
+
- `@owlmeans/file-inspector` (detection, re-exported), `@owlmeans/storage-resource` (the bucket
|
|
167
|
+
model), `@owlmeans/storage-common` (errors, instances), `@owlmeans/context`, `@owlmeans/error`,
|
|
168
|
+
`@owlmeans/log`
|
|
169
|
+
- `@aws-sdk/client-s3`, `socks`, `tar-stream`, `yauzl`, `yazl`
|
|
170
|
+
|
|
171
|
+
## Related
|
|
172
|
+
|
|
173
|
+
- [[file-inspector]] — what a file is: verdicts, policies, kinds, active PDF, safe names
|
|
174
|
+
- [[storage-resource]] — the upload resource and the bucket model
|
|
175
|
+
- [[storage-common]] — stored-file types and errors, `FileScope` and `fileScopeHelper`
|
|
176
|
+
- [[logging]] — scopes `file-streams:<area>`; URLs are logged redacted, contents never
|
|
177
|
+
|
|
178
|
+
## Parts
|
|
179
|
+
|
|
180
|
+
| Part | Load it when |
|
|
181
|
+
|---|---|
|
|
182
|
+
| [network](parts/network.md) | changing what may be reached, routing through proxies, or downloading Google Drive and Docs links |
|
|
183
|
+
| [workspaces](parts/workspaces.md) | keeping files in a service tmp folder by scope, mirroring them to a bucket, restoring after a restart, listing or deleting a bucket prefix, or reading files back from a workload |
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: network
|
|
3
|
+
description: What the HTTP client may reach — the connect-time network policy, redirects and sensitive headers, the HTTP and SOCKS proxy plugins and their pinning, why the client is node:http over its own socket under Bun, and the Google Drive and Docs resolver.
|
|
4
|
+
metadata:
|
|
5
|
+
purposes: config, ops
|
|
6
|
+
mentions: createNetworkPolicy, allowCidrs, allowPrivate, createProxyRouter, createHttpProxyPlugin, createSocksProxyPlugin, trustProxyDns, createGoogleDriveResolver, googleDriveResolver, ResolverError, NetworkPolicyError
|
|
7
|
+
---
|
|
8
|
+
<!-- AUTO-GENERATED — do not edit. Regenerate via sync-agent-meta. -->
|
|
9
|
+
|
|
10
|
+
# @owlmeans/file-streams — Network policy, proxies and Google Drive
|
|
11
|
+
|
|
12
|
+
## Network policy and proxies
|
|
13
|
+
|
|
14
|
+
- Loopback, private, link-local, CGNAT, multicast, reserved and documentation ranges are refused,
|
|
15
|
+
and the IPv4 inside a mapped, NAT64, 6to4 or Teredo address is checked too; internal names
|
|
16
|
+
(`localhost`, `.local`, `.internal`, `.svc`, `.cluster.local`, `.home.arpa`) are refused before DNS.
|
|
17
|
+
- The check is pinned to the connect: a direct socket goes to the address just resolved and
|
|
18
|
+
checked, a proxy tunnel carries that address (TLS keeps the host name for SNI and certificates).
|
|
19
|
+
Every redirect hop is checked again; https → http is refused; `Authorization`, `Cookie`,
|
|
20
|
+
`Proxy-Authorization` and `sensitiveHeaders` never cross origins.
|
|
21
|
+
- A local stand-in or an in-cluster origin needs `network: { allowCidrs: [...] }` (or
|
|
22
|
+
`allowPrivate`). S3 traffic through the SDK is not policed — its endpoint is operator config.
|
|
23
|
+
- Built-in plugins: `http`/`https` endpoints (CONNECT for https targets, absolute-form forward for
|
|
24
|
+
plain http — which a proxy resolves itself, so pinning holds only through CONNECT:
|
|
25
|
+
`createHttpProxyPlugin({ httpTargets: 'connect' })`), and `socks4`/`socks4a`/`socks5`/`socks5h`.
|
|
26
|
+
Remote-DNS endpoints (`socks5h`, `socks4a`) still pre-check the name locally and are refused when
|
|
27
|
+
it does not resolve unless `trustProxyDns`.
|
|
28
|
+
- Bun's own `fetch({ proxy })` speaks only http/https proxies and undici dispatchers are unreliable
|
|
29
|
+
under Bun, so the client is `node:http(s)` over a socket it connects itself — Bun 1.4.2 honours
|
|
30
|
+
the `createConnection` option, an Agent's `createConnection`, the `lookup` option and TLS over an
|
|
31
|
+
existing socket.
|
|
32
|
+
|
|
33
|
+
## Google Drive
|
|
34
|
+
|
|
35
|
+
Share links (`/file/d/<id>/…`, `open?id=`, `uc?id=`) become `drive.usercontent.google.com/download`;
|
|
36
|
+
the "can't scan this file for viruses" page is followed through its own `download-form` (action must
|
|
37
|
+
be a Google origin; hidden `confirm`/`uuid` carried), then the legacy `confirm=` link, then a
|
|
38
|
+
`download_warning` cookie. Docs, Sheets, Slides and Drawings links become export URLs (docx, xlsx,
|
|
39
|
+
pptx, png by default; csv/tsv keep the `gid`). Folders, quota pages, sign-in redirects and forms
|
|
40
|
+
posting elsewhere fail as `ResolverError('google-drive:<reason>')`. The page flow is the one gdown
|
|
41
|
+
documents; it is reverse-engineered, so `FILE_STREAMS_LIVE_DRIVE_URL` exercises it against Google.
|
|
42
|
+
|
|
43
|
+
## External docs
|
|
44
|
+
|
|
45
|
+
- https://bun.com/docs/runtime/networking/fetch — Bun `fetch({ proxy })` accepts http/https proxy URLs only (no SOCKS) (Bun 1.4)
|
|
46
|
+
- https://undici.nodejs.org/api/Socks5ProxyAgent — undici's SOCKS5 agent is experimental; dispatchers are not relied on under Bun
|
|
47
|
+
- https://docs.rs/modde-sources/latest/src/modde_sources/gdrive/mod.rs.html — the usercontent download host and the virus-scan form flow (`confirm`, `uuid`) public Drive downloads take; no official Google documentation exists for it
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: workspaces
|
|
3
|
+
description: Scoped temporary workspaces nested by organization/project/purpose, workspaces mirrored to a bucket (put, restore, evict, drop), bucket prefix listing and removal, reading files back from a Viable workload, and the service's workspace config.
|
|
4
|
+
metadata:
|
|
5
|
+
purposes: data, config, ops
|
|
6
|
+
mentions: createWorkspace, createBackedWorkspace, makeWorkloadSource, WorkloadReader, readBinaryFile, removePrefix, fileScopeHelper, FileScope, WorkspaceScope, BackedScope, BackedWorkspace, WORKSPACE_SCOPE_MARKER
|
|
7
|
+
---
|
|
8
|
+
<!-- AUTO-GENERATED — do not edit. Regenerate via sync-agent-meta. -->
|
|
9
|
+
|
|
10
|
+
# @owlmeans/file-streams — Workspaces, backed workspaces, prefixes and workload reads
|
|
11
|
+
|
|
12
|
+
## Scopes
|
|
13
|
+
|
|
14
|
+
A `FileScope` (`@owlmeans/storage-common`, re-exported here) is `{ organization, project?, purpose }`:
|
|
15
|
+
the organization's `entitySlug`, an optional project id or slug, the task (`new-project`, `upload`).
|
|
16
|
+
Each is one segment — 1–128 characters of `[A-Za-z0-9][A-Za-z0-9._-]*`, so never `.` or `..` —
|
|
17
|
+
and `fileScopeHelper.pathOf(scope)` nests them as `org/project/purpose` or `org/purpose`
|
|
18
|
+
(`parse` reads a path back, `assert` throws `FileScopeError('<field>')`). Claim a scope once, store
|
|
19
|
+
it with the records it serves and read it back; never recompute it from names that can change.
|
|
20
|
+
|
|
21
|
+
## Workspace
|
|
22
|
+
|
|
23
|
+
```typescript
|
|
24
|
+
import { createWorkspace, transferHelper, archiveHelper, httpClient, FileKind } from '@owlmeans/file-streams'
|
|
25
|
+
|
|
26
|
+
const workspace = createWorkspace() // <os.tmpdir()>/owlmeans-workspace
|
|
27
|
+
const files = workspace.scope({ organization: entitySlug, project: slotId, purpose: 'new-project' })
|
|
28
|
+
await transferHelper.copy(httpClient.source(url), files.entry('brand/logo.png'), { verify: { allow: [FileKind.Image] } })
|
|
29
|
+
await archiveHelper.extract(source, files.tree(), { onEntryError: 'skip' })
|
|
30
|
+
await files.list() // [{ path: 'brand/logo.png', size, modifiedAt }, …]
|
|
31
|
+
await files.drop() // the folder and the parents it leaves empty
|
|
32
|
+
await workspace.prune({ olderThanMs: 24 * 3600_000 })
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
- **Root**: absolute, or relative to `os.tmpdir()` (which `TMPDIR` moves) — never to the cwd.
|
|
36
|
+
Folders are created `0o700`, files written `0o600` (`dirMode`, `fileMode`). Nothing survives a pod
|
|
37
|
+
restart; that is what a backed workspace is for.
|
|
38
|
+
- **Scope folders**: `workspace.scope(scope)` does no I/O. The first write creates
|
|
39
|
+
`<root>/<scope path>` and its marker `.owlmeans-scope` (`WORKSPACE_SCOPE_MARKER`, the scope as
|
|
40
|
+
JSON); `scopes(organization?)` answers the marked folders, sorted by path.
|
|
41
|
+
- **Confinement**: every path goes through the package's path rules (`\` → `/`, NFC; refused:
|
|
42
|
+
`traversal`, `absolute`, `invalid-char`, `too-long`, `too-deep`, `empty`), a segment named like the
|
|
43
|
+
marker is `reserved`, and any existing symlink from the workspace root down to the target is
|
|
44
|
+
refused (`PathError('symlink')`) — checked before and after folders are created, and on every
|
|
45
|
+
`source` call. `list()` neither follows nor shows links and hides in-flight temporary files.
|
|
46
|
+
- **Writes**: `entry(path, { onConflict })` (default `error` → `PathError('exists')`) and
|
|
47
|
+
`tree({ onConflict })` — a confined directory sink for archives, refusing case-fold twins within
|
|
48
|
+
one tree. Each file commits atomically. `source(path, claims?)` is re-openable with random access.
|
|
49
|
+
- **Reads and removal**: `has(path)` is true for a regular file only; `remove(path)` takes a file or a
|
|
50
|
+
folder and the parents it empties (the scope folder stays), `false` when absent; `drop()` removes
|
|
51
|
+
the scope folder and every parent it empties, up to — never including — the root.
|
|
52
|
+
- **Overlap**: `org/<purpose>` contains `org/<project>/<purpose>` when a project segment equals a
|
|
53
|
+
purpose. The scope on disk first keeps the folder; the other's first write throws
|
|
54
|
+
`PathError('scope-overlap')`. A bucket cannot see this (it has no markers), so purposes never equal
|
|
55
|
+
a project segment.
|
|
56
|
+
- **Prune** drops whole scopes whose newest file, scope folder and marker are all older than
|
|
57
|
+
`olderThanMs` (never single files, so a scope is never half there), answers how many and logs it at
|
|
58
|
+
info. Writes to different scopes are independent; a drop that empties a shared parent while a
|
|
59
|
+
sibling scope is being created is retried once.
|
|
60
|
+
|
|
61
|
+
## Backed workspace
|
|
62
|
+
|
|
63
|
+
```typescript
|
|
64
|
+
import { createBackedWorkspace, createS3Bucket, createWorkspace } from '@owlmeans/file-streams'
|
|
65
|
+
|
|
66
|
+
const backed = createBackedWorkspace({ workspace: createWorkspace(), bucket: createS3Bucket(cfg.storageBuckets.projects), prefix: 'materials' })
|
|
67
|
+
const files = backed.scope(scope)
|
|
68
|
+
const { local, remote, transfer } = await files.put('brand/logo.png', source, { verify: { refuse: [FileKind.Executable] } })
|
|
69
|
+
files.key('brand/logo.png') // '<basePrefix>/materials/<org>/<project>/<purpose>/brand/logo.png'
|
|
70
|
+
files.prefixOf('brand/logo.png') // the same without the base prefix — what bucket.source/sink/head/urlOf take
|
|
71
|
+
const { restored, kept } = await files.restore() // after a restart: the bucket is the authority
|
|
72
|
+
await files.drop() // local folder + every object under the scope
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
- **Layout**: an object lands at `<basePrefix>/<prefix?>/<scope path>/<path>` — the same nesting as
|
|
76
|
+
the folder. Give the bucket entry `acl: false` (or a private bucket): the files are people's uploads,
|
|
77
|
+
and storage-resource's default ACL is `public-read`.
|
|
78
|
+
- **`put(path, source, options)`** copies the source into the local entry through the transfer helper
|
|
79
|
+
(`verify`, `limits`, `hash`, `onProgress` pass through; `onConflict` defaults to `overwrite`, like the
|
|
80
|
+
bucket), then uploads the LOCAL file, with the Content-Type and disposition of the copy's verdict
|
|
81
|
+
(`options.sink` sets cache control or metadata for every object). A failed upload removes the local
|
|
82
|
+
copy and rethrows, so everything local is backed. A refused verification writes nothing anywhere.
|
|
83
|
+
- **`restore()`** lists the scope's prefix and downloads every object that is missing locally or has
|
|
84
|
+
another size — byte-identical, unverified (it was verified on the way in); the rest are `kept`.
|
|
85
|
+
Local files the bucket does not name are left alone; folder placeholder keys (ending `/`) are skipped.
|
|
86
|
+
- **`list()`** is the remote listing with paths relative to the scope; **`remove(path)`** deletes the
|
|
87
|
+
local file or folder, the object and anything under `<path>/`; **`evict()`** drops the local copy
|
|
88
|
+
only; **`drop()`** drops the local copy and every object under the scope.
|
|
89
|
+
|
|
90
|
+
## Bucket prefixes
|
|
91
|
+
|
|
92
|
+
- `bucket.list(prefix)` answers every object under `<prefix>/` (ListObjectsV2, all pages — a sibling
|
|
93
|
+
`prefix-x` is not included) as `{ key, path, size, etag, modifiedAt }`, `path` relative to the
|
|
94
|
+
prefix; a listing carries no media type. An empty prefix is refused (`PathError('empty')`), so no call
|
|
95
|
+
ever addresses the whole base prefix.
|
|
96
|
+
- `bucket.removePrefix(prefix)` lists, then deletes in DeleteObjects batches of at most 1000 keys
|
|
97
|
+
(`S3_DELETE_BATCH_SIZE`) and answers the count. DeleteObjects requires a checksum: the request carries
|
|
98
|
+
Content-MD5 in place of the SDK's flexible checksum headers (the SDK's own MD5 fallback), which S3 and
|
|
99
|
+
stores that predate flexible checksums both accept; every other operation keeps the client's
|
|
100
|
+
`WHEN_REQUIRED` checksums. A key the store refuses throws `S3TransferError('delete-objects:<code>')`;
|
|
101
|
+
removal is not atomic, and running it again is safe.
|
|
102
|
+
|
|
103
|
+
## Workload reads
|
|
104
|
+
|
|
105
|
+
```typescript
|
|
106
|
+
const materials = makeWorkloadSource(fileHelper, { root: 'docs/materials' }) // WorkloadReader: readBinaryFile(filePath) → base64
|
|
107
|
+
await transferHelper.copy(materials.entry('logo.png'), files.entry('logo.png'), { verify: { allow: [FileKind.Image] } })
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
The publisher's `readBinaryFile { filePath } → { base64 }` command, reached through viable-agent's
|
|
111
|
+
`FileHelper` (pass the helper itself). Each `open` is one read of the whole file; `info()` reads
|
|
112
|
+
nothing and knows the size once a read has happened; `ranges()` serves the bytes of one read, so
|
|
113
|
+
probes and zip extraction work. Files above 12 MiB (`WORKLOAD_MAX_FILE_BYTES`) or a lower
|
|
114
|
+
`maxFileBytes` are refused before decoding (`WorkloadError('too-large')`); a failed read is retried
|
|
115
|
+
once (`retries`), then `WorkloadError('read-failed')`.
|
|
116
|
+
|
|
117
|
+
## Service
|
|
118
|
+
|
|
119
|
+
`cfg.fileStreams.workspace: { root?, dirMode?, fileMode? }` → `service.workspace()`;
|
|
120
|
+
`service.backed(storageKey, { prefix? })` is one backed workspace per storage entry and prefix, over
|
|
121
|
+
that workspace and the service's transfer helper (its configured limits). A server context reads
|
|
122
|
+
every string leaf that starts with `/` as a secret file, so `root` is relative to `os.tmpdir()`
|
|
123
|
+
(`root: 'agent-workspace'`), or a file that holds an absolute path.
|
|
124
|
+
|
|
125
|
+
## External docs
|
|
126
|
+
|
|
127
|
+
- https://github.com/aws/aws-sdk-js-v3/blob/main/supplemental-docs/MD5_FALLBACK.md — since v3.729 the SDK checksums with CRC32; DeleteObjects is the one operation that requires a checksum, and the documented fallback strips the flexible checksum headers and adds Content-MD5 after `flexibleChecksumsMiddleware`
|
|
128
|
+
- https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html — at most 1000 keys per request; Content-MD5 or an `x-amz-checksum-*` header is required
|
|
129
|
+
- https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html — at most 1000 keys per page, `IsTruncated` + `NextContinuationToken`
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { ArchiveLimits } from './types.js';
|
|
2
|
+
/** The archive formats read and written; detected from bytes, never from names. */
|
|
3
|
+
export declare enum ArchiveFormat {
|
|
4
|
+
Zip = "zip",
|
|
5
|
+
Tar = "tar",
|
|
6
|
+
TarGz = "tar.gz",
|
|
7
|
+
Gzip = "gz"
|
|
8
|
+
}
|
|
9
|
+
export declare const DEFAULT_ARCHIVE_LIMITS: Readonly<ArchiveLimits>;
|
|
10
|
+
/** Expansion ratios are judged only past this many produced bytes. */
|
|
11
|
+
export declare const RATIO_FLOOR_BYTES: number;
|
|
12
|
+
/** A zip local file header without its name and extra field. */
|
|
13
|
+
export declare const ZIP_LOCAL_HEADER_BYTES = 30;
|
|
14
|
+
/** Names that mark a nested archive. */
|
|
15
|
+
export declare const NESTED_ARCHIVE_EXTENSIONS: readonly string[];
|
|
16
|
+
//# sourceMappingURL=consts.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"consts.d.ts","sourceRoot":"","sources":["../../src/archive/consts.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAE/C,mFAAmF;AACnF,oBAAY,aAAa;IACvB,GAAG,QAAQ;IACX,GAAG,QAAQ;IACX,KAAK,WAAW;IAChB,IAAI,OAAO;CACZ;AAED,eAAO,MAAM,sBAAsB,EAAE,QAAQ,CAAC,aAAa,CAQzD,CAAA;AAEF,sEAAsE;AACtE,eAAO,MAAM,iBAAiB,QAAc,CAAA;AAE5C,gEAAgE;AAChE,eAAO,MAAM,sBAAsB,KAAK,CAAA;AAExC,wCAAwC;AACxC,eAAO,MAAM,yBAAyB,EAAE,SAAS,MAAM,EAAyD,CAAA"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/** The archive formats read and written; detected from bytes, never from names. */
|
|
2
|
+
export var ArchiveFormat;
|
|
3
|
+
(function (ArchiveFormat) {
|
|
4
|
+
ArchiveFormat["Zip"] = "zip";
|
|
5
|
+
ArchiveFormat["Tar"] = "tar";
|
|
6
|
+
ArchiveFormat["TarGz"] = "tar.gz";
|
|
7
|
+
ArchiveFormat["Gzip"] = "gz";
|
|
8
|
+
})(ArchiveFormat || (ArchiveFormat = {}));
|
|
9
|
+
export const DEFAULT_ARCHIVE_LIMITS = Object.freeze({
|
|
10
|
+
maxEntries: 10_000,
|
|
11
|
+
maxEntryBytes: 512 * 1024 * 1024,
|
|
12
|
+
maxTotalBytes: 2 * 1024 * 1024 * 1024,
|
|
13
|
+
maxRatio: 200,
|
|
14
|
+
maxDepth: 1,
|
|
15
|
+
maxPathBytes: 1024,
|
|
16
|
+
maxPathSegments: 64,
|
|
17
|
+
});
|
|
18
|
+
/** Expansion ratios are judged only past this many produced bytes. */
|
|
19
|
+
export const RATIO_FLOOR_BYTES = 1024 * 1024;
|
|
20
|
+
/** A zip local file header without its name and extra field. */
|
|
21
|
+
export const ZIP_LOCAL_HEADER_BYTES = 30;
|
|
22
|
+
/** Names that mark a nested archive. */
|
|
23
|
+
export const NESTED_ARCHIVE_EXTENSIONS = Object.freeze(['zip', 'tar', 'tar.gz', 'tgz', 'gz']);
|
|
24
|
+
//# sourceMappingURL=consts.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"consts.js","sourceRoot":"","sources":["../../src/archive/consts.ts"],"names":[],"mappings":"AAEA,mFAAmF;AACnF,MAAM,CAAN,IAAY,aAKX;AALD,WAAY,aAAa;IACvB,4BAAW,CAAA;IACX,4BAAW,CAAA;IACX,iCAAgB,CAAA;IAChB,4BAAW,CAAA;AACb,CAAC,EALW,aAAa,KAAb,aAAa,QAKxB;AAED,MAAM,CAAC,MAAM,sBAAsB,GAA4B,MAAM,CAAC,MAAM,CAAC;IAC3E,UAAU,EAAE,MAAM;IAClB,aAAa,EAAE,GAAG,GAAG,IAAI,GAAG,IAAI;IAChC,aAAa,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI;IACrC,QAAQ,EAAE,GAAG;IACb,QAAQ,EAAE,CAAC;IACX,YAAY,EAAE,IAAI;IAClB,eAAe,EAAE,EAAE;CACpB,CAAC,CAAA;AAEF,sEAAsE;AACtE,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,GAAG,IAAI,CAAA;AAE5C,gEAAgE;AAChE,MAAM,CAAC,MAAM,sBAAsB,GAAG,EAAE,CAAA;AAExC,wCAAwC;AACxC,MAAM,CAAC,MAAM,yBAAyB,GAAsB,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,CAAA"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"formats.d.ts","sourceRoot":"","sources":["../../src/archive/formats.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAErD,+CAA+C;AAC/C,eAAO,MAAM,mBAAmB,QAAO,aAqDtC,CAAA"}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { gunzipSync } from 'node:zlib';
|
|
2
|
+
import { constants } from 'node:zlib';
|
|
3
|
+
import { ArchiveFormat } from './consts.js';
|
|
4
|
+
/** Archive formats told from leading bytes. */
|
|
5
|
+
export const createFormatSniffer = () => {
|
|
6
|
+
const isTarHeader = (block) => {
|
|
7
|
+
if (block.length < 512) {
|
|
8
|
+
return false;
|
|
9
|
+
}
|
|
10
|
+
if (Buffer.from(block.subarray(257, 262)).toString('latin1') === 'ustar') {
|
|
11
|
+
return true;
|
|
12
|
+
}
|
|
13
|
+
const stored = parseInt(Buffer.from(block.subarray(148, 156)).toString('latin1').replace(/\0.*$/, '').trim(), 8);
|
|
14
|
+
if (Number.isNaN(stored)) {
|
|
15
|
+
return false;
|
|
16
|
+
}
|
|
17
|
+
let sum = 0;
|
|
18
|
+
for (let index = 0; index < 512; index++) {
|
|
19
|
+
sum += index >= 148 && index < 156 ? 0x20 : block[index];
|
|
20
|
+
}
|
|
21
|
+
return sum === stored;
|
|
22
|
+
};
|
|
23
|
+
const inflateHead = (head) => {
|
|
24
|
+
try {
|
|
25
|
+
return gunzipSync(head, { finishFlush: constants.Z_SYNC_FLUSH, maxOutputLength: 64 * 1024 });
|
|
26
|
+
}
|
|
27
|
+
catch {
|
|
28
|
+
return new Uint8Array(0);
|
|
29
|
+
}
|
|
30
|
+
};
|
|
31
|
+
const formatOf = (head) => {
|
|
32
|
+
if (head[0] === 0x50 && head[1] === 0x4b && [0x0304, 0x0506, 0x0708].includes((head[2] << 8) | head[3])) {
|
|
33
|
+
return ArchiveFormat.Zip;
|
|
34
|
+
}
|
|
35
|
+
if (head[0] === 0x1f && head[1] === 0x8b) {
|
|
36
|
+
return isTarHeader(inflateHead(head)) ? ArchiveFormat.TarGz : ArchiveFormat.Gzip;
|
|
37
|
+
}
|
|
38
|
+
return isTarHeader(head) ? ArchiveFormat.Tar : null;
|
|
39
|
+
};
|
|
40
|
+
const gzipName = (head) => {
|
|
41
|
+
if (head[0] !== 0x1f || head[1] !== 0x8b || (head[3] & 0x08) === 0) {
|
|
42
|
+
return undefined;
|
|
43
|
+
}
|
|
44
|
+
let offset = 10;
|
|
45
|
+
if ((head[3] & 0x04) !== 0) {
|
|
46
|
+
offset += 2 + (head[10] | (head[11] << 8));
|
|
47
|
+
}
|
|
48
|
+
const end = head.indexOf(0, offset);
|
|
49
|
+
return end > offset ? Buffer.from(head.subarray(offset, end)).toString('latin1') : undefined;
|
|
50
|
+
};
|
|
51
|
+
return { formatOf, gzipName };
|
|
52
|
+
};
|
|
53
|
+
//# sourceMappingURL=formats.js.map
|