@owlmeans/file-streams 0.0.0-stage → 0.2.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +71 -2
- package/agent-meta/manifest.json +64 -0
- package/agent-meta/skills/file-streams/SKILL.md +183 -0
- package/agent-meta/skills/file-streams/parts/network.md +47 -0
- package/agent-meta/skills/file-streams/parts/workspaces.md +129 -0
- package/build/archive/consts.d.ts +16 -0
- package/build/archive/consts.d.ts.map +1 -0
- package/build/archive/consts.js +24 -0
- package/build/archive/consts.js.map +1 -0
- package/build/archive/formats.d.ts +4 -0
- package/build/archive/formats.d.ts.map +1 -0
- package/build/archive/formats.js +53 -0
- package/build/archive/formats.js.map +1 -0
- package/build/archive/guard.d.ts +5 -0
- package/build/archive/guard.d.ts.map +1 -0
- package/build/archive/guard.js +62 -0
- package/build/archive/guard.js.map +1 -0
- package/build/archive/ratio.d.ts +5 -0
- package/build/archive/ratio.d.ts.map +1 -0
- package/build/archive/ratio.js +30 -0
- package/build/archive/ratio.js.map +1 -0
- package/build/archive/tar.d.ts +4 -0
- package/build/archive/tar.d.ts.map +1 -0
- package/build/archive/tar.js +142 -0
- package/build/archive/tar.js.map +1 -0
- package/build/archive/types.d.ts +95 -0
- package/build/archive/types.d.ts.map +1 -0
- package/build/archive/types.js +2 -0
- package/build/archive/types.js.map +1 -0
- package/build/archive/types.local.d.ts +64 -0
- package/build/archive/types.local.d.ts.map +1 -0
- package/build/archive/types.local.js +2 -0
- package/build/archive/types.local.js.map +1 -0
- package/build/archive/zip.d.ts +4 -0
- package/build/archive/zip.d.ts.map +1 -0
- package/build/archive/zip.js +135 -0
- package/build/archive/zip.js.map +1 -0
- package/build/archive.d.ts +4 -0
- package/build/archive.d.ts.map +1 -0
- package/build/archive.js +265 -0
- package/build/archive.js.map +1 -0
- package/build/backed-workspace/types.d.ts +67 -0
- package/build/backed-workspace/types.d.ts.map +1 -0
- package/build/backed-workspace/types.js +2 -0
- package/build/backed-workspace/types.js.map +1 -0
- package/build/backed-workspace.d.ts +4 -0
- package/build/backed-workspace.d.ts.map +1 -0
- package/build/backed-workspace.js +63 -0
- package/build/backed-workspace.js.map +1 -0
- package/build/consts.d.ts +3 -0
- package/build/consts.d.ts.map +1 -0
- package/build/consts.js +3 -0
- package/build/consts.js.map +1 -0
- package/build/consts.local.d.ts +3 -0
- package/build/consts.local.d.ts.map +1 -0
- package/build/consts.local.js +3 -0
- package/build/consts.local.js.map +1 -0
- package/build/errors.d.ts +59 -0
- package/build/errors.d.ts.map +1 -0
- package/build/errors.js +105 -0
- package/build/errors.js.map +1 -0
- package/build/http/connector.d.ts +11 -0
- package/build/http/connector.d.ts.map +1 -0
- package/build/http/connector.js +124 -0
- package/build/http/connector.js.map +1 -0
- package/build/http/consts.d.ts +9 -0
- package/build/http/consts.d.ts.map +1 -0
- package/build/http/consts.js +8 -0
- package/build/http/consts.js.map +1 -0
- package/build/http/source.d.ts +5 -0
- package/build/http/source.d.ts.map +1 -0
- package/build/http/source.js +76 -0
- package/build/http/source.js.map +1 -0
- package/build/http/types.d.ts +67 -0
- package/build/http/types.d.ts.map +1 -0
- package/build/http/types.js +2 -0
- package/build/http/types.js.map +1 -0
- package/build/http/types.local.d.ts +21 -0
- package/build/http/types.local.d.ts.map +1 -0
- package/build/http/types.local.js +2 -0
- package/build/http/types.local.js.map +1 -0
- package/build/http.d.ts +4 -0
- package/build/http.d.ts.map +1 -0
- package/build/http.js +248 -0
- package/build/http.js.map +1 -0
- package/build/index.d.ts +53 -0
- package/build/index.d.ts.map +1 -0
- package/build/index.js +33 -0
- package/build/index.js.map +1 -0
- package/build/network/consts.d.ts +12 -0
- package/build/network/consts.d.ts.map +1 -0
- package/build/network/consts.js +20 -0
- package/build/network/consts.js.map +1 -0
- package/build/network/types.d.ts +43 -0
- package/build/network/types.d.ts.map +1 -0
- package/build/network/types.js +2 -0
- package/build/network/types.js.map +1 -0
- package/build/network.d.ts +4 -0
- package/build/network.d.ts.map +1 -0
- package/build/network.js +165 -0
- package/build/network.js.map +1 -0
- package/build/proxy/consts.d.ts +7 -0
- package/build/proxy/consts.d.ts.map +1 -0
- package/build/proxy/consts.js +9 -0
- package/build/proxy/consts.js.map +1 -0
- package/build/proxy/endpoint.d.ts +4 -0
- package/build/proxy/endpoint.d.ts.map +1 -0
- package/build/proxy/endpoint.js +27 -0
- package/build/proxy/endpoint.js.map +1 -0
- package/build/proxy/http.d.ts +5 -0
- package/build/proxy/http.d.ts.map +1 -0
- package/build/proxy/http.js +95 -0
- package/build/proxy/http.js.map +1 -0
- package/build/proxy/router.d.ts +4 -0
- package/build/proxy/router.d.ts.map +1 -0
- package/build/proxy/router.js +85 -0
- package/build/proxy/router.js.map +1 -0
- package/build/proxy/socks.d.ts +5 -0
- package/build/proxy/socks.d.ts.map +1 -0
- package/build/proxy/socks.js +44 -0
- package/build/proxy/socks.js.map +1 -0
- package/build/proxy/types.d.ts +99 -0
- package/build/proxy/types.d.ts.map +1 -0
- package/build/proxy/types.js +2 -0
- package/build/proxy/types.js.map +1 -0
- package/build/resolvers/google-drive/consts.d.ts +10 -0
- package/build/resolvers/google-drive/consts.d.ts.map +1 -0
- package/build/resolvers/google-drive/consts.js +16 -0
- package/build/resolvers/google-drive/consts.js.map +1 -0
- package/build/resolvers/google-drive/types.d.ts +21 -0
- package/build/resolvers/google-drive/types.d.ts.map +1 -0
- package/build/resolvers/google-drive/types.js +2 -0
- package/build/resolvers/google-drive/types.js.map +1 -0
- package/build/resolvers/google-drive.d.ts +10 -0
- package/build/resolvers/google-drive.d.ts.map +1 -0
- package/build/resolvers/google-drive.js +130 -0
- package/build/resolvers/google-drive.js.map +1 -0
- package/build/resolvers/types.d.ts +39 -0
- package/build/resolvers/types.d.ts.map +1 -0
- package/build/resolvers/types.js +2 -0
- package/build/resolvers/types.js.map +1 -0
- package/build/s3/consts.d.ts +9 -0
- package/build/s3/consts.d.ts.map +1 -0
- package/build/s3/consts.js +9 -0
- package/build/s3/consts.js.map +1 -0
- package/build/s3/prefix.d.ts +4 -0
- package/build/s3/prefix.d.ts.map +1 -0
- package/build/s3/prefix.js +87 -0
- package/build/s3/prefix.js.map +1 -0
- package/build/s3/sink.d.ts +6 -0
- package/build/s3/sink.d.ts.map +1 -0
- package/build/s3/sink.js +30 -0
- package/build/s3/sink.js.map +1 -0
- package/build/s3/source.d.ts +5 -0
- package/build/s3/source.d.ts.map +1 -0
- package/build/s3/source.js +81 -0
- package/build/s3/source.js.map +1 -0
- package/build/s3/types.d.ts +66 -0
- package/build/s3/types.d.ts.map +1 -0
- package/build/s3/types.js +2 -0
- package/build/s3/types.js.map +1 -0
- package/build/s3/types.local.d.ts +43 -0
- package/build/s3/types.local.d.ts.map +1 -0
- package/build/s3/types.local.js +2 -0
- package/build/s3/types.local.js.map +1 -0
- package/build/s3/upload.d.ts +4 -0
- package/build/s3/upload.d.ts.map +1 -0
- package/build/s3/upload.js +118 -0
- package/build/s3/upload.js.map +1 -0
- package/build/s3.d.ts +5 -0
- package/build/s3.d.ts.map +1 -0
- package/build/s3.js +68 -0
- package/build/s3.js.map +1 -0
- package/build/service/types.d.ts +66 -0
- package/build/service/types.d.ts.map +1 -0
- package/build/service/types.js +2 -0
- package/build/service/types.js.map +1 -0
- package/build/service/types.local.d.ts +11 -0
- package/build/service/types.local.d.ts.map +1 -0
- package/build/service/types.local.js +2 -0
- package/build/service/types.local.js.map +1 -0
- package/build/service.d.ts +6 -0
- package/build/service.d.ts.map +1 -0
- package/build/service.js +73 -0
- package/build/service.js.map +1 -0
- package/build/sinks/buffer/consts.d.ts +2 -0
- package/build/sinks/buffer/consts.d.ts.map +1 -0
- package/build/sinks/buffer/consts.js +2 -0
- package/build/sinks/buffer/consts.js.map +1 -0
- package/build/sinks/buffer/types.d.ts +11 -0
- package/build/sinks/buffer/types.d.ts.map +1 -0
- package/build/sinks/buffer/types.js +2 -0
- package/build/sinks/buffer/types.js.map +1 -0
- package/build/sinks/buffer.d.ts +4 -0
- package/build/sinks/buffer.d.ts.map +1 -0
- package/build/sinks/buffer.js +13 -0
- package/build/sinks/buffer.js.map +1 -0
- package/build/sinks/directory/types.d.ts +7 -0
- package/build/sinks/directory/types.d.ts.map +1 -0
- package/build/sinks/directory/types.js +2 -0
- package/build/sinks/directory/types.js.map +1 -0
- package/build/sinks/directory.d.ts +8 -0
- package/build/sinks/directory.d.ts.map +1 -0
- package/build/sinks/directory.js +43 -0
- package/build/sinks/directory.js.map +1 -0
- package/build/sinks/file.d.ts +5 -0
- package/build/sinks/file.d.ts.map +1 -0
- package/build/sinks/file.js +46 -0
- package/build/sinks/file.js.map +1 -0
- package/build/sinks/types.d.ts +11 -0
- package/build/sinks/types.d.ts.map +1 -0
- package/build/sinks/types.js +2 -0
- package/build/sinks/types.js.map +1 -0
- package/build/sinks/workload/consts.d.ts +5 -0
- package/build/sinks/workload/consts.d.ts.map +1 -0
- package/build/sinks/workload/consts.js +5 -0
- package/build/sinks/workload/consts.js.map +1 -0
- package/build/sinks/workload/types.d.ts +17 -0
- package/build/sinks/workload/types.d.ts.map +1 -0
- package/build/sinks/workload/types.js +2 -0
- package/build/sinks/workload/types.js.map +1 -0
- package/build/sinks/workload.d.ts +8 -0
- package/build/sinks/workload.d.ts.map +1 -0
- package/build/sinks/workload.js +51 -0
- package/build/sinks/workload.js.map +1 -0
- package/build/sources/buffer.d.ts +5 -0
- package/build/sources/buffer.d.ts.map +1 -0
- package/build/sources/buffer.js +20 -0
- package/build/sources/buffer.js.map +1 -0
- package/build/sources/file.d.ts +5 -0
- package/build/sources/file.d.ts.map +1 -0
- package/build/sources/file.js +50 -0
- package/build/sources/file.js.map +1 -0
- package/build/sources/stream.d.ts +5 -0
- package/build/sources/stream.d.ts.map +1 -0
- package/build/sources/stream.js +19 -0
- package/build/sources/stream.js.map +1 -0
- package/build/sources/types.d.ts +10 -0
- package/build/sources/types.d.ts.map +1 -0
- package/build/sources/types.js +2 -0
- package/build/sources/types.js.map +1 -0
- package/build/sources/workload/types.d.ts +29 -0
- package/build/sources/workload/types.d.ts.map +1 -0
- package/build/sources/workload/types.js +2 -0
- package/build/sources/workload/types.js.map +1 -0
- package/build/sources/workload.d.ts +7 -0
- package/build/sources/workload.d.ts.map +1 -0
- package/build/sources/workload.js +67 -0
- package/build/sources/workload.js.map +1 -0
- package/build/transfer/consts.d.ts +5 -0
- package/build/transfer/consts.d.ts.map +1 -0
- package/build/transfer/consts.js +8 -0
- package/build/transfer/consts.js.map +1 -0
- package/build/transfer/types.d.ts +49 -0
- package/build/transfer/types.d.ts.map +1 -0
- package/build/transfer/types.js +2 -0
- package/build/transfer/types.js.map +1 -0
- package/build/transfer.d.ts +4 -0
- package/build/transfer.d.ts.map +1 -0
- package/build/transfer.js +75 -0
- package/build/transfer.js.map +1 -0
- package/build/types.d.ts +99 -0
- package/build/types.d.ts.map +1 -0
- package/build/types.js +2 -0
- package/build/types.js.map +1 -0
- package/build/utils/names/consts.d.ts +5 -0
- package/build/utils/names/consts.d.ts.map +1 -0
- package/build/utils/names/consts.js +5 -0
- package/build/utils/names/consts.js.map +1 -0
- package/build/utils/names/types.d.ts +13 -0
- package/build/utils/names/types.d.ts.map +1 -0
- package/build/utils/names/types.js +2 -0
- package/build/utils/names/types.js.map +1 -0
- package/build/utils/names.d.ts +4 -0
- package/build/utils/names.d.ts.map +1 -0
- package/build/utils/names.js +60 -0
- package/build/utils/names.js.map +1 -0
- package/build/utils/paths/consts.d.ts +3 -0
- package/build/utils/paths/consts.d.ts.map +1 -0
- package/build/utils/paths/consts.js +2 -0
- package/build/utils/paths/consts.js.map +1 -0
- package/build/utils/paths/types.d.ts +20 -0
- package/build/utils/paths/types.d.ts.map +1 -0
- package/build/utils/paths/types.js +2 -0
- package/build/utils/paths/types.js.map +1 -0
- package/build/utils/paths.d.ts +4 -0
- package/build/utils/paths.d.ts.map +1 -0
- package/build/utils/paths.js +67 -0
- package/build/utils/paths.js.map +1 -0
- package/build/utils/range/types.d.ts +11 -0
- package/build/utils/range/types.d.ts.map +1 -0
- package/build/utils/range/types.js +2 -0
- package/build/utils/range/types.js.map +1 -0
- package/build/utils/range.d.ts +4 -0
- package/build/utils/range.d.ts.map +1 -0
- package/build/utils/range.js +64 -0
- package/build/utils/range.js.map +1 -0
- package/build/utils/redact/types.d.ts +16 -0
- package/build/utils/redact/types.d.ts.map +1 -0
- package/build/utils/redact/types.js +2 -0
- package/build/utils/redact/types.js.map +1 -0
- package/build/utils/redact.d.ts +4 -0
- package/build/utils/redact.d.ts.map +1 -0
- package/build/utils/redact.js +18 -0
- package/build/utils/redact.js.map +1 -0
- package/build/utils/signals/types.d.ts +17 -0
- package/build/utils/signals/types.d.ts.map +1 -0
- package/build/utils/signals/types.js +2 -0
- package/build/utils/signals/types.js.map +1 -0
- package/build/utils/signals.d.ts +4 -0
- package/build/utils/signals.d.ts.map +1 -0
- package/build/utils/signals.js +52 -0
- package/build/utils/signals.js.map +1 -0
- package/build/utils/streams/consts.d.ts +3 -0
- package/build/utils/streams/consts.d.ts.map +1 -0
- package/build/utils/streams/consts.js +3 -0
- package/build/utils/streams/consts.js.map +1 -0
- package/build/utils/streams/types.d.ts +59 -0
- package/build/utils/streams/types.d.ts.map +1 -0
- package/build/utils/streams/types.js +2 -0
- package/build/utils/streams/types.js.map +1 -0
- package/build/utils/streams.d.ts +4 -0
- package/build/utils/streams.d.ts.map +1 -0
- package/build/utils/streams.js +167 -0
- package/build/utils/streams.js.map +1 -0
- package/build/utils/tree/types.d.ts +6 -0
- package/build/utils/tree/types.d.ts.map +1 -0
- package/build/utils/tree/types.js +2 -0
- package/build/utils/tree/types.js.map +1 -0
- package/build/utils/tree.d.ts +4 -0
- package/build/utils/tree.d.ts.map +1 -0
- package/build/utils/tree.js +14 -0
- package/build/utils/tree.js.map +1 -0
- package/build/workspace/consts.d.ts +12 -0
- package/build/workspace/consts.d.ts.map +1 -0
- package/build/workspace/consts.js +12 -0
- package/build/workspace/consts.js.map +1 -0
- package/build/workspace/consts.local.d.ts +3 -0
- package/build/workspace/consts.local.d.ts.map +1 -0
- package/build/workspace/consts.local.js +3 -0
- package/build/workspace/consts.local.js.map +1 -0
- package/build/workspace/fs.d.ts +4 -0
- package/build/workspace/fs.d.ts.map +1 -0
- package/build/workspace/fs.js +45 -0
- package/build/workspace/fs.js.map +1 -0
- package/build/workspace/scope.d.ts +6 -0
- package/build/workspace/scope.d.ts.map +1 -0
- package/build/workspace/scope.js +180 -0
- package/build/workspace/scope.js.map +1 -0
- package/build/workspace/types.d.ts +66 -0
- package/build/workspace/types.d.ts.map +1 -0
- package/build/workspace/types.js +2 -0
- package/build/workspace/types.js.map +1 -0
- package/build/workspace/types.local.d.ts +20 -0
- package/build/workspace/types.local.d.ts.map +1 -0
- package/build/workspace/types.local.js +2 -0
- package/build/workspace/types.local.js.map +1 -0
- package/build/workspace.d.ts +7 -0
- package/build/workspace.d.ts.map +1 -0
- package/build/workspace.js +87 -0
- package/build/workspace.js.map +1 -0
- package/package.json +61 -4
- package/src/archive/consts.ts +28 -0
- package/src/archive/formats.ts +60 -0
- package/src/archive/guard.ts +68 -0
- package/src/archive/ratio.ts +35 -0
- package/src/archive/tar.ts +147 -0
- package/src/archive/types.local.ts +72 -0
- package/src/archive/types.ts +107 -0
- package/src/archive/zip.ts +142 -0
- package/src/archive.ts +298 -0
- package/src/backed-workspace/types.ts +72 -0
- package/src/backed-workspace.ts +85 -0
- package/src/consts.local.ts +2 -0
- package/src/consts.ts +2 -0
- package/src/errors.ts +128 -0
- package/src/http/connector.ts +136 -0
- package/src/http/consts.ts +13 -0
- package/src/http/source.ts +89 -0
- package/src/http/types.local.ts +23 -0
- package/src/http/types.ts +74 -0
- package/src/http.ts +276 -0
- package/src/index.ts +65 -0
- package/src/network/consts.ts +22 -0
- package/src/network/types.ts +45 -0
- package/src/network.ts +179 -0
- package/src/proxy/consts.ts +11 -0
- package/src/proxy/endpoint.ts +28 -0
- package/src/proxy/http.ts +108 -0
- package/src/proxy/router.ts +93 -0
- package/src/proxy/socks.ts +48 -0
- package/src/proxy/types.ts +114 -0
- package/src/resolvers/google-drive/consts.ts +22 -0
- package/src/resolvers/google-drive/types.ts +22 -0
- package/src/resolvers/google-drive.ts +153 -0
- package/src/resolvers/types.ts +45 -0
- package/src/s3/consts.ts +12 -0
- package/src/s3/prefix.ts +100 -0
- package/src/s3/sink.ts +36 -0
- package/src/s3/source.ts +90 -0
- package/src/s3/types.local.ts +49 -0
- package/src/s3/types.ts +70 -0
- package/src/s3/upload.ts +130 -0
- package/src/s3.ts +83 -0
- package/src/service/types.local.ts +9 -0
- package/src/service/types.ts +71 -0
- package/src/service.ts +97 -0
- package/src/sinks/buffer/consts.ts +1 -0
- package/src/sinks/buffer/types.ts +12 -0
- package/src/sinks/buffer.ts +18 -0
- package/src/sinks/directory/types.ts +7 -0
- package/src/sinks/directory.ts +49 -0
- package/src/sinks/file.ts +52 -0
- package/src/sinks/types.ts +12 -0
- package/src/sinks/workload/consts.ts +5 -0
- package/src/sinks/workload/types.ts +17 -0
- package/src/sinks/workload.ts +59 -0
- package/src/sources/buffer.ts +27 -0
- package/src/sources/file.ts +57 -0
- package/src/sources/stream.ts +26 -0
- package/src/sources/types.ts +12 -0
- package/src/sources/workload/types.ts +31 -0
- package/src/sources/workload.ts +79 -0
- package/src/transfer/consts.ts +10 -0
- package/src/transfer/types.ts +55 -0
- package/src/transfer.ts +89 -0
- package/src/types.ts +106 -0
- package/src/utils/names/consts.ts +5 -0
- package/src/utils/names/types.ts +12 -0
- package/src/utils/names.ts +70 -0
- package/src/utils/paths/consts.ts +3 -0
- package/src/utils/paths/types.ts +20 -0
- package/src/utils/paths.ts +75 -0
- package/src/utils/range/types.ts +11 -0
- package/src/utils/range.ts +80 -0
- package/src/utils/redact/types.ts +16 -0
- package/src/utils/redact.ts +23 -0
- package/src/utils/signals/types.ts +17 -0
- package/src/utils/signals.ts +63 -0
- package/src/utils/streams/consts.ts +2 -0
- package/src/utils/streams/types.ts +65 -0
- package/src/utils/streams.ts +188 -0
- package/src/utils/tree/types.ts +6 -0
- package/src/utils/tree.ts +18 -0
- package/src/workspace/consts.local.ts +2 -0
- package/src/workspace/consts.ts +14 -0
- package/src/workspace/fs.ts +48 -0
- package/src/workspace/scope.ts +196 -0
- package/src/workspace/types.local.ts +21 -0
- package/src/workspace/types.ts +71 -0
- package/src/workspace.ts +102 -0
- package/tests/archive.spec.ts +134 -0
- package/tests/backed.spec.ts +147 -0
- package/tests/context.ts +114 -0
- package/tests/fixtures.ts +190 -0
- package/tests/google-drive.spec.ts +76 -0
- package/tests/harness/drive.ts +92 -0
- package/tests/harness/http-proxy.ts +96 -0
- package/tests/harness/origin.ts +134 -0
- package/tests/harness/socks5.ts +135 -0
- package/tests/harness/workload.ts +118 -0
- package/tests/http.spec.ts +101 -0
- package/tests/inspector.spec.ts +40 -0
- package/tests/live.spec.ts +27 -0
- package/tests/network.spec.ts +79 -0
- package/tests/proxy.spec.ts +149 -0
- package/tests/s3.spec.ts +133 -0
- package/tests/service.spec.ts +77 -0
- package/tests/sources-sinks.spec.ts +96 -0
- package/tests/transfer.spec.ts +99 -0
- package/tests/tsconfig.json +12 -0
- package/tests/workload.spec.ts +140 -0
- package/tests/workspace.spec.ts +158 -0
- package/tsconfig.json +16 -0
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { connect as connectTcp, isIP } from 'node:net'
|
|
2
|
+
import type { Socket } from 'node:net'
|
|
3
|
+
import type { Duplex } from 'node:stream'
|
|
4
|
+
import { connect as connectTls } from 'node:tls'
|
|
5
|
+
import { logger } from '@owlmeans/log'
|
|
6
|
+
import { HttpError, NetworkPolicyError, ProxyError } from '../errors.js'
|
|
7
|
+
import type { NetworkPolicy, ResolvedAddress } from '../network/types.js'
|
|
8
|
+
import type { ProxyRouter } from '../proxy/types.js'
|
|
9
|
+
import { redactUtils } from '../utils/redact.js'
|
|
10
|
+
import { signalUtils } from '../utils/signals.js'
|
|
11
|
+
import type { HttpTimeouts } from './types.js'
|
|
12
|
+
import type { Connection, HttpConnector } from './types.local.js'
|
|
13
|
+
|
|
14
|
+
const log = logger('file-streams:http')
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Connections the policy approved: a direct socket goes to an address that was resolved and
|
|
18
|
+
* checked a moment before (no second lookup to rebind), a proxied one carries that address into
|
|
19
|
+
* the tunnel; TLS is layered on top with the host name for SNI and certificate checks.
|
|
20
|
+
*/
|
|
21
|
+
export const makeHttpConnector = (
|
|
22
|
+
policy: NetworkPolicy, router: ProxyRouter, timeouts: HttpTimeouts, ca?: string | string[]
|
|
23
|
+
): HttpConnector => {
|
|
24
|
+
const tcp = (address: ResolvedAddress, port: number, signal?: AbortSignal): Promise<Socket> => new Promise((resolve, reject) => {
|
|
25
|
+
const socket = connectTcp({ host: address.address, port, family: address.family })
|
|
26
|
+
const timer = setTimeout(() => fail(new HttpError('connect-timeout')), timeouts.connectMs)
|
|
27
|
+
const release = signalUtils.onAbort(signal, reason => fail(reason as Error))
|
|
28
|
+
function fail (error: Error): void {
|
|
29
|
+
clearTimeout(timer)
|
|
30
|
+
release()
|
|
31
|
+
socket.destroy()
|
|
32
|
+
reject(error)
|
|
33
|
+
}
|
|
34
|
+
socket.once('connect', () => {
|
|
35
|
+
clearTimeout(timer)
|
|
36
|
+
release()
|
|
37
|
+
socket.removeAllListeners('error')
|
|
38
|
+
resolve(socket)
|
|
39
|
+
})
|
|
40
|
+
socket.once('error', () => fail(new HttpError('connect')))
|
|
41
|
+
})
|
|
42
|
+
|
|
43
|
+
const direct = async (addresses: ResolvedAddress[], port: number, signal?: AbortSignal): Promise<Socket> => {
|
|
44
|
+
let last: unknown
|
|
45
|
+
for (const address of addresses) {
|
|
46
|
+
try {
|
|
47
|
+
return await tcp(address, port, signal)
|
|
48
|
+
} catch (error) {
|
|
49
|
+
if (signal?.aborted === true) {
|
|
50
|
+
throw error
|
|
51
|
+
}
|
|
52
|
+
last = error
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
throw last ?? new HttpError('connect')
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const tls = (socket: Duplex, hostname: string, signal?: AbortSignal): Promise<Duplex> => new Promise((resolve, reject) => {
|
|
59
|
+
const literal = isIP(hostname) !== 0
|
|
60
|
+
const secure = connectTls({
|
|
61
|
+
socket, host: hostname, servername: literal ? undefined : hostname, ALPNProtocols: ['http/1.1'], minVersion: 'TLSv1.2', ca,
|
|
62
|
+
})
|
|
63
|
+
const timer = setTimeout(() => fail(new HttpError('tls-timeout')), timeouts.connectMs)
|
|
64
|
+
const release = signalUtils.onAbort(signal, reason => fail(reason as Error))
|
|
65
|
+
function fail (error: Error): void {
|
|
66
|
+
clearTimeout(timer)
|
|
67
|
+
release()
|
|
68
|
+
secure.destroy()
|
|
69
|
+
socket.destroy()
|
|
70
|
+
reject(error)
|
|
71
|
+
}
|
|
72
|
+
secure.once('secureConnect', () => {
|
|
73
|
+
clearTimeout(timer)
|
|
74
|
+
release()
|
|
75
|
+
secure.removeAllListeners('error')
|
|
76
|
+
resolve(secure)
|
|
77
|
+
})
|
|
78
|
+
secure.once('error', error => fail(error instanceof HttpError ? error : new HttpError(`tls:${(error as NodeJS.ErrnoException).code ?? 'failed'}`)))
|
|
79
|
+
})
|
|
80
|
+
|
|
81
|
+
const proxied = async (url: URL, hostname: string, port: number, secure: boolean, proxyKey: string | undefined, signal?: AbortSignal): Promise<Connection | null> => {
|
|
82
|
+
let attempts = 0
|
|
83
|
+
for (;;) {
|
|
84
|
+
const selection = await router.select({ key: proxyKey, url })
|
|
85
|
+
if (selection == null) {
|
|
86
|
+
return null
|
|
87
|
+
}
|
|
88
|
+
const { endpoint, plugin } = selection
|
|
89
|
+
let host = hostname
|
|
90
|
+
if (plugin.remoteDns(endpoint)) {
|
|
91
|
+
try {
|
|
92
|
+
await policy.resolve(hostname, signal)
|
|
93
|
+
} catch (error) {
|
|
94
|
+
if (!(error instanceof NetworkPolicyError) || error.message !== 'file-streams:policy:unresolvable' || !policy.trustProxyDns) {
|
|
95
|
+
throw error
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
} else {
|
|
99
|
+
host = (await policy.resolve(hostname, signal))[0].address
|
|
100
|
+
}
|
|
101
|
+
try {
|
|
102
|
+
const connection = await plugin.connect(endpoint, { host, port, hostname, secure }, { signal, timeoutMs: timeouts.connectMs })
|
|
103
|
+
router.report(endpoint, true)
|
|
104
|
+
log.debug('Connected through a proxy', { proxy: redactUtils.proxy(endpoint), mode: connection.mode })
|
|
105
|
+
return { socket: connection.socket, mode: connection.mode, headers: connection.headers }
|
|
106
|
+
} catch (error) {
|
|
107
|
+
if (signal?.aborted === true) {
|
|
108
|
+
throw error
|
|
109
|
+
}
|
|
110
|
+
router.report(endpoint, false)
|
|
111
|
+
if (++attempts > router.retries) {
|
|
112
|
+
throw error instanceof ProxyError ? error : new ProxyError('connect')
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
const connect = async (url: URL, proxyKey: string | undefined, signal?: AbortSignal): Promise<Connection> => {
|
|
119
|
+
const secure = url.protocol === 'https:'
|
|
120
|
+
const hostname = url.hostname.replace(/^\[|\]$/g, '')
|
|
121
|
+
const port = url.port !== '' ? Number(url.port) : secure ? 443 : 80
|
|
122
|
+
|
|
123
|
+
let connection = await proxied(url, hostname, port, secure, proxyKey, signal)
|
|
124
|
+
if (connection == null) {
|
|
125
|
+
const addresses = await policy.resolve(hostname, signal)
|
|
126
|
+
connection = { socket: await direct(addresses, port, signal), mode: 'direct' }
|
|
127
|
+
}
|
|
128
|
+
if (secure && connection.mode !== 'forward') {
|
|
129
|
+
connection = { ...connection, socket: await tls(connection.socket, hostname, signal) }
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
return connection
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
return { connect }
|
|
136
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { HttpTimeouts } from './types.js'
|
|
2
|
+
|
|
3
|
+
export const DEFAULT_HTTP_TIMEOUTS: Readonly<HttpTimeouts> = Object.freeze({ connectMs: 10_000, headersMs: 30_000, idleMs: 30_000 })
|
|
4
|
+
|
|
5
|
+
export const DEFAULT_MAX_REDIRECTS = 5
|
|
6
|
+
|
|
7
|
+
export const REDIRECT_STATUSES: readonly number[] = Object.freeze([301, 302, 303, 307, 308])
|
|
8
|
+
|
|
9
|
+
/** Headers never carried to another origin. */
|
|
10
|
+
export const ALWAYS_SENSITIVE_HEADERS: readonly string[] = Object.freeze(['authorization', 'cookie', 'proxy-authorization'])
|
|
11
|
+
|
|
12
|
+
/** Bytes a resolver may read from a page by default. */
|
|
13
|
+
export const DEFAULT_RESOLVER_PAGE_BYTES = 256 * 1024
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { SourceError } from '../errors.js'
|
|
2
|
+
import type { FileInfo, FileSource, OpenedFile, OpenOptions, RangeReader, SignalOptions } from '../types.js'
|
|
3
|
+
import { nameUtils } from '../utils/names.js'
|
|
4
|
+
import { streamUtils } from '../utils/streams.js'
|
|
5
|
+
import type { HttpClient, HttpRequestOptions, HttpSourceOptions } from './types.js'
|
|
6
|
+
import type { KnownHttpFile } from './types.local.js'
|
|
7
|
+
|
|
8
|
+
/** A URL as a source: re-openable; Range-capable when the server answers ranges. */
|
|
9
|
+
export const makeHttpSource = (client: HttpClient, url: URL, options: HttpSourceOptions = {}): FileSource => {
|
|
10
|
+
let known: Promise<KnownHttpFile> | undefined
|
|
11
|
+
|
|
12
|
+
const base = (signal?: AbortSignal): HttpRequestOptions => ({ headers: options.headers, proxyKey: options.proxyKey, raw: options.raw, signal })
|
|
13
|
+
|
|
14
|
+
const named = (info: FileInfo): FileInfo => ({ ...info, name: nameUtils.sanitize(options.name) ?? info.name })
|
|
15
|
+
|
|
16
|
+
const discover = async (signal?: AbortSignal): Promise<KnownHttpFile> => {
|
|
17
|
+
const response = await client.request(url, { ...base(signal), range: { start: 0, end: 0 } })
|
|
18
|
+
streamUtils.discard(response.body)
|
|
19
|
+
const etag = response.info.etag
|
|
20
|
+
return {
|
|
21
|
+
info: named(response.info),
|
|
22
|
+
rangeable: response.status === 206 && response.info.size != null,
|
|
23
|
+
final: response.url,
|
|
24
|
+
validator: etag != null && !etag.startsWith('W/') ? etag : undefined,
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const learn = async (signal?: AbortSignal): Promise<KnownHttpFile> => {
|
|
29
|
+
known ??= discover(signal)
|
|
30
|
+
try {
|
|
31
|
+
return await known
|
|
32
|
+
} catch (error) {
|
|
33
|
+
known = undefined
|
|
34
|
+
throw error
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const info = async (infoOptions: SignalOptions = {}): Promise<FileInfo> => (await learn(infoOptions.signal)).info
|
|
39
|
+
|
|
40
|
+
const open = async (openOptions: OpenOptions = {}): Promise<OpenedFile> => {
|
|
41
|
+
const response = await client.request(url, { ...base(openOptions.signal), range: openOptions.range })
|
|
42
|
+
if (openOptions.range != null && response.status !== 206) {
|
|
43
|
+
streamUtils.discard(response.body)
|
|
44
|
+
throw new SourceError('range-unsupported')
|
|
45
|
+
}
|
|
46
|
+
const opened: OpenedFile = { stream: response.body, info: named(response.info) }
|
|
47
|
+
const span = /bytes (\d+)-(\d+)\/(\d+)/.exec(String(response.headers['content-range'] ?? ''))
|
|
48
|
+
if (span != null) {
|
|
49
|
+
opened.range = { start: Number(span[1]), end: Number(span[2]), total: Number(span[3]) }
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
return opened
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const ranges = async (rangeOptions: SignalOptions = {}): Promise<RangeReader | null> => {
|
|
56
|
+
const current = await learn(rangeOptions.signal)
|
|
57
|
+
if (!current.rangeable || current.info.size == null) {
|
|
58
|
+
return null
|
|
59
|
+
}
|
|
60
|
+
const size = current.info.size
|
|
61
|
+
|
|
62
|
+
const fetchRange = async (start: number, end: number, signal?: AbortSignal) => {
|
|
63
|
+
const response = await client.request(current.final, {
|
|
64
|
+
...base(signal), raw: true, range: { start, end }, ifRange: current.validator,
|
|
65
|
+
})
|
|
66
|
+
if (response.status !== 206) {
|
|
67
|
+
streamUtils.discard(response.body)
|
|
68
|
+
throw new SourceError('changed')
|
|
69
|
+
}
|
|
70
|
+
return response
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
return {
|
|
74
|
+
size,
|
|
75
|
+
read: async (offset, length, signal) => {
|
|
76
|
+
const end = Math.min(offset + length, size) - 1
|
|
77
|
+
if (end < offset) {
|
|
78
|
+
return new Uint8Array(0)
|
|
79
|
+
}
|
|
80
|
+
const response = await fetchRange(offset, end, signal)
|
|
81
|
+
return await streamUtils.collect(response.body, end - offset + 1)
|
|
82
|
+
},
|
|
83
|
+
stream: async (start, end, signal) => (await fetchRange(start, end - 1, signal)).body,
|
|
84
|
+
close: async () => undefined,
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
return { kind: 'http', info, open, ranges }
|
|
89
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { Duplex } from 'node:stream'
|
|
2
|
+
import type { FileInfo } from '../types.js'
|
|
3
|
+
|
|
4
|
+
export interface Connection {
|
|
5
|
+
socket: Duplex
|
|
6
|
+
/** `direct` and `tunnel` carry origin-form requests; `forward` absolute-form ones. */
|
|
7
|
+
mode: 'direct' | 'tunnel' | 'forward'
|
|
8
|
+
headers?: Record<string, string>
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export interface HttpConnector {
|
|
12
|
+
/** A connected (and, for https, TLS-wrapped) socket to the URL's origin. */
|
|
13
|
+
connect: (url: URL, proxyKey: string | undefined, signal?: AbortSignal) => Promise<Connection>
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/** What a source learned from its first response. */
|
|
17
|
+
export interface KnownHttpFile {
|
|
18
|
+
info: FileInfo
|
|
19
|
+
rangeable: boolean
|
|
20
|
+
/** The URL that served the bytes, after redirects and resolvers. */
|
|
21
|
+
final: URL
|
|
22
|
+
validator?: string
|
|
23
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import type { Readable } from 'node:stream'
|
|
2
|
+
import type { NetworkPolicy, NetworkPolicyOptions } from '../network/types.js'
|
|
3
|
+
import type { ProxyRouter, ProxyRouterOptions } from '../proxy/types.js'
|
|
4
|
+
import type { UrlResolver } from '../resolvers/types.js'
|
|
5
|
+
import type { ByteRange, FileInfo, FileSource, SignalOptions } from '../types.js'
|
|
6
|
+
|
|
7
|
+
export interface HttpTimeouts {
|
|
8
|
+
/** TCP (and proxy handshake) connect. Default 10 000 ms. */
|
|
9
|
+
connectMs: number
|
|
10
|
+
/** From the request to the response head. Default 30 000 ms. */
|
|
11
|
+
headersMs: number
|
|
12
|
+
/** Longest silence while the body flows. Default 30 000 ms. */
|
|
13
|
+
idleMs: number
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export interface HttpClientOptions {
|
|
17
|
+
network?: NetworkPolicy | NetworkPolicyOptions
|
|
18
|
+
proxies?: ProxyRouter | ProxyRouterOptions
|
|
19
|
+
/** Default `[googleDriveResolver]`; `[]` turns resolving off. */
|
|
20
|
+
resolvers?: UrlResolver[]
|
|
21
|
+
timeouts?: Partial<HttpTimeouts>
|
|
22
|
+
/** Default 5. */
|
|
23
|
+
maxRedirects?: number
|
|
24
|
+
userAgent?: string
|
|
25
|
+
/** Sent with every request. */
|
|
26
|
+
headers?: Record<string, string>
|
|
27
|
+
/** Dropped on a cross-origin redirect, beside `authorization`, `cookie` and `proxy-authorization`. */
|
|
28
|
+
sensitiveHeaders?: string[]
|
|
29
|
+
/** Extra CAs (verification is never turned off). */
|
|
30
|
+
ca?: string | string[]
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** The two methods a download issues. */
|
|
34
|
+
// A closed pair of HTTP verbs — a union.
|
|
35
|
+
export type HttpMethod = 'GET' | 'HEAD'
|
|
36
|
+
|
|
37
|
+
export interface HttpRequestOptions extends SignalOptions {
|
|
38
|
+
method?: HttpMethod
|
|
39
|
+
headers?: Record<string, string>
|
|
40
|
+
range?: ByteRange
|
|
41
|
+
/** Routing key for `by-key` pools and proxy providers. */
|
|
42
|
+
proxyKey?: string
|
|
43
|
+
/** Skip the URL resolvers. */
|
|
44
|
+
raw?: boolean
|
|
45
|
+
/** Send `If-Range` with this strong validator. */
|
|
46
|
+
ifRange?: string
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export interface HttpResponse {
|
|
50
|
+
/** The final URL; redact before logging. */
|
|
51
|
+
url: URL
|
|
52
|
+
status: number
|
|
53
|
+
headers: Record<string, string | string[] | undefined>
|
|
54
|
+
/** The body, transport-decoded (never a gzip file itself), idle-guarded, length-checked. */
|
|
55
|
+
body: Readable
|
|
56
|
+
info: FileInfo
|
|
57
|
+
redirects: number
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export interface HttpSourceOptions {
|
|
61
|
+
headers?: Record<string, string>
|
|
62
|
+
proxyKey?: string
|
|
63
|
+
raw?: boolean
|
|
64
|
+
/** The file name to claim, over what the response says. */
|
|
65
|
+
name?: string
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** Downloads under the network policy, through proxies, redirects and resolvers. */
|
|
69
|
+
export interface HttpClient {
|
|
70
|
+
/** One GET or HEAD; a non-2xx answer is `HttpError('status:<code>')`. */
|
|
71
|
+
request: (url: string | URL, options?: HttpRequestOptions) => Promise<HttpResponse>
|
|
72
|
+
/** A re-openable source; Range-capable when the server answers 206 with Content-Range. */
|
|
73
|
+
source: (url: string | URL, options?: HttpSourceOptions) => FileSource
|
|
74
|
+
}
|
package/src/http.ts
ADDED
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
import { request as httpRequest } from 'node:http'
|
|
2
|
+
import type { IncomingMessage, RequestOptions } from 'node:http'
|
|
3
|
+
import { request as httpsRequest } from 'node:https'
|
|
4
|
+
import type { Socket } from 'node:net'
|
|
5
|
+
import { pipeline, Transform } from 'node:stream'
|
|
6
|
+
import type { Readable } from 'node:stream'
|
|
7
|
+
import { createBrotliDecompress, createGunzip, createInflate } from 'node:zlib'
|
|
8
|
+
import { logger } from '@owlmeans/log'
|
|
9
|
+
import { NON_INFORMATIVE_MIMES } from '@owlmeans/file-inspector'
|
|
10
|
+
import { HttpError, ProxyError, ResolverError } from './errors.js'
|
|
11
|
+
import { makeHttpConnector } from './http/connector.js'
|
|
12
|
+
import { ALWAYS_SENSITIVE_HEADERS, DEFAULT_HTTP_TIMEOUTS, DEFAULT_MAX_REDIRECTS, DEFAULT_RESOLVER_PAGE_BYTES, REDIRECT_STATUSES } from './http/consts.js'
|
|
13
|
+
import { makeHttpSource } from './http/source.js'
|
|
14
|
+
import type { HttpClient, HttpClientOptions, HttpRequestOptions, HttpResponse } from './http/types.js'
|
|
15
|
+
import type { Connection } from './http/types.local.js'
|
|
16
|
+
import { createNetworkPolicy } from './network.js'
|
|
17
|
+
import type { NetworkPolicy } from './network/types.js'
|
|
18
|
+
import { createProxyRouter } from './proxy/router.js'
|
|
19
|
+
import type { ProxyRouter } from './proxy/types.js'
|
|
20
|
+
import { googleDriveResolver } from './resolvers/google-drive.js'
|
|
21
|
+
import type { UrlResolver } from './resolvers/types.js'
|
|
22
|
+
import type { FileInfo } from './types.js'
|
|
23
|
+
import { nameUtils } from './utils/names.js'
|
|
24
|
+
import { redactUtils } from './utils/redact.js'
|
|
25
|
+
import { signalUtils } from './utils/signals.js'
|
|
26
|
+
import { streamUtils } from './utils/streams.js'
|
|
27
|
+
import type { PeekedStream } from './utils/streams/types.js'
|
|
28
|
+
import { PACKAGE_USER_AGENT } from './consts.local.js'
|
|
29
|
+
|
|
30
|
+
const log = logger('file-streams:http')
|
|
31
|
+
|
|
32
|
+
export const createHttpClient = (options: HttpClientOptions = {}): HttpClient => {
|
|
33
|
+
const policy: NetworkPolicy = options.network != null && 'assertUrl' in options.network
|
|
34
|
+
? options.network
|
|
35
|
+
: createNetworkPolicy(options.network)
|
|
36
|
+
const router: ProxyRouter = options.proxies != null && 'select' in options.proxies && typeof options.proxies.select === 'function'
|
|
37
|
+
? options.proxies as ProxyRouter
|
|
38
|
+
: createProxyRouter(options.proxies as Parameters<typeof createProxyRouter>[0])
|
|
39
|
+
const resolvers: UrlResolver[] = options.resolvers ?? [googleDriveResolver]
|
|
40
|
+
const timeouts = { ...DEFAULT_HTTP_TIMEOUTS, ...options.timeouts }
|
|
41
|
+
const maxRedirects = options.maxRedirects ?? DEFAULT_MAX_REDIRECTS
|
|
42
|
+
const sensitive = [...ALWAYS_SENSITIVE_HEADERS, ...(options.sensitiveHeaders ?? []).map(name => name.toLowerCase())]
|
|
43
|
+
const connector = makeHttpConnector(policy, router, timeouts, options.ca)
|
|
44
|
+
|
|
45
|
+
const header = (headers: IncomingMessage['headers'], name: string): string | undefined => {
|
|
46
|
+
const value = headers[name]
|
|
47
|
+
return Array.isArray(value) ? value[0] : value
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const mimeOf = (value: string | undefined): string | undefined => {
|
|
51
|
+
const mime = value?.split(';')[0].trim().toLowerCase()
|
|
52
|
+
return mime == null || mime === '' || NON_INFORMATIVE_MIMES.includes(mime) ? undefined : mime
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const isGzipFile = (url: URL, response: IncomingMessage): boolean => {
|
|
56
|
+
const mime = mimeOf(header(response.headers, 'content-type'))
|
|
57
|
+
return mime === 'application/gzip' || mime === 'application/x-gzip' || /\.(t?gz)$/i.test(url.pathname)
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
const infoOf = (url: URL, response: IncomingMessage, decoded: boolean): FileInfo => {
|
|
61
|
+
const status = response.statusCode ?? 0
|
|
62
|
+
let size: number | undefined
|
|
63
|
+
if (status === 206) {
|
|
64
|
+
const total = /\/(\d+)\s*$/.exec(header(response.headers, 'content-range') ?? '')?.[1]
|
|
65
|
+
size = total != null ? Number(total) : undefined
|
|
66
|
+
} else if (!decoded) {
|
|
67
|
+
const length = header(response.headers, 'content-length')
|
|
68
|
+
size = length != null && /^\d+$/.test(length) ? Number(length) : undefined
|
|
69
|
+
}
|
|
70
|
+
const modified = header(response.headers, 'last-modified')
|
|
71
|
+
const info: FileInfo = {
|
|
72
|
+
origin: redactUtils.url(url),
|
|
73
|
+
name: nameUtils.fromDisposition(header(response.headers, 'content-disposition')) ?? nameUtils.fromUrl(url),
|
|
74
|
+
size,
|
|
75
|
+
mime: mimeOf(header(response.headers, 'content-type')),
|
|
76
|
+
etag: header(response.headers, 'etag'),
|
|
77
|
+
}
|
|
78
|
+
if (modified != null && !Number.isNaN(Date.parse(modified))) {
|
|
79
|
+
info.modifiedAt = new Date(modified)
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
return info
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const send = (url: URL, method: string, headers: Record<string, string>, connection: Connection, signal?: AbortSignal): Promise<IncomingMessage> =>
|
|
86
|
+
new Promise((resolve, reject) => {
|
|
87
|
+
const secure = url.protocol === 'https:'
|
|
88
|
+
const forward = connection.mode === 'forward'
|
|
89
|
+
const requestOptions: RequestOptions = {
|
|
90
|
+
method,
|
|
91
|
+
host: url.hostname.replace(/^\[|\]$/g, ''),
|
|
92
|
+
port: url.port !== '' ? Number(url.port) : secure ? 443 : 80,
|
|
93
|
+
path: forward ? url.href : `${url.pathname}${url.search}`,
|
|
94
|
+
headers: { ...headers, ...connection.headers },
|
|
95
|
+
createConnection: (() => connection.socket as Socket) as unknown as RequestOptions['createConnection'],
|
|
96
|
+
}
|
|
97
|
+
const request = (secure ? httpsRequest : httpRequest)(requestOptions)
|
|
98
|
+
const timer = setTimeout(() => request.destroy(new HttpError('headers-timeout')), timeouts.headersMs)
|
|
99
|
+
const release = signalUtils.onAbort(signal, reason => request.destroy(reason as Error))
|
|
100
|
+
request.once('response', response => {
|
|
101
|
+
clearTimeout(timer)
|
|
102
|
+
release()
|
|
103
|
+
resolve(response)
|
|
104
|
+
})
|
|
105
|
+
request.once('error', error => {
|
|
106
|
+
clearTimeout(timer)
|
|
107
|
+
release()
|
|
108
|
+
connection.socket.destroy()
|
|
109
|
+
reject(error instanceof HttpError || signal?.aborted === true ? error : new HttpError('request'))
|
|
110
|
+
})
|
|
111
|
+
request.end()
|
|
112
|
+
})
|
|
113
|
+
|
|
114
|
+
const bodyOf = (url: URL, response: IncomingMessage): { body: Readable, decoded: boolean } => {
|
|
115
|
+
const encoding = header(response.headers, 'content-encoding')?.trim().toLowerCase()
|
|
116
|
+
const stages: NodeJS.ReadWriteStream[] = []
|
|
117
|
+
let decoded = false
|
|
118
|
+
if (encoding != null && encoding !== '' && encoding !== 'identity' && !isGzipFile(url, response)) {
|
|
119
|
+
decoded = true
|
|
120
|
+
if (encoding === 'gzip' || encoding === 'x-gzip') {
|
|
121
|
+
stages.push(createGunzip())
|
|
122
|
+
} else if (encoding === 'deflate') {
|
|
123
|
+
stages.push(createInflate())
|
|
124
|
+
} else if (encoding === 'br') {
|
|
125
|
+
stages.push(createBrotliDecompress())
|
|
126
|
+
} else {
|
|
127
|
+
response.destroy()
|
|
128
|
+
throw new HttpError('encoding')
|
|
129
|
+
}
|
|
130
|
+
} else {
|
|
131
|
+
const length = header(response.headers, 'content-length')
|
|
132
|
+
if (length != null && /^\d+$/.test(length)) {
|
|
133
|
+
const expected = Number(length)
|
|
134
|
+
let seen = 0
|
|
135
|
+
stages.push(new Transform({
|
|
136
|
+
transform: (chunk: Buffer, _encoding, callback) => {
|
|
137
|
+
seen += chunk.length
|
|
138
|
+
callback(null, chunk)
|
|
139
|
+
},
|
|
140
|
+
flush: callback => callback(seen < expected ? new HttpError('truncated') : null),
|
|
141
|
+
}))
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
stages.push(streamUtils.meter({ idleTimeoutMs: timeouts.idleMs }).stream)
|
|
145
|
+
response.once('aborted', () => response.destroy(new HttpError('truncated')))
|
|
146
|
+
const body = (pipeline as unknown as (...streams: unknown[]) => Readable)(response, ...stages, () => undefined)
|
|
147
|
+
|
|
148
|
+
return { body, decoded }
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
const request = async (input: string | URL, requestOptions: HttpRequestOptions = {}): Promise<HttpResponse> => {
|
|
152
|
+
const signal = requestOptions.signal
|
|
153
|
+
const method = requestOptions.method ?? 'GET'
|
|
154
|
+
let current = new URL(input)
|
|
155
|
+
policy.assertUrl(current)
|
|
156
|
+
|
|
157
|
+
const resolver = requestOptions.raw === true ? undefined : resolvers.find(candidate => candidate.matches(current))
|
|
158
|
+
if (resolver != null && requestOptions.range != null) {
|
|
159
|
+
// A resolver reads pages whole: resolve first, then ask the bytes' own URL for the range.
|
|
160
|
+
const resolved = await request(current, { ...requestOptions, range: undefined, ifRange: undefined })
|
|
161
|
+
streamUtils.discard(resolved.body)
|
|
162
|
+
return await request(resolved.url, { ...requestOptions, raw: true })
|
|
163
|
+
}
|
|
164
|
+
if (resolver != null) {
|
|
165
|
+
current = resolver.rewrite(current)
|
|
166
|
+
policy.assertUrl(current)
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
const headers: Record<string, string> = {
|
|
170
|
+
'user-agent': options.userAgent ?? PACKAGE_USER_AGENT,
|
|
171
|
+
accept: '*/*',
|
|
172
|
+
'accept-encoding': 'identity',
|
|
173
|
+
connection: 'close',
|
|
174
|
+
...lowerKeys(options.headers),
|
|
175
|
+
...lowerKeys(requestOptions.headers),
|
|
176
|
+
}
|
|
177
|
+
if (requestOptions.range != null) {
|
|
178
|
+
headers.range = `bytes=${requestOptions.range.start}-${requestOptions.range.end ?? ''}`
|
|
179
|
+
if (requestOptions.ifRange != null) {
|
|
180
|
+
headers['if-range'] = requestOptions.ifRange
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
let redirects = 0
|
|
185
|
+
let steps = 0
|
|
186
|
+
let cookie: string | undefined
|
|
187
|
+
for (;;) {
|
|
188
|
+
signalUtils.throwIfAborted(signal)
|
|
189
|
+
const connection = await connector.connect(current, requestOptions.proxyKey, signal)
|
|
190
|
+
const outgoing: Record<string, string> = { ...headers, host: current.host }
|
|
191
|
+
if (cookie != null && resolver?.owns(current) === true) {
|
|
192
|
+
outgoing.cookie = cookie
|
|
193
|
+
}
|
|
194
|
+
const response = await send(current, method, outgoing, connection, signal)
|
|
195
|
+
const status = response.statusCode ?? 0
|
|
196
|
+
const location = header(response.headers, 'location')
|
|
197
|
+
|
|
198
|
+
if (REDIRECT_STATUSES.includes(status) && location != null) {
|
|
199
|
+
response.resume()
|
|
200
|
+
response.destroy()
|
|
201
|
+
if (++redirects > maxRedirects) {
|
|
202
|
+
throw new HttpError('redirects')
|
|
203
|
+
}
|
|
204
|
+
let next: URL
|
|
205
|
+
try {
|
|
206
|
+
next = new URL(location, current)
|
|
207
|
+
} catch {
|
|
208
|
+
throw new HttpError('location')
|
|
209
|
+
}
|
|
210
|
+
policy.assertUrl(next, current)
|
|
211
|
+
if (next.origin !== current.origin) {
|
|
212
|
+
sensitive.forEach(name => { delete headers[name] })
|
|
213
|
+
}
|
|
214
|
+
current = next
|
|
215
|
+
continue
|
|
216
|
+
}
|
|
217
|
+
if (status < 200 || status >= 300) {
|
|
218
|
+
response.resume()
|
|
219
|
+
response.destroy()
|
|
220
|
+
if (status === 407 && connection.mode === 'forward') {
|
|
221
|
+
throw new ProxyError('auth-failed')
|
|
222
|
+
}
|
|
223
|
+
throw new HttpError(`status:${status}`)
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
let { body, decoded } = method === 'HEAD'
|
|
227
|
+
? { body: response, decoded: false }
|
|
228
|
+
: bodyOf(current, response)
|
|
229
|
+
|
|
230
|
+
if (resolver != null && method === 'GET') {
|
|
231
|
+
let peeked: PeekedStream | undefined
|
|
232
|
+
const readBody = async (maxBytes: number = DEFAULT_RESOLVER_PAGE_BYTES): Promise<string> => {
|
|
233
|
+
peeked ??= await streamUtils.peek(body, maxBytes)
|
|
234
|
+
return Buffer.from(peeked.head.subarray(0, maxBytes)).toString('utf8')
|
|
235
|
+
}
|
|
236
|
+
const step = await resolver.inspect({ url: current, status, headers: response.headers }, readBody)
|
|
237
|
+
if (peeked != null) {
|
|
238
|
+
body = peeked.stream
|
|
239
|
+
}
|
|
240
|
+
if (step.action === 'follow') {
|
|
241
|
+
streamUtils.discard(body)
|
|
242
|
+
if (++steps > resolver.maxSteps) {
|
|
243
|
+
throw new ResolverError(`${resolver.name}:steps`)
|
|
244
|
+
}
|
|
245
|
+
policy.assertUrl(step.url, current)
|
|
246
|
+
if (!resolver.owns(step.url)) {
|
|
247
|
+
throw new ResolverError(`${resolver.name}:foreign-follow`)
|
|
248
|
+
}
|
|
249
|
+
cookie = step.headers?.cookie ?? cookie
|
|
250
|
+
current = step.url
|
|
251
|
+
continue
|
|
252
|
+
}
|
|
253
|
+
if (step.action === 'fail') {
|
|
254
|
+
streamUtils.discard(body)
|
|
255
|
+
throw new ResolverError(`${resolver.name}:${step.reason}`)
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
log.debug('Response', { url: redactUtils.url(current), status, redirects })
|
|
260
|
+
|
|
261
|
+
return { url: current, status, headers: response.headers, body, info: infoOf(current, response, decoded), redirects }
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
const lowerKeys = (headers: Record<string, string> | undefined): Record<string, string> =>
|
|
266
|
+
Object.fromEntries(Object.entries(headers ?? {}).map(([name, value]) => [name.toLowerCase(), value]))
|
|
267
|
+
|
|
268
|
+
const client: HttpClient = {
|
|
269
|
+
request,
|
|
270
|
+
source: (url, sourceOptions) => makeHttpSource(client, new URL(url), sourceOptions),
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
return client
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
export const httpClient = createHttpClient()
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
export type * from './types.js'
|
|
2
|
+
export type * from './archive/types.js'
|
|
3
|
+
export type * from './http/types.js'
|
|
4
|
+
export type * from './network/types.js'
|
|
5
|
+
export type * from './proxy/types.js'
|
|
6
|
+
export type * from './resolvers/types.js'
|
|
7
|
+
export type * from './resolvers/google-drive/types.js'
|
|
8
|
+
export type * from './s3/types.js'
|
|
9
|
+
export type * from './sources/types.js'
|
|
10
|
+
export type * from './sinks/types.js'
|
|
11
|
+
export type * from './sinks/directory/types.js'
|
|
12
|
+
export type * from './sinks/buffer/types.js'
|
|
13
|
+
export type * from './sinks/workload/types.js'
|
|
14
|
+
export type * from './sources/workload/types.js'
|
|
15
|
+
export type * from './workspace/types.js'
|
|
16
|
+
export type * from './backed-workspace/types.js'
|
|
17
|
+
export type * from './transfer/types.js'
|
|
18
|
+
export type * from './service/types.js'
|
|
19
|
+
|
|
20
|
+
export * from './consts.js'
|
|
21
|
+
export * from './archive/consts.js'
|
|
22
|
+
export * from './network/consts.js'
|
|
23
|
+
export * from './resolvers/google-drive/consts.js'
|
|
24
|
+
export * from './s3/consts.js'
|
|
25
|
+
export * from './sinks/workload/consts.js'
|
|
26
|
+
export * from './transfer/consts.js'
|
|
27
|
+
export * from './workspace/consts.js'
|
|
28
|
+
export * from './errors.js'
|
|
29
|
+
|
|
30
|
+
export * from './archive.js'
|
|
31
|
+
export * from './network.js'
|
|
32
|
+
export * from './http.js'
|
|
33
|
+
export * from './proxy/endpoint.js'
|
|
34
|
+
export * from './proxy/router.js'
|
|
35
|
+
export * from './proxy/http.js'
|
|
36
|
+
export * from './proxy/socks.js'
|
|
37
|
+
export * from './resolvers/google-drive.js'
|
|
38
|
+
export * from './transfer.js'
|
|
39
|
+
export * from './s3.js'
|
|
40
|
+
export * from './sources/file.js'
|
|
41
|
+
export * from './sources/buffer.js'
|
|
42
|
+
export * from './sources/stream.js'
|
|
43
|
+
export * from './sinks/file.js'
|
|
44
|
+
export * from './sinks/directory.js'
|
|
45
|
+
export * from './sinks/buffer.js'
|
|
46
|
+
export * from './sinks/workload.js'
|
|
47
|
+
export * from './sources/workload.js'
|
|
48
|
+
export * from './workspace.js'
|
|
49
|
+
export * from './backed-workspace.js'
|
|
50
|
+
export * from './service.js'
|
|
51
|
+
|
|
52
|
+
// Detection lives in @owlmeans/file-inspector; these names stay importable from here.
|
|
53
|
+
export type {
|
|
54
|
+
DeclaredType, DetectedType, FileClaims, FileDetails, FileInspector, FileInspectorOptions, FileVerdict, InspectOptions, ProbeOptions,
|
|
55
|
+
VerdictStatus, VerifyPolicy,
|
|
56
|
+
} from '@owlmeans/file-inspector'
|
|
57
|
+
export {
|
|
58
|
+
CODE_LANGUAGE_BY_EXT, CONTAINER_EXTENSIONS, createFileInspector, DEFAULT_PROBE_BYTES, DEFAULT_SAMPLE_BYTES, EXTENSION_GROUPS, EXTENSION_MIME,
|
|
59
|
+
FileKind, FileKindError, FileMimicryError, fileInspector, KIND_BY_MIME, KIND_BY_MIME_PREFIX, MIME_ALIASES, NON_INFORMATIVE_MIMES,
|
|
60
|
+
OFFICE_KINDS, PASSIVE_KINDS, PDF_TRAILER_BYTES, SHEBANG_LANGUAGES, TEXT_FAMILY,
|
|
61
|
+
} from '@owlmeans/file-inspector'
|
|
62
|
+
|
|
63
|
+
// Scopes are storage-common's vocabulary; workspaces are addressed by them.
|
|
64
|
+
export type { FileScope, FileScopeHelper } from '@owlmeans/storage-common'
|
|
65
|
+
export { FileScopeError, FileScopeSchema, fileScopeHelper } from '@owlmeans/storage-common'
|