@owlmeans/file-streams 0.0.0-stage → 0.2.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +71 -2
- package/agent-meta/manifest.json +64 -0
- package/agent-meta/skills/file-streams/SKILL.md +183 -0
- package/agent-meta/skills/file-streams/parts/network.md +47 -0
- package/agent-meta/skills/file-streams/parts/workspaces.md +129 -0
- package/build/archive/consts.d.ts +16 -0
- package/build/archive/consts.d.ts.map +1 -0
- package/build/archive/consts.js +24 -0
- package/build/archive/consts.js.map +1 -0
- package/build/archive/formats.d.ts +4 -0
- package/build/archive/formats.d.ts.map +1 -0
- package/build/archive/formats.js +53 -0
- package/build/archive/formats.js.map +1 -0
- package/build/archive/guard.d.ts +5 -0
- package/build/archive/guard.d.ts.map +1 -0
- package/build/archive/guard.js +62 -0
- package/build/archive/guard.js.map +1 -0
- package/build/archive/ratio.d.ts +5 -0
- package/build/archive/ratio.d.ts.map +1 -0
- package/build/archive/ratio.js +30 -0
- package/build/archive/ratio.js.map +1 -0
- package/build/archive/tar.d.ts +4 -0
- package/build/archive/tar.d.ts.map +1 -0
- package/build/archive/tar.js +142 -0
- package/build/archive/tar.js.map +1 -0
- package/build/archive/types.d.ts +95 -0
- package/build/archive/types.d.ts.map +1 -0
- package/build/archive/types.js +2 -0
- package/build/archive/types.js.map +1 -0
- package/build/archive/types.local.d.ts +64 -0
- package/build/archive/types.local.d.ts.map +1 -0
- package/build/archive/types.local.js +2 -0
- package/build/archive/types.local.js.map +1 -0
- package/build/archive/zip.d.ts +4 -0
- package/build/archive/zip.d.ts.map +1 -0
- package/build/archive/zip.js +135 -0
- package/build/archive/zip.js.map +1 -0
- package/build/archive.d.ts +4 -0
- package/build/archive.d.ts.map +1 -0
- package/build/archive.js +265 -0
- package/build/archive.js.map +1 -0
- package/build/backed-workspace/types.d.ts +67 -0
- package/build/backed-workspace/types.d.ts.map +1 -0
- package/build/backed-workspace/types.js +2 -0
- package/build/backed-workspace/types.js.map +1 -0
- package/build/backed-workspace.d.ts +4 -0
- package/build/backed-workspace.d.ts.map +1 -0
- package/build/backed-workspace.js +63 -0
- package/build/backed-workspace.js.map +1 -0
- package/build/consts.d.ts +3 -0
- package/build/consts.d.ts.map +1 -0
- package/build/consts.js +3 -0
- package/build/consts.js.map +1 -0
- package/build/consts.local.d.ts +3 -0
- package/build/consts.local.d.ts.map +1 -0
- package/build/consts.local.js +3 -0
- package/build/consts.local.js.map +1 -0
- package/build/errors.d.ts +59 -0
- package/build/errors.d.ts.map +1 -0
- package/build/errors.js +105 -0
- package/build/errors.js.map +1 -0
- package/build/http/connector.d.ts +11 -0
- package/build/http/connector.d.ts.map +1 -0
- package/build/http/connector.js +124 -0
- package/build/http/connector.js.map +1 -0
- package/build/http/consts.d.ts +9 -0
- package/build/http/consts.d.ts.map +1 -0
- package/build/http/consts.js +8 -0
- package/build/http/consts.js.map +1 -0
- package/build/http/source.d.ts +5 -0
- package/build/http/source.d.ts.map +1 -0
- package/build/http/source.js +76 -0
- package/build/http/source.js.map +1 -0
- package/build/http/types.d.ts +67 -0
- package/build/http/types.d.ts.map +1 -0
- package/build/http/types.js +2 -0
- package/build/http/types.js.map +1 -0
- package/build/http/types.local.d.ts +21 -0
- package/build/http/types.local.d.ts.map +1 -0
- package/build/http/types.local.js +2 -0
- package/build/http/types.local.js.map +1 -0
- package/build/http.d.ts +4 -0
- package/build/http.d.ts.map +1 -0
- package/build/http.js +248 -0
- package/build/http.js.map +1 -0
- package/build/index.d.ts +53 -0
- package/build/index.d.ts.map +1 -0
- package/build/index.js +33 -0
- package/build/index.js.map +1 -0
- package/build/network/consts.d.ts +12 -0
- package/build/network/consts.d.ts.map +1 -0
- package/build/network/consts.js +20 -0
- package/build/network/consts.js.map +1 -0
- package/build/network/types.d.ts +43 -0
- package/build/network/types.d.ts.map +1 -0
- package/build/network/types.js +2 -0
- package/build/network/types.js.map +1 -0
- package/build/network.d.ts +4 -0
- package/build/network.d.ts.map +1 -0
- package/build/network.js +165 -0
- package/build/network.js.map +1 -0
- package/build/proxy/consts.d.ts +7 -0
- package/build/proxy/consts.d.ts.map +1 -0
- package/build/proxy/consts.js +9 -0
- package/build/proxy/consts.js.map +1 -0
- package/build/proxy/endpoint.d.ts +4 -0
- package/build/proxy/endpoint.d.ts.map +1 -0
- package/build/proxy/endpoint.js +27 -0
- package/build/proxy/endpoint.js.map +1 -0
- package/build/proxy/http.d.ts +5 -0
- package/build/proxy/http.d.ts.map +1 -0
- package/build/proxy/http.js +95 -0
- package/build/proxy/http.js.map +1 -0
- package/build/proxy/router.d.ts +4 -0
- package/build/proxy/router.d.ts.map +1 -0
- package/build/proxy/router.js +85 -0
- package/build/proxy/router.js.map +1 -0
- package/build/proxy/socks.d.ts +5 -0
- package/build/proxy/socks.d.ts.map +1 -0
- package/build/proxy/socks.js +44 -0
- package/build/proxy/socks.js.map +1 -0
- package/build/proxy/types.d.ts +99 -0
- package/build/proxy/types.d.ts.map +1 -0
- package/build/proxy/types.js +2 -0
- package/build/proxy/types.js.map +1 -0
- package/build/resolvers/google-drive/consts.d.ts +10 -0
- package/build/resolvers/google-drive/consts.d.ts.map +1 -0
- package/build/resolvers/google-drive/consts.js +16 -0
- package/build/resolvers/google-drive/consts.js.map +1 -0
- package/build/resolvers/google-drive/types.d.ts +21 -0
- package/build/resolvers/google-drive/types.d.ts.map +1 -0
- package/build/resolvers/google-drive/types.js +2 -0
- package/build/resolvers/google-drive/types.js.map +1 -0
- package/build/resolvers/google-drive.d.ts +10 -0
- package/build/resolvers/google-drive.d.ts.map +1 -0
- package/build/resolvers/google-drive.js +130 -0
- package/build/resolvers/google-drive.js.map +1 -0
- package/build/resolvers/types.d.ts +39 -0
- package/build/resolvers/types.d.ts.map +1 -0
- package/build/resolvers/types.js +2 -0
- package/build/resolvers/types.js.map +1 -0
- package/build/s3/consts.d.ts +9 -0
- package/build/s3/consts.d.ts.map +1 -0
- package/build/s3/consts.js +9 -0
- package/build/s3/consts.js.map +1 -0
- package/build/s3/prefix.d.ts +4 -0
- package/build/s3/prefix.d.ts.map +1 -0
- package/build/s3/prefix.js +87 -0
- package/build/s3/prefix.js.map +1 -0
- package/build/s3/sink.d.ts +6 -0
- package/build/s3/sink.d.ts.map +1 -0
- package/build/s3/sink.js +30 -0
- package/build/s3/sink.js.map +1 -0
- package/build/s3/source.d.ts +5 -0
- package/build/s3/source.d.ts.map +1 -0
- package/build/s3/source.js +81 -0
- package/build/s3/source.js.map +1 -0
- package/build/s3/types.d.ts +66 -0
- package/build/s3/types.d.ts.map +1 -0
- package/build/s3/types.js +2 -0
- package/build/s3/types.js.map +1 -0
- package/build/s3/types.local.d.ts +43 -0
- package/build/s3/types.local.d.ts.map +1 -0
- package/build/s3/types.local.js +2 -0
- package/build/s3/types.local.js.map +1 -0
- package/build/s3/upload.d.ts +4 -0
- package/build/s3/upload.d.ts.map +1 -0
- package/build/s3/upload.js +118 -0
- package/build/s3/upload.js.map +1 -0
- package/build/s3.d.ts +5 -0
- package/build/s3.d.ts.map +1 -0
- package/build/s3.js +68 -0
- package/build/s3.js.map +1 -0
- package/build/service/types.d.ts +66 -0
- package/build/service/types.d.ts.map +1 -0
- package/build/service/types.js +2 -0
- package/build/service/types.js.map +1 -0
- package/build/service/types.local.d.ts +11 -0
- package/build/service/types.local.d.ts.map +1 -0
- package/build/service/types.local.js +2 -0
- package/build/service/types.local.js.map +1 -0
- package/build/service.d.ts +6 -0
- package/build/service.d.ts.map +1 -0
- package/build/service.js +73 -0
- package/build/service.js.map +1 -0
- package/build/sinks/buffer/consts.d.ts +2 -0
- package/build/sinks/buffer/consts.d.ts.map +1 -0
- package/build/sinks/buffer/consts.js +2 -0
- package/build/sinks/buffer/consts.js.map +1 -0
- package/build/sinks/buffer/types.d.ts +11 -0
- package/build/sinks/buffer/types.d.ts.map +1 -0
- package/build/sinks/buffer/types.js +2 -0
- package/build/sinks/buffer/types.js.map +1 -0
- package/build/sinks/buffer.d.ts +4 -0
- package/build/sinks/buffer.d.ts.map +1 -0
- package/build/sinks/buffer.js +13 -0
- package/build/sinks/buffer.js.map +1 -0
- package/build/sinks/directory/types.d.ts +7 -0
- package/build/sinks/directory/types.d.ts.map +1 -0
- package/build/sinks/directory/types.js +2 -0
- package/build/sinks/directory/types.js.map +1 -0
- package/build/sinks/directory.d.ts +8 -0
- package/build/sinks/directory.d.ts.map +1 -0
- package/build/sinks/directory.js +43 -0
- package/build/sinks/directory.js.map +1 -0
- package/build/sinks/file.d.ts +5 -0
- package/build/sinks/file.d.ts.map +1 -0
- package/build/sinks/file.js +46 -0
- package/build/sinks/file.js.map +1 -0
- package/build/sinks/types.d.ts +11 -0
- package/build/sinks/types.d.ts.map +1 -0
- package/build/sinks/types.js +2 -0
- package/build/sinks/types.js.map +1 -0
- package/build/sinks/workload/consts.d.ts +5 -0
- package/build/sinks/workload/consts.d.ts.map +1 -0
- package/build/sinks/workload/consts.js +5 -0
- package/build/sinks/workload/consts.js.map +1 -0
- package/build/sinks/workload/types.d.ts +17 -0
- package/build/sinks/workload/types.d.ts.map +1 -0
- package/build/sinks/workload/types.js +2 -0
- package/build/sinks/workload/types.js.map +1 -0
- package/build/sinks/workload.d.ts +8 -0
- package/build/sinks/workload.d.ts.map +1 -0
- package/build/sinks/workload.js +51 -0
- package/build/sinks/workload.js.map +1 -0
- package/build/sources/buffer.d.ts +5 -0
- package/build/sources/buffer.d.ts.map +1 -0
- package/build/sources/buffer.js +20 -0
- package/build/sources/buffer.js.map +1 -0
- package/build/sources/file.d.ts +5 -0
- package/build/sources/file.d.ts.map +1 -0
- package/build/sources/file.js +50 -0
- package/build/sources/file.js.map +1 -0
- package/build/sources/stream.d.ts +5 -0
- package/build/sources/stream.d.ts.map +1 -0
- package/build/sources/stream.js +19 -0
- package/build/sources/stream.js.map +1 -0
- package/build/sources/types.d.ts +10 -0
- package/build/sources/types.d.ts.map +1 -0
- package/build/sources/types.js +2 -0
- package/build/sources/types.js.map +1 -0
- package/build/sources/workload/types.d.ts +29 -0
- package/build/sources/workload/types.d.ts.map +1 -0
- package/build/sources/workload/types.js +2 -0
- package/build/sources/workload/types.js.map +1 -0
- package/build/sources/workload.d.ts +7 -0
- package/build/sources/workload.d.ts.map +1 -0
- package/build/sources/workload.js +67 -0
- package/build/sources/workload.js.map +1 -0
- package/build/transfer/consts.d.ts +5 -0
- package/build/transfer/consts.d.ts.map +1 -0
- package/build/transfer/consts.js +8 -0
- package/build/transfer/consts.js.map +1 -0
- package/build/transfer/types.d.ts +49 -0
- package/build/transfer/types.d.ts.map +1 -0
- package/build/transfer/types.js +2 -0
- package/build/transfer/types.js.map +1 -0
- package/build/transfer.d.ts +4 -0
- package/build/transfer.d.ts.map +1 -0
- package/build/transfer.js +75 -0
- package/build/transfer.js.map +1 -0
- package/build/types.d.ts +99 -0
- package/build/types.d.ts.map +1 -0
- package/build/types.js +2 -0
- package/build/types.js.map +1 -0
- package/build/utils/names/consts.d.ts +5 -0
- package/build/utils/names/consts.d.ts.map +1 -0
- package/build/utils/names/consts.js +5 -0
- package/build/utils/names/consts.js.map +1 -0
- package/build/utils/names/types.d.ts +13 -0
- package/build/utils/names/types.d.ts.map +1 -0
- package/build/utils/names/types.js +2 -0
- package/build/utils/names/types.js.map +1 -0
- package/build/utils/names.d.ts +4 -0
- package/build/utils/names.d.ts.map +1 -0
- package/build/utils/names.js +60 -0
- package/build/utils/names.js.map +1 -0
- package/build/utils/paths/consts.d.ts +3 -0
- package/build/utils/paths/consts.d.ts.map +1 -0
- package/build/utils/paths/consts.js +2 -0
- package/build/utils/paths/consts.js.map +1 -0
- package/build/utils/paths/types.d.ts +20 -0
- package/build/utils/paths/types.d.ts.map +1 -0
- package/build/utils/paths/types.js +2 -0
- package/build/utils/paths/types.js.map +1 -0
- package/build/utils/paths.d.ts +4 -0
- package/build/utils/paths.d.ts.map +1 -0
- package/build/utils/paths.js +67 -0
- package/build/utils/paths.js.map +1 -0
- package/build/utils/range/types.d.ts +11 -0
- package/build/utils/range/types.d.ts.map +1 -0
- package/build/utils/range/types.js +2 -0
- package/build/utils/range/types.js.map +1 -0
- package/build/utils/range.d.ts +4 -0
- package/build/utils/range.d.ts.map +1 -0
- package/build/utils/range.js +64 -0
- package/build/utils/range.js.map +1 -0
- package/build/utils/redact/types.d.ts +16 -0
- package/build/utils/redact/types.d.ts.map +1 -0
- package/build/utils/redact/types.js +2 -0
- package/build/utils/redact/types.js.map +1 -0
- package/build/utils/redact.d.ts +4 -0
- package/build/utils/redact.d.ts.map +1 -0
- package/build/utils/redact.js +18 -0
- package/build/utils/redact.js.map +1 -0
- package/build/utils/signals/types.d.ts +17 -0
- package/build/utils/signals/types.d.ts.map +1 -0
- package/build/utils/signals/types.js +2 -0
- package/build/utils/signals/types.js.map +1 -0
- package/build/utils/signals.d.ts +4 -0
- package/build/utils/signals.d.ts.map +1 -0
- package/build/utils/signals.js +52 -0
- package/build/utils/signals.js.map +1 -0
- package/build/utils/streams/consts.d.ts +3 -0
- package/build/utils/streams/consts.d.ts.map +1 -0
- package/build/utils/streams/consts.js +3 -0
- package/build/utils/streams/consts.js.map +1 -0
- package/build/utils/streams/types.d.ts +59 -0
- package/build/utils/streams/types.d.ts.map +1 -0
- package/build/utils/streams/types.js +2 -0
- package/build/utils/streams/types.js.map +1 -0
- package/build/utils/streams.d.ts +4 -0
- package/build/utils/streams.d.ts.map +1 -0
- package/build/utils/streams.js +167 -0
- package/build/utils/streams.js.map +1 -0
- package/build/utils/tree/types.d.ts +6 -0
- package/build/utils/tree/types.d.ts.map +1 -0
- package/build/utils/tree/types.js +2 -0
- package/build/utils/tree/types.js.map +1 -0
- package/build/utils/tree.d.ts +4 -0
- package/build/utils/tree.d.ts.map +1 -0
- package/build/utils/tree.js +14 -0
- package/build/utils/tree.js.map +1 -0
- package/build/workspace/consts.d.ts +12 -0
- package/build/workspace/consts.d.ts.map +1 -0
- package/build/workspace/consts.js +12 -0
- package/build/workspace/consts.js.map +1 -0
- package/build/workspace/consts.local.d.ts +3 -0
- package/build/workspace/consts.local.d.ts.map +1 -0
- package/build/workspace/consts.local.js +3 -0
- package/build/workspace/consts.local.js.map +1 -0
- package/build/workspace/fs.d.ts +4 -0
- package/build/workspace/fs.d.ts.map +1 -0
- package/build/workspace/fs.js +45 -0
- package/build/workspace/fs.js.map +1 -0
- package/build/workspace/scope.d.ts +6 -0
- package/build/workspace/scope.d.ts.map +1 -0
- package/build/workspace/scope.js +180 -0
- package/build/workspace/scope.js.map +1 -0
- package/build/workspace/types.d.ts +66 -0
- package/build/workspace/types.d.ts.map +1 -0
- package/build/workspace/types.js +2 -0
- package/build/workspace/types.js.map +1 -0
- package/build/workspace/types.local.d.ts +20 -0
- package/build/workspace/types.local.d.ts.map +1 -0
- package/build/workspace/types.local.js +2 -0
- package/build/workspace/types.local.js.map +1 -0
- package/build/workspace.d.ts +7 -0
- package/build/workspace.d.ts.map +1 -0
- package/build/workspace.js +87 -0
- package/build/workspace.js.map +1 -0
- package/package.json +61 -4
- package/src/archive/consts.ts +28 -0
- package/src/archive/formats.ts +60 -0
- package/src/archive/guard.ts +68 -0
- package/src/archive/ratio.ts +35 -0
- package/src/archive/tar.ts +147 -0
- package/src/archive/types.local.ts +72 -0
- package/src/archive/types.ts +107 -0
- package/src/archive/zip.ts +142 -0
- package/src/archive.ts +298 -0
- package/src/backed-workspace/types.ts +72 -0
- package/src/backed-workspace.ts +85 -0
- package/src/consts.local.ts +2 -0
- package/src/consts.ts +2 -0
- package/src/errors.ts +128 -0
- package/src/http/connector.ts +136 -0
- package/src/http/consts.ts +13 -0
- package/src/http/source.ts +89 -0
- package/src/http/types.local.ts +23 -0
- package/src/http/types.ts +74 -0
- package/src/http.ts +276 -0
- package/src/index.ts +65 -0
- package/src/network/consts.ts +22 -0
- package/src/network/types.ts +45 -0
- package/src/network.ts +179 -0
- package/src/proxy/consts.ts +11 -0
- package/src/proxy/endpoint.ts +28 -0
- package/src/proxy/http.ts +108 -0
- package/src/proxy/router.ts +93 -0
- package/src/proxy/socks.ts +48 -0
- package/src/proxy/types.ts +114 -0
- package/src/resolvers/google-drive/consts.ts +22 -0
- package/src/resolvers/google-drive/types.ts +22 -0
- package/src/resolvers/google-drive.ts +153 -0
- package/src/resolvers/types.ts +45 -0
- package/src/s3/consts.ts +12 -0
- package/src/s3/prefix.ts +100 -0
- package/src/s3/sink.ts +36 -0
- package/src/s3/source.ts +90 -0
- package/src/s3/types.local.ts +49 -0
- package/src/s3/types.ts +70 -0
- package/src/s3/upload.ts +130 -0
- package/src/s3.ts +83 -0
- package/src/service/types.local.ts +9 -0
- package/src/service/types.ts +71 -0
- package/src/service.ts +97 -0
- package/src/sinks/buffer/consts.ts +1 -0
- package/src/sinks/buffer/types.ts +12 -0
- package/src/sinks/buffer.ts +18 -0
- package/src/sinks/directory/types.ts +7 -0
- package/src/sinks/directory.ts +49 -0
- package/src/sinks/file.ts +52 -0
- package/src/sinks/types.ts +12 -0
- package/src/sinks/workload/consts.ts +5 -0
- package/src/sinks/workload/types.ts +17 -0
- package/src/sinks/workload.ts +59 -0
- package/src/sources/buffer.ts +27 -0
- package/src/sources/file.ts +57 -0
- package/src/sources/stream.ts +26 -0
- package/src/sources/types.ts +12 -0
- package/src/sources/workload/types.ts +31 -0
- package/src/sources/workload.ts +79 -0
- package/src/transfer/consts.ts +10 -0
- package/src/transfer/types.ts +55 -0
- package/src/transfer.ts +89 -0
- package/src/types.ts +106 -0
- package/src/utils/names/consts.ts +5 -0
- package/src/utils/names/types.ts +12 -0
- package/src/utils/names.ts +70 -0
- package/src/utils/paths/consts.ts +3 -0
- package/src/utils/paths/types.ts +20 -0
- package/src/utils/paths.ts +75 -0
- package/src/utils/range/types.ts +11 -0
- package/src/utils/range.ts +80 -0
- package/src/utils/redact/types.ts +16 -0
- package/src/utils/redact.ts +23 -0
- package/src/utils/signals/types.ts +17 -0
- package/src/utils/signals.ts +63 -0
- package/src/utils/streams/consts.ts +2 -0
- package/src/utils/streams/types.ts +65 -0
- package/src/utils/streams.ts +188 -0
- package/src/utils/tree/types.ts +6 -0
- package/src/utils/tree.ts +18 -0
- package/src/workspace/consts.local.ts +2 -0
- package/src/workspace/consts.ts +14 -0
- package/src/workspace/fs.ts +48 -0
- package/src/workspace/scope.ts +196 -0
- package/src/workspace/types.local.ts +21 -0
- package/src/workspace/types.ts +71 -0
- package/src/workspace.ts +102 -0
- package/tests/archive.spec.ts +134 -0
- package/tests/backed.spec.ts +147 -0
- package/tests/context.ts +114 -0
- package/tests/fixtures.ts +190 -0
- package/tests/google-drive.spec.ts +76 -0
- package/tests/harness/drive.ts +92 -0
- package/tests/harness/http-proxy.ts +96 -0
- package/tests/harness/origin.ts +134 -0
- package/tests/harness/socks5.ts +135 -0
- package/tests/harness/workload.ts +118 -0
- package/tests/http.spec.ts +101 -0
- package/tests/inspector.spec.ts +40 -0
- package/tests/live.spec.ts +27 -0
- package/tests/network.spec.ts +79 -0
- package/tests/proxy.spec.ts +149 -0
- package/tests/s3.spec.ts +133 -0
- package/tests/service.spec.ts +77 -0
- package/tests/sources-sinks.spec.ts +96 -0
- package/tests/transfer.spec.ts +99 -0
- package/tests/tsconfig.json +12 -0
- package/tests/workload.spec.ts +140 -0
- package/tests/workspace.spec.ts +158 -0
- package/tsconfig.json +16 -0
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/** IPv4 ranges never reached unless allowed: this host, private, CGNAT, link-local, documentation, multicast, reserved. */
|
|
2
|
+
export const DENIED_IPV4_CIDRS: readonly string[] = Object.freeze([
|
|
3
|
+
'0.0.0.0/8', '10.0.0.0/8', '100.64.0.0/10', '127.0.0.0/8', '169.254.0.0/16', '172.16.0.0/12', '192.0.0.0/24',
|
|
4
|
+
'192.0.2.0/24', '192.88.99.0/24', '192.168.0.0/16', '198.18.0.0/15', '198.51.100.0/24', '203.0.113.0/24',
|
|
5
|
+
'224.0.0.0/4', '240.0.0.0/4', '255.255.255.255/32',
|
|
6
|
+
])
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* IPv6 ranges never reached unless allowed. The IPv4-mapped and IPv4-compatible ranges are not
|
|
10
|
+
* listed — a block list matches every IPv4 address against them — the embedded IPv4 of such an
|
|
11
|
+
* address is extracted and checked against the IPv4 list instead.
|
|
12
|
+
*/
|
|
13
|
+
export const DENIED_IPV6_CIDRS: readonly string[] = Object.freeze([
|
|
14
|
+
'::/128', '::1/128', '64:ff9b:1::/48', '100::/64', '2001:db8::/32', 'fc00::/7', 'fe80::/10', 'fec0::/10', 'ff00::/8',
|
|
15
|
+
])
|
|
16
|
+
|
|
17
|
+
/** Names that only mean something inside a network. */
|
|
18
|
+
export const DENIED_HOST_SUFFIXES: readonly string[] = Object.freeze([
|
|
19
|
+
'localhost', '.localhost', '.local', '.internal', '.svc', '.cluster.local', '.home.arpa', '.lan', '.intranet',
|
|
20
|
+
])
|
|
21
|
+
|
|
22
|
+
export const DEFAULT_SCHEMES: readonly string[] = Object.freeze(['https:', 'http:'])
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
export interface ResolvedAddress {
|
|
2
|
+
address: string
|
|
3
|
+
family: 4 | 6
|
|
4
|
+
}
|
|
5
|
+
|
|
6
|
+
/** A name resolver (DoH, split horizon, a test seam); it must answer every address of the name. */
|
|
7
|
+
export interface AddressResolver {
|
|
8
|
+
(hostname: string, signal?: AbortSignal): Promise<ResolvedAddress[]>
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export interface NetworkPolicyOptions {
|
|
12
|
+
/** URL schemes allowed. Default `['https:', 'http:']`. */
|
|
13
|
+
schemes?: string[]
|
|
14
|
+
/** Allow loopback, private, link-local, CGNAT, multicast, reserved and internal names. Default `false`. */
|
|
15
|
+
allowPrivate?: boolean
|
|
16
|
+
/** CIDRs allowed even inside the denied ranges (`'127.0.0.1/32'` for a local stand-in). */
|
|
17
|
+
allowCidrs?: string[]
|
|
18
|
+
/** Extra CIDRs to deny. */
|
|
19
|
+
denyCidrs?: string[]
|
|
20
|
+
/** When set, only these hosts (`x.com`, `*.x.com`). */
|
|
21
|
+
allowHosts?: string[]
|
|
22
|
+
/** Hosts never reached. */
|
|
23
|
+
denyHosts?: string[]
|
|
24
|
+
/** Ports allowed. Default any. */
|
|
25
|
+
ports?: number[]
|
|
26
|
+
/** Follow an https → http redirect. Default `false`. */
|
|
27
|
+
allowDowngrade?: boolean
|
|
28
|
+
/** Accept `user:pass@` in URLs. Default `false`. */
|
|
29
|
+
allowUrlCredentials?: boolean
|
|
30
|
+
/** Accept a name the local resolver cannot resolve when a remote-DNS proxy will. Default `false`. */
|
|
31
|
+
trustProxyDns?: boolean
|
|
32
|
+
resolver?: AddressResolver
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** What the package may reach. Every connection goes through it. */
|
|
36
|
+
export interface NetworkPolicy {
|
|
37
|
+
/** Whether a remote-DNS proxy may resolve names this side cannot. */
|
|
38
|
+
readonly trustProxyDns: boolean
|
|
39
|
+
/** Scheme, credentials, host patterns, port and downgrade (`from` is the previous hop) — no DNS. */
|
|
40
|
+
assertUrl: (url: URL, from?: URL) => void
|
|
41
|
+
/** Every address of a name, checked; one refused address refuses the name. */
|
|
42
|
+
resolve: (hostname: string, signal?: AbortSignal) => Promise<ResolvedAddress[]>
|
|
43
|
+
/** Whether one address may be connected to. */
|
|
44
|
+
allowsAddress: (address: string) => boolean
|
|
45
|
+
}
|
package/src/network.ts
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
import { lookup } from 'node:dns/promises'
|
|
2
|
+
import { BlockList, isIP } from 'node:net'
|
|
3
|
+
import { logger } from '@owlmeans/log'
|
|
4
|
+
import { NetworkPolicyError } from './errors.js'
|
|
5
|
+
import { DEFAULT_SCHEMES, DENIED_HOST_SUFFIXES, DENIED_IPV4_CIDRS, DENIED_IPV6_CIDRS } from './network/consts.js'
|
|
6
|
+
import type { NetworkPolicy, NetworkPolicyOptions, ResolvedAddress } from './network/types.js'
|
|
7
|
+
|
|
8
|
+
const log = logger('file-streams:network')
|
|
9
|
+
|
|
10
|
+
export const createNetworkPolicy = (options: NetworkPolicyOptions = {}): NetworkPolicy => {
|
|
11
|
+
const schemes = options.schemes ?? DEFAULT_SCHEMES
|
|
12
|
+
|
|
13
|
+
const listOf = (cidrs: readonly string[]): BlockList => {
|
|
14
|
+
const list = new BlockList()
|
|
15
|
+
for (const cidr of cidrs) {
|
|
16
|
+
const [address, prefix] = cidr.split('/')
|
|
17
|
+
const family = isIP(address) === 6 ? 'ipv6' : 'ipv4'
|
|
18
|
+
if (prefix == null) {
|
|
19
|
+
list.addAddress(address, family)
|
|
20
|
+
} else {
|
|
21
|
+
list.addSubnet(address, Number(prefix), family)
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
return list
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const denied = listOf([...DENIED_IPV4_CIDRS, ...DENIED_IPV6_CIDRS])
|
|
28
|
+
const extraDenied = listOf(options.denyCidrs ?? [])
|
|
29
|
+
const allowed = listOf(options.allowCidrs ?? [])
|
|
30
|
+
|
|
31
|
+
const ipv6Bytes = (address: string): number[] | undefined => {
|
|
32
|
+
let text = address.toLowerCase().replace(/^\[|\]$/g, '').split('%')[0]
|
|
33
|
+
const dotted = /(\d+\.\d+\.\d+\.\d+)$/.exec(text)
|
|
34
|
+
if (dotted != null) {
|
|
35
|
+
const octets = dotted[1].split('.').map(Number)
|
|
36
|
+
text = `${text.slice(0, -dotted[1].length)}${((octets[0] << 8) | octets[1]).toString(16)}:${((octets[2] << 8) | octets[3]).toString(16)}`
|
|
37
|
+
}
|
|
38
|
+
const halves = text.split('::')
|
|
39
|
+
if (halves.length > 2) {
|
|
40
|
+
return undefined
|
|
41
|
+
}
|
|
42
|
+
const head = halves[0] === '' ? [] : halves[0].split(':')
|
|
43
|
+
const tail = halves.length === 2 && halves[1] !== '' ? halves[1].split(':') : []
|
|
44
|
+
const groups = halves.length === 2 ? [...head, ...new Array(8 - head.length - tail.length).fill('0'), ...tail] : head
|
|
45
|
+
if (groups.length !== 8) {
|
|
46
|
+
return undefined
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
return groups.flatMap(group => {
|
|
50
|
+
const value = parseInt(group, 16)
|
|
51
|
+
return [(value >> 8) & 0xff, value & 0xff]
|
|
52
|
+
})
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** The IPv4 an IPv6 address carries: mapped, NAT64, 6to4, Teredo, IPv4-compatible. */
|
|
56
|
+
const embeddedIpv4 = (address: string): string | undefined => {
|
|
57
|
+
const bytes = ipv6Bytes(address)
|
|
58
|
+
if (bytes == null) {
|
|
59
|
+
return undefined
|
|
60
|
+
}
|
|
61
|
+
const quad = (offset: number, mask = 0): string => bytes.slice(offset, offset + 4).map(byte => byte ^ mask).join('.')
|
|
62
|
+
const zeros = (from: number, to: number): boolean => bytes.slice(from, to).every(byte => byte === 0)
|
|
63
|
+
if (zeros(0, 10) && bytes[10] === 0xff && bytes[11] === 0xff) {
|
|
64
|
+
return quad(12)
|
|
65
|
+
}
|
|
66
|
+
if (bytes[0] === 0x00 && bytes[1] === 0x64 && bytes[2] === 0xff && bytes[3] === 0x9b) {
|
|
67
|
+
return quad(12)
|
|
68
|
+
}
|
|
69
|
+
if (bytes[0] === 0x20 && bytes[1] === 0x02) {
|
|
70
|
+
return quad(2)
|
|
71
|
+
}
|
|
72
|
+
if (bytes[0] === 0x20 && bytes[1] === 0x01 && bytes[2] === 0x00 && bytes[3] === 0x00) {
|
|
73
|
+
return quad(12, 0xff)
|
|
74
|
+
}
|
|
75
|
+
if (zeros(0, 12) && !zeros(12, 16)) {
|
|
76
|
+
return quad(12)
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
return undefined
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const allowsAddress = (address: string): boolean => {
|
|
83
|
+
const family = isIP(address)
|
|
84
|
+
if (family === 0) {
|
|
85
|
+
return false
|
|
86
|
+
}
|
|
87
|
+
const type = family === 6 ? 'ipv6' : 'ipv4'
|
|
88
|
+
if (allowed.check(address, type)) {
|
|
89
|
+
return true
|
|
90
|
+
}
|
|
91
|
+
if (extraDenied.check(address, type)) {
|
|
92
|
+
return false
|
|
93
|
+
}
|
|
94
|
+
if (options.allowPrivate === true) {
|
|
95
|
+
return true
|
|
96
|
+
}
|
|
97
|
+
if (denied.check(address, type)) {
|
|
98
|
+
return false
|
|
99
|
+
}
|
|
100
|
+
if (family === 6) {
|
|
101
|
+
const inner = embeddedIpv4(address)
|
|
102
|
+
if (inner != null) {
|
|
103
|
+
return allowsAddress(inner)
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
return true
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
const matches = (hostname: string, pattern: string): boolean => {
|
|
111
|
+
const host = hostname.toLowerCase()
|
|
112
|
+
const rule = pattern.toLowerCase()
|
|
113
|
+
return rule.startsWith('*.') ? host.endsWith(rule.slice(1)) : host === rule
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
const hostOf = (url: URL): string => url.hostname.replace(/^\[|\]$/g, '')
|
|
117
|
+
|
|
118
|
+
const assertUrl = (url: URL, from?: URL): void => {
|
|
119
|
+
if (!schemes.includes(url.protocol)) {
|
|
120
|
+
throw new NetworkPolicyError('scheme')
|
|
121
|
+
}
|
|
122
|
+
if ((url.username !== '' || url.password !== '') && options.allowUrlCredentials !== true) {
|
|
123
|
+
throw new NetworkPolicyError('credentials')
|
|
124
|
+
}
|
|
125
|
+
if (from?.protocol === 'https:' && url.protocol === 'http:' && options.allowDowngrade !== true) {
|
|
126
|
+
throw new NetworkPolicyError('downgrade')
|
|
127
|
+
}
|
|
128
|
+
const host = hostOf(url)
|
|
129
|
+
if (options.allowHosts != null && !options.allowHosts.some(pattern => matches(host, pattern))) {
|
|
130
|
+
throw new NetworkPolicyError('blocked-host')
|
|
131
|
+
}
|
|
132
|
+
if (options.denyHosts?.some(pattern => matches(host, pattern)) === true) {
|
|
133
|
+
throw new NetworkPolicyError('blocked-host')
|
|
134
|
+
}
|
|
135
|
+
const port = url.port !== '' ? Number(url.port) : url.protocol === 'https:' ? 443 : 80
|
|
136
|
+
if (options.ports != null && !options.ports.includes(port)) {
|
|
137
|
+
throw new NetworkPolicyError('port')
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
const resolve = async (hostname: string, signal?: AbortSignal): Promise<ResolvedAddress[]> => {
|
|
142
|
+
const host = hostname.replace(/^\[|\]$/g, '').toLowerCase()
|
|
143
|
+
const literal = isIP(host)
|
|
144
|
+
let addresses: ResolvedAddress[]
|
|
145
|
+
if (literal !== 0) {
|
|
146
|
+
addresses = [{ address: host, family: literal as 4 | 6 }]
|
|
147
|
+
} else {
|
|
148
|
+
const explicitlyAllowed = options.allowHosts?.some(pattern => matches(host, pattern)) === true
|
|
149
|
+
if (options.allowPrivate !== true && !explicitlyAllowed
|
|
150
|
+
&& DENIED_HOST_SUFFIXES.some(suffix => suffix.startsWith('.') ? host.endsWith(suffix) : host === suffix)) {
|
|
151
|
+
log.warn('Refused an internal host name', { host }, { event: 'file-streams.policy.blocked' })
|
|
152
|
+
throw new NetworkPolicyError('blocked-host')
|
|
153
|
+
}
|
|
154
|
+
try {
|
|
155
|
+
addresses = options.resolver != null
|
|
156
|
+
? await options.resolver(host, signal)
|
|
157
|
+
: (await lookup(host, { all: true, verbatim: true })).map(entry => ({ address: entry.address, family: entry.family as 4 | 6 }))
|
|
158
|
+
} catch (error) {
|
|
159
|
+
if (error instanceof NetworkPolicyError) {
|
|
160
|
+
throw error
|
|
161
|
+
}
|
|
162
|
+
throw new NetworkPolicyError('unresolvable')
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
if (addresses.length === 0) {
|
|
166
|
+
throw new NetworkPolicyError('unresolvable')
|
|
167
|
+
}
|
|
168
|
+
if (addresses.some(entry => !allowsAddress(entry.address))) {
|
|
169
|
+
log.warn('Refused a blocked address', { host }, { event: 'file-streams.policy.blocked' })
|
|
170
|
+
throw new NetworkPolicyError('blocked-address')
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
return addresses
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
return { trustProxyDns: options.trustProxyDns === true, assertUrl, resolve, allowsAddress }
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
export const networkPolicy = createNetworkPolicy()
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export const DEFAULT_PROXY_RETRIES = 1
|
|
2
|
+
|
|
3
|
+
export const DEFAULT_PROXY_COOLDOWN_MS = 30_000
|
|
4
|
+
|
|
5
|
+
/** The longest CONNECT reply head a proxy may send. */
|
|
6
|
+
export const CONNECT_MAX_HEADER_BYTES = 16 * 1024
|
|
7
|
+
|
|
8
|
+
/** Default ports by proxy type. */
|
|
9
|
+
export const DEFAULT_PROXY_PORTS: Readonly<Record<string, number>> = Object.freeze({
|
|
10
|
+
http: 8080, https: 443, socks4: 1080, socks4a: 1080, socks5: 1080, socks5h: 1080,
|
|
11
|
+
})
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { ProxyError } from '../errors.js'
|
|
2
|
+
import { DEFAULT_PROXY_PORTS } from './consts.js'
|
|
3
|
+
import type { ProxyEndpoint } from './types.js'
|
|
4
|
+
|
|
5
|
+
/** A proxy URL (`socks5h://user:pass@host:1080`) read as an endpoint. */
|
|
6
|
+
export const parseProxyEndpoint = (value: ProxyEndpoint | string): ProxyEndpoint => {
|
|
7
|
+
if (typeof value !== 'string') {
|
|
8
|
+
return value
|
|
9
|
+
}
|
|
10
|
+
let url: URL
|
|
11
|
+
try {
|
|
12
|
+
url = new URL(value)
|
|
13
|
+
} catch {
|
|
14
|
+
throw new ProxyError('endpoint')
|
|
15
|
+
}
|
|
16
|
+
const type = url.protocol.replace(/:$/, '').toLowerCase()
|
|
17
|
+
const port = url.port !== '' ? Number(url.port) : DEFAULT_PROXY_PORTS[type]
|
|
18
|
+
if (url.hostname === '' || port == null) {
|
|
19
|
+
throw new ProxyError('endpoint')
|
|
20
|
+
}
|
|
21
|
+
const endpoint: ProxyEndpoint = { type, host: url.hostname.replace(/^\[|\]$/g, ''), port }
|
|
22
|
+
if (url.username !== '') {
|
|
23
|
+
endpoint.username = decodeURIComponent(url.username)
|
|
24
|
+
endpoint.password = decodeURIComponent(url.password)
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
return endpoint
|
|
28
|
+
}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import { connect as connectTcp } from 'node:net'
|
|
2
|
+
import type { Socket } from 'node:net'
|
|
3
|
+
import { connect as connectTls } from 'node:tls'
|
|
4
|
+
import { ProxyError } from '../errors.js'
|
|
5
|
+
import { signalUtils } from '../utils/signals.js'
|
|
6
|
+
import { CONNECT_MAX_HEADER_BYTES } from './consts.js'
|
|
7
|
+
import type { HttpProxyPluginOptions, ProxyConnectContext, ProxyConnection, ProxyEndpoint, ProxyPlugin, TunnelTarget } from './types.js'
|
|
8
|
+
|
|
9
|
+
/** HTTP(S) proxies: CONNECT tunnels, and forward mode for plain-http targets. */
|
|
10
|
+
export const createHttpProxyPlugin = (options: HttpProxyPluginOptions = {}): ProxyPlugin => {
|
|
11
|
+
const authorization = (endpoint: ProxyEndpoint): string | undefined => endpoint.username != null
|
|
12
|
+
? `Basic ${Buffer.from(`${endpoint.username}:${endpoint.password ?? ''}`).toString('base64')}`
|
|
13
|
+
: undefined
|
|
14
|
+
|
|
15
|
+
const open = (endpoint: ProxyEndpoint, context: ProxyConnectContext): Promise<Socket> => new Promise((resolve, reject) => {
|
|
16
|
+
const socket: Socket = endpoint.type === 'https'
|
|
17
|
+
? connectTls({ host: endpoint.host, port: endpoint.port, servername: endpoint.host, ca: options.ca, minVersion: 'TLSv1.2' })
|
|
18
|
+
: connectTcp({ host: endpoint.host, port: endpoint.port })
|
|
19
|
+
const timer = setTimeout(() => fail(new ProxyError('connect')), context.timeoutMs)
|
|
20
|
+
const release = signalUtils.onAbort(context.signal, reason => fail(reason as Error))
|
|
21
|
+
const done = (): void => {
|
|
22
|
+
clearTimeout(timer)
|
|
23
|
+
release()
|
|
24
|
+
socket.removeListener('error', fail)
|
|
25
|
+
}
|
|
26
|
+
function fail (error: Error): void {
|
|
27
|
+
done()
|
|
28
|
+
socket.destroy()
|
|
29
|
+
reject(error instanceof ProxyError || context.signal?.aborted === true ? error : new ProxyError('connect'))
|
|
30
|
+
}
|
|
31
|
+
socket.once(endpoint.type === 'https' ? 'secureConnect' : 'connect', () => {
|
|
32
|
+
done()
|
|
33
|
+
resolve(socket)
|
|
34
|
+
})
|
|
35
|
+
socket.once('error', fail)
|
|
36
|
+
})
|
|
37
|
+
|
|
38
|
+
const tunnel = (socket: Socket, endpoint: ProxyEndpoint, target: TunnelTarget, context: ProxyConnectContext): Promise<void> =>
|
|
39
|
+
new Promise((resolve, reject) => {
|
|
40
|
+
const authority = target.host.includes(':') ? `[${target.host}]:${target.port}` : `${target.host}:${target.port}`
|
|
41
|
+
const auth = authorization(endpoint)
|
|
42
|
+
const chunks: Buffer[] = []
|
|
43
|
+
let length = 0
|
|
44
|
+
const timer = setTimeout(() => fail(new ProxyError('handshake')), context.timeoutMs)
|
|
45
|
+
const release = signalUtils.onAbort(context.signal, reason => fail(reason as Error))
|
|
46
|
+
|
|
47
|
+
const cleanup = (): void => {
|
|
48
|
+
clearTimeout(timer)
|
|
49
|
+
release()
|
|
50
|
+
socket.removeListener('data', onData)
|
|
51
|
+
socket.removeListener('error', onError)
|
|
52
|
+
socket.removeListener('close', onClose)
|
|
53
|
+
}
|
|
54
|
+
function fail (error: Error): void {
|
|
55
|
+
cleanup()
|
|
56
|
+
socket.destroy()
|
|
57
|
+
reject(error)
|
|
58
|
+
}
|
|
59
|
+
const onError = (): void => fail(new ProxyError('handshake'))
|
|
60
|
+
const onClose = (): void => fail(new ProxyError('handshake'))
|
|
61
|
+
function onData (chunk: Buffer): void {
|
|
62
|
+
chunks.push(chunk)
|
|
63
|
+
length += chunk.length
|
|
64
|
+
const buffer = Buffer.concat(chunks, length)
|
|
65
|
+
const end = buffer.indexOf('\r\n\r\n')
|
|
66
|
+
if (end === -1) {
|
|
67
|
+
if (length > CONNECT_MAX_HEADER_BYTES) {
|
|
68
|
+
fail(new ProxyError('handshake'))
|
|
69
|
+
}
|
|
70
|
+
return
|
|
71
|
+
}
|
|
72
|
+
cleanup()
|
|
73
|
+
const status = Number(/^HTTP\/1\.[01] (\d{3})/.exec(buffer.subarray(0, end).toString('latin1'))?.[1])
|
|
74
|
+
if (status === 200) {
|
|
75
|
+
const rest = buffer.subarray(end + 4)
|
|
76
|
+
if (rest.length > 0) {
|
|
77
|
+
socket.unshift(rest)
|
|
78
|
+
}
|
|
79
|
+
resolve()
|
|
80
|
+
return
|
|
81
|
+
}
|
|
82
|
+
socket.destroy()
|
|
83
|
+
reject(new ProxyError(status === 407 ? 'auth-failed' : `refused:${Number.isNaN(status) ? 'invalid' : status}`))
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
socket.on('data', onData)
|
|
87
|
+
socket.once('error', onError)
|
|
88
|
+
socket.once('close', onClose)
|
|
89
|
+
socket.write(
|
|
90
|
+
`CONNECT ${authority} HTTP/1.1\r\nHost: ${authority}\r\n${auth != null ? `Proxy-Authorization: ${auth}\r\n` : ''}\r\n`
|
|
91
|
+
)
|
|
92
|
+
})
|
|
93
|
+
|
|
94
|
+
const connect = async (endpoint: ProxyEndpoint, target: TunnelTarget, context: ProxyConnectContext): Promise<ProxyConnection> => {
|
|
95
|
+
const socket = await open(endpoint, context)
|
|
96
|
+
if (!target.secure && options.httpTargets !== 'connect') {
|
|
97
|
+
const auth = authorization(endpoint)
|
|
98
|
+
return { socket, mode: 'forward', headers: auth != null ? { 'proxy-authorization': auth } : undefined }
|
|
99
|
+
}
|
|
100
|
+
await tunnel(socket, endpoint, target, context)
|
|
101
|
+
|
|
102
|
+
return { socket, mode: 'tunnel' }
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
return { types: ['http', 'https'], remoteDns: () => options.remoteDns === true, connect }
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
export const httpProxyPlugin = createHttpProxyPlugin()
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto'
|
|
2
|
+
import { logger } from '@owlmeans/log'
|
|
3
|
+
import { ProxyError } from '../errors.js'
|
|
4
|
+
import { redactUtils } from '../utils/redact.js'
|
|
5
|
+
import { DEFAULT_PROXY_COOLDOWN_MS, DEFAULT_PROXY_RETRIES } from './consts.js'
|
|
6
|
+
import { parseProxyEndpoint } from './endpoint.js'
|
|
7
|
+
import { httpProxyPlugin } from './http.js'
|
|
8
|
+
import { socksProxyPlugin } from './socks.js'
|
|
9
|
+
import type { ProxyEndpoint, ProxyRequest, ProxyRouter, ProxyRouterOptions, ProxySelection } from './types.js'
|
|
10
|
+
|
|
11
|
+
const log = logger('file-streams:proxy')
|
|
12
|
+
|
|
13
|
+
/** Chooses a proxy per connection from a list (with a strategy) or a provider callback. */
|
|
14
|
+
export const createProxyRouter = (options: ProxyRouterOptions = {}): ProxyRouter => {
|
|
15
|
+
const plugins = [...(options.plugins ?? []), httpProxyPlugin, socksProxyPlugin]
|
|
16
|
+
const list = Array.isArray(options.proxies) ? options.proxies.map(parseProxyEndpoint) : undefined
|
|
17
|
+
const cooldownMs = options.cooldownMs ?? DEFAULT_PROXY_COOLDOWN_MS
|
|
18
|
+
const cooling = new Map<string, number>()
|
|
19
|
+
let cursor = 0
|
|
20
|
+
|
|
21
|
+
const idOf = (endpoint: ProxyEndpoint): string => redactUtils.proxy(endpoint)
|
|
22
|
+
|
|
23
|
+
const bypassed = (url: URL): boolean => {
|
|
24
|
+
const host = url.hostname.replace(/^\[|\]$/g, '').toLowerCase()
|
|
25
|
+
return options.bypass?.some(rule => {
|
|
26
|
+
const pattern = rule.toLowerCase()
|
|
27
|
+
return pattern.startsWith('*.') ? host.endsWith(pattern.slice(1)) : host === pattern
|
|
28
|
+
}) === true
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const fromList = (members: ProxyEndpoint[], request: ProxyRequest): ProxyEndpoint => {
|
|
32
|
+
const now = Date.now()
|
|
33
|
+
const warm = members.filter(member => (cooling.get(idOf(member)) ?? 0) <= now)
|
|
34
|
+
const pool = warm.length > 0 ? warm : members
|
|
35
|
+
switch (options.strategy) {
|
|
36
|
+
case 'random':
|
|
37
|
+
return pool[Math.floor(Math.random() * pool.length)]
|
|
38
|
+
case 'failover':
|
|
39
|
+
return pool[0]
|
|
40
|
+
case 'by-key':
|
|
41
|
+
if (request.key != null) {
|
|
42
|
+
const index = createHash('sha256').update(request.key).digest().readUInt32BE(0) % members.length
|
|
43
|
+
const sticky = members[index]
|
|
44
|
+
return pool.includes(sticky) ? sticky : pool[index % pool.length]
|
|
45
|
+
}
|
|
46
|
+
return pool[cursor++ % pool.length]
|
|
47
|
+
default:
|
|
48
|
+
return pool[cursor++ % pool.length]
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const select = async (request: ProxyRequest): Promise<ProxySelection | null> => {
|
|
53
|
+
if (options.proxies == null || bypassed(request.url)) {
|
|
54
|
+
return null
|
|
55
|
+
}
|
|
56
|
+
let endpoint: ProxyEndpoint
|
|
57
|
+
if (list != null) {
|
|
58
|
+
if (list.length === 0) {
|
|
59
|
+
return null
|
|
60
|
+
}
|
|
61
|
+
endpoint = fromList(list, request)
|
|
62
|
+
} else {
|
|
63
|
+
const provided = await (options.proxies as Exclude<ProxyRouterOptions['proxies'], unknown[] | undefined>)(request)
|
|
64
|
+
if (provided == null) {
|
|
65
|
+
return null
|
|
66
|
+
}
|
|
67
|
+
endpoint = parseProxyEndpoint(provided)
|
|
68
|
+
}
|
|
69
|
+
if (endpoint.username == null && options.credentials != null) {
|
|
70
|
+
const credentials = await options.credentials(endpoint, request)
|
|
71
|
+
if (credentials != null) {
|
|
72
|
+
endpoint = { ...endpoint, username: credentials.username, password: credentials.password }
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
const plugin = plugins.find(candidate => candidate.types.includes(endpoint.type))
|
|
76
|
+
if (plugin == null) {
|
|
77
|
+
throw new ProxyError('no-plugin')
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
return { endpoint, plugin }
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const report = (endpoint: ProxyEndpoint, ok: boolean): void => {
|
|
84
|
+
if (ok) {
|
|
85
|
+
cooling.delete(idOf(endpoint))
|
|
86
|
+
return
|
|
87
|
+
}
|
|
88
|
+
cooling.set(idOf(endpoint), Date.now() + cooldownMs)
|
|
89
|
+
log.warn('Proxy failed, cooling down', { proxy: idOf(endpoint), cooldownMs })
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
return { retries: options.retries ?? DEFAULT_PROXY_RETRIES, select, report }
|
|
93
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { SocksClient } from 'socks'
|
|
2
|
+
import { ProxyError } from '../errors.js'
|
|
3
|
+
import { signalUtils } from '../utils/signals.js'
|
|
4
|
+
import type { ProxyConnectContext, ProxyConnection, ProxyEndpoint, ProxyPlugin, SocksProxyPluginOptions, TunnelTarget } from './types.js'
|
|
5
|
+
|
|
6
|
+
/** SOCKS4, SOCKS4a, SOCKS5 and SOCKS5h proxies, with user/password authentication. */
|
|
7
|
+
export const createSocksProxyPlugin = (options: SocksProxyPluginOptions = {}): ProxyPlugin => {
|
|
8
|
+
const remoteDns = (endpoint: ProxyEndpoint): boolean =>
|
|
9
|
+
endpoint.type === 'socks5h' || endpoint.type === 'socks4a' || options.remoteDns === true
|
|
10
|
+
|
|
11
|
+
const connect = async (endpoint: ProxyEndpoint, target: TunnelTarget, context: ProxyConnectContext): Promise<ProxyConnection> => {
|
|
12
|
+
signalUtils.throwIfAborted(context.signal)
|
|
13
|
+
const pending = SocksClient.createConnection({
|
|
14
|
+
command: 'connect',
|
|
15
|
+
destination: { host: target.host, port: target.port },
|
|
16
|
+
proxy: {
|
|
17
|
+
host: endpoint.host,
|
|
18
|
+
port: endpoint.port,
|
|
19
|
+
type: endpoint.type.startsWith('socks4') ? 4 : 5,
|
|
20
|
+
userId: endpoint.username,
|
|
21
|
+
password: endpoint.password,
|
|
22
|
+
},
|
|
23
|
+
timeout: context.timeoutMs,
|
|
24
|
+
})
|
|
25
|
+
|
|
26
|
+
let release = (): void => undefined
|
|
27
|
+
const aborted = new Promise<never>((_resolve, reject) => {
|
|
28
|
+
release = signalUtils.onAbort(context.signal, reason => reject(reason))
|
|
29
|
+
})
|
|
30
|
+
try {
|
|
31
|
+
const { socket } = await Promise.race([pending, aborted])
|
|
32
|
+
return { socket, mode: 'tunnel' }
|
|
33
|
+
} catch (error) {
|
|
34
|
+
pending.then(({ socket }) => socket.destroy(), () => undefined)
|
|
35
|
+
if (context.signal?.aborted === true) {
|
|
36
|
+
throw error
|
|
37
|
+
}
|
|
38
|
+
const message = (error as Error).message ?? ''
|
|
39
|
+
throw new ProxyError(/auth/i.test(message) ? 'auth-failed' : /rejected|refused|failed/i.test(message) ? 'refused:socks' : 'connect')
|
|
40
|
+
} finally {
|
|
41
|
+
release()
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
return { types: ['socks4', 'socks4a', 'socks5', 'socks5h'], remoteDns, connect }
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export const socksProxyPlugin = createSocksProxyPlugin()
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import type { Duplex } from 'node:stream'
|
|
2
|
+
|
|
3
|
+
/** One proxy. `type` names the plugin that speaks to it. */
|
|
4
|
+
export interface ProxyEndpoint {
|
|
5
|
+
/** `http`, `https`, `socks4`, `socks4a`, `socks5`, `socks5h`, or a custom plugin's type. */
|
|
6
|
+
type: string
|
|
7
|
+
host: string
|
|
8
|
+
port: number
|
|
9
|
+
username?: string
|
|
10
|
+
password?: string
|
|
11
|
+
/** A credential-free name for logs; default `<type>://<host>:<port>`. */
|
|
12
|
+
id?: string
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export interface ProxyCredentials {
|
|
16
|
+
username: string
|
|
17
|
+
password: string
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** What a proxy is chosen for: the caller's routing key and the URL. */
|
|
21
|
+
export interface ProxyRequest {
|
|
22
|
+
key?: string
|
|
23
|
+
url: URL
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** A pool as a callback: the proxy (credentials included) for a key and a URL, or nothing to go direct. */
|
|
27
|
+
export interface ProxyProvider {
|
|
28
|
+
(request: ProxyRequest): ProxyEndpoint | string | null | undefined | Promise<ProxyEndpoint | string | null | undefined>
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** Credentials for a chosen endpoint that carries none (a per-key session user, a vault lookup). */
|
|
32
|
+
export interface ProxyCredentialsProvider {
|
|
33
|
+
(endpoint: ProxyEndpoint, request: ProxyRequest): ProxyCredentials | null | undefined | Promise<ProxyCredentials | null | undefined>
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** How a list of proxies is walked. */
|
|
37
|
+
// A closed set of strategies — a union.
|
|
38
|
+
export type ProxyStrategy = 'round-robin' | 'random' | 'failover' | 'by-key'
|
|
39
|
+
|
|
40
|
+
export interface ProxyRouterOptions {
|
|
41
|
+
/** Endpoints or URLs (`socks5h://user:pass@egress:1080`), or a provider callback. */
|
|
42
|
+
proxies?: Array<ProxyEndpoint | string> | ProxyProvider
|
|
43
|
+
credentials?: ProxyCredentialsProvider
|
|
44
|
+
/** Default `round-robin`; `by-key` keeps one key on one proxy. */
|
|
45
|
+
strategy?: ProxyStrategy
|
|
46
|
+
/** Hosts (`x.com`, `*.x.com`) never proxied. */
|
|
47
|
+
bypass?: string[]
|
|
48
|
+
/** Plugins tried before the built-in http and socks ones. */
|
|
49
|
+
plugins?: ProxyPlugin[]
|
|
50
|
+
/** Further proxies tried after a failed connect. Default 1. */
|
|
51
|
+
retries?: number
|
|
52
|
+
/** How long a failed proxy is skipped. Default 30 000 ms. */
|
|
53
|
+
cooldownMs?: number
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export interface ProxySelection {
|
|
57
|
+
endpoint: ProxyEndpoint
|
|
58
|
+
plugin: ProxyPlugin
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** Chooses the proxy of each connection. */
|
|
62
|
+
export interface ProxyRouter {
|
|
63
|
+
/** Further attempts after a failed connect. */
|
|
64
|
+
readonly retries: number
|
|
65
|
+
/** The proxy for a request, or `null` to connect directly. */
|
|
66
|
+
select: (request: ProxyRequest) => Promise<ProxySelection | null>
|
|
67
|
+
/** How a connect through `endpoint` went; failures cool it down. */
|
|
68
|
+
report: (endpoint: ProxyEndpoint, ok: boolean) => void
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export interface TunnelTarget {
|
|
72
|
+
/** The checked IP, or the host name when the proxy resolves it. */
|
|
73
|
+
host: string
|
|
74
|
+
port: number
|
|
75
|
+
/** The host name (SNI, `Host`). */
|
|
76
|
+
hostname: string
|
|
77
|
+
/** TLS rides inside the tunnel; a plugin never terminates it. */
|
|
78
|
+
secure: boolean
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export interface ProxyConnectContext {
|
|
82
|
+
signal?: AbortSignal
|
|
83
|
+
timeoutMs: number
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export interface ProxyConnection {
|
|
87
|
+
socket: Duplex
|
|
88
|
+
/** `forward`: absolute-form plain HTTP through the proxy (http targets only). */
|
|
89
|
+
mode: 'tunnel' | 'forward'
|
|
90
|
+
/** Headers a forward-mode request carries (`Proxy-Authorization`). */
|
|
91
|
+
headers?: Record<string, string>
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** Speaks one or more proxy protocols. */
|
|
95
|
+
export interface ProxyPlugin {
|
|
96
|
+
readonly types: string[]
|
|
97
|
+
/** Whether the proxy resolves names itself for this endpoint. */
|
|
98
|
+
remoteDns: (endpoint: ProxyEndpoint) => boolean
|
|
99
|
+
connect: (endpoint: ProxyEndpoint, target: TunnelTarget, context: ProxyConnectContext) => Promise<ProxyConnection>
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export interface HttpProxyPluginOptions {
|
|
103
|
+
/** How plain-http targets travel. Default `forward` (proxies often refuse CONNECT to port 80). */
|
|
104
|
+
httpTargets?: 'forward' | 'connect'
|
|
105
|
+
/** Put the host name, not the checked IP, in CONNECT. Default `false`. */
|
|
106
|
+
remoteDns?: boolean
|
|
107
|
+
/** Extra CAs for `https` proxy endpoints. */
|
|
108
|
+
ca?: string | string[]
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
export interface SocksProxyPluginOptions {
|
|
112
|
+
/** Let socks4/socks5 endpoints resolve names too; socks4a/socks5h always do. Default `false`. */
|
|
113
|
+
remoteDns?: boolean
|
|
114
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { GoogleDriveOrigins, GoogleExportFormats } from './types.js'
|
|
2
|
+
|
|
3
|
+
export const GOOGLE_DRIVE_RESOLVER = 'google-drive'
|
|
4
|
+
|
|
5
|
+
export const DEFAULT_GOOGLE_ORIGINS: Readonly<Required<GoogleDriveOrigins>> = Object.freeze({
|
|
6
|
+
drive: 'https://drive.google.com',
|
|
7
|
+
usercontent: 'https://drive.usercontent.google.com',
|
|
8
|
+
docs: 'https://docs.google.com',
|
|
9
|
+
accounts: 'https://accounts.google.com',
|
|
10
|
+
})
|
|
11
|
+
|
|
12
|
+
export const DEFAULT_EXPORT_FORMATS: Readonly<Required<GoogleExportFormats>> = Object.freeze({
|
|
13
|
+
document: 'docx', spreadsheet: 'xlsx', presentation: 'pptx', drawing: 'png',
|
|
14
|
+
})
|
|
15
|
+
|
|
16
|
+
/** What a Drive file id looks like. */
|
|
17
|
+
export const DRIVE_ID_PATTERN = /^[A-Za-z0-9_-]{10,}$/
|
|
18
|
+
|
|
19
|
+
/** Follow-ups one Drive download may take (interstitial form, legacy confirm, cookie confirm). */
|
|
20
|
+
export const MAX_DRIVE_STEPS = 3
|
|
21
|
+
|
|
22
|
+
export const DEFAULT_DRIVE_PAGE_BYTES = 256 * 1024
|