@owlmeans/file-streams 0.0.0-stage → 0.2.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +71 -2
- package/agent-meta/manifest.json +64 -0
- package/agent-meta/skills/file-streams/SKILL.md +183 -0
- package/agent-meta/skills/file-streams/parts/network.md +47 -0
- package/agent-meta/skills/file-streams/parts/workspaces.md +129 -0
- package/build/archive/consts.d.ts +16 -0
- package/build/archive/consts.d.ts.map +1 -0
- package/build/archive/consts.js +24 -0
- package/build/archive/consts.js.map +1 -0
- package/build/archive/formats.d.ts +4 -0
- package/build/archive/formats.d.ts.map +1 -0
- package/build/archive/formats.js +53 -0
- package/build/archive/formats.js.map +1 -0
- package/build/archive/guard.d.ts +5 -0
- package/build/archive/guard.d.ts.map +1 -0
- package/build/archive/guard.js +62 -0
- package/build/archive/guard.js.map +1 -0
- package/build/archive/ratio.d.ts +5 -0
- package/build/archive/ratio.d.ts.map +1 -0
- package/build/archive/ratio.js +30 -0
- package/build/archive/ratio.js.map +1 -0
- package/build/archive/tar.d.ts +4 -0
- package/build/archive/tar.d.ts.map +1 -0
- package/build/archive/tar.js +142 -0
- package/build/archive/tar.js.map +1 -0
- package/build/archive/types.d.ts +95 -0
- package/build/archive/types.d.ts.map +1 -0
- package/build/archive/types.js +2 -0
- package/build/archive/types.js.map +1 -0
- package/build/archive/types.local.d.ts +64 -0
- package/build/archive/types.local.d.ts.map +1 -0
- package/build/archive/types.local.js +2 -0
- package/build/archive/types.local.js.map +1 -0
- package/build/archive/zip.d.ts +4 -0
- package/build/archive/zip.d.ts.map +1 -0
- package/build/archive/zip.js +135 -0
- package/build/archive/zip.js.map +1 -0
- package/build/archive.d.ts +4 -0
- package/build/archive.d.ts.map +1 -0
- package/build/archive.js +265 -0
- package/build/archive.js.map +1 -0
- package/build/backed-workspace/types.d.ts +67 -0
- package/build/backed-workspace/types.d.ts.map +1 -0
- package/build/backed-workspace/types.js +2 -0
- package/build/backed-workspace/types.js.map +1 -0
- package/build/backed-workspace.d.ts +4 -0
- package/build/backed-workspace.d.ts.map +1 -0
- package/build/backed-workspace.js +63 -0
- package/build/backed-workspace.js.map +1 -0
- package/build/consts.d.ts +3 -0
- package/build/consts.d.ts.map +1 -0
- package/build/consts.js +3 -0
- package/build/consts.js.map +1 -0
- package/build/consts.local.d.ts +3 -0
- package/build/consts.local.d.ts.map +1 -0
- package/build/consts.local.js +3 -0
- package/build/consts.local.js.map +1 -0
- package/build/errors.d.ts +59 -0
- package/build/errors.d.ts.map +1 -0
- package/build/errors.js +105 -0
- package/build/errors.js.map +1 -0
- package/build/http/connector.d.ts +11 -0
- package/build/http/connector.d.ts.map +1 -0
- package/build/http/connector.js +124 -0
- package/build/http/connector.js.map +1 -0
- package/build/http/consts.d.ts +9 -0
- package/build/http/consts.d.ts.map +1 -0
- package/build/http/consts.js +8 -0
- package/build/http/consts.js.map +1 -0
- package/build/http/source.d.ts +5 -0
- package/build/http/source.d.ts.map +1 -0
- package/build/http/source.js +76 -0
- package/build/http/source.js.map +1 -0
- package/build/http/types.d.ts +67 -0
- package/build/http/types.d.ts.map +1 -0
- package/build/http/types.js +2 -0
- package/build/http/types.js.map +1 -0
- package/build/http/types.local.d.ts +21 -0
- package/build/http/types.local.d.ts.map +1 -0
- package/build/http/types.local.js +2 -0
- package/build/http/types.local.js.map +1 -0
- package/build/http.d.ts +4 -0
- package/build/http.d.ts.map +1 -0
- package/build/http.js +248 -0
- package/build/http.js.map +1 -0
- package/build/index.d.ts +53 -0
- package/build/index.d.ts.map +1 -0
- package/build/index.js +33 -0
- package/build/index.js.map +1 -0
- package/build/network/consts.d.ts +12 -0
- package/build/network/consts.d.ts.map +1 -0
- package/build/network/consts.js +20 -0
- package/build/network/consts.js.map +1 -0
- package/build/network/types.d.ts +43 -0
- package/build/network/types.d.ts.map +1 -0
- package/build/network/types.js +2 -0
- package/build/network/types.js.map +1 -0
- package/build/network.d.ts +4 -0
- package/build/network.d.ts.map +1 -0
- package/build/network.js +165 -0
- package/build/network.js.map +1 -0
- package/build/proxy/consts.d.ts +7 -0
- package/build/proxy/consts.d.ts.map +1 -0
- package/build/proxy/consts.js +9 -0
- package/build/proxy/consts.js.map +1 -0
- package/build/proxy/endpoint.d.ts +4 -0
- package/build/proxy/endpoint.d.ts.map +1 -0
- package/build/proxy/endpoint.js +27 -0
- package/build/proxy/endpoint.js.map +1 -0
- package/build/proxy/http.d.ts +5 -0
- package/build/proxy/http.d.ts.map +1 -0
- package/build/proxy/http.js +95 -0
- package/build/proxy/http.js.map +1 -0
- package/build/proxy/router.d.ts +4 -0
- package/build/proxy/router.d.ts.map +1 -0
- package/build/proxy/router.js +85 -0
- package/build/proxy/router.js.map +1 -0
- package/build/proxy/socks.d.ts +5 -0
- package/build/proxy/socks.d.ts.map +1 -0
- package/build/proxy/socks.js +44 -0
- package/build/proxy/socks.js.map +1 -0
- package/build/proxy/types.d.ts +99 -0
- package/build/proxy/types.d.ts.map +1 -0
- package/build/proxy/types.js +2 -0
- package/build/proxy/types.js.map +1 -0
- package/build/resolvers/google-drive/consts.d.ts +10 -0
- package/build/resolvers/google-drive/consts.d.ts.map +1 -0
- package/build/resolvers/google-drive/consts.js +16 -0
- package/build/resolvers/google-drive/consts.js.map +1 -0
- package/build/resolvers/google-drive/types.d.ts +21 -0
- package/build/resolvers/google-drive/types.d.ts.map +1 -0
- package/build/resolvers/google-drive/types.js +2 -0
- package/build/resolvers/google-drive/types.js.map +1 -0
- package/build/resolvers/google-drive.d.ts +10 -0
- package/build/resolvers/google-drive.d.ts.map +1 -0
- package/build/resolvers/google-drive.js +130 -0
- package/build/resolvers/google-drive.js.map +1 -0
- package/build/resolvers/types.d.ts +39 -0
- package/build/resolvers/types.d.ts.map +1 -0
- package/build/resolvers/types.js +2 -0
- package/build/resolvers/types.js.map +1 -0
- package/build/s3/consts.d.ts +9 -0
- package/build/s3/consts.d.ts.map +1 -0
- package/build/s3/consts.js +9 -0
- package/build/s3/consts.js.map +1 -0
- package/build/s3/prefix.d.ts +4 -0
- package/build/s3/prefix.d.ts.map +1 -0
- package/build/s3/prefix.js +87 -0
- package/build/s3/prefix.js.map +1 -0
- package/build/s3/sink.d.ts +6 -0
- package/build/s3/sink.d.ts.map +1 -0
- package/build/s3/sink.js +30 -0
- package/build/s3/sink.js.map +1 -0
- package/build/s3/source.d.ts +5 -0
- package/build/s3/source.d.ts.map +1 -0
- package/build/s3/source.js +81 -0
- package/build/s3/source.js.map +1 -0
- package/build/s3/types.d.ts +66 -0
- package/build/s3/types.d.ts.map +1 -0
- package/build/s3/types.js +2 -0
- package/build/s3/types.js.map +1 -0
- package/build/s3/types.local.d.ts +43 -0
- package/build/s3/types.local.d.ts.map +1 -0
- package/build/s3/types.local.js +2 -0
- package/build/s3/types.local.js.map +1 -0
- package/build/s3/upload.d.ts +4 -0
- package/build/s3/upload.d.ts.map +1 -0
- package/build/s3/upload.js +118 -0
- package/build/s3/upload.js.map +1 -0
- package/build/s3.d.ts +5 -0
- package/build/s3.d.ts.map +1 -0
- package/build/s3.js +68 -0
- package/build/s3.js.map +1 -0
- package/build/service/types.d.ts +66 -0
- package/build/service/types.d.ts.map +1 -0
- package/build/service/types.js +2 -0
- package/build/service/types.js.map +1 -0
- package/build/service/types.local.d.ts +11 -0
- package/build/service/types.local.d.ts.map +1 -0
- package/build/service/types.local.js +2 -0
- package/build/service/types.local.js.map +1 -0
- package/build/service.d.ts +6 -0
- package/build/service.d.ts.map +1 -0
- package/build/service.js +73 -0
- package/build/service.js.map +1 -0
- package/build/sinks/buffer/consts.d.ts +2 -0
- package/build/sinks/buffer/consts.d.ts.map +1 -0
- package/build/sinks/buffer/consts.js +2 -0
- package/build/sinks/buffer/consts.js.map +1 -0
- package/build/sinks/buffer/types.d.ts +11 -0
- package/build/sinks/buffer/types.d.ts.map +1 -0
- package/build/sinks/buffer/types.js +2 -0
- package/build/sinks/buffer/types.js.map +1 -0
- package/build/sinks/buffer.d.ts +4 -0
- package/build/sinks/buffer.d.ts.map +1 -0
- package/build/sinks/buffer.js +13 -0
- package/build/sinks/buffer.js.map +1 -0
- package/build/sinks/directory/types.d.ts +7 -0
- package/build/sinks/directory/types.d.ts.map +1 -0
- package/build/sinks/directory/types.js +2 -0
- package/build/sinks/directory/types.js.map +1 -0
- package/build/sinks/directory.d.ts +8 -0
- package/build/sinks/directory.d.ts.map +1 -0
- package/build/sinks/directory.js +43 -0
- package/build/sinks/directory.js.map +1 -0
- package/build/sinks/file.d.ts +5 -0
- package/build/sinks/file.d.ts.map +1 -0
- package/build/sinks/file.js +46 -0
- package/build/sinks/file.js.map +1 -0
- package/build/sinks/types.d.ts +11 -0
- package/build/sinks/types.d.ts.map +1 -0
- package/build/sinks/types.js +2 -0
- package/build/sinks/types.js.map +1 -0
- package/build/sinks/workload/consts.d.ts +5 -0
- package/build/sinks/workload/consts.d.ts.map +1 -0
- package/build/sinks/workload/consts.js +5 -0
- package/build/sinks/workload/consts.js.map +1 -0
- package/build/sinks/workload/types.d.ts +17 -0
- package/build/sinks/workload/types.d.ts.map +1 -0
- package/build/sinks/workload/types.js +2 -0
- package/build/sinks/workload/types.js.map +1 -0
- package/build/sinks/workload.d.ts +8 -0
- package/build/sinks/workload.d.ts.map +1 -0
- package/build/sinks/workload.js +51 -0
- package/build/sinks/workload.js.map +1 -0
- package/build/sources/buffer.d.ts +5 -0
- package/build/sources/buffer.d.ts.map +1 -0
- package/build/sources/buffer.js +20 -0
- package/build/sources/buffer.js.map +1 -0
- package/build/sources/file.d.ts +5 -0
- package/build/sources/file.d.ts.map +1 -0
- package/build/sources/file.js +50 -0
- package/build/sources/file.js.map +1 -0
- package/build/sources/stream.d.ts +5 -0
- package/build/sources/stream.d.ts.map +1 -0
- package/build/sources/stream.js +19 -0
- package/build/sources/stream.js.map +1 -0
- package/build/sources/types.d.ts +10 -0
- package/build/sources/types.d.ts.map +1 -0
- package/build/sources/types.js +2 -0
- package/build/sources/types.js.map +1 -0
- package/build/sources/workload/types.d.ts +29 -0
- package/build/sources/workload/types.d.ts.map +1 -0
- package/build/sources/workload/types.js +2 -0
- package/build/sources/workload/types.js.map +1 -0
- package/build/sources/workload.d.ts +7 -0
- package/build/sources/workload.d.ts.map +1 -0
- package/build/sources/workload.js +67 -0
- package/build/sources/workload.js.map +1 -0
- package/build/transfer/consts.d.ts +5 -0
- package/build/transfer/consts.d.ts.map +1 -0
- package/build/transfer/consts.js +8 -0
- package/build/transfer/consts.js.map +1 -0
- package/build/transfer/types.d.ts +49 -0
- package/build/transfer/types.d.ts.map +1 -0
- package/build/transfer/types.js +2 -0
- package/build/transfer/types.js.map +1 -0
- package/build/transfer.d.ts +4 -0
- package/build/transfer.d.ts.map +1 -0
- package/build/transfer.js +75 -0
- package/build/transfer.js.map +1 -0
- package/build/types.d.ts +99 -0
- package/build/types.d.ts.map +1 -0
- package/build/types.js +2 -0
- package/build/types.js.map +1 -0
- package/build/utils/names/consts.d.ts +5 -0
- package/build/utils/names/consts.d.ts.map +1 -0
- package/build/utils/names/consts.js +5 -0
- package/build/utils/names/consts.js.map +1 -0
- package/build/utils/names/types.d.ts +13 -0
- package/build/utils/names/types.d.ts.map +1 -0
- package/build/utils/names/types.js +2 -0
- package/build/utils/names/types.js.map +1 -0
- package/build/utils/names.d.ts +4 -0
- package/build/utils/names.d.ts.map +1 -0
- package/build/utils/names.js +60 -0
- package/build/utils/names.js.map +1 -0
- package/build/utils/paths/consts.d.ts +3 -0
- package/build/utils/paths/consts.d.ts.map +1 -0
- package/build/utils/paths/consts.js +2 -0
- package/build/utils/paths/consts.js.map +1 -0
- package/build/utils/paths/types.d.ts +20 -0
- package/build/utils/paths/types.d.ts.map +1 -0
- package/build/utils/paths/types.js +2 -0
- package/build/utils/paths/types.js.map +1 -0
- package/build/utils/paths.d.ts +4 -0
- package/build/utils/paths.d.ts.map +1 -0
- package/build/utils/paths.js +67 -0
- package/build/utils/paths.js.map +1 -0
- package/build/utils/range/types.d.ts +11 -0
- package/build/utils/range/types.d.ts.map +1 -0
- package/build/utils/range/types.js +2 -0
- package/build/utils/range/types.js.map +1 -0
- package/build/utils/range.d.ts +4 -0
- package/build/utils/range.d.ts.map +1 -0
- package/build/utils/range.js +64 -0
- package/build/utils/range.js.map +1 -0
- package/build/utils/redact/types.d.ts +16 -0
- package/build/utils/redact/types.d.ts.map +1 -0
- package/build/utils/redact/types.js +2 -0
- package/build/utils/redact/types.js.map +1 -0
- package/build/utils/redact.d.ts +4 -0
- package/build/utils/redact.d.ts.map +1 -0
- package/build/utils/redact.js +18 -0
- package/build/utils/redact.js.map +1 -0
- package/build/utils/signals/types.d.ts +17 -0
- package/build/utils/signals/types.d.ts.map +1 -0
- package/build/utils/signals/types.js +2 -0
- package/build/utils/signals/types.js.map +1 -0
- package/build/utils/signals.d.ts +4 -0
- package/build/utils/signals.d.ts.map +1 -0
- package/build/utils/signals.js +52 -0
- package/build/utils/signals.js.map +1 -0
- package/build/utils/streams/consts.d.ts +3 -0
- package/build/utils/streams/consts.d.ts.map +1 -0
- package/build/utils/streams/consts.js +3 -0
- package/build/utils/streams/consts.js.map +1 -0
- package/build/utils/streams/types.d.ts +59 -0
- package/build/utils/streams/types.d.ts.map +1 -0
- package/build/utils/streams/types.js +2 -0
- package/build/utils/streams/types.js.map +1 -0
- package/build/utils/streams.d.ts +4 -0
- package/build/utils/streams.d.ts.map +1 -0
- package/build/utils/streams.js +167 -0
- package/build/utils/streams.js.map +1 -0
- package/build/utils/tree/types.d.ts +6 -0
- package/build/utils/tree/types.d.ts.map +1 -0
- package/build/utils/tree/types.js +2 -0
- package/build/utils/tree/types.js.map +1 -0
- package/build/utils/tree.d.ts +4 -0
- package/build/utils/tree.d.ts.map +1 -0
- package/build/utils/tree.js +14 -0
- package/build/utils/tree.js.map +1 -0
- package/build/workspace/consts.d.ts +12 -0
- package/build/workspace/consts.d.ts.map +1 -0
- package/build/workspace/consts.js +12 -0
- package/build/workspace/consts.js.map +1 -0
- package/build/workspace/consts.local.d.ts +3 -0
- package/build/workspace/consts.local.d.ts.map +1 -0
- package/build/workspace/consts.local.js +3 -0
- package/build/workspace/consts.local.js.map +1 -0
- package/build/workspace/fs.d.ts +4 -0
- package/build/workspace/fs.d.ts.map +1 -0
- package/build/workspace/fs.js +45 -0
- package/build/workspace/fs.js.map +1 -0
- package/build/workspace/scope.d.ts +6 -0
- package/build/workspace/scope.d.ts.map +1 -0
- package/build/workspace/scope.js +180 -0
- package/build/workspace/scope.js.map +1 -0
- package/build/workspace/types.d.ts +66 -0
- package/build/workspace/types.d.ts.map +1 -0
- package/build/workspace/types.js +2 -0
- package/build/workspace/types.js.map +1 -0
- package/build/workspace/types.local.d.ts +20 -0
- package/build/workspace/types.local.d.ts.map +1 -0
- package/build/workspace/types.local.js +2 -0
- package/build/workspace/types.local.js.map +1 -0
- package/build/workspace.d.ts +7 -0
- package/build/workspace.d.ts.map +1 -0
- package/build/workspace.js +87 -0
- package/build/workspace.js.map +1 -0
- package/package.json +61 -4
- package/src/archive/consts.ts +28 -0
- package/src/archive/formats.ts +60 -0
- package/src/archive/guard.ts +68 -0
- package/src/archive/ratio.ts +35 -0
- package/src/archive/tar.ts +147 -0
- package/src/archive/types.local.ts +72 -0
- package/src/archive/types.ts +107 -0
- package/src/archive/zip.ts +142 -0
- package/src/archive.ts +298 -0
- package/src/backed-workspace/types.ts +72 -0
- package/src/backed-workspace.ts +85 -0
- package/src/consts.local.ts +2 -0
- package/src/consts.ts +2 -0
- package/src/errors.ts +128 -0
- package/src/http/connector.ts +136 -0
- package/src/http/consts.ts +13 -0
- package/src/http/source.ts +89 -0
- package/src/http/types.local.ts +23 -0
- package/src/http/types.ts +74 -0
- package/src/http.ts +276 -0
- package/src/index.ts +65 -0
- package/src/network/consts.ts +22 -0
- package/src/network/types.ts +45 -0
- package/src/network.ts +179 -0
- package/src/proxy/consts.ts +11 -0
- package/src/proxy/endpoint.ts +28 -0
- package/src/proxy/http.ts +108 -0
- package/src/proxy/router.ts +93 -0
- package/src/proxy/socks.ts +48 -0
- package/src/proxy/types.ts +114 -0
- package/src/resolvers/google-drive/consts.ts +22 -0
- package/src/resolvers/google-drive/types.ts +22 -0
- package/src/resolvers/google-drive.ts +153 -0
- package/src/resolvers/types.ts +45 -0
- package/src/s3/consts.ts +12 -0
- package/src/s3/prefix.ts +100 -0
- package/src/s3/sink.ts +36 -0
- package/src/s3/source.ts +90 -0
- package/src/s3/types.local.ts +49 -0
- package/src/s3/types.ts +70 -0
- package/src/s3/upload.ts +130 -0
- package/src/s3.ts +83 -0
- package/src/service/types.local.ts +9 -0
- package/src/service/types.ts +71 -0
- package/src/service.ts +97 -0
- package/src/sinks/buffer/consts.ts +1 -0
- package/src/sinks/buffer/types.ts +12 -0
- package/src/sinks/buffer.ts +18 -0
- package/src/sinks/directory/types.ts +7 -0
- package/src/sinks/directory.ts +49 -0
- package/src/sinks/file.ts +52 -0
- package/src/sinks/types.ts +12 -0
- package/src/sinks/workload/consts.ts +5 -0
- package/src/sinks/workload/types.ts +17 -0
- package/src/sinks/workload.ts +59 -0
- package/src/sources/buffer.ts +27 -0
- package/src/sources/file.ts +57 -0
- package/src/sources/stream.ts +26 -0
- package/src/sources/types.ts +12 -0
- package/src/sources/workload/types.ts +31 -0
- package/src/sources/workload.ts +79 -0
- package/src/transfer/consts.ts +10 -0
- package/src/transfer/types.ts +55 -0
- package/src/transfer.ts +89 -0
- package/src/types.ts +106 -0
- package/src/utils/names/consts.ts +5 -0
- package/src/utils/names/types.ts +12 -0
- package/src/utils/names.ts +70 -0
- package/src/utils/paths/consts.ts +3 -0
- package/src/utils/paths/types.ts +20 -0
- package/src/utils/paths.ts +75 -0
- package/src/utils/range/types.ts +11 -0
- package/src/utils/range.ts +80 -0
- package/src/utils/redact/types.ts +16 -0
- package/src/utils/redact.ts +23 -0
- package/src/utils/signals/types.ts +17 -0
- package/src/utils/signals.ts +63 -0
- package/src/utils/streams/consts.ts +2 -0
- package/src/utils/streams/types.ts +65 -0
- package/src/utils/streams.ts +188 -0
- package/src/utils/tree/types.ts +6 -0
- package/src/utils/tree.ts +18 -0
- package/src/workspace/consts.local.ts +2 -0
- package/src/workspace/consts.ts +14 -0
- package/src/workspace/fs.ts +48 -0
- package/src/workspace/scope.ts +196 -0
- package/src/workspace/types.local.ts +21 -0
- package/src/workspace/types.ts +71 -0
- package/src/workspace.ts +102 -0
- package/tests/archive.spec.ts +134 -0
- package/tests/backed.spec.ts +147 -0
- package/tests/context.ts +114 -0
- package/tests/fixtures.ts +190 -0
- package/tests/google-drive.spec.ts +76 -0
- package/tests/harness/drive.ts +92 -0
- package/tests/harness/http-proxy.ts +96 -0
- package/tests/harness/origin.ts +134 -0
- package/tests/harness/socks5.ts +135 -0
- package/tests/harness/workload.ts +118 -0
- package/tests/http.spec.ts +101 -0
- package/tests/inspector.spec.ts +40 -0
- package/tests/live.spec.ts +27 -0
- package/tests/network.spec.ts +79 -0
- package/tests/proxy.spec.ts +149 -0
- package/tests/s3.spec.ts +133 -0
- package/tests/service.spec.ts +77 -0
- package/tests/sources-sinks.spec.ts +96 -0
- package/tests/transfer.spec.ts +99 -0
- package/tests/tsconfig.json +12 -0
- package/tests/workload.spec.ts +140 -0
- package/tests/workspace.spec.ts +158 -0
- package/tsconfig.json +16 -0
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/** The format each Google editor file exports as. */
|
|
2
|
+
export interface GoogleExportFormats {
|
|
3
|
+
document?: 'docx' | 'pdf' | 'odt' | 'rtf' | 'txt' | 'epub' | 'md' | 'html'
|
|
4
|
+
spreadsheet?: 'xlsx' | 'ods' | 'pdf' | 'csv' | 'tsv'
|
|
5
|
+
presentation?: 'pptx' | 'odp' | 'pdf' | 'txt'
|
|
6
|
+
drawing?: 'png' | 'svg' | 'pdf' | 'jpg'
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
/** Where Google answers; tests point these at a stand-in. */
|
|
10
|
+
export interface GoogleDriveOrigins {
|
|
11
|
+
drive?: string
|
|
12
|
+
usercontent?: string
|
|
13
|
+
docs?: string
|
|
14
|
+
accounts?: string
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export interface GoogleDriveResolverOptions {
|
|
18
|
+
exports?: GoogleExportFormats
|
|
19
|
+
origins?: GoogleDriveOrigins
|
|
20
|
+
/** Bytes of an interstitial page read. Default 256 KiB. */
|
|
21
|
+
maxPageBytes?: number
|
|
22
|
+
}
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
import { logger } from '@owlmeans/log'
|
|
2
|
+
import { ResolverError } from '../errors.js'
|
|
3
|
+
import type { ResolverBodyReader, ResolverResponseHead, ResolverStep, UrlResolver } from './types.js'
|
|
4
|
+
import {
|
|
5
|
+
DEFAULT_DRIVE_PAGE_BYTES, DEFAULT_EXPORT_FORMATS, DEFAULT_GOOGLE_ORIGINS, DRIVE_ID_PATTERN, GOOGLE_DRIVE_RESOLVER, MAX_DRIVE_STEPS,
|
|
6
|
+
} from './google-drive/consts.js'
|
|
7
|
+
import type { GoogleDriveResolverOptions } from './google-drive/types.js'
|
|
8
|
+
|
|
9
|
+
const log = logger('file-streams:drive')
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Google Drive and Docs links to the bytes behind them: share links become the usercontent
|
|
13
|
+
* download, the "can't scan for viruses" page is followed through its own form, and editor files
|
|
14
|
+
* become export URLs in the configured formats. Folders, quotas and non-public files fail.
|
|
15
|
+
*/
|
|
16
|
+
export const createGoogleDriveResolver = (options: GoogleDriveResolverOptions = {}): UrlResolver => {
|
|
17
|
+
const origins = { ...DEFAULT_GOOGLE_ORIGINS, ...options.origins }
|
|
18
|
+
const drive = new URL(origins.drive).origin
|
|
19
|
+
const usercontent = new URL(origins.usercontent).origin
|
|
20
|
+
const docs = new URL(origins.docs).origin
|
|
21
|
+
const accounts = new URL(origins.accounts).origin
|
|
22
|
+
const formats = { ...DEFAULT_EXPORT_FORMATS, ...options.exports }
|
|
23
|
+
const maxPageBytes = options.maxPageBytes ?? DEFAULT_DRIVE_PAGE_BYTES
|
|
24
|
+
|
|
25
|
+
const owns = (url: URL): boolean => url.origin === drive || url.origin === usercontent || url.origin === docs
|
|
26
|
+
|
|
27
|
+
const matches = owns
|
|
28
|
+
|
|
29
|
+
const fail = (reason: string): ResolverError => new ResolverError(`${GOOGLE_DRIVE_RESOLVER}:${reason}`)
|
|
30
|
+
|
|
31
|
+
const exportOf = (url: URL): URL | undefined => {
|
|
32
|
+
const match = /^\/(document|spreadsheets|presentation|drawings)\/d\/([^/]+)/.exec(url.pathname)
|
|
33
|
+
if (match == null || !DRIVE_ID_PATTERN.test(match[2])) {
|
|
34
|
+
return undefined
|
|
35
|
+
}
|
|
36
|
+
const [, type, id] = match
|
|
37
|
+
switch (type) {
|
|
38
|
+
case 'document':
|
|
39
|
+
return new URL(`/document/d/${id}/export?format=${formats.document}`, docs)
|
|
40
|
+
case 'spreadsheets': {
|
|
41
|
+
const exported = new URL(`/spreadsheets/d/${id}/export?format=${formats.spreadsheet}`, docs)
|
|
42
|
+
const gid = url.searchParams.get('gid') ?? /gid=(\d+)/.exec(url.hash)?.[1]
|
|
43
|
+
if (gid != null && (formats.spreadsheet === 'csv' || formats.spreadsheet === 'tsv')) {
|
|
44
|
+
exported.searchParams.set('gid', gid)
|
|
45
|
+
}
|
|
46
|
+
return exported
|
|
47
|
+
}
|
|
48
|
+
case 'presentation':
|
|
49
|
+
return new URL(`/presentation/d/${id}/export/${formats.presentation}`, docs)
|
|
50
|
+
default:
|
|
51
|
+
return new URL(`/drawings/d/${id}/export/${formats.drawing}`, docs)
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const rewrite = (url: URL): URL => {
|
|
56
|
+
if (url.origin === docs) {
|
|
57
|
+
const exported = exportOf(url)
|
|
58
|
+
if (exported != null) {
|
|
59
|
+
return exported
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
if (url.origin === drive) {
|
|
63
|
+
if (/\/drive\/(u\/\d+\/)?folders\//.test(url.pathname) || url.pathname.startsWith('/embeddedfolderview')) {
|
|
64
|
+
throw fail('folder')
|
|
65
|
+
}
|
|
66
|
+
const id = /\/file\/d\/([^/]+)/.exec(url.pathname)?.[1] ?? url.searchParams.get('id')
|
|
67
|
+
if (id == null || !DRIVE_ID_PATTERN.test(id)) {
|
|
68
|
+
return url
|
|
69
|
+
}
|
|
70
|
+
const download = new URL('/download', usercontent)
|
|
71
|
+
download.searchParams.set('id', id)
|
|
72
|
+
download.searchParams.set('export', 'download')
|
|
73
|
+
const resourceKey = url.searchParams.get('resourcekey')
|
|
74
|
+
if (resourceKey != null) {
|
|
75
|
+
download.searchParams.set('resourcekey', resourceKey)
|
|
76
|
+
}
|
|
77
|
+
return download
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
return url
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const decodeHtml = (value: string): string => value
|
|
84
|
+
.replace(/&/g, '&').replace(/"/g, '"').replace(/'|'/g, "'").replace(/</g, '<').replace(/>/g, '>')
|
|
85
|
+
|
|
86
|
+
const attribute = (tag: string, name: string): string | undefined =>
|
|
87
|
+
new RegExp(`\\s${name}\\s*=\\s*("([^"]*)"|'([^']*)')`, 'i').exec(tag)?.slice(2).find(value => value != null)
|
|
88
|
+
|
|
89
|
+
const headerOf = (head: ResolverResponseHead, name: string): string[] => {
|
|
90
|
+
const value = head.headers[name]
|
|
91
|
+
return value == null ? [] : Array.isArray(value) ? value : [value]
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const inspect = async (head: ResolverResponseHead, readBody: ResolverBodyReader): Promise<ResolverStep> => {
|
|
95
|
+
if (head.url.origin === accounts) {
|
|
96
|
+
return { action: 'fail', reason: 'not-public' }
|
|
97
|
+
}
|
|
98
|
+
const type = headerOf(head, 'content-type')[0]?.toLowerCase() ?? ''
|
|
99
|
+
const disposition = headerOf(head, 'content-disposition')[0]?.toLowerCase() ?? ''
|
|
100
|
+
if (disposition.includes('attachment') || !type.includes('text/html')) {
|
|
101
|
+
return { action: 'accept' }
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
const page = await readBody(maxPageBytes)
|
|
105
|
+
const formTag = /<form\b[^>]*\bid\s*=\s*["']download-form["'][^>]*>/i.exec(page)
|
|
106
|
+
if (formTag != null) {
|
|
107
|
+
const action = attribute(formTag[0], 'action')
|
|
108
|
+
if (action != null) {
|
|
109
|
+
const next = new URL(decodeHtml(action), head.url)
|
|
110
|
+
if (!owns(next)) {
|
|
111
|
+
return { action: 'fail', reason: 'foreign-form' }
|
|
112
|
+
}
|
|
113
|
+
const body = page.slice(formTag.index, page.indexOf('</form>', formTag.index) === -1 ? undefined : page.indexOf('</form>', formTag.index))
|
|
114
|
+
for (const input of body.match(/<input\b[^>]*>/gi) ?? []) {
|
|
115
|
+
const name = attribute(input, 'name')
|
|
116
|
+
if (name != null && (attribute(input, 'type') ?? '').toLowerCase() === 'hidden') {
|
|
117
|
+
next.searchParams.set(name, decodeHtml(attribute(input, 'value') ?? ''))
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
log.debug('Following the Drive confirmation form')
|
|
121
|
+
return { action: 'follow', url: next }
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
const legacy = /href\s*=\s*"(\/uc\?export=download[^"]*confirm=[^"]+)"/i.exec(page)
|
|
126
|
+
if (legacy != null) {
|
|
127
|
+
return { action: 'follow', url: new URL(decodeHtml(legacy[1]), head.url) }
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
const warning = headerOf(head, 'set-cookie').map(cookie => /^(download_warning[^=]*)=([^;]+)/.exec(cookie)).find(match => match != null)
|
|
131
|
+
if (warning != null) {
|
|
132
|
+
const next = new URL(head.url)
|
|
133
|
+
next.searchParams.set('confirm', warning[2])
|
|
134
|
+
return { action: 'follow', url: next, headers: { cookie: `${warning[1]}=${warning[2]}` } }
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
if (/quota|too many users have viewed or downloaded/i.test(page)) {
|
|
138
|
+
return { action: 'fail', reason: 'quota' }
|
|
139
|
+
}
|
|
140
|
+
if (/too large to be exported/i.test(page)) {
|
|
141
|
+
return { action: 'fail', reason: 'export-too-large' }
|
|
142
|
+
}
|
|
143
|
+
if (/accounts\.google\.com\/(ServiceLogin|v3\/signin)|you need access|request access/i.test(page)) {
|
|
144
|
+
return { action: 'fail', reason: 'not-public' }
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
return { action: 'fail', reason: 'unrecognized' }
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
return { name: GOOGLE_DRIVE_RESOLVER, maxSteps: MAX_DRIVE_STEPS, matches, rewrite, inspect, owns }
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
export const googleDriveResolver = createGoogleDriveResolver()
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/** The head of a response a resolver inspects. */
|
|
2
|
+
export interface ResolverResponseHead {
|
|
3
|
+
url: URL
|
|
4
|
+
status: number
|
|
5
|
+
headers: Record<string, string | string[] | undefined>
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
export interface ResolverAccept {
|
|
9
|
+
action: 'accept'
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export interface ResolverFollow {
|
|
13
|
+
action: 'follow'
|
|
14
|
+
url: URL
|
|
15
|
+
/** Only `cookie` is honoured, and only towards the resolver's own origins. */
|
|
16
|
+
headers?: Record<string, string>
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export interface ResolverFail {
|
|
20
|
+
action: 'fail'
|
|
21
|
+
reason: string
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** What a resolver decides about a response. */
|
|
25
|
+
// Three distinct steps a discriminated union names best.
|
|
26
|
+
export type ResolverStep = ResolverAccept | ResolverFollow | ResolverFail
|
|
27
|
+
|
|
28
|
+
/** Reads at most `maxBytes` of the body as text; the bytes are replayed to the caller on `accept`. */
|
|
29
|
+
export interface ResolverBodyReader {
|
|
30
|
+
(maxBytes?: number): Promise<string>
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Turns a page or share link into the bytes behind it (Google Drive, Docs exports). */
|
|
34
|
+
export interface UrlResolver {
|
|
35
|
+
readonly name: string
|
|
36
|
+
/** Follow-ups one download may take. */
|
|
37
|
+
readonly maxSteps: number
|
|
38
|
+
matches: (url: URL) => boolean
|
|
39
|
+
/** The URL that serves the bytes; throws `ResolverError` for links that cannot (folders). */
|
|
40
|
+
rewrite: (url: URL) => URL
|
|
41
|
+
/** Decides on a final (post-redirect) response before its body reaches the caller. */
|
|
42
|
+
inspect: (head: ResolverResponseHead, readBody: ResolverBodyReader) => Promise<ResolverStep>
|
|
43
|
+
/** Whether a follow-up URL belongs to this resolver (cookies are sent only there). */
|
|
44
|
+
owns: (url: URL) => boolean
|
|
45
|
+
}
|
package/src/s3/consts.ts
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export const DEFAULT_PART_SIZE = 8 * 1024 * 1024
|
|
2
|
+
|
|
3
|
+
/** S3's smallest part (all but the last). */
|
|
4
|
+
export const MIN_PART_SIZE = 5 * 1024 * 1024
|
|
5
|
+
|
|
6
|
+
/** S3's most parts per upload. */
|
|
7
|
+
export const MAX_PARTS = 10_000
|
|
8
|
+
|
|
9
|
+
export const DEFAULT_UPLOAD_CONCURRENCY = 2
|
|
10
|
+
|
|
11
|
+
/** S3's most keys per DeleteObjects request. */
|
|
12
|
+
export const S3_DELETE_BATCH_SIZE = 1000
|
package/src/s3/prefix.ts
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto'
|
|
2
|
+
import { DeleteObjectsCommand, ListObjectsV2Command } from '@aws-sdk/client-s3'
|
|
3
|
+
import type { ListObjectsV2CommandOutput } from '@aws-sdk/client-s3'
|
|
4
|
+
import { logger } from '@owlmeans/log'
|
|
5
|
+
import { S3TransferError } from '../errors.js'
|
|
6
|
+
import type { SignalOptions } from '../types.js'
|
|
7
|
+
import { S3_DELETE_BATCH_SIZE } from './consts.js'
|
|
8
|
+
import type { S3ListedObject } from './types.js'
|
|
9
|
+
import type { S3BuildHandler, S3Core, S3PrefixOps } from './types.local.js'
|
|
10
|
+
|
|
11
|
+
const log = logger('file-streams:s3')
|
|
12
|
+
|
|
13
|
+
/** Listing and deleting everything under one prefix of the bucket. */
|
|
14
|
+
export const makeS3PrefixOps = (core: S3Core): S3PrefixOps => {
|
|
15
|
+
const failure = (error: unknown, op: string, signal?: AbortSignal): unknown => {
|
|
16
|
+
if (signal?.aborted === true) {
|
|
17
|
+
return signal.reason
|
|
18
|
+
}
|
|
19
|
+
log.error('S3 prefix operation failed', error as Error)
|
|
20
|
+
return new S3TransferError(op)
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
// DeleteObjects requires a checksum. The SDK's flexible checksums (CRC32) are not understood by
|
|
24
|
+
// every S3-compatible store; Content-MD5 is, by all of them and by S3 itself — the SDK's documented
|
|
25
|
+
// MD5 fallback: the flexible checksum headers go, a Content-MD5 of the body takes their place.
|
|
26
|
+
const contentMd5 = (next: S3BuildHandler): S3BuildHandler => async args => {
|
|
27
|
+
const request = args.request as { headers?: Record<string, string>, body?: unknown }
|
|
28
|
+
if (request.headers != null && (typeof request.body === 'string' || request.body instanceof Uint8Array)) {
|
|
29
|
+
for (const header of Object.keys(request.headers)) {
|
|
30
|
+
if (/^x-amz-(sdk-)?checksum-/i.test(header)) {
|
|
31
|
+
delete request.headers[header]
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
request.headers['content-md5'] = createHash('md5').update(request.body).digest('base64')
|
|
35
|
+
}
|
|
36
|
+
return await next(args)
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const withContentMd5 = (command: DeleteObjectsCommand): DeleteObjectsCommand => {
|
|
40
|
+
// A plain build handler: the SDK's middleware union cannot be inferred from it, hence the cast.
|
|
41
|
+
command.middlewareStack.addRelativeTo(contentMd5 as never, {
|
|
42
|
+
relation: 'after', toMiddleware: 'flexibleChecksumsMiddleware', name: 'owlmeansContentMd5',
|
|
43
|
+
})
|
|
44
|
+
return command
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const list = async (prefix: string, options: SignalOptions = {}): Promise<S3ListedObject[]> => {
|
|
48
|
+
const base = `${core.keyOf(prefix)}/`
|
|
49
|
+
const items: S3ListedObject[] = []
|
|
50
|
+
let token: string | undefined
|
|
51
|
+
do {
|
|
52
|
+
let page: ListObjectsV2CommandOutput
|
|
53
|
+
try {
|
|
54
|
+
page = await core.client().send(
|
|
55
|
+
new ListObjectsV2Command({ Bucket: core.model.bucket, Prefix: base, ...(token != null ? { ContinuationToken: token } : {}) }),
|
|
56
|
+
{ abortSignal: options.signal }
|
|
57
|
+
)
|
|
58
|
+
} catch (error) {
|
|
59
|
+
throw failure(error, 'list', options.signal)
|
|
60
|
+
}
|
|
61
|
+
for (const object of page.Contents ?? []) {
|
|
62
|
+
if (object.Key != null && object.Key.startsWith(base) && object.Key.length > base.length) {
|
|
63
|
+
items.push({
|
|
64
|
+
key: object.Key, path: object.Key.slice(base.length), size: object.Size ?? 0, etag: object.ETag, modifiedAt: object.LastModified,
|
|
65
|
+
})
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
token = page.IsTruncated === true ? page.NextContinuationToken : undefined
|
|
69
|
+
} while (token != null)
|
|
70
|
+
|
|
71
|
+
return items
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const removePrefix = async (prefix: string, options: SignalOptions = {}): Promise<number> => {
|
|
75
|
+
const keys = (await list(prefix, options)).map(item => item.key)
|
|
76
|
+
let removed = 0
|
|
77
|
+
for (let index = 0; index < keys.length; index += S3_DELETE_BATCH_SIZE) {
|
|
78
|
+
const batch = keys.slice(index, index + S3_DELETE_BATCH_SIZE)
|
|
79
|
+
let errors
|
|
80
|
+
try {
|
|
81
|
+
const result = await core.client().send(withContentMd5(new DeleteObjectsCommand({
|
|
82
|
+
Bucket: core.model.bucket, Delete: { Objects: batch.map(Key => ({ Key })), Quiet: true },
|
|
83
|
+
})), { abortSignal: options.signal })
|
|
84
|
+
errors = result.Errors ?? []
|
|
85
|
+
} catch (error) {
|
|
86
|
+
throw failure(error, 'delete-objects', options.signal)
|
|
87
|
+
}
|
|
88
|
+
if (errors.length > 0) {
|
|
89
|
+
log.error('S3 refused deleting keys', { refused: errors.length, code: errors[0].Code })
|
|
90
|
+
throw new S3TransferError(`delete-objects:${errors[0].Code ?? 'error'}`)
|
|
91
|
+
}
|
|
92
|
+
removed += batch.length
|
|
93
|
+
}
|
|
94
|
+
log.debug('Prefix removed', { objects: removed })
|
|
95
|
+
|
|
96
|
+
return removed
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
return { list, removePrefix }
|
|
100
|
+
}
|
package/src/s3/sink.ts
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { EXTENSION_MIME } from '@owlmeans/file-inspector'
|
|
2
|
+
import type { FileSink, SinkInput, SinkResult, SinkWriteOptions } from '../types.js'
|
|
3
|
+
import { nameUtils } from '../utils/names.js'
|
|
4
|
+
import { streamUtils } from '../utils/streams.js'
|
|
5
|
+
import type { S3SinkOptions } from './types.js'
|
|
6
|
+
import type { S3Core, S3Uploader } from './types.local.js'
|
|
7
|
+
|
|
8
|
+
/** One object as a sink: its type from the verdict, never inline when it carries scripts. */
|
|
9
|
+
export const makeS3Sink = (core: S3Core, uploader: S3Uploader, prefix: string, options: S3SinkOptions = {}): FileSink => {
|
|
10
|
+
const key = core.keyOf(prefix)
|
|
11
|
+
|
|
12
|
+
const write = async (input: SinkInput, writeOptions: SinkWriteOptions = {}): Promise<SinkResult> => {
|
|
13
|
+
const ext = nameUtils.extOf(prefix)
|
|
14
|
+
const contentType = options.contentType ?? input.verdict?.detected?.mime ?? input.info.mime
|
|
15
|
+
?? (ext != null ? EXTENSION_MIME[ext] : undefined) ?? 'application/octet-stream'
|
|
16
|
+
const active = input.verdict?.details.active === true && options.allowActiveInline !== true
|
|
17
|
+
const disposition = active ? 'attachment' : options.disposition ?? 'inline'
|
|
18
|
+
const meter = streamUtils.meter({ maxBytes: writeOptions.maxBytes })
|
|
19
|
+
const stream = input.stream.pipe(meter.stream)
|
|
20
|
+
input.stream.once('error', error => meter.stream.destroy(error))
|
|
21
|
+
|
|
22
|
+
let result
|
|
23
|
+
try {
|
|
24
|
+
result = await uploader.upload({
|
|
25
|
+
key, stream, contentType, disposition, cacheControl: options.cacheControl, metadata: options.metadata, signal: writeOptions.signal,
|
|
26
|
+
})
|
|
27
|
+
} catch (error) {
|
|
28
|
+
streamUtils.discard(input.stream)
|
|
29
|
+
throw error
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
return { kind: 's3', location: core.location(key), bytes: result.bytes, url: core.model.urlOf(prefix), etag: result.etag, mime: contentType }
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
return { kind: 's3', write }
|
|
36
|
+
}
|
package/src/s3/source.ts
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { GetObjectCommand, HeadObjectCommand } from '@aws-sdk/client-s3'
|
|
2
|
+
import type { Readable } from 'node:stream'
|
|
3
|
+
import { S3TransferError, SourceError } from '../errors.js'
|
|
4
|
+
import type { FileInfo, FileSource, OpenedFile, OpenOptions, RangeReader, SignalOptions } from '../types.js'
|
|
5
|
+
import { nameUtils } from '../utils/names.js'
|
|
6
|
+
import { streamUtils } from '../utils/streams.js'
|
|
7
|
+
import type { S3Core } from './types.local.js'
|
|
8
|
+
|
|
9
|
+
/** One object as a source: re-openable, with random access through Range and If-Match. */
|
|
10
|
+
export const makeS3Source = (core: S3Core, key: string): FileSource => {
|
|
11
|
+
let known: FileInfo | undefined
|
|
12
|
+
|
|
13
|
+
const statusOf = (error: unknown): number | undefined => (error as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode
|
|
14
|
+
|
|
15
|
+
const failure = (error: unknown, op: string, signal?: AbortSignal): unknown => {
|
|
16
|
+
if (signal?.aborted === true) {
|
|
17
|
+
return signal.reason
|
|
18
|
+
}
|
|
19
|
+
const status = statusOf(error)
|
|
20
|
+
if (status === 404 || (error as Error).name === 'NoSuchKey' || (error as Error).name === 'NotFound') {
|
|
21
|
+
return new SourceError('not-found')
|
|
22
|
+
}
|
|
23
|
+
if (status === 412 || (error as Error).name === 'PreconditionFailed') {
|
|
24
|
+
return new SourceError('changed')
|
|
25
|
+
}
|
|
26
|
+
return new S3TransferError(op)
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
const describe = (head: { ContentLength?: number, ContentType?: string, ETag?: string, LastModified?: Date }): FileInfo => ({
|
|
30
|
+
origin: core.location(key),
|
|
31
|
+
name: nameUtils.fromKey(key),
|
|
32
|
+
size: head.ContentLength,
|
|
33
|
+
mime: head.ContentType?.split(';')[0].trim().toLowerCase(),
|
|
34
|
+
etag: head.ETag,
|
|
35
|
+
modifiedAt: head.LastModified,
|
|
36
|
+
})
|
|
37
|
+
|
|
38
|
+
const info = async (options: SignalOptions = {}): Promise<FileInfo> => {
|
|
39
|
+
if (known == null) {
|
|
40
|
+
try {
|
|
41
|
+
known = describe(await core.client().send(new HeadObjectCommand({ Bucket: core.model.bucket, Key: key }), { abortSignal: options.signal }))
|
|
42
|
+
} catch (error) {
|
|
43
|
+
throw failure(error, 'head', options.signal)
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
return known
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const get = async (range: string | undefined, ifMatch: string | undefined, signal?: AbortSignal) => {
|
|
50
|
+
try {
|
|
51
|
+
return await core.client().send(new GetObjectCommand({
|
|
52
|
+
Bucket: core.model.bucket, Key: key, ...(range != null ? { Range: range } : {}), ...(ifMatch != null ? { IfMatch: ifMatch } : {}),
|
|
53
|
+
}), { abortSignal: signal })
|
|
54
|
+
} catch (error) {
|
|
55
|
+
throw failure(error, 'get', signal)
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const open = async (options: OpenOptions = {}): Promise<OpenedFile> => {
|
|
60
|
+
const range = options.range != null ? `bytes=${options.range.start}-${options.range.end ?? ''}` : undefined
|
|
61
|
+
const response = await get(range, undefined, options.signal)
|
|
62
|
+
const opened: OpenedFile = { stream: response.Body as Readable, info: { ...describe(response), size: known?.size ?? response.ContentLength } }
|
|
63
|
+
const span = /bytes (\d+)-(\d+)\/(\d+)/.exec(response.ContentRange ?? '')
|
|
64
|
+
if (span != null) {
|
|
65
|
+
opened.range = { start: Number(span[1]), end: Number(span[2]), total: Number(span[3]) }
|
|
66
|
+
opened.info.size = Number(span[3])
|
|
67
|
+
}
|
|
68
|
+
return opened
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const ranges = async (options: SignalOptions = {}): Promise<RangeReader> => {
|
|
72
|
+
const current = await info(options)
|
|
73
|
+
const size = current.size ?? 0
|
|
74
|
+
return {
|
|
75
|
+
size,
|
|
76
|
+
read: async (offset, length, signal) => {
|
|
77
|
+
const end = Math.min(offset + length, size) - 1
|
|
78
|
+
if (end < offset) {
|
|
79
|
+
return new Uint8Array(0)
|
|
80
|
+
}
|
|
81
|
+
const response = await get(`bytes=${offset}-${end}`, current.etag, signal)
|
|
82
|
+
return await streamUtils.collect(response.Body as Readable, end - offset + 1)
|
|
83
|
+
},
|
|
84
|
+
stream: async (start, end, signal) => (await get(`bytes=${start}-${end - 1}`, current.etag, signal)).Body as Readable,
|
|
85
|
+
close: async () => undefined,
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
return { kind: 's3', info, open, ranges }
|
|
90
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { S3Client } from '@aws-sdk/client-s3'
|
|
2
|
+
import type { Readable } from 'node:stream'
|
|
3
|
+
import type { StorageBucketModel } from '@owlmeans/storage-resource'
|
|
4
|
+
import type { SignalOptions } from '../types.js'
|
|
5
|
+
import type { S3ListedObject } from './types.js'
|
|
6
|
+
|
|
7
|
+
/** What the bucket's parts share. */
|
|
8
|
+
export interface S3Core {
|
|
9
|
+
readonly model: StorageBucketModel
|
|
10
|
+
client: () => S3Client
|
|
11
|
+
keyOf: (prefix: string) => string
|
|
12
|
+
location: (key: string) => string
|
|
13
|
+
partSize: number
|
|
14
|
+
concurrency: number
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export interface S3UploadRequest {
|
|
18
|
+
key: string
|
|
19
|
+
stream: Readable
|
|
20
|
+
contentType: string
|
|
21
|
+
disposition: string
|
|
22
|
+
cacheControl?: string
|
|
23
|
+
metadata?: Record<string, string>
|
|
24
|
+
signal?: AbortSignal
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export interface S3UploadResult {
|
|
28
|
+
bytes: number
|
|
29
|
+
etag?: string
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface S3Uploader {
|
|
33
|
+
/** One object from a stream of unknown length. */
|
|
34
|
+
upload: (request: S3UploadRequest) => Promise<S3UploadResult>
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export interface S3PrefixOps {
|
|
38
|
+
list: (prefix: string, options?: SignalOptions) => Promise<S3ListedObject[]>
|
|
39
|
+
removePrefix: (prefix: string, options?: SignalOptions) => Promise<number>
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** The part of an SDK build-step handler's arguments the Content-MD5 middleware touches. */
|
|
43
|
+
export interface S3BuildArgs {
|
|
44
|
+
request: unknown
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export interface S3BuildHandler {
|
|
48
|
+
(args: S3BuildArgs): Promise<unknown>
|
|
49
|
+
}
|
package/src/s3/types.ts
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import type { StorageBucketModel } from '@owlmeans/storage-resource'
|
|
2
|
+
import type { StoredFileInstance } from '@owlmeans/storage-common'
|
|
3
|
+
import type { FileSink, FileSource, SignalOptions, SinkResult, TreeSink } from '../types.js'
|
|
4
|
+
|
|
5
|
+
export interface S3BucketOptions {
|
|
6
|
+
/** Multipart part size. Default 8 MiB, at least 5 MiB. */
|
|
7
|
+
partSize?: number
|
|
8
|
+
/** Parts in flight. Default 2. */
|
|
9
|
+
concurrency?: number
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export interface S3SinkOptions {
|
|
13
|
+
/** Overrides the type taken from the verdict or the source. */
|
|
14
|
+
contentType?: string
|
|
15
|
+
/** Default `inline`; active svg/html is written as `attachment` unless allowed. */
|
|
16
|
+
disposition?: 'inline' | 'attachment'
|
|
17
|
+
allowActiveInline?: boolean
|
|
18
|
+
cacheControl?: string
|
|
19
|
+
metadata?: Record<string, string>
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export interface S3ObjectInfo {
|
|
23
|
+
key: string
|
|
24
|
+
size: number
|
|
25
|
+
etag?: string
|
|
26
|
+
mime?: string
|
|
27
|
+
modifiedAt?: Date
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** One object of a prefix listing (a listing carries no media type). */
|
|
31
|
+
export interface S3ListedObject extends S3ObjectInfo {
|
|
32
|
+
/** The key relative to the listed prefix — `media/a.png` for `<basePrefix>/<prefix>/media/a.png`. */
|
|
33
|
+
path: string
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* One storage bucket as sources and sinks, addressed exactly as `@owlmeans/storage-resource`
|
|
38
|
+
* addresses it: a `prefix` lands at `<basePrefix>/<prefix>` and answers on the same public URL.
|
|
39
|
+
*/
|
|
40
|
+
export interface S3Bucket {
|
|
41
|
+
readonly model: StorageBucketModel
|
|
42
|
+
/** The object key of a prefix (normalized; `..` and absolute prefixes refused). */
|
|
43
|
+
keyOf: (prefix: string) => string
|
|
44
|
+
/** The public URL of a prefix. */
|
|
45
|
+
urlOf: (prefix: string) => string
|
|
46
|
+
/** GetObject; random access through Range with If-Match. */
|
|
47
|
+
source: (prefix: string) => FileSource
|
|
48
|
+
/** PutObject when the body fits one part, a multipart upload (aborted on failure) otherwise. */
|
|
49
|
+
sink: (prefix: string, options?: S3SinkOptions) => FileSink
|
|
50
|
+
/** Many objects under one prefix. */
|
|
51
|
+
tree: (prefix: string, options?: S3SinkOptions) => TreeSink
|
|
52
|
+
head: (prefix: string, options?: SignalOptions) => Promise<S3ObjectInfo | null>
|
|
53
|
+
remove: (prefix: string, options?: SignalOptions) => Promise<void>
|
|
54
|
+
/**
|
|
55
|
+
* Every object under `<prefix>/` — ListObjectsV2, all pages — with its full `key` and its `path`
|
|
56
|
+
* relative to the prefix. An empty prefix is refused (`PathError('empty')`), never the whole bucket.
|
|
57
|
+
*/
|
|
58
|
+
list: (prefix: string, options?: SignalOptions) => Promise<S3ListedObject[]>
|
|
59
|
+
/**
|
|
60
|
+
* Deletes every object under `<prefix>/` — DeleteObjects in batches of at most 1000 keys, each
|
|
61
|
+
* checksummed by Content-MD5 (the SDK's MD5 fallback, which S3-compatible stores accept) — and
|
|
62
|
+
* answers how many went.
|
|
63
|
+
* @throws {S3TransferError} `delete-objects:<code>` when the store refuses any key of a batch.
|
|
64
|
+
*/
|
|
65
|
+
removePrefix: (prefix: string, options?: SignalOptions) => Promise<number>
|
|
66
|
+
/** A sink result as a storage-common file instance. */
|
|
67
|
+
instanceOf: (result: SinkResult, alias: string) => StoredFileInstance
|
|
68
|
+
/** Releases the client. */
|
|
69
|
+
close: () => void
|
|
70
|
+
}
|