@owlmeans/file-streams 0.2.0-alpha.0 → 0.2.1-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +437 -21
- package/agent-meta/manifest.json +77 -8
- package/agent-meta/skills/file-streams/SKILL.md +84 -105
- package/agent-meta/skills/file-streams/parts/archives.md +80 -0
- package/agent-meta/skills/file-streams/parts/extending.md +117 -0
- package/agent-meta/skills/file-streams/parts/network.md +77 -22
- package/agent-meta/skills/file-streams/parts/workload.md +51 -0
- package/agent-meta/skills/file-streams/parts/workspaces.md +9 -22
- package/build/errors.d.ts +5 -0
- package/build/errors.d.ts.map +1 -1
- package/build/errors.js +5 -0
- package/build/errors.js.map +1 -1
- package/build/fetch.d.ts +12 -0
- package/build/fetch.d.ts.map +1 -0
- package/build/fetch.js +6 -0
- package/build/fetch.js.map +1 -0
- package/build/http/types.d.ts +19 -1
- package/build/http/types.d.ts.map +1 -1
- package/build/http.d.ts.map +1 -1
- package/build/http.js +7 -4
- package/build/http.js.map +1 -1
- package/build/index.d.ts +2 -0
- package/build/index.d.ts.map +1 -1
- package/build/index.js +1 -0
- package/build/index.js.map +1 -1
- package/build/pinned-fetch/consts.local.d.ts +9 -0
- package/build/pinned-fetch/consts.local.d.ts.map +1 -0
- package/build/pinned-fetch/consts.local.js +13 -0
- package/build/pinned-fetch/consts.local.js.map +1 -0
- package/build/pinned-fetch/types.d.ts +16 -0
- package/build/pinned-fetch/types.d.ts.map +1 -0
- package/build/pinned-fetch/types.js +2 -0
- package/build/pinned-fetch/types.js.map +1 -0
- package/build/pinned-fetch.d.ts +9 -0
- package/build/pinned-fetch.d.ts.map +1 -0
- package/build/pinned-fetch.js +82 -0
- package/build/pinned-fetch.js.map +1 -0
- package/package.json +18 -11
- package/src/errors.ts +6 -0
- package/src/fetch.ts +14 -0
- package/src/http/types.ts +21 -1
- package/src/http.ts +7 -4
- package/src/index.ts +2 -0
- package/src/pinned-fetch/consts.local.ts +15 -0
- package/src/pinned-fetch/types.ts +17 -0
- package/src/pinned-fetch.ts +93 -0
- package/tests/fetch.spec.ts +163 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@owlmeans/file-streams",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.1-alpha.0",
|
|
4
4
|
"description": "Server-side file streaming: URL downloads with connect-time SSRF pinning, HTTP/SOCKS proxy plugins and a Google Drive resolver; S3, filesystem, buffer and Viable-workload destinations; zip/tar/gzip with traversal and bomb guards; file-type verification without full downloads.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"homepage": "https://owlmeans.com/docs/",
|
|
@@ -30,26 +30,33 @@
|
|
|
30
30
|
"default": "./build/index.js",
|
|
31
31
|
"module": "./build/index.js",
|
|
32
32
|
"types": "./build/index.d.ts"
|
|
33
|
+
},
|
|
34
|
+
"./fetch": {
|
|
35
|
+
"import": "./build/fetch.js",
|
|
36
|
+
"require": "./build/fetch.js",
|
|
37
|
+
"default": "./build/fetch.js",
|
|
38
|
+
"module": "./build/fetch.js",
|
|
39
|
+
"types": "./build/fetch.d.ts"
|
|
33
40
|
}
|
|
34
41
|
},
|
|
35
42
|
"dependencies": {
|
|
36
43
|
"@aws-sdk/client-s3": "^3.1158.0",
|
|
37
|
-
"@owlmeans/context": "^0.2.
|
|
38
|
-
"@owlmeans/error": "^0.2.
|
|
39
|
-
"@owlmeans/file-inspector": "^0.2.
|
|
40
|
-
"@owlmeans/log": "^0.2.
|
|
41
|
-
"@owlmeans/storage-common": "^0.2.
|
|
42
|
-
"@owlmeans/storage-resource": "^0.2.
|
|
44
|
+
"@owlmeans/context": "^0.2.1-alpha.0",
|
|
45
|
+
"@owlmeans/error": "^0.2.1-alpha.0",
|
|
46
|
+
"@owlmeans/file-inspector": "^0.2.1-alpha.0",
|
|
47
|
+
"@owlmeans/log": "^0.2.1-alpha.0",
|
|
48
|
+
"@owlmeans/storage-common": "^0.2.1-alpha.0",
|
|
49
|
+
"@owlmeans/storage-resource": "^0.2.1-alpha.0",
|
|
43
50
|
"socks": "^2.8.10",
|
|
44
51
|
"tar-stream": "^3.2.2",
|
|
45
52
|
"yauzl": "^3.4.0",
|
|
46
53
|
"yazl": "^3.3.1"
|
|
47
54
|
},
|
|
48
55
|
"devDependencies": {
|
|
49
|
-
"@owlmeans/dep-config": "^0.2.
|
|
50
|
-
"@owlmeans/server-context": "^0.2.
|
|
51
|
-
"@owlmeans/test": "^0.2.
|
|
52
|
-
"@owlmeans/test-integration": "^0.2.
|
|
56
|
+
"@owlmeans/dep-config": "^0.2.1-alpha.0",
|
|
57
|
+
"@owlmeans/server-context": "^0.2.1-alpha.0",
|
|
58
|
+
"@owlmeans/test": "^0.2.1-alpha.0",
|
|
59
|
+
"@owlmeans/test-integration": "^0.2.1-alpha.0",
|
|
53
60
|
"@types/bun": "^1.4.4",
|
|
54
61
|
"@types/node": "^26.6.5",
|
|
55
62
|
"@types/yauzl": "^3.4.0",
|
package/src/errors.ts
CHANGED
|
@@ -13,6 +13,12 @@ export class FileStreamsError extends ResilientError {
|
|
|
13
13
|
export class NetworkPolicyError extends FileStreamsError {
|
|
14
14
|
public static override typeName: string = `${FileStreamsError.typeName}NetworkPolicy`
|
|
15
15
|
|
|
16
|
+
/**
|
|
17
|
+
* Marks a refusal, not a network failure — what a consumer of the pinned fetch (`./fetch`) reads
|
|
18
|
+
* to tell "the target may not be reached" apart from "the target could not be reached".
|
|
19
|
+
*/
|
|
20
|
+
public readonly refused: boolean = true
|
|
21
|
+
|
|
16
22
|
constructor(message: string = 'error') {
|
|
17
23
|
super(`policy:${message}`)
|
|
18
24
|
this.type = NetworkPolicyError.typeName
|
package/src/fetch.ts
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
// The aws-free subpath `@owlmeans/file-streams/fetch`: a pinned WHATWG `fetch` over the HTTP client,
|
|
2
|
+
// with the policy, proxy and resolver vocabulary it is configured by. Nothing reachable from here
|
|
3
|
+
// imports the AWS SDK, storage-resource or the S3 modules (`tests/fetch.spec.ts` walks the graph).
|
|
4
|
+
export type * from './pinned-fetch/types.js'
|
|
5
|
+
export type { HttpClient, HttpClientOptions, HttpTimeouts } from './http/types.js'
|
|
6
|
+
export type { AddressResolver, NetworkPolicy, NetworkPolicyOptions, ResolvedAddress } from './network/types.js'
|
|
7
|
+
export type { ProxyRouter, ProxyRouterOptions } from './proxy/types.js'
|
|
8
|
+
export type { UrlResolver } from './resolvers/types.js'
|
|
9
|
+
export type { GoogleDriveResolverOptions } from './resolvers/google-drive/types.js'
|
|
10
|
+
export { createPinnedFetch } from './pinned-fetch.js'
|
|
11
|
+
export { createHttpClient } from './http.js'
|
|
12
|
+
export { createNetworkPolicy } from './network.js'
|
|
13
|
+
export { createGoogleDriveResolver, googleDriveResolver } from './resolvers/google-drive.js'
|
|
14
|
+
export { HttpError, NetworkPolicyError, ProxyError, ResolverError } from './errors.js'
|
package/src/http/types.ts
CHANGED
|
@@ -34,6 +34,13 @@ export interface HttpClientOptions {
|
|
|
34
34
|
// A closed pair of HTTP verbs — a union.
|
|
35
35
|
export type HttpMethod = 'GET' | 'HEAD'
|
|
36
36
|
|
|
37
|
+
/**
|
|
38
|
+
* What a 3xx answer does: `follow` (default) goes on to its `Location` under the policy; `manual`
|
|
39
|
+
* returns the 3xx response itself, `Location` included, for a caller that checks every hop itself.
|
|
40
|
+
*/
|
|
41
|
+
// A closed pair of modes — a union.
|
|
42
|
+
export type HttpRedirectMode = 'follow' | 'manual'
|
|
43
|
+
|
|
37
44
|
export interface HttpRequestOptions extends SignalOptions {
|
|
38
45
|
method?: HttpMethod
|
|
39
46
|
headers?: Record<string, string>
|
|
@@ -44,6 +51,17 @@ export interface HttpRequestOptions extends SignalOptions {
|
|
|
44
51
|
raw?: boolean
|
|
45
52
|
/** Send `If-Range` with this strong validator. */
|
|
46
53
|
ifRange?: string
|
|
54
|
+
/**
|
|
55
|
+
* Default `follow`. `manual` returns a 3xx answer (with its `Location`) instead of following it.
|
|
56
|
+
* A resolver's own flow (a Drive virus-scan page, its hops) still runs inside the client under the
|
|
57
|
+
* policy, because it carries its cookie across those hops.
|
|
58
|
+
*/
|
|
59
|
+
redirect?: HttpRedirectMode
|
|
60
|
+
/**
|
|
61
|
+
* Return every status instead of throwing `HttpError('status:<code>')` — a 404 page is an answer.
|
|
62
|
+
* A forward proxy's 407 still throws `ProxyError('auth-failed')`: that is the proxy, not the target.
|
|
63
|
+
*/
|
|
64
|
+
anyStatus?: boolean
|
|
47
65
|
}
|
|
48
66
|
|
|
49
67
|
export interface HttpResponse {
|
|
@@ -55,6 +73,8 @@ export interface HttpResponse {
|
|
|
55
73
|
body: Readable
|
|
56
74
|
info: FileInfo
|
|
57
75
|
redirects: number
|
|
76
|
+
/** The body was transport-decoded (`Content-Encoding` undone); its `Content-Length` no longer applies. */
|
|
77
|
+
decoded?: boolean
|
|
58
78
|
}
|
|
59
79
|
|
|
60
80
|
export interface HttpSourceOptions {
|
|
@@ -67,7 +87,7 @@ export interface HttpSourceOptions {
|
|
|
67
87
|
|
|
68
88
|
/** Downloads under the network policy, through proxies, redirects and resolvers. */
|
|
69
89
|
export interface HttpClient {
|
|
70
|
-
/** One GET or HEAD; a non-2xx answer is `HttpError('status:<code>')
|
|
90
|
+
/** One GET or HEAD; a non-2xx answer is `HttpError('status:<code>')` unless `anyStatus` (or a 3xx under `redirect: 'manual'`). */
|
|
71
91
|
request: (url: string | URL, options?: HttpRequestOptions) => Promise<HttpResponse>
|
|
72
92
|
/** A re-openable source; Range-capable when the server answers 206 with Content-Range. */
|
|
73
93
|
source: (url: string | URL, options?: HttpSourceOptions) => FileSource
|
package/src/http.ts
CHANGED
|
@@ -195,7 +195,9 @@ export const createHttpClient = (options: HttpClientOptions = {}): HttpClient =>
|
|
|
195
195
|
const status = response.statusCode ?? 0
|
|
196
196
|
const location = header(response.headers, 'location')
|
|
197
197
|
|
|
198
|
-
|
|
198
|
+
// A resolver's flow carries its cookie across hops, so it follows them itself even under `manual`.
|
|
199
|
+
const returned = REDIRECT_STATUSES.includes(status) && location != null && requestOptions.redirect === 'manual' && resolver == null
|
|
200
|
+
if (REDIRECT_STATUSES.includes(status) && location != null && !returned) {
|
|
199
201
|
response.resume()
|
|
200
202
|
response.destroy()
|
|
201
203
|
if (++redirects > maxRedirects) {
|
|
@@ -214,7 +216,8 @@ export const createHttpClient = (options: HttpClientOptions = {}): HttpClient =>
|
|
|
214
216
|
current = next
|
|
215
217
|
continue
|
|
216
218
|
}
|
|
217
|
-
|
|
219
|
+
const success = status >= 200 && status < 300
|
|
220
|
+
if (!success && !returned && (requestOptions.anyStatus !== true || (status === 407 && connection.mode === 'forward'))) {
|
|
218
221
|
response.resume()
|
|
219
222
|
response.destroy()
|
|
220
223
|
if (status === 407 && connection.mode === 'forward') {
|
|
@@ -227,7 +230,7 @@ export const createHttpClient = (options: HttpClientOptions = {}): HttpClient =>
|
|
|
227
230
|
? { body: response, decoded: false }
|
|
228
231
|
: bodyOf(current, response)
|
|
229
232
|
|
|
230
|
-
if (resolver != null && method === 'GET') {
|
|
233
|
+
if (resolver != null && method === 'GET' && success) {
|
|
231
234
|
let peeked: PeekedStream | undefined
|
|
232
235
|
const readBody = async (maxBytes: number = DEFAULT_RESOLVER_PAGE_BYTES): Promise<string> => {
|
|
233
236
|
peeked ??= await streamUtils.peek(body, maxBytes)
|
|
@@ -258,7 +261,7 @@ export const createHttpClient = (options: HttpClientOptions = {}): HttpClient =>
|
|
|
258
261
|
|
|
259
262
|
log.debug('Response', { url: redactUtils.url(current), status, redirects })
|
|
260
263
|
|
|
261
|
-
return { url: current, status, headers: response.headers, body, info: infoOf(current, response, decoded), redirects }
|
|
264
|
+
return { url: current, status, headers: response.headers, body, info: infoOf(current, response, decoded), redirects, decoded }
|
|
262
265
|
}
|
|
263
266
|
}
|
|
264
267
|
|
package/src/index.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
export type * from './types.js'
|
|
2
2
|
export type * from './archive/types.js'
|
|
3
3
|
export type * from './http/types.js'
|
|
4
|
+
export type * from './pinned-fetch/types.js'
|
|
4
5
|
export type * from './network/types.js'
|
|
5
6
|
export type * from './proxy/types.js'
|
|
6
7
|
export type * from './resolvers/types.js'
|
|
@@ -30,6 +31,7 @@ export * from './errors.js'
|
|
|
30
31
|
export * from './archive.js'
|
|
31
32
|
export * from './network.js'
|
|
32
33
|
export * from './http.js'
|
|
34
|
+
export * from './pinned-fetch.js'
|
|
33
35
|
export * from './proxy/endpoint.js'
|
|
34
36
|
export * from './proxy/router.js'
|
|
35
37
|
export * from './proxy/http.js'
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/** Request headers the pinned fetch never forwards: hop-by-hop, the host it sets itself, and any encoding but identity. */
|
|
2
|
+
export const PINNED_FETCH_DROPPED_REQUEST_HEADERS: readonly string[] = Object.freeze([
|
|
3
|
+
'connection', 'keep-alive', 'proxy-connection', 'transfer-encoding', 'upgrade', 'te', 'host', 'accept-encoding', 'content-length',
|
|
4
|
+
])
|
|
5
|
+
|
|
6
|
+
/** Response headers that describe the connection, not the answer. */
|
|
7
|
+
export const PINNED_FETCH_DROPPED_RESPONSE_HEADERS: readonly string[] = Object.freeze([
|
|
8
|
+
'connection', 'keep-alive', 'proxy-connection', 'transfer-encoding', 'upgrade',
|
|
9
|
+
])
|
|
10
|
+
|
|
11
|
+
/** Statuses a `Response` must carry without a body. */
|
|
12
|
+
export const NULL_BODY_STATUSES: readonly number[] = Object.freeze([101, 103, 204, 205, 304])
|
|
13
|
+
|
|
14
|
+
/** The methods the pinned fetch carries. */
|
|
15
|
+
export const PINNED_FETCH_METHODS: readonly string[] = Object.freeze(['GET', 'HEAD'])
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { HttpClient, HttpClientOptions } from '../http/types.js'
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* A WHATWG-`fetch`-compatible function over the package's HTTP client: every connection is pinned
|
|
5
|
+
* to an address the network policy checked a moment before (no second lookup to rebind), every
|
|
6
|
+
* redirect hop is checked again, URL resolvers (Google Drive) apply, and bodies stream. Only GET and
|
|
7
|
+
* HEAD are carried; any status comes back as a `Response` (a 404 page is an answer), a refusal of the
|
|
8
|
+
* policy rejects with `NetworkPolicyError` (`refused === true`).
|
|
9
|
+
*/
|
|
10
|
+
export interface PinnedFetch {
|
|
11
|
+
(input: string | URL | Request, init?: RequestInit): Promise<Response>
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export interface PinnedFetchOptions extends HttpClientOptions {
|
|
15
|
+
/** A ready client instead of one built from these options. */
|
|
16
|
+
client?: HttpClient
|
|
17
|
+
}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { Readable } from 'node:stream'
|
|
2
|
+
import { HttpError } from './errors.js'
|
|
3
|
+
import {
|
|
4
|
+
NULL_BODY_STATUSES, PINNED_FETCH_DROPPED_REQUEST_HEADERS, PINNED_FETCH_DROPPED_RESPONSE_HEADERS, PINNED_FETCH_METHODS,
|
|
5
|
+
} from './pinned-fetch/consts.local.js'
|
|
6
|
+
import type { PinnedFetch, PinnedFetchOptions } from './pinned-fetch/types.js'
|
|
7
|
+
import { createHttpClient } from './http.js'
|
|
8
|
+
import { REDIRECT_STATUSES } from './http/consts.js'
|
|
9
|
+
import type { HttpMethod } from './http/types.js'
|
|
10
|
+
import { signalUtils } from './utils/signals.js'
|
|
11
|
+
import { streamUtils } from './utils/streams.js'
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* A `fetch` whose connections are pinned at connect time — hand it to code written against the
|
|
15
|
+
* WHATWG API (`@owlmeans/llm-reader`, a renderer) so its traffic gets the policy, the proxies and
|
|
16
|
+
* the resolvers of {@link createHttpClient}. This module and its imports never load the AWS SDK or
|
|
17
|
+
* the S3 modules: import it from `@owlmeans/file-streams/fetch`.
|
|
18
|
+
*/
|
|
19
|
+
export const createPinnedFetch = (options: PinnedFetchOptions = {}): PinnedFetch => {
|
|
20
|
+
const { client: given, ...clientOptions } = options
|
|
21
|
+
const client = given ?? createHttpClient(clientOptions)
|
|
22
|
+
|
|
23
|
+
const headersOf = (input: string | URL | Request, init?: RequestInit): Record<string, string> => {
|
|
24
|
+
const merged = new Headers(input instanceof Request ? input.headers : undefined)
|
|
25
|
+
new Headers(init?.headers).forEach((value, name) => merged.set(name, value))
|
|
26
|
+
const headers: Record<string, string> = {}
|
|
27
|
+
merged.forEach((value, name) => {
|
|
28
|
+
if (!PINNED_FETCH_DROPPED_REQUEST_HEADERS.includes(name)) {
|
|
29
|
+
headers[name] = value
|
|
30
|
+
}
|
|
31
|
+
})
|
|
32
|
+
|
|
33
|
+
return headers
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
return async (input, init) => {
|
|
37
|
+
const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url
|
|
38
|
+
const method = (init?.method ?? (input instanceof Request ? input.method : 'GET')).toUpperCase()
|
|
39
|
+
if (!PINNED_FETCH_METHODS.includes(method)) {
|
|
40
|
+
throw new HttpError(`method:${method}`)
|
|
41
|
+
}
|
|
42
|
+
if (init?.body != null) {
|
|
43
|
+
throw new HttpError('body')
|
|
44
|
+
}
|
|
45
|
+
const mode = init?.redirect ?? (input instanceof Request ? input.redirect : 'follow')
|
|
46
|
+
const signal = init?.signal ?? (input instanceof Request ? input.signal : undefined) ?? undefined
|
|
47
|
+
signalUtils.throwIfAborted(signal)
|
|
48
|
+
|
|
49
|
+
const response = await client.request(url, {
|
|
50
|
+
method: method as HttpMethod,
|
|
51
|
+
headers: headersOf(input, init),
|
|
52
|
+
signal,
|
|
53
|
+
redirect: mode === 'follow' ? 'follow' : 'manual',
|
|
54
|
+
anyStatus: true,
|
|
55
|
+
})
|
|
56
|
+
const isRedirect = REDIRECT_STATUSES.includes(response.status) && response.headers.location != null
|
|
57
|
+
if (mode === 'error' && isRedirect) {
|
|
58
|
+
streamUtils.discard(response.body)
|
|
59
|
+
throw new HttpError('redirect')
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const headers = new Headers()
|
|
63
|
+
for (const [name, value] of Object.entries(response.headers)) {
|
|
64
|
+
if (value == null || PINNED_FETCH_DROPPED_RESPONSE_HEADERS.includes(name)) {
|
|
65
|
+
continue
|
|
66
|
+
}
|
|
67
|
+
if (response.decoded === true && (name === 'content-encoding' || name === 'content-length')) {
|
|
68
|
+
continue
|
|
69
|
+
}
|
|
70
|
+
for (const one of Array.isArray(value) ? value : [value]) {
|
|
71
|
+
headers.append(name, one)
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
let body: ReadableStream<Uint8Array> | null = null
|
|
76
|
+
if (method === 'HEAD' || NULL_BODY_STATUSES.includes(response.status)) {
|
|
77
|
+
streamUtils.discard(response.body)
|
|
78
|
+
} else {
|
|
79
|
+
// The client stops listening once the head arrived; an abort must still end the body.
|
|
80
|
+
const release = signalUtils.onAbort(signal, reason => response.body.destroy(reason as Error))
|
|
81
|
+
response.body.once('close', release)
|
|
82
|
+
body = Readable.toWeb(response.body) as unknown as ReadableStream<Uint8Array>
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const answer = new Response(body, { status: response.status, headers })
|
|
86
|
+
Object.defineProperty(answer, 'url', { value: response.url.href })
|
|
87
|
+
if (response.redirects > 0) {
|
|
88
|
+
Object.defineProperty(answer, 'redirected', { value: true })
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
return answer
|
|
92
|
+
}
|
|
93
|
+
}
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
import { afterAll, beforeAll, describe, expect, test } from 'bun:test'
|
|
2
|
+
import { readFileSync } from 'node:fs'
|
|
3
|
+
import { dirname, join, resolve } from 'node:path'
|
|
4
|
+
import { fileURLToPath } from 'node:url'
|
|
5
|
+
import { createGoogleDriveResolver, createPinnedFetch, NetworkPolicyError } from '../src/fetch.js'
|
|
6
|
+
import { noise, PNG } from './fixtures.js'
|
|
7
|
+
import { startFakeDrive } from './harness/drive.js'
|
|
8
|
+
import type { FakeDrive } from './harness/drive.js'
|
|
9
|
+
import { startOrigin } from './harness/origin.js'
|
|
10
|
+
import type { Origin } from './harness/origin.js'
|
|
11
|
+
import { LOCAL_NETWORK, localClient } from './context.js'
|
|
12
|
+
|
|
13
|
+
const TIMEOUTS = { connectMs: 3_000, headersMs: 3_000, idleMs: 3_000 }
|
|
14
|
+
const BUILD = fileURLToPath(new URL('../build/', import.meta.url))
|
|
15
|
+
|
|
16
|
+
/** Every module `entry` reaches by relative import, and every bare specifier those modules name. */
|
|
17
|
+
const importGraph = (entry: string): { files: string[], bare: Set<string> } => {
|
|
18
|
+
const files: string[] = []
|
|
19
|
+
const bare = new Set<string>()
|
|
20
|
+
const visit = (file: string): void => {
|
|
21
|
+
if (files.includes(file)) return
|
|
22
|
+
files.push(file)
|
|
23
|
+
const source = readFileSync(file, 'utf8')
|
|
24
|
+
for (const match of source.matchAll(/(?:import|export)\s[^'"]*?from\s+['"]([^'"]+)['"]|import\s*\(\s*['"]([^'"]+)['"]\s*\)|import\s+['"]([^'"]+)['"]/g)) {
|
|
25
|
+
const specifier = match[1] ?? match[2] ?? match[3]!
|
|
26
|
+
if (specifier.startsWith('.')) visit(resolve(dirname(file), specifier))
|
|
27
|
+
else bare.add(specifier)
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
visit(entry)
|
|
31
|
+
return { files, bare }
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
describe('@owlmeans/file-streams — pinned fetch', () => {
|
|
35
|
+
let origin: Origin
|
|
36
|
+
let foreign: Origin
|
|
37
|
+
let drive: FakeDrive
|
|
38
|
+
const big = noise(6 * 1024 * 1024, 3)
|
|
39
|
+
beforeAll(() => {
|
|
40
|
+
origin = startOrigin()
|
|
41
|
+
foreign = startOrigin()
|
|
42
|
+
origin.files.set('big.bin', { bytes: big })
|
|
43
|
+
origin.files.set('page.html', { bytes: Buffer.from('<h1>Hi</h1>'), type: 'text/html; charset=utf-8' })
|
|
44
|
+
drive = startFakeDrive({ small: PNG, large: noise(300_000), docx: PNG, xlsx: PNG, pptx: PNG }, foreign)
|
|
45
|
+
})
|
|
46
|
+
afterAll(() => {
|
|
47
|
+
origin.stop()
|
|
48
|
+
foreign.stop()
|
|
49
|
+
drive.stop()
|
|
50
|
+
})
|
|
51
|
+
|
|
52
|
+
const pinned = () => createPinnedFetch({ network: LOCAL_NETWORK, resolvers: [], timeouts: TIMEOUTS })
|
|
53
|
+
|
|
54
|
+
test('answers like fetch: status, headers, url and body; any status is a Response', async () => {
|
|
55
|
+
const page = await pinned()(origin.url('/files/page.html'), { headers: { 'x-trace': 't1' } })
|
|
56
|
+
expect(page.status).toBe(200)
|
|
57
|
+
expect(page.headers.get('content-type')).toBe('text/html; charset=utf-8')
|
|
58
|
+
expect(page.url).toBe(origin.url('/files/page.html'))
|
|
59
|
+
expect(await page.text()).toBe('<h1>Hi</h1>')
|
|
60
|
+
expect(origin.requests.at(-1)!.headers['x-trace']).toBe('t1')
|
|
61
|
+
|
|
62
|
+
const missing = await pinned()(origin.url('/status/404'))
|
|
63
|
+
expect(missing.status).toBe(404)
|
|
64
|
+
expect(await missing.text()).toBe('status page')
|
|
65
|
+
const head = await pinned()(origin.url('/files/page.html'), { method: 'HEAD' })
|
|
66
|
+
expect(head.status).toBe(200)
|
|
67
|
+
expect(head.body).toBeNull()
|
|
68
|
+
await expect(pinned()(origin.url('/files/page.html'), { method: 'POST' })).rejects.toThrow('http:method:POST')
|
|
69
|
+
})
|
|
70
|
+
|
|
71
|
+
test('redirect: manual returns the 3xx with its Location; follow (the default) goes on', async () => {
|
|
72
|
+
const hop = await pinned()(origin.url('/redirect/2'), { redirect: 'manual' })
|
|
73
|
+
expect(hop.status).toBe(302)
|
|
74
|
+
expect(hop.headers.get('location')).toBe('/redirect/1')
|
|
75
|
+
expect(hop.redirected).toBe(false)
|
|
76
|
+
|
|
77
|
+
const followed = await pinned()(origin.url('/redirect/2'))
|
|
78
|
+
expect(followed.status).toBe(200)
|
|
79
|
+
expect(followed.redirected).toBe(true)
|
|
80
|
+
expect(followed.url).toBe(origin.url('/redirect/0'))
|
|
81
|
+
expect(await followed.text()).toBe('arrived')
|
|
82
|
+
await expect(pinned()(origin.url('/redirect/1'), { redirect: 'error' })).rejects.toThrow('http:redirect')
|
|
83
|
+
|
|
84
|
+
// The client itself: a 3xx returned under `manual`, any status under `anyStatus`.
|
|
85
|
+
const client = localClient()
|
|
86
|
+
const returned = await client.request(origin.url('/redirect/1'), { redirect: 'manual' })
|
|
87
|
+
expect([returned.status, returned.headers.location, returned.redirects]).toEqual([302, '/redirect/0', 0])
|
|
88
|
+
await expect(client.request(origin.url('/status/404'))).rejects.toThrow('http:status:404')
|
|
89
|
+
expect((await client.request(origin.url('/status/404'), { anyStatus: true })).status).toBe(404)
|
|
90
|
+
})
|
|
91
|
+
|
|
92
|
+
test('private space is refused unless allowed — pinned at connect, so a name rebound to loopback is refused too', async () => {
|
|
93
|
+
const strict = createPinnedFetch({ resolvers: [], timeouts: TIMEOUTS })
|
|
94
|
+
const refusal = await strict(origin.url('/files/page.html')).catch((error: unknown) => error)
|
|
95
|
+
expect(refusal).toBeInstanceOf(NetworkPolicyError)
|
|
96
|
+
expect((refusal as NetworkPolicyError).refused).toBe(true)
|
|
97
|
+
|
|
98
|
+
// A name whose resolution turns private at connect time: the policy checks the address it dials.
|
|
99
|
+
const rebound = createPinnedFetch({
|
|
100
|
+
resolvers: [], timeouts: TIMEOUTS, network: { resolver: async () => [{ address: '127.0.0.1', family: 4 }] },
|
|
101
|
+
})
|
|
102
|
+
const port = new URL(origin.base).port
|
|
103
|
+
await expect(rebound(`http://public.example:${port}/files/page.html`)).rejects.toBeInstanceOf(NetworkPolicyError)
|
|
104
|
+
const allowed = createPinnedFetch({
|
|
105
|
+
resolvers: [], timeouts: TIMEOUTS, network: { ...LOCAL_NETWORK, resolver: async () => [{ address: '127.0.0.1', family: 4 }] },
|
|
106
|
+
})
|
|
107
|
+
expect(await (await allowed(`http://public.example:${port}/files/page.html`)).text()).toBe('<h1>Hi</h1>')
|
|
108
|
+
})
|
|
109
|
+
|
|
110
|
+
test('the Google Drive resolver applies through the fetch', async () => {
|
|
111
|
+
const fetch = createPinnedFetch({
|
|
112
|
+
network: LOCAL_NETWORK, timeouts: TIMEOUTS,
|
|
113
|
+
resolvers: [createGoogleDriveResolver({ origins: { drive: drive.base, usercontent: drive.base, docs: drive.base, accounts: drive.accounts } })],
|
|
114
|
+
})
|
|
115
|
+
const small = await fetch(`${drive.base}/file/d/SMALLFILE0001/view?usp=sharing`)
|
|
116
|
+
expect(small.headers.get('content-disposition')).toContain('small.png')
|
|
117
|
+
expect(new Uint8Array(await small.arrayBuffer())).toEqual(new Uint8Array(PNG))
|
|
118
|
+
// The virus-scan page of a large file is followed inside the resolver, even for a manual caller.
|
|
119
|
+
const large = await fetch(`${drive.base}/file/d/LARGEFILE0001/view`, { redirect: 'manual' })
|
|
120
|
+
expect(new Uint8Array(await large.arrayBuffer())).toEqual(new Uint8Array(noise(300_000)))
|
|
121
|
+
})
|
|
122
|
+
|
|
123
|
+
test('a multi-megabyte body streams in chunks', async () => {
|
|
124
|
+
const response = await pinned()(origin.url('/files/big.bin'))
|
|
125
|
+
expect(Number(response.headers.get('content-length'))).toBe(big.length)
|
|
126
|
+
const reader = response.body!.getReader()
|
|
127
|
+
let size = 0
|
|
128
|
+
let chunks = 0
|
|
129
|
+
for (;;) {
|
|
130
|
+
const { done, value } = await reader.read()
|
|
131
|
+
if (done) break
|
|
132
|
+
expect(Buffer.from(value).equals(big.subarray(size, size + value.byteLength))).toBe(true)
|
|
133
|
+
size += value.byteLength
|
|
134
|
+
chunks++
|
|
135
|
+
}
|
|
136
|
+
expect(size).toBe(big.length)
|
|
137
|
+
expect(chunks).toBeGreaterThan(1)
|
|
138
|
+
})
|
|
139
|
+
|
|
140
|
+
test('an abort ends a request waiting for its head and a body in flight', async () => {
|
|
141
|
+
const early = new AbortController()
|
|
142
|
+
const waiting = pinned()(origin.url('/stall-headers'), { signal: early.signal })
|
|
143
|
+
setTimeout(() => early.abort(new Error('stop-early')), 100)
|
|
144
|
+
await expect(waiting).rejects.toThrow('stop-early')
|
|
145
|
+
|
|
146
|
+
const late = new AbortController()
|
|
147
|
+
const response = await pinned()(origin.url('/stall-body'), { signal: late.signal })
|
|
148
|
+
const reader = response.body!.getReader()
|
|
149
|
+
expect([...(await reader.read()).value!]).toEqual([1, 2, 3])
|
|
150
|
+
late.abort(new Error('stop-late'))
|
|
151
|
+
await expect(reader.read()).rejects.toThrow()
|
|
152
|
+
})
|
|
153
|
+
|
|
154
|
+
test('the ./fetch subpath never loads the AWS SDK, storage-resource or the S3 modules', () => {
|
|
155
|
+
const { files, bare } = importGraph(join(BUILD, 'fetch.js'))
|
|
156
|
+
expect(files.length).toBeGreaterThan(5)
|
|
157
|
+
expect(files.some(file => /[\\/](s3|service|backed-workspace|workspace|transfer|archive)(\.js|[\\/])/.test(file.slice(BUILD.length)))).toBe(false)
|
|
158
|
+
expect([...bare].filter(name => name.startsWith('@aws-sdk/') || name.startsWith('@owlmeans/storage-resource'))).toEqual([])
|
|
159
|
+
expect([...bare].filter(name => !name.startsWith('node:')).sort()).toEqual(
|
|
160
|
+
['@owlmeans/error', '@owlmeans/file-inspector', '@owlmeans/log', '@owlmeans/storage-common', 'socks'],
|
|
161
|
+
)
|
|
162
|
+
})
|
|
163
|
+
})
|