@owlmeans/file-streams 0.2.0-alpha.0 → 0.2.1-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/README.md +437 -21
  2. package/agent-meta/manifest.json +77 -8
  3. package/agent-meta/skills/file-streams/SKILL.md +84 -105
  4. package/agent-meta/skills/file-streams/parts/archives.md +80 -0
  5. package/agent-meta/skills/file-streams/parts/extending.md +117 -0
  6. package/agent-meta/skills/file-streams/parts/network.md +77 -22
  7. package/agent-meta/skills/file-streams/parts/workload.md +51 -0
  8. package/agent-meta/skills/file-streams/parts/workspaces.md +9 -22
  9. package/build/errors.d.ts +5 -0
  10. package/build/errors.d.ts.map +1 -1
  11. package/build/errors.js +5 -0
  12. package/build/errors.js.map +1 -1
  13. package/build/fetch.d.ts +12 -0
  14. package/build/fetch.d.ts.map +1 -0
  15. package/build/fetch.js +6 -0
  16. package/build/fetch.js.map +1 -0
  17. package/build/http/types.d.ts +19 -1
  18. package/build/http/types.d.ts.map +1 -1
  19. package/build/http.d.ts.map +1 -1
  20. package/build/http.js +7 -4
  21. package/build/http.js.map +1 -1
  22. package/build/index.d.ts +2 -0
  23. package/build/index.d.ts.map +1 -1
  24. package/build/index.js +1 -0
  25. package/build/index.js.map +1 -1
  26. package/build/pinned-fetch/consts.local.d.ts +9 -0
  27. package/build/pinned-fetch/consts.local.d.ts.map +1 -0
  28. package/build/pinned-fetch/consts.local.js +13 -0
  29. package/build/pinned-fetch/consts.local.js.map +1 -0
  30. package/build/pinned-fetch/types.d.ts +16 -0
  31. package/build/pinned-fetch/types.d.ts.map +1 -0
  32. package/build/pinned-fetch/types.js +2 -0
  33. package/build/pinned-fetch/types.js.map +1 -0
  34. package/build/pinned-fetch.d.ts +9 -0
  35. package/build/pinned-fetch.d.ts.map +1 -0
  36. package/build/pinned-fetch.js +82 -0
  37. package/build/pinned-fetch.js.map +1 -0
  38. package/package.json +18 -11
  39. package/src/errors.ts +6 -0
  40. package/src/fetch.ts +14 -0
  41. package/src/http/types.ts +21 -1
  42. package/src/http.ts +7 -4
  43. package/src/index.ts +2 -0
  44. package/src/pinned-fetch/consts.local.ts +15 -0
  45. package/src/pinned-fetch/types.ts +17 -0
  46. package/src/pinned-fetch.ts +93 -0
  47. package/tests/fetch.spec.ts +163 -0
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@owlmeans/file-streams",
3
- "version": "0.2.0-alpha.0",
3
+ "version": "0.2.1-alpha.0",
4
4
  "description": "Server-side file streaming: URL downloads with connect-time SSRF pinning, HTTP/SOCKS proxy plugins and a Google Drive resolver; S3, filesystem, buffer and Viable-workload destinations; zip/tar/gzip with traversal and bomb guards; file-type verification without full downloads.",
5
5
  "license": "MIT",
6
6
  "homepage": "https://owlmeans.com/docs/",
@@ -30,26 +30,33 @@
30
30
  "default": "./build/index.js",
31
31
  "module": "./build/index.js",
32
32
  "types": "./build/index.d.ts"
33
+ },
34
+ "./fetch": {
35
+ "import": "./build/fetch.js",
36
+ "require": "./build/fetch.js",
37
+ "default": "./build/fetch.js",
38
+ "module": "./build/fetch.js",
39
+ "types": "./build/fetch.d.ts"
33
40
  }
34
41
  },
35
42
  "dependencies": {
36
43
  "@aws-sdk/client-s3": "^3.1158.0",
37
- "@owlmeans/context": "^0.2.0-alpha.0",
38
- "@owlmeans/error": "^0.2.0-alpha.0",
39
- "@owlmeans/file-inspector": "^0.2.0-alpha.0",
40
- "@owlmeans/log": "^0.2.0-alpha.0",
41
- "@owlmeans/storage-common": "^0.2.0-alpha.1",
42
- "@owlmeans/storage-resource": "^0.2.0-alpha.1",
44
+ "@owlmeans/context": "^0.2.1-alpha.0",
45
+ "@owlmeans/error": "^0.2.1-alpha.0",
46
+ "@owlmeans/file-inspector": "^0.2.1-alpha.0",
47
+ "@owlmeans/log": "^0.2.1-alpha.0",
48
+ "@owlmeans/storage-common": "^0.2.1-alpha.0",
49
+ "@owlmeans/storage-resource": "^0.2.1-alpha.0",
43
50
  "socks": "^2.8.10",
44
51
  "tar-stream": "^3.2.2",
45
52
  "yauzl": "^3.4.0",
46
53
  "yazl": "^3.3.1"
47
54
  },
48
55
  "devDependencies": {
49
- "@owlmeans/dep-config": "^0.2.0-alpha.0",
50
- "@owlmeans/server-context": "^0.2.0-alpha.1",
51
- "@owlmeans/test": "^0.2.0-alpha.1",
52
- "@owlmeans/test-integration": "^0.2.0-alpha.1",
56
+ "@owlmeans/dep-config": "^0.2.1-alpha.0",
57
+ "@owlmeans/server-context": "^0.2.1-alpha.0",
58
+ "@owlmeans/test": "^0.2.1-alpha.0",
59
+ "@owlmeans/test-integration": "^0.2.1-alpha.0",
53
60
  "@types/bun": "^1.4.4",
54
61
  "@types/node": "^26.6.5",
55
62
  "@types/yauzl": "^3.4.0",
package/src/errors.ts CHANGED
@@ -13,6 +13,12 @@ export class FileStreamsError extends ResilientError {
13
13
  export class NetworkPolicyError extends FileStreamsError {
14
14
  public static override typeName: string = `${FileStreamsError.typeName}NetworkPolicy`
15
15
 
16
+ /**
17
+ * Marks a refusal, not a network failure — what a consumer of the pinned fetch (`./fetch`) reads
18
+ * to tell "the target may not be reached" apart from "the target could not be reached".
19
+ */
20
+ public readonly refused: boolean = true
21
+
16
22
  constructor(message: string = 'error') {
17
23
  super(`policy:${message}`)
18
24
  this.type = NetworkPolicyError.typeName
package/src/fetch.ts ADDED
@@ -0,0 +1,14 @@
1
+ // The aws-free subpath `@owlmeans/file-streams/fetch`: a pinned WHATWG `fetch` over the HTTP client,
2
+ // with the policy, proxy and resolver vocabulary it is configured by. Nothing reachable from here
3
+ // imports the AWS SDK, storage-resource or the S3 modules (`tests/fetch.spec.ts` walks the graph).
4
+ export type * from './pinned-fetch/types.js'
5
+ export type { HttpClient, HttpClientOptions, HttpTimeouts } from './http/types.js'
6
+ export type { AddressResolver, NetworkPolicy, NetworkPolicyOptions, ResolvedAddress } from './network/types.js'
7
+ export type { ProxyRouter, ProxyRouterOptions } from './proxy/types.js'
8
+ export type { UrlResolver } from './resolvers/types.js'
9
+ export type { GoogleDriveResolverOptions } from './resolvers/google-drive/types.js'
10
+ export { createPinnedFetch } from './pinned-fetch.js'
11
+ export { createHttpClient } from './http.js'
12
+ export { createNetworkPolicy } from './network.js'
13
+ export { createGoogleDriveResolver, googleDriveResolver } from './resolvers/google-drive.js'
14
+ export { HttpError, NetworkPolicyError, ProxyError, ResolverError } from './errors.js'
package/src/http/types.ts CHANGED
@@ -34,6 +34,13 @@ export interface HttpClientOptions {
34
34
  // A closed pair of HTTP verbs — a union.
35
35
  export type HttpMethod = 'GET' | 'HEAD'
36
36
 
37
+ /**
38
+ * What a 3xx answer does: `follow` (default) goes on to its `Location` under the policy; `manual`
39
+ * returns the 3xx response itself, `Location` included, for a caller that checks every hop itself.
40
+ */
41
+ // A closed pair of modes — a union.
42
+ export type HttpRedirectMode = 'follow' | 'manual'
43
+
37
44
  export interface HttpRequestOptions extends SignalOptions {
38
45
  method?: HttpMethod
39
46
  headers?: Record<string, string>
@@ -44,6 +51,17 @@ export interface HttpRequestOptions extends SignalOptions {
44
51
  raw?: boolean
45
52
  /** Send `If-Range` with this strong validator. */
46
53
  ifRange?: string
54
+ /**
55
+ * Default `follow`. `manual` returns a 3xx answer (with its `Location`) instead of following it.
56
+ * A resolver's own flow (a Drive virus-scan page, its hops) still runs inside the client under the
57
+ * policy, because it carries its cookie across those hops.
58
+ */
59
+ redirect?: HttpRedirectMode
60
+ /**
61
+ * Return every status instead of throwing `HttpError('status:<code>')` — a 404 page is an answer.
62
+ * A forward proxy's 407 still throws `ProxyError('auth-failed')`: that is the proxy, not the target.
63
+ */
64
+ anyStatus?: boolean
47
65
  }
48
66
 
49
67
  export interface HttpResponse {
@@ -55,6 +73,8 @@ export interface HttpResponse {
55
73
  body: Readable
56
74
  info: FileInfo
57
75
  redirects: number
76
+ /** The body was transport-decoded (`Content-Encoding` undone); its `Content-Length` no longer applies. */
77
+ decoded?: boolean
58
78
  }
59
79
 
60
80
  export interface HttpSourceOptions {
@@ -67,7 +87,7 @@ export interface HttpSourceOptions {
67
87
 
68
88
  /** Downloads under the network policy, through proxies, redirects and resolvers. */
69
89
  export interface HttpClient {
70
- /** One GET or HEAD; a non-2xx answer is `HttpError('status:<code>')`. */
90
+ /** One GET or HEAD; a non-2xx answer is `HttpError('status:<code>')` unless `anyStatus` (or a 3xx under `redirect: 'manual'`). */
71
91
  request: (url: string | URL, options?: HttpRequestOptions) => Promise<HttpResponse>
72
92
  /** A re-openable source; Range-capable when the server answers 206 with Content-Range. */
73
93
  source: (url: string | URL, options?: HttpSourceOptions) => FileSource
package/src/http.ts CHANGED
@@ -195,7 +195,9 @@ export const createHttpClient = (options: HttpClientOptions = {}): HttpClient =>
195
195
  const status = response.statusCode ?? 0
196
196
  const location = header(response.headers, 'location')
197
197
 
198
- if (REDIRECT_STATUSES.includes(status) && location != null) {
198
+ // A resolver's flow carries its cookie across hops, so it follows them itself even under `manual`.
199
+ const returned = REDIRECT_STATUSES.includes(status) && location != null && requestOptions.redirect === 'manual' && resolver == null
200
+ if (REDIRECT_STATUSES.includes(status) && location != null && !returned) {
199
201
  response.resume()
200
202
  response.destroy()
201
203
  if (++redirects > maxRedirects) {
@@ -214,7 +216,8 @@ export const createHttpClient = (options: HttpClientOptions = {}): HttpClient =>
214
216
  current = next
215
217
  continue
216
218
  }
217
- if (status < 200 || status >= 300) {
219
+ const success = status >= 200 && status < 300
220
+ if (!success && !returned && (requestOptions.anyStatus !== true || (status === 407 && connection.mode === 'forward'))) {
218
221
  response.resume()
219
222
  response.destroy()
220
223
  if (status === 407 && connection.mode === 'forward') {
@@ -227,7 +230,7 @@ export const createHttpClient = (options: HttpClientOptions = {}): HttpClient =>
227
230
  ? { body: response, decoded: false }
228
231
  : bodyOf(current, response)
229
232
 
230
- if (resolver != null && method === 'GET') {
233
+ if (resolver != null && method === 'GET' && success) {
231
234
  let peeked: PeekedStream | undefined
232
235
  const readBody = async (maxBytes: number = DEFAULT_RESOLVER_PAGE_BYTES): Promise<string> => {
233
236
  peeked ??= await streamUtils.peek(body, maxBytes)
@@ -258,7 +261,7 @@ export const createHttpClient = (options: HttpClientOptions = {}): HttpClient =>
258
261
 
259
262
  log.debug('Response', { url: redactUtils.url(current), status, redirects })
260
263
 
261
- return { url: current, status, headers: response.headers, body, info: infoOf(current, response, decoded), redirects }
264
+ return { url: current, status, headers: response.headers, body, info: infoOf(current, response, decoded), redirects, decoded }
262
265
  }
263
266
  }
264
267
 
package/src/index.ts CHANGED
@@ -1,6 +1,7 @@
1
1
  export type * from './types.js'
2
2
  export type * from './archive/types.js'
3
3
  export type * from './http/types.js'
4
+ export type * from './pinned-fetch/types.js'
4
5
  export type * from './network/types.js'
5
6
  export type * from './proxy/types.js'
6
7
  export type * from './resolvers/types.js'
@@ -30,6 +31,7 @@ export * from './errors.js'
30
31
  export * from './archive.js'
31
32
  export * from './network.js'
32
33
  export * from './http.js'
34
+ export * from './pinned-fetch.js'
33
35
  export * from './proxy/endpoint.js'
34
36
  export * from './proxy/router.js'
35
37
  export * from './proxy/http.js'
@@ -0,0 +1,15 @@
1
+ /** Request headers the pinned fetch never forwards: hop-by-hop, the host it sets itself, and any encoding but identity. */
2
+ export const PINNED_FETCH_DROPPED_REQUEST_HEADERS: readonly string[] = Object.freeze([
3
+ 'connection', 'keep-alive', 'proxy-connection', 'transfer-encoding', 'upgrade', 'te', 'host', 'accept-encoding', 'content-length',
4
+ ])
5
+
6
+ /** Response headers that describe the connection, not the answer. */
7
+ export const PINNED_FETCH_DROPPED_RESPONSE_HEADERS: readonly string[] = Object.freeze([
8
+ 'connection', 'keep-alive', 'proxy-connection', 'transfer-encoding', 'upgrade',
9
+ ])
10
+
11
+ /** Statuses a `Response` must carry without a body. */
12
+ export const NULL_BODY_STATUSES: readonly number[] = Object.freeze([101, 103, 204, 205, 304])
13
+
14
+ /** The methods the pinned fetch carries. */
15
+ export const PINNED_FETCH_METHODS: readonly string[] = Object.freeze(['GET', 'HEAD'])
@@ -0,0 +1,17 @@
1
+ import type { HttpClient, HttpClientOptions } from '../http/types.js'
2
+
3
+ /**
4
+ * A WHATWG-`fetch`-compatible function over the package's HTTP client: every connection is pinned
5
+ * to an address the network policy checked a moment before (no second lookup to rebind), every
6
+ * redirect hop is checked again, URL resolvers (Google Drive) apply, and bodies stream. Only GET and
7
+ * HEAD are carried; any status comes back as a `Response` (a 404 page is an answer), a refusal of the
8
+ * policy rejects with `NetworkPolicyError` (`refused === true`).
9
+ */
10
+ export interface PinnedFetch {
11
+ (input: string | URL | Request, init?: RequestInit): Promise<Response>
12
+ }
13
+
14
+ export interface PinnedFetchOptions extends HttpClientOptions {
15
+ /** A ready client instead of one built from these options. */
16
+ client?: HttpClient
17
+ }
@@ -0,0 +1,93 @@
1
+ import { Readable } from 'node:stream'
2
+ import { HttpError } from './errors.js'
3
+ import {
4
+ NULL_BODY_STATUSES, PINNED_FETCH_DROPPED_REQUEST_HEADERS, PINNED_FETCH_DROPPED_RESPONSE_HEADERS, PINNED_FETCH_METHODS,
5
+ } from './pinned-fetch/consts.local.js'
6
+ import type { PinnedFetch, PinnedFetchOptions } from './pinned-fetch/types.js'
7
+ import { createHttpClient } from './http.js'
8
+ import { REDIRECT_STATUSES } from './http/consts.js'
9
+ import type { HttpMethod } from './http/types.js'
10
+ import { signalUtils } from './utils/signals.js'
11
+ import { streamUtils } from './utils/streams.js'
12
+
13
+ /**
14
+ * A `fetch` whose connections are pinned at connect time — hand it to code written against the
15
+ * WHATWG API (`@owlmeans/llm-reader`, a renderer) so its traffic gets the policy, the proxies and
16
+ * the resolvers of {@link createHttpClient}. This module and its imports never load the AWS SDK or
17
+ * the S3 modules: import it from `@owlmeans/file-streams/fetch`.
18
+ */
19
+ export const createPinnedFetch = (options: PinnedFetchOptions = {}): PinnedFetch => {
20
+ const { client: given, ...clientOptions } = options
21
+ const client = given ?? createHttpClient(clientOptions)
22
+
23
+ const headersOf = (input: string | URL | Request, init?: RequestInit): Record<string, string> => {
24
+ const merged = new Headers(input instanceof Request ? input.headers : undefined)
25
+ new Headers(init?.headers).forEach((value, name) => merged.set(name, value))
26
+ const headers: Record<string, string> = {}
27
+ merged.forEach((value, name) => {
28
+ if (!PINNED_FETCH_DROPPED_REQUEST_HEADERS.includes(name)) {
29
+ headers[name] = value
30
+ }
31
+ })
32
+
33
+ return headers
34
+ }
35
+
36
+ return async (input, init) => {
37
+ const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url
38
+ const method = (init?.method ?? (input instanceof Request ? input.method : 'GET')).toUpperCase()
39
+ if (!PINNED_FETCH_METHODS.includes(method)) {
40
+ throw new HttpError(`method:${method}`)
41
+ }
42
+ if (init?.body != null) {
43
+ throw new HttpError('body')
44
+ }
45
+ const mode = init?.redirect ?? (input instanceof Request ? input.redirect : 'follow')
46
+ const signal = init?.signal ?? (input instanceof Request ? input.signal : undefined) ?? undefined
47
+ signalUtils.throwIfAborted(signal)
48
+
49
+ const response = await client.request(url, {
50
+ method: method as HttpMethod,
51
+ headers: headersOf(input, init),
52
+ signal,
53
+ redirect: mode === 'follow' ? 'follow' : 'manual',
54
+ anyStatus: true,
55
+ })
56
+ const isRedirect = REDIRECT_STATUSES.includes(response.status) && response.headers.location != null
57
+ if (mode === 'error' && isRedirect) {
58
+ streamUtils.discard(response.body)
59
+ throw new HttpError('redirect')
60
+ }
61
+
62
+ const headers = new Headers()
63
+ for (const [name, value] of Object.entries(response.headers)) {
64
+ if (value == null || PINNED_FETCH_DROPPED_RESPONSE_HEADERS.includes(name)) {
65
+ continue
66
+ }
67
+ if (response.decoded === true && (name === 'content-encoding' || name === 'content-length')) {
68
+ continue
69
+ }
70
+ for (const one of Array.isArray(value) ? value : [value]) {
71
+ headers.append(name, one)
72
+ }
73
+ }
74
+
75
+ let body: ReadableStream<Uint8Array> | null = null
76
+ if (method === 'HEAD' || NULL_BODY_STATUSES.includes(response.status)) {
77
+ streamUtils.discard(response.body)
78
+ } else {
79
+ // The client stops listening once the head arrived; an abort must still end the body.
80
+ const release = signalUtils.onAbort(signal, reason => response.body.destroy(reason as Error))
81
+ response.body.once('close', release)
82
+ body = Readable.toWeb(response.body) as unknown as ReadableStream<Uint8Array>
83
+ }
84
+
85
+ const answer = new Response(body, { status: response.status, headers })
86
+ Object.defineProperty(answer, 'url', { value: response.url.href })
87
+ if (response.redirects > 0) {
88
+ Object.defineProperty(answer, 'redirected', { value: true })
89
+ }
90
+
91
+ return answer
92
+ }
93
+ }
@@ -0,0 +1,163 @@
1
+ import { afterAll, beforeAll, describe, expect, test } from 'bun:test'
2
+ import { readFileSync } from 'node:fs'
3
+ import { dirname, join, resolve } from 'node:path'
4
+ import { fileURLToPath } from 'node:url'
5
+ import { createGoogleDriveResolver, createPinnedFetch, NetworkPolicyError } from '../src/fetch.js'
6
+ import { noise, PNG } from './fixtures.js'
7
+ import { startFakeDrive } from './harness/drive.js'
8
+ import type { FakeDrive } from './harness/drive.js'
9
+ import { startOrigin } from './harness/origin.js'
10
+ import type { Origin } from './harness/origin.js'
11
+ import { LOCAL_NETWORK, localClient } from './context.js'
12
+
13
+ const TIMEOUTS = { connectMs: 3_000, headersMs: 3_000, idleMs: 3_000 }
14
+ const BUILD = fileURLToPath(new URL('../build/', import.meta.url))
15
+
16
+ /** Every module `entry` reaches by relative import, and every bare specifier those modules name. */
17
+ const importGraph = (entry: string): { files: string[], bare: Set<string> } => {
18
+ const files: string[] = []
19
+ const bare = new Set<string>()
20
+ const visit = (file: string): void => {
21
+ if (files.includes(file)) return
22
+ files.push(file)
23
+ const source = readFileSync(file, 'utf8')
24
+ for (const match of source.matchAll(/(?:import|export)\s[^'"]*?from\s+['"]([^'"]+)['"]|import\s*\(\s*['"]([^'"]+)['"]\s*\)|import\s+['"]([^'"]+)['"]/g)) {
25
+ const specifier = match[1] ?? match[2] ?? match[3]!
26
+ if (specifier.startsWith('.')) visit(resolve(dirname(file), specifier))
27
+ else bare.add(specifier)
28
+ }
29
+ }
30
+ visit(entry)
31
+ return { files, bare }
32
+ }
33
+
34
+ describe('@owlmeans/file-streams — pinned fetch', () => {
35
+ let origin: Origin
36
+ let foreign: Origin
37
+ let drive: FakeDrive
38
+ const big = noise(6 * 1024 * 1024, 3)
39
+ beforeAll(() => {
40
+ origin = startOrigin()
41
+ foreign = startOrigin()
42
+ origin.files.set('big.bin', { bytes: big })
43
+ origin.files.set('page.html', { bytes: Buffer.from('<h1>Hi</h1>'), type: 'text/html; charset=utf-8' })
44
+ drive = startFakeDrive({ small: PNG, large: noise(300_000), docx: PNG, xlsx: PNG, pptx: PNG }, foreign)
45
+ })
46
+ afterAll(() => {
47
+ origin.stop()
48
+ foreign.stop()
49
+ drive.stop()
50
+ })
51
+
52
+ const pinned = () => createPinnedFetch({ network: LOCAL_NETWORK, resolvers: [], timeouts: TIMEOUTS })
53
+
54
+ test('answers like fetch: status, headers, url and body; any status is a Response', async () => {
55
+ const page = await pinned()(origin.url('/files/page.html'), { headers: { 'x-trace': 't1' } })
56
+ expect(page.status).toBe(200)
57
+ expect(page.headers.get('content-type')).toBe('text/html; charset=utf-8')
58
+ expect(page.url).toBe(origin.url('/files/page.html'))
59
+ expect(await page.text()).toBe('<h1>Hi</h1>')
60
+ expect(origin.requests.at(-1)!.headers['x-trace']).toBe('t1')
61
+
62
+ const missing = await pinned()(origin.url('/status/404'))
63
+ expect(missing.status).toBe(404)
64
+ expect(await missing.text()).toBe('status page')
65
+ const head = await pinned()(origin.url('/files/page.html'), { method: 'HEAD' })
66
+ expect(head.status).toBe(200)
67
+ expect(head.body).toBeNull()
68
+ await expect(pinned()(origin.url('/files/page.html'), { method: 'POST' })).rejects.toThrow('http:method:POST')
69
+ })
70
+
71
+ test('redirect: manual returns the 3xx with its Location; follow (the default) goes on', async () => {
72
+ const hop = await pinned()(origin.url('/redirect/2'), { redirect: 'manual' })
73
+ expect(hop.status).toBe(302)
74
+ expect(hop.headers.get('location')).toBe('/redirect/1')
75
+ expect(hop.redirected).toBe(false)
76
+
77
+ const followed = await pinned()(origin.url('/redirect/2'))
78
+ expect(followed.status).toBe(200)
79
+ expect(followed.redirected).toBe(true)
80
+ expect(followed.url).toBe(origin.url('/redirect/0'))
81
+ expect(await followed.text()).toBe('arrived')
82
+ await expect(pinned()(origin.url('/redirect/1'), { redirect: 'error' })).rejects.toThrow('http:redirect')
83
+
84
+ // The client itself: a 3xx returned under `manual`, any status under `anyStatus`.
85
+ const client = localClient()
86
+ const returned = await client.request(origin.url('/redirect/1'), { redirect: 'manual' })
87
+ expect([returned.status, returned.headers.location, returned.redirects]).toEqual([302, '/redirect/0', 0])
88
+ await expect(client.request(origin.url('/status/404'))).rejects.toThrow('http:status:404')
89
+ expect((await client.request(origin.url('/status/404'), { anyStatus: true })).status).toBe(404)
90
+ })
91
+
92
+ test('private space is refused unless allowed — pinned at connect, so a name rebound to loopback is refused too', async () => {
93
+ const strict = createPinnedFetch({ resolvers: [], timeouts: TIMEOUTS })
94
+ const refusal = await strict(origin.url('/files/page.html')).catch((error: unknown) => error)
95
+ expect(refusal).toBeInstanceOf(NetworkPolicyError)
96
+ expect((refusal as NetworkPolicyError).refused).toBe(true)
97
+
98
+ // A name whose resolution turns private at connect time: the policy checks the address it dials.
99
+ const rebound = createPinnedFetch({
100
+ resolvers: [], timeouts: TIMEOUTS, network: { resolver: async () => [{ address: '127.0.0.1', family: 4 }] },
101
+ })
102
+ const port = new URL(origin.base).port
103
+ await expect(rebound(`http://public.example:${port}/files/page.html`)).rejects.toBeInstanceOf(NetworkPolicyError)
104
+ const allowed = createPinnedFetch({
105
+ resolvers: [], timeouts: TIMEOUTS, network: { ...LOCAL_NETWORK, resolver: async () => [{ address: '127.0.0.1', family: 4 }] },
106
+ })
107
+ expect(await (await allowed(`http://public.example:${port}/files/page.html`)).text()).toBe('<h1>Hi</h1>')
108
+ })
109
+
110
+ test('the Google Drive resolver applies through the fetch', async () => {
111
+ const fetch = createPinnedFetch({
112
+ network: LOCAL_NETWORK, timeouts: TIMEOUTS,
113
+ resolvers: [createGoogleDriveResolver({ origins: { drive: drive.base, usercontent: drive.base, docs: drive.base, accounts: drive.accounts } })],
114
+ })
115
+ const small = await fetch(`${drive.base}/file/d/SMALLFILE0001/view?usp=sharing`)
116
+ expect(small.headers.get('content-disposition')).toContain('small.png')
117
+ expect(new Uint8Array(await small.arrayBuffer())).toEqual(new Uint8Array(PNG))
118
+ // The virus-scan page of a large file is followed inside the resolver, even for a manual caller.
119
+ const large = await fetch(`${drive.base}/file/d/LARGEFILE0001/view`, { redirect: 'manual' })
120
+ expect(new Uint8Array(await large.arrayBuffer())).toEqual(new Uint8Array(noise(300_000)))
121
+ })
122
+
123
+ test('a multi-megabyte body streams in chunks', async () => {
124
+ const response = await pinned()(origin.url('/files/big.bin'))
125
+ expect(Number(response.headers.get('content-length'))).toBe(big.length)
126
+ const reader = response.body!.getReader()
127
+ let size = 0
128
+ let chunks = 0
129
+ for (;;) {
130
+ const { done, value } = await reader.read()
131
+ if (done) break
132
+ expect(Buffer.from(value).equals(big.subarray(size, size + value.byteLength))).toBe(true)
133
+ size += value.byteLength
134
+ chunks++
135
+ }
136
+ expect(size).toBe(big.length)
137
+ expect(chunks).toBeGreaterThan(1)
138
+ })
139
+
140
+ test('an abort ends a request waiting for its head and a body in flight', async () => {
141
+ const early = new AbortController()
142
+ const waiting = pinned()(origin.url('/stall-headers'), { signal: early.signal })
143
+ setTimeout(() => early.abort(new Error('stop-early')), 100)
144
+ await expect(waiting).rejects.toThrow('stop-early')
145
+
146
+ const late = new AbortController()
147
+ const response = await pinned()(origin.url('/stall-body'), { signal: late.signal })
148
+ const reader = response.body!.getReader()
149
+ expect([...(await reader.read()).value!]).toEqual([1, 2, 3])
150
+ late.abort(new Error('stop-late'))
151
+ await expect(reader.read()).rejects.toThrow()
152
+ })
153
+
154
+ test('the ./fetch subpath never loads the AWS SDK, storage-resource or the S3 modules', () => {
155
+ const { files, bare } = importGraph(join(BUILD, 'fetch.js'))
156
+ expect(files.length).toBeGreaterThan(5)
157
+ expect(files.some(file => /[\\/](s3|service|backed-workspace|workspace|transfer|archive)(\.js|[\\/])/.test(file.slice(BUILD.length)))).toBe(false)
158
+ expect([...bare].filter(name => name.startsWith('@aws-sdk/') || name.startsWith('@owlmeans/storage-resource'))).toEqual([])
159
+ expect([...bare].filter(name => !name.startsWith('node:')).sort()).toEqual(
160
+ ['@owlmeans/error', '@owlmeans/file-inspector', '@owlmeans/log', '@owlmeans/storage-common', 'socks'],
161
+ )
162
+ })
163
+ })