@filelayer/core 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +338 -0
- package/LICENSE +202 -0
- package/MIGRATIONS.md +328 -0
- package/NOTICE +37 -0
- package/README.md +343 -0
- package/SEMANTICS.md +729 -0
- package/dist/authz.d.ts +524 -0
- package/dist/authz.d.ts.map +1 -0
- package/dist/authz.js +889 -0
- package/dist/authz.js.map +1 -0
- package/dist/db.d.ts +145 -0
- package/dist/db.d.ts.map +1 -0
- package/dist/db.js +217 -0
- package/dist/db.js.map +1 -0
- package/dist/delivery.d.ts +293 -0
- package/dist/delivery.d.ts.map +1 -0
- package/dist/delivery.js +519 -0
- package/dist/delivery.js.map +1 -0
- package/dist/errors.d.ts +16 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +21 -0
- package/dist/errors.js.map +1 -0
- package/dist/filelayer.d.ts +542 -0
- package/dist/filelayer.d.ts.map +1 -0
- package/dist/filelayer.js +1360 -0
- package/dist/filelayer.js.map +1 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +8 -0
- package/dist/index.js.map +1 -0
- package/dist/simple.d.ts +297 -0
- package/dist/simple.d.ts.map +1 -0
- package/dist/simple.js +492 -0
- package/dist/simple.js.map +1 -0
- package/dist/storage.d.ts +269 -0
- package/dist/storage.d.ts.map +1 -0
- package/dist/storage.js +700 -0
- package/dist/storage.js.map +1 -0
- package/dist/store.d.ts +432 -0
- package/dist/store.d.ts.map +1 -0
- package/dist/store.js +862 -0
- package/dist/store.js.map +1 -0
- package/package.json +77 -0
- package/schema.sql +1190 -0
- package/src/authz.ts +1398 -0
- package/src/db.ts +271 -0
- package/src/delivery.ts +737 -0
- package/src/errors.ts +24 -0
- package/src/filelayer.ts +1836 -0
- package/src/index.ts +7 -0
- package/src/simple.ts +666 -0
- package/src/storage.ts +917 -0
- package/src/store.ts +1072 -0
- package/test/delivery.test.ts +0 -0
- package/test/group-subjects.test.ts +1072 -0
- package/test/helpers.ts +65 -0
- package/test/listing.test.ts +689 -0
- package/test/local-s3.d.mts +33 -0
- package/test/local-s3.mjs +400 -0
- package/test/persistence.test.ts +953 -0
- package/test/regression.test.ts +619 -0
- package/test/s3-live.test.ts +322 -0
- package/test/security.test.ts +1652 -0
- package/test/semantics.test.ts +888 -0
- package/test/storage.test.ts +437 -0
- package/test/tiers.test.ts +432 -0
- package/test/vault-example.test.ts +302 -0
- package/tsconfig.build.json +29 -0
- package/tsconfig.json +19 -0
package/src/delivery.ts
ADDED
|
@@ -0,0 +1,737 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BYTE DELIVERY -- the library owns the response, not the developer.
|
|
3
|
+
*
|
|
4
|
+
* A security review found three live defects here and correctly identified their
|
|
5
|
+
* common root cause: `read()` and `redeem()` handed back a `Uint8Array` and
|
|
6
|
+
* left everything that happens to those bytes on the way to a browser as the
|
|
7
|
+
* application's problem. Three of the four security-sensitive decisions the
|
|
8
|
+
* library was still leaving to the developer lived in that gap:
|
|
9
|
+
*
|
|
10
|
+
* - no `X-Content-Type-Options: nosniff` / `Content-Disposition` on the
|
|
11
|
+
* authenticated read -> a user-uploaded .html or .svg executes in OUR OWN
|
|
12
|
+
* origin. Stored XSS with full session access, cross-tenant reach.
|
|
13
|
+
* - no `Cache-Control: no-store` on the share download -> a revoked link is
|
|
14
|
+
* replayable from disk cache or an intermediary. This one is worse than it
|
|
15
|
+
* sounds: immediate revocation is the product's headline property, and a
|
|
16
|
+
* cache replay partially defeats it.
|
|
17
|
+
* - the share password in the query string -> the secret AND the password
|
|
18
|
+
* land in access logs, proxy logs, browser history, and the `Referer`
|
|
19
|
+
* header of any outbound link inside the delivered document.
|
|
20
|
+
*
|
|
21
|
+
* The fix is not "document the headers". A decision the developer can forget is
|
|
22
|
+
* a decision they will eventually forget, which is the entire premise of this
|
|
23
|
+
* design. So:
|
|
24
|
+
*
|
|
25
|
+
* 1. `read()` and `redeem()` now return a DELIVERY DESCRIPTOR -- bytes plus
|
|
26
|
+
* the exact headers required to serve them safely. The headers are
|
|
27
|
+
* computed by the library from the file record; there is no argument that
|
|
28
|
+
* turns them off.
|
|
29
|
+
* 2. Framework-agnostic writers are provided (`toResponse` for anything with
|
|
30
|
+
* a WHATWG `Response`, `sendNodeResponse` for `node:http`), so the
|
|
31
|
+
* developer writes one line and cannot write the wrong headers.
|
|
32
|
+
* 3. For the share path the library owns the ROUTE, not just the response,
|
|
33
|
+
* because the password-in-the-URL defect is a routing decision rather than
|
|
34
|
+
* a response decision. `shareDownloadRoute()` reads the password from the
|
|
35
|
+
* request body and REFUSES, loudly, with 400, if a credential appears in
|
|
36
|
+
* the query string. A silent decision has been converted into a noisy one,
|
|
37
|
+
* which by our own definition means it stops being a security-sensitive
|
|
38
|
+
* decision at all.
|
|
39
|
+
*
|
|
40
|
+
* Residual, stated plainly: `delivery.body` is still a public field, so a
|
|
41
|
+
* developer who ignores the helpers and hand-writes `res.end(d.body)` gets the
|
|
42
|
+
* old behaviour. This is "hard to get wrong", not "impossible to get wrong".
|
|
43
|
+
* Making it impossible would mean never exposing the bytes, which would break
|
|
44
|
+
* every non-HTTP consumer (a queue worker, a virus scanner, a thumbnailer).
|
|
45
|
+
* We chose the weaker guarantee deliberately and state it rather than hide it.
|
|
46
|
+
*/
|
|
47
|
+
|
|
48
|
+
import type { IncomingMessage, ServerResponse } from 'node:http';
|
|
49
|
+
import type { Filelayer } from './filelayer.ts';
|
|
50
|
+
import type { Principal } from './authz.ts';
|
|
51
|
+
import { FilelayerError } from './errors.ts';
|
|
52
|
+
|
|
53
|
+
export type Disposition = 'attachment' | 'inline';
|
|
54
|
+
|
|
55
|
+
export interface DeliverableFile {
|
|
56
|
+
name: string;
|
|
57
|
+
contentType: string;
|
|
58
|
+
sizeBytes?: number | null;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export interface FileDelivery {
|
|
62
|
+
file: DeliverableFile;
|
|
63
|
+
body: Uint8Array;
|
|
64
|
+
/**
|
|
65
|
+
* Everything that must be on the response. Lowercased, ready to spread into
|
|
66
|
+
* `res.writeHead()` or a `Headers` init.
|
|
67
|
+
*/
|
|
68
|
+
headers: Record<string, string>;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
// =============================================================================
|
|
72
|
+
// DELIVERY MODES
|
|
73
|
+
// =============================================================================
|
|
74
|
+
//
|
|
75
|
+
// There are exactly two, they have different security properties, and the
|
|
76
|
+
// difference is deliberately impossible to stumble into.
|
|
77
|
+
//
|
|
78
|
+
// 'proxy' (DEFAULT, and the only mode available unless you configure the
|
|
79
|
+
// other one) -- the bytes flow through this process. Every request
|
|
80
|
+
// is authorized. Revocation is immediate, full stop: the next
|
|
81
|
+
// request after a revoke gets 404, including one already in flight
|
|
82
|
+
// only in the sense that it has not yet reached us.
|
|
83
|
+
//
|
|
84
|
+
// 'redirect' (OPT-IN, and the opt-in is verbose on purpose) -- we authorize,
|
|
85
|
+
// we audit, and we answer 302 to a short-lived presigned URL. The
|
|
86
|
+
// bytes never touch this process, so it is CDN-cacheable for a
|
|
87
|
+
// public asset and costs no egress and no heap.
|
|
88
|
+
//
|
|
89
|
+
// THE TRADE, STATED THE WAY AN AUDITOR NEEDS IT STATED:
|
|
90
|
+
//
|
|
91
|
+
// Revocation is immediate at DECISION time, plus up to `ttlSeconds` of
|
|
92
|
+
// in-flight window.
|
|
93
|
+
//
|
|
94
|
+
// Concretely: if a grant is revoked at T, no NEW redirect is issued from T
|
|
95
|
+
// onward -- that part is as immediate as the proxied path. But a redirect
|
|
96
|
+
// issued at T-1 hands out a URL the object store will honour until
|
|
97
|
+
// T-1+ttlSeconds, and the object store has never heard of a grant. There is no
|
|
98
|
+
// way to recall it; AWS's own documented answer is "rotate the signing
|
|
99
|
+
// credential", which revokes every URL for every tenant at once and is not a
|
|
100
|
+
// per-grant control. So the window is real and bounded, and the bound is the
|
|
101
|
+
// TTL, and the TTL is bounded by us.
|
|
102
|
+
//
|
|
103
|
+
// Consequences that are enforced rather than documented:
|
|
104
|
+
// - the config will not typecheck without the acknowledgement string;
|
|
105
|
+
// - `ttlSeconds` is clamped to MAX_REDIRECT_TTL_SECONDS whatever you pass;
|
|
106
|
+
// - by default ONLY anonymous (published/public) grants may be redirected,
|
|
107
|
+
// because a public asset's residual window is a window onto something
|
|
108
|
+
// already public. Private grants stay proxied unless you widen the scope
|
|
109
|
+
// explicitly;
|
|
110
|
+
// - every redirected delivery writes a `file.deliver` audit event carrying
|
|
111
|
+
// `mode: 'redirect'`, the TTL and the resulting window, so a compliance auditor can
|
|
112
|
+
// answer "which deliveries were proxied and which were redirected" from the
|
|
113
|
+
// log rather than from a config file they have to trust;
|
|
114
|
+
// - the presigned URL pins `response-content-type` and
|
|
115
|
+
// `response-content-disposition`, so the object store serves the same
|
|
116
|
+
// neutralised type and `attachment` disposition the proxied path would
|
|
117
|
+
// have. A redirect does not lose the response-header protections.
|
|
118
|
+
|
|
119
|
+
export type DeliveryMode = 'proxy' | 'redirect';
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* The literal a developer must type to enable redirect delivery.
|
|
123
|
+
*
|
|
124
|
+
* A boolean would be typed once and forgotten. A sentence has to be read to be
|
|
125
|
+
* copied, it appears verbatim in the diff, and it shows up in a grep of the
|
|
126
|
+
* codebase when someone asks "do we ever hand out URLs that outlive a
|
|
127
|
+
* revocation?" -- which is the question this whole mode exists to make
|
|
128
|
+
* answerable.
|
|
129
|
+
*/
|
|
130
|
+
export const REDIRECT_ACKNOWLEDGEMENT =
|
|
131
|
+
'I accept a revocation window of up to ttlSeconds on redirected deliveries';
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Our ceiling, not the object store's. Five minutes is long enough for a client
|
|
135
|
+
* to follow a redirect on a bad mobile connection and short enough that the
|
|
136
|
+
* residual window is something you can put in a compliance document without
|
|
137
|
+
* flinching.
|
|
138
|
+
*/
|
|
139
|
+
export const MAX_REDIRECT_TTL_SECONDS = 300;
|
|
140
|
+
export const DEFAULT_REDIRECT_TTL_SECONDS = 60;
|
|
141
|
+
|
|
142
|
+
export interface RedirectDeliveryConfig {
|
|
143
|
+
/** Must be exactly `REDIRECT_ACKNOWLEDGEMENT`. Checked at runtime too. */
|
|
144
|
+
acknowledgeRevocationWindow: typeof REDIRECT_ACKNOWLEDGEMENT;
|
|
145
|
+
/** Clamped to [1, MAX_REDIRECT_TTL_SECONDS]. */
|
|
146
|
+
ttlSeconds?: number;
|
|
147
|
+
/**
|
|
148
|
+
* 'anonymous-grants-only' (DEFAULT) -- only a delivery authorized by an
|
|
149
|
+
* anonymous grant, i.e. something the customer has deliberately published,
|
|
150
|
+
* may be redirected. This is the setting where the residual window is a
|
|
151
|
+
* window onto an already-public object.
|
|
152
|
+
*
|
|
153
|
+
* 'all-grants' -- link and actor grants too. This is the setting that trades
|
|
154
|
+
* a real revocation window on genuinely private data for zero-proxy
|
|
155
|
+
* delivery. It is not the default and it never will be.
|
|
156
|
+
*/
|
|
157
|
+
scope?: 'anonymous-grants-only' | 'all-grants';
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
export interface ResolvedRedirectConfig {
|
|
161
|
+
ttlSeconds: number;
|
|
162
|
+
scope: 'anonymous-grants-only' | 'all-grants';
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
export function resolveRedirectConfig(cfg: RedirectDeliveryConfig): ResolvedRedirectConfig {
|
|
166
|
+
if (cfg.acknowledgeRevocationWindow !== REDIRECT_ACKNOWLEDGEMENT) {
|
|
167
|
+
throw new FilelayerError(
|
|
168
|
+
500,
|
|
169
|
+
'redirect_not_acknowledged',
|
|
170
|
+
'redirect delivery requires the verbatim REDIRECT_ACKNOWLEDGEMENT string',
|
|
171
|
+
);
|
|
172
|
+
}
|
|
173
|
+
const requested = cfg.ttlSeconds ?? DEFAULT_REDIRECT_TTL_SECONDS;
|
|
174
|
+
if (!Number.isFinite(requested) || requested < 1) {
|
|
175
|
+
throw new FilelayerError(500, 'redirect_bad_ttl', 'ttlSeconds must be >= 1');
|
|
176
|
+
}
|
|
177
|
+
return {
|
|
178
|
+
// Clamped, not rejected: a config that asks for a day gets five minutes and
|
|
179
|
+
// keeps working. Rejecting would tempt someone to "fix" it by removing the
|
|
180
|
+
// bound.
|
|
181
|
+
ttlSeconds: Math.min(Math.floor(requested), MAX_REDIRECT_TTL_SECONDS),
|
|
182
|
+
scope: cfg.scope ?? 'anonymous-grants-only',
|
|
183
|
+
};
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/** A delivery whose bytes flow through this process. */
|
|
187
|
+
export interface ProxyDelivery {
|
|
188
|
+
mode: 'proxy';
|
|
189
|
+
file: DeliverableFile;
|
|
190
|
+
headers: Record<string, string>;
|
|
191
|
+
/** The bytes, as a stream. Nothing here is ever fully resident. */
|
|
192
|
+
body: ReadableStream<Uint8Array>;
|
|
193
|
+
/** Known length, when the store reported one. */
|
|
194
|
+
bytes: number | null;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/** A delivery answered with a 302 to a short-lived presigned URL. */
|
|
198
|
+
export interface RedirectDelivery {
|
|
199
|
+
mode: 'redirect';
|
|
200
|
+
file: DeliverableFile;
|
|
201
|
+
headers: Record<string, string>;
|
|
202
|
+
status: 302;
|
|
203
|
+
url: string;
|
|
204
|
+
/** When the presigned URL stops working. */
|
|
205
|
+
expiresAt: Date;
|
|
206
|
+
/**
|
|
207
|
+
* The number the compliance document needs: revocation is immediate at
|
|
208
|
+
* decision time, PLUS up to this many seconds of in-flight window.
|
|
209
|
+
*/
|
|
210
|
+
revocationWindowSeconds: number;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
export type StreamedDelivery = ProxyDelivery | RedirectDelivery;
|
|
214
|
+
|
|
215
|
+
/**
|
|
216
|
+
* Content types that execute, or can be made to execute, in the origin that
|
|
217
|
+
* serves them. Served inline, any of these is stored XSS against your own
|
|
218
|
+
* application. `Content-Disposition: attachment` already defeats this in every
|
|
219
|
+
* current browser, so this list is the second layer: when a caller explicitly
|
|
220
|
+
* asks for `inline` (PDF preview, image thumbnails -- both legitimate), an
|
|
221
|
+
* active type is downgraded to `application/octet-stream` and the disposition
|
|
222
|
+
* is forced back to `attachment`.
|
|
223
|
+
*
|
|
224
|
+
* The predicate is deliberately broader than the list: anything whose type or
|
|
225
|
+
* subtype mentions html, xml, svg or script is treated as active, because the
|
|
226
|
+
* failure mode of being too strict is "the PDF downloads instead of previewing"
|
|
227
|
+
* and the failure mode of being too lax is session theft.
|
|
228
|
+
*/
|
|
229
|
+
const ACTIVE_TYPE_RE = /(html|xml|svg|script|xsl|ecmascript)/i;
|
|
230
|
+
|
|
231
|
+
export function isActiveContentType(contentType: string): boolean {
|
|
232
|
+
return ACTIVE_TYPE_RE.test(contentType.split(';')[0] ?? '');
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/**
|
|
236
|
+
* A content type we are willing to put on a response.
|
|
237
|
+
*
|
|
238
|
+
* A `content-type` is attacker-controlled: it is whatever the uploader sent.
|
|
239
|
+
* Anything with a CR, an LF or a NUL is a header-injection attempt and is not
|
|
240
|
+
* negotiable; anything that is not a plausible media type is served as bytes.
|
|
241
|
+
*/
|
|
242
|
+
export function safeContentType(contentType: string, disposition: Disposition): string {
|
|
243
|
+
const raw = contentType.trim();
|
|
244
|
+
if (!/^[a-zA-Z0-9!#$&^_.+-]{1,127}\/[a-zA-Z0-9!#$&^_.+-]{1,127}$/.test(raw)) {
|
|
245
|
+
return 'application/octet-stream';
|
|
246
|
+
}
|
|
247
|
+
if (disposition === 'inline' && isActiveContentType(raw)) return 'application/octet-stream';
|
|
248
|
+
return raw;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
// eslint-disable-next-line no-control-regex
|
|
252
|
+
const CONTROL_CHARS = /[\u0000-\u001f\u007f]/g;
|
|
253
|
+
|
|
254
|
+
/**
|
|
255
|
+
* RFC 6266 `Content-Disposition`, with the filename encoded rather than
|
|
256
|
+
* interpolated.
|
|
257
|
+
*
|
|
258
|
+
* NEW FINDING, found while doing this work: the example app wrote
|
|
259
|
+
* `filename="${file.name}"` directly. `file.name` is whatever the uploader
|
|
260
|
+
* sent. A name containing a double quote truncates the header; a name
|
|
261
|
+
* containing CR/LF injects an arbitrary header, including a second
|
|
262
|
+
* `Content-Type` or a `Set-Cookie`. That is a response-splitting bug in the
|
|
263
|
+
* shipped example, and it is not one the comparable integrations have, because
|
|
264
|
+
* none of them interpolate the name unescaped. It is fixed here, once, for
|
|
265
|
+
* every caller.
|
|
266
|
+
*/
|
|
267
|
+
export function contentDisposition(name: string, disposition: Disposition): string {
|
|
268
|
+
// ASCII fallback: printable ASCII only, no quote, no backslash, no path
|
|
269
|
+
// separators, no control characters. Never empty.
|
|
270
|
+
// Control characters (CR/LF are the header-injection vector) are dropped
|
|
271
|
+
// outright; quote, backslash and path separators become underscores; anything
|
|
272
|
+
// outside printable ASCII is replaced, because the extended form below is
|
|
273
|
+
// what actually carries a non-ASCII name.
|
|
274
|
+
const stripped = name.replace(CONTROL_CHARS, '');
|
|
275
|
+
const ascii =
|
|
276
|
+
stripped
|
|
277
|
+
.replace(/[\\"/]/g, '_')
|
|
278
|
+
.replace(/[^\x20-\x7e]/g, '_')
|
|
279
|
+
.trim()
|
|
280
|
+
.slice(0, 120) || 'download';
|
|
281
|
+
// RFC 5987 extended form, which carries the real (possibly non-ASCII) name.
|
|
282
|
+
const utf8 = encodeURIComponent(stripped.slice(0, 200)).replace(
|
|
283
|
+
/['()*]/g,
|
|
284
|
+
(c) => '%' + c.charCodeAt(0).toString(16).toUpperCase(),
|
|
285
|
+
);
|
|
286
|
+
return `${disposition}; filename="${ascii}"; filename*=UTF-8''${utf8}`;
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
/**
|
|
290
|
+
* The headers every Filelayer byte response carries. There is no option to
|
|
291
|
+
* omit any of them.
|
|
292
|
+
*
|
|
293
|
+
* nosniff - stops content-type sniffing turning a .txt into HTML.
|
|
294
|
+
* Content-Disposition- attachment by default; the browser never renders it.
|
|
295
|
+
* Cache-Control - `private, no-store` + `no-cache` + `must-revalidate`.
|
|
296
|
+
* This is the one that matters most: it is what makes
|
|
297
|
+
* "revocation is immediate" true at the browser and at
|
|
298
|
+
* every intermediary, not just at our origin.
|
|
299
|
+
* Pragma / Expires - the HTTP/1.0 spelling of the same thing, because
|
|
300
|
+
* corporate proxies are real.
|
|
301
|
+
* Referrer-Policy - `no-referrer`. A share link's secret is in the URL. Any
|
|
302
|
+
* outbound link inside a delivered document would leak it
|
|
303
|
+
* in the Referer header. This closes the third vector of
|
|
304
|
+
* the credential-in-the-URL defect, the one that survives
|
|
305
|
+
* moving the password out of the query string.
|
|
306
|
+
* CSP sandbox - defence in depth for the inline case and for any future
|
|
307
|
+
* caller that overrides disposition.
|
|
308
|
+
* X-Frame-Options - the delivered bytes cannot be framed by a third party.
|
|
309
|
+
*/
|
|
310
|
+
export function deliveryHeaders(
|
|
311
|
+
file: DeliverableFile,
|
|
312
|
+
opts: { disposition?: Disposition } = {},
|
|
313
|
+
): Record<string, string> {
|
|
314
|
+
const requested = opts.disposition ?? 'attachment';
|
|
315
|
+
// An active type is never served inline, whatever was asked for.
|
|
316
|
+
const disposition =
|
|
317
|
+
requested === 'inline' && isActiveContentType(file.contentType) ? 'attachment' : requested;
|
|
318
|
+
|
|
319
|
+
const headers: Record<string, string> = {
|
|
320
|
+
// Found by test/tiers.test.ts: `disposition` here has ALREADY been
|
|
321
|
+
// flipped to 'attachment' for an active type, so passing it to
|
|
322
|
+
// `safeContentType` meant the `disposition === 'inline' && isActive`
|
|
323
|
+
// branch could never fire and the type neutralisation was dead code. It
|
|
324
|
+
// must be evaluated against what the CALLER ASKED FOR, not against the
|
|
325
|
+
// value we corrected it to. Belt and braces are both wanted here: the
|
|
326
|
+
// disposition defeats current browsers, the neutralised type defeats a
|
|
327
|
+
// future one (and any non-browser client that ignores disposition).
|
|
328
|
+
'content-type': safeContentType(file.contentType, requested),
|
|
329
|
+
'content-disposition': contentDisposition(file.name, disposition),
|
|
330
|
+
'x-content-type-options': 'nosniff',
|
|
331
|
+
'cache-control': 'private, no-store, no-cache, must-revalidate, max-age=0',
|
|
332
|
+
pragma: 'no-cache',
|
|
333
|
+
expires: '0',
|
|
334
|
+
'referrer-policy': 'no-referrer',
|
|
335
|
+
'content-security-policy': "default-src 'none'; sandbox",
|
|
336
|
+
'x-frame-options': 'DENY',
|
|
337
|
+
};
|
|
338
|
+
if (file.sizeBytes != null) headers['content-length'] = String(file.sizeBytes);
|
|
339
|
+
return headers;
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
/** The same headers, minus the body ones, for a JSON error on a delivery path. */
|
|
343
|
+
export function errorHeaders(): Record<string, string> {
|
|
344
|
+
return {
|
|
345
|
+
'content-type': 'application/json',
|
|
346
|
+
'cache-control': 'private, no-store, no-cache, must-revalidate, max-age=0',
|
|
347
|
+
pragma: 'no-cache',
|
|
348
|
+
'referrer-policy': 'no-referrer',
|
|
349
|
+
'x-content-type-options': 'nosniff',
|
|
350
|
+
};
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
/**
|
|
354
|
+
* The headers on a 302 to a presigned URL.
|
|
355
|
+
*
|
|
356
|
+
* `cacheable` is true for an ANONYMOUS grant and only for an anonymous grant.
|
|
357
|
+
* That is the "public/anonymous grants should be able to use a cacheable path"
|
|
358
|
+
* requirement, and the negative half of it is the important half: nothing that
|
|
359
|
+
* was not already public becomes cacheable by a shared cache, ever.
|
|
360
|
+
*
|
|
361
|
+
* The cache lifetime is HALF the presigned URL's TTL. A cached 302 hands a
|
|
362
|
+
* client a URL that is already partly used up, so caching for the full TTL
|
|
363
|
+
* would let a client receive a redirect with milliseconds of life left and see
|
|
364
|
+
* a spurious 403. Half guarantees at least half the TTL remains.
|
|
365
|
+
*
|
|
366
|
+
* The 302 itself carries no bytes, so it carries no content-type protections;
|
|
367
|
+
* those are pinned INTO the presigned URL by the caller (see
|
|
368
|
+
* `Filelayer.readStream`), which is what stops a redirect from being a way to
|
|
369
|
+
* lose them.
|
|
370
|
+
*/
|
|
371
|
+
export function redirectHeaders(
|
|
372
|
+
url: string,
|
|
373
|
+
opts: { ttlSeconds: number; cacheable: boolean },
|
|
374
|
+
): Record<string, string> {
|
|
375
|
+
return {
|
|
376
|
+
location: url,
|
|
377
|
+
'cache-control': opts.cacheable
|
|
378
|
+
? `public, max-age=${Math.max(1, Math.floor(opts.ttlSeconds / 2))}`
|
|
379
|
+
: 'private, no-store, no-cache, must-revalidate, max-age=0',
|
|
380
|
+
// The presigned URL is a credential. It is in `Location`, so it will be in
|
|
381
|
+
// the browser's address bar and therefore in `Referer` on any onward
|
|
382
|
+
// navigation unless we say otherwise.
|
|
383
|
+
'referrer-policy': 'no-referrer',
|
|
384
|
+
'x-content-type-options': 'nosniff',
|
|
385
|
+
...(opts.cacheable ? {} : { pragma: 'no-cache', expires: '0' }),
|
|
386
|
+
};
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
// -----------------------------------------------------------------------------
|
|
390
|
+
// Writers
|
|
391
|
+
// -----------------------------------------------------------------------------
|
|
392
|
+
|
|
393
|
+
/** WHATWG `Response` -- Workers, Deno, Bun, Next.js route handlers, Hono. */
|
|
394
|
+
export function toResponse(delivery: FileDelivery, status = 200): Response {
|
|
395
|
+
return new Response(delivery.body as unknown as BodyInit, {
|
|
396
|
+
status,
|
|
397
|
+
headers: delivery.headers,
|
|
398
|
+
});
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
/**
|
|
402
|
+
* 206 when the body is a byte range, 200 when it is the whole object.
|
|
403
|
+
*
|
|
404
|
+
* `readStream()` and `redeemStream()` accept a byte range, and when the store
|
|
405
|
+
* serves one they attach `Content-Range`. A partial body sent under a 200 is
|
|
406
|
+
* silent data corruption: every HTTP client on earth treats 200 as "this is the
|
|
407
|
+
* complete representation", so it stores, caches, hashes and hands on a
|
|
408
|
+
* truncated file without a single error anywhere. The one thing that makes it
|
|
409
|
+
* a range is the status code.
|
|
410
|
+
*
|
|
411
|
+
* So the status is DERIVED from the headers rather than left to the caller.
|
|
412
|
+
* There is no value a caller could pass that this does not already know, and
|
|
413
|
+
* every value they could pass by mistake is wrong.
|
|
414
|
+
*/
|
|
415
|
+
function proxyStatus(delivery: ProxyDelivery): 200 | 206 {
|
|
416
|
+
return delivery.headers['content-range'] ? 206 : 200;
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
/**
|
|
420
|
+
* The streaming/redirect equivalent. One function for both modes, because the
|
|
421
|
+
* developer must not have to branch on the mode -- branching is where the
|
|
422
|
+
* `Cache-Control` gets copied from the wrong arm.
|
|
423
|
+
*/
|
|
424
|
+
export function toStreamResponse(delivery: StreamedDelivery): Response {
|
|
425
|
+
if (delivery.mode === 'redirect') {
|
|
426
|
+
return new Response(null, { status: delivery.status, headers: delivery.headers });
|
|
427
|
+
}
|
|
428
|
+
return new Response(delivery.body as unknown as BodyInit, {
|
|
429
|
+
status: proxyStatus(delivery),
|
|
430
|
+
headers: delivery.headers,
|
|
431
|
+
});
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
/** `node:http` / Express. */
|
|
435
|
+
export function sendNodeResponse(
|
|
436
|
+
res: ServerResponse,
|
|
437
|
+
delivery: FileDelivery,
|
|
438
|
+
status = 200,
|
|
439
|
+
): void {
|
|
440
|
+
// content-length is recomputed from the actual buffer rather than trusted
|
|
441
|
+
// from the record: a size column that disagrees with the object would
|
|
442
|
+
// otherwise truncate or hang the response.
|
|
443
|
+
const headers = { ...delivery.headers, 'content-length': String(delivery.body.byteLength) };
|
|
444
|
+
res.writeHead(status, headers);
|
|
445
|
+
res.end(delivery.body);
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
/**
|
|
449
|
+
* `node:http`, streaming. Nothing is buffered: the object's bytes go from the
|
|
450
|
+
* store's socket to the client's socket a chunk at a time.
|
|
451
|
+
*
|
|
452
|
+
* The `content-length` from the record is dropped unless the STORE reported one
|
|
453
|
+
* for this response, for the same reason `sendNodeResponse` recomputes it: a
|
|
454
|
+
* `size_bytes` that disagrees with the object truncates or hangs the response,
|
|
455
|
+
* and on a streamed response the hang is the one you notice in production
|
|
456
|
+
* rather than in a test.
|
|
457
|
+
*/
|
|
458
|
+
export async function sendNodeStream(
|
|
459
|
+
res: ServerResponse,
|
|
460
|
+
delivery: StreamedDelivery,
|
|
461
|
+
): Promise<void> {
|
|
462
|
+
if (delivery.mode === 'redirect') {
|
|
463
|
+
res.writeHead(delivery.status, delivery.headers);
|
|
464
|
+
res.end();
|
|
465
|
+
return;
|
|
466
|
+
}
|
|
467
|
+
const headers = { ...delivery.headers };
|
|
468
|
+
if (delivery.bytes === null) delete headers['content-length'];
|
|
469
|
+
else headers['content-length'] = String(delivery.bytes);
|
|
470
|
+
// 206 if this is a byte range. See `proxyStatus` -- a partial body under a
|
|
471
|
+
// 200 is a truncated file that no client can detect.
|
|
472
|
+
res.writeHead(proxyStatus(delivery), headers);
|
|
473
|
+
|
|
474
|
+
const reader = delivery.body.getReader();
|
|
475
|
+
try {
|
|
476
|
+
for (;;) {
|
|
477
|
+
const { done, value } = await reader.read();
|
|
478
|
+
if (done) break;
|
|
479
|
+
if (!value) continue;
|
|
480
|
+
if (!res.write(value)) {
|
|
481
|
+
await new Promise<void>((resolve) => res.once('drain', resolve));
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
res.end();
|
|
485
|
+
} catch (err) {
|
|
486
|
+
// Headers are already sent, so there is no way to turn this into a status
|
|
487
|
+
// code. Destroying the socket is the only honest signal that the body is
|
|
488
|
+
// incomplete; ending it normally would deliver a truncated file that looks
|
|
489
|
+
// like a successful download.
|
|
490
|
+
res.destroy(err instanceof Error ? err : new Error(String(err)));
|
|
491
|
+
} finally {
|
|
492
|
+
reader.releaseLock();
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
function sendNodeError(res: ServerResponse, err: unknown): void {
|
|
497
|
+
const e =
|
|
498
|
+
err instanceof FilelayerError ? err : new FilelayerError(500, 'internal');
|
|
499
|
+
res.writeHead(e.status, errorHeaders());
|
|
500
|
+
res.end(JSON.stringify({ error: e.code }));
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
// -----------------------------------------------------------------------------
|
|
504
|
+
// Routes the library owns
|
|
505
|
+
// -----------------------------------------------------------------------------
|
|
506
|
+
|
|
507
|
+
/**
|
|
508
|
+
* Credential names that must never appear in a URL. `secret` is not listed:
|
|
509
|
+
* the share secret IS the path segment, which is the standard design and is
|
|
510
|
+
* what `Referrer-Policy: no-referrer` above is for.
|
|
511
|
+
*/
|
|
512
|
+
const FORBIDDEN_QUERY_KEYS = ['password', 'pw', 'pass', 'passwd', 'token', 'secret', 'key'];
|
|
513
|
+
|
|
514
|
+
async function readBody(req: IncomingMessage, limit = 64 * 1024): Promise<string> {
|
|
515
|
+
const chunks: Buffer[] = [];
|
|
516
|
+
let size = 0;
|
|
517
|
+
for await (const c of req) {
|
|
518
|
+
size += (c as Buffer).byteLength;
|
|
519
|
+
if (size > limit) throw new FilelayerError(413, 'payload_too_large');
|
|
520
|
+
chunks.push(c as Buffer);
|
|
521
|
+
}
|
|
522
|
+
return Buffer.concat(chunks).toString('utf8');
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
/** JSON or form-urlencoded, whichever the client sent. Never the query string. */
|
|
526
|
+
function extractPassword(raw: string, contentType: string): string | undefined {
|
|
527
|
+
if (raw.length === 0) return undefined;
|
|
528
|
+
if (/application\/x-www-form-urlencoded/i.test(contentType)) {
|
|
529
|
+
const v = new URLSearchParams(raw).get('password');
|
|
530
|
+
return v === null ? undefined : v;
|
|
531
|
+
}
|
|
532
|
+
try {
|
|
533
|
+
const parsed = JSON.parse(raw) as Record<string, unknown>;
|
|
534
|
+
const v = parsed['password'];
|
|
535
|
+
return typeof v === 'string' ? v : undefined;
|
|
536
|
+
} catch {
|
|
537
|
+
throw new FilelayerError(400, 'bad_request');
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
export interface ShareRouteOptions {
|
|
542
|
+
/** Path prefix the share links are served under. Must match `baseUrl`. */
|
|
543
|
+
prefix?: string;
|
|
544
|
+
disposition?: Disposition;
|
|
545
|
+
/**
|
|
546
|
+
* 'auto' (default) applies the INSTANCE's redirect policy -- which is "never"
|
|
547
|
+
* unless `redirectDelivery` was configured and acknowledged, so the default
|
|
548
|
+
* is proxying for everybody who has not opted in. 'proxy' forces proxying for
|
|
549
|
+
* this route even on an instance that has opted in.
|
|
550
|
+
*
|
|
551
|
+
* There is no 'redirect' value. A route cannot demand a mode the instance was
|
|
552
|
+
* not configured (and acknowledged) for; the opt-in lives in exactly one
|
|
553
|
+
* place and it is the place with the acknowledgement string next to it.
|
|
554
|
+
*/
|
|
555
|
+
mode?: 'proxy' | 'auto';
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
/**
|
|
559
|
+
* `GET|POST <prefix>/:secret` -- the entire public share-link download path.
|
|
560
|
+
*
|
|
561
|
+
* The application supplies nothing. Identity is the secret itself, the password
|
|
562
|
+
* is read from the body, the counter is consumed by `redeem()`, and the
|
|
563
|
+
* response headers come from `deliveryHeaders()`. There is no parameter here
|
|
564
|
+
* whose wrong value leaks data.
|
|
565
|
+
*
|
|
566
|
+
* Returns true if it handled the request, so it can be dropped into any router.
|
|
567
|
+
*/
|
|
568
|
+
export function shareDownloadRoute(
|
|
569
|
+
fl: Filelayer,
|
|
570
|
+
opts: ShareRouteOptions = {},
|
|
571
|
+
): (req: IncomingMessage, res: ServerResponse) => Promise<boolean> {
|
|
572
|
+
const prefix = (opts.prefix ?? '/d').replace(/\/+$/, '');
|
|
573
|
+
|
|
574
|
+
return async (req, res) => {
|
|
575
|
+
const url = new URL(req.url ?? '/', 'http://filelayer.invalid');
|
|
576
|
+
const segments = url.pathname.split('/').filter(Boolean);
|
|
577
|
+
const prefixSegments = prefix.split('/').filter(Boolean);
|
|
578
|
+
if (segments.length !== prefixSegments.length + 1) return false;
|
|
579
|
+
if (prefixSegments.some((s, i) => segments[i] !== s)) return false;
|
|
580
|
+
if (req.method !== 'GET' && req.method !== 'POST') return false;
|
|
581
|
+
|
|
582
|
+
const secret = decodeURIComponent(segments[prefixSegments.length]!);
|
|
583
|
+
|
|
584
|
+
try {
|
|
585
|
+
// THE FIX FOR THE CREDENTIAL-IN-THE-URL DEFECT, and the reason this
|
|
586
|
+
// route exists.
|
|
587
|
+
// A password (or any other credential) in the query string ends up in
|
|
588
|
+
// access logs, proxy logs and browser history. The old example accepted
|
|
589
|
+
// one. We refuse -- loudly, before doing any work, and without consuming
|
|
590
|
+
// a download -- so the mistake cannot be made silently.
|
|
591
|
+
for (const k of FORBIDDEN_QUERY_KEYS) {
|
|
592
|
+
if (url.searchParams.has(k)) {
|
|
593
|
+
throw new FilelayerError(400, 'credential_in_query', `query_param:${k}`);
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
let password: string | undefined;
|
|
598
|
+
if (req.method === 'POST') {
|
|
599
|
+
password = extractPassword(await readBody(req), String(req.headers['content-type'] ?? ''));
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
// STREAMED, not buffered. This route is the one every share link goes
|
|
603
|
+
// through, so buffering here was the whole-file-in-memory tax on the
|
|
604
|
+
// busiest path in the product.
|
|
605
|
+
const delivery = await fl.redeemStream(secret, {
|
|
606
|
+
...(password !== undefined ? { password } : {}),
|
|
607
|
+
...(req.socket.remoteAddress ? { ip: req.socket.remoteAddress } : {}),
|
|
608
|
+
...(req.headers['user-agent'] ? { userAgent: String(req.headers['user-agent']) } : {}),
|
|
609
|
+
...(opts.disposition ? { disposition: opts.disposition } : {}),
|
|
610
|
+
...(opts.mode ? { mode: opts.mode } : {}),
|
|
611
|
+
});
|
|
612
|
+
|
|
613
|
+
await sendNodeStream(res, {
|
|
614
|
+
...delivery,
|
|
615
|
+
headers: {
|
|
616
|
+
...delivery.headers,
|
|
617
|
+
// Useful to the recipient and safe to expose: they already hold the
|
|
618
|
+
// credential this counts against.
|
|
619
|
+
'x-downloads-remaining': String(delivery.remainingDownloads ?? ''),
|
|
620
|
+
'x-filelayer-delivery': delivery.mode,
|
|
621
|
+
},
|
|
622
|
+
} as StreamedDelivery);
|
|
623
|
+
return true;
|
|
624
|
+
} catch (err) {
|
|
625
|
+
// 401 means "this link has a password". The client must re-issue as a
|
|
626
|
+
// POST with the password in the body; there is no supported way to put it
|
|
627
|
+
// in the URL.
|
|
628
|
+
if (err instanceof FilelayerError && err.status === 401) {
|
|
629
|
+
res.writeHead(401, { ...errorHeaders(), 'www-authenticate': 'FilelayerShare' });
|
|
630
|
+
res.end(JSON.stringify({ error: err.code, retry: { method: 'POST', field: 'password' } }));
|
|
631
|
+
return true;
|
|
632
|
+
}
|
|
633
|
+
sendNodeError(res, err);
|
|
634
|
+
return true;
|
|
635
|
+
}
|
|
636
|
+
};
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
export interface FileRouteOptions {
|
|
640
|
+
prefix?: string;
|
|
641
|
+
disposition?: Disposition;
|
|
642
|
+
/** See `ShareRouteOptions.mode`. */
|
|
643
|
+
mode?: 'proxy' | 'auto';
|
|
644
|
+
/** The application's own authentication. Filelayer never guesses identity. */
|
|
645
|
+
principal: (req: IncomingMessage) => Principal | Promise<Principal>;
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
/**
|
|
649
|
+
* `GET <prefix>/:fileId` -- the authenticated read path, headers included.
|
|
650
|
+
*
|
|
651
|
+
* The one thing the application must supply is who the caller is, because that
|
|
652
|
+
* is the one thing Filelayer cannot know. Everything downstream of that -- the
|
|
653
|
+
* decision, the audit event, the status code, and every response header -- is
|
|
654
|
+
* the library's.
|
|
655
|
+
*/
|
|
656
|
+
export function fileDownloadRoute(
|
|
657
|
+
fl: Filelayer,
|
|
658
|
+
opts: FileRouteOptions,
|
|
659
|
+
): (req: IncomingMessage, res: ServerResponse) => Promise<boolean> {
|
|
660
|
+
const prefix = (opts.prefix ?? '/files').replace(/\/+$/, '');
|
|
661
|
+
|
|
662
|
+
return async (req, res) => {
|
|
663
|
+
const url = new URL(req.url ?? '/', 'http://filelayer.invalid');
|
|
664
|
+
const segments = url.pathname.split('/').filter(Boolean);
|
|
665
|
+
const prefixSegments = prefix.split('/').filter(Boolean);
|
|
666
|
+
if (req.method !== 'GET') return false;
|
|
667
|
+
if (segments.length !== prefixSegments.length + 1) return false;
|
|
668
|
+
if (prefixSegments.some((s, i) => segments[i] !== s)) return false;
|
|
669
|
+
|
|
670
|
+
try {
|
|
671
|
+
const delivery = await fl.readStream(
|
|
672
|
+
await opts.principal(req),
|
|
673
|
+
segments[prefixSegments.length]!,
|
|
674
|
+
{
|
|
675
|
+
...(opts.disposition ? { disposition: opts.disposition } : {}),
|
|
676
|
+
...(opts.mode ? { mode: opts.mode } : {}),
|
|
677
|
+
},
|
|
678
|
+
);
|
|
679
|
+
await sendNodeStream(res, {
|
|
680
|
+
...delivery,
|
|
681
|
+
headers: { ...delivery.headers, 'x-filelayer-delivery': delivery.mode },
|
|
682
|
+
} as StreamedDelivery);
|
|
683
|
+
} catch (err) {
|
|
684
|
+
sendNodeError(res, err);
|
|
685
|
+
}
|
|
686
|
+
return true;
|
|
687
|
+
};
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
export interface DeliveryHandlerOptions {
|
|
691
|
+
/** Where authorized file reads are served. Must match `publicUrl()`. */
|
|
692
|
+
filePrefix?: string;
|
|
693
|
+
/** Where share links are served. Must match `baseUrl` + `/d`. */
|
|
694
|
+
sharePrefix?: string;
|
|
695
|
+
disposition?: Disposition;
|
|
696
|
+
/** See `ShareRouteOptions.mode`. Applies to both routes. */
|
|
697
|
+
mode?: 'proxy' | 'auto';
|
|
698
|
+
/**
|
|
699
|
+
* The application's authentication. Omitted means every request to the file
|
|
700
|
+
* path is ANONYMOUS -- which is safe, because an anonymous principal can only
|
|
701
|
+
* reach a file carrying an explicit `anonymous` grant (P1). It is not a
|
|
702
|
+
* "public mode"; there is no public mode.
|
|
703
|
+
*/
|
|
704
|
+
principal?: (req: IncomingMessage) => Principal | Promise<Principal>;
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
/**
|
|
708
|
+
* The whole byte-delivery surface as one `node:http` request listener.
|
|
709
|
+
*
|
|
710
|
+
* `createServer(deliveryHandler(fl))` is a complete, correct file server: both
|
|
711
|
+
* delivery paths, every security header, the query-string credential refusal,
|
|
712
|
+
* and a 404 for everything else. The developer writes no header and no status
|
|
713
|
+
* code, which means there is no header or status code for them to get wrong.
|
|
714
|
+
*/
|
|
715
|
+
export function deliveryHandler(
|
|
716
|
+
fl: Filelayer,
|
|
717
|
+
opts: DeliveryHandlerOptions = {},
|
|
718
|
+
): (req: IncomingMessage, res: ServerResponse) => Promise<void> {
|
|
719
|
+
const files = fileDownloadRoute(fl, {
|
|
720
|
+
prefix: opts.filePrefix ?? '/f',
|
|
721
|
+
principal: opts.principal ?? (() => ({ actorId: null })),
|
|
722
|
+
...(opts.disposition ? { disposition: opts.disposition } : {}),
|
|
723
|
+
...(opts.mode ? { mode: opts.mode } : {}),
|
|
724
|
+
});
|
|
725
|
+
const shares = shareDownloadRoute(fl, {
|
|
726
|
+
prefix: opts.sharePrefix ?? '/d',
|
|
727
|
+
...(opts.disposition ? { disposition: opts.disposition } : {}),
|
|
728
|
+
...(opts.mode ? { mode: opts.mode } : {}),
|
|
729
|
+
});
|
|
730
|
+
|
|
731
|
+
return async (req, res) => {
|
|
732
|
+
if (await files(req, res)) return;
|
|
733
|
+
if (await shares(req, res)) return;
|
|
734
|
+
res.writeHead(404, errorHeaders());
|
|
735
|
+
res.end(JSON.stringify({ error: 'not_found' }));
|
|
736
|
+
};
|
|
737
|
+
}
|