@felipedsvit/s3node 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +143 -0
- package/bin/s3node.js +101 -0
- package/docs/plan.md +340 -0
- package/package.json +50 -0
- package/src/auth/chunked.js +277 -0
- package/src/auth/credentials.js +41 -0
- package/src/auth/sigv4.js +293 -0
- package/src/errors.js +65 -0
- package/src/features/cors.js +115 -0
- package/src/features/encryption.js +201 -0
- package/src/features/formdata.js +169 -0
- package/src/features/lifecycle.js +175 -0
- package/src/features/notifications.js +161 -0
- package/src/features/policy.js +0 -0
- package/src/features/postpolicy.js +150 -0
- package/src/features/tagging.js +64 -0
- package/src/handlers/bucket.js +197 -0
- package/src/handlers/config.js +129 -0
- package/src/handlers/index.js +6 -0
- package/src/handlers/multipart.js +109 -0
- package/src/handlers/object.js +211 -0
- package/src/handlers/post.js +106 -0
- package/src/handlers/service.js +10 -0
- package/src/handlers/shared.js +84 -0
- package/src/http.js +243 -0
- package/src/index.js +30 -0
- package/src/router.js +128 -0
- package/src/server.js +282 -0
- package/src/storage/blobs.js +135 -0
- package/src/storage/metadata.js +616 -0
- package/src/storage/store.js +548 -0
- package/src/util/bytes.js +35 -0
- package/src/util/crc.js +54 -0
- package/src/util/hash.js +64 -0
- package/src/xml.js +205 -0
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { S3Error } from '../errors.js'
|
|
2
|
+
import { childText, childrenNamed, document, parseXml, text } from '../xml.js'
|
|
3
|
+
|
|
4
|
+
const MAX_RULES = 100
|
|
5
|
+
|
|
6
|
+
function ruleValues(rule, name) {
|
|
7
|
+
return childrenNamed(rule, name).map((child) => child.text)
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export function parseCorsXml(body) {
|
|
11
|
+
const root = parseXml(body)
|
|
12
|
+
if (root.name !== 'CORSConfiguration') {
|
|
13
|
+
throw new S3Error('MalformedXML', 'Expected a CORSConfiguration element')
|
|
14
|
+
}
|
|
15
|
+
const rules = childrenNamed(root, 'CORSRule').map((rule) => {
|
|
16
|
+
const allowedOrigins = ruleValues(rule, 'AllowedOrigin')
|
|
17
|
+
const allowedMethods = ruleValues(rule, 'AllowedMethod').map((method) => method.toUpperCase())
|
|
18
|
+
if (allowedOrigins.length === 0 || allowedMethods.length === 0) {
|
|
19
|
+
throw new S3Error('MalformedXML', 'Each CORSRule requires AllowedOrigin and AllowedMethod')
|
|
20
|
+
}
|
|
21
|
+
const maxAge = childText(rule, 'MaxAgeSeconds')
|
|
22
|
+
return {
|
|
23
|
+
id: childText(rule, 'ID'),
|
|
24
|
+
allowedOrigins,
|
|
25
|
+
allowedMethods,
|
|
26
|
+
allowedHeaders: ruleValues(rule, 'AllowedHeader').map((header) => header.toLowerCase()),
|
|
27
|
+
exposeHeaders: ruleValues(rule, 'ExposeHeader'),
|
|
28
|
+
maxAgeSeconds: maxAge === undefined ? undefined : Number.parseInt(maxAge, 10),
|
|
29
|
+
}
|
|
30
|
+
})
|
|
31
|
+
if (rules.length === 0) throw new S3Error('MalformedXML', 'At least one CORSRule is required')
|
|
32
|
+
if (rules.length > MAX_RULES) {
|
|
33
|
+
throw new S3Error('MalformedXML', `A maximum of ${MAX_RULES} CORS rules is allowed`)
|
|
34
|
+
}
|
|
35
|
+
return { rules }
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export function corsXml(config) {
|
|
39
|
+
const rules = (config?.rules ?? []).map((rule) => (
|
|
40
|
+
'<CORSRule>' +
|
|
41
|
+
(rule.id ? text('ID', rule.id) : '') +
|
|
42
|
+
rule.allowedOrigins.map((origin) => text('AllowedOrigin', origin)).join('') +
|
|
43
|
+
rule.allowedMethods.map((method) => text('AllowedMethod', method)).join('') +
|
|
44
|
+
rule.allowedHeaders.map((header) => text('AllowedHeader', header)).join('') +
|
|
45
|
+
rule.exposeHeaders.map((header) => text('ExposeHeader', header)).join('') +
|
|
46
|
+
(rule.maxAgeSeconds === undefined ? '' : text('MaxAgeSeconds', rule.maxAgeSeconds)) +
|
|
47
|
+
'</CORSRule>'
|
|
48
|
+
)).join('')
|
|
49
|
+
return document('CORSConfiguration', rules)
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** A single `*` may appear anywhere in an allowed origin. */
|
|
53
|
+
function originMatches(pattern, origin) {
|
|
54
|
+
if (pattern === '*') return true
|
|
55
|
+
const star = pattern.indexOf('*')
|
|
56
|
+
if (star === -1) return pattern === origin
|
|
57
|
+
const prefix = pattern.slice(0, star)
|
|
58
|
+
const suffix = pattern.slice(star + 1)
|
|
59
|
+
return origin.length >= prefix.length + suffix.length &&
|
|
60
|
+
origin.startsWith(prefix) && origin.endsWith(suffix)
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function headersAllowed(rule, requested) {
|
|
64
|
+
if (requested.length === 0) return true
|
|
65
|
+
if (rule.allowedHeaders.includes('*')) return true
|
|
66
|
+
return requested.every((header) => rule.allowedHeaders.includes(header))
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export function matchRule(config, { origin, method, requestHeaders = [] }) {
|
|
70
|
+
if (!config?.rules || !origin) return null
|
|
71
|
+
for (const rule of config.rules) {
|
|
72
|
+
if (!rule.allowedOrigins.some((pattern) => originMatches(pattern, origin))) continue
|
|
73
|
+
if (!rule.allowedMethods.includes(method)) continue
|
|
74
|
+
if (!headersAllowed(rule, requestHeaders)) continue
|
|
75
|
+
return rule
|
|
76
|
+
}
|
|
77
|
+
return null
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export function parseRequestHeaders(header) {
|
|
81
|
+
return String(header ?? '')
|
|
82
|
+
.split(',')
|
|
83
|
+
.map((entry) => entry.trim().toLowerCase())
|
|
84
|
+
.filter(Boolean)
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Headers for an actual (non-preflight) cross-origin response. Vary is always
|
|
89
|
+
* set so caches do not serve one origin's response to another.
|
|
90
|
+
*/
|
|
91
|
+
export function responseHeaders(rule, origin) {
|
|
92
|
+
if (!rule) return { Vary: 'Origin' }
|
|
93
|
+
const headers = {
|
|
94
|
+
'Access-Control-Allow-Origin': rule.allowedOrigins.includes('*') ? '*' : origin,
|
|
95
|
+
'Access-Control-Allow-Methods': rule.allowedMethods.join(', '),
|
|
96
|
+
Vary: 'Origin',
|
|
97
|
+
}
|
|
98
|
+
if (rule.exposeHeaders.length) {
|
|
99
|
+
headers['Access-Control-Expose-Headers'] = rule.exposeHeaders.join(', ')
|
|
100
|
+
}
|
|
101
|
+
if (rule.maxAgeSeconds !== undefined) {
|
|
102
|
+
headers['Access-Control-Max-Age'] = String(rule.maxAgeSeconds)
|
|
103
|
+
}
|
|
104
|
+
return headers
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
export function preflightHeaders(rule, origin, requestHeaders) {
|
|
108
|
+
const headers = responseHeaders(rule, origin)
|
|
109
|
+
if (requestHeaders.length) {
|
|
110
|
+
headers['Access-Control-Allow-Headers'] = rule.allowedHeaders.includes('*')
|
|
111
|
+
? requestHeaders.join(', ')
|
|
112
|
+
: rule.allowedHeaders.filter((header) => requestHeaders.includes(header)).join(', ')
|
|
113
|
+
}
|
|
114
|
+
return headers
|
|
115
|
+
}
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
import { createCipheriv, createDecipheriv, createHash, randomBytes, timingSafeEqual } from 'node:crypto'
|
|
2
|
+
import { readFile, writeFile } from 'node:fs/promises'
|
|
3
|
+
import { Transform } from 'node:stream'
|
|
4
|
+
import { S3Error } from '../errors.js'
|
|
5
|
+
|
|
6
|
+
const KEY_BYTES = 32
|
|
7
|
+
const BLOCK_BYTES = 16
|
|
8
|
+
const DATA_CIPHER = 'aes-256-ctr'
|
|
9
|
+
const WRAP_CIPHER = 'aes-256-gcm'
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Object data is encrypted with AES-256-CTR rather than GCM because CTR is
|
|
13
|
+
* seekable: a range read can jump straight to the block containing the first
|
|
14
|
+
* requested byte. GCM would force decrypting from byte zero on every ranged
|
|
15
|
+
* GET. Integrity still comes from the stored MD5/checksums, and the SSE-S3 data
|
|
16
|
+
* key itself is wrapped with GCM.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
/** 128-bit big-endian counter addition, used to seek and to space parts apart. */
|
|
20
|
+
export function advanceIv(iv, blocks) {
|
|
21
|
+
const value = (BigInt(`0x${iv.toString('hex')}`) + BigInt(blocks)) & ((1n << 128n) - 1n)
|
|
22
|
+
return Buffer.from(value.toString(16).padStart(32, '0'), 'hex')
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Each multipart part gets its own counter window, spaced 2^64 blocks apart.
|
|
27
|
+
* A part caps out at 5 GiB (2^28 blocks), so windows can never overlap and no
|
|
28
|
+
* per-part IV has to be stored.
|
|
29
|
+
*/
|
|
30
|
+
export function partIv(baseIv, partNumber) {
|
|
31
|
+
return partNumber ? advanceIv(baseIv, BigInt(partNumber) << 64n) : Buffer.from(baseIv)
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** Discards the first `count` bytes of a stream. */
|
|
35
|
+
class SkipBytes extends Transform {
|
|
36
|
+
constructor(count) {
|
|
37
|
+
super()
|
|
38
|
+
this.remaining = count
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
_transform(chunk, _encoding, callback) {
|
|
42
|
+
if (this.remaining >= chunk.length) {
|
|
43
|
+
this.remaining -= chunk.length
|
|
44
|
+
callback()
|
|
45
|
+
return
|
|
46
|
+
}
|
|
47
|
+
const output = this.remaining ? chunk.subarray(this.remaining) : chunk
|
|
48
|
+
this.remaining = 0
|
|
49
|
+
callback(null, output)
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export class EncryptionManager {
|
|
54
|
+
constructor(masterKey) {
|
|
55
|
+
this.masterKey = masterKey
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Loads the SSE-S3 master key, generating and persisting one on first use.
|
|
60
|
+
* Losing this file makes every SSE-S3 object unreadable.
|
|
61
|
+
*/
|
|
62
|
+
static async load(path, provided = null) {
|
|
63
|
+
if (provided) {
|
|
64
|
+
const key = Buffer.isBuffer(provided) ? provided : Buffer.from(provided, 'base64')
|
|
65
|
+
if (key.length !== KEY_BYTES) {
|
|
66
|
+
throw new TypeError(`encryptionMasterKey must be ${KEY_BYTES} bytes`)
|
|
67
|
+
}
|
|
68
|
+
return new EncryptionManager(key)
|
|
69
|
+
}
|
|
70
|
+
try {
|
|
71
|
+
const stored = Buffer.from(await readFile(path, 'utf8'), 'base64')
|
|
72
|
+
if (stored.length === KEY_BYTES) return new EncryptionManager(stored)
|
|
73
|
+
} catch {
|
|
74
|
+
// Fall through and generate one.
|
|
75
|
+
}
|
|
76
|
+
const key = randomBytes(KEY_BYTES)
|
|
77
|
+
await writeFile(path, key.toString('base64'), { mode: 0o600 })
|
|
78
|
+
return new EncryptionManager(key)
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Reads the SSE headers off a request. Returns null when the request asks for
|
|
83
|
+
* no encryption.
|
|
84
|
+
*/
|
|
85
|
+
parseRequest(headers, { prefix = 'x-amz-server-side-encryption' } = {}) {
|
|
86
|
+
const customerAlgorithm = headers[`${prefix}-customer-algorithm`]
|
|
87
|
+
if (customerAlgorithm) {
|
|
88
|
+
if (String(customerAlgorithm) !== 'AES256') {
|
|
89
|
+
throw new S3Error('InvalidArgument', 'Unsupported customer encryption algorithm')
|
|
90
|
+
}
|
|
91
|
+
const rawKey = headers[`${prefix}-customer-key`]
|
|
92
|
+
const declaredMd5 = headers[`${prefix}-customer-key-md5`]
|
|
93
|
+
if (!rawKey || !declaredMd5) {
|
|
94
|
+
throw new S3Error('InvalidRequest', 'SSE-C requires both the customer key and its MD5')
|
|
95
|
+
}
|
|
96
|
+
const key = Buffer.from(String(rawKey), 'base64')
|
|
97
|
+
if (key.length !== KEY_BYTES) {
|
|
98
|
+
throw new S3Error('InvalidArgument', 'The customer key must be 256 bits')
|
|
99
|
+
}
|
|
100
|
+
const keyMd5 = createHash('md5').update(key).digest('base64')
|
|
101
|
+
if (keyMd5 !== String(declaredMd5)) {
|
|
102
|
+
throw new S3Error('InvalidDigest', 'The customer key MD5 does not match the key')
|
|
103
|
+
}
|
|
104
|
+
return { mode: 'SSE-C', algorithm: 'AES256', key, keyMd5 }
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
const managed = headers[prefix]
|
|
108
|
+
if (managed) {
|
|
109
|
+
if (String(managed) !== 'AES256') {
|
|
110
|
+
throw new S3Error('InvalidArgument', 'Unsupported server-side encryption algorithm')
|
|
111
|
+
}
|
|
112
|
+
return { mode: 'SSE-S3', algorithm: 'AES256' }
|
|
113
|
+
}
|
|
114
|
+
return null
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** Builds the stored encryption context and the data key used to write. */
|
|
118
|
+
create(request) {
|
|
119
|
+
const iv = randomBytes(BLOCK_BYTES)
|
|
120
|
+
if (request.mode === 'SSE-C') {
|
|
121
|
+
return {
|
|
122
|
+
key: request.key,
|
|
123
|
+
context: { mode: 'SSE-C', algorithm: 'AES256', iv: iv.toString('base64'), keyMd5: request.keyMd5 },
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
const dataKey = randomBytes(KEY_BYTES)
|
|
127
|
+
const wrapIv = randomBytes(12)
|
|
128
|
+
const wrapper = createCipheriv(WRAP_CIPHER, this.masterKey, wrapIv)
|
|
129
|
+
const wrapped = Buffer.concat([wrapper.update(dataKey), wrapper.final()])
|
|
130
|
+
return {
|
|
131
|
+
key: dataKey,
|
|
132
|
+
context: {
|
|
133
|
+
mode: 'SSE-S3',
|
|
134
|
+
algorithm: 'AES256',
|
|
135
|
+
iv: iv.toString('base64'),
|
|
136
|
+
wrappedKey: wrapped.toString('base64'),
|
|
137
|
+
wrapIv: wrapIv.toString('base64'),
|
|
138
|
+
wrapTag: wrapper.getAuthTag().toString('base64'),
|
|
139
|
+
},
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* Recovers the data key for a stored object. SSE-C objects require the client
|
|
145
|
+
* to present the same key again; the server never stored it.
|
|
146
|
+
*/
|
|
147
|
+
resolveKey(context, request) {
|
|
148
|
+
if (!context) return null
|
|
149
|
+
if (context.mode === 'SSE-C') {
|
|
150
|
+
if (!request || request.mode !== 'SSE-C') {
|
|
151
|
+
throw new S3Error('InvalidRequest',
|
|
152
|
+
'The object was stored with SSE-C; the same customer key must be supplied to read it')
|
|
153
|
+
}
|
|
154
|
+
const stored = Buffer.from(context.keyMd5, 'base64')
|
|
155
|
+
const offered = Buffer.from(request.keyMd5, 'base64')
|
|
156
|
+
if (stored.length !== offered.length || !timingSafeEqual(stored, offered)) {
|
|
157
|
+
throw new S3Error('AccessDenied', 'The customer key does not match the one used to store the object')
|
|
158
|
+
}
|
|
159
|
+
return request.key
|
|
160
|
+
}
|
|
161
|
+
const unwrapper = createDecipheriv(WRAP_CIPHER, this.masterKey, Buffer.from(context.wrapIv, 'base64'))
|
|
162
|
+
unwrapper.setAuthTag(Buffer.from(context.wrapTag, 'base64'))
|
|
163
|
+
return Buffer.concat([
|
|
164
|
+
unwrapper.update(Buffer.from(context.wrappedKey, 'base64')),
|
|
165
|
+
unwrapper.final(),
|
|
166
|
+
])
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
createEncryptStream(key, context, partNumber = 0) {
|
|
170
|
+
const iv = partIv(Buffer.from(context.iv, 'base64'), partNumber)
|
|
171
|
+
return createCipheriv(DATA_CIPHER, key, iv)
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* Decrypts starting at `offset` bytes into the (part's) plaintext. The caller
|
|
176
|
+
* must have started reading at `blockAlignedOffset(offset)`.
|
|
177
|
+
*/
|
|
178
|
+
createDecryptStreams(key, context, offset = 0, partNumber = 0) {
|
|
179
|
+
const base = partIv(Buffer.from(context.iv, 'base64'), partNumber)
|
|
180
|
+
const block = Math.floor(offset / BLOCK_BYTES)
|
|
181
|
+
const decipher = createDecipheriv(DATA_CIPHER, key, advanceIv(base, block))
|
|
182
|
+
const skip = offset - block * BLOCK_BYTES
|
|
183
|
+
return skip ? [decipher, new SkipBytes(skip)] : [decipher]
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/** Ciphertext must be read from a block boundary for CTR seeking to line up. */
|
|
188
|
+
export function blockAlignedOffset(offset) {
|
|
189
|
+
return Math.floor(offset / BLOCK_BYTES) * BLOCK_BYTES
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
export function encryptionResponseHeaders(context) {
|
|
193
|
+
if (!context) return {}
|
|
194
|
+
if (context.mode === 'SSE-C') {
|
|
195
|
+
return {
|
|
196
|
+
'x-amz-server-side-encryption-customer-algorithm': context.algorithm,
|
|
197
|
+
'x-amz-server-side-encryption-customer-key-MD5': context.keyMd5,
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
return { 'x-amz-server-side-encryption': context.algorithm }
|
|
201
|
+
}
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
import { PassThrough } from 'node:stream'
|
|
2
|
+
import { pipeline } from 'node:stream/promises'
|
|
3
|
+
import { S3Error } from '../errors.js'
|
|
4
|
+
|
|
5
|
+
const MAX_HEADER_BYTES = 8192
|
|
6
|
+
const DEFAULT_MAX_FIELD_BYTES = 64 * 1024
|
|
7
|
+
const DEFAULT_MAX_FIELDS = 100
|
|
8
|
+
|
|
9
|
+
export function parseBoundary(contentType) {
|
|
10
|
+
const match = /;\s*boundary=(?:"([^"]+)"|([^;\s]+))/i.exec(String(contentType ?? ''))
|
|
11
|
+
const boundary = match?.[1] ?? match?.[2]
|
|
12
|
+
if (!boundary) throw new S3Error('MalformedPOSTRequest', 'The Content-Type is missing a boundary')
|
|
13
|
+
return boundary
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/** Pull-based buffer over an async iterable of Buffers. */
|
|
17
|
+
class Scanner {
|
|
18
|
+
constructor(iterator) {
|
|
19
|
+
this.iterator = iterator
|
|
20
|
+
this.buffer = Buffer.alloc(0)
|
|
21
|
+
this.exhausted = false
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
async fill() {
|
|
25
|
+
if (this.exhausted) return false
|
|
26
|
+
const { value, done } = await this.iterator.next()
|
|
27
|
+
if (done) {
|
|
28
|
+
this.exhausted = true
|
|
29
|
+
return false
|
|
30
|
+
}
|
|
31
|
+
this.buffer = this.buffer.length ? Buffer.concat([this.buffer, value]) : value
|
|
32
|
+
return true
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
async readUntil(pattern, maxBytes) {
|
|
36
|
+
for (;;) {
|
|
37
|
+
const index = this.buffer.indexOf(pattern)
|
|
38
|
+
if (index !== -1) {
|
|
39
|
+
const chunk = this.buffer.subarray(0, index)
|
|
40
|
+
this.buffer = this.buffer.subarray(index + pattern.length)
|
|
41
|
+
return chunk
|
|
42
|
+
}
|
|
43
|
+
if (this.buffer.length > maxBytes) {
|
|
44
|
+
throw new S3Error('MalformedPOSTRequest', 'A form part exceeded the maximum accepted size')
|
|
45
|
+
}
|
|
46
|
+
if (!(await this.fill())) return null
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
async ensure(count) {
|
|
51
|
+
while (this.buffer.length < count) {
|
|
52
|
+
if (!(await this.fill())) return false
|
|
53
|
+
}
|
|
54
|
+
return true
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function parsePartHeaders(raw) {
|
|
59
|
+
const headers = {}
|
|
60
|
+
for (const line of raw.toString('latin1').split('\r\n')) {
|
|
61
|
+
const separator = line.indexOf(':')
|
|
62
|
+
if (separator === -1) continue
|
|
63
|
+
headers[line.slice(0, separator).trim().toLowerCase()] = line.slice(separator + 1).trim()
|
|
64
|
+
}
|
|
65
|
+
const disposition = headers['content-disposition'] ?? ''
|
|
66
|
+
const name = /;\s*name="([^"]*)"/i.exec(disposition)?.[1]
|
|
67
|
+
const filenameMatch = /;\s*filename="([^"]*)"/i.exec(disposition)
|
|
68
|
+
return {
|
|
69
|
+
name,
|
|
70
|
+
filename: filenameMatch?.[1],
|
|
71
|
+
hasFilename: filenameMatch !== null,
|
|
72
|
+
contentType: headers['content-type'],
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
async function pumpFile(scanner, delimiter, out) {
|
|
77
|
+
const retain = delimiter.length - 1
|
|
78
|
+
for (;;) {
|
|
79
|
+
const index = scanner.buffer.indexOf(delimiter)
|
|
80
|
+
if (index !== -1) {
|
|
81
|
+
const tail = scanner.buffer.subarray(0, index)
|
|
82
|
+
scanner.buffer = scanner.buffer.subarray(index + delimiter.length)
|
|
83
|
+
if (tail.length && !out.write(tail)) await once(out, 'drain')
|
|
84
|
+
out.end()
|
|
85
|
+
return
|
|
86
|
+
}
|
|
87
|
+
// Hold back enough bytes that a delimiter split across chunks is still found.
|
|
88
|
+
if (scanner.buffer.length > retain) {
|
|
89
|
+
const emit = scanner.buffer.subarray(0, scanner.buffer.length - retain)
|
|
90
|
+
scanner.buffer = scanner.buffer.subarray(scanner.buffer.length - retain)
|
|
91
|
+
if (!out.write(emit)) await once(out, 'drain')
|
|
92
|
+
}
|
|
93
|
+
if (!(await scanner.fill())) {
|
|
94
|
+
out.destroy(new S3Error('MalformedPOSTRequest', 'The form data ended before the final boundary'))
|
|
95
|
+
return
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function once(emitter, event) {
|
|
101
|
+
return new Promise((resolve, reject) => {
|
|
102
|
+
emitter.once(event, resolve)
|
|
103
|
+
emitter.once('error', reject)
|
|
104
|
+
})
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function sourceIterator(sources) {
|
|
108
|
+
if (sources.length === 1) return sources[0][Symbol.asyncIterator]()
|
|
109
|
+
const merged = new PassThrough()
|
|
110
|
+
pipeline(...sources, merged).catch((err) => merged.destroy(err))
|
|
111
|
+
return merged[Symbol.asyncIterator]()
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Streaming multipart/form-data reader for browser POST uploads.
|
|
116
|
+
*
|
|
117
|
+
* Resolves as soon as the file part is reached, so the (possibly very large)
|
|
118
|
+
* file body is never buffered: it is handed back as a stream that keeps being
|
|
119
|
+
* fed while the caller writes it to storage. S3 requires the file field to come
|
|
120
|
+
* last, which is what makes this possible.
|
|
121
|
+
*/
|
|
122
|
+
export async function parseFormData(sources, {
|
|
123
|
+
boundary,
|
|
124
|
+
maxFieldBytes = DEFAULT_MAX_FIELD_BYTES,
|
|
125
|
+
maxFields = DEFAULT_MAX_FIELDS,
|
|
126
|
+
} = {}) {
|
|
127
|
+
const scanner = new Scanner(sourceIterator(Array.isArray(sources) ? sources : [sources]))
|
|
128
|
+
const dashBoundary = Buffer.from(`--${boundary}`)
|
|
129
|
+
const delimiter = Buffer.from(`\r\n--${boundary}`)
|
|
130
|
+
const fields = new Map()
|
|
131
|
+
|
|
132
|
+
// Skip the preamble up to the first boundary.
|
|
133
|
+
if ((await scanner.readUntil(dashBoundary, maxFieldBytes)) === null) {
|
|
134
|
+
throw new S3Error('MalformedPOSTRequest', 'No form boundary was found')
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
for (;;) {
|
|
138
|
+
if (!(await scanner.ensure(2))) {
|
|
139
|
+
throw new S3Error('MalformedPOSTRequest', 'Truncated form data')
|
|
140
|
+
}
|
|
141
|
+
if (scanner.buffer[0] === 0x2d && scanner.buffer[1] === 0x2d) {
|
|
142
|
+
return { fields, file: null } // closing "--"
|
|
143
|
+
}
|
|
144
|
+
scanner.buffer = scanner.buffer.subarray(2) // the CRLF after the boundary
|
|
145
|
+
|
|
146
|
+
const rawHeaders = await scanner.readUntil(Buffer.from('\r\n\r\n'), MAX_HEADER_BYTES)
|
|
147
|
+
if (rawHeaders === null) throw new S3Error('MalformedPOSTRequest', 'Truncated form part headers')
|
|
148
|
+
const part = parsePartHeaders(rawHeaders)
|
|
149
|
+
if (!part.name) throw new S3Error('MalformedPOSTRequest', 'A form part is missing its name')
|
|
150
|
+
|
|
151
|
+
if (part.hasFilename || part.name.toLowerCase() === 'file') {
|
|
152
|
+
const stream = new PassThrough()
|
|
153
|
+
// Deliberately not awaited: the caller consumes the stream while this
|
|
154
|
+
// keeps feeding it.
|
|
155
|
+
pumpFile(scanner, delimiter, stream).catch((err) => stream.destroy(err))
|
|
156
|
+
return {
|
|
157
|
+
fields,
|
|
158
|
+
file: { name: part.name, filename: part.filename ?? '', contentType: part.contentType, stream },
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
const value = await scanner.readUntil(delimiter, maxFieldBytes)
|
|
163
|
+
if (value === null) throw new S3Error('MalformedPOSTRequest', 'Truncated form field')
|
|
164
|
+
if (fields.size >= maxFields) {
|
|
165
|
+
throw new S3Error('MalformedPOSTRequest', `A form may carry at most ${maxFields} fields`)
|
|
166
|
+
}
|
|
167
|
+
fields.set(part.name.toLowerCase(), value.toString('utf8'))
|
|
168
|
+
}
|
|
169
|
+
}
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import { S3Error } from '../errors.js'
|
|
2
|
+
import { childNamed, childText, childrenNamed, document, parseXml, text } from '../xml.js'
|
|
3
|
+
import { toKeyBuffer } from '../util/bytes.js'
|
|
4
|
+
|
|
5
|
+
const DAY_MS = 24 * 60 * 60 * 1000
|
|
6
|
+
const MAX_RULES = 1000
|
|
7
|
+
|
|
8
|
+
function optionalInt(node, name) {
|
|
9
|
+
const value = childText(node, name)
|
|
10
|
+
if (value === undefined) return undefined
|
|
11
|
+
const parsed = Number.parseInt(value, 10)
|
|
12
|
+
if (!Number.isInteger(parsed) || parsed < 0) {
|
|
13
|
+
throw new S3Error('MalformedXML', `Invalid ${name}`)
|
|
14
|
+
}
|
|
15
|
+
return parsed
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function parseFilter(rule) {
|
|
19
|
+
// Rules may carry a bare <Prefix> (legacy) or a <Filter>.
|
|
20
|
+
const legacyPrefix = childText(rule, 'Prefix')
|
|
21
|
+
const filter = childNamed(rule, 'Filter')
|
|
22
|
+
if (!filter) return { prefix: legacyPrefix ?? '' }
|
|
23
|
+
const and = childNamed(filter, 'And')
|
|
24
|
+
const scope = and ?? filter
|
|
25
|
+
const tags = {}
|
|
26
|
+
for (const tag of childrenNamed(scope, 'Tag')) {
|
|
27
|
+
const key = childText(tag, 'Key')
|
|
28
|
+
if (key) tags[key] = childText(tag, 'Value') ?? ''
|
|
29
|
+
}
|
|
30
|
+
return { prefix: childText(scope, 'Prefix') ?? '', tags }
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function parseLifecycleXml(body) {
|
|
34
|
+
const root = parseXml(body)
|
|
35
|
+
if (root.name !== 'LifecycleConfiguration' && root.name !== 'BucketLifecycleConfiguration') {
|
|
36
|
+
throw new S3Error('MalformedXML', 'Expected a LifecycleConfiguration element')
|
|
37
|
+
}
|
|
38
|
+
const rules = childrenNamed(root, 'Rule').map((rule, index) => {
|
|
39
|
+
const status = childText(rule, 'Status')
|
|
40
|
+
if (status !== 'Enabled' && status !== 'Disabled') {
|
|
41
|
+
throw new S3Error('MalformedXML', 'Each Rule requires a Status of Enabled or Disabled')
|
|
42
|
+
}
|
|
43
|
+
const expiration = childNamed(rule, 'Expiration')
|
|
44
|
+
const noncurrent = childNamed(rule, 'NoncurrentVersionExpiration')
|
|
45
|
+
const abort = childNamed(rule, 'AbortIncompleteMultipartUpload')
|
|
46
|
+
if (!expiration && !noncurrent && !abort) {
|
|
47
|
+
throw new S3Error('MalformedXML', 'Each Rule requires at least one action')
|
|
48
|
+
}
|
|
49
|
+
const expirationDate = expiration ? childText(expiration, 'Date') : undefined
|
|
50
|
+
if (expirationDate !== undefined && Number.isNaN(Date.parse(expirationDate))) {
|
|
51
|
+
throw new S3Error('MalformedXML', 'Invalid Expiration Date')
|
|
52
|
+
}
|
|
53
|
+
return {
|
|
54
|
+
id: childText(rule, 'ID') ?? `rule-${index}`,
|
|
55
|
+
status,
|
|
56
|
+
filter: parseFilter(rule),
|
|
57
|
+
expirationDays: expiration ? optionalInt(expiration, 'Days') : undefined,
|
|
58
|
+
expirationDate,
|
|
59
|
+
expiredObjectDeleteMarker: expiration
|
|
60
|
+
? childText(expiration, 'ExpiredObjectDeleteMarker') === 'true'
|
|
61
|
+
: false,
|
|
62
|
+
noncurrentDays: noncurrent ? optionalInt(noncurrent, 'NoncurrentDays') : undefined,
|
|
63
|
+
abortAfterDays: abort ? optionalInt(abort, 'DaysAfterInitiation') : undefined,
|
|
64
|
+
}
|
|
65
|
+
})
|
|
66
|
+
if (rules.length === 0) throw new S3Error('MalformedXML', 'At least one Rule is required')
|
|
67
|
+
if (rules.length > MAX_RULES) {
|
|
68
|
+
throw new S3Error('MalformedXML', `A maximum of ${MAX_RULES} lifecycle rules is allowed`)
|
|
69
|
+
}
|
|
70
|
+
return { rules }
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export function lifecycleXml(config) {
|
|
74
|
+
const rules = (config?.rules ?? []).map((rule) => {
|
|
75
|
+
const filter = `<Filter>${rule.filter.prefix ? text('Prefix', rule.filter.prefix) : ''}${
|
|
76
|
+
Object.entries(rule.filter.tags ?? {})
|
|
77
|
+
.map(([key, value]) => `<Tag>${text('Key', key)}${text('Value', value)}</Tag>`).join('')
|
|
78
|
+
}</Filter>`
|
|
79
|
+
const expiration = rule.expirationDays !== undefined || rule.expirationDate || rule.expiredObjectDeleteMarker
|
|
80
|
+
? `<Expiration>${
|
|
81
|
+
rule.expirationDays !== undefined ? text('Days', rule.expirationDays) : ''
|
|
82
|
+
}${rule.expirationDate ? text('Date', rule.expirationDate) : ''
|
|
83
|
+
}${rule.expiredObjectDeleteMarker ? text('ExpiredObjectDeleteMarker', 'true') : ''}</Expiration>`
|
|
84
|
+
: ''
|
|
85
|
+
const noncurrent = rule.noncurrentDays !== undefined
|
|
86
|
+
? `<NoncurrentVersionExpiration>${text('NoncurrentDays', rule.noncurrentDays)}</NoncurrentVersionExpiration>`
|
|
87
|
+
: ''
|
|
88
|
+
const abort = rule.abortAfterDays !== undefined
|
|
89
|
+
? `<AbortIncompleteMultipartUpload>${
|
|
90
|
+
text('DaysAfterInitiation', rule.abortAfterDays)}</AbortIncompleteMultipartUpload>`
|
|
91
|
+
: ''
|
|
92
|
+
return `<Rule>${text('ID', rule.id)}${filter}${text('Status', rule.status)}${expiration}${noncurrent}${abort}</Rule>`
|
|
93
|
+
}).join('')
|
|
94
|
+
return document('LifecycleConfiguration', rules)
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function ruleMatches(rule, record) {
|
|
98
|
+
const prefix = toKeyBuffer(rule.filter.prefix ?? '')
|
|
99
|
+
if (prefix.length && !record.key.subarray(0, prefix.length).equals(prefix)) return false
|
|
100
|
+
for (const [key, value] of Object.entries(rule.filter.tags ?? {})) {
|
|
101
|
+
if ((record.tags ?? {})[key] !== value) return false
|
|
102
|
+
}
|
|
103
|
+
return true
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function isExpired(rule, timestamp, now) {
|
|
107
|
+
if (rule.expirationDate !== undefined) return now >= Date.parse(rule.expirationDate)
|
|
108
|
+
if (rule.expirationDays !== undefined) return now - timestamp >= rule.expirationDays * DAY_MS
|
|
109
|
+
return false
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Applies expiration rules across every bucket that has a lifecycle
|
|
114
|
+
* configuration. Returns a summary so callers can log or assert on it.
|
|
115
|
+
*
|
|
116
|
+
* Deliberately explicit rather than implicit: `ObjectStore` does not run this
|
|
117
|
+
* on its own, the server schedules it.
|
|
118
|
+
*/
|
|
119
|
+
export async function runLifecycle(store, { now = Date.now() } = {}) {
|
|
120
|
+
const summary = { expiredObjects: 0, expiredVersions: 0, abortedUploads: 0 }
|
|
121
|
+
|
|
122
|
+
for (const { bucket, value: config } of store.metadata.bucketsWithConfig('lifecycle')) {
|
|
123
|
+
if (!store.metadata.getBucket(bucket)) continue
|
|
124
|
+
const rules = (config.rules ?? []).filter((rule) => rule.status === 'Enabled')
|
|
125
|
+
if (rules.length === 0) continue
|
|
126
|
+
|
|
127
|
+
for (const rule of rules) {
|
|
128
|
+
if (rule.abortAfterDays !== undefined) {
|
|
129
|
+
const cutoff = now - rule.abortAfterDays * DAY_MS
|
|
130
|
+
for (const upload of store.metadata.uploadsStartedBefore(bucket, cutoff)) {
|
|
131
|
+
if (!ruleMatches(rule, { key: upload.key, tags: upload.tags })) continue
|
|
132
|
+
await store.abortMultipartUpload({
|
|
133
|
+
bucket, key: upload.key.toString('utf8'), uploadId: upload.uploadId,
|
|
134
|
+
})
|
|
135
|
+
summary.abortedUploads++
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// Walk every version once and let each rule vote on it.
|
|
141
|
+
for (const record of store.metadata.objectsModifiedBefore(bucket, now + 1)) {
|
|
142
|
+
for (const rule of rules) {
|
|
143
|
+
if (!ruleMatches(rule, record)) continue
|
|
144
|
+
|
|
145
|
+
if (!record.isLatest && rule.noncurrentDays !== undefined) {
|
|
146
|
+
if (now - record.lastModified.getTime() >= rule.noncurrentDays * DAY_MS) {
|
|
147
|
+
await store.deleteObject(bucket, record.key.toString('utf8'), record.versionId)
|
|
148
|
+
summary.expiredVersions++
|
|
149
|
+
break
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
if (record.isLatest && !record.isDeleteMarker &&
|
|
154
|
+
isExpired(rule, record.lastModified.getTime(), now)) {
|
|
155
|
+
await store.deleteObject(bucket, record.key.toString('utf8'))
|
|
156
|
+
summary.expiredObjects++
|
|
157
|
+
break
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
if (record.isLatest && record.isDeleteMarker && rule.expiredObjectDeleteMarker) {
|
|
161
|
+
const remaining = store.metadata.allVersionsOfKey(bucket, record.key)
|
|
162
|
+
if (remaining.length === 1) {
|
|
163
|
+
await store.deleteObject(bucket, record.key.toString('utf8'), record.versionId)
|
|
164
|
+
summary.expiredVersions++
|
|
165
|
+
break
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
return summary
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
export { DAY_MS }
|