@felipedsvit/s3node 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +143 -0
- package/bin/s3node.js +101 -0
- package/docs/plan.md +340 -0
- package/package.json +50 -0
- package/src/auth/chunked.js +277 -0
- package/src/auth/credentials.js +41 -0
- package/src/auth/sigv4.js +293 -0
- package/src/errors.js +65 -0
- package/src/features/cors.js +115 -0
- package/src/features/encryption.js +201 -0
- package/src/features/formdata.js +169 -0
- package/src/features/lifecycle.js +175 -0
- package/src/features/notifications.js +161 -0
- package/src/features/policy.js +0 -0
- package/src/features/postpolicy.js +150 -0
- package/src/features/tagging.js +64 -0
- package/src/handlers/bucket.js +197 -0
- package/src/handlers/config.js +129 -0
- package/src/handlers/index.js +6 -0
- package/src/handlers/multipart.js +109 -0
- package/src/handlers/object.js +211 -0
- package/src/handlers/post.js +106 -0
- package/src/handlers/service.js +10 -0
- package/src/handlers/shared.js +84 -0
- package/src/http.js +243 -0
- package/src/index.js +30 -0
- package/src/router.js +128 -0
- package/src/server.js +282 -0
- package/src/storage/blobs.js +135 -0
- package/src/storage/metadata.js +616 -0
- package/src/storage/store.js +548 -0
- package/src/util/bytes.js +35 -0
- package/src/util/crc.js +54 -0
- package/src/util/hash.js +64 -0
- package/src/xml.js +205 -0
package/src/http.js
ADDED
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
import { randomBytes } from 'node:crypto'
|
|
2
|
+
import { Writable } from 'node:stream'
|
|
3
|
+
import { pipeline } from 'node:stream/promises'
|
|
4
|
+
import { S3Error } from './errors.js'
|
|
5
|
+
import { errorDocument } from './xml.js'
|
|
6
|
+
|
|
7
|
+
export const MAX_XML_BODY_BYTES = 1024 * 1024
|
|
8
|
+
|
|
9
|
+
export function requestId() {
|
|
10
|
+
return randomBytes(8).toString('hex').toUpperCase()
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export function hostId() {
|
|
14
|
+
return randomBytes(24).toString('base64')
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/** RFC 1123, as required for Last-Modified and Date headers. */
|
|
18
|
+
export function httpDate(date) {
|
|
19
|
+
return date.toUTCString()
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** ISO 8601 with milliseconds, as required inside S3 XML payloads. */
|
|
23
|
+
export function isoDate(date) {
|
|
24
|
+
return date.toISOString()
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function safeDecode(value) {
|
|
28
|
+
try {
|
|
29
|
+
return decodeURIComponent(value)
|
|
30
|
+
} catch {
|
|
31
|
+
return value
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function parseQueryPairs(rawQuery) {
|
|
36
|
+
if (!rawQuery) return []
|
|
37
|
+
return rawQuery.split('&').filter(Boolean).map((pair) => {
|
|
38
|
+
const eq = pair.indexOf('=')
|
|
39
|
+
return eq === -1
|
|
40
|
+
? [safeDecode(pair), '']
|
|
41
|
+
: [safeDecode(pair.slice(0, eq)), safeDecode(pair.slice(eq + 1))]
|
|
42
|
+
})
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Resolve bucket and key.
|
|
47
|
+
*
|
|
48
|
+
* Virtual-host style is only attempted when a base domain is configured. SigV4
|
|
49
|
+
* signs the Host header, so a proxy that rewrites Host invalidates every
|
|
50
|
+
* signature — see docs/plan.md section 4.6.
|
|
51
|
+
*/
|
|
52
|
+
function resolveTarget(rawPath, headers, virtualHostDomain) {
|
|
53
|
+
const decodedPath = safeDecode(rawPath)
|
|
54
|
+
if (virtualHostDomain) {
|
|
55
|
+
const host = String(headers.host ?? '').split(':')[0].toLowerCase()
|
|
56
|
+
const suffix = `.${virtualHostDomain.toLowerCase()}`
|
|
57
|
+
if (host.endsWith(suffix) && host.length > suffix.length) {
|
|
58
|
+
return {
|
|
59
|
+
bucket: host.slice(0, -suffix.length),
|
|
60
|
+
key: decodedPath.length > 1 ? decodedPath.slice(1) : '',
|
|
61
|
+
style: 'virtual-host',
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
const trimmed = decodedPath.startsWith('/') ? decodedPath.slice(1) : decodedPath
|
|
66
|
+
if (trimmed === '') return { bucket: '', key: '', style: 'path' }
|
|
67
|
+
const slash = trimmed.indexOf('/')
|
|
68
|
+
return slash === -1
|
|
69
|
+
? { bucket: trimmed, key: '', style: 'path' }
|
|
70
|
+
: { bucket: trimmed.slice(0, slash), key: trimmed.slice(slash + 1), style: 'path' }
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export function createContext(req, { virtualHostDomain = null } = {}) {
|
|
74
|
+
const rawUrl = req.url ?? '/'
|
|
75
|
+
const queryStart = rawUrl.indexOf('?')
|
|
76
|
+
const rawPath = queryStart === -1 ? rawUrl : rawUrl.slice(0, queryStart)
|
|
77
|
+
const rawQuery = queryStart === -1 ? '' : rawUrl.slice(queryStart + 1)
|
|
78
|
+
const queryPairs = parseQueryPairs(rawQuery)
|
|
79
|
+
|
|
80
|
+
const query = new Map()
|
|
81
|
+
for (const [name, value] of queryPairs) if (!query.has(name)) query.set(name, value)
|
|
82
|
+
|
|
83
|
+
const { bucket, key, style } = resolveTarget(rawPath, req.headers, virtualHostDomain)
|
|
84
|
+
|
|
85
|
+
return {
|
|
86
|
+
req,
|
|
87
|
+
method: req.method,
|
|
88
|
+
rawPath,
|
|
89
|
+
rawQuery,
|
|
90
|
+
queryPairs,
|
|
91
|
+
query,
|
|
92
|
+
headers: req.headers,
|
|
93
|
+
bucket,
|
|
94
|
+
key,
|
|
95
|
+
style,
|
|
96
|
+
requestId: requestId(),
|
|
97
|
+
hostId: hostId(),
|
|
98
|
+
bodyStreams: [req],
|
|
99
|
+
trailers: null,
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** User metadata carried on `x-amz-meta-*` headers. */
|
|
104
|
+
export function userMetadata(headers) {
|
|
105
|
+
const metadata = {}
|
|
106
|
+
for (const [name, value] of Object.entries(headers)) {
|
|
107
|
+
if (name.startsWith('x-amz-meta-')) metadata[name.slice('x-amz-meta-'.length)] = String(value)
|
|
108
|
+
}
|
|
109
|
+
return metadata
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export async function collectBody(sources, maxBytes = MAX_XML_BODY_BYTES) {
|
|
113
|
+
const chunks = []
|
|
114
|
+
let total = 0
|
|
115
|
+
const sink = new Writable({
|
|
116
|
+
write(chunk, _encoding, callback) {
|
|
117
|
+
total += chunk.length
|
|
118
|
+
if (total > maxBytes) {
|
|
119
|
+
callback(new S3Error('MalformedXML', 'Request body exceeds the maximum accepted size'))
|
|
120
|
+
return
|
|
121
|
+
}
|
|
122
|
+
chunks.push(chunk)
|
|
123
|
+
callback()
|
|
124
|
+
},
|
|
125
|
+
})
|
|
126
|
+
await pipeline(...sources, sink)
|
|
127
|
+
return Buffer.concat(chunks)
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
export function baseHeaders(ctx) {
|
|
131
|
+
return {
|
|
132
|
+
'x-amz-request-id': ctx.requestId,
|
|
133
|
+
'x-amz-id-2': ctx.hostId,
|
|
134
|
+
Date: httpDate(new Date()),
|
|
135
|
+
Server: 'S3Node',
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
export function sendXml(ctx, res, status, body, extraHeaders = {}) {
|
|
140
|
+
const payload = Buffer.from(body, 'utf8')
|
|
141
|
+
res.writeHead(status, {
|
|
142
|
+
...baseHeaders(ctx),
|
|
143
|
+
'Content-Type': 'application/xml',
|
|
144
|
+
'Content-Length': payload.length,
|
|
145
|
+
...extraHeaders,
|
|
146
|
+
})
|
|
147
|
+
res.end(ctx.method === 'HEAD' ? undefined : payload)
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
export function sendEmpty(ctx, res, status, extraHeaders = {}) {
|
|
151
|
+
res.writeHead(status, { ...baseHeaders(ctx), 'Content-Length': 0, ...extraHeaders })
|
|
152
|
+
res.end()
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
export function sendError(ctx, res, error) {
|
|
156
|
+
const s3 = error instanceof S3Error ? error : new S3Error('InternalError', error?.message)
|
|
157
|
+
if (res.headersSent) {
|
|
158
|
+
res.destroy()
|
|
159
|
+
return s3
|
|
160
|
+
}
|
|
161
|
+
const body = errorDocument({
|
|
162
|
+
code: s3.code,
|
|
163
|
+
message: s3.message,
|
|
164
|
+
resource: ctx?.rawPath ?? '/',
|
|
165
|
+
requestId: ctx?.requestId ?? requestId(),
|
|
166
|
+
hostId: ctx?.hostId ?? hostId(),
|
|
167
|
+
})
|
|
168
|
+
const payload = Buffer.from(body, 'utf8')
|
|
169
|
+
const headers = {
|
|
170
|
+
'x-amz-request-id': ctx?.requestId ?? requestId(),
|
|
171
|
+
'x-amz-id-2': ctx?.hostId ?? hostId(),
|
|
172
|
+
Date: httpDate(new Date()),
|
|
173
|
+
Server: 'S3Node',
|
|
174
|
+
'Content-Type': 'application/xml',
|
|
175
|
+
'Content-Length': payload.length,
|
|
176
|
+
}
|
|
177
|
+
if (s3.detail?.contentRange) headers['Content-Range'] = s3.detail.contentRange
|
|
178
|
+
Object.assign(headers, s3.detail?.headers ?? {})
|
|
179
|
+
res.writeHead(s3.statusCode, headers)
|
|
180
|
+
// A 304 or a HEAD response must not carry a body.
|
|
181
|
+
res.end(ctx?.method === 'HEAD' || s3.statusCode === 304 ? undefined : payload)
|
|
182
|
+
return s3
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/** Parse a single-range `Range: bytes=` header. Returns null when absent. */
|
|
186
|
+
export function parseRange(header, size) {
|
|
187
|
+
if (!header) return null
|
|
188
|
+
const match = /^bytes=(\d*)-(\d*)$/.exec(String(header).trim())
|
|
189
|
+
if (!match) return null
|
|
190
|
+
const [, rawStart, rawEnd] = match
|
|
191
|
+
if (rawStart === '' && rawEnd === '') return null
|
|
192
|
+
|
|
193
|
+
let start
|
|
194
|
+
let end
|
|
195
|
+
if (rawStart === '') {
|
|
196
|
+
const suffix = Number.parseInt(rawEnd, 10)
|
|
197
|
+
if (suffix <= 0) throw new S3Error('InvalidRange', undefined, { contentRange: `bytes */${size}` })
|
|
198
|
+
start = Math.max(size - suffix, 0)
|
|
199
|
+
end = size - 1
|
|
200
|
+
} else {
|
|
201
|
+
start = Number.parseInt(rawStart, 10)
|
|
202
|
+
end = rawEnd === '' ? size - 1 : Number.parseInt(rawEnd, 10)
|
|
203
|
+
if (end >= size) end = size - 1
|
|
204
|
+
}
|
|
205
|
+
if (start >= size || start > end) {
|
|
206
|
+
throw new S3Error('InvalidRange', undefined, { contentRange: `bytes */${size}` })
|
|
207
|
+
}
|
|
208
|
+
return { start, end }
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
function etagMatches(candidate, etag) {
|
|
212
|
+
const normalize = (value) => value.trim().replace(/^W\//, '').replaceAll('"', '')
|
|
213
|
+
return candidate.split(',').some((entry) => {
|
|
214
|
+
const trimmed = entry.trim()
|
|
215
|
+
return trimmed === '*' || normalize(trimmed) === normalize(etag)
|
|
216
|
+
})
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* RFC 7232 precondition evaluation. Order matters: If-Match wins over
|
|
221
|
+
* If-Unmodified-Since, If-None-Match over If-Modified-Since.
|
|
222
|
+
*/
|
|
223
|
+
export function evaluatePreconditions(headers, { etag, lastModified }) {
|
|
224
|
+
const modifiedSeconds = Math.floor(lastModified.getTime() / 1000)
|
|
225
|
+
|
|
226
|
+
if (headers['if-match'] !== undefined) {
|
|
227
|
+
if (!etagMatches(headers['if-match'], etag)) throw new S3Error('PreconditionFailed')
|
|
228
|
+
} else if (headers['if-unmodified-since'] !== undefined) {
|
|
229
|
+
const since = Date.parse(headers['if-unmodified-since'])
|
|
230
|
+
if (Number.isFinite(since) && modifiedSeconds > Math.floor(since / 1000)) {
|
|
231
|
+
throw new S3Error('PreconditionFailed')
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
if (headers['if-none-match'] !== undefined) {
|
|
236
|
+
if (etagMatches(headers['if-none-match'], etag)) return 'not-modified'
|
|
237
|
+
} else if (headers['if-modified-since'] !== undefined) {
|
|
238
|
+
const since = Date.parse(headers['if-modified-since'])
|
|
239
|
+
if (Number.isFinite(since) && modifiedSeconds <= Math.floor(since / 1000)) return 'not-modified'
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
return 'ok'
|
|
243
|
+
}
|
package/src/index.js
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
export { S3NodeServer } from './server.js'
|
|
2
|
+
export { CredentialStore, generateCredential } from './auth/credentials.js'
|
|
3
|
+
export { ObjectStore } from './storage/store.js'
|
|
4
|
+
export { BlobStore } from './storage/blobs.js'
|
|
5
|
+
export { MetadataStore, NULL_VERSION, SCHEMA_VERSION } from './storage/metadata.js'
|
|
6
|
+
export { S3Error, ERROR_CODES } from './errors.js'
|
|
7
|
+
export { ChunkedDecoder, encodeChunked } from './auth/chunked.js'
|
|
8
|
+
export { EncryptionManager } from './features/encryption.js'
|
|
9
|
+
export { evaluatePolicy, parsePolicy } from './features/policy.js'
|
|
10
|
+
export { runLifecycle } from './features/lifecycle.js'
|
|
11
|
+
export { NotificationDispatcher, buildEvent } from './features/notifications.js'
|
|
12
|
+
export * as sigv4 from './auth/sigv4.js'
|
|
13
|
+
|
|
14
|
+
import { S3NodeServer } from './server.js'
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Create and start a server in one call.
|
|
18
|
+
*
|
|
19
|
+
* Running in-process is the point of shipping this on Node: tests can boot a
|
|
20
|
+
* real S3 endpoint without a container or a Go/Rust binary (docs/plan.md 5.5).
|
|
21
|
+
*
|
|
22
|
+
* const s3 = await createServer({ dataDir: './data', credentials: [...] })
|
|
23
|
+
* // s3.endpoint -> http://127.0.0.1:54321
|
|
24
|
+
*/
|
|
25
|
+
export async function createServer(options = {}) {
|
|
26
|
+
const server = await S3NodeServer.create(options)
|
|
27
|
+
const address = await server.listen(options.port ?? 0, options.host ?? '127.0.0.1')
|
|
28
|
+
server.endpoint = address.endpoint
|
|
29
|
+
return server
|
|
30
|
+
}
|
package/src/router.js
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { S3Error } from './errors.js'
|
|
2
|
+
import * as handlers from './handlers/index.js'
|
|
3
|
+
import { bucketArn, objectArn } from './features/policy.js'
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Sub-resources that are recognised but not implemented. Returning
|
|
7
|
+
* NotImplemented is deliberate: silently succeeding would let a client believe
|
|
8
|
+
* an ACL or a replication rule had been applied.
|
|
9
|
+
*/
|
|
10
|
+
const NOT_IMPLEMENTED_SUBRESOURCES = [
|
|
11
|
+
'acl', 'website', 'replication', 'encryption', 'object-lock', 'accelerate',
|
|
12
|
+
'logging', 'requestPayment', 'analytics', 'inventory', 'metrics', 'publicAccessBlock',
|
|
13
|
+
'intelligent-tiering', 'ownershipControls', 'legal-hold', 'retention', 'restore', 'select',
|
|
14
|
+
]
|
|
15
|
+
|
|
16
|
+
function rejectUnimplemented(ctx) {
|
|
17
|
+
for (const subresource of NOT_IMPLEMENTED_SUBRESOURCES) {
|
|
18
|
+
if (ctx.query.has(subresource)) {
|
|
19
|
+
throw new S3Error('NotImplemented', `The ${subresource} subresource is not implemented`)
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const route = (handler, action) => ({ handler, action })
|
|
25
|
+
|
|
26
|
+
function bucketRoute(ctx) {
|
|
27
|
+
const { method, query } = ctx
|
|
28
|
+
|
|
29
|
+
if (method === 'PUT') {
|
|
30
|
+
if (query.has('versioning')) return route(handlers.putBucketVersioning, 's3:PutBucketVersioning')
|
|
31
|
+
if (query.has('policy')) return route(handlers.putBucketPolicy, 's3:PutBucketPolicy')
|
|
32
|
+
if (query.has('cors')) return route(handlers.putBucketCors, 's3:PutBucketCORS')
|
|
33
|
+
if (query.has('lifecycle')) return route(handlers.putBucketLifecycle, 's3:PutLifecycleConfiguration')
|
|
34
|
+
if (query.has('tagging')) return route(handlers.putBucketTagging, 's3:PutBucketTagging')
|
|
35
|
+
if (query.has('notification')) return route(handlers.putBucketNotification, 's3:PutBucketNotification')
|
|
36
|
+
return route(handlers.createBucket, 's3:CreateBucket')
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
if (method === 'DELETE') {
|
|
40
|
+
if (query.has('policy')) return route(handlers.deleteBucketPolicy, 's3:DeleteBucketPolicy')
|
|
41
|
+
if (query.has('cors')) return route(handlers.deleteBucketCors, 's3:PutBucketCORS')
|
|
42
|
+
if (query.has('lifecycle')) return route(handlers.deleteBucketLifecycle, 's3:PutLifecycleConfiguration')
|
|
43
|
+
if (query.has('tagging')) return route(handlers.deleteBucketTagging, 's3:PutBucketTagging')
|
|
44
|
+
return route(handlers.deleteBucket, 's3:DeleteBucket')
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
if (method === 'HEAD') return route(handlers.headBucket, 's3:ListBucket')
|
|
48
|
+
|
|
49
|
+
if (method === 'POST') {
|
|
50
|
+
if (query.has('delete')) return route(handlers.deleteObjects, 's3:DeleteObject')
|
|
51
|
+
return route(handlers.postObject, 's3:PutObject')
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
if (method === 'GET') {
|
|
55
|
+
if (query.has('location')) return route(handlers.getBucketLocation, 's3:GetBucketLocation')
|
|
56
|
+
if (query.has('versioning')) return route(handlers.getBucketVersioning, 's3:GetBucketVersioning')
|
|
57
|
+
if (query.has('policy')) return route(handlers.getBucketPolicy, 's3:GetBucketPolicy')
|
|
58
|
+
if (query.has('cors')) return route(handlers.getBucketCors, 's3:GetBucketCORS')
|
|
59
|
+
if (query.has('lifecycle')) return route(handlers.getBucketLifecycle, 's3:GetLifecycleConfiguration')
|
|
60
|
+
if (query.has('tagging')) return route(handlers.getBucketTagging, 's3:GetBucketTagging')
|
|
61
|
+
if (query.has('notification')) return route(handlers.getBucketNotification, 's3:GetBucketNotification')
|
|
62
|
+
if (query.has('versions')) return route(handlers.listObjectVersions, 's3:ListBucketVersions')
|
|
63
|
+
if (query.has('uploads')) return route(handlers.listMultipartUploads, 's3:ListBucketMultipartUploads')
|
|
64
|
+
return query.get('list-type') === '2'
|
|
65
|
+
? route(handlers.listObjectsV2, 's3:ListBucket')
|
|
66
|
+
: route(handlers.listObjectsV1, 's3:ListBucket')
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
throw new S3Error('MethodNotAllowed')
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function objectRoute(ctx) {
|
|
73
|
+
const { method, query, headers } = ctx
|
|
74
|
+
const uploadId = query.get('uploadId')
|
|
75
|
+
|
|
76
|
+
if (method === 'PUT') {
|
|
77
|
+
if (query.has('tagging')) return route(handlers.putObjectTagging, 's3:PutObjectTagging')
|
|
78
|
+
if (uploadId && query.has('partNumber')) return route(handlers.uploadPart, 's3:PutObject')
|
|
79
|
+
if (headers['x-amz-copy-source']) return route(handlers.copyObject, 's3:PutObject')
|
|
80
|
+
return route(handlers.putObject, 's3:PutObject')
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
if (method === 'POST') {
|
|
84
|
+
if (query.has('uploads')) return route(handlers.createMultipartUpload, 's3:PutObject')
|
|
85
|
+
if (uploadId) return route(handlers.completeMultipartUpload, 's3:PutObject')
|
|
86
|
+
throw new S3Error('MethodNotAllowed')
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
if (method === 'GET') {
|
|
90
|
+
if (query.has('tagging')) return route(handlers.getObjectTagging, 's3:GetObjectTagging')
|
|
91
|
+
if (uploadId) return route(handlers.listParts, 's3:ListMultipartUploadParts')
|
|
92
|
+
return route(handlers.getObject, 's3:GetObject')
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
if (method === 'HEAD') return route(handlers.headObject, 's3:GetObject')
|
|
96
|
+
|
|
97
|
+
if (method === 'DELETE') {
|
|
98
|
+
if (query.has('tagging')) return route(handlers.deleteObjectTagging, 's3:DeleteObjectTagging')
|
|
99
|
+
if (uploadId) return route(handlers.abortMultipartUpload, 's3:AbortMultipartUpload')
|
|
100
|
+
return route(handlers.deleteObject, 's3:DeleteObject')
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
throw new S3Error('MethodNotAllowed')
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* S3 routing is unusual: sub-resources arrive as query-string flags rather than
|
|
108
|
+
* path segments, so dispatch is a decision tree rather than a path table
|
|
109
|
+
* (docs/plan.md section 8).
|
|
110
|
+
*
|
|
111
|
+
* Returns the handler alongside the IAM action it performs and the ARN it acts
|
|
112
|
+
* on, which is what bucket-policy evaluation needs.
|
|
113
|
+
*/
|
|
114
|
+
export function resolveRoute(ctx) {
|
|
115
|
+
if (!ctx.bucket) {
|
|
116
|
+
if (ctx.method === 'GET') {
|
|
117
|
+
return { ...route(handlers.listBuckets, 's3:ListAllMyBuckets'), resource: 'arn:aws:s3:::*' }
|
|
118
|
+
}
|
|
119
|
+
throw new S3Error('MethodNotAllowed')
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
rejectUnimplemented(ctx)
|
|
123
|
+
const resolved = ctx.key ? objectRoute(ctx) : bucketRoute(ctx)
|
|
124
|
+
return {
|
|
125
|
+
...resolved,
|
|
126
|
+
resource: ctx.key ? objectArn(ctx.bucket, ctx.key) : bucketArn(ctx.bucket),
|
|
127
|
+
}
|
|
128
|
+
}
|
package/src/server.js
ADDED
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
import { createServer as createHttpServer } from 'node:http'
|
|
2
|
+
import { ChunkedDecoder } from './auth/chunked.js'
|
|
3
|
+
import { CredentialStore } from './auth/credentials.js'
|
|
4
|
+
import {
|
|
5
|
+
STREAMING_PAYLOADS,
|
|
6
|
+
STREAMING_SIGNED,
|
|
7
|
+
STREAMING_SIGNED_TRAILER,
|
|
8
|
+
verifyRequest,
|
|
9
|
+
} from './auth/sigv4.js'
|
|
10
|
+
import { S3Error } from './errors.js'
|
|
11
|
+
import {
|
|
12
|
+
matchRule,
|
|
13
|
+
parseRequestHeaders,
|
|
14
|
+
preflightHeaders,
|
|
15
|
+
responseHeaders as corsResponseHeaders,
|
|
16
|
+
} from './features/cors.js'
|
|
17
|
+
import { runLifecycle } from './features/lifecycle.js'
|
|
18
|
+
import { NotificationDispatcher } from './features/notifications.js'
|
|
19
|
+
import { evaluatePolicy, objectArn } from './features/policy.js'
|
|
20
|
+
import { createContext, sendEmpty, sendError } from './http.js'
|
|
21
|
+
import { resolveRoute } from './router.js'
|
|
22
|
+
import { ObjectStore } from './storage/store.js'
|
|
23
|
+
|
|
24
|
+
const DEFAULT_REGION = 'us-east-1'
|
|
25
|
+
|
|
26
|
+
export class S3NodeServer {
|
|
27
|
+
constructor({
|
|
28
|
+
store, credentials, region = DEFAULT_REGION, virtualHostDomain = null, logger = null,
|
|
29
|
+
lifecycleIntervalMs = 0,
|
|
30
|
+
}) {
|
|
31
|
+
this.store = store
|
|
32
|
+
this.credentials = credentials
|
|
33
|
+
this.region = region
|
|
34
|
+
this.virtualHostDomain = virtualHostDomain
|
|
35
|
+
this.logger = logger
|
|
36
|
+
this.notifications = new NotificationDispatcher(store, { region, logger })
|
|
37
|
+
this.lifecycleTimer = null
|
|
38
|
+
this.http = createHttpServer()
|
|
39
|
+
|
|
40
|
+
this.http.on('request', (req, res) => {
|
|
41
|
+
if (req.headers.expect) return // handled by the checkContinue listener
|
|
42
|
+
this._handle(req, res, false)
|
|
43
|
+
})
|
|
44
|
+
this.http.on('checkContinue', (req, res) => {
|
|
45
|
+
this._handle(req, res, true)
|
|
46
|
+
})
|
|
47
|
+
this.http.on('clientError', (err, socket) => {
|
|
48
|
+
if (socket.writable) socket.end('HTTP/1.1 400 Bad Request\r\nConnection: close\r\n\r\n')
|
|
49
|
+
})
|
|
50
|
+
|
|
51
|
+
if (lifecycleIntervalMs > 0) {
|
|
52
|
+
this.lifecycleTimer = setInterval(() => {
|
|
53
|
+
this.runLifecycle().catch((err) =>
|
|
54
|
+
this.logger?.error?.({ message: 'lifecycle sweep failed', error: err.message }))
|
|
55
|
+
}, lifecycleIntervalMs)
|
|
56
|
+
this.lifecycleTimer.unref?.()
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
static async create({
|
|
61
|
+
dataDir,
|
|
62
|
+
credentials = [],
|
|
63
|
+
region = DEFAULT_REGION,
|
|
64
|
+
virtualHostDomain = null,
|
|
65
|
+
minPartSize,
|
|
66
|
+
encryptionMasterKey = null,
|
|
67
|
+
lifecycleIntervalMs = 0,
|
|
68
|
+
logger = null,
|
|
69
|
+
} = {}) {
|
|
70
|
+
if (!dataDir) throw new TypeError('dataDir is required')
|
|
71
|
+
const store = await ObjectStore.open({ dataDir, region, minPartSize, encryptionMasterKey })
|
|
72
|
+
return new S3NodeServer({
|
|
73
|
+
store,
|
|
74
|
+
credentials: credentials instanceof CredentialStore ? credentials : new CredentialStore(credentials),
|
|
75
|
+
region,
|
|
76
|
+
virtualHostDomain,
|
|
77
|
+
logger,
|
|
78
|
+
lifecycleIntervalMs,
|
|
79
|
+
})
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** Applies expiration rules once. Also exposed so callers can trigger it. */
|
|
83
|
+
runLifecycle(options) {
|
|
84
|
+
return runLifecycle(this.store, options)
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* `Expect: 100-continue` is answered only after the request is both
|
|
89
|
+
* authenticated and authorised, so a rejected client is turned away before it
|
|
90
|
+
* uploads the body rather than after (docs/plan.md section 4.8).
|
|
91
|
+
*/
|
|
92
|
+
async _handle(req, res, expectContinue) {
|
|
93
|
+
let ctx = null
|
|
94
|
+
try {
|
|
95
|
+
ctx = createContext(req, { virtualHostDomain: this.virtualHostDomain })
|
|
96
|
+
|
|
97
|
+
if (req.method === 'OPTIONS') {
|
|
98
|
+
this._handlePreflight(ctx, res)
|
|
99
|
+
return
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
const route = resolveRoute(ctx)
|
|
103
|
+
// A browser form upload carries its credential and signature in the form
|
|
104
|
+
// body, so it authenticates itself once the body has been parsed.
|
|
105
|
+
const selfAuthenticating = route.handler.name === 'postObject'
|
|
106
|
+
|
|
107
|
+
ctx.auth = selfAuthenticating
|
|
108
|
+
? { anonymous: false, deferred: true, payloadHash: null }
|
|
109
|
+
: this._authenticate(ctx)
|
|
110
|
+
|
|
111
|
+
if (!selfAuthenticating) this.authorize(ctx, route.action, route.resource)
|
|
112
|
+
|
|
113
|
+
if (expectContinue) res.writeContinue()
|
|
114
|
+
|
|
115
|
+
this._attachBody(ctx)
|
|
116
|
+
this._applyCors(ctx, res)
|
|
117
|
+
await route.handler(ctx, res, { store: this.store, server: this })
|
|
118
|
+
} catch (err) {
|
|
119
|
+
const rendered = sendError(ctx, res, err)
|
|
120
|
+
this.logger?.error?.({
|
|
121
|
+
requestId: ctx?.requestId,
|
|
122
|
+
method: req.method,
|
|
123
|
+
url: req.url,
|
|
124
|
+
code: rendered.code,
|
|
125
|
+
message: rendered.message,
|
|
126
|
+
canonicalRequest: rendered.detail?.canonicalRequest,
|
|
127
|
+
stringToSign: rendered.detail?.stringToSign,
|
|
128
|
+
})
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
_authenticate(ctx) {
|
|
133
|
+
const signed = ctx.headers.authorization || ctx.query.get('X-Amz-Signature') !== undefined
|
|
134
|
+
if (!signed) {
|
|
135
|
+
// Anonymous is allowed to reach policy evaluation; the default there is
|
|
136
|
+
// deny, so an unauthenticated request only proceeds if a bucket policy
|
|
137
|
+
// explicitly permits it.
|
|
138
|
+
return {
|
|
139
|
+
anonymous: true,
|
|
140
|
+
payloadHash: ctx.headers['x-amz-content-sha256'] ?? 'UNSIGNED-PAYLOAD',
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
return {
|
|
144
|
+
anonymous: false,
|
|
145
|
+
...verifyRequest(ctx, { lookupCredential: this.credentials.lookup, region: this.region }),
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
_bucketConfig(bucket, name) {
|
|
150
|
+
if (!bucket) return null
|
|
151
|
+
try {
|
|
152
|
+
return this.store.metadata.getConfig(bucket, name)
|
|
153
|
+
} catch {
|
|
154
|
+
return null
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
_conditionContext(ctx) {
|
|
159
|
+
const socket = ctx.req.socket
|
|
160
|
+
return {
|
|
161
|
+
principal: ctx.auth.anonymous ? '*' : `arn:aws:iam::s3node:user/${ctx.auth.accessKeyId}`,
|
|
162
|
+
'aws:sourceip': socket?.remoteAddress ?? '',
|
|
163
|
+
'aws:securetransport': String(Boolean(socket?.encrypted)),
|
|
164
|
+
'aws:useragent': ctx.headers['user-agent'],
|
|
165
|
+
'aws:referer': ctx.headers.referer,
|
|
166
|
+
'aws:principaltype': ctx.auth.anonymous ? 'Anonymous' : 'User',
|
|
167
|
+
's3:prefix': ctx.query.get('prefix'),
|
|
168
|
+
's3:delimiter': ctx.query.get('delimiter'),
|
|
169
|
+
's3:max-keys': ctx.query.get('max-keys'),
|
|
170
|
+
's3:x-amz-acl': ctx.headers['x-amz-acl'],
|
|
171
|
+
's3:x-amz-server-side-encryption': ctx.headers['x-amz-server-side-encryption'],
|
|
172
|
+
's3:x-amz-content-sha256': ctx.headers['x-amz-content-sha256'],
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* Bucket policy evaluation. Explicit Deny always wins; when the policy is
|
|
178
|
+
* silent the holder of a valid credential is allowed and anonymous is not.
|
|
179
|
+
*/
|
|
180
|
+
authorize(ctx, action, resource, overrides = {}) {
|
|
181
|
+
const policy = this._bucketConfig(ctx.bucket, 'policy')
|
|
182
|
+
const context = { ...this._conditionContext(ctx), ...overrides }
|
|
183
|
+
const decision = evaluatePolicy(policy, { action, resource, context })
|
|
184
|
+
|
|
185
|
+
if (decision === 'Deny') throw new S3Error('AccessDenied')
|
|
186
|
+
if (decision === 'Allow') return
|
|
187
|
+
if (ctx.auth.anonymous) throw new S3Error('AccessDenied')
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/** Used by the POST upload handler once the form has revealed its principal. */
|
|
191
|
+
authorizePostUpload(ctx, key, accessKeyId) {
|
|
192
|
+
const previous = ctx.auth
|
|
193
|
+
ctx.auth = { anonymous: false, accessKeyId }
|
|
194
|
+
try {
|
|
195
|
+
this.authorize(ctx, 's3:PutObject', objectArn(ctx.bucket, key))
|
|
196
|
+
} finally {
|
|
197
|
+
ctx.auth = previous
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/* -------------------------------- CORS -------------------------------- */
|
|
202
|
+
|
|
203
|
+
_handlePreflight(ctx, res) {
|
|
204
|
+
const origin = ctx.headers.origin
|
|
205
|
+
const method = ctx.headers['access-control-request-method']
|
|
206
|
+
if (!origin || !method) {
|
|
207
|
+
throw new S3Error('InvalidRequest', 'This is not a valid CORS preflight request')
|
|
208
|
+
}
|
|
209
|
+
const requestHeaders = parseRequestHeaders(ctx.headers['access-control-request-headers'])
|
|
210
|
+
const rule = matchRule(this._bucketConfig(ctx.bucket, 'cors'), {
|
|
211
|
+
origin, method: String(method).toUpperCase(), requestHeaders,
|
|
212
|
+
})
|
|
213
|
+
// Preflight is intentionally unauthenticated: browsers never attach
|
|
214
|
+
// credentials to it.
|
|
215
|
+
if (!rule) throw new S3Error('AccessForbidden')
|
|
216
|
+
sendEmpty(ctx, res, 200, preflightHeaders(rule, origin, requestHeaders))
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
_applyCors(ctx, res) {
|
|
220
|
+
const origin = ctx.headers.origin
|
|
221
|
+
if (!origin || !ctx.bucket) return
|
|
222
|
+
const rule = matchRule(this._bucketConfig(ctx.bucket, 'cors'), { origin, method: ctx.method })
|
|
223
|
+
// Headers set here survive writeHead(): Node merges them, with writeHead's
|
|
224
|
+
// own object taking precedence.
|
|
225
|
+
for (const [name, value] of Object.entries(corsResponseHeaders(rule, origin))) {
|
|
226
|
+
res.setHeader(name, value)
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Wraps the request in the `aws-chunked` decoder when the payload is
|
|
232
|
+
* streamed. Without this the framing bytes are written into the object and
|
|
233
|
+
* every aws-cli upload is silently corrupted (docs/plan.md section 4.1).
|
|
234
|
+
*/
|
|
235
|
+
_attachBody(ctx) {
|
|
236
|
+
const payloadHash = ctx.auth.payloadHash
|
|
237
|
+
if (!payloadHash || !STREAMING_PAYLOADS.has(payloadHash)) return
|
|
238
|
+
|
|
239
|
+
const declaredLength = ctx.headers['x-amz-decoded-content-length']
|
|
240
|
+
const expectedLength = declaredLength === undefined ? null : Number.parseInt(declaredLength, 10)
|
|
241
|
+
if (declaredLength !== undefined && !Number.isInteger(expectedLength)) {
|
|
242
|
+
throw new S3Error('InvalidArgument', 'Invalid x-amz-decoded-content-length')
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
const decoder = new ChunkedDecoder({
|
|
246
|
+
signed: !ctx.auth.anonymous &&
|
|
247
|
+
(payloadHash === STREAMING_SIGNED || payloadHash === STREAMING_SIGNED_TRAILER),
|
|
248
|
+
seedSignature: ctx.auth.seedSignature,
|
|
249
|
+
signingKey: ctx.auth.signingKey,
|
|
250
|
+
scope: ctx.auth.scope,
|
|
251
|
+
amzDate: ctx.auth.amzDate,
|
|
252
|
+
expectedLength,
|
|
253
|
+
})
|
|
254
|
+
|
|
255
|
+
ctx.trailers = decoder.trailers
|
|
256
|
+
ctx.bodyStreams = [ctx.req, decoder]
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
listen(port = 0, host = '127.0.0.1') {
|
|
260
|
+
return new Promise((resolve, reject) => {
|
|
261
|
+
this.http.once('error', reject)
|
|
262
|
+
this.http.listen(port, host, () => {
|
|
263
|
+
this.http.removeListener('error', reject)
|
|
264
|
+
resolve(this.address())
|
|
265
|
+
})
|
|
266
|
+
})
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
address() {
|
|
270
|
+
const address = this.http.address()
|
|
271
|
+
if (!address) return null
|
|
272
|
+
const host = address.family === 'IPv6' ? `[${address.address}]` : address.address
|
|
273
|
+
return { ...address, endpoint: `http://${host}:${address.port}` }
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
async close() {
|
|
277
|
+
if (this.lifecycleTimer) clearInterval(this.lifecycleTimer)
|
|
278
|
+
await new Promise((resolve) => this.http.close(resolve))
|
|
279
|
+
await this.notifications.drain()
|
|
280
|
+
this.store.close()
|
|
281
|
+
}
|
|
282
|
+
}
|