@carlonicora/nestjs-neo4jsonapi 4.17.3 → 4.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bootstrap/bootstrap.d.ts.map +1 -1
- package/dist/bootstrap/bootstrap.js +5 -25
- package/dist/bootstrap/bootstrap.js.map +1 -1
- package/dist/bootstrap/defaults.d.ts +1 -0
- package/dist/bootstrap/defaults.d.ts.map +1 -1
- package/dist/bootstrap/defaults.js +3 -0
- package/dist/bootstrap/defaults.js.map +1 -1
- package/dist/bootstrap/graceful-shutdown.d.ts +25 -0
- package/dist/bootstrap/graceful-shutdown.d.ts.map +1 -0
- package/dist/bootstrap/graceful-shutdown.js +72 -0
- package/dist/bootstrap/graceful-shutdown.js.map +1 -0
- package/dist/bootstrap/raw-body.config.d.ts +13 -0
- package/dist/bootstrap/raw-body.config.d.ts.map +1 -1
- package/dist/bootstrap/raw-body.config.js +26 -1
- package/dist/bootstrap/raw-body.config.js.map +1 -1
- package/dist/config/base.config.d.ts.map +1 -1
- package/dist/config/base.config.js +7 -0
- package/dist/config/base.config.js.map +1 -1
- package/dist/config/interfaces/config.s3.interface.d.ts +11 -0
- package/dist/config/interfaces/config.s3.interface.d.ts.map +1 -1
- package/dist/core/health/indicators/s3.health.d.ts.map +1 -1
- package/dist/core/health/indicators/s3.health.js +8 -38
- package/dist/core/health/indicators/s3.health.js.map +1 -1
- package/dist/core/llm/services/ai-connection-resolver.service.d.ts +8 -2
- package/dist/core/llm/services/ai-connection-resolver.service.d.ts.map +1 -1
- package/dist/core/llm/services/ai-connection-resolver.service.js +9 -0
- package/dist/core/llm/services/ai-connection-resolver.service.js.map +1 -1
- package/dist/foundations/chunker/services/chunker.service.d.ts +12 -4
- package/dist/foundations/chunker/services/chunker.service.d.ts.map +1 -1
- package/dist/foundations/chunker/services/chunker.service.js +22 -14
- package/dist/foundations/chunker/services/chunker.service.js.map +1 -1
- package/dist/foundations/s3/cipher/index.d.ts +2 -0
- package/dist/foundations/s3/cipher/index.d.ts.map +1 -0
- package/dist/foundations/s3/cipher/index.js +18 -0
- package/dist/foundations/s3/cipher/index.js.map +1 -0
- package/dist/foundations/s3/cipher/s3-cipher.core.d.ts +85 -0
- package/dist/foundations/s3/cipher/s3-cipher.core.d.ts.map +1 -0
- package/dist/foundations/s3/cipher/s3-cipher.core.js +359 -0
- package/dist/foundations/s3/cipher/s3-cipher.core.js.map +1 -0
- package/dist/foundations/s3/controllers/s3.controller.d.ts +25 -1
- package/dist/foundations/s3/controllers/s3.controller.d.ts.map +1 -1
- package/dist/foundations/s3/controllers/s3.controller.js +143 -4
- package/dist/foundations/s3/controllers/s3.controller.js.map +1 -1
- package/dist/foundations/s3/entities/s3.meta.d.ts +2 -0
- package/dist/foundations/s3/entities/s3.meta.d.ts.map +1 -1
- package/dist/foundations/s3/entities/s3.meta.js +3 -1
- package/dist/foundations/s3/entities/s3.meta.js.map +1 -1
- package/dist/foundations/s3/index.d.ts +3 -0
- package/dist/foundations/s3/index.d.ts.map +1 -1
- package/dist/foundations/s3/index.js +3 -0
- package/dist/foundations/s3/index.js.map +1 -1
- package/dist/foundations/s3/s3.service.module.d.ts +1 -1
- package/dist/foundations/s3/s3.service.module.d.ts.map +1 -1
- package/dist/foundations/s3/s3.service.module.js +4 -3
- package/dist/foundations/s3/s3.service.module.js.map +1 -1
- package/dist/foundations/s3/services/s3-cipher.service.d.ts +33 -0
- package/dist/foundations/s3/services/s3-cipher.service.d.ts.map +1 -0
- package/dist/foundations/s3/services/s3-cipher.service.js +66 -0
- package/dist/foundations/s3/services/s3-cipher.service.js.map +1 -0
- package/dist/foundations/s3/services/s3.service.d.ts +37 -5
- package/dist/foundations/s3/services/s3.service.d.ts.map +1 -1
- package/dist/foundations/s3/services/s3.service.js +274 -207
- package/dist/foundations/s3/services/s3.service.js.map +1 -1
- package/dist/foundations/s3/storage-client.d.ts +8 -0
- package/dist/foundations/s3/storage-client.d.ts.map +1 -0
- package/dist/foundations/s3/storage-client.js +45 -0
- package/dist/foundations/s3/storage-client.js.map +1 -0
- package/package.json +2 -1
|
@@ -12,7 +12,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
12
12
|
exports.S3Service = void 0;
|
|
13
13
|
const client_s3_1 = require("@aws-sdk/client-s3");
|
|
14
14
|
const lib_storage_1 = require("@aws-sdk/lib-storage");
|
|
15
|
-
const
|
|
15
|
+
const stream_1 = require("stream");
|
|
16
16
|
const storage_blob_1 = require("@azure/storage-blob");
|
|
17
17
|
const common_1 = require("@nestjs/common");
|
|
18
18
|
const config_1 = require("@nestjs/config");
|
|
@@ -21,16 +21,117 @@ const nestjs_cls_1 = require("nestjs-cls");
|
|
|
21
21
|
const jsonapi_service_1 = require("../../../core/jsonapi/services/jsonapi.service");
|
|
22
22
|
const logging_service_1 = require("../../../core/logging/services/logging.service");
|
|
23
23
|
const s3_model_1 = require("../../s3/entities/s3.model");
|
|
24
|
+
const s3_meta_1 = require("../../s3/entities/s3.meta");
|
|
25
|
+
const s3_cipher_core_1 = require("../cipher/s3-cipher.core");
|
|
26
|
+
const s3_cipher_service_1 = require("./s3-cipher.service");
|
|
24
27
|
let S3Service = class S3Service {
|
|
25
|
-
constructor(builder, clsService, configService, logger) {
|
|
28
|
+
constructor(builder, clsService, configService, logger, cipher) {
|
|
26
29
|
this.builder = builder;
|
|
27
30
|
this.clsService = clsService;
|
|
28
31
|
this.configService = configService;
|
|
29
32
|
this.logger = logger;
|
|
33
|
+
this.cipher = cipher;
|
|
30
34
|
}
|
|
31
35
|
get s3Config() {
|
|
32
36
|
return this.configService.get("s3");
|
|
33
37
|
}
|
|
38
|
+
/** Token-verified api URL served by S3Controller: `${api.url}s3/objects/<token>`. */
|
|
39
|
+
_apiObjectUrl(token) {
|
|
40
|
+
return `${this.configService.get("api").url}${s3_meta_1.s3Meta.endpoint}/${s3_meta_1.S3_OBJECTS_SEGMENT}/${token}`;
|
|
41
|
+
}
|
|
42
|
+
/** Object metadata flag for an encrypted write, or undefined when the key is stored as plaintext. */
|
|
43
|
+
_metadataFor(key) {
|
|
44
|
+
return this.cipher.shouldEncrypt(key) ? { [s3_cipher_core_1.S3_ENCRYPTION_METADATA_KEY]: s3_cipher_core_1.S3_ENCRYPTION_METADATA_VALUE } : undefined;
|
|
45
|
+
}
|
|
46
|
+
/** Body and metadata to write for `key`: ciphertext plus the flag when encrypting, the buffer untouched otherwise. */
|
|
47
|
+
async _prepareBody(key, buffer) {
|
|
48
|
+
const metadata = this._metadataFor(key);
|
|
49
|
+
if (!metadata)
|
|
50
|
+
return { body: buffer, metadata: undefined };
|
|
51
|
+
return { body: await this.cipher.encrypt(buffer), metadata };
|
|
52
|
+
}
|
|
53
|
+
/** Throws when a flagged object cannot be opened: ciphertext is never returned to a caller. */
|
|
54
|
+
_assertCanDecrypt(key) {
|
|
55
|
+
if (!this.cipher.canDecrypt)
|
|
56
|
+
throw new s3_cipher_core_1.S3CipherKeyMissingError([], key);
|
|
57
|
+
}
|
|
58
|
+
/** The cipher does not know the storage key; name it in a missing-key error. */
|
|
59
|
+
_withObjectKey(error, key) {
|
|
60
|
+
if (error instanceof s3_cipher_core_1.S3CipherKeyMissingError && !error.objectKey) {
|
|
61
|
+
return new s3_cipher_core_1.S3CipherKeyMissingError(error.keyIds, key);
|
|
62
|
+
}
|
|
63
|
+
return error;
|
|
64
|
+
}
|
|
65
|
+
/** Decrypts the stored bytes when the object metadata (S3 `Metadata` / Azure `metadata`) carries the flag. */
|
|
66
|
+
async _readBody(key, metadata, bytes) {
|
|
67
|
+
if (!(0, s3_cipher_core_1.isEncryptedObjectMetadata)(metadata))
|
|
68
|
+
return bytes;
|
|
69
|
+
this._assertCanDecrypt(key);
|
|
70
|
+
try {
|
|
71
|
+
return await this.cipher.decrypt(bytes);
|
|
72
|
+
}
|
|
73
|
+
catch (error) {
|
|
74
|
+
throw this._withObjectKey(error, key);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
/** Streaming counterpart of `_readBody`: a flagged object is piped through the cipher's decrypt stream. */
|
|
78
|
+
_readStream(key, metadata, stream) {
|
|
79
|
+
if (!(0, s3_cipher_core_1.isEncryptedObjectMetadata)(metadata))
|
|
80
|
+
return stream;
|
|
81
|
+
this._assertCanDecrypt(key);
|
|
82
|
+
const decrypting = this.cipher.decryptStream();
|
|
83
|
+
(0, stream_1.pipeline)(stream, decrypting, (error) => {
|
|
84
|
+
if (error)
|
|
85
|
+
this.logger.error(`Failed to decrypt object stream ${key}`, this._withObjectKey(error, key));
|
|
86
|
+
});
|
|
87
|
+
return decrypting;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* `_readStream`, but for a flagged object it waits for the first decrypted chunk (or the end), so a
|
|
91
|
+
* failure before any byte (a key that is not configured) rejects here, before a response starts,
|
|
92
|
+
* instead of breaking a response that is already under way.
|
|
93
|
+
*/
|
|
94
|
+
async _readStreamReady(key, metadata, body) {
|
|
95
|
+
const stream = this._readStream(key, metadata, body);
|
|
96
|
+
if (stream === body)
|
|
97
|
+
return stream;
|
|
98
|
+
try {
|
|
99
|
+
return await new Promise((resolve, reject) => {
|
|
100
|
+
const cleanup = () => {
|
|
101
|
+
stream.off("data", onData);
|
|
102
|
+
stream.off("end", onEnd);
|
|
103
|
+
stream.off("error", onError);
|
|
104
|
+
};
|
|
105
|
+
const onData = (chunk) => {
|
|
106
|
+
cleanup();
|
|
107
|
+
stream.pause();
|
|
108
|
+
stream.unshift(chunk);
|
|
109
|
+
resolve(stream);
|
|
110
|
+
};
|
|
111
|
+
const onEnd = () => {
|
|
112
|
+
cleanup();
|
|
113
|
+
resolve(stream_1.Readable.from([]));
|
|
114
|
+
};
|
|
115
|
+
const onError = (error) => {
|
|
116
|
+
cleanup();
|
|
117
|
+
reject(error);
|
|
118
|
+
};
|
|
119
|
+
stream.on("data", onData);
|
|
120
|
+
stream.on("end", onEnd);
|
|
121
|
+
stream.on("error", onError);
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
catch (error) {
|
|
125
|
+
throw this._withObjectKey(error, key);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
async _streamToBuffer(stream) {
|
|
129
|
+
const chunks = [];
|
|
130
|
+
for await (const chunk of stream) {
|
|
131
|
+
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
|
132
|
+
}
|
|
133
|
+
return Buffer.concat(chunks);
|
|
134
|
+
}
|
|
34
135
|
async _loadConfiguration() {
|
|
35
136
|
const type = this.s3Config.type;
|
|
36
137
|
this._storageType = type;
|
|
@@ -102,61 +203,25 @@ let S3Service = class S3Service {
|
|
|
102
203
|
break;
|
|
103
204
|
}
|
|
104
205
|
}
|
|
105
|
-
async _prepareUrl(params) {
|
|
106
|
-
if (this._storageType === "azure")
|
|
107
|
-
return await this._prepareAzureUrl(params);
|
|
108
|
-
const isPublic = params.isPublic ?? false;
|
|
109
|
-
const command = isPublic
|
|
110
|
-
? new client_s3_1.PutObjectCommand({
|
|
111
|
-
Bucket: this._bucket,
|
|
112
|
-
Key: params.key,
|
|
113
|
-
ContentType: params.contentType,
|
|
114
|
-
ACL: "public-read",
|
|
115
|
-
})
|
|
116
|
-
: new client_s3_1.PutObjectCommand({
|
|
117
|
-
Bucket: this._bucket,
|
|
118
|
-
ContentType: params.contentType,
|
|
119
|
-
Key: params.key,
|
|
120
|
-
});
|
|
121
|
-
const signedUrl = await (0, s3_request_presigner_1.getSignedUrl)(this.s3Client, command, {
|
|
122
|
-
expiresIn: 3600,
|
|
123
|
-
});
|
|
124
|
-
return signedUrl;
|
|
125
|
-
}
|
|
126
|
-
async _prepareAzureUrl(params) {
|
|
127
|
-
const blobClient = this.containerClient.getBlockBlobClient(params.key);
|
|
128
|
-
// For Azure, we need to check if we have shared key credentials to generate SAS
|
|
129
|
-
const credential = this.blobServiceClient.credential;
|
|
130
|
-
if (credential instanceof storage_blob_1.StorageSharedKeyCredential) {
|
|
131
|
-
const sasToken = (0, storage_blob_1.generateBlobSASQueryParameters)({
|
|
132
|
-
containerName: this._bucket,
|
|
133
|
-
blobName: params.key,
|
|
134
|
-
permissions: storage_blob_1.BlobSASPermissions.parse("racw"), // Read, Add, Create, Write permissions
|
|
135
|
-
startsOn: new Date(new Date().valueOf() - 15 * 60 * 1000), // Start 15 minutes ago to account for clock skew
|
|
136
|
-
expiresOn: new Date(new Date().valueOf() + 3600 * 1000), // 1 hour from now
|
|
137
|
-
protocol: storage_blob_1.SASProtocol.Https,
|
|
138
|
-
contentType: params.contentType, // This helps Azure validate the content type
|
|
139
|
-
}, credential);
|
|
140
|
-
return `${blobClient.url}?${sasToken}`;
|
|
141
|
-
}
|
|
142
|
-
else {
|
|
143
|
-
// If we don't have shared key credentials, we can't generate SAS tokens
|
|
144
|
-
// This would happen if using connection string with SAS or other auth methods
|
|
145
|
-
throw new Error("Cannot generate SAS token: Shared key credentials required for presigned URLs");
|
|
146
|
-
}
|
|
147
|
-
}
|
|
148
206
|
async generatePresignedUrl(params) {
|
|
149
207
|
await this._loadConfiguration();
|
|
150
208
|
if (!this._endpoint)
|
|
151
209
|
return null;
|
|
152
|
-
|
|
210
|
+
// Uploads always go through the api: S3Controller writes the bytes, encrypting them when
|
|
211
|
+
// `shouldEncrypt`, passing them through otherwise. Put tokens are not windowed.
|
|
212
|
+
const token = this.cipher.signObjectToken({
|
|
213
|
+
op: "put",
|
|
214
|
+
key: params.key,
|
|
215
|
+
contentType: params.contentType,
|
|
216
|
+
expiresInSeconds: s3_cipher_core_1.S3_OBJECT_TOKEN_TTL_SECONDS,
|
|
217
|
+
});
|
|
153
218
|
const data = {
|
|
154
219
|
id: (0, crypto_1.randomUUID)(),
|
|
155
|
-
url:
|
|
156
|
-
storageType:
|
|
220
|
+
url: this._apiObjectUrl(token),
|
|
221
|
+
storageType: "api",
|
|
157
222
|
contentType: params.contentType,
|
|
158
|
-
blobType:
|
|
159
|
-
acl:
|
|
223
|
+
blobType: undefined,
|
|
224
|
+
acl: undefined,
|
|
160
225
|
createdAt: new Date(),
|
|
161
226
|
updatedAt: new Date(),
|
|
162
227
|
};
|
|
@@ -241,7 +306,7 @@ let S3Service = class S3Service {
|
|
|
241
306
|
* characters that would break the header (quotes, path separators, control chars) while
|
|
242
307
|
* `filename*` carries the original UTF-8 name for browsers that support it.
|
|
243
308
|
*/
|
|
244
|
-
|
|
309
|
+
contentDispositionAttachment(filename) {
|
|
245
310
|
const ascii = filename.replace(/[^\x20-\x7e]/g, "_").replace(/["\\/\r\n]/g, "_");
|
|
246
311
|
return `attachment; filename="${ascii}"; filename*=UTF-8''${encodeURIComponent(filename)}`;
|
|
247
312
|
}
|
|
@@ -260,69 +325,25 @@ let S3Service = class S3Service {
|
|
|
260
325
|
await this._loadConfiguration();
|
|
261
326
|
if (!this._endpoint)
|
|
262
327
|
return null;
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
...(params.filename
|
|
276
|
-
? { ResponseContentDisposition: this._contentDispositionAttachment(params.filename) }
|
|
277
|
-
: {}),
|
|
278
|
-
});
|
|
279
|
-
// Sign against the start of the current window so the same key yields the same URL
|
|
280
|
-
// for the whole window and browser/image caches can hit. The window is added to the
|
|
281
|
-
// expiry so the URL stays valid for at least the requested ttl from now.
|
|
282
|
-
const configuredWindow = this.s3Config.signingWindowSeconds;
|
|
283
|
-
const windowSeconds = configuredWindow > 0 ? configuredWindow : 3600;
|
|
284
|
-
const windowMs = windowSeconds * 1000;
|
|
285
|
-
const signedUrl = await (0, s3_request_presigner_1.getSignedUrl)(this.s3Client, command, {
|
|
286
|
-
signingDate: new Date(Math.floor(Date.now() / windowMs) * windowMs),
|
|
287
|
-
expiresIn: (params.ttl ?? 3600) + windowSeconds,
|
|
288
|
-
});
|
|
289
|
-
return signedUrl;
|
|
290
|
-
}
|
|
291
|
-
}
|
|
292
|
-
catch (error) {
|
|
293
|
-
throw new Error(`Failed to generate URL: ${error.message}`);
|
|
294
|
-
}
|
|
328
|
+
// Downloads always go through the api, for every key; the api decrypts flagged objects and
|
|
329
|
+
// passes the rest through. The expiry is anchored to the start of the current signing window
|
|
330
|
+
// so the same key yields the same URL for the whole window and browser/image caches can hit;
|
|
331
|
+
// the window is added so the URL stays valid for at least the requested ttl from now.
|
|
332
|
+
// Public assets get no expiry.
|
|
333
|
+
const token = this.cipher.signObjectToken({
|
|
334
|
+
op: "get",
|
|
335
|
+
key: params.key,
|
|
336
|
+
filename: params.filename,
|
|
337
|
+
expiresAt: params.isPublic ? undefined : this._windowedExpiry(params.ttl ?? s3_cipher_core_1.S3_OBJECT_TOKEN_TTL_SECONDS),
|
|
338
|
+
});
|
|
339
|
+
return this._apiObjectUrl(token);
|
|
295
340
|
}
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
else {
|
|
303
|
-
// Check if we have shared key credentials to generate SAS
|
|
304
|
-
const credential = this.blobServiceClient.credential;
|
|
305
|
-
if (credential instanceof storage_blob_1.StorageSharedKeyCredential) {
|
|
306
|
-
const sasToken = (0, storage_blob_1.generateBlobSASQueryParameters)({
|
|
307
|
-
containerName: this._bucket,
|
|
308
|
-
blobName: params.key,
|
|
309
|
-
permissions: storage_blob_1.BlobSASPermissions.parse("r"), // Read permission
|
|
310
|
-
startsOn: new Date(new Date().valueOf() - 15 * 60 * 1000), // Start 15 minutes ago to account for clock skew
|
|
311
|
-
expiresOn: new Date(new Date().valueOf() + (params.ttl ?? 3600) * 1000),
|
|
312
|
-
protocol: storage_blob_1.SASProtocol.Https,
|
|
313
|
-
...(params.filename ? { contentDisposition: this._contentDispositionAttachment(params.filename) } : {}),
|
|
314
|
-
}, credential);
|
|
315
|
-
return `${blobClient.url}?${sasToken}`;
|
|
316
|
-
}
|
|
317
|
-
else {
|
|
318
|
-
// If no shared key credentials, return the blob URL (works if container is public)
|
|
319
|
-
return blobClient.url;
|
|
320
|
-
}
|
|
321
|
-
}
|
|
322
|
-
}
|
|
323
|
-
catch (error) {
|
|
324
|
-
throw new Error(`Failed to generate Azure URL: ${error.message}`);
|
|
325
|
-
}
|
|
341
|
+
/** Unix seconds: start of the current signing window + ttl + window (`s3.signingWindowSeconds`, default 3600). */
|
|
342
|
+
_windowedExpiry(ttl) {
|
|
343
|
+
const configuredWindow = this.s3Config.signingWindowSeconds;
|
|
344
|
+
const windowSeconds = configuredWindow > 0 ? configuredWindow : 3600;
|
|
345
|
+
const windowStart = Math.floor(Math.floor(Date.now() / 1000) / windowSeconds) * windowSeconds;
|
|
346
|
+
return windowStart + ttl + windowSeconds;
|
|
326
347
|
}
|
|
327
348
|
async uploadFile(params) {
|
|
328
349
|
await this._loadConfiguration();
|
|
@@ -345,18 +366,15 @@ let S3Service = class S3Service {
|
|
|
345
366
|
};
|
|
346
367
|
const extension = extensionMapping[baseContentType] || (baseContentType.includes("/") ? baseContentType.split("/")[1] : "bin");
|
|
347
368
|
const finalKey = `${params.key}.${extension}`;
|
|
348
|
-
const presignedUrl = await this._prepareUrl({
|
|
349
|
-
key: finalKey,
|
|
350
|
-
contentType: fileContentType,
|
|
351
|
-
});
|
|
352
369
|
try {
|
|
353
|
-
await
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
370
|
+
const { body, metadata } = await this._prepareBody(finalKey, params.buffer);
|
|
371
|
+
await this.s3Client.send(new client_s3_1.PutObjectCommand({
|
|
372
|
+
Bucket: this._bucket,
|
|
373
|
+
Key: finalKey,
|
|
374
|
+
Body: body,
|
|
375
|
+
ContentType: fileContentType,
|
|
376
|
+
Metadata: metadata,
|
|
377
|
+
}));
|
|
360
378
|
}
|
|
361
379
|
catch (error) {
|
|
362
380
|
this.logger.error("File upload error", error);
|
|
@@ -382,11 +400,13 @@ let S3Service = class S3Service {
|
|
|
382
400
|
const extension = extensionMapping[baseContentType] || (baseContentType.includes("/") ? baseContentType.split("/")[1] : "bin");
|
|
383
401
|
const finalKey = `${params.key}.${extension}`;
|
|
384
402
|
try {
|
|
403
|
+
const { body, metadata } = await this._prepareBody(finalKey, params.buffer);
|
|
385
404
|
const blobClient = this.containerClient.getBlockBlobClient(finalKey);
|
|
386
|
-
await blobClient.upload(
|
|
405
|
+
await blobClient.upload(body, body.length, {
|
|
387
406
|
blobHTTPHeaders: {
|
|
388
407
|
blobContentType: fileContentType,
|
|
389
408
|
},
|
|
409
|
+
metadata,
|
|
390
410
|
});
|
|
391
411
|
const fileUrl = await this.generateSignedUrl({ key: finalKey });
|
|
392
412
|
return { filePath: finalKey, fileUrl };
|
|
@@ -407,18 +427,15 @@ let S3Service = class S3Service {
|
|
|
407
427
|
throw new Error("Unsupported image format");
|
|
408
428
|
}
|
|
409
429
|
const finalKey = `${params.key}.${contentType.split("/")[1]}`;
|
|
410
|
-
const presignedUrl = await this._prepareUrl({
|
|
411
|
-
key: finalKey,
|
|
412
|
-
contentType: contentType,
|
|
413
|
-
});
|
|
414
430
|
try {
|
|
415
|
-
await
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
431
|
+
const { body, metadata } = await this._prepareBody(finalKey, params.buffer);
|
|
432
|
+
await this.s3Client.send(new client_s3_1.PutObjectCommand({
|
|
433
|
+
Bucket: this._bucket,
|
|
434
|
+
Key: finalKey,
|
|
435
|
+
Body: body,
|
|
436
|
+
ContentType: contentType,
|
|
437
|
+
Metadata: metadata,
|
|
438
|
+
}));
|
|
422
439
|
}
|
|
423
440
|
catch (error) {
|
|
424
441
|
this.logger.error("Image upload error", error);
|
|
@@ -433,11 +450,13 @@ let S3Service = class S3Service {
|
|
|
433
450
|
}
|
|
434
451
|
const finalKey = `${params.key}.${contentType.split("/")[1]}`;
|
|
435
452
|
try {
|
|
453
|
+
const { body, metadata } = await this._prepareBody(finalKey, params.buffer);
|
|
436
454
|
const blobClient = this.containerClient.getBlockBlobClient(finalKey);
|
|
437
|
-
await blobClient.upload(
|
|
455
|
+
await blobClient.upload(body, body.length, {
|
|
438
456
|
blobHTTPHeaders: {
|
|
439
457
|
blobContentType: contentType,
|
|
440
458
|
},
|
|
459
|
+
metadata,
|
|
441
460
|
});
|
|
442
461
|
const publicUrl = await this.generateSignedUrl({ key: finalKey });
|
|
443
462
|
return { imagePath: finalKey, imageUrl: publicUrl };
|
|
@@ -474,11 +493,13 @@ let S3Service = class S3Service {
|
|
|
474
493
|
}
|
|
475
494
|
const finalKey = params.key;
|
|
476
495
|
try {
|
|
496
|
+
const { body, metadata } = await this._prepareBody(finalKey, params.buffer);
|
|
477
497
|
const blobClient = this.containerClient.getBlockBlobClient(finalKey);
|
|
478
|
-
await blobClient.upload(
|
|
498
|
+
await blobClient.upload(body, body.length, {
|
|
479
499
|
blobHTTPHeaders: {
|
|
480
500
|
blobContentType: params.contentType,
|
|
481
501
|
},
|
|
502
|
+
metadata,
|
|
482
503
|
});
|
|
483
504
|
const fileUrl = await this.generateSignedUrl({ key: finalKey });
|
|
484
505
|
return { filePath: finalKey, fileUrl };
|
|
@@ -492,19 +513,22 @@ let S3Service = class S3Service {
|
|
|
492
513
|
await this._loadConfiguration();
|
|
493
514
|
if (!this._endpoint)
|
|
494
515
|
return;
|
|
516
|
+
const { body, metadata } = await this._prepareBody(params.key, params.buffer);
|
|
495
517
|
if (this._storageType === "azure") {
|
|
496
518
|
const blobClient = this.containerClient.getBlockBlobClient(params.key);
|
|
497
|
-
await blobClient.upload(
|
|
519
|
+
await blobClient.upload(body, body.length, {
|
|
498
520
|
blobHTTPHeaders: { blobContentType: params.contentType },
|
|
521
|
+
metadata,
|
|
499
522
|
});
|
|
500
523
|
return;
|
|
501
524
|
}
|
|
502
525
|
const command = new client_s3_1.PutObjectCommand({
|
|
503
526
|
Bucket: this._bucket,
|
|
504
527
|
Key: params.key,
|
|
505
|
-
Body:
|
|
528
|
+
Body: body,
|
|
506
529
|
ContentType: params.contentType,
|
|
507
530
|
ACL: params.isPublic ? "public-read" : undefined,
|
|
531
|
+
Metadata: metadata,
|
|
508
532
|
});
|
|
509
533
|
await this.s3Client.send(command);
|
|
510
534
|
}
|
|
@@ -521,12 +545,8 @@ let S3Service = class S3Service {
|
|
|
521
545
|
});
|
|
522
546
|
try {
|
|
523
547
|
const response = await this.s3Client.send(command);
|
|
524
|
-
const
|
|
525
|
-
|
|
526
|
-
for await (const chunk of stream) {
|
|
527
|
-
chunks.push(Buffer.from(chunk));
|
|
528
|
-
}
|
|
529
|
-
return Buffer.concat(chunks).toString("utf-8");
|
|
548
|
+
const bytes = await this._streamToBuffer(response.Body);
|
|
549
|
+
return (await this._readBody(params.key, response.Metadata, bytes)).toString("utf-8");
|
|
530
550
|
}
|
|
531
551
|
catch (error) {
|
|
532
552
|
console.error(`Failed to get object content for key ${params.key}:`, error);
|
|
@@ -537,12 +557,8 @@ let S3Service = class S3Service {
|
|
|
537
557
|
try {
|
|
538
558
|
const blobClient = this.containerClient.getBlobClient(params.key);
|
|
539
559
|
const downloadResponse = await blobClient.download();
|
|
540
|
-
const
|
|
541
|
-
|
|
542
|
-
for await (const chunk of stream) {
|
|
543
|
-
chunks.push(Buffer.from(chunk));
|
|
544
|
-
}
|
|
545
|
-
return Buffer.concat(chunks).toString("utf-8");
|
|
560
|
+
const bytes = await this._streamToBuffer(downloadResponse.readableStreamBody);
|
|
561
|
+
return (await this._readBody(params.key, downloadResponse.metadata, bytes)).toString("utf-8");
|
|
546
562
|
}
|
|
547
563
|
catch (error) {
|
|
548
564
|
console.error(`Failed to get Azure object content for key ${params.key}:`, error);
|
|
@@ -591,12 +607,9 @@ let S3Service = class S3Service {
|
|
|
591
607
|
if (!downloadResponse.readableStreamBody) {
|
|
592
608
|
throw new Error(`Azure blob not found: ${params.key}`);
|
|
593
609
|
}
|
|
594
|
-
const
|
|
595
|
-
for await (const chunk of downloadResponse.readableStreamBody) {
|
|
596
|
-
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
|
597
|
-
}
|
|
610
|
+
const bytes = await this._streamToBuffer(downloadResponse.readableStreamBody);
|
|
598
611
|
return {
|
|
599
|
-
buffer:
|
|
612
|
+
buffer: await this._readBody(params.key, downloadResponse.metadata, bytes),
|
|
600
613
|
contentType: downloadResponse.contentType ?? undefined,
|
|
601
614
|
};
|
|
602
615
|
}
|
|
@@ -608,12 +621,9 @@ let S3Service = class S3Service {
|
|
|
608
621
|
if (!response.Body) {
|
|
609
622
|
throw new Error(`S3 object not found: ${params.key}`);
|
|
610
623
|
}
|
|
611
|
-
const
|
|
612
|
-
for await (const chunk of response.Body) {
|
|
613
|
-
chunks.push(Buffer.from(chunk));
|
|
614
|
-
}
|
|
624
|
+
const bytes = await this._streamToBuffer(response.Body);
|
|
615
625
|
return {
|
|
616
|
-
buffer:
|
|
626
|
+
buffer: await this._readBody(params.key, response.Metadata, bytes),
|
|
617
627
|
contentType: response.ContentType,
|
|
618
628
|
};
|
|
619
629
|
}
|
|
@@ -625,10 +635,21 @@ let S3Service = class S3Service {
|
|
|
625
635
|
await this._loadConfiguration();
|
|
626
636
|
if (!this._endpoint)
|
|
627
637
|
return;
|
|
638
|
+
const metadata = this._metadataFor(params.key);
|
|
639
|
+
let body = params.stream;
|
|
640
|
+
if (metadata) {
|
|
641
|
+
const encrypting = this.cipher.encryptStream();
|
|
642
|
+
(0, stream_1.pipeline)(params.stream, encrypting, (error) => {
|
|
643
|
+
if (error)
|
|
644
|
+
this.logger.error(`Failed to encrypt object stream ${params.key}`, error);
|
|
645
|
+
});
|
|
646
|
+
body = encrypting;
|
|
647
|
+
}
|
|
628
648
|
if (this._storageType === "azure") {
|
|
629
649
|
const blobClient = this.containerClient.getBlockBlobClient(params.key);
|
|
630
|
-
await blobClient.uploadStream(
|
|
650
|
+
await blobClient.uploadStream(body, 8 * 1024 * 1024, 4, {
|
|
631
651
|
blobHTTPHeaders: { blobContentType: params.contentType },
|
|
652
|
+
metadata,
|
|
632
653
|
});
|
|
633
654
|
return;
|
|
634
655
|
}
|
|
@@ -638,8 +659,9 @@ let S3Service = class S3Service {
|
|
|
638
659
|
params: {
|
|
639
660
|
Bucket: this._bucket,
|
|
640
661
|
Key: params.key,
|
|
641
|
-
Body:
|
|
662
|
+
Body: body,
|
|
642
663
|
ContentType: params.contentType,
|
|
664
|
+
Metadata: metadata,
|
|
643
665
|
},
|
|
644
666
|
partSize: 10 * 1024 * 1024,
|
|
645
667
|
queueSize: 4,
|
|
@@ -720,35 +742,36 @@ let S3Service = class S3Service {
|
|
|
720
742
|
if (this._storageType === "azure") {
|
|
721
743
|
return await this._downloadAzureFileAsString(params);
|
|
722
744
|
}
|
|
745
|
+
let response;
|
|
723
746
|
try {
|
|
724
|
-
|
|
725
|
-
if (response.Body) {
|
|
726
|
-
return await response.Body.transformToString();
|
|
727
|
-
}
|
|
728
|
-
return null;
|
|
747
|
+
response = await this.s3Client.send(new client_s3_1.GetObjectCommand({ Bucket: this._bucket, Key: params.key }));
|
|
729
748
|
}
|
|
730
749
|
catch (error) {
|
|
731
750
|
this.logger.error(`Failed to download file ${params.key}`, error);
|
|
732
751
|
return null;
|
|
733
752
|
}
|
|
753
|
+
if (!response.Body)
|
|
754
|
+
return null;
|
|
755
|
+
if (!(0, s3_cipher_core_1.isEncryptedObjectMetadata)(response.Metadata))
|
|
756
|
+
return await response.Body.transformToString();
|
|
757
|
+
const bytes = Buffer.from(await response.Body.transformToByteArray());
|
|
758
|
+
return (await this._readBody(params.key, response.Metadata, bytes)).toString("utf-8");
|
|
734
759
|
}
|
|
735
760
|
async _downloadAzureFileAsString(params) {
|
|
761
|
+
let downloadResponse;
|
|
762
|
+
let bytes;
|
|
736
763
|
try {
|
|
737
764
|
const blobClient = this.containerClient.getBlobClient(params.key);
|
|
738
|
-
|
|
739
|
-
if (downloadResponse.readableStreamBody)
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
|
743
|
-
}
|
|
744
|
-
return Buffer.concat(chunks).toString("utf-8");
|
|
745
|
-
}
|
|
746
|
-
return null;
|
|
765
|
+
downloadResponse = await blobClient.download();
|
|
766
|
+
if (!downloadResponse.readableStreamBody)
|
|
767
|
+
return null;
|
|
768
|
+
bytes = await this._streamToBuffer(downloadResponse.readableStreamBody);
|
|
747
769
|
}
|
|
748
770
|
catch (error) {
|
|
749
771
|
this.logger.error(`Failed to download Azure file ${params.key}`, error);
|
|
750
772
|
return null;
|
|
751
773
|
}
|
|
774
|
+
return (await this._readBody(params.key, downloadResponse.metadata, bytes)).toString("utf-8");
|
|
752
775
|
}
|
|
753
776
|
/**
|
|
754
777
|
* Download a file from S3/Azure and return its content as a Buffer.
|
|
@@ -761,36 +784,35 @@ let S3Service = class S3Service {
|
|
|
761
784
|
if (this._storageType === "azure") {
|
|
762
785
|
return await this._downloadAzureFileAsBuffer(params);
|
|
763
786
|
}
|
|
787
|
+
let response;
|
|
788
|
+
let bytes;
|
|
764
789
|
try {
|
|
765
|
-
|
|
766
|
-
if (response.Body)
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
}
|
|
770
|
-
return null;
|
|
790
|
+
response = await this.s3Client.send(new client_s3_1.GetObjectCommand({ Bucket: this._bucket, Key: params.key }));
|
|
791
|
+
if (!response.Body)
|
|
792
|
+
return null;
|
|
793
|
+
bytes = Buffer.from(await response.Body.transformToByteArray());
|
|
771
794
|
}
|
|
772
795
|
catch (error) {
|
|
773
796
|
this.logger.error(`Failed to download file as buffer ${params.key}`, error);
|
|
774
797
|
return null;
|
|
775
798
|
}
|
|
799
|
+
return await this._readBody(params.key, response.Metadata, bytes);
|
|
776
800
|
}
|
|
777
801
|
async _downloadAzureFileAsBuffer(params) {
|
|
802
|
+
let downloadResponse;
|
|
803
|
+
let bytes;
|
|
778
804
|
try {
|
|
779
805
|
const blobClient = this.containerClient.getBlobClient(params.key);
|
|
780
|
-
|
|
781
|
-
if (downloadResponse.readableStreamBody)
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
|
785
|
-
}
|
|
786
|
-
return Buffer.concat(chunks);
|
|
787
|
-
}
|
|
788
|
-
return null;
|
|
806
|
+
downloadResponse = await blobClient.download();
|
|
807
|
+
if (!downloadResponse.readableStreamBody)
|
|
808
|
+
return null;
|
|
809
|
+
bytes = await this._streamToBuffer(downloadResponse.readableStreamBody);
|
|
789
810
|
}
|
|
790
811
|
catch (error) {
|
|
791
812
|
this.logger.error(`Failed to download Azure file as buffer ${params.key}`, error);
|
|
792
813
|
return null;
|
|
793
814
|
}
|
|
815
|
+
return await this._readBody(params.key, downloadResponse.metadata, bytes);
|
|
794
816
|
}
|
|
795
817
|
/**
|
|
796
818
|
* Download a file from S3/Azure and return a readable stream.
|
|
@@ -803,36 +825,77 @@ let S3Service = class S3Service {
|
|
|
803
825
|
if (this._storageType === "azure") {
|
|
804
826
|
return await this._downloadAzureFileAsStream(params);
|
|
805
827
|
}
|
|
828
|
+
let response;
|
|
806
829
|
try {
|
|
807
830
|
const command = new client_s3_1.GetObjectCommand({
|
|
808
831
|
Bucket: this._bucket,
|
|
809
832
|
Key: params.key,
|
|
810
833
|
});
|
|
811
|
-
|
|
812
|
-
if (response.Body) {
|
|
813
|
-
// AWS SDK v3 Body is SdkStream which extends Readable
|
|
814
|
-
return response.Body;
|
|
815
|
-
}
|
|
816
|
-
return null;
|
|
834
|
+
response = await this.s3Client.send(command);
|
|
817
835
|
}
|
|
818
836
|
catch (error) {
|
|
819
837
|
this.logger.error(`Failed to download file as stream ${params.key}`, error);
|
|
820
838
|
return null;
|
|
821
839
|
}
|
|
840
|
+
if (!response.Body)
|
|
841
|
+
return null;
|
|
842
|
+
// AWS SDK v3 Body is SdkStream which extends Readable
|
|
843
|
+
return this._readStream(params.key, response.Metadata, response.Body);
|
|
844
|
+
}
|
|
845
|
+
/**
|
|
846
|
+
* Stream an object for the token-verified api route. Returns null when the object does not exist;
|
|
847
|
+
* a flagged object comes back decrypted, and throws when no configured key can open it.
|
|
848
|
+
*/
|
|
849
|
+
async downloadObject(params) {
|
|
850
|
+
await this._loadConfiguration();
|
|
851
|
+
if (!this._endpoint)
|
|
852
|
+
return null;
|
|
853
|
+
if (this._storageType === "azure") {
|
|
854
|
+
let downloadResponse;
|
|
855
|
+
try {
|
|
856
|
+
downloadResponse = await this.containerClient.getBlobClient(params.key).download();
|
|
857
|
+
}
|
|
858
|
+
catch (error) {
|
|
859
|
+
if (error?.statusCode === 404)
|
|
860
|
+
return null;
|
|
861
|
+
throw error;
|
|
862
|
+
}
|
|
863
|
+
if (!downloadResponse.readableStreamBody)
|
|
864
|
+
return null;
|
|
865
|
+
return {
|
|
866
|
+
stream: await this._readStreamReady(params.key, downloadResponse.metadata, downloadResponse.readableStreamBody),
|
|
867
|
+
contentType: downloadResponse.contentType ?? undefined,
|
|
868
|
+
};
|
|
869
|
+
}
|
|
870
|
+
let response;
|
|
871
|
+
try {
|
|
872
|
+
response = await this.s3Client.send(new client_s3_1.GetObjectCommand({ Bucket: this._bucket, Key: params.key }));
|
|
873
|
+
}
|
|
874
|
+
catch (error) {
|
|
875
|
+
if (error?.name === "NoSuchKey" || error?.name === "NotFound" || error?.$metadata?.httpStatusCode === 404)
|
|
876
|
+
return null;
|
|
877
|
+
throw error;
|
|
878
|
+
}
|
|
879
|
+
if (!response.Body)
|
|
880
|
+
return null;
|
|
881
|
+
return {
|
|
882
|
+
stream: await this._readStreamReady(params.key, response.Metadata, response.Body),
|
|
883
|
+
contentType: response.ContentType,
|
|
884
|
+
};
|
|
822
885
|
}
|
|
823
886
|
async _downloadAzureFileAsStream(params) {
|
|
887
|
+
let downloadResponse;
|
|
824
888
|
try {
|
|
825
889
|
const blobClient = this.containerClient.getBlobClient(params.key);
|
|
826
|
-
|
|
827
|
-
if (downloadResponse.readableStreamBody) {
|
|
828
|
-
return downloadResponse.readableStreamBody;
|
|
829
|
-
}
|
|
830
|
-
return null;
|
|
890
|
+
downloadResponse = await blobClient.download();
|
|
831
891
|
}
|
|
832
892
|
catch (error) {
|
|
833
893
|
this.logger.error(`Failed to download Azure file as stream ${params.key}`, error);
|
|
834
894
|
return null;
|
|
835
895
|
}
|
|
896
|
+
if (!downloadResponse.readableStreamBody)
|
|
897
|
+
return null;
|
|
898
|
+
return this._readStream(params.key, downloadResponse.metadata, downloadResponse.readableStreamBody);
|
|
836
899
|
}
|
|
837
900
|
/**
|
|
838
901
|
* Upload a buffer with an exact key — NO extension appending.
|
|
@@ -843,18 +906,21 @@ let S3Service = class S3Service {
|
|
|
843
906
|
await this._loadConfiguration();
|
|
844
907
|
if (!this._endpoint)
|
|
845
908
|
return;
|
|
909
|
+
const { body, metadata } = await this._prepareBody(params.key, params.buffer);
|
|
846
910
|
if (this._storageType === "azure") {
|
|
847
911
|
const blobClient = this.containerClient.getBlockBlobClient(params.key);
|
|
848
|
-
await blobClient.upload(
|
|
912
|
+
await blobClient.upload(body, body.length, {
|
|
849
913
|
blobHTTPHeaders: { blobContentType: params.contentType },
|
|
914
|
+
metadata,
|
|
850
915
|
});
|
|
851
916
|
return;
|
|
852
917
|
}
|
|
853
918
|
await this.s3Client.send(new client_s3_1.PutObjectCommand({
|
|
854
919
|
Bucket: this._bucket,
|
|
855
920
|
Key: params.key,
|
|
856
|
-
Body:
|
|
921
|
+
Body: body,
|
|
857
922
|
ContentType: params.contentType,
|
|
923
|
+
Metadata: metadata,
|
|
858
924
|
}));
|
|
859
925
|
}
|
|
860
926
|
};
|
|
@@ -864,6 +930,7 @@ exports.S3Service = S3Service = __decorate([
|
|
|
864
930
|
__metadata("design:paramtypes", [jsonapi_service_1.JsonApiService,
|
|
865
931
|
nestjs_cls_1.ClsService,
|
|
866
932
|
config_1.ConfigService,
|
|
867
|
-
logging_service_1.AppLoggingService
|
|
933
|
+
logging_service_1.AppLoggingService,
|
|
934
|
+
s3_cipher_service_1.S3CipherService])
|
|
868
935
|
], S3Service);
|
|
869
936
|
//# sourceMappingURL=s3.service.js.map
|