@h-ai/storage 1.0.0-beta.17 → 1.0.0-beta.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +4 -0
- package/README.ja.md +4 -0
- package/README.md +7 -1
- package/dist/browser.d.ts +2 -1
- package/dist/browser.js +3 -3
- package/dist/{chunk-P65WYXJG.js → chunk-6YG3N3CW.js} +8 -4
- package/dist/chunk-6YG3N3CW.js.map +1 -0
- package/dist/{chunk-JSTMXNYI.js → chunk-FK7KRCML.js} +4 -4
- package/dist/{chunk-JSTMXNYI.js.map → chunk-FK7KRCML.js.map} +1 -1
- package/dist/{chunk-TLZY4IP6.js → chunk-JAJYHGLI.js} +61 -17
- package/dist/chunk-JAJYHGLI.js.map +1 -0
- package/dist/client/index.d.ts +2 -1
- package/dist/client/index.js +1 -1
- package/dist/node.js +333 -99
- package/dist/node.js.map +1 -1
- package/dist/schemas.d.ts +2 -1
- package/dist/schemas.js +1 -1
- package/package.json +12 -12
- package/dist/chunk-P65WYXJG.js.map +0 -1
- package/dist/chunk-TLZY4IP6.js.map +0 -1
package/dist/node.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { HaiStorageError, StorageConfigSchema, PresignUploadOptionsSchema, PresignOptionsSchema } from './chunk-
|
|
2
|
-
export { HaiStorageError, LocalConfigSchema, PresignOptionsSchema, PresignUploadOptionsSchema, S3ConfigSchema, StorageConfigSchema, StorageOperationLogConfigSchema, StorageTypeSchema } from './chunk-
|
|
3
|
-
import { storageM, MIME_TYPES, MIME_TYPE_DEFAULT } from './chunk-
|
|
4
|
-
export { downloadAndSave, downloadWithPresignedUrl, formatFileSize, getFileExtension, getMimeType, uploadWithPresignedUrl } from './chunk-
|
|
5
|
-
export { STORAGE_DELETE_MAX_KEYS, STORAGE_KEY_MAX_BYTES, StorageDeleteKeysSchema, StorageKeySchema, StoragePrefixSchema, StoragePresignExpiresInSchema, getStorageKeyIssue } from './chunk-
|
|
1
|
+
import { HaiStorageError, StorageConfigSchema, PresignUploadOptionsSchema, PresignOptionsSchema } from './chunk-FK7KRCML.js';
|
|
2
|
+
export { HaiStorageError, LocalConfigSchema, PresignOptionsSchema, PresignUploadOptionsSchema, S3ConfigSchema, StorageConfigSchema, StorageOperationLogConfigSchema, StorageTypeSchema } from './chunk-FK7KRCML.js';
|
|
3
|
+
import { storageM, MIME_TYPES, MIME_TYPE_DEFAULT } from './chunk-JAJYHGLI.js';
|
|
4
|
+
export { downloadAndSave, downloadWithPresignedUrl, formatFileSize, getFileExtension, getMimeType, uploadWithPresignedUrl } from './chunk-JAJYHGLI.js';
|
|
5
|
+
export { STORAGE_DELETE_MAX_KEYS, STORAGE_KEY_MAX_BYTES, StorageDeleteKeysSchema, StorageKeySchema, StoragePrefixSchema, StoragePresignExpiresInSchema, getStorageKeyIssue } from './chunk-6YG3N3CW.js';
|
|
6
6
|
import process from 'node:process';
|
|
7
7
|
import { core, err, ok } from '@h-ai/core';
|
|
8
8
|
import { Buffer } from 'node:buffer';
|
|
@@ -23,7 +23,9 @@ function toStorageDeleteResult(summary) {
|
|
|
23
23
|
success: false,
|
|
24
24
|
error: {
|
|
25
25
|
...HaiStorageError.OPERATION_FAILED,
|
|
26
|
-
message: storageM("storage_deleteManyFailed", {
|
|
26
|
+
message: storageM("storage_deleteManyFailed", {
|
|
27
|
+
params: { count: summary.failures.length }
|
|
28
|
+
}),
|
|
27
29
|
ext: { deleteMany: summary }
|
|
28
30
|
}
|
|
29
31
|
};
|
|
@@ -113,22 +115,33 @@ function logStorageOperation(logger4, config, category, operation, payload) {
|
|
|
113
115
|
// src/providers/storage-provider-local.ts
|
|
114
116
|
var logger = core.logger.child({ module: "storage", scope: "provider-local" });
|
|
115
117
|
var PATH_SEGMENT_SPLIT_REGEX = /[\\/]+/;
|
|
118
|
+
var METADATA_DIRECTORY = ".hai-storage-meta";
|
|
116
119
|
function toStorageError(error, key) {
|
|
117
120
|
const e = error;
|
|
118
121
|
if (e.code === "EEXIST") {
|
|
119
|
-
return {
|
|
122
|
+
return {
|
|
123
|
+
...HaiStorageError.ALREADY_EXISTS,
|
|
124
|
+
message: storageM("storage_fileAlreadyExists", {
|
|
125
|
+
params: { key: key ?? "" }
|
|
126
|
+
}),
|
|
127
|
+
cause: error
|
|
128
|
+
};
|
|
120
129
|
}
|
|
121
130
|
if (e.code === "ENOENT") {
|
|
122
131
|
return {
|
|
123
132
|
...HaiStorageError.NOT_FOUND,
|
|
124
|
-
message: storageM("storage_fileNotFound", {
|
|
133
|
+
message: storageM("storage_fileNotFound", {
|
|
134
|
+
params: { key: key ?? "" }
|
|
135
|
+
}),
|
|
125
136
|
cause: error
|
|
126
137
|
};
|
|
127
138
|
}
|
|
128
139
|
if (e.code === "EACCES" || e.code === "EPERM") {
|
|
129
140
|
return {
|
|
130
141
|
...HaiStorageError.PERMISSION_DENIED,
|
|
131
|
-
message: storageM("storage_permissionDenied", {
|
|
142
|
+
message: storageM("storage_permissionDenied", {
|
|
143
|
+
params: { key: key ?? "" }
|
|
144
|
+
}),
|
|
132
145
|
cause: error
|
|
133
146
|
};
|
|
134
147
|
}
|
|
@@ -142,7 +155,9 @@ function toStorageError(error, key) {
|
|
|
142
155
|
if (e.code === "EISDIR") {
|
|
143
156
|
return {
|
|
144
157
|
...HaiStorageError.INVALID_PATH,
|
|
145
|
-
message: storageM("storage_pathIsDir", {
|
|
158
|
+
message: storageM("storage_pathIsDir", {
|
|
159
|
+
params: { key: key ?? "" }
|
|
160
|
+
}),
|
|
146
161
|
cause: error
|
|
147
162
|
};
|
|
148
163
|
}
|
|
@@ -154,8 +169,8 @@ function toStorageError(error, key) {
|
|
|
154
169
|
};
|
|
155
170
|
}
|
|
156
171
|
return {
|
|
157
|
-
|
|
158
|
-
message: storageM("storage_ioError"
|
|
172
|
+
...HaiStorageError.IO_ERROR,
|
|
173
|
+
message: storageM("storage_ioError"),
|
|
159
174
|
cause: error
|
|
160
175
|
};
|
|
161
176
|
}
|
|
@@ -164,11 +179,33 @@ function getMimeType2(filePath) {
|
|
|
164
179
|
return MIME_TYPES[ext] || MIME_TYPE_DEFAULT;
|
|
165
180
|
}
|
|
166
181
|
function calculateEtag(stat2) {
|
|
167
|
-
|
|
182
|
+
const modified = "mtimeNs" in stat2 ? Number(stat2.mtimeNs) / 1e6 : stat2.mtimeMs;
|
|
183
|
+
return `"${stat2.size.toString(16)}-${modified.toString(16)}"`;
|
|
184
|
+
}
|
|
185
|
+
function metadataGeneration(stat2) {
|
|
186
|
+
return crypto.createHash("sha256").update([
|
|
187
|
+
stat2.dev,
|
|
188
|
+
stat2.ino,
|
|
189
|
+
stat2.birthtimeNs,
|
|
190
|
+
stat2.mtimeNs,
|
|
191
|
+
stat2.size
|
|
192
|
+
].join(":")).digest("hex");
|
|
193
|
+
}
|
|
194
|
+
function isMetadataRecord(value) {
|
|
195
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
196
|
+
return false;
|
|
197
|
+
if ("contentType" in value && typeof value.contentType !== "string")
|
|
198
|
+
return false;
|
|
199
|
+
if ("metadata" in value) {
|
|
200
|
+
const metadata = value.metadata;
|
|
201
|
+
if (!metadata || typeof metadata !== "object" || Array.isArray(metadata) || Object.values(metadata).some((item) => typeof item !== "string")) {
|
|
202
|
+
return false;
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
return true;
|
|
168
206
|
}
|
|
169
207
|
function isMetaFile(key) {
|
|
170
|
-
|
|
171
|
-
return normalized.endsWith(".meta.json") || normalized.endsWith(".upload.tmp");
|
|
208
|
+
return key.toLowerCase().split(PATH_SEGMENT_SPLIT_REGEX).some((segment) => segment === METADATA_DIRECTORY || segment.endsWith(".meta.json") || segment.endsWith(".upload.tmp"));
|
|
172
209
|
}
|
|
173
210
|
function createInvalidPathError() {
|
|
174
211
|
const error = new Error(storageM("storage_pathTraversal"));
|
|
@@ -224,48 +261,76 @@ function createLocalProvider() {
|
|
|
224
261
|
function logOperation(category, operation, payload) {
|
|
225
262
|
logStorageOperation(logger, config?.operationLog, category, operation, payload);
|
|
226
263
|
}
|
|
264
|
+
function metadataKey(filePath, stat2) {
|
|
265
|
+
return path.relative(getConfig().root, path.join(path.dirname(filePath), METADATA_DIRECTORY, `${metadataGeneration(stat2)}.meta.json`));
|
|
266
|
+
}
|
|
227
267
|
const file = {
|
|
228
268
|
async put(key, data, options = {}) {
|
|
229
269
|
if (isMetaFile(key)) {
|
|
230
270
|
return err(
|
|
231
271
|
HaiStorageError.INVALID_PATH,
|
|
232
|
-
storageM("storage_metaFileAccess", {
|
|
272
|
+
storageM("storage_metaFileAccess", {
|
|
273
|
+
params: { key }
|
|
274
|
+
})
|
|
233
275
|
);
|
|
234
276
|
}
|
|
235
277
|
let temporaryPath;
|
|
278
|
+
let preparedMetadataPath;
|
|
279
|
+
let published = false;
|
|
236
280
|
try {
|
|
237
281
|
const filePath = await fullPath(key);
|
|
238
|
-
const
|
|
282
|
+
const cfg = getConfig();
|
|
283
|
+
let previousMetadataKey;
|
|
284
|
+
try {
|
|
285
|
+
previousMetadataKey = metadataKey(filePath, await fsp.stat(filePath, { bigint: true }));
|
|
286
|
+
} catch (error) {
|
|
287
|
+
if (error.code !== "ENOENT")
|
|
288
|
+
throw error;
|
|
289
|
+
}
|
|
239
290
|
logOperation("write", "file.put", { key, options });
|
|
240
291
|
const dir2 = path.dirname(filePath);
|
|
241
|
-
await fsp.mkdir(dir2, { recursive: true, mode:
|
|
292
|
+
await fsp.mkdir(dir2, { recursive: true, mode: cfg.directoryMode });
|
|
242
293
|
temporaryPath = `${filePath}.${crypto.randomUUID()}.upload.tmp`;
|
|
243
294
|
await pipeline(
|
|
244
295
|
Readable.from(toByteStream(data, options.signal), { objectMode: false }),
|
|
245
|
-
fs.createWriteStream(temporaryPath, { flags: "wx", mode:
|
|
296
|
+
fs.createWriteStream(temporaryPath, { flags: "wx", mode: cfg.fileMode }),
|
|
246
297
|
{ signal: options.signal }
|
|
247
298
|
);
|
|
248
299
|
options.signal?.throwIfAborted();
|
|
300
|
+
const stat2 = await fsp.stat(temporaryPath, { bigint: true });
|
|
301
|
+
const metaPath = await fullPath(metadataKey(filePath, stat2));
|
|
302
|
+
const serializedMetadata = JSON.stringify({ contentType: options.contentType, metadata: options.metadata });
|
|
303
|
+
await fsp.mkdir(path.dirname(metaPath), { recursive: true, mode: cfg.directoryMode });
|
|
304
|
+
const metadataFile = await fsp.open(metaPath, "wx", cfg.fileMode);
|
|
305
|
+
preparedMetadataPath = metaPath;
|
|
306
|
+
try {
|
|
307
|
+
await metadataFile.writeFile(serializedMetadata);
|
|
308
|
+
} finally {
|
|
309
|
+
await metadataFile.close();
|
|
310
|
+
}
|
|
311
|
+
options.signal?.throwIfAborted();
|
|
249
312
|
if (options.ifNoneMatch === "*")
|
|
250
313
|
await fsp.link(temporaryPath, filePath);
|
|
251
314
|
else
|
|
252
315
|
await fsp.rename(temporaryPath, filePath);
|
|
253
|
-
|
|
316
|
+
published = true;
|
|
254
317
|
const metadata = {
|
|
255
318
|
key,
|
|
256
|
-
size: stat2.size,
|
|
319
|
+
size: Number(stat2.size),
|
|
257
320
|
contentType: options.contentType || getMimeType2(key),
|
|
258
321
|
lastModified: stat2.mtime,
|
|
259
322
|
etag: calculateEtag(stat2),
|
|
260
323
|
metadata: options.metadata
|
|
261
324
|
};
|
|
262
|
-
if (
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
325
|
+
if (previousMetadataKey) {
|
|
326
|
+
try {
|
|
327
|
+
const previousMetadataPath = await fullPath(previousMetadataKey);
|
|
328
|
+
if (previousMetadataPath !== metaPath)
|
|
329
|
+
await fsp.unlink(previousMetadataPath);
|
|
330
|
+
} catch (error) {
|
|
331
|
+
if (error.code !== "ENOENT")
|
|
332
|
+
logger.warn("Obsolete local metadata cleanup failed", { key });
|
|
333
|
+
}
|
|
269
334
|
}
|
|
270
335
|
return ok(metadata);
|
|
271
336
|
} catch (error) {
|
|
@@ -275,6 +340,8 @@ function createLocalProvider() {
|
|
|
275
340
|
if (temporaryPath)
|
|
276
341
|
await fsp.unlink(temporaryPath).catch(() => {
|
|
277
342
|
});
|
|
343
|
+
if (preparedMetadataPath && !published)
|
|
344
|
+
await fsp.unlink(preparedMetadataPath).catch(() => logger.warn("Unpublished local metadata cleanup failed", { key }));
|
|
278
345
|
}
|
|
279
346
|
},
|
|
280
347
|
async get(key, options = {}) {
|
|
@@ -294,7 +361,9 @@ function createLocalProvider() {
|
|
|
294
361
|
if (isMetaFile(key)) {
|
|
295
362
|
return err(
|
|
296
363
|
HaiStorageError.INVALID_PATH,
|
|
297
|
-
storageM("storage_metaFileAccess", {
|
|
364
|
+
storageM("storage_metaFileAccess", {
|
|
365
|
+
params: { key }
|
|
366
|
+
})
|
|
298
367
|
);
|
|
299
368
|
}
|
|
300
369
|
try {
|
|
@@ -302,8 +371,11 @@ function createLocalProvider() {
|
|
|
302
371
|
logOperation("read", "file.getStream", { key, options });
|
|
303
372
|
options.signal?.throwIfAborted();
|
|
304
373
|
const stat2 = await fsp.stat(filePath);
|
|
305
|
-
if (stat2.isDirectory())
|
|
306
|
-
return err(HaiStorageError.INVALID_PATH, storageM("storage_pathIsDir", {
|
|
374
|
+
if (stat2.isDirectory()) {
|
|
375
|
+
return err(HaiStorageError.INVALID_PATH, storageM("storage_pathIsDir", {
|
|
376
|
+
params: { key }
|
|
377
|
+
}));
|
|
378
|
+
}
|
|
307
379
|
async function* read() {
|
|
308
380
|
const stream = fs.createReadStream(filePath, { start: options.rangeStart, end: options.rangeEnd, signal: options.signal });
|
|
309
381
|
for await (const chunk of stream)
|
|
@@ -318,35 +390,59 @@ function createLocalProvider() {
|
|
|
318
390
|
if (isMetaFile(key)) {
|
|
319
391
|
return err(
|
|
320
392
|
HaiStorageError.INVALID_PATH,
|
|
321
|
-
storageM("storage_metaFileAccess", {
|
|
393
|
+
storageM("storage_metaFileAccess", {
|
|
394
|
+
params: { key }
|
|
395
|
+
})
|
|
322
396
|
);
|
|
323
397
|
}
|
|
324
398
|
try {
|
|
325
399
|
const filePath = await fullPath(key);
|
|
326
400
|
logOperation("read", "file.head", { key });
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
HaiStorageError.INVALID_PATH,
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
401
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
402
|
+
const stat2 = await fsp.stat(filePath, { bigint: true });
|
|
403
|
+
if (stat2.isDirectory()) {
|
|
404
|
+
return err(HaiStorageError.INVALID_PATH, storageM("storage_pathIsDir", {
|
|
405
|
+
params: { key }
|
|
406
|
+
}));
|
|
407
|
+
}
|
|
408
|
+
let serializedMetadata;
|
|
409
|
+
try {
|
|
410
|
+
const metaPath = await fullPath(metadataKey(filePath, stat2));
|
|
411
|
+
serializedMetadata = await fsp.readFile(metaPath, "utf8");
|
|
412
|
+
} catch (error) {
|
|
413
|
+
if (!["ENOENT", "ENOTDIR"].includes(error.code ?? ""))
|
|
414
|
+
throw error;
|
|
415
|
+
if (metadataGeneration(await fsp.stat(filePath, { bigint: true })) !== metadataGeneration(stat2))
|
|
416
|
+
continue;
|
|
417
|
+
return err(HaiStorageError.OPERATION_FAILED, storageM("storage_metadataRequired"));
|
|
418
|
+
}
|
|
419
|
+
let customMetadata;
|
|
420
|
+
try {
|
|
421
|
+
customMetadata = JSON.parse(serializedMetadata);
|
|
422
|
+
} catch {
|
|
423
|
+
logger.warn("Invalid local metadata record", {
|
|
424
|
+
key,
|
|
425
|
+
generation: metadataGeneration(stat2)
|
|
426
|
+
});
|
|
427
|
+
return err(HaiStorageError.OPERATION_FAILED, storageM("storage_metadataInvalid"));
|
|
428
|
+
}
|
|
429
|
+
if (!isMetadataRecord(customMetadata)) {
|
|
430
|
+
logger.warn("Invalid local metadata shape", {
|
|
431
|
+
key,
|
|
432
|
+
generation: metadataGeneration(stat2)
|
|
433
|
+
});
|
|
434
|
+
return err(HaiStorageError.OPERATION_FAILED, storageM("storage_metadataInvalid"));
|
|
435
|
+
}
|
|
436
|
+
return ok({
|
|
437
|
+
key,
|
|
438
|
+
size: Number(stat2.size),
|
|
439
|
+
contentType: customMetadata.contentType || getMimeType2(key),
|
|
440
|
+
lastModified: stat2.mtime,
|
|
441
|
+
etag: calculateEtag(stat2),
|
|
442
|
+
metadata: customMetadata.metadata
|
|
443
|
+
});
|
|
340
444
|
}
|
|
341
|
-
|
|
342
|
-
return ok({
|
|
343
|
-
key,
|
|
344
|
-
size: stat2.size,
|
|
345
|
-
contentType: customMetadata.contentType || getMimeType2(key),
|
|
346
|
-
lastModified: stat2.mtime,
|
|
347
|
-
etag,
|
|
348
|
-
metadata: customMetadata.metadata
|
|
349
|
-
});
|
|
445
|
+
return err(HaiStorageError.OPERATION_FAILED, storageM("storage_fileChanged"));
|
|
350
446
|
} catch (error) {
|
|
351
447
|
return err(toStorageError(error, key));
|
|
352
448
|
}
|
|
@@ -355,7 +451,9 @@ function createLocalProvider() {
|
|
|
355
451
|
if (isMetaFile(key)) {
|
|
356
452
|
return err(
|
|
357
453
|
HaiStorageError.INVALID_PATH,
|
|
358
|
-
storageM("storage_metaFileAccess", {
|
|
454
|
+
storageM("storage_metaFileAccess", {
|
|
455
|
+
params: { key }
|
|
456
|
+
})
|
|
359
457
|
);
|
|
360
458
|
}
|
|
361
459
|
try {
|
|
@@ -374,17 +472,22 @@ function createLocalProvider() {
|
|
|
374
472
|
if (isMetaFile(key)) {
|
|
375
473
|
return err(
|
|
376
474
|
HaiStorageError.INVALID_PATH,
|
|
377
|
-
storageM("storage_metaFileAccess", {
|
|
475
|
+
storageM("storage_metaFileAccess", {
|
|
476
|
+
params: { key }
|
|
477
|
+
})
|
|
378
478
|
);
|
|
379
479
|
}
|
|
380
480
|
try {
|
|
381
481
|
const filePath = await fullPath(key);
|
|
482
|
+
const generationKey = metadataKey(filePath, await fsp.stat(filePath, { bigint: true }));
|
|
382
483
|
logOperation("write", "file.delete", { key });
|
|
383
484
|
logger.debug("Deleting file", { key });
|
|
384
485
|
await fsp.unlink(filePath);
|
|
385
486
|
try {
|
|
386
|
-
await fsp.unlink(
|
|
387
|
-
} catch {
|
|
487
|
+
await fsp.unlink(await fullPath(generationKey));
|
|
488
|
+
} catch (error) {
|
|
489
|
+
if (error.code !== "ENOENT")
|
|
490
|
+
logger.warn("Deleted object metadata cleanup failed", { key });
|
|
388
491
|
}
|
|
389
492
|
return ok(void 0);
|
|
390
493
|
} catch (error) {
|
|
@@ -398,18 +501,31 @@ function createLocalProvider() {
|
|
|
398
501
|
async deleteMany(keys) {
|
|
399
502
|
logOperation("write", "file.deleteMany", { count: keys.length, keys });
|
|
400
503
|
const summary = { deletedKeys: [], failures: [] };
|
|
401
|
-
const uniqueKeys = [
|
|
504
|
+
const uniqueKeys = [
|
|
505
|
+
...new Set(keys)
|
|
506
|
+
];
|
|
402
507
|
for (let offset = 0; offset < uniqueKeys.length; offset += 32) {
|
|
403
508
|
const batch = uniqueKeys.slice(offset, offset + 32);
|
|
404
509
|
const results = await Promise.allSettled(batch.map((key) => file.delete(key)));
|
|
405
510
|
for (const [index, result] of results.entries()) {
|
|
406
511
|
const key = batch[index];
|
|
407
|
-
if (result.status === "rejected")
|
|
408
|
-
summary.failures.push({
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
512
|
+
if (result.status === "rejected") {
|
|
513
|
+
summary.failures.push({
|
|
514
|
+
key,
|
|
515
|
+
code: "UnexpectedDeleteFailure",
|
|
516
|
+
retryable: false,
|
|
517
|
+
outcome: "failed"
|
|
518
|
+
});
|
|
519
|
+
} else if (!result.value.success) {
|
|
520
|
+
summary.failures.push({
|
|
521
|
+
key,
|
|
522
|
+
code: String(result.value.error.code),
|
|
523
|
+
retryable: false,
|
|
524
|
+
outcome: "failed"
|
|
525
|
+
});
|
|
526
|
+
} else {
|
|
412
527
|
summary.deletedKeys.push(key);
|
|
528
|
+
}
|
|
413
529
|
}
|
|
414
530
|
}
|
|
415
531
|
return toStorageDeleteResult(summary);
|
|
@@ -501,7 +617,12 @@ function createLocalProvider() {
|
|
|
501
617
|
commonPrefixes.push(logicalKey);
|
|
502
618
|
} else {
|
|
503
619
|
const stat2 = await fsp.stat(entry.filePath);
|
|
504
|
-
files.push({
|
|
620
|
+
files.push({
|
|
621
|
+
key: entry.key,
|
|
622
|
+
size: stat2.size,
|
|
623
|
+
contentType: getMimeType2(entry.key),
|
|
624
|
+
lastModified: stat2.mtime
|
|
625
|
+
});
|
|
505
626
|
}
|
|
506
627
|
count++;
|
|
507
628
|
last = logicalKey;
|
|
@@ -510,13 +631,20 @@ function createLocalProvider() {
|
|
|
510
631
|
files,
|
|
511
632
|
commonPrefixes,
|
|
512
633
|
isTruncated,
|
|
513
|
-
...isTruncated ? {
|
|
634
|
+
...isTruncated ? {
|
|
635
|
+
nextContinuationToken: Buffer.from(JSON.stringify([1, scope, last])).toString("base64url")
|
|
636
|
+
} : {}
|
|
514
637
|
});
|
|
515
638
|
} catch (error) {
|
|
516
639
|
return err(toStorageError(error));
|
|
517
640
|
}
|
|
518
641
|
},
|
|
519
642
|
async delete(prefix) {
|
|
643
|
+
if (isMetaFile(prefix)) {
|
|
644
|
+
return err(HaiStorageError.INVALID_PATH, storageM("storage_metaFileAccess", {
|
|
645
|
+
params: { key: prefix }
|
|
646
|
+
}));
|
|
647
|
+
}
|
|
520
648
|
try {
|
|
521
649
|
const dirPath = await fullPath(prefix);
|
|
522
650
|
logOperation("write", "dir.delete", { prefix });
|
|
@@ -584,22 +712,38 @@ var logger2 = core.logger.child({ module: "storage", scope: "provider-s3" });
|
|
|
584
712
|
var TRAILING_SLASHES_REGEX = /\/+$/;
|
|
585
713
|
var LEADING_SLASHES_REGEX = /^\/+/;
|
|
586
714
|
var DELETE_BATCH_SIZE = 1e3;
|
|
587
|
-
var RETRYABLE_DELETE_CODES = /* @__PURE__ */ new Set([
|
|
715
|
+
var RETRYABLE_DELETE_CODES = /* @__PURE__ */ new Set([
|
|
716
|
+
"InternalError",
|
|
717
|
+
"ServiceUnavailable",
|
|
718
|
+
"SlowDown",
|
|
719
|
+
"RequestTimeout"
|
|
720
|
+
]);
|
|
588
721
|
function toStorageError2(error, key) {
|
|
589
722
|
const e = error;
|
|
590
|
-
if (e.name === "PreconditionFailed" || e.$metadata?.httpStatusCode === 412)
|
|
591
|
-
return {
|
|
723
|
+
if (e.name === "PreconditionFailed" || e.$metadata?.httpStatusCode === 412) {
|
|
724
|
+
return {
|
|
725
|
+
...HaiStorageError.ALREADY_EXISTS,
|
|
726
|
+
message: storageM("storage_fileAlreadyExists", {
|
|
727
|
+
params: { key: key ?? "" }
|
|
728
|
+
}),
|
|
729
|
+
cause: error
|
|
730
|
+
};
|
|
731
|
+
}
|
|
592
732
|
if (e.name === "NoSuchKey" || e.Code === "NoSuchKey" || e.name === "NotFound" || e.Code === "NotFound" || e.name === "404" || e.$metadata?.httpStatusCode === 404) {
|
|
593
733
|
return {
|
|
594
734
|
...HaiStorageError.NOT_FOUND,
|
|
595
|
-
message: storageM("storage_fileNotFound", {
|
|
735
|
+
message: storageM("storage_fileNotFound", {
|
|
736
|
+
params: { key: key ?? "" }
|
|
737
|
+
}),
|
|
596
738
|
cause: error
|
|
597
739
|
};
|
|
598
740
|
}
|
|
599
741
|
if (e.name === "AccessDenied" || e.Code === "AccessDenied") {
|
|
600
742
|
return {
|
|
601
743
|
...HaiStorageError.PERMISSION_DENIED,
|
|
602
|
-
message: storageM("storage_permissionDenied", {
|
|
744
|
+
message: storageM("storage_permissionDenied", {
|
|
745
|
+
params: { key: key ?? "" }
|
|
746
|
+
}),
|
|
603
747
|
cause: error
|
|
604
748
|
};
|
|
605
749
|
}
|
|
@@ -619,7 +763,9 @@ function toStorageError2(error, key) {
|
|
|
619
763
|
}
|
|
620
764
|
return {
|
|
621
765
|
...HaiStorageError.OPERATION_FAILED,
|
|
622
|
-
message: storageM("storage_operationFailed", {
|
|
766
|
+
message: storageM("storage_operationFailed", {
|
|
767
|
+
params: { error: e.message ?? "" }
|
|
768
|
+
}),
|
|
623
769
|
cause: error
|
|
624
770
|
};
|
|
625
771
|
}
|
|
@@ -719,7 +865,12 @@ function createS3Provider() {
|
|
|
719
865
|
ContentDisposition: options.contentDisposition,
|
|
720
866
|
IfNoneMatch: options.ifNoneMatch
|
|
721
867
|
};
|
|
722
|
-
logOperation("write", "file.put", {
|
|
868
|
+
logOperation("write", "file.put", {
|
|
869
|
+
key,
|
|
870
|
+
fullPath,
|
|
871
|
+
byteLength: buffered ? size : void 0,
|
|
872
|
+
options
|
|
873
|
+
});
|
|
723
874
|
if (buffered) {
|
|
724
875
|
await s3Client.send(new PutObjectCommand({ ...params, Body: body }), { abortSignal: options.signal });
|
|
725
876
|
} else {
|
|
@@ -737,7 +888,13 @@ function createS3Provider() {
|
|
|
737
888
|
if (options.signal?.aborted)
|
|
738
889
|
abort();
|
|
739
890
|
const { Upload } = await import('@aws-sdk/lib-storage');
|
|
740
|
-
await new Upload({
|
|
891
|
+
await new Upload({
|
|
892
|
+
client: s3Client,
|
|
893
|
+
params: { ...params, Body: stream },
|
|
894
|
+
queueSize: 2,
|
|
895
|
+
leavePartsOnError: false,
|
|
896
|
+
abortController: controller
|
|
897
|
+
}).done();
|
|
741
898
|
} finally {
|
|
742
899
|
options.signal?.removeEventListener("abort", abort);
|
|
743
900
|
stream.destroy();
|
|
@@ -852,7 +1009,9 @@ function createS3Provider() {
|
|
|
852
1009
|
const s3Client = getClient("deleteMany");
|
|
853
1010
|
const s3Config = getConfig();
|
|
854
1011
|
logOperation("write", "file.deleteMany", { count: keys.length, keys });
|
|
855
|
-
const uniqueKeys = [
|
|
1012
|
+
const uniqueKeys = [
|
|
1013
|
+
...new Set(keys)
|
|
1014
|
+
];
|
|
856
1015
|
const summary = { deletedKeys: [], failures: [] };
|
|
857
1016
|
for (let offset = 0; offset < uniqueKeys.length; offset += DELETE_BATCH_SIZE) {
|
|
858
1017
|
let pending = uniqueKeys.slice(offset, offset + DELETE_BATCH_SIZE);
|
|
@@ -861,7 +1020,12 @@ function createS3Provider() {
|
|
|
861
1020
|
try {
|
|
862
1021
|
response = await s3Client.send(new DeleteObjectsCommand({
|
|
863
1022
|
Bucket: s3Config.bucket,
|
|
864
|
-
Delete: {
|
|
1023
|
+
Delete: {
|
|
1024
|
+
Objects: [
|
|
1025
|
+
...new Set(pending.map(fullKey))
|
|
1026
|
+
].map((Key) => ({ Key })),
|
|
1027
|
+
Quiet: false
|
|
1028
|
+
}
|
|
865
1029
|
}));
|
|
866
1030
|
} catch (cause) {
|
|
867
1031
|
const failure = cause && typeof cause === "object" ? cause : {};
|
|
@@ -870,7 +1034,12 @@ function createS3Provider() {
|
|
|
870
1034
|
const retryable = status === void 0 || status === 408 || status === 429 || status >= 500;
|
|
871
1035
|
const outcome = status !== void 0 && status >= 400 && status < 500 && status !== 408 ? "failed" : "unknown";
|
|
872
1036
|
summary.failures.push(...pending.map((key) => ({ key, code, retryable, outcome })));
|
|
873
|
-
summary.failures.push(...uniqueKeys.slice(offset + DELETE_BATCH_SIZE).map((key) => ({
|
|
1037
|
+
summary.failures.push(...uniqueKeys.slice(offset + DELETE_BATCH_SIZE).map((key) => ({
|
|
1038
|
+
key,
|
|
1039
|
+
code,
|
|
1040
|
+
retryable,
|
|
1041
|
+
outcome: "notAttempted"
|
|
1042
|
+
})));
|
|
874
1043
|
return toStorageDeleteResult(summary);
|
|
875
1044
|
}
|
|
876
1045
|
const outcomes = /* @__PURE__ */ new Map();
|
|
@@ -878,7 +1047,10 @@ function createS3Provider() {
|
|
|
878
1047
|
let malformed = false;
|
|
879
1048
|
const items = [
|
|
880
1049
|
...(response.Deleted ?? []).map((item) => ({ key: item.Key, code: null })),
|
|
881
|
-
...(response.Errors ?? []).map((item) => ({
|
|
1050
|
+
...(response.Errors ?? []).map((item) => ({
|
|
1051
|
+
key: item.Key,
|
|
1052
|
+
code: item.Code && /^[\w.-]{1,80}$/.test(item.Code) ? item.Code : "UnknownError"
|
|
1053
|
+
}))
|
|
882
1054
|
];
|
|
883
1055
|
for (const item of items) {
|
|
884
1056
|
if (!item.key || !requested.has(item.key) || outcomes.has(item.key)) {
|
|
@@ -888,19 +1060,35 @@ function createS3Provider() {
|
|
|
888
1060
|
outcomes.set(item.key, item.code);
|
|
889
1061
|
}
|
|
890
1062
|
if (malformed || outcomes.size !== requested.size) {
|
|
891
|
-
summary.failures.push(...pending.map((key) => ({
|
|
892
|
-
|
|
1063
|
+
summary.failures.push(...pending.map((key) => ({
|
|
1064
|
+
key,
|
|
1065
|
+
code: "InvalidDeleteResponse",
|
|
1066
|
+
retryable: false,
|
|
1067
|
+
outcome: "unknown"
|
|
1068
|
+
})));
|
|
1069
|
+
summary.failures.push(...uniqueKeys.slice(offset + DELETE_BATCH_SIZE).map((key) => ({
|
|
1070
|
+
key,
|
|
1071
|
+
code: "InvalidDeleteResponse",
|
|
1072
|
+
retryable: false,
|
|
1073
|
+
outcome: "notAttempted"
|
|
1074
|
+
})));
|
|
893
1075
|
return toStorageDeleteResult(summary);
|
|
894
1076
|
}
|
|
895
1077
|
const retry = [];
|
|
896
1078
|
for (const key of pending) {
|
|
897
1079
|
const code = outcomes.get(fullKey(key));
|
|
898
|
-
if (code === null)
|
|
1080
|
+
if (code === null) {
|
|
899
1081
|
summary.deletedKeys.push(key);
|
|
900
|
-
else if (code && RETRYABLE_DELETE_CODES.has(code) && attempt < 2)
|
|
1082
|
+
} else if (code && RETRYABLE_DELETE_CODES.has(code) && attempt < 2) {
|
|
901
1083
|
retry.push(key);
|
|
902
|
-
else
|
|
903
|
-
summary.failures.push({
|
|
1084
|
+
} else {
|
|
1085
|
+
summary.failures.push({
|
|
1086
|
+
key,
|
|
1087
|
+
code: code ?? "UnknownError",
|
|
1088
|
+
retryable: !!code && RETRYABLE_DELETE_CODES.has(code),
|
|
1089
|
+
outcome: "failed"
|
|
1090
|
+
});
|
|
1091
|
+
}
|
|
904
1092
|
}
|
|
905
1093
|
pending = retry;
|
|
906
1094
|
if (pending.length > 0)
|
|
@@ -918,10 +1106,20 @@ function createS3Provider() {
|
|
|
918
1106
|
const s3Config = getConfig();
|
|
919
1107
|
const sourceFullPath = fullKey(sourceKey);
|
|
920
1108
|
const destFullPath = fullKey(destKey);
|
|
921
|
-
|
|
1109
|
+
const copySource = [
|
|
1110
|
+
s3Config.bucket,
|
|
1111
|
+
...sourceFullPath.split("/")
|
|
1112
|
+
].map((part) => encodeURIComponent(part).replace(/[!'()*]/g, (character) => `%${character.charCodeAt(0).toString(16).toUpperCase()}`)).join("/");
|
|
1113
|
+
logOperation("write", "file.copy", {
|
|
1114
|
+
sourceKey,
|
|
1115
|
+
destKey,
|
|
1116
|
+
sourceFullPath,
|
|
1117
|
+
destFullPath,
|
|
1118
|
+
options
|
|
1119
|
+
});
|
|
922
1120
|
await s3Client.send(new CopyObjectCommand({
|
|
923
1121
|
Bucket: s3Config.bucket,
|
|
924
|
-
CopySource:
|
|
1122
|
+
CopySource: copySource,
|
|
925
1123
|
Key: destFullPath,
|
|
926
1124
|
ContentType: options.contentType,
|
|
927
1125
|
Metadata: options.metadata,
|
|
@@ -1092,7 +1290,11 @@ function createS3Provider() {
|
|
|
1092
1290
|
if (cfg.publicEndpoint) {
|
|
1093
1291
|
const endpoint = new URL(cfg.publicEndpoint);
|
|
1094
1292
|
publicBasePath = endpoint.pathname.replace(TRAILING_SLASHES_REGEX, "");
|
|
1095
|
-
presignClient = new S3Client({
|
|
1293
|
+
presignClient = new S3Client({
|
|
1294
|
+
...clientConfig,
|
|
1295
|
+
endpoint: endpoint.origin,
|
|
1296
|
+
forcePathStyle: publicBasePath ? true : cfg.forcePathStyle
|
|
1297
|
+
});
|
|
1096
1298
|
}
|
|
1097
1299
|
if (cfg.createBucket) {
|
|
1098
1300
|
try {
|
|
@@ -1105,7 +1307,9 @@ function createS3Provider() {
|
|
|
1105
1307
|
await client.send(new CreateBucketCommand({
|
|
1106
1308
|
Bucket: cfg.bucket,
|
|
1107
1309
|
// AWS us-east-1 要省略 LocationConstraint;兼容服务可使用自定义区域名。
|
|
1108
|
-
...cfg.region === "us-east-1" ? {} : {
|
|
1310
|
+
...cfg.region === "us-east-1" ? {} : {
|
|
1311
|
+
CreateBucketConfiguration: { LocationConstraint: cfg.region }
|
|
1312
|
+
}
|
|
1109
1313
|
}));
|
|
1110
1314
|
logger2.info("S3 bucket created", { bucket: cfg.bucket });
|
|
1111
1315
|
} catch (createError) {
|
|
@@ -1131,7 +1335,10 @@ function createS3Provider() {
|
|
|
1131
1335
|
await closeClients();
|
|
1132
1336
|
return err(original);
|
|
1133
1337
|
} catch (cleanupError) {
|
|
1134
|
-
return err({
|
|
1338
|
+
return err({
|
|
1339
|
+
...original,
|
|
1340
|
+
ext: { ...original.ext, cleanupPending: true, cleanupError }
|
|
1341
|
+
});
|
|
1135
1342
|
}
|
|
1136
1343
|
}
|
|
1137
1344
|
},
|
|
@@ -1143,7 +1350,13 @@ function createS3Provider() {
|
|
|
1143
1350
|
}
|
|
1144
1351
|
var HEX_CHECKSUM = /^[\da-f]+$/i;
|
|
1145
1352
|
var BASE64_CHECKSUM = /^(?:[A-Z\d+/]{4})*(?:[A-Z\d+/]{2}==|[A-Z\d+/]{3}=)?$/i;
|
|
1146
|
-
var ALGORITHMS = /* @__PURE__ */ new Set([
|
|
1353
|
+
var ALGORITHMS = /* @__PURE__ */ new Set([
|
|
1354
|
+
"sha256",
|
|
1355
|
+
"sha384",
|
|
1356
|
+
"sha512",
|
|
1357
|
+
"sha1",
|
|
1358
|
+
"md5"
|
|
1359
|
+
]);
|
|
1147
1360
|
async function generate(data, options = {}) {
|
|
1148
1361
|
try {
|
|
1149
1362
|
const algorithm = options.algorithm ?? "sha256";
|
|
@@ -1195,9 +1408,16 @@ async function closeOwnedProvider() {
|
|
|
1195
1408
|
return ok(void 0);
|
|
1196
1409
|
} catch (error) {
|
|
1197
1410
|
logger3.error("Provider cleanup failed; retaining resource for retry", { error });
|
|
1198
|
-
return err({
|
|
1199
|
-
|
|
1200
|
-
|
|
1411
|
+
return err({
|
|
1412
|
+
...HaiStorageError.CONNECTION_FAILED,
|
|
1413
|
+
message: storageM("storage_operationFailed", {
|
|
1414
|
+
params: {
|
|
1415
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1416
|
+
}
|
|
1417
|
+
}),
|
|
1418
|
+
cause: error,
|
|
1419
|
+
ext: { cleanupPending: true }
|
|
1420
|
+
});
|
|
1201
1421
|
}
|
|
1202
1422
|
}
|
|
1203
1423
|
async function failInitialization(result) {
|
|
@@ -1208,9 +1428,15 @@ async function failInitialization(result) {
|
|
|
1208
1428
|
const ext = { ...result.error.ext };
|
|
1209
1429
|
delete ext.cleanupPending;
|
|
1210
1430
|
delete ext.cleanupError;
|
|
1211
|
-
return err({
|
|
1431
|
+
return err({
|
|
1432
|
+
...result.error,
|
|
1433
|
+
ext: Object.keys(ext).length > 0 ? ext : void 0
|
|
1434
|
+
});
|
|
1212
1435
|
}
|
|
1213
|
-
return err({
|
|
1436
|
+
return err({
|
|
1437
|
+
...result.error,
|
|
1438
|
+
ext: { ...result.error.ext, cleanupPending: true, cleanupError: cleanup.error }
|
|
1439
|
+
});
|
|
1214
1440
|
}
|
|
1215
1441
|
function createProvider(config) {
|
|
1216
1442
|
switch (config.type) {
|
|
@@ -1219,7 +1445,9 @@ function createProvider(config) {
|
|
|
1219
1445
|
case "local":
|
|
1220
1446
|
return createLocalProvider();
|
|
1221
1447
|
default:
|
|
1222
|
-
throw new Error(storageM("storage_unsupportedType", {
|
|
1448
|
+
throw new Error(storageM("storage_unsupportedType", {
|
|
1449
|
+
params: { type: config.type }
|
|
1450
|
+
}));
|
|
1223
1451
|
}
|
|
1224
1452
|
}
|
|
1225
1453
|
var notInitialized = core.module.createNotInitializedKit(
|
|
@@ -1260,7 +1488,9 @@ var storage = {
|
|
|
1260
1488
|
logger3.warn("Storage init already in progress, skipping concurrent call");
|
|
1261
1489
|
return err(
|
|
1262
1490
|
HaiStorageError.OPERATION_FAILED,
|
|
1263
|
-
storageM("storage_operationFailed", {
|
|
1491
|
+
storageM("storage_operationFailed", {
|
|
1492
|
+
params: { error: "Concurrent initialization detected" }
|
|
1493
|
+
})
|
|
1264
1494
|
);
|
|
1265
1495
|
}
|
|
1266
1496
|
initInProgress = true;
|
|
@@ -1278,7 +1508,9 @@ var storage = {
|
|
|
1278
1508
|
logger3.error("Storage config validation failed", { error: parseResult.error.message });
|
|
1279
1509
|
return err(
|
|
1280
1510
|
HaiStorageError.CONFIG_ERROR,
|
|
1281
|
-
storageM("storage_configError", {
|
|
1511
|
+
storageM("storage_configError", {
|
|
1512
|
+
params: { error: parseResult.error.message }
|
|
1513
|
+
}),
|
|
1282
1514
|
parseResult.error
|
|
1283
1515
|
);
|
|
1284
1516
|
}
|
|
@@ -1305,7 +1537,9 @@ var storage = {
|
|
|
1305
1537
|
return await failInitialization(err(
|
|
1306
1538
|
HaiStorageError.CONNECTION_FAILED,
|
|
1307
1539
|
storageM("storage_operationFailed", {
|
|
1308
|
-
params: {
|
|
1540
|
+
params: {
|
|
1541
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1542
|
+
}
|
|
1309
1543
|
}),
|
|
1310
1544
|
error
|
|
1311
1545
|
));
|