@h-ai/storage 0.1.0-alpha.66 → 0.1.0-alpha.68
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +100 -25
- package/README.ja.md +101 -26
- package/README.md +141 -26
- package/dist/browser.d.ts +399 -93
- package/dist/browser.js +3 -2
- package/dist/{chunk-KXE6OO5T.js → chunk-C7FR45VO.js} +8 -5
- package/dist/chunk-C7FR45VO.js.map +1 -0
- package/dist/chunk-I4BSES63.js +357 -0
- package/dist/chunk-I4BSES63.js.map +1 -0
- package/dist/chunk-P65WYXJG.js +33 -0
- package/dist/chunk-P65WYXJG.js.map +1 -0
- package/dist/client/index.d.ts +57 -48
- package/dist/client/index.js +1 -1
- package/dist/node.d.ts +15 -11
- package/dist/node.js +296 -136
- package/dist/node.js.map +1 -1
- package/dist/schemas.d.ts +103 -0
- package/dist/schemas.js +3 -0
- package/dist/schemas.js.map +1 -0
- package/package.json +7 -2
- package/dist/chunk-25GYTRA7.js +0 -274
- package/dist/chunk-25GYTRA7.js.map +0 -1
- package/dist/chunk-KXE6OO5T.js.map +0 -1
package/dist/node.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
import { HaiStorageError, StorageConfigSchema } from './chunk-
|
|
2
|
-
export { HaiStorageError, LocalConfigSchema, PresignOptionsSchema, PresignUploadOptionsSchema, S3ConfigSchema, StorageConfigSchema, StorageOperationLogConfigSchema, StorageTypeSchema } from './chunk-
|
|
3
|
-
import { storageM, MIME_TYPES, MIME_TYPE_DEFAULT } from './chunk-
|
|
4
|
-
export { downloadAndSave, downloadWithPresignedUrl, formatFileSize, getFileExtension, getMimeType, uploadWithPresignedUrl } from './chunk-
|
|
1
|
+
import { HaiStorageError, StorageConfigSchema, PresignUploadOptionsSchema, PresignOptionsSchema } from './chunk-C7FR45VO.js';
|
|
2
|
+
export { HaiStorageError, LocalConfigSchema, PresignOptionsSchema, PresignUploadOptionsSchema, S3ConfigSchema, StorageConfigSchema, StorageOperationLogConfigSchema, StorageTypeSchema } from './chunk-C7FR45VO.js';
|
|
3
|
+
import { storageM, MIME_TYPES, MIME_TYPE_DEFAULT } from './chunk-I4BSES63.js';
|
|
4
|
+
export { downloadAndSave, downloadWithPresignedUrl, formatFileSize, getFileExtension, getMimeType, uploadWithPresignedUrl } from './chunk-I4BSES63.js';
|
|
5
|
+
export { STORAGE_DELETE_MAX_KEYS, STORAGE_KEY_MAX_BYTES, StorageDeleteKeysSchema, StorageKeySchema, StoragePrefixSchema, StoragePresignExpiresInSchema, getStorageKeyIssue } from './chunk-P65WYXJG.js';
|
|
6
|
+
import process from 'node:process';
|
|
5
7
|
import { core, err, ok } from '@h-ai/core';
|
|
6
8
|
import { Buffer } from 'node:buffer';
|
|
7
9
|
import * as crypto from 'node:crypto';
|
|
@@ -14,6 +16,18 @@ import { pipeline } from 'node:stream/promises';
|
|
|
14
16
|
import { S3Client, ListObjectsV2Command, PutObjectCommand, GetObjectCommand, CopyObjectCommand, DeleteObjectsCommand, DeleteObjectCommand, HeadObjectCommand } from '@aws-sdk/client-s3';
|
|
15
17
|
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
|
|
16
18
|
|
|
19
|
+
function toStorageDeleteResult(summary) {
|
|
20
|
+
if (summary.failures.length === 0)
|
|
21
|
+
return ok(summary);
|
|
22
|
+
return {
|
|
23
|
+
success: false,
|
|
24
|
+
error: {
|
|
25
|
+
...HaiStorageError.OPERATION_FAILED,
|
|
26
|
+
message: storageM("storage_deleteManyFailed", { params: { count: summary.failures.length } }),
|
|
27
|
+
ext: { deleteMany: summary }
|
|
28
|
+
}
|
|
29
|
+
};
|
|
30
|
+
}
|
|
17
31
|
async function* toByteStream(data, signal) {
|
|
18
32
|
signal?.throwIfAborted();
|
|
19
33
|
if (typeof data === "string" || data instanceof Uint8Array) {
|
|
@@ -366,21 +380,22 @@ function createLocalProvider() {
|
|
|
366
380
|
},
|
|
367
381
|
async deleteMany(keys) {
|
|
368
382
|
logOperation("write", "file.deleteMany", { count: keys.length, keys });
|
|
369
|
-
const
|
|
370
|
-
const
|
|
371
|
-
for (
|
|
372
|
-
|
|
373
|
-
|
|
383
|
+
const summary = { deletedKeys: [], failures: [] };
|
|
384
|
+
const uniqueKeys = [...new Set(keys)];
|
|
385
|
+
for (let offset = 0; offset < uniqueKeys.length; offset += 32) {
|
|
386
|
+
const batch = uniqueKeys.slice(offset, offset + 32);
|
|
387
|
+
const results = await Promise.allSettled(batch.map((key) => file.delete(key)));
|
|
388
|
+
for (const [index, result] of results.entries()) {
|
|
389
|
+
const key = batch[index];
|
|
390
|
+
if (result.status === "rejected")
|
|
391
|
+
summary.failures.push({ key, code: "UnexpectedDeleteFailure", retryable: false, outcome: "failed" });
|
|
392
|
+
else if (!result.value.success)
|
|
393
|
+
summary.failures.push({ key, code: String(result.value.error.code), retryable: false, outcome: "failed" });
|
|
394
|
+
else
|
|
395
|
+
summary.deletedKeys.push(key);
|
|
374
396
|
}
|
|
375
397
|
}
|
|
376
|
-
|
|
377
|
-
return err(
|
|
378
|
-
HaiStorageError.OPERATION_FAILED,
|
|
379
|
-
storageM("storage_deleteManyFailed", { params: { count: errors.length } }),
|
|
380
|
-
errors
|
|
381
|
-
);
|
|
382
|
-
}
|
|
383
|
-
return ok(void 0);
|
|
398
|
+
return toStorageDeleteResult(summary);
|
|
384
399
|
},
|
|
385
400
|
async copy(sourceKey, destKey, options = {}) {
|
|
386
401
|
if (isMetaFile(sourceKey) || isMetaFile(destKey)) {
|
|
@@ -423,49 +438,85 @@ function createLocalProvider() {
|
|
|
423
438
|
async list(options = {}) {
|
|
424
439
|
try {
|
|
425
440
|
const cfg = getConfig();
|
|
426
|
-
const prefix = options.prefix
|
|
427
|
-
const delimiter = options.delimiter
|
|
428
|
-
const maxKeys = options.maxKeys
|
|
441
|
+
const prefix = options.prefix === void 0 ? "" : options.prefix;
|
|
442
|
+
const delimiter = options.delimiter === void 0 ? "" : options.delimiter;
|
|
443
|
+
const maxKeys = options.maxKeys === void 0 ? 1e3 : options.maxKeys;
|
|
444
|
+
if (typeof prefix !== "string" || typeof delimiter !== "string" || !Number.isInteger(maxKeys) || maxKeys < 1 || maxKeys > 1e3 || Buffer.from(prefix).toString("utf8") !== prefix || Buffer.from(delimiter).toString("utf8") !== delimiter) {
|
|
445
|
+
return err(HaiStorageError.INVALID_LIST_OPTIONS, storageM("storage_listOptionsInvalid"));
|
|
446
|
+
}
|
|
429
447
|
logOperation("read", "dir.list", { options });
|
|
430
|
-
|
|
431
|
-
const
|
|
432
|
-
const
|
|
433
|
-
|
|
434
|
-
|
|
448
|
+
safePath(cfg.root, prefix);
|
|
449
|
+
const root = await fsp.realpath(cfg.root);
|
|
450
|
+
const scope = crypto.createHash("sha256").update(JSON.stringify([root, prefix, delimiter])).digest("hex");
|
|
451
|
+
let after = "";
|
|
452
|
+
if (options.continuationToken !== void 0) {
|
|
453
|
+
const token = options.continuationToken;
|
|
435
454
|
try {
|
|
436
|
-
|
|
455
|
+
if (typeof token !== "string" || token.length === 0 || token.length > 262144)
|
|
456
|
+
throw new Error("Invalid cursor encoding");
|
|
457
|
+
const bytes = Buffer.from(token, "base64url");
|
|
458
|
+
if (bytes.toString("base64url") !== token)
|
|
459
|
+
throw new Error("Invalid cursor encoding");
|
|
460
|
+
const decoded = JSON.parse(bytes.toString("utf8"));
|
|
461
|
+
if (!Array.isArray(decoded) || decoded.length !== 3 || decoded[0] !== 1 || decoded[1] !== scope || typeof decoded[2] !== "string" || decoded[2].length === 0 || !decoded[2].startsWith(prefix)) {
|
|
462
|
+
throw new Error("Invalid cursor scope");
|
|
463
|
+
}
|
|
464
|
+
after = decoded[2];
|
|
437
465
|
} catch {
|
|
438
|
-
return;
|
|
466
|
+
return err(HaiStorageError.INVALID_LIST_OPTIONS, storageM("storage_listCursorInvalid"));
|
|
439
467
|
}
|
|
468
|
+
}
|
|
469
|
+
const files = [];
|
|
470
|
+
const commonPrefixes = [];
|
|
471
|
+
const compare = (left, right) => Buffer.compare(Buffer.from(left), Buffer.from(right));
|
|
472
|
+
async function* readDir(dirPath, directoryKey) {
|
|
473
|
+
const entries = await fsp.readdir(dirPath, { withFileTypes: true });
|
|
474
|
+
entries.sort((left, right) => compare(left.name + (left.isDirectory() ? "/" : ""), right.name + (right.isDirectory() ? "/" : "")));
|
|
440
475
|
for (const entry of entries) {
|
|
441
|
-
if (files.length >= maxKeys)
|
|
442
|
-
return;
|
|
443
|
-
const fullEntryPath = path.join(dirPath, entry.name);
|
|
444
|
-
const key = path.relative(cfg.root, fullEntryPath).replace(/\\/g, "/");
|
|
445
476
|
if (isMetaFile(entry.name))
|
|
446
477
|
continue;
|
|
478
|
+
const key = directoryKey + entry.name;
|
|
479
|
+
const filePath = path.join(dirPath, entry.name);
|
|
447
480
|
if (entry.isDirectory()) {
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
await readDir(fullEntryPath);
|
|
481
|
+
const childPrefix = `${key}/`;
|
|
482
|
+
if (prefix.startsWith(childPrefix) || childPrefix.startsWith(prefix)) {
|
|
483
|
+
yield* readDir(filePath, childPrefix);
|
|
452
484
|
}
|
|
453
|
-
} else if (entry.isFile()) {
|
|
454
|
-
|
|
455
|
-
files.push({
|
|
456
|
-
key,
|
|
457
|
-
size: stat2.size,
|
|
458
|
-
contentType: getMimeType2(key),
|
|
459
|
-
lastModified: stat2.mtime
|
|
460
|
-
});
|
|
485
|
+
} else if (entry.isFile() && key.startsWith(prefix)) {
|
|
486
|
+
yield { key, filePath };
|
|
461
487
|
}
|
|
462
488
|
}
|
|
463
489
|
}
|
|
464
|
-
|
|
490
|
+
let previous = "";
|
|
491
|
+
let last = "";
|
|
492
|
+
let count = 0;
|
|
493
|
+
let isTruncated = false;
|
|
494
|
+
for await (const entry of readDir(root, "")) {
|
|
495
|
+
const index = delimiter ? entry.key.indexOf(delimiter, prefix.length) : -1;
|
|
496
|
+
const logicalKey = index < 0 ? entry.key : entry.key.slice(0, index + delimiter.length);
|
|
497
|
+
if (logicalKey === previous)
|
|
498
|
+
continue;
|
|
499
|
+
previous = logicalKey;
|
|
500
|
+
if (compare(logicalKey, after) <= 0)
|
|
501
|
+
continue;
|
|
502
|
+
if (count === maxKeys) {
|
|
503
|
+
isTruncated = true;
|
|
504
|
+
break;
|
|
505
|
+
}
|
|
506
|
+
if (index >= 0) {
|
|
507
|
+
commonPrefixes.push(logicalKey);
|
|
508
|
+
} else {
|
|
509
|
+
const stat2 = await fsp.stat(entry.filePath);
|
|
510
|
+
files.push({ key: entry.key, size: stat2.size, contentType: getMimeType2(entry.key), lastModified: stat2.mtime });
|
|
511
|
+
}
|
|
512
|
+
count++;
|
|
513
|
+
last = logicalKey;
|
|
514
|
+
}
|
|
465
515
|
return ok({
|
|
466
516
|
files,
|
|
467
|
-
commonPrefixes
|
|
468
|
-
isTruncated
|
|
517
|
+
commonPrefixes,
|
|
518
|
+
isTruncated,
|
|
519
|
+
...isTruncated ? { nextContinuationToken: Buffer.from(JSON.stringify([1, scope, last])).toString("base64url") } : {}
|
|
469
520
|
});
|
|
470
521
|
} catch (error) {
|
|
471
522
|
return err(toStorageError(error));
|
|
@@ -488,17 +539,11 @@ function createLocalProvider() {
|
|
|
488
539
|
const presign = {
|
|
489
540
|
async getUrl(key, options) {
|
|
490
541
|
logOperation("read", "presign.getUrl", { key, options });
|
|
491
|
-
|
|
492
|
-
const expires = Math.floor(Date.now() / 1e3) + expiresIn;
|
|
493
|
-
const signature = crypto.createHash("sha256").update(`${key}:${expires}`).digest("hex").slice(0, 16);
|
|
494
|
-
return ok(`local://${key}?expires=${expires}&signature=${signature}`);
|
|
542
|
+
return err(HaiStorageError.UNSUPPORTED_OPERATION, storageM("storage_localPresignUnsupported"));
|
|
495
543
|
},
|
|
496
544
|
async putUrl(key, options) {
|
|
497
545
|
logOperation("write", "presign.putUrl", { key, options });
|
|
498
|
-
|
|
499
|
-
const expires = Math.floor(Date.now() / 1e3) + expiresIn;
|
|
500
|
-
const signature = crypto.createHash("sha256").update(`put:${key}:${expires}`).digest("hex").slice(0, 16);
|
|
501
|
-
return ok(`local://${key}?action=put&expires=${expires}&signature=${signature}`);
|
|
546
|
+
return err(HaiStorageError.UNSUPPORTED_OPERATION, storageM("storage_localPresignUnsupported"));
|
|
502
547
|
},
|
|
503
548
|
publicUrl(key) {
|
|
504
549
|
logOperation("read", "presign.publicUrl", { key });
|
|
@@ -543,18 +588,8 @@ function createLocalProvider() {
|
|
|
543
588
|
var logger2 = core.logger.child({ module: "storage", scope: "provider-s3" });
|
|
544
589
|
var TRAILING_SLASHES_REGEX = /\/+$/;
|
|
545
590
|
var LEADING_SLASHES_REGEX = /^\/+/;
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
const parsed = new URL(url);
|
|
549
|
-
if (parsed.username)
|
|
550
|
-
parsed.username = "***";
|
|
551
|
-
if (parsed.password)
|
|
552
|
-
parsed.password = "***";
|
|
553
|
-
return parsed.toString();
|
|
554
|
-
} catch {
|
|
555
|
-
return "(invalid url)";
|
|
556
|
-
}
|
|
557
|
-
}
|
|
591
|
+
var DELETE_BATCH_SIZE = 1e3;
|
|
592
|
+
var RETRYABLE_DELETE_CODES = /* @__PURE__ */ new Set(["InternalError", "ServiceUnavailable", "SlowDown", "RequestTimeout"]);
|
|
558
593
|
function toStorageError2(error, key) {
|
|
559
594
|
const e = error;
|
|
560
595
|
if (e.name === "PreconditionFailed" || e.$metadata?.httpStatusCode === 412)
|
|
@@ -609,15 +644,46 @@ function withoutPrefix(key, prefix) {
|
|
|
609
644
|
}
|
|
610
645
|
function createS3Provider() {
|
|
611
646
|
let client = null;
|
|
647
|
+
let deleteClient = null;
|
|
612
648
|
let config = null;
|
|
613
|
-
|
|
614
|
-
|
|
649
|
+
let ready = false;
|
|
650
|
+
let closing = null;
|
|
651
|
+
function closeClients() {
|
|
652
|
+
ready = false;
|
|
653
|
+
config = null;
|
|
654
|
+
if (closing)
|
|
655
|
+
return closing;
|
|
656
|
+
closing = Promise.resolve().then(() => {
|
|
657
|
+
const failures = [];
|
|
658
|
+
for (const kind of ["normal", "delete"]) {
|
|
659
|
+
const owned = kind === "normal" ? client : deleteClient;
|
|
660
|
+
if (!owned)
|
|
661
|
+
continue;
|
|
662
|
+
try {
|
|
663
|
+
owned.destroy();
|
|
664
|
+
if (kind === "normal")
|
|
665
|
+
client = null;
|
|
666
|
+
else deleteClient = null;
|
|
667
|
+
} catch (error) {
|
|
668
|
+
failures.push(error);
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
if (failures.length > 0)
|
|
672
|
+
throw new AggregateError(failures, "S3 client cleanup failed");
|
|
673
|
+
}).finally(() => {
|
|
674
|
+
closing = null;
|
|
675
|
+
});
|
|
676
|
+
return closing;
|
|
677
|
+
}
|
|
678
|
+
function getClient(operation) {
|
|
679
|
+
const selected = operation === "deleteMany" ? deleteClient : client;
|
|
680
|
+
if (!ready || !selected) {
|
|
615
681
|
throw new Error(storageM("storage_s3ClientNotInitialized"));
|
|
616
682
|
}
|
|
617
|
-
return
|
|
683
|
+
return selected;
|
|
618
684
|
}
|
|
619
685
|
function getConfig() {
|
|
620
|
-
if (!config) {
|
|
686
|
+
if (!ready || !config) {
|
|
621
687
|
throw new Error(storageM("storage_s3ConfigNotInitialized"));
|
|
622
688
|
}
|
|
623
689
|
return config;
|
|
@@ -777,19 +843,65 @@ function createS3Provider() {
|
|
|
777
843
|
},
|
|
778
844
|
async deleteMany(keys) {
|
|
779
845
|
try {
|
|
780
|
-
const s3Client = getClient();
|
|
846
|
+
const s3Client = getClient("deleteMany");
|
|
781
847
|
const s3Config = getConfig();
|
|
782
|
-
if (keys.length === 0) {
|
|
783
|
-
return ok(void 0);
|
|
784
|
-
}
|
|
785
848
|
logOperation("write", "file.deleteMany", { count: keys.length, keys });
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
849
|
+
const uniqueKeys = [...new Set(keys)];
|
|
850
|
+
const summary = { deletedKeys: [], failures: [] };
|
|
851
|
+
for (let offset = 0; offset < uniqueKeys.length; offset += DELETE_BATCH_SIZE) {
|
|
852
|
+
let pending = uniqueKeys.slice(offset, offset + DELETE_BATCH_SIZE);
|
|
853
|
+
for (let attempt = 0; pending.length > 0 && attempt < 3; attempt++) {
|
|
854
|
+
let response;
|
|
855
|
+
try {
|
|
856
|
+
response = await s3Client.send(new DeleteObjectsCommand({
|
|
857
|
+
Bucket: s3Config.bucket,
|
|
858
|
+
Delete: { Objects: [...new Set(pending.map(fullKey))].map((Key) => ({ Key })), Quiet: false }
|
|
859
|
+
}));
|
|
860
|
+
} catch (cause) {
|
|
861
|
+
const failure = cause && typeof cause === "object" ? cause : {};
|
|
862
|
+
const status = failure.$metadata?.httpStatusCode;
|
|
863
|
+
const code = failure.name && /^[\w.-]{1,80}$/.test(failure.name) ? failure.name : "RequestFailed";
|
|
864
|
+
const retryable = status === void 0 || status === 408 || status === 429 || status >= 500;
|
|
865
|
+
const outcome = status !== void 0 && status >= 400 && status < 500 && status !== 408 ? "failed" : "unknown";
|
|
866
|
+
summary.failures.push(...pending.map((key) => ({ key, code, retryable, outcome })));
|
|
867
|
+
summary.failures.push(...uniqueKeys.slice(offset + DELETE_BATCH_SIZE).map((key) => ({ key, code, retryable, outcome: "notAttempted" })));
|
|
868
|
+
return toStorageDeleteResult(summary);
|
|
869
|
+
}
|
|
870
|
+
const outcomes = /* @__PURE__ */ new Map();
|
|
871
|
+
const requested = new Set(pending.map(fullKey));
|
|
872
|
+
let malformed = false;
|
|
873
|
+
const items = [
|
|
874
|
+
...(response.Deleted ?? []).map((item) => ({ key: item.Key, code: null })),
|
|
875
|
+
...(response.Errors ?? []).map((item) => ({ key: item.Key, code: item.Code && /^[\w.-]{1,80}$/.test(item.Code) ? item.Code : "UnknownError" }))
|
|
876
|
+
];
|
|
877
|
+
for (const item of items) {
|
|
878
|
+
if (!item.key || !requested.has(item.key) || outcomes.has(item.key)) {
|
|
879
|
+
malformed = true;
|
|
880
|
+
break;
|
|
881
|
+
}
|
|
882
|
+
outcomes.set(item.key, item.code);
|
|
883
|
+
}
|
|
884
|
+
if (malformed || outcomes.size !== requested.size) {
|
|
885
|
+
summary.failures.push(...pending.map((key) => ({ key, code: "InvalidDeleteResponse", retryable: false, outcome: "unknown" })));
|
|
886
|
+
summary.failures.push(...uniqueKeys.slice(offset + DELETE_BATCH_SIZE).map((key) => ({ key, code: "InvalidDeleteResponse", retryable: false, outcome: "notAttempted" })));
|
|
887
|
+
return toStorageDeleteResult(summary);
|
|
888
|
+
}
|
|
889
|
+
const retry = [];
|
|
890
|
+
for (const key of pending) {
|
|
891
|
+
const code = outcomes.get(fullKey(key));
|
|
892
|
+
if (code === null)
|
|
893
|
+
summary.deletedKeys.push(key);
|
|
894
|
+
else if (code && RETRYABLE_DELETE_CODES.has(code) && attempt < 2)
|
|
895
|
+
retry.push(key);
|
|
896
|
+
else
|
|
897
|
+
summary.failures.push({ key, code: code ?? "UnknownError", retryable: !!code && RETRYABLE_DELETE_CODES.has(code), outcome: "failed" });
|
|
898
|
+
}
|
|
899
|
+
pending = retry;
|
|
900
|
+
if (pending.length > 0)
|
|
901
|
+
await new Promise((resolve2) => setTimeout(resolve2, 100 * 2 ** attempt));
|
|
790
902
|
}
|
|
791
|
-
}
|
|
792
|
-
return
|
|
903
|
+
}
|
|
904
|
+
return toStorageDeleteResult(summary);
|
|
793
905
|
} catch (error) {
|
|
794
906
|
return err(toStorageError2(error));
|
|
795
907
|
}
|
|
@@ -882,6 +994,7 @@ function createS3Provider() {
|
|
|
882
994
|
const presign = {
|
|
883
995
|
async getUrl(key, options) {
|
|
884
996
|
try {
|
|
997
|
+
const parsed = PresignOptionsSchema.parse(options ?? {});
|
|
885
998
|
const s3Client = getClient();
|
|
886
999
|
const s3Config = getConfig();
|
|
887
1000
|
const fullPath = fullKey(key);
|
|
@@ -889,11 +1002,11 @@ function createS3Provider() {
|
|
|
889
1002
|
const command = new GetObjectCommand({
|
|
890
1003
|
Bucket: s3Config.bucket,
|
|
891
1004
|
Key: fullPath,
|
|
892
|
-
ResponseContentType:
|
|
893
|
-
ResponseContentDisposition:
|
|
1005
|
+
ResponseContentType: parsed.responseContentType,
|
|
1006
|
+
ResponseContentDisposition: parsed.responseContentDisposition
|
|
894
1007
|
});
|
|
895
1008
|
const url = await getSignedUrl(s3Client, command, {
|
|
896
|
-
expiresIn:
|
|
1009
|
+
expiresIn: parsed.expiresIn
|
|
897
1010
|
});
|
|
898
1011
|
return ok(url);
|
|
899
1012
|
} catch (error) {
|
|
@@ -906,6 +1019,7 @@ function createS3Provider() {
|
|
|
906
1019
|
},
|
|
907
1020
|
async putUrl(key, options) {
|
|
908
1021
|
try {
|
|
1022
|
+
const parsed = PresignUploadOptionsSchema.parse(options ?? {});
|
|
909
1023
|
const s3Client = getClient();
|
|
910
1024
|
const s3Config = getConfig();
|
|
911
1025
|
const fullPath = fullKey(key);
|
|
@@ -913,10 +1027,10 @@ function createS3Provider() {
|
|
|
913
1027
|
const command = new PutObjectCommand({
|
|
914
1028
|
Bucket: s3Config.bucket,
|
|
915
1029
|
Key: fullPath,
|
|
916
|
-
ContentType:
|
|
1030
|
+
ContentType: parsed.contentType
|
|
917
1031
|
});
|
|
918
1032
|
const url = await getSignedUrl(s3Client, command, {
|
|
919
|
-
expiresIn:
|
|
1033
|
+
expiresIn: parsed.expiresIn
|
|
920
1034
|
});
|
|
921
1035
|
return ok(url);
|
|
922
1036
|
} catch (error) {
|
|
@@ -950,13 +1064,14 @@ function createS3Provider() {
|
|
|
950
1064
|
);
|
|
951
1065
|
}
|
|
952
1066
|
try {
|
|
1067
|
+
await closeClients();
|
|
953
1068
|
config = cfg;
|
|
954
1069
|
logger2.info("Connecting S3 provider", {
|
|
955
1070
|
bucket: cfg.bucket,
|
|
956
1071
|
region: cfg.region,
|
|
957
|
-
endpoint: cfg.endpoint ?
|
|
1072
|
+
endpoint: cfg.endpoint ? core.sanitize.sanitizeUrl(cfg.endpoint) : void 0
|
|
958
1073
|
});
|
|
959
|
-
|
|
1074
|
+
const clientConfig = {
|
|
960
1075
|
region: cfg.region,
|
|
961
1076
|
endpoint: cfg.endpoint,
|
|
962
1077
|
forcePathStyle: cfg.forcePathStyle,
|
|
@@ -967,30 +1082,31 @@ function createS3Provider() {
|
|
|
967
1082
|
accessKeyId: cfg.accessKeyId,
|
|
968
1083
|
secretAccessKey: cfg.secretAccessKey
|
|
969
1084
|
}
|
|
970
|
-
}
|
|
1085
|
+
};
|
|
1086
|
+
client = new S3Client(clientConfig);
|
|
1087
|
+
deleteClient = new S3Client({ ...clientConfig, maxAttempts: 1 });
|
|
971
1088
|
await client.send(new ListObjectsV2Command({
|
|
972
1089
|
Bucket: cfg.bucket,
|
|
973
1090
|
MaxKeys: 1
|
|
974
1091
|
}));
|
|
1092
|
+
config = cfg;
|
|
1093
|
+
ready = true;
|
|
975
1094
|
logger2.info("S3 provider connected", { bucket: cfg.bucket });
|
|
976
1095
|
return ok(void 0);
|
|
977
1096
|
} catch (error) {
|
|
978
1097
|
logger2.error("S3 provider connect failed", { error });
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
client.destroy();
|
|
987
|
-
client = null;
|
|
988
|
-
config = null;
|
|
989
|
-
logger2.info("S3 provider disconnected");
|
|
1098
|
+
const original = toStorageError2(error);
|
|
1099
|
+
try {
|
|
1100
|
+
await closeClients();
|
|
1101
|
+
return err(original);
|
|
1102
|
+
} catch (cleanupError) {
|
|
1103
|
+
return err({ ...original, ext: { ...original.ext, cleanupPending: true, cleanupError } });
|
|
1104
|
+
}
|
|
990
1105
|
}
|
|
991
1106
|
},
|
|
1107
|
+
close: closeClients,
|
|
992
1108
|
isConnected() {
|
|
993
|
-
return client !== null && config !== null;
|
|
1109
|
+
return ready && client !== null && deleteClient !== null && config !== null;
|
|
994
1110
|
}
|
|
995
1111
|
};
|
|
996
1112
|
}
|
|
@@ -1031,6 +1147,40 @@ var logger3 = core.logger.child({ module: "storage", scope: "main" });
|
|
|
1031
1147
|
var currentProvider = null;
|
|
1032
1148
|
var currentConfig = null;
|
|
1033
1149
|
var initInProgress = false;
|
|
1150
|
+
var pendingProvider = null;
|
|
1151
|
+
var initCompletion = null;
|
|
1152
|
+
var closingPromise = null;
|
|
1153
|
+
async function closeOwnedProvider() {
|
|
1154
|
+
if (currentProvider) {
|
|
1155
|
+
pendingProvider = currentProvider;
|
|
1156
|
+
currentProvider = null;
|
|
1157
|
+
}
|
|
1158
|
+
currentConfig = null;
|
|
1159
|
+
if (!pendingProvider)
|
|
1160
|
+
return ok(void 0);
|
|
1161
|
+
try {
|
|
1162
|
+
await pendingProvider.close();
|
|
1163
|
+
pendingProvider = null;
|
|
1164
|
+
return ok(void 0);
|
|
1165
|
+
} catch (error) {
|
|
1166
|
+
logger3.error("Provider cleanup failed; retaining resource for retry", { error });
|
|
1167
|
+
return err({ ...HaiStorageError.CONNECTION_FAILED, message: storageM("storage_operationFailed", {
|
|
1168
|
+
params: { error: error instanceof Error ? error.message : String(error) }
|
|
1169
|
+
}), cause: error, ext: { cleanupPending: true } });
|
|
1170
|
+
}
|
|
1171
|
+
}
|
|
1172
|
+
async function failInitialization(result) {
|
|
1173
|
+
const cleanup = await closeOwnedProvider();
|
|
1174
|
+
if (result.success)
|
|
1175
|
+
return result;
|
|
1176
|
+
if (cleanup.success) {
|
|
1177
|
+
const ext = { ...result.error.ext };
|
|
1178
|
+
delete ext.cleanupPending;
|
|
1179
|
+
delete ext.cleanupError;
|
|
1180
|
+
return err({ ...result.error, ext: Object.keys(ext).length > 0 ? ext : void 0 });
|
|
1181
|
+
}
|
|
1182
|
+
return err({ ...result.error, ext: { ...result.error.ext, cleanupPending: true, cleanupError: cleanup.error } });
|
|
1183
|
+
}
|
|
1034
1184
|
function createProvider(config) {
|
|
1035
1185
|
switch (config.type) {
|
|
1036
1186
|
case "s3":
|
|
@@ -1060,13 +1210,22 @@ var storage = {
|
|
|
1060
1210
|
* 初始化存储连接。
|
|
1061
1211
|
*
|
|
1062
1212
|
* 如果当前已有活跃连接,会先自动 close 再重新初始化。
|
|
1063
|
-
*
|
|
1213
|
+
* 配置通过 Zod Schema 校验;校验、连接或清理失败均返回 HaiResult,不发布失败候选。
|
|
1064
1214
|
*
|
|
1065
1215
|
* @param config 存储配置(S3 或本地),支持省略带默认值的字段。
|
|
1066
1216
|
* @returns 成功时返回 ok(undefined);失败时返回包含错误码和消息的 err。
|
|
1217
|
+
* @example
|
|
1218
|
+
* ```ts
|
|
1219
|
+
* import { storage } from '@h-ai/storage'
|
|
1220
|
+
*
|
|
1221
|
+
* const initialized = await storage.init({ type: 'local', root: './data/uploads' })
|
|
1222
|
+
* if (!initialized.success) throw new Error(initialized.error.message)
|
|
1223
|
+
* const closed = await storage.close()
|
|
1224
|
+
* if (!closed.success) throw new Error(closed.error.message)
|
|
1225
|
+
* ```
|
|
1067
1226
|
*/
|
|
1068
1227
|
async init(config) {
|
|
1069
|
-
if (initInProgress) {
|
|
1228
|
+
if (initInProgress || closingPromise) {
|
|
1070
1229
|
logger3.warn("Storage init already in progress, skipping concurrent call");
|
|
1071
1230
|
return err(
|
|
1072
1231
|
HaiStorageError.OPERATION_FAILED,
|
|
@@ -1074,14 +1233,14 @@ var storage = {
|
|
|
1074
1233
|
);
|
|
1075
1234
|
}
|
|
1076
1235
|
initInProgress = true;
|
|
1236
|
+
let resolveInit;
|
|
1237
|
+
initCompletion = new Promise((resolve2) => {
|
|
1238
|
+
resolveInit = resolve2;
|
|
1239
|
+
});
|
|
1077
1240
|
try {
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
if (!closeResult.success) {
|
|
1082
|
-
return closeResult;
|
|
1083
|
-
}
|
|
1084
|
-
}
|
|
1241
|
+
const cleanup = await closeOwnedProvider();
|
|
1242
|
+
if (!cleanup.success)
|
|
1243
|
+
return cleanup;
|
|
1085
1244
|
logger3.info("Initializing storage module");
|
|
1086
1245
|
const parseResult = StorageConfigSchema.safeParse(config);
|
|
1087
1246
|
if (!parseResult.success) {
|
|
@@ -1093,30 +1252,36 @@ var storage = {
|
|
|
1093
1252
|
);
|
|
1094
1253
|
}
|
|
1095
1254
|
const parsed = parseResult.data;
|
|
1255
|
+
if (core.module.requiresSharedState(process.env) && parsed.type === "local")
|
|
1256
|
+
return err(HaiStorageError.CONFIG_ERROR, storageM("storage_sharedBackendRequired"));
|
|
1096
1257
|
const provider = createProvider(parsed);
|
|
1258
|
+
pendingProvider = provider;
|
|
1097
1259
|
const connectResult = await provider.connect(parsed);
|
|
1098
1260
|
if (!connectResult.success) {
|
|
1099
1261
|
logger3.error("Storage module initialization failed", {
|
|
1100
1262
|
code: connectResult.error.code,
|
|
1101
1263
|
message: connectResult.error.message
|
|
1102
1264
|
});
|
|
1103
|
-
return connectResult;
|
|
1265
|
+
return await failInitialization(connectResult);
|
|
1104
1266
|
}
|
|
1267
|
+
pendingProvider = null;
|
|
1105
1268
|
currentProvider = provider;
|
|
1106
1269
|
currentConfig = parsed;
|
|
1107
1270
|
logger3.info("Storage module initialized");
|
|
1108
1271
|
return ok(void 0);
|
|
1109
1272
|
} catch (error) {
|
|
1110
1273
|
logger3.error("Storage module initialization failed", { error });
|
|
1111
|
-
return err(
|
|
1274
|
+
return await failInitialization(err(
|
|
1112
1275
|
HaiStorageError.CONNECTION_FAILED,
|
|
1113
1276
|
storageM("storage_operationFailed", {
|
|
1114
1277
|
params: { error: error instanceof Error ? error.message : String(error) }
|
|
1115
1278
|
}),
|
|
1116
1279
|
error
|
|
1117
|
-
);
|
|
1280
|
+
));
|
|
1118
1281
|
} finally {
|
|
1119
1282
|
initInProgress = false;
|
|
1283
|
+
initCompletion = null;
|
|
1284
|
+
resolveInit();
|
|
1120
1285
|
}
|
|
1121
1286
|
},
|
|
1122
1287
|
/** 文件操作接口。未初始化时所有方法返回 NOT_INITIALIZED 错误 */
|
|
@@ -1137,38 +1302,33 @@ var storage = {
|
|
|
1137
1302
|
},
|
|
1138
1303
|
/** 是否已完成初始化 */
|
|
1139
1304
|
get isInitialized() {
|
|
1140
|
-
return currentProvider !== null;
|
|
1305
|
+
return currentProvider !== null && currentProvider.isConnected();
|
|
1141
1306
|
},
|
|
1142
1307
|
/**
|
|
1143
1308
|
* 关闭存储连接并释放资源
|
|
1144
1309
|
*
|
|
1145
1310
|
* 关闭后 file/dir/presign 操作将返回 NOT_INITIALIZED 错误。
|
|
1146
|
-
*
|
|
1311
|
+
* 等待初始化并合并并发关闭;失败时保留资源供 close/init 重试。
|
|
1312
|
+
* @returns 所有客户端释放成功才返回成功。
|
|
1313
|
+
* @example
|
|
1314
|
+
* ```ts
|
|
1315
|
+
* import { storage } from '@h-ai/storage'
|
|
1316
|
+
*
|
|
1317
|
+
* const closed = await storage.close()
|
|
1318
|
+
* if (!closed.success) throw new Error(closed.error.message)
|
|
1319
|
+
* ```
|
|
1147
1320
|
*/
|
|
1148
1321
|
async close() {
|
|
1149
|
-
if (!
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
logger3.info("Storage module closed");
|
|
1158
|
-
return ok(void 0);
|
|
1159
|
-
} catch (error) {
|
|
1160
|
-
logger3.error("Storage module close failed", { error });
|
|
1161
|
-
return err(
|
|
1162
|
-
HaiStorageError.CONNECTION_FAILED,
|
|
1163
|
-
storageM("storage_operationFailed", {
|
|
1164
|
-
params: { error: error instanceof Error ? error.message : String(error) }
|
|
1165
|
-
}),
|
|
1166
|
-
error
|
|
1167
|
-
);
|
|
1168
|
-
} finally {
|
|
1169
|
-
currentProvider = null;
|
|
1170
|
-
currentConfig = null;
|
|
1322
|
+
if (!closingPromise) {
|
|
1323
|
+
closingPromise = (async () => {
|
|
1324
|
+
if (initCompletion)
|
|
1325
|
+
await initCompletion;
|
|
1326
|
+
return await closeOwnedProvider();
|
|
1327
|
+
})().finally(() => {
|
|
1328
|
+
closingPromise = null;
|
|
1329
|
+
});
|
|
1171
1330
|
}
|
|
1331
|
+
return await closingPromise;
|
|
1172
1332
|
}
|
|
1173
1333
|
};
|
|
1174
1334
|
|