@h-ai/storage 0.1.0-alpha.67 → 0.1.0-alpha.69
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +92 -16
- package/README.ja.md +92 -16
- package/README.md +132 -16
- package/dist/browser.d.ts +401 -93
- package/dist/browser.js +3 -2
- package/dist/{chunk-KXE6OO5T.js → chunk-B4Q2NMST.js} +10 -5
- package/dist/chunk-B4Q2NMST.js.map +1 -0
- package/dist/chunk-I4BSES63.js +357 -0
- package/dist/chunk-I4BSES63.js.map +1 -0
- package/dist/chunk-P65WYXJG.js +33 -0
- package/dist/chunk-P65WYXJG.js.map +1 -0
- package/dist/client/index.d.ts +57 -48
- package/dist/client/index.js +1 -1
- package/dist/node.d.ts +15 -11
- package/dist/node.js +318 -137
- package/dist/node.js.map +1 -1
- package/dist/schemas.d.ts +103 -0
- package/dist/schemas.js +3 -0
- package/dist/schemas.js.map +1 -0
- package/package.json +8 -3
- package/dist/chunk-25GYTRA7.js +0 -274
- package/dist/chunk-25GYTRA7.js.map +0 -1
- package/dist/chunk-KXE6OO5T.js.map +0 -1
package/dist/node.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
import { HaiStorageError, StorageConfigSchema } from './chunk-
|
|
2
|
-
export { HaiStorageError, LocalConfigSchema, PresignOptionsSchema, PresignUploadOptionsSchema, S3ConfigSchema, StorageConfigSchema, StorageOperationLogConfigSchema, StorageTypeSchema } from './chunk-
|
|
3
|
-
import { storageM, MIME_TYPES, MIME_TYPE_DEFAULT } from './chunk-
|
|
4
|
-
export { downloadAndSave, downloadWithPresignedUrl, formatFileSize, getFileExtension, getMimeType, uploadWithPresignedUrl } from './chunk-
|
|
1
|
+
import { HaiStorageError, StorageConfigSchema, PresignUploadOptionsSchema, PresignOptionsSchema } from './chunk-B4Q2NMST.js';
|
|
2
|
+
export { HaiStorageError, LocalConfigSchema, PresignOptionsSchema, PresignUploadOptionsSchema, S3ConfigSchema, StorageConfigSchema, StorageOperationLogConfigSchema, StorageTypeSchema } from './chunk-B4Q2NMST.js';
|
|
3
|
+
import { storageM, MIME_TYPES, MIME_TYPE_DEFAULT } from './chunk-I4BSES63.js';
|
|
4
|
+
export { downloadAndSave, downloadWithPresignedUrl, formatFileSize, getFileExtension, getMimeType, uploadWithPresignedUrl } from './chunk-I4BSES63.js';
|
|
5
|
+
export { STORAGE_DELETE_MAX_KEYS, STORAGE_KEY_MAX_BYTES, StorageDeleteKeysSchema, StorageKeySchema, StoragePrefixSchema, StoragePresignExpiresInSchema, getStorageKeyIssue } from './chunk-P65WYXJG.js';
|
|
6
|
+
import process from 'node:process';
|
|
5
7
|
import { core, err, ok } from '@h-ai/core';
|
|
6
8
|
import { Buffer } from 'node:buffer';
|
|
7
9
|
import * as crypto from 'node:crypto';
|
|
@@ -11,9 +13,21 @@ import * as fsp from 'node:fs/promises';
|
|
|
11
13
|
import * as path from 'node:path';
|
|
12
14
|
import { Readable } from 'node:stream';
|
|
13
15
|
import { pipeline } from 'node:stream/promises';
|
|
14
|
-
import { S3Client, ListObjectsV2Command, PutObjectCommand, GetObjectCommand, CopyObjectCommand, DeleteObjectsCommand, DeleteObjectCommand, HeadObjectCommand } from '@aws-sdk/client-s3';
|
|
16
|
+
import { S3Client, HeadBucketCommand, CreateBucketCommand, ListObjectsV2Command, PutObjectCommand, GetObjectCommand, CopyObjectCommand, DeleteObjectsCommand, DeleteObjectCommand, HeadObjectCommand } from '@aws-sdk/client-s3';
|
|
15
17
|
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
|
|
16
18
|
|
|
19
|
+
function toStorageDeleteResult(summary) {
|
|
20
|
+
if (summary.failures.length === 0)
|
|
21
|
+
return ok(summary);
|
|
22
|
+
return {
|
|
23
|
+
success: false,
|
|
24
|
+
error: {
|
|
25
|
+
...HaiStorageError.OPERATION_FAILED,
|
|
26
|
+
message: storageM("storage_deleteManyFailed", { params: { count: summary.failures.length } }),
|
|
27
|
+
ext: { deleteMany: summary }
|
|
28
|
+
}
|
|
29
|
+
};
|
|
30
|
+
}
|
|
17
31
|
async function* toByteStream(data, signal) {
|
|
18
32
|
signal?.throwIfAborted();
|
|
19
33
|
if (typeof data === "string" || data instanceof Uint8Array) {
|
|
@@ -366,21 +380,22 @@ function createLocalProvider() {
|
|
|
366
380
|
},
|
|
367
381
|
async deleteMany(keys) {
|
|
368
382
|
logOperation("write", "file.deleteMany", { count: keys.length, keys });
|
|
369
|
-
const
|
|
370
|
-
const
|
|
371
|
-
for (
|
|
372
|
-
|
|
373
|
-
|
|
383
|
+
const summary = { deletedKeys: [], failures: [] };
|
|
384
|
+
const uniqueKeys = [...new Set(keys)];
|
|
385
|
+
for (let offset = 0; offset < uniqueKeys.length; offset += 32) {
|
|
386
|
+
const batch = uniqueKeys.slice(offset, offset + 32);
|
|
387
|
+
const results = await Promise.allSettled(batch.map((key) => file.delete(key)));
|
|
388
|
+
for (const [index, result] of results.entries()) {
|
|
389
|
+
const key = batch[index];
|
|
390
|
+
if (result.status === "rejected")
|
|
391
|
+
summary.failures.push({ key, code: "UnexpectedDeleteFailure", retryable: false, outcome: "failed" });
|
|
392
|
+
else if (!result.value.success)
|
|
393
|
+
summary.failures.push({ key, code: String(result.value.error.code), retryable: false, outcome: "failed" });
|
|
394
|
+
else
|
|
395
|
+
summary.deletedKeys.push(key);
|
|
374
396
|
}
|
|
375
397
|
}
|
|
376
|
-
|
|
377
|
-
return err(
|
|
378
|
-
HaiStorageError.OPERATION_FAILED,
|
|
379
|
-
storageM("storage_deleteManyFailed", { params: { count: errors.length } }),
|
|
380
|
-
errors
|
|
381
|
-
);
|
|
382
|
-
}
|
|
383
|
-
return ok(void 0);
|
|
398
|
+
return toStorageDeleteResult(summary);
|
|
384
399
|
},
|
|
385
400
|
async copy(sourceKey, destKey, options = {}) {
|
|
386
401
|
if (isMetaFile(sourceKey) || isMetaFile(destKey)) {
|
|
@@ -423,49 +438,85 @@ function createLocalProvider() {
|
|
|
423
438
|
async list(options = {}) {
|
|
424
439
|
try {
|
|
425
440
|
const cfg = getConfig();
|
|
426
|
-
const prefix = options.prefix
|
|
427
|
-
const delimiter = options.delimiter
|
|
428
|
-
const maxKeys = options.maxKeys
|
|
441
|
+
const prefix = options.prefix === void 0 ? "" : options.prefix;
|
|
442
|
+
const delimiter = options.delimiter === void 0 ? "" : options.delimiter;
|
|
443
|
+
const maxKeys = options.maxKeys === void 0 ? 1e3 : options.maxKeys;
|
|
444
|
+
if (typeof prefix !== "string" || typeof delimiter !== "string" || !Number.isInteger(maxKeys) || maxKeys < 1 || maxKeys > 1e3 || Buffer.from(prefix).toString("utf8") !== prefix || Buffer.from(delimiter).toString("utf8") !== delimiter) {
|
|
445
|
+
return err(HaiStorageError.INVALID_LIST_OPTIONS, storageM("storage_listOptionsInvalid"));
|
|
446
|
+
}
|
|
429
447
|
logOperation("read", "dir.list", { options });
|
|
430
|
-
|
|
431
|
-
const
|
|
432
|
-
const
|
|
433
|
-
|
|
434
|
-
|
|
448
|
+
safePath(cfg.root, prefix);
|
|
449
|
+
const root = await fsp.realpath(cfg.root);
|
|
450
|
+
const scope = crypto.createHash("sha256").update(JSON.stringify([root, prefix, delimiter])).digest("hex");
|
|
451
|
+
let after = "";
|
|
452
|
+
if (options.continuationToken !== void 0) {
|
|
453
|
+
const token = options.continuationToken;
|
|
435
454
|
try {
|
|
436
|
-
|
|
455
|
+
if (typeof token !== "string" || token.length === 0 || token.length > 262144)
|
|
456
|
+
throw new Error("Invalid cursor encoding");
|
|
457
|
+
const bytes = Buffer.from(token, "base64url");
|
|
458
|
+
if (bytes.toString("base64url") !== token)
|
|
459
|
+
throw new Error("Invalid cursor encoding");
|
|
460
|
+
const decoded = JSON.parse(bytes.toString("utf8"));
|
|
461
|
+
if (!Array.isArray(decoded) || decoded.length !== 3 || decoded[0] !== 1 || decoded[1] !== scope || typeof decoded[2] !== "string" || decoded[2].length === 0 || !decoded[2].startsWith(prefix)) {
|
|
462
|
+
throw new Error("Invalid cursor scope");
|
|
463
|
+
}
|
|
464
|
+
after = decoded[2];
|
|
437
465
|
} catch {
|
|
438
|
-
return;
|
|
466
|
+
return err(HaiStorageError.INVALID_LIST_OPTIONS, storageM("storage_listCursorInvalid"));
|
|
439
467
|
}
|
|
468
|
+
}
|
|
469
|
+
const files = [];
|
|
470
|
+
const commonPrefixes = [];
|
|
471
|
+
const compare = (left, right) => Buffer.compare(Buffer.from(left), Buffer.from(right));
|
|
472
|
+
async function* readDir(dirPath, directoryKey) {
|
|
473
|
+
const entries = await fsp.readdir(dirPath, { withFileTypes: true });
|
|
474
|
+
entries.sort((left, right) => compare(left.name + (left.isDirectory() ? "/" : ""), right.name + (right.isDirectory() ? "/" : "")));
|
|
440
475
|
for (const entry of entries) {
|
|
441
|
-
if (files.length >= maxKeys)
|
|
442
|
-
return;
|
|
443
|
-
const fullEntryPath = path.join(dirPath, entry.name);
|
|
444
|
-
const key = path.relative(cfg.root, fullEntryPath).replace(/\\/g, "/");
|
|
445
476
|
if (isMetaFile(entry.name))
|
|
446
477
|
continue;
|
|
478
|
+
const key = directoryKey + entry.name;
|
|
479
|
+
const filePath = path.join(dirPath, entry.name);
|
|
447
480
|
if (entry.isDirectory()) {
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
await readDir(fullEntryPath);
|
|
481
|
+
const childPrefix = `${key}/`;
|
|
482
|
+
if (prefix.startsWith(childPrefix) || childPrefix.startsWith(prefix)) {
|
|
483
|
+
yield* readDir(filePath, childPrefix);
|
|
452
484
|
}
|
|
453
|
-
} else if (entry.isFile()) {
|
|
454
|
-
|
|
455
|
-
files.push({
|
|
456
|
-
key,
|
|
457
|
-
size: stat2.size,
|
|
458
|
-
contentType: getMimeType2(key),
|
|
459
|
-
lastModified: stat2.mtime
|
|
460
|
-
});
|
|
485
|
+
} else if (entry.isFile() && key.startsWith(prefix)) {
|
|
486
|
+
yield { key, filePath };
|
|
461
487
|
}
|
|
462
488
|
}
|
|
463
489
|
}
|
|
464
|
-
|
|
490
|
+
let previous = "";
|
|
491
|
+
let last = "";
|
|
492
|
+
let count = 0;
|
|
493
|
+
let isTruncated = false;
|
|
494
|
+
for await (const entry of readDir(root, "")) {
|
|
495
|
+
const index = delimiter ? entry.key.indexOf(delimiter, prefix.length) : -1;
|
|
496
|
+
const logicalKey = index < 0 ? entry.key : entry.key.slice(0, index + delimiter.length);
|
|
497
|
+
if (logicalKey === previous)
|
|
498
|
+
continue;
|
|
499
|
+
previous = logicalKey;
|
|
500
|
+
if (compare(logicalKey, after) <= 0)
|
|
501
|
+
continue;
|
|
502
|
+
if (count === maxKeys) {
|
|
503
|
+
isTruncated = true;
|
|
504
|
+
break;
|
|
505
|
+
}
|
|
506
|
+
if (index >= 0) {
|
|
507
|
+
commonPrefixes.push(logicalKey);
|
|
508
|
+
} else {
|
|
509
|
+
const stat2 = await fsp.stat(entry.filePath);
|
|
510
|
+
files.push({ key: entry.key, size: stat2.size, contentType: getMimeType2(entry.key), lastModified: stat2.mtime });
|
|
511
|
+
}
|
|
512
|
+
count++;
|
|
513
|
+
last = logicalKey;
|
|
514
|
+
}
|
|
465
515
|
return ok({
|
|
466
516
|
files,
|
|
467
|
-
commonPrefixes
|
|
468
|
-
isTruncated
|
|
517
|
+
commonPrefixes,
|
|
518
|
+
isTruncated,
|
|
519
|
+
...isTruncated ? { nextContinuationToken: Buffer.from(JSON.stringify([1, scope, last])).toString("base64url") } : {}
|
|
469
520
|
});
|
|
470
521
|
} catch (error) {
|
|
471
522
|
return err(toStorageError(error));
|
|
@@ -488,17 +539,11 @@ function createLocalProvider() {
|
|
|
488
539
|
const presign = {
|
|
489
540
|
async getUrl(key, options) {
|
|
490
541
|
logOperation("read", "presign.getUrl", { key, options });
|
|
491
|
-
|
|
492
|
-
const expires = Math.floor(Date.now() / 1e3) + expiresIn;
|
|
493
|
-
const signature = crypto.createHash("sha256").update(`${key}:${expires}`).digest("hex").slice(0, 16);
|
|
494
|
-
return ok(`local://${key}?expires=${expires}&signature=${signature}`);
|
|
542
|
+
return err(HaiStorageError.UNSUPPORTED_OPERATION, storageM("storage_localPresignUnsupported"));
|
|
495
543
|
},
|
|
496
544
|
async putUrl(key, options) {
|
|
497
545
|
logOperation("write", "presign.putUrl", { key, options });
|
|
498
|
-
|
|
499
|
-
const expires = Math.floor(Date.now() / 1e3) + expiresIn;
|
|
500
|
-
const signature = crypto.createHash("sha256").update(`put:${key}:${expires}`).digest("hex").slice(0, 16);
|
|
501
|
-
return ok(`local://${key}?action=put&expires=${expires}&signature=${signature}`);
|
|
546
|
+
return err(HaiStorageError.UNSUPPORTED_OPERATION, storageM("storage_localPresignUnsupported"));
|
|
502
547
|
},
|
|
503
548
|
publicUrl(key) {
|
|
504
549
|
logOperation("read", "presign.publicUrl", { key });
|
|
@@ -543,18 +588,8 @@ function createLocalProvider() {
|
|
|
543
588
|
var logger2 = core.logger.child({ module: "storage", scope: "provider-s3" });
|
|
544
589
|
var TRAILING_SLASHES_REGEX = /\/+$/;
|
|
545
590
|
var LEADING_SLASHES_REGEX = /^\/+/;
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
const parsed = new URL(url);
|
|
549
|
-
if (parsed.username)
|
|
550
|
-
parsed.username = "***";
|
|
551
|
-
if (parsed.password)
|
|
552
|
-
parsed.password = "***";
|
|
553
|
-
return parsed.toString();
|
|
554
|
-
} catch {
|
|
555
|
-
return "(invalid url)";
|
|
556
|
-
}
|
|
557
|
-
}
|
|
591
|
+
var DELETE_BATCH_SIZE = 1e3;
|
|
592
|
+
var RETRYABLE_DELETE_CODES = /* @__PURE__ */ new Set(["InternalError", "ServiceUnavailable", "SlowDown", "RequestTimeout"]);
|
|
558
593
|
function toStorageError2(error, key) {
|
|
559
594
|
const e = error;
|
|
560
595
|
if (e.name === "PreconditionFailed" || e.$metadata?.httpStatusCode === 412)
|
|
@@ -609,15 +644,46 @@ function withoutPrefix(key, prefix) {
|
|
|
609
644
|
}
|
|
610
645
|
function createS3Provider() {
|
|
611
646
|
let client = null;
|
|
647
|
+
let deleteClient = null;
|
|
612
648
|
let config = null;
|
|
613
|
-
|
|
614
|
-
|
|
649
|
+
let ready = false;
|
|
650
|
+
let closing = null;
|
|
651
|
+
function closeClients() {
|
|
652
|
+
ready = false;
|
|
653
|
+
config = null;
|
|
654
|
+
if (closing)
|
|
655
|
+
return closing;
|
|
656
|
+
closing = Promise.resolve().then(() => {
|
|
657
|
+
const failures = [];
|
|
658
|
+
for (const kind of ["normal", "delete"]) {
|
|
659
|
+
const owned = kind === "normal" ? client : deleteClient;
|
|
660
|
+
if (!owned)
|
|
661
|
+
continue;
|
|
662
|
+
try {
|
|
663
|
+
owned.destroy();
|
|
664
|
+
if (kind === "normal")
|
|
665
|
+
client = null;
|
|
666
|
+
else deleteClient = null;
|
|
667
|
+
} catch (error) {
|
|
668
|
+
failures.push(error);
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
if (failures.length > 0)
|
|
672
|
+
throw new AggregateError(failures, "S3 client cleanup failed");
|
|
673
|
+
}).finally(() => {
|
|
674
|
+
closing = null;
|
|
675
|
+
});
|
|
676
|
+
return closing;
|
|
677
|
+
}
|
|
678
|
+
function getClient(operation) {
|
|
679
|
+
const selected = operation === "deleteMany" ? deleteClient : client;
|
|
680
|
+
if (!ready || !selected) {
|
|
615
681
|
throw new Error(storageM("storage_s3ClientNotInitialized"));
|
|
616
682
|
}
|
|
617
|
-
return
|
|
683
|
+
return selected;
|
|
618
684
|
}
|
|
619
685
|
function getConfig() {
|
|
620
|
-
if (!config) {
|
|
686
|
+
if (!ready || !config) {
|
|
621
687
|
throw new Error(storageM("storage_s3ConfigNotInitialized"));
|
|
622
688
|
}
|
|
623
689
|
return config;
|
|
@@ -777,19 +843,65 @@ function createS3Provider() {
|
|
|
777
843
|
},
|
|
778
844
|
async deleteMany(keys) {
|
|
779
845
|
try {
|
|
780
|
-
const s3Client = getClient();
|
|
846
|
+
const s3Client = getClient("deleteMany");
|
|
781
847
|
const s3Config = getConfig();
|
|
782
|
-
if (keys.length === 0) {
|
|
783
|
-
return ok(void 0);
|
|
784
|
-
}
|
|
785
848
|
logOperation("write", "file.deleteMany", { count: keys.length, keys });
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
849
|
+
const uniqueKeys = [...new Set(keys)];
|
|
850
|
+
const summary = { deletedKeys: [], failures: [] };
|
|
851
|
+
for (let offset = 0; offset < uniqueKeys.length; offset += DELETE_BATCH_SIZE) {
|
|
852
|
+
let pending = uniqueKeys.slice(offset, offset + DELETE_BATCH_SIZE);
|
|
853
|
+
for (let attempt = 0; pending.length > 0 && attempt < 3; attempt++) {
|
|
854
|
+
let response;
|
|
855
|
+
try {
|
|
856
|
+
response = await s3Client.send(new DeleteObjectsCommand({
|
|
857
|
+
Bucket: s3Config.bucket,
|
|
858
|
+
Delete: { Objects: [...new Set(pending.map(fullKey))].map((Key) => ({ Key })), Quiet: false }
|
|
859
|
+
}));
|
|
860
|
+
} catch (cause) {
|
|
861
|
+
const failure = cause && typeof cause === "object" ? cause : {};
|
|
862
|
+
const status = failure.$metadata?.httpStatusCode;
|
|
863
|
+
const code = failure.name && /^[\w.-]{1,80}$/.test(failure.name) ? failure.name : "RequestFailed";
|
|
864
|
+
const retryable = status === void 0 || status === 408 || status === 429 || status >= 500;
|
|
865
|
+
const outcome = status !== void 0 && status >= 400 && status < 500 && status !== 408 ? "failed" : "unknown";
|
|
866
|
+
summary.failures.push(...pending.map((key) => ({ key, code, retryable, outcome })));
|
|
867
|
+
summary.failures.push(...uniqueKeys.slice(offset + DELETE_BATCH_SIZE).map((key) => ({ key, code, retryable, outcome: "notAttempted" })));
|
|
868
|
+
return toStorageDeleteResult(summary);
|
|
869
|
+
}
|
|
870
|
+
const outcomes = /* @__PURE__ */ new Map();
|
|
871
|
+
const requested = new Set(pending.map(fullKey));
|
|
872
|
+
let malformed = false;
|
|
873
|
+
const items = [
|
|
874
|
+
...(response.Deleted ?? []).map((item) => ({ key: item.Key, code: null })),
|
|
875
|
+
...(response.Errors ?? []).map((item) => ({ key: item.Key, code: item.Code && /^[\w.-]{1,80}$/.test(item.Code) ? item.Code : "UnknownError" }))
|
|
876
|
+
];
|
|
877
|
+
for (const item of items) {
|
|
878
|
+
if (!item.key || !requested.has(item.key) || outcomes.has(item.key)) {
|
|
879
|
+
malformed = true;
|
|
880
|
+
break;
|
|
881
|
+
}
|
|
882
|
+
outcomes.set(item.key, item.code);
|
|
883
|
+
}
|
|
884
|
+
if (malformed || outcomes.size !== requested.size) {
|
|
885
|
+
summary.failures.push(...pending.map((key) => ({ key, code: "InvalidDeleteResponse", retryable: false, outcome: "unknown" })));
|
|
886
|
+
summary.failures.push(...uniqueKeys.slice(offset + DELETE_BATCH_SIZE).map((key) => ({ key, code: "InvalidDeleteResponse", retryable: false, outcome: "notAttempted" })));
|
|
887
|
+
return toStorageDeleteResult(summary);
|
|
888
|
+
}
|
|
889
|
+
const retry = [];
|
|
890
|
+
for (const key of pending) {
|
|
891
|
+
const code = outcomes.get(fullKey(key));
|
|
892
|
+
if (code === null)
|
|
893
|
+
summary.deletedKeys.push(key);
|
|
894
|
+
else if (code && RETRYABLE_DELETE_CODES.has(code) && attempt < 2)
|
|
895
|
+
retry.push(key);
|
|
896
|
+
else
|
|
897
|
+
summary.failures.push({ key, code: code ?? "UnknownError", retryable: !!code && RETRYABLE_DELETE_CODES.has(code), outcome: "failed" });
|
|
898
|
+
}
|
|
899
|
+
pending = retry;
|
|
900
|
+
if (pending.length > 0)
|
|
901
|
+
await new Promise((resolve2) => setTimeout(resolve2, 100 * 2 ** attempt));
|
|
790
902
|
}
|
|
791
|
-
}
|
|
792
|
-
return
|
|
903
|
+
}
|
|
904
|
+
return toStorageDeleteResult(summary);
|
|
793
905
|
} catch (error) {
|
|
794
906
|
return err(toStorageError2(error));
|
|
795
907
|
}
|
|
@@ -882,6 +994,7 @@ function createS3Provider() {
|
|
|
882
994
|
const presign = {
|
|
883
995
|
async getUrl(key, options) {
|
|
884
996
|
try {
|
|
997
|
+
const parsed = PresignOptionsSchema.parse(options ?? {});
|
|
885
998
|
const s3Client = getClient();
|
|
886
999
|
const s3Config = getConfig();
|
|
887
1000
|
const fullPath = fullKey(key);
|
|
@@ -889,11 +1002,11 @@ function createS3Provider() {
|
|
|
889
1002
|
const command = new GetObjectCommand({
|
|
890
1003
|
Bucket: s3Config.bucket,
|
|
891
1004
|
Key: fullPath,
|
|
892
|
-
ResponseContentType:
|
|
893
|
-
ResponseContentDisposition:
|
|
1005
|
+
ResponseContentType: parsed.responseContentType,
|
|
1006
|
+
ResponseContentDisposition: parsed.responseContentDisposition
|
|
894
1007
|
});
|
|
895
1008
|
const url = await getSignedUrl(s3Client, command, {
|
|
896
|
-
expiresIn:
|
|
1009
|
+
expiresIn: parsed.expiresIn
|
|
897
1010
|
});
|
|
898
1011
|
return ok(url);
|
|
899
1012
|
} catch (error) {
|
|
@@ -906,6 +1019,7 @@ function createS3Provider() {
|
|
|
906
1019
|
},
|
|
907
1020
|
async putUrl(key, options) {
|
|
908
1021
|
try {
|
|
1022
|
+
const parsed = PresignUploadOptionsSchema.parse(options ?? {});
|
|
909
1023
|
const s3Client = getClient();
|
|
910
1024
|
const s3Config = getConfig();
|
|
911
1025
|
const fullPath = fullKey(key);
|
|
@@ -913,10 +1027,10 @@ function createS3Provider() {
|
|
|
913
1027
|
const command = new PutObjectCommand({
|
|
914
1028
|
Bucket: s3Config.bucket,
|
|
915
1029
|
Key: fullPath,
|
|
916
|
-
ContentType:
|
|
1030
|
+
ContentType: parsed.contentType
|
|
917
1031
|
});
|
|
918
1032
|
const url = await getSignedUrl(s3Client, command, {
|
|
919
|
-
expiresIn:
|
|
1033
|
+
expiresIn: parsed.expiresIn
|
|
920
1034
|
});
|
|
921
1035
|
return ok(url);
|
|
922
1036
|
} catch (error) {
|
|
@@ -950,13 +1064,14 @@ function createS3Provider() {
|
|
|
950
1064
|
);
|
|
951
1065
|
}
|
|
952
1066
|
try {
|
|
1067
|
+
await closeClients();
|
|
953
1068
|
config = cfg;
|
|
954
1069
|
logger2.info("Connecting S3 provider", {
|
|
955
1070
|
bucket: cfg.bucket,
|
|
956
1071
|
region: cfg.region,
|
|
957
|
-
endpoint: cfg.endpoint ?
|
|
1072
|
+
endpoint: cfg.endpoint ? core.sanitize.sanitizeUrl(cfg.endpoint) : void 0
|
|
958
1073
|
});
|
|
959
|
-
|
|
1074
|
+
const clientConfig = {
|
|
960
1075
|
region: cfg.region,
|
|
961
1076
|
endpoint: cfg.endpoint,
|
|
962
1077
|
forcePathStyle: cfg.forcePathStyle,
|
|
@@ -967,30 +1082,52 @@ function createS3Provider() {
|
|
|
967
1082
|
accessKeyId: cfg.accessKeyId,
|
|
968
1083
|
secretAccessKey: cfg.secretAccessKey
|
|
969
1084
|
}
|
|
970
|
-
}
|
|
1085
|
+
};
|
|
1086
|
+
client = new S3Client(clientConfig);
|
|
1087
|
+
deleteClient = new S3Client({ ...clientConfig, maxAttempts: 1 });
|
|
1088
|
+
if (cfg.createBucket) {
|
|
1089
|
+
try {
|
|
1090
|
+
await client.send(new HeadBucketCommand({ Bucket: cfg.bucket }));
|
|
1091
|
+
} catch (error) {
|
|
1092
|
+
if (!(error && typeof error === "object" && "$metadata" in error && error.$metadata && typeof error.$metadata === "object" && "httpStatusCode" in error.$metadata && error.$metadata.httpStatusCode === 404)) {
|
|
1093
|
+
throw error;
|
|
1094
|
+
}
|
|
1095
|
+
try {
|
|
1096
|
+
await client.send(new CreateBucketCommand({
|
|
1097
|
+
Bucket: cfg.bucket,
|
|
1098
|
+
// AWS us-east-1 要省略 LocationConstraint;兼容服务可使用自定义区域名。
|
|
1099
|
+
...cfg.region === "us-east-1" ? {} : { CreateBucketConfiguration: { LocationConstraint: cfg.region } }
|
|
1100
|
+
}));
|
|
1101
|
+
logger2.info("S3 bucket created", { bucket: cfg.bucket });
|
|
1102
|
+
} catch (createError) {
|
|
1103
|
+
if (!(createError instanceof Error && createError.name === "BucketAlreadyOwnedByYou"))
|
|
1104
|
+
throw createError;
|
|
1105
|
+
await client.send(new HeadBucketCommand({ Bucket: cfg.bucket }));
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
}
|
|
971
1109
|
await client.send(new ListObjectsV2Command({
|
|
972
1110
|
Bucket: cfg.bucket,
|
|
973
1111
|
MaxKeys: 1
|
|
974
1112
|
}));
|
|
1113
|
+
config = cfg;
|
|
1114
|
+
ready = true;
|
|
975
1115
|
logger2.info("S3 provider connected", { bucket: cfg.bucket });
|
|
976
1116
|
return ok(void 0);
|
|
977
1117
|
} catch (error) {
|
|
978
1118
|
logger2.error("S3 provider connect failed", { error });
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
client.destroy();
|
|
987
|
-
client = null;
|
|
988
|
-
config = null;
|
|
989
|
-
logger2.info("S3 provider disconnected");
|
|
1119
|
+
const original = toStorageError2(error);
|
|
1120
|
+
try {
|
|
1121
|
+
await closeClients();
|
|
1122
|
+
return err(original);
|
|
1123
|
+
} catch (cleanupError) {
|
|
1124
|
+
return err({ ...original, ext: { ...original.ext, cleanupPending: true, cleanupError } });
|
|
1125
|
+
}
|
|
990
1126
|
}
|
|
991
1127
|
},
|
|
1128
|
+
close: closeClients,
|
|
992
1129
|
isConnected() {
|
|
993
|
-
return client !== null && config !== null;
|
|
1130
|
+
return ready && client !== null && deleteClient !== null && config !== null;
|
|
994
1131
|
}
|
|
995
1132
|
};
|
|
996
1133
|
}
|
|
@@ -1031,6 +1168,40 @@ var logger3 = core.logger.child({ module: "storage", scope: "main" });
|
|
|
1031
1168
|
var currentProvider = null;
|
|
1032
1169
|
var currentConfig = null;
|
|
1033
1170
|
var initInProgress = false;
|
|
1171
|
+
var pendingProvider = null;
|
|
1172
|
+
var initCompletion = null;
|
|
1173
|
+
var closingPromise = null;
|
|
1174
|
+
async function closeOwnedProvider() {
|
|
1175
|
+
if (currentProvider) {
|
|
1176
|
+
pendingProvider = currentProvider;
|
|
1177
|
+
currentProvider = null;
|
|
1178
|
+
}
|
|
1179
|
+
currentConfig = null;
|
|
1180
|
+
if (!pendingProvider)
|
|
1181
|
+
return ok(void 0);
|
|
1182
|
+
try {
|
|
1183
|
+
await pendingProvider.close();
|
|
1184
|
+
pendingProvider = null;
|
|
1185
|
+
return ok(void 0);
|
|
1186
|
+
} catch (error) {
|
|
1187
|
+
logger3.error("Provider cleanup failed; retaining resource for retry", { error });
|
|
1188
|
+
return err({ ...HaiStorageError.CONNECTION_FAILED, message: storageM("storage_operationFailed", {
|
|
1189
|
+
params: { error: error instanceof Error ? error.message : String(error) }
|
|
1190
|
+
}), cause: error, ext: { cleanupPending: true } });
|
|
1191
|
+
}
|
|
1192
|
+
}
|
|
1193
|
+
async function failInitialization(result) {
|
|
1194
|
+
const cleanup = await closeOwnedProvider();
|
|
1195
|
+
if (result.success)
|
|
1196
|
+
return result;
|
|
1197
|
+
if (cleanup.success) {
|
|
1198
|
+
const ext = { ...result.error.ext };
|
|
1199
|
+
delete ext.cleanupPending;
|
|
1200
|
+
delete ext.cleanupError;
|
|
1201
|
+
return err({ ...result.error, ext: Object.keys(ext).length > 0 ? ext : void 0 });
|
|
1202
|
+
}
|
|
1203
|
+
return err({ ...result.error, ext: { ...result.error.ext, cleanupPending: true, cleanupError: cleanup.error } });
|
|
1204
|
+
}
|
|
1034
1205
|
function createProvider(config) {
|
|
1035
1206
|
switch (config.type) {
|
|
1036
1207
|
case "s3":
|
|
@@ -1060,13 +1231,22 @@ var storage = {
|
|
|
1060
1231
|
* 初始化存储连接。
|
|
1061
1232
|
*
|
|
1062
1233
|
* 如果当前已有活跃连接,会先自动 close 再重新初始化。
|
|
1063
|
-
*
|
|
1234
|
+
* 配置通过 Zod Schema 校验;校验、连接或清理失败均返回 HaiResult,不发布失败候选。
|
|
1064
1235
|
*
|
|
1065
1236
|
* @param config 存储配置(S3 或本地),支持省略带默认值的字段。
|
|
1066
1237
|
* @returns 成功时返回 ok(undefined);失败时返回包含错误码和消息的 err。
|
|
1238
|
+
* @example
|
|
1239
|
+
* ```ts
|
|
1240
|
+
* import { storage } from '@h-ai/storage'
|
|
1241
|
+
*
|
|
1242
|
+
* const initialized = await storage.init({ type: 'local', root: './data/uploads' })
|
|
1243
|
+
* if (!initialized.success) throw new Error(initialized.error.message)
|
|
1244
|
+
* const closed = await storage.close()
|
|
1245
|
+
* if (!closed.success) throw new Error(closed.error.message)
|
|
1246
|
+
* ```
|
|
1067
1247
|
*/
|
|
1068
1248
|
async init(config) {
|
|
1069
|
-
if (initInProgress) {
|
|
1249
|
+
if (initInProgress || closingPromise) {
|
|
1070
1250
|
logger3.warn("Storage init already in progress, skipping concurrent call");
|
|
1071
1251
|
return err(
|
|
1072
1252
|
HaiStorageError.OPERATION_FAILED,
|
|
@@ -1074,14 +1254,14 @@ var storage = {
|
|
|
1074
1254
|
);
|
|
1075
1255
|
}
|
|
1076
1256
|
initInProgress = true;
|
|
1257
|
+
let resolveInit;
|
|
1258
|
+
initCompletion = new Promise((resolve2) => {
|
|
1259
|
+
resolveInit = resolve2;
|
|
1260
|
+
});
|
|
1077
1261
|
try {
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
if (!closeResult.success) {
|
|
1082
|
-
return closeResult;
|
|
1083
|
-
}
|
|
1084
|
-
}
|
|
1262
|
+
const cleanup = await closeOwnedProvider();
|
|
1263
|
+
if (!cleanup.success)
|
|
1264
|
+
return cleanup;
|
|
1085
1265
|
logger3.info("Initializing storage module");
|
|
1086
1266
|
const parseResult = StorageConfigSchema.safeParse(config);
|
|
1087
1267
|
if (!parseResult.success) {
|
|
@@ -1093,30 +1273,36 @@ var storage = {
|
|
|
1093
1273
|
);
|
|
1094
1274
|
}
|
|
1095
1275
|
const parsed = parseResult.data;
|
|
1276
|
+
if (core.module.requiresSharedState(process.env) && parsed.type === "local")
|
|
1277
|
+
return err(HaiStorageError.CONFIG_ERROR, storageM("storage_sharedBackendRequired"));
|
|
1096
1278
|
const provider = createProvider(parsed);
|
|
1279
|
+
pendingProvider = provider;
|
|
1097
1280
|
const connectResult = await provider.connect(parsed);
|
|
1098
1281
|
if (!connectResult.success) {
|
|
1099
1282
|
logger3.error("Storage module initialization failed", {
|
|
1100
1283
|
code: connectResult.error.code,
|
|
1101
1284
|
message: connectResult.error.message
|
|
1102
1285
|
});
|
|
1103
|
-
return connectResult;
|
|
1286
|
+
return await failInitialization(connectResult);
|
|
1104
1287
|
}
|
|
1288
|
+
pendingProvider = null;
|
|
1105
1289
|
currentProvider = provider;
|
|
1106
1290
|
currentConfig = parsed;
|
|
1107
1291
|
logger3.info("Storage module initialized");
|
|
1108
1292
|
return ok(void 0);
|
|
1109
1293
|
} catch (error) {
|
|
1110
1294
|
logger3.error("Storage module initialization failed", { error });
|
|
1111
|
-
return err(
|
|
1295
|
+
return await failInitialization(err(
|
|
1112
1296
|
HaiStorageError.CONNECTION_FAILED,
|
|
1113
1297
|
storageM("storage_operationFailed", {
|
|
1114
1298
|
params: { error: error instanceof Error ? error.message : String(error) }
|
|
1115
1299
|
}),
|
|
1116
1300
|
error
|
|
1117
|
-
);
|
|
1301
|
+
));
|
|
1118
1302
|
} finally {
|
|
1119
1303
|
initInProgress = false;
|
|
1304
|
+
initCompletion = null;
|
|
1305
|
+
resolveInit();
|
|
1120
1306
|
}
|
|
1121
1307
|
},
|
|
1122
1308
|
/** 文件操作接口。未初始化时所有方法返回 NOT_INITIALIZED 错误 */
|
|
@@ -1137,38 +1323,33 @@ var storage = {
|
|
|
1137
1323
|
},
|
|
1138
1324
|
/** 是否已完成初始化 */
|
|
1139
1325
|
get isInitialized() {
|
|
1140
|
-
return currentProvider !== null;
|
|
1326
|
+
return currentProvider !== null && currentProvider.isConnected();
|
|
1141
1327
|
},
|
|
1142
1328
|
/**
|
|
1143
1329
|
* 关闭存储连接并释放资源
|
|
1144
1330
|
*
|
|
1145
1331
|
* 关闭后 file/dir/presign 操作将返回 NOT_INITIALIZED 错误。
|
|
1146
|
-
*
|
|
1332
|
+
* 等待初始化并合并并发关闭;失败时保留资源供 close/init 重试。
|
|
1333
|
+
* @returns 所有客户端释放成功才返回成功。
|
|
1334
|
+
* @example
|
|
1335
|
+
* ```ts
|
|
1336
|
+
* import { storage } from '@h-ai/storage'
|
|
1337
|
+
*
|
|
1338
|
+
* const closed = await storage.close()
|
|
1339
|
+
* if (!closed.success) throw new Error(closed.error.message)
|
|
1340
|
+
* ```
|
|
1147
1341
|
*/
|
|
1148
1342
|
async close() {
|
|
1149
|
-
if (!
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
logger3.info("Storage module closed");
|
|
1158
|
-
return ok(void 0);
|
|
1159
|
-
} catch (error) {
|
|
1160
|
-
logger3.error("Storage module close failed", { error });
|
|
1161
|
-
return err(
|
|
1162
|
-
HaiStorageError.CONNECTION_FAILED,
|
|
1163
|
-
storageM("storage_operationFailed", {
|
|
1164
|
-
params: { error: error instanceof Error ? error.message : String(error) }
|
|
1165
|
-
}),
|
|
1166
|
-
error
|
|
1167
|
-
);
|
|
1168
|
-
} finally {
|
|
1169
|
-
currentProvider = null;
|
|
1170
|
-
currentConfig = null;
|
|
1343
|
+
if (!closingPromise) {
|
|
1344
|
+
closingPromise = (async () => {
|
|
1345
|
+
if (initCompletion)
|
|
1346
|
+
await initCompletion;
|
|
1347
|
+
return await closeOwnedProvider();
|
|
1348
|
+
})().finally(() => {
|
|
1349
|
+
closingPromise = null;
|
|
1350
|
+
});
|
|
1171
1351
|
}
|
|
1352
|
+
return await closingPromise;
|
|
1172
1353
|
}
|
|
1173
1354
|
};
|
|
1174
1355
|
|