@h-ai/storage 0.1.0-alpha.60 → 0.1.0-alpha.62

package/dist/node.js

@@ -1,16 +1,63 @@
-import { HaiStorageError, StorageConfigSchema } from './chunk-W44BX2GI.js';
-export { HaiStorageError, LocalConfigSchema, PresignOptionsSchema, PresignUploadOptionsSchema, S3ConfigSchema, StorageConfigSchema, StorageOperationLogConfigSchema, StorageTypeSchema } from './chunk-W44BX2GI.js';
-import { storageM, MIME_TYPES, MIME_TYPE_DEFAULT } from './chunk-U427BQIQ.js';
-export { downloadAndSave, downloadWithPresignedUrl, formatFileSize, getFileExtension, getMimeType, uploadWithPresignedUrl } from './chunk-U427BQIQ.js';
-import { core, ok, err } from '@h-ai/core';
-import { Buffer } from 'buffer';
-import * as crypto from 'crypto';
-import * as fs from 'fs';
-import * as fsp from 'fs/promises';
-import * as path from 'path';
+import { HaiStorageError, StorageConfigSchema } from './chunk-KXE6OO5T.js';
+export { HaiStorageError, LocalConfigSchema, PresignOptionsSchema, PresignUploadOptionsSchema, S3ConfigSchema, StorageConfigSchema, StorageOperationLogConfigSchema, StorageTypeSchema } from './chunk-KXE6OO5T.js';
+import { storageM, MIME_TYPES, MIME_TYPE_DEFAULT } from './chunk-25GYTRA7.js';
+export { downloadAndSave, downloadWithPresignedUrl, formatFileSize, getFileExtension, getMimeType, uploadWithPresignedUrl } from './chunk-25GYTRA7.js';
+import { core, err, ok } from '@h-ai/core';
+import { Buffer } from 'node:buffer';
+import * as crypto from 'node:crypto';
+import { createHash, timingSafeEqual } from 'node:crypto';
+import * as fs from 'node:fs';
+import * as fsp from 'node:fs/promises';
+import * as path from 'node:path';
+import { Readable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
 import { S3Client, ListObjectsV2Command, PutObjectCommand, GetObjectCommand, CopyObjectCommand, DeleteObjectsCommand, DeleteObjectCommand, HeadObjectCommand } from '@aws-sdk/client-s3';
 import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
 
+async function* toByteStream(data, signal) {
+  signal?.throwIfAborted();
+  if (typeof data === "string" || data instanceof Uint8Array) {
+    yield typeof data === "string" ? Buffer.from(data) : data;
+    return;
+  }
+  const iterator = data[Symbol.asyncIterator]();
+  let completed = false;
+  try {
+    while (true) {
+      signal?.throwIfAborted();
+      let result;
+      let onAbort;
+      try {
+        const cancelled = signal ? new Promise((_, reject) => {
+          onAbort = () => reject(signal.reason);
+          signal.addEventListener("abort", onAbort, { once: true });
+        }) : void 0;
+        const next = iterator.next();
+        result = await (cancelled ? Promise.race([next, cancelled]) : next);
+      } finally {
+        if (onAbort)
+          signal?.removeEventListener("abort", onAbort);
+      }
+      if (result.done) {
+        completed = true;
+        break;
+      }
+      if (!(result.value instanceof Uint8Array))
+        throw new TypeError(storageM("storage_streamInvalidChunk"));
+      yield result.value;
+    }
+    signal?.throwIfAborted();
+  } finally {
+    if (!completed && iterator.return) {
+      if (signal?.aborted)
+        void Promise.resolve(iterator.return()).catch(() => {
+        });
+      else
+        await iterator.return();
+    }
+  }
+}
+
 // src/providers/storage-operation-log.ts
 function normalizePayloadValue(value) {
   if (value instanceof Uint8Array) {
@@ -54,6 +101,9 @@ var logger = core.logger.child({ module: "storage", scope: "provider-local" });
 var PATH_SEGMENT_SPLIT_REGEX = /[\\/]+/;
 function toStorageError(error, key) {
   const e = error;
+  if (e.code === "EEXIST") {
+    return { ...HaiStorageError.ALREADY_EXISTS, message: storageM("storage_fileAlreadyExists", { params: { key: key ?? "" } }), cause: error };
+  }
   if (e.code === "ENOENT") {
     return {
       ...HaiStorageError.NOT_FOUND,
@@ -103,7 +153,7 @@ function calculateEtag(stat2) {
   return `"${stat2.size.toString(16)}-${stat2.mtimeMs.toString(16)}"`;
 }
 function isMetaFile(key) {
-  return key.endsWith(".meta.json");
+  return key.endsWith(".meta.json") || key.endsWith(".upload.tmp");
 }
 function createInvalidPathError() {
   const error = new Error(storageM("storage_pathTraversal"));
@@ -145,15 +195,6 @@ function createLocalProvider() {
   function logOperation(category, operation, payload) {
     logStorageOperation(logger, config?.operationLog, category, operation, payload);
   }
-  function toBuffer(data) {
-    if (typeof data === "string") {
-      return Buffer.from(data);
-    }
-    if (data instanceof Buffer) {
-      return data;
-    }
-    return Buffer.from(data);
-  }
   const file = {
     async put(key, data, options = {}) {
       if (isMetaFile(key)) {
@@ -162,14 +203,23 @@ function createLocalProvider() {
           storageM("storage_metaFileAccess", { params: { key } })
         );
       }
+      let temporaryPath;
       try {
         const filePath = fullPath(key);
-        const buffer = toBuffer(data);
-        logOperation("write", "file.put", { key, byteLength: buffer.length, options });
-        logger.debug("Putting file", { key, size: buffer.length });
+        logOperation("write", "file.put", { key, options });
         const dir2 = path.dirname(filePath);
         await fsp.mkdir(dir2, { recursive: true, mode: getConfig().directoryMode });
-        await fsp.writeFile(filePath, buffer, { mode: getConfig().fileMode });
+        temporaryPath = `${filePath}.${crypto.randomUUID()}.upload.tmp`;
+        await pipeline(
+          Readable.from(toByteStream(data, options.signal), { objectMode: false }),
+          fs.createWriteStream(temporaryPath, { flags: "wx", mode: getConfig().fileMode }),
+          { signal: options.signal }
+        );
+        options.signal?.throwIfAborted();
+        if (options.ifNoneMatch === "*")
+          await fsp.link(temporaryPath, filePath);
+        else
+          await fsp.rename(temporaryPath, filePath);
         const stat2 = await fsp.stat(filePath);
         const metadata = {
           key,
@@ -179,7 +229,7 @@ function createLocalProvider() {
           etag: calculateEtag(stat2),
           metadata: options.metadata
         };
-        if (options.metadata) {
+        if (options.metadata || options.contentType) {
           const metaPath = `${filePath}.meta.json`;
           await fsp.writeFile(metaPath, JSON.stringify({
             contentType: options.contentType,
@@ -190,9 +240,26 @@ function createLocalProvider() {
       } catch (error) {
         logger.warn("Put file failed", { key, error });
         return err(toStorageError(error, key));
+      } finally {
+        if (temporaryPath)
+          await fsp.unlink(temporaryPath).catch(() => {
+          });
       }
     },
     async get(key, options = {}) {
+      const streamed = await file.getStream(key, options);
+      if (!streamed.success)
+        return streamed;
+      try {
+        const chunks = [];
+        for await (const chunk of streamed.data)
+          chunks.push(chunk);
+        return ok(Buffer.concat(chunks));
+      } catch (error) {
+        return err(toStorageError(error, key));
+      }
+    },
+    async getStream(key, options = {}) {
       if (isMetaFile(key)) {
         return err(
           HaiStorageError.INVALID_PATH,
@@ -201,21 +268,17 @@ function createLocalProvider() {
       }
       try {
         const filePath = fullPath(key);
-        logOperation("read", "file.get", { key, options });
-        if (options.rangeStart !== void 0 || options.rangeEnd !== void 0) {
-          const stat2 = await fsp.stat(filePath);
-          const start = options.rangeStart ?? 0;
-          const end = options.rangeEnd !== void 0 ? options.rangeEnd : stat2.size - 1;
-          return new Promise((resolve2) => {
-            const chunks = [];
-            const stream = fs.createReadStream(filePath, { start, end });
-            stream.on("data", (chunk) => chunks.push(chunk));
-            stream.on("end", () => resolve2(ok(Buffer.concat(chunks))));
-            stream.on("error", (error) => resolve2(err(toStorageError(error, key))));
-          });
+        logOperation("read", "file.getStream", { key, options });
+        options.signal?.throwIfAborted();
+        const stat2 = await fsp.stat(filePath);
+        if (stat2.isDirectory())
+          return err(HaiStorageError.INVALID_PATH, storageM("storage_pathIsDir", { params: { key } }));
+        async function* read() {
+          const stream = fs.createReadStream(filePath, { start: options.rangeStart, end: options.rangeEnd, signal: options.signal });
+          for await (const chunk of stream)
+            yield chunk;
         }
-        const data = await fsp.readFile(filePath);
-        return ok(data);
+        return ok(read());
       } catch (error) {
         return err(toStorageError(error, key));
       }
@@ -379,7 +442,7 @@ function createLocalProvider() {
               return;
             const fullEntryPath = path.join(dirPath, entry.name);
             const key = path.relative(cfg.root, fullEntryPath).replace(/\\/g, "/");
-            if (entry.name.endsWith(".meta.json"))
+            if (isMetaFile(entry.name))
               continue;
             if (entry.isDirectory()) {
               if (delimiter) {
@@ -494,6 +557,8 @@ function sanitizeEndpointUrl(url) {
 }
 function toStorageError2(error, key) {
   const e = error;
+  if (e.name === "PreconditionFailed" || e.$metadata?.httpStatusCode === 412)
+    return { ...HaiStorageError.ALREADY_EXISTS, message: storageM("storage_fileAlreadyExists", { params: { key: key ?? "" } }), cause: error };
   if (e.name === "NoSuchKey" || e.Code === "NoSuchKey" || e.name === "NotFound" || e.Code === "NotFound" || e.name === "404" || e.$metadata?.httpStatusCode === 404) {
     return {
       ...HaiStorageError.NOT_FOUND,
@@ -569,24 +634,50 @@ function createS3Provider() {
         const s3Client = getClient();
         const s3Config = getConfig();
         const fullPath = fullKey(key);
+        options.signal?.throwIfAborted();
         const body = typeof data === "string" ? Buffer.from(data) : data;
-        logOperation("write", "file.put", { key, fullPath, byteLength: body.length, options });
-        await s3Client.send(new PutObjectCommand({
+        const buffered = body instanceof Uint8Array;
+        let size = buffered ? body.byteLength : 0;
+        const params = {
           Bucket: s3Config.bucket,
           Key: fullPath,
-          Body: body,
           ContentType: options.contentType,
           Metadata: options.metadata,
           CacheControl: options.cacheControl,
-          ContentDisposition: options.contentDisposition
-        }));
+          ContentDisposition: options.contentDisposition,
+          IfNoneMatch: options.ifNoneMatch
+        };
+        logOperation("write", "file.put", { key, fullPath, byteLength: buffered ? size : void 0, options });
+        if (buffered) {
+          await s3Client.send(new PutObjectCommand({ ...params, Body: body }), { abortSignal: options.signal });
+        } else {
+          async function* counted() {
+            for await (const chunk of toByteStream(data, options.signal)) {
+              size += chunk.byteLength;
+              yield chunk;
+            }
+          }
+          const stream = Readable.from(counted(), { objectMode: false });
+          const controller = new AbortController();
+          const abort = () => controller.abort();
+          options.signal?.addEventListener("abort", abort, { once: true });
+          try {
+            if (options.signal?.aborted)
+              abort();
+            const { Upload } = await import('@aws-sdk/lib-storage');
+            await new Upload({ client: s3Client, params: { ...params, Body: stream }, queueSize: 2, leavePartsOnError: false, abortController: controller }).done();
+          } finally {
+            options.signal?.removeEventListener("abort", abort);
+            stream.destroy();
+          }
+        }
         const headResult = await file.head(key);
         if (headResult.success) {
           return headResult;
         }
         return ok({
           key,
-          size: body.length,
+          size,
           contentType: options.contentType || "application/octet-stream",
           lastModified: /* @__PURE__ */ new Date()
         });
@@ -595,11 +686,24 @@ function createS3Provider() {
       }
     },
     async get(key, options = {}) {
+      const streamed = await file.getStream(key, options);
+      if (!streamed.success)
+        return streamed;
+      try {
+        const chunks = [];
+        for await (const chunk of streamed.data)
+          chunks.push(chunk);
+        return ok(Buffer.concat(chunks));
+      } catch (error) {
+        return err(toStorageError2(error, key));
+      }
+    },
+    async getStream(key, options = {}) {
       try {
         const s3Client = getClient();
         const s3Config = getConfig();
         const fullPath = fullKey(key);
-        logOperation("read", "file.get", { key, fullPath, options });
+        logOperation("read", "file.getStream", { key, fullPath, options });
         let Range;
         if (options.rangeStart !== void 0 || options.rangeEnd !== void 0) {
           const start = options.rangeStart ?? 0;
@@ -610,19 +714,15 @@ function createS3Provider() {
           Bucket: s3Config.bucket,
           Key: fullPath,
           Range
-        }));
+        }), { abortSignal: options.signal });
         if (!response.Body) {
           return err(
             HaiStorageError.OPERATION_FAILED,
             storageM("storage_responseBodyEmpty")
           );
         }
-        const chunks = [];
         const stream = response.Body;
-        for await (const chunk of stream) {
-          chunks.push(chunk);
-        }
-        return ok(Buffer.concat(chunks));
+        return ok(stream);
       } catch (error) {
         return err(toStorageError2(error, key));
       }
@@ -860,6 +960,9 @@ function createS3Provider() {
           region: cfg.region,
           endpoint: cfg.endpoint,
           forcePathStyle: cfg.forcePathStyle,
+          // 不强制兼容服务实现 AWS checksum 扩展；校验工具按实际字节独立计算。
+          requestChecksumCalculation: "WHEN_REQUIRED",
+          responseChecksumValidation: "WHEN_REQUIRED",
           credentials: {
             accessKeyId: cfg.accessKeyId,
             secretAccessKey: cfg.secretAccessKey
@@ -891,6 +994,37 @@ function createS3Provider() {
     }
   };
 }
+var HEX_CHECKSUM = /^[\da-f]+$/i;
+var BASE64_CHECKSUM = /^(?:[A-Z\d+/]{4})*(?:[A-Z\d+/]{2}==|[A-Z\d+/]{3}=)?$/i;
+var ALGORITHMS = /* @__PURE__ */ new Set(["sha256", "sha384", "sha512", "sha1", "md5"]);
+async function generate(data, options = {}) {
+  try {
+    const algorithm = options.algorithm ?? "sha256";
+    if (!ALGORITHMS.has(algorithm))
+      return err(HaiStorageError.INVALID_CHECKSUM, storageM("storage_checksumInvalid"));
+    const hash = createHash(algorithm);
+    for await (const chunk of toByteStream(data, options.signal))
+      hash.update(chunk);
+    return ok(hash.digest(options.encoding ?? "hex"));
+  } catch (error) {
+    return err(HaiStorageError.CHECKSUM_FAILED, storageM("storage_checksumFailed"), error);
+  }
+}
+var checksum = {
+  generate,
+  async verify(data, expected, options = {}) {
+    const encoding = options.encoding ?? "hex";
+    const valid = encoding === "hex" ? expected.length % 2 === 0 && HEX_CHECKSUM.test(expected) : BASE64_CHECKSUM.test(expected) && expected.length > 0;
+    if (!valid)
+      return err(HaiStorageError.INVALID_CHECKSUM, storageM("storage_checksumInvalid"));
+    const calculated = await generate(data, options);
+    if (!calculated.success)
+      return calculated;
+    const actualBytes = Buffer.from(calculated.data, encoding);
+    const expectedBytes = Buffer.from(expected, encoding);
+    return ok(actualBytes.length === expectedBytes.length && timingSafeEqual(actualBytes, expectedBytes));
+  }
+};
 
 // src/storage-main.ts
 var logger3 = core.logger.child({ module: "storage", scope: "main" });
@@ -921,6 +1055,7 @@ var notInitializedPresign = new Proxy(
   }
 );
 var storage = {
+  checksum,
   /**
    * 初始化存储连接。
    *