@hanzo/s3 0.6.3

package/lib/index.js

@@ -0,0 +1,450 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var __decorateClass = (decorators, target, key, kind) => {
+  var result = kind > 1 ? void 0 : kind ? __getOwnPropDesc(target, key) : target;
+  for (var i = decorators.length - 1, decorator; i >= 0; i--)
+    if (decorator = decorators[i])
+      result = (kind ? decorator(target, key, result) : decorator(result)) || result;
+  if (kind && result) __defProp(target, key, result);
+  return result;
+};
+var index_exports = {};
+__export(index_exports, {
+  CONFIG_KIND: () => CONFIG_KIND,
+  S3Service: () => S3Service,
+  processConfigFromEnv: () => processConfigFromEnv
+});
+module.exports = __toCommonJS(index_exports);
+var import_client_s3 = require("@aws-sdk/client-s3");
+var import_lib_storage = require("@aws-sdk/lib-storage");
+var import_node_http_handler = require("@smithy/node-http-handler");
+var import_http = require("http");
+var import_https = require("https");
+var import_core = __toESM(require("@hanzo/core"));
+var import_platform = require("@hanzo/platform");
+var import_server_core = __toESM(require("@hanzo/server-core"));
+var import_stream = require("stream");
+var import_storage = require("@hanzo/storage");
+const CONFIG_KIND = "s3";
+class S3Service {
+  constructor(opt) {
+    this.opt = opt;
+    this.client = new import_client_s3.S3({
+      endpoint: opt.endpoint,
+      credentials: {
+        accessKeyId: opt.accessKey,
+        secretAccessKey: opt.secretKey
+      },
+      region: opt.region ?? "auto",
+      requestHandler: new import_node_http_handler.NodeHttpHandler({
+        connectionTimeout: 5e3,
+        socketTimeout: 12e4,
+        httpAgent: new import_http.Agent({ maxSockets: 500, keepAlive: true }),
+        httpsAgent: new import_https.Agent({ maxSockets: 500, keepAlive: true })
+      })
+    });
+    this.expireTime = parseInt(this.opt.expireTime ?? "168") * 3600;
+  }
+  static {
+    __name(this, "S3Service");
+  }
+  expireTime;
+  client;
+  async initialize(ctx, wsIds) {
+  }
+  /**
+   * @public
+   */
+  getBucketId(wsIds) {
+    return this.opt.rootBucket ?? (this.opt.bucketPrefix ?? "") + (0, import_server_core.getDataId)(wsIds);
+  }
+  getBucketFolder(wsIds) {
+    return (0, import_server_core.getDataId)(wsIds);
+  }
+  async close() {
+  }
+  async exists(ctx, wsIds) {
+    try {
+      const result = await this.client.headBucket({
+        Bucket: this.getBucketId(wsIds)
+      });
+      return result.$metadata.httpStatusCode === 200;
+    } catch (err) {
+      if (err.name === "400") {
+        return false;
+      }
+    }
+    return false;
+  }
+  async make(ctx, wsIds) {
+    try {
+      await this.client.createBucket({
+        Bucket: this.getBucketId(wsIds)
+      });
+    } catch (err) {
+      if (err.Code === "BucketAlreadyOwnedByYou") {
+        return;
+      }
+      ctx.error("error during create bucket", { err });
+    }
+  }
+  async listBuckets(ctx) {
+    try {
+      if (this.opt.rootBucket !== void 0) {
+        const info = /* @__PURE__ */ new Map();
+        let token;
+        while (true) {
+          const res = await this.client.listObjectsV2({
+            Bucket: this.opt.rootBucket,
+            Prefix: "",
+            Delimiter: "/",
+            ContinuationToken: token
+          });
+          for (const data of res.CommonPrefixes ?? []) {
+            const wsDataId = data.Prefix?.split("/")?.[0];
+            const wsIds = {
+              uuid: wsDataId,
+              dataId: wsDataId,
+              url: ""
+            };
+            if (wsDataId !== void 0 && !info.has(wsDataId)) {
+              info.set(wsDataId, {
+                name: wsDataId,
+                delete: /* @__PURE__ */ __name(async () => {
+                  await this.delete(ctx, wsIds);
+                }, "delete"),
+                list: /* @__PURE__ */ __name(async () => await this.listStream(ctx, wsIds), "list")
+              });
+            }
+          }
+          if (res.IsTruncated === true) {
+            token = res.NextContinuationToken;
+          } else {
+            break;
+          }
+        }
+        return Array.from(info.values());
+      } else {
+        const productPostfix = this.getBucketFolder({
+          uuid: "",
+          dataId: "",
+          url: ""
+        });
+        const buckets = await this.client.listBuckets();
+        return (buckets.Buckets ?? []).filter((it) => it.Name !== void 0 && it.Name.endsWith(productPostfix)).map((it) => {
+          let name = it.Name ?? "";
+          name = name.slice(0, name.length - productPostfix.length);
+          const wsIds = {
+            uuid: name,
+            dataId: name,
+            url: ""
+          };
+          return {
+            name,
+            delete: /* @__PURE__ */ __name(async () => {
+              await this.delete(ctx, wsIds);
+            }, "delete"),
+            list: /* @__PURE__ */ __name(async () => await this.listStream(ctx, wsIds), "list")
+          };
+        });
+      }
+    } catch (err) {
+      if (err.Code === "NoSuchBucket") {
+        return [];
+      }
+      ctx.error("failed to list buckets", { rootBucket: this.opt.rootBucket });
+      console.error(err);
+      return [];
+    }
+  }
+  getDocumentKey(wsIds, name) {
+    return this.opt.rootBucket === void 0 ? name : `${this.getBucketFolder(wsIds)}/${name}`;
+  }
+  async remove(ctx, wsIds, objectNames) {
+    await this.client.deleteObjects({
+      Bucket: this.getBucketId(wsIds),
+      Delete: {
+        Objects: objectNames.map((it) => ({ Key: this.getDocumentKey(wsIds, it) }))
+      }
+    });
+  }
+  async delete(ctx, wsIds) {
+    try {
+      await (0, import_storage.removeAllObjects)(ctx, this, wsIds);
+    } catch (err) {
+      ctx.error("failed t oclean all objecrs", { error: err });
+    }
+    if (this.opt.rootBucket === void 0) {
+      await this.client.deleteBucket({
+        Bucket: this.getBucketId(wsIds)
+      });
+    }
+  }
+  stripPrefix(prefix, key) {
+    if (prefix !== void 0 && key.startsWith(prefix)) {
+      return key.slice(prefix.length);
+    }
+    return key;
+  }
+  rootPrefix(wsIds) {
+    return this.opt.rootBucket !== void 0 ? this.getBucketFolder(wsIds) + "/" : void 0;
+  }
+  async copy(sourceId, targetId, objectName) {
+    const copyOp = new import_client_s3.CopyObjectCommand({
+      Bucket: this.getBucketId(targetId),
+      Key: this.getDocumentKey(targetId, objectName),
+      CopySource: `${this.getBucketId(sourceId)}/${this.getDocumentKey(sourceId, objectName)}`
+    });
+    await this.client.send(copyOp);
+  }
+  async listStream(ctx, wsIds) {
+    let hasMore = true;
+    const buffer = [];
+    let token;
+    const rootPrefix = this.rootPrefix(wsIds);
+    return {
+      next: /* @__PURE__ */ __name(async () => {
+        try {
+          while (hasMore && buffer.length < 50) {
+            const res = await this.client.listObjectsV2({
+              Bucket: this.getBucketId(wsIds),
+              Prefix: rootPrefix ?? "",
+              ContinuationToken: token
+            });
+            if (res.IsTruncated === true) {
+              token = res.NextContinuationToken;
+            } else {
+              hasMore = false;
+            }
+            for (const data of res.Contents ?? []) {
+              const _id = this.stripPrefix(rootPrefix, data.Key ?? "");
+              buffer.push({
+                _id,
+                _class: import_core.default.class.Blob,
+                etag: data.ETag ?? "",
+                size: data.Size ?? 0,
+                provider: this.opt.name,
+                space: import_core.default.space.Configuration,
+                modifiedBy: import_core.default.account.ConfigUser,
+                modifiedOn: data.LastModified?.getTime() ?? 0
+              });
+            }
+          }
+        } catch (err) {
+          ctx.error("Failed to get list", { error: err, wsIds });
+        }
+        return buffer.splice(0, 50);
+      }, "next"),
+      close: /* @__PURE__ */ __name(async () => {
+      }, "close")
+    };
+  }
+  async stat(ctx, wsIds, objectName) {
+    try {
+      const result = await this.client.headObject({
+        Bucket: this.getBucketId(wsIds),
+        Key: this.getDocumentKey(wsIds, objectName)
+      });
+      const rootPrefix = this.rootPrefix(wsIds);
+      return {
+        provider: "",
+        _class: import_core.default.class.Blob,
+        _id: this.stripPrefix(rootPrefix, objectName),
+        contentType: result.ContentType ?? "",
+        size: result.ContentLength ?? 0,
+        etag: result.ETag ?? "",
+        space: import_core.default.space.Configuration,
+        modifiedBy: import_core.default.account.System,
+        modifiedOn: result.LastModified?.getTime() ?? 0,
+        version: result.VersionId ?? null
+      };
+    } catch (err) {
+      if (err?.$metadata?.httpStatusCode !== 404) {
+        ctx.warn("no object found", { error: err, objectName, wsIds });
+      }
+    }
+  }
+  async get(ctx, wsIds, objectName) {
+    return await this.doGet(ctx, wsIds, objectName);
+  }
+  async doGet(ctx, wsIds, objectName, range) {
+    try {
+      const res = await this.client.getObject({
+        Bucket: this.getBucketId(wsIds),
+        Key: this.getDocumentKey(wsIds, objectName),
+        Range: range
+      });
+      const stream = res.Body?.transformToWebStream();
+      if (stream !== void 0) {
+        return import_stream.Readable.fromWeb(stream);
+      } else {
+        const readable = new import_stream.Readable();
+        readable._read = () => {
+        };
+        readable.push(null);
+        return readable;
+      }
+    } catch (err) {
+      throw new import_server_core.NoSuchKeyError(`uuid=${wsIds.uuid} dataId=${wsIds.dataId} missing ${objectName}`, err);
+    }
+  }
+  put(ctx, wsIds, objectName, stream, contentType, size) {
+    if (size !== void 0 && size < 1024 * 1024 * 5) {
+      return ctx.with(
+        "simple-put",
+        {},
+        async () => {
+          const cmd = new import_client_s3.PutObjectCommand({
+            Bucket: this.getBucketId(wsIds),
+            Key: this.getDocumentKey(wsIds, objectName),
+            ContentType: contentType,
+            ContentLength: size,
+            Body: stream
+          });
+          const response = await this.client.send(cmd);
+          return {
+            etag: response.ETag ?? "",
+            versionId: response.VersionId ?? null
+          };
+        },
+        { size, objectName, wsIds }
+      );
+    } else {
+      return ctx.with(
+        "multipart-upload",
+        {},
+        async () => {
+          const uploadTask = new import_lib_storage.Upload({
+            client: this.client,
+            params: {
+              Bucket: this.getBucketId(wsIds),
+              Key: this.getDocumentKey(wsIds, objectName),
+              ContentType: contentType,
+              Body: stream
+            },
+            // (optional) concurrency configuration
+            // queueSize: 1,
+            // (optional) size of each part, in bytes, at least 5MB
+            partSize: 1024 * 1024 * 5,
+            leavePartsOnError: false
+          });
+          const output = await uploadTask.done();
+          return {
+            etag: output.ETag ?? "",
+            versionId: output.VersionId ?? null
+          };
+        },
+        { size, objectName, wsIds }
+      );
+    }
+  }
+  async read(ctx, wsIds, name) {
+    const data = await this.doGet(ctx, wsIds, name);
+    const chunks = [];
+    await new Promise((resolve, reject) => {
+      data.on("data", (chunk) => {
+        chunks.push(chunk);
+      });
+      data.on("end", () => {
+        data.destroy();
+        resolve(null);
+      });
+      data.on("error", (err) => {
+        data.destroy();
+        reject(err);
+      });
+    });
+    return chunks;
+  }
+  async partial(ctx, wsIds, objectName, offset, length) {
+    const range = length !== void 0 ? `bytes=${offset}-${offset + length}` : `bytes=${offset}-`;
+    return await this.doGet(ctx, wsIds, objectName, range);
+  }
+  async getUrl(ctx, wsIds, objectName) {
+    const filesUrl = (0, import_platform.getMetadata)(import_server_core.default.metadata.FilesUrl) ?? "";
+    return filesUrl.replaceAll(":workspace", (0, import_server_core.getDataId)(wsIds)).replaceAll(":blobId", objectName);
+  }
+}
+__decorateClass([
+  (0, import_core.withContext)("make")
+], S3Service.prototype, "make", 1);
+__decorateClass([
+  (0, import_core.withContext)("remove")
+], S3Service.prototype, "remove", 1);
+__decorateClass([
+  (0, import_core.withContext)("delete")
+], S3Service.prototype, "delete", 1);
+__decorateClass([
+  (0, import_core.withContext)("listStream")
+], S3Service.prototype, "listStream", 1);
+__decorateClass([
+  (0, import_core.withContext)("stat")
+], S3Service.prototype, "stat", 1);
+__decorateClass([
+  (0, import_core.withContext)("get")
+], S3Service.prototype, "get", 1);
+__decorateClass([
+  (0, import_core.withContext)("put")
+], S3Service.prototype, "put", 1);
+__decorateClass([
+  (0, import_core.withContext)("read")
+], S3Service.prototype, "read", 1);
+__decorateClass([
+  (0, import_core.withContext)("partial")
+], S3Service.prototype, "partial", 1);
+__decorateClass([
+  (0, import_core.withContext)("getUrl")
+], S3Service.prototype, "getUrl", 1);
+function processConfigFromEnv(storageConfig) {
+  const endpoint = process.env.S3_ENDPOINT;
+  if (endpoint === void 0) {
+    return "S3_ENDPOINT";
+  }
+  const accessKey = process.env.S3_ACCESS_KEY;
+  if (accessKey === void 0) {
+    return "S3_ACCESS_KEY";
+  }
+  const secretKey = process.env.S3_SECRET_KEY;
+  if (secretKey === void 0) {
+    return "S3_SECRET_KEY";
+  }
+  const config = {
+    kind: "s3",
+    name: "s3",
+    region: "auto",
+    endpoint,
+    accessKey,
+    secretKey
+  };
+  storageConfig.storages.push(config);
+  storageConfig.default = "s3";
+}
+__name(processConfigFromEnv, "processConfigFromEnv");
+//# sourceMappingURL=index.js.map

package/lib/index.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/index.ts"],
+  "sourcesContent": ["//\n// Copyright \u00A9 2024 Hardcore Engineering Inc.\n//\n// Licensed under the Eclipse Public License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License. You may\n// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n//\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//\n\nimport { CopyObjectCommand, PutObjectCommand, S3 } from '@aws-sdk/client-s3'\nimport { Upload } from '@aws-sdk/lib-storage'\nimport { NodeHttpHandler } from '@smithy/node-http-handler'\nimport { Agent as HttpAgent } from 'http'\nimport { Agent as HttpsAgent } from 'https'\n\nimport core, {\n  withContext,\n  type WorkspaceIds,\n  type Blob,\n  type MeasureContext,\n  type Ref,\n  type WorkspaceDataId,\n  type WorkspaceUuid\n} from '@hanzo/core'\nimport { getMetadata } from '@hanzo/platform'\nimport serverCore, {\n  NoSuchKeyError,\n  getDataId,\n  type BlobStorageIterator,\n  type ListBlobResult,\n  type StorageAdapter,\n  type StorageConfig,\n  type StorageConfiguration,\n  type UploadedObjectInfo\n} from '@hanzo/server-core'\nimport { Readable } from 'stream'\n\nimport { removeAllObjects, type BucketInfo } from '@hanzo/storage'\nimport type { ReadableStream } from 'stream/web'\n\nexport interface S3Config extends StorageConfig {\n  kind: 's3'\n  accessKey: string\n  secretKey: string\n  region?: string\n\n  // If defined, all resources will be inside selected root bucket.\n  rootBucket?: string\n\n  // A prefix string to be added to a bucketId in case rootBucket not used\n  bucketPrefix?: string\n\n  // If not specified will be enabled\n  allowPresign?: string\n  // Expire time for presigned URIs\n  expireTime?: string\n}\n\nexport const CONFIG_KIND = 's3'\n\n/**\n * @public\n */\nexport class S3Service implements StorageAdapter {\n  expireTime: number\n  client: S3\n  constructor (readonly opt: S3Config) {\n    this.client = new S3({\n      endpoint: opt.endpoint,\n      credentials: {\n        accessKeyId: opt.accessKey,\n        secretAccessKey: opt.secretKey\n      },\n      region: opt.region ?? 'auto',\n      requestHandler: new NodeHttpHandler({\n        connectionTimeout: 5000,\n        socketTimeout: 120000,\n        httpAgent: new HttpAgent({ maxSockets: 500, keepAlive: true }),\n        httpsAgent: new HttpsAgent({ maxSockets: 500, keepAlive: true })\n      })\n    })\n\n    this.expireTime = parseInt(this.opt.expireTime ?? '168') * 3600 // use 7 * 24 - hours as default value for expireF\n  }\n\n  async initialize (ctx: MeasureContext, wsIds: WorkspaceIds): Promise<void> {}\n\n  /**\n   * @public\n   */\n  getBucketId (wsIds: WorkspaceIds): string {\n    return this.opt.rootBucket ?? (this.opt.bucketPrefix ?? '') + getDataId(wsIds)\n  }\n\n  getBucketFolder (wsIds: WorkspaceIds): string {\n    return getDataId(wsIds)\n  }\n\n  async close (): Promise<void> {}\n\n  async exists (ctx: MeasureContext, wsIds: WorkspaceIds): Promise<boolean> {\n    try {\n      const result = await this.client.headBucket({\n        Bucket: this.getBucketId(wsIds)\n      })\n      return result.$metadata.httpStatusCode === 200\n    } catch (err: any) {\n      if (err.name === '400') {\n        // No bucket exisrs\n        return false\n      }\n    }\n    // No API to check is bucket exists or not, so we need to call make and check if it already exists.\n    return false\n  }\n\n  @withContext('make')\n  async make (ctx: MeasureContext, wsIds: WorkspaceIds): Promise<void> {\n    try {\n      await this.client.createBucket({\n        Bucket: this.getBucketId(wsIds)\n      })\n    } catch (err: any) {\n      if (err.Code === 'BucketAlreadyOwnedByYou') {\n        return\n      }\n      ctx.error('error during create bucket', { err })\n    }\n  }\n\n  async listBuckets (ctx: MeasureContext): Promise<BucketInfo[]> {\n    try {\n      if (this.opt.rootBucket !== undefined) {\n        const info = new Map<string, BucketInfo>()\n        let token: string | undefined\n\n        while (true) {\n          const res = await this.client.listObjectsV2({\n            Bucket: this.opt.rootBucket,\n            Prefix: '',\n            Delimiter: '/',\n            ContinuationToken: token\n          })\n          for (const data of res.CommonPrefixes ?? []) {\n            const wsDataId = data.Prefix?.split('/')?.[0] as WorkspaceDataId\n            const wsIds = {\n              uuid: wsDataId as unknown as WorkspaceUuid,\n              dataId: wsDataId,\n              url: ''\n            }\n            if (wsDataId !== undefined && !info.has(wsDataId)) {\n              info.set(wsDataId, {\n                name: wsDataId,\n                delete: async () => {\n                  await this.delete(ctx, wsIds)\n                },\n                list: async () => await this.listStream(ctx, wsIds)\n              })\n            }\n          }\n          if (res.IsTruncated === true) {\n            token = res.NextContinuationToken\n          } else {\n            break\n          }\n        }\n        return Array.from(info.values())\n      } else {\n        const productPostfix = this.getBucketFolder({\n          uuid: '' as WorkspaceUuid,\n          dataId: '' as WorkspaceDataId,\n          url: ''\n        })\n        const buckets = await this.client.listBuckets()\n        return (buckets.Buckets ?? [])\n          .filter((it) => it.Name !== undefined && it.Name.endsWith(productPostfix))\n          .map((it) => {\n            let name = (it.Name ?? '') as WorkspaceDataId\n            name = name.slice(0, name.length - productPostfix.length) as WorkspaceDataId\n            const wsIds = {\n              uuid: name as unknown as WorkspaceUuid,\n              dataId: name,\n              url: ''\n            }\n            return {\n              name,\n              delete: async () => {\n                await this.delete(ctx, wsIds)\n              },\n              list: async () => await this.listStream(ctx, wsIds)\n            }\n          })\n      }\n    } catch (err: any) {\n      if (err.Code === 'NoSuchBucket') {\n        return []\n      }\n      ctx.error('failed to list buckets', { rootBucket: this.opt.rootBucket })\n      console.error(err)\n      return []\n    }\n  }\n\n  getDocumentKey (wsIds: WorkspaceIds, name: string): string {\n    return this.opt.rootBucket === undefined ? name : `${this.getBucketFolder(wsIds)}/${name}`\n  }\n\n  @withContext('remove')\n  async remove (ctx: MeasureContext, wsIds: WorkspaceIds, objectNames: string[]): Promise<void> {\n    await this.client.deleteObjects({\n      Bucket: this.getBucketId(wsIds),\n      Delete: {\n        Objects: objectNames.map((it) => ({ Key: this.getDocumentKey(wsIds, it) }))\n      }\n    })\n  }\n\n  @withContext('delete')\n  async delete (ctx: MeasureContext, wsIds: WorkspaceIds): Promise<void> {\n    try {\n      await removeAllObjects(ctx, this, wsIds)\n    } catch (err: any) {\n      ctx.error('failed t oclean all objecrs', { error: err })\n    }\n    if (this.opt.rootBucket === undefined) {\n      // We should also delete bucket\n      await this.client.deleteBucket({\n        Bucket: this.getBucketId(wsIds)\n      })\n    }\n  }\n\n  stripPrefix (prefix: string | undefined, key: string): string {\n    if (prefix !== undefined && key.startsWith(prefix)) {\n      return key.slice(prefix.length)\n    }\n    return key\n  }\n\n  rootPrefix (wsIds: WorkspaceIds): string | undefined {\n    return this.opt.rootBucket !== undefined ? this.getBucketFolder(wsIds) + '/' : undefined\n  }\n\n  async copy (sourceId: WorkspaceIds, targetId: WorkspaceIds, objectName: string): Promise<void> {\n    const copyOp = new CopyObjectCommand({\n      Bucket: this.getBucketId(targetId),\n      Key: this.getDocumentKey(targetId, objectName),\n      CopySource: `${this.getBucketId(sourceId)}/${this.getDocumentKey(sourceId, objectName)}`\n    })\n    await this.client.send(copyOp)\n  }\n\n  @withContext('listStream')\n  async listStream (ctx: MeasureContext, wsIds: WorkspaceIds): Promise<BlobStorageIterator> {\n    let hasMore = true\n    const buffer: ListBlobResult[] = []\n    let token: string | undefined\n\n    const rootPrefix = this.rootPrefix(wsIds)\n    return {\n      next: async (): Promise<ListBlobResult[]> => {\n        try {\n          while (hasMore && buffer.length < 50) {\n            const res = await this.client.listObjectsV2({\n              Bucket: this.getBucketId(wsIds),\n              Prefix: rootPrefix ?? '',\n              ContinuationToken: token\n            })\n            if (res.IsTruncated === true) {\n              token = res.NextContinuationToken\n            } else {\n              hasMore = false\n            }\n\n            for (const data of res.Contents ?? []) {\n              const _id = this.stripPrefix(rootPrefix, data.Key ?? '')\n              buffer.push({\n                _id: _id as Ref<Blob>,\n                _class: core.class.Blob,\n                etag: data.ETag ?? '',\n                size: data.Size ?? 0,\n                provider: this.opt.name,\n                space: core.space.Configuration,\n                modifiedBy: core.account.ConfigUser,\n                modifiedOn: data.LastModified?.getTime() ?? 0\n              })\n            }\n          }\n        } catch (err: any) {\n          ctx.error('Failed to get list', { error: err, wsIds })\n        }\n        return buffer.splice(0, 50)\n      },\n      close: async () => {}\n    }\n  }\n\n  @withContext('stat')\n  async stat (ctx: MeasureContext, wsIds: WorkspaceIds, objectName: string): Promise<Blob | undefined> {\n    try {\n      const result = await this.client.headObject({\n        Bucket: this.getBucketId(wsIds),\n        Key: this.getDocumentKey(wsIds, objectName)\n      })\n      const rootPrefix = this.rootPrefix(wsIds)\n      return {\n        provider: '',\n        _class: core.class.Blob,\n        _id: this.stripPrefix(rootPrefix, objectName) as Ref<Blob>,\n        contentType: result.ContentType ?? '',\n        size: result.ContentLength ?? 0,\n        etag: result.ETag ?? '',\n        space: core.space.Configuration,\n        modifiedBy: core.account.System,\n        modifiedOn: result.LastModified?.getTime() ?? 0,\n        version: result.VersionId ?? null\n      }\n    } catch (err: any) {\n      if (err?.$metadata?.httpStatusCode !== 404) {\n        ctx.warn('no object found', { error: err, objectName, wsIds })\n      }\n    }\n  }\n\n  @withContext('get')\n  async get (ctx: MeasureContext, wsIds: WorkspaceIds, objectName: string): Promise<Readable> {\n    return await this.doGet(ctx, wsIds, objectName)\n  }\n\n  async doGet (ctx: MeasureContext, wsIds: WorkspaceIds, objectName: string, range?: string): Promise<Readable> {\n    try {\n      const res = await this.client.getObject({\n        Bucket: this.getBucketId(wsIds),\n        Key: this.getDocumentKey(wsIds, objectName),\n        Range: range\n      })\n\n      const stream = res.Body?.transformToWebStream()\n\n      if (stream !== undefined) {\n        return Readable.fromWeb(stream as ReadableStream<any>)\n      } else {\n        const readable = new Readable()\n        readable._read = () => {}\n        readable.push(null)\n        return readable\n      }\n    } catch (err: any) {\n      // In case of error return undefined\n      throw new NoSuchKeyError(`uuid=${wsIds.uuid} dataId=${wsIds.dataId} missing ${objectName}`, err)\n    }\n  }\n\n  @withContext('put')\n  put (\n    ctx: MeasureContext,\n    wsIds: WorkspaceIds,\n    objectName: string,\n    stream: Readable | Buffer | string,\n    contentType: string,\n    size?: number\n  ): Promise<UploadedObjectInfo> {\n    if (size !== undefined && size < 1024 * 1024 * 5) {\n      return ctx.with(\n        'simple-put',\n        {},\n        async () => {\n          const cmd = new PutObjectCommand({\n            Bucket: this.getBucketId(wsIds),\n            Key: this.getDocumentKey(wsIds, objectName),\n            ContentType: contentType,\n            ContentLength: size,\n            Body: stream\n          })\n          const response = await this.client.send(cmd)\n          return {\n            etag: response.ETag ?? '',\n            versionId: response.VersionId ?? null\n          }\n        },\n        { size, objectName, wsIds }\n      )\n      // Less 5Mb\n    } else {\n      return ctx.with(\n        'multipart-upload',\n        {},\n        async () => {\n          const uploadTask = new Upload({\n            client: this.client,\n            params: {\n              Bucket: this.getBucketId(wsIds),\n              Key: this.getDocumentKey(wsIds, objectName),\n              ContentType: contentType,\n              Body: stream\n            },\n\n            // (optional) concurrency configuration\n            // queueSize: 1,\n\n            // (optional) size of each part, in bytes, at least 5MB\n            partSize: 1024 * 1024 * 5,\n            leavePartsOnError: false\n          })\n\n          const output = await uploadTask.done()\n          return {\n            etag: output.ETag ?? '',\n            versionId: output.VersionId ?? null\n          }\n        },\n        { size, objectName, wsIds }\n      )\n    }\n  }\n\n  @withContext('read')\n  async read (ctx: MeasureContext, wsIds: WorkspaceIds, name: string): Promise<Buffer[]> {\n    const data = await this.doGet(ctx, wsIds, name)\n    const chunks: Buffer[] = []\n\n    await new Promise((resolve, reject) => {\n      data.on('data', (chunk) => {\n        chunks.push(chunk)\n      })\n\n      data.on('end', () => {\n        data.destroy()\n        resolve(null)\n      })\n      data.on('error', (err) => {\n        data.destroy()\n        reject(err)\n      })\n    })\n    return chunks\n  }\n\n  @withContext('partial')\n  async partial (\n    ctx: MeasureContext,\n    wsIds: WorkspaceIds,\n    objectName: string,\n    offset: number,\n    length?: number\n  ): Promise<Readable> {\n    const range = length !== undefined ? `bytes=${offset}-${offset + length}` : `bytes=${offset}-`\n    return await this.doGet(ctx, wsIds, objectName, range)\n  }\n\n  @withContext('getUrl')\n  async getUrl (ctx: MeasureContext, wsIds: WorkspaceIds, objectName: string): Promise<string> {\n    const filesUrl = getMetadata(serverCore.metadata.FilesUrl) ?? ''\n    return filesUrl.replaceAll(':workspace', getDataId(wsIds)).replaceAll(':blobId', objectName)\n  }\n}\n\nexport function processConfigFromEnv (storageConfig: StorageConfiguration): string | undefined {\n  const endpoint = process.env.S3_ENDPOINT\n  if (endpoint === undefined) {\n    return 'S3_ENDPOINT'\n  }\n  const accessKey = process.env.S3_ACCESS_KEY\n  if (accessKey === undefined) {\n    return 'S3_ACCESS_KEY'\n  }\n\n  const secretKey = process.env.S3_SECRET_KEY\n  if (secretKey === undefined) {\n    return 'S3_SECRET_KEY'\n  }\n\n  const config: S3Config = {\n    kind: 's3',\n    name: 's3',\n    region: 'auto',\n    endpoint,\n    accessKey,\n    secretKey\n  }\n  storageConfig.storages.push(config)\n  storageConfig.default = 's3'\n}\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAeA,uBAAwD;AACxD,yBAAuB;AACvB,+BAAgC;AAChC,kBAAmC;AACnC,mBAAoC;AAEpC,kBAQO;AACP,sBAA4B;AAC5B,yBASO;AACP,oBAAyB;AAEzB,qBAAkD;AAqB3C,MAAM,cAAc;AAKpB,MAAM,UAAoC;AAAA,EAG/C,YAAsB,KAAe;AAAf;AACpB,SAAK,SAAS,IAAI,oBAAG;AAAA,MACnB,UAAU,IAAI;AAAA,MACd,aAAa;AAAA,QACX,aAAa,IAAI;AAAA,QACjB,iBAAiB,IAAI;AAAA,MACvB;AAAA,MACA,QAAQ,IAAI,UAAU;AAAA,MACtB,gBAAgB,IAAI,yCAAgB;AAAA,QAClC,mBAAmB;AAAA,QACnB,eAAe;AAAA,QACf,WAAW,IAAI,YAAAA,MAAU,EAAE,YAAY,KAAK,WAAW,KAAK,CAAC;AAAA,QAC7D,YAAY,IAAI,aAAAC,MAAW,EAAE,YAAY,KAAK,WAAW,KAAK,CAAC;AAAA,MACjE,CAAC;AAAA,IACH,CAAC;AAED,SAAK,aAAa,SAAS,KAAK,IAAI,cAAc,KAAK,IAAI;AAAA,EAC7D;AAAA,EAzFF,OAqEiD;AAAA;AAAA;AAAA,EAC/C;AAAA,EACA;AAAA,EAoBA,MAAM,WAAY,KAAqB,OAAoC;AAAA,EAAC;AAAA;AAAA;AAAA;AAAA,EAK5E,YAAa,OAA6B;AACxC,WAAO,KAAK,IAAI,eAAe,KAAK,IAAI,gBAAgB,UAAM,8BAAU,KAAK;AAAA,EAC/E;AAAA,EAEA,gBAAiB,OAA6B;AAC5C,eAAO,8BAAU,KAAK;AAAA,EACxB;AAAA,EAEA,MAAM,QAAwB;AAAA,EAAC;AAAA,EAE/B,MAAM,OAAQ,KAAqB,OAAuC;AACxE,QAAI;AACF,YAAM,SAAS,MAAM,KAAK,OAAO,WAAW;AAAA,QAC1C,QAAQ,KAAK,YAAY,KAAK;AAAA,MAChC,CAAC;AACD,aAAO,OAAO,UAAU,mBAAmB;AAAA,IAC7C,SAAS,KAAU;AACjB,UAAI,IAAI,SAAS,OAAO;AAEtB,eAAO;AAAA,MACT;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAGA,MAAM,KAAM,KAAqB,OAAoC;AACnE,QAAI;AACF,YAAM,KAAK,OAAO,aAAa;AAAA,QAC7B,QAAQ,KAAK,YAAY,KAAK;AAAA,MAChC,CAAC;AAAA,IACH,SAAS,KAAU;AACjB,UAAI,IAAI,SAAS,2BAA2B;AAC1C;AAAA,MACF;AACA,UAAI,MAAM,8BAA8B,EAAE,IAAI,CAAC;AAAA,IACjD;AAAA,EACF;AAAA,EAEA,MAAM,YAAa,KAA4C;AAC7D,QAAI;AACF,UAAI,KAAK,IAAI,eAAe,QAAW;AACrC,cAAM,OAAO,oBAAI,IAAwB;AACzC,YAAI;AAEJ,eAAO,MAAM;AACX,gBAAM,MAAM,MAAM,KAAK,OAAO,cAAc;AAAA,YAC1C,QAAQ,KAAK,IAAI;AAAA,YACjB,QAAQ;AAAA,YACR,WAAW;AAAA,YACX,mBAAmB;AAAA,UACrB,CAAC;AACD,qBAAW,QAAQ,IAAI,kBAAkB,CAAC,GAAG;AAC3C,kBAAM,WAAW,KAAK,QAAQ,MAAM,GAAG,IAAI,CAAC;AAC5C,kBAAM,QAAQ;AAAA,cACZ,MAAM;AAAA,cACN,QAAQ;AAAA,cACR,KAAK;AAAA,YACP;AACA,gBAAI,aAAa,UAAa,CAAC,KAAK,IAAI,QAAQ,GAAG;AACjD,mBAAK,IAAI,UAAU;AAAA,gBACjB,MAAM;AAAA,gBACN,QAAQ,mCAAY;AAClB,wBAAM,KAAK,OAAO,KAAK,KAAK;AAAA,gBAC9B,GAFQ;AAAA,gBAGR,MAAM,mCAAY,MAAM,KAAK,WAAW,KAAK,KAAK,GAA5C;AAAA,cACR,CAAC;AAAA,YACH;AAAA,UACF;AACA,cAAI,IAAI,gBAAgB,MAAM;AAC5B,oBAAQ,IAAI;AAAA,UACd,OAAO;AACL;AAAA,UACF;AAAA,QACF;AACA,eAAO,MAAM,KAAK,KAAK,OAAO,CAAC;AAAA,MACjC,OAAO;AACL,cAAM,iBAAiB,KAAK,gBAAgB;AAAA,UAC1C,MAAM;AAAA,UACN,QAAQ;AAAA,UACR,KAAK;AAAA,QACP,CAAC;AACD,cAAM,UAAU,MAAM,KAAK,OAAO,YAAY;AAC9C,gBAAQ,QAAQ,WAAW,CAAC,GACzB,OAAO,CAAC,OAAO,GAAG,SAAS,UAAa,GAAG,KAAK,SAAS,cAAc,CAAC,EACxE,IAAI,CAAC,OAAO;AACX,cAAI,OAAQ,GAAG,QAAQ;AACvB,iBAAO,KAAK,MAAM,GAAG,KAAK,SAAS,eAAe,MAAM;AACxD,gBAAM,QAAQ;AAAA,YACZ,MAAM;AAAA,YACN,QAAQ;AAAA,YACR,KAAK;AAAA,UACP;AACA,iBAAO;AAAA,YACL;AAAA,YACA,QAAQ,mCAAY;AAClB,oBAAM,KAAK,OAAO,KAAK,KAAK;AAAA,YAC9B,GAFQ;AAAA,YAGR,MAAM,mCAAY,MAAM,KAAK,WAAW,KAAK,KAAK,GAA5C;AAAA,UACR;AAAA,QACF,CAAC;AAAA,MACL;AAAA,IACF,SAAS,KAAU;AACjB,UAAI,IAAI,SAAS,gBAAgB;AAC/B,eAAO,CAAC;AAAA,MACV;AACA,UAAI,MAAM,0BAA0B,EAAE,YAAY,KAAK,IAAI,WAAW,CAAC;AACvE,cAAQ,MAAM,GAAG;AACjB,aAAO,CAAC;AAAA,IACV;AAAA,EACF;AAAA,EAEA,eAAgB,OAAqB,MAAsB;AACzD,WAAO,KAAK,IAAI,eAAe,SAAY,OAAO,GAAG,KAAK,gBAAgB,KAAK,CAAC,IAAI,IAAI;AAAA,EAC1F;AAAA,EAGA,MAAM,OAAQ,KAAqB,OAAqB,aAAsC;AAC5F,UAAM,KAAK,OAAO,cAAc;AAAA,MAC9B,QAAQ,KAAK,YAAY,KAAK;AAAA,MAC9B,QAAQ;AAAA,QACN,SAAS,YAAY,IAAI,CAAC,QAAQ,EAAE,KAAK,KAAK,eAAe,OAAO,EAAE,EAAE,EAAE;AAAA,MAC5E;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAGA,MAAM,OAAQ,KAAqB,OAAoC;AACrE,QAAI;AACF,gBAAM,iCAAiB,KAAK,MAAM,KAAK;AAAA,IACzC,SAAS,KAAU;AACjB,UAAI,MAAM,+BAA+B,EAAE,OAAO,IAAI,CAAC;AAAA,IACzD;AACA,QAAI,KAAK,IAAI,eAAe,QAAW;AAErC,YAAM,KAAK,OAAO,aAAa;AAAA,QAC7B,QAAQ,KAAK,YAAY,KAAK;AAAA,MAChC,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,YAAa,QAA4B,KAAqB;AAC5D,QAAI,WAAW,UAAa,IAAI,WAAW,MAAM,GAAG;AAClD,aAAO,IAAI,MAAM,OAAO,MAAM;AAAA,IAChC;AACA,WAAO;AAAA,EACT;AAAA,EAEA,WAAY,OAAyC;AACnD,WAAO,KAAK,IAAI,eAAe,SAAY,KAAK,gBAAgB,KAAK,IAAI,MAAM;AAAA,EACjF;AAAA,EAEA,MAAM,KAAM,UAAwB,UAAwB,YAAmC;AAC7F,UAAM,SAAS,IAAI,mCAAkB;AAAA,MACnC,QAAQ,KAAK,YAAY,QAAQ;AAAA,MACjC,KAAK,KAAK,eAAe,UAAU,UAAU;AAAA,MAC7C,YAAY,GAAG,KAAK,YAAY,QAAQ,CAAC,IAAI,KAAK,eAAe,UAAU,UAAU,CAAC;AAAA,IACxF,CAAC;AACD,UAAM,KAAK,OAAO,KAAK,MAAM;AAAA,EAC/B;AAAA,EAGA,MAAM,WAAY,KAAqB,OAAmD;AACxF,QAAI,UAAU;AACd,UAAM,SAA2B,CAAC;AAClC,QAAI;AAEJ,UAAM,aAAa,KAAK,WAAW,KAAK;AACxC,WAAO;AAAA,MACL,MAAM,mCAAuC;AAC3C,YAAI;AACF,iBAAO,WAAW,OAAO,SAAS,IAAI;AACpC,kBAAM,MAAM,MAAM,KAAK,OAAO,cAAc;AAAA,cAC1C,QAAQ,KAAK,YAAY,KAAK;AAAA,cAC9B,QAAQ,cAAc;AAAA,cACtB,mBAAmB;AAAA,YACrB,CAAC;AACD,gBAAI,IAAI,gBAAgB,MAAM;AAC5B,sBAAQ,IAAI;AAAA,YACd,OAAO;AACL,wBAAU;AAAA,YACZ;AAEA,uBAAW,QAAQ,IAAI,YAAY,CAAC,GAAG;AACrC,oBAAM,MAAM,KAAK,YAAY,YAAY,KAAK,OAAO,EAAE;AACvD,qBAAO,KAAK;AAAA,gBACV;AAAA,gBACA,QAAQ,YAAAC,QAAK,MAAM;AAAA,gBACnB,MAAM,KAAK,QAAQ;AAAA,gBACnB,MAAM,KAAK,QAAQ;AAAA,gBACnB,UAAU,KAAK,IAAI;AAAA,gBACnB,OAAO,YAAAA,QAAK,MAAM;AAAA,gBAClB,YAAY,YAAAA,QAAK,QAAQ;AAAA,gBACzB,YAAY,KAAK,cAAc,QAAQ,KAAK;AAAA,cAC9C,CAAC;AAAA,YACH;AAAA,UACF;AAAA,QACF,SAAS,KAAU;AACjB,cAAI,MAAM,sBAAsB,EAAE,OAAO,KAAK,MAAM,CAAC;AAAA,QACvD;AACA,eAAO,OAAO,OAAO,GAAG,EAAE;AAAA,MAC5B,GAhCM;AAAA,MAiCN,OAAO,mCAAY;AAAA,MAAC,GAAb;AAAA,IACT;AAAA,EACF;AAAA,EAGA,MAAM,KAAM,KAAqB,OAAqB,YAA+C;AACnG,QAAI;AACF,YAAM,SAAS,MAAM,KAAK,OAAO,WAAW;AAAA,QAC1C,QAAQ,KAAK,YAAY,KAAK;AAAA,QAC9B,KAAK,KAAK,eAAe,OAAO,UAAU;AAAA,MAC5C,CAAC;AACD,YAAM,aAAa,KAAK,WAAW,KAAK;AACxC,aAAO;AAAA,QACL,UAAU;AAAA,QACV,QAAQ,YAAAA,QAAK,MAAM;AAAA,QACnB,KAAK,KAAK,YAAY,YAAY,UAAU;AAAA,QAC5C,aAAa,OAAO,eAAe;AAAA,QACnC,MAAM,OAAO,iBAAiB;AAAA,QAC9B,MAAM,OAAO,QAAQ;AAAA,QACrB,OAAO,YAAAA,QAAK,MAAM;AAAA,QAClB,YAAY,YAAAA,QAAK,QAAQ;AAAA,QACzB,YAAY,OAAO,cAAc,QAAQ,KAAK;AAAA,QAC9C,SAAS,OAAO,aAAa;AAAA,MAC/B;AAAA,IACF,SAAS,KAAU;AACjB,UAAI,KAAK,WAAW,mBAAmB,KAAK;AAC1C,YAAI,KAAK,mBAAmB,EAAE,OAAO,KAAK,YAAY,MAAM,CAAC;AAAA,MAC/D;AAAA,IACF;AAAA,EACF;AAAA,EAGA,MAAM,IAAK,KAAqB,OAAqB,YAAuC;AAC1F,WAAO,MAAM,KAAK,MAAM,KAAK,OAAO,UAAU;AAAA,EAChD;AAAA,EAEA,MAAM,MAAO,KAAqB,OAAqB,YAAoB,OAAmC;AAC5G,QAAI;AACF,YAAM,MAAM,MAAM,KAAK,OAAO,UAAU;AAAA,QACtC,QAAQ,KAAK,YAAY,KAAK;AAAA,QAC9B,KAAK,KAAK,eAAe,OAAO,UAAU;AAAA,QAC1C,OAAO;AAAA,MACT,CAAC;AAED,YAAM,SAAS,IAAI,MAAM,qBAAqB;AAE9C,UAAI,WAAW,QAAW;AACxB,eAAO,uBAAS,QAAQ,MAA6B;AAAA,MACvD,OAAO;AACL,cAAM,WAAW,IAAI,uBAAS;AAC9B,iBAAS,QAAQ,MAAM;AAAA,QAAC;AACxB,iBAAS,KAAK,IAAI;AAClB,eAAO;AAAA,MACT;AAAA,IACF,SAAS,KAAU;AAEjB,YAAM,IAAI,kCAAe,QAAQ,MAAM,IAAI,WAAW,MAAM,MAAM,YAAY,UAAU,IAAI,GAAG;AAAA,IACjG;AAAA,EACF;AAAA,EAGA,IACE,KACA,OACA,YACA,QACA,aACA,MAC6B;AAC7B,QAAI,SAAS,UAAa,OAAO,OAAO,OAAO,GAAG;AAChD,aAAO,IAAI;AAAA,QACT;AAAA,QACA,CAAC;AAAA,QACD,YAAY;AACV,gBAAM,MAAM,IAAI,kCAAiB;AAAA,YAC/B,QAAQ,KAAK,YAAY,KAAK;AAAA,YAC9B,KAAK,KAAK,eAAe,OAAO,UAAU;AAAA,YAC1C,aAAa;AAAA,YACb,eAAe;AAAA,YACf,MAAM;AAAA,UACR,CAAC;AACD,gBAAM,WAAW,MAAM,KAAK,OAAO,KAAK,GAAG;AAC3C,iBAAO;AAAA,YACL,MAAM,SAAS,QAAQ;AAAA,YACvB,WAAW,SAAS,aAAa;AAAA,UACnC;AAAA,QACF;AAAA,QACA,EAAE,MAAM,YAAY,MAAM;AAAA,MAC5B;AAAA,IAEF,OAAO;AACL,aAAO,IAAI;AAAA,QACT;AAAA,QACA,CAAC;AAAA,QACD,YAAY;AACV,gBAAM,aAAa,IAAI,0BAAO;AAAA,YAC5B,QAAQ,KAAK;AAAA,YACb,QAAQ;AAAA,cACN,QAAQ,KAAK,YAAY,KAAK;AAAA,cAC9B,KAAK,KAAK,eAAe,OAAO,UAAU;AAAA,cAC1C,aAAa;AAAA,cACb,MAAM;AAAA,YACR;AAAA;AAAA;AAAA;AAAA,YAMA,UAAU,OAAO,OAAO;AAAA,YACxB,mBAAmB;AAAA,UACrB,CAAC;AAED,gBAAM,SAAS,MAAM,WAAW,KAAK;AACrC,iBAAO;AAAA,YACL,MAAM,OAAO,QAAQ;AAAA,YACrB,WAAW,OAAO,aAAa;AAAA,UACjC;AAAA,QACF;AAAA,QACA,EAAE,MAAM,YAAY,MAAM;AAAA,MAC5B;AAAA,IACF;AAAA,EACF;AAAA,EAGA,MAAM,KAAM,KAAqB,OAAqB,MAAiC;AACrF,UAAM,OAAO,MAAM,KAAK,MAAM,KAAK,OAAO,IAAI;AAC9C,UAAM,SAAmB,CAAC;AAE1B,UAAM,IAAI,QAAQ,CAAC,SAAS,WAAW;AACrC,WAAK,GAAG,QAAQ,CAAC,UAAU;AACzB,eAAO,KAAK,KAAK;AAAA,MACnB,CAAC;AAED,WAAK,GAAG,OAAO,MAAM;AACnB,aAAK,QAAQ;AACb,gBAAQ,IAAI;AAAA,MACd,CAAC;AACD,WAAK,GAAG,SAAS,CAAC,QAAQ;AACxB,aAAK,QAAQ;AACb,eAAO,GAAG;AAAA,MACZ,CAAC;AAAA,IACH,CAAC;AACD,WAAO;AAAA,EACT;AAAA,EAGA,MAAM,QACJ,KACA,OACA,YACA,QACA,QACmB;AACnB,UAAM,QAAQ,WAAW,SAAY,SAAS,MAAM,IAAI,SAAS,MAAM,KAAK,SAAS,MAAM;AAC3F,WAAO,MAAM,KAAK,MAAM,KAAK,OAAO,YAAY,KAAK;AAAA,EACvD;AAAA,EAGA,MAAM,OAAQ,KAAqB,OAAqB,YAAqC;AAC3F,UAAM,eAAW,6BAAY,mBAAAC,QAAW,SAAS,QAAQ,KAAK;AAC9D,WAAO,SAAS,WAAW,kBAAc,8BAAU,KAAK,CAAC,EAAE,WAAW,WAAW,UAAU;AAAA,EAC7F;AACF;AAlVQ;AAAA,MADL,yBAAY,MAAM;AAAA,GArDR,UAsDL;AA2FA;AAAA,MADL,yBAAY,QAAQ;AAAA,GAhJV,UAiJL;AAUA;AAAA,MADL,yBAAY,QAAQ;AAAA,GA1JV,UA2JL;AAmCA;AAAA,MADL,yBAAY,YAAY;AAAA,GA7Ld,UA8LL;AA6CA;AAAA,MADL,yBAAY,MAAM;AAAA,GA1OR,UA2OL;AA2BA;AAAA,MADL,yBAAY,KAAK;AAAA,GArQP,UAsQL;AA6BN;AAAA,MADC,yBAAY,KAAK;AAAA,GAlSP,UAmSX;AA+DM;AAAA,MADL,yBAAY,MAAM;AAAA,GAjWR,UAkWL;AAsBA;AAAA,MADL,yBAAY,SAAS;AAAA,GAvXX,UAwXL;AAYA;AAAA,MADL,yBAAY,QAAQ;AAAA,GAnYV,UAoYL;AAMD,SAAS,qBAAsB,eAAyD;AAC7F,QAAM,WAAW,QAAQ,IAAI;AAC7B,MAAI,aAAa,QAAW;AAC1B,WAAO;AAAA,EACT;AACA,QAAM,YAAY,QAAQ,IAAI;AAC9B,MAAI,cAAc,QAAW;AAC3B,WAAO;AAAA,EACT;AAEA,QAAM,YAAY,QAAQ,IAAI;AAC9B,MAAI,cAAc,QAAW;AAC3B,WAAO;AAAA,EACT;AAEA,QAAM,SAAmB;AAAA,IACvB,MAAM;AAAA,IACN,MAAM;AAAA,IACN,QAAQ;AAAA,IACR;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,gBAAc,SAAS,KAAK,MAAM;AAClC,gBAAc,UAAU;AAC1B;AAzBgB;",
+  "names": ["HttpAgent", "HttpsAgent", "core", "serverCore"]
+}

package/lib/perfTest.js

@@ -0,0 +1,91 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
+var import_core = require("@hanzo/core");
+var import__ = require(".");
+const MB = 1024 * 1024;
+const config = { default: "minio", storages: [] };
+const minioConfigVar = (0, import__.processConfigFromEnv)(config);
+if (minioConfigVar !== void 0 || config.storages[0] === void 0) {
+  console.error("No S3 config env is configured:" + minioConfigVar);
+  it.skip("No S3 config env is configured", async () => {
+  });
+  process.exit(1);
+}
+const toolCtx = new import_core.MeasureMetricsContext("test", {});
+const storageService = new import__.S3Service({ ...config.storages[0], rootBucket: "haiodo-test-bucket" });
+async function doTest() {
+  const existingTestBuckets = await storageService.listBuckets(toolCtx);
+  for (const b of existingTestBuckets) {
+    await b.delete();
+  }
+  const genWorkspaceId1 = (0, import_core.generateId)();
+  const wsIds1 = {
+    uuid: genWorkspaceId1,
+    dataId: genWorkspaceId1,
+    url: ""
+  };
+  await storageService.make(toolCtx, wsIds1);
+  let st1 = Date.now();
+  const sz = 10;
+  const stream = Buffer.alloc(sz * 1024 * 1024);
+  for (let i = 0; i < 10; i++) {
+    const st = Date.now();
+    await storageService.put(toolCtx, wsIds1, `testObject.${i}`, stream, "application/octet-stream", stream.length);
+    console.log("upload time", Date.now() - st);
+  }
+  let now = Date.now();
+  console.log(`upload performance: ${Math.round(sz * 10 * 1e3 * 100 / (now - st1)) / 100} mb per second`);
+  st1 = Date.now();
+  for (let i = 0; i < 10; i++) {
+    const st = Date.now();
+    await storageService.read(toolCtx, wsIds1, `testObject.${i}`);
+    console.log("download time", Date.now() - st);
+  }
+  now = Date.now();
+  console.log(`download performance: ${Math.round(sz * 10 * 1e3 * 100 / (now - st1)) / 100} mb per second`);
+  st1 = Date.now();
+  for (let i = 0; i < 10; i++) {
+    const st = Date.now();
+    const readable = await storageService.get(toolCtx, wsIds1, `testObject.${i}`);
+    const chunks = [];
+    readable.on("data", (chunk) => {
+      chunks.push(chunk);
+    });
+    await new Promise((resolve) => {
+      readable.on("end", () => {
+        resolve();
+        readable.destroy();
+      });
+    });
+    console.log("download time 2", Date.now() - st);
+  }
+  now = Date.now();
+  console.log(`download performance: ${Math.round(sz * 10 * 1e3 * 100 / (now - st1)) / 100} mb per second`);
+  st1 = Date.now();
+  for (let i = 0; i < 10; i++) {
+    const st = Date.now();
+    for (let i2 = 0; i2 < sz; i2++) {
+      const readable = await storageService.partial(toolCtx, wsIds1, `testObject.${i2}`, i2 * MB, MB);
+      const chunks = [];
+      readable.on("data", (chunk) => {
+        chunks.push(chunk);
+      });
+      await new Promise((resolve) => {
+        readable.on("end", () => {
+          resolve();
+          readable.destroy();
+        });
+      });
+    }
+    console.log("download time 2", Date.now() - st);
+  }
+  now = Date.now();
+  console.log(`download performance: ${Math.round(sz * 10 * 1e3 * 100 / (now - st1)) / 100} mb per second`);
+}
+__name(doTest, "doTest");
+void doTest().catch((err) => {
+  console.error(err);
+});
+console.log("done");
+//# sourceMappingURL=perfTest.js.map

package/lib/perfTest.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/perfTest.ts"],
+  "sourcesContent": ["import { MeasureMetricsContext, type WorkspaceDataId, type WorkspaceUuid, generateId } from '@hanzo/core'\nimport type { StorageConfiguration } from '@hanzo/server-core'\nimport { S3Service, processConfigFromEnv, type S3Config } from '.'\n\nconst MB = 1024 * 1024\n\nconst config: StorageConfiguration = { default: 'minio', storages: [] }\nconst minioConfigVar = processConfigFromEnv(config)\nif (minioConfigVar !== undefined || config.storages[0] === undefined) {\n  console.error('No S3 config env is configured:' + minioConfigVar)\n  it.skip('No S3 config env is configured', async () => {})\n  process.exit(1)\n}\nconst toolCtx = new MeasureMetricsContext('test', {})\nconst storageService = new S3Service({ ...(config.storages[0] as S3Config), rootBucket: 'haiodo-test-bucket' })\n\nasync function doTest (): Promise<void> {\n  const existingTestBuckets = await storageService.listBuckets(toolCtx)\n  // Delete old buckets\n  for (const b of existingTestBuckets) {\n    await b.delete()\n  }\n\n  const genWorkspaceId1 = generateId() as unknown as WorkspaceDataId\n\n  const wsIds1 = {\n    uuid: genWorkspaceId1 as unknown as WorkspaceUuid,\n    dataId: genWorkspaceId1,\n    url: ''\n  }\n  await storageService.make(toolCtx, wsIds1)\n  /// /////// Uploads\n  let st1 = Date.now()\n  const sz = 10\n  const stream = Buffer.alloc(sz * 1024 * 1024)\n  for (let i = 0; i < 10; i++) {\n    // We need 1Mb random file to check upload speed.\n    const st = Date.now()\n    await storageService.put(toolCtx, wsIds1, `testObject.${i}`, stream, 'application/octet-stream', stream.length)\n    console.log('upload time', Date.now() - st)\n  }\n  let now = Date.now()\n  console.log(`upload performance: ${Math.round((sz * 10 * 1000 * 100) / (now - st1)) / 100} mb per second`)\n\n  /// // Downloads 1\n  st1 = Date.now()\n  for (let i = 0; i < 10; i++) {\n    // We need 1Mb random file to check upload speed.\n    const st = Date.now()\n    await storageService.read(toolCtx, wsIds1, `testObject.${i}`)\n    console.log('download time', Date.now() - st)\n  }\n\n  now = Date.now()\n  console.log(`download performance: ${Math.round((sz * 10 * 1000 * 100) / (now - st1)) / 100} mb per second`)\n\n  /// Downloads 2\n  st1 = Date.now()\n  for (let i = 0; i < 10; i++) {\n    // We need 1Mb random file to check upload speed.\n    const st = Date.now()\n    const readable = await storageService.get(toolCtx, wsIds1, `testObject.${i}`)\n    const chunks: Buffer[] = []\n    readable.on('data', (chunk) => {\n      chunks.push(chunk)\n    })\n    await new Promise<void>((resolve) => {\n      readable.on('end', () => {\n        resolve()\n        readable.destroy()\n      })\n    })\n    console.log('download time 2', Date.now() - st)\n  }\n\n  now = Date.now()\n  console.log(`download performance: ${Math.round((sz * 10 * 1000 * 100) / (now - st1)) / 100} mb per second`)\n\n  /// Downloads 3\n  st1 = Date.now()\n  for (let i = 0; i < 10; i++) {\n    // We need 1Mb random file to check upload speed.\n    const st = Date.now()\n    for (let i = 0; i < sz; i++) {\n      const readable = await storageService.partial(toolCtx, wsIds1, `testObject.${i}`, i * MB, MB)\n      const chunks: Buffer[] = []\n      readable.on('data', (chunk) => {\n        chunks.push(chunk)\n      })\n      await new Promise<void>((resolve) => {\n        readable.on('end', () => {\n          resolve()\n          readable.destroy()\n        })\n      })\n    }\n    console.log('download time 2', Date.now() - st)\n  }\n\n  now = Date.now()\n  console.log(`download performance: ${Math.round((sz * 10 * 1000 * 100) / (now - st1)) / 100} mb per second`)\n}\nvoid doTest().catch((err) => {\n  console.error(err)\n})\nconsole.log('done')\n"],
+  "mappings": ";;;AAAA,kBAA4F;AAE5F,eAA+D;AAE/D,MAAM,KAAK,OAAO;AAElB,MAAM,SAA+B,EAAE,SAAS,SAAS,UAAU,CAAC,EAAE;AACtE,MAAM,qBAAiB,+BAAqB,MAAM;AAClD,IAAI,mBAAmB,UAAa,OAAO,SAAS,CAAC,MAAM,QAAW;AACpE,UAAQ,MAAM,oCAAoC,cAAc;AAChE,KAAG,KAAK,kCAAkC,YAAY;AAAA,EAAC,CAAC;AACxD,UAAQ,KAAK,CAAC;AAChB;AACA,MAAM,UAAU,IAAI,kCAAsB,QAAQ,CAAC,CAAC;AACpD,MAAM,iBAAiB,IAAI,mBAAU,EAAE,GAAI,OAAO,SAAS,CAAC,GAAgB,YAAY,qBAAqB,CAAC;AAE9G,eAAe,SAAyB;AACtC,QAAM,sBAAsB,MAAM,eAAe,YAAY,OAAO;AAEpE,aAAW,KAAK,qBAAqB;AACnC,UAAM,EAAE,OAAO;AAAA,EACjB;AAEA,QAAM,sBAAkB,wBAAW;AAEnC,QAAM,SAAS;AAAA,IACb,MAAM;AAAA,IACN,QAAQ;AAAA,IACR,KAAK;AAAA,EACP;AACA,QAAM,eAAe,KAAK,SAAS,MAAM;AAEzC,MAAI,MAAM,KAAK,IAAI;AACnB,QAAM,KAAK;AACX,QAAM,SAAS,OAAO,MAAM,KAAK,OAAO,IAAI;AAC5C,WAAS,IAAI,GAAG,IAAI,IAAI,KAAK;AAE3B,UAAM,KAAK,KAAK,IAAI;AACpB,UAAM,eAAe,IAAI,SAAS,QAAQ,cAAc,CAAC,IAAI,QAAQ,4BAA4B,OAAO,MAAM;AAC9G,YAAQ,IAAI,eAAe,KAAK,IAAI,IAAI,EAAE;AAAA,EAC5C;AACA,MAAI,MAAM,KAAK,IAAI;AACnB,UAAQ,IAAI,uBAAuB,KAAK,MAAO,KAAK,KAAK,MAAO,OAAQ,MAAM,IAAI,IAAI,GAAG,gBAAgB;AAGzG,QAAM,KAAK,IAAI;AACf,WAAS,IAAI,GAAG,IAAI,IAAI,KAAK;AAE3B,UAAM,KAAK,KAAK,IAAI;AACpB,UAAM,eAAe,KAAK,SAAS,QAAQ,cAAc,CAAC,EAAE;AAC5D,YAAQ,IAAI,iBAAiB,KAAK,IAAI,IAAI,EAAE;AAAA,EAC9C;AAEA,QAAM,KAAK,IAAI;AACf,UAAQ,IAAI,yBAAyB,KAAK,MAAO,KAAK,KAAK,MAAO,OAAQ,MAAM,IAAI,IAAI,GAAG,gBAAgB;AAG3G,QAAM,KAAK,IAAI;AACf,WAAS,IAAI,GAAG,IAAI,IAAI,KAAK;AAE3B,UAAM,KAAK,KAAK,IAAI;AACpB,UAAM,WAAW,MAAM,eAAe,IAAI,SAAS,QAAQ,cAAc,CAAC,EAAE;AAC5E,UAAM,SAAmB,CAAC;AAC1B,aAAS,GAAG,QAAQ,CAAC,UAAU;AAC7B,aAAO,KAAK,KAAK;AAAA,IACnB,CAAC;AACD,UAAM,IAAI,QAAc,CAAC,YAAY;AACnC,eAAS,GAAG,OAAO,MAAM;AACvB,gBAAQ;AACR,iBAAS,QAAQ;AAAA,MACnB,CAAC;AAAA,IACH,CAAC;AACD,YAAQ,IAAI,mBAAmB,KAAK,IAAI,IAAI,EAAE;AAAA,EAChD;AAEA,QAAM,KAAK,IAAI;AACf,UAAQ,IAAI,yBAAyB,KAAK,MAAO,KAAK,KAAK,MAAO,OAAQ,MAAM,IAAI,IAAI,GAAG,gBAAgB;AAG3G,QAAM,KAAK,IAAI;AACf,WAAS,IAAI,GAAG,IAAI,IAAI,KAAK;AAE3B,UAAM,KAAK,KAAK,IAAI;AACpB,aAASA,KAAI,GAAGA,KAAI,IAAIA,MAAK;AAC3B,YAAM,WAAW,MAAM,eAAe,QAAQ,SAAS,QAAQ,cAAcA,EAAC,IAAIA,KAAI,IAAI,EAAE;AAC5F,YAAM,SAAmB,CAAC;AAC1B,eAAS,GAAG,QAAQ,CAAC,UAAU;AAC7B,eAAO,KAAK,KAAK;AAAA,MACnB,CAAC;AACD,YAAM,IAAI,QAAc,CAAC,YAAY;AACnC,iBAAS,GAAG,OAAO,MAAM;AACvB,kBAAQ;AACR,mBAAS,QAAQ;AAAA,QACnB,CAAC;AAAA,MACH,CAAC;AAAA,IACH;AACA,YAAQ,IAAI,mBAAmB,KAAK,IAAI,IAAI,EAAE;AAAA,EAChD;AAEA,QAAM,KAAK,IAAI;AACf,UAAQ,IAAI,yBAAyB,KAAK,MAAO,KAAK,KAAK,MAAO,OAAQ,MAAM,IAAI,IAAI,GAAG,gBAAgB;AAC7G;AArFe;AAsFf,KAAK,OAAO,EAAE,MAAM,CAAC,QAAQ;AAC3B,UAAQ,MAAM,GAAG;AACnB,CAAC;AACD,QAAQ,IAAI,MAAM;",
+  "names": ["i"]
+}

package/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "@hanzo/s3",
+  "version": "0.6.3",
+  "main": "lib/index.js",
+  "svelte": "src/index.ts",
+  "types": "types/index.d.ts",
+  "author": "Hanzo <dev@hanzo.ai>",
+  "template": "@hanzo/node-package",
+  "license": "EPL-2.0",
+  "scripts": {
+    "build": "compile",
+    "build:watch": "compile",
+    "test": "jest --passWithNoTests --silent --forceExit",
+    "format": "format src",
+    "_phase:build": "compile transpile src",
+    "_phase:test": "jest --passWithNoTests --silent --forceExit",
+    "_phase:format": "format src",
+    "_phase:validate": "compile validate"
+  },
+  "devDependencies": {
+    "@hanzo/platform-rig": "^0.6.0",
+    "@typescript-eslint/eslint-plugin": "^6.11.0",
+    "eslint-plugin-import": "^2.26.0",
+    "eslint-plugin-promise": "^6.1.1",
+    "eslint-plugin-n": "^15.4.0",
+    "eslint": "^8.54.0",
+    "@typescript-eslint/parser": "^6.11.0",
+    "eslint-config-standard-with-typescript": "^40.0.0",
+    "prettier": "^3.1.0",
+    "typescript": "^5.3.3",
+    "@types/node": "~20.11.16",
+    "jest": "^29.7.0",
+    "ts-jest": "^29.1.1",
+    "@types/jest": "^29.5.5"
+  },
+  "dependencies": {
+    "@hanzo/core": "^0.6.37",
+    "@hanzo/platform": "^0.6.15",
+    "@hanzo/server-core": "^0.6.4",
+    "@hanzo/storage": "^0.6.3",
+    "@aws-sdk/client-s3": "^3.738.0",
+    "@aws-sdk/s3-request-presigner": "^3.738.0",
+    "@aws-sdk/lib-storage": "^3.738.0",
+    "@smithy/node-http-handler": "^4.0.2"
+  }
+}