@super-protocol/sdk-js 0.15.0 → 0.15.2-beta.0

package/build/providers/storage/S3StorageProvider.js

@@ -8,33 +8,6 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-var __generator = (this && this.__generator) || function (thisArg, body) {
-    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
-    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
-    function verb(n) { return function (v) { return step([n, v]); }; }
-    function step(op) {
-        if (f) throw new TypeError("Generator is already executing.");
-        while (g && (g = 0, op[0] && (_ = 0)), _) try {
-            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
-            if (y = 0, t) op = [op[0] & 2, t.value];
-            switch (op[0]) {
-                case 0: case 1: t = op; break;
-                case 4: _.label++; return { value: op[1], done: false };
-                case 5: _.label++; y = op[1]; op = [0]; continue;
-                case 7: op = _.ops.pop(); _.trys.pop(); continue;
-                default:
-                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
-                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
-                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
-                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
-                    if (t[2]) _.ops.pop();
-                    _.trys.pop(); continue;
-            }
-            op = body.call(thisArg, _);
-        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
-        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
-    }
-};
 var __asyncValues = (this && this.__asyncValues) || function (o) {
     if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
     var m = o[Symbol.asyncIterator], i;
@@ -44,11 +17,11 @@ var __asyncValues = (this && this.__asyncValues) || function (o) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.S3StorageProvider = void 0;
-var aws_sdk_1 = require("aws-sdk");
-var S3StorageProvider = /** @class */ (function () {
-    function S3StorageProvider(storageAccess) {
+const aws_sdk_1 = require("aws-sdk");
+class S3StorageProvider {
+    constructor(storageAccess) {
         this.multipartChunkSizeInBytes = 64 * 1024 * 1024; // 64MB
-        var accessKeyId = storageAccess.accessKeyId, secretAccessKey = storageAccess.secretAccessKey, endpoint = storageAccess.endpoint, bucket = storageAccess.bucket;
+        const { accessKeyId, secretAccessKey, endpoint, bucket } = storageAccess;
         if (!accessKeyId)
             throw new Error("Access key id is undefined");
         if (!secretAccessKey)
@@ -60,89 +33,62 @@ var S3StorageProvider = /** @class */ (function () {
         this.bucket = bucket;
         this.s3Client = new aws_sdk_1.S3({
             credentials: {
-                accessKeyId: accessKeyId,
-                secretAccessKey: secretAccessKey,
+                accessKeyId,
+                secretAccessKey,
             },
-            endpoint: endpoint,
+            endpoint,
             s3ForcePathStyle: true,
             signatureVersion: "v4",
             httpOptions: { timeout: 0 },
         });
     }
-    S3StorageProvider.prototype.uploadFile = function (inputStream, remotePath, contentLength, progressListener) {
-        return __awaiter(this, void 0, void 0, function () {
-            var result;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        // For performance & cost optimization
-                        // https://docs.storj.io/dcs/api-reference/s3-compatible-gateway/multipart-upload/multipart-part-size
-                        if (inputStream.readableHighWaterMark >= this.multipartChunkSizeInBytes) {
-                            return [2 /*return*/, this.multipartUpload(inputStream, remotePath, contentLength, progressListener)];
-                        }
-                        return [4 /*yield*/, this.s3Client
-                                .putObject({
-                                Body: inputStream,
-                                Bucket: this.bucket,
-                                Key: remotePath,
-                                ContentLength: contentLength,
-                            })
-                                .on("httpUploadProgress", function (_a) {
-                                var total = _a.total, loaded = _a.loaded;
-                                return progressListener === null || progressListener === void 0 ? void 0 : progressListener(total, loaded);
-                            })
-                                .promise()];
-                    case 1:
-                        result = _a.sent();
-                        if (result.$response.error) {
-                            throw result.$response.error;
-                        }
-                        return [2 /*return*/];
-                }
-            });
+    uploadFile(inputStream, remotePath, contentLength, progressListener) {
+        return __awaiter(this, void 0, void 0, function* () {
+            // For performance & cost optimization
+            // https://docs.storj.io/dcs/api-reference/s3-compatible-gateway/multipart-upload/multipart-part-size
+            if (inputStream.readableHighWaterMark >= this.multipartChunkSizeInBytes) {
+                return this.multipartUpload(inputStream, remotePath, contentLength, progressListener);
+            }
+            const result = yield this.s3Client
+                .putObject({
+                Body: inputStream,
+                Bucket: this.bucket,
+                Key: remotePath,
+                ContentLength: contentLength,
+            })
+                .on("httpUploadProgress", ({ total, loaded }) => progressListener === null || progressListener === void 0 ? void 0 : progressListener(total, loaded))
+                .promise();
+            if (result.$response.error) {
+                throw result.$response.error;
+            }
         });
-    };
-    S3StorageProvider.prototype.multipartUpload = function (inputStream, remotePath, contentLength, progressListener) {
+    }
+    multipartUpload(inputStream, remotePath, contentLength, progressListener) {
         var _a, inputStream_1, inputStream_1_1;
         var _b, e_1, _c, _d;
-        return __awaiter(this, void 0, void 0, function () {
-            var multipart, totalWritten, partNumber, uploadId, parts, buffer, response, e_1_1, doneParams, uploadingError_1;
-            return __generator(this, function (_e) {
-                switch (_e.label) {
-                    case 0: return [4 /*yield*/, this.s3Client
-                            .createMultipartUpload({
-                            Bucket: this.bucket,
-                            Key: remotePath,
-                        })
-                            .promise()];
-                    case 1:
-                        multipart = _e.sent();
-                        if (!multipart.UploadId) {
-                            throw new Error("UploadId property is empty");
-                        }
-                        _e.label = 2;
-                    case 2:
-                        _e.trys.push([2, 19, , 21]);
-                        totalWritten = 0;
-                        partNumber = 0;
-                        uploadId = multipart.UploadId;
-                        parts = [];
-                        _e.label = 3;
-                    case 3:
-                        _e.trys.push([3, 11, 12, 17]);
-                        _a = true, inputStream_1 = __asyncValues(inputStream);
-                        _e.label = 4;
-                    case 4: return [4 /*yield*/, inputStream_1.next()];
-                    case 5:
-                        if (!(inputStream_1_1 = _e.sent(), _b = inputStream_1_1.done, !_b)) return [3 /*break*/, 10];
+        return __awaiter(this, void 0, void 0, function* () {
+            const multipart = yield this.s3Client
+                .createMultipartUpload({
+                Bucket: this.bucket,
+                Key: remotePath,
+            })
+                .promise();
+            if (!multipart.UploadId) {
+                throw new Error("UploadId property is empty");
+            }
+            try {
+                let totalWritten = 0;
+                let partNumber = 0;
+                const uploadId = multipart.UploadId;
+                const parts = [];
+                try {
+                    for (_a = true, inputStream_1 = __asyncValues(inputStream); inputStream_1_1 = yield inputStream_1.next(), _b = inputStream_1_1.done, !_b;) {
                         _d = inputStream_1_1.value;
                         _a = false;
-                        _e.label = 6;
-                    case 6:
-                        _e.trys.push([6, , 8, 9]);
-                        buffer = _d;
-                        partNumber++;
-                        return [4 /*yield*/, this.s3Client
+                        try {
+                            const buffer = _d;
+                            partNumber++;
+                            const response = yield this.s3Client
                                 .uploadPart({
                                 Body: buffer,
                                 Bucket: this.bucket,
@@ -150,208 +96,137 @@ var S3StorageProvider = /** @class */ (function () {
                                 UploadId: uploadId,
                                 PartNumber: partNumber,
                             })
-                                .promise()];
-                    case 7:
-                        response = _e.sent();
-                        parts.push({
-                            ETag: response.ETag,
-                            PartNumber: partNumber,
-                        });
-                        totalWritten += buffer.length;
-                        if (!!progressListener) {
-                            progressListener(contentLength, totalWritten);
+                                .promise();
+                            parts.push({
+                                ETag: response.ETag,
+                                PartNumber: partNumber,
+                            });
+                            totalWritten += buffer.length;
+                            if (!!progressListener) {
+                                progressListener(contentLength, totalWritten);
+                            }
                         }
-                        return [3 /*break*/, 9];
-                    case 8:
-                        _a = true;
-                        return [7 /*endfinally*/];
-                    case 9: return [3 /*break*/, 4];
-                    case 10: return [3 /*break*/, 17];
-                    case 11:
-                        e_1_1 = _e.sent();
-                        e_1 = { error: e_1_1 };
-                        return [3 /*break*/, 17];
-                    case 12:
-                        _e.trys.push([12, , 15, 16]);
-                        if (!(!_a && !_b && (_c = inputStream_1.return))) return [3 /*break*/, 14];
-                        return [4 /*yield*/, _c.call(inputStream_1)];
-                    case 13:
-                        _e.sent();
-                        _e.label = 14;
-                    case 14: return [3 /*break*/, 16];
-                    case 15:
-                        if (e_1) throw e_1.error;
-                        return [7 /*endfinally*/];
-                    case 16: return [7 /*endfinally*/];
-                    case 17:
-                        doneParams = {
-                            Bucket: this.bucket,
-                            Key: remotePath,
-                            UploadId: multipart.UploadId,
-                            MultipartUpload: { Parts: parts },
-                        };
-                        return [4 /*yield*/, this.s3Client.completeMultipartUpload(doneParams).promise()];
-                    case 18:
-                        _e.sent();
-                        return [3 /*break*/, 21];
-                    case 19:
-                        uploadingError_1 = _e.sent();
-                        return [4 /*yield*/, this.s3Client
-                                .abortMultipartUpload({
-                                Bucket: this.bucket,
-                                Key: remotePath,
-                                UploadId: multipart.UploadId,
-                            })
-                                .promise()];
-                    case 20:
-                        _e.sent();
-                        throw uploadingError_1;
-                    case 21: return [2 /*return*/];
-                }
-            });
-        });
-    };
-    S3StorageProvider.prototype.deleteObject = function (remotePath) {
-        return __awaiter(this, void 0, void 0, function () {
-            var deleteResult;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0: return [4 /*yield*/, this.s3Client
-                            .deleteObject({
-                            Bucket: this.bucket,
-                            Key: remotePath,
-                        })
-                            .promise()];
-                    case 1:
-                        deleteResult = _a.sent();
-                        if (deleteResult.$response.error) {
-                            throw deleteResult.$response.error;
+                        finally {
+                            _a = true;
                         }
-                        return [2 /*return*/];
+                    }
                 }
-            });
-        });
-    };
-    S3StorageProvider.prototype.downloadFile = function (remotePath, config, progressListener) {
-        return __awaiter(this, void 0, void 0, function () {
-            var getObjectParams, start, end, downloadStream, current, fileBytesSize_1, _a;
-            return __generator(this, function (_b) {
-                switch (_b.label) {
-                    case 0:
-                        getObjectParams = {
-                            Bucket: this.bucket,
-                            Key: remotePath,
-                        };
-                        if (config) {
-                            start = config.offset || 0;
-                            end = start + (config.length || 0);
-                            getObjectParams.Range = "bytes=".concat(start, "-").concat(end || "");
-                        }
-                        downloadStream = this.s3Client.getObject(getObjectParams).createReadStream();
-                        current = 0;
-                        if (!progressListener) return [3 /*break*/, 3];
-                        _a = config.length;
-                        if (_a) return [3 /*break*/, 2];
-                        return [4 /*yield*/, this.getObjectSize(remotePath)];
-                    case 1:
-                        _a = (_b.sent());
-                        _b.label = 2;
-                    case 2:
-                        fileBytesSize_1 = _a;
-                        downloadStream.on("data", function (chunk) {
-                            current += chunk.length;
-                            progressListener(fileBytesSize_1, current);
-                        });
-                        _b.label = 3;
-                    case 3: return [2 /*return*/, downloadStream];
+                catch (e_1_1) { e_1 = { error: e_1_1 }; }
+                finally {
+                    try {
+                        if (!_a && !_b && (_c = inputStream_1.return)) yield _c.call(inputStream_1);
+                    }
+                    finally { if (e_1) throw e_1.error; }
                 }
-            });
+                const doneParams = {
+                    Bucket: this.bucket,
+                    Key: remotePath,
+                    UploadId: multipart.UploadId,
+                    MultipartUpload: { Parts: parts },
+                };
+                yield this.s3Client.completeMultipartUpload(doneParams).promise();
+            }
+            catch (uploadingError) {
+                yield this.s3Client
+                    .abortMultipartUpload({
+                    Bucket: this.bucket,
+                    Key: remotePath,
+                    UploadId: multipart.UploadId,
+                })
+                    .promise();
+                throw uploadingError;
+            }
         });
-    };
-    S3StorageProvider.prototype.listObjects = function (remotePath) {
-        return __awaiter(this, void 0, void 0, function () {
-            var prefix, listObjects, result;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        prefix = remotePath.endsWith("/") ? remotePath : "".concat(remotePath, "/");
-                        return [4 /*yield*/, this.s3Client
-                                .listObjectsV2({
-                                Bucket: this.bucket,
-                                Prefix: prefix,
-                            })
-                                .promise()];
-                    case 1:
-                        listObjects = _a.sent();
-                        if (listObjects.$response.error) {
-                            throw listObjects.$response.error;
-                        }
-                        result = [];
-                        if (listObjects.Contents) {
-                            result = listObjects.Contents.map(function (object) { return ({
-                                name: object.Key || "",
-                                createdAt: object.LastModified || new Date(),
-                                size: object.Size || 0,
-                            }); });
-                        }
-                        return [2 /*return*/, result];
-                }
-            });
+    }
+    deleteObject(remotePath) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const deleteResult = yield this.s3Client
+                .deleteObject({
+                Bucket: this.bucket,
+                Key: remotePath,
+            })
+                .promise();
+            if (deleteResult.$response.error) {
+                throw deleteResult.$response.error;
+            }
         });
-    };
-    S3StorageProvider.prototype.getMetadata = function (remotePath) {
-        return __awaiter(this, void 0, void 0, function () {
-            var getObjectParams, metadata;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        getObjectParams = {
-                            Bucket: this.bucket,
-                            Key: remotePath,
-                        };
-                        return [4 /*yield*/, this.s3Client.headObject(getObjectParams).promise()];
-                    case 1:
-                        metadata = _a.sent();
-                        if (metadata.$response.error) {
-                            throw metadata.$response.error;
-                        }
-                        return [2 /*return*/, metadata];
-                }
-            });
+    }
+    downloadFile(remotePath, config, progressListener) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const getObjectParams = {
+                Bucket: this.bucket,
+                Key: remotePath,
+            };
+            if (config) {
+                const start = config.offset || 0;
+                const end = start + (config.length || 0);
+                getObjectParams.Range = `bytes=${start}-${end || ""}`;
+            }
+            const downloadStream = this.s3Client.getObject(getObjectParams).createReadStream();
+            let current = 0;
+            if (progressListener) {
+                const fileBytesSize = config.length || (yield this.getObjectSize(remotePath));
+                downloadStream.on("data", (chunk) => {
+                    current += chunk.length;
+                    progressListener(fileBytesSize, current);
+                });
+            }
+            return downloadStream;
         });
-    };
-    S3StorageProvider.prototype.getObjectSize = function (remotePath) {
-        return __awaiter(this, void 0, void 0, function () {
-            var metadata;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0: return [4 /*yield*/, this.getMetadata(remotePath)];
-                    case 1:
-                        metadata = _a.sent();
-                        if (metadata.ContentLength === undefined) {
-                            throw new Error("ContentLength property is empty");
-                        }
-                        return [2 /*return*/, metadata.ContentLength || 0];
-                }
-            });
+    }
+    listObjects(remotePath) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const prefix = remotePath.endsWith("/") ? remotePath : `${remotePath}/`;
+            const listObjects = yield this.s3Client
+                .listObjectsV2({
+                Bucket: this.bucket,
+                Prefix: prefix,
+            })
+                .promise();
+            if (listObjects.$response.error) {
+                throw listObjects.$response.error;
+            }
+            let result = [];
+            if (listObjects.Contents) {
+                result = listObjects.Contents.map((object) => ({
+                    name: object.Key || "",
+                    createdAt: object.LastModified || new Date(),
+                    size: object.Size || 0,
+                }));
+            }
+            return result;
         });
-    };
-    S3StorageProvider.prototype.getLastModified = function (remotePath) {
-        return __awaiter(this, void 0, void 0, function () {
-            var metadata;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0: return [4 /*yield*/, this.getMetadata(remotePath)];
-                    case 1:
-                        metadata = _a.sent();
-                        if (!metadata.LastModified) {
-                            throw new Error("LastModified property is empty");
-                        }
-                        return [2 /*return*/, metadata.LastModified];
-                }
-            });
+    }
+    getMetadata(remotePath) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const getObjectParams = {
+                Bucket: this.bucket,
+                Key: remotePath,
+            };
+            const metadata = yield this.s3Client.headObject(getObjectParams).promise();
+            if (metadata.$response.error) {
+                throw metadata.$response.error;
+            }
+            return metadata;
         });
-    };
-    return S3StorageProvider;
-}());
+    }
+    getObjectSize(remotePath) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const metadata = yield this.getMetadata(remotePath);
+            if (metadata.ContentLength === undefined) {
+                throw new Error("ContentLength property is empty");
+            }
+            return metadata.ContentLength || 0;
+        });
+    }
+    getLastModified(remotePath) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const metadata = yield this.getMetadata(remotePath);
+            if (!metadata.LastModified) {
+                throw new Error("LastModified property is empty");
+            }
+            return metadata.LastModified;
+        });
+    }
+}
 exports.S3StorageProvider = S3StorageProvider;

package/build/providers/storage/StorageAdapter.d.ts

@@ -0,0 +1,60 @@
+/// <reference types="node" />
+import StorageAccess from "../../types/storage/StorageAccess";
+import { Performance } from "./types";
+export interface LRUCacheConfig {
+    max: number;
+}
+export interface StorageAdapterConfig {
+    lruCache: LRUCacheConfig;
+    writeInterval: number;
+    readInterval: number;
+    objectDeletedFlag: string;
+    readMetadataConcurrency?: number;
+    performance?: Performance;
+    showLogs?: boolean;
+}
+export declare enum CacheEvents {
+    INSTANCES_CHANGED = "INSTANCES_CHANGED",
+    KEY_DELETED = "KEY_DELETED"
+}
+export default class StorageAdapter<V extends object> {
+    private readonly logger?;
+    private readonly storageKeyValueAdapter;
+    private readonly cache;
+    private readonly encryptionKeys;
+    private readonly contentWriter;
+    private readonly metadataReader;
+    private readonly instanceId;
+    private timeout;
+    private readonly readInterval;
+    private readonly queues;
+    private readonly queueReadMetadata;
+    private readonly isUpdating;
+    private readonly pubSub;
+    private readonly eventName;
+    private readonly performance?;
+    constructor(storageAccess: StorageAccess, config: StorageAdapterConfig);
+    private generateHash;
+    subscribe(cb: (props: {
+        type: CacheEvents;
+        message: unknown;
+    }) => void): Promise<() => Promise<void>>;
+    private publish;
+    has(key: string): Promise<boolean>;
+    private getEnryptionKey;
+    set(key: string, value: V, encryptionKeyBuffer: Buffer): Promise<void>;
+    private setByInstance;
+    delete(key: string): Promise<void>;
+    get(key: string, encryptionKeyBuffer: Buffer): Promise<(V | null)[] | null>;
+    private getQueue;
+    private clearQueue;
+    private cacheHasNullInstances;
+    private fetchNullValues;
+    clear(): void;
+    run(): void;
+    private checkUpdates;
+    private startUpdatesChecking;
+    private stopUpdatesChecking;
+    stop(): void;
+    shutdown(): Promise<void>;
+}