@spinajs/fs-s3 2.0.180 → 2.0.182

package/lib/cjs/index.js

@@ -1,345 +1,382 @@
-"use strict";
-var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
-    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
-    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
-    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
-    return c > 3 && r && Object.defineProperty(target, key, r), r;
-};
-var __metadata = (this && this.__metadata) || function (k, v) {
-    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.fsS3 = void 0;
-const di_1 = require("@spinajs/di");
-const log_common_1 = require("@spinajs/log-common");
-const fs_1 = require("@spinajs/fs");
-const client_s3_1 = require("@aws-sdk/client-s3");
-const lib_storage_1 = require("@aws-sdk/lib-storage");
-const configuration_1 = require("@spinajs/configuration");
-const archiver_1 = __importDefault(require("archiver"));
-const path_1 = require("path");
-const exceptions_1 = require("@spinajs/exceptions");
-const fs_2 = require("fs");
-const luxon_1 = require("luxon");
-const stream_1 = require("stream");
-/**
- * Abstract layer for file operations.
- * Basic implementation is just wrapper for native node fs functions
- *
- * It allows to wrap other libs eg. aws s3, ftp
- * and inject it into code without changing logic that use them.
- *
- * TODO: map errors to some kind of common errors shared with other implementations
- */
-let fsS3 = class fsS3 extends fs_1.fs {
-    /**
-     * Name of provider. We can have multiple providers of the same type but with different options.
-     * Also used in InjectService decorator for mapping
-     */
-    get Name() {
-        return this.Options.name;
-    }
-    constructor(Options) {
-        super();
-        this.Options = Options;
-    }
-    async resolve() {
-        this.S3 = new client_s3_1.S3Client(this.AwsConfig);
-    }
-    /**
-     *
-     * Tries to download file to local filesystem, then returns local filesystem path.
-     * Native implementation simply returns local path and does nothing.
-     *
-     * @param path - file to download
-     */
-    async download(path) {
-        const tmpName = this.TempFs.tmppath();
-        const wStream = await this.TempFs.writeStream(tmpName);
-        const command = new client_s3_1.GetObjectCommand({
-            Bucket: this.Options.bucket,
-            Key: path,
-        });
-        const result = await this.S3.send(command);
-        return new Promise((resolve, reject) => {
-            if (result.Body instanceof stream_1.Readable) {
-                result.Body
-                    .pipe(wStream)
-                    .on("error", (err) => reject(err))
-                    .on("close", () => resolve(tmpName));
-            }
-            else {
-                reject(new exceptions_1.IOFail(`Cannot download file ${path}, empty response`));
-            }
-        });
-    }
-    /**
-     * read all content of file
-     */
-    async read(path, encoding) {
-        const fLocal = await this.download(path);
-        const content = await this.TempFs.read(fLocal, encoding);
-        await this.TempFs.unlink(fLocal);
-        return content;
-    }
-    async readStream(path, encoding) {
-        const fLocal = await this.download(path);
-        return this.TempFs.readStream(fLocal, encoding);
-    }
-    /**
-     * Write to file string or buffer
-     */
-    async write(path, data, encoding) {
-        const upload = new lib_storage_1.Upload({
-            client: this.S3,
-            params: {
-                Bucket: this.Options.bucket,
-                Key: path,
-                Body: data,
-                ContentEncoding: encoding
-            },
-        });
-        await upload.done();
-    }
-    async append(path, data, encoding) {
-        /**
-         * We cannot append to file in s3 directly,
-         * we have to download file first, append locally, then upload again new file
-         */
-        const fLocal = await this.download(path);
-        await this.TempFs.append(fLocal, data, encoding);
-        const wStream = await this.writeStream(path, encoding);
-        const rStream = await this.TempFs.readStream(fLocal, encoding);
-        return new Promise((resolve, reject) => {
-            rStream
-                .pipe(wStream)
-                .on('end', () => {
-                this.TempFs.rm(fLocal)
-                    .then(() => {
-                    return resolve();
-                })
-                    .catch(() => {
-                    resolve();
-                });
-            })
-                .on('error', (err) => {
-                // eslint-disable-next-line promise/no-promise-in-callback
-                this.TempFs.rm(fLocal)
-                    .then(() => {
-                    return reject(err);
-                })
-                    .catch(() => {
-                    reject(err);
-                });
-            });
-        });
-    }
-    async upload(srcPath, destPath) {
-        if (!(0, fs_2.existsSync)(srcPath)) {
-            throw new exceptions_1.IOFail(`file ${srcPath} does not exists`);
-        }
-        const dPath = this.resolvePath(destPath ?? (0, path_1.basename)(srcPath));
-        const rStream = (0, fs_2.createReadStream)(srcPath);
-        await this.writeStream(dPath, rStream);
-    }
-    async writeStream(path, rStream, encoding) {
-        if (!(rStream instanceof fs_2.ReadStream)) {
-            throw new exceptions_1.InvalidArgument(`rStream should be readable stream`);
-        }
-        const result = new lib_storage_1.Upload({
-            client: this.S3,
-            params: {
-                Bucket: this.Options.bucket,
-                Key: path,
-                Body: rStream,
-                ContentEncoding: encoding,
-            },
-        });
-        await result.done();
-    }
-    /**
-     * Checks if file existst
-     * @param path - path to check
-     */
-    async exists(path) {
-        try {
-            const command = new client_s3_1.HeadObjectCommand({
-                Bucket: this.Options.bucket,
-                Key: path,
-            });
-            await this.S3.send(command);
-        }
-        catch (err) {
-            if (err.name === 'NotFound')
-                return false;
-        }
-        return true;
-    }
-    async dirExists() {
-        // s3 does not have concept of folders
-        // we assume that all exists
-        return Promise.resolve(true);
-    }
-    /**
-     * Copy file to another location
-     * @param path - src path
-     * @param dest - dest path
-     */
-    async copy(path, dest) {
-        const command = new client_s3_1.CopyObjectCommand({
-            Bucket: this.Options.bucket,
-            CopySource: this.Options.bucket + '/' + path,
-            Key: dest,
-        });
-        await this.S3.send(command);
-    }
-    /**
-     * Copy file to another location and deletes src file
-     */
-    async move(oldPath, newPath) {
-        await this.copy(oldPath, newPath);
-        await this.unlink(oldPath);
-    }
-    /**
-     * Change name of a file
-     */
-    async rename(oldPath, newPath) {
-        return this.move(oldPath, newPath);
-    }
-    /**
-     * Deletes file permanently
-     *
-     * @param path - path to file that will be deleted
-     * @param onlyTemp - remote filesystems need to download file before, if so, calling unlink with this flag removes only local temp file after we finished processing
-     */
-    async unlink(path, onlyTemp) {
-        if (onlyTemp) {
-            await this.TempFs.unlink(path);
-            return;
-        }
-        const command = new client_s3_1.DeleteObjectCommand({
-            Key: path,
-            Bucket: this.Options.bucket,
-        });
-        await this.S3.send(command);
-    }
-    /**
-     *
-     * Deletes dir recursively & all contents inside
-     *
-     * @param path - dir to remove
-     */
-    async rm(_path) {
-        const command = new client_s3_1.DeleteObjectCommand({
-            Bucket: this.Options.bucket,
-            Key: _path
-        });
-        await this.S3.send(command);
-    }
-    /**
-     *
-     * Creates directory, recursively
-     *
-     */
-    async mkdir() {
-        // s3 dont need to create folders
-    }
-    /**
-     * Returns file statistics, not all fields may be accesible
-     */
-    async stat(path) {
-        const command = new client_s3_1.HeadObjectCommand({
-            Bucket: this.Options.bucket,
-            Key: path
-        });
-        const result = await this.S3.send(command);
-        return {
-            // no directories in s3
-            IsDirectory: false,
-            // only files can be stored in s3
-            IsFile: true,
-            // no creation time
-            CreationTime: luxon_1.DateTime.min(),
-            ModifiedTime: luxon_1.DateTime.fromJSDate(result.LastModified),
-            // no access time in s3s
-            AccessTime: luxon_1.DateTime.min(),
-            Size: result.ContentLength,
-        };
-    }
-    // protected async getSignedUrl(path: string) {
-    //   return this.S3.getSignedUrlPromise('getObject', {
-    //     Bucket: this.Options.bucket,
-    //     Key: path,
-    //     Expires: 24 * 60 * 60,
-    //   });
-    // }
-    tmppath() {
-        throw new exceptions_1.MethodNotImplemented('fs s3 does not support temporary paths');
-    }
-    /**
-     * List content of directory
-     *
-     * @param path - path to directory
-     */
-    async list(path) {
-        const command = new client_s3_1.ListObjectsV2Command({
-            Bucket: this.Options.bucket,
-            Delimiter: '/',
-            Prefix: path,
-        });
-        const result = await this.S3.send(command);
-        return result.Contents.map((x) => x.Key);
-    }
-    async unzip(_path, _destPath) {
-        throw new Error('not implemented');
-    }
-    async zip(path, zName) {
-        const zTmpName = this.TempFs.tmppath();
-        const output = await this.TempFs.writeStream(zTmpName);
-        const archive = (0, archiver_1.default)('zip', {
-            zlib: { level: 9 }, // Sets the compression level.
-        });
-        // pipe archive data to the file
-        archive.pipe(output);
-        const tFile = await this.download(path);
-        archive.file(tFile, { name: zName ?? (0, path_1.basename)(path) });
-        await archive.finalize();
-        return {
-            asFilePath: () => {
-                return zTmpName;
-            },
-            asStream: (encoding) => {
-                return (0, fs_2.createReadStream)(`${zTmpName}`, encoding);
-            },
-            asBase64: () => {
-                return (0, fs_2.readFileSync)(`${zTmpName}`, 'base64');
-            },
-        };
-    }
-    // eslint-disable-next-line @typescript-eslint/no-empty-function
-    resolvePath(_path) {
-        throw new exceptions_1.MethodNotImplemented('fs s3 does not support path resolving');
-    }
-};
-__decorate([
-    (0, log_common_1.Logger)('fs'),
-    __metadata("design:type", log_common_1.Log)
-], fsS3.prototype, "Logger", void 0);
-__decorate([
-    (0, configuration_1.Config)('fs.s3.config'),
-    __metadata("design:type", Object)
-], fsS3.prototype, "AwsConfig", void 0);
-__decorate([
-    (0, fs_1.FileSystem)('fs-temp'),
-    __metadata("design:type", fs_1.fs)
-], fsS3.prototype, "TempFs", void 0);
-fsS3 = __decorate([
-    (0, di_1.Injectable)('fs'),
-    (0, di_1.PerInstanceCheck)(),
-    __metadata("design:paramtypes", [Object])
-], fsS3);
-exports.fsS3 = fsS3;
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.fsS3 = void 0;
+const di_1 = require("@spinajs/di");
+const log_common_1 = require("@spinajs/log-common");
+const fs_1 = require("@spinajs/fs");
+const client_s3_1 = require("@aws-sdk/client-s3");
+const lib_storage_1 = require("@aws-sdk/lib-storage");
+const configuration_1 = require("@spinajs/configuration");
+const path_1 = __importStar(require("path"));
+const exceptions_1 = require("@spinajs/exceptions");
+const fs_2 = require("fs");
+const luxon_1 = require("luxon");
+const stream_1 = require("stream");
+const iconv_lite_1 = __importDefault(require("iconv-lite"));
+/**
+ * Abstract layer for file operations.
+ * Basic implementation is just wrapper for native node fs functions
+ *
+ * It allows to wrap other libs eg. aws s3, ftp
+ * and inject it into code without changing logic that use them.
+ *
+ * TODO: map errors to some kind of common errors shared with other implementations
+ */
+let fsS3 = class fsS3 extends fs_1.fs {
+    /**
+     * Name of provider. We can have multiple providers of the same type but with different options.
+     * Also used in InjectService decorator for mapping
+     */
+    get Name() {
+        return this.Options.name;
+    }
+    constructor(Options) {
+        super();
+        this.Options = Options;
+    }
+    async resolve() {
+        this.S3 = new client_s3_1.S3Client(Object.assign({}, this.AwsConfig, {
+            endpoint: this.AwsConfig.endpoint ?? undefined,
+            logger: {
+                trace: (msg) => this.Logger.trace(msg),
+                debug: (msg) => this.Logger.debug(msg),
+                info: (msg) => this.Logger.info(msg),
+                warn: (msg) => this.Logger.warn(msg),
+                error: (msg) => this.Logger.error(msg),
+            },
+        }));
+    }
+    /**
+     *
+     * Tries to download file to local filesystem, then returns local filesystem path.
+     * Native implementation simply returns local path and does nothing.
+     *
+     * @param path - file to download
+     */
+    async download(path) {
+        const tmpName = this.TempFs.tmppath();
+        const wStream = await this.TempFs.writeStream(tmpName);
+        const command = new client_s3_1.GetObjectCommand({
+            Bucket: this.Options.bucket,
+            Key: path,
+        });
+        const result = await this.S3.send(command);
+        return new Promise((resolve, reject) => {
+            if (result.Body instanceof stream_1.Readable) {
+                result.Body.pipe(wStream)
+                    .on('error', (err) => reject(err))
+                    .on('close', () => resolve(tmpName));
+            }
+            else {
+                reject(new exceptions_1.IOFail(`Cannot download file ${path}, empty response`));
+            }
+        });
+    }
+    /**
+     * read all content of file
+     */
+    async read(path, encoding) {
+        const fLocal = await this.download(path);
+        const content = await this.TempFs.read(fLocal, encoding);
+        await this.TempFs.rm(fLocal);
+        return content;
+    }
+    /**
+     *
+     * @param path
+     * @param _encoding
+     * @returns
+     */
+    async readStream(path, encoding) {
+        const command = new client_s3_1.GetObjectCommand({
+            Bucket: this.Options.bucket,
+            Key: path,
+        });
+        const result = await this.S3.send(command);
+        const rStream = result.Body;
+        if (encoding) {
+            const encodedStream = rStream.pipe(iconv_lite_1.default.decodeStream(encoding));
+            return encodedStream;
+        }
+        return rStream;
+    }
+    /**
+     * Write to file string or buffer
+     */
+    async write(path, data, encoding) {
+        const upload = new lib_storage_1.Upload({
+            client: this.S3,
+            params: {
+                Bucket: this.Options.bucket,
+                Key: path,
+                Body: data,
+                ContentEncoding: encoding,
+            },
+        });
+        await upload.done();
+    }
+    /**
+     * NOTE: append on s3 downloads file, appends to it, then uploads it again
+     * so it can be slow on large files
+     *
+     * @param path
+     * @param data
+     * @param encoding
+     */
+    async append(path, data, encoding) {
+        /**
+         * We cannot append to file in s3 directly,
+         * we have to download file first, append locally, then upload again new file
+         */
+        const fLocal = await this.download(path);
+        await this.TempFs.append(fLocal, data, encoding);
+        await this.upload(fLocal, path);
+    }
+    async upload(srcPath, destPath) {
+        if (!(0, fs_2.existsSync)(srcPath)) {
+            throw new exceptions_1.IOFail(`file ${srcPath} does not exists`);
+        }
+        const dPath = destPath ?? (0, path_1.basename)(srcPath);
+        const rStream = (0, fs_2.createReadStream)(srcPath);
+        const hash = await this.hash(srcPath, 'md5');
+        const fInfo = await this.FileInfo.getInfo(this.resolvePath(srcPath));
+        const upload = new lib_storage_1.Upload({
+            client: this.S3,
+            params: {
+                Bucket: this.Options.bucket,
+                Key: dPath,
+                Body: rStream,
+                // content md5 header is always base64 encoded
+                ContentMD5: Buffer.from(hash, 'hex').toString('base64'),
+                // convert all metadata values to string, and back to object with key-value pair of strings
+                Metadata: Object.fromEntries(Object.entries(fInfo).map(([key, value]) => [key, String(value)])),
+            },
+        });
+        await upload.done();
+    }
+    /**
+     *
+     * Gets metadata of file in s3 bucket
+     *
+     * @param path path to file
+     * @returns
+     */
+    async getMetadata(path) {
+        const command = new client_s3_1.HeadObjectCommand({
+            Bucket: this.Options.bucket,
+            Key: path,
+        });
+        const result = await this.S3.send(command);
+        return result.Metadata;
+    }
+    /**
+     *
+     * Returns writable stream for given path
+     *
+     * @param path file path ( relative to base path of provider)
+     * @param rStream readable stream, must be provided beforehand
+     * @param encoding optional stream encoding
+     */
+    async writeStream(_path, _encoding) {
+        throw new exceptions_1.IOFail('Method not implemented, s3 does not support writable streams');
+    }
+    /**
+     * Checks if file existst
+     * @param path - path to check
+     */
+    async exists(path) {
+        try {
+            const command = new client_s3_1.HeadObjectCommand({
+                Bucket: this.Options.bucket,
+                Key: path,
+            });
+            await this.S3.send(command);
+        }
+        catch (err) {
+            if (err.name === 'NotFound')
+                return false;
+            throw err;
+        }
+        return true;
+    }
+    async dirExists() {
+        throw new exceptions_1.IOFail('Method not implemented, s3 does not support directories');
+    }
+    /**
+     * Copy file to another location
+     * @param path - src path
+     * @param dest - dest path
+     */
+    async copy(path, dest, dstFs) {
+        // if dest fs is set
+        // copy using it
+        if (dstFs) {
+            const file = await this.download(path);
+            await dstFs.upload(file, dest);
+        }
+        else {
+            // we copy file in s3 by copying it to another location
+            const command = new client_s3_1.CopyObjectCommand({
+                Bucket: this.Options.bucket,
+                CopySource: this.Options.bucket + '/' + path,
+                Key: dest,
+            });
+            await this.S3.send(command);
+        }
+    }
+    /**
+     * Copy file to another location and deletes src file
+     */
+    async move(oldPath, newPath, dstFs) {
+        await this.copy(oldPath, newPath, dstFs);
+        await this.rm(oldPath);
+    }
+    /**
+     * Change name of a file
+     */
+    async rename(oldPath, newPath) {
+        return this.move(oldPath, newPath);
+    }
+    /**
+     *
+     * Deletes dir recursively & all contents inside
+     *
+     * @param path - dir to remove
+     */
+    async rm(_path) {
+        const command = new client_s3_1.DeleteObjectCommand({
+            Bucket: this.Options.bucket,
+            Key: _path,
+        });
+        await this.S3.send(command);
+    }
+    /**
+     *
+     * Creates directory, recursively
+     *
+     */
+    async mkdir() {
+        throw new exceptions_1.IOFail('Method not implemented, s3 does not support directories');
+    }
+    async isDir(_path) {
+        throw new exceptions_1.IOFail('Method not implemented, s3 does not support directories');
+    }
+    /**
+     * Returns file statistics, not all fields may be accesible
+     */
+    async stat(path) {
+        const command = new client_s3_1.HeadObjectCommand({
+            Bucket: this.Options.bucket,
+            Key: path,
+        });
+        const result = await this.S3.send(command);
+        return {
+            // no directories in s3
+            IsDirectory: false,
+            // only files can be stored in s3
+            IsFile: true,
+            // no creation time
+            CreationTime: luxon_1.DateTime.min(),
+            ModifiedTime: luxon_1.DateTime.fromJSDate(result.LastModified),
+            // no access time in s3s
+            AccessTime: luxon_1.DateTime.min(),
+            Size: result.ContentLength,
+        };
+    }
+    // protected async getSignedUrl(path: string) {
+    //   return this.S3.getSignedUrlPromise('getObject', {
+    //     Bucket: this.Options.bucket,
+    //     Key: path,
+    //     Expires: 24 * 60 * 60,
+    //   });
+    // }
+    tmppath() {
+        throw new exceptions_1.MethodNotImplemented('fs s3 does not support temporary paths');
+    }
+    /**
+     * List content of directory
+     *
+     * @param path - path to directory
+     */
+    async list(path) {
+        const command = new client_s3_1.ListObjectsV2Command({
+            Bucket: this.Options.bucket,
+            Delimiter: '/',
+            Prefix: path,
+        });
+        const result = await this.S3.send(command);
+        return result.Contents.map((x) => x.Key);
+    }
+    async unzip(_path, _destPath) {
+        throw new exceptions_1.IOFail('Method not implemented, you should download zipped file first, then unzip it');
+    }
+    async zip(_path, _dstFs, _dstFile) {
+        throw new exceptions_1.IOFail('Method not implemented, you should zip files locally, then upload it');
+    }
+    resolvePath(_path) {
+        // we checek if path is absolute
+        // for hash function
+        if (path_1.default.isAbsolute(_path)) {
+            return _path;
+        }
+        throw new exceptions_1.MethodNotImplemented('fs s3 does not support path resolving');
+    }
+};
+exports.fsS3 = fsS3;
+__decorate([
+    (0, log_common_1.Logger)('fs'),
+    __metadata("design:type", log_common_1.Log)
+], fsS3.prototype, "Logger", void 0);
+__decorate([
+    (0, configuration_1.Config)('fs.s3.config'),
+    __metadata("design:type", Object)
+], fsS3.prototype, "AwsConfig", void 0);
+__decorate([
+    (0, fs_1.FileSystem)('fs-temp-s3'),
+    __metadata("design:type", fs_1.fs)
+], fsS3.prototype, "TempFs", void 0);
+__decorate([
+    (0, di_1.Autoinject)(),
+    __metadata("design:type", fs_1.FileInfoService)
+], fsS3.prototype, "FileInfo", void 0);
+exports.fsS3 = fsS3 = __decorate([
+    (0, di_1.Injectable)('fs'),
+    (0, di_1.PerInstanceCheck)(),
+    __metadata("design:paramtypes", [Object])
+], fsS3);
 //# sourceMappingURL=index.js.map