@spinajs/fs-s3 2.0.179 → 2.0.181

package/lib/mjs/index.js

@@ -1,339 +1,353 @@
-var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
-    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
-    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
-    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
-    return c > 3 && r && Object.defineProperty(target, key, r), r;
-};
-var __metadata = (this && this.__metadata) || function (k, v) {
-    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
-};
-import { Injectable, PerInstanceCheck } from '@spinajs/di';
-import { Log, Logger } from '@spinajs/log-common';
-import { fs, FileSystem } from '@spinajs/fs';
-import { S3Client, HeadObjectCommand, CopyObjectCommand, DeleteObjectCommand, ListObjectsV2Command, GetObjectCommand } from '@aws-sdk/client-s3';
-import { Upload } from '@aws-sdk/lib-storage';
-import { Config } from '@spinajs/configuration';
-import archiver from 'archiver';
-import { basename } from 'path';
-import { InvalidArgument, IOFail, MethodNotImplemented } from '@spinajs/exceptions';
-import { createReadStream, existsSync, readFileSync, ReadStream } from 'fs';
-import { DateTime } from 'luxon';
-import { Readable } from 'stream';
-/**
- * Abstract layer for file operations.
- * Basic implementation is just wrapper for native node fs functions
- *
- * It allows to wrap other libs eg. aws s3, ftp
- * and inject it into code without changing logic that use them.
- *
- * TODO: map errors to some kind of common errors shared with other implementations
- */
-let fsS3 = class fsS3 extends fs {
-    /**
-     * Name of provider. We can have multiple providers of the same type but with different options.
-     * Also used in InjectService decorator for mapping
-     */
-    get Name() {
-        return this.Options.name;
-    }
-    constructor(Options) {
-        super();
-        this.Options = Options;
-    }
-    async resolve() {
-        this.S3 = new S3Client(this.AwsConfig);
-    }
-    /**
-     *
-     * Tries to download file to local filesystem, then returns local filesystem path.
-     * Native implementation simply returns local path and does nothing.
-     *
-     * @param path - file to download
-     */
-    async download(path) {
-        const tmpName = this.TempFs.tmppath();
-        const wStream = await this.TempFs.writeStream(tmpName);
-        const command = new GetObjectCommand({
-            Bucket: this.Options.bucket,
-            Key: path,
-        });
-        const result = await this.S3.send(command);
-        return new Promise((resolve, reject) => {
-            if (result.Body instanceof Readable) {
-                result.Body
-                    .pipe(wStream)
-                    .on("error", (err) => reject(err))
-                    .on("close", () => resolve(tmpName));
-            }
-            else {
-                reject(new IOFail(`Cannot download file ${path}, empty response`));
-            }
-        });
-    }
-    /**
-     * read all content of file
-     */
-    async read(path, encoding) {
-        const fLocal = await this.download(path);
-        const content = await this.TempFs.read(fLocal, encoding);
-        await this.TempFs.unlink(fLocal);
-        return content;
-    }
-    async readStream(path, encoding) {
-        const fLocal = await this.download(path);
-        return this.TempFs.readStream(fLocal, encoding);
-    }
-    /**
-     * Write to file string or buffer
-     */
-    async write(path, data, encoding) {
-        const upload = new Upload({
-            client: this.S3,
-            params: {
-                Bucket: this.Options.bucket,
-                Key: path,
-                Body: data,
-                ContentEncoding: encoding
-            },
-        });
-        await upload.done();
-    }
-    async append(path, data, encoding) {
-        /**
-         * We cannot append to file in s3 directly,
-         * we have to download file first, append locally, then upload again new file
-         */
-        const fLocal = await this.download(path);
-        await this.TempFs.append(fLocal, data, encoding);
-        const wStream = await this.writeStream(path, encoding);
-        const rStream = await this.TempFs.readStream(fLocal, encoding);
-        return new Promise((resolve, reject) => {
-            rStream
-                .pipe(wStream)
-                .on('end', () => {
-                this.TempFs.rm(fLocal)
-                    .then(() => {
-                    return resolve();
-                })
-                    .catch(() => {
-                    resolve();
-                });
-            })
-                .on('error', (err) => {
-                // eslint-disable-next-line promise/no-promise-in-callback
-                this.TempFs.rm(fLocal)
-                    .then(() => {
-                    return reject(err);
-                })
-                    .catch(() => {
-                    reject(err);
-                });
-            });
-        });
-    }
-    async upload(srcPath, destPath) {
-        if (!existsSync(srcPath)) {
-            throw new IOFail(`file ${srcPath} does not exists`);
-        }
-        const dPath = this.resolvePath(destPath ?? basename(srcPath));
-        const rStream = createReadStream(srcPath);
-        await this.writeStream(dPath, rStream);
-    }
-    async writeStream(path, rStream, encoding) {
-        if (!(rStream instanceof ReadStream)) {
-            throw new InvalidArgument(`rStream should be readable stream`);
-        }
-        const result = new Upload({
-            client: this.S3,
-            params: {
-                Bucket: this.Options.bucket,
-                Key: path,
-                Body: rStream,
-                ContentEncoding: encoding,
-            },
-        });
-        await result.done();
-    }
-    /**
-     * Checks if file existst
-     * @param path - path to check
-     */
-    async exists(path) {
-        try {
-            const command = new HeadObjectCommand({
-                Bucket: this.Options.bucket,
-                Key: path,
-            });
-            await this.S3.send(command);
-        }
-        catch (err) {
-            if (err.name === 'NotFound')
-                return false;
-        }
-        return true;
-    }
-    async dirExists() {
-        // s3 does not have concept of folders
-        // we assume that all exists
-        return Promise.resolve(true);
-    }
-    /**
-     * Copy file to another location
-     * @param path - src path
-     * @param dest - dest path
-     */
-    async copy(path, dest) {
-        const command = new CopyObjectCommand({
-            Bucket: this.Options.bucket,
-            CopySource: this.Options.bucket + '/' + path,
-            Key: dest,
-        });
-        await this.S3.send(command);
-    }
-    /**
-     * Copy file to another location and deletes src file
-     */
-    async move(oldPath, newPath) {
-        await this.copy(oldPath, newPath);
-        await this.unlink(oldPath);
-    }
-    /**
-     * Change name of a file
-     */
-    async rename(oldPath, newPath) {
-        return this.move(oldPath, newPath);
-    }
-    /**
-     * Deletes file permanently
-     *
-     * @param path - path to file that will be deleted
-     * @param onlyTemp - remote filesystems need to download file before, if so, calling unlink with this flag removes only local temp file after we finished processing
-     */
-    async unlink(path, onlyTemp) {
-        if (onlyTemp) {
-            await this.TempFs.unlink(path);
-            return;
-        }
-        const command = new DeleteObjectCommand({
-            Key: path,
-            Bucket: this.Options.bucket,
-        });
-        await this.S3.send(command);
-    }
-    /**
-     *
-     * Deletes dir recursively & all contents inside
-     *
-     * @param path - dir to remove
-     */
-    async rm(_path) {
-        const command = new DeleteObjectCommand({
-            Bucket: this.Options.bucket,
-            Key: _path
-        });
-        await this.S3.send(command);
-    }
-    /**
-     *
-     * Creates directory, recursively
-     *
-     */
-    async mkdir() {
-        // s3 dont need to create folders
-    }
-    /**
-     * Returns file statistics, not all fields may be accesible
-     */
-    async stat(path) {
-        const command = new HeadObjectCommand({
-            Bucket: this.Options.bucket,
-            Key: path
-        });
-        const result = await this.S3.send(command);
-        return {
-            // no directories in s3
-            IsDirectory: false,
-            // only files can be stored in s3
-            IsFile: true,
-            // no creation time
-            CreationTime: DateTime.min(),
-            ModifiedTime: DateTime.fromJSDate(result.LastModified),
-            // no access time in s3s
-            AccessTime: DateTime.min(),
-            Size: result.ContentLength,
-        };
-    }
-    // protected async getSignedUrl(path: string) {
-    //   return this.S3.getSignedUrlPromise('getObject', {
-    //     Bucket: this.Options.bucket,
-    //     Key: path,
-    //     Expires: 24 * 60 * 60,
-    //   });
-    // }
-    tmppath() {
-        throw new MethodNotImplemented('fs s3 does not support temporary paths');
-    }
-    /**
-     * List content of directory
-     *
-     * @param path - path to directory
-     */
-    async list(path) {
-        const command = new ListObjectsV2Command({
-            Bucket: this.Options.bucket,
-            Delimiter: '/',
-            Prefix: path,
-        });
-        const result = await this.S3.send(command);
-        return result.Contents.map((x) => x.Key);
-    }
-    async unzip(_path, _destPath) {
-        throw new Error('not implemented');
-    }
-    async zip(path, zName) {
-        const zTmpName = this.TempFs.tmppath();
-        const output = await this.TempFs.writeStream(zTmpName);
-        const archive = archiver('zip', {
-            zlib: { level: 9 }, // Sets the compression level.
-        });
-        // pipe archive data to the file
-        archive.pipe(output);
-        const tFile = await this.download(path);
-        archive.file(tFile, { name: zName ?? basename(path) });
-        await archive.finalize();
-        return {
-            asFilePath: () => {
-                return zTmpName;
-            },
-            asStream: (encoding) => {
-                return createReadStream(`${zTmpName}`, encoding);
-            },
-            asBase64: () => {
-                return readFileSync(`${zTmpName}`, 'base64');
-            },
-        };
-    }
-    // eslint-disable-next-line @typescript-eslint/no-empty-function
-    resolvePath(_path) {
-        throw new MethodNotImplemented('fs s3 does not support path resolving');
-    }
-};
-__decorate([
-    Logger('fs'),
-    __metadata("design:type", Log)
-], fsS3.prototype, "Logger", void 0);
-__decorate([
-    Config('fs.s3.config'),
-    __metadata("design:type", Object)
-], fsS3.prototype, "AwsConfig", void 0);
-__decorate([
-    FileSystem('fs-temp'),
-    __metadata("design:type", fs)
-], fsS3.prototype, "TempFs", void 0);
-fsS3 = __decorate([
-    Injectable('fs'),
-    PerInstanceCheck(),
-    __metadata("design:paramtypes", [Object])
-], fsS3);
-export { fsS3 };
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+import { Autoinject, Injectable, PerInstanceCheck } from '@spinajs/di';
+import { Log, Logger } from '@spinajs/log-common';
+import { fs, FileSystem, FileInfoService } from '@spinajs/fs';
+import { S3Client, HeadObjectCommand, CopyObjectCommand, DeleteObjectCommand, ListObjectsV2Command, GetObjectCommand, } from '@aws-sdk/client-s3';
+import { Upload } from '@aws-sdk/lib-storage';
+import { Config } from '@spinajs/configuration';
+import path, { basename } from 'path';
+import { IOFail, MethodNotImplemented } from '@spinajs/exceptions';
+import { createReadStream, existsSync } from 'fs';
+import { DateTime } from 'luxon';
+import { Readable } from 'stream';
+import iconv from 'iconv-lite';
+/**
+ * Abstract layer for file operations.
+ * Basic implementation is just wrapper for native node fs functions
+ *
+ * It allows to wrap other libs eg. aws s3, ftp
+ * and inject it into code without changing logic that use them.
+ *
+ * TODO: map errors to some kind of common errors shared with other implementations
+ */
+let fsS3 = class fsS3 extends fs {
+    /**
+     * Name of provider. We can have multiple providers of the same type but with different options.
+     * Also used in InjectService decorator for mapping
+     */
+    get Name() {
+        return this.Options.name;
+    }
+    constructor(Options) {
+        super();
+        this.Options = Options;
+    }
+    async resolve() {
+        this.S3 = new S3Client(Object.assign({}, this.AwsConfig, {
+            endpoint: this.AwsConfig.endpoint ?? undefined,
+            logger: {
+                trace: (msg) => this.Logger.trace(msg),
+                debug: (msg) => this.Logger.debug(msg),
+                info: (msg) => this.Logger.info(msg),
+                warn: (msg) => this.Logger.warn(msg),
+                error: (msg) => this.Logger.error(msg),
+            },
+        }));
+    }
+    /**
+     *
+     * Tries to download file to local filesystem, then returns local filesystem path.
+     * Native implementation simply returns local path and does nothing.
+     *
+     * @param path - file to download
+     */
+    async download(path) {
+        const tmpName = this.TempFs.tmppath();
+        const wStream = await this.TempFs.writeStream(tmpName);
+        const command = new GetObjectCommand({
+            Bucket: this.Options.bucket,
+            Key: path,
+        });
+        const result = await this.S3.send(command);
+        return new Promise((resolve, reject) => {
+            if (result.Body instanceof Readable) {
+                result.Body.pipe(wStream)
+                    .on('error', (err) => reject(err))
+                    .on('close', () => resolve(tmpName));
+            }
+            else {
+                reject(new IOFail(`Cannot download file ${path}, empty response`));
+            }
+        });
+    }
+    /**
+     * read all content of file
+     */
+    async read(path, encoding) {
+        const fLocal = await this.download(path);
+        const content = await this.TempFs.read(fLocal, encoding);
+        await this.TempFs.rm(fLocal);
+        return content;
+    }
+    /**
+     *
+     * @param path
+     * @param _encoding
+     * @returns
+     */
+    async readStream(path, encoding) {
+        const command = new GetObjectCommand({
+            Bucket: this.Options.bucket,
+            Key: path,
+        });
+        const result = await this.S3.send(command);
+        const rStream = result.Body;
+        if (encoding) {
+            const encodedStream = rStream.pipe(iconv.decodeStream(encoding));
+            return encodedStream;
+        }
+        return rStream;
+    }
+    /**
+     * Write to file string or buffer
+     */
+    async write(path, data, encoding) {
+        const upload = new Upload({
+            client: this.S3,
+            params: {
+                Bucket: this.Options.bucket,
+                Key: path,
+                Body: data,
+                ContentEncoding: encoding,
+            },
+        });
+        await upload.done();
+    }
+    /**
+     * NOTE: append on s3 downloads file, appends to it, then uploads it again
+     * so it can be slow on large files
+     *
+     * @param path
+     * @param data
+     * @param encoding
+     */
+    async append(path, data, encoding) {
+        /**
+         * We cannot append to file in s3 directly,
+         * we have to download file first, append locally, then upload again new file
+         */
+        const fLocal = await this.download(path);
+        await this.TempFs.append(fLocal, data, encoding);
+        await this.upload(fLocal, path);
+    }
+    async upload(srcPath, destPath) {
+        if (!existsSync(srcPath)) {
+            throw new IOFail(`file ${srcPath} does not exists`);
+        }
+        const dPath = destPath ?? basename(srcPath);
+        const rStream = createReadStream(srcPath);
+        const hash = await this.hash(srcPath, 'md5');
+        const fInfo = await this.FileInfo.getInfo(this.resolvePath(srcPath));
+        const upload = new Upload({
+            client: this.S3,
+            params: {
+                Bucket: this.Options.bucket,
+                Key: dPath,
+                Body: rStream,
+                // content md5 header is always base64 encoded
+                ContentMD5: Buffer.from(hash, 'hex').toString('base64'),
+                // convert all metadata values to string, and back to object with key-value pair of strings
+                Metadata: Object.fromEntries(Object.entries(fInfo).map(([key, value]) => [key, String(value)])),
+            },
+        });
+        await upload.done();
+    }
+    /**
+     *
+     * Gets metadata of file in s3 bucket
+     *
+     * @param path path to file
+     * @returns
+     */
+    async getMetadata(path) {
+        const command = new HeadObjectCommand({
+            Bucket: this.Options.bucket,
+            Key: path,
+        });
+        const result = await this.S3.send(command);
+        return result.Metadata;
+    }
+    /**
+     *
+     * Returns writable stream for given path
+     *
+     * @param path file path ( relative to base path of provider)
+     * @param rStream readable stream, must be provided beforehand
+     * @param encoding optional stream encoding
+     */
+    async writeStream(_path, _encoding) {
+        throw new IOFail('Method not implemented, s3 does not support writable streams');
+    }
+    /**
+     * Checks if file existst
+     * @param path - path to check
+     */
+    async exists(path) {
+        try {
+            const command = new HeadObjectCommand({
+                Bucket: this.Options.bucket,
+                Key: path,
+            });
+            await this.S3.send(command);
+        }
+        catch (err) {
+            if (err.name === 'NotFound')
+                return false;
+            throw err;
+        }
+        return true;
+    }
+    async dirExists() {
+        throw new IOFail('Method not implemented, s3 does not support directories');
+    }
+    /**
+     * Copy file to another location
+     * @param path - src path
+     * @param dest - dest path
+     */
+    async copy(path, dest, dstFs) {
+        // if dest fs is set
+        // copy using it
+        if (dstFs) {
+            const file = await this.download(path);
+            await dstFs.upload(file, dest);
+        }
+        else {
+            // we copy file in s3 by copying it to another location
+            const command = new CopyObjectCommand({
+                Bucket: this.Options.bucket,
+                CopySource: this.Options.bucket + '/' + path,
+                Key: dest,
+            });
+            await this.S3.send(command);
+        }
+    }
+    /**
+     * Copy file to another location and deletes src file
+     */
+    async move(oldPath, newPath, dstFs) {
+        await this.copy(oldPath, newPath, dstFs);
+        await this.rm(oldPath);
+    }
+    /**
+     * Change name of a file
+     */
+    async rename(oldPath, newPath) {
+        return this.move(oldPath, newPath);
+    }
+    /**
+     *
+     * Deletes dir recursively & all contents inside
+     *
+     * @param path - dir to remove
+     */
+    async rm(_path) {
+        const command = new DeleteObjectCommand({
+            Bucket: this.Options.bucket,
+            Key: _path,
+        });
+        await this.S3.send(command);
+    }
+    /**
+     *
+     * Creates directory, recursively
+     *
+     */
+    async mkdir() {
+        throw new IOFail('Method not implemented, s3 does not support directories');
+    }
+    async isDir(_path) {
+        throw new IOFail('Method not implemented, s3 does not support directories');
+    }
+    /**
+     * Returns file statistics, not all fields may be accesible
+     */
+    async stat(path) {
+        const command = new HeadObjectCommand({
+            Bucket: this.Options.bucket,
+            Key: path,
+        });
+        const result = await this.S3.send(command);
+        return {
+            // no directories in s3
+            IsDirectory: false,
+            // only files can be stored in s3
+            IsFile: true,
+            // no creation time
+            CreationTime: DateTime.min(),
+            ModifiedTime: DateTime.fromJSDate(result.LastModified),
+            // no access time in s3s
+            AccessTime: DateTime.min(),
+            Size: result.ContentLength,
+        };
+    }
+    // protected async getSignedUrl(path: string) {
+    //   return this.S3.getSignedUrlPromise('getObject', {
+    //     Bucket: this.Options.bucket,
+    //     Key: path,
+    //     Expires: 24 * 60 * 60,
+    //   });
+    // }
+    tmppath() {
+        throw new MethodNotImplemented('fs s3 does not support temporary paths');
+    }
+    /**
+     * List content of directory
+     *
+     * @param path - path to directory
+     */
+    async list(path) {
+        const command = new ListObjectsV2Command({
+            Bucket: this.Options.bucket,
+            Delimiter: '/',
+            Prefix: path,
+        });
+        const result = await this.S3.send(command);
+        return result.Contents.map((x) => x.Key);
+    }
+    async unzip(_path, _destPath) {
+        throw new IOFail('Method not implemented, you should download zipped file first, then unzip it');
+    }
+    async zip(_path, _dstFs, _dstFile) {
+        throw new IOFail('Method not implemented, you should zip files locally, then upload it');
+    }
+    resolvePath(_path) {
+        // we checek if path is absolute
+        // for hash function
+        if (path.isAbsolute(_path)) {
+            return _path;
+        }
+        throw new MethodNotImplemented('fs s3 does not support path resolving');
+    }
+};
+__decorate([
+    Logger('fs'),
+    __metadata("design:type", Log)
+], fsS3.prototype, "Logger", void 0);
+__decorate([
+    Config('fs.s3.config'),
+    __metadata("design:type", Object)
+], fsS3.prototype, "AwsConfig", void 0);
+__decorate([
+    FileSystem('fs-temp-s3'),
+    __metadata("design:type", fs)
+], fsS3.prototype, "TempFs", void 0);
+__decorate([
+    Autoinject(),
+    __metadata("design:type", FileInfoService)
+], fsS3.prototype, "FileInfo", void 0);
+fsS3 = __decorate([
+    Injectable('fs'),
+    PerInstanceCheck(),
+    __metadata("design:paramtypes", [Object])
+], fsS3);
+export { fsS3 };
 //# sourceMappingURL=index.js.map