@flystorage/file-storage 0.0.2 → 0.0.4

package/README.md

@@ -0,0 +1,128 @@
+<img src="https://avatars.githubusercontent.com/u/151840999" width="50px" height="50px" />
+
+# Flystorage
+Flystorage is a file storage abstraction for NodeJS and TypeScript. It is an 80/20 solution
+that is built around a set of goals:
+
+- Provide a straight-forward API that is easy to use.
+- Allow application code to be unaware WHERE files are stored.
+- Pragmatically smooth over underlying storage differences.
+- Expose an async/await based API, promises all the way.
+- Abstract over file permissions using "visibility".
+- Actually tested using real integrations, _mocks are not welcome_.
+- Stand on the shoulders of giants, use official vendor packages when possible.
+
+### What is Flystorage NOT:
+Flystorage is meant to be used in cases for generic file storage use-cases. It's not an API for
+any  specific filesystem. It's a generalised solution and will not implement feature only
+specific to one particular storage implementation. There will be use-cases that are not catered
+to, simply because they cannot be abstracted over in a reasonable manner.
+
+## Capabilities
+
+### Implemented
+- [x] Write files using string | buffer | readable/stream
+- [x] Read files as stream, string, or Uint8Array
+- [x] Set permissions using abstracted visibility
+- [x] List the contents of a directory/prefix, (shallow and deep).
+- [x] Delete files without failing when they don't exist.
+- [x] Delete directories (and any files it contains)
+- [x] Generate public URLs.
+- [x] Generate temporary (signed) URLs.
+- [x] Expose or calculate checksums for files.
+- [x] Mime-type resolving
+- [x] Last modified fetching
+- [x] File size
+
+### Planned
+- [ ] Moving files
+- [ ] Copying files
+
+## Implementations / Adapters
+
+### Implemented
+- [x] Local Filesystem
+- [x] AWS S3 (using the V3 SDK)
+
+### Planned
+
+#### Prio 1
+- [ ] Azure Blob Storage
+- [ ] Test implementation (in-memory, with staged errors)
+- [ ] Google Cloud Storage
+
+### Prio 2
+- [ ] FTP (using `basic-ftp`)
+- [ ] SFTP (?)
+
+## Usage
+Install the main package and any adapters you might need:
+
+```bash
+npm i -S @flystorage/file-storage
+
+# for using AWS S3
+npm i -S @flystorage/aws-s3
+
+# for using the local filesystem
+npm i -S @flystorage/local-fs
+```
+
+## Local Usage
+```typescript
+import {resolve} from 'node:path';
+import {createReadStream} from 'node:fs';
+import {FileStorage, Visibility} from '@flystorage/file-storage';
+import {LocalFileStorage} from '@flystorage/local-fs';
+
+/**
+ * SETUP
+ **/
+
+const rootDirectory = resolve(process.cwd(), 'my-files');
+const storage = new FileStorage(new LocalFileStorage(rootDirectory));
+
+/**
+ * USAGE
+ **/
+
+// Write using a string
+await storage.write('write-from-a-string.txt', 'file contents');
+
+// Write using a stream
+const stream = createReadStream(resolve(process.cwd(), 'test-files/picture.png'));
+await storage.write('picture.png', stream);
+
+// Write with visibility (permissions).
+await storage.write('public.txt', 'debug', {
+    visibility: Visibility.PUBLIC, // mode: 0o644
+});
+await storage.write('private.txt', 'debug', {
+    visibility: Visibility.PRIVATE, // mode: 0o600
+});
+
+// List directory contents
+const contentsAsAsyncGenerator = storage.list('', {deep: true});
+
+for await (const item of contentsAsAsyncGenerator) {
+    console.log(item.path);
+
+    if (item.isFile) {
+        // do something with the file
+    } else if (item.isDirectory) {
+        // do something with the directory
+    }
+}
+
+// Delete a file
+await storage.deleteFile('some-file.txt');
+
+// Delete a directory (with all contents)
+await storage.deleteDirectory('some-directory');
+```
+
+## Author
+Flystorage is built by the maintainer of [Flysystem](https://flysystem.thephpleague.com), a
+filesystem abstraction for PHP. This brings along more than
+a decade of smoothing over filesystem implementation differences
+and weighing trade-offs to make a usable API.

package/dist/cjs/errors.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.UnableToListDirectory = exports.UnableToCheckDirectoryExistence = exports.UnableToCheckFileExistence = exports.UnableToDeleteFile = exports.UnableToDeleteDirectory = exports.UnableToCreateDirectory = exports.UnableToGetStat = exports.UnableToGetTemporaryUrl = exports.UnableToGetPublicUrl = exports.UnableToGetVisibility = exports.UnableToSetVisibility = exports.UnableToReadFile = exports.UnableToWriteFile = exports.UnableToGetChecksum = exports.ChecksumIsNotAvailable = exports.FlystorageError = exports.errorToMessage = void 0;
+exports.UnableToListDirectory = exports.UnableToCheckDirectoryExistence = exports.UnableToCheckFileExistence = exports.UnableToDeleteFile = exports.UnableToDeleteDirectory = exports.UnableToCreateDirectory = exports.UnableToGetStat = exports.UnableToGetTemporaryUrl = exports.UnableToGetPublicUrl = exports.UnableToGetVisibility = exports.UnableToSetVisibility = exports.UnableToReadFile = exports.UnableToWriteFile = exports.UnableToGetFileSize = exports.UnableToGetLastModified = exports.UnableToGetMimeType = exports.UnableToGetChecksum = exports.ChecksumIsNotAvailable = exports.FlystorageError = exports.errorToMessage = void 0;
 function errorToMessage(error) {
     return error instanceof Error ? error.message : String(error);
 }
@@ -35,6 +35,21 @@ class UnableToGetChecksum extends FlystorageError {
     static because = (reason, { context = {}, cause = undefined }) => new UnableToGetChecksum(`Unable to write the file. Reason: ${reason}`, context, cause);
 }
 exports.UnableToGetChecksum = UnableToGetChecksum;
+class UnableToGetMimeType extends FlystorageError {
+    code = 'flystorage.unable_to_get_mimetype';
+    static because = (reason, { context = {}, cause = undefined }) => new UnableToGetMimeType(`Unable to get mime-type. Reason: ${reason}`, context, cause);
+}
+exports.UnableToGetMimeType = UnableToGetMimeType;
+class UnableToGetLastModified extends FlystorageError {
+    code = 'flystorage.unable_to_get_last_modified';
+    static because = (reason, { context = {}, cause = undefined }) => new UnableToGetLastModified(`Unable to get last modified. Reason: ${reason}`, context, cause);
+}
+exports.UnableToGetLastModified = UnableToGetLastModified;
+class UnableToGetFileSize extends FlystorageError {
+    code = 'flystorage.unable_to_get_file_size';
+    static because = (reason, { context = {}, cause = undefined }) => new UnableToGetFileSize(`Unable to get file size. Reason: ${reason}`, context, cause);
+}
+exports.UnableToGetFileSize = UnableToGetFileSize;
 class UnableToWriteFile extends FlystorageError {
     code = 'flystorage.unable_to_write_file';
     static because = (reason, { context = {}, cause = undefined }) => new UnableToWriteFile(`Unable to write the file. Reason: ${reason}`, context, cause);

package/dist/cjs/file-storage.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.readableToUint8Array = exports.readableToString = exports.closeReadable = exports.normalizeExpiryToMilliseconds = exports.normalizeExpiryToDate = exports.FileStorage = exports.isDirectory = exports.isFile = void 0;
+exports.readableToUint8Array = exports.readableToString = exports.closeReadable = exports.normalizeExpiryToMilliseconds = exports.normalizeExpiryToDate = exports.FileStorage = exports.DirectoryListing = exports.isDirectory = exports.isFile = void 0;
 const stream_1 = require("stream");
 const checksum_from_stream_js_1 = require("./checksum-from-stream.js");
 const errors = require("./errors.js");
@@ -14,6 +14,45 @@ function isDirectory(stat) {
     return stat.isDirectory;
 }
 exports.isDirectory = isDirectory;
+class DirectoryListing {
+    listing;
+    path;
+    deep;
+    constructor(listing, path, deep) {
+        this.listing = listing;
+        this.path = path;
+        this.deep = deep;
+    }
+    async toArray(sorted = true) {
+        const items = [];
+        for await (const item of this.listing) {
+            items.push(item);
+        }
+        return sorted ? items.sort((a, b) => naturalSorting.compare(a.path, b.path)) : items;
+    }
+    filter(filter) {
+        const listing = this.listing;
+        const filtered = (async function* () {
+            for await (const entry of listing) {
+                if (filter(entry)) {
+                    yield entry;
+                }
+            }
+        })();
+        return new DirectoryListing(filtered, this.path, this.deep);
+    }
+    async *[Symbol.asyncIterator]() {
+        try {
+            for await (const item of this.listing) {
+                yield item;
+            }
+        }
+        catch (error) {
+            throw errors.UnableToListDirectory.because(errors.errorToMessage(error), { cause: error, context: { path: this.path, deep: this.deep } });
+        }
+    }
+}
+exports.DirectoryListing = DirectoryListing;
 function toReadable(contents) {
     if (contents instanceof stream_1.Readable) {
         return contents;
@@ -57,6 +96,9 @@ class FileStorage {
     async readToUint8Array(path) {
         return await readableToUint8Array(await this.read(path));
     }
+    async readToBuffer(path) {
+        return Buffer.from(await this.readToUint8Array(path));
+    }
     async deleteFile(path) {
         try {
             await this.adapter.deleteFile(this.pathNormalizer.normalizePath(path));
@@ -114,26 +156,7 @@ class FileStorage {
         }
     }
     list(path, { deep = false } = {}) {
-        const listing = this.adapter.list(this.pathNormalizer.normalizePath(path), { deep });
-        return {
-            async toArray(sorted = true) {
-                const items = [];
-                for await (const item of listing) {
-                    items.push(item);
-                }
-                return sorted ? items.sort((a, b) => naturalSorting.compare(a.path, b.path)) : items;
-            },
-            async *[Symbol.asyncIterator]() {
-                try {
-                    for await (const item of listing) {
-                        yield item;
-                    }
-                }
-                catch (error) {
-                    throw errors.UnableToListDirectory.because(errors.errorToMessage(error), { cause: error, context: { path, deep } });
-                }
-            }
-        };
+        return new DirectoryListing(this.adapter.list(this.pathNormalizer.normalizePath(path), { deep }), path, deep);
     }
     async statFile(path) {
         const stat = await this.stat(path);
@@ -177,6 +200,30 @@ class FileStorage {
             throw errors.UnableToGetChecksum.because(errors.errorToMessage(error), { cause: error, context: { path, options } });
         }
     }
+    async mimeType(path, options = {}) {
+        try {
+            return await this.adapter.mimeType(this.pathNormalizer.normalizePath(path), options);
+        }
+        catch (error) {
+            throw errors.UnableToGetMimeType.because(errors.errorToMessage(error), { cause: error, context: { path, options } });
+        }
+    }
+    async lastModified(path) {
+        try {
+            return await this.adapter.lastModified(this.pathNormalizer.normalizePath(path));
+        }
+        catch (error) {
+            throw errors.UnableToGetLastModified.because(errors.errorToMessage(error), { cause: error, context: { path } });
+        }
+    }
+    async fileSize(path) {
+        try {
+            return await this.adapter.fileSize(this.pathNormalizer.normalizePath(path));
+        }
+        catch (error) {
+            throw errors.UnableToGetFileSize.because(errors.errorToMessage(error), { cause: error, context: { path } });
+        }
+    }
     async calculateChecksum(path, options) {
         try {
             return await (0, checksum_from_stream_js_1.checksumFromStream)(await this.read(path), options);

package/dist/esm/errors.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.UnableToListDirectory = exports.UnableToCheckDirectoryExistence = exports.UnableToCheckFileExistence = exports.UnableToDeleteFile = exports.UnableToDeleteDirectory = exports.UnableToCreateDirectory = exports.UnableToGetStat = exports.UnableToGetTemporaryUrl = exports.UnableToGetPublicUrl = exports.UnableToGetVisibility = exports.UnableToSetVisibility = exports.UnableToReadFile = exports.UnableToWriteFile = exports.UnableToGetChecksum = exports.ChecksumIsNotAvailable = exports.FlystorageError = exports.errorToMessage = void 0;
+exports.UnableToListDirectory = exports.UnableToCheckDirectoryExistence = exports.UnableToCheckFileExistence = exports.UnableToDeleteFile = exports.UnableToDeleteDirectory = exports.UnableToCreateDirectory = exports.UnableToGetStat = exports.UnableToGetTemporaryUrl = exports.UnableToGetPublicUrl = exports.UnableToGetVisibility = exports.UnableToSetVisibility = exports.UnableToReadFile = exports.UnableToWriteFile = exports.UnableToGetFileSize = exports.UnableToGetLastModified = exports.UnableToGetMimeType = exports.UnableToGetChecksum = exports.ChecksumIsNotAvailable = exports.FlystorageError = exports.errorToMessage = void 0;
 function errorToMessage(error) {
     return error instanceof Error ? error.message : String(error);
 }
@@ -35,6 +35,21 @@ class UnableToGetChecksum extends FlystorageError {
     static because = (reason, { context = {}, cause = undefined }) => new UnableToGetChecksum(`Unable to write the file. Reason: ${reason}`, context, cause);
 }
 exports.UnableToGetChecksum = UnableToGetChecksum;
+class UnableToGetMimeType extends FlystorageError {
+    code = 'flystorage.unable_to_get_mimetype';
+    static because = (reason, { context = {}, cause = undefined }) => new UnableToGetMimeType(`Unable to get mime-type. Reason: ${reason}`, context, cause);
+}
+exports.UnableToGetMimeType = UnableToGetMimeType;
+class UnableToGetLastModified extends FlystorageError {
+    code = 'flystorage.unable_to_get_last_modified';
+    static because = (reason, { context = {}, cause = undefined }) => new UnableToGetLastModified(`Unable to get last modified. Reason: ${reason}`, context, cause);
+}
+exports.UnableToGetLastModified = UnableToGetLastModified;
+class UnableToGetFileSize extends FlystorageError {
+    code = 'flystorage.unable_to_get_file_size';
+    static because = (reason, { context = {}, cause = undefined }) => new UnableToGetFileSize(`Unable to get file size. Reason: ${reason}`, context, cause);
+}
+exports.UnableToGetFileSize = UnableToGetFileSize;
 class UnableToWriteFile extends FlystorageError {
     code = 'flystorage.unable_to_write_file';
     static because = (reason, { context = {}, cause = undefined }) => new UnableToWriteFile(`Unable to write the file. Reason: ${reason}`, context, cause);

package/dist/esm/file-storage.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.readableToUint8Array = exports.readableToString = exports.closeReadable = exports.normalizeExpiryToMilliseconds = exports.normalizeExpiryToDate = exports.FileStorage = exports.isDirectory = exports.isFile = void 0;
+exports.readableToUint8Array = exports.readableToString = exports.closeReadable = exports.normalizeExpiryToMilliseconds = exports.normalizeExpiryToDate = exports.FileStorage = exports.DirectoryListing = exports.isDirectory = exports.isFile = void 0;
 const stream_1 = require("stream");
 const checksum_from_stream_js_1 = require("./checksum-from-stream.js");
 const errors = require("./errors.js");
@@ -14,6 +14,45 @@ function isDirectory(stat) {
     return stat.isDirectory;
 }
 exports.isDirectory = isDirectory;
+class DirectoryListing {
+    listing;
+    path;
+    deep;
+    constructor(listing, path, deep) {
+        this.listing = listing;
+        this.path = path;
+        this.deep = deep;
+    }
+    async toArray(sorted = true) {
+        const items = [];
+        for await (const item of this.listing) {
+            items.push(item);
+        }
+        return sorted ? items.sort((a, b) => naturalSorting.compare(a.path, b.path)) : items;
+    }
+    filter(filter) {
+        const listing = this.listing;
+        const filtered = (async function* () {
+            for await (const entry of listing) {
+                if (filter(entry)) {
+                    yield entry;
+                }
+            }
+        })();
+        return new DirectoryListing(filtered, this.path, this.deep);
+    }
+    async *[Symbol.asyncIterator]() {
+        try {
+            for await (const item of this.listing) {
+                yield item;
+            }
+        }
+        catch (error) {
+            throw errors.UnableToListDirectory.because(errors.errorToMessage(error), { cause: error, context: { path: this.path, deep: this.deep } });
+        }
+    }
+}
+exports.DirectoryListing = DirectoryListing;
 function toReadable(contents) {
     if (contents instanceof stream_1.Readable) {
         return contents;
@@ -57,6 +96,9 @@ class FileStorage {
     async readToUint8Array(path) {
         return await readableToUint8Array(await this.read(path));
     }
+    async readToBuffer(path) {
+        return Buffer.from(await this.readToUint8Array(path));
+    }
     async deleteFile(path) {
         try {
             await this.adapter.deleteFile(this.pathNormalizer.normalizePath(path));
@@ -114,26 +156,7 @@ class FileStorage {
         }
     }
     list(path, { deep = false } = {}) {
-        const listing = this.adapter.list(this.pathNormalizer.normalizePath(path), { deep });
-        return {
-            async toArray(sorted = true) {
-                const items = [];
-                for await (const item of listing) {
-                    items.push(item);
-                }
-                return sorted ? items.sort((a, b) => naturalSorting.compare(a.path, b.path)) : items;
-            },
-            async *[Symbol.asyncIterator]() {
-                try {
-                    for await (const item of listing) {
-                        yield item;
-                    }
-                }
-                catch (error) {
-                    throw errors.UnableToListDirectory.because(errors.errorToMessage(error), { cause: error, context: { path, deep } });
-                }
-            }
-        };
+        return new DirectoryListing(this.adapter.list(this.pathNormalizer.normalizePath(path), { deep }), path, deep);
     }
     async statFile(path) {
         const stat = await this.stat(path);
@@ -177,6 +200,30 @@ class FileStorage {
             throw errors.UnableToGetChecksum.because(errors.errorToMessage(error), { cause: error, context: { path, options } });
         }
     }
+    async mimeType(path, options = {}) {
+        try {
+            return await this.adapter.mimeType(this.pathNormalizer.normalizePath(path), options);
+        }
+        catch (error) {
+            throw errors.UnableToGetMimeType.because(errors.errorToMessage(error), { cause: error, context: { path, options } });
+        }
+    }
+    async lastModified(path) {
+        try {
+            return await this.adapter.lastModified(this.pathNormalizer.normalizePath(path));
+        }
+        catch (error) {
+            throw errors.UnableToGetLastModified.because(errors.errorToMessage(error), { cause: error, context: { path } });
+        }
+    }
+    async fileSize(path) {
+        try {
+            return await this.adapter.fileSize(this.pathNormalizer.normalizePath(path));
+        }
+        catch (error) {
+            throw errors.UnableToGetFileSize.because(errors.errorToMessage(error), { cause: error, context: { path } });
+        }
+    }
     async calculateChecksum(path, options) {
         try {
             return await (0, checksum_from_stream_js_1.checksumFromStream)(await this.read(path), options);

package/dist/types/errors.d.ts

@@ -31,6 +31,27 @@ export declare class UnableToGetChecksum extends FlystorageError {
         cause?: unknown;
     }) => UnableToGetChecksum;
 }
+export declare class UnableToGetMimeType extends FlystorageError {
+    readonly code = "flystorage.unable_to_get_mimetype";
+    static because: (reason: string, { context, cause }: {
+        context?: ErrorContext | undefined;
+        cause?: unknown;
+    }) => UnableToGetMimeType;
+}
+export declare class UnableToGetLastModified extends FlystorageError {
+    readonly code = "flystorage.unable_to_get_last_modified";
+    static because: (reason: string, { context, cause }: {
+        context?: ErrorContext | undefined;
+        cause?: unknown;
+    }) => UnableToGetLastModified;
+}
+export declare class UnableToGetFileSize extends FlystorageError {
+    readonly code = "flystorage.unable_to_get_file_size";
+    static because: (reason: string, { context, cause }: {
+        context?: ErrorContext | undefined;
+        cause?: unknown;
+    }) => UnableToGetFileSize;
+}
 export declare class UnableToWriteFile extends FlystorageError {
     readonly code = "flystorage.unable_to_write_file";
     static because: (reason: string, { context, cause }: {

package/dist/types/file-storage.d.ts

@@ -1,6 +1,7 @@
 /// <reference types="node" />
 /// <reference types="node" />
 /// <reference types="node" />
+/// <reference types="node" />
 import { BinaryToTextEncoding } from 'crypto';
 import { Readable } from 'stream';
 import { PathNormalizer } from './path-normalizer.js';
@@ -41,14 +42,27 @@ export interface StorageAdapter {
     publicUrl(path: string, options: PublicUrlOptions): Promise<string>;
     temporaryUrl(path: string, options: TemporaryUrlOptions): Promise<string>;
     checksum(path: string, options: ChecksumOptions): Promise<string>;
+    mimeType(path: string, options: MimeTypeOptions): Promise<string>;
+    lastModified(path: string): Promise<number>;
+    fileSize(path: string): Promise<number>;
 }
-export interface DirectoryListing extends AsyncIterable<StatEntry> {
+export declare class DirectoryListing implements AsyncIterable<StatEntry> {
+    private readonly listing;
+    private readonly path;
+    private readonly deep;
+    constructor(listing: AsyncGenerator<StatEntry>, path: string, deep: boolean);
     toArray(sorted?: boolean): Promise<StatEntry[]>;
+    filter(filter: (entry: StatEntry) => boolean): DirectoryListing;
+    [Symbol.asyncIterator](): AsyncGenerator<StatEntry, void, unknown>;
 }
 export type FileContents = Iterable<any> | AsyncIterable<any> | NodeJS.ReadableStream | Readable;
 export type MiscellaneousOptions = {
     [option: string]: any;
 };
+export type MimeTypeOptions = MiscellaneousOptions & {
+    disallowFallback?: boolean;
+    fallbackMethod?: 'contents' | 'path';
+};
 export type VisibilityOptions = {
     visibility?: string;
     directoryVisibility?: string;
@@ -79,6 +93,7 @@ export declare class FileStorage {
     read(path: string): Promise<Readable>;
     readToString(path: string): Promise<string>;
     readToUint8Array(path: string): Promise<Uint8Array>;
+    readToBuffer(path: string): Promise<Buffer>;
     deleteFile(path: string): Promise<void>;
     createDirectory(path: string, options?: CreateDirectoryOptions): Promise<void>;
     deleteDirectory(path: string): Promise<void>;
@@ -94,6 +109,9 @@ export declare class FileStorage {
     publicUrl(path: string, options?: PublicUrlOptions): Promise<string>;
     temporaryUrl(path: string, options: TemporaryUrlOptions): Promise<string>;
     checksum(path: string, options?: ChecksumOptions): Promise<string>;
+    mimeType(path: string, options?: MimeTypeOptions): Promise<string>;
+    lastModified(path: string): Promise<number>;
+    fileSize(path: string): Promise<number>;
     private calculateChecksum;
 }
 export type TimestampMs = number;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flystorage/file-storage",
-  "version": "0.0.2",
+  "version": "0.0.4",
   "description": "File-storage abstraction: multiple filesystems, one API.",
   "main": "./dist/cjs/index.js",
   "types": "./dist/types/index.d.ts",
@@ -24,6 +24,11 @@
     "watch": "tsc --watch"
   },
   "author": "Frank de Jonge (https://frankdejonge.nl)",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/flystorage/flystorage.git",
+    "directory": "packages/file-storage"
+  },
   "keywords": ["fs", "file", "files", "filesystem", "filesystems", "storage"],
   "license": "MIT"
 }