@directus/storage-driver-s3 9.21.2

package/dist/index.js

@@ -0,0 +1,136 @@
+import { CopyObjectCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, ListObjectsV2Command, S3Client, } from '@aws-sdk/client-s3';
+import { Upload } from '@aws-sdk/lib-storage';
+import { normalizePath } from '@directus/utils';
+import { isReadableStream } from '@directus/utils/node';
+import { join } from 'node:path';
+export class DriverS3 {
+    root;
+    client;
+    bucket;
+    acl;
+    serverSideEncryption;
+    constructor(config) {
+        const s3ClientConfig = {
+            credentials: {
+                accessKeyId: config.key,
+                secretAccessKey: config.secret,
+            },
+        };
+        if (config.endpoint) {
+            const protocol = config.endpoint.startsWith('https://') ? 'https:' : 'http:';
+            const hostname = config.endpoint.replace('https://', '').replace('http://', '');
+            s3ClientConfig.endpoint = {
+                hostname,
+                protocol,
+                path: '/',
+            };
+            s3ClientConfig.forcePathStyle = true;
+        }
+        this.client = new S3Client(s3ClientConfig);
+        this.bucket = config.bucket;
+        this.acl = config.acl;
+        this.serverSideEncryption = config.serverSideEncryption;
+        this.root = config.root ? normalizePath(config.root, { removeLeading: true }) : '';
+    }
+    fullPath(filepath) {
+        return normalizePath(join(this.root, filepath));
+    }
+    async read(filepath, range) {
+        const commandInput = {
+            Key: this.fullPath(filepath),
+            Bucket: this.bucket,
+        };
+        if (range) {
+            commandInput.Range = `bytes=${range.start ?? ''}-${range.end ?? ''}`;
+        }
+        const { Body: stream } = await this.client.send(new GetObjectCommand(commandInput));
+        if (!stream || !isReadableStream(stream)) {
+            throw new Error(`No stream returned for file "${filepath}"`);
+        }
+        return stream;
+    }
+    async stat(filepath) {
+        const { ContentLength, LastModified } = await this.client.send(new HeadObjectCommand({
+            Key: this.fullPath(filepath),
+            Bucket: this.bucket,
+        }));
+        return {
+            size: ContentLength,
+            modified: LastModified,
+        };
+    }
+    async exists(filepath) {
+        try {
+            await this.stat(filepath);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    async move(src, dest) {
+        await this.copy(src, dest);
+        await this.delete(src);
+    }
+    async copy(src, dest) {
+        const params = {
+            Key: this.fullPath(dest),
+            Bucket: this.bucket,
+            CopySource: `/${this.bucket}/${this.fullPath(src)}`,
+        };
+        if (this.serverSideEncryption) {
+            params.ServerSideEncryption = this.serverSideEncryption;
+        }
+        if (this.acl) {
+            params.ACL = this.acl;
+        }
+        await this.client.send(new CopyObjectCommand(params));
+    }
+    async write(filepath, content, type) {
+        const params = {
+            Key: this.fullPath(filepath),
+            Body: content,
+            Bucket: this.bucket,
+        };
+        if (type) {
+            params.ContentType = type;
+        }
+        if (this.acl) {
+            params.ACL = this.acl;
+        }
+        if (this.serverSideEncryption) {
+            params.ServerSideEncryption = this.serverSideEncryption;
+        }
+        const upload = new Upload({
+            client: this.client,
+            params,
+        });
+        await upload.done();
+    }
+    async delete(filepath) {
+        await this.client.send(new DeleteObjectCommand({ Key: this.fullPath(filepath), Bucket: this.bucket }));
+    }
+    async *list(prefix = '') {
+        let continuationToken = undefined;
+        do {
+            const listObjectsV2CommandInput = {
+                Bucket: this.bucket,
+                Prefix: this.fullPath(prefix),
+                MaxKeys: 1000,
+            };
+            if (continuationToken) {
+                listObjectsV2CommandInput.ContinuationToken = continuationToken;
+            }
+            const response = await this.client.send(new ListObjectsV2Command(listObjectsV2CommandInput));
+            continuationToken = response.NextContinuationToken;
+            if (response.Contents) {
+                for (const file of response.Contents) {
+                    if (file.Key) {
+                        yield file.Key.substring(this.root.length);
+                    }
+                }
+            }
+        } while (continuationToken);
+    }
+}
+export default DriverS3;

package/dist/index.test.js

@@ -0,0 +1,471 @@
+import { CopyObjectCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, ListObjectsV2Command, S3Client, } from '@aws-sdk/client-s3';
+import { Upload } from '@aws-sdk/lib-storage';
+import { normalizePath } from '@directus/utils';
+import { isReadableStream } from '@directus/utils/node';
+import { randAlphaNumeric, randDirectoryPath, randDomainName, randFilePath, randFileType, randGitBranch as randBucket, randGitShortSha as randUnique, randNumber, randPastDate, randText, randWord, } from '@ngneat/falso';
+import { join } from 'node:path';
+import { PassThrough, Readable } from 'node:stream';
+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
+import { DriverS3 } from './index.js';
+vi.mock('@directus/utils/node');
+vi.mock('@directus/utils');
+vi.mock('@aws-sdk/client-s3');
+vi.mock('@aws-sdk/lib-storage');
+vi.mock('node:path');
+let sample;
+let driver;
+beforeEach(() => {
+    sample = {
+        config: {
+            key: randAlphaNumeric({ length: 20 }).join(''),
+            secret: randAlphaNumeric({ length: 40 }).join(''),
+            bucket: randBucket(),
+            acl: randWord(),
+            serverSideEncryption: randWord(),
+            root: randDirectoryPath(),
+            endpoint: randDomainName(),
+        },
+        path: {
+            input: randUnique() + randFilePath(),
+            inputFull: randUnique() + randFilePath(),
+            src: randUnique() + randFilePath(),
+            srcFull: randUnique() + randFilePath(),
+            dest: randUnique() + randFilePath(),
+            destFull: randUnique() + randFilePath(),
+        },
+        range: {
+            start: randNumber(),
+            end: randNumber(),
+        },
+        stream: new PassThrough(),
+        text: randText(),
+        file: {
+            type: randFileType(),
+            size: randNumber(),
+            modified: randPastDate(),
+        },
+    };
+    driver = new DriverS3({
+        key: sample.config.key,
+        secret: sample.config.secret,
+        bucket: sample.config.bucket,
+    });
+    driver['fullPath'] = vi.fn().mockImplementation((input) => {
+        if (input === sample.path.src)
+            return sample.path.srcFull;
+        if (input === sample.path.dest)
+            return sample.path.destFull;
+        if (input === sample.path.input)
+            return sample.path.inputFull;
+        return '';
+    });
+});
+afterEach(() => {
+    vi.resetAllMocks();
+});
+describe('#constructor', () => {
+    test('Creates S3Client with key / secret configuration', () => {
+        expect(S3Client).toHaveBeenCalledWith({
+            credentials: {
+                accessKeyId: sample.config.key,
+                secretAccessKey: sample.config.secret,
+            },
+        });
+        expect(driver['client']).toBeInstanceOf(S3Client);
+    });
+    test('Sets private bucket reference based on config', () => {
+        expect(driver['bucket']).toBe(sample.config.bucket);
+    });
+    test('Sets private acl reference based on config', () => {
+        const driver = new DriverS3({
+            key: sample.config.key,
+            secret: sample.config.secret,
+            bucket: sample.config.bucket,
+            acl: sample.config.acl,
+        });
+        expect(driver['acl']).toBe(sample.config.acl);
+    });
+    test('Sets private serverSideEncryption reference based on config', () => {
+        const driver = new DriverS3({
+            key: sample.config.key,
+            secret: sample.config.secret,
+            bucket: sample.config.bucket,
+            serverSideEncryption: sample.config.serverSideEncryption,
+        });
+        expect(driver['serverSideEncryption']).toBe(sample.config.serverSideEncryption);
+    });
+    test('Defaults root to empty string', () => {
+        expect(driver['root']).toBe('');
+    });
+    test('Sets http endpoints', () => {
+        const sampleDomain = randDomainName();
+        const sampleHttpEndpoint = `http://${sampleDomain}`;
+        new DriverS3({
+            key: sample.config.key,
+            secret: sample.config.secret,
+            bucket: sample.config.bucket,
+            endpoint: sampleHttpEndpoint,
+        });
+        expect(S3Client).toHaveBeenCalledWith({
+            forcePathStyle: true,
+            endpoint: {
+                hostname: sampleDomain,
+                protocol: 'http:',
+                path: '/',
+            },
+            credentials: {
+                accessKeyId: sample.config.key,
+                secretAccessKey: sample.config.secret,
+            },
+        });
+    });
+    test('Sets https endpoints', () => {
+        const sampleDomain = randDomainName();
+        const sampleHttpEndpoint = `https://${sampleDomain}`;
+        new DriverS3({
+            key: sample.config.key,
+            secret: sample.config.secret,
+            bucket: sample.config.bucket,
+            endpoint: sampleHttpEndpoint,
+        });
+        expect(S3Client).toHaveBeenCalledWith({
+            forcePathStyle: true,
+            endpoint: {
+                hostname: sampleDomain,
+                protocol: 'https:',
+                path: '/',
+            },
+            credentials: {
+                accessKeyId: sample.config.key,
+                secretAccessKey: sample.config.secret,
+            },
+        });
+    });
+    test('Normalizes config path when root is given', () => {
+        const mockRoot = randDirectoryPath();
+        vi.mocked(normalizePath).mockReturnValue(mockRoot);
+        const driver = new DriverS3({
+            key: sample.config.key,
+            secret: sample.config.secret,
+            bucket: sample.config.bucket,
+            root: sample.config.root,
+        });
+        expect(normalizePath).toHaveBeenCalledWith(sample.config.root, { removeLeading: true });
+        expect(driver['root']).toBe(mockRoot);
+    });
+});
+describe('#fullPath', () => {
+    test('Returns normalized joined path', () => {
+        const driver = new DriverS3({
+            key: sample.config.key,
+            secret: sample.config.secret,
+            bucket: sample.config.bucket,
+        });
+        vi.mocked(join).mockReturnValue(sample.path.inputFull);
+        vi.mocked(normalizePath).mockReturnValue(sample.path.inputFull);
+        driver['root'] = sample.config.root;
+        const result = driver['fullPath'](sample.path.input);
+        expect(join).toHaveBeenCalledWith(sample.config.root, sample.path.input);
+        expect(normalizePath).toHaveBeenCalledWith(sample.path.inputFull);
+        expect(result).toBe(sample.path.inputFull);
+    });
+});
+describe('#read', () => {
+    beforeEach(() => {
+        vi.mocked(driver['client'].send).mockReturnValue({ Body: new Readable() });
+        vi.mocked(isReadableStream).mockReturnValue(true);
+    });
+    test('Uses fullPath key / bucket in command input', async () => {
+        await driver.read(sample.path.input);
+        expect(driver['fullPath']).toHaveBeenCalledWith(sample.path.input);
+        expect(GetObjectCommand).toHaveBeenCalledWith({
+            Key: sample.path.inputFull,
+            Bucket: sample.config.bucket,
+        });
+    });
+    test('Optionally allows setting start range offset', async () => {
+        await driver.read(sample.path.input, { start: sample.range.start });
+        expect(GetObjectCommand).toHaveBeenCalledWith({
+            Key: sample.path.inputFull,
+            Bucket: sample.config.bucket,
+            Range: `bytes=${sample.range.start}-`,
+        });
+    });
+    test('Optionally allows setting end range offset', async () => {
+        await driver.read(sample.path.input, { end: sample.range.end });
+        expect(GetObjectCommand).toHaveBeenCalledWith({
+            Key: sample.path.inputFull,
+            Bucket: sample.config.bucket,
+            Range: `bytes=-${sample.range.end}`,
+        });
+    });
+    test('Optionally allows setting start and end range offset', async () => {
+        await driver.read(sample.path.input, sample.range);
+        expect(GetObjectCommand).toHaveBeenCalledWith({
+            Key: sample.path.inputFull,
+            Bucket: sample.config.bucket,
+            Range: `bytes=${sample.range.start}-${sample.range.end}`,
+        });
+    });
+    test('Throws an error when no stream is returned', async () => {
+        vi.mocked(driver['client'].send).mockReturnValue({ Body: undefined });
+        try {
+            await driver.read(sample.path.input, sample.range);
+        }
+        catch (err) {
+            expect(err).toBeInstanceOf(Error);
+            expect(err.message).toBe(`No stream returned for file "${sample.path.input}"`);
+        }
+    });
+    test('Throws an error when returned stream is not a readable stream', async () => {
+        vi.mocked(isReadableStream).mockReturnValue(false);
+        expect(driver.read(sample.path.input, sample.range)).rejects.toThrowError(new Error(`No stream returned for file "${sample.path.input}"`));
+    });
+    test('Returns stream from S3 client', async () => {
+        const mockGetObjectCommand = {};
+        vi.mocked(driver['client'].send).mockReturnValue({ Body: sample.stream });
+        vi.mocked(GetObjectCommand).mockReturnValue(mockGetObjectCommand);
+        const stream = await driver.read(sample.path.input, sample.range);
+        expect(driver['client'].send).toHaveBeenCalledWith(mockGetObjectCommand);
+        expect(stream).toBe(sample.stream);
+    });
+});
+describe('#stat', () => {
+    beforeEach(() => {
+        vi.mocked(driver['client'].send).mockResolvedValue({
+            ContentLength: sample.file.size,
+            LastModified: sample.file.modified,
+        });
+    });
+    test('Uses HeadObjectCommand with fullPath', async () => {
+        await driver.stat(sample.path.input);
+        expect(driver['fullPath']).toHaveBeenCalledWith(sample.path.input);
+        expect(HeadObjectCommand).toHaveBeenCalledWith({
+            Key: sample.path.inputFull,
+            Bucket: sample.config.bucket,
+        });
+    });
+    test('Calls #send with HeadObjectCommand', async () => {
+        const mockHeadObjectCommand = {};
+        vi.mocked(HeadObjectCommand).mockReturnValue(mockHeadObjectCommand);
+        await driver.stat(sample.path.input);
+        expect(driver['client'].send).toHaveBeenCalledWith(mockHeadObjectCommand);
+    });
+    test('Returns size/modified from returned send data', async () => {
+        const result = await driver.stat(sample.path.input);
+        expect(result).toStrictEqual({
+            size: sample.file.size,
+            modified: sample.file.modified,
+        });
+    });
+});
+describe('#exists', () => {
+    beforeEach(() => {
+        driver.stat = vi.fn();
+    });
+    test('Returns true if stat returns the stats', async () => {
+        vi.mocked(driver.stat).mockResolvedValue({ size: sample.file.size, modified: sample.file.modified });
+        const exists = await driver.exists(sample.path.input);
+        expect(exists).toBe(true);
+    });
+    test('Returns false if stat throws an error', async () => {
+        vi.mocked(driver.stat).mockRejectedValue(new Error());
+        const exists = await driver.exists(sample.path.input);
+        expect(exists).toBe(false);
+    });
+});
+describe('#move', () => {
+    beforeEach(async () => {
+        driver.copy = vi.fn();
+        driver.delete = vi.fn();
+        await driver.move(sample.path.src, sample.path.dest);
+    });
+    test('Calls copy with given src and dest', async () => {
+        expect(driver.copy).toHaveBeenCalledWith(sample.path.src, sample.path.dest);
+    });
+    test('Calls delete on successful copy', async () => {
+        expect(driver.delete).toHaveBeenCalledWith(sample.path.src);
+    });
+});
+describe('#copy', () => {
+    test('Constructs params object based on config', async () => {
+        await driver.copy(sample.path.src, sample.path.dest);
+        expect(CopyObjectCommand).toHaveBeenCalledWith({
+            Key: sample.path.destFull,
+            Bucket: sample.config.bucket,
+            CopySource: `/${sample.config.bucket}/${sample.path.srcFull}`,
+        });
+    });
+    test('Optionally sets ServerSideEncryption', async () => {
+        driver['serverSideEncryption'] = sample.config.serverSideEncryption;
+        await driver.copy(sample.path.src, sample.path.dest);
+        expect(CopyObjectCommand).toHaveBeenCalledWith({
+            Key: sample.path.destFull,
+            Bucket: sample.config.bucket,
+            CopySource: `/${sample.config.bucket}/${sample.path.srcFull}`,
+            ServerSideEncryption: sample.config.serverSideEncryption,
+        });
+    });
+    test('Optionally sets ACL', async () => {
+        driver['acl'] = sample.config.acl;
+        await driver.copy(sample.path.src, sample.path.dest);
+        expect(CopyObjectCommand).toHaveBeenCalledWith({
+            Key: sample.path.destFull,
+            Bucket: sample.config.bucket,
+            CopySource: `/${sample.config.bucket}/${sample.path.srcFull}`,
+            ACL: sample.config.acl,
+        });
+    });
+    test('Executes CopyObjectCommand', async () => {
+        const mockCommand = {};
+        vi.mocked(CopyObjectCommand).mockReturnValue(mockCommand);
+        await driver.copy(sample.path.src, sample.path.dest);
+        expect(driver['client'].send).toHaveBeenCalledWith(mockCommand);
+    });
+});
+describe('#write', () => {
+    test('Passes streams to body as is', async () => {
+        await driver.write(sample.path.input, sample.stream);
+        expect(Upload).toHaveBeenCalledWith({
+            client: driver['client'],
+            params: {
+                Key: sample.path.inputFull,
+                Bucket: sample.config.bucket,
+                Body: sample.stream,
+            },
+        });
+    });
+    test('Optionally sets ContentType', async () => {
+        await driver.write(sample.path.input, sample.stream, sample.file.type);
+        expect(Upload).toHaveBeenCalledWith({
+            client: driver['client'],
+            params: {
+                Key: sample.path.inputFull,
+                Bucket: sample.config.bucket,
+                Body: sample.stream,
+                ContentType: sample.file.type,
+            },
+        });
+    });
+    test('Optionally sets ServerSideEncryption', async () => {
+        driver['serverSideEncryption'] = sample.config.serverSideEncryption;
+        await driver.write(sample.path.input, sample.stream);
+        expect(Upload).toHaveBeenCalledWith({
+            client: driver['client'],
+            params: {
+                Key: sample.path.inputFull,
+                Bucket: sample.config.bucket,
+                Body: sample.stream,
+                ServerSideEncryption: sample.config.serverSideEncryption,
+            },
+        });
+    });
+    test('Optionally sets ACL', async () => {
+        driver['acl'] = sample.config.acl;
+        await driver.write(sample.path.input, sample.stream);
+        expect(Upload).toHaveBeenCalledWith({
+            client: driver['client'],
+            params: {
+                Key: sample.path.inputFull,
+                Bucket: sample.config.bucket,
+                Body: sample.stream,
+                ACL: sample.config.acl,
+            },
+        });
+    });
+    test('Waits for upload to be done', async () => {
+        const mockUpload = { done: vi.fn() };
+        vi.mocked(Upload).mockReturnValue(mockUpload);
+        await driver.write(sample.path.input, sample.stream);
+        expect(mockUpload.done).toHaveBeenCalledOnce();
+    });
+});
+describe('#delete', () => {
+    test('Constructs params based on input', async () => {
+        await driver.delete(sample.path.input);
+        expect(DeleteObjectCommand).toHaveBeenCalledWith({
+            Key: sample.path.inputFull,
+            Bucket: sample.config.bucket,
+        });
+    });
+    test('Executes DeleteObjectCommand', async () => {
+        const mockDeleteObjectCommand = {};
+        vi.mocked(DeleteObjectCommand).mockReturnValue(mockDeleteObjectCommand);
+        await driver.delete(sample.path.input);
+        expect(driver['client'].send).toHaveBeenCalledWith(mockDeleteObjectCommand);
+    });
+});
+describe('#list', () => {
+    test('Constructs list objects params based on input prefix', async () => {
+        vi.mocked(driver['client'].send).mockResolvedValue({});
+        await driver.list(sample.path.input).next();
+        expect(ListObjectsV2Command).toHaveBeenCalledWith({
+            Bucket: sample.config.bucket,
+            Prefix: sample.path.inputFull,
+            MaxKeys: 1000,
+        });
+    });
+    test('Calls send with the command', async () => {
+        const mockListObjectsV2Command = {};
+        vi.mocked(ListObjectsV2Command).mockReturnValue(mockListObjectsV2Command);
+        vi.mocked(driver['client'].send).mockResolvedValue({});
+        await driver.list(sample.path.input).next();
+        expect(driver['client'].send).toHaveBeenCalledWith(mockListObjectsV2Command);
+    });
+    test('Yields file Key omitting root', async () => {
+        const sampleRoot = randDirectoryPath();
+        const sampleFile = randFilePath();
+        const sampleFull = `${sampleRoot}${sampleFile}`;
+        vi.mocked(driver['client'].send).mockResolvedValue({
+            Contents: [
+                {
+                    Key: sampleFull,
+                },
+            ],
+        });
+        driver['root'] = sampleRoot;
+        const iterator = driver.list(sample.path.input);
+        const output = [];
+        for await (const filepath of iterator) {
+            output.push(filepath);
+        }
+        expect(output).toStrictEqual([sampleFile]);
+    });
+    test('Continuously fetches until all pages are returned', async () => {
+        vi.mocked(driver['client'].send)
+            .mockResolvedValueOnce({
+            NextContinuationToken: randWord(),
+            Contents: [
+                {
+                    Key: randFilePath(),
+                },
+                {
+                    Key: randFilePath(),
+                },
+            ],
+        })
+            .mockResolvedValueOnce({
+            NextContinuationToken: randWord(),
+            Contents: [
+                {
+                    Key: randFilePath(),
+                },
+            ],
+        })
+            .mockResolvedValueOnce({
+            NextContinuationToken: undefined,
+            Contents: [
+                {
+                    Key: randFilePath(),
+                },
+            ],
+        });
+        const iterator = driver.list(sample.path.input);
+        const output = [];
+        for await (const filepath of iterator) {
+            output.push(filepath);
+        }
+        expect(driver['client'].send).toHaveBeenCalledTimes(3);
+        expect(output.length).toBe(4);
+    });
+});

package/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "@directus/storage-driver-s3",
+  "version": "9.21.2",
+  "type": "module",
+  "description": "S3 file storage abstraction for `@directus/storage`",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/directus/directus.git",
+    "directory": "packages/storage-driver-s3"
+  },
+  "funding": "https://github.com/directus/directus?sponsor=1",
+  "license": "GPL-3.0",
+  "author": "Rijk van Zanten <rijkvanzanten@me.com>",
+  "exports": {
+    ".": "./dist/index.js",
+    "./package.json": "./package.json"
+  },
+  "main": "dist/index.js",
+  "files": [
+    "dist",
+    "!**/*.d.ts?(.map)"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@aws-sdk/client-s3": "3.224.0",
+    "@aws-sdk/lib-storage": "3.234.0",
+    "@directus/storage": "9.21.2",
+    "@directus/utils": "9.21.2"
+  },
+  "devDependencies": {
+    "@directus/tsconfig": "0.0.6",
+    "@ngneat/falso": "6.3.0",
+    "@vitest/coverage-c8": "0.25.3",
+    "typescript": "4.9.3",
+    "vitest": "0.25.3"
+  },
+  "scripts": {
+    "build": "tsc --build",
+    "dev": "tsc --watch",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:coverage": "vitest run --coverage"
+  }
+}

package/readme.md

@@ -0,0 +1,3 @@
+# `@directus/storage-driver-s3`
+
+S3 file storage driver for `@directus/storage`