@openstax/ts-utils 1.33.1 → 1.34.0

package/packages/utils/src/services/fileServer/localFileServer.spec.ts

@@ -0,0 +1,182 @@
+import fs from 'fs';
+import path from 'path';
+import { localFileServer } from './localFileServer';
+import { FileServerAdapter } from '.';
+
+jest.mock('https', () => ({
+  createServer: () => ({
+    listen: jest.fn(),
+    once: jest.fn(),
+  })
+}));
+
+jest.mock('uuid', () => ({
+  v4: () => 'test-uuid'
+}));
+
+describe('localFileServer', () => {
+  const dataDir = path.join(__dirname, '../../..');
+  let fileServer: FileServerAdapter;
+
+  beforeEach(() => {
+    Object.assign(process.env, {
+      SSL_KEY_FILE: path.join(dataDir, 'dist', 'cjs/index.d.ts'),
+      SSL_CRT_FILE: path.join(dataDir, 'dist', 'cjs/index.d.ts'),
+    });
+    fileServer = localFileServer({ configSpace: 'test', dataDir })({ test: { storagePrefix: 'dist', port: '8080', host: 'cool-host' } });
+  });
+
+  it('gets local file contents', async() => {
+    const content = await fileServer.getFileContent({
+      dataType: 'file',
+      mimeType: 'text/typescript',
+      path: 'cjs/index.d.ts',
+      label: 'index.d.ts',
+    });
+    expect(content.toString()).toBeTruthy();
+  });
+
+  it('throws if the file does not exist', async() => {
+    await expect(fileServer.getFileContent({
+      dataType: 'file',
+      mimeType: 'text/typescript',
+      path: 'cjs/does-not-exist.d.ts',
+      label: 'does-not-exist.d.ts',
+    })).rejects.toThrow('ENOENT');
+  });
+
+  it('gets signed url', async() => {
+    const url = await fileServer.getSignedViewerUrl({
+      dataType: 'file',
+      mimeType: 'text/typescript',
+      path: 'cjs/index.d.ts',
+      label: 'index.d.ts',
+    });
+    expect(url).toBe('https://cool-host:8080/cjs/index.d.ts');
+  });
+
+  it('gets public viewer url', async() => {
+    const url = await fileServer.getPublicViewerUrl({
+      dataType: 'file',
+      mimeType: 'text/typescript',
+      path: 'cjs/index.d.ts',
+      label: 'index.d.ts',
+    });
+    expect(url).toBe('https://cool-host:8080/cjs/index.d.ts');
+  });
+
+  it('puts file content', async() => {
+    fileServer = localFileServer({ configSpace: 'test', dataDir })({ test: { storagePrefix: 'dist' } });
+    await expect(fileServer.putFileContent({
+      dataType: 'file',
+      mimeType: 'text/typescript',
+      path: 'cjs/coolFile.d.ts',
+      label: 'coolFile.d.ts',
+    }, 'content')).resolves.not.toThrow();
+  });
+
+  it('gets signed upload config', async() => {
+    const config = await fileServer.getSignedFileUploadConfig();
+    expect(config).toEqual({
+      url: 'https://cool-host:8080/',
+      payload: { key: 'uploads/test-uuid/${filename}' }
+    });
+  });
+
+  it('copies file to destination', async() => {
+    // Create a source file to copy
+    const sourceFile = {
+      dataType: 'file' as const,
+      mimeType: 'text/typescript',
+      path: 'cjs/index.d.ts',
+      label: 'index.d.ts',
+    };
+    
+    const result = await fileServer.copyFileTo(sourceFile, 'cjs/copied-index.d.ts');
+    
+    expect(result).toEqual({
+      dataType: 'file',
+      mimeType: 'text/typescript', 
+      path: 'cjs/copied-index.d.ts',
+      label: 'index.d.ts',
+    });
+    
+    // Verify the file was actually copied
+    const copiedFilePath = path.join(dataDir, 'dist', 'cjs/copied-index.d.ts');
+    expect(fs.existsSync(copiedFilePath)).toBe(true);
+    
+    // Clean up
+    await fs.promises.unlink(copiedFilePath);
+  });
+
+  it('copies file to directory', async() => {
+    const sourceFile = {
+      dataType: 'file' as const,
+      mimeType: 'text/typescript',
+      path: 'cjs/index.d.ts',
+      label: 'index.d.ts',
+    };
+    
+    const result = await fileServer.copyFileToDirectory(sourceFile, 'cjs/copied/');
+    
+    expect(result).toEqual({
+      dataType: 'file',
+      mimeType: 'text/typescript',
+      path: 'cjs/copied/index.d.ts',
+      label: 'index.d.ts',
+    });
+    
+    // Clean up
+    const copiedFilePath = path.join(dataDir, 'dist', 'cjs/copied/index.d.ts');
+    if (fs.existsSync(copiedFilePath)) {
+      await fs.promises.unlink(copiedFilePath);
+      await fs.promises.rmdir(path.dirname(copiedFilePath));
+    }
+  });
+
+  it('identifies temporary uploads', () => {
+    expect(fileServer.isTemporaryUpload({
+      dataType: 'file',
+      mimeType: 'text/html',
+      path: 'uploads/test-file.html',
+      label: 'test-file.html',
+    })).toBe(true);
+
+    expect(fileServer.isTemporaryUpload({
+      dataType: 'file',
+      mimeType: 'text/html',
+      path: 'permanent/test-file.html',
+      label: 'test-file.html',
+    })).toBe(false);
+  });
+
+  it('gets file checksum', async() => {
+    const checksum = await fileServer.getFileChecksum({
+      dataType: 'file',
+      mimeType: 'text/typescript',
+      path: 'cjs/index.d.ts',
+      label: 'index.d.ts',
+    });
+    expect(typeof checksum).toBe('string');
+    expect(checksum).toHaveLength(32); // MD5 hash length
+  });
+
+  it('compares files for equality', async() => {
+    const fileA = {
+      dataType: 'file' as const,
+      mimeType: 'text/typescript',
+      path: 'cjs/index.d.ts',
+      label: 'index.d.ts',
+    };
+    
+    const fileB = {
+      dataType: 'file' as const,
+      mimeType: 'text/typescript',
+      path: 'cjs/index.d.ts', // Same file
+      label: 'index.d.ts',
+    };
+
+    const equal = await fileServer.filesEqual(fileA, fileB);
+    expect(equal).toBe(true);
+  });
+});

package/packages/utils/src/services/fileServer/localFileServer.ts

@@ -0,0 +1,159 @@
+/* cspell:ignore originalname */
+import crypto from 'crypto';
+import fs from 'fs';
+import https from 'https';
+import path from 'path';
+import cors from 'cors';
+import express from 'express';
+import multer from 'multer';
+import { v4 as uuid } from 'uuid';
+import { assertString } from '../../assertions';
+import { ConfigProviderForConfig, resolveConfigValue } from '../../config';
+import { ifDefined } from '../../guards';
+import { memoize } from '../../misc/helpers';
+import { FileServerAdapter, FileValue } from '.';
+
+export type Config = {
+  port?: string;
+  host?: string;
+  storagePrefix: string;
+};
+interface Initializer<C> {
+  dataDir: string;
+  configSpace?: C;
+}
+
+/* istanbul ignore next */
+const startServer = memoize((port: string, uploadDir: string) => {
+  // TODO - re-evaluate the `preservePath` behavior to match whatever s3 does
+  const upload = multer({dest: uploadDir, preservePath: true});
+  const fileServerApp = express();
+
+  fileServerApp.use(cors());
+  fileServerApp.use(express.static(uploadDir));
+  fileServerApp.post('/', upload.single('file'), async (req, res) => {
+    const file = req.file;
+
+    if (!file) {
+      return res.status(400).send({message: 'file is required'});
+    }
+
+    const destinationName = req.body.key.replace('${filename}', file.originalname);
+    const destinationPath = path.join(uploadDir, destinationName);
+    const destinationDirectory = path.dirname(destinationPath);
+
+    await fs.promises.mkdir(destinationDirectory, { recursive: true });
+    await fs.promises.rename(file.path, destinationPath);
+    res.status(201).send();
+  });
+
+  const server = https.createServer({
+    key: fs.readFileSync(assertString(process.env.SSL_KEY_FILE, new Error('ssl key is required for localFileServer')), 'utf8'),
+    cert: fs.readFileSync(assertString(process.env.SSL_CRT_FILE, new Error('ssl key is required for localFileServer')), 'utf8'),
+  }, fileServerApp);
+
+  server.once('error', function(err) {
+    if ((err as any).code === 'EADDRINUSE') {
+      // when the local dev server reloads files on every request it doesn't
+      // actually tear down the old modules, so this server only starts on the
+      // first execution and changes in its code will not be reloaded
+      return;
+    }
+    throw err;
+  });
+
+  server.listen(port);
+
+  return true;
+});
+
+export const localFileServer = <C extends string = 'local'>(initializer: Initializer<C>) => (configProvider: {[_key in C]: ConfigProviderForConfig<Config>}): FileServerAdapter => {
+  const config = configProvider[ifDefined(initializer.configSpace, 'local' as C)];
+  const port = resolveConfigValue(config.port || '');
+  const host = resolveConfigValue(config.host || '');
+  const storagePrefix = resolveConfigValue(config.storagePrefix);
+
+  const fileDir = storagePrefix.then((prefix) => path.join(initializer.dataDir, prefix));
+
+  Promise.all([port, fileDir])
+    .then(([port, fileDir]) => port && startServer(port, fileDir));
+
+  const getSignedViewerUrl = async (source: FileValue) => {
+    return `https://${await host}:${await port}/${source.path}`;
+  };
+  const getPublicViewerUrl = async (source: FileValue) => {
+    return `https://${await host}:${await port}/${source.path}`;
+  };
+  const getFileContent = async (source: FileValue) => {
+    const filePath = path.join(await fileDir, source.path);
+    return fs.promises.readFile(filePath);
+  };
+  const putFileContent = async (source: FileValue, content: string) => {
+    const filePath = path.join(await fileDir, source.path);
+    const directory = path.dirname(filePath);
+    await fs.promises.mkdir(directory, { recursive: true });
+    await fs.promises.writeFile(filePath, content);
+    return source;
+  };
+
+  const getSignedFileUploadConfig = async() => {
+    const prefix = 'uploads/' + uuid();
+    
+    return {
+      url: `https://${await host}:${await port}/`,
+      payload: {
+        key: prefix + '/${filename}',
+      }
+    };
+  };
+
+  const copyFileTo = async (source: FileValue, destinationPath: string) => {
+    const sourcePath = path.join(await fileDir, source.path);
+    const destPath = path.join(await fileDir, destinationPath);
+    const destDirectory = path.dirname(destPath);
+    
+    await fs.promises.mkdir(destDirectory, { recursive: true });
+    await fs.promises.copyFile(sourcePath, destPath);
+    
+    return {
+      ...source,
+      path: destinationPath
+    };
+  };
+
+  const copyFileToDirectory = async (source: FileValue, destination: string) => {
+    const destinationPath = path.join(destination, source.label);
+    return copyFileTo(source, destinationPath);
+  };
+
+  const isTemporaryUpload = (source: FileValue) => {
+    return source.path.indexOf('uploads/') === 0;
+  };
+
+  const getFileChecksum = async (source: FileValue) => {
+    const filePath = path.join(await fileDir, source.path);
+    const fileContent = await fs.promises.readFile(filePath);
+    return crypto.createHash('md5').update(fileContent).digest('hex');
+  };
+
+  const filesEqual = async (sourceA: FileValue, sourceB: FileValue) => {
+    const [aSum, bSum] = await Promise.all([
+      getFileChecksum(sourceA),
+      getFileChecksum(sourceB)
+    ]);
+    return aSum === bSum;
+  };
+
+  return {
+    getSignedViewerUrl,
+    getPublicViewerUrl,
+    getFileContent,
+    putFileContent,
+    getSignedFileUploadConfig,
+    copyFileTo,
+    copyFileToDirectory,
+    isTemporaryUpload,
+    getFileChecksum,
+    filesEqual,
+  };
+};

package/packages/utils/src/services/fileServer/s3FileServer.spec.ts

@@ -0,0 +1,266 @@
+/* cspell:ignore presigner */
+import * as awsSdk from '@aws-sdk/client-s3';
+import { s3FileServer } from './s3FileServer';
+
+jest.mock('@aws-sdk/client-s3', () => {
+  const sendSpy = jest.fn(() => Promise.reject(new Error('error'))) as any;
+   class S3Client {
+    static sendSpy = sendSpy;
+    send(...a: any[]) { return S3Client.sendSpy(...a); }
+  }
+  return {
+    ...jest.requireActual('@aws-sdk/client-s3'),
+    S3Client: S3Client,
+  };
+});
+
+jest.mock('@aws-sdk/s3-request-presigner', () => ({
+  getSignedUrl: () => Promise.resolve('cool url')
+}));
+
+jest.mock('@aws-sdk/s3-presigned-post', () => ({
+  createPresignedPost: () => Promise.resolve({
+    url: 'https://test.s3.amazonaws.com/',
+    fields: { key: 'uploads/test-uuid/${filename}', acl: 'private' }
+  })
+}));
+
+jest.mock('uuid', () => ({
+  v4: () => 'test-uuid'
+}));
+
+describe('s3FileServer', () => {
+  const bucketRegion = 'us-east-1';
+  const bucketName = 'test';
+  const path = 'build/index.html';
+  const defaultSendSpy: jest.SpyInstance = (awsSdk.S3Client as any).sendSpy;
+
+  const body = 'contents';
+  const Body = { transformToByteArray: () => Buffer.from(body) };
+  let send: jest.SpyInstance;
+  let getS3Client: () => awsSdk.S3Client;
+
+  beforeEach(() => {
+    send = jest.fn(({ input }) => {
+      const { Bucket, Key } = input;
+      if (Bucket !== bucketName) {
+        throw new awsSdk.NoSuchBucket({ $metadata: {}, message: 'The specified bucket does not exist.' });
+      }
+      if (Key !== path) {
+        throw new awsSdk.NoSuchKey({ $metadata: {}, message: 'The specified key does not exist.' });
+      }
+      return { Body };
+    });
+
+    getS3Client = (() => ({send})) as unknown as typeof getS3Client;
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('when the S3 bucket exists', () => {
+    let fileServer: ReturnType<ReturnType<typeof s3FileServer>>;
+
+    beforeEach(() => {
+      fileServer = s3FileServer({ configSpace: 'test', getS3Client })({ test: { bucketName, bucketRegion } });
+    });
+
+    it('gets S3 file contents', async() => {
+      const content = await fileServer.getFileContent({
+        dataType: 'file',
+        mimeType: 'text/html',
+        path,
+        label: 'index.html',
+      });
+      expect(content.toString()).toBe(body);
+    });
+
+    it('throws if the key does not exist', async() => {
+      await expect(fileServer.getFileContent({
+        dataType: 'file',
+        mimeType: 'text/html',
+        path: 'build/does-not-exist.html',
+        label: 'does-not-exist.html',
+      })).rejects.toThrow('The specified key does not exist.');
+    });
+
+    it('gets signed url', async() => {
+      const url = await fileServer.getSignedViewerUrl({
+        dataType: 'file',
+        mimeType: 'text/typescript',
+        path: 'cjs/index.d.ts',
+        label: 'index.d.ts',
+      });
+      expect(url).toBe('cool url');
+    });
+
+    it('throws when getting public viewer url without publicViewerDomain configured', async() => {
+      await expect(fileServer.getPublicViewerUrl({
+        dataType: 'file',
+        mimeType: 'text/typescript',
+        path: 'cjs/index.d.ts',
+        label: 'index.d.ts',
+      })).rejects.toThrow('Tried to get public viewer URL for cjs/index.d.ts but no publicViewerDomain configured');
+    });
+
+    it('puts file content', async() => {
+      send.mockImplementation(() => {});
+      await expect(fileServer.putFileContent({
+        dataType: 'file',
+        mimeType: 'text/typescript',
+        path: 'cjs/coolFile.d.ts',
+        label: 'coolFile.d.ts',
+      }, 'content')).resolves.not.toThrow();
+    });
+
+    it('gets signed upload config', async() => {
+      const config = await fileServer.getSignedFileUploadConfig();
+      expect(config).toEqual({
+        url: 'https://test.s3.amazonaws.com/',
+        payload: { key: 'uploads/test-uuid/${filename}', acl: 'private' }
+      });
+    });
+
+    it('copies file to destination', async() => {
+      send.mockImplementation(() => {});
+      const result = await fileServer.copyFileTo({
+        dataType: 'file',
+        mimeType: 'text/html',
+        path: 'uploads/test-file.html',
+        label: 'test-file.html',
+      }, 'permanent/test-file.html');
+      
+      expect(result).toEqual({
+        dataType: 'file',
+        mimeType: 'text/html',
+        path: 'permanent/test-file.html',
+        label: 'test-file.html',
+      });
+    });
+
+    it('copies file to directory', async() => {
+      send.mockImplementation(() => {});
+      const result = await fileServer.copyFileToDirectory({
+        dataType: 'file',
+        mimeType: 'text/html',
+        path: 'uploads/test-file.html',
+        label: 'test-file.html',
+      }, 'permanent/');
+      
+      expect(result).toEqual({
+        dataType: 'file',
+        mimeType: 'text/html',
+        path: 'permanent/test-file.html',
+        label: 'test-file.html',
+      });
+    });
+
+    it('identifies temporary uploads', () => {
+      expect(fileServer.isTemporaryUpload({
+        dataType: 'file',
+        mimeType: 'text/html',
+        path: 'uploads/test-file.html',
+        label: 'test-file.html',
+      })).toBe(true);
+
+      expect(fileServer.isTemporaryUpload({
+        dataType: 'file',
+        mimeType: 'text/html',
+        path: 'permanent/test-file.html',
+        label: 'test-file.html',
+      })).toBe(false);
+    });
+
+    it('gets file checksum', async() => {
+      send.mockImplementation(() => ({ ETag: '"checksum123"' }));
+      const checksum = await fileServer.getFileChecksum({
+        dataType: 'file',
+        mimeType: 'text/html',
+        path: 'test-file.html',
+        label: 'test-file.html',
+      });
+      expect(checksum).toBe('"checksum123"');
+    });
+
+    it('compares files for equality', async() => {
+      send.mockImplementation(() => ({ ETag: '"checksum123"' }));
+      
+      const fileA = {
+        dataType: 'file' as const,
+        mimeType: 'text/html',
+        path: 'fileA.html',
+        label: 'fileA.html',
+      };
+      
+      const fileB = {
+        dataType: 'file' as const,
+        mimeType: 'text/html', 
+        path: 'fileB.html',
+        label: 'fileB.html',
+      };
+
+      const equal = await fileServer.filesEqual(fileA, fileB);
+      expect(equal).toBe(true);
+    });
+  });
+
+  it('works without client override', async() => {
+    defaultSendSpy.mockImplementation(({ input }) => {
+      const { Bucket, Key } = input;
+      if (Bucket !== bucketName) {
+        throw new awsSdk.NoSuchBucket({ $metadata: {}, message: 'The specified bucket does not exist.' });
+      }
+      if (Key !== path) {
+        throw new awsSdk.NoSuchKey({ $metadata: {}, message: 'The specified key does not exist.' });
+      }
+      return { Body };
+    });
+
+    const fileServer = s3FileServer({ configSpace: 'test' })({ test: { bucketName, bucketRegion } });
+
+    const content = await fileServer.getFileContent({
+      dataType: 'file',
+      mimeType: 'text/html',
+      path,
+      label: 'index.html',
+    });
+    expect(content.toString()).toBe(body);
+
+    expect(send).not.toHaveBeenCalled();
+    expect(defaultSendSpy).toHaveBeenCalled();
+  });
+
+  it('throws if the bucket does not exist', async() => {
+    const fileServer = s3FileServer({ configSpace: 'test', getS3Client })(
+      { test: { bucketName: 'does-not-exist', bucketRegion } }
+    );
+
+    await expect(fileServer.getFileContent({
+      dataType: 'file',
+      mimeType: 'text/html',
+      path,
+      label: 'index.html',
+    })).rejects.toThrow('The specified bucket does not exist.');
+  });
+
+  describe('with publicViewerDomain configured', () => {
+    let fileServer: ReturnType<ReturnType<typeof s3FileServer>>;
+
+    beforeEach(() => {
+      fileServer = s3FileServer({ configSpace: 'test', getS3Client })({ 
+        test: { bucketName, bucketRegion, publicViewerDomain: 'cdn.example.com' } 
+      });
+    });
+
+    it('gets public viewer url', async() => {
+      const url = await fileServer.getPublicViewerUrl({
+        dataType: 'file',
+        mimeType: 'text/typescript',
+        path: 'cjs/index.d.ts',
+        label: 'index.d.ts',
+      });
+      expect(url).toBe('https://cdn.example.com/cjs/index.d.ts');
+    });
+  });
+});