@jpmorganchase/mosaic-core 0.1.0-beta.3

package/src/__tests__/WorkerSubscription.test.ts

@@ -0,0 +1,308 @@
+import { EventEmitter } from 'stream';
+import { Worker } from 'worker_threads';
+
+import WorkerSubscription, { EVENT } from '../WorkerSubscription';
+
+jest.mock('worker_threads');
+
+const utf8Encoder = new TextEncoder();
+
+describe('GIVEN WorkerSubscription', () => {
+  let subscription: WorkerSubscription;
+
+  beforeEach(() => {
+    subscription = new WorkerSubscription({
+      modulePath: 'module',
+      plugins: [],
+      options: { primary: true }
+    });
+  });
+
+  describe('WHEN calling `stop`', () => {
+    beforeEach(() => {
+      jest.spyOn(Worker.prototype, 'terminate');
+      subscription = new WorkerSubscription({
+        modulePath: 'module',
+        options: {},
+        plugins: []
+      });
+    });
+    afterEach(() => {
+      (Worker.prototype.terminate as jest.MockedFunction<any>).mockRestore();
+    });
+    test('THEN the worker should throw if the source is already stopped', async () => {
+      subscription.stop();
+      expect(() => subscription.stop()).toThrow(
+        new Error('Cannot stop source - subscription is already closed')
+      );
+    });
+    test('THEN the worker should be closed', () => {
+      subscription.stop();
+      expect(Worker.prototype.terminate).toHaveBeenCalledTimes(1);
+    });
+  });
+
+  describe('WHEN events fire', () => {
+    const workerHandlers: {
+      exit: (code: number) => void;
+      message: ({ type, data }: { type: string; data: string }) => void;
+      error: (error: Error) => void;
+    } = {
+      exit: null,
+      message: null,
+      error: null
+    };
+
+    afterEach(() => {
+      (Worker.prototype.on as jest.MockedFunction<any>).mockRestore();
+      (Worker.prototype.once as jest.MockedFunction<any>).mockRestore();
+      (Worker.prototype.terminate as jest.MockedFunction<any>).mockRestore();
+    });
+    beforeEach(() => {
+      jest.spyOn(Worker.prototype, 'terminate');
+      jest.spyOn(Worker.prototype, 'on').mockImplementation(function on(event: EVENT, handler) {
+        workerHandlers[event] = handler;
+        return this as EventEmitter;
+      } as any);
+      jest.spyOn(Worker.prototype, 'once').mockImplementation(function on(event: EVENT, handler) {
+        workerHandlers[event] = handler;
+        return this as EventEmitter;
+      } as any);
+      subscription = new WorkerSubscription({
+        modulePath: 'module',
+        options: {},
+        plugins: []
+      });
+    });
+
+    describe('AND exceptions occur', () => {
+      beforeEach(() => {
+        (Worker.prototype.terminate as jest.MockedFunction<any>).mockClear();
+      });
+      test('THEN the worker should terminate on exit event', () => {
+        workerHandlers.exit(0);
+        expect(Worker.prototype.terminate).toHaveBeenCalled();
+      });
+      test('THEN the worker should terminate on errors', () => {
+        workerHandlers.error(new Error('something broke'));
+        expect(Worker.prototype.terminate).toHaveBeenCalled();
+      });
+      test('THEN start handler should fire', () => {
+        const startSpy = jest.fn();
+        subscription.on(EVENT.START, startSpy);
+        workerHandlers.message({ data: utf8Encoder.encode('{ "key": "value"}'), type: 'init' });
+        workerHandlers.message({ data: utf8Encoder.encode('{ "key": "value"}'), type: 'init' });
+        expect(startSpy).toHaveBeenCalledTimes(2);
+      });
+      test('THEN exit handler should only fire once', () => {
+        const exitSpy = jest.fn();
+        subscription.on(EVENT.EXIT, exitSpy);
+        workerHandlers.exit(0);
+        workerHandlers.exit(0);
+        expect(exitSpy).toHaveBeenCalledTimes(1);
+      });
+      test('THEN error handler should only fire once', () => {
+        const errorSpy = jest.fn();
+        subscription.on(EVENT.ERROR, errorSpy);
+        workerHandlers.error(new Error('an exception that should only trigger an event once'));
+        workerHandlers.error(new Error('an exception that should only trigger an event once'));
+        expect(errorSpy).toHaveBeenCalledTimes(1);
+      });
+    });
+
+    describe('AND event handlers are attached', () => {
+      test('THEN the update handler should return a cleanup function', () => {
+        const updateSpy = jest.fn();
+        const cleanup = subscription.on(EVENT.UPDATE, updateSpy);
+        cleanup();
+        workerHandlers.message({ data: utf8Encoder.encode('{ "key": "value"}'), type: 'message' });
+        expect(updateSpy).not.toHaveBeenCalled();
+      });
+      test('THEN the error handler should return a cleanup function', () => {
+        const errorSpy = jest.fn();
+        const cleanup = subscription.on(EVENT.ERROR, errorSpy);
+        cleanup();
+        workerHandlers.error(new Error('something broke'));
+        expect(errorSpy).not.toHaveBeenCalled();
+      });
+      test('THEN the update handler should be fired ONCE for a once message handler', () => {
+        const updateSpy = jest.fn();
+        subscription.once(EVENT.UPDATE, updateSpy);
+        workerHandlers.message({ type: 'message', data: utf8Encoder.encode('{ "key": "value"}') });
+        workerHandlers.message({ type: 'message', data: utf8Encoder.encode('{ "key": "value"}') });
+        expect(updateSpy).toHaveBeenCalledTimes(1);
+      });
+      test('THEN the update handler should be fired on a message', () => {
+        const updateSpy = jest.fn();
+        subscription.on(EVENT.UPDATE, updateSpy);
+        workerHandlers.message({ type: 'message', data: utf8Encoder.encode('{ "key": "value"}') });
+        expect(updateSpy).toHaveBeenCalledWith({ data: { key: 'value' } });
+      });
+
+      test('THEN `stop` should unsubscribe all handlers', () => {
+        const exitSpy = jest.fn();
+        const errorSpy = jest.fn();
+        const updateSpy = jest.fn();
+        const startSpy = jest.fn();
+        subscription.on(EVENT.ERROR, errorSpy);
+        subscription.on(EVENT.EXIT, exitSpy);
+        subscription.on(EVENT.START, startSpy);
+        subscription.on(EVENT.UPDATE, updateSpy);
+        subscription.stop();
+        workerHandlers.error(new Error('i should never be handled'));
+        workerHandlers.message({ type: 'init', data: utf8Encoder.encode('{ "key": "value"}') });
+        workerHandlers.message({ type: 'message', data: utf8Encoder.encode('{ "key": "value"}') });
+        workerHandlers.exit(0);
+        expect(exitSpy).toHaveBeenCalled();
+        expect(errorSpy).not.toHaveBeenCalled();
+        expect(updateSpy).not.toHaveBeenCalled();
+        expect(startSpy).not.toHaveBeenCalled();
+      });
+
+      test('THEN the error handler should unsubscribe all other handlers and invoke exit', async () => {
+        const exitSpy = jest.fn();
+        const errorSpy = jest.fn();
+        subscription.on(EVENT.ERROR, errorSpy);
+        subscription.on(EVENT.EXIT, exitSpy);
+        workerHandlers.error(new Error('some error'));
+        workerHandlers.exit(0);
+        workerHandlers.error(new Error('i should never be handled'));
+        expect(exitSpy).toHaveBeenCalledTimes(1);
+        expect(errorSpy).toHaveBeenCalledTimes(1);
+      });
+
+      test('THEN the exit handler should unsubscribe all other handlers', async () => {
+        const exitSpy = jest.fn();
+        const errorSpy = jest.fn();
+        const updateSpy = jest.fn();
+        const startSpy = jest.fn();
+        subscription.on(EVENT.ERROR, errorSpy);
+        subscription.on(EVENT.EXIT, exitSpy);
+        subscription.on(EVENT.START, startSpy);
+        subscription.on(EVENT.UPDATE, updateSpy);
+        workerHandlers.exit(0);
+        workerHandlers.error(new Error('i should never be handled'));
+        workerHandlers.message({ type: 'message', data: 'i should be ignored' });
+        workerHandlers.message({ data: utf8Encoder.encode('{ "key": "value"}'), type: 'init' });
+        expect(exitSpy).toHaveBeenCalledTimes(1);
+        expect(errorSpy).not.toHaveBeenCalled();
+        expect(startSpy).not.toHaveBeenCalled();
+        expect(updateSpy).not.toHaveBeenCalled();
+      });
+
+      test('THEN an exception should throw for an invalid signal', async () => {
+        const errorSpy = jest.fn();
+        subscription.on(EVENT.ERROR, errorSpy);
+        const updateSpy = jest.fn();
+        subscription.on(EVENT.UPDATE, updateSpy);
+        workerHandlers.message({ data: 'something', type: 'invalid-signal' });
+        workerHandlers.message({ data: 'should-not-be-received', type: 'message' });
+        expect(errorSpy).toHaveBeenCalledWith(
+          new Error(
+            "Unsupported type 'invalid-signal' with value of 'something' received from source worker."
+          )
+        );
+        expect(updateSpy).not.toHaveBeenCalled();
+      });
+
+      test('THEN the start handler should be fired when a worker received an init signal', () => {
+        const startSpy = jest.fn();
+        subscription.on(EVENT.START, startSpy);
+        workerHandlers.message({ data: utf8Encoder.encode('{ "key": "value"}'), type: 'init' });
+        expect(startSpy).toHaveBeenCalledTimes(1);
+      });
+
+      test('THEN the exit handler should be fired when a worker emits an error', () => {
+        const exitSpy = jest.fn();
+        subscription.on(EVENT.EXIT, exitSpy);
+        workerHandlers.error(new Error('an exception that should cause an exit'));
+        expect(exitSpy).toHaveBeenCalledTimes(1);
+      });
+
+      test('THEN the exit handler should be fired when a worker emits an error (exit code > 0)', () => {
+        const exitSpy = jest.fn();
+        subscription.on(EVENT.EXIT, exitSpy);
+        const errorSpy = jest.fn();
+        subscription.on(EVENT.ERROR, errorSpy);
+        workerHandlers.exit(1);
+        expect(errorSpy).toHaveBeenCalledWith(new Error('mosaic source stopped with exit code 1'));
+        expect(exitSpy).toHaveBeenCalledTimes(1);
+      });
+
+      test('THEN the exit handler should be fired when a worker is complete', () => {
+        const exitSpy = jest.fn();
+        subscription.on(EVENT.EXIT, exitSpy);
+        workerHandlers.exit(0);
+        expect(exitSpy).toHaveBeenCalledTimes(1);
+      });
+
+      test('THEN the error handler should be fired on an error', () => {
+        const error = new Error('something broke');
+        const errorSpy = jest.fn();
+        subscription.on(EVENT.ERROR, errorSpy);
+        workerHandlers.error(error);
+        expect(errorSpy).toHaveBeenCalledWith(error);
+      });
+    });
+  });
+
+  test('THEN it should instantiate correctly', () => {
+    expect(
+      new WorkerSubscription({
+        modulePath: 'module',
+        options: {},
+        plugins: []
+      })
+    ).toBeDefined();
+  });
+
+  test('THEN it should have a once method', () => {
+    expect(
+      new WorkerSubscription({
+        modulePath: 'module',
+        options: {},
+        plugins: []
+      })
+    ).toHaveProperty('once');
+  });
+
+  test('THEN it should have a on method', () => {
+    expect(
+      new WorkerSubscription({
+        modulePath: 'module',
+        options: {},
+        plugins: []
+      })
+    ).toHaveProperty('on');
+  });
+
+  test('THEN it should have a close getter', () => {
+    expect(
+      new WorkerSubscription({
+        modulePath: 'module',
+        options: {},
+        plugins: []
+      })
+    ).toHaveProperty('closed');
+  });
+
+  test('THEN it should have a stop method', () => {
+    expect(
+      new WorkerSubscription({
+        modulePath: 'module',
+        options: {},
+        plugins: []
+      })
+    ).toHaveProperty('stop');
+  });
+
+  test('THEN the child worker should be created', () => {
+    expect(Worker).toHaveBeenCalledWith(expect.any(String), {
+      workerData: {
+        plugins: [],
+        modulePath: 'module',
+        options: { primary: true }
+      }
+    });
+  });
+});

package/src/filesystems/FileAccess.ts

@@ -0,0 +1,256 @@
+import path from 'path';
+import glob from 'fast-glob';
+import { PathLike } from 'fs';
+import { createFsFromVolume } from 'memfs';
+import type { DirectoryJSON, IRealpathOptions, Volume } from 'memfs/lib/volume';
+import type { TDataOut } from 'memfs/lib/encoding';
+
+import type IFileAccess from '@jpmorganchase/mosaic-types/dist/IFileAccess';
+
+export default class FileAccess implements IFileAccess {
+  #adapter: Volume;
+  #hooks: ((filepath: PathLike, fileData: TDataOut) => Promise<TDataOut>)[] = [];
+  #rawReadonlyFs;
+  #symlinks: { [key: string]: { target: string; type: string }[] } = {};
+  #frozen = false;
+  #cachedPages = new Map();
+
+  constructor(filesystemAdapter: Volume) {
+    this.#adapter = filesystemAdapter;
+  }
+
+  async readFile(file) {
+    const resolvedPath = await this.#resolvePath(file);
+    return await this.#getFileFromCache(resolvedPath);
+    //return this.#adapter.promises.readFile(resolvedPath);
+  }
+
+  async glob(pattern, options) {
+    // We need a created filesystem, not a Volume, for glob
+    this.#rawReadonlyFs = this.#rawReadonlyFs || createFsFromVolume(this.#adapter as any);
+    const paths = Array.from(
+      new Set(
+        await Promise.all(
+          (
+            await glob(pattern, {
+              ...options,
+              absolute: true,
+              fs: this.#rawReadonlyFs
+            })
+          ).map(filepath => this.#resolvePath(filepath))
+        )
+      )
+    );
+
+    return paths;
+  }
+
+  $$addReadFileHook(hook) {
+    // if (this.#frozen) {
+    //   throw new Error('This file system has been frozen. Mutations are not allowed.');
+    // }
+    this.#hooks.push(hook);
+  }
+
+  readdir(dir, options) {
+    return this.#adapter.promises.readdir(dir, options);
+  }
+
+  async stat(file, options) {
+    return this.#adapter.promises.stat(file, options);
+  }
+
+  get $$frozen() {
+    return this.#frozen;
+  }
+
+  realpath(target, options?: string | IRealpathOptions) {
+    return this.#adapter.promises.realpath(target, options);
+  }
+
+  async $$symlinksFromJSON(symlinks) {
+    this.#symlinks = symlinks;
+    if (this.#frozen) {
+      throw new Error('This file system has been frozen. Mutations are not allowed.');
+    }
+    for (const alias in symlinks) {
+      for (const { target, type = 'file' } of symlinks[alias]) {
+        if (!(await this.exists(alias))) {
+          if (!(await this.exists(path.dirname(alias)))) {
+            // TODO: This works... But why are the subdirs not being passed from the child process as `null`s?
+            await this.#adapter.promises.mkdir(path.dirname(alias), { recursive: true });
+          }
+          await this.#adapter.promises.symlink(target, alias, type);
+        }
+      }
+    }
+  }
+
+  async unlink(target) {
+    if (this.#frozen) {
+      throw new Error('This file system has been frozen. Mutations are not allowed.');
+    }
+    await this.#adapter.promises.unlink(target);
+    // If path refers to a symbolic link, then the link is removed without affecting the file or directory to which that link refers.
+    this.#cachedPages.delete(target);
+  }
+
+  symlink(target, alias, type) {
+    if (this.#frozen) {
+      throw new Error('This file system has been frozen. Mutations are not allowed.');
+    }
+    this.#symlinks[alias] = this.#symlinks[alias] || [];
+    this.#symlinks[alias].push({ target, type });
+    // console.debug('[PullDocs]', target, 'aliased to', alias);
+    return this.#adapter.promises.symlink(target, alias, type);
+  }
+
+  async writeFile(file, data) {
+    if (this.#frozen) {
+      throw new Error('This file system has been frozen. Mutations are not allowed.');
+    }
+    await this.#adapter.promises.writeFile(file, data);
+
+    this.#cachedPages.delete(file);
+    // TODO: Do we need?
+    this.#cachedPages.delete(await this.#resolvePath(file));
+  }
+
+  async mkdir(dir, options) {
+    if (this.#frozen) {
+      throw new Error('This file system has been frozen. Mutations are not allowed.');
+    }
+    return this.#adapter.promises.mkdir(dir, options);
+  }
+
+  async exists(file) {
+    try {
+      return !!(await this.#adapter.promises.realpath(file));
+    } catch (e) {
+      if (e.code !== 'ENOENT') {
+        throw e;
+      }
+    }
+    return false;
+  }
+
+  toJSON() {
+    return this.#adapter.toJSON();
+  }
+
+  symlinksToJSON() {
+    return this.#symlinks;
+  }
+
+  async #resolvePath(file) {
+    const realPath = await this.#adapter.promises.realpath(file);
+
+    if (
+      realPath !== file &&
+      !(await this.#adapter.promises.stat(realPath, { throwIfNoEntry: true }))
+    ) {
+      throw new Error(`ENOENT: no such file or directory, stat '${file}'`);
+    }
+
+    const stat = await this.#adapter.promises.stat(realPath, { throwIfNoEntry: true });
+    // Enable this for /index directories ... but should a filesystem really rewrite URLs like that?
+    // The downside is that relative links can break when /index is missing from the end of a URL
+    // e.g. if /home/dir is actually /home/dir/index any relative links will be from ./home not ./home/dir
+    // if (stat.isDirectory()) {
+    //   try {
+    //     const resolved = await this.#resolvePath(path.join(String(realPath), 'index'));
+
+    //     return resolved;
+    //   } catch {}
+    // }
+    if (!stat.isFile()) {
+      return file;
+      //throw new Error(`EISDIR: illegal operation on a directory, open '${file}'`);
+    }
+    return realPath;
+  }
+
+  async #getFileFromCache(file) {
+    const stat = await this.#adapter.promises.stat(file, { throwIfNoEntry: true });
+
+    if (!stat.isFile()) {
+      return null;
+    }
+
+    if (await this.#isNonHiddenPageCached(file)) {
+      const cachedItem = this.#cachedPages.get(file);
+      return cachedItem;
+    }
+
+    // Bit of parallel programming needed here, to avoid 2+ requests triggering a file load at the same time and making a race condition
+    const loadPagePromise = new Promise(async resolve => {
+      const rawFile = await this.#adapter.promises.readFile(file);
+
+      // if (!this.#pageTest(file)) {
+      //   return resolve(rawFile);
+      // }
+      let fileData = rawFile;
+
+      for (const hook of this.#hooks) {
+        fileData = await hook(file, fileData);
+      }
+      return resolve(fileData);
+    });
+
+    this.#cachedPages.set(file, loadPagePromise);
+
+    const result = await loadPagePromise;
+
+    if (this.#frozen) {
+      this.#cachedPages.set(file, result);
+    } else {
+      // Clear the `currentlyResolving` entry
+      this.#cachedPages.delete(file);
+    }
+
+    return result;
+
+    // this.#cachedPages.add(file);
+    //return this.#adapter.promises.writeFile(file, JSON.stringify(currentPage));
+  }
+
+  async #isNonHiddenPageCached(fileArg) {
+    const file = await this.#adapter.promises.realpath(fileArg);
+
+    return this.#cachedPages.has(file);
+  }
+
+  $$freeze() {
+    if (this.#frozen) {
+      throw new Error('This file system has already been frozen.');
+    }
+    this.#frozen = true;
+  }
+
+  $$unfreeze() {
+    if (!this.#frozen) {
+      throw new Error('This file system is not frozen.');
+    }
+    this.#frozen = false;
+  }
+
+  $$clearCache() {
+    this.#cachedPages.clear();
+  }
+
+  async $$reset() {
+    this.#rawReadonlyFs = null;
+    this.#frozen = false;
+    this.#symlinks = {};
+    this.#hooks = [];
+    this.#adapter.reset();
+    this.#cachedPages.clear();
+  }
+
+  fromJSON(json: DirectoryJSON) {
+    if (this.#frozen) {
+      throw new Error('This file system has been frozen. Mutations are not allowed.');
+    }
+    this.#adapter.fromJSON(json);
+  }
+}

package/src/filesystems/ImmutableVolume.ts

@@ -0,0 +1,45 @@
+import type { IVolumeImmutable } from '@jpmorganchase/mosaic-types/dist/Volume';
+import type IFileAccess from '@jpmorganchase/mosaic-types/dist/IFileAccess';
+
+export default class ImmutableVolume implements IVolumeImmutable {
+  #vfs: IFileAccess;
+  namespace: string;
+  
+  constructor(vfs, namespace) {
+    this.#vfs = vfs;
+    this.namespace = namespace;
+  }
+
+  promises: IVolumeImmutable['promises'] = {
+    readFile: file => {
+      return this.#vfs.readFile(file);
+    },
+    readdir: (dir, options) => {
+      return this.#vfs.readdir(dir, options);
+    },
+    stat: (file, options) => {
+      return this.#vfs.stat(file, options);
+    },
+    realpath: (target, options) => {
+      return this.#vfs.realpath(target, options);
+    },
+    exists: file => {
+      return this.#vfs.exists(file);
+    },
+    glob: (pattern, options) => {
+      return this.#vfs.glob(pattern, options);
+    }
+  };
+
+  get frozen() {
+    return this.#vfs.$$frozen;
+  }
+
+  symlinksToJSON(): { [key: string]: { target: string; type: string }[] } {
+    return this.#vfs.symlinksToJSON();
+  }
+
+  toJSON() {
+    return this.#vfs.toJSON();
+  }
+}

package/src/filesystems/MutableVolume.ts

@@ -0,0 +1,51 @@
+import type { DirectoryJSON } from 'memfs/lib/volume';
+
+import type { IVolumeMutable } from '@jpmorganchase/mosaic-types/dist/Volume';
+import type IFileAccess from '@jpmorganchase/mosaic-types/dist/IFileAccess';
+
+import FileSystemRestricted from './RestrictedVolume';
+
+export default class MutableVolume extends FileSystemRestricted implements IVolumeMutable {
+  #vfs: IFileAccess;
+
+  constructor(vfs, namespace) {
+    super(vfs, namespace);
+    this.#vfs = vfs;
+  }
+
+  asRestricted() {
+    return new FileSystemRestricted(this.#vfs, super.namespace);
+  }
+
+  freeze() {
+    this.#vfs.$$freeze();
+  }
+
+  clearCache() {
+    this.#vfs.$$clearCache();
+  }
+
+  unfreeze() {
+    this.#vfs.$$unfreeze();
+  }
+
+  __internal_do_not_use_addReadFileHook(hook) {
+    this.#vfs.$$addReadFileHook(hook);
+  }
+
+  symlinksToJSON(): { [key: string]: { target: string; type: string }[] } {
+    return this.#vfs.symlinksToJSON();
+  }
+
+  async symlinksFromJSON(json: { [key: string]: { target: string; type: string }[] }) {
+    return await this.#vfs.$$symlinksFromJSON(json);
+  }
+
+  fromJSON(json: DirectoryJSON) {
+    return this.#vfs.fromJSON(json);
+  }
+
+  reset() {
+    this.#vfs.$$reset();
+  }
+}