@jpmorganchase/mosaic-core 0.1.0-beta.3

package/src/filesystems/RestrictedVolume.ts

@@ -0,0 +1,57 @@
+import type IFileAccess from '@jpmorganchase/mosaic-types/dist/IFileAccess';
+import type { IVolumePartiallyMutable } from '@jpmorganchase/mosaic-types/dist/Volume';
+
+import ImmutableFileSystem from './ImmutableVolume';
+import { create } from 'lodash';
+
+/**
+ * Restricted filesystems can be written to, but cannot be frozen, updated or reset
+ */
+class RestrictedVolume extends ImmutableFileSystem implements IVolumePartiallyMutable {
+  #vfs: IFileAccess;
+
+  constructor(vfs, namespace) {
+    super(vfs, namespace);
+    this.#vfs = vfs;
+  }
+
+  promises = create(this.promises, {
+    unlink: target => {
+      return this.#vfs.unlink(target);
+    },
+    symlink: (target, alias, type) => {
+      return this.#vfs.symlink(target, alias, type);
+    },
+    writeFile: (file, data) => {
+      return this.#vfs.writeFile(file, data);
+    },
+    mkdir: (dir, options) => {
+      return this.#vfs.mkdir(dir, options);
+    }
+  });
+
+  /**
+   * Do not use this method on restricted volumes, as it may disrupt the read/cache flow of files.
+   */
+  __internal_do_not_use_addReadFileHook(readFileHook) {
+    return this.#vfs.$$addReadFileHook(readFileHook);
+  }
+
+  symlinksToJSON(): { [key: string]: { target: string; type: string }[] } {
+    return this.#vfs.symlinksToJSON();
+  }
+
+  toJSON() {
+    return this.#vfs.toJSON();
+  }
+
+  get frozen() {
+    return this.#vfs.$$frozen;
+  }
+
+  asReadOnly() {
+    return new ImmutableFileSystem(this.#vfs, super.namespace);
+  }
+}
+
+export default RestrictedVolume;

package/src/filesystems/UnionFileAccess.ts

@@ -0,0 +1,77 @@
+import { IUnionFs, Union } from 'unionfs';
+import merge from 'lodash/merge';
+
+import type { IVolumeMutable } from '@jpmorganchase/mosaic-types/dist/Volume';
+
+import FileAccess from './FileAccess';
+import { IFS } from 'unionfs/lib/fs';
+
+export default class UnionFileAccess extends FileAccess {
+  #ufs: IUnionFs & { fss: IVolumeMutable[] };
+
+  constructor(ufs) {
+    super(ufs);
+    this.#ufs = ufs;
+  }
+  // We have to override this here, so we can hand-hold the glob and make sure it fires on all filesystems
+  async glob(pattern, options) {
+    return this.#ufs.fss.reduce(
+      async (globResults, childFs) => [
+        ...(await globResults),
+        ...(await childFs.promises.glob(pattern, options))
+      ],
+      Promise.resolve([])
+    );
+  }
+  /**
+   * Scopes a filesystem to specific namespaces
+   * @param {string[]} namespaces Array of namespaces to limit this filesystem to
+   * @returns A new immutable union filesystem
+   */
+  scope(namespaces: string[]) {
+    const scopedUnion: IUnionFs & { fss: IVolumeMutable[] } = new Union() as IUnionFs & {
+      fss: IVolumeMutable[];
+    };
+    const matchingNamespaceFSs = this.#ufs.fss.filter(fss => namespaces.includes(fss.namespace));
+
+    if (!matchingNamespaceFSs.length) {
+      throw new Error(
+        `Scoping failed. No matching filesystems for ${namespaces
+          .map(namespace => `"${namespace}"`)
+          .join(', ')}.`
+      );
+    }
+    matchingNamespaceFSs.forEach(fss => scopedUnion.use(fss as unknown as IFS));
+    return new UnionFileAccess(scopedUnion);
+  }
+  /**
+   * Reads 1 or more files (if multiple files exist at the same location within the union filesystem)
+   * @param file Path
+   * @param options.includeConflicts If multiple files exist at the same location in the filesystem, return them all as an array
+   * @returns Promise<TDataOut[]>
+   */
+  readFile(file, options?: { includeConflicts?: boolean }) {
+    if (!options?.includeConflicts) {
+      return super.readFile(file);
+    }
+    return this.#ufs.fss.reduce(
+      async (fileResults, childFs) => [
+        ...(await fileResults),
+        ...((await childFs.promises.exists(file)) ? [await childFs.promises.readFile(file)] : [])
+      ],
+      Promise.resolve([])
+    );
+  }
+  symlinksToJSON() {
+    return this.#ufs.fss.reduce(
+      (merged, filesystem) => merge(merged, filesystem.symlinksToJSON()),
+      {}
+    );
+  }
+  toJSON() {
+    return this.#ufs.fss.reduce(
+      (merged, filesystem) => ({ ...merged, ...filesystem.toJSON() }),
+      {}
+    );
+  }
+}

package/src/filesystems/UnionVolume.ts

@@ -0,0 +1,28 @@
+import type { IUnionVolume } from '@jpmorganchase/mosaic-types/dist/Volume';
+
+import type UnionFileAccess from './UnionFileAccess';
+import { create } from 'lodash';
+import { TDataOut } from 'memfs/lib/encoding';
+import MutableVolume from './MutableVolume';
+
+export default class UnionVolume extends MutableVolume implements IUnionVolume {
+  #vfs: UnionFileAccess;
+
+  constructor(vfs, namespace) {
+    super(vfs, namespace);
+    this.#vfs = vfs;
+  }
+
+  promises = create(this.promises, {
+    readFile: (file, options) => {
+      if (options?.includeConflicts) {
+        return this.#vfs.readFile(file, options) as Promise<TDataOut[]>;
+      }
+      return this.#vfs.readFile(file) as Promise<TDataOut>;
+    }
+  });
+
+  scope(namespaces: string[]) {
+    return new UnionVolume(this.#vfs.scope(namespaces), super.namespace);
+  }
+}

package/src/helpers/createConfig.ts

@@ -0,0 +1,52 @@
+import type MutableData from '@jpmorganchase/mosaic-types/dist/MutableData';
+import { ImmutableData } from '@jpmorganchase/mosaic-types/dist/MutableData';
+import { merge } from 'lodash';
+import path from 'path';
+
+export default function createConfig<T = {}>(initialData: Partial<T> = {}): MutableData<T> {
+  let data: { refs?: {}; globalRefs?: {}; aliases?: {} } = { refs: {}, globalRefs: {}, aliases: {}, ...initialData };
+
+  const configReadOnly = {
+    get data() {
+      return Object.freeze({ ...data });
+    }
+  } as ImmutableData<T>;
+
+  return {
+    asReadOnly() {
+      return configReadOnly;
+    },
+    setTags(fullPath: string, tags: string[]) {
+      data.aliases[fullPath] = new Set<string>(data.aliases[fullPath] || []);
+      tags.forEach(tag => data.aliases[fullPath].add(path.join('/.tags', tag, fullPath)));
+    },
+    setGlobalRef(targetPath, targetPropPath, refValue) {
+      data.globalRefs[targetPath] = data.globalRefs[targetPath] || [];
+      data.globalRefs[targetPath].push({
+        $$path: targetPropPath,
+        $$value: refValue
+      });
+    },
+    setRef(targetPath, targetPropPath, refValue) {
+      data.refs[targetPath] = data.refs[targetPath] || [];
+      data.refs[targetPath].push({
+        $$path: targetPropPath,
+        $$value: refValue
+      });
+    },
+    setAliases(fullPath: string, aliases: string[]) {
+      data.aliases[fullPath] = new Set<string>(data.aliases[fullPath] || []);
+      aliases.forEach(alias => data.aliases[fullPath].add(alias));
+    },
+    setData(value, overwrite = false) {
+      if (overwrite) {
+        data = value;
+      } else {
+        merge(data, value);
+      }
+    },
+    get data() {
+      return configReadOnly.data;
+    }
+  };
+}

package/src/index.ts

@@ -0,0 +1 @@
+export { default } from './PullDocs';

package/src/operators/ObservableWorker.ts

@@ -0,0 +1,24 @@
+import { Worker } from 'worker_threads';
+
+import { Observable } from 'rxjs';
+
+import type WorkerData from '@jpmorganchase/mosaic-types/dist/WorkerData';
+
+export function from<T>(workerData: WorkerData) {
+  return new Observable<T>(observer => {
+    const worker = new Worker(require.resolve('../worker/Source.worker'), { workerData });
+    worker.on('message', message => observer.next(message));
+    worker.once('error', error => observer.error(error));
+    worker.once('exit', code => {
+      if (code !== 0) {
+        observer.error(new Error(`mosaic source stopped with exit code ${code}`));
+      } else {
+        observer.complete();
+      }
+    });
+
+    return () => {
+      worker.terminate();
+    };
+  });
+}

package/src/plugin/createPluginAPI.ts

@@ -0,0 +1,75 @@
+import type { LoadedPlugin } from '@jpmorganchase/mosaic-types/dist/Plugin';
+import type { LoadedSerialiser } from '@jpmorganchase/mosaic-types/dist/Serialiser';
+import type Plugin from '@jpmorganchase/mosaic-types/dist/Plugin';
+import type PluginModuleDefinition from '@jpmorganchase/mosaic-types/dist/PluginModuleDefinition';
+import type Serialiser from '@jpmorganchase/mosaic-types/dist/Serialiser';
+import type Page from '@jpmorganchase/mosaic-types/dist/Page';
+
+import loadDefinitionModules from './loadDefinitionModules';
+import serialiserRunner from './serialiserRunner';
+import pluginRunner from './pluginRunner';
+
+function createProxyBaseSerialiserAPI(): Serialiser {
+  return {
+    serialise() {
+      throw new Error('This is just for the interface on the Proxy and should never be invoked.');
+    },
+    deserialise() {
+      throw new Error('This is just for the interface on the Proxy and should never be invoked.');
+    }
+  };
+}
+
+function createProxyBaseAPI<ConfigData>(): Plugin<ConfigData> {
+  return {
+    $afterSource() {
+      throw new Error('This is just for the interface on the Proxy and should never be invoked.');
+    },
+    shouldClearCache() {
+      throw new Error('This is just for the interface on the Proxy and should never be invoked.');
+    },
+    afterUpdate() {
+      throw new Error('This is just for the interface on the Proxy and should never be invoked.');
+    },
+    $beforeSend() {
+      throw new Error('This is just for the interface on the Proxy and should never be invoked.');
+    }
+  };
+}
+
+export default async function createPluginAPI<PluginInput, ConfigData = {}>(
+  plugins: PluginModuleDefinition[]
+): Promise<Plugin<ConfigData>> {
+  const loadedPlugins: LoadedPlugin[] | LoadedSerialiser[] = await loadDefinitionModules(plugins);
+  const baseObj = createProxyBaseAPI<ConfigData>();
+  const baseSerialiserObj = createProxyBaseSerialiserAPI();
+
+  return new Proxy(baseObj, {
+    get(obj, propOrLifecycleName) {
+      if (baseSerialiserObj.hasOwnProperty(propOrLifecycleName)) {
+        return async (pagePath, ...args: [string | Page, {}]) =>
+          await serialiserRunner(
+            {
+              serialiserMethod: String(propOrLifecycleName),
+              loadedPlugins: loadedPlugins as LoadedSerialiser[]
+            },
+            pagePath,
+            ...args
+          );
+      }
+      if (baseObj.hasOwnProperty(propOrLifecycleName)) {
+        return async (...args: [PluginInput, {}]) =>
+          await pluginRunner(
+            {
+              loadedPlugins: loadedPlugins as LoadedPlugin[],
+              lifecycleName: String(propOrLifecycleName)
+            },
+            ...args
+          );
+      }
+      if (typeof obj[propOrLifecycleName] !== 'undefined') {
+        return obj[propOrLifecycleName];
+      }
+    }
+  });
+}

package/src/plugin/index.ts

@@ -0,0 +1,41 @@
+import type { TDataOut } from 'memfs/lib/encoding';
+
+import type Page from '@jpmorganchase/mosaic-types/dist/Page';
+import type Serialiser from '@jpmorganchase/mosaic-types/dist/Serialiser';
+import type Plugin from '@jpmorganchase/mosaic-types/dist/Plugin';
+import type { IVolumeMutable } from '@jpmorganchase/mosaic-types/dist/Volume';
+import type PluginModuleDefinition from '@jpmorganchase/mosaic-types/dist/PluginModuleDefinition';
+
+import loadPlugins from './createPluginAPI';
+
+export async function bindSerialiser(serialisers): Promise<Serialiser> {
+  const pluginApi = (await loadPlugins<Page>(serialisers)) as Serialiser;
+
+  return {
+    async serialise(fullPath, page) {
+      return await pluginApi.serialise(fullPath, page);
+    },
+    async deserialise(fullPath, data) {
+      return await pluginApi.deserialise(fullPath, data);
+    }
+  };
+}
+
+export async function bindPluginMethods(plugins: PluginModuleDefinition[]): Promise<Plugin> {
+  const pluginApi = await loadPlugins<Page[] | TDataOut | IVolumeMutable>(plugins);
+
+  return {
+    async $afterSource(pages, args) {
+      return await pluginApi.$afterSource(pages, args);
+    },
+    async shouldClearCache(lastAfterUpdateReturn, args) {
+      return await pluginApi.shouldClearCache(lastAfterUpdateReturn, args);
+    },
+    async afterUpdate(mutableFilesystem, args) {
+      return await pluginApi.afterUpdate(mutableFilesystem, args);
+    },
+    async $beforeSend(mutableFilesystem, args) {
+      await pluginApi.$beforeSend(mutableFilesystem, args);
+    }
+  };
+}

package/src/plugin/loadDefinitionModules.ts

@@ -0,0 +1,58 @@
+import type PluginModuleDefinition from '@jpmorganchase/mosaic-types/dist/PluginModuleDefinition';
+import type SerialiserModuleDefinition from '@jpmorganchase/mosaic-types/dist/SerialiserModuleDefinition';
+import type Plugin from '@jpmorganchase/mosaic-types/dist/Plugin';
+import type { LoadedSerialiser, Serialiser } from '@jpmorganchase/mosaic-types/dist/Serialiser';
+import type { LoadedPlugin } from '@jpmorganchase/mosaic-types/dist/Plugin';
+
+const loadedPluginsAndSerialisers: { [key: string]: LoadedPlugin | LoadedSerialiser } = {};
+
+export default async function loadDefinitionModules(
+  plugins: (PluginModuleDefinition | SerialiserModuleDefinition)[]
+): Promise<LoadedPlugin[] | LoadedSerialiser[]> {
+  const results = [];
+
+  try {
+    for (const plugin of plugins) {
+      const { modulePath }: { modulePath: string } = plugin;
+      if (!loadedPluginsAndSerialisers[modulePath]) {
+        const {
+          default: definitionExports
+        }: {
+          default: Partial<Plugin | Serialiser> | { __esModule: boolean; default: Partial<Plugin | Serialiser> };
+        } = await import(modulePath);
+        const pluginApi: Partial<Plugin | Serialiser> =
+          '__esModule' in definitionExports && 'default' in definitionExports
+            ? definitionExports.default
+            : definitionExports;
+        if (!pluginApi) {
+          throw new Error(`Plugin or serialiser '${modulePath}' did not have a default export.`);
+        }
+
+        loadedPluginsAndSerialisers[modulePath] = createInvoker(plugin, pluginApi);
+      }
+      results.push(loadedPluginsAndSerialisers[modulePath]);
+    }
+    return results;
+  } catch (e) {
+    throw new Error(`Could not load plugin/serialiser modules due to error.
+${e}`);
+  }
+}
+
+function createInvoker(plugin, api) {
+  return new Proxy(plugin, {
+    get(obj, prop) {
+      if (typeof obj[prop] !== 'undefined') {
+        return obj[prop];
+      }
+
+      if (typeof api === 'function') {
+        return api;
+      }
+
+      if (prop in api) {
+        return api[prop];
+      }
+    }
+  });
+}

package/src/plugin/pluginRunner.ts

@@ -0,0 +1,52 @@
+import type { LoadedPlugin } from '@jpmorganchase/mosaic-types/dist/Plugin';
+
+import PluginError from '../PluginError';
+
+export default async function pluginRunner(
+  { loadedPlugins, lifecycleName }: { loadedPlugins: LoadedPlugin[]; lifecycleName: string },
+  input,
+  ...args
+) {
+  let transformedInput = input;
+
+  for (const plugin of loadedPlugins) {
+    try {
+      if (typeof input === 'undefined') {
+        throw new Error('Input was undefined.');
+      }
+
+      if (typeof plugin[lifecycleName] !== 'function') {
+        continue;
+      }
+
+      // console.debug(
+      //   `[PullDocs] Applying plugin method \`${lifecycleName}\`${lifecycleName.startsWith('$') ? ' (in a child worker)' : ''} for '${plugin.modulePath}'.`
+      // );
+
+      const result = await plugin[lifecycleName](
+        lifecycleName === '$afterSource' ? transformedInput : input,
+        ...args,
+        plugin.options
+      );
+
+      if (result && lifecycleName !== '$afterSource' && lifecycleName !== 'shouldClearCache') {
+        console.warn(
+          `[PullDocs] \`${lifecycleName}\` plugin should not return a value - this lifecycle phase expects mutation to occur directly on the filesystem instance. This will be ignored.`
+        );
+      }
+
+      transformedInput = result;
+    } catch (e) {
+      // This check will stop nested errors from ending up with multiple 'Plugin X threw an exception' headers from
+      // being prefixed to the messages
+      if (e instanceof PluginError) {
+        throw e;
+      }
+      throw new PluginError(
+        `Plugin '${plugin.modulePath}' threw an exception running \`${lifecycleName}\`. See below:
+  ${e.stack}`
+      );
+    }
+  }
+  return transformedInput;
+}

package/src/plugin/serialiserRunner.ts

@@ -0,0 +1,36 @@
+import type { LoadedSerialiser } from '@jpmorganchase/mosaic-types/dist/Serialiser';
+
+import PluginError from '../PluginError';
+
+export default async function serialiserRunner(
+    { loadedPlugins, serialiserMethod }: { serialiserMethod: string; loadedPlugins: LoadedSerialiser[] },
+    fullPath: string,
+    ...args
+  ) {
+    if (typeof fullPath !== 'string' || !/\/[^/]+\.\w{1,4}$/.test(fullPath)) {
+      throw new Error(`Invalid serialiser input. '${fullPath}' provided but a path string (with a valid file extension) was expected.`);
+    }
+    for (const serialiser of loadedPlugins) {
+      try {
+        if (!serialiser.filter.test(String(fullPath))) {
+          // Serialiser failed the path filter test
+          continue;
+        }
+        const result = await serialiser[serialiserMethod](fullPath, ...args, serialiser.options);
+        return result;
+      } catch (e) {
+        // This check will stop nested errors from ending up with multiple 'Plugin X threw an exception' headers from
+        // being prefixed to the messages
+        if (e instanceof PluginError) {
+          throw e;
+        }
+        throw new PluginError(
+          `Serialiser '${serialiser.modulePath}' threw an exception calling \`serialiser.${serialiserMethod}\` on '${fullPath}'.
+          
+  Is this the correct serialiser to use for this file type? See error below:
+  ${e.stack}`
+        );
+      }
+    }
+    throw new Error(`Could not find a suitable serialiser for file '${fullPath}'.`);
+  }

package/src/worker/Source.worker.ts

@@ -0,0 +1,124 @@
+import { parentPort, isMainThread, workerData as unTypedWorkerData } from 'worker_threads';
+
+import path from 'path';
+import fs from 'fs';
+import { map, tap, switchMap } from 'rxjs';
+import { DirectoryJSON, Volume } from 'memfs';
+
+import type WorkerData from '@jpmorganchase/mosaic-types/dist/WorkerData';
+import type Page from '@jpmorganchase/mosaic-types/dist/Page';
+
+import createSourceObservable from './helpers/createSourceObservable';
+import { bindSerialiser, bindPluginMethods } from '../plugin';
+import createConfig from '../helpers/createConfig';
+import FileAccess from '../filesystems/FileAccess';
+import MutableVolume from '../filesystems/MutableVolume';
+
+const workerData: WorkerData<{ cache: boolean }> = unTypedWorkerData;
+
+if (isMainThread) {
+  throw new Error('This module can only be called from a child process.');
+}
+(async () => {
+  let config;
+  let filesystem;
+  const serialiser = await bindSerialiser(workerData.serialisers);
+  const pluginApi = await bindPluginMethods(workerData.plugins);
+  const cachePath = path.join(process.cwd(), '.tmp', '.cache', `${workerData.name}.json`);
+
+  if (workerData.options.cache !== false) {
+  await fs.promises.mkdir(path.dirname(cachePath), { recursive: true });
+  }
+
+  (await createSourceObservable(workerData, serialiser))
+    .pipe(
+      tap(() => {
+        config = createConfig();
+      }),
+      switchMap((pages: Page[]) =>
+        pluginApi.$afterSource(pages, {
+          config,
+          serialiser,
+          ignorePages: workerData.ignorePages,
+          pageExtensions: workerData.pageExtensions
+        })
+      ),
+      switchMap((pages: Page[]) =>
+        pages.reduce(async (mergedPagesPromise, page) => {
+          const mergedPages = await mergedPagesPromise;
+          mergedPages[page.fullPath] = String(await serialiser.serialise(page.fullPath, page));
+          return mergedPages;
+        }, Promise.resolve({}))
+      ),
+      switchMap(async (mergedPages: DirectoryJSON) => {
+        filesystem = new MutableVolume(
+          new FileAccess(Volume.fromJSON(mergedPages)),
+          workerData.namespace
+        );
+        await pluginApi.$beforeSend(filesystem.asRestricted(), {
+          config,
+          ignorePages: workerData.ignorePages,
+          pageExtensions: workerData.pageExtensions,
+          serialiser
+        });
+        // In the main thread we would freeze the filesystem here, but since we throw it away after sending it to the parent process,
+        // we don't bother freezing
+        // Turn data into buffer
+        return Buffer.from(JSON.stringify({ pages: filesystem.toJSON(), data: config.data, symlinks: filesystem.symlinksToJSON() }));
+      }),
+      tap(() => {
+        // Reset filesystem and config memory
+        config = null;
+        filesystem.reset();
+        filesystem = null;
+      })
+    )
+    .subscribe(async (pagesAndSymlinks: Buffer) => {
+      if (workerData.options.cache !== false) {
+        console.info(`[PullDocs] Saving cached filesystem of ${workerData.name}`);
+        await fs.promises.writeFile(
+          cachePath,
+          pagesAndSymlinks
+        );
+      }
+
+      parentPort.postMessage({
+        type: 'message',
+        data: pagesAndSymlinks
+      }, /* transferList */ [pagesAndSymlinks.buffer]);
+    });
+
+  if (workerData.options.cache !== false) {
+    try {
+      if (await fs.promises.stat(cachePath)) {
+        const data = await fs.promises.readFile(cachePath);
+        console.info(`Restoring cached filesystem for ${workerData.name}`);
+        parentPort.postMessage({
+          type: 'init',
+          data
+        }, /* transferList */ [data.buffer]);
+      }
+      // Important: Return to avoid sending another init signal on L107
+      return;
+    } catch (e) {
+      // Does not exist
+    }
+  }
+  parentPort.postMessage({ type: 'init' });
+})();
+
+// Catch any unhandled errors and re-throw them
+process.once('uncaughtException', e => {
+  throw e;
+});
+
+// async function getPagesWithoutContent(pages: DirectoryJSON, serialiser: Serialiser): Promise<DirectoryJSON> {
+//   const pagesWithoutContent = {};
+//   for (const fullPath in pages) {
+//     const serialisedPage = pages[fullPath];
+//     const page = await serialiser.deserialise(fullPath, serialisedPage);
+//     page.content = '';
+//     pagesWithoutContent[fullPath] = String(await serialiser.serialise(fullPath, page));
+//   }
+//   return pagesWithoutContent;
+// }