@jpmorganchase/mosaic-plugins 0.1.0-beta.3

package/.turbo/turbo-build.log

@@ -0,0 +1,14 @@
+@jpmorganchase/mosaic-plugins:build: cache hit, replaying output 1ab365469776992b
+@jpmorganchase/mosaic-plugins:build: $ yarn clean && tsc && swc ./src/*.mjs.ts -d ./dist -C module.type=es6 -C sourceMaps=true && renamer --find '.mjs.js' --replace '.mjs' './dist/*.mjs.js' && renamer --find '.mjs.js.map' --replace '.mjs.map' './dist/*.mjs.js.map'
+@jpmorganchase/mosaic-plugins:build: $ rm -fr dist
+@jpmorganchase/mosaic-plugins:build: Successfully compiled: 2 files with swc (30.78ms)
+@jpmorganchase/mosaic-plugins:build: ✔︎ ./dist/ReadingTimePlugin.mjs.js → dist/ReadingTimePlugin.mjs 
+@jpmorganchase/mosaic-plugins:build: ✔︎ ./dist/TableOfContentsPlugin.mjs.js → dist/TableOfContentsPlugin.mjs 
+@jpmorganchase/mosaic-plugins:build: 
+@jpmorganchase/mosaic-plugins:build: Rename complete: 2 of 2 files renamed.
+@jpmorganchase/mosaic-plugins:build: 
+@jpmorganchase/mosaic-plugins:build: ✔︎ ./dist/ReadingTimePlugin.mjs.js.map → dist/ReadingTimePlugin.mjs.map 
+@jpmorganchase/mosaic-plugins:build: ✔︎ ./dist/TableOfContentsPlugin.mjs.js.map → dist/TableOfContentsPlugin.mjs.map 
+@jpmorganchase/mosaic-plugins:build: 
+@jpmorganchase/mosaic-plugins:build: Rename complete: 2 of 2 files renamed.
+@jpmorganchase/mosaic-plugins:build: 

package/CHANGELOG.md

@@ -0,0 +1,34 @@
+# @jpmorganchase/mosaic-plugins
+
+## 0.1.0-beta.3
+
+### Patch Changes
+
+- 457df5e: switch to public package
+- Updated dependencies [457df5e]
+  - @jpmorganchase/mosaic-types@0.1.0-beta.3
+
+## 0.1.0-beta.2
+
+### Patch Changes
+
+- e1bbbe7: Initial release of Mosaic Core file-system.
+
+  Mosaic is a content aggregating service which pulls content from heterogeneous data sources.
+
+- 4b2bc51: pipped to 0.1.0-beta.1 for publishing to NPM
+- Updated dependencies [e1bbbe7]
+- Updated dependencies [4b2bc51]
+  - @jpmorganchase/mosaic-types@0.1.0-beta.2
+
+## 0.1.0
+
+### Patch Changes
+
+- e1bbbe7: Initial release of Mosaic Core file-system.
+
+  Mosaic is a content aggregating service which pulls content from heterogeneous data sources.
+
+- Updated dependencies [e1bbbe7]
+- Updated dependencies [bbc853c]
+  - @jpmorganchase/mosaic-types@0.1.0

package/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "@jpmorganchase/mosaic-plugins",
+  "version": "0.1.0-beta.3",
+  "author": "",
+  "description": "",
+  "license": "Apache-2.0",
+  "main": "./dist/index.js",
+  "engines": {
+    "node": ">= 12.0.0"
+  },
+  "scripts": {
+    "build": "yarn clean && tsc && swc ./src/*.mjs.ts -d ./dist -C module.type=es6 -C sourceMaps=true && renamer --find '.mjs.js' --replace '.mjs' './dist/*.mjs.js' && renamer --find '.mjs.js.map' --replace '.mjs.map' './dist/*.mjs.js.map'",
+    "clean": "rm -fr dist"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git@github.com:jpmorganchase/mosaic.git"
+  },
+  "devDependencies": {
+    "@swc/cli": "^0.1.55",
+    "@swc/core": "^1.2.237",
+    "renamer": "^4.0.0"
+  },
+  "dependencies": {
+    "@apidevtools/json-schema-ref-parser": "^9.0.9",
+    "@jpmorganchase/mosaic-types": "^0.1.0-beta.3",
+    "@types/github-slugger": "^1.3.0",
+    "github-slugger": "^1.4.0",
+    "hast-util-to-string": "^2.0.0",
+    "lodash": "^4.17.21",
+    "memfs": "^3.3.0",
+    "reading-time": "^1.5.0",
+    "rxjs": "^7.5.5"
+  }
+}

package/src/$AliasPlugin.ts

@@ -0,0 +1,62 @@
+import path from 'path';
+
+import type Page from '@jpmorganchase/mosaic-types/dist/Page';
+import type PluginType from '@jpmorganchase/mosaic-types/dist/Plugin';
+import { escapeRegExp } from 'lodash';
+
+/**
+ * Plugin that scrapes `aliases` from page metadata and also applies all aliases stored in `config.data.aliases`
+ * Other plugins can use `setAliases` to apply new aliases, as long as they call it before this plugin has reaches `$beforeSend`
+ */
+const $AliasPlugin: PluginType<{ aliases: { [key: string]: Set<string> } }> = {
+  async $afterSource(pages: Page<{ aliases?: string[] }>[], { config, ignorePages, pageExtensions }, options) {
+    const isNonHiddenPage = createPageTest(ignorePages, pageExtensions);
+
+    // Group together all aliases defined in the frontmatter and store them in the alias config object
+    for (const page of pages) {
+      if (!isNonHiddenPage(page.fullPath)) {
+        continue;
+      }
+      if (page.aliases) {
+        config.setAliases(page.fullPath, page.aliases);
+      }
+      delete page.aliases;
+    }
+    return pages;
+  },
+
+  async $beforeSend(mutableFilesystem, { config }) {
+    // Prevent any more aliases being set after this lifecycle has been called - as they won't take effect
+    config.setAliases = () => {
+      throw new Error('Cannot set aliases after `$AliasPlugin` has reached `$beforeSend` lifecycle phase.');
+    };    if (!config.data.aliases) {
+      return;
+    }
+    for (const fullPath in config.data.aliases) {
+      for (const alias of config.data.aliases[fullPath]) {
+        const aliasPath = path.resolve(path.dirname(fullPath), alias).toLowerCase();
+        const aliasDir = path.dirname(aliasPath);
+        if (!(await mutableFilesystem.promises.exists(aliasDir))) {
+          await mutableFilesystem.promises.mkdir(aliasDir, {
+            recursive: true
+          });
+        }
+        if (!(await mutableFilesystem.promises.exists(aliasPath))) {
+          await mutableFilesystem.promises.symlink(fullPath, aliasPath);
+        } else {
+          console.warn(`Alias '${aliasPath}' already exists. Is there a duplicate alias, or a page with the same name but different file extensions?`);
+        }
+      }
+    }
+  }
+};
+
+export default $AliasPlugin;
+
+function createPageTest(ignorePages, pageExtensions) {
+  const extTest = new RegExp(`${pageExtensions.map(escapeRegExp).join('|')}$`);
+  const ignoreTest = new RegExp(`${ignorePages.map(escapeRegExp).join('|')}$`);
+  return file => {
+    return !ignoreTest.test(file) && extTest.test(file) && !path.basename(file).startsWith('.');
+  };
+}

package/src/$CodeModPlugin.ts

@@ -0,0 +1,34 @@
+import type PluginType from '@jpmorganchase/mosaic-types/dist/Plugin';
+import type Page from '@jpmorganchase/mosaic-types/dist/Page';
+import { escapeRegExp } from 'lodash';
+import path from 'path';
+
+const $CodeModPlugin: PluginType<{
+  nextPrev: { [key: string]: string[] };
+  refs: { [key: string]: { $$path: (number | string)[]; $$value: string }[] };
+}> = {
+  async $afterSource(pages: Page[], { ignorePages, pageExtensions }) {
+    const isNonHiddenPage = createPageTest(ignorePages, pageExtensions);
+
+    for (const page of pages) {
+      if (!isNonHiddenPage(page.fullPath)) {
+        continue;
+      }
+      if (page.frameOverrides) {
+        page.sharedConfig = page.frameOverrides;
+        page.frameOverrides = { $ref: '#/sharedConfig' };
+      }
+    }
+    return pages;
+  }
+};
+
+export default $CodeModPlugin;
+
+function createPageTest(ignorePages, pageExtensions) {
+  const extTest = new RegExp(`${pageExtensions.map(escapeRegExp).join('|')}$`);
+  const ignoreTest = new RegExp(`${ignorePages.map(escapeRegExp).join('|')}$`);
+  return file => {
+    return !ignoreTest.test(file) && extTest.test(file) && !path.basename(file).startsWith('.');
+  };
+}

package/src/$RefPlugin.ts

@@ -0,0 +1,200 @@
+import path from 'path';
+
+import reduce from 'lodash/reduce';
+import omit from 'lodash/omit';
+
+import escapeRegExp from 'lodash/escapeRegExp';
+import $RefParser from '@apidevtools/json-schema-ref-parser';
+
+import type PluginType from '@jpmorganchase/mosaic-types/dist/Plugin';
+import type Page from '@jpmorganchase/mosaic-types/dist/Page';
+
+import normaliseRefs from './utils/normaliseRefs';
+
+/**
+ * Plugin that scrapes `$ref` properties from page metadata and also applies all refs stored in `config.data.refs`.
+ * Other plugins can use `setRef` to apply new refs, as long as they call it before this plugin has reaches `$beforeSend`
+ */
+const $RefPlugin: PluginType<{
+  refs: { [key: string]: { $$path: string[]; $$value: string }[] };
+  globalRefs: { [key: string]: { $$path: string[]; $$value: string }[] };
+  subscribedTags: string[];
+}> = {
+  async afterUpdate(
+    mutableFilesystem,
+    { ignorePages, globalFilesystem, serialiser, pageExtensions, globalConfig }
+  ) {
+    mutableFilesystem.__internal_do_not_use_addReadFileHook(async (pagePath: string, fileData) => {
+      if (globalConfig.data.globalRefs[pagePath]) {
+        const normalisedRef = {
+          ...(await serialiser.deserialise(pagePath, fileData)),
+          ...(await normaliseRefs(
+            String(pagePath),
+            globalConfig.data.globalRefs[pagePath],
+            globalFilesystem,
+            pageExtensions,
+            ignorePages
+          ))
+        } as any;
+        const resolve = createRefResolver({
+          [pagePath]: normalisedRef
+        }, serialiser, globalFilesystem);
+        const refParser = new $RefParser();
+
+        try {
+          const resolved: $RefParser.JSONSchema = await refParser.dereference(
+            String(pagePath),
+            normalisedRef,
+            {
+              resolve,
+              dereference: { circular: false }
+            }
+          );
+          return serialiser.serialise(pagePath, { ...normalisedRef, ...resolved });
+        } catch (e) {
+          console.warn(
+            `Error resolving ref(s) for page '${pagePath}'. ${e.message.replace(/\.$/, '')} in '${
+              e.source
+            }'`
+          );
+          throw e;
+        }
+      }
+
+      return fileData;
+    });
+  },
+  async $beforeSend(
+    mutableFilesystem,
+    { config, serialiser, pageExtensions, ignorePages },
+    options
+  ) {
+    // Prevent any more refs being set after this lifecycle has been called - as they won't take effect
+    config.setRef = () => {
+      throw new Error(
+        'Cannot set refs after `RefPlugin` has reached `$beforeSend` lifecycle phase.'
+      );
+    };
+    const normalisedRefs = {};
+
+    if (config.data.refs) {
+      // First pass of refs, normalises any $ref entries to expand wildcards
+      for (const fullPath in config.data.refs) {
+        const page: Page = await serialiser.deserialise(
+          fullPath,
+          await mutableFilesystem.promises.readFile(fullPath)
+        );
+
+        normalisedRefs[fullPath] = {
+          ...page,
+          ...(await normaliseRefs(
+            String(fullPath),
+            config.data.refs[fullPath],
+            mutableFilesystem,
+            pageExtensions,
+            ignorePages
+          ))
+        };
+      }
+
+      const resolve = createRefResolver(normalisedRefs, serialiser, mutableFilesystem);
+      const refParser = new $RefParser();
+
+      // Second pass of refs, resolves the refs
+      for (const fullPath in normalisedRefs) {
+        const page = normalisedRefs[fullPath];
+        try {
+          const resolved: $RefParser.JSONSchema = await refParser.dereference(
+            String(fullPath),
+            page,
+            {
+              resolve,
+              dereference: { circular: false }
+            }
+          );
+          normalisedRefs[fullPath] = { ...page, ...resolved };
+        } catch (e) {
+          console.warn(
+            `Error resolving ref(s) for page '${fullPath}'. ${e.message.replace(/\.$/, '')} in '${
+              e.source
+            }'`
+          );
+          throw e;
+        }
+      }
+      // Third pass, write out the files
+      for (const fullPath in normalisedRefs) {
+        await mutableFilesystem.promises.writeFile(
+          fullPath,
+          await serialiser.serialise(fullPath, normalisedRefs[fullPath])
+        );
+      }
+    }
+  },
+  async $afterSource(pages: Page[], { config, ignorePages, pageExtensions }) {
+    const isNonHiddenPage = createPageTest(ignorePages, pageExtensions);
+
+    for (const page of pages) {
+      if (!isNonHiddenPage(page.fullPath)) {
+        continue;
+      }
+      const refs = findKeys(page, '$ref');
+
+      if (refs.length) {
+        for (const ref of refs) {
+          config.setRef(page.fullPath, ...ref);
+        }
+      }
+    }
+    return pages;
+  }
+};
+
+export default $RefPlugin;
+
+function createRefResolver(normalisedRefs, serialiser, mutableFilesystem) {
+  return {
+    file: {
+      canRead: true,
+      order: 1,
+      async read(file, callback) {
+        try {
+          const refedPage =
+            normalisedRefs[file.url] ||
+            (await serialiser.deserialise(
+              file.url,
+              await mutableFilesystem.promises.readFile(file.url)
+            ));
+          return callback(null, omit(refedPage, 'content'));
+        } catch (e) {
+          return callback(e, null);
+        }
+      }
+    }
+  };
+}
+
+const findKeys = (obj, targetProp, pathParts: string[] = []): [string[], string][] => {
+  return reduce<string, [string[], string][]>(
+    obj,
+    (found, value, key) => {
+      if (key === targetProp) {
+        return found.concat([[pathParts.concat(String(key)), value]]);
+      }
+      if (typeof value === 'object') {
+        const foundParents = findKeys(value, targetProp, pathParts.concat(String(key)));
+        return found.concat(foundParents);
+      }
+      return found;
+    },
+    []
+  );
+};
+
+function createPageTest(ignorePages, pageExtensions) {
+  const extTest = new RegExp(`${pageExtensions.map(escapeRegExp).join('|')}$`);
+  const ignoreTest = new RegExp(`${ignorePages.map(escapeRegExp).join('|')}$`);
+  return file => {
+    return !ignoreTest.test(file) && extTest.test(file) && !path.basename(file).startsWith('.');
+  };
+}

package/src/$TagPlugin.ts

@@ -0,0 +1,99 @@
+import path from 'path';
+
+import reduce from 'lodash/reduce';
+import escapeRegExp from 'lodash/escapeRegExp';
+
+import type PluginType from '@jpmorganchase/mosaic-types/dist/Plugin';
+import type Page from '@jpmorganchase/mosaic-types/dist/Page';
+import type Meta from '@jpmorganchase/mosaic-types/dist/Meta';
+
+/**
+ * Plugin that scrapes `$tag` from page metadata and also applies all aliases stored in `config.data.tags`.
+ * Tags ultimately resolve down into refs, but are different from normal refs, in that they are applied to the
+ * union filesystem (all merged filesystems), not to the individual source filesystem that they were defined on.
+ * Other plugins can use `setData()` and modify the `tags` property, to apply new global refs, as long as
+ * they do so before this plugin has reaches `$beforeSend`
+ */
+const $TagPlugin: PluginType<{
+  tagRefs: { [key: string]: { $$path: string[]; $$value: string[] }[] };
+  globalRefs: { [key: string]: { $$path: string[]; $$value: string[] }[] };
+  subscribedTags: string[];
+}> = {
+  // Check if the updated source has any /.tag aliases that we care about - if it does, we should re-run `afterUpdate` to
+  // make sure we pull in the latest pages
+  async shouldClearCache(updatedSourceFilesystem, { config }) {
+    if (
+      !config.data?.subscribedTags ||
+      !(await updatedSourceFilesystem.promises.exists('/.tags'))
+    ) {
+      return false;
+    }
+    // Does the updated source have tags we are interested in?
+    return ((await updatedSourceFilesystem.promises.readdir('/.tags')) as string[]).some(tag =>
+      config.data.subscribedTags.includes(tag)
+    );
+  },
+  // Every source with tags will create symlinks with the tagged pages, inside /.tags
+  async $afterSource(pages: Page<{ tags?: string[] }>[], { config, ignorePages, pageExtensions }) {
+    const isNonHiddenPage = createPageTest(ignorePages, pageExtensions);
+
+    for (const page of pages) {
+      if (!isNonHiddenPage(page.fullPath)) {
+        continue;
+      }
+      const meta = page as Meta<{ tags?: string[] }>;
+      // Symlink every `tag` item to '/.tags' folder
+      if (meta.tags?.length) {
+        config.setTags(page.fullPath, meta.tags);
+        delete meta.tags;
+      }
+
+      // Find any references to $tag
+      const foundTags = findKeys(page, '$tag');
+      if (foundTags.length) {
+        const tags = new Set<string>();
+
+        foundTags.forEach(ref => {
+          const [tag, fragment = ''] = ref.$$value.split('#');
+
+          tags.add(tag);
+
+          config.setGlobalRef(
+            page.fullPath,
+            ref.$$path,
+            `${path.join('/.tags', tag, '**')}#${fragment}`
+          );
+        });
+        config.setData({ subscribedTags: Array.from(tags) });
+      }
+    }
+    return pages;
+  }
+};
+
+export default $TagPlugin;
+
+const findKeys = (obj, targetProp, pathParts: string[] = []) => {
+  return reduce<string, { $$path: string[]; $$value: string }[]>(
+    obj,
+    (found, value, key) => {
+      if (key === targetProp) {
+        return found.concat({ $$path: pathParts.concat(String(key)), $$value: value });
+      }
+      if (typeof value === 'object') {
+        const foundParents = findKeys(value, targetProp, pathParts.concat(String(key)));
+        return found.concat(...foundParents);
+      }
+      return found;
+    },
+    []
+  );
+};
+
+function createPageTest(ignorePages, pageExtensions) {
+  const extTest = new RegExp(`${pageExtensions.map(escapeRegExp).join('|')}$`);
+  const ignoreTest = new RegExp(`${ignorePages.map(escapeRegExp).join('|')}$`);
+  return file => {
+    return !ignoreTest.test(file) && extTest.test(file) && !path.basename(file).startsWith('.');
+  };
+}

package/src/BreadcrumbsPlugin.ts

@@ -0,0 +1,43 @@
+import type PluginType from '@jpmorganchase/mosaic-types/dist/Plugin';
+import type Page from '@jpmorganchase/mosaic-types/dist/Page';
+import path from 'path';
+
+export type Breadcrumb = { label: string; path: string; id: string };
+
+/**
+ * Calculates breadcrumbs for pages then embeds a `breadcrumbs` property to the metadata
+ */
+const BreadcrumbsPlugin: PluginType<{}, { indexPageName: string }> = {
+  async $afterSource(pages: Page[], {}, options) {
+    for (const page of pages) {
+      const breadcrumbs: Array<Breadcrumb> = [];
+      let currentPage = page;
+      let parentDir = path.posix.dirname(currentPage.fullPath);
+
+      while (currentPage !== undefined) {
+        if (breadcrumbs.findIndex(breadcrumb => breadcrumb.id === currentPage.fullPath) === -1) {
+          breadcrumbs.unshift({
+            label: currentPage.title,
+            path: currentPage.route,
+            id: currentPage.fullPath
+          });
+        }
+
+        currentPage = pages.find(
+          page => page.fullPath === path.posix.join(parentDir, options.indexPageName)
+        );
+        if (currentPage) {
+          parentDir = path.posix.dirname(path.posix.join(String(currentPage.fullPath), '..'));
+        }
+      }
+
+      if (!page.breadcrumbs) {
+        page.breadcrumbs = breadcrumbs;
+      }
+    }
+
+    return pages;
+  }
+};
+
+export default BreadcrumbsPlugin;

package/src/LazyPagePlugin.ts

@@ -0,0 +1,101 @@
+import type PluginType from '@jpmorganchase/mosaic-types/dist/Plugin';
+import fs from 'fs';
+import fsExtra from 'fs-extra';
+import merge from 'lodash/merge';
+import escapeRegExp from 'lodash/escapeRegExp';
+import path from 'path';
+import { TDataOut } from 'memfs/lib/encoding';
+
+const LazyPagePlugin: PluginType<
+  { aliases: { [key: string]: Set<string> }; hddPaths: { [key: string]: string } },
+  { cacheDir: string }
+> = {
+  async afterUpdate(mutableFilesystem, { ignorePages, config, pageExtensions, serialiser }) {
+    const isNonHiddenPage = createPageTest(ignorePages, pageExtensions);
+
+    // This adds a hook that triggers everytime a non-cached page is read from the filesystem.
+    mutableFilesystem.__internal_do_not_use_addReadFileHook(
+      async (pagePath: string, fileData: TDataOut) => {
+        // If this is a 'page' - read the original file from disk and try to inject anything that's missing.
+        if (isNonHiddenPage(pagePath)) {
+          const currentPage = await serialiser.deserialise(pagePath, fileData);
+          if (config.data.hddPaths[pagePath]) {
+            const page = await serialiser.deserialise(
+              pagePath,
+              await fs.promises.readFile(config.data.hddPaths[pagePath])
+            );
+            const completePage = await serialiser.serialise(
+              pagePath,
+              merge(page, currentPage, { content: page.content })
+            );
+            return completePage;
+          }
+        }
+        return fileData;
+      }
+    );
+  },
+  async $beforeSend(
+    mutableFilesystem,
+    { config, serialiser, ignorePages, pageExtensions },
+    options
+  ) {
+    const hddPaths = {};
+    let originalDiskSize = 0;
+    let newDiskSize = 0;
+    const allPages = await mutableFilesystem.promises.glob(createFileGlob('**', pageExtensions), {
+      ignore: ignorePages.map(ignore => `**/${ignore}`),
+      cwd: '/',
+      onlyFiles: true
+    });
+    const baseDir = path.join(
+      process.cwd(),
+      options.cacheDir || '.mosaic-lazy-page-plugin-cache'
+    );
+    await fsExtra.ensureDir(baseDir);
+    for (const filePath of allPages) {
+      const rawPage = await mutableFilesystem.promises.readFile(String(filePath));
+      const page = await serialiser.deserialise(String(filePath), rawPage);
+      originalDiskSize += rawPage.length;
+      await fs.promises.mkdir(path.dirname(path.join(baseDir, String(filePath))), {
+        recursive: true
+      });
+      await fs.promises.writeFile(
+        path.join(baseDir, String(filePath)),
+        await serialiser.serialise(String(filePath), page)
+      );
+      const { content, ...pageRest } = page;
+      hddPaths[String(filePath)] = path.join(baseDir, String(filePath));
+      const redactedPage = await serialiser.serialise(String(filePath), pageRest);
+      newDiskSize += redactedPage.length;
+    }
+
+    config.setData({hddPaths});
+
+    const reduction = originalDiskSize - newDiskSize;
+    console.info(
+      `[PullDocs] LazyPagePlugin moved ${((reduction / originalDiskSize) * 100).toFixed(
+        2
+      )}% of virtual filesystem content to the hard disk. This reduced the virtual filesystem size from ${(
+        originalDiskSize / 1000000
+      ).toFixed(2)}mb to ${(newDiskSize / 1000000).toFixed(2)}mb (yay!)`
+    );
+  }
+};
+
+function createFileGlob(url, pageExtensions) {
+  if (pageExtensions.length === 1) {
+    return `${url}${pageExtensions[0]}`;
+  }
+  return `${url}{${pageExtensions.join(',')}}`;
+}
+
+export default LazyPagePlugin;
+
+function createPageTest(ignorePages, pageExtensions) {
+  const extTest = new RegExp(`${pageExtensions.map(escapeRegExp).join('|')}$`);
+  const ignoreTest = new RegExp(`${ignorePages.map(escapeRegExp).join('|')}$`);
+  return file => {
+    return !ignoreTest.test(file) && extTest.test(file) && !path.basename(file).startsWith('.');
+  };
+}

package/src/NextPrevPagePlugin.ts

@@ -0,0 +1,80 @@
+import type PluginType from '@jpmorganchase/mosaic-types/dist/Plugin';
+import { escapeRegExp } from 'lodash';
+
+/**
+ * Sorts the pages in a folder alphabetically and then embeds a `navigation` property to the metadata with ordered
+ * next/prev pages
+ */
+const NextPrevPlugin: PluginType<
+  {
+    nextPrev: { [key: string]: string[] };
+    refs: { [key: string]: { $$path: (number | string)[]; $$value: string }[] };
+  },
+  { indexFirst?: boolean; sortBy?: 'a-z' }
+> = {
+  async $beforeSend(
+    mutableFilesystem,
+    { config, pageExtensions, ignorePages },
+    { indexFirst, sortBy }
+  ) {
+    const pageDirs = (await mutableFilesystem.promises.glob('**', {
+      onlyDirectories: true,
+      ignore: ignorePages.map(ignore => `**/${ignore}`),
+      cwd: '/'
+    })) as string[];
+    for (const dirName of pageDirs) {
+      const pages = (
+        await mutableFilesystem.promises.glob(createFileGlob('*', pageExtensions), { onlyFiles: true, cwd: dirName, ignore: ignorePages })
+      ).sort(createSortFn(pageExtensions, { indexFirst, sortBy })) as string[];
+      for (let i = 0; i < pages.length; i++) {
+        if (i > 0) {
+          config.setRef(
+            pages[i],
+            ['navigation', 'prev', 'title', '$ref'],
+            `${pages[i - 1]}#/title`
+          );
+          config.setRef(
+            pages[i],
+            ['navigation', 'prev', 'fullPath', '$ref'],
+            `${pages[i - 1]}#/route`
+          );
+        }
+        if (i < pages.length - 1) {
+          config.setRef(
+            pages[i],
+            ['navigation', 'next', 'title', '$ref'],
+            `${pages[i + 1]}#/title`
+          );
+          config.setRef(
+            pages[i],
+            ['navigation', 'next', 'fullPath', '$ref'],
+            `${pages[i + 1]}#/route`
+          );
+        }
+      }
+    }
+  }
+};
+
+export default NextPrevPlugin;
+
+function createFileGlob(url, pageExtensions) {
+  if (pageExtensions.length === 1) {
+    return `${url}${pageExtensions[0]}`;
+  }
+  return `${url}{${pageExtensions.join(',')}}`;
+}
+
+function createSortFn(pageExtensions, { sortBy: _sortBy, indexFirst }) {
+  const indexRegExp =
+    indexFirst && new RegExp(`index${pageExtensions.map(escapeRegExp).join('|')}$`);
+
+  // TODO: Switch out the algorithm here, based on `sortBy`
+  return function sortFn(fullPathA, fullPathB) {
+    // Always pin /index to the front
+    if (indexFirst && indexRegExp.test(fullPathA)) {
+      return -1;
+    }
+    return fullPathA.localeCompare(fullPathB);
+  };
+}

package/src/PagesWithoutFileExtPlugin.ts

@@ -0,0 +1,38 @@
+import type Page from '@jpmorganchase/mosaic-types/dist/Page';
+import type PluginType from '@jpmorganchase/mosaic-types/dist/Plugin';
+import { escapeRegExp } from 'lodash';
+import path from 'path';
+
+/**
+ * Plugin that creates aliases without the file extension for every page emitted by the source.
+ * This allows pages to be retrieved from the filesystem without specifying `.mdx` (or whatever file extension is specified in `options.stripExt`).
+ * The plugin also modifies the `route` to point to the shorter alias. Friendly fullPaths are the shortest possible paths that
+ * will point to the page (see `Page` docs)
+ */
+const PagesWithoutFileExtPlugin: PluginType<
+  {}
+> = {
+  async $afterSource(pages: Page[], { config, ignorePages, pageExtensions }) {
+    const isNonHiddenPage = createPageTest(ignorePages, pageExtensions);
+    const pageTest = new RegExp(`${pageExtensions.map(escapeRegExp).join('|')}$`);
+
+    for (const page of pages) {
+      if (!isNonHiddenPage(page.fullPath)) {
+        continue;
+      }
+      page.route = page.route.replace(pageTest, '');
+      config.setAliases(page.fullPath, [page.route]);
+    }
+    return pages;
+  }
+};
+
+export default PagesWithoutFileExtPlugin;
+
+function createPageTest(ignorePages, pageExtensions) {
+  const extTest = new RegExp(`${pageExtensions.map(escapeRegExp).join('|')}$`);
+  const ignoreTest = new RegExp(`${ignorePages.map(escapeRegExp).join('|')}$`);
+  return file => {
+    return !ignoreTest.test(file) && extTest.test(file) && !path.basename(file).startsWith('.');
+  };
+}

package/src/ReadingTimePlugin.mjs.ts

@@ -0,0 +1,32 @@
+import type PluginType from '@jpmorganchase/mosaic-types/dist/Plugin';
+import type Page from '@jpmorganchase/mosaic-types/dist/Page';
+import getReadingTime from 'reading-time';
+import { unified } from 'unified';
+import { visit } from 'unist-util-visit';
+import markdown from 'remark-parse';
+
+/**
+ * Calculates reading time for pages and adds to frontmatter
+ */
+const ReadingTimePlugin: PluginType<{}> = {
+  async $afterSource(pages: Page[]) {
+    const processor = unified().use(markdown);
+    for (const page of pages) {
+      const tree = await processor.parse(page.content);
+      let textContent = '';
+
+      visit(
+        tree,
+        node => node.type === 'text' || node.type === 'code',
+        node => {
+          textContent += node.value;
+        }
+      );
+      page.readingTime = getReadingTime(textContent);
+    }
+
+    return pages;
+  }
+};
+
+export default ReadingTimePlugin;

package/src/SharedConfigPlugin.ts

@@ -0,0 +1,49 @@
+import type PluginType from '@jpmorganchase/mosaic-types/dist/Plugin';
+import flatten from 'lodash/flatten';
+import path from 'path';
+
+/**
+ * Plugin that crawls the page hierarchy to find the closest `sharedConfig` from any parent index's page metadata.
+ * It then exports a JSON file (name: `options.filename`) into each directory with the merged config for that level
+ */
+const SharedConfigPlugin: PluginType<{}, { filename: string }> = {
+  async $beforeSend(
+    mutableFilesystem,
+    { config, serialiser, ignorePages, pageExtensions },
+    options
+  ) {
+    const pagePaths = await mutableFilesystem.promises.glob(
+      createFileGlob('**/index,/index', pageExtensions),
+      {
+        ignore: [options.filename, ...flatten(ignorePages.map(ignore => [ignore, `**/${ignore}`]))],
+        cwd: '/'
+      }
+    );
+
+    for (const pagePath of pagePaths) {
+      const sharedConfigFile = path.join(path.dirname(String(pagePath)), options.filename);
+
+      const page = await serialiser.deserialise(
+        String(pagePath),
+        await mutableFilesystem.promises.readFile(String(pagePath))
+      );
+      if (page.sharedConfig) {
+        config.setRef(sharedConfigFile, ['config', '$ref'], `${String(pagePath)}#/sharedConfig`);
+        await mutableFilesystem.promises.writeFile(sharedConfigFile, '{}');
+      } else {
+        const baseDir = path.resolve(path.dirname(String(pagePath)), '../');
+
+        config.setAliases(path.join(baseDir, options.filename), [sharedConfigFile]);
+      }
+    }
+  }
+};
+
+export default SharedConfigPlugin;
+
+function createFileGlob(url, pageExtensions) {
+  if (pageExtensions.length === 1) {
+    return `${url}${pageExtensions[0]}`;
+  }
+  return `${url}{${pageExtensions.join(',')}}`;
+}

package/src/SidebarPlugin.ts

@@ -0,0 +1,154 @@
+import path from 'path';
+import type PluginType from '@jpmorganchase/mosaic-types/dist/Plugin';
+
+/**
+ * Sorts the pages in a folder by priority and then exports a JSON file (name: `options.filename`) with the
+ * sidebar tree from that directory downwards and adds sidebar data into frontmatter for each page.
+ */
+
+const SidebarPlugin: PluginType<
+  {
+    dirs: string[];
+    refs: { [key: string]: { $$path: (number | string)[]; $$value: string }[] };
+  },
+  { filename: string }
+> = {
+  async $beforeSend(
+    mutableFilesystem,
+    { config, serialiser, ignorePages, pageExtensions },
+    options
+  ) {
+    const dirs = await mutableFilesystem.promises.glob('**', {
+      onlyDirectories: true,
+      cwd: '/'
+    });
+    for (const dirName of dirs) {
+      const sidebarFilePath = path.posix.join(String(dirName), options.filename);
+      const pages = (
+        await Promise.all(
+          ((await mutableFilesystem.promises.glob(
+            createFileGlob(['*', '*/index'], pageExtensions),
+            {
+              onlyFiles: true,
+              cwd: String(dirName),
+              ignore: ignorePages.map(ignore => `**/${ignore}`)
+            }
+          )) as string[]).map(async pagePath => {
+            return await serialiser.deserialise(
+              pagePath,
+              await mutableFilesystem.promises.readFile(pagePath)
+            );
+          })
+        )
+      )
+        .filter(page => filterFn(page))
+        .sort((pageA, pageB) => sortFn(pageA, pageB, dirName));
+
+      for (let i = 0; i < pages.length; i++) {
+        setPageRefs(sidebarFilePath, pages[i], i);
+        setChildPageRefs(pages[i], dirName, i, sidebarFilePath);
+        addSidebarDataToFrontmatter(pages[i], dirName);
+      }
+      await mutableFilesystem.promises.writeFile(sidebarFilePath, '[]');
+    }
+
+    function setPageRefs(sidebarFilePath, page, i) {
+      config.setRef(sidebarFilePath, ['pages', `${i}`, 'id', '$ref'], `${page.fullPath}#/route`);
+      page.sidebar && page.sidebar.label
+        ? config.setRef(
+            sidebarFilePath,
+            ['pages', `${i}`, 'name', '$ref'],
+            `${page.fullPath}#/sidebar/label`
+          )
+        : config.setRef(
+            sidebarFilePath,
+            ['pages', `${i}`, 'name', '$ref'],
+            `${page.fullPath}#/title`
+          );
+      config.setRef(
+        sidebarFilePath,
+        ['pages', `${i}`, 'data', 'link', '$ref'],
+        `${page.fullPath}#/route`
+      );
+    }
+
+    async function setChildPageRefs(page, dirName, i, sidebarFilePath) {
+      if (isSubfolderIndex(page, dirName)) {
+        const childDirName = path.posix.dirname(page.fullPath);
+        const childNodeSidebarPath = path.posix.join(childDirName, options.filename);
+        const childPages = (
+          await Promise.all(
+            ((await mutableFilesystem.promises.glob(
+              createFileGlob(['*', '*/index'], pageExtensions),
+              {
+                onlyFiles: true,
+                cwd: String(childDirName),
+                ignore: ignorePages.map(ignore => `**/${ignore}`)
+              }
+            )) as string[]).map(async pagePath => {
+              return await serialiser.deserialise(
+                pagePath,
+                await mutableFilesystem.promises.readFile(pagePath)
+              );
+            })
+          )
+        ).filter(page => filterFn(page));
+        if (childPages.length > 1) {
+          for (let j = 1; j < childPages.length; j++) {
+            config.setRef(
+              sidebarFilePath,
+              ['pages', `${i}`, 'childNodes', `${j - 1}`, '$ref'],
+              `${childNodeSidebarPath}#/pages/${j}`
+            );
+          }
+        }
+      }
+    }
+
+    function addSidebarDataToFrontmatter(page, dirName) {
+      const dirNameLevels = String(dirName)
+        .split('/')
+        .filter(level => level);
+      const sidebarRootDir =
+        `/` + dirNameLevels.slice(0, dirNameLevels.length < 3 ? dirNameLevels.length : 3).join(`/`);
+      config.setRef(
+        String(page.fullPath),
+        ['sidebarData', '$ref'],
+        path.posix.join(String(sidebarRootDir), options.filename, '#', 'pages')
+      );
+    }
+  }
+};
+export default SidebarPlugin;
+
+function createFileGlob(patterns, pageExtensions) {
+  if (Array.isArray(patterns)) {
+    return patterns.map(pattern => createFileGlob(pattern, pageExtensions));
+  }
+  if (pageExtensions.length === 1) {
+    return `${patterns}${pageExtensions[0]}`;
+  }
+  return `${patterns}{${pageExtensions.join(',')}}`;
+}
+
+function isSubfolderIndex(page, dirName) {
+  return page.fullPath.split('/').length == String(dirName).split('/').length + 2;
+}
+
+function sortFn(pageA, pageB, dirName) {
+  // Always pin /index to the front
+  if (pageA.route === dirName + '/index') {
+    return -1;
+  }
+  if (pageB.route === dirName + '/index') {
+    return 1;
+  }
+  return (
+    (pageB.sidebar && pageB.sidebar.priority ? pageB.sidebar.priority : -1) -
+    (pageA.sidebar && pageA.sidebar.priority ? pageA.sidebar.priority : -1)
+  );
+}
+
+function filterFn(page) {
+  return !(page.sidebar && page.sidebar.exclude);
+}

package/src/SiteMapPlugin.ts

@@ -0,0 +1,44 @@
+import type PluginType from '@jpmorganchase/mosaic-types/dist/Plugin';
+
+const SiteMapPlugin: PluginType<
+  { sitemaps: string[][] },
+  {}
+> = {
+  // Merge together all of the individual sitemaps from each source
+  async afterUpdate(mutableFilesystem, { sharedFilesystem, globalConfig }) {
+    if (!Array.isArray(globalConfig.data.sitemaps)) {
+      return;
+    }
+    await sharedFilesystem.promises.writeFile(
+      '/sitemap.xml',
+      '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n' +
+        globalConfig.data.sitemaps
+          .map(items =>
+            items
+              .map(
+                item =>
+                  `\t<url>\n\t\t<loc>${String(
+                    item
+                  )}</loc>\n\t\t<changefreq>weekly</changefreq>\n\t\t<priority>0.5</priority>\n\t</url>`
+              )
+              .join('\n')
+          )
+          .join('') +
+        '\n</urlset>'
+    );
+  },
+  // Store a sitemap for just this source
+  async $beforeSend(mutableFilesystem, { config, ignorePages }, options) {
+    config.setData({
+      sitemaps: [
+        (await mutableFilesystem.promises.glob('**', {
+          onlyFiles: true,
+          ignore: ignorePages.map(ignore => `**/${ignore}`),
+          cwd: '/'
+        })) as string[]
+      ]
+    });
+  }
+};
+
+export default SiteMapPlugin;

package/src/TableOfContentsPlugin.mjs.ts

@@ -0,0 +1,52 @@
+import type PluginType from '@jpmorganchase/mosaic-types/dist/Plugin';
+import type Page from '@jpmorganchase/mosaic-types/dist/Page';
+import { visit } from 'unist-util-visit';
+import markdown from 'remark-parse';
+import { unified } from 'unified';
+import Slugger from 'github-slugger';
+import { toString } from 'hast-util-to-string';
+
+type TOCItem = { level: number; id: string; text: string };
+
+/**
+ * Calculates table of contents from page headings
+ */
+const TableOfContentsPlugin: PluginType<{}, { minRank: 2; maxRank: 4 }> = {
+  async $afterSource(pages: Page[], {}, { minRank, maxRank }) {
+    const processor = unified().use(markdown);
+    const slugger = new Slugger();
+    for (const page of pages) {
+      const tree = await processor.parse(page.content);
+      const items: TOCItem[] = [];
+      visit(
+        tree,
+        node => test(node),
+        node => {
+          const text = toString(node).trim();
+          items.push({
+            level: node.depth,
+            id: slugger.slug(text),
+            text: text
+          });
+        }
+      );
+      page.tableOfContents = items;
+    }
+
+    return pages;
+
+    function test(node) {
+      return node.type === 'heading' && isHeadingInRange(node);
+    }
+
+    function isHeadingInRange(node) {
+      const depth = node.depth;
+      if (depth < minRank || depth > maxRank) {
+        return false;
+      }
+      return true;
+    }
+  }
+};
+
+export default TableOfContentsPlugin;

package/src/utils/normaliseRefs.ts

@@ -0,0 +1,85 @@
+import path from 'path';
+
+import glob from 'fast-glob';
+import set from 'lodash/set';
+
+function createFileGlob(url, pageExtensions) {
+  if (pageExtensions.length === 1) {
+    return `${url}${pageExtensions[0]}`;
+  }
+  return `${url}{${pageExtensions.join(',')}}`;
+}
+
+export default async function normaliseRefs(filepath, refs, filesystem, pageExtensions, ignorePages) {
+  const configWithoutGlobs = {};
+
+  for (let i = 0; i < refs.length; i++) {
+    const ref = refs[i];
+    const [url, fragment = ''] = ref.$$value.split('#');
+
+    if (!url) {
+      continue;
+    }
+
+    const fullUrl = path.resolve(path.dirname(filepath), url);
+
+    if (!glob.isDynamicPattern(url)) {
+      if (!(await filesystem.promises.exists(fullUrl))) {
+        throw new Error(
+          `A \`$ref\` for '${filepath}' pointed to page '${fullUrl}' - which could not be found.`
+        );
+      }
+      const refRealPath = await filesystem.promises.realpath(
+        fullUrl
+        //path.resolve(path.dirname(filepath), url)
+      );
+      const stat = await filesystem.promises.stat(refRealPath);
+      // Turn directory refs into full index paths
+      // TODO: But DO we want to do this?
+      if (stat.isDirectory()) {
+        // && !/\/index(\.\w{1,4})?$/.test(url)) {
+        set(configWithoutGlobs, ref.$$path.slice(0, -1), {
+          $ref: `${await filesystem.promises.realpath(path.join(refRealPath, 'index'))}#${fragment}`
+        });
+      } else {
+        set(configWithoutGlobs, ref.$$path.slice(0, -1), {
+          $ref: `${refRealPath}#${fragment}`
+        });
+      }
+      continue;
+    }
+
+    try {
+      let globResolvedRefs =
+        // If the pattern doesn't have an explicit file extension - then we'll make it just return pages
+        await filesystem.promises.glob(
+          !/\/[^/]+\.\w{2,4}$/.test(url) ? createFileGlob(url, pageExtensions) : url,
+          {
+            ignore: [filepath, ...ignorePages.map(ignore => `**/${ignore}`)],
+            cwd: path.dirname(filepath),
+            absolute: true
+          }
+        );
+
+      // Use a set so we de-dupe any symlinks that point to the same page
+      const resolvedRefs = new Set();
+
+      for (const value of globResolvedRefs) {
+        const refRealPath = await filesystem.promises.realpath(path.resolve(filepath, value));
+        if (refRealPath !== (await filesystem.promises.realpath(filepath))) {
+          resolvedRefs.add(`${refRealPath}#${fragment}`);
+        }
+      }
+
+      set(
+        configWithoutGlobs,
+        ref.$$path.slice(0, -1),
+        Array.from(resolvedRefs).map($ref => ({ $ref }))
+      );
+    } catch (e) {
+      throw e;
+    }
+  }
+
+  return configWithoutGlobs;
+}

package/tsconfig.json

@@ -0,0 +1,21 @@
+{
+  "compilerOptions": {
+    "module": "commonjs",
+    "allowJs": true,
+    "target": "es2015",
+    "esModuleInterop": true,
+    "moduleResolution": "NodeNext",
+    "outDir": "dist",
+    "skipLibCheck": true,
+    "sourceMap": true,
+    "rootDir": "./src"
+  },
+  "include": ["src/**/*"],
+  "exclude": [
+    "node_modules",
+    "src/**/__tests__/**/*",
+    "src/__tests__/**/*",
+    "src/**/*.mjs.*",
+    "**/dist/**/*"
+  ]
+}