@jpmorganchase/mosaic-plugins 0.1.0-beta.3 → 0.1.0-beta.31

package/src/$RefPlugin.ts

@@ -1,200 +0,0 @@
-import path from 'path';
-
-import reduce from 'lodash/reduce';
-import omit from 'lodash/omit';
-
-import escapeRegExp from 'lodash/escapeRegExp';
-import $RefParser from '@apidevtools/json-schema-ref-parser';
-
-import type PluginType from '@jpmorganchase/mosaic-types/dist/Plugin';
-import type Page from '@jpmorganchase/mosaic-types/dist/Page';
-
-import normaliseRefs from './utils/normaliseRefs';
-
-/**
- * Plugin that scrapes `$ref` properties from page metadata and also applies all refs stored in `config.data.refs`.
- * Other plugins can use `setRef` to apply new refs, as long as they call it before this plugin has reaches `$beforeSend`
- */
-const $RefPlugin: PluginType<{
-  refs: { [key: string]: { $$path: string[]; $$value: string }[] };
-  globalRefs: { [key: string]: { $$path: string[]; $$value: string }[] };
-  subscribedTags: string[];
-}> = {
-  async afterUpdate(
-    mutableFilesystem,
-    { ignorePages, globalFilesystem, serialiser, pageExtensions, globalConfig }
-  ) {
-    mutableFilesystem.__internal_do_not_use_addReadFileHook(async (pagePath: string, fileData) => {
-      if (globalConfig.data.globalRefs[pagePath]) {
-        const normalisedRef = {
-          ...(await serialiser.deserialise(pagePath, fileData)),
-          ...(await normaliseRefs(
-            String(pagePath),
-            globalConfig.data.globalRefs[pagePath],
-            globalFilesystem,
-            pageExtensions,
-            ignorePages
-          ))
-        } as any;
-        const resolve = createRefResolver({
-          [pagePath]: normalisedRef
-        }, serialiser, globalFilesystem);
-        const refParser = new $RefParser();
-
-        try {
-          const resolved: $RefParser.JSONSchema = await refParser.dereference(
-            String(pagePath),
-            normalisedRef,
-            {
-              resolve,
-              dereference: { circular: false }
-            }
-          );
-          return serialiser.serialise(pagePath, { ...normalisedRef, ...resolved });
-        } catch (e) {
-          console.warn(
-            `Error resolving ref(s) for page '${pagePath}'. ${e.message.replace(/\.$/, '')} in '${
-              e.source
-            }'`
-          );
-          throw e;
-        }
-      }
-
-      return fileData;
-    });
-  },
-  async $beforeSend(
-    mutableFilesystem,
-    { config, serialiser, pageExtensions, ignorePages },
-    options
-  ) {
-    // Prevent any more refs being set after this lifecycle has been called - as they won't take effect
-    config.setRef = () => {
-      throw new Error(
-        'Cannot set refs after `RefPlugin` has reached `$beforeSend` lifecycle phase.'
-      );
-    };
-    const normalisedRefs = {};
-
-    if (config.data.refs) {
-      // First pass of refs, normalises any $ref entries to expand wildcards
-      for (const fullPath in config.data.refs) {
-        const page: Page = await serialiser.deserialise(
-          fullPath,
-          await mutableFilesystem.promises.readFile(fullPath)
-        );
-
-        normalisedRefs[fullPath] = {
-          ...page,
-          ...(await normaliseRefs(
-            String(fullPath),
-            config.data.refs[fullPath],
-            mutableFilesystem,
-            pageExtensions,
-            ignorePages
-          ))
-        };
-      }
-
-      const resolve = createRefResolver(normalisedRefs, serialiser, mutableFilesystem);
-      const refParser = new $RefParser();
-
-      // Second pass of refs, resolves the refs
-      for (const fullPath in normalisedRefs) {
-        const page = normalisedRefs[fullPath];
-        try {
-          const resolved: $RefParser.JSONSchema = await refParser.dereference(
-            String(fullPath),
-            page,
-            {
-              resolve,
-              dereference: { circular: false }
-            }
-          );
-          normalisedRefs[fullPath] = { ...page, ...resolved };
-        } catch (e) {
-          console.warn(
-            `Error resolving ref(s) for page '${fullPath}'. ${e.message.replace(/\.$/, '')} in '${
-              e.source
-            }'`
-          );
-          throw e;
-        }
-      }
-      // Third pass, write out the files
-      for (const fullPath in normalisedRefs) {
-        await mutableFilesystem.promises.writeFile(
-          fullPath,
-          await serialiser.serialise(fullPath, normalisedRefs[fullPath])
-        );
-      }
-    }
-  },
-  async $afterSource(pages: Page[], { config, ignorePages, pageExtensions }) {
-    const isNonHiddenPage = createPageTest(ignorePages, pageExtensions);
-
-    for (const page of pages) {
-      if (!isNonHiddenPage(page.fullPath)) {
-        continue;
-      }
-      const refs = findKeys(page, '$ref');
-
-      if (refs.length) {
-        for (const ref of refs) {
-          config.setRef(page.fullPath, ...ref);
-        }
-      }
-    }
-    return pages;
-  }
-};
-
-export default $RefPlugin;
-
-function createRefResolver(normalisedRefs, serialiser, mutableFilesystem) {
-  return {
-    file: {
-      canRead: true,
-      order: 1,
-      async read(file, callback) {
-        try {
-          const refedPage =
-            normalisedRefs[file.url] ||
-            (await serialiser.deserialise(
-              file.url,
-              await mutableFilesystem.promises.readFile(file.url)
-            ));
-          return callback(null, omit(refedPage, 'content'));
-        } catch (e) {
-          return callback(e, null);
-        }
-      }
-    }
-  };
-}
-
-const findKeys = (obj, targetProp, pathParts: string[] = []): [string[], string][] => {
-  return reduce<string, [string[], string][]>(
-    obj,
-    (found, value, key) => {
-      if (key === targetProp) {
-        return found.concat([[pathParts.concat(String(key)), value]]);
-      }
-      if (typeof value === 'object') {
-        const foundParents = findKeys(value, targetProp, pathParts.concat(String(key)));
-        return found.concat(foundParents);
-      }
-      return found;
-    },
-    []
-  );
-};
-
-function createPageTest(ignorePages, pageExtensions) {
-  const extTest = new RegExp(`${pageExtensions.map(escapeRegExp).join('|')}$`);
-  const ignoreTest = new RegExp(`${ignorePages.map(escapeRegExp).join('|')}$`);
-  return file => {
-    return !ignoreTest.test(file) && extTest.test(file) && !path.basename(file).startsWith('.');
-  };
-}

package/src/$TagPlugin.ts

@@ -1,99 +0,0 @@
-import path from 'path';
-
-import reduce from 'lodash/reduce';
-import escapeRegExp from 'lodash/escapeRegExp';
-
-import type PluginType from '@jpmorganchase/mosaic-types/dist/Plugin';
-import type Page from '@jpmorganchase/mosaic-types/dist/Page';
-import type Meta from '@jpmorganchase/mosaic-types/dist/Meta';
-
-/**
- * Plugin that scrapes `$tag` from page metadata and also applies all aliases stored in `config.data.tags`.
- * Tags ultimately resolve down into refs, but are different from normal refs, in that they are applied to the
- * union filesystem (all merged filesystems), not to the individual source filesystem that they were defined on.
- * Other plugins can use `setData()` and modify the `tags` property, to apply new global refs, as long as
- * they do so before this plugin has reaches `$beforeSend`
- */
-const $TagPlugin: PluginType<{
-  tagRefs: { [key: string]: { $$path: string[]; $$value: string[] }[] };
-  globalRefs: { [key: string]: { $$path: string[]; $$value: string[] }[] };
-  subscribedTags: string[];
-}> = {
-  // Check if the updated source has any /.tag aliases that we care about - if it does, we should re-run `afterUpdate` to
-  // make sure we pull in the latest pages
-  async shouldClearCache(updatedSourceFilesystem, { config }) {
-    if (
-      !config.data?.subscribedTags ||
-      !(await updatedSourceFilesystem.promises.exists('/.tags'))
-    ) {
-      return false;
-    }
-    // Does the updated source have tags we are interested in?
-    return ((await updatedSourceFilesystem.promises.readdir('/.tags')) as string[]).some(tag =>
-      config.data.subscribedTags.includes(tag)
-    );
-  },
-  // Every source with tags will create symlinks with the tagged pages, inside /.tags
-  async $afterSource(pages: Page<{ tags?: string[] }>[], { config, ignorePages, pageExtensions }) {
-    const isNonHiddenPage = createPageTest(ignorePages, pageExtensions);
-
-    for (const page of pages) {
-      if (!isNonHiddenPage(page.fullPath)) {
-        continue;
-      }
-      const meta = page as Meta<{ tags?: string[] }>;
-      // Symlink every `tag` item to '/.tags' folder
-      if (meta.tags?.length) {
-        config.setTags(page.fullPath, meta.tags);
-        delete meta.tags;
-      }
-
-      // Find any references to $tag
-      const foundTags = findKeys(page, '$tag');
-      if (foundTags.length) {
-        const tags = new Set<string>();
-
-        foundTags.forEach(ref => {
-          const [tag, fragment = ''] = ref.$$value.split('#');
-
-          tags.add(tag);
-
-          config.setGlobalRef(
-            page.fullPath,
-            ref.$$path,
-            `${path.join('/.tags', tag, '**')}#${fragment}`
-          );
-        });
-        config.setData({ subscribedTags: Array.from(tags) });
-      }
-    }
-    return pages;
-  }
-};
-
-export default $TagPlugin;
-
-const findKeys = (obj, targetProp, pathParts: string[] = []) => {
-  return reduce<string, { $$path: string[]; $$value: string }[]>(
-    obj,
-    (found, value, key) => {
-      if (key === targetProp) {
-        return found.concat({ $$path: pathParts.concat(String(key)), $$value: value });
-      }
-      if (typeof value === 'object') {
-        const foundParents = findKeys(value, targetProp, pathParts.concat(String(key)));
-        return found.concat(...foundParents);
-      }
-      return found;
-    },
-    []
-  );
-};
-
-function createPageTest(ignorePages, pageExtensions) {
-  const extTest = new RegExp(`${pageExtensions.map(escapeRegExp).join('|')}$`);
-  const ignoreTest = new RegExp(`${ignorePages.map(escapeRegExp).join('|')}$`);
-  return file => {
-    return !ignoreTest.test(file) && extTest.test(file) && !path.basename(file).startsWith('.');
-  };
-}

package/src/BreadcrumbsPlugin.ts

@@ -1,43 +0,0 @@
-import type PluginType from '@jpmorganchase/mosaic-types/dist/Plugin';
-import type Page from '@jpmorganchase/mosaic-types/dist/Page';
-import path from 'path';
-
-export type Breadcrumb = { label: string; path: string; id: string };
-
-/**
- * Calculates breadcrumbs for pages then embeds a `breadcrumbs` property to the metadata
- */
-const BreadcrumbsPlugin: PluginType<{}, { indexPageName: string }> = {
-  async $afterSource(pages: Page[], {}, options) {
-    for (const page of pages) {
-      const breadcrumbs: Array<Breadcrumb> = [];
-      let currentPage = page;
-      let parentDir = path.posix.dirname(currentPage.fullPath);
-
-      while (currentPage !== undefined) {
-        if (breadcrumbs.findIndex(breadcrumb => breadcrumb.id === currentPage.fullPath) === -1) {
-          breadcrumbs.unshift({
-            label: currentPage.title,
-            path: currentPage.route,
-            id: currentPage.fullPath
-          });
-        }
-
-        currentPage = pages.find(
-          page => page.fullPath === path.posix.join(parentDir, options.indexPageName)
-        );
-        if (currentPage) {
-          parentDir = path.posix.dirname(path.posix.join(String(currentPage.fullPath), '..'));
-        }
-      }
-
-      if (!page.breadcrumbs) {
-        page.breadcrumbs = breadcrumbs;
-      }
-    }
-
-    return pages;
-  }
-};
-
-export default BreadcrumbsPlugin;

package/src/LazyPagePlugin.ts

@@ -1,101 +0,0 @@
-import type PluginType from '@jpmorganchase/mosaic-types/dist/Plugin';
-import fs from 'fs';
-import fsExtra from 'fs-extra';
-import merge from 'lodash/merge';
-import escapeRegExp from 'lodash/escapeRegExp';
-import path from 'path';
-import { TDataOut } from 'memfs/lib/encoding';
-
-const LazyPagePlugin: PluginType<
-  { aliases: { [key: string]: Set<string> }; hddPaths: { [key: string]: string } },
-  { cacheDir: string }
-> = {
-  async afterUpdate(mutableFilesystem, { ignorePages, config, pageExtensions, serialiser }) {
-    const isNonHiddenPage = createPageTest(ignorePages, pageExtensions);
-
-    // This adds a hook that triggers everytime a non-cached page is read from the filesystem.
-    mutableFilesystem.__internal_do_not_use_addReadFileHook(
-      async (pagePath: string, fileData: TDataOut) => {
-        // If this is a 'page' - read the original file from disk and try to inject anything that's missing.
-        if (isNonHiddenPage(pagePath)) {
-          const currentPage = await serialiser.deserialise(pagePath, fileData);
-          if (config.data.hddPaths[pagePath]) {
-            const page = await serialiser.deserialise(
-              pagePath,
-              await fs.promises.readFile(config.data.hddPaths[pagePath])
-            );
-            const completePage = await serialiser.serialise(
-              pagePath,
-              merge(page, currentPage, { content: page.content })
-            );
-            return completePage;
-          }
-        }
-        return fileData;
-      }
-    );
-  },
-  async $beforeSend(
-    mutableFilesystem,
-    { config, serialiser, ignorePages, pageExtensions },
-    options
-  ) {
-    const hddPaths = {};
-    let originalDiskSize = 0;
-    let newDiskSize = 0;
-    const allPages = await mutableFilesystem.promises.glob(createFileGlob('**', pageExtensions), {
-      ignore: ignorePages.map(ignore => `**/${ignore}`),
-      cwd: '/',
-      onlyFiles: true
-    });
-    const baseDir = path.join(
-      process.cwd(),
-      options.cacheDir || '.mosaic-lazy-page-plugin-cache'
-    );
-    await fsExtra.ensureDir(baseDir);
-    for (const filePath of allPages) {
-      const rawPage = await mutableFilesystem.promises.readFile(String(filePath));
-      const page = await serialiser.deserialise(String(filePath), rawPage);
-      originalDiskSize += rawPage.length;
-      await fs.promises.mkdir(path.dirname(path.join(baseDir, String(filePath))), {
-        recursive: true
-      });
-      await fs.promises.writeFile(
-        path.join(baseDir, String(filePath)),
-        await serialiser.serialise(String(filePath), page)
-      );
-      const { content, ...pageRest } = page;
-      hddPaths[String(filePath)] = path.join(baseDir, String(filePath));
-      const redactedPage = await serialiser.serialise(String(filePath), pageRest);
-      newDiskSize += redactedPage.length;
-    }
-
-    config.setData({hddPaths});
-
-    const reduction = originalDiskSize - newDiskSize;
-    console.info(
-      `[PullDocs] LazyPagePlugin moved ${((reduction / originalDiskSize) * 100).toFixed(
-        2
-      )}% of virtual filesystem content to the hard disk. This reduced the virtual filesystem size from ${(
-        originalDiskSize / 1000000
-      ).toFixed(2)}mb to ${(newDiskSize / 1000000).toFixed(2)}mb (yay!)`
-    );
-  }
-};
-
-function createFileGlob(url, pageExtensions) {
-  if (pageExtensions.length === 1) {
-    return `${url}${pageExtensions[0]}`;
-  }
-  return `${url}{${pageExtensions.join(',')}}`;
-}
-
-export default LazyPagePlugin;
-
-function createPageTest(ignorePages, pageExtensions) {
-  const extTest = new RegExp(`${pageExtensions.map(escapeRegExp).join('|')}$`);
-  const ignoreTest = new RegExp(`${ignorePages.map(escapeRegExp).join('|')}$`);
-  return file => {
-    return !ignoreTest.test(file) && extTest.test(file) && !path.basename(file).startsWith('.');
-  };
-}

package/src/NextPrevPagePlugin.ts

@@ -1,80 +0,0 @@
-import type PluginType from '@jpmorganchase/mosaic-types/dist/Plugin';
-import { escapeRegExp } from 'lodash';
-
-/**
- * Sorts the pages in a folder alphabetically and then embeds a `navigation` property to the metadata with ordered
- * next/prev pages
- */
-const NextPrevPlugin: PluginType<
-  {
-    nextPrev: { [key: string]: string[] };
-    refs: { [key: string]: { $$path: (number | string)[]; $$value: string }[] };
-  },
-  { indexFirst?: boolean; sortBy?: 'a-z' }
-> = {
-  async $beforeSend(
-    mutableFilesystem,
-    { config, pageExtensions, ignorePages },
-    { indexFirst, sortBy }
-  ) {
-    const pageDirs = (await mutableFilesystem.promises.glob('**', {
-      onlyDirectories: true,
-      ignore: ignorePages.map(ignore => `**/${ignore}`),
-      cwd: '/'
-    })) as string[];
-    for (const dirName of pageDirs) {
-      const pages = (
-        await mutableFilesystem.promises.glob(createFileGlob('*', pageExtensions), { onlyFiles: true, cwd: dirName, ignore: ignorePages })
-      ).sort(createSortFn(pageExtensions, { indexFirst, sortBy })) as string[];
-      for (let i = 0; i < pages.length; i++) {
-        if (i > 0) {
-          config.setRef(
-            pages[i],
-            ['navigation', 'prev', 'title', '$ref'],
-            `${pages[i - 1]}#/title`
-          );
-          config.setRef(
-            pages[i],
-            ['navigation', 'prev', 'fullPath', '$ref'],
-            `${pages[i - 1]}#/route`
-          );
-        }
-        if (i < pages.length - 1) {
-          config.setRef(
-            pages[i],
-            ['navigation', 'next', 'title', '$ref'],
-            `${pages[i + 1]}#/title`
-          );
-          config.setRef(
-            pages[i],
-            ['navigation', 'next', 'fullPath', '$ref'],
-            `${pages[i + 1]}#/route`
-          );
-        }
-      }
-    }
-  }
-};
-
-export default NextPrevPlugin;
-
-function createFileGlob(url, pageExtensions) {
-  if (pageExtensions.length === 1) {
-    return `${url}${pageExtensions[0]}`;
-  }
-  return `${url}{${pageExtensions.join(',')}}`;
-}
-
-function createSortFn(pageExtensions, { sortBy: _sortBy, indexFirst }) {
-  const indexRegExp =
-    indexFirst && new RegExp(`index${pageExtensions.map(escapeRegExp).join('|')}$`);
-
-  // TODO: Switch out the algorithm here, based on `sortBy`
-  return function sortFn(fullPathA, fullPathB) {
-    // Always pin /index to the front
-    if (indexFirst && indexRegExp.test(fullPathA)) {
-      return -1;
-    }
-    return fullPathA.localeCompare(fullPathB);
-  };
-}

package/src/PagesWithoutFileExtPlugin.ts

@@ -1,38 +0,0 @@
-import type Page from '@jpmorganchase/mosaic-types/dist/Page';
-import type PluginType from '@jpmorganchase/mosaic-types/dist/Plugin';
-import { escapeRegExp } from 'lodash';
-import path from 'path';
-
-/**
- * Plugin that creates aliases without the file extension for every page emitted by the source.
- * This allows pages to be retrieved from the filesystem without specifying `.mdx` (or whatever file extension is specified in `options.stripExt`).
- * The plugin also modifies the `route` to point to the shorter alias. Friendly fullPaths are the shortest possible paths that
- * will point to the page (see `Page` docs)
- */
-const PagesWithoutFileExtPlugin: PluginType<
-  {}
-> = {
-  async $afterSource(pages: Page[], { config, ignorePages, pageExtensions }) {
-    const isNonHiddenPage = createPageTest(ignorePages, pageExtensions);
-    const pageTest = new RegExp(`${pageExtensions.map(escapeRegExp).join('|')}$`);
-
-    for (const page of pages) {
-      if (!isNonHiddenPage(page.fullPath)) {
-        continue;
-      }
-      page.route = page.route.replace(pageTest, '');
-      config.setAliases(page.fullPath, [page.route]);
-    }
-    return pages;
-  }
-};
-
-export default PagesWithoutFileExtPlugin;
-
-function createPageTest(ignorePages, pageExtensions) {
-  const extTest = new RegExp(`${pageExtensions.map(escapeRegExp).join('|')}$`);
-  const ignoreTest = new RegExp(`${ignorePages.map(escapeRegExp).join('|')}$`);
-  return file => {
-    return !ignoreTest.test(file) && extTest.test(file) && !path.basename(file).startsWith('.');
-  };
-}

package/src/ReadingTimePlugin.mjs.ts

@@ -1,32 +0,0 @@
-import type PluginType from '@jpmorganchase/mosaic-types/dist/Plugin';
-import type Page from '@jpmorganchase/mosaic-types/dist/Page';
-import getReadingTime from 'reading-time';
-import { unified } from 'unified';
-import { visit } from 'unist-util-visit';
-import markdown from 'remark-parse';
-
-/**
- * Calculates reading time for pages and adds to frontmatter
- */
-const ReadingTimePlugin: PluginType<{}> = {
-  async $afterSource(pages: Page[]) {
-    const processor = unified().use(markdown);
-    for (const page of pages) {
-      const tree = await processor.parse(page.content);
-      let textContent = '';
-
-      visit(
-        tree,
-        node => node.type === 'text' || node.type === 'code',
-        node => {
-          textContent += node.value;
-        }
-      );
-      page.readingTime = getReadingTime(textContent);
-    }
-
-    return pages;
-  }
-};
-
-export default ReadingTimePlugin;

package/src/SharedConfigPlugin.ts

@@ -1,49 +0,0 @@
-import type PluginType from '@jpmorganchase/mosaic-types/dist/Plugin';
-import flatten from 'lodash/flatten';
-import path from 'path';
-
-/**
- * Plugin that crawls the page hierarchy to find the closest `sharedConfig` from any parent index's page metadata.
- * It then exports a JSON file (name: `options.filename`) into each directory with the merged config for that level
- */
-const SharedConfigPlugin: PluginType<{}, { filename: string }> = {
-  async $beforeSend(
-    mutableFilesystem,
-    { config, serialiser, ignorePages, pageExtensions },
-    options
-  ) {
-    const pagePaths = await mutableFilesystem.promises.glob(
-      createFileGlob('**/index,/index', pageExtensions),
-      {
-        ignore: [options.filename, ...flatten(ignorePages.map(ignore => [ignore, `**/${ignore}`]))],
-        cwd: '/'
-      }
-    );
-
-    for (const pagePath of pagePaths) {
-      const sharedConfigFile = path.join(path.dirname(String(pagePath)), options.filename);
-
-      const page = await serialiser.deserialise(
-        String(pagePath),
-        await mutableFilesystem.promises.readFile(String(pagePath))
-      );
-      if (page.sharedConfig) {
-        config.setRef(sharedConfigFile, ['config', '$ref'], `${String(pagePath)}#/sharedConfig`);
-        await mutableFilesystem.promises.writeFile(sharedConfigFile, '{}');
-      } else {
-        const baseDir = path.resolve(path.dirname(String(pagePath)), '../');
-
-        config.setAliases(path.join(baseDir, options.filename), [sharedConfigFile]);
-      }
-    }
-  }
-};
-
-export default SharedConfigPlugin;
-
-function createFileGlob(url, pageExtensions) {
-  if (pageExtensions.length === 1) {
-    return `${url}${pageExtensions[0]}`;
-  }
-  return `${url}{${pageExtensions.join(',')}}`;
-}