@static-suite/static-suite-data-server 0.0.1

package/src/lib/store/dump/dumpManager.types.ts

@@ -0,0 +1,61 @@
+/**
+ * Service that dumps resolved data (with resolved includes) into a dump directory.
+ */
+export type DumpManager = {
+  /**
+   * Dump resolved data (with resolved includes) into a dump directory.
+   *
+   * @remarks
+   * Raw data stored in Data Server does not resolve its includes until
+   * each file is consumed (and therefore stringified). A dump consumes
+   * that raw data and stores it in a directory, along its metadata
+   * (a list of changed/deleted files), so a later process can store
+   * those changes in another service like AWS S3.
+   *
+   * @param options - Configuration options
+   */
+  dump(options?: { incremental: boolean }): Dump;
+
+  /**
+   * Reset dump metadata.
+   *
+   * @remarks
+   * Resets dump metadata, removing any dump information older than the passed timestamp.
+   *
+   * @param timestamp - Timestamp
+   */
+  reset(timestamp: number): void;
+};
+
+/**
+ * A group of files to by dumped to dump directory.
+ */
+export type Dump = {
+  /**
+   * Timestamp in milliseconds.
+   */
+  since: number;
+
+  /**
+   * A list of updated files, keyed by file path. Its value is an object with two keys,
+   * old public URL and new public URL (the value stored at data.content.url.path) before
+   * and after updating file in dumpDir.
+   */
+  updated: Map<
+    string,
+    { oldPublicUrl: string | null; newPublicUrl: string | null }
+  >;
+
+  /**
+   * A list of deleted files, keyed by file path. Its value is an object with two keys,
+   * old public URL and new public URL (the value stored at data.content.url.path) before
+   * and after updating file in dumpDir. New public URL is always null, since file is
+   * deleted. We keep the same structure for both updated and deleted items.
+   */
+  deleted: Map<string, { oldPublicUrl: string | null; newPublicUrl: null }>;
+
+  /**
+   * Optional execution time taken by the dump, in milliseconds.
+   */
+  execTimeMs?: number;
+};

package/src/lib/store/dump/index.ts

@@ -0,0 +1 @@
+export * from './dumpManager';

package/src/lib/store/hook/hook.types.ts

@@ -0,0 +1,232 @@
+import { ConfigOptions } from '@lib/config/config.types';
+import { Store } from '@lib/store/store.types';
+import { FileType } from '@lib/utils/fs/fs.types';
+import { DependencyTagger } from '../dependency/dependency.types';
+import { Dump } from '../dump/dumpManager.types';
+import { ChangedFiles } from '../workDir/workDir.types';
+
+/**
+ * A manager for a group of user-land modules.
+ *
+ * @typeParam ModuleType - The type of module being managed.
+ */
+export type HookManager = {
+  reset(): void;
+
+  // Events when store is created/loaded for the first time.
+  invokeOnStoreLoadStart(): void;
+  invokeOnProcessFile(options: FileHookOptions): FileType;
+  invokeOnStoreItemAdd(options: FileHookOptions): void;
+  invokeOnStoreLoadDone(): void;
+
+  // Events when store changes, leading to update or remove actions.
+  invokeOnStoreChangeStart(changedFiles: ChangedFiles): void;
+  invokeOnStoreItemBeforeUpdate(options: FileHookOptions): void;
+  invokeOnStoreItemAfterUpdate(options: FileHookOptions): void;
+  invokeOnStoreItemDelete(options: FileHookOptions): void;
+  invokeOnStoreChangeDone(changedFiles: ChangedFiles): void;
+
+  // Dump related events
+  invokeOnDumpCreate(dump: Dump): Dump;
+};
+
+/**
+ * Options passed to an onModuleLoad hook.
+ */
+export interface OnModuleLoadHookOptions {
+  /**
+   * Configuration options.
+   */
+  config: ConfigOptions;
+
+  /**
+   * The data store.
+   */
+  store: Store;
+}
+
+/**
+ * Options passed to a hook.
+ *
+ * @remarks
+ * Since hooks are user-land modules, they do not have access to configuration
+ * or any other part of the Data Server. All data they need to function must be
+ * passed as parameters.
+ */
+export interface BaseHookOptions {
+  /**
+   * The data store.
+   */
+  store: Store;
+
+  /**
+   * The dependency tagger service.
+   */
+  dependencyTagger: DependencyTagger;
+}
+
+/**
+ * Options passed to a file hook.
+ */
+export interface FileHookOptions {
+  /**
+   * Relative file path inside the data dir.
+   */
+  relativeFilepath: string;
+
+  /**
+   * File contents, an object with "raw" and "json" members.
+   */
+  fileContent: FileType;
+}
+
+/**
+ * Full options passed to a hook.
+ */
+export interface FullHookOptions extends BaseHookOptions, FileHookOptions {}
+
+export interface ChangedFilesHookOptions extends BaseHookOptions {
+  /**
+   * A group of changed files in Static Suite's data dir.
+   */
+  changedFiles: ChangedFiles;
+}
+
+export interface OnDumpHookOptions {
+  /**
+   * Path to the dump directory.
+   */
+  dumpDir: string;
+
+  /**
+   * The data store.
+   */
+  store: Store;
+
+  /**
+   * The dump to be processed.
+   */
+  dump: Dump;
+}
+
+/**
+ * A module that defines several hooks.
+ */
+export type HookModule = {
+  /**
+   * A hook executed when a hook module is loaded or reloaded.
+   *
+   * @remarks
+   * This hook is aimed at bootstrapping some data structure, or establishing some connection
+   * to an external system before other hooks are run.
+   */
+  onModuleLoad?(options: OnModuleLoadHookOptions): void;
+
+  /**
+   * A hook executed before the store starts loading.
+   *
+   * @remarks
+   * This hook is aimed at bootstrapping some data structure, or establishing some connection
+   * to an external system before other hooks are run.
+   *
+   * @param options - An object with options passed to the hook. @see {@link HookOptions}
+   */
+  onStoreLoadStart?(options: BaseHookOptions): void;
+
+  /**
+   * A hook executed after a file is read from disk, before adding it to the store.
+   *
+   * @remarks
+   * This hook is aimed at altering the contents of the file before it being added to the store.
+   *
+   * @param options - An object with options passed to the hook. @see {@link FileHookOptions}
+   *
+   * @returns The file contents, and object with "raw" and "json" members.
+   */
+  onProcessFile?(options: FullHookOptions): FileType;
+
+  /**
+   * A hook executed after a file is added into the store.
+   *
+   * @remarks
+   * This hook is aimed at adding some information to the store, e.g.- indexing all contents
+   * by their taxonomy.
+   *
+   * @param options - An object with options passed to the hook. @see {@link FileHookOptions}
+   */
+  onStoreItemAdd?(options: FullHookOptions): void;
+
+  /**
+   * A hook executed after the store finishes loading.
+   *
+   * @remarks
+   * This hook is aimed at adding some final information to the store, e.g.- counting all images, etc
+   *
+   * @param options - An object with options passed to the hook. @see {@link HookOptions}
+   */
+  onStoreLoadDone?(options: BaseHookOptions): void;
+
+  /**
+   * A hook executed before the store starts updating.
+   *
+   * @remarks
+   * This hook is aimed at bootstrapping or updating some data structure
+   * before a set of update/delete operations starts.
+   *
+   * @param options - An object with options passed to the hook. @see {@link HookOptions}
+   */
+  onStoreChangeStart?(options: ChangedFilesHookOptions): void;
+
+  /**
+   * A hook executed before a file is updated in the store.
+   *
+   * @remarks
+   * This hook is aimed at adding some information to the store, e.g.- indexing all contents
+   * by their taxonomy.
+   *
+   * @param options - An object with options passed to the hook. @see {@link FileHookOptions}
+   */
+  onStoreItemBeforeUpdate?(options: FullHookOptions): void;
+
+  /**
+   * A hook executed after a file is updated in the store.
+   *
+   * @remarks
+   * This hook is aimed at adding some information to the store, e.g.- indexing all contents
+   * by their taxonomy.
+   *
+   * @param options - An object with options passed to the hook. @see {@link FileHookOptions}
+   */
+  onStoreItemAfterUpdate?(options: FullHookOptions): void;
+
+  /**
+   * A hook executed after a file is deleted from the store.
+   *
+   * @remarks
+   * This hook is aimed at reverting the actions taken in storeAdd hook, e.g.- removing a file
+   * from a taxonomy index.
+   *
+   * @param options - An object with options passed to the hook. @see {@link FileHookOptions}
+   */
+  onStoreItemDelete?(options: FullHookOptions): void;
+
+  /**
+   * A hook executed after the store ends updating.
+   *
+   * @remarks
+   * This hook is run after a set of update/delete operations finishes.
+   *
+   * @param options - An object with options passed to the hook. @see {@link HookOptions}
+   */
+  onStoreChangeDone?(options: ChangedFilesHookOptions): void;
+
+  /**
+   * A hook executed after a dump object is created.
+   *
+   * @remarks
+   * This hook can alter the contents of the dump being saved, and add/remove items.
+   *
+   * @param options - An object with options passed to the hook. @see {@link OnDumpHookOptions}
+   */
+  onDumpCreate?(options: OnDumpHookOptions): Dump;
+};

package/src/lib/store/hook/hookManager.ts

@@ -0,0 +1,180 @@
+import { config } from '@lib/config';
+import { store } from '@lib/store';
+import { dependencyTagger } from '@lib/store/dependency/dependencyTagger';
+import { FileType } from '@lib/utils/fs/fs.types';
+import { dirBasedModuleGroupManager } from '@lib/utils/module';
+import { HookManager, HookModule } from './hook.types';
+import { Dump } from '../dump/dumpManager.types';
+
+/**
+ * A manager for hooks stored in a user-land directory defined by configuration.
+ *
+ * @remarks
+ * It extends dirBasedModuleGroupManager, which manages groups of modules based
+ * on a directory: @see {@link dirBasedModuleGroupManager}
+ */
+const moduleManager = dirBasedModuleGroupManager<HookModule>('hook');
+
+/**
+ * A specialized hook manager with invoke helpers for all hooks
+ */
+export const hookManager: HookManager = {
+  reset() {
+    moduleManager.reset();
+  },
+
+  invokeOnStoreLoadStart() {
+    const hookModulesInfo = moduleManager.getModuleGroupInfo();
+    hookModulesInfo.forEach(hookInfo => {
+      const hookModule = hookInfo.getModule();
+      if (hookModule.onStoreLoadStart) {
+        hookModule.onStoreLoadStart({
+          store,
+          dependencyTagger,
+        });
+      }
+    });
+  },
+
+  invokeOnProcessFile({ relativeFilepath, fileContent }) {
+    let processedFileContent = fileContent;
+    let returnValue: FileType | null = null;
+    const hookModulesInfo = moduleManager.getModuleGroupInfo();
+    hookModulesInfo.forEach(hookInfo => {
+      const hookModule = hookInfo.getModule();
+      if (hookModule.onProcessFile) {
+        returnValue = hookModule.onProcessFile({
+          relativeFilepath,
+          fileContent: processedFileContent,
+          store,
+          dependencyTagger,
+        });
+        if (returnValue) {
+          processedFileContent = returnValue;
+        }
+      }
+    });
+    return processedFileContent;
+  },
+
+  invokeOnStoreItemAdd({ relativeFilepath, fileContent }): void {
+    const hookModulesInfo = moduleManager.getModuleGroupInfo();
+    hookModulesInfo.forEach(hookInfo => {
+      const hookModule = hookInfo.getModule();
+      if (hookModule.onStoreItemAdd) {
+        hookModule.onStoreItemAdd({
+          relativeFilepath,
+          fileContent,
+          store,
+          dependencyTagger,
+        });
+      }
+    });
+  },
+
+  invokeOnStoreLoadDone() {
+    const hookModulesInfo = moduleManager.getModuleGroupInfo();
+    hookModulesInfo.forEach(hookInfo => {
+      const hookModule = hookInfo.getModule();
+      if (hookModule.onStoreLoadDone) {
+        hookModule.onStoreLoadDone({
+          store,
+          dependencyTagger,
+        });
+      }
+    });
+  },
+
+  invokeOnStoreChangeStart(changedFiles) {
+    const hookModulesInfo = moduleManager.getModuleGroupInfo();
+    hookModulesInfo.forEach(hookInfo => {
+      const hookModule = hookInfo.getModule();
+      if (hookModule.onStoreChangeStart) {
+        hookModule.onStoreChangeStart({
+          store,
+          dependencyTagger,
+          changedFiles,
+        });
+      }
+    });
+  },
+
+  invokeOnStoreItemBeforeUpdate({ relativeFilepath, fileContent }) {
+    const hookModulesInfo = moduleManager.getModuleGroupInfo();
+    hookModulesInfo.forEach(hookInfo => {
+      const hookModule = hookInfo.getModule();
+      if (hookModule.onStoreItemBeforeUpdate) {
+        hookModule.onStoreItemBeforeUpdate({
+          store,
+          dependencyTagger,
+          relativeFilepath,
+          fileContent,
+        });
+      }
+    });
+  },
+
+  invokeOnStoreItemAfterUpdate({ relativeFilepath, fileContent }) {
+    const hookModulesInfo = moduleManager.getModuleGroupInfo();
+    hookModulesInfo.forEach(hookInfo => {
+      const hookModule = hookInfo.getModule();
+      if (hookModule.onStoreItemAfterUpdate) {
+        hookModule.onStoreItemAfterUpdate({
+          store,
+          dependencyTagger,
+          relativeFilepath,
+          fileContent,
+        });
+      }
+    });
+  },
+
+  invokeOnStoreItemDelete({ relativeFilepath, fileContent }) {
+    const hookModulesInfo = moduleManager.getModuleGroupInfo();
+    hookModulesInfo.forEach(hookInfo => {
+      const hookModule = hookInfo.getModule();
+      if (hookModule.onStoreItemDelete) {
+        hookModule.onStoreItemDelete({
+          store,
+          dependencyTagger,
+          relativeFilepath,
+          fileContent,
+        });
+      }
+    });
+  },
+
+  invokeOnStoreChangeDone(changedFiles) {
+    const hookModulesInfo = moduleManager.getModuleGroupInfo();
+    hookModulesInfo.forEach(hookInfo => {
+      const hookModule = hookInfo.getModule();
+      if (hookModule.onStoreChangeDone) {
+        hookModule.onStoreChangeDone({
+          store,
+          dependencyTagger,
+          changedFiles,
+        });
+      }
+    });
+  },
+
+  invokeOnDumpCreate(dump) {
+    let processedDump = dump;
+    let returnValue: Dump | null = null;
+    const hookModulesInfo = moduleManager.getModuleGroupInfo();
+    hookModulesInfo.forEach(hookInfo => {
+      const hookModule = hookInfo.getModule();
+      if (config.dumpDir && hookModule.onDumpCreate) {
+        returnValue = hookModule.onDumpCreate({
+          dumpDir: config.dumpDir,
+          store,
+          dump: processedDump,
+        });
+        if (returnValue) {
+          processedDump = returnValue;
+        }
+      }
+    });
+    return processedDump;
+  },
+};

package/src/lib/store/hook/index.ts

@@ -0,0 +1 @@
+export * from './hookManager';

package/src/lib/store/include/__tests__/includeParser.test.ts

@@ -0,0 +1,80 @@
+import fs from 'fs';
+import path from 'path';
+import { config } from '@lib/config';
+import { store } from '@lib/store';
+import { includeParser } from '../includeParser';
+
+const staticFileContent = {
+  data: {
+    content: {
+      id: '58724',
+      title: 'Title',
+      author: {
+        entity: {
+          entityInclude: 'article2.json',
+        },
+      },
+      image: {
+        entity: {
+          entityInclude: 'article2.json',
+        },
+      },
+      queryInclude: 'lastPublishedContents',
+    },
+  },
+  metadata: {
+    includes: {
+      static: {
+        'data.content.author.entity.entityInclude': 'article2.json',
+        'data.content.image.entity.entityInclude': 'article2.json',
+      },
+      dynamic: { 'data.content.queryInclude': 'lastPublishedContents' },
+    },
+  },
+};
+store.data.set('article2.json', {
+  data: {
+    content: {
+      id: '58728',
+      title: 'Title 2',
+    },
+  },
+});
+
+describe('staticIncludeParser', () => {
+  it('sets correct value by reference', () => {
+    includeParser.static('irrelevantFilePath', staticFileContent);
+    expect(staticFileContent.data.content.author.entity).toBe(
+      store.data.get('article2.json').data.content,
+    );
+    store.data.get('article2.json').data.content.title = 'Title 3';
+    expect(staticFileContent.data.content.author.entity).toBe(
+      store.data.get('article2.json').data.content,
+    );
+  });
+});
+
+const dynamicFileContent = {
+  data: {
+    content: {
+      queryInclude: 'query2?y=10',
+    },
+  },
+  metadata: {
+    includes: {
+      dynamic: { 'data.content.queryInclude': 'query2?y=10' },
+    },
+  },
+};
+config.queryDir = fs.realpathSync(path.resolve('src/__tests__/fixtures/query'));
+
+describe('queryIncludeParser', () => {
+  it('sets correct value', () => {
+    includeParser.dynamic(dynamicFileContent);
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-ignore
+    expect(dynamicFileContent.data.content.query.data).toStrictEqual([
+      { id: '1' },
+    ]);
+  });
+});

package/src/lib/store/include/includeParser.ts

@@ -0,0 +1,54 @@
+import { Json } from '@lib/utils/object/object.types';
+import { IncludeParser } from './includeParser.types';
+import {
+  configIncludeParser,
+  customIncludeParser,
+  entityIncludeParser,
+  localeIncludeParser,
+  queryIncludeParser,
+  // queryIncludeParser,
+} from './parsers';
+import { store } from '..';
+
+export const includeParser: IncludeParser = {
+  static: (relativeFilepath: string, host: Json): void => {
+    const staticIncludes = host?.metadata?.includes?.static;
+    if (!staticIncludes) {
+      return;
+    }
+    Object.keys(staticIncludes).forEach((includePath: string) => {
+      const mountPath = includePath.split('.');
+      const normalizedIncludePath = includePath.toLowerCase();
+      const includeKey = mountPath.pop();
+      if (includeKey) {
+        // All static includes share the same strategy, where the
+        // includePath gives access to the final target to be included.
+        const targetKey = staticIncludes[includePath];
+        const target = store.data.get(targetKey);
+        if (normalizedIncludePath.endsWith('configinclude')) {
+          configIncludeParser({ host, target, mountPath, includeKey });
+        }
+        if (normalizedIncludePath.endsWith('entityinclude')) {
+          entityIncludeParser({ host, target, mountPath, includeKey });
+        }
+        if (normalizedIncludePath.endsWith('custominclude')) {
+          customIncludeParser({ host, target, mountPath, includeKey });
+        }
+        if (normalizedIncludePath.endsWith('localeinclude')) {
+          localeIncludeParser({ host, target, mountPath, includeKey });
+        }
+      }
+    });
+  },
+
+  dynamic: (host: Json): void => {
+    const dynamicIncludes = host?.metadata?.includes?.dynamic;
+    if (!dynamicIncludes) {
+      return;
+    }
+    Object.keys(dynamicIncludes).forEach((includePath: string) => {
+      // No need to check if it is a query include.
+      queryIncludeParser({ host, includePath });
+    });
+  },
+};