@static-suite/static-suite-data-server 0.0.1

package/src/lib/store/dependency/dependencyFileHelper.ts

@@ -0,0 +1,50 @@
+import { Json } from '@lib/utils/object/object.types';
+import { dependencyTagger } from './dependencyTagger';
+
+export const dependencyIncludeHelper = {
+  setIncludeDependencies: (filepath: string, json: Json): void => {
+    const includes = json?.metadata?.includes;
+
+    if (includes?.static) {
+      const staticIncludes = Object.values(includes?.static);
+      dependencyTagger.setDependency(filepath, staticIncludes);
+    }
+
+    if (includes?.dynamic) {
+      const dynamicIncludes = Object.values(includes?.dynamic);
+      dependencyTagger.setDependency(filepath, dynamicIncludes);
+      // Queries are invalidated by any data change until they are executed,
+      // when they return their dependencies. They are "globally invalidable" when:
+      // 1) Are added for the first time
+      // 2) Are updated: file can contain another query include,
+      //    or different parameters, etc
+      // In another words:
+      // 1) When a file with a query is loaded for he first time,
+      //    that query has a dependency on "*"
+      // 2) When that query is executed, it has a dependency on "node:1234"
+      // 3) When that file with a query is updated, the query has again a dependency on "*"
+      dynamicIncludes.forEach(queryDefinition => {
+        dependencyTagger.setDependency(queryDefinition, ['*']);
+      });
+    }
+  },
+
+  deleteIncludeDependencies: (filepath: string, json: Json): void => {
+    const includes = json?.metadata?.includes;
+
+    if (includes?.static) {
+      const staticIncludes = Object.values(includes?.static);
+      dependencyTagger.deleteDependency(filepath, staticIncludes);
+    }
+
+    if (includes?.dynamic) {
+      const dynamicIncludes = Object.values(includes?.dynamic);
+      dependencyTagger.deleteDependency(filepath, dynamicIncludes);
+      // Queries are invalidated by any data change until they are executed,
+      // when they return their dependencies. Remove that dependency if still present.
+      dynamicIncludes.forEach(queryDefinition => {
+        dependencyTagger.deleteDependency(queryDefinition, ['*']);
+      });
+    }
+  },
+};

package/src/lib/store/dependency/dependencyManager.ts

@@ -0,0 +1,82 @@
+import { store } from '../store';
+import { DependencyManager } from './dependency.types';
+import { getReversedDependencyTree, invalidatedTags } from './dependencyTagger';
+
+const invalidatedFilepaths = {
+  updated: new Set<string>(),
+  deleted: new Set<string>(),
+};
+
+/**
+ * Gets a list of parents of a dependency tag
+ */
+const getTagParents = (tag: string, parents = new Set<string>()) => {
+  getReversedDependencyTree()
+    .get(tag)
+    ?.forEach(parent => {
+      if (!parents.has(parent)) {
+        parents.add(parent);
+        getTagParents(parent, parents);
+      }
+    });
+  return parents;
+};
+
+/**
+ * Gets a recursive list of invalidated tags.
+ *
+ * @remarks
+ * Iterates over all explicitly invalidated tags, and obtains all
+ * its parents.
+ */
+const getInvalidatedTags = (): string[] => {
+  const allInvalidatedTags = new Set<string>();
+  const invalidatedTagsArray = Array.from(invalidatedTags);
+  if (invalidatedTagsArray.length > 0) {
+    invalidatedTagsArray.push('*');
+  }
+  invalidatedTagsArray.forEach(invalidatedTag => {
+    // Mark this tag as invalidated.
+    if (invalidatedTag !== '*') {
+      allInvalidatedTags.add(invalidatedTag);
+    }
+
+    // Search for parents of this invalidated tag.
+    getTagParents(invalidatedTag).forEach(parentTag => {
+      allInvalidatedTags.add(parentTag);
+    });
+  });
+  return Array.from(allInvalidatedTags);
+};
+
+/**
+ * Manager that keeps control of dependencies between data.
+ */
+export const dependencyManager: DependencyManager = {
+  trackInvalidatedFilepaths: () => {
+    const allInvalidatedTags = getInvalidatedTags();
+    allInvalidatedTags.forEach(invalidatedTag => {
+      if (store.deleted.has(invalidatedTag)) {
+        invalidatedFilepaths.deleted.add(invalidatedTag);
+        invalidatedFilepaths.updated.delete(invalidatedTag);
+      } else if (store.data.has(invalidatedTag)) {
+        invalidatedFilepaths.updated.add(invalidatedTag);
+        invalidatedFilepaths.deleted.delete(invalidatedTag);
+      }
+    });
+  },
+
+  getInvalidatedFilepaths: () => {
+    dependencyManager.trackInvalidatedFilepaths();
+    return {
+      updated: Array.from(invalidatedFilepaths.updated),
+      deleted: Array.from(invalidatedFilepaths.deleted),
+    };
+  },
+
+  reset: () => {
+    invalidatedTags.clear();
+    invalidatedFilepaths.updated.clear();
+    invalidatedFilepaths.deleted.clear();
+  },
+};

package/src/lib/store/dependency/dependencyTagger.ts

@@ -0,0 +1,105 @@
+import { DependencyTagger, DependencyTree } from './dependency.types';
+
+export const dependencyTree: DependencyTree = new Map<string, Set<string>>();
+
+export const invalidatedTags = new Set<string>();
+
+const reversedDependencyTree: DependencyTree = new Map<string, Set<string>>();
+
+let reversedDependencyTreeIsValid = false;
+
+/**
+ * Makes a reverted version of the dependency tree
+ *
+ * @remarks Required to be able to search for dependencies in a fast way.
+ */
+export const getReversedDependencyTree = (): DependencyTree => {
+  if (!reversedDependencyTreeIsValid) {
+    reversedDependencyTree.clear();
+    dependencyTree.forEach((children, parent) => {
+      children.forEach(child => {
+        const previousData = reversedDependencyTree.get(child);
+        if (previousData) {
+          previousData.add(parent);
+        } else {
+          reversedDependencyTree.set(child, new Set([parent]));
+        }
+      });
+    });
+    reversedDependencyTreeIsValid = true;
+  }
+
+  return reversedDependencyTree;
+};
+
+export const dependencyTagger: DependencyTagger = {
+  setDependency: (tag, dependsOnTags) => {
+    dependencyTree.set(tag, new Set(dependsOnTags));
+    reversedDependencyTreeIsValid = false;
+  },
+
+  addDependency: (tag, dependsOnTags) => {
+    const previousData = dependencyTree.get(tag);
+    if (previousData) {
+      dependsOnTags.forEach(dependsOnTag => previousData.add(dependsOnTag));
+    } else {
+      dependencyTree.set(tag, new Set(dependsOnTags));
+    }
+    reversedDependencyTreeIsValid = false;
+  },
+
+  deleteDependency: (tag, dependsOnTags) => {
+    const previousData = dependencyTree.get(tag);
+    if (previousData) {
+      dependsOnTags?.forEach(dependsOnTag => previousData.delete(dependsOnTag));
+      if (previousData.size === 0) {
+        dependencyTree.delete(tag);
+      }
+    }
+    reversedDependencyTreeIsValid = false;
+  },
+
+  invalidateTags: tags => {
+    tags.forEach(tag => {
+      invalidatedTags.add(tag);
+    });
+  },
+};
+
+/*
+import { DependencyTagger } from './dependency.types';
+
+export const dependencyTree = new Map<string, Set<string>>();
+
+export const invalidatedTags = new Set<string>();
+
+export const dependencyTagger: DependencyTagger = {
+  setDependency: (tag, dependsOnTags) => {
+    dependsOnTags.forEach(dependsOnTag => {
+      const previousData = dependencyTree.get(dependsOnTag);
+      if (previousData) {
+        previousData.add(tag);
+      } else {
+        dependencyTree.set(dependsOnTag, new Set<string>([tag]));
+      }
+    });
+  },
+
+  deleteDependency: (tag, dependsOnTags) => {
+    dependsOnTags.forEach(dependsOnTag => {
+      const previousData = dependencyTree.get(dependsOnTag);
+      if (previousData) {
+        previousData.delete(tag);
+        if (previousData.size === 0) {
+          dependencyTree.delete(dependsOnTag);
+        }
+      }
+    });
+  },
+
+  invalidateTags: tags => {
+    tags.forEach(tag => {
+      invalidatedTags.add(tag);
+    });
+  },
+}; */

package/src/lib/store/diff/diffManager.ts

@@ -0,0 +1,122 @@
+import microtime from 'microtime';
+import { logger } from '@lib/utils/logger';
+import { store } from '../store';
+import { DiffManager, Diff } from './diffManager.types';
+import { dataDirManager } from '../dataDir/dataDirManager';
+import { dependencyManager } from '../dependency/dependencyManager';
+
+let lastDiffDate: Date | null = null;
+
+export const diffManager: DiffManager = {
+  reset(date: Date): void {
+    lastDiffDate = date;
+    dependencyManager.reset();
+  },
+
+  getDiff(options = { incremental: true }): Diff {
+    const startDate = microtime.now();
+
+    // Before getting any diff data, update any pending changes from data dir.
+    dataDirManager.update();
+
+    let updated = new Set<string>();
+    let deleted = new Set<string>();
+    if (options.incremental && lastDiffDate) {
+      logger.debug(`Getting incremental diff using date "${lastDiffDate}"`);
+      const invalidatedFilepaths = dependencyManager.getInvalidatedFilepaths();
+      updated = new Set<string>(invalidatedFilepaths.updated);
+      deleted = new Set<string>(invalidatedFilepaths.deleted);
+    } else {
+      // If no sinceDate is passed, return all files
+      logger.debug(`Getting full diff with no date`);
+      updated = new Set<string>(store.data.keys());
+    }
+
+    const diff: Diff = {
+      since: lastDiffDate ? lastDiffDate.getTime() : new Date(0).getTime(),
+      updated,
+      deleted,
+    };
+
+    logger.info(
+      `Diff created in ${(microtime.now() - startDate) / 1000} ms. Updated: ${
+        diff.updated.size
+      } / Deleted: ${diff.deleted.size}`,
+    );
+
+    // Log diff if not empty
+    // if (diff.updated.length || diff.deleted.length) {
+    // logger.debug(`Diff: "${JSON.stringify(jsonify(diff))}"`);
+    // }
+
+    return diff;
+  },
+
+  /*
+  getDiff(options = { incremental: true }): Diff {
+    const startDate = microtime.now();
+
+    // Before getting any diff data, update any pending changes from data dir.
+    const changedFiles = dataDirManager.update();
+
+    let updated = new Set<string>();
+    let deleted = new Set<string>();
+    const sinceDate = getLastDiffDate();
+    if (options.incremental && sinceDate) {
+      logger.debug(`Getting incremental diff using date "${sinceDate}"`);
+      // Only process changedFiles if not empty.
+      if (changedFiles.updated.length || changedFiles.deleted.length) {
+        // Update tracked files so no affected parent is missed.
+        changedFiles.updated.forEach(file => {
+          tracker.trackChangedFile(file);
+        });
+        changedFiles.deleted.forEach(file => {
+          tracker.trackChangedFile(file);
+        });
+      }
+
+      // Create the resulting set of updated and deleted files.
+      // "updated" includes all affected parents tracked down
+      // by tracker, without any deleted file.
+      updated = new Set<string>(tracker.getChangedFiles());
+      changedFiles.deleted.forEach(file => updated.delete(file));
+
+      // "deleted" includes only deleted files, and is based only
+      // on data coming from changedFiles, because all deleted files
+      // are stored on Static Suite Data Server log
+      deleted = new Set<string>(changedFiles.deleted);
+
+      // Add updated files affected by queries only if "updated" or "deleted"
+      // contain any changes.
+      if (updated.size > 0 || deleted.size > 0) {
+        queryTagManager
+          .getInvalidFilepaths()
+          .forEach(file => updated.add(file));
+        queryTagManager.resetInvalidatedTags();
+      }
+    } else {
+      // If no sinceDate is passed, return all files
+      logger.debug(`Getting full diff with no date`);
+      updated = new Set<string>(store.data.keys());
+    }
+
+    const diff: Diff = {
+      since: sinceDate ? sinceDate.getTime() : new Date(0).getTime(),
+      updated,
+      deleted,
+    };
+
+    logger.info(
+      `Diff created in ${(microtime.now() - startDate) / 1000} ms. Updated: ${
+        diff.updated.size
+      } / Deleted: ${diff.deleted.size}`,
+    );
+
+    // Log diff if not empty
+    // if (diff.updated.length || diff.deleted.length) {
+    // logger.debug(`Diff: "${JSON.stringify(jsonify(diff))}"`);
+    // }
+
+    return diff;
+  }, */
+};

package/src/lib/store/diff/diffManager.types.ts

@@ -0,0 +1,51 @@
+/**
+ * Service that handles the incremental differences in data (includes and queries).
+ */
+export type DiffManager = {
+  /**
+   * Clears intermediate changes tracked by trackChangedFile().
+   *
+   * @param date - Date to be used as a timestamp for the next diff.
+   */
+  reset(date: Date): void;
+
+  /**
+   * Gets a list of changed files affected by includes and queries.
+   *
+   * @remarks
+   * This function does not need any date passed as argument to get a diff from.
+   * It simply uses the timestamp of the last generated diff. Calling this function does
+   * not automatically update that timestamp, because diff data must be stored in
+   * disk (a process called "dump"). Once a dump is finished, we can use the new
+   * timestamp for further diffs.
+   *
+   * To obtain a diff, we need changed files from:
+   *
+   * 1) includes: every time a file changes, all its parents are affected.
+   * 2) queries: there is no easy way to know if a changed file will alter a query,
+   *   so we run all of them and check if something has changed.
+
+   * @param options - Configuration options
+   */
+  getDiff(options?: { incremental: boolean }): Diff;
+};
+
+/**
+ * A group of changed files in Static Suite's data dir.
+ */
+export type Diff = {
+  /**
+   * Timestamp in milliseconds.
+   */
+  since: number;
+
+  /**
+   * A list of updated files (both newly added and changed files).
+   */
+  updated: Set<string>;
+
+  /**
+   * A list of deleted files.
+   */
+  deleted: Set<string>;
+};

package/src/lib/store/diff/index.ts

@@ -0,0 +1 @@
+export * from './diffManager';

package/src/lib/store/dump/dumpManager.ts

@@ -0,0 +1,235 @@
+import fs from 'fs';
+import path from 'path';
+import microtime from 'microtime';
+import { logger } from '@lib/utils/logger';
+import { jsonify } from '@lib/utils/object';
+import { getFileContent, isJsonFile, readFile } from '@lib/utils/fs';
+import { config } from '@lib/config';
+import { store } from '@lib/store';
+import { diffManager } from '../diff/diffManager';
+import { Diff } from '../diff/diffManager.types';
+import { Dump, DumpManager } from './dumpManager.types';
+import { hookManager } from '../hook';
+
+const removeEmptyDirsUpwards = (dir: string): void => {
+  const isEmpty = fs.readdirSync(dir).length === 0;
+  if (isEmpty) {
+    try {
+      fs.rmdirSync(dir);
+    } catch (e) {
+      logger.debug(`Error deleting empty directory "${dir}": ${e}`);
+    }
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    removeEmptyDirsUpwards(path.dirname(dir));
+  }
+};
+
+const storeUpdatedFiles = (diff: Diff, dump: Dump, dumpDir: string): void => {
+  diff.updated.forEach(relativeFilepath => {
+    const storeFileContentData = store.data.get(relativeFilepath);
+    if (storeFileContentData) {
+      const absoluteFilepathInDumpDir = `${dumpDir}/${relativeFilepath}`;
+      try {
+        // Create parent directories if they are missing.
+        const dir = path.dirname(absoluteFilepathInDumpDir);
+        if (!fs.existsSync(dir)) {
+          fs.mkdirSync(dir, { recursive: true });
+        }
+
+        // If store data is an object, stringify it to execute its
+        // queries and resolve its includes.
+        const isJson = isJsonFile(relativeFilepath);
+        const storeFileContentString = isJson
+          ? JSON.stringify(storeFileContentData)
+          : storeFileContentData;
+
+        // Before overwriting a file, check it has changed.
+        let needsSave = true;
+        let oldPublicUrl: string | null = null;
+        const newPublicUrl: string | null =
+          storeFileContentData.data?.content?.url?.path || null;
+        if (fs.existsSync(absoluteFilepathInDumpDir)) {
+          const dumpFileContentString = readFile(absoluteFilepathInDumpDir);
+          if (storeFileContentString === dumpFileContentString) {
+            // No need to save it, file has not changed.
+            needsSave = false;
+          } else if (isJson && dumpFileContentString) {
+            // Get old public URL before it changes on disk.
+            oldPublicUrl =
+              JSON.parse(dumpFileContentString)?.data?.content?.url?.path ||
+              null;
+            /*
+            fs.renameSync(
+              absoluteFilepathInDumpDir,
+              absoluteFilepathInDumpDir.replace('.json', '.2.json'),
+            );
+            */
+          }
+        }
+        if (needsSave) {
+          // Save it.
+          fs.writeFileSync(absoluteFilepathInDumpDir, storeFileContentString);
+
+          // Mark it as updated.
+          dump.updated.set(relativeFilepath, {
+            oldPublicUrl,
+            newPublicUrl,
+          });
+        }
+      } catch (e) {
+        logger.error(
+          `Dump: error writing file "${absoluteFilepathInDumpDir}": ${e}`,
+        );
+      }
+    } else {
+      logger.error(
+        `Dump: store data for file "${relativeFilepath}" not found.`,
+      );
+    }
+  });
+};
+
+const removeDeletedFiles = (diff: Diff, dump: Dump, dumpDir: string): void => {
+  diff.deleted.forEach(relativeFilepath => {
+    const absoluteFilepath = `${dumpDir}/${relativeFilepath}`;
+    try {
+      if (fs.existsSync(absoluteFilepath)) {
+        // Get old public URLs before they change on disk.
+        const dumpContent = getFileContent(absoluteFilepath);
+        const oldPublicUrl: string | null =
+          dumpContent.json?.data?.content?.url?.path;
+
+        // Delete it.
+        fs.unlinkSync(absoluteFilepath);
+
+        // Mark it as deleted.
+        dump.deleted.set(relativeFilepath, {
+          oldPublicUrl,
+          newPublicUrl: null,
+        });
+      }
+    } catch (e) {
+      logger.error(`Dump: error deleting file "${absoluteFilepath}": ${e}`);
+    }
+  });
+};
+
+const storeDumpMetadata = (
+  metadataFilepath: string,
+  dump: Dump,
+  diffResetDate: Date,
+) => {
+  let currentDumpMetadata: Dump[] = [];
+  try {
+    currentDumpMetadata = JSON.parse(
+      fs.readFileSync(metadataFilepath).toString(),
+    );
+  } catch (e) {
+    currentDumpMetadata = [];
+  }
+  currentDumpMetadata.push(jsonify(dump));
+  try {
+    const currentDumpMetadataString = JSON.stringify(currentDumpMetadata);
+    fs.writeFileSync(metadataFilepath, currentDumpMetadataString);
+    // Resetting the diff must happen when dump metadata is successfully
+    // stored into disk.
+    diffManager.reset(diffResetDate);
+  } catch (e) {
+    logger.error(
+      `Dump: error saving dump metadata to "${metadataFilepath}": ${e}`,
+    );
+  }
+};
+
+export const dumpManager: DumpManager = {
+  dump(options = { incremental: true }): Dump {
+    const startDate = microtime.now();
+
+    // Create a diff reset date just before consuming a diff.
+    const diffResetDate = new Date();
+    const diff = diffManager.getDiff({ incremental: options.incremental });
+
+    // Diff data is processed and transformed into a dump object.
+    let dump: Dump = {
+      since: diff.since,
+      updated: new Map(),
+      deleted: new Map(),
+    };
+
+    if (config.dumpDir) {
+      const dumpDir = `${config.dumpDir}/files`;
+      const metadataFilepath = `${config.dumpDir}/metadata.json`;
+
+      if (diff.updated.size || diff.deleted.size) {
+        // Store updated files.
+        storeUpdatedFiles(diff, dump, dumpDir);
+
+        // Remove deleted files.
+        removeDeletedFiles(diff, dump, dumpDir);
+
+        // Invoke "onDumpCreate" hook.
+        dump = hookManager.invokeOnDumpCreate(dump);
+
+        const execTimeMs = (microtime.now() - startDate) / 1000;
+        dump.execTimeMs = execTimeMs;
+
+        // Merge and store dump metadata if any.
+        if (dump.updated.size || dump.deleted.size) {
+          storeDumpMetadata(metadataFilepath, dump, diffResetDate);
+
+          logger.info(
+            `Dump created in ${execTimeMs} ms. Updated: ${dump.updated.size} / Deleted: ${dump.deleted.size}`,
+          );
+
+          // Log dump if not empty
+          if (dump.updated.size || dump.deleted.size) {
+            logger.debug(`Dump: "${JSON.stringify(jsonify(dump))}"`);
+          }
+        } else {
+          // Resetting the diff must happen when no other operations are pending.
+          diffManager.reset(diffResetDate);
+          logger.info(
+            `Dump done in ${execTimeMs} ms without changes stored into disk.`,
+          );
+        }
+      } else {
+        logger.info('Dump not stored into disk due to an empty diff.');
+      }
+    } else {
+      logger.error('"dumpDir" option not provided. Dump cannot be executed.');
+    }
+
+    return dump;
+  },
+
+  reset(timestamp: number): void {
+    if (config.dumpDir) {
+      const metadataFilepath = `${config.dumpDir}/metadata.json`;
+
+      const metadata = JSON.parse(
+        fs.existsSync(metadataFilepath)
+          ? fs.readFileSync(metadataFilepath).toString()
+          : '[]',
+      );
+
+      // Remove any dump data older that timestamp
+      const resetMetadata = metadata.filter(
+        (dump: Dump) => dump.since > timestamp,
+      );
+
+      logger.debug(`Dump reset : "${JSON.stringify(resetMetadata)}"`);
+
+      try {
+        fs.writeFileSync(metadataFilepath, JSON.stringify(resetMetadata));
+      } catch (e) {
+        logger.error(
+          `Dump: error resetting dump metadata to "${metadataFilepath}": ${e}`,
+        );
+      }
+    } else {
+      logger.error(
+        '"dumpDir" option not provided. Dump reset cannot be executed.',
+      );
+    }
+  },
+};