@static-suite/static-suite-data-server 0.0.1 → 0.0.3

package/src/lib/store/dump/dumpManager.ts

@@ -2,27 +2,19 @@ import fs from 'fs';
 import path from 'path';
 import microtime from 'microtime';
 import { logger } from '@lib/utils/logger';
-import { jsonify } from '@lib/utils/object';
-import { getFileContent, isJsonFile, readFile } from '@lib/utils/fs';
+import {
+  getFileContent,
+  isJsonFile,
+  readFile,
+  removeEmptyDirsUpwards,
+} from '@lib/utils/fs';
 import { config } from '@lib/config';
 import { store } from '@lib/store';
 import { diffManager } from '../diff/diffManager';
-import { Diff } from '../diff/diffManager.types';
-import { Dump, DumpManager } from './dumpManager.types';
+import { Diff } from '../diff/diff.types';
+import { Dump, DumpManager } from './dump.types';
 import { hookManager } from '../hook';
-
-const removeEmptyDirsUpwards = (dir: string): void => {
-  const isEmpty = fs.readdirSync(dir).length === 0;
-  if (isEmpty) {
-    try {
-      fs.rmdirSync(dir);
-    } catch (e) {
-      logger.debug(`Error deleting empty directory "${dir}": ${e}`);
-    }
-    // eslint-disable-next-line @typescript-eslint/no-unused-vars
-    removeEmptyDirsUpwards(path.dirname(dir));
-  }
-};
+import { dumpMetadataHelper } from './dumpMetadataHelper';
 
 const storeUpdatedFiles = (diff: Diff, dump: Dump, dumpDir: string): void => {
   diff.updated.forEach(relativeFilepath => {
@@ -59,9 +51,19 @@ const storeUpdatedFiles = (diff: Diff, dump: Dump, dumpDir: string): void => {
               JSON.parse(dumpFileContentString)?.data?.content?.url?.path ||
               null;
             /*
+            // Save a copy for later reference.
+            const backupAbsoluteFilepathInDumpDir =
+              absoluteFilepathInDumpDir.replace(
+                '/dump/files/',
+                '/dump/files.bak/',
+              );
+            const backupDir = path.dirname(backupAbsoluteFilepathInDumpDir);
+            if (!fs.existsSync(backupDir)) {
+              fs.mkdirSync(backupDir, { recursive: true });
+            }
             fs.renameSync(
               absoluteFilepathInDumpDir,
-              absoluteFilepathInDumpDir.replace('.json', '.2.json'),
+              backupAbsoluteFilepathInDumpDir,
             );
             */
           }
@@ -102,6 +104,9 @@ const removeDeletedFiles = (diff: Diff, dump: Dump, dumpDir: string): void => {
         // Delete it.
         fs.unlinkSync(absoluteFilepath);
 
+        // Delete any empty directory.
+        removeEmptyDirsUpwards(path.dirname(absoluteFilepath));
+
         // Mark it as deleted.
         dump.deleted.set(relativeFilepath, {
           oldPublicUrl,
@@ -114,51 +119,23 @@ const removeDeletedFiles = (diff: Diff, dump: Dump, dumpDir: string): void => {
   });
 };
 
-const storeDumpMetadata = (
-  metadataFilepath: string,
-  dump: Dump,
-  diffResetDate: Date,
-) => {
-  let currentDumpMetadata: Dump[] = [];
-  try {
-    currentDumpMetadata = JSON.parse(
-      fs.readFileSync(metadataFilepath).toString(),
-    );
-  } catch (e) {
-    currentDumpMetadata = [];
-  }
-  currentDumpMetadata.push(jsonify(dump));
-  try {
-    const currentDumpMetadataString = JSON.stringify(currentDumpMetadata);
-    fs.writeFileSync(metadataFilepath, currentDumpMetadataString);
-    // Resetting the diff must happen when dump metadata is successfully
-    // stored into disk.
-    diffManager.reset(diffResetDate);
-  } catch (e) {
-    logger.error(
-      `Dump: error saving dump metadata to "${metadataFilepath}": ${e}`,
-    );
-  }
-};
-
 export const dumpManager: DumpManager = {
   dump(options = { incremental: true }): Dump {
     const startDate = microtime.now();
 
-    // Create a diff reset date just before consuming a diff.
-    const diffResetDate = new Date();
-    const diff = diffManager.getDiff({ incremental: options.incremental });
-
     // Diff data is processed and transformed into a dump object.
+    const diff = diffManager.getDiff({ incremental: options.incremental });
     let dump: Dump = {
-      since: diff.since,
+      execTimeMs: 0,
+      fromUniqueId: diff.fromUniqueId,
+      toUniqueId: diff.toUniqueId,
       updated: new Map(),
       deleted: new Map(),
+      diff,
     };
 
     if (config.dumpDir) {
       const dumpDir = `${config.dumpDir}/files`;
-      const metadataFilepath = `${config.dumpDir}/metadata.json`;
 
       if (diff.updated.size || diff.deleted.size) {
         // Store updated files.
@@ -175,19 +152,26 @@ export const dumpManager: DumpManager = {
 
         // Merge and store dump metadata if any.
         if (dump.updated.size || dump.deleted.size) {
-          storeDumpMetadata(metadataFilepath, dump, diffResetDate);
+          const storeSuccessful = dumpMetadataHelper.storeDumpMetadata(dump);
+          if (storeSuccessful) {
+            // Resetting the diff must happen when dump metadata is successfully
+            // stored into disk.
+            diffManager.reset(diff.toUniqueId);
+          }
 
           logger.info(
             `Dump created in ${execTimeMs} ms. Updated: ${dump.updated.size} / Deleted: ${dump.deleted.size}`,
           );
 
+          /*
           // Log dump if not empty
           if (dump.updated.size || dump.deleted.size) {
             logger.debug(`Dump: "${JSON.stringify(jsonify(dump))}"`);
           }
+          */
         } else {
           // Resetting the diff must happen when no other operations are pending.
-          diffManager.reset(diffResetDate);
+          diffManager.reset(diff.toUniqueId);
           logger.info(
             `Dump done in ${execTimeMs} ms without changes stored into disk.`,
           );
@@ -202,30 +186,11 @@ export const dumpManager: DumpManager = {
     return dump;
   },
 
-  reset(timestamp: number): void {
+  reset(uniqueId): void {
     if (config.dumpDir) {
-      const metadataFilepath = `${config.dumpDir}/metadata.json`;
-
-      const metadata = JSON.parse(
-        fs.existsSync(metadataFilepath)
-          ? fs.readFileSync(metadataFilepath).toString()
-          : '[]',
-      );
-
-      // Remove any dump data older that timestamp
-      const resetMetadata = metadata.filter(
-        (dump: Dump) => dump.since > timestamp,
-      );
-
+      const resetMetadata =
+        dumpMetadataHelper.removeDumpDataOlderThan(uniqueId);
       logger.debug(`Dump reset : "${JSON.stringify(resetMetadata)}"`);
-
-      try {
-        fs.writeFileSync(metadataFilepath, JSON.stringify(resetMetadata));
-      } catch (e) {
-        logger.error(
-          `Dump: error resetting dump metadata to "${metadataFilepath}": ${e}`,
-        );
-      }
     } else {
       logger.error(
         '"dumpDir" option not provided. Dump reset cannot be executed.',

package/src/lib/store/dump/dumpMetadataHelper.ts

@@ -0,0 +1,73 @@
+import fs from 'fs';
+import { logger } from '@lib/utils/logger';
+import { jsonify } from '@lib/utils/object';
+import { config } from '@lib/config';
+import { Dump, DumpMetadata } from './dump.types';
+import { unixEpochUniqueId } from '../workDir';
+
+const getMetadataFilepath = () => `${config.dumpDir}/metadata.json`;
+
+export const dumpMetadataHelper = {
+  storeDumpMetadata: (dump: Dump): boolean => {
+    const metadataFilepath = getMetadataFilepath();
+    const fallbackDumpMetadata: DumpMetadata = {
+      current: unixEpochUniqueId,
+      dumps: [],
+    };
+    let currentDumpMetadata = fallbackDumpMetadata;
+    try {
+      currentDumpMetadata = JSON.parse(
+        fs.readFileSync(metadataFilepath).toString(),
+      );
+    } catch (e) {
+      currentDumpMetadata = fallbackDumpMetadata;
+    }
+    currentDumpMetadata.current = dump.toUniqueId;
+    currentDumpMetadata.dumps.push(jsonify(dump));
+    try {
+      const currentDumpMetadataString = JSON.stringify(currentDumpMetadata);
+      fs.writeFileSync(metadataFilepath, currentDumpMetadataString);
+      return true;
+    } catch (e) {
+      logger.error(
+        `Dump: error saving dump metadata to "${metadataFilepath}": ${e}`,
+      );
+    }
+    return false;
+  },
+
+  removeDumpDataOlderThan(uniqueId: string): DumpMetadata {
+    const metadataFilepath = getMetadataFilepath();
+
+    const metadata = JSON.parse(
+      fs.existsSync(metadataFilepath)
+        ? fs.readFileSync(metadataFilepath).toString()
+        : '[]',
+    ) as DumpMetadata;
+
+    // Remove any dump data older than or equal to unique id.
+    metadata.dumps = metadata.dumps.filter(dump => dump.toUniqueId > uniqueId);
+
+    try {
+      fs.writeFileSync(metadataFilepath, JSON.stringify(metadata));
+    } catch (e) {
+      logger.error(
+        `Dump: error removing dump metadata older than "${uniqueId}" from "${metadataFilepath}": ${e}`,
+      );
+    }
+
+    return metadata;
+  },
+
+  getCurrentDumpUniqueId(): string {
+    const metadataFilepath = getMetadataFilepath();
+
+    const metadata = JSON.parse(
+      fs.existsSync(metadataFilepath)
+        ? fs.readFileSync(metadataFilepath).toString()
+        : '[]',
+    ) as DumpMetadata;
+
+    return metadata.current || unixEpochUniqueId;
+  },
+};

package/src/lib/store/hook/hook.types.ts

@@ -1,8 +1,10 @@
 import { ConfigOptions } from '@lib/config/config.types';
 import { Store } from '@lib/store/store.types';
 import { FileType } from '@lib/utils/fs/fs.types';
+import { Logger } from '@lib/utils/logger/logger.types';
+import { Json } from '@lib/utils/object/object.types';
 import { DependencyTagger } from '../dependency/dependency.types';
-import { Dump } from '../dump/dumpManager.types';
+import { Dump } from '../dump/dump.types';
 import { ChangedFiles } from '../workDir/workDir.types';
 
 /**
@@ -15,15 +17,15 @@ export type HookManager = {
 
   // Events when store is created/loaded for the first time.
   invokeOnStoreLoadStart(): void;
-  invokeOnProcessFile(options: FileHookOptions): FileType;
-  invokeOnStoreItemAdd(options: FileHookOptions): void;
+  invokeOnProcessFile(options: FileTypeHookOptions): FileType;
+  invokeOnStoreItemAdd(options: StoreItemHookOptions): void;
   invokeOnStoreLoadDone(): void;
 
   // Events when store changes, leading to update or remove actions.
   invokeOnStoreChangeStart(changedFiles: ChangedFiles): void;
-  invokeOnStoreItemBeforeUpdate(options: FileHookOptions): void;
-  invokeOnStoreItemAfterUpdate(options: FileHookOptions): void;
-  invokeOnStoreItemDelete(options: FileHookOptions): void;
+  invokeOnStoreItemBeforeUpdate(options: StoreItemHookOptions): void;
+  invokeOnStoreItemAfterUpdate(options: StoreItemHookOptions): void;
+  invokeOnStoreItemDelete(options: StoreItemHookOptions): void;
   invokeOnStoreChangeDone(changedFiles: ChangedFiles): void;
 
   // Dump related events
@@ -43,6 +45,11 @@ export interface OnModuleLoadHookOptions {
    * The data store.
    */
   store: Store;
+
+  /**
+   * The logger service.
+   */
+  logger: Logger;
 }
 
 /**
@@ -59,6 +66,11 @@ export interface BaseHookOptions {
    */
   store: Store;
 
+  /**
+   * The logger service.
+   */
+  logger: Logger;
+
   /**
    * The dependency tagger service.
    */
@@ -66,24 +78,47 @@ export interface BaseHookOptions {
 }
 
 /**
- * Options passed to a file hook.
+ * Options passed to a store item hook.
  */
-export interface FileHookOptions {
+export interface StoreItemHookOptions {
   /**
    * Relative file path inside the data dir.
    */
   relativeFilepath: string;
 
   /**
-   * File contents, an object with "raw" and "json" members.
+   * Contents of the store item.
    */
-  fileContent: FileType;
+  storeItem: Json | string | null;
+
+  /**
+   * Contents of the previous store item before store was updated.
+   */
+  previousStoreItem?: Json | string | null;
 }
 
+export interface FullStoreItemHookOptions
+  extends StoreItemHookOptions,
+    BaseHookOptions {}
+
 /**
- * Full options passed to a hook.
+ * Options passed to a file type hook.
  */
-export interface FullHookOptions extends BaseHookOptions, FileHookOptions {}
+export interface FileTypeHookOptions {
+  /**
+   * Relative file path inside the data dir.
+   */
+  relativeFilepath: string;
+
+  /**
+   * File contents, an object with "raw" and "json" members.
+   */
+  fileContent: FileType;
+}
+
+export interface FullFileTypeHookOptions
+  extends FileTypeHookOptions,
+    BaseHookOptions {}
 
 export interface ChangedFilesHookOptions extends BaseHookOptions {
   /**
@@ -139,11 +174,11 @@ export type HookModule = {
    * @remarks
    * This hook is aimed at altering the contents of the file before it being added to the store.
    *
-   * @param options - An object with options passed to the hook. @see {@link FileHookOptions}
+   * @param options - An object with options passed to the hook. @see {@link FullFileTypeHookOptions}
    *
-   * @returns The file contents, and object with "raw" and "json" members.
+   * @returns The file contents, an object with "raw" and "json" members.
    */
-  onProcessFile?(options: FullHookOptions): FileType;
+  onProcessFile?(options: FullFileTypeHookOptions): FileType;
 
   /**
    * A hook executed after a file is added into the store.
@@ -152,9 +187,9 @@ export type HookModule = {
    * This hook is aimed at adding some information to the store, e.g.- indexing all contents
    * by their taxonomy.
    *
-   * @param options - An object with options passed to the hook. @see {@link FileHookOptions}
+   * @param options - An object with options passed to the hook. @see {@link StoreItemHookOptions}
    */
-  onStoreItemAdd?(options: FullHookOptions): void;
+  onStoreItemAdd?(options: FullStoreItemHookOptions): void;
 
   /**
    * A hook executed after the store finishes loading.
@@ -184,9 +219,9 @@ export type HookModule = {
    * This hook is aimed at adding some information to the store, e.g.- indexing all contents
    * by their taxonomy.
    *
-   * @param options - An object with options passed to the hook. @see {@link FileHookOptions}
+   * @param options - An object with options passed to the hook. @see {@link StoreItemHookOptions}
    */
-  onStoreItemBeforeUpdate?(options: FullHookOptions): void;
+  onStoreItemBeforeUpdate?(options: FullStoreItemHookOptions): void;
 
   /**
    * A hook executed after a file is updated in the store.
@@ -195,9 +230,9 @@ export type HookModule = {
    * This hook is aimed at adding some information to the store, e.g.- indexing all contents
    * by their taxonomy.
    *
-   * @param options - An object with options passed to the hook. @see {@link FileHookOptions}
+   * @param options - An object with options passed to the hook. @see {@link StoreItemHookOptions}
    */
-  onStoreItemAfterUpdate?(options: FullHookOptions): void;
+  onStoreItemAfterUpdate?(options: FullStoreItemHookOptions): void;
 
   /**
    * A hook executed after a file is deleted from the store.
@@ -206,9 +241,9 @@ export type HookModule = {
    * This hook is aimed at reverting the actions taken in storeAdd hook, e.g.- removing a file
    * from a taxonomy index.
    *
-   * @param options - An object with options passed to the hook. @see {@link FileHookOptions}
+   * @param options - An object with options passed to the hook. @see {@link StoreItemHookOptions}
    */
-  onStoreItemDelete?(options: FullHookOptions): void;
+  onStoreItemDelete?(options: FullStoreItemHookOptions): void;
 
   /**
    * A hook executed after the store ends updating.

package/src/lib/store/hook/hookManager.ts

@@ -1,10 +1,11 @@
 import { config } from '@lib/config';
 import { store } from '@lib/store';
+import { logger } from '@lib/utils/logger';
 import { dependencyTagger } from '@lib/store/dependency/dependencyTagger';
 import { FileType } from '@lib/utils/fs/fs.types';
 import { dirBasedModuleGroupManager } from '@lib/utils/module';
 import { HookManager, HookModule } from './hook.types';
-import { Dump } from '../dump/dumpManager.types';
+import { Dump } from '../dump/dump.types';
 
 /**
  * A manager for hooks stored in a user-land directory defined by configuration.
@@ -30,6 +31,7 @@ export const hookManager: HookManager = {
       if (hookModule.onStoreLoadStart) {
         hookModule.onStoreLoadStart({
           store,
+          logger,
           dependencyTagger,
         });
       }
@@ -47,6 +49,7 @@ export const hookManager: HookManager = {
           relativeFilepath,
           fileContent: processedFileContent,
           store,
+          logger,
           dependencyTagger,
         });
         if (returnValue) {
@@ -57,16 +60,17 @@ export const hookManager: HookManager = {
     return processedFileContent;
   },
 
-  invokeOnStoreItemAdd({ relativeFilepath, fileContent }): void {
+  invokeOnStoreItemAdd({ relativeFilepath, storeItem }): void {
     const hookModulesInfo = moduleManager.getModuleGroupInfo();
     hookModulesInfo.forEach(hookInfo => {
       const hookModule = hookInfo.getModule();
       if (hookModule.onStoreItemAdd) {
         hookModule.onStoreItemAdd({
           relativeFilepath,
-          fileContent,
           store,
+          logger,
           dependencyTagger,
+          storeItem,
         });
       }
     });
@@ -79,6 +83,7 @@ export const hookManager: HookManager = {
       if (hookModule.onStoreLoadDone) {
         hookModule.onStoreLoadDone({
           store,
+          logger,
           dependencyTagger,
         });
       }
@@ -92,6 +97,7 @@ export const hookManager: HookManager = {
       if (hookModule.onStoreChangeStart) {
         hookModule.onStoreChangeStart({
           store,
+          logger,
           dependencyTagger,
           changedFiles,
         });
@@ -99,46 +105,54 @@ export const hookManager: HookManager = {
     });
   },
 
-  invokeOnStoreItemBeforeUpdate({ relativeFilepath, fileContent }) {
+  invokeOnStoreItemBeforeUpdate({ relativeFilepath, storeItem }) {
     const hookModulesInfo = moduleManager.getModuleGroupInfo();
     hookModulesInfo.forEach(hookInfo => {
       const hookModule = hookInfo.getModule();
       if (hookModule.onStoreItemBeforeUpdate) {
         hookModule.onStoreItemBeforeUpdate({
           store,
+          logger,
           dependencyTagger,
           relativeFilepath,
-          fileContent,
+          storeItem,
         });
       }
     });
   },
 
-  invokeOnStoreItemAfterUpdate({ relativeFilepath, fileContent }) {
+  invokeOnStoreItemAfterUpdate({
+    relativeFilepath,
+    storeItem,
+    previousStoreItem,
+  }) {
     const hookModulesInfo = moduleManager.getModuleGroupInfo();
     hookModulesInfo.forEach(hookInfo => {
       const hookModule = hookInfo.getModule();
       if (hookModule.onStoreItemAfterUpdate) {
         hookModule.onStoreItemAfterUpdate({
           store,
+          logger,
           dependencyTagger,
           relativeFilepath,
-          fileContent,
+          storeItem,
+          previousStoreItem,
         });
       }
     });
   },
 
-  invokeOnStoreItemDelete({ relativeFilepath, fileContent }) {
+  invokeOnStoreItemDelete({ relativeFilepath, storeItem }) {
     const hookModulesInfo = moduleManager.getModuleGroupInfo();
     hookModulesInfo.forEach(hookInfo => {
       const hookModule = hookInfo.getModule();
       if (hookModule.onStoreItemDelete) {
         hookModule.onStoreItemDelete({
           store,
+          logger,
           dependencyTagger,
           relativeFilepath,
-          fileContent,
+          storeItem,
         });
       }
     });
@@ -151,6 +165,7 @@ export const hookManager: HookManager = {
       if (hookModule.onStoreChangeDone) {
         hookModule.onStoreChangeDone({
           store,
+          logger,
           dependencyTagger,
           changedFiles,
         });

package/src/lib/store/store.ts

@@ -8,6 +8,7 @@ import {
   StoreSubset,
   StoreSubsetOptions,
 } from './store.types';
+import { unixEpochUniqueId } from './workDir';
 
 // Instantiate the subset cache so it can be accessed faster.
 const subsetCache = cache.bin<StoreSubset>('store-subset');
@@ -22,7 +23,8 @@ const initIndex = () => {
 
 const initData = () => new Map<string, any>() as StoreData<string, any>;
 export const store: Store = {
-  syncDate: null,
+  initialUniqueId: unixEpochUniqueId,
+  currentUniqueId: unixEpochUniqueId,
   data: initData(),
   deleted: new Set<string>(),
   index: initIndex(),
@@ -39,7 +41,7 @@ store.data.subset = (options: StoreSubsetOptions): StoreSubset => {
   };
   const opts = { ...defaultOptions, ...options };
 
-  let subset: StoreSubset = { filenames: [], items: [] };
+  let subset: StoreSubset = { map: new Map(), filenames: [], items: [] };
 
   const dirWithTrimmedSlashes = opts.dir?.replace(/^\//, '').replace(/\/$/, '');
   const dirPart = dirWithTrimmedSlashes ? `${dirWithTrimmedSlashes}/` : '';
@@ -77,8 +79,10 @@ store.data.subset = (options: StoreSubsetOptions): StoreSubset => {
     const mapKeys = Array.from(store.data.keys());
     mapKeys.forEach(k => {
       if (k.match(regex)) {
+        const storeItem = store.data.get(k);
+        subset.map.set(k, storeItem);
         subset.filenames.push(k);
-        subset.items.push(store.data.get(k));
+        subset.items.push(storeItem);
       }
     });
     subsetCache.set(pattern, subset);
@@ -96,7 +100,8 @@ store.data.subset = (options: StoreSubsetOptions): StoreSubset => {
  * Resets store and deletes all loaded data.
  */
 export const resetStore = (): void => {
-  store.syncDate = null;
+  store.initialUniqueId = unixEpochUniqueId;
+  store.currentUniqueId = unixEpochUniqueId;
   const previousSubset = store.data.subset;
   store.data = initData();
   store.data.subset = previousSubset;