@static-suite/static-suite-data-server 0.0.1 → 0.0.3

package/src/http/routes/index.ts

@@ -20,6 +20,15 @@ import {
   dumpMetadataShow,
   dumpMetadataReset,
 } from './dump';
+import {
+  dependencyIndex,
+  dependencyInvalidatedFilepaths,
+  dependencyExplicitlyInvalidatedTags,
+  dependencyAllInvalidatedTags,
+  dependencyTreeReversed,
+  dependencyTreeRoute,
+  dependencyTagParents,
+} from './dependency';
 
 const routes = Router();
 
@@ -42,6 +51,13 @@ routes.get('/dump/incremental', dumpIncremental);
 routes.get('/dump/full', dumpFull);
 routes.get('/dump/metadata/show', dumpMetadataShow);
 routes.get('/dump/metadata/reset', dumpMetadataReset);
+routes.get('/dependency/tree/reversed', dependencyTreeReversed);
+routes.get('/dependency/tree', dependencyTreeRoute);
+routes.get('/dependency/invalidated/tags', dependencyExplicitlyInvalidatedTags);
+routes.get('/dependency/invalidated/tags/all', dependencyAllInvalidatedTags);
+routes.get('/dependency/invalidated/filepaths', dependencyInvalidatedFilepaths);
+routes.get('/dependency/tag/parents', dependencyTagParents);
+routes.get('/dependency', dependencyIndex);
 routes.get(['/data/*', '/data'], data);
 
 export default routes;

package/src/http/routes/status.ts

@@ -8,11 +8,6 @@ import { store } from '@lib/store';
 import { jsonify } from '@lib/utils/object';
 import { diffManager } from '@lib/store/diff/diffManager';
 import { ObjectType } from '@lib/utils/object/object.types';
-import { dependencyManager } from '@lib/store/dependency/dependencyManager';
-import {
-  dependencyTree,
-  getReversedDependencyTree,
-} from '@lib/store/dependency/dependencyTagger';
 
 const statusIndex = (req: Request, res: Response): void => {
   res.render('statusIndex', {
@@ -32,14 +27,11 @@ const statusBasic = (req: Request, res: Response): void => {
 
   const response = {
     config,
-    dataDirLastUpdate: dataDirManager.getModificationDate(),
+    dataDirLastUpdate: dataDirManager.getModificationUniqueId(),
     query: {
       numberOfExecutions: queryRunner.getCount(),
       numberOfCachedQueries: cache.bin('query').size,
     },
-    dependencyTree: jsonify(dependencyTree),
-    reversedDependencyTree: jsonify(getReversedDependencyTree()),
-    invalidatedFilepaths: dependencyManager.getInvalidatedFilepaths(),
   };
 
   res.status(200);

package/src/http/views/dependencyIndex.pug

@@ -0,0 +1,14 @@
+extends includes/layout.pug
+
+block vars
+  - var title = "Dependency | Static Suite Data Server"
+
+block content
+  h1(class="title is-1")="Dependency"
+  table(class="table is-fullwidth is-hoverable")
+    tbody
+      each description, link in links
+        tr
+          td
+            a(href=link)= link
+          td=description

package/src/lib/dataServer.ts

@@ -7,6 +7,7 @@ import { store } from '@lib/store';
 import { Store } from '@lib/store/store.types';
 import { queryRunner } from '@lib/query';
 import { taskRunner } from '@lib/task/taskRunner';
+import { TaskRunner } from '@lib/task/task.types';
 import {
   QueryRunner,
   QuerySuccessfulResponse,
@@ -77,5 +78,6 @@ export {
   QueryErrorResponse,
   QueryArgs,
   QueryModule,
+  TaskRunner,
   CacheStatus,
 };

package/src/lib/query/query.types.ts

@@ -99,8 +99,8 @@ export type QueryModuleResult = {
  * @public
  */
 export enum CacheStatus {
-  MISS,
-  HIT,
+  MISS = 'miss',
+  HIT = 'hit',
 }
 
 /**

package/src/lib/query/queryRunner.ts

@@ -74,12 +74,14 @@ export const queryRunner: QueryRunner = {
           queryCache.set(cacheId, queryResult);
         }
 
-        // Set tags for this query.
+        // Set tags for this query, replacing any other, because query
+        // dependencies are set only by their "tags" field.
         const queryTags =
           queryResponse.tags && queryResponse.tags?.length > 0
             ? queryResponse.tags
             : ['*'];
         dependencyTagger.setDependency(queryDefinition, queryTags);
+        queryResult.tags = queryTags;
       } catch (e) {
         // Log error and rethrow.
         logger.error(

package/src/lib/store/dataDir/dataDir.types.ts

@@ -23,16 +23,16 @@ export type DataDirManager = {
   update(): ChangedFiles;
 
   /**
-   * Get date of last modification of data directory.
+   * Get unique id of last modification of data directory.
    *
    * @remarks
    * Metadata about when and how is the data directory updated is
    * stored in the work directory. If that directory is not present or
-   * not initialized with proper data, it returns the sync date of the store.
-   * If the store is not yet loaded, it returns a date in the past,
+   * not initialized with proper data, it returns the unique id of the store.
+   * If the store is not yet loaded, it returns a unique id in the past,
    * the Unix Epoch (00:00:00 UTC on 1 January 1970).
    *
-   * @returns The date of last modification of data directory
+   * @returns The unique id of last modification of data directory
    */
-  getModificationDate(): Date;
+  getModificationUniqueId(): string;
 };

package/src/lib/store/dataDir/dataDirManager.ts

@@ -3,7 +3,7 @@ import { config } from '@lib/config';
 import { store } from '@lib/store';
 import { findFilesInDir } from '@lib/utils/fs';
 import { logger } from '@lib/utils/logger';
-import { workDirHelper } from '@lib/store/workDir';
+import { unixEpochUniqueId, workDirHelper } from '@lib/store/workDir';
 import { cache } from '@lib/utils/cache';
 import { storeManager } from '@lib/store/storeManager';
 import { hookManager } from '@lib/store/hook';
@@ -13,6 +13,8 @@ import { DataDirManager } from './dataDir.types';
 import { ChangedFiles } from '../workDir/workDir.types';
 import { dependencyManager } from '../dependency/dependencyManager';
 
+let initialUniqueIdAlreadySet = false;
+
 export const dataDirManager: DataDirManager = {
   load: () => {
     logger.info('Loading data dir...');
@@ -28,11 +30,18 @@ export const dataDirManager: DataDirManager = {
     hookManager.reset();
     queryManager.reset();
     taskManager.reset();
+    dependencyManager.reset();
 
     // Invoke "store load start" hook.
     hookManager.invokeOnStoreLoadStart();
 
-    store.syncDate = dataDirManager.getModificationDate();
+    const modificationUniqueId = dataDirManager.getModificationUniqueId();
+    if (!initialUniqueIdAlreadySet) {
+      store.initialUniqueId = modificationUniqueId;
+      initialUniqueIdAlreadySet = true;
+    }
+    store.currentUniqueId = modificationUniqueId;
+    logger.info(`Found unique id: ${store.currentUniqueId}`);
 
     // Add all files, one by one.
     const relativeFilePaths = findFilesInDir(config.dataDir);
@@ -62,29 +71,34 @@ export const dataDirManager: DataDirManager = {
   },
 
   update: () => {
-    const dataDirModificationDate = dataDirManager.getModificationDate();
+    const dataDirModificationUniqueId =
+      dataDirManager.getModificationUniqueId();
 
     let changedFiles: ChangedFiles = {
       updated: [],
       deleted: [],
+      fromUniqueId: store.currentUniqueId,
+      toUniqueId: dataDirModificationUniqueId,
     };
 
-    if (store.syncDate) {
-      if (dataDirModificationDate > store.syncDate) {
-        // Save store.syncDate to a variable and set it ASAP to avoid two concurrent
-        // processes updating the data directory at the same time.
-        const storeLastSyncDate = store.syncDate;
-        store.syncDate = dataDirModificationDate;
+    if (store.currentUniqueId) {
+      if (dataDirModificationUniqueId > store.currentUniqueId) {
+        // Save store.syncDate to a variable and set it ASAP.
+        const storeLastUniqueId = store.currentUniqueId;
+        store.currentUniqueId = dataDirModificationUniqueId;
         const startDate = microtime.now();
         logger.debug(
-          `Data dir outdated. Current data loaded at ${storeLastSyncDate.toISOString()} but last updated at ${dataDirModificationDate.toISOString()}`,
+          `Data dir outdated. Current unique id in memory is ${storeLastUniqueId} but data dir is ${dataDirModificationUniqueId}`,
         );
 
         // Before updating anything, track down which filepaths are invalidated with
         // the current set of dependencies, before those dependencies change.
         dependencyManager.trackInvalidatedFilepaths();
 
-        changedFiles = workDirHelper.getChangedFilesSince(storeLastSyncDate);
+        changedFiles = workDirHelper.getChangedFilesBetween(
+          storeLastUniqueId,
+          dataDirModificationUniqueId,
+        );
         hookManager.invokeOnStoreChangeStart(changedFiles);
         changedFiles.updated.forEach(file => {
           storeManager.update(file);
@@ -107,7 +121,7 @@ export const dataDirManager: DataDirManager = {
         logger.debug(`Data dir updated in ${execTimeMs}ms.`);
       } else {
         logger.debug(
-          `Data dir up to date. Current data loaded at ${store.syncDate.toISOString()} and last updated at ${dataDirModificationDate.toISOString()}`,
+          `Data dir up to date. Current unique id in memory is ${store.currentUniqueId} and data dir is ${dataDirModificationUniqueId}`,
         );
       }
     } else {
@@ -116,6 +130,8 @@ export const dataDirManager: DataDirManager = {
     return changedFiles;
   },
 
-  getModificationDate: (): Date =>
-    workDirHelper.getModificationDate() || store.syncDate || new Date(0),
+  getModificationUniqueId: (): string =>
+    workDirHelper.getModificationUniqueId() ||
+    store.currentUniqueId ||
+    unixEpochUniqueId,
 };

package/src/lib/store/dependency/dependency.types.ts

@@ -37,11 +37,6 @@ export type DependencyTagger = {
    * @param tags - The list of tags for which to invalidate dependency items.
    */
   invalidateTags(tags: string[]): void;
-
-  /**
-   * Resets the list of invalidated tags once consumed.
-   */
-  // resetInvalidatedTags(): void;
 };
 
 /**

package/src/lib/store/dependency/dependencyFileHelper.ts

@@ -2,17 +2,17 @@ import { Json } from '@lib/utils/object/object.types';
 import { dependencyTagger } from './dependencyTagger';
 
 export const dependencyIncludeHelper = {
-  setIncludeDependencies: (filepath: string, json: Json): void => {
+  addIncludeDependencies: (filepath: string, json: Json): void => {
     const includes = json?.metadata?.includes;
 
     if (includes?.static) {
       const staticIncludes = Object.values(includes?.static);
-      dependencyTagger.setDependency(filepath, staticIncludes);
+      dependencyTagger.addDependency(filepath, staticIncludes);
     }
 
     if (includes?.dynamic) {
       const dynamicIncludes = Object.values(includes?.dynamic);
-      dependencyTagger.setDependency(filepath, dynamicIncludes);
+      dependencyTagger.addDependency(filepath, dynamicIncludes);
       // Queries are invalidated by any data change until they are executed,
       // when they return their dependencies. They are "globally invalidable" when:
       // 1) Are added for the first time
@@ -24,6 +24,8 @@ export const dependencyIncludeHelper = {
       // 2) When that query is executed, it has a dependency on "node:1234"
       // 3) When that file with a query is updated, the query has again a dependency on "*"
       dynamicIncludes.forEach(queryDefinition => {
+        // This is explicity a "set" operation, because query dependencies are
+        // only set from a query's "tags" field.
         dependencyTagger.setDependency(queryDefinition, ['*']);
       });
     }

package/src/lib/store/dependency/dependencyManager.ts

@@ -10,7 +10,10 @@ const invalidatedFilepaths = {
 /**
  * Gets a list of parents of a dependency tag
  */
-const getTagParents = (tag: string, parents = new Set<string>()) => {
+export const getTagParents = (
+  tag: string,
+  parents = new Set<string>(),
+): Set<string> => {
   getReversedDependencyTree()
     .get(tag)
     ?.forEach(parent => {
@@ -29,7 +32,7 @@ const getTagParents = (tag: string, parents = new Set<string>()) => {
  * Iterates over all explicitly invalidated tags, and obtains all
  * its parents.
  */
-const getInvalidatedTags = (): string[] => {
+export const getAllInvalidatedTags = (): string[] => {
   const allInvalidatedTags = new Set<string>();
   const invalidatedTagsArray = Array.from(invalidatedTags);
   if (invalidatedTagsArray.length > 0) {
@@ -54,7 +57,7 @@ const getInvalidatedTags = (): string[] => {
  */
 export const dependencyManager: DependencyManager = {
   trackInvalidatedFilepaths: () => {
-    const allInvalidatedTags = getInvalidatedTags();
+    const allInvalidatedTags = getAllInvalidatedTags();
     allInvalidatedTags.forEach(invalidatedTag => {
       if (store.deleted.has(invalidatedTag)) {
         invalidatedFilepaths.deleted.add(invalidatedTag);

package/src/lib/store/dependency/dependencyTagger.ts

@@ -61,45 +61,9 @@ export const dependencyTagger: DependencyTagger = {
 
   invalidateTags: tags => {
     tags.forEach(tag => {
-      invalidatedTags.add(tag);
-    });
-  },
-};
-
-/*
-import { DependencyTagger } from './dependency.types';
-
-export const dependencyTree = new Map<string, Set<string>>();
-
-export const invalidatedTags = new Set<string>();
-
-export const dependencyTagger: DependencyTagger = {
-  setDependency: (tag, dependsOnTags) => {
-    dependsOnTags.forEach(dependsOnTag => {
-      const previousData = dependencyTree.get(dependsOnTag);
-      if (previousData) {
-        previousData.add(tag);
-      } else {
-        dependencyTree.set(dependsOnTag, new Set<string>([tag]));
-      }
-    });
-  },
-
-  deleteDependency: (tag, dependsOnTags) => {
-    dependsOnTags.forEach(dependsOnTag => {
-      const previousData = dependencyTree.get(dependsOnTag);
-      if (previousData) {
-        previousData.delete(tag);
-        if (previousData.size === 0) {
-          dependencyTree.delete(dependsOnTag);
-        }
+      if (tag) {
+        invalidatedTags.add(tag);
       }
     });
   },
-
-  invalidateTags: tags => {
-    tags.forEach(tag => {
-      invalidatedTags.add(tag);
-    });
-  },
-}; */
+};

package/src/lib/store/diff/{diffManager.types.ts → diff.types.ts}

@@ -3,11 +3,11 @@
  */
 export type DiffManager = {
   /**
-   * Clears intermediate changes tracked by trackChangedFile().
+   * Clears intermediate changes tracked by dependency manager.
    *
-   * @param date - Date to be used as a timestamp for the next diff.
+   * @param uniqueId - Unique id to be used for the next diff.
    */
-  reset(date: Date): void;
+  reset(uniqueId: string): void;
 
   /**
    * Gets a list of changed files affected by includes and queries.
@@ -35,9 +35,18 @@ export type DiffManager = {
  */
 export type Diff = {
   /**
-   * Timestamp in milliseconds.
+   * Execution time taken by the diff, in milliseconds.
    */
-  since: number;
+  execTimeMs: number;
+  /**
+   * A unique id representing the date from which this diff is obtained.
+   */
+  fromUniqueId: string;
+
+  /**
+   * A unique id representing the date until which this diff is obtained.
+   */
+  toUniqueId: string;
 
   /**
    * A list of updated files (both newly added and changed files).

package/src/lib/store/diff/diffManager.ts

@@ -1,115 +1,76 @@
 import microtime from 'microtime';
 import { logger } from '@lib/utils/logger';
 import { store } from '../store';
-import { DiffManager, Diff } from './diffManager.types';
+import { DiffManager, Diff } from './diff.types';
 import { dataDirManager } from '../dataDir/dataDirManager';
 import { dependencyManager } from '../dependency/dependencyManager';
+import { unixEpochUniqueId, workDirHelper } from '../workDir';
+import { dumpMetadataHelper } from '../dump/dumpMetadataHelper';
 
-let lastDiffDate: Date | null = null;
+let lastDiffUniqueId: string = unixEpochUniqueId;
 
 export const diffManager: DiffManager = {
-  reset(date: Date): void {
-    lastDiffDate = date;
+  reset(uniqueId: string): void {
+    lastDiffUniqueId = uniqueId;
     dependencyManager.reset();
   },
 
-  getDiff(options = { incremental: true }): Diff {
-    const startDate = microtime.now();
-
-    // Before getting any diff data, update any pending changes from data dir.
-    dataDirManager.update();
-
-    let updated = new Set<string>();
-    let deleted = new Set<string>();
-    if (options.incremental && lastDiffDate) {
-      logger.debug(`Getting incremental diff using date "${lastDiffDate}"`);
-      const invalidatedFilepaths = dependencyManager.getInvalidatedFilepaths();
-      updated = new Set<string>(invalidatedFilepaths.updated);
-      deleted = new Set<string>(invalidatedFilepaths.deleted);
-    } else {
-      // If no sinceDate is passed, return all files
-      logger.debug(`Getting full diff with no date`);
-      updated = new Set<string>(store.data.keys());
-    }
-
-    const diff: Diff = {
-      since: lastDiffDate ? lastDiffDate.getTime() : new Date(0).getTime(),
-      updated,
-      deleted,
-    };
-
-    logger.info(
-      `Diff created in ${(microtime.now() - startDate) / 1000} ms. Updated: ${
-        diff.updated.size
-      } / Deleted: ${diff.deleted.size}`,
-    );
-
-    // Log diff if not empty
-    // if (diff.updated.length || diff.deleted.length) {
-    // logger.debug(`Diff: "${JSON.stringify(jsonify(diff))}"`);
-    // }
-
-    return diff;
-  },
-
-  /*
   getDiff(options = { incremental: true }): Diff {
     const startDate = microtime.now();
 
     // Before getting any diff data, update any pending changes from data dir.
     const changedFiles = dataDirManager.update();
 
-    let updated = new Set<string>();
-    let deleted = new Set<string>();
-    const sinceDate = getLastDiffDate();
-    if (options.incremental && sinceDate) {
-      logger.debug(`Getting incremental diff using date "${sinceDate}"`);
-      // Only process changedFiles if not empty.
-      if (changedFiles.updated.length || changedFiles.deleted.length) {
-        // Update tracked files so no affected parent is missed.
-        changedFiles.updated.forEach(file => {
-          tracker.trackChangedFile(file);
-        });
-        changedFiles.deleted.forEach(file => {
-          tracker.trackChangedFile(file);
-        });
+    const updated = new Set<string>();
+    const deleted = new Set<string>();
+
+    const currentDumpUniqueId = dumpMetadataHelper.getCurrentDumpUniqueId();
+    const isFirstDiffAfterReboot = lastDiffUniqueId === unixEpochUniqueId;
+    const isADumpAvailable = currentDumpUniqueId !== unixEpochUniqueId;
+    // If both checks passes, check if something has changed after last dump to avoid a full diff:
+    if (isFirstDiffAfterReboot && isADumpAvailable) {
+      const dataDirModificationUniqueId = changedFiles.toUniqueId;
+      if (currentDumpUniqueId === store.initialUniqueId) {
+        // If nothing changed, use store.initialUniqueId as lastDiffUniqueId, to
+        // execute an incremental dump.
+        lastDiffUniqueId = store.initialUniqueId;
+      } else {
+        // If something changed, keep lastDiffUniqueId as is, and get the list of
+        // changed files from last dump. From that list, we need deleted files, since
+        // those files are no more in data dir and we need to remove them from dump dir.
+        const changedFilesSinceLastDump = workDirHelper.getChangedFilesBetween(
+          currentDumpUniqueId,
+          dataDirModificationUniqueId,
+        );
+        changedFilesSinceLastDump.deleted.forEach(filepath =>
+          deleted.add(filepath),
+        );
       }
+    }
 
-      // Create the resulting set of updated and deleted files.
-      // "updated" includes all affected parents tracked down
-      // by tracker, without any deleted file.
-      updated = new Set<string>(tracker.getChangedFiles());
-      changedFiles.deleted.forEach(file => updated.delete(file));
-
-      // "deleted" includes only deleted files, and is based only
-      // on data coming from changedFiles, because all deleted files
-      // are stored on Static Suite Data Server log
-      deleted = new Set<string>(changedFiles.deleted);
-
-      // Add updated files affected by queries only if "updated" or "deleted"
-      // contain any changes.
-      if (updated.size > 0 || deleted.size > 0) {
-        queryTagManager
-          .getInvalidFilepaths()
-          .forEach(file => updated.add(file));
-        queryTagManager.resetInvalidatedTags();
-      }
+    if (options.incremental && lastDiffUniqueId !== unixEpochUniqueId) {
+      logger.debug(`Getting incremental diff using date "${lastDiffUniqueId}"`);
+      const invalidatedFilepaths = dependencyManager.getInvalidatedFilepaths();
+      invalidatedFilepaths.updated.forEach(filepath => updated.add(filepath));
+      invalidatedFilepaths.deleted.forEach(filepath => deleted.add(filepath));
     } else {
-      // If no sinceDate is passed, return all files
+      // Return all files
       logger.debug(`Getting full diff with no date`);
-      updated = new Set<string>(store.data.keys());
+      Array.from(store.data.keys()).forEach(key => updated.add(key));
     }
 
+    const execTimeMs = (microtime.now() - startDate) / 1000;
+
     const diff: Diff = {
-      since: sinceDate ? sinceDate.getTime() : new Date(0).getTime(),
+      execTimeMs,
+      fromUniqueId: lastDiffUniqueId,
+      toUniqueId: changedFiles.toUniqueId,
       updated,
       deleted,
     };
 
     logger.info(
-      `Diff created in ${(microtime.now() - startDate) / 1000} ms. Updated: ${
-        diff.updated.size
-      } / Deleted: ${diff.deleted.size}`,
+      `Diff created in ${execTimeMs} ms. Updated: ${diff.updated.size} / Deleted: ${diff.deleted.size}`,
     );
 
     // Log diff if not empty
@@ -118,5 +79,5 @@ export const diffManager: DiffManager = {
     // }
 
     return diff;
-  }, */
+  },
 };

package/src/lib/store/dump/{dumpManager.types.ts → dump.types.ts}

@@ -1,3 +1,5 @@
+import { Diff } from '../diff/diff.types';
+
 /**
  * Service that dumps resolved data (with resolved includes) into a dump directory.
  */
@@ -20,11 +22,11 @@ export type DumpManager = {
    * Reset dump metadata.
    *
    * @remarks
-   * Resets dump metadata, removing any dump information older than the passed timestamp.
+   * Resets dump metadata, removing any dump information older than the passed unique id.
    *
-   * @param timestamp - Timestamp
+   * @param uniqueId - Unique id
    */
-  reset(timestamp: number): void;
+  reset(uniqueId: string): void;
 };
 
 /**
@@ -32,9 +34,19 @@ export type DumpManager = {
  */
 export type Dump = {
   /**
-   * Timestamp in milliseconds.
+   * Execution time taken by the dump, in milliseconds.
    */
-  since: number;
+  execTimeMs: number;
+
+  /**
+   * A unique id representing the date from which this dump is obtained.
+   */
+  fromUniqueId: string;
+
+  /**
+   * A unique id representing the date until which this dump is obtained.
+   */
+  toUniqueId: string;
 
   /**
    * A list of updated files, keyed by file path. Its value is an object with two keys,
@@ -55,7 +67,12 @@ export type Dump = {
   deleted: Map<string, { oldPublicUrl: string | null; newPublicUrl: null }>;
 
   /**
-   * Optional execution time taken by the dump, in milliseconds.
+   * The diff used to generate this dump.
    */
-  execTimeMs?: number;
+  diff: Diff;
+};
+
+export type DumpMetadata = {
+  current: string;
+  dumps: Dump[];
 };