@static-suite/static-suite-data-server 0.0.1

package/src/lib/query/query.types.ts

@@ -0,0 +1,231 @@
+import { Store } from '@lib/store/store.types';
+
+/**
+ * A service to handle the execution of queries.
+ *
+ * @public
+ */
+export type QueryRunner = {
+  /**
+   * Runs a query and returns its results with metadata.
+   *
+   * @remarks
+   * It checks several validations before a query is run, and then it tries to get
+   * the query result from cache. If cache is empty, query is executed and saved
+   * into the cache.
+   *
+   * It logs an error and throws an exception if any problem occurs during the
+   * query execution.
+   *
+   * @param queryDefinition - An URL containing a query ID (filename without extension
+   * and suffix) of the query to be executed, plus and optional query string with arguments.
+   * @see {@link ModuleInfo#id | Definition of a module ID}
+   *
+   * @returns The result of the query as a JSON object with data and metadata keys,
+   * or an error response if some validation is not met.
+   *
+   * @example
+   * Here's a result example:
+   * ```
+   * {
+   * data: ['title 1', 'title 2'],
+   *  metadata: {
+   *    contentType: 'application/json',
+   *    execTime: 2,
+   *    cache: ['miss'|'hit'],
+   *    num: 2,
+   *  }
+   * }
+   * ```
+
+  * @throws
+   * Exception thrown if any problem occurs during the query execution.
+   */
+  run(queryDefinition: string): QuerySuccessfulResponse | QueryErrorResponse;
+
+  /**
+   * Gets number of executed queries.
+   *
+   * @remarks
+   * A simple counter that stores the number of executed queries, of any type.
+   *
+   * @returns Number of executed queries.
+   */
+  getCount(): number;
+};
+
+/**
+ * An object with query arguments.
+ *
+ * @public
+ */
+export type QueryArgs = Record<string, any>;
+
+/**
+ * The result that a query returns after being executed.
+ */
+export type QueryModuleResult = {
+  /**
+   * A result, which can be of any kind (and array, an object, a string, etc)
+   */
+  result: any;
+
+  /**
+   * Optional content type for the data available in "result"
+   *
+   * @remarks
+   * Data coming from the query can be anything (XML, a string, etc).
+   * This content type is added to the metadata section of the QueryResponse.
+   * @see {@link QuerySuccessfulResponse#metadata}.
+   *
+   * @defaultValue 'application/json'
+   */
+  contentType?: string;
+
+  /**
+   * Optional flag that tells whether this result can be cached or not.
+   */
+  cacheable?: boolean;
+
+  /**
+   * A list of tags to get this query invalidated.
+   */
+  tags?: string[];
+};
+
+/**
+ * Possible values for a cache get operation: miss or hit.
+ *
+ * @public
+ */
+export enum CacheStatus {
+  MISS,
+  HIT,
+}
+
+/**
+ * A successful response returned after executing a query.
+ *
+ * @remarks
+ * It takes the results from a QueryModule#queryHandler, and wraps them into an
+ * structure with metadata. @see {@link QueryModule#default}
+ *
+ * @public
+ */
+export type QuerySuccessfulResponse = {
+  /**
+   * Data returned by the query. @see {@link QueryModule#default}
+   */
+  data: any;
+
+  /**
+   * Query metadata
+   */
+  metadata: {
+    /**
+     * Content type for the data available in "data"
+     *
+     * @remarks
+     * Even thought this structure is a JSON object, data coming from the
+     * query can be anything (XML, a string, etc). It must be specified so any
+     * consumer of this data knows what is being returned.
+     */
+    contentType: string;
+
+    /**
+     * Execution time taken by the query, in milliseconds.
+     */
+    execTimeMs: number;
+
+    /**
+     * Tells wether the query results comes from a miss or hit.
+     */
+    cache: CacheStatus;
+
+    /**
+     * A list of tags to get this query invalidated.
+     */
+    tags?: string[];
+  };
+};
+
+/**
+ * A module that defines a query.
+ */
+export type QueryModule = {
+  /**
+   * The handler that executes a query.
+   *
+   * @remarks
+   * It receives the store and a set of arguments, executes the query
+   * and returns its results.
+   *
+   * @param options - An object with options for the query: store and args
+   */
+  default(options: {
+    /**
+     * Store data to be used in the query
+     */
+    store: Store;
+    args: QueryArgs;
+  }): QueryModuleResult;
+};
+
+/**
+ * An error response returned after executing a query.
+ *
+ * @remarks
+ * It takes an string from QueryModule#queryHandler, and wraps them into an
+ * structure with metadata. @see {@link QueryModule#default}
+ *
+ * @public
+ */
+export type QueryErrorResponse = {
+  /**
+   * Error message from the query, usually coming from a failed validation.
+   */
+  error: string;
+};
+
+/**
+ * Tells whether a query response is erroneous or not.
+ *
+ * @param queryResponse - A response (successful or erroneous) from the query handler.
+ * @returns True if queryResponse is erroneous.
+ */
+export const isQueryErrorResponse = (
+  queryResponse: QuerySuccessfulResponse | QueryErrorResponse,
+): queryResponse is QueryErrorResponse =>
+  (queryResponse as QueryErrorResponse).error !== undefined;
+
+export type QueryTagManager = {
+  /**
+   * Marks query items with any of the specified tags as invalid.
+   *
+   * @param tags - The list of tags for which to invalidate query items.
+   */
+  invalidateTags(tags: string[]): void;
+
+  /**
+   * Resets the list of invalidated tags once consumed.
+   */
+  resetInvalidatedTags(): void;
+
+  /**
+   * Adds a list of query tags to a query, or removes it if tags are empty.
+   *
+   * @param queryDefinition - The string containing the query id and its arguments.
+   * @param tags - The list of tags for which to invalidate query items.
+   */
+  setTagsToQuery(
+    queryDefinition: string,
+    tags: Set<string> | null | undefined,
+  ): void;
+
+  /**
+   * Gets a list of filepaths which contain an invalidated query.
+   *
+   * @returns A list of filepaths which contain an invalidated query.
+   */
+  getInvalidFilepaths(): string[];
+};

package/src/lib/query/queryManager.ts

@@ -0,0 +1,11 @@
+import { dirBasedModuleGroupManager } from '@lib/utils/module';
+import { QueryModule } from './query.types';
+
+/**
+ * A manager for queries stored in a user-land directory defined by configuration.
+ *
+ * @remarks
+ * It extends dirBasedModuleGroupManager, which manages groups of modules based
+ * on a directory: @see {@link dirBasedModuleGroupManager}
+ */
+export const queryManager = dirBasedModuleGroupManager<QueryModule>('query');

package/src/lib/query/queryRunner.ts

@@ -0,0 +1,110 @@
+import microtime from 'microtime';
+import { config } from '@lib/config';
+import { store } from '@lib/store';
+import { logger } from '@lib/utils/logger';
+import { cache } from '@lib/utils/cache';
+import { parseURLSearchParams } from '@lib/utils/string';
+import { dependencyTagger } from '@lib/store/dependency/dependencyTagger';
+import { queryManager } from './queryManager';
+import {
+  QueryModuleResult,
+  QueryRunner,
+  CacheStatus,
+  QueryErrorResponse,
+  QuerySuccessfulResponse,
+  QueryArgs,
+} from './query.types';
+
+let count = 0;
+
+/**
+ * Creates a QueryErrorResponse object ready to be returned to clients.
+ *
+ * @param message - Error message
+ *
+ * @returns A JSON object with an error key
+ *
+ * @internal
+ */
+const createErrorResponse = (message: string): QueryErrorResponse => {
+  logger.error(message);
+  return { error: message };
+};
+
+export const queryRunner: QueryRunner = {
+  run: (queryDefinition): QuerySuccessfulResponse | QueryErrorResponse => {
+    if (!config.queryDir) {
+      return createErrorResponse('No query directory ("queryDir") configured');
+    }
+
+    const [queryId, queryArgsString] = queryDefinition.split('?');
+    const queryArgs: QueryArgs = queryArgsString
+      ? parseURLSearchParams(queryArgsString)
+      : {};
+
+    if (!queryId) {
+      return createErrorResponse('No query ID received');
+    }
+    const queryModuleInfo = queryManager.getModuleGroupInfo().get(queryId);
+    if (!queryModuleInfo) {
+      return createErrorResponse(`Query module for ID ${queryId} not found`);
+    }
+
+    count += 1;
+    const startDate = microtime.now();
+
+    // Implement a query cache.
+    const cacheId = queryDefinition;
+    let isCacheMiss = false;
+    const queryCache = cache.bin<QueryModuleResult>('query');
+    let queryResult = queryCache.get(cacheId);
+    if (!queryResult) {
+      try {
+        const queryModule = queryModuleInfo.getModule();
+        const queryResponse = queryModule.default({
+          store,
+          args: queryArgs,
+        });
+        queryResult = {
+          result: queryResponse?.result,
+          contentType: queryResponse?.contentType || 'application/json',
+        };
+        isCacheMiss = true;
+        if (queryResponse?.cacheable !== false) {
+          queryCache.set(cacheId, queryResult);
+        }
+
+        // Set tags for this query.
+        const queryTags =
+          queryResponse.tags && queryResponse.tags?.length > 0
+            ? queryResponse.tags
+            : ['*'];
+        dependencyTagger.setDependency(queryDefinition, queryTags);
+      } catch (e) {
+        // Log error and rethrow.
+        logger.error(
+          `#${count} Query error: "${queryId}", args "${queryArgsString}": ${e}`,
+        );
+        throw e;
+      }
+    }
+    const execTimeMs = (microtime.now() - startDate) / 1000;
+    const response: QuerySuccessfulResponse = {
+      data: queryResult.result,
+      metadata: {
+        contentType: queryResult?.contentType || 'application/json',
+        execTimeMs,
+        cache: isCacheMiss ? CacheStatus.MISS : CacheStatus.HIT,
+        tags: queryResult.tags ? Array.from(queryResult.tags) : undefined,
+      },
+    };
+
+    /*     logger.debug(
+      `#${count} Query "${queryId}" args "${argsString}" took ${execTimeMs} ms.`,
+    ); */
+
+    return response;
+  },
+
+  getCount: (): number => count,
+};

package/src/lib/query/queryTagManager.ts

@@ -0,0 +1,53 @@
+import { store } from '@lib/store/store';
+import { Json } from '@lib/utils/object/object.types';
+import { QueryTagManager } from './query.types';
+
+const invalidatedTags = new Set<string>();
+
+const tagsByQueryIndex = new Map<string, Set<string>>();
+
+export const queryTagManager: QueryTagManager = {
+  invalidateTags: tags => {
+    tags.forEach(tag => {
+      invalidatedTags.add(tag);
+    });
+  },
+
+  resetInvalidatedTags: () => {
+    invalidatedTags.clear();
+  },
+
+  setTagsToQuery: (queryDefinition, tags) => {
+    if (tags) {
+      tagsByQueryIndex.set(queryDefinition, tags);
+    } else {
+      tagsByQueryIndex.delete(queryDefinition);
+    }
+  },
+
+  getInvalidFilepaths: () => {
+    const invalidFilepaths = new Set<string>();
+    store.data.forEach((json: Json, relativeFilepath: string) => {
+      if (json?.metadata?.includes?.dynamic) {
+        Object.values(json.metadata.includes.dynamic).forEach(
+          queryDefinition => {
+            const tagsByQuery = tagsByQueryIndex.get(queryDefinition);
+            if (tagsByQuery) {
+              // eslint-disable-next-line no-restricted-syntax
+              for (const invalidatedTag of invalidatedTags) {
+                if (tagsByQuery.has(invalidatedTag)) {
+                  invalidFilepaths.add(relativeFilepath);
+                  break;
+                }
+              }
+            } else {
+              invalidFilepaths.add(relativeFilepath);
+            }
+          },
+        );
+      }
+    });
+
+    return Array.from(invalidFilepaths);
+  },
+};

package/src/lib/store/dataDir/__tests__/dataDirManager.test.ts

@@ -0,0 +1,15 @@
+import fs from 'fs';
+import path from 'path';
+import { config } from '@lib/config';
+
+beforeEach(() => {
+  config.dataDir = fs.realpathSync(path.resolve('src/__tests__/fixtures/data'));
+});
+
+describe('dataDirManager test', () => {
+  describe('loadDataDir', () => {
+    it(`Adds to store all files in datadir`, () => {
+      expect('1').toEqual('1');
+    });
+  });
+});

package/src/lib/store/dataDir/dataDir.types.ts

@@ -0,0 +1,38 @@
+import { ChangedFiles } from '../workDir/workDir.types';
+
+/**
+ * A manager for the data dir from Static Suite.
+ */
+export type DataDirManager = {
+  /**
+   * Loads all files inside the data directory into the store.
+   *
+   */
+  load(): void;
+
+  /**
+   * Updates the store with changed files since last sync.
+   *
+   * @remarks
+   * It's main difference with load() is that, instead of loading the whole store,
+   * update() checks if something has changed, not doing anything if nothing
+   * has changed. Thus, this method is way faster than load().
+   *
+   * @returns A group of changed files in Static Suite's data dir.
+   */
+  update(): ChangedFiles;
+
+  /**
+   * Get date of last modification of data directory.
+   *
+   * @remarks
+   * Metadata about when and how is the data directory updated is
+   * stored in the work directory. If that directory is not present or
+   * not initialized with proper data, it returns the sync date of the store.
+   * If the store is not yet loaded, it returns a date in the past,
+   * the Unix Epoch (00:00:00 UTC on 1 January 1970).
+   *
+   * @returns The date of last modification of data directory
+   */
+  getModificationDate(): Date;
+};

package/src/lib/store/dataDir/dataDirManager.ts

@@ -0,0 +1,121 @@
+import microtime from 'microtime';
+import { config } from '@lib/config';
+import { store } from '@lib/store';
+import { findFilesInDir } from '@lib/utils/fs';
+import { logger } from '@lib/utils/logger';
+import { workDirHelper } from '@lib/store/workDir';
+import { cache } from '@lib/utils/cache';
+import { storeManager } from '@lib/store/storeManager';
+import { hookManager } from '@lib/store/hook';
+import { queryManager } from '@lib/query';
+import { taskManager } from '@lib/task';
+import { DataDirManager } from './dataDir.types';
+import { ChangedFiles } from '../workDir/workDir.types';
+import { dependencyManager } from '../dependency/dependencyManager';
+
+export const dataDirManager: DataDirManager = {
+  load: () => {
+    logger.info('Loading data dir...');
+    const startDate = Date.now();
+
+    // Reset store, just in case this load is called on an already loaded store.
+    storeManager.reset();
+
+    // Clear all caches.
+    cache.clear();
+
+    // Reset all managers.
+    hookManager.reset();
+    queryManager.reset();
+    taskManager.reset();
+
+    // Invoke "store load start" hook.
+    hookManager.invokeOnStoreLoadStart();
+
+    store.syncDate = dataDirManager.getModificationDate();
+
+    // Add all files, one by one.
+    const relativeFilePaths = findFilesInDir(config.dataDir);
+    const storeHydrationStartDate = Date.now();
+    relativeFilePaths.forEach(relativeFilePath => {
+      storeManager.add(relativeFilePath);
+    });
+
+    logger.debug(
+      `Store map hydrated in ${Date.now() - storeHydrationStartDate}ms.`,
+    );
+
+    const includeParserStartDate = Date.now();
+    storeManager.parseIncludes();
+    logger.debug(
+      `Store include parser done in ${Date.now() - includeParserStartDate}ms.`,
+    );
+
+    // Invoke "store load done" hook.
+    hookManager.invokeOnStoreLoadDone();
+
+    logger.info(
+      `${relativeFilePaths.length} files loaded in ${
+        Date.now() - startDate
+      }ms.`,
+    );
+  },
+
+  update: () => {
+    const dataDirModificationDate = dataDirManager.getModificationDate();
+
+    let changedFiles: ChangedFiles = {
+      updated: [],
+      deleted: [],
+    };
+
+    if (store.syncDate) {
+      if (dataDirModificationDate > store.syncDate) {
+        // Save store.syncDate to a variable and set it ASAP to avoid two concurrent
+        // processes updating the data directory at the same time.
+        const storeLastSyncDate = store.syncDate;
+        store.syncDate = dataDirModificationDate;
+        const startDate = microtime.now();
+        logger.debug(
+          `Data dir outdated. Current data loaded at ${storeLastSyncDate.toISOString()} but last updated at ${dataDirModificationDate.toISOString()}`,
+        );
+
+        // Before updating anything, track down which filepaths are invalidated with
+        // the current set of dependencies, before those dependencies change.
+        dependencyManager.trackInvalidatedFilepaths();
+
+        changedFiles = workDirHelper.getChangedFilesSince(storeLastSyncDate);
+        hookManager.invokeOnStoreChangeStart(changedFiles);
+        changedFiles.updated.forEach(file => {
+          storeManager.update(file);
+          const fileContent = store.data.get(file);
+          storeManager.parseSingleFileIncludes(file, fileContent);
+        });
+        changedFiles.deleted.forEach(file => {
+          storeManager.remove(file);
+        });
+        // Clear all store subsets and queries, since they are stale.
+        // In fact, the subset cache should be cleared only when files
+        // are added or deleted, but not when they are simply updated.
+        // Anyway, we currently do not have any means of distinguishing
+        // added files from updated ones, so we must clear the subset
+        // cache in all situations.
+        cache.bin('store-subset').clear();
+        cache.bin('query').clear();
+        hookManager.invokeOnStoreChangeDone(changedFiles);
+        const execTimeMs = (microtime.now() - startDate) / 1000;
+        logger.debug(`Data dir updated in ${execTimeMs}ms.`);
+      } else {
+        logger.debug(
+          `Data dir up to date. Current data loaded at ${store.syncDate.toISOString()} and last updated at ${dataDirModificationDate.toISOString()}`,
+        );
+      }
+    } else {
+      logger.debug(`Data dir not yet loaded, so it cannot be updated.`);
+    }
+    return changedFiles;
+  },
+
+  getModificationDate: (): Date =>
+    workDirHelper.getModificationDate() || store.syncDate || new Date(0),
+};

package/src/lib/store/dataDir/index.ts

@@ -0,0 +1 @@
+export * from './dataDirManager';

package/src/lib/store/dependency/dependency.types.ts

@@ -0,0 +1,78 @@
+/**
+ * Service to define dependencies between data.
+ */
+export type DependencyTagger = {
+  /**
+   * Sets a dependency relationship between a tag that depends on another tags.
+   *
+   * @remarks It replaces any previous dependency relationship.
+   *
+   * @param tag - A tag to set a dependency for.
+   * @param dependsOnTags - An array of tags that depends on first parameter tag.
+   */
+  setDependency(tag: string, dependsOnTags: string[]): void;
+
+  /**
+   * Adds a dependency relationship between a tag that depends on another tags.
+   *
+   * @remarks It adds data to any previous dependency relationship.
+   *
+   * @param tag - A tag to set a dependency for.
+   * @param dependsOnTags - An array of tags that depends on first parameter tag.
+   */
+  addDependency(tag: string, dependsOnTags: string[]): void;
+
+  /**
+   * Deletes a dependency relationship between a tag that depends on another tags.
+   *
+   * @param tag - A tag to delete a dependency for.
+   * @param dependsOnTags - Optional array of tags that depends on first parameter
+   * tag. If not provided, it deletes all dependencies for tag.
+   */
+  deleteDependency(tag: string, dependsOnTags?: string[]): void;
+
+  /**
+   * Marks dependency items with any of the specified tags as invalid.
+   *
+   * @param tags - The list of tags for which to invalidate dependency items.
+   */
+  invalidateTags(tags: string[]): void;
+
+  /**
+   * Resets the list of invalidated tags once consumed.
+   */
+  // resetInvalidatedTags(): void;
+};
+
+/**
+ * Manager that keeps control of dependencies between data.
+ */
+export type DependencyManager = {
+  /**
+   * Calculates and tracks down a list of filepaths which contain an invalidated tag.
+   *
+   * @remarks
+   * It calculates which filepaths have been invalidated, and stores them so they
+   * can be consumed later. This function should be called before the store is updated,
+   * before any dependency data changes, to avoid losing the set of invalidated filepaths
+   * between updates.
+   */
+  trackInvalidatedFilepaths(): void;
+
+  /**
+   * Gets a list of filepaths which contain an invalidated tag.
+   *
+   * @remarks
+   * It uses the set of tracked down filepaths from
+   *
+   * @returns A list of filepaths which contain an invalidated tag.
+   */
+  getInvalidatedFilepaths(): { updated: string[]; deleted: string[] };
+
+  /**
+   * Resets the list of invalidated tags once consumed.
+   */
+  reset(): void;
+};
+
+export type DependencyTree = Map<string, Set<string>>;