@static-suite/static-suite-data-server 0.0.1

package/src/lib/store/store.types.ts

@@ -0,0 +1,251 @@
+export interface StoreData<K, V> extends Map<K, V> {
+  /**
+   * Create a subset with all files in store that match the given arguments.
+   *
+   * @remarks
+   * All subsets are automatically cached, so passing the same arguments
+   * will return the same subset from the cache.
+   *
+   * This is the preferred way of getting a subset of the store files,
+   * since is a simple function that most of the times only requires
+   * the first argument.
+   *
+   * @example
+   * ```
+   * // Get a subset of all nodes with "json" extension.
+   * dataServer.store.data.subset({ dir: 'en/entity/node/', variant: null });
+   *
+   * // Get a subset of all articles in all languages with "json" extension.
+   * dataServer.store.data.subset({ dir: '.+/entity/node/article/' });
+   *
+   * // Get a subset of all articles regardless of their extension.
+   * dataServer.store.data.subset({ dir: 'en/entity/node/article/', ext: null });
+   *
+   * // Get a subset of all card variants for articles with "json" extension.
+   * dataServer.store.data.subset({ dir: 'en/entity/node/article/', variant: null });
+   *
+   * // Get a subset of english articles, with "yml" extension, non-recursively.
+   * dataServer.store.data.subset({ dir: 'en/entity/node/article/', ext: 'yml', recursive: false });
+   * ```
+   *
+   * @param options - Object with options for creating a store subset.
+   *
+   * @returns An object with "filenames" and "items".
+   */
+  subset(options: StoreSubsetOptions): StoreSubset;
+}
+
+/**
+ * The store that holds all data.
+ */
+export type Store = {
+  /**
+   * Date to tell when was the store last synced.
+   *
+   * Every time the store is updated, syncDate is set with the last modification
+   * date of the data directory, not the date when the store has finished updating.
+   *
+   * Given this scenario:
+   * - Data directory modified at 12:30:00
+   * - Syncing the store takes 3 seconds and finishes at 12:30:03
+   *
+   * ... syncDate will be 12:30:00, not 12:30:03.
+   *
+   * Doing it the other way round can cause data inconsistencies, because syncing files
+   * to the store takes time, it is done sequentially, and in the meanwhile, other
+   * modifications can happen:
+   * - Store last syncDate is 12:29:00
+   * - Data directory is modified at 12:30:00
+   * - We ask for modified files after 12:29:00 and got 3 files (1.json, 2.json and 3.json)
+   * - File "1.json" is processed and added to the store at 12:30:01
+   * - While "2.json" is being processed and added to the store, data directory is modified
+   * again at 12:30:02 and "1.json" is changed.
+   * - File "3.json" is processed and added to the store at 12:30:03
+   *
+   * At this moment, there are two options:
+   *
+   * 1) Setting syncDate to the current date:
+   * - Store syncDate is set to 12:30:03
+   * - Data directory is modified again at 12:30:10 and only "3.json" is changed
+   * - We ask for modified files after 12:30:03 and get only "3.json"
+   * - Modification done at 12:30:02 to "1.json" is lost and cannot be recovered unless "1.json" is
+   * modified again, or Data Server is restarted
+   *
+   * 2) Setting syncDate to the last modification date of the data directory:
+   * - Store syncDate is set to 12:30:00
+   * - Data directory is modified again at 12:30:10 and only "3.json" is changed
+   * - We ask for modified files after 12:30:00 and get "1.json" and "3.json"
+   * - Modification done at 12:30:02 to "1.json" is properly processed
+   */
+  syncDate: Date | null;
+
+  /**
+   * Map that holds all data for all files.
+   *
+   * @remarks
+   * This map contains files, keyed by their relative filepath. No directory or
+   * any other structure is stored here.
+   *
+   * When executing a query, a common pattern is to create a subset of this
+   * map (e.g.- to get all articles of an specific language) and the execute the
+   * query against that subset. To do so, see {@link Store#subset}
+   *
+   * @example
+   * ```
+   * // Find an specific article.
+   * const article = store.data.get('en/entity/node/article/123.json');
+   * const article = store.index.url.get('public/pretty/url');
+   * const article = store.index.uuid.get('nl').get('ce736c8b-a6ea-43c4-9a6b-ad1fc498017c');
+   * const article = store.index.custom.get('taxonomies').get('12234');
+   *
+   * // Create a subset of all english articles.
+   * const results = store.data.subset({ dir: 'en/entity/node/article/', recursive: true });
+   * ```
+   */
+  data: StoreData<string, any>;
+
+  /**
+   * A list of deleted files from store, to be able to track them.
+   */
+  deleted: Set<string>;
+
+  /**
+   * An object to hold accessory index data.
+   */
+  index: {
+    /**
+     * An index that holds all data for all files, keyed by their URL.
+     */
+    url: Map<string, any>;
+
+    /**
+     * An index that holds all data for all files, keyed by their lang and uuid.
+     */
+    uuid: Map<string, any>;
+
+    /**
+     * An index to hold custom data defined in hooks or queries.
+     */
+    custom: Map<string, any>;
+  };
+};
+
+/**
+ * Options for subset() function.
+ */
+export type StoreSubsetOptions = {
+  /**
+   * Optional base directory to filter files.
+   *
+   * @remarks
+   * It requires a trailing slash, but not a leading slash, e.g.- "en/entity/node/article/"
+   */
+  dir?: string;
+
+  /**
+   * Optional name of a variant file.
+   *
+   * @remarks
+   * Use '_main' to obtain the default variant.
+   * Use any other string, e.g.- 'card', to obtain the card variant.
+   * Use null to avoid searching for any variant.
+   * Defaults to '_main'.
+   */
+  variant?: string;
+
+  /**
+   * Optional file extension, without dots. Defaults to 'json'.
+   */
+  ext?: string;
+
+  /**
+   * Optional flag to search for files recursively. True by default.
+   */
+  recursive?: boolean;
+};
+
+/**
+ * Object holding a subset of items from the store.
+ */
+export type StoreSubset = {
+  /**
+   * Array of filenames in this subset.
+   */
+  filenames: Array<string>;
+
+  /**
+   * Array of items in this subset.
+   */
+  items: Array<any>;
+};
+
+/**
+ * The manager that handles the store data.
+ */
+export type StoreManager = {
+  /**
+   * Adds a file to the store, into a Map structure keyed by its relative filepath.
+   *
+   * @remarks
+   * It invokes "process file" and "store add" hooks.
+   *
+   * @param relativeFilepath - Relative file path, inside dataDir, to the file to be added.
+   *
+   * @returns The store manager, to allow chaining.
+   */
+  add(relativeFilepath: string): StoreManager;
+
+  /**
+   * Removes a file from the store.
+   *
+   * @remarks
+   * It removes the file from the map structure.
+   * It invokes the "store remove" hook.
+   *
+   * @param file - Relative path to the file to be removed.
+   *
+   * @returns The store manager, to allow chaining.
+   */
+  remove(file: string): StoreManager;
+
+  /**
+   * Updates a file from the store.
+   *
+   * @remarks
+   * It removes the file from the map structure, and adds it again.
+   *
+   * @param file - Relative path, inside dataDir, to the file to be updated.
+   *
+   * @returns The store manager, to allow chaining.
+   */
+  update(file: string): StoreManager;
+
+  /**
+   * Parses all static and dynamic includes (entity, config, locale, custom and query)
+   * from data stored in "store.data".
+   *
+   * @returns The store manager, to allow chaining.
+   */
+  parseIncludes(): StoreManager;
+
+  /**
+   * Parses all static and dynamic includes (entity, config, locale, custom and query)
+   * from one file.
+   *
+   * @param relativeFilepath - Relative file path, inside dataDir, to the file to be parsed.
+   * @param fileContent - The contents of a file.
+   *
+   * @returns The store manager, to allow chaining.
+   */
+  parseSingleFileIncludes(
+    relativeFilepath: string,
+    fileContent: any,
+  ): StoreManager;
+
+  /**
+   * Resets store and deletes all loaded data.
+   *
+   * @returns The store manager, to allow chaining.
+   */
+  reset(): StoreManager;
+};

package/src/lib/store/storeManager.ts

@@ -0,0 +1,187 @@
+import { config } from '@lib/config';
+import { getFileContent } from '@lib/utils/fs';
+import { FileType } from '@lib/utils/fs/fs.types';
+import { StoreManager } from './store.types';
+import { hookManager } from './hook';
+import { store, resetStore } from '.';
+import { includeParser } from './include';
+import { dependencyIncludeHelper } from './dependency/dependencyFileHelper';
+import { dependencyTagger } from './dependency/dependencyTagger';
+
+/**
+ * Sets a file into the store, regardless of being already added or not.
+ *
+ * @param relativeFilepath - Relative file path, inside dataDir, to the file to be added.
+ * @param options - Configuration options.
+ *
+ * @returns Object with two properties, "raw" and "json".
+ */
+const setFileIntoStore = (relativeFilepath: string): FileType => {
+  const absoluteFilePath = `${config.dataDir}/${relativeFilepath}`;
+  let fileContent = getFileContent(absoluteFilePath);
+
+  // Invoke "process file" hook.
+  fileContent = hookManager.invokeOnProcessFile({
+    relativeFilepath,
+    fileContent,
+  });
+
+  const dataToStore = fileContent.json || fileContent.raw;
+  if (fileContent.json) {
+    // Check if the object already exists to make sure we don't break the reference
+    const previousData = store.data.get(relativeFilepath);
+    if (dataToStore && typeof dataToStore === 'object') {
+      if (previousData) {
+        // Delete include dependencies.
+        dependencyIncludeHelper.deleteIncludeDependencies(
+          relativeFilepath,
+          dataToStore,
+        );
+
+        // Remove previous data from URL index.
+        const url = previousData.data?.content?.url?.path;
+        if (url) {
+          store.index.url.delete(url);
+        }
+
+        // Remove previous data from UUID index.
+        const uuid = previousData.data?.content?.uuid;
+        const langcode = previousData.data?.content?.langcode?.value;
+        if (uuid && langcode) {
+          store.index.uuid.get(langcode)?.delete(uuid);
+        }
+
+        // Delete all referenced object properties
+        Object.keys(previousData).forEach(key => {
+          delete previousData[key];
+        });
+        // hydrate new object properties into referenced object
+        Object.keys(dataToStore).forEach(key => {
+          previousData[key] = dataToStore[key];
+        });
+      }
+
+      // Add data to UUID index.
+      const uuid = dataToStore.data?.content?.uuid;
+      const langcode = dataToStore.data?.content?.langcode?.value;
+      if (uuid && langcode) {
+        const langcodeMap =
+          store.index.uuid.get(langcode) ||
+          store.index.uuid.set(langcode, new Map<string, any>()).get(langcode);
+        langcodeMap.set(uuid, dataToStore);
+      }
+
+      // Add data to URL index.
+      const url = dataToStore.data?.content?.url?.path;
+      if (url) {
+        store.index.url.set(url, dataToStore);
+      }
+
+      // Add include dependencies.
+      dependencyIncludeHelper.setIncludeDependencies(
+        relativeFilepath,
+        dataToStore,
+      );
+    }
+  }
+  store.data.set(relativeFilepath, dataToStore);
+
+  // Remove this path from store.deleted
+  store.deleted.delete(relativeFilepath);
+
+  return fileContent;
+};
+
+export const storeManager: StoreManager = {
+  add: (relativeFilepath: string): StoreManager => {
+    const fileContent = setFileIntoStore(relativeFilepath);
+
+    // Invoke "store add" hook.
+    hookManager.invokeOnStoreItemAdd({ relativeFilepath, fileContent });
+
+    return storeManager;
+  },
+
+  update: (relativeFilepath: string): StoreManager => {
+    const storedData = store.data.get(relativeFilepath);
+    if (storedData) {
+      hookManager.invokeOnStoreItemBeforeUpdate({
+        relativeFilepath,
+        fileContent: storedData,
+      });
+    }
+
+    const fileContent = setFileIntoStore(relativeFilepath);
+
+    // Invalidate this item.
+    dependencyTagger.invalidateTags([relativeFilepath]);
+
+    // Invoke "store update" hook.
+    hookManager.invokeOnStoreItemAfterUpdate({
+      relativeFilepath,
+      fileContent,
+    });
+
+    return storeManager;
+  },
+
+  remove: (relativeFilepath: string): StoreManager => {
+    // Get stored data before removing it from store.
+    const storedData = store.data.get(relativeFilepath);
+
+    // Remove data from URL index.
+    const url = storedData.data?.content?.url?.path;
+    if (url) {
+      store.index.url.delete(url);
+    }
+
+    // Remove data from UUID index.
+    const uuid = storedData.data?.content?.uuid;
+    const langcode = storedData.data?.content?.langcode?.value;
+    if (uuid && langcode) {
+      store.index.uuid.get(langcode)?.delete(uuid);
+    }
+
+    // Delete file contents from store.
+    store.data.delete(relativeFilepath);
+
+    // Save its path in store.deleted for future reference.
+    store.deleted.add(relativeFilepath);
+
+    // Invalidate this item.
+    dependencyTagger.invalidateTags([relativeFilepath]);
+
+    // Invoke "store remove" hook.
+    hookManager.invokeOnStoreItemDelete({
+      relativeFilepath,
+      fileContent: storedData,
+    });
+
+    return storeManager;
+  },
+
+  parseIncludes: (): StoreManager => {
+    // Parses static includes.
+    store.data.forEach((fileContent, relativeFilepath) => {
+      includeParser.static(relativeFilepath, fileContent);
+    });
+
+    // Parses dynamic includes.
+    store.data.forEach(fileContent => {
+      includeParser.dynamic(fileContent);
+    });
+
+    return storeManager;
+  },
+
+  parseSingleFileIncludes: (relativeFilepath, fileContent): StoreManager => {
+    includeParser.static(relativeFilepath, fileContent);
+    includeParser.dynamic(fileContent);
+    return storeManager;
+  },
+
+  reset: () => {
+    resetStore();
+    return storeManager;
+  },
+};

package/src/lib/store/workDir/__tests__/workDirHelper.test.ts

@@ -0,0 +1,25 @@
+import path from 'path';
+import fs from 'fs';
+import { config } from '@lib/config';
+import { workDirHelper } from '../workDirHelper';
+
+describe('workDirHelper test', () => {
+  describe('getChangedFilesSince', () => {
+    config.workDir = fs.realpathSync(
+      path.resolve('src/__tests__/fixtures/work'),
+    );
+
+    it(`Gets modified and deleted files from static-suite log`, () => {
+      expect(
+        workDirHelper.getChangedFilesSince(new Date('2021-08-01')),
+      ).toEqual({
+        deleted: ['es/entity/node/article/10.json'],
+        updated: [
+          'en/entity/node/article/10.json',
+          'es/config/static_export.settings.json',
+          'es/entity/node/recipe/9.json',
+        ],
+      });
+    });
+  });
+});

package/src/lib/store/workDir/index.ts

@@ -0,0 +1 @@
+export * from './workDirHelper';

package/src/lib/store/workDir/storage/__tests__/getChangedLinesSince.test.ts

@@ -0,0 +1,23 @@
+import fs from 'fs';
+import path from 'path';
+import { config } from '@lib/config';
+import { getChangedLinesSince } from '../getChangedLinesSince';
+
+describe('workDir storage getChangedLinesSince test', () => {
+  describe('getChangedLinesSince', () => {
+    config.workDir = fs.realpathSync(
+      path.resolve('src/__tests__/fixtures/work'),
+    );
+
+    it(`Gets lines of log since given date`, () => {
+      expect(getChangedLinesSince(new Date('2021-08-01'))).toEqual([
+        '2021-08-16_07-11-55.666963__3504 delete [ID: 10] Prueba y cultiva tus propias hierbas | static-local://es/entity/node/article/10.json',
+        '2021-08-16_07-12-45.771095__5959 write [ID: 10] Give it a go and grow your own herbs | static-local://en/entity/node/article/10.json',
+        '2021-08-16_07-12-46.771095__5959 delete [ID: 10] Give it a go and grow your own herbs | static-local://en/entity/node/article/10.json',
+        '2021-08-16_07-12-47.771095__5959 write [ID: 10] Give it a go and grow your own herbs | static-local://en/entity/node/article/10.json',
+        '2021-08-17_08-49-26.273927__5991 write [ID: static_export.settings] static_export.settings | static-local://es/config/static_export.settings.json',
+        '2021-08-17_08-50-00.235800__1900 write [ID: 9] Salsa de chile ardiente | static-local://es/entity/node/recipe/9.json',
+      ]);
+    });
+  });
+});

package/src/lib/store/workDir/storage/__tests__/getDataFromLogLine.test.ts

@@ -0,0 +1,21 @@
+import { getDataFromLogLine } from '..';
+
+describe('workDir storage getDataFromLogLine test', () => {
+  describe('getDataFromLogLine', () => {
+    it(`Gets data from line of log`, () => {
+      expect(
+        getDataFromLogLine(
+          '2021-08-16_07-12-45.771095__5959 write [ID: 10] Give it a go and grow your own herbs | static-local://en/entity/node/article/10.json',
+        ),
+      ).toStrictEqual({
+        file: {
+          id: '10',
+          label: 'Give it a go and grow your own herbs',
+          relativePath: 'en/entity/node/article/10.json',
+        },
+        operation: 'write',
+        uniqueId: '2021-08-16_07-12-45.771095__5959',
+      });
+    });
+  });
+});

package/src/lib/store/workDir/storage/getChangedLinesSince.ts

@@ -0,0 +1,47 @@
+import fs from 'fs';
+import { logger } from '@lib/utils/logger';
+import { getLogFile } from './getLogFile';
+
+/**
+   * Get changed files since a date.
+   *
+   * @param sinceDate - Date to search
+
+  * @returns Array of changed lines
+   */
+export const getChangedLinesSince = (sinceDate: Date): string[] => {
+  let allLines: string[] = [];
+  const logFile = getLogFile();
+  if (logFile) {
+    try {
+      allLines = fs.readFileSync(logFile).toString().split('\n');
+    } catch (e) {
+      logger.error(
+        `Error reading metadata log file located at ${`logFile`}: ${e}`,
+      );
+    }
+  }
+
+  // todo - Do not use dates and use timestamps in all cases
+  //  to avoid having to fix offsets.
+  const date = new Date();
+  const dateOffset = -(date.getTimezoneOffset() * 60 * 1000);
+
+  const changedLines = allLines.filter(line => {
+    const uniqueId = line.substring(0, 32);
+    if (!uniqueId) {
+      return false;
+    }
+    const dateString = uniqueId.replace(
+      /(\d{4})-(\d{2})-(\d{2})_(\d{2})-(\d{2})-(\d{2})\.(\d{3}).*/,
+      '$1-$2-$3T$4:$5:$6.$7',
+    );
+    const uniqueIdDate = Date.parse(dateString) + dateOffset;
+    return uniqueIdDate && uniqueIdDate > sinceDate.getTime();
+  });
+  logger.debug(`Found ${changedLines.length} changed lines`);
+  changedLines.forEach(line => {
+    logger.debug(`Found changed line: ${line}`);
+  });
+  return changedLines;
+};

package/src/lib/store/workDir/storage/getDataFromLogLine.ts

@@ -0,0 +1,31 @@
+import { LogLineData } from '../workDir.types';
+
+/**
+ * Extract information from Static Suite's log line.
+ *
+ * @param line - Log line.
+ *
+ * @returns An object with parsed data, or null if line format is not valid.
+ */
+export const getDataFromLogLine = (line: string): LogLineData | null => {
+  const matches = line.match(/^(\S+) (\S+) \[ID: ([^\]]+)] (.+)/);
+  if (matches) {
+    const uniqueId = matches[1];
+    const operation = matches[2];
+    const fileId = matches[3];
+    const [fileLabel, uri] = matches[4].split(' | ');
+    const [, uriTarget] = uri.split('://');
+    if (operation === 'write' || operation === 'delete') {
+      return {
+        uniqueId,
+        operation,
+        file: {
+          id: fileId,
+          label: fileLabel,
+          relativePath: uriTarget,
+        },
+      };
+    }
+  }
+  return null;
+};

package/src/lib/store/workDir/storage/getLogFile.ts

@@ -0,0 +1,11 @@
+import { config } from '@lib/config';
+
+/**
+ * Gets the absolute path to Static Suite's log file.
+ *
+ * @returns The absolute path to Static Suite's log file.
+ */
+const getLogFile = (): string | null =>
+  config.workDir ? `${config.workDir}/lock-executed.log` : null;
+
+export { getLogFile };

package/src/lib/store/workDir/storage/index.ts

@@ -0,0 +1,8 @@
+/*
+These functions are separated from workDirHelper, since they are specific
+to the current storage backend (log files saved on disk) and they can change
+in the future.
+*/
+export * from './getChangedLinesSince';
+export * from './getDataFromLogLine';
+export * from './getLogFile';

package/src/lib/store/workDir/workDir.types.ts

@@ -0,0 +1,48 @@
+/**
+ * A group of changed files in Static Suite's data dir.
+ */
+export type ChangedFiles = {
+  /**
+   * A list of updated files (both newly added and changed files).
+   */
+  updated: string[];
+  /**
+   * A list of deleted files.
+   */
+  deleted: string[];
+};
+
+/**
+ * Information from a Static Suite's log line.
+ */
+export type LogLineData = {
+  /**
+   * The unique ID of the write/delete operation from Static Suite.
+   */
+  uniqueId: string;
+
+  /**
+   * The type of operation executed in the log line.
+   */
+  operation: 'write' | 'delete';
+
+  /**
+   * Information about the affected file
+   */
+  file: {
+    /**
+     * File ID, a string identifying the file in a meaningful way, usually a numeric ID
+     */
+    id: string;
+
+    /**
+     * A long text identifying the file in a meaningful way, usually the content title.
+     */
+    label: string;
+
+    /**
+     * Relative file path inside data dir.
+     */
+    relativePath: string;
+  };
+};