@static-suite/static-suite-data-server 0.0.1

package/dist/lib/watcher/watcher.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Initializes a watcher on any file inside queryDir, hookDir and taskDir.
+ *
+ * @remarks
+ * If runMode is DEV and queryDir, hookDir and/or taskDir are defined by current
+ * configuration, a watcher is initialized on them.
+ *
+ * When something changes inside query directory:
+ * 1) Remove all modules inside the query directory, so they are required again.
+ * 2) Clear the query cache, since that is not done by the queryManager.
+ *
+ * When something changes inside hook directory:
+ * 1) Remove all modules inside the hook directory, so they are required again.
+ * 2) Reload the whole data directory so hooks can be reapplied (this implies clearing the query cache).
+ */
+export declare const initWatcher: () => void;
+//# sourceMappingURL=watcher.d.ts.map

package/dist/lib/watcher/watcher.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"watcher.d.ts","sourceRoot":"","sources":["../../../src/lib/watcher/watcher.ts"],"names":[],"mappings":"AASA;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,WAAW,QAAO,IAyC9B,CAAC"}

package/dist/lib/watcher/watcher.js

@@ -0,0 +1,65 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.initWatcher = void 0;
+const config_1 = require("@lib/config");
+const dataDirManager_1 = require("@lib/store/dataDir/dataDirManager");
+const logger_1 = require("@lib/utils/logger");
+const cache_1 = require("@lib/utils/cache");
+const fs_1 = require("@lib/utils/fs");
+const hook_1 = require("@lib/store/hook");
+const query_1 = require("@lib/query");
+const task_1 = require("@lib/task");
+/**
+ * Initializes a watcher on any file inside queryDir, hookDir and taskDir.
+ *
+ * @remarks
+ * If runMode is DEV and queryDir, hookDir and/or taskDir are defined by current
+ * configuration, a watcher is initialized on them.
+ *
+ * When something changes inside query directory:
+ * 1) Remove all modules inside the query directory, so they are required again.
+ * 2) Clear the query cache, since that is not done by the queryManager.
+ *
+ * When something changes inside hook directory:
+ * 1) Remove all modules inside the hook directory, so they are required again.
+ * 2) Reload the whole data directory so hooks can be reapplied (this implies clearing the query cache).
+ */
+const initWatcher = () => {
+    const paths = [];
+    if (config_1.config.queryDir) {
+        paths.push(config_1.config.queryDir);
+    }
+    if (config_1.config.hookDir) {
+        paths.push(config_1.config.hookDir);
+    }
+    if (config_1.config.taskDir) {
+        paths.push(config_1.config.taskDir);
+    }
+    const listener = (filePath) => {
+        // Remove all modules inside the query directory.
+        if (config_1.config.queryDir && filePath.startsWith(config_1.config.queryDir)) {
+            query_1.queryManager.reset();
+            // Clear all queries, since they are stale.
+            // No need to clear the store subset cache, since no
+            // file has been added/updated/deleted.
+            cache_1.cache.bin('query').clear();
+        }
+        // Remove all modules inside the hook directory and reload the whole
+        // data directory so hooks can be reapplied.
+        if (config_1.config.hookDir && filePath.startsWith(config_1.config.hookDir)) {
+            hook_1.hookManager.reset();
+            dataDirManager_1.dataDirManager.load();
+            logger_1.logger.debug('Re-building store done');
+        }
+        // Remove all modules inside the task directory.
+        if (config_1.config.taskDir && filePath.startsWith(config_1.config.taskDir)) {
+            task_1.taskManager.reset();
+        }
+    };
+    (0, fs_1.watch)(paths, {
+        add: listener,
+        change: listener,
+        unlink: listener,
+    });
+};
+exports.initWatcher = initWatcher;

package/dist/static-suite-data-server.d.ts

@@ -0,0 +1,392 @@
+import { CacheStatus } from '@lib/query/query.types';
+import { LogFile } from '@lib/utils/logger/logger.types';
+import { LogLevel } from '@lib/utils/logger/logger.types';
+import { QueryArgs } from '@lib/query/query.types';
+import { QueryErrorResponse } from '@lib/query/query.types';
+import { QueryModule } from '@lib/query/query.types';
+import { QueryRunner } from '@lib/query/query.types';
+import { QuerySuccessfulResponse } from '@lib/query/query.types';
+import { Store } from '@lib/store/store.types';
+
+export { CacheStatus }
+
+/**
+ * The Data Server instance.
+ *
+ * @public
+ */
+export declare const dataServer: {
+    /**
+     * Initializes the Data Server.
+     *
+     * @param options - Configuration options
+     *
+     * @returns An object with the data store and the queryRunner service.
+     */
+    init: (options: DataServerInitOptions) => DataServerReturn;
+};
+
+/**
+ * Init options for Data Server.
+ *
+ * @public
+ */
+export declare type DataServerInitOptions = {
+    /**
+     * Log level. Optional. @see {@link LogLevel}
+     */
+    logLevel?: LogLevel;
+    /**
+     * Path to the log file. Optional. @see {@link LogFile}
+     */
+    logFile?: LogFile;
+    /**
+     * Log level for the log file. Optional. @see {@link LogLevel}
+     */
+    logFileLevel?: LogLevel;
+    /**
+     * Path to the directory where data is stored.
+     */
+    dataDir: string;
+    /**
+     * Path to the directory where work data is stored. Optional.
+     */
+    workDir?: string;
+    /**
+     * Path to the directory where queries are stored. Optional.
+     */
+    queryDir?: string;
+    /**
+     * Path to the directory where hooks are stored. Optional.
+     */
+    hookDir?: string;
+    /**
+     * Path to the directory where tasks are stored. Optional.
+     */
+    taskDir?: string;
+    /**
+     * Path to the directory where dumps are stored. Optional.
+     */
+    dumpDir?: string;
+    /**
+     * Run mode (dev or prod).
+     */
+    runMode: RunMode;
+};
+
+/**
+ * Data returned by Data Server once initialized.
+ *
+ * @public
+ */
+export declare type DataServerReturn = {
+    /**
+     * The data store.
+     */
+    store: Store_2;
+    /**
+     * The query runner, to be able to run queries on demand.
+     */
+    queryRunner: QueryRunner;
+    /**
+     * The task runner, to be able to run tasks on demand.
+     */
+    taskRunner: TaskRunner;
+};
+
+export { LogFile }
+
+export { LogLevel }
+
+export { QueryArgs }
+
+export { QueryErrorResponse }
+
+export { QueryModule }
+
+export { QueryRunner }
+
+export { QuerySuccessfulResponse }
+
+/**
+ * Available run modes: dev or prod.
+ *
+ * @public
+ */
+export declare enum RunMode {
+    DEV = "dev",
+    PROD = "prod"
+}
+
+export { Store }
+
+/**
+ * The store that holds all data.
+ */
+declare type Store_2 = {
+    /**
+     * Date to tell when was the store last synced.
+     *
+     * Every time the store is updated, syncDate is set with the last modification
+     * date of the data directory, not the date when the store has finished updating.
+     *
+     * Given this scenario:
+     * - Data directory modified at 12:30:00
+     * - Syncing the store takes 3 seconds and finishes at 12:30:03
+     *
+     * ... syncDate will be 12:30:00, not 12:30:03.
+     *
+     * Doing it the other way round can cause data inconsistencies, because syncing files
+     * to the store takes time, it is done sequentially, and in the meanwhile, other
+     * modifications can happen:
+     * - Store last syncDate is 12:29:00
+     * - Data directory is modified at 12:30:00
+     * - We ask for modified files after 12:29:00 and got 3 files (1.json, 2.json and 3.json)
+     * - File "1.json" is processed and added to the store at 12:30:01
+     * - While "2.json" is being processed and added to the store, data directory is modified
+     * again at 12:30:02 and "1.json" is changed.
+     * - File "3.json" is processed and added to the store at 12:30:03
+     *
+     * At this moment, there are two options:
+     *
+     * 1) Setting syncDate to the current date:
+     * - Store syncDate is set to 12:30:03
+     * - Data directory is modified again at 12:30:10 and only "3.json" is changed
+     * - We ask for modified files after 12:30:03 and get only "3.json"
+     * - Modification done at 12:30:02 to "1.json" is lost and cannot be recovered unless "1.json" is
+     * modified again, or Data Server is restarted
+     *
+     * 2) Setting syncDate to the last modification date of the data directory:
+     * - Store syncDate is set to 12:30:00
+     * - Data directory is modified again at 12:30:10 and only "3.json" is changed
+     * - We ask for modified files after 12:30:00 and get "1.json" and "3.json"
+     * - Modification done at 12:30:02 to "1.json" is properly processed
+     */
+    syncDate: Date | null;
+    /**
+     * Map that holds all data for all files.
+     *
+     * @remarks
+     * This map contains files, keyed by their relative filepath. No directory or
+     * any other structure is stored here.
+     *
+     * When executing a query, a common pattern is to create a subset of this
+     * map (e.g.- to get all articles of an specific language) and the execute the
+     * query against that subset. To do so, see {@link Store#subset}
+     *
+     * @example
+     * ```
+     * // Find an specific article.
+     * const article = store.data.get('en/entity/node/article/123.json');
+     * const article = store.index.url.get('public/pretty/url');
+     * const article = store.index.uuid.get('nl').get('ce736c8b-a6ea-43c4-9a6b-ad1fc498017c');
+     * const article = store.index.custom.get('taxonomies').get('12234');
+     *
+     * // Create a subset of all english articles.
+     * const results = store.data.subset({ dir: 'en/entity/node/article/', recursive: true });
+     * ```
+     */
+    data: StoreData<string, any>;
+    /**
+     * A list of deleted files from store, to be able to track them.
+     */
+    deleted: Set<string>;
+    /**
+     * An object to hold accessory index data.
+     */
+    index: {
+        /**
+         * An index that holds all data for all files, keyed by their URL.
+         */
+        url: Map<string, any>;
+        /**
+         * An index that holds all data for all files, keyed by their lang and uuid.
+         */
+        uuid: Map<string, any>;
+        /**
+         * An index to hold custom data defined in hooks or queries.
+         */
+        custom: Map<string, any>;
+    };
+};
+
+declare interface StoreData<K, V> extends Map<K, V> {
+    /**
+     * Create a subset with all files in store that match the given arguments.
+     *
+     * @remarks
+     * All subsets are automatically cached, so passing the same arguments
+     * will return the same subset from the cache.
+     *
+     * This is the preferred way of getting a subset of the store files,
+     * since is a simple function that most of the times only requires
+     * the first argument.
+     *
+     * @example
+     * ```
+     * // Get a subset of all nodes with "json" extension.
+     * dataServer.store.data.subset({ dir: 'en/entity/node/', variant: null });
+     *
+     * // Get a subset of all articles in all languages with "json" extension.
+     * dataServer.store.data.subset({ dir: '.+/entity/node/article/' });
+     *
+     * // Get a subset of all articles regardless of their extension.
+     * dataServer.store.data.subset({ dir: 'en/entity/node/article/', ext: null });
+     *
+     * // Get a subset of all card variants for articles with "json" extension.
+     * dataServer.store.data.subset({ dir: 'en/entity/node/article/', variant: null });
+     *
+     * // Get a subset of english articles, with "yml" extension, non-recursively.
+     * dataServer.store.data.subset({ dir: 'en/entity/node/article/', ext: 'yml', recursive: false });
+     * ```
+     *
+     * @param options - Object with options for creating a store subset.
+     *
+     * @returns An object with "filenames" and "items".
+     */
+    subset(options: StoreSubsetOptions): StoreSubset;
+}
+
+/**
+ * Object holding a subset of items from the store.
+ */
+declare type StoreSubset = {
+    /**
+     * Array of filenames in this subset.
+     */
+    filenames: Array<string>;
+    /**
+     * Array of items in this subset.
+     */
+    items: Array<any>;
+};
+
+/**
+ * Options for subset() function.
+ */
+declare type StoreSubsetOptions = {
+    /**
+     * Optional base directory to filter files.
+     *
+     * @remarks
+     * It requires a trailing slash, but not a leading slash, e.g.- "en/entity/node/article/"
+     */
+    dir?: string;
+    /**
+     * Optional name of a variant file.
+     *
+     * @remarks
+     * Use '_main' to obtain the default variant.
+     * Use any other string, e.g.- 'card', to obtain the card variant.
+     * Use null to avoid searching for any variant.
+     * Defaults to '_main'.
+     */
+    variant?: string;
+    /**
+     * Optional file extension, without dots. Defaults to 'json'.
+     */
+    ext?: string;
+    /**
+     * Optional flag to search for files recursively. True by default.
+     */
+    recursive?: boolean;
+};
+
+/**
+ * An object with task arguments.
+ *
+ * @public
+ */
+declare type TaskArgs = Record<string, any>;
+
+/**
+ * An error response returned after executing a task.
+ *
+ * @remarks
+ * It takes an string from TaskModule#taskHandler, and wraps them into an
+ * structure with metadata. @see {@link TaskModule#default}
+ *
+ * @public
+ */
+declare type TaskErrorResponse = {
+    /**
+     * Error message from the task, usually coming from a failed validation.
+     */
+    error: string;
+};
+
+/**
+ * A service to handle the execution of tasks.
+ *
+ * @public
+ */
+declare type TaskRunner = {
+    /**
+     * Runs a task and returns its results with metadata.
+     *
+     * @remarks
+     * It checks several validations before a task is run, and then task is executed.
+     *
+     * It logs an error and throws an exception if any problem occurs during the
+     * task execution.
+     *
+     * @param taskId - Task ID (filename without extension and suffix) of the task
+     * to be executed. @see {@link ModuleInfo#id | Definition of a module ID}
+     * @param args - Optional object with task arguments.
+     *
+     * @returns The result of the task as a JSON object with data and metadata keys,
+     * or an error response if some validation is not met.
+     *
+     * @example
+     * Here's a result example:
+     * ```
+     * {
+     * data: ['title 1', 'title 2'],
+     *  metadata: {
+     *    contentType: 'application/json',
+     *    execTime: 2,
+     *  }
+     * }
+     * ```
+
+     * @throws
+     * Exception thrown if any problem occurs during the task execution.
+     */
+    run(taskId: string, args: TaskArgs): TaskSuccessfulResponse | TaskErrorResponse;
+};
+
+/**
+ * A successful response returned after executing a task.
+ *
+ * @remarks
+ * It takes the results from a TaskModule#taskHandler, and wraps them into an
+ * structure with metadata. @see {@link TaskModule#default}
+ *
+ * @public
+ */
+declare type TaskSuccessfulResponse = {
+    /**
+     * Data returned by the task. @see {@link TaskModule#default}
+     */
+    data: any;
+    /**
+     * Task metadata
+     */
+    metadata: {
+        /**
+         * Content type for the data available in "data"
+         *
+         * @remarks
+         * Even thought this structure is a JSON object, data coming from the
+         * task can be anything (XML, a string, etc). It must be specified so any
+         * consumer of this data knows what is being returned.
+         */
+        contentType: string;
+        /**
+         * Execution time taken by the task, in milliseconds.
+         */
+        execTimeMs: number;
+    };
+};
+
+export { }

package/package.json

@@ -0,0 +1,102 @@
+{
+  "name": "@static-suite/static-suite-data-server",
+  "version": "0.0.1",
+  "description": "A flexible, fast and developer friendly Data Server for Static Suite",
+  "main": "./dist/lib/index.js",
+  "typings": "./dist/lib/index.d.ts",
+  "license": "GPL-2.0+",
+  "dependencies": {
+    "@creativebulma/bulma-tooltip": "^1.2.0",
+    "@microsoft/api-extractor": "^7.18.19",
+    "@types/jest": "^27.0.2",
+    "@types/tap": "^15.0.5",
+    "bulma": "^0.9.3",
+    "chokidar": "^3.5.2",
+    "clear-module": "^4.1.1",
+    "clipboard": "^2.0.8",
+    "copyfiles": "^2.4.1",
+    "cors": "^2.8.5",
+    "decache": "^4.6.0",
+    "express": "^4.17.1",
+    "fast-glob": "^3.2.7",
+    "marked": "^3.0.4",
+    "microtime": "^3.0.0",
+    "mime-types": "^2.1.33",
+    "module-alias": "^2.2.2",
+    "nodemon": "^2.0.13",
+    "pug": "^3.0.2",
+    "simple-datatables": "^3.1.2",
+    "tsc-watch": "^4.5.0",
+    "typescript": "^4.4.3",
+    "winston": "^3.3.3",
+    "yargs": "^17.2.1"
+  },
+  "bin": {
+    "static-suite-data-server": "./dist/bin/serve.js"
+  },
+  "scripts": {
+    "dev": "tsc-watch --onSuccess \"yarn lint\"",
+    "build": "yarn copy-http-files-to-src && yarn lint && tsc && yarn api-extractor run --local --verbose && yarn post-build && yarn copy-http-files-to-dist",
+    "post-build": "yarn rimraf dist/**/*__tests__",
+    "serve": "node ./dist/bin/serve.js http --port=57471 --data-dir=./src/__tests__/fixtures/data --work-dir=./src/__tests__/fixtures/work --query-dir=./src/__tests__/fixtures/query  --hook-dir=./src/__tests__/fixtures/hook --task-dir=./src/__tests__/fixtures/task --run-mode=dev --log-level=debug",
+    "lint": "eslint . --ext .ts",
+    "docs": "yarn typedoc --out docs src/lib/",
+    "prettier-format": "prettier --config .prettierrc 'src/**/*.ts' --write",
+    "test": "jest --watch",
+    "copy-http-files-to-src": "node ./scripts/copy-http-files-to-src.js",
+    "copy-http-files-to-dist": "cp -r ./src/http/public/ ./dist/http/ && cp -r ./src/http/views/ ./dist/http/ && cp README.md ./dist/http/"
+  },
+  "devDependencies": {
+    "@types/cors": "^2.8.12",
+    "@types/express": "^4.17.13",
+    "@types/marked": "^3.0.2",
+    "@types/microtime": "^2.1.0",
+    "@types/mime-types": "^2.1.1",
+    "@types/module-alias": "^2.0.1",
+    "@types/node": "^16.10.2",
+    "@types/yargs": "^17.0.3",
+    "@typescript-eslint/eslint-plugin": "^4.33.0",
+    "@typescript-eslint/parser": "^4.33.0",
+    "eslint": "^7.32.0",
+    "eslint-config-airbnb-typescript-prettier": "^4.2.0",
+    "eslint-import-resolver-typescript": "^2.5.0",
+    "eslint-plugin-import": "^2.25.2",
+    "eslint-plugin-prettier": "^4.0.0",
+    "eslint-plugin-tsdoc": "^0.2.14",
+    "husky": "^4.3.7",
+    "jest": "^27.3.1",
+    "prettier": "^2.4.1",
+    "ts-jest": "^27.0.7",
+    "typedoc": "^0.22.6",
+    "yarn-upgrade-all": "^0.5.4"
+  },
+  "_moduleAliases": {
+    "@bin": "dist/bin",
+    "@http": "dist/http",
+    "@lib": "dist/lib",
+    "@__tests__": "dist/__tests__"
+  },
+  "jest": {
+    "preset": "ts-jest",
+    "verbose": true,
+    "moduleDirectories": [
+      "node_modules",
+      "src"
+    ],
+    "testPathIgnorePatterns": [
+      "src/__tests__/fixtures/*",
+      "src/__tests__/mocks/*"
+    ],
+    "moduleNameMapper": {
+      "^@bin/(.*)$": "<rootDir>/src/bin/$1",
+      "^@http/(.*)$": "<rootDir>/src/http/$1",
+      "^@lib/(.*)$": "<rootDir>/src/lib/$1",
+      "^@__tests__/(.*)$": "<rootDir>/src/__tests__/$1"
+    }
+  },
+  "husky": {
+    "hooks": {
+      "pre-commit": "yarn build"
+    }
+  }
+}

package/scripts/copy-http-files-to-src.js

@@ -0,0 +1,45 @@
+/**
+ * Script to run after npm install
+ *
+ * Copy selected files to assets directory
+ */
+
+const fs = require('fs');
+
+// Copy files from Bulma to assets directory
+const bulmaSrcDir = 'node_modules/bulma/css/';
+const bulmaTargetDir = `${process.env.INIT_CWD}/src/http/public/assets/bulma/`;
+fs.mkdirSync(bulmaTargetDir, { recursive: true });
+fs.copyFileSync(
+  `${bulmaSrcDir}/bulma.min.css`,
+  `${bulmaTargetDir}/bulma.min.css`,
+);
+
+// Copy files from Bulma tooltip to assets directory
+const bulmaTooltipSrcDir = 'node_modules/@creativebulma/bulma-tooltip/dist/';
+fs.copyFileSync(
+  `${bulmaTooltipSrcDir}/bulma-tooltip.min.css`,
+  `${bulmaTargetDir}/bulma-tooltip.min.css`,
+);
+
+// Copy files from Clipboard to assets directory
+const clipboardSrcDir = 'node_modules/clipboard/dist/';
+const clipboardTargetDir = `${process.env.INIT_CWD}/src/http/public/assets/clipboard/`;
+fs.mkdirSync(clipboardTargetDir, { recursive: true });
+fs.copyFileSync(
+  `${clipboardSrcDir}/clipboard.min.js`,
+  `${clipboardTargetDir}/clipboard.min.js`,
+);
+
+// Copy files from datatables to assets directory
+const dataTablesSrcDir = 'node_modules/simple-datatables/dist/';
+const dataTablesTargetDir = `${process.env.INIT_CWD}/src/http/public/assets/simple-datatables/`;
+fs.mkdirSync(dataTablesTargetDir, { recursive: true });
+fs.copyFileSync(
+  `${dataTablesSrcDir}/umd/simple-datatables.js`,
+  `${dataTablesTargetDir}/index.js`,
+);
+fs.copyFileSync(
+  `${dataTablesSrcDir}/style.css`,
+  `${dataTablesTargetDir}/style.css`,
+);

package/src/__tests__/fixtures/data/en/en.json

@@ -0,0 +1,3 @@
+{
+  "name": "en"
+}

package/src/__tests__/fixtures/data/en/entity/entity.json

@@ -0,0 +1,3 @@
+{
+  "name": "entity"
+}

package/src/__tests__/fixtures/data/en/entity/node/article/article1--card.json

@@ -0,0 +1,3 @@
+{
+  "name": "article1--card"
+}

package/src/__tests__/fixtures/data/en/entity/node/article/article1--small.json

@@ -0,0 +1,3 @@
+{
+  "name": "article1--small"
+}

package/src/__tests__/fixtures/data/en/entity/node/article/article1.json

@@ -0,0 +1,40 @@
+{
+  "data": {
+    "content": {
+      "id": "58724",
+      "title": "Title 1",
+      "author": {
+        "entity": {
+          "entityInclude": "en/entity/node/article/article2.json"
+        }
+      },
+      "image": {
+        "entity": {
+          "entityInclude": "en/entity/node/article/article2.json"
+        }
+      },
+      "queryInclude": "query1?y=33",
+      "article2CustomInclude": "en/entity/node/article/article3.json",
+      "customInclude": "en/entity/node/article/article3.json",
+      "localeInclude": "en/entity/node/article/article3.json",
+      "configInclude": "en/entity/node/article/article3.json",
+      "thisConfigInclude": "en/entity/node/article/article3.json"
+    }
+  },
+  "metadata": {
+    "includes": {
+      "static": {
+        "data.content.author.entity.entityInclude": "en/entity/node/article/article2.json",
+        "data.content.image.entity.entityInclude": "en/entity/node/article/article2.json",
+        "data.content.article2CustomInclude": "en/entity/node/article/article3.json",
+        "data.content.customInclude": "en/entity/node/article/article3.json",
+        "data.content.localeInclude": "en/entity/node/article/article3.json",
+        "data.content.configInclude": "en/entity/node/article/article3.json",
+        "data.content.thisConfigInclude": "en/entity/node/article/article3.json"
+      },
+      "dynamic": {
+        "data.content.queryInclude": "query1?y=33"
+      }
+    }
+  }
+}

package/src/__tests__/fixtures/data/en/entity/node/article/article2--card.json

@@ -0,0 +1,3 @@
+{
+  "name": "article2--card"
+}