@static-suite/static-suite-data-server 0.0.1

package/src/__tests__/fixtures/data/en/entity/node/article/article2--small.json

@@ -0,0 +1,3 @@
+{
+  "name": "article2--small"
+}

package/src/__tests__/fixtures/data/en/entity/node/article/article2.json

@@ -0,0 +1,40 @@
+{
+  "data": {
+    "content": {
+      "id": "58725",
+      "title": "Title 2",
+      "author": {
+        "entity": {
+          "entityInclude": "en/entity/node/article/article3.json"
+        }
+      },
+      "image": {
+        "entity": {
+          "entityInclude": "en/entity/node/article/article3.json"
+        }
+      },
+      "queryInclude": "query1?y=33",
+      "article2CustomInclude": "en/entity/node/article/article3.json",
+      "customInclude": "en/entity/node/article/article3.json",
+      "localeInclude": "en/entity/node/article/article3.json",
+      "configInclude": "en/entity/node/article/article3.json",
+      "thisConfigInclude": "en/entity/node/article/article3.json"
+    }
+  },
+  "metadata": {
+    "includes": {
+      "static": {
+        "data.content.author.entity.entityInclude": "en/entity/node/article/article3.json",
+        "data.content.image.entity.entityInclude": "en/entity/node/article/article3.json",
+        "data.content.article2CustomInclude": "en/entity/node/article/article3.json",
+        "data.content.customInclude": "en/entity/node/article/article3.json",
+        "data.content.localeInclude": "en/entity/node/article/article3.json",
+        "data.content.configInclude": "en/entity/node/article/article3.json",
+        "data.content.thisConfigInclude": "en/entity/node/article/article3.json"
+      },
+      "dynamic": {
+        "data.content.queryInclude": "query2"
+      }
+    }
+  }
+}

package/src/__tests__/fixtures/data/en/entity/node/article/article3--card.json

@@ -0,0 +1,3 @@
+{
+  "name": "article3--card"
+}

package/src/__tests__/fixtures/data/en/entity/node/article/article3--small.json

@@ -0,0 +1,3 @@
+{
+  "name": "article3--small"
+}

package/src/__tests__/fixtures/data/en/entity/node/article/article3.json

@@ -0,0 +1,7 @@
+{
+  "data": {
+    "content": {
+      "name": "article3"
+    }
+  }
+}

package/src/__tests__/fixtures/data/en/entity/node/node.json

@@ -0,0 +1,3 @@
+{
+  "name": "node"
+}

package/src/__tests__/fixtures/data/en/entity/node/page/page1--card.json

@@ -0,0 +1,3 @@
+{
+  "name": "page1--card"
+}

package/src/__tests__/fixtures/data/en/entity/node/page/page1--small.json

@@ -0,0 +1,3 @@
+{
+  "name": "page1--small"
+}

package/src/__tests__/fixtures/data/en/entity/node/page/page1.json

@@ -0,0 +1,3 @@
+{
+  "name": "page1"
+}

package/src/__tests__/fixtures/data/en/entity/node/page/page2--card.json

@@ -0,0 +1,3 @@
+{
+  "name": "page2--card"
+}

package/src/__tests__/fixtures/data/en/entity/node/page/page2--small.json

@@ -0,0 +1,3 @@
+{
+  "name": "page2--small"
+}

package/src/__tests__/fixtures/data/en/entity/node/page/page2.json

@@ -0,0 +1,3 @@
+{
+  "name": "page2"
+}

package/src/__tests__/fixtures/data/global--block.json

@@ -0,0 +1,3 @@
+{
+  "name": "global--block"
+}

package/src/__tests__/fixtures/data/global.json

@@ -0,0 +1,3 @@
+{
+  "name": "global"
+}

package/src/__tests__/fixtures/hook/main.hook.js

@@ -0,0 +1,15 @@
+exports.onProcessFile = ({ fileContent }) => {
+  // Do something...
+  return {
+    raw: fileContent.raw,
+    json: fileContent.json,
+  };
+};
+
+exports.onStoreAdd = (/* _dataDir, _file, _fileContent, _store */) => {
+  // Do something...
+};
+
+exports.onStoreRemove = (/* _file, _store */) => {
+  // Do something...
+};

package/src/__tests__/fixtures/hook/taxonomy/main.hook.js

@@ -0,0 +1,15 @@
+exports.onProcessFile = ({ fileContent }) => {
+  // Do something...
+  return {
+    raw: fileContent.raw,
+    json: fileContent.json,
+  };
+};
+
+exports.onStoreAdd = (/* _dataDir, _file, _fileContent, _store */) => {
+  // Do something...
+};
+
+exports.onStoreRemove = (/* _file, _store */) => {
+  // Do something...
+};

package/src/__tests__/fixtures/hook/taxonomy/taxo.hook.js

@@ -0,0 +1,15 @@
+exports.onProcessFile = ({ fileContent }) => {
+  // Do something...
+  return {
+    raw: fileContent.raw,
+    json: fileContent.json,
+  };
+};
+
+exports.onStoreAdd = (/* _dataDir, _file, _fileContent, _store */) => {
+  // Do something...
+};
+
+exports.onStoreRemove = (/* _file, _store */) => {
+  // Do something...
+};

package/src/__tests__/fixtures/query/error.query.js

@@ -0,0 +1,6 @@
+const error = () => {
+  // eslint-disable-next-line no-throw-literal
+  throw new Error('oops');
+};
+
+module.exports.default = error;

package/src/__tests__/fixtures/query/noHandler.query.js

@@ -0,0 +1,14 @@
+const query2 = () => {
+  const result = [
+    {
+      id: '1',
+    },
+  ];
+
+  return {
+    result,
+    cacheable: true,
+  };
+};
+
+module.exports = query2;

package/src/__tests__/fixtures/query/query1.query.js

@@ -0,0 +1,15 @@
+const query1 = ({ args }) => {
+  const result = [
+    {
+      id: args.y,
+      random: Math.random(),
+    },
+  ];
+
+  return {
+    result,
+    cacheable: true,
+  };
+};
+
+module.exports.default = query1;

package/src/__tests__/fixtures/query/query2.query.js

@@ -0,0 +1,14 @@
+const query2 = () => {
+  const result = [
+    {
+      id: '1',
+    },
+  ];
+
+  return {
+    result,
+    cacheable: true,
+  };
+};
+
+module.exports.default = query2;

package/src/__tests__/fixtures/query/wrongNamequery.js

@@ -0,0 +1,14 @@
+const query1 = () => {
+  const result = [
+    {
+      id: '1',
+    },
+  ];
+
+  return {
+    result,
+    cacheable: true,
+  };
+};
+
+module.exports.default = query1;

package/src/__tests__/fixtures/task/error.task.js

@@ -0,0 +1,6 @@
+const error = () => {
+  // eslint-disable-next-line no-throw-literal
+  throw new Error('oops');
+};
+
+module.exports.default = error;

package/src/__tests__/fixtures/task/noHandler.task.js

@@ -0,0 +1,13 @@
+const task2 = () => {
+  const result = [
+    {
+      id: '1',
+    },
+  ];
+
+  return {
+    result,
+  };
+};
+
+module.exports = task2;

package/src/__tests__/fixtures/task/task1.task.js

@@ -0,0 +1,14 @@
+const task1 = ({ args }) => {
+  const result = [
+    {
+      id: args.y,
+      random: Math.random(),
+    },
+  ];
+
+  return {
+    result,
+  };
+};
+
+module.exports.default = task1;

package/src/__tests__/fixtures/task/task2.task.js

@@ -0,0 +1,13 @@
+const task2 = () => {
+  const result = [
+    {
+      id: '1',
+    },
+  ];
+
+  return {
+    result,
+  };
+};
+
+module.exports.default = task2;

package/src/__tests__/fixtures/task/wrongNametask.js

@@ -0,0 +1,13 @@
+const task1 = () => {
+  const result = [
+    {
+      id: '1',
+    },
+  ];
+
+  return {
+    result,
+  };
+};
+
+module.exports.default = task1;

package/src/bin/serve.ts

@@ -0,0 +1,107 @@
+#!/usr/bin/env node
+/* eslint-disable import/first */
+
+import yargs from 'yargs/yargs';
+import { hideBin } from 'yargs/helpers';
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+require('module-alias')(`${__dirname}/../..`);
+
+import { httpServer } from '@http/httpServer';
+import { dataServer } from '@lib/dataServer';
+import { RunMode, RunModeStrings } from '@lib/dataServer.types';
+import { LogLevel, LogLevelStrings } from '@lib/utils/logger/logger.types';
+
+const argv = yargs(hideBin(process.argv))
+  .usage('Usage: $0 http --data-dir [path]')
+  .command(['http [--port]'], 'Start an HTTP server')
+  .demandCommand()
+  .positional('--port', {
+    describe: 'Port number',
+    type: 'number',
+    default: 57471,
+  })
+  .options({
+    'data-dir': {
+      describe: 'Path to Static Suite data directory',
+      demandOption: true,
+      type: 'string',
+    },
+    'work-dir': {
+      describe: 'Path to Static Suite work directory',
+      demandOption: false,
+      type: 'string',
+    },
+    'query-dir': {
+      describe: 'Path to the directory where queries are stored',
+      demandOption: false,
+      type: 'string',
+    },
+    'hook-dir': {
+      describe: 'Path to the directory where hooks are stored',
+      type: 'string',
+    },
+    'task-dir': {
+      describe: 'Path to the directory where tasks are stored',
+      type: 'string',
+    },
+    'dump-dir': {
+      describe: 'Path to the directory where dumps are stored',
+      type: 'string',
+    },
+    'run-mode': {
+      demandOption: true,
+      default: 'prod',
+      describe:
+        'Run mode (dev or prod). Dev mode watches for changes on user-land modules (queries, hooks and tasks)',
+      choices: ['dev', 'prod'],
+    },
+    'log-level': {
+      describe: 'Log level verbosity',
+      choices: Object.values(LogLevel),
+      default: 'info',
+    },
+    'log-file': {
+      describe: 'Path to log file',
+      type: 'string',
+    },
+    'log-file-level': {
+      describe: 'Log file level verbosity',
+      choices: Object.values(LogLevel),
+    },
+    help: {
+      alias: 'h',
+      describe: 'Show help',
+      type: 'boolean',
+    },
+  })
+  .completion('completion', (_current, _argv, done) => {
+    setTimeout(() => {
+      done(['query-dir', 'data-dir']);
+    }, 500);
+  })
+  .parseSync();
+
+const logLevel = LogLevel[argv['log-level'].toUpperCase() as LogLevelStrings];
+const logFileLevel = argv['log-file-level']
+  ? LogLevel[argv['log-file-level'].toUpperCase() as LogLevelStrings]
+  : logLevel;
+
+// Configure data server.
+dataServer.init({
+  logLevel,
+  logFile: argv['log-file']
+    ? { path: argv['log-file'], level: logFileLevel }
+    : undefined,
+  dataDir: argv['data-dir'],
+  workDir: argv['work-dir'],
+  queryDir: argv['query-dir'],
+  hookDir: argv['hook-dir'],
+  taskDir: argv['task-dir'],
+  dumpDir: argv['dump-dir'],
+  runMode: RunMode[argv['run-mode'].toUpperCase() as RunModeStrings],
+});
+
+// Start server.
+if (argv._.includes('http')) {
+  httpServer.start(argv['--port']);
+}

package/src/http/httpServer.ts

@@ -0,0 +1,21 @@
+import path from 'path';
+import express from 'express';
+import cors from 'cors';
+import { logger } from '@lib/utils/logger';
+import routes from './routes';
+
+export const httpServer = {
+  start: (port: number): void => {
+    const app = express();
+    app.use(express.static(`${__dirname}/public`));
+    app.use(cors());
+    app.set('views', path.join(__dirname, 'views'));
+    app.set('view engine', 'pug');
+
+    app.use(routes);
+
+    app.listen(port, () => {
+      logger.info(`Data Server listening at http://localhost:${port}`);
+    });
+  },
+};

package/src/http/routes/cache.ts

@@ -0,0 +1,17 @@
+import { Request, Response } from 'express';
+import { cache } from '@lib/utils/cache';
+
+export const cacheIndex = (req: Request, res: Response): void => {
+  const cacheBins = Array.from(cache.keys());
+  res.render('cacheIndex', { cacheBins });
+};
+
+export const cacheClear = (req: Request, res: Response): void => {
+  const cacheBinId = req.params[0];
+  let done = false;
+  if (cache.has(cacheBinId)) {
+    cache.bin(cacheBinId).clear();
+    done = true;
+  }
+  res.render('cacheClear', { cacheBinId, done });
+};

package/src/http/routes/data.ts

@@ -0,0 +1,114 @@
+import path from 'path';
+import { Request, Response } from 'express';
+import mime from 'mime-types';
+import { store } from '@lib/store';
+import { logger } from '@lib/utils/logger';
+import { dataDirManager } from '@lib/store/dataDir';
+
+const data = (req: Request, res: Response): void => {
+  dataDirManager.update();
+  const storeKey = req.params[0];
+  const storeKeyParts =
+    !storeKey || storeKey === '' ? null : storeKey.split('/');
+
+  const storeFile = store.data.get(storeKey);
+
+  if (storeFile === undefined) {
+    // Render a directory
+    let dirKey = '';
+    if (storeKey) {
+      dirKey = storeKey.endsWith('/') ? storeKey : `${storeKey}/`;
+    }
+    logger.debug(`Rendering directory "${dirKey}"`);
+    const storeDirSubset = store.data.subset({
+      dir: dirKey,
+      variant: undefined,
+      ext: undefined,
+      recursive: false,
+    });
+
+    // Obtain breadcrumbs.
+    const breadcrumbs = storeKeyParts
+      ? storeKeyParts.map((keyPart, index) => ({
+          title: keyPart,
+          url: storeKeyParts.slice(0, index + 1).join('/'),
+        }))
+      : [];
+
+    type Item = {
+      name: string;
+      type: string;
+      info?: any;
+    };
+    const items: Array<Item> = [];
+
+    // Find directories, so they appear first on screen.
+    const mapKeys = Array.from(store.data.keys());
+    const foundDirs: Map<string, boolean> = new Map();
+    const pattern = `^${dirKey}([^/]+)/.+$`;
+    const regex = new RegExp(pattern);
+    mapKeys.forEach(k => {
+      const result = k.match(regex);
+      const dir = result?.[1];
+      if (dir && !foundDirs.has(dir)) {
+        foundDirs.set(dir, true);
+        items.push({
+          name: result[1],
+          type: 'directory',
+        });
+      }
+    });
+
+    // Add files so they appear after directories.
+    let hasContentInfo = false;
+    let info = null;
+    storeDirSubset.items.forEach((item, key) => {
+      const filename = storeDirSubset.filenames[key];
+      info = {
+        filename,
+      };
+
+      hasContentInfo = true;
+      const content = item.data?.content;
+      if (content) {
+        const { id, type, bundle, isPublished, title } = content;
+        if (id || type || bundle || title) {
+          info = {
+            id,
+            type,
+            bundle,
+            isPublished,
+            title,
+            filename,
+          };
+          hasContentInfo = true;
+        }
+      }
+
+      items.push({
+        name: path.basename(filename),
+        type: Array.isArray(item) ? 'array' : typeof item,
+        info,
+      });
+    });
+    const vars = {
+      base: dirKey,
+      path: dirKey || '/',
+      breadcrumbs,
+      items,
+      count: items.length,
+      hasContentInfo,
+    };
+    res.render('data', vars);
+  } else {
+    // Render a single file.
+    logger.debug(`Rendering file "${storeKey}", type ${typeof storeFile}`);
+    res.status(200);
+    res.set({
+      'Content-Type': mime.lookup(storeKey) || 'text/plain',
+    });
+    res.send(storeFile);
+  }
+};
+
+export { data };

package/src/http/routes/docs.ts

@@ -0,0 +1,9 @@
+import { Request, Response } from 'express';
+import path from 'path';
+import marked from 'marked';
+import { readFile } from '@lib/utils/fs/fsUtils';
+
+export const docs = (req: Request, res: Response): void => {
+  const readme = readFile(path.join(__dirname, '../README.md'));
+  res.render('docs', { marked, readme });
+};

package/src/http/routes/dump.ts

@@ -0,0 +1,61 @@
+import fs from 'fs';
+import { Request, Response } from 'express';
+import { dumpManager } from '@lib/store/dump';
+import { jsonify } from '@lib/utils/object';
+import { config } from '@lib/config';
+
+const dumpIndex = (req: Request, res: Response): void => {
+  res.render('statusIndex', {
+    links: {
+      '/dump/incremental': 'Execute an incremental dump',
+      '/dump/full': 'Execute a full dump',
+      '/dump/metadata/show': 'Show dump metadata info',
+      '/dump/metadata/reset': 'Reset dump metadata info',
+    },
+  });
+};
+
+const dumpIncremental = (req: Request, res: Response): void => {
+  const dump = dumpManager.dump({ incremental: true });
+  res.status(200);
+  res.set({ 'Content-Type': 'application/json' });
+  res.send(jsonify(dump));
+};
+
+const dumpFull = (req: Request, res: Response): void => {
+  const dump = dumpManager.dump({ incremental: false });
+  res.status(200);
+  res.set({ 'Content-Type': 'application/json' });
+  res.send(jsonify(dump));
+};
+
+const getMetadata = () => {
+  const metadataFilepath = `${config.dumpDir}/metadata.json`;
+  return fs.existsSync(metadataFilepath)
+    ? fs.readFileSync(metadataFilepath).toString()
+    : '[]';
+};
+
+const dumpMetadataShow = (req: Request, res: Response): void => {
+  res.status(200);
+  res.set({ 'Content-Type': 'application/json' });
+  res.send(getMetadata());
+};
+
+const dumpMetadataReset = (req: Request, res: Response): void => {
+  const args: any = req.query;
+  if (args?.timestamp) {
+    dumpManager.reset(parseInt(args.timestamp, 10));
+  }
+  res.status(200);
+  res.set({ 'Content-Type': 'application/json' });
+  res.send(getMetadata());
+};
+
+export {
+  dumpIndex,
+  dumpIncremental,
+  dumpFull,
+  dumpMetadataShow,
+  dumpMetadataReset,
+};

package/src/http/routes/home.ts

@@ -0,0 +1,16 @@
+import { Request, Response } from 'express';
+
+export const home = (req: Request, res: Response): void => {
+  res.render('home', {
+    links: {
+      '/data': 'Browse all data loaded into the server',
+      '/dump':
+        'Dump resolved data (includes and queries) into the dump directory',
+      '/query': 'List of available queries',
+      '/task': 'List of available tasks',
+      '/cache': 'List of available cache bins',
+      '/status': 'All things nerd',
+      '/docs': 'Documentation',
+    },
+  });
+};