@rubeneschauzier/solidbench 2.1.0

package/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright © 2021 - now Ruben Taelman
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

package/README.md

@@ -0,0 +1,211 @@
+# SolidBench.js
+
+[![Build status](https://github.com/SolidBench/SolidBench.js/workflows/CI/badge.svg)](https://github.com/SolidBench/SolidBench.js/actions?query=workflow%3ACI)
+[![Coverage Status](https://coveralls.io/repos/github/SolidBench/SolidBench.js/badge.svg?branch=master)](https://coveralls.io/github/SolidBench/SolidBench.js?branch=master)
+[![npm version](https://badge.fury.io/js/solidbench.svg)](https://www.npmjs.com/package/solidbench)
+
+### A benchmark for [Solid](https://solidproject.org/) to simulate vaults with social network data.
+
+This benchmark allows you to generate a large amount of Solid data vaults with **simulated social network data**,
+and serve them over HTTP using a built-in [Solid Community Server](https://github.com/CommunitySolidServer/CommunitySolidServer) instance.
+Furthermore, different SPARQL queries will be generated to simulate workloads of social network apps for Solid.
+
+This benchmark is based on the [LDBC SNB](https://github.com/ldbc/ldbc_snb_datagen_hadoop) **social network** dataset.
+
+## Requirements
+
+* [Node.js](https://nodejs.org/en/) _(16 or higher)_
+* [Docker](https://www.docker.com/) _(required for invoking [LDBC SNB generator](https://github.com/ldbc/ldbc_snb_datagen_hadoop))_
+
+## Installation
+
+```bash
+$ npm install -g solidbench
+```
+or
+```bash
+$ yarn global add solidbench --ignore-engines
+```
+
+## Quick start
+
+1. `$ solidbench generate`: Generate Solid data vaults with social network data.
+2. `$ solidbench serve`: Serve datasets over HTTP.
+3. Initiate HTTP requests over `http://localhost:3000/`, such as `$ curl http://localhost:3000/pods/00000000000000000933/profile/card`
+
+## Usage
+
+### 1. Generate
+
+The social network data can be generated using the default options:
+
+```bash
+$ solidbench generate
+```
+
+**Full usage options:**
+
+```bash
+solidbench generate
+
+Generate social network data
+
+Options:
+      --version                    Show version number                 [boolean]
+      --cwd                        The current working directory
+                                                           [string] [default: .]
+      --verbose                    If more output should be printed    [boolean]
+      --help                       Show help                           [boolean]
+  -o, --overwrite                  If existing files should be overwritten
+                                                       [string] [default: false]
+  -s, --scale                      The SNB scale factor  [number] [default: 0.1]
+  -e, --enhancementConfig          Path to enhancement config
+                                  [string] [default: enhancer-config-pod.json]
+  -f, --fragmentConfig             Path to fragmentation config
+                                  [string] [default: fragmenter-config-pod.json]
+  -q, --queryConfig                Path to query instantiation config
+                                           [string] [default: query-config.json]
+      --validationParams           URL of the validation parameters zip file
+                           [string] [default: https://.../validation_params.zip]
+  -v, --validationConfig           Path to validation generator config
+                                      [string] [default: validation-config.json]
+      --hadoopMemory               Memory limit for Hadoop
+                                                        [string] [default: "4G"]
+```
+
+**Memory usage**
+
+For certain scale factors, you may have to increase your default Node memory limit.
+You can do this as follows (set RAM limit to 8GB):
+
+```bash
+NODE_OPTIONS=--max-old-space-size=8192 solidbench.js generate
+```
+
+**What does this do?**
+
+This generate command will first use the (interactive) [LDBC SNB generator](https://github.com/ldbc/ldbc_snb_datagen_hadoop)
+to **create one large Turtle file** with a given scale factor (defaults to `0.1`, allowed values: `[0.1, 0.3, 1, 3, 10, 30, 100, 300, 1000]).
+The default scale factor of `0.1` produces around 5M triples, and requires around 15 minutes for full generation on an average machine.
+
+Then, **auxiliary data** will be generated using [`ldbc-snb-enhancer.js`](https://github.com/SolidBench/ldbc-snb-enhancer.js/)
+based on the given enhancement config (defaults to an empty config).
+
+Next, this Turtle file will be **fragmented** using [`rdf-dataset-fragmenter.js`](https://github.com/SolidBench/rdf-dataset-fragmenter.js)
+and the given fragmentation strategy config (defaults to a Solid vault-based fragmentation).
+
+Then, **query** templates will be instantiated based on the generated data.
+This is done using [`sparql-query-parameter-instantiator.js`](https://github.com/SolidBench/sparql-query-parameter-instantiator.js)
+with the given query instantiation config (defaults to a config instantiating [all LDBC SNB interactive queries](https://github.com/SolidBench/SolidBench.js/tree/master/templates/queries)).
+
+Finally, **validation queries and results** will be generated.
+This is done using [`ldbc-snb-validation-generator.js`](https://github.com/SolidBench/ldbc-snb-validation-generator.js/) with the given validation config.
+This defaults to a config instantiating all queries and results from the `validation_params-sf1-without-updates.csv` file from LDBC SNB.
+This default config will produce queries and expected results in the `out-validate/` directory,
+which are expected to be executed on a scale factor of `1`.
+
+### 2. Serve
+
+The social network data can be served over HTTP as actual Solid vaults:
+
+```bash
+$ solidbench serve
+```
+
+**Full usage options:**
+
+```bash
+solidbench serve
+
+Serves the fragmented dataset via an HTTP server
+
+Options:
+      --version       Show version number                              [boolean]
+      --cwd           The current working directory        [string] [default: .]
+      --verbose       If more output should be printed                 [boolean]
+      --help          Show help                                        [boolean]
+  -p, --port          The HTTP port to run on           [number] [default: 3000]
+  -b, --baseUrl       The base URL of the server                        [string]
+  -r, --rootFilePath  Path to the root of the files to serve
+                        [string] [default: "out-fragments/http/localhost_3000/"]
+  -c, --config        Path to server config
+                                          [string] [default: server-config.json]
+  -l, --logLevel      Logging level (error, warn, info, verbose, debug, silly)
+                                                      [string] [default: "info"]
+```
+
+**What does this do?**
+
+The fragmented dataset from the preparation phase is loaded into the [Solid Community Server](https://github.com/CommunitySolidServer/CommunitySolidServer)
+so that it can be served over HTTP.
+
+The provided server config uses a simple file-based mapping, so that for example the file in `out-fragments/http/localhost:3000/pods/00000000000000000933/profile/card` is served on `http://localhost:3000/pods/00000000000000000933/profile/card`.
+Once the server is live, you can perform requests such as:
+
+```bash
+$ curl http://localhost:3000/pods/00000000000000000933/profile/card
+```
+
+## Data model
+
+By default, the following data model is used where all triples are placed in the document identified by their subject URL.
+
+![](https://raw.githubusercontent.com/ldbc/ldbc_snb_docs/dev/figures/schema-comfortable.png)
+
+Query templates can be found in [`templates/queries/`](https://github.com/rubensworks/solidbench.js/tree/master/templates/queries).
+
+## Pod-based fragmentation
+
+By default, data will be fragmented into files resembling [Solid data pods](https://solidproject.org/).
+
+For example, a generated pod can contain the following files:
+
+```text
+pods/00000000000000000290/
+    comments/
+        2010-12-02
+        2012-07-14
+    noise/
+        NOISE-1411
+        NOISE-83603
+    posts/
+        2010-02-14
+        2012-09-09
+    profile/
+        card.
+    settings/
+        publicTypeIndex
+```
+
+All files are serialized using the [N-Quads serialization](https://www.w3.org/TR/n-quads/).
+
+The `noise/` directory contains dummy triples with the main purpose of increasing the size of a pod.
+The amount of noise that is produced can be configured using the enhancement config file.
+
+Below, a minimalized example of the contents of a profile can be found:
+```turtle
+<http://localhost:3000/pods/00000000000000000290/profile/card#me> <http://www.w3.org/ns/solid/terms#publicTypeIndex> <http://localhost:3000/pods/00000000000000000290/settings/publicTypeIndex> .
+<http://localhost:3000/pods/00000000000000000290/profile/card#me> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://localhost:3000/www.ldbc.eu/ldbc_socialnet/1.0/vocabulary/Person> .
+<http://localhost:3000/pods/00000000000000000290/profile/card#me> <http://www.w3.org/ns/pim/space#storage> <http://localhost:3000/pods/00000000000000000290/> .
+<http://localhost:3000/pods/00000000000000000290/profile/card#me> <http://localhost:3000/www.ldbc.eu/ldbc_socialnet/1.0/vocabulary/id> "290"^^<http://www.w3.org/2001/XMLSchema#long> .
+<http://localhost:3000/pods/00000000000000000290/profile/card#me> <http://localhost:3000/www.ldbc.eu/ldbc_socialnet/1.0/vocabulary/firstName> "Ayesha" .
+<http://localhost:3000/pods/00000000000000000290/profile/card#me> <http://localhost:3000/www.ldbc.eu/ldbc_socialnet/1.0/vocabulary/lastName> "Baloch" .
+<http://localhost:3000/pods/00000000000000000290/profile/card#me> <http://localhost:3000/www.ldbc.eu/ldbc_socialnet/1.0/vocabulary/gender> "male" .
+<http://localhost:3000/pods/00000000000000000290/profile/card#me> <http://localhost:3000/www.ldbc.eu/ldbc_socialnet/1.0/vocabulary/hasInterest> <http://localhost:3000/www.ldbc.eu/ldbc_socialnet/1.0/tag/John_the_Baptist> .
+<http://localhost:3000/pods/00000000000000000290/profile/card#me> <http://localhost:3000/www.ldbc.eu/ldbc_socialnet/1.0/vocabulary/knows> _:b4_knows00000000000000124063 .
+_:b4_knows00000000000000124063 <http://localhost:3000/www.ldbc.eu/ldbc_socialnet/1.0/vocabulary/hasPerson> <http://localhost:3000/pods/00000000000000001753/profile/card#me> .
+```
+
+## Limitations
+
+At this stage, this benchmark has the following limitations:
+
+- Vaults don't make use of authentication, and all data is readable by everyone without authentication.
+- SPARQL update queries for modifying data within vaults are not being generated yet: https://github.com/SolidBench/SolidBench.js/issues/3
+- All vaults make use of the same vocabulary: https://github.com/SolidBench/SolidBench.js/issues/4
+
+## License
+
+This software is written by [Ruben Taelman](https://rubensworks.net/).
+
+This code is released under the [MIT license](http://opensource.org/licenses/MIT).

package/bin/solidbench

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+require('./solidbench.js')

package/bin/solidbench.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/bin/solidbench.js

@@ -0,0 +1,6 @@
+#!/usr/bin/env node
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const CliRunner_1 = require("../lib/CliRunner");
+(0, CliRunner_1.runCli)(process.cwd(), process.argv);
+//# sourceMappingURL=solidbench.js.map

package/bin/solidbench.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"solidbench.js","sourceRoot":"","sources":["solidbench.ts"],"names":[],"mappings":";;;AACA,gDAA0C;AAE1C,IAAA,kBAAM,EAAC,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC","sourcesContent":["#!/usr/bin/env node\nimport { runCli } from '../lib/CliRunner';\n\nrunCli(process.cwd(), process.argv);\n"]}

package/lib/CliRunner.d.ts

@@ -0,0 +1 @@
+export declare function runCli(cwd: string, argv: string[]): void;

package/lib/CliRunner.js

@@ -0,0 +1,20 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runCli = runCli;
+const yargs_1 = __importDefault(require("yargs"));
+const helpers_1 = require("yargs/helpers");
+function runCli(cwd, argv) {
+    const _ = (0, yargs_1.default)((0, helpers_1.hideBin)(argv))
+        .options({
+        cwd: { type: 'string', default: cwd, describe: 'The current working directory', defaultDescription: '.' },
+        verbose: { type: 'boolean', describe: 'If more output should be printed' },
+    })
+        .commandDir('commands')
+        .demandCommand()
+        .help()
+        .parse();
+}
+//# sourceMappingURL=CliRunner.js.map

package/lib/CliRunner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CliRunner.js","sourceRoot":"","sources":["CliRunner.ts"],"names":[],"mappings":";;;;;AAGA,wBAUC;AAbD,kDAA0B;AAC1B,2CAAwC;AAExC,SAAgB,MAAM,CAAC,GAAW,EAAE,IAAc;IAChD,MAAM,CAAC,GAAG,IAAA,eAAK,EAAC,IAAA,iBAAO,EAAC,IAAI,CAAC,CAAC;SAC3B,OAAO,CAAC;QACP,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,+BAA+B,EAAE,kBAAkB,EAAE,GAAG,EAAE;QACzG,OAAO,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,kCAAkC,EAAE;KAC3E,CAAC;SACD,UAAU,CAAC,UAAU,CAAC;SACtB,aAAa,EAAE;SACf,IAAI,EAAE;SACN,KAAK,EAAE,CAAC;AACb,CAAC","sourcesContent":["import yargs from 'yargs';\nimport { hideBin } from 'yargs/helpers';\n\nexport function runCli(cwd: string, argv: string[]): void {\n  const _ = yargs(hideBin(argv))\n    .options({\n      cwd: { type: 'string', default: cwd, describe: 'The current working directory', defaultDescription: '.' },\n      verbose: { type: 'boolean', describe: 'If more output should be printed' },\n    })\n    .commandDir('commands')\n    .demandCommand()\n    .help()\n    .parse();\n}\n"]}

package/lib/Generator.d.ts

@@ -0,0 +1,76 @@
+/**
+ * Generates decentralized social network data in different phases.
+ */
+export declare class Generator {
+    static readonly COLOR_RESET: string;
+    static readonly COLOR_RED: string;
+    static readonly COLOR_GREEN: string;
+    static readonly COLOR_YELLOW: string;
+    static readonly COLOR_BLUE: string;
+    static readonly COLOR_MAGENTA: string;
+    static readonly COLOR_CYAN: string;
+    static readonly COLOR_GRAY: string;
+    static readonly LDBC_SNB_DATAGEN_DOCKER_IMAGE: string;
+    private readonly cwd;
+    private readonly verbose;
+    private readonly overwrite;
+    private readonly scale;
+    private readonly enhancementConfig;
+    private readonly fragmentConfig;
+    private readonly queryConfig;
+    private readonly validationParams;
+    private readonly validationConfig;
+    private readonly hadoopMemory;
+    private readonly mainModulePath;
+    constructor(opts: IGeneratorOptions);
+    protected targetExists(path: string): Promise<boolean>;
+    protected log(phase: string, status: string): void;
+    protected runPhase(name: string, directory: string, runner: () => Promise<void>): Promise<void>;
+    /**
+     * Run all generator phases.
+     */
+    generate(): Promise<void>;
+    /**
+     * Invoke the LDBC SNB generator.
+     */
+    generateSnbDataset(): Promise<void>;
+    /**
+     * Enhance the generated LDBC SNB dataset.
+     */
+    enhanceSnbDataset(): Promise<void>;
+    /**
+     * Fragment the generated and enhanced LDBC SNB datasets.
+     */
+    fragmentSnbDataset(): Promise<void>;
+    /**
+     * Instantiate queries based on the LDBC SNB datasets.
+     */
+    instantiateQueries(): Promise<void>;
+    /**
+     * Download validation parameters
+     */
+    downloadValidationParams(): Promise<void>;
+    /**
+     * Generate validation queries and results.
+     */
+    generateValidation(): Promise<void>;
+    protected generateVariables(): Promise<Record<string, string>>;
+    /**
+     * Return a string in a given color
+     * @param str The string that should be printed in
+     * @param color A given color
+     */
+    static withColor(str: any, color: string): string;
+}
+export interface IGeneratorOptions {
+    cwd: string;
+    verbose: boolean;
+    overwrite: boolean;
+    scale: string;
+    enhancementConfig: string;
+    fragmentConfig: string;
+    queryConfig: string;
+    validationParams?: string;
+    validationConfig?: string;
+    hadoopMemory: string;
+}

package/lib/Generator.js

@@ -0,0 +1,231 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Generator = void 0;
+const promises_1 = require("node:fs/promises");
+const node_https_1 = require("node:https");
+const node_path_1 = require("node:path");
+const dockerode_1 = __importDefault(require("dockerode"));
+const ldbc_snb_enhancer_1 = require("ldbc-snb-enhancer");
+const ldbc_snb_validation_generator_1 = require("ldbc-snb-validation-generator");
+const rdf_dataset_fragmenter_1 = require("rdf-dataset-fragmenter");
+const sparql_query_parameter_instantiator_1 = require("sparql-query-parameter-instantiator");
+const unzipper_1 = require("unzipper");
+/**
+ * Generates decentralized social network data in different phases.
+ */
+class Generator {
+    constructor(opts) {
+        this.cwd = opts.cwd;
+        this.verbose = opts.verbose;
+        this.overwrite = opts.overwrite;
+        this.scale = opts.scale;
+        this.enhancementConfig = opts.enhancementConfig;
+        this.fragmentConfig = opts.fragmentConfig;
+        this.queryConfig = opts.queryConfig;
+        this.validationParams = opts.validationParams;
+        this.validationConfig = opts.validationConfig;
+        this.hadoopMemory = opts.hadoopMemory;
+        this.mainModulePath = (0, node_path_1.join)(__dirname, '..');
+    }
+    async targetExists(path) {
+        try {
+            await (0, promises_1.stat)(path);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    log(phase, status) {
+        process.stdout.write(`${Generator.withColor(`[${phase}]`, Generator.COLOR_CYAN)} ${status}\n`);
+    }
+    async runPhase(name, directory, runner) {
+        if (this.overwrite || !await this.targetExists((0, node_path_1.join)(this.cwd, directory))) {
+            this.log(name, 'Started');
+            const timeStart = process.hrtime();
+            await runner();
+            const timeEnd = process.hrtime(timeStart);
+            this.log(name, `Done in ${timeEnd[0] + (timeEnd[1] / 1_000_000_000)} seconds`);
+        }
+        else {
+            this.log(name, `Skipped (/${directory} already exists, remove to regenerate)`);
+        }
+    }
+    /**
+     * Run all generator phases.
+     */
+    async generate() {
+        const timeStart = process.hrtime();
+        await this.runPhase('SNB dataset generator', 'out-snb', () => this.generateSnbDataset());
+        await this.runPhase('SNB dataset enhancer', 'out-enhanced', () => this.enhanceSnbDataset());
+        await this.runPhase('SNB dataset fragmenter', 'out-fragments', () => this.fragmentSnbDataset());
+        await this.runPhase('SPARQL query instantiator', 'out-queries', () => this.instantiateQueries());
+        if (this.validationParams && this.validationConfig) {
+            await this.runPhase('SNB validation downloader', 'out-validate-params', () => this.downloadValidationParams());
+            await this.runPhase('SNB validation generator', 'out-validate', () => this.generateValidation());
+        }
+        const timeEnd = process.hrtime(timeStart);
+        this.log('All', `Done in ${timeEnd[0] + (timeEnd[1] / 1_000_000_000)} seconds`);
+    }
+    /**
+     * Invoke the LDBC SNB generator.
+     */
+    async generateSnbDataset() {
+        // Create params.ini file
+        const paramsTemplate = await (0, promises_1.readFile)((0, node_path_1.join)(__dirname, '../templates/params.ini'), 'utf8');
+        const paramsPath = (0, node_path_1.join)(this.cwd, 'params.ini');
+        await (0, promises_1.writeFile)(paramsPath, paramsTemplate.replaceAll('SCALE', this.scale), 'utf8');
+        // Pull the base Docker image
+        const dockerode = new dockerode_1.default();
+        const buildStream = await dockerode.pull(Generator.LDBC_SNB_DATAGEN_DOCKER_IMAGE);
+        await new Promise((resolve, reject) => {
+            dockerode.modem.followProgress(buildStream, (err, res) => err ? reject(err) : resolve(res));
+        });
+        // Start Docker container
+        const container = await dockerode.createContainer({
+            Image: Generator.LDBC_SNB_DATAGEN_DOCKER_IMAGE,
+            Tty: true,
+            AttachStdout: true,
+            AttachStderr: true,
+            Env: [`HADOOP_CLIENT_OPTS=-Xmx${this.hadoopMemory}`],
+            HostConfig: {
+                Binds: [
+                    `${this.cwd}/out-snb/:/opt/ldbc_snb_datagen/out`,
+                    `${paramsPath}:/opt/ldbc_snb_datagen/params.ini`,
+                ],
+            },
+        });
+        await container.start();
+        // Stop process on force-exit
+        let containerEnded = false;
+        // eslint-disable-next-line ts/no-misused-promises
+        process.on('SIGINT', async () => {
+            if (!containerEnded) {
+                await container.kill();
+                await cleanup();
+            }
+        });
+        async function cleanup() {
+            await container.remove();
+            await (0, promises_1.unlink)(paramsPath);
+        }
+        // Attach output to stdout
+        const out = await container.attach({
+            stream: true,
+            stdout: true,
+            stderr: true,
+        });
+        if (this.verbose) {
+            out.pipe(process.stdout);
+        }
+        else {
+            out.resume();
+        }
+        // Wait until generation ends
+        await new Promise((resolve, reject) => {
+            out.on('end', resolve);
+            out.on('error', reject);
+        });
+        containerEnded = true;
+        // Cleanup
+        await cleanup();
+    }
+    /**
+     * Enhance the generated LDBC SNB dataset.
+     */
+    async enhanceSnbDataset() {
+        // Create target directory
+        await (0, promises_1.mkdir)((0, node_path_1.join)(this.cwd, 'out-enhanced'));
+        // Run enhancer
+        const oldCwd = process.cwd();
+        process.chdir(this.cwd);
+        await (0, ldbc_snb_enhancer_1.runConfig)(this.enhancementConfig, { mainModulePath: this.mainModulePath });
+        process.chdir(oldCwd);
+    }
+    /**
+     * Fragment the generated and enhanced LDBC SNB datasets.
+     */
+    async fragmentSnbDataset() {
+        const oldCwd = process.cwd();
+        process.chdir(this.cwd);
+        // Initial fragmentation
+        await (0, rdf_dataset_fragmenter_1.runConfig)(this.fragmentConfig, { mainModulePath: this.mainModulePath });
+        process.chdir(oldCwd);
+    }
+    /**
+     * Instantiate queries based on the LDBC SNB datasets.
+     */
+    async instantiateQueries() {
+        // Create target directory
+        await (0, promises_1.mkdir)((0, node_path_1.join)(this.cwd, 'out-queries'));
+        // Run instantiator
+        const oldCwd = process.cwd();
+        process.chdir(this.cwd);
+        await (0, sparql_query_parameter_instantiator_1.runConfig)(this.queryConfig, { mainModulePath: this.mainModulePath }, {
+            variables: await this.generateVariables(),
+        });
+        process.chdir(oldCwd);
+    }
+    /**
+     * Download validation parameters
+     */
+    async downloadValidationParams() {
+        // Create target directory
+        const target = (0, node_path_1.join)(this.cwd, 'out-validate-params');
+        await (0, promises_1.mkdir)(target);
+        // Download and extract zip file
+        return new Promise((resolve, reject) => {
+            (0, node_https_1.request)(this.validationParams, (res) => {
+                res
+                    .on('error', reject)
+                    .pipe((0, unzipper_1.Extract)({ path: target }))
+                    .on('error', reject)
+                    .on('close', resolve);
+            }).end();
+        });
+    }
+    /**
+     * Generate validation queries and results.
+     */
+    async generateValidation() {
+        // Create target directory
+        await (0, promises_1.mkdir)((0, node_path_1.join)(this.cwd, 'out-validate'));
+        // Run generator
+        const oldCwd = process.cwd();
+        process.chdir(this.cwd);
+        await (0, ldbc_snb_validation_generator_1.runConfig)(this.validationConfig, { mainModulePath: this.mainModulePath }, {
+            variables: await this.generateVariables(),
+        });
+        process.chdir(oldCwd);
+    }
+    async generateVariables() {
+        const templateMappings = (await (0, promises_1.readdir)((0, node_path_1.join)(__dirname, '../templates/queries/')))
+            .map(name => [`urn:variables:query-templates:${name}`, (0, node_path_1.join)(__dirname, `../templates/queries/${name}`)]);
+        const refinementMappings = (await (0, promises_1.readdir)((0, node_path_1.join)(__dirname, '../templates/refinements/')))
+            .map(name => [`urn:variables:query-refinements:${name}`, (0, node_path_1.join)(__dirname, `../templates/refinements/${name}`)]);
+        const result = Object.fromEntries([...templateMappings, ...refinementMappings]);
+        return result;
+    }
+    /**
+     * Return a string in a given color
+     * @param str The string that should be printed in
+     * @param color A given color
+     */
+    static withColor(str, color) {
+        return `${color}${str}${Generator.COLOR_RESET}`;
+    }
+}
+exports.Generator = Generator;
+Generator.COLOR_RESET = '\u001B[0m';
+Generator.COLOR_RED = '\u001B[31m';
+Generator.COLOR_GREEN = '\u001B[32m';
+Generator.COLOR_YELLOW = '\u001B[33m';
+Generator.COLOR_BLUE = '\u001B[34m';
+Generator.COLOR_MAGENTA = '\u001B[35m';
+Generator.COLOR_CYAN = '\u001B[36m';
+Generator.COLOR_GRAY = '\u001B[90m';
+Generator.LDBC_SNB_DATAGEN_DOCKER_IMAGE = 'rubensworks/ldbc_snb_datagen:latest';
+//# sourceMappingURL=Generator.js.map

package/lib/Generator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Generator.js","sourceRoot":"","sources":["Generator.ts"],"names":[],"mappings":";;;;;;AAAA,+CAAqF;AACrF,2CAAqC;AACrC,yCAAiC;AACjC,0DAAkC;AAClC,yDAA6D;AAC7D,iFAAoF;AACpF,mEAAoE;AACpE,6FAAwF;AACxF,uCAAmC;AAEnC;;GAEG;AACH,MAAa,SAAS;IAuBpB,YAAmB,IAAuB;QACxC,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC;QACpB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QAChC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC;QAChD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC;QACpC,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,CAAC;QAC9C,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,CAAC;QAC9C,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QACtC,IAAI,CAAC,cAAc,GAAG,IAAA,gBAAI,EAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAC9C,CAAC;IAES,KAAK,CAAC,YAAY,CAAC,IAAY;QACvC,IAAI,CAAC;YACH,MAAM,IAAA,eAAI,EAAC,IAAI,CAAC,CAAC;YACjB,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAES,GAAG,CAAC,KAAa,EAAE,MAAc;QACzC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,SAAS,CAAC,SAAS,CAAC,IAAI,KAAK,GAAG,EAAE,SAAS,CAAC,UAAU,CAAC,IAAI,MAAM,IAAI,CAAC,CAAC;IACjG,CAAC;IAES,KAAK,CAAC,QAAQ,CAAC,IAAY,EAAE,SAAiB,EAAE,MAA2B;QACnF,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,IAAA,gBAAI,EAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC;YAC1E,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;YAC1B,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;YACnC,MAAM,MAAM,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAC1C,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,WAAW,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;QACjF,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,aAAa,SAAS,wCAAwC,CAAC,CAAC;QACjF,CAAC;IACH,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,QAAQ;QACnB,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;QACnC,MAAM,IAAI,CAAC,QAAQ,CAAC,uBAAuB,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC,CAAC;QACzF,MAAM,IAAI,CAAC,QAAQ,CAAC,sBAAsB,EAAE,cAAc,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC;QAC5F,MAAM,IAAI,CAAC,QAAQ,CAAC,wBAAwB,EAAE,eAAe,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC,CAAC;QAChG,MAAM,IAAI,CAAC,QAAQ,CAAC,2BAA2B,EAAE,aAAa,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC,CAAC;QACjG,IAAI,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACnD,MAAM,IAAI,CAAC,QAAQ,CAAC,2BAA2B,EAAE,qBAAqB,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,wBAAwB,EAAE,CAAC,CAAC;YAC/G,MAAM,IAAI,CAAC,QAAQ,CAAC,0BAA0B,EAAE,cAAc,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC,CAAC;QACnG,CAAC;QACD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC1C,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,WAAW,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;IAClF,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,kBAAkB;QAC7B,yBAAyB;QACzB,MAAM,cAAc,GAAG,MAAM,IAAA,mBAAQ,EAAC,IAAA,gBAAI,EAAC,SAAS,EAAE,yBAAyB,CAAC,EAAE,MAAM,CAAC,CAAC;QAC1F,MAAM,UAAU,GAAG,IAAA,gBAAI,EAAC,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QAChD,MAAM,IAAA,oBAAS,EAAC,UAAU,EAAE,cAAc,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,CAAC;QAEpF,6BAA6B;QAC7B,MAAM,SAAS,GAAG,IAAI,mBAAS,EAAE,CAAC;QAClC,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,6BAA6B,CAAC,CAAC;QAClF,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACpC,SAAS,CAAC,KAAK,CAAC,cAAc,CAAC,WAAW,EAAE,CAAC,GAAiB,EAAE,GAAU,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;QACnH,CAAC,CAAC,CAAC;QAEH,yBAAyB;QACzB,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC;YAChD,KAAK,EAAE,SAAS,CAAC,6BAA6B;YAC9C,GAAG,EAAE,IAAI;YACT,YAAY,EAAE,IAAI;YAClB,YAAY,EAAE,IAAI;YAClB,GAAG,EAAE,CAAE,0BAA0B,IAAI,CAAC,YAAY,EAAE,CAAE;YACtD,UAAU,EAAE;gBACV,KAAK,EAAE;oBACL,GAAG,IAAI,CAAC,GAAG,qCAAqC;oBAChD,GAAG,UAAU,mCAAmC;iBACjD;aACF;SACF,CAAC,CAAC;QACH,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;QAExB,6BAA6B;QAC7B,IAAI,cAAc,GAAG,KAAK,CAAC;QAC3B,kDAAkD;QAClD,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,KAAK,IAAG,EAAE;YAC7B,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;gBACvB,MAAM,OAAO,EAAE,CAAC;YAClB,CAAC;QACH,CAAC,CAAC,CAAC;QACH,KAAK,UAAU,OAAO;YACpB,MAAM,SAAS,CAAC,MAAM,EAAE,CAAC;YACzB,MAAM,IAAA,iBAAM,EAAC,UAAU,CAAC,CAAC;QAC3B,CAAC;QAED,0BAA0B;QAC1B,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC;YACjC,MAAM,EAAE,IAAI;YACZ,MAAM,EAAE,IAAI;YACZ,MAAM,EAAE,IAAI;SACb,CAAC,CAAC;QACH,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC3B,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,MAAM,EAAE,CAAC;QACf,CAAC;QAED,6BAA6B;QAC7B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACpC,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YACvB,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QACH,cAAc,GAAG,IAAI,CAAC;QAEtB,UAAU;QACV,MAAM,OAAO,EAAE,CAAC;IAClB,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,iBAAiB;QAC5B,0BAA0B;QAC1B,MAAM,IAAA,gBAAK,EAAC,IAAA,gBAAI,EAAC,IAAI,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC,CAAC;QAE5C,eAAe;QACf,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QAC7B,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxB,MAAM,IAAA,6BAAW,EAAC,IAAI,CAAC,iBAAiB,EAAE,EAAE,cAAc,EAAE,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC;QACnF,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACxB,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,kBAAkB;QAC7B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QAC7B,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAExB,wBAAwB;QACxB,MAAM,IAAA,kCAAa,EAAC,IAAI,CAAC,cAAc,EAAE,EAAE,cAAc,EAAE,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC;QAElF,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACxB,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,kBAAkB;QAC7B,0BAA0B;QAC1B,MAAM,IAAA,gBAAK,EAAC,IAAA,gBAAI,EAAC,IAAI,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC,CAAC;QAE3C,mBAAmB;QACnB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QAC7B,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxB,MAAM,IAAA,+CAAoB,EAAC,IAAI,CAAC,WAAW,EAAE,EAAE,cAAc,EAAE,IAAI,CAAC,cAAc,EAAE,EAAE;YACpF,SAAS,EAAE,MAAM,IAAI,CAAC,iBAAiB,EAAE;SAC1C,CAAC,CAAC;QACH,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACxB,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,wBAAwB;QACnC,0BAA0B;QAC1B,MAAM,MAAM,GAAG,IAAA,gBAAI,EAAC,IAAI,CAAC,GAAG,EAAE,qBAAqB,CAAC,CAAC;QACrD,MAAM,IAAA,gBAAK,EAAC,MAAM,CAAC,CAAC;QAEpB,gCAAgC;QAChC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,IAAA,oBAAO,EAAC,IAAI,CAAC,gBAAiB,EAAE,CAAC,GAAG,EAAE,EAAE;gBACtC,GAAG;qBACA,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC;qBACnB,IAAI,CAAC,IAAA,kBAAO,EAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;qBAC/B,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC;qBACnB,EAAE,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC1B,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QACX,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,kBAAkB;QAC7B,0BAA0B;QAC1B,MAAM,IAAA,gBAAK,EAAC,IAAA,gBAAI,EAAC,IAAI,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC,CAAC;QAE5C,gBAAgB;QAChB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QAC7B,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxB,MAAM,IAAA,yCAAsB,EAAC,IAAI,CAAC,gBAAiB,EAAE,EAAE,cAAc,EAAE,IAAI,CAAC,cAAc,EAAE,EAAE;YAC5F,SAAS,EAAE,MAAM,IAAI,CAAC,iBAAiB,EAAE;SAC1C,CAAC,CAAC;QACH,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACxB,CAAC;IAES,KAAK,CAAC,iBAAiB;QAC/B,MAAM,gBAAgB,GAAe,CAAC,MAAM,IAAA,kBAAO,EAAC,IAAA,gBAAI,EAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC,CAAC;aAC3F,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAE,iCAAiC,IAAI,EAAE,EAAE,IAAA,gBAAI,EAAC,SAAS,EAAE,wBAAwB,IAAI,EAAE,CAAC,CAAE,CAAC,CAAC;QAC7G,MAAM,kBAAkB,GAAe,CAAC,MAAM,IAAA,kBAAO,EAAC,IAAA,gBAAI,EAAC,SAAS,EAAE,2BAA2B,CAAC,CAAC,CAAC;aACjG,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAE,mCAAmC,IAAI,EAAE,EAAE,IAAA,gBAAI,EAAC,SAAS,EAAE,4BAA4B,IAAI,EAAE,CAAC,CAAE,CAAC,CAAC;QACnH,MAAM,MAAM,GAA4B,MAAM,CAAC,WAAW,CAAC,CAAE,GAAG,gBAAgB,EAAE,GAAG,kBAAkB,CAAE,CAAC,CAAC;QAC3G,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;OAIG;IACI,MAAM,CAAC,SAAS,CAAC,GAAQ,EAAE,KAAa;QAC7C,OAAO,GAAG,KAAK,GAAG,GAAG,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC;IAClD,CAAC;;AAnPH,8BAoPC;AAnPwB,qBAAW,GAAW,WAAW,CAAC;AAClC,mBAAS,GAAW,YAAY,CAAC;AACjC,qBAAW,GAAW,YAAY,CAAC;AACnC,sBAAY,GAAW,YAAY,CAAC;AACpC,oBAAU,GAAW,YAAY,CAAC;AAClC,uBAAa,GAAW,YAAY,CAAC;AACrC,oBAAU,GAAW,YAAY,CAAC;AAClC,oBAAU,GAAW,YAAY,CAAC;AAClC,uCAA6B,GAAW,qCAAqC,CAAC","sourcesContent":["import { stat, unlink, mkdir, readdir, readFile, writeFile } from 'node:fs/promises';\nimport { request } from 'node:https';\nimport { join } from 'node:path';\nimport Dockerode from 'dockerode';\nimport { runConfig as runEnhancer } from 'ldbc-snb-enhancer';\nimport { runConfig as runValidationGenerator } from 'ldbc-snb-validation-generator';\nimport { runConfig as runFragmenter } from 'rdf-dataset-fragmenter';\nimport { runConfig as runQueryInstantiator } from 'sparql-query-parameter-instantiator';\nimport { Extract } from 'unzipper';\n\n/**\n * Generates decentralized social network data in different phases.\n */\nexport class Generator {\n  public static readonly COLOR_RESET: string = '\\u001B[0m';\n  public static readonly COLOR_RED: string = '\\u001B[31m';\n  public static readonly COLOR_GREEN: string = '\\u001B[32m';\n  public static readonly COLOR_YELLOW: string = '\\u001B[33m';\n  public static readonly COLOR_BLUE: string = '\\u001B[34m';\n  public static readonly COLOR_MAGENTA: string = '\\u001B[35m';\n  public static readonly COLOR_CYAN: string = '\\u001B[36m';\n  public static readonly COLOR_GRAY: string = '\\u001B[90m';\n  public static readonly LDBC_SNB_DATAGEN_DOCKER_IMAGE: string = 'rubensworks/ldbc_snb_datagen:latest';\n\n  private readonly cwd: string;\n  private readonly verbose: boolean;\n  private readonly overwrite: boolean;\n  private readonly scale: string;\n  private readonly enhancementConfig: string;\n  private readonly fragmentConfig: string;\n  private readonly queryConfig: string;\n  private readonly validationParams: string | undefined;\n  private readonly validationConfig: string | undefined;\n  private readonly hadoopMemory: string;\n  private readonly mainModulePath: string;\n\n  public constructor(opts: IGeneratorOptions) {\n    this.cwd = opts.cwd;\n    this.verbose = opts.verbose;\n    this.overwrite = opts.overwrite;\n    this.scale = opts.scale;\n    this.enhancementConfig = opts.enhancementConfig;\n    this.fragmentConfig = opts.fragmentConfig;\n    this.queryConfig = opts.queryConfig;\n    this.validationParams = opts.validationParams;\n    this.validationConfig = opts.validationConfig;\n    this.hadoopMemory = opts.hadoopMemory;\n    this.mainModulePath = join(__dirname, '..');\n  }\n\n  protected async targetExists(path: string): Promise<boolean> {\n    try {\n      await stat(path);\n      return true;\n    } catch {\n      return false;\n    }\n  }\n\n  protected log(phase: string, status: string): void {\n    process.stdout.write(`${Generator.withColor(`[${phase}]`, Generator.COLOR_CYAN)} ${status}\\n`);\n  }\n\n  protected async runPhase(name: string, directory: string, runner: () => Promise<void>): Promise<void> {\n    if (this.overwrite || !await this.targetExists(join(this.cwd, directory))) {\n      this.log(name, 'Started');\n      const timeStart = process.hrtime();\n      await runner();\n      const timeEnd = process.hrtime(timeStart);\n      this.log(name, `Done in ${timeEnd[0] + (timeEnd[1] / 1_000_000_000)} seconds`);\n    } else {\n      this.log(name, `Skipped (/${directory} already exists, remove to regenerate)`);\n    }\n  }\n\n  /**\n   * Run all generator phases.\n   */\n  public async generate(): Promise<void> {\n    const timeStart = process.hrtime();\n    await this.runPhase('SNB dataset generator', 'out-snb', () => this.generateSnbDataset());\n    await this.runPhase('SNB dataset enhancer', 'out-enhanced', () => this.enhanceSnbDataset());\n    await this.runPhase('SNB dataset fragmenter', 'out-fragments', () => this.fragmentSnbDataset());\n    await this.runPhase('SPARQL query instantiator', 'out-queries', () => this.instantiateQueries());\n    if (this.validationParams && this.validationConfig) {\n      await this.runPhase('SNB validation downloader', 'out-validate-params', () => this.downloadValidationParams());\n      await this.runPhase('SNB validation generator', 'out-validate', () => this.generateValidation());\n    }\n    const timeEnd = process.hrtime(timeStart);\n    this.log('All', `Done in ${timeEnd[0] + (timeEnd[1] / 1_000_000_000)} seconds`);\n  }\n\n  /**\n   * Invoke the LDBC SNB generator.\n   */\n  public async generateSnbDataset(): Promise<void> {\n    // Create params.ini file\n    const paramsTemplate = await readFile(join(__dirname, '../templates/params.ini'), 'utf8');\n    const paramsPath = join(this.cwd, 'params.ini');\n    await writeFile(paramsPath, paramsTemplate.replaceAll('SCALE', this.scale), 'utf8');\n\n    // Pull the base Docker image\n    const dockerode = new Dockerode();\n    const buildStream = await dockerode.pull(Generator.LDBC_SNB_DATAGEN_DOCKER_IMAGE);\n    await new Promise((resolve, reject) => {\n      dockerode.modem.followProgress(buildStream, (err: Error | null, res: any[]) => err ? reject(err) : resolve(res));\n    });\n\n    // Start Docker container\n    const container = await dockerode.createContainer({\n      Image: Generator.LDBC_SNB_DATAGEN_DOCKER_IMAGE,\n      Tty: true,\n      AttachStdout: true,\n      AttachStderr: true,\n      Env: [ `HADOOP_CLIENT_OPTS=-Xmx${this.hadoopMemory}` ],\n      HostConfig: {\n        Binds: [\n          `${this.cwd}/out-snb/:/opt/ldbc_snb_datagen/out`,\n          `${paramsPath}:/opt/ldbc_snb_datagen/params.ini`,\n        ],\n      },\n    });\n    await container.start();\n\n    // Stop process on force-exit\n    let containerEnded = false;\n    // eslint-disable-next-line ts/no-misused-promises\n    process.on('SIGINT', async() => {\n      if (!containerEnded) {\n        await container.kill();\n        await cleanup();\n      }\n    });\n    async function cleanup(): Promise<void> {\n      await container.remove();\n      await unlink(paramsPath);\n    }\n\n    // Attach output to stdout\n    const out = await container.attach({\n      stream: true,\n      stdout: true,\n      stderr: true,\n    });\n    if (this.verbose) {\n      out.pipe(process.stdout);\n    } else {\n      out.resume();\n    }\n\n    // Wait until generation ends\n    await new Promise((resolve, reject) => {\n      out.on('end', resolve);\n      out.on('error', reject);\n    });\n    containerEnded = true;\n\n    // Cleanup\n    await cleanup();\n  }\n\n  /**\n   * Enhance the generated LDBC SNB dataset.\n   */\n  public async enhanceSnbDataset(): Promise<void> {\n    // Create target directory\n    await mkdir(join(this.cwd, 'out-enhanced'));\n\n    // Run enhancer\n    const oldCwd = process.cwd();\n    process.chdir(this.cwd);\n    await runEnhancer(this.enhancementConfig, { mainModulePath: this.mainModulePath });\n    process.chdir(oldCwd);\n  }\n\n  /**\n   * Fragment the generated and enhanced LDBC SNB datasets.\n   */\n  public async fragmentSnbDataset(): Promise<void> {\n    const oldCwd = process.cwd();\n    process.chdir(this.cwd);\n\n    // Initial fragmentation\n    await runFragmenter(this.fragmentConfig, { mainModulePath: this.mainModulePath });\n\n    process.chdir(oldCwd);\n  }\n\n  /**\n   * Instantiate queries based on the LDBC SNB datasets.\n   */\n  public async instantiateQueries(): Promise<void> {\n    // Create target directory\n    await mkdir(join(this.cwd, 'out-queries'));\n\n    // Run instantiator\n    const oldCwd = process.cwd();\n    process.chdir(this.cwd);\n    await runQueryInstantiator(this.queryConfig, { mainModulePath: this.mainModulePath }, {\n      variables: await this.generateVariables(),\n    });\n    process.chdir(oldCwd);\n  }\n\n  /**\n   * Download validation parameters\n   */\n  public async downloadValidationParams(): Promise<void> {\n    // Create target directory\n    const target = join(this.cwd, 'out-validate-params');\n    await mkdir(target);\n\n    // Download and extract zip file\n    return new Promise((resolve, reject) => {\n      request(this.validationParams!, (res) => {\n        res\n          .on('error', reject)\n          .pipe(Extract({ path: target }))\n          .on('error', reject)\n          .on('close', resolve);\n      }).end();\n    });\n  }\n\n  /**\n   * Generate validation queries and results.\n   */\n  public async generateValidation(): Promise<void> {\n    // Create target directory\n    await mkdir(join(this.cwd, 'out-validate'));\n\n    // Run generator\n    const oldCwd = process.cwd();\n    process.chdir(this.cwd);\n    await runValidationGenerator(this.validationConfig!, { mainModulePath: this.mainModulePath }, {\n      variables: await this.generateVariables(),\n    });\n    process.chdir(oldCwd);\n  }\n\n  protected async generateVariables(): Promise<Record<string, string>> {\n    const templateMappings: string[][] = (await readdir(join(__dirname, '../templates/queries/')))\n      .map(name => [ `urn:variables:query-templates:${name}`, join(__dirname, `../templates/queries/${name}`) ]);\n    const refinementMappings: string[][] = (await readdir(join(__dirname, '../templates/refinements/')))\n      .map(name => [ `urn:variables:query-refinements:${name}`, join(__dirname, `../templates/refinements/${name}`) ]);\n    const result = <Record<string, string>> Object.fromEntries([ ...templateMappings, ...refinementMappings ]);\n    return result;\n  }\n\n  /**\n   * Return a string in a given color\n   * @param str The string that should be printed in\n   * @param color A given color\n   */\n  public static withColor(str: any, color: string): string {\n    return `${color}${str}${Generator.COLOR_RESET}`;\n  }\n}\n\nexport interface IGeneratorOptions {\n  cwd: string;\n  verbose: boolean;\n  overwrite: boolean;\n  scale: string;\n  enhancementConfig: string;\n  fragmentConfig: string;\n  queryConfig: string;\n  validationParams?: string;\n  validationConfig?: string;\n  hadoopMemory: string;\n}\n"]}

package/lib/Server.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Serves generated fragments over HTTP.
+ */
+export declare class Server {
+    private readonly configPath;
+    private readonly port;
+    private readonly baseUrl;
+    private readonly rootFilePath;
+    private readonly logLevel;
+    constructor(options: IServerOptions);
+    serve(): Promise<void>;
+}
+export interface IServerOptions {
+    configPath: string;
+    port: number;
+    baseUrl: string | undefined;
+    rootFilePath: string;
+    logLevel: string;
+}