@lde/sparql-qlever 0.3.2

package/README.md

@@ -0,0 +1,3 @@
+# SPARQL QLever
+
+An adapter for the [QLever](https://github.com/ad-freiburg/qlever) SPARQL server.

package/dist/importer.d.ts

@@ -0,0 +1,31 @@
+import { Importer as ImporterInterface, ImportFailed, ImportSuccessful, NotSupported } from '@lde/sparql-importer';
+import { Dataset } from '@lde/dataset';
+import { Downloader } from '@lde/distribution-downloader';
+import { TaskRunner } from '@lde/task-runner';
+export interface Options {
+    taskRunner: TaskRunner<unknown>;
+    indexName?: string;
+    downloader?: Downloader;
+    qleverOptions?: {
+        'ascii-prefixes-only': boolean;
+        'num-triples-per-batch': number;
+    };
+    port?: number;
+}
+/**
+ * Import RDF to a QLever SPARQL server.
+ *
+ * - Use the QLever CLI because the Graph Store Protocol is not parallelized.
+ */
+export declare class Importer implements ImporterInterface {
+    private indexName;
+    private taskRunner;
+    private downloader;
+    private qleverOptions;
+    constructor({ taskRunner, downloader, indexName, qleverOptions }: Options);
+    import(dataset: Dataset): Promise<NotSupported | ImportSuccessful | ImportFailed>;
+    private doImport;
+    private fileFormatFromMimeType;
+    private index;
+}
+//# sourceMappingURL=importer.d.ts.map

package/dist/importer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,IAAI,iBAAiB,EAC7B,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACb,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,UAAU,EAA0B,MAAM,8BAA8B,CAAC;AAGlF,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,MAAM,WAAW,OAAO;IACtB,UAAU,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,aAAa,CAAC,EAAE;QACd,qBAAqB,EAAE,OAAO,CAAC;QAC/B,uBAAuB,EAAE,MAAM,CAAC;KACjC,CAAC;IACF,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;;;GAIG;AACH,qBAAa,QAAS,YAAW,iBAAiB;IAChD,OAAO,CAAC,SAAS,CAAC;IAClB,OAAO,CAAC,UAAU,CAAsB;IACxC,OAAO,CAAC,UAAU,CAAC;IACnB,OAAO,CAAC,aAAa,CAAC;gBAEV,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,EAAE,OAAO;IAU5D,MAAM,CACjB,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,YAAY,GAAG,gBAAgB,GAAG,YAAY,CAAC;YA2B5C,QAAQ;IAYtB,OAAO,CAAC,sBAAsB;YAgBhB,KAAK;CAmBpB"}

package/dist/importer.js

@@ -0,0 +1,78 @@
+import { ImportFailed, ImportSuccessful, NotSupported, } from '@lde/sparql-importer';
+import { LastModifiedDownloader } from '@lde/distribution-downloader';
+import { basename, dirname } from 'path';
+import { writeFile } from 'node:fs/promises';
+/**
+ * Import RDF to a QLever SPARQL server.
+ *
+ * - Use the QLever CLI because the Graph Store Protocol is not parallelized.
+ */
+export class Importer {
+    indexName;
+    taskRunner;
+    downloader;
+    qleverOptions;
+    constructor({ taskRunner, downloader, indexName, qleverOptions }) {
+        this.taskRunner = taskRunner;
+        this.downloader = downloader ?? new LastModifiedDownloader();
+        this.indexName = indexName ?? 'data';
+        this.qleverOptions = qleverOptions ?? {
+            'ascii-prefixes-only': true,
+            'num-triples-per-batch': 100000,
+        };
+    }
+    async import(dataset) {
+        const downloadDistributions = dataset.getDownloadDistributions();
+        if (downloadDistributions.length === 0) {
+            return new NotSupported();
+        }
+        let result;
+        for (const downloadDistribution of downloadDistributions) {
+            try {
+                result = await this.doImport(downloadDistribution);
+                if (result instanceof ImportSuccessful) {
+                    return result;
+                }
+            }
+            catch (error) {
+                let errorMessage;
+                if (error instanceof AggregateError) {
+                    errorMessage = error.errors.join(' / ');
+                }
+                else {
+                    errorMessage = error.message;
+                }
+                result = new ImportFailed(downloadDistribution, errorMessage);
+            }
+        }
+        return result;
+    }
+    async doImport(distribution) {
+        const localFile = await this.downloader.download(distribution);
+        await this.index(localFile, this.fileFormatFromMimeType(distribution.mimeType));
+        return new ImportSuccessful(distribution);
+    }
+    fileFormatFromMimeType(mimeType) {
+        switch (mimeType) {
+            case 'application/n-triples':
+            case 'application/n-triples+gzip':
+                return 'nt';
+            case 'application/n-quads':
+            case 'application/n-quads+gzip':
+                return 'nq';
+            case 'text/turtle':
+            case 'text/turtle+gzip':
+                return 'ttl';
+            default:
+                throw new Error(`Unsupported media type: ${mimeType}`);
+        }
+    }
+    async index(file, format) {
+        const workingDir = dirname(file);
+        const settingsFile = 'index.settings.json';
+        await writeFile(`${workingDir}/${settingsFile}`, JSON.stringify(this.qleverOptions));
+        // TODO: write index to named volume instead of bind mount for better performance.
+        const indexTask = await this.taskRunner.run(`(zcat '${basename(file)}' 2>/dev/null || cat '${basename(file)}') | IndexBuilderMain -i ${this.indexName} -s ${settingsFile} -F ${format} -f -`);
+        await this.taskRunner.wait(indexTask);
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+export * from './importer.js';
+export * from './server.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAC;AAC9B,cAAc,aAAa,CAAC"}

package/dist/index.js

@@ -0,0 +1,2 @@
+export * from './importer.js';
+export * from './server.js';

package/dist/server.d.ts

@@ -0,0 +1,18 @@
+import { SparqlServer } from '@lde/sparql-server';
+import { TaskRunner } from '@lde/task-runner';
+export declare class Server<Task> implements SparqlServer {
+    private taskRunner;
+    private readonly indexName;
+    private task?;
+    private readonly port;
+    constructor({ taskRunner, indexName, port }: Arguments<Task>);
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    get queryEndpoint(): URL;
+}
+export interface Arguments<Task> {
+    taskRunner: TaskRunner<Task>;
+    indexName: string;
+    port?: number;
+}
+//# sourceMappingURL=server.d.ts.map

package/dist/server.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,qBAAa,MAAM,CAAC,IAAI,CAAE,YAAW,YAAY;IAC/C,OAAO,CAAC,UAAU,CAAmB;IACrC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,IAAI,CAAC,CAAO;IACpB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;gBAElB,EAAE,UAAU,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,SAAS,CAAC,IAAI,CAAC;IAM/C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAQtB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAOlC,IAAW,aAAa,IAAI,GAAG,CAE9B;CACF;AAED,MAAM,WAAW,SAAS,CAAC,IAAI;IAC7B,UAAU,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf"}

package/dist/server.js

@@ -0,0 +1,24 @@
+export class Server {
+    taskRunner;
+    indexName;
+    task;
+    port;
+    constructor({ taskRunner, indexName, port }) {
+        this.taskRunner = taskRunner;
+        this.indexName = indexName;
+        this.port = port ?? 7001;
+    }
+    async start() {
+        // TODO prevent double starts.
+        this.task = await this.taskRunner.run(`ServerMain --index-basename ${this.indexName} --memory-max-size 6G --port ${this.port}`);
+    }
+    async stop() {
+        if (this.task) {
+            await this.taskRunner.stop(this.task);
+            this.task = undefined;
+        }
+    }
+    get queryEndpoint() {
+        return new URL(`http://localhost:${this.port}/sparql`);
+    }
+}

package/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "@lde/sparql-qlever",
+  "version": "0.3.2",
+  "type": "module",
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "development": "./src/index.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "!**/*.tsbuildinfo"
+  ],
+  "dependencies": {
+    "tslib": "^2.3.0",
+    "@lde/sparql-importer": "0.0.5",
+    "@lde/dataset": "0.3.1",
+    "@lde/distribution-downloader": "0.2.3",
+    "@lde/task-runner": "0.0.4",
+    "@lde/sparql-server": "0.2.1"
+  }
+}