@lde/sparql-qlever 0.10.2 → 0.12.0

package/dist/createQlever.d.ts

@@ -1,5 +1,5 @@
 import { Downloader } from '@lde/distribution-downloader';
-import { Importer } from './importer.js';
+import { Importer, QleverIndexOptions } from './importer.js';
 import { Server } from './server.js';
 export type QleverOptions = {
     /** Directory where downloaded data files are stored. */
@@ -8,8 +8,9 @@ export type QleverOptions = {
     /** @default 7001 */
     port?: number;
     downloader?: Downloader;
-    /** Cache QLever indices and skip re-indexing when source data is unchanged. Defaults to `true`. */
+    /** Cache QLever indices and skip re-indexing when source data is unchanged. @default true */
     cacheIndex?: boolean;
+    qleverOptions?: QleverIndexOptions;
 } & ({
     mode: 'docker';
     image: string;
@@ -17,6 +18,12 @@ export type QleverOptions = {
 } | {
     mode: 'native';
 });
+/**
+ * Create a paired QLever {@link Importer} and {@link Server} that share a
+ * single {@link TaskRunner}. In pipeline setups the importer and server must
+ * use the same runner (and therefore the same Docker container or working
+ * directory) so that the server can serve the index the importer built.
+ */
 export declare function createQlever(options: QleverOptions): {
     importer: Importer;
     server: Server<unknown>;

package/dist/createQlever.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"createQlever.d.ts","sourceRoot":"","sources":["../src/createQlever.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,UAAU,EAEX,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,MAAM,MAAM,aAAa,GAAG;IAC1B,wDAAwD;IACxD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,oBAAoB;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,mGAAmG;IACnG,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB,GAAG,CACA;IACE,IAAI,EAAE,QAAQ,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,GACD;IAAE,IAAI,EAAE,QAAQ,CAAA;CAAE,CACrB,CAAC;AAEF,wBAAgB,YAAY,CAAC,OAAO,EAAE,aAAa;;;EA0BlD"}
+{"version":3,"file":"createQlever.d.ts","sourceRoot":"","sources":["../src/createQlever.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,UAAU,EAEX,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAC7D,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,MAAM,MAAM,aAAa,GAAG;IAC1B,wDAAwD;IACxD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,oBAAoB;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,6FAA6F;IAC7F,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,aAAa,CAAC,EAAE,kBAAkB,CAAC;CACpC,GAAG,CACA;IACE,IAAI,EAAE,QAAQ,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,GACD;IAAE,IAAI,EAAE,QAAQ,CAAA;CAAE,CACrB,CAAC;AAEF;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,aAAa;;;EA2BlD"}

package/dist/createQlever.js

@@ -3,6 +3,12 @@ import { NativeTaskRunner } from '@lde/task-runner-native';
 import { LastModifiedDownloader, } from '@lde/distribution-downloader';
 import { Importer } from './importer.js';
 import { Server } from './server.js';
+/**
+ * Create a paired QLever {@link Importer} and {@link Server} that share a
+ * single {@link TaskRunner}. In pipeline setups the importer and server must
+ * use the same runner (and therefore the same Docker container or working
+ * directory) so that the server can serve the index the importer built.
+ */
 export function createQlever(options) {
     const port = options.port ?? 7001;
     const taskRunner = options.mode === 'docker'
@@ -19,6 +25,7 @@ export function createQlever(options) {
             indexName: options.indexName,
             downloader: options.downloader ?? new LastModifiedDownloader(options.dataDir),
             cacheIndex: options.cacheIndex,
+            qleverOptions: options.qleverOptions,
         }),
         server: new Server({
             taskRunner,

package/dist/importer.d.ts

@@ -1,34 +1,27 @@
-import { Importer as ImporterInterface, ImportFailed, ImportSuccessful, NotSupported } from '@lde/sparql-importer';
+import { Importer as ImporterInterface, ImporterOptions, ImportFailed, ImportSuccessful, NotSupported } from '@lde/sparql-importer';
 import { Distribution } from '@lde/dataset';
-import { Downloader } from '@lde/distribution-downloader';
-import { TaskRunner } from '@lde/task-runner';
-export interface Options {
-    taskRunner: TaskRunner<unknown>;
+export interface QleverIndexOptions {
+    /** @default true */
+    'ascii-prefixes-only'?: boolean;
+    /** @default 3_000_000 */
+    'num-triples-per-batch'?: number;
+    /** Memory budget for sorting during the index build. @default '10G' */
+    'stxxl-memory'?: string;
+    /** @default true */
+    'parse-parallel'?: boolean;
+    /** Build only PSO and POS permutations. Faster, but queries with predicate variables won't work. Also disables pattern precomputation. @default false */
+    'only-pso-and-pos-permutations'?: boolean;
+}
+export interface QleverImporterOptions extends ImporterOptions {
+    /** @default 'data' */
     indexName?: string;
-    downloader?: Downloader;
-    qleverOptions?: {
-        'ascii-prefixes-only': boolean;
-        'num-triples-per-batch': number;
-    };
-    port?: number;
-    /** Cache QLever indices and skip re-indexing when source data is unchanged. Defaults to `true`. */
-    cacheIndex?: boolean;
+    qleverOptions?: QleverIndexOptions;
 }
-/**
- * Import RDF to a QLever SPARQL server.
- *
- * - Use the QLever CLI because the Graph Store Protocol is not parallelized.
- */
 export declare class Importer implements ImporterInterface {
-    private indexName;
-    private taskRunner;
-    private downloader;
-    private qleverOptions;
-    private cacheIndex;
-    constructor({ taskRunner, downloader, indexName, qleverOptions, cacheIndex, }: Options);
+    private readonly options;
+    constructor(options: QleverImporterOptions);
     import(distributions: Distribution[]): Promise<NotSupported | ImportSuccessful | ImportFailed>;
     private doImport;
-    private fileFormatFromMimeType;
     private parseTripleCount;
     private cacheInfoPath;
     /**

package/dist/importer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,IAAI,iBAAiB,EAC7B,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACb,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EACL,UAAU,EAEX,MAAM,8BAA8B,CAAC;AAGtC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAU9C,MAAM,WAAW,OAAO;IACtB,UAAU,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,aAAa,CAAC,EAAE;QACd,qBAAqB,EAAE,OAAO,CAAC;QAC/B,uBAAuB,EAAE,MAAM,CAAC;KACjC,CAAC;IACF,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,mGAAmG;IACnG,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAMD;;;;GAIG;AACH,qBAAa,QAAS,YAAW,iBAAiB;IAChD,OAAO,CAAC,SAAS,CAAC;IAClB,OAAO,CAAC,UAAU,CAAsB;IACxC,OAAO,CAAC,UAAU,CAAC;IACnB,OAAO,CAAC,aAAa,CAAC;IACtB,OAAO,CAAC,UAAU,CAAC;gBAEP,EACV,UAAU,EACV,UAAU,EACV,SAAS,EACT,aAAa,EACb,UAAU,GACX,EAAE,OAAO;IAWG,MAAM,CACjB,aAAa,EAAE,YAAY,EAAE,GAC5B,OAAO,CAAC,YAAY,GAAG,gBAAgB,GAAG,YAAY,CAAC;YA+B5C,QAAQ;IAkCtB,OAAO,CAAC,sBAAsB;IAQ9B,OAAO,CAAC,gBAAgB;IAQxB,OAAO,CAAC,aAAa;IAIrB;;OAEG;YACW,eAAe;IA0B7B,yDAAyD;YAC3C,eAAe;YAaf,cAAc;YAKd,KAAK;CAmBpB"}
+{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,IAAI,iBAAiB,EAC7B,eAAe,EACf,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACb,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAK5C,MAAM,WAAW,kBAAkB;IACjC,oBAAoB;IACpB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,yBAAyB;IACzB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,uEAAuE;IACvE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,oBAAoB;IACpB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,yJAAyJ;IACzJ,+BAA+B,CAAC,EAAE,OAAO,CAAC;CAC3C;AAED,MAAM,WAAW,qBAAsB,SAAQ,eAAe;IAC5D,sBAAsB;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,kBAAkB,CAAC;CACpC;AAWD,qBAAa,QAAS,YAAW,iBAAiB;IAChD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkB;gBAE9B,OAAO,EAAE,qBAAqB;IAa7B,MAAM,CACjB,aAAa,EAAE,YAAY,EAAE,GAC5B,OAAO,CAAC,YAAY,GAAG,gBAAgB,GAAG,YAAY,CAAC;YA+B5C,QAAQ;IA4CtB,OAAO,CAAC,gBAAgB;IAQxB,OAAO,CAAC,aAAa;IAIrB;;OAEG;YACW,eAAe;IA0B7B,yDAAyD;YAC3C,eAAe;YAaf,cAAc;YAKd,KAAK;CA2CpB"}

package/dist/importer.js

@@ -1,32 +1,20 @@
 import { ImportFailed, ImportSuccessful, NotSupported, } from '@lde/sparql-importer';
-import { LastModifiedDownloader, } from '@lde/distribution-downloader';
+import { LastModifiedDownloader } from '@lde/distribution-downloader';
 import { basename, dirname, join } from 'path';
 import { readFile, stat, writeFile } from 'node:fs/promises';
-const supportedFormats = new Map([
-    ['application/n-triples', 'nt'],
-    ['application/n-quads', 'nq'],
-    ['text/turtle', 'ttl'],
-]);
-/**
- * Import RDF to a QLever SPARQL server.
- *
- * - Use the QLever CLI because the Graph Store Protocol is not parallelized.
- */
 export class Importer {
-    indexName;
-    taskRunner;
-    downloader;
-    qleverOptions;
-    cacheIndex;
-    constructor({ taskRunner, downloader, indexName, qleverOptions, cacheIndex, }) {
-        this.taskRunner = taskRunner;
-        this.downloader = downloader ?? new LastModifiedDownloader();
-        this.indexName = indexName ?? 'data';
-        this.qleverOptions = qleverOptions ?? {
-            'ascii-prefixes-only': true,
-            'num-triples-per-batch': 100000,
+    options;
+    constructor(options) {
+        this.options = {
+            ...options,
+            indexName: options.indexName ?? 'data',
+            downloader: options.downloader ?? new LastModifiedDownloader(),
+            cacheIndex: options.cacheIndex ?? true,
+            qleverOptions: {
+                ...defaultQleverIndexOptions,
+                ...options.qleverOptions,
+            },
         };
-        this.cacheIndex = cacheIndex ?? true;
     }
     async import(distributions) {
         const downloadDistributions = distributions.filter((distribution) => distribution.mimeType !== undefined &&
@@ -56,7 +44,7 @@ export class Importer {
         return result;
     }
     async doImport(distribution) {
-        const localFile = await this.downloader.download(distribution);
+        const localFile = await this.options.downloader.download(distribution);
         if (await this.isIndexUpToDate(localFile)) {
             const tripleCount = await this.readTripleCount(localFile);
             if (tripleCount === 0) {
@@ -64,7 +52,20 @@ export class Importer {
             }
             return new ImportSuccessful(distribution, undefined, tripleCount);
         }
-        const logs = await this.index(localFile, this.fileFormatFromMimeType(distribution.mimeType));
+        const format = fileFormatFromMimeType(distribution.mimeType);
+        let logs;
+        try {
+            logs = await this.index(localFile, format);
+        }
+        catch (error) {
+            if (format === 'ttl' &&
+                error.message?.includes('multiline string literal')) {
+                logs = await this.index(localFile, format, false);
+            }
+            else {
+                throw error;
+            }
+        }
         const tripleCount = this.parseTripleCount(logs);
         if (tripleCount === 0) {
             return new ImportFailed(distribution, 'Indexed 0 triples from distribution');
@@ -72,13 +73,6 @@ export class Importer {
         await this.writeCacheInfo(localFile);
         return new ImportSuccessful(distribution, undefined, tripleCount);
     }
-    fileFormatFromMimeType(mimeType) {
-        const format = supportedFormats.get(mimeType);
-        if (format === undefined) {
-            throw new Error(`Unsupported media type: ${mimeType}`);
-        }
-        return format;
-    }
     parseTripleCount(logs) {
         // Extract num-triples.normal from the metadata JSON that the index
         // command cats to stdout. Use a regex rather than JSON.parse because
@@ -87,13 +81,13 @@ export class Importer {
         return match ? Number(match[1]) : undefined;
     }
     cacheInfoPath(dataFile) {
-        return join(dirname(dataFile), `${this.indexName}.cache-info.json`);
+        return join(dirname(dataFile), `${this.options.indexName}.cache-info.json`);
     }
     /**
      * Check whether the cached index is still up to date.
      */
     async isIndexUpToDate(dataFile) {
-        if (!this.cacheIndex)
+        if (!this.options.cacheIndex)
             return false;
         let cacheInfo;
         try {
@@ -118,7 +112,7 @@ export class Importer {
     /** Read the triple count from QLever's metadata file. */
     async readTripleCount(dataFile) {
         try {
-            const metadataPath = join(dirname(dataFile), `${this.indexName}.meta-data.json`);
+            const metadataPath = join(dirname(dataFile), `${this.options.indexName}.meta-data.json`);
             const raw = await readFile(metadataPath, 'utf-8');
             return this.parseTripleCount(raw);
         }
@@ -130,12 +124,50 @@ export class Importer {
         const info = { sourceFile: basename(dataFile) };
         await writeFile(this.cacheInfoPath(dataFile), JSON.stringify(info));
     }
-    async index(file, format) {
+    async index(file, format, parseParallel) {
         const settingsFile = 'index.settings.json';
-        await writeFile(`${dirname(file)}/${settingsFile}`, JSON.stringify(this.qleverOptions));
+        const settings = {
+            'ascii-prefixes-only': this.options.qleverOptions['ascii-prefixes-only'],
+            'num-triples-per-batch': this.options.qleverOptions['num-triples-per-batch'],
+        };
+        await writeFile(`${dirname(file)}/${settingsFile}`, JSON.stringify(settings));
         // TODO: write index to named volume instead of bind mount for better performance.
-        const metadataFile = `${this.indexName}.meta-data.json`;
-        const indexTask = await this.taskRunner.run(`(gunzip -c '${basename(file)}' 2>/dev/null || cat '${basename(file)}') | qlever-index -i ${this.indexName} -s ${settingsFile} -F ${format} -p true -f - && cat ${metadataFile}`);
-        return await this.taskRunner.wait(indexTask);
+        const parallel = parseParallel ?? this.options.qleverOptions['parse-parallel'];
+        const flags = [
+            `-i ${this.options.indexName}`,
+            `-s ${settingsFile}`,
+            `-F ${format}`,
+            `-p true`,
+            `-m ${this.options.qleverOptions['stxxl-memory']}`,
+            parallel ? '' : '--parse-parallel false',
+            this.options.qleverOptions['only-pso-and-pos-permutations']
+                ? '-o --no-patterns'
+                : '',
+            '-f -',
+        ]
+            .filter(Boolean)
+            .join(' ');
+        const metadataFile = `${this.options.indexName}.meta-data.json`;
+        const indexTask = await this.options.taskRunner.run(`(gunzip -c '${basename(file)}' 2>/dev/null || cat '${basename(file)}') | qlever-index ${flags} && cat ${metadataFile}`);
+        return await this.options.taskRunner.wait(indexTask);
+    }
+}
+const supportedFormats = new Map([
+    ['application/n-triples', 'nt'],
+    ['application/n-quads', 'nq'],
+    ['text/turtle', 'ttl'],
+]);
+const defaultQleverIndexOptions = {
+    'ascii-prefixes-only': true,
+    'num-triples-per-batch': 3_000_000,
+    'stxxl-memory': '10G',
+    'parse-parallel': true,
+    'only-pso-and-pos-permutations': false,
+};
+function fileFormatFromMimeType(mimeType) {
+    const format = supportedFormats.get(mimeType);
+    if (format === undefined) {
+        throw new Error(`Unsupported media type: ${mimeType}`);
     }
+    return format;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/sparql-qlever",
-  "version": "0.10.2",
+  "version": "0.12.0",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/sparql-qlever"
@@ -25,7 +25,7 @@
   "dependencies": {
     "@lde/dataset": "0.7.2",
     "@lde/distribution-downloader": "0.5.3",
-    "@lde/sparql-importer": "0.4.2",
+    "@lde/sparql-importer": "0.5.0",
     "@lde/sparql-server": "0.4.10",
     "@lde/task-runner": "0.2.10",
     "@lde/task-runner-docker": "0.2.11",