@lde/sparql-qlever 0.12.2 → 0.14.0

package/README.md

@@ -9,3 +9,41 @@ Building a QLever index is slow. To avoid rebuilding it on every pipeline run, t
 Only **one** index is cached at a time. In a multi-dataset pipeline, each dataset overwrites the previous index. On re-run, the last-indexed dataset gets a cache hit while the others rebuild.
 
 Caching is enabled by default. Disable it by passing `cacheIndex: false` to `createQlever()` or the `Importer` constructor (e.g. driven by a `QLEVER_CACHE_INDEX=false` environment variable).
+
+## Configuration
+
+`createQlever()` accepts `indexOptions` and `serverOptions` to tune QLever's index builder and server respectively.
+
+### Server options (`serverOptions`)
+
+Passed to `qlever-server` at startup.
+
+| Option                  | Description                                      | Default |
+| ----------------------- | ------------------------------------------------ | ------- |
+| `memory-max-size`       | Maximum memory for query processing and caching. | `'4G'`  |
+| `default-query-timeout` | Default query timeout.                           | `'30s'` |
+
+Example:
+
+```ts
+const { importer, server } = createQlever({
+  mode: 'docker',
+  image: 'adfreiburg/qlever:latest',
+  serverOptions: {
+    'memory-max-size': '12G',
+    'default-query-timeout': '120s',
+  },
+});
+```
+
+### Index options (`indexOptions`)
+
+Passed to `qlever-index` during import.
+
+| Option                          | Description                                                                                   | Default     |
+| ------------------------------- | --------------------------------------------------------------------------------------------- | ----------- |
+| `ascii-prefixes-only`           | Enable faster parsing for well-behaved TTL files.                                             | `true`      |
+| `num-triples-per-batch`         | Triples per batch; lower values reduce memory usage.                                          | `3_000_000` |
+| `stxxl-memory`                  | Memory budget for sorting during the index build.                                             | `'10G'`     |
+| `parse-parallel`                | Parse input in parallel.                                                                      | `true`      |
+| `only-pso-and-pos-permutations` | Build only PSO and POS permutations. Faster, but queries with predicate variables won't work. | `false`     |

package/dist/createQlever.d.ts

@@ -1,6 +1,6 @@
 import { Downloader } from '@lde/distribution-downloader';
 import { Importer, QleverIndexOptions } from './importer.js';
-import { Server } from './server.js';
+import { QleverServerOptions, Server } from './server.js';
 export type QleverOptions = {
     /** Directory where downloaded data files are stored. */
     dataDir?: string;
@@ -10,9 +10,10 @@ export type QleverOptions = {
     downloader?: Downloader;
     /** Cache QLever indices and skip re-indexing when source data is unchanged. @default true */
     cacheIndex?: boolean;
-    /** QLever `--default-query-timeout` value (e.g. '30s', '5min'). @default '30s' */
-    queryTimeout?: string;
-    qleverOptions?: QleverIndexOptions;
+    /** Options for `qlever-index` (index building). */
+    indexOptions?: QleverIndexOptions;
+    /** Options for `qlever-server` (query processing). */
+    serverOptions?: QleverServerOptions;
 } & ({
     mode: 'docker';
     image: string;

package/dist/createQlever.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"createQlever.d.ts","sourceRoot":"","sources":["../src/createQlever.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,UAAU,EAEX,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAC7D,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,MAAM,MAAM,aAAa,GAAG;IAC1B,wDAAwD;IACxD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,oBAAoB;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,6FAA6F;IAC7F,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,kFAAkF;IAClF,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,kBAAkB,CAAC;CACpC,GAAG,CACA;IACE,IAAI,EAAE,QAAQ,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,GACD;IAAE,IAAI,EAAE,QAAQ,CAAA;CAAE,CACrB,CAAC;AAEF;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,aAAa;;;EA4BlD"}
+{"version":3,"file":"createQlever.d.ts","sourceRoot":"","sources":["../src/createQlever.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,UAAU,EAEX,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAC7D,OAAO,EAAE,mBAAmB,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAE1D,MAAM,MAAM,aAAa,GAAG;IAC1B,wDAAwD;IACxD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,oBAAoB;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,6FAA6F;IAC7F,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,mDAAmD;IACnD,YAAY,CAAC,EAAE,kBAAkB,CAAC;IAClC,sDAAsD;IACtD,aAAa,CAAC,EAAE,mBAAmB,CAAC;CACrC,GAAG,CACA;IACE,IAAI,EAAE,QAAQ,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,GACD;IAAE,IAAI,EAAE,QAAQ,CAAA;CAAE,CACrB,CAAC;AAEF;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,aAAa;;;EA4BlD"}

package/dist/createQlever.js

@@ -25,13 +25,13 @@ export function createQlever(options) {
             indexName: options.indexName,
             downloader: options.downloader ?? new LastModifiedDownloader(options.dataDir),
             cacheIndex: options.cacheIndex,
-            qleverOptions: options.qleverOptions,
+            qleverOptions: options.indexOptions,
         }),
         server: new Server({
             taskRunner,
             indexName: options.indexName ?? 'data',
             port,
-            queryTimeout: options.queryTimeout,
+            qleverOptions: options.serverOptions,
         }),
     };
 }

package/dist/importer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,IAAI,iBAAiB,EAC7B,eAAe,EACf,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACb,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAK5C,MAAM,WAAW,kBAAkB;IACjC,oBAAoB;IACpB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,yBAAyB;IACzB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,uEAAuE;IACvE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,oBAAoB;IACpB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,yJAAyJ;IACzJ,+BAA+B,CAAC,EAAE,OAAO,CAAC;CAC3C;AAED,MAAM,WAAW,qBAAsB,SAAQ,eAAe;IAC5D,sBAAsB;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,kBAAkB,CAAC;CACpC;AAWD,qBAAa,QAAS,YAAW,iBAAiB;IAChD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkB;gBAE9B,OAAO,EAAE,qBAAqB;IAa7B,MAAM,CACjB,aAAa,EAAE,YAAY,EAAE,GAC5B,OAAO,CAAC,YAAY,GAAG,gBAAgB,GAAG,YAAY,CAAC;YA+B5C,QAAQ;IA4CtB,OAAO,CAAC,gBAAgB;IAQxB,OAAO,CAAC,aAAa;IAIrB;;OAEG;YACW,eAAe;IA0B7B,yDAAyD;YAC3C,eAAe;YAaf,cAAc;YAKd,KAAK;CA0CpB"}
+{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,IAAI,iBAAiB,EAC7B,eAAe,EACf,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACb,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAK5C,MAAM,WAAW,kBAAkB;IACjC,oBAAoB;IACpB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,yBAAyB;IACzB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,uEAAuE;IACvE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,oBAAoB;IACpB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,yJAAyJ;IACzJ,+BAA+B,CAAC,EAAE,OAAO,CAAC;CAC3C;AAED,MAAM,WAAW,qBAAsB,SAAQ,eAAe;IAC5D,sBAAsB;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,kBAAkB,CAAC;CACpC;AAWD,qBAAa,QAAS,YAAW,iBAAiB;IAChD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkB;gBAE9B,OAAO,EAAE,qBAAqB;IAa7B,MAAM,CACjB,aAAa,EAAE,YAAY,EAAE,GAC5B,OAAO,CAAC,YAAY,GAAG,gBAAgB,GAAG,YAAY,CAAC;YA+B5C,QAAQ;IAkDtB,OAAO,CAAC,gBAAgB;IAQxB,OAAO,CAAC,aAAa;IAIrB;;OAEG;YACW,eAAe;IA0B7B,yDAAyD;YAC3C,eAAe;YAaf,cAAc;YAKd,KAAK;CA0CpB"}

package/dist/importer.js

@@ -44,7 +44,7 @@ export class Importer {
         return result;
     }
     async doImport(distribution) {
-        const localFile = await this.options.downloader.download(distribution);
+        const { path: localFile, headers } = await this.options.downloader.download(distribution);
         if (await this.isIndexUpToDate(localFile)) {
             const tripleCount = await this.readTripleCount(localFile);
             if (tripleCount === 0) {
@@ -52,7 +52,7 @@ export class Importer {
             }
             return new ImportSuccessful(distribution, undefined, tripleCount);
         }
-        const format = fileFormatFromMimeType(distribution.mimeType);
+        const { format, warning } = fileFormatFor(distribution.mimeType, basename(localFile), headers.get('Content-Type') ?? undefined);
         let logs;
         try {
             logs = await this.index(localFile, format);
@@ -71,7 +71,8 @@ export class Importer {
             return new ImportFailed(distribution, 'Indexed 0 triples from distribution');
         }
         await this.writeCacheInfo(localFile);
-        return new ImportSuccessful(distribution, undefined, tripleCount);
+        const warnings = warning ? [warning] : [];
+        return new ImportSuccessful(distribution, undefined, tripleCount, warnings);
     }
     parseTripleCount(logs) {
         // Extract num-triples.normal from the metadata JSON that the index
@@ -163,10 +164,54 @@ const defaultQleverIndexOptions = {
     'parse-parallel': true,
     'only-pso-and-pos-permutations': false,
 };
-function fileFormatFromMimeType(mimeType) {
-    const format = supportedFormats.get(mimeType);
-    if (format === undefined) {
-        throw new Error(`Unsupported media type: ${mimeType}`);
+const extensionFormats = new Map([
+    ['.nt', 'nt'],
+    ['.nq', 'nq'],
+    ['.ttl', 'ttl'],
+]);
+const compressionTypes = new Set([
+    'application/gzip',
+    'application/x-gzip',
+    'application/octet-stream',
+]);
+/**
+ * Determine the QLever format flag for a distribution.
+ *
+ * Priority:
+ * 1. Server Content-Type (if it maps to a supported RDF format)
+ * 2. File extension (fallback when Content-Type is a compression type or missing)
+ * 3. Declared MIME type from the dataset registry (last resort)
+ */
+function fileFormatFor(declaredMimeType, filename, serverContentType) {
+    const declaredFormat = supportedFormats.get(declaredMimeType);
+    if (declaredFormat === undefined) {
+        throw new Error(`Unsupported media type: ${declaredMimeType}`);
+    }
+    // Try server Content-Type first (strip parameters like "; charset=utf-8").
+    if (serverContentType) {
+        const actualType = serverContentType.split(';')[0].trim();
+        if (!compressionTypes.has(actualType)) {
+            const serverFormat = supportedFormats.get(actualType);
+            if (serverFormat !== undefined && serverFormat !== declaredFormat) {
+                return {
+                    format: serverFormat,
+                    warning: `Server Content-Type ${actualType} does not match declared media type ${declaredMimeType}; using ${serverFormat} format`,
+                };
+            }
+            if (serverFormat !== undefined) {
+                return { format: serverFormat };
+            }
+        }
+    }
+    // Fall back to file extension.
+    const stripped = filename.replace(/\.(gz|bz2|xz|zst)$/i, '');
+    const extension = stripped.slice(stripped.lastIndexOf('.'));
+    const extensionFormat = extensionFormats.get(extension);
+    if (extensionFormat !== undefined && extensionFormat !== declaredFormat) {
+        return {
+            format: extensionFormat,
+            warning: `Declared media type ${declaredMimeType} does not match file extension ${extension}; using ${extensionFormat} format`,
+        };
     }
-    return format;
+    return { format: declaredFormat };
 }

package/dist/server.d.ts

@@ -1,12 +1,18 @@
 import { SparqlServer } from '@lde/sparql-server';
 import { TaskRunner } from '@lde/task-runner';
+export interface QleverServerOptions {
+    /** Maximum memory for query processing and caching. @default '4G' */
+    'memory-max-size'?: string;
+    /** Default query timeout. @default '30s' */
+    'default-query-timeout'?: string;
+}
 export declare class Server<Task> implements SparqlServer {
     private readonly taskRunner;
     private readonly indexName;
     private task?;
     private readonly port;
-    private readonly queryTimeout;
-    constructor({ taskRunner, indexName, port, queryTimeout }: Arguments<Task>);
+    private readonly qleverOptions;
+    constructor({ taskRunner, indexName, port, qleverOptions }: Arguments<Task>);
     start(): Promise<void>;
     stop(): Promise<void>;
     get queryEndpoint(): URL;
@@ -16,7 +22,6 @@ export interface Arguments<Task> {
     indexName: string;
     /** @default 7001 */
     port?: number;
-    /** QLever `--default-query-timeout` value (e.g. '30s', '5min'). @default '30s' */
-    queryTimeout?: string;
+    qleverOptions?: QleverServerOptions;
 }
 //# sourceMappingURL=server.d.ts.map

package/dist/server.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAG9C,qBAAa,MAAM,CAAC,IAAI,CAAE,YAAW,YAAY;IAC/C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAmB;IAC9C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,IAAI,CAAC,CAAO;IACpB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;gBAE1B,EAAE,UAAU,EAAE,SAAS,EAAE,IAAI,EAAE,YAAY,EAAE,EAAE,SAAS,CAAC,IAAI,CAAC;IAO7D,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAStB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAOlC,IAAW,aAAa,IAAI,GAAG,CAE9B;CACF;AAED,MAAM,WAAW,SAAS,CAAC,IAAI;IAC7B,UAAU,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,oBAAoB;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kFAAkF;IAClF,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB"}
+{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAG9C,MAAM,WAAW,mBAAmB;IAClC,qEAAqE;IACrE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,4CAA4C;IAC5C,uBAAuB,CAAC,EAAE,MAAM,CAAC;CAClC;AAOD,qBAAa,MAAM,CAAC,IAAI,CAAE,YAAW,YAAY;IAC/C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAmB;IAC9C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,IAAI,CAAC,CAAO;IACpB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAgC;gBAElD,EAAE,UAAU,EAAE,SAAS,EAAE,IAAI,EAAE,aAAa,EAAE,EAAE,SAAS,CAAC,IAAI,CAAC;IAO9D,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAStB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAOlC,IAAW,aAAa,IAAI,GAAG,CAE9B;CACF;AAED,MAAM,WAAW,SAAS,CAAC,IAAI;IAC7B,UAAU,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,oBAAoB;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE,mBAAmB,CAAC;CACrC"}

package/dist/server.js

@@ -1,19 +1,23 @@
 import { waitForSparqlEndpointAvailable } from '@lde/wait-for-sparql';
+const defaultQleverServerOptions = {
+    'memory-max-size': '4G',
+    'default-query-timeout': '30s',
+};
 export class Server {
     taskRunner;
     indexName;
     task;
     port;
-    queryTimeout;
-    constructor({ taskRunner, indexName, port, queryTimeout }) {
+    qleverOptions;
+    constructor({ taskRunner, indexName, port, qleverOptions }) {
         this.taskRunner = taskRunner;
         this.indexName = indexName;
         this.port = port ?? 7001;
-        this.queryTimeout = queryTimeout ?? '30s';
+        this.qleverOptions = { ...defaultQleverServerOptions, ...qleverOptions };
     }
     async start() {
         // TODO prevent double starts.
-        this.task = await this.taskRunner.run(`qlever-server --index-basename ${this.indexName} --memory-max-size 6G --default-query-timeout ${this.queryTimeout} --port ${this.port}`);
+        this.task = await this.taskRunner.run(`qlever-server --index-basename ${this.indexName} --memory-max-size ${this.qleverOptions['memory-max-size']} --default-query-timeout ${this.qleverOptions['default-query-timeout']} --port ${this.port}`);
         await waitForSparqlEndpointAvailable(this.queryEndpoint.toString());
     }
     async stop() {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/sparql-qlever",
-  "version": "0.12.2",
+  "version": "0.14.0",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/sparql-qlever"
@@ -25,8 +25,8 @@
   ],
   "dependencies": {
     "@lde/dataset": "0.7.2",
-    "@lde/distribution-downloader": "0.5.3",
-    "@lde/sparql-importer": "0.5.0",
+    "@lde/distribution-downloader": "0.6.0",
+    "@lde/sparql-importer": "0.6.0",
     "@lde/sparql-server": "0.4.10",
     "@lde/task-runner": "0.2.10",
     "@lde/task-runner-docker": "0.2.11",