@lde/sparql-qlever 0.12.1 → 0.13.0

package/README.md

@@ -9,3 +9,41 @@ Building a QLever index is slow. To avoid rebuilding it on every pipeline run, t
 Only **one** index is cached at a time. In a multi-dataset pipeline, each dataset overwrites the previous index. On re-run, the last-indexed dataset gets a cache hit while the others rebuild.
 
 Caching is enabled by default. Disable it by passing `cacheIndex: false` to `createQlever()` or the `Importer` constructor (e.g. driven by a `QLEVER_CACHE_INDEX=false` environment variable).
+
+## Configuration
+
+`createQlever()` accepts `indexOptions` and `serverOptions` to tune QLever's index builder and server respectively.
+
+### Server options (`serverOptions`)
+
+Passed to `qlever-server` at startup.
+
+| Option                  | Description                                      | Default |
+| ----------------------- | ------------------------------------------------ | ------- |
+| `memory-max-size`       | Maximum memory for query processing and caching. | `'4G'`  |
+| `default-query-timeout` | Default query timeout.                           | `'30s'` |
+
+Example:
+
+```ts
+const { importer, server } = createQlever({
+  mode: 'docker',
+  image: 'adfreiburg/qlever:latest',
+  serverOptions: {
+    'memory-max-size': '12G',
+    'default-query-timeout': '120s',
+  },
+});
+```
+
+### Index options (`indexOptions`)
+
+Passed to `qlever-index` during import.
+
+| Option                          | Description                                                                                   | Default     |
+| ------------------------------- | --------------------------------------------------------------------------------------------- | ----------- |
+| `ascii-prefixes-only`           | Enable faster parsing for well-behaved TTL files.                                             | `true`      |
+| `num-triples-per-batch`         | Triples per batch; lower values reduce memory usage.                                          | `3_000_000` |
+| `stxxl-memory`                  | Memory budget for sorting during the index build.                                             | `'10G'`     |
+| `parse-parallel`                | Parse input in parallel.                                                                      | `true`      |
+| `only-pso-and-pos-permutations` | Build only PSO and POS permutations. Faster, but queries with predicate variables won't work. | `false`     |

package/dist/createQlever.d.ts

@@ -1,6 +1,6 @@
 import { Downloader } from '@lde/distribution-downloader';
 import { Importer, QleverIndexOptions } from './importer.js';
-import { Server } from './server.js';
+import { QleverServerOptions, Server } from './server.js';
 export type QleverOptions = {
     /** Directory where downloaded data files are stored. */
     dataDir?: string;
@@ -10,7 +10,10 @@ export type QleverOptions = {
     downloader?: Downloader;
     /** Cache QLever indices and skip re-indexing when source data is unchanged. @default true */
     cacheIndex?: boolean;
-    qleverOptions?: QleverIndexOptions;
+    /** Options for `qlever-index` (index building). */
+    indexOptions?: QleverIndexOptions;
+    /** Options for `qlever-server` (query processing). */
+    serverOptions?: QleverServerOptions;
 } & ({
     mode: 'docker';
     image: string;

package/dist/createQlever.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"createQlever.d.ts","sourceRoot":"","sources":["../src/createQlever.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,UAAU,EAEX,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAC7D,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,MAAM,MAAM,aAAa,GAAG;IAC1B,wDAAwD;IACxD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,oBAAoB;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,6FAA6F;IAC7F,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,aAAa,CAAC,EAAE,kBAAkB,CAAC;CACpC,GAAG,CACA;IACE,IAAI,EAAE,QAAQ,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,GACD;IAAE,IAAI,EAAE,QAAQ,CAAA;CAAE,CACrB,CAAC;AAEF;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,aAAa;;;EA2BlD"}
+{"version":3,"file":"createQlever.d.ts","sourceRoot":"","sources":["../src/createQlever.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,UAAU,EAEX,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAC7D,OAAO,EAAE,mBAAmB,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAE1D,MAAM,MAAM,aAAa,GAAG;IAC1B,wDAAwD;IACxD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,oBAAoB;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,6FAA6F;IAC7F,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,mDAAmD;IACnD,YAAY,CAAC,EAAE,kBAAkB,CAAC;IAClC,sDAAsD;IACtD,aAAa,CAAC,EAAE,mBAAmB,CAAC;CACrC,GAAG,CACA;IACE,IAAI,EAAE,QAAQ,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,GACD;IAAE,IAAI,EAAE,QAAQ,CAAA;CAAE,CACrB,CAAC;AAEF;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,aAAa;;;EA4BlD"}

package/dist/createQlever.js

@@ -25,12 +25,13 @@ export function createQlever(options) {
             indexName: options.indexName,
             downloader: options.downloader ?? new LastModifiedDownloader(options.dataDir),
             cacheIndex: options.cacheIndex,
-            qleverOptions: options.qleverOptions,
+            qleverOptions: options.indexOptions,
         }),
         server: new Server({
             taskRunner,
             indexName: options.indexName ?? 'data',
             port,
+            qleverOptions: options.serverOptions,
         }),
     };
 }

package/dist/server.d.ts

@@ -1,11 +1,18 @@
 import { SparqlServer } from '@lde/sparql-server';
 import { TaskRunner } from '@lde/task-runner';
+export interface QleverServerOptions {
+    /** Maximum memory for query processing and caching. @default '4G' */
+    'memory-max-size'?: string;
+    /** Default query timeout. @default '30s' */
+    'default-query-timeout'?: string;
+}
 export declare class Server<Task> implements SparqlServer {
     private readonly taskRunner;
     private readonly indexName;
     private task?;
     private readonly port;
-    constructor({ taskRunner, indexName, port }: Arguments<Task>);
+    private readonly qleverOptions;
+    constructor({ taskRunner, indexName, port, qleverOptions }: Arguments<Task>);
     start(): Promise<void>;
     stop(): Promise<void>;
     get queryEndpoint(): URL;
@@ -13,6 +20,8 @@ export declare class Server<Task> implements SparqlServer {
 export interface Arguments<Task> {
     taskRunner: TaskRunner<Task>;
     indexName: string;
+    /** @default 7001 */
     port?: number;
+    qleverOptions?: QleverServerOptions;
 }
 //# sourceMappingURL=server.d.ts.map

package/dist/server.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAG9C,qBAAa,MAAM,CAAC,IAAI,CAAE,YAAW,YAAY;IAC/C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAmB;IAC9C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,IAAI,CAAC,CAAO;IACpB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;gBAElB,EAAE,UAAU,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,SAAS,CAAC,IAAI,CAAC;IAM/C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAStB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAOlC,IAAW,aAAa,IAAI,GAAG,CAE9B;CACF;AAED,MAAM,WAAW,SAAS,CAAC,IAAI;IAC7B,UAAU,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf"}
+{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAG9C,MAAM,WAAW,mBAAmB;IAClC,qEAAqE;IACrE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,4CAA4C;IAC5C,uBAAuB,CAAC,EAAE,MAAM,CAAC;CAClC;AAOD,qBAAa,MAAM,CAAC,IAAI,CAAE,YAAW,YAAY;IAC/C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAmB;IAC9C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,IAAI,CAAC,CAAO;IACpB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAgC;gBAElD,EAAE,UAAU,EAAE,SAAS,EAAE,IAAI,EAAE,aAAa,EAAE,EAAE,SAAS,CAAC,IAAI,CAAC;IAO9D,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAStB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAOlC,IAAW,aAAa,IAAI,GAAG,CAE9B;CACF;AAED,MAAM,WAAW,SAAS,CAAC,IAAI;IAC7B,UAAU,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,oBAAoB;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE,mBAAmB,CAAC;CACrC"}

package/dist/server.js

@@ -1,17 +1,23 @@
 import { waitForSparqlEndpointAvailable } from '@lde/wait-for-sparql';
+const defaultQleverServerOptions = {
+    'memory-max-size': '4G',
+    'default-query-timeout': '30s',
+};
 export class Server {
     taskRunner;
     indexName;
     task;
     port;
-    constructor({ taskRunner, indexName, port }) {
+    qleverOptions;
+    constructor({ taskRunner, indexName, port, qleverOptions }) {
         this.taskRunner = taskRunner;
         this.indexName = indexName;
         this.port = port ?? 7001;
+        this.qleverOptions = { ...defaultQleverServerOptions, ...qleverOptions };
     }
     async start() {
         // TODO prevent double starts.
-        this.task = await this.taskRunner.run(`qlever-server --index-basename ${this.indexName} --memory-max-size 6G --port ${this.port}`);
+        this.task = await this.taskRunner.run(`qlever-server --index-basename ${this.indexName} --memory-max-size ${this.qleverOptions['memory-max-size']} --default-query-timeout ${this.qleverOptions['default-query-timeout']} --port ${this.port}`);
         await waitForSparqlEndpointAvailable(this.queryEndpoint.toString());
     }
     async stop() {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/sparql-qlever",
-  "version": "0.12.1",
+  "version": "0.13.0",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/sparql-qlever"