@lde/search-indexer 0.30.3 → 0.31.0

package/dist/composition.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"composition.d.ts","sourceRoot":"","sources":["../src/composition.ts"],"names":[],"mappings":"AAKA,OAAO,EAOL,KAAK,eAAe,EACpB,KAAK,oBAAoB,EACzB,KAAK,MAAM,EACZ,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EACV,QAAQ,EACR,cAAc,EACd,YAAY,EAEb,MAAM,aAAa,CAAC;AAMrB,OAAO,EAAgB,KAAK,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACtE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAEjD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,aAAa,GAAG,eAAe,CAiB1E;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,aAAa,GACpB,CAAC,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,KAAK,MAAM,CAAC,cAAc,CAAC,CAmBxE;AAED;;;;;;;;;GASG;AACH,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,aAAa,EACrB,oBAAoB,CAAC,EAAE,aAAa,CAAC,sBAAsB,CAAC,GAC3D,oBAAoB,GAAG,SAAS,CAmClC"}
+{"version":3,"file":"composition.d.ts","sourceRoot":"","sources":["../src/composition.ts"],"names":[],"mappings":"AAKA,OAAO,EAOL,KAAK,eAAe,EACpB,KAAK,oBAAoB,EACzB,KAAK,MAAM,EACZ,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EACV,QAAQ,EACR,cAAc,EACd,YAAY,EAEb,MAAM,aAAa,CAAC;AAMrB,OAAO,EAAgB,KAAK,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACtE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAEjD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,aAAa,GAAG,eAAe,CAiB1E;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,aAAa,GACpB,CAAC,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,KAAK,MAAM,CAAC,cAAc,CAAC,CAmBxE;AAED;;;;;;;;;GASG;AACH,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,aAAa,EACrB,oBAAoB,CAAC,EAAE,aAAa,CAAC,sBAAsB,CAAC,GAC3D,oBAAoB,GAAG,SAAS,CA0ClC"}

package/dist/composition.js

@@ -69,11 +69,15 @@ export function distributionResolverFrom(config, replaceJsonLdContext) {
     if (!config.qlever) {
         return undefined;
     }
+    const serverOptions = config.qlever.memoryMaxSize === undefined
+        ? undefined
+        : { 'memory-max-size': config.qlever.memoryMaxSize };
     const { importer, server } = config.qlever.mode === 'remote'
         ? createQlever({
             mode: 'remote',
             dataDir: config.qlever.dataDir,
             replaceJsonLdContext,
+            serverOptions,
         })
         : config.qlever.network
             ? createQlever({
@@ -87,12 +91,14 @@ export function distributionResolverFrom(config, replaceJsonLdContext) {
                 // one indexer per network.
                 containerName: `qlever-${config.qlever.network}`,
                 replaceJsonLdContext,
+                serverOptions,
             })
             : createQlever({
                 mode: 'docker',
                 image: config.qlever.image,
                 dataDir: config.qlever.dataDir,
                 replaceJsonLdContext,
+                serverOptions,
             });
     return new ImportResolver(new SparqlDistributionResolver(), {
         importer,

package/dist/config.d.ts

@@ -48,7 +48,7 @@ export interface IndexerConfig {
      *  (`PROVENANCE_FILE` + `PIPELINE_VERSION`). */
     readonly provenance?: ProvenanceConfig;
     /** The QLever import path (`QLEVER_MODE` + `QLEVER_IMAGE` +
-     *  `IMPORT_STRATEGY` + `DATA_DIR`): import data dumps into a
+     *  `IMPORT_STRATEGY` + `DATA_DIR` + `QLEVER_MEMORY_MAX_SIZE`): import data dumps into a
      *  pipeline-controlled QLever instead of relying on live SPARQL endpoints
      *  alone. */
     readonly qlever?: QleverConfig;
@@ -78,6 +78,14 @@ export type QleverConfig = {
     readonly strategy: 'sparql' | 'sparqlWithImportFallback' | 'import';
     /** Directory for downloaded dumps and QLever index caches. */
     readonly dataDir: string;
+    /**
+     * The memory QLever may reserve for answering queries
+     * (`QLEVER_MEMORY_MAX_SIZE`, QLever’s `--memory-max-size`), e.g. `8G`.
+     * QLever refuses a query that would exceed it, with an HTTP 500 such as
+     * “Tried to allocate 8 MB, but only 5.8 MB were available”. Unset, the
+     * `@lde/sparql-qlever` default applies.
+     */
+    readonly memoryMaxSize?: string;
 } & ({
     /** A sibling container the indexer starts over the Docker socket. */
     readonly mode: 'docker';

package/dist/config.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAInE;;;;;GAKG;AACH,MAAM,WAAW,aAAa;IAC5B,uEAAuE;IACvE,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC;;2CAEuC;IACvC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,GAAG,CAAC;IAChC;;;;;;;OAOG;IACH,QAAQ,CAAC,iBAAiB,EAAE,SAAS,MAAM,EAAE,CAAC;IAC9C;;eAEW;IACX,QAAQ,CAAC,eAAe,EAAE,cAAc,CAAC;IACzC;;2DAEuD;IACvD,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,yDAAyD;IACzD,QAAQ,CAAC,SAAS,EAAE,mBAAmB,CAAC;IACxC;wEACoE;IACpE,QAAQ,CAAC,WAAW,EAAE,UAAU,GAAG,YAAY,CAAC;IAChD;wEACoE;IACpE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC;;;;;OAKG;IACH,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC;oDACgD;IAChD,QAAQ,CAAC,UAAU,CAAC,EAAE,gBAAgB,CAAC;IACvC;;;iBAGa;IACb,QAAQ,CAAC,MAAM,CAAC,EAAE,YAAY,CAAC;CAChC;AAED,sEAAsE;AACtE,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;IACpC,uEAAuE;IACvE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;sEACsE;AACtE,MAAM,WAAW,gBAAgB;IAC/B,sEAAsE;IACtE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,yEAAyE;IACzE,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;CAClC;AAED;;;GAGG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB,sEAAsE;IACtE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,GAAG,0BAA0B,GAAG,QAAQ,CAAC;IACpE,8DAA8D;IAC9D,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,GAAG,CACA;IACE,qEAAqE;IACrE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,mEAAmE;IACnE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;iDAG6C;IAC7C,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;CAC3B,GACD;IACE;8DAC0D;IAC1D,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;CACzB,CACJ,CAAC;AAUF;;;;GAIG;AACH,wBAAgB,qBAAqB,CACnC,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,GACxD,aAAa,CAkGf"}
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAInE;;;;;GAKG;AACH,MAAM,WAAW,aAAa;IAC5B,uEAAuE;IACvE,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC;;2CAEuC;IACvC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,GAAG,CAAC;IAChC;;;;;;;OAOG;IACH,QAAQ,CAAC,iBAAiB,EAAE,SAAS,MAAM,EAAE,CAAC;IAC9C;;eAEW;IACX,QAAQ,CAAC,eAAe,EAAE,cAAc,CAAC;IACzC;;2DAEuD;IACvD,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,yDAAyD;IACzD,QAAQ,CAAC,SAAS,EAAE,mBAAmB,CAAC;IACxC;wEACoE;IACpE,QAAQ,CAAC,WAAW,EAAE,UAAU,GAAG,YAAY,CAAC;IAChD;wEACoE;IACpE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC;;;;;OAKG;IACH,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC;oDACgD;IAChD,QAAQ,CAAC,UAAU,CAAC,EAAE,gBAAgB,CAAC;IACvC;;;iBAGa;IACb,QAAQ,CAAC,MAAM,CAAC,EAAE,YAAY,CAAC;CAChC;AAED,sEAAsE;AACtE,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;IACpC,uEAAuE;IACvE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;sEACsE;AACtE,MAAM,WAAW,gBAAgB;IAC/B,sEAAsE;IACtE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,yEAAyE;IACzE,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;CAClC;AAED;;;GAGG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB,sEAAsE;IACtE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,GAAG,0BAA0B,GAAG,QAAQ,CAAC;IACpE,8DAA8D;IAC9D,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB;;;;;;OAMG;IACH,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;CACjC,GAAG,CACA;IACE,qEAAqE;IACrE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,mEAAmE;IACnE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;iDAG6C;IAC7C,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;CAC3B,GACD;IACE;8DAC0D;IAC1D,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;CACzB,CACJ,CAAC;AAcF;;;;GAIG;AACH,wBAAgB,qBAAqB,CACnC,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,GACxD,aAAa,CAkGf"}

package/dist/config.js

@@ -2,6 +2,9 @@ import { Dataset } from '@lde/dataset';
 import { dirname } from 'node:path';
 import { fileURLToPath } from 'node:url';
 const QLEVER_MODES = ['docker', 'remote'];
+/** A size QLever’s `--memory-max-size` reads: a number and a unit, such as
+ *  `8G`, `8 GB` or `512M`. */
+const MEMORY_SIZE = /^\d+(\.\d+)?\s?[kmgt]b?$/i;
 const IMPORT_STRATEGIES = [
     'sparql',
     'sparqlWithImportFallback',
@@ -193,6 +196,7 @@ function qleverFromEnvironment(environment, problems) {
     const image = environment['QLEVER_IMAGE'];
     const strategy = environment['IMPORT_STRATEGY'];
     const network = environment['QLEVER_NETWORK'];
+    const memoryMaxSize = environment['QLEVER_MEMORY_MAX_SIZE'];
     // QLEVER_IMAGE alone is how the Docker mode has always been enabled. An
     // empty QLEVER_MODE, as a template renders an unset one, counts as unset.
     const mode = environment['QLEVER_MODE'] || (image ? 'docker' : undefined);
@@ -203,6 +207,9 @@ function qleverFromEnvironment(environment, problems) {
         if (network) {
             problems.push('QLEVER_NETWORK requires QLEVER_IMAGE: without an import engine there is no QLever container to attach');
         }
+        if (memoryMaxSize) {
+            problems.push('QLEVER_MEMORY_MAX_SIZE requires QLEVER_IMAGE or QLEVER_MODE=remote: without an import engine there is no QLever to size');
+        }
         return undefined;
     }
     if (!QLEVER_MODES.includes(mode)) {
@@ -213,9 +220,13 @@ function qleverFromEnvironment(environment, problems) {
         !IMPORT_STRATEGIES.includes(strategy)) {
         problems.push(`IMPORT_STRATEGY must be one of ${IMPORT_STRATEGIES.map((name) => `“${name}”`).join(', ')}, got “${strategy}”`);
     }
+    if (memoryMaxSize && !MEMORY_SIZE.test(memoryMaxSize)) {
+        problems.push(`QLEVER_MEMORY_MAX_SIZE must be a size such as “8G” or “8 GB”, got “${memoryMaxSize}”`);
+    }
     const common = {
         strategy: (strategy ?? 'sparql'),
         dataDir: environment['DATA_DIR'] ?? '/data',
+        ...(memoryMaxSize ? { memoryMaxSize } : {}),
     };
     if (mode === 'remote') {
         if (image) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/search-indexer",
-  "version": "0.30.3",
+  "version": "0.31.0",
   "description": "The @lde/search-pipeline indexer as a bootable process and prebuilt Docker image: loads a mounted schema-declaration module, dataset selection, Typesense connection and rebuild mode from the environment, then runs the object-grain searchIndexerPipeline once to completion. The composition layer that binds the engine-agnostic @lde/search-pipeline stages to @lde/search-typesense – the write-path counterpart of @lde/search-api-server.",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
@@ -30,12 +30,12 @@
   "dependencies": {
     "@lde/dataset": "^0.10.0",
     "@lde/dataset-registry-client": "^0.12.0",
-    "@lde/pipeline": "^0.41.0",
-    "@lde/pipeline-console-reporter": "^0.32.0",
+    "@lde/pipeline": "^0.42.0",
+    "@lde/pipeline-console-reporter": "^0.33.0",
     "@lde/search": "^0.36.3",
-    "@lde/search-pipeline": "^0.40.3",
-    "@lde/search-typesense": "^0.43.3",
-    "@lde/sparql-qlever": "^0.20.0",
+    "@lde/search-pipeline": "^0.41.0",
+    "@lde/search-typesense": "^0.44.0",
+    "@lde/sparql-qlever": "^0.21.0",
     "commander": "^15.0.0",
     "tslib": "^2.3.0",
     "typesense": "^3.1.1"