@lde/sparql-qlever 0.9.3 → 0.10.0

package/README.md

@@ -1,3 +1,11 @@
 # SPARQL QLever
 
 An adapter for the [QLever](https://github.com/ad-freiburg/qlever) SPARQL server.
+
+## Index caching
+
+Building a QLever index is slow. To avoid rebuilding it on every pipeline run, the importer caches a single index and reuses it when the source data hasn't changed. On subsequent runs, indexing is skipped when the source file matches and hasn't been re-downloaded.
+
+Only **one** index is cached at a time. In a multi-dataset pipeline, each dataset overwrites the previous index. On re-run, the last-indexed dataset gets a cache hit while the others rebuild.
+
+Caching is enabled by default. Disable it by passing `cacheIndex: false` to `createQlever()` or the `Importer` constructor (e.g. driven by a `QLEVER_CACHE_INDEX=false` environment variable).

package/dist/createQlever.d.ts

@@ -8,6 +8,8 @@ export type QleverOptions = {
     /** @default 7001 */
     port?: number;
     downloader?: Downloader;
+    /** Cache QLever indices and skip re-indexing when source data is unchanged. Defaults to `true`. */
+    cacheIndex?: boolean;
 } & ({
     mode: 'docker';
     image: string;

package/dist/createQlever.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"createQlever.d.ts","sourceRoot":"","sources":["../src/createQlever.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,UAAU,EAEX,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,MAAM,MAAM,aAAa,GAAG;IAC1B,wDAAwD;IACxD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,oBAAoB;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,UAAU,CAAC;CACzB,GAAG,CACA;IACE,IAAI,EAAE,QAAQ,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,GACD;IAAE,IAAI,EAAE,QAAQ,CAAA;CAAE,CACrB,CAAC;AAEF,wBAAgB,YAAY,CAAC,OAAO,EAAE,aAAa;;;EAyBlD"}
+{"version":3,"file":"createQlever.d.ts","sourceRoot":"","sources":["../src/createQlever.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,UAAU,EAEX,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,MAAM,MAAM,aAAa,GAAG;IAC1B,wDAAwD;IACxD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,oBAAoB;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,mGAAmG;IACnG,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB,GAAG,CACA;IACE,IAAI,EAAE,QAAQ,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,GACD;IAAE,IAAI,EAAE,QAAQ,CAAA;CAAE,CACrB,CAAC;AAEF,wBAAgB,YAAY,CAAC,OAAO,EAAE,aAAa;;;EA0BlD"}

package/dist/createQlever.js

@@ -18,6 +18,7 @@ export function createQlever(options) {
             taskRunner,
             indexName: options.indexName,
             downloader: options.downloader ?? new LastModifiedDownloader(options.dataDir),
+            cacheIndex: options.cacheIndex,
         }),
         server: new Server({
             taskRunner,

package/dist/importer.d.ts

@@ -11,6 +11,8 @@ export interface Options {
         'num-triples-per-batch': number;
     };
     port?: number;
+    /** Cache QLever indices and skip re-indexing when source data is unchanged. Defaults to `true`. */
+    cacheIndex?: boolean;
 }
 /**
  * Import RDF to a QLever SPARQL server.
@@ -22,11 +24,20 @@ export declare class Importer implements ImporterInterface {
     private taskRunner;
     private downloader;
     private qleverOptions;
-    constructor({ taskRunner, downloader, indexName, qleverOptions }: Options);
+    private cacheIndex;
+    constructor({ taskRunner, downloader, indexName, qleverOptions, cacheIndex, }: Options);
     import(dataset: Dataset): Promise<NotSupported | ImportSuccessful | ImportFailed>;
     private doImport;
     private fileFormatFromMimeType;
     private parseTripleCount;
+    private cacheInfoPath;
+    /**
+     * Check whether the cached index is still up to date.
+     */
+    private isIndexUpToDate;
+    /** Read the triple count from QLever's metadata file. */
+    private readTripleCount;
+    private writeCacheInfo;
     private index;
 }
 //# sourceMappingURL=importer.d.ts.map

package/dist/importer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,IAAI,iBAAiB,EAC7B,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACb,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AACrD,OAAO,EACL,UAAU,EAEX,MAAM,8BAA8B,CAAC;AAGtC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAU9C,MAAM,WAAW,OAAO;IACtB,UAAU,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,aAAa,CAAC,EAAE;QACd,qBAAqB,EAAE,OAAO,CAAC;QAC/B,uBAAuB,EAAE,MAAM,CAAC;KACjC,CAAC;IACF,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;;;GAIG;AACH,qBAAa,QAAS,YAAW,iBAAiB;IAChD,OAAO,CAAC,SAAS,CAAC;IAClB,OAAO,CAAC,UAAU,CAAsB;IACxC,OAAO,CAAC,UAAU,CAAC;IACnB,OAAO,CAAC,aAAa,CAAC;gBAEV,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,EAAE,OAAO;IAU5D,MAAM,CACjB,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,YAAY,GAAG,gBAAgB,GAAG,YAAY,CAAC;YAiC5C,QAAQ;IAatB,OAAO,CAAC,sBAAsB;IAQ9B,OAAO,CAAC,gBAAgB;YAQV,KAAK;CAmBpB"}
+{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,IAAI,iBAAiB,EAC7B,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACb,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AACrD,OAAO,EACL,UAAU,EAEX,MAAM,8BAA8B,CAAC;AAGtC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAU9C,MAAM,WAAW,OAAO;IACtB,UAAU,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,aAAa,CAAC,EAAE;QACd,qBAAqB,EAAE,OAAO,CAAC;QAC/B,uBAAuB,EAAE,MAAM,CAAC;KACjC,CAAC;IACF,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,mGAAmG;IACnG,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAMD;;;;GAIG;AACH,qBAAa,QAAS,YAAW,iBAAiB;IAChD,OAAO,CAAC,SAAS,CAAC;IAClB,OAAO,CAAC,UAAU,CAAsB;IACxC,OAAO,CAAC,UAAU,CAAC;IACnB,OAAO,CAAC,aAAa,CAAC;IACtB,OAAO,CAAC,UAAU,CAAC;gBAEP,EACV,UAAU,EACV,UAAU,EACV,SAAS,EACT,aAAa,EACb,UAAU,GACX,EAAE,OAAO;IAWG,MAAM,CACjB,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,YAAY,GAAG,gBAAgB,GAAG,YAAY,CAAC;YAiC5C,QAAQ;IAqBtB,OAAO,CAAC,sBAAsB;IAQ9B,OAAO,CAAC,gBAAgB;IAQxB,OAAO,CAAC,aAAa;IAIrB;;OAEG;YACW,eAAe;IA0B7B,yDAAyD;YAC3C,eAAe;YAaf,cAAc;YAKd,KAAK;CAmBpB"}

package/dist/importer.js

@@ -1,7 +1,7 @@
 import { ImportFailed, ImportSuccessful, NotSupported, } from '@lde/sparql-importer';
 import { LastModifiedDownloader, } from '@lde/distribution-downloader';
-import { basename, dirname } from 'path';
-import { writeFile } from 'node:fs/promises';
+import { basename, dirname, join } from 'path';
+import { readFile, stat, writeFile } from 'node:fs/promises';
 const supportedFormats = new Map([
     ['application/n-triples', 'nt'],
     ['application/n-quads', 'nq'],
@@ -17,7 +17,8 @@ export class Importer {
     taskRunner;
     downloader;
     qleverOptions;
-    constructor({ taskRunner, downloader, indexName, qleverOptions }) {
+    cacheIndex;
+    constructor({ taskRunner, downloader, indexName, qleverOptions, cacheIndex, }) {
         this.taskRunner = taskRunner;
         this.downloader = downloader ?? new LastModifiedDownloader();
         this.indexName = indexName ?? 'data';
@@ -25,6 +26,7 @@ export class Importer {
             'ascii-prefixes-only': true,
             'num-triples-per-batch': 100000,
         };
+        this.cacheIndex = cacheIndex ?? true;
     }
     async import(dataset) {
         const downloadDistributions = dataset
@@ -57,8 +59,13 @@ export class Importer {
     }
     async doImport(distribution) {
         const localFile = await this.downloader.download(distribution);
+        if (await this.isIndexUpToDate(localFile)) {
+            const tripleCount = await this.readTripleCount(localFile);
+            return new ImportSuccessful(distribution, undefined, tripleCount);
+        }
         const logs = await this.index(localFile, this.fileFormatFromMimeType(distribution.mimeType));
         const tripleCount = this.parseTripleCount(logs);
+        await this.writeCacheInfo(localFile);
         return new ImportSuccessful(distribution, undefined, tripleCount);
     }
     fileFormatFromMimeType(mimeType) {
@@ -75,6 +82,50 @@ export class Importer {
         const match = logs.match(/"num-triples":\{[^}]*"normal":(\d+)/);
         return match ? Number(match[1]) : undefined;
     }
+    cacheInfoPath(dataFile) {
+        return join(dirname(dataFile), `${this.indexName}.cache-info.json`);
+    }
+    /**
+     * Check whether the cached index is still up to date.
+     */
+    async isIndexUpToDate(dataFile) {
+        if (!this.cacheIndex)
+            return false;
+        let cacheInfo;
+        try {
+            const raw = await readFile(this.cacheInfoPath(dataFile), 'utf-8');
+            cacheInfo = JSON.parse(raw);
+        }
+        catch {
+            return false; // No cache marker — first run.
+        }
+        if (cacheInfo.sourceFile !== basename(dataFile)) {
+            return false; // Different dataset was last indexed.
+        }
+        const [cacheInfoStat, dataFileStat] = await Promise.all([
+            stat(this.cacheInfoPath(dataFile)),
+            stat(dataFile),
+        ]);
+        if (dataFileStat.mtimeMs > cacheInfoStat.mtimeMs) {
+            return false; // Data was re-downloaded.
+        }
+        return true;
+    }
+    /** Read the triple count from QLever's metadata file. */
+    async readTripleCount(dataFile) {
+        try {
+            const metadataPath = join(dirname(dataFile), `${this.indexName}.meta-data.json`);
+            const raw = await readFile(metadataPath, 'utf-8');
+            return this.parseTripleCount(raw);
+        }
+        catch {
+            return undefined;
+        }
+    }
+    async writeCacheInfo(dataFile) {
+        const info = { sourceFile: basename(dataFile) };
+        await writeFile(this.cacheInfoPath(dataFile), JSON.stringify(info));
+    }
     async index(file, format) {
         const settingsFile = 'index.settings.json';
         await writeFile(`${dirname(file)}/${settingsFile}`, JSON.stringify(this.qleverOptions));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/sparql-qlever",
-  "version": "0.9.3",
+  "version": "0.10.0",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/sparql-qlever"