@lde/sparql-qlever 0.8.5 → 0.9.0

package/dist/importer.d.ts

@@ -26,6 +26,7 @@ export declare class Importer implements ImporterInterface {
     import(dataset: Dataset): Promise<NotSupported | ImportSuccessful | ImportFailed>;
     private doImport;
     private fileFormatFromMimeType;
+    private parseTripleCount;
     private index;
 }
 //# sourceMappingURL=importer.d.ts.map

package/dist/importer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,IAAI,iBAAiB,EAC7B,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACb,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AACrD,OAAO,EACL,UAAU,EAEX,MAAM,8BAA8B,CAAC;AAGtC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,MAAM,WAAW,OAAO;IACtB,UAAU,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,aAAa,CAAC,EAAE;QACd,qBAAqB,EAAE,OAAO,CAAC;QAC/B,uBAAuB,EAAE,MAAM,CAAC;KACjC,CAAC;IACF,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;;;GAIG;AACH,qBAAa,QAAS,YAAW,iBAAiB;IAChD,OAAO,CAAC,SAAS,CAAC;IAClB,OAAO,CAAC,UAAU,CAAsB;IACxC,OAAO,CAAC,UAAU,CAAC;IACnB,OAAO,CAAC,aAAa,CAAC;gBAEV,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,EAAE,OAAO;IAU5D,MAAM,CACjB,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,YAAY,GAAG,gBAAgB,GAAG,YAAY,CAAC;YAgC5C,QAAQ;IAYtB,OAAO,CAAC,sBAAsB;YAahB,KAAK;CAsBpB"}
+{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,IAAI,iBAAiB,EAC7B,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACb,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AACrD,OAAO,EACL,UAAU,EAEX,MAAM,8BAA8B,CAAC;AAGtC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAU9C,MAAM,WAAW,OAAO;IACtB,UAAU,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,aAAa,CAAC,EAAE;QACd,qBAAqB,EAAE,OAAO,CAAC;QAC/B,uBAAuB,EAAE,MAAM,CAAC;KACjC,CAAC;IACF,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;;;GAIG;AACH,qBAAa,QAAS,YAAW,iBAAiB;IAChD,OAAO,CAAC,SAAS,CAAC;IAClB,OAAO,CAAC,UAAU,CAAsB;IACxC,OAAO,CAAC,UAAU,CAAC;IACnB,OAAO,CAAC,aAAa,CAAC;gBAEV,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,EAAE,OAAO;IAU5D,MAAM,CACjB,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,YAAY,GAAG,gBAAgB,GAAG,YAAY,CAAC;YAiC5C,QAAQ;IAatB,OAAO,CAAC,sBAAsB;IAQ9B,OAAO,CAAC,gBAAgB;YAQV,KAAK;CAuBpB"}

package/dist/importer.js

@@ -2,6 +2,11 @@ import { ImportFailed, ImportSuccessful, NotSupported, } from '@lde/sparql-impor
 import { LastModifiedDownloader, } from '@lde/distribution-downloader';
 import { basename, dirname } from 'path';
 import { writeFile } from 'node:fs/promises';
+const supportedFormats = new Map([
+    ['application/n-triples', 'nt'],
+    ['application/n-quads', 'nq'],
+    ['text/turtle', 'ttl'],
+]);
 /**
  * Import RDF to a QLever SPARQL server.
  *
@@ -24,7 +29,8 @@ export class Importer {
     async import(dataset) {
         const downloadDistributions = dataset
             .getDownloadDistributions()
-            .filter((distribution) => distribution.mimeType !== undefined);
+            .filter((distribution) => distribution.mimeType !== undefined &&
+            supportedFormats.has(distribution.mimeType));
         if (downloadDistributions.length === 0) {
             return new NotSupported();
         }
@@ -51,20 +57,23 @@ export class Importer {
     }
     async doImport(distribution) {
         const localFile = await this.downloader.download(distribution);
-        await this.index(localFile, this.fileFormatFromMimeType(distribution.mimeType));
-        return new ImportSuccessful(distribution);
+        const logs = await this.index(localFile, this.fileFormatFromMimeType(distribution.mimeType));
+        const tripleCount = this.parseTripleCount(logs);
+        return new ImportSuccessful(distribution, undefined, tripleCount);
     }
     fileFormatFromMimeType(mimeType) {
-        switch (mimeType) {
-            case 'application/n-triples':
-                return 'nt';
-            case 'application/n-quads':
-                return 'nq';
-            case 'text/turtle':
-                return 'ttl';
-            default:
-                throw new Error(`Unsupported media type: ${mimeType}`);
+        const format = supportedFormats.get(mimeType);
+        if (format === undefined) {
+            throw new Error(`Unsupported media type: ${mimeType}`);
         }
+        return format;
+    }
+    parseTripleCount(logs) {
+        // Extract num-triples.normal from the metadata JSON that the index
+        // command cats to stdout. Use a regex rather than JSON.parse because
+        // Docker log multiplexing prepends binary frame headers to each chunk.
+        const match = logs.match(/"num-triples":\{[^}]*"normal":(\d+)/);
+        return match ? Number(match[1]) : undefined;
     }
     async index(file, format) {
         const workingDir = dirname(file);
@@ -76,7 +85,8 @@ export class Importer {
             : this.qleverOptions;
         await writeFile(`${workingDir}/${settingsFile}`, JSON.stringify(settings));
         // TODO: write index to named volume instead of bind mount for better performance.
-        const indexTask = await this.taskRunner.run(`(zcat '${basename(file)}' 2>/dev/null || cat '${basename(file)}') | qlever-index -i ${this.indexName} -s ${settingsFile} -F ${format} -f -`);
-        await this.taskRunner.wait(indexTask);
+        const metadataFile = `${this.indexName}.meta-data.json`;
+        const indexTask = await this.taskRunner.run(`(zcat '${basename(file)}' 2>/dev/null || cat '${basename(file)}') | qlever-index -i ${this.indexName} -s ${settingsFile} -F ${format} -f - && cat ${metadataFile}`);
+        return await this.taskRunner.wait(indexTask);
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/sparql-qlever",
-  "version": "0.8.5",
+  "version": "0.9.0",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/sparql-qlever"
@@ -25,7 +25,7 @@
   "dependencies": {
     "@lde/dataset": "0.7.1",
     "@lde/distribution-downloader": "0.5.1",
-    "@lde/sparql-importer": "0.3.1",
+    "@lde/sparql-importer": "0.4.0",
     "@lde/sparql-server": "0.4.10",
     "@lde/task-runner": "0.2.10",
     "@lde/task-runner-docker": "0.2.11",