@lde/sparql-qlever 0.8.6 → 0.9.0

package/dist/importer.d.ts

@@ -26,6 +26,7 @@ export declare class Importer implements ImporterInterface {
     import(dataset: Dataset): Promise<NotSupported | ImportSuccessful | ImportFailed>;
     private doImport;
     private fileFormatFromMimeType;
+    private parseTripleCount;
     private index;
 }
 //# sourceMappingURL=importer.d.ts.map

package/dist/importer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,IAAI,iBAAiB,EAC7B,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACb,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AACrD,OAAO,EACL,UAAU,EAEX,MAAM,8BAA8B,CAAC;AAGtC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAU9C,MAAM,WAAW,OAAO;IACtB,UAAU,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,aAAa,CAAC,EAAE;QACd,qBAAqB,EAAE,OAAO,CAAC;QAC/B,uBAAuB,EAAE,MAAM,CAAC;KACjC,CAAC;IACF,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;;;GAIG;AACH,qBAAa,QAAS,YAAW,iBAAiB;IAChD,OAAO,CAAC,SAAS,CAAC;IAClB,OAAO,CAAC,UAAU,CAAsB;IACxC,OAAO,CAAC,UAAU,CAAC;IACnB,OAAO,CAAC,aAAa,CAAC;gBAEV,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,EAAE,OAAO;IAU5D,MAAM,CACjB,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,YAAY,GAAG,gBAAgB,GAAG,YAAY,CAAC;YAiC5C,QAAQ;IAYtB,OAAO,CAAC,sBAAsB;YAQhB,KAAK;CAsBpB"}
+{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,IAAI,iBAAiB,EAC7B,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACb,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AACrD,OAAO,EACL,UAAU,EAEX,MAAM,8BAA8B,CAAC;AAGtC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAU9C,MAAM,WAAW,OAAO;IACtB,UAAU,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,aAAa,CAAC,EAAE;QACd,qBAAqB,EAAE,OAAO,CAAC;QAC/B,uBAAuB,EAAE,MAAM,CAAC;KACjC,CAAC;IACF,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;;;GAIG;AACH,qBAAa,QAAS,YAAW,iBAAiB;IAChD,OAAO,CAAC,SAAS,CAAC;IAClB,OAAO,CAAC,UAAU,CAAsB;IACxC,OAAO,CAAC,UAAU,CAAC;IACnB,OAAO,CAAC,aAAa,CAAC;gBAEV,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,EAAE,OAAO;IAU5D,MAAM,CACjB,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,YAAY,GAAG,gBAAgB,GAAG,YAAY,CAAC;YAiC5C,QAAQ;IAatB,OAAO,CAAC,sBAAsB;IAQ9B,OAAO,CAAC,gBAAgB;YAQV,KAAK;CAuBpB"}

package/dist/importer.js

@@ -57,8 +57,9 @@ export class Importer {
     }
     async doImport(distribution) {
         const localFile = await this.downloader.download(distribution);
-        await this.index(localFile, this.fileFormatFromMimeType(distribution.mimeType));
-        return new ImportSuccessful(distribution);
+        const logs = await this.index(localFile, this.fileFormatFromMimeType(distribution.mimeType));
+        const tripleCount = this.parseTripleCount(logs);
+        return new ImportSuccessful(distribution, undefined, tripleCount);
     }
     fileFormatFromMimeType(mimeType) {
         const format = supportedFormats.get(mimeType);
@@ -67,6 +68,13 @@ export class Importer {
         }
         return format;
     }
+    parseTripleCount(logs) {
+        // Extract num-triples.normal from the metadata JSON that the index
+        // command cats to stdout. Use a regex rather than JSON.parse because
+        // Docker log multiplexing prepends binary frame headers to each chunk.
+        const match = logs.match(/"num-triples":\{[^}]*"normal":(\d+)/);
+        return match ? Number(match[1]) : undefined;
+    }
     async index(file, format) {
         const workingDir = dirname(file);
         const settingsFile = 'index.settings.json';
@@ -77,7 +85,8 @@ export class Importer {
             : this.qleverOptions;
         await writeFile(`${workingDir}/${settingsFile}`, JSON.stringify(settings));
         // TODO: write index to named volume instead of bind mount for better performance.
-        const indexTask = await this.taskRunner.run(`(zcat '${basename(file)}' 2>/dev/null || cat '${basename(file)}') | qlever-index -i ${this.indexName} -s ${settingsFile} -F ${format} -f -`);
-        await this.taskRunner.wait(indexTask);
+        const metadataFile = `${this.indexName}.meta-data.json`;
+        const indexTask = await this.taskRunner.run(`(zcat '${basename(file)}' 2>/dev/null || cat '${basename(file)}') | qlever-index -i ${this.indexName} -s ${settingsFile} -F ${format} -f - && cat ${metadataFile}`);
+        return await this.taskRunner.wait(indexTask);
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/sparql-qlever",
-  "version": "0.8.6",
+  "version": "0.9.0",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/sparql-qlever"
@@ -25,7 +25,7 @@
   "dependencies": {
     "@lde/dataset": "0.7.1",
     "@lde/distribution-downloader": "0.5.1",
-    "@lde/sparql-importer": "0.3.1",
+    "@lde/sparql-importer": "0.4.0",
     "@lde/sparql-server": "0.4.10",
     "@lde/task-runner": "0.2.10",
     "@lde/task-runner-docker": "0.2.11",