@lde/sparql-qlever 0.10.1 → 0.11.0

package/dist/importer.d.ts

@@ -1,5 +1,5 @@
 import { Importer as ImporterInterface, ImportFailed, ImportSuccessful, NotSupported } from '@lde/sparql-importer';
-import { Dataset } from '@lde/dataset';
+import { Distribution } from '@lde/dataset';
 import { Downloader } from '@lde/distribution-downloader';
 import { TaskRunner } from '@lde/task-runner';
 export interface Options {
@@ -26,7 +26,7 @@ export declare class Importer implements ImporterInterface {
     private qleverOptions;
     private cacheIndex;
     constructor({ taskRunner, downloader, indexName, qleverOptions, cacheIndex, }: Options);
-    import(dataset: Dataset): Promise<NotSupported | ImportSuccessful | ImportFailed>;
+    import(distributions: Distribution[]): Promise<NotSupported | ImportSuccessful | ImportFailed>;
     private doImport;
     private fileFormatFromMimeType;
     private parseTripleCount;

package/dist/importer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,IAAI,iBAAiB,EAC7B,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACb,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AACrD,OAAO,EACL,UAAU,EAEX,MAAM,8BAA8B,CAAC;AAGtC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAU9C,MAAM,WAAW,OAAO;IACtB,UAAU,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,aAAa,CAAC,EAAE;QACd,qBAAqB,EAAE,OAAO,CAAC;QAC/B,uBAAuB,EAAE,MAAM,CAAC;KACjC,CAAC;IACF,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,mGAAmG;IACnG,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAMD;;;;GAIG;AACH,qBAAa,QAAS,YAAW,iBAAiB;IAChD,OAAO,CAAC,SAAS,CAAC;IAClB,OAAO,CAAC,UAAU,CAAsB;IACxC,OAAO,CAAC,UAAU,CAAC;IACnB,OAAO,CAAC,aAAa,CAAC;IACtB,OAAO,CAAC,UAAU,CAAC;gBAEP,EACV,UAAU,EACV,UAAU,EACV,SAAS,EACT,aAAa,EACb,UAAU,GACX,EAAE,OAAO;IAWG,MAAM,CACjB,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,YAAY,GAAG,gBAAgB,GAAG,YAAY,CAAC;YAiC5C,QAAQ;IAqBtB,OAAO,CAAC,sBAAsB;IAQ9B,OAAO,CAAC,gBAAgB;IAQxB,OAAO,CAAC,aAAa;IAIrB;;OAEG;YACW,eAAe;IA0B7B,yDAAyD;YAC3C,eAAe;YAaf,cAAc;YAKd,KAAK;CAmBpB"}
+{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,IAAI,iBAAiB,EAC7B,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACb,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EACL,UAAU,EAEX,MAAM,8BAA8B,CAAC;AAGtC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAU9C,MAAM,WAAW,OAAO;IACtB,UAAU,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,aAAa,CAAC,EAAE;QACd,qBAAqB,EAAE,OAAO,CAAC;QAC/B,uBAAuB,EAAE,MAAM,CAAC;KACjC,CAAC;IACF,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,mGAAmG;IACnG,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAMD;;;;GAIG;AACH,qBAAa,QAAS,YAAW,iBAAiB;IAChD,OAAO,CAAC,SAAS,CAAC;IAClB,OAAO,CAAC,UAAU,CAAsB;IACxC,OAAO,CAAC,UAAU,CAAC;IACnB,OAAO,CAAC,aAAa,CAAC;IACtB,OAAO,CAAC,UAAU,CAAC;gBAEP,EACV,UAAU,EACV,UAAU,EACV,SAAS,EACT,aAAa,EACb,UAAU,GACX,EAAE,OAAO;IAWG,MAAM,CACjB,aAAa,EAAE,YAAY,EAAE,GAC5B,OAAO,CAAC,YAAY,GAAG,gBAAgB,GAAG,YAAY,CAAC;YA+B5C,QAAQ;IA4CtB,OAAO,CAAC,sBAAsB;IAQ9B,OAAO,CAAC,gBAAgB;IAQxB,OAAO,CAAC,aAAa;IAIrB;;OAEG;YACW,eAAe;IA0B7B,yDAAyD;YAC3C,eAAe;YAaf,cAAc;YAKd,KAAK;CAyBpB"}

package/dist/importer.js

@@ -28,10 +28,8 @@ export class Importer {
         };
         this.cacheIndex = cacheIndex ?? true;
     }
-    async import(dataset) {
-        const downloadDistributions = dataset
-            .getDownloadDistributions()
-            .filter((distribution) => distribution.mimeType !== undefined &&
+    async import(distributions) {
+        const downloadDistributions = distributions.filter((distribution) => distribution.mimeType !== undefined &&
             supportedFormats.has(distribution.mimeType));
         if (downloadDistributions.length === 0) {
             return new NotSupported();
@@ -61,10 +59,29 @@ export class Importer {
         const localFile = await this.downloader.download(distribution);
         if (await this.isIndexUpToDate(localFile)) {
             const tripleCount = await this.readTripleCount(localFile);
+            if (tripleCount === 0) {
+                return new ImportFailed(distribution, 'Index is cached but contains 0 triples');
+            }
             return new ImportSuccessful(distribution, undefined, tripleCount);
         }
-        const logs = await this.index(localFile, this.fileFormatFromMimeType(distribution.mimeType));
+        const format = this.fileFormatFromMimeType(distribution.mimeType);
+        let logs;
+        try {
+            logs = await this.index(localFile, format);
+        }
+        catch (error) {
+            if (format === 'ttl' &&
+                error.message?.includes('multiline string literal')) {
+                logs = await this.index(localFile, format, false);
+            }
+            else {
+                throw error;
+            }
+        }
         const tripleCount = this.parseTripleCount(logs);
+        if (tripleCount === 0) {
+            return new ImportFailed(distribution, 'Indexed 0 triples from distribution');
+        }
         await this.writeCacheInfo(localFile);
         return new ImportSuccessful(distribution, undefined, tripleCount);
     }
@@ -126,12 +143,12 @@ export class Importer {
         const info = { sourceFile: basename(dataFile) };
         await writeFile(this.cacheInfoPath(dataFile), JSON.stringify(info));
     }
-    async index(file, format) {
+    async index(file, format, parseParallel = true) {
         const settingsFile = 'index.settings.json';
         await writeFile(`${dirname(file)}/${settingsFile}`, JSON.stringify(this.qleverOptions));
         // TODO: write index to named volume instead of bind mount for better performance.
         const metadataFile = `${this.indexName}.meta-data.json`;
-        const indexTask = await this.taskRunner.run(`(gunzip -c '${basename(file)}' 2>/dev/null || cat '${basename(file)}') | qlever-index -i ${this.indexName} -s ${settingsFile} -F ${format} -p true -f - && cat ${metadataFile}`);
+        const indexTask = await this.taskRunner.run(`(gunzip -c '${basename(file)}' 2>/dev/null || cat '${basename(file)}') | qlever-index -i ${this.indexName} -s ${settingsFile} -F ${format} -p true${parseParallel ? '' : ' --parse-parallel false'} -f - && cat ${metadataFile}`);
         return await this.taskRunner.wait(indexTask);
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/sparql-qlever",
-  "version": "0.10.1",
+  "version": "0.11.0",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/sparql-qlever"
@@ -25,7 +25,7 @@
   "dependencies": {
     "@lde/dataset": "0.7.2",
     "@lde/distribution-downloader": "0.5.3",
-    "@lde/sparql-importer": "0.4.1",
+    "@lde/sparql-importer": "0.4.2",
     "@lde/sparql-server": "0.4.10",
     "@lde/task-runner": "0.2.10",
     "@lde/task-runner-docker": "0.2.11",