@lde/sparql-qlever 0.10.2 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/importer.d.ts.map +1 -1
- package/dist/importer.js +16 -3
- package/package.json +1 -1
package/dist/importer.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,IAAI,iBAAiB,EAC7B,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACb,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EACL,UAAU,EAEX,MAAM,8BAA8B,CAAC;AAGtC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAU9C,MAAM,WAAW,OAAO;IACtB,UAAU,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,aAAa,CAAC,EAAE;QACd,qBAAqB,EAAE,OAAO,CAAC;QAC/B,uBAAuB,EAAE,MAAM,CAAC;KACjC,CAAC;IACF,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,mGAAmG;IACnG,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAMD;;;;GAIG;AACH,qBAAa,QAAS,YAAW,iBAAiB;IAChD,OAAO,CAAC,SAAS,CAAC;IAClB,OAAO,CAAC,UAAU,CAAsB;IACxC,OAAO,CAAC,UAAU,CAAC;IACnB,OAAO,CAAC,aAAa,CAAC;IACtB,OAAO,CAAC,UAAU,CAAC;gBAEP,EACV,UAAU,EACV,UAAU,EACV,SAAS,EACT,aAAa,EACb,UAAU,GACX,EAAE,OAAO;IAWG,MAAM,CACjB,aAAa,EAAE,YAAY,EAAE,GAC5B,OAAO,CAAC,YAAY,GAAG,gBAAgB,GAAG,YAAY,CAAC;YA+B5C,QAAQ;
|
|
1
|
+
{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,IAAI,iBAAiB,EAC7B,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACb,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EACL,UAAU,EAEX,MAAM,8BAA8B,CAAC;AAGtC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAU9C,MAAM,WAAW,OAAO;IACtB,UAAU,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,aAAa,CAAC,EAAE;QACd,qBAAqB,EAAE,OAAO,CAAC;QAC/B,uBAAuB,EAAE,MAAM,CAAC;KACjC,CAAC;IACF,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,mGAAmG;IACnG,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAMD;;;;GAIG;AACH,qBAAa,QAAS,YAAW,iBAAiB;IAChD,OAAO,CAAC,SAAS,CAAC;IAClB,OAAO,CAAC,UAAU,CAAsB;IACxC,OAAO,CAAC,UAAU,CAAC;IACnB,OAAO,CAAC,aAAa,CAAC;IACtB,OAAO,CAAC,UAAU,CAAC;gBAEP,EACV,UAAU,EACV,UAAU,EACV,SAAS,EACT,aAAa,EACb,UAAU,GACX,EAAE,OAAO;IAWG,MAAM,CACjB,aAAa,EAAE,YAAY,EAAE,GAC5B,OAAO,CAAC,YAAY,GAAG,gBAAgB,GAAG,YAAY,CAAC;YA+B5C,QAAQ;IA4CtB,OAAO,CAAC,sBAAsB;IAQ9B,OAAO,CAAC,gBAAgB;IAQxB,OAAO,CAAC,aAAa;IAIrB;;OAEG;YACW,eAAe;IA0B7B,yDAAyD;YAC3C,eAAe;YAaf,cAAc;YAKd,KAAK;CAyBpB"}
|
package/dist/importer.js
CHANGED
|
@@ -64,7 +64,20 @@ export class Importer {
|
|
|
64
64
|
}
|
|
65
65
|
return new ImportSuccessful(distribution, undefined, tripleCount);
|
|
66
66
|
}
|
|
67
|
-
const
|
|
67
|
+
const format = this.fileFormatFromMimeType(distribution.mimeType);
|
|
68
|
+
let logs;
|
|
69
|
+
try {
|
|
70
|
+
logs = await this.index(localFile, format);
|
|
71
|
+
}
|
|
72
|
+
catch (error) {
|
|
73
|
+
if (format === 'ttl' &&
|
|
74
|
+
error.message?.includes('multiline string literal')) {
|
|
75
|
+
logs = await this.index(localFile, format, false);
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
throw error;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
68
81
|
const tripleCount = this.parseTripleCount(logs);
|
|
69
82
|
if (tripleCount === 0) {
|
|
70
83
|
return new ImportFailed(distribution, 'Indexed 0 triples from distribution');
|
|
@@ -130,12 +143,12 @@ export class Importer {
|
|
|
130
143
|
const info = { sourceFile: basename(dataFile) };
|
|
131
144
|
await writeFile(this.cacheInfoPath(dataFile), JSON.stringify(info));
|
|
132
145
|
}
|
|
133
|
-
async index(file, format) {
|
|
146
|
+
async index(file, format, parseParallel = true) {
|
|
134
147
|
const settingsFile = 'index.settings.json';
|
|
135
148
|
await writeFile(`${dirname(file)}/${settingsFile}`, JSON.stringify(this.qleverOptions));
|
|
136
149
|
// TODO: write index to named volume instead of bind mount for better performance.
|
|
137
150
|
const metadataFile = `${this.indexName}.meta-data.json`;
|
|
138
|
-
const indexTask = await this.taskRunner.run(`(gunzip -c '${basename(file)}' 2>/dev/null || cat '${basename(file)}') | qlever-index -i ${this.indexName} -s ${settingsFile} -F ${format} -p true -f - && cat ${metadataFile}`);
|
|
151
|
+
const indexTask = await this.taskRunner.run(`(gunzip -c '${basename(file)}' 2>/dev/null || cat '${basename(file)}') | qlever-index -i ${this.indexName} -s ${settingsFile} -F ${format} -p true${parseParallel ? '' : ' --parse-parallel false'} -f - && cat ${metadataFile}`);
|
|
139
152
|
return await this.taskRunner.wait(indexTask);
|
|
140
153
|
}
|
|
141
154
|
}
|