@lde/sparql-qlever 0.14.11 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/importer.d.ts +1 -1
- package/dist/importer.d.ts.map +1 -1
- package/dist/importer.js +10 -8
- package/package.json +8 -8
package/README.md
CHANGED
|
@@ -22,7 +22,7 @@ Passed to `qlever-server` at startup.
|
|
|
22
22
|
| ----------------------- | ------------------------------------------------ | ------- |
|
|
23
23
|
| `memory-max-size` | Maximum memory for query processing and caching. | `'4G'` |
|
|
24
24
|
| `default-query-timeout` | Default query timeout. | `'30s'` |
|
|
25
|
-
| `cache-max-size` | Maximum cache size for query results. |
|
|
25
|
+
| `cache-max-size` | Maximum cache size for query results. | – |
|
|
26
26
|
|
|
27
27
|
Example:
|
|
28
28
|
|
|
@@ -46,5 +46,5 @@ Passed to `qlever-index` during import.
|
|
|
46
46
|
| `ascii-prefixes-only` | Enable faster parsing for well-behaved TTL files. | `true` |
|
|
47
47
|
| `num-triples-per-batch` | Triples per batch; lower values reduce memory usage. | `3_000_000` |
|
|
48
48
|
| `stxxl-memory` | Memory budget for sorting during the index build. | `'10G'` |
|
|
49
|
-
| `
|
|
49
|
+
| `parallel-parsing` | Parse input in parallel. | `true` |
|
|
50
50
|
| `only-pso-and-pos-permutations` | Build only PSO and POS permutations. Faster, but queries with predicate variables won't work. | `false` |
|
package/dist/importer.d.ts
CHANGED
|
@@ -8,7 +8,7 @@ export interface QleverIndexOptions {
|
|
|
8
8
|
/** Memory budget for sorting during the index build. @default '10G' */
|
|
9
9
|
'stxxl-memory'?: string;
|
|
10
10
|
/** @default true */
|
|
11
|
-
'
|
|
11
|
+
'parallel-parsing'?: boolean;
|
|
12
12
|
/** Build only PSO and POS permutations. Faster, but queries with predicate variables won't work. Also disables pattern precomputation. @default false */
|
|
13
13
|
'only-pso-and-pos-permutations'?: boolean;
|
|
14
14
|
}
|
package/dist/importer.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,IAAI,iBAAiB,EAC7B,eAAe,EACf,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACb,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAyB,YAAY,EAAE,MAAM,cAAc,CAAC;AAMnE,MAAM,WAAW,kBAAkB;IACjC,oBAAoB;IACpB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,yBAAyB;IACzB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,uEAAuE;IACvE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,oBAAoB;IACpB,
|
|
1
|
+
{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,IAAI,iBAAiB,EAC7B,eAAe,EACf,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACb,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAyB,YAAY,EAAE,MAAM,cAAc,CAAC;AAMnE,MAAM,WAAW,kBAAkB;IACjC,oBAAoB;IACpB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,yBAAyB;IACzB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,uEAAuE;IACvE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,oBAAoB;IACpB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,yJAAyJ;IACzJ,+BAA+B,CAAC,EAAE,OAAO,CAAC;CAC3C;AAED,MAAM,WAAW,qBAAsB,SAAQ,eAAe;IAC5D,sBAAsB;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,kBAAkB,CAAC;CACpC;AAWD,qBAAa,QAAS,YAAW,iBAAiB;IAChD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkB;gBAE9B,OAAO,EAAE,qBAAqB;IAa7B,MAAM,CACjB,aAAa,EAAE,YAAY,EAAE,GAC5B,OAAO,CAAC,YAAY,GAAG,gBAAgB,GAAG,YAAY,CAAC;YAqC5C,QAAQ;IA6DtB,OAAO,CAAC,gBAAgB;IAQxB,OAAO,CAAC,aAAa;IAIrB;;OAEG;YACW,eAAe;IA0B7B,yDAAyD;YAC3C,eAAe;YAaf,cAAc;YAKd,KAAK;CA8CpB"}
|
package/dist/importer.js
CHANGED
|
@@ -114,7 +114,7 @@ export class Importer {
|
|
|
114
114
|
cacheInfo = JSON.parse(raw);
|
|
115
115
|
}
|
|
116
116
|
catch {
|
|
117
|
-
return false; // No cache marker
|
|
117
|
+
return false; // No cache marker – first run.
|
|
118
118
|
}
|
|
119
119
|
if (cacheInfo.sourceFile !== basename(dataFile)) {
|
|
120
120
|
return false; // Different dataset was last indexed.
|
|
@@ -151,12 +151,14 @@ export class Importer {
|
|
|
151
151
|
};
|
|
152
152
|
await writeFile(`${dirname(file)}/${settingsFile}`, JSON.stringify(settings));
|
|
153
153
|
// TODO: write index to named volume instead of bind mount for better performance.
|
|
154
|
-
const parallel = parseParallel ?? this.options.qleverOptions['
|
|
154
|
+
const parallel = parseParallel ?? this.options.qleverOptions['parallel-parsing'];
|
|
155
155
|
const flags = [
|
|
156
156
|
`-i ${this.options.indexName}`,
|
|
157
157
|
`-s ${settingsFile}`,
|
|
158
158
|
`-F ${format}`,
|
|
159
|
-
|
|
159
|
+
// Short flag because the long flag is version-dependent: current QLever
|
|
160
|
+
// spells it --parallel-parsing, older versions --parse-parallel.
|
|
161
|
+
`-p ${parallel}`,
|
|
160
162
|
`-m ${this.options.qleverOptions['stxxl-memory']}`,
|
|
161
163
|
this.options.qleverOptions['only-pso-and-pos-permutations']
|
|
162
164
|
? '-o --no-patterns'
|
|
@@ -178,9 +180,9 @@ export class Importer {
|
|
|
178
180
|
* POSIX-quote a value for safe interpolation into a shell command: wrap it in
|
|
179
181
|
* single quotes and escape any embedded single quote as `'\''`.
|
|
180
182
|
*
|
|
181
|
-
* Without this, a data filename containing an apostrophe
|
|
183
|
+
* Without this, a data filename containing an apostrophe – e.g. a dataset
|
|
182
184
|
* titled `'s-Hertogenbosch`, whose distribution URL maps to a local file like
|
|
183
|
-
* `…Erfgoed+'s-Hertogenbosch.nt`
|
|
185
|
+
* `…Erfgoed+'s-Hertogenbosch.nt` – would terminate the surrounding quotes, so
|
|
184
186
|
* `cat`/`gunzip` would read a non-existent path and feed `qlever-index` empty
|
|
185
187
|
* input. The index then "succeeds" with 0 triples, the import is treated as
|
|
186
188
|
* failed, and every distribution (and the JSON-LD fallback) fails the same way.
|
|
@@ -189,7 +191,7 @@ function shellQuote(value) {
|
|
|
189
191
|
return `'${value.replace(/'/g, "'\\''")}'`;
|
|
190
192
|
}
|
|
191
193
|
/**
|
|
192
|
-
* Native QLever index formats
|
|
194
|
+
* Native QLever index formats – `qlever-index -F <flag>` consumes these
|
|
193
195
|
* directly. JSON-LD is not here: it is preprocessed to N-Quads first (see
|
|
194
196
|
* {@link preprocess}).
|
|
195
197
|
*/
|
|
@@ -204,7 +206,7 @@ const nativeFormats = new Map([
|
|
|
204
206
|
* skip the Node-side preprocessor; TriG comes last so a streaming native dump
|
|
205
207
|
* is always preferred when a dataset offers both.
|
|
206
208
|
*
|
|
207
|
-
* `application/zip` is intentionally absent
|
|
209
|
+
* `application/zip` is intentionally absent – the inner RDF format must be
|
|
208
210
|
* declared via `mediaType` with `application/zip` appearing only as the
|
|
209
211
|
* `compressFormat`, so we know what is inside.
|
|
210
212
|
*/
|
|
@@ -220,7 +222,7 @@ const defaultQleverIndexOptions = {
|
|
|
220
222
|
'ascii-prefixes-only': true,
|
|
221
223
|
'num-triples-per-batch': 3_000_000,
|
|
222
224
|
'stxxl-memory': '10G',
|
|
223
|
-
'
|
|
225
|
+
'parallel-parsing': true,
|
|
224
226
|
'only-pso-and-pos-permutations': false,
|
|
225
227
|
};
|
|
226
228
|
const extensionFormats = new Map([
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lde/sparql-qlever",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.15.0",
|
|
4
4
|
"repository": {
|
|
5
5
|
"url": "git+https://github.com/ldelements/lde.git",
|
|
6
6
|
"directory": "packages/sparql-qlever"
|
|
@@ -25,13 +25,13 @@
|
|
|
25
25
|
],
|
|
26
26
|
"dependencies": {
|
|
27
27
|
"@lde/dataset": "^0.7.8",
|
|
28
|
-
"@lde/distribution-downloader": "^0.6.
|
|
29
|
-
"@lde/sparql-importer": "^0.6.
|
|
30
|
-
"@lde/sparql-server": "^0.4.
|
|
31
|
-
"@lde/task-runner": "^0.2.
|
|
32
|
-
"@lde/task-runner-docker": "^0.2.
|
|
33
|
-
"@lde/task-runner-native": "^0.2.
|
|
34
|
-
"@lde/wait-for-sparql": "^0.2.
|
|
28
|
+
"@lde/distribution-downloader": "^0.6.7",
|
|
29
|
+
"@lde/sparql-importer": "^0.6.7",
|
|
30
|
+
"@lde/sparql-server": "^0.4.12",
|
|
31
|
+
"@lde/task-runner": "^0.2.12",
|
|
32
|
+
"@lde/task-runner-docker": "^0.2.14",
|
|
33
|
+
"@lde/task-runner-native": "^0.2.15",
|
|
34
|
+
"@lde/wait-for-sparql": "^0.2.15",
|
|
35
35
|
"rdf-parse": "^5.0.0",
|
|
36
36
|
"rdf-serialize": "^5.1.0",
|
|
37
37
|
"tslib": "^2.3.0",
|