@lde/sparql-qlever 0.15.3 → 0.15.5

package/README.md

@@ -1,45 +1,13 @@
-# SPARQL QLever
+# @lde/sparql-qlever
 
 An adapter for the [QLever](https://github.com/ad-freiburg/qlever) SPARQL server.
 
-## Index caching
+## Installation
 
-Building a QLever index is slow. To avoid rebuilding it on every pipeline run, the importer caches a single index and reuses it when the source data hasn't changed. On subsequent runs, indexing is skipped when the source file matches and hasn't been re-downloaded.
-
-Only **one** index is cached at a time. In a multi-dataset pipeline, each dataset overwrites the previous index. On re-run, the last-indexed dataset gets a cache hit while the others rebuild.
-
-Caching is enabled by default. Disable it by passing `cacheIndex: false` to `createQlever()` or the `Importer` constructor (e.g. driven by a `QLEVER_CACHE_INDEX=false` environment variable).
-
-## Networking (Docker mode)
-
-By default `createQlever({ mode: 'docker' })` publishes QLever’s port on the host and addresses the query endpoint as `localhost` – correct when the consumer runs on the host, or shares the QLever container’s network namespace. A consumer that itself runs in a container on a bridge network cannot reach a host-published port; pass `network` (a Docker network the consumer is attached to) together with `containerName`, and QLever joins that network with the endpoint addressed by container name instead:
-
-```ts
-const { importer, server } = createQlever({
-  mode: 'docker',
-  image: 'adfreiburg/qlever:latest',
-  network: 'app_default',
-  containerName: 'qlever',
-});
-server.queryEndpoint.toString(); // 'http://qlever:7001/sparql'
+```sh
+npm install @lde/sparql-qlever
 ```
 
-## Configuration
-
-`createQlever()` accepts `indexOptions` and `serverOptions` to tune QLever's index builder and server respectively.
-
-### Server options (`serverOptions`)
-
-Passed to `qlever-server` at startup.
-
-| Option                  | Description                                      | Default |
-| ----------------------- | ------------------------------------------------ | ------- |
-| `memory-max-size`       | Maximum memory for query processing and caching. | `'4G'`  |
-| `default-query-timeout` | Default query timeout.                           | `'30s'` |
-| `cache-max-size`        | Maximum cache size for query results.            | –       |
-
-Example:
-
 ```ts
 const { importer, server } = createQlever({
   mode: 'docker',
@@ -51,14 +19,6 @@ const { importer, server } = createQlever({
 });
 ```
 
-### Index options (`indexOptions`)
-
-Passed to `qlever-index` during import.
+## Documentation
 
-| Option                          | Description                                                                                   | Default     |
-| ------------------------------- | --------------------------------------------------------------------------------------------- | ----------- |
-| `ascii-prefixes-only`           | Enable faster parsing for well-behaved TTL files.                                             | `true`      |
-| `num-triples-per-batch`         | Triples per batch; lower values reduce memory usage.                                          | `3_000_000` |
-| `stxxl-memory`                  | Memory budget for sorting during the index build.                                             | `'10G'`     |
-| `parallel-parsing`              | Parse input in parallel.                                                                      | `true`      |
-| `only-pso-and-pos-permutations` | Build only PSO and POS permutations. Faster, but queries with predicate variables won't work. | `false`     |
+See the [full documentation](https://ldelements.org/reference/sparql-qlever).

package/dist/preprocess.d.ts

@@ -12,8 +12,8 @@ export interface PreprocessResult {
  * JSON-LD, RDF/XML and TriG distributions return `true`: `qlever-index` cannot
  * parse any of them, so we stream them through `rdf-parse` into N-Quads first.
  *
- * Native RDF formats (`nt`, `nq`, `ttl`) — including when wrapped in
- * `application/gzip` or `application/zip` — go straight through the shell
+ * Native RDF formats (`nt`, `nq`, `ttl`) – including when wrapped in
+ * `application/gzip` or `application/zip` – go straight through the shell
  * pipeline in `index()`, which uses `gunzip -c` or `unzip -p` as appropriate.
  * Standalone `mediaType=application/zip` is rejected upstream: the inner
  * format must be declared.
@@ -25,8 +25,8 @@ export declare function needsPreprocessing(distribution: Distribution): boolean;
  *
  * Streams the source through `rdf-parse` → `rdf-serialize` so memory use
  * stays bounded regardless of input size. Handles gzip transparently
- * (declared `compressFormat` or `.gz` filename) and zip containers (folds
- * every parseable entry into the output stream in order).
+ * (detected from the file’s magic bytes, not from declared metadata) and zip
+ * containers (folds every parseable entry into the output stream in order).
  *
  * Cached: if the output is newer than the input, it is reused as-is.
  */

package/dist/preprocess.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"preprocess.d.ts","sourceRoot":"","sources":["../src/preprocess.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AA6B5C,MAAM,WAAW,gBAAgB;IAC/B,iEAAiE;IACjE,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,IAAI,CAAC;IACb,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,kBAAkB,CAAC,YAAY,EAAE,YAAY,GAAG,OAAO,CAKtE;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,UAAU,CAC9B,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,YAAY,GACzB,OAAO,CAAC,gBAAgB,CAAC,CAyB3B"}
+{"version":3,"file":"preprocess.d.ts","sourceRoot":"","sources":["../src/preprocess.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AA2B5C,MAAM,WAAW,gBAAgB;IAC/B,iEAAiE;IACjE,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,IAAI,CAAC;IACb,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,kBAAkB,CAAC,YAAY,EAAE,YAAY,GAAG,OAAO,CAKtE;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,UAAU,CAC9B,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,YAAY,GACzB,OAAO,CAAC,gBAAgB,CAAC,CAyB3B"}

package/dist/preprocess.js

@@ -2,7 +2,7 @@ import { rdfParser } from 'rdf-parse';
 import { rdfSerializer } from 'rdf-serialize';
 import { createGunzip } from 'node:zlib';
 import { createReadStream, createWriteStream } from 'node:fs';
-import { rm, stat } from 'node:fs/promises';
+import { open, rm, stat } from 'node:fs/promises';
 import { finished } from 'node:stream/promises';
 import { promisify } from 'node:util';
 import yauzl from 'yauzl';
@@ -10,8 +10,6 @@ const JSONLD_MIME = 'application/ld+json';
 const RDFXML_MIME = 'application/rdf+xml';
 const TRIG_MIME = 'application/trig';
 const ZIP_MIME = 'application/zip';
-const GZIP_MIME = 'application/gzip';
-const GZIP_MIME_LEGACY = 'application/x-gzip';
 /**
  * RDF media types `qlever-index` cannot read natively, keyed by `rdf-parse`
  * `contentType` with a label for warnings. TriG belongs here too: its
@@ -29,8 +27,8 @@ const preprocessFormats = new Map([
  * JSON-LD, RDF/XML and TriG distributions return `true`: `qlever-index` cannot
  * parse any of them, so we stream them through `rdf-parse` into N-Quads first.
  *
- * Native RDF formats (`nt`, `nq`, `ttl`) — including when wrapped in
- * `application/gzip` or `application/zip` — go straight through the shell
+ * Native RDF formats (`nt`, `nq`, `ttl`) – including when wrapped in
+ * `application/gzip` or `application/zip` – go straight through the shell
  * pipeline in `index()`, which uses `gunzip -c` or `unzip -p` as appropriate.
  * Standalone `mediaType=application/zip` is rejected upstream: the inner
  * format must be declared.
@@ -45,8 +43,8 @@ export function needsPreprocessing(distribution) {
  *
  * Streams the source through `rdf-parse` → `rdf-serialize` so memory use
  * stays bounded regardless of input size. Handles gzip transparently
- * (declared `compressFormat` or `.gz` filename) and zip containers (folds
- * every parseable entry into the output stream in order).
+ * (detected from the file’s magic bytes, not from declared metadata) and zip
+ * containers (folds every parseable entry into the output stream in order).
  *
  * Cached: if the output is newer than the input, it is reused as-is.
  */
@@ -66,7 +64,7 @@ export async function preprocess(localFile, distribution) {
         await streamRdfZip(localFile, outputFile, contentType, label, warnings);
     }
     else {
-        await streamRdfFile(localFile, outputFile, contentType, distribution);
+        await streamRdfFile(localFile, outputFile, contentType);
     }
     return { path: outputFile, format: 'nq', warnings };
 }
@@ -102,12 +100,11 @@ async function closeWritable(output) {
         output.end();
     });
 }
-async function streamRdfFile(localFile, outputFile, contentType, distribution) {
-    const isGzipped = distribution.compressMimeType === GZIP_MIME ||
-        distribution.compressMimeType === GZIP_MIME_LEGACY ||
-        localFile.toLowerCase().endsWith('.gz');
+async function streamRdfFile(localFile, outputFile, contentType) {
     const source = createReadStream(localFile);
-    const input = isGzipped ? source.pipe(createGunzip()) : source;
+    const input = (await isGzipFile(localFile))
+        ? source.pipe(createGunzip())
+        : source;
     const output = createWriteStream(outputFile);
     try {
         await pipeRdfToWritable(input, output, contentType);
@@ -116,6 +113,25 @@ async function streamRdfFile(localFile, outputFile, contentType, distribution) {
         await closeWritable(output);
     }
 }
+/**
+ * Whether the file begins with the gzip magic number (RFC 1952 §2.3.1).
+ *
+ * Sniffing the bytes is the only reliable signal for a downloaded dump.
+ * `dcat:compressFormat` is frequently absent, and the local filename is derived
+ * from the access URL, so a dump behind a query string (`…/download.trig.gz?graph=…`)
+ * carries no `.gz` suffix. Conversely, a `.gz` name says nothing once the server
+ * declared a real `Content-Encoding: gzip` and `fetch` already inflated the body.
+ */
+async function isGzipFile(path) {
+    const file = await open(path);
+    try {
+        const { buffer, bytesRead } = await file.read(Buffer.alloc(2), 0, 2, 0);
+        return bytesRead === 2 && buffer[0] === 0x1f && buffer[1] === 0x8b;
+    }
+    finally {
+        await file.close();
+    }
+}
 const openZip = promisify(yauzl.open);
 /**
  * Fold every parseable entry of a zip into the N-Quads output, in order. The

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/sparql-qlever",
-  "version": "0.15.3",
+  "version": "0.15.5",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/sparql-qlever"
@@ -31,7 +31,7 @@
     "@lde/task-runner": "^0.2.13",
     "@lde/task-runner-docker": "^0.2.16",
     "@lde/task-runner-native": "^0.2.16",
-    "@lde/wait-for-sparql": "^0.2.16",
+    "@lde/wait-for-sparql": "^0.2.17",
     "rdf-parse": "^5.0.0",
     "rdf-serialize": "^5.1.0",
     "tslib": "^2.3.0",