@lde/sparql-qlever 0.15.4 → 0.15.6

package/dist/importer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,IAAI,iBAAiB,EAC7B,eAAe,EACf,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACb,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAyB,YAAY,EAAE,MAAM,cAAc,CAAC;AAMnE,MAAM,WAAW,kBAAkB;IACjC,oBAAoB;IACpB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,yBAAyB;IACzB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,uEAAuE;IACvE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,oBAAoB;IACpB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,yJAAyJ;IACzJ,+BAA+B,CAAC,EAAE,OAAO,CAAC;CAC3C;AAED,MAAM,WAAW,qBAAsB,SAAQ,eAAe;IAC5D,sBAAsB;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,kBAAkB,CAAC;CACpC;AAWD,qBAAa,QAAS,YAAW,iBAAiB;IAChD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkB;gBAE9B,OAAO,EAAE,qBAAqB;IAa7B,MAAM,CACjB,aAAa,EAAE,YAAY,EAAE,GAC5B,OAAO,CAAC,YAAY,GAAG,gBAAgB,GAAG,YAAY,CAAC;YAqC5C,QAAQ;IA6DtB,OAAO,CAAC,gBAAgB;IAQxB,OAAO,CAAC,aAAa;IAIrB;;OAEG;YACW,eAAe;IA0B7B,yDAAyD;YAC3C,eAAe;YAaf,cAAc;YAKd,KAAK;CA8CpB"}
+{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,IAAI,iBAAiB,EAC7B,eAAe,EACf,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACb,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAyB,YAAY,EAAE,MAAM,cAAc,CAAC;AAOnE,MAAM,WAAW,kBAAkB;IACjC,oBAAoB;IACpB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,yBAAyB;IACzB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,uEAAuE;IACvE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,oBAAoB;IACpB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,yJAAyJ;IACzJ,+BAA+B,CAAC,EAAE,OAAO,CAAC;CAC3C;AAED,MAAM,WAAW,qBAAsB,SAAQ,eAAe;IAC5D,sBAAsB;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,kBAAkB,CAAC;CACpC;AAWD,qBAAa,QAAS,YAAW,iBAAiB;IAChD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkB;gBAE9B,OAAO,EAAE,qBAAqB;IAa7B,MAAM,CACjB,aAAa,EAAE,YAAY,EAAE,GAC5B,OAAO,CAAC,YAAY,GAAG,gBAAgB,GAAG,YAAY,CAAC;YAqC5C,QAAQ;IA6DtB,OAAO,CAAC,gBAAgB;IAQxB,OAAO,CAAC,aAAa;IAIrB;;OAEG;YACW,eAAe;IA0B7B,yDAAyD;YAC3C,eAAe;YAaf,cAAc;YAKd,KAAK;CA8CpB"}

package/dist/importer.js

@@ -1,6 +1,7 @@
 import { ImportFailed, ImportSuccessful, NotSupported, } from '@lde/sparql-importer';
 import { compressionMediaTypes } from '@lde/dataset';
 import { LastModifiedDownloader } from '@lde/distribution-downloader';
+import { shellQuote } from '@lde/task-runner';
 import { basename, dirname, join } from 'path';
 import { readFile, stat, writeFile } from 'node:fs/promises';
 import { needsPreprocessing, preprocess } from './preprocess.js';
@@ -176,20 +177,6 @@ export class Importer {
         return await this.options.taskRunner.wait(indexTask);
     }
 }
-/**
- * POSIX-quote a value for safe interpolation into a shell command: wrap it in
- * single quotes and escape any embedded single quote as `'\''`.
- *
- * Without this, a data filename containing an apostrophe – e.g. a dataset
- * titled `'s-Hertogenbosch`, whose distribution URL maps to a local file like
- * `…Erfgoed+'s-Hertogenbosch.nt` – would terminate the surrounding quotes, so
- * `cat`/`gunzip` would read a non-existent path and feed `qlever-index` empty
- * input. The index then "succeeds" with 0 triples, the import is treated as
- * failed, and every distribution (and the JSON-LD fallback) fails the same way.
- */
-function shellQuote(value) {
-    return `'${value.replace(/'/g, "'\\''")}'`;
-}
 /**
  * Native QLever index formats – `qlever-index -F <flag>` consumes these
  * directly. JSON-LD is not here: it is preprocessed to N-Quads first (see

package/dist/preprocess.d.ts

@@ -12,8 +12,8 @@ export interface PreprocessResult {
  * JSON-LD, RDF/XML and TriG distributions return `true`: `qlever-index` cannot
  * parse any of them, so we stream them through `rdf-parse` into N-Quads first.
  *
- * Native RDF formats (`nt`, `nq`, `ttl`) — including when wrapped in
- * `application/gzip` or `application/zip` — go straight through the shell
+ * Native RDF formats (`nt`, `nq`, `ttl`) – including when wrapped in
+ * `application/gzip` or `application/zip` – go straight through the shell
  * pipeline in `index()`, which uses `gunzip -c` or `unzip -p` as appropriate.
  * Standalone `mediaType=application/zip` is rejected upstream: the inner
  * format must be declared.
@@ -25,8 +25,8 @@ export declare function needsPreprocessing(distribution: Distribution): boolean;
  *
  * Streams the source through `rdf-parse` → `rdf-serialize` so memory use
  * stays bounded regardless of input size. Handles gzip transparently
- * (declared `compressFormat` or `.gz` filename) and zip containers (folds
- * every parseable entry into the output stream in order).
+ * (detected from the file’s magic bytes, not from declared metadata) and zip
+ * containers (folds every parseable entry into the output stream in order).
  *
  * Cached: if the output is newer than the input, it is reused as-is.
  */

package/dist/preprocess.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"preprocess.d.ts","sourceRoot":"","sources":["../src/preprocess.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AA6B5C,MAAM,WAAW,gBAAgB;IAC/B,iEAAiE;IACjE,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,IAAI,CAAC;IACb,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,kBAAkB,CAAC,YAAY,EAAE,YAAY,GAAG,OAAO,CAKtE;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,UAAU,CAC9B,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,YAAY,GACzB,OAAO,CAAC,gBAAgB,CAAC,CAyB3B"}
+{"version":3,"file":"preprocess.d.ts","sourceRoot":"","sources":["../src/preprocess.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AA2B5C,MAAM,WAAW,gBAAgB;IAC/B,iEAAiE;IACjE,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,IAAI,CAAC;IACb,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,kBAAkB,CAAC,YAAY,EAAE,YAAY,GAAG,OAAO,CAKtE;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,UAAU,CAC9B,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,YAAY,GACzB,OAAO,CAAC,gBAAgB,CAAC,CAyB3B"}

package/dist/preprocess.js

@@ -2,7 +2,7 @@ import { rdfParser } from 'rdf-parse';
 import { rdfSerializer } from 'rdf-serialize';
 import { createGunzip } from 'node:zlib';
 import { createReadStream, createWriteStream } from 'node:fs';
-import { rm, stat } from 'node:fs/promises';
+import { open, rm, stat } from 'node:fs/promises';
 import { finished } from 'node:stream/promises';
 import { promisify } from 'node:util';
 import yauzl from 'yauzl';
@@ -10,8 +10,6 @@ const JSONLD_MIME = 'application/ld+json';
 const RDFXML_MIME = 'application/rdf+xml';
 const TRIG_MIME = 'application/trig';
 const ZIP_MIME = 'application/zip';
-const GZIP_MIME = 'application/gzip';
-const GZIP_MIME_LEGACY = 'application/x-gzip';
 /**
  * RDF media types `qlever-index` cannot read natively, keyed by `rdf-parse`
  * `contentType` with a label for warnings. TriG belongs here too: its
@@ -29,8 +27,8 @@ const preprocessFormats = new Map([
  * JSON-LD, RDF/XML and TriG distributions return `true`: `qlever-index` cannot
  * parse any of them, so we stream them through `rdf-parse` into N-Quads first.
  *
- * Native RDF formats (`nt`, `nq`, `ttl`) — including when wrapped in
- * `application/gzip` or `application/zip` — go straight through the shell
+ * Native RDF formats (`nt`, `nq`, `ttl`) – including when wrapped in
+ * `application/gzip` or `application/zip` – go straight through the shell
  * pipeline in `index()`, which uses `gunzip -c` or `unzip -p` as appropriate.
  * Standalone `mediaType=application/zip` is rejected upstream: the inner
  * format must be declared.
@@ -45,8 +43,8 @@ export function needsPreprocessing(distribution) {
  *
  * Streams the source through `rdf-parse` → `rdf-serialize` so memory use
  * stays bounded regardless of input size. Handles gzip transparently
- * (declared `compressFormat` or `.gz` filename) and zip containers (folds
- * every parseable entry into the output stream in order).
+ * (detected from the file’s magic bytes, not from declared metadata) and zip
+ * containers (folds every parseable entry into the output stream in order).
  *
  * Cached: if the output is newer than the input, it is reused as-is.
  */
@@ -66,7 +64,7 @@ export async function preprocess(localFile, distribution) {
         await streamRdfZip(localFile, outputFile, contentType, label, warnings);
     }
     else {
-        await streamRdfFile(localFile, outputFile, contentType, distribution);
+        await streamRdfFile(localFile, outputFile, contentType);
     }
     return { path: outputFile, format: 'nq', warnings };
 }
@@ -102,12 +100,11 @@ async function closeWritable(output) {
         output.end();
     });
 }
-async function streamRdfFile(localFile, outputFile, contentType, distribution) {
-    const isGzipped = distribution.compressMimeType === GZIP_MIME ||
-        distribution.compressMimeType === GZIP_MIME_LEGACY ||
-        localFile.toLowerCase().endsWith('.gz');
+async function streamRdfFile(localFile, outputFile, contentType) {
     const source = createReadStream(localFile);
-    const input = isGzipped ? source.pipe(createGunzip()) : source;
+    const input = (await isGzipFile(localFile))
+        ? source.pipe(createGunzip())
+        : source;
     const output = createWriteStream(outputFile);
     try {
         await pipeRdfToWritable(input, output, contentType);
@@ -116,6 +113,25 @@ async function streamRdfFile(localFile, outputFile, contentType, distribution) {
         await closeWritable(output);
     }
 }
+/**
+ * Whether the file begins with the gzip magic number (RFC 1952 §2.3.1).
+ *
+ * Sniffing the bytes is the only reliable signal for a downloaded dump.
+ * `dcat:compressFormat` is frequently absent, and the local filename is derived
+ * from the access URL, so a dump behind a query string (`…/download.trig.gz?graph=…`)
+ * carries no `.gz` suffix. Conversely, a `.gz` name says nothing once the server
+ * declared a real `Content-Encoding: gzip` and `fetch` already inflated the body.
+ */
+async function isGzipFile(path) {
+    const file = await open(path);
+    try {
+        const { buffer, bytesRead } = await file.read(Buffer.alloc(2), 0, 2, 0);
+        return bytesRead === 2 && buffer[0] === 0x1f && buffer[1] === 0x8b;
+    }
+    finally {
+        await file.close();
+    }
+}
 const openZip = promisify(yauzl.open);
 /**
  * Fold every parseable entry of a zip into the N-Quads output, in order. The

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/sparql-qlever",
-  "version": "0.15.4",
+  "version": "0.15.6",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/sparql-qlever"
@@ -26,11 +26,11 @@
   "dependencies": {
     "@lde/dataset": "^0.7.9",
     "@lde/distribution-downloader": "^0.6.8",
-    "@lde/sparql-importer": "^0.6.9",
+    "@lde/sparql-importer": "^0.6.10",
     "@lde/sparql-server": "^0.4.14",
-    "@lde/task-runner": "^0.2.13",
-    "@lde/task-runner-docker": "^0.2.16",
-    "@lde/task-runner-native": "^0.2.16",
+    "@lde/task-runner": "^0.2.14",
+    "@lde/task-runner-docker": "^0.2.17",
+    "@lde/task-runner-native": "^0.2.17",
     "@lde/wait-for-sparql": "^0.2.17",
     "rdf-parse": "^5.0.0",
     "rdf-serialize": "^5.1.0",