@lde/sparql-qlever 0.14.3 → 0.14.5

package/dist/importer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,IAAI,iBAAiB,EAC7B,eAAe,EACf,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACb,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAK5C,MAAM,WAAW,kBAAkB;IACjC,oBAAoB;IACpB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,yBAAyB;IACzB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,uEAAuE;IACvE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,oBAAoB;IACpB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,yJAAyJ;IACzJ,+BAA+B,CAAC,EAAE,OAAO,CAAC;CAC3C;AAED,MAAM,WAAW,qBAAsB,SAAQ,eAAe;IAC5D,sBAAsB;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,kBAAkB,CAAC;CACpC;AAWD,qBAAa,QAAS,YAAW,iBAAiB;IAChD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkB;gBAE9B,OAAO,EAAE,qBAAqB;IAa7B,MAAM,CACjB,aAAa,EAAE,YAAY,EAAE,GAC5B,OAAO,CAAC,YAAY,GAAG,gBAAgB,GAAG,YAAY,CAAC;YA+B5C,QAAQ;IAkDtB,OAAO,CAAC,gBAAgB;IAQxB,OAAO,CAAC,aAAa;IAIrB;;OAEG;YACW,eAAe;IA0B7B,yDAAyD;YAC3C,eAAe;YAaf,cAAc;YAKd,KAAK;CA0CpB"}
+{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,IAAI,iBAAiB,EAC7B,eAAe,EACf,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACb,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAyB,YAAY,EAAE,MAAM,cAAc,CAAC;AAMnE,MAAM,WAAW,kBAAkB;IACjC,oBAAoB;IACpB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,yBAAyB;IACzB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,uEAAuE;IACvE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,oBAAoB;IACpB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,yJAAyJ;IACzJ,+BAA+B,CAAC,EAAE,OAAO,CAAC;CAC3C;AAED,MAAM,WAAW,qBAAsB,SAAQ,eAAe;IAC5D,sBAAsB;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,kBAAkB,CAAC;CACpC;AAWD,qBAAa,QAAS,YAAW,iBAAiB;IAChD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkB;gBAE9B,OAAO,EAAE,qBAAqB;IAa7B,MAAM,CACjB,aAAa,EAAE,YAAY,EAAE,GAC5B,OAAO,CAAC,YAAY,GAAG,gBAAgB,GAAG,YAAY,CAAC;YAqC5C,QAAQ;IA6DtB,OAAO,CAAC,gBAAgB;IAQxB,OAAO,CAAC,aAAa;IAIrB;;OAEG;YACW,eAAe;IA0B7B,yDAAyD;YAC3C,eAAe;YAaf,cAAc;YAKd,KAAK;CA4CpB"}

package/dist/importer.js

@@ -1,7 +1,9 @@
 import { ImportFailed, ImportSuccessful, NotSupported, } from '@lde/sparql-importer';
+import { compressionMediaTypes } from '@lde/dataset';
 import { LastModifiedDownloader } from '@lde/distribution-downloader';
 import { basename, dirname, join } from 'path';
 import { readFile, stat, writeFile } from 'node:fs/promises';
+import { needsPreprocessing, preprocess } from './preprocess.js';
 export class Importer {
     options;
     constructor(options) {
@@ -17,8 +19,11 @@ export class Importer {
         };
     }
     async import(distributions) {
-        const downloadDistributions = distributions.filter((distribution) => distribution.mimeType !== undefined &&
-            supportedFormats.has(distribution.mimeType));
+        const downloadDistributions = distributions
+            .filter((distribution) => distribution.mimeType !== undefined &&
+            acceptedMediaTypes.includes(distribution.mimeType))
+            .sort((a, b) => acceptedMediaTypes.indexOf(a.mimeType) -
+            acceptedMediaTypes.indexOf(b.mimeType));
         if (downloadDistributions.length === 0) {
             return new NotSupported();
         }
@@ -52,15 +57,29 @@ export class Importer {
             }
             return new ImportSuccessful(distribution, undefined, tripleCount);
         }
-        const { format, warning } = fileFormatFor(distribution.mimeType, basename(localFile), headers.get('Content-Type') ?? undefined);
+        let indexFile = localFile;
+        let format;
+        const warnings = [];
+        if (needsPreprocessing(distribution)) {
+            const result = await preprocess(localFile, distribution);
+            indexFile = result.path;
+            format = result.format;
+            warnings.push(...result.warnings);
+        }
+        else {
+            const resolved = fileFormatFor(distribution.mimeType, basename(localFile), headers.get('Content-Type') ?? undefined);
+            format = resolved.format;
+            if (resolved.warning)
+                warnings.push(resolved.warning);
+        }
         let logs;
         try {
-            logs = await this.index(localFile, format);
+            logs = await this.index(indexFile, format);
         }
         catch (error) {
             if (format === 'ttl' &&
                 error.message?.includes('multiline string literal')) {
-                logs = await this.index(localFile, format, false);
+                logs = await this.index(indexFile, format, false);
             }
             else {
                 throw error;
@@ -71,7 +90,6 @@ export class Importer {
             return new ImportFailed(distribution, 'Indexed 0 triples from distribution');
         }
         await this.writeCacheInfo(localFile);
-        const warnings = warning ? [warning] : [];
         return new ImportSuccessful(distribution, undefined, tripleCount, warnings);
     }
     parseTripleCount(logs) {
@@ -148,15 +166,53 @@ export class Importer {
             .filter(Boolean)
             .join(' ');
         const metadataFile = `${this.options.indexName}.meta-data.json`;
-        const indexTask = await this.options.taskRunner.run(`(gunzip -c '${basename(file)}' 2>/dev/null || cat '${basename(file)}') | qlever-index ${flags} && cat ${metadataFile}`);
+        const localName = basename(file);
+        const decompressCommand = localName.toLowerCase().endsWith('.zip')
+            ? `unzip -p ${shellQuote(localName)}`
+            : `(gunzip -c ${shellQuote(localName)} 2>/dev/null || cat ${shellQuote(localName)})`;
+        const indexTask = await this.options.taskRunner.run(`${decompressCommand} | qlever-index ${flags} && cat ${shellQuote(metadataFile)}`);
         return await this.options.taskRunner.wait(indexTask);
     }
 }
-const supportedFormats = new Map([
+/**
+ * POSIX-quote a value for safe interpolation into a shell command: wrap it in
+ * single quotes and escape any embedded single quote as `'\''`.
+ *
+ * Without this, a data filename containing an apostrophe — e.g. a dataset
+ * titled `'s-Hertogenbosch`, whose distribution URL maps to a local file like
+ * `…Erfgoed+'s-Hertogenbosch.nt` — would terminate the surrounding quotes, so
+ * `cat`/`gunzip` would read a non-existent path and feed `qlever-index` empty
+ * input. The index then "succeeds" with 0 triples, the import is treated as
+ * failed, and every distribution (and the JSON-LD fallback) fails the same way.
+ */
+function shellQuote(value) {
+    return `'${value.replace(/'/g, "'\\''")}'`;
+}
+/**
+ * Native QLever index formats — `qlever-index -F <flag>` consumes these
+ * directly. JSON-LD is not here: it is preprocessed to N-Quads first (see
+ * {@link preprocess}).
+ */
+const nativeFormats = new Map([
     ['application/n-triples', 'nt'],
     ['application/n-quads', 'nq'],
     ['text/turtle', 'ttl'],
 ]);
+/**
+ * Accepted distribution media types, in preference order: the first match is
+ * tried first. Native formats win over JSON-LD because they skip the Node-side
+ * preprocessor.
+ *
+ * `application/zip` is intentionally absent — the inner RDF format must be
+ * declared via `mediaType` with `application/zip` appearing only as the
+ * `compressFormat`, so we know what is inside.
+ */
+const acceptedMediaTypes = [
+    'application/n-quads',
+    'application/n-triples',
+    'text/turtle',
+    'application/ld+json',
+];
 const defaultQleverIndexOptions = {
     'ascii-prefixes-only': true,
     'num-triples-per-batch': 3_000_000,
@@ -169,11 +225,6 @@ const extensionFormats = new Map([
     ['.nq', 'nq'],
     ['.ttl', 'ttl'],
 ]);
-const compressionTypes = new Set([
-    'application/gzip',
-    'application/x-gzip',
-    'application/octet-stream',
-]);
 /**
  * Determine the QLever format flag for a distribution.
  *
@@ -183,15 +234,15 @@ const compressionTypes = new Set([
  * 3. Declared MIME type from the dataset registry (last resort)
  */
 function fileFormatFor(declaredMimeType, filename, serverContentType) {
-    const declaredFormat = supportedFormats.get(declaredMimeType);
+    const declaredFormat = nativeFormats.get(declaredMimeType);
     if (declaredFormat === undefined) {
         throw new Error(`Unsupported media type: ${declaredMimeType}`);
     }
     // Try server Content-Type first (strip parameters like "; charset=utf-8").
     if (serverContentType) {
         const actualType = serverContentType.split(';')[0].trim();
-        if (!compressionTypes.has(actualType)) {
-            const serverFormat = supportedFormats.get(actualType);
+        if (!compressionMediaTypes.has(actualType)) {
+            const serverFormat = nativeFormats.get(actualType);
             if (serverFormat !== undefined && serverFormat !== declaredFormat) {
                 return {
                     format: serverFormat,

package/dist/preprocess.d.ts

@@ -0,0 +1,33 @@
+import { Distribution } from '@lde/dataset';
+export interface PreprocessResult {
+    /** Path to the file ready for `qlever-index`. Always N-Quads. */
+    path: string;
+    format: 'nq';
+    warnings: string[];
+}
+/**
+ * Whether a distribution needs Node-side preprocessing before `qlever-index`
+ * can read it.
+ *
+ * Only JSON-LD distributions return `true`: `qlever-index` cannot parse
+ * JSON-LD, so we stream it through a JSON-LD parser into N-Quads first.
+ *
+ * Native RDF formats (`nt`, `nq`, `ttl`) — including when wrapped in
+ * `application/gzip` or `application/zip` — go straight through the shell
+ * pipeline in `index()`, which uses `gunzip -c` or `unzip -p` as appropriate.
+ * Standalone `mediaType=application/zip` is rejected upstream: the inner
+ * format must be declared.
+ */
+export declare function needsPreprocessing(distribution: Distribution): boolean;
+/**
+ * Convert a JSON-LD distribution to N-Quads alongside the source file.
+ *
+ * Streams the source through `rdf-parse` → `rdf-serialize` so memory use
+ * stays bounded regardless of input size. Handles gzip transparently
+ * (declared `compressFormat` or `.gz` filename) and zip containers (folds
+ * each JSON-LD entry into the output stream in order).
+ *
+ * Cached: if the output is newer than the input, it is reused as-is.
+ */
+export declare function preprocess(localFile: string, distribution: Distribution): Promise<PreprocessResult>;
+//# sourceMappingURL=preprocess.d.ts.map

package/dist/preprocess.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"preprocess.d.ts","sourceRoot":"","sources":["../src/preprocess.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAmB5C,MAAM,WAAW,gBAAgB;IAC/B,iEAAiE;IACjE,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,IAAI,CAAC;IACb,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,kBAAkB,CAAC,YAAY,EAAE,YAAY,GAAG,OAAO,CAEtE;AAED;;;;;;;;;GASG;AACH,wBAAsB,UAAU,CAC9B,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,YAAY,GACzB,OAAO,CAAC,gBAAgB,CAAC,CAsB3B"}

package/dist/preprocess.js

@@ -0,0 +1,158 @@
+import { rdfParser } from 'rdf-parse';
+import { rdfSerializer } from 'rdf-serialize';
+import { createGunzip } from 'node:zlib';
+import { createReadStream, createWriteStream } from 'node:fs';
+import { rm, stat } from 'node:fs/promises';
+import { extname } from 'node:path';
+import { finished } from 'node:stream/promises';
+import { promisify } from 'node:util';
+import yauzl from 'yauzl';
+const JSONLD_MIME = 'application/ld+json';
+const ZIP_MIME = 'application/zip';
+const GZIP_MIME = 'application/gzip';
+const GZIP_MIME_LEGACY = 'application/x-gzip';
+const JSONLD_ZIP_EXTENSIONS = ['.jsonld', '.json'];
+/**
+ * Whether a distribution needs Node-side preprocessing before `qlever-index`
+ * can read it.
+ *
+ * Only JSON-LD distributions return `true`: `qlever-index` cannot parse
+ * JSON-LD, so we stream it through a JSON-LD parser into N-Quads first.
+ *
+ * Native RDF formats (`nt`, `nq`, `ttl`) — including when wrapped in
+ * `application/gzip` or `application/zip` — go straight through the shell
+ * pipeline in `index()`, which uses `gunzip -c` or `unzip -p` as appropriate.
+ * Standalone `mediaType=application/zip` is rejected upstream: the inner
+ * format must be declared.
+ */
+export function needsPreprocessing(distribution) {
+    return distribution.mimeType === JSONLD_MIME;
+}
+/**
+ * Convert a JSON-LD distribution to N-Quads alongside the source file.
+ *
+ * Streams the source through `rdf-parse` → `rdf-serialize` so memory use
+ * stays bounded regardless of input size. Handles gzip transparently
+ * (declared `compressFormat` or `.gz` filename) and zip containers (folds
+ * each JSON-LD entry into the output stream in order).
+ *
+ * Cached: if the output is newer than the input, it is reused as-is.
+ */
+export async function preprocess(localFile, distribution) {
+    if (!needsPreprocessing(distribution)) {
+        throw new Error(`preprocess called for distribution that does not need preprocessing: mediaType=${distribution.mimeType}`);
+    }
+    const outputFile = `${localFile}.preprocessed.nq`;
+    if (await outputIsUpToDate(localFile, outputFile)) {
+        return { path: outputFile, format: 'nq', warnings: [] };
+    }
+    await rm(outputFile, { force: true });
+    const warnings = [];
+    if (distribution.compressMimeType === ZIP_MIME) {
+        await streamJsonldZip(localFile, outputFile, warnings);
+    }
+    else {
+        await streamJsonldFile(localFile, outputFile, distribution);
+    }
+    return { path: outputFile, format: 'nq', warnings };
+}
+async function outputIsUpToDate(inputFile, outputFile) {
+    try {
+        const [inputStat, outputStat] = await Promise.all([
+            stat(inputFile),
+            stat(outputFile),
+        ]);
+        return outputStat.mtimeMs > inputStat.mtimeMs && outputStat.size > 0;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Pipe one JSON-LD source through parse → N-Quads serialize into an already
+ * open writable, without closing it. Back-pressure is handled by Node's
+ * built-in `.pipe()`; the caller manages `output`'s lifecycle.
+ */
+async function pipeJsonldToWritable(input, output) {
+    const quads = rdfParser.parse(input, { contentType: JSONLD_MIME });
+    const bytes = rdfSerializer.serialize(quads, {
+        contentType: 'application/n-quads',
+    });
+    bytes.pipe(output, { end: false });
+    await finished(bytes);
+}
+async function closeWritable(output) {
+    await new Promise((resolve, reject) => {
+        output.once('close', resolve);
+        output.once('error', reject);
+        output.end();
+    });
+}
+async function streamJsonldFile(localFile, outputFile, distribution) {
+    const isGzipped = distribution.compressMimeType === GZIP_MIME ||
+        distribution.compressMimeType === GZIP_MIME_LEGACY ||
+        localFile.toLowerCase().endsWith('.gz');
+    const source = createReadStream(localFile);
+    const input = isGzipped ? source.pipe(createGunzip()) : source;
+    const output = createWriteStream(outputFile);
+    try {
+        await pipeJsonldToWritable(input, output);
+    }
+    finally {
+        await closeWritable(output);
+    }
+}
+const openZip = promisify(yauzl.open);
+async function streamJsonldZip(zipFile, outputFile, warnings) {
+    const zip = await openZip(zipFile, { lazyEntries: true });
+    const output = createWriteStream(outputFile);
+    let entriesProcessed = 0;
+    try {
+        await new Promise((resolve, reject) => {
+            zip.on('error', reject);
+            zip.on('end', resolve);
+            zip.on('entry', (entry) => {
+                void (async () => {
+                    try {
+                        if (entry.fileName.endsWith('/')) {
+                            zip.readEntry();
+                            return;
+                        }
+                        const extension = extname(entry.fileName).toLowerCase();
+                        if (!JSONLD_ZIP_EXTENSIONS.includes(extension)) {
+                            warnings.push(`Skipping zip entry ${entry.fileName}: extension ${extension || '(none)'} is not JSON-LD`);
+                            zip.readEntry();
+                            return;
+                        }
+                        const stream = await openZipEntry(zip, entry);
+                        await pipeJsonldToWritable(stream, output);
+                        entriesProcessed++;
+                        zip.readEntry();
+                    }
+                    catch (error) {
+                        reject(error);
+                    }
+                })();
+            });
+            zip.readEntry();
+        });
+    }
+    finally {
+        zip.close();
+        await closeWritable(output);
+    }
+    if (entriesProcessed === 0) {
+        throw new Error(`Zip ${zipFile} contains no JSON-LD entries`);
+    }
+}
+function openZipEntry(zip, entry) {
+    return new Promise((resolve, reject) => {
+        zip.openReadStream(entry, (error, stream) => {
+            if (error || stream === undefined) {
+                reject(error ?? new Error('Failed to open zip entry'));
+                return;
+            }
+            resolve(stream);
+        });
+    });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/sparql-qlever",
-  "version": "0.14.3",
+  "version": "0.14.5",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/sparql-qlever"
@@ -24,14 +24,21 @@
     "!**/*.tsbuildinfo"
   ],
   "dependencies": {
-    "@lde/dataset": "0.7.3",
-    "@lde/distribution-downloader": "0.6.1",
-    "@lde/sparql-importer": "0.6.1",
+    "@lde/dataset": "0.7.4",
+    "@lde/distribution-downloader": "0.6.2",
+    "@lde/sparql-importer": "0.6.2",
     "@lde/sparql-server": "0.4.11",
     "@lde/task-runner": "0.2.11",
     "@lde/task-runner-docker": "0.2.13",
     "@lde/task-runner-native": "0.2.14",
     "@lde/wait-for-sparql": "0.2.12",
-    "tslib": "^2.3.0"
+    "rdf-parse": "^5.0.0",
+    "rdf-serialize": "^5.1.0",
+    "tslib": "^2.3.0",
+    "yauzl": "^3.3.1"
+  },
+  "devDependencies": {
+    "@rdfjs/types": "^2.0.0",
+    "@types/yauzl": "^2.10.3"
   }
 }