@lde/sparql-qlever 0.14.7 → 0.14.9

package/dist/importer.js

@@ -200,8 +200,9 @@ const nativeFormats = new Map([
 ]);
 /**
  * Accepted distribution media types, in preference order: the first match is
- * tried first. Native formats win over JSON-LD and RDF/XML because they skip
- * the Node-side preprocessor.
+ * tried first. Native formats win over JSON-LD, RDF/XML and TriG because they
+ * skip the Node-side preprocessor; TriG comes last so a streaming native dump
+ * is always preferred when a dataset offers both.
  *
  * `application/zip` is intentionally absent — the inner RDF format must be
  * declared via `mediaType` with `application/zip` appearing only as the
@@ -213,6 +214,7 @@ const acceptedMediaTypes = [
     'text/turtle',
     'application/ld+json',
     'application/rdf+xml',
+    'application/trig',
 ];
 const defaultQleverIndexOptions = {
     'ascii-prefixes-only': true,

package/dist/preprocess.d.ts

@@ -9,8 +9,8 @@ export interface PreprocessResult {
  * Whether a distribution needs Node-side preprocessing before `qlever-index`
  * can read it.
  *
- * JSON-LD and RDF/XML distributions return `true`: `qlever-index` cannot parse
- * either, so we stream them through `rdf-parse` into N-Quads first.
+ * JSON-LD, RDF/XML and TriG distributions return `true`: `qlever-index` cannot
+ * parse any of them, so we stream them through `rdf-parse` into N-Quads first.
  *
  * Native RDF formats (`nt`, `nq`, `ttl`) — including when wrapped in
  * `application/gzip` or `application/zip` — go straight through the shell
@@ -20,13 +20,13 @@ export interface PreprocessResult {
  */
 export declare function needsPreprocessing(distribution: Distribution): boolean;
 /**
- * Convert a JSON-LD or RDF/XML distribution to N-Quads alongside the source
- * file.
+ * Convert a JSON-LD, RDF/XML or TriG distribution to N-Quads alongside the
+ * source file.
  *
  * Streams the source through `rdf-parse` → `rdf-serialize` so memory use
  * stays bounded regardless of input size. Handles gzip transparently
  * (declared `compressFormat` or `.gz` filename) and zip containers (folds
- * each matching entry into the output stream in order).
+ * every parseable entry into the output stream in order).
  *
  * Cached: if the output is newer than the input, it is reused as-is.
  */

package/dist/preprocess.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"preprocess.d.ts","sourceRoot":"","sources":["../src/preprocess.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAmC5C,MAAM,WAAW,gBAAgB;IAC/B,iEAAiE;IACjE,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,IAAI,CAAC;IACb,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,kBAAkB,CAAC,YAAY,EAAE,YAAY,GAAG,OAAO,CAKtE;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,UAAU,CAC9B,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,YAAY,GACzB,OAAO,CAAC,gBAAgB,CAAC,CA2B3B"}
+{"version":3,"file":"preprocess.d.ts","sourceRoot":"","sources":["../src/preprocess.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AA6B5C,MAAM,WAAW,gBAAgB;IAC/B,iEAAiE;IACjE,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,IAAI,CAAC;IACb,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,kBAAkB,CAAC,YAAY,EAAE,YAAY,GAAG,OAAO,CAKtE;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,UAAU,CAC9B,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,YAAY,GACzB,OAAO,CAAC,gBAAgB,CAAC,CAyB3B"}

package/dist/preprocess.js

@@ -3,30 +3,31 @@ import { rdfSerializer } from 'rdf-serialize';
 import { createGunzip } from 'node:zlib';
 import { createReadStream, createWriteStream } from 'node:fs';
 import { rm, stat } from 'node:fs/promises';
-import { extname } from 'node:path';
 import { finished } from 'node:stream/promises';
 import { promisify } from 'node:util';
 import yauzl from 'yauzl';
 const JSONLD_MIME = 'application/ld+json';
 const RDFXML_MIME = 'application/rdf+xml';
+const TRIG_MIME = 'application/trig';
 const ZIP_MIME = 'application/zip';
 const GZIP_MIME = 'application/gzip';
 const GZIP_MIME_LEGACY = 'application/x-gzip';
 /**
- * RDF media types that `qlever-index` cannot read natively, so they are streamed
- * through `rdf-parse` → N-Quads first. The map key is the `rdf-parse`
- * `contentType`; the value carries the metadata needed to handle zip containers.
+ * RDF media types `qlever-index` cannot read natively, keyed by `rdf-parse`
+ * `contentType` with a label for warnings. TriG belongs here too: its
+ * `<graph> { … }` blocks parse as neither N-Quads nor Turtle.
  */
 const preprocessFormats = new Map([
-    [JSONLD_MIME, { label: 'JSON-LD', zipExtensions: ['.jsonld', '.json'] }],
-    [RDFXML_MIME, { label: 'RDF/XML', zipExtensions: ['.rdf', '.xml', '.owl'] }],
+    [JSONLD_MIME, 'JSON-LD'],
+    [RDFXML_MIME, 'RDF/XML'],
+    [TRIG_MIME, 'TriG'],
 ]);
 /**
  * Whether a distribution needs Node-side preprocessing before `qlever-index`
  * can read it.
  *
- * JSON-LD and RDF/XML distributions return `true`: `qlever-index` cannot parse
- * either, so we stream them through `rdf-parse` into N-Quads first.
+ * JSON-LD, RDF/XML and TriG distributions return `true`: `qlever-index` cannot
+ * parse any of them, so we stream them through `rdf-parse` into N-Quads first.
  *
  * Native RDF formats (`nt`, `nq`, `ttl`) — including when wrapped in
  * `application/gzip` or `application/zip` — go straight through the shell
@@ -39,22 +40,20 @@ export function needsPreprocessing(distribution) {
         preprocessFormats.has(distribution.mimeType));
 }
 /**
- * Convert a JSON-LD or RDF/XML distribution to N-Quads alongside the source
- * file.
+ * Convert a JSON-LD, RDF/XML or TriG distribution to N-Quads alongside the
+ * source file.
  *
  * Streams the source through `rdf-parse` → `rdf-serialize` so memory use
  * stays bounded regardless of input size. Handles gzip transparently
  * (declared `compressFormat` or `.gz` filename) and zip containers (folds
- * each matching entry into the output stream in order).
+ * every parseable entry into the output stream in order).
  *
  * Cached: if the output is newer than the input, it is reused as-is.
  */
 export async function preprocess(localFile, distribution) {
     const contentType = distribution.mimeType;
-    const format = contentType === undefined
-        ? undefined
-        : preprocessFormats.get(contentType);
-    if (contentType === undefined || format === undefined) {
+    const label = contentType === undefined ? undefined : preprocessFormats.get(contentType);
+    if (contentType === undefined || label === undefined) {
         throw new Error(`preprocess called for distribution that does not need preprocessing: mediaType=${distribution.mimeType}`);
     }
     const outputFile = `${localFile}.preprocessed.nq`;
@@ -64,7 +63,7 @@ export async function preprocess(localFile, distribution) {
     await rm(outputFile, { force: true });
     const warnings = [];
     if (distribution.compressMimeType === ZIP_MIME) {
-        await streamRdfZip(localFile, outputFile, contentType, format, warnings);
+        await streamRdfZip(localFile, outputFile, contentType, label, warnings);
     }
     else {
         await streamRdfFile(localFile, outputFile, contentType, distribution);
@@ -118,7 +117,12 @@ async function streamRdfFile(localFile, outputFile, contentType, distribution) {
     }
 }
 const openZip = promisify(yauzl.open);
-async function streamRdfZip(zipFile, outputFile, contentType, format, warnings) {
+/**
+ * Fold every parseable entry of a zip into the N-Quads output, in order. The
+ * declared `contentType` drives the parser; an entry that fails to parse (a
+ * sidecar, OS metadata) is skipped with a warning. Throws if nothing parses.
+ */
+async function streamRdfZip(zipFile, outputFile, contentType, label, warnings) {
     const zip = await openZip(zipFile, { lazyEntries: true });
     const output = createWriteStream(outputFile);
     let entriesProcessed = 0;
@@ -128,25 +132,25 @@ async function streamRdfZip(zipFile, outputFile, contentType, format, warnings)
             zip.on('end', resolve);
             zip.on('entry', (entry) => {
                 void (async () => {
+                    if (entry.fileName.endsWith('/')) {
+                        zip.readEntry();
+                        return;
+                    }
                     try {
-                        if (entry.fileName.endsWith('/')) {
-                            zip.readEntry();
-                            return;
+                        const stream = await openZipEntry(zip, entry);
+                        try {
+                            await pipeRdfToWritable(stream, output, contentType);
+                            entriesProcessed++;
                         }
-                        const extension = extname(entry.fileName).toLowerCase();
-                        if (!format.zipExtensions.includes(extension)) {
-                            warnings.push(`Skipping zip entry ${entry.fileName}: extension ${extension || '(none)'} is not ${format.label}`);
-                            zip.readEntry();
-                            return;
+                        finally {
+                            // yauzl lazyEntries won't advance until this stream is released.
+                            stream.destroy();
                         }
-                        const stream = await openZipEntry(zip, entry);
-                        await pipeRdfToWritable(stream, output, contentType);
-                        entriesProcessed++;
-                        zip.readEntry();
                     }
                     catch (error) {
-                        reject(error);
+                        warnings.push(`Skipping zip entry ${entry.fileName}: not valid ${label} (${error.message})`);
                     }
+                    zip.readEntry();
                 })();
             });
             zip.readEntry();
@@ -157,7 +161,7 @@ async function streamRdfZip(zipFile, outputFile, contentType, format, warnings)
         await closeWritable(output);
     }
     if (entriesProcessed === 0) {
-        throw new Error(`Zip ${zipFile} contains no ${format.label} entries`);
+        throw new Error(`Zip ${zipFile} contains no valid ${label} entries`);
     }
 }
 function openZipEntry(zip, entry) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/sparql-qlever",
-  "version": "0.14.7",
+  "version": "0.14.9",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/sparql-qlever"
@@ -24,9 +24,9 @@
     "!**/*.tsbuildinfo"
   ],
   "dependencies": {
-    "@lde/dataset": "0.7.5",
-    "@lde/distribution-downloader": "0.6.3",
-    "@lde/sparql-importer": "0.6.3",
+    "@lde/dataset": "0.7.7",
+    "@lde/distribution-downloader": "0.6.5",
+    "@lde/sparql-importer": "0.6.5",
     "@lde/sparql-server": "0.4.11",
     "@lde/task-runner": "0.2.11",
     "@lde/task-runner-docker": "0.2.13",
@@ -35,10 +35,10 @@
     "rdf-parse": "^5.0.0",
     "rdf-serialize": "^5.1.0",
     "tslib": "^2.3.0",
-    "yauzl": "^3.3.2"
+    "yauzl": "^3.4.0"
   },
   "devDependencies": {
     "@rdfjs/types": "^2.0.0",
-    "@types/yauzl": "^2.10.3"
+    "@types/yauzl": "^3.3.0"
   }
 }