@lde/sparql-qlever 0.13.0 → 0.14.0

package/dist/importer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,IAAI,iBAAiB,EAC7B,eAAe,EACf,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACb,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAK5C,MAAM,WAAW,kBAAkB;IACjC,oBAAoB;IACpB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,yBAAyB;IACzB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,uEAAuE;IACvE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,oBAAoB;IACpB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,yJAAyJ;IACzJ,+BAA+B,CAAC,EAAE,OAAO,CAAC;CAC3C;AAED,MAAM,WAAW,qBAAsB,SAAQ,eAAe;IAC5D,sBAAsB;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,kBAAkB,CAAC;CACpC;AAWD,qBAAa,QAAS,YAAW,iBAAiB;IAChD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkB;gBAE9B,OAAO,EAAE,qBAAqB;IAa7B,MAAM,CACjB,aAAa,EAAE,YAAY,EAAE,GAC5B,OAAO,CAAC,YAAY,GAAG,gBAAgB,GAAG,YAAY,CAAC;YA+B5C,QAAQ;IA4CtB,OAAO,CAAC,gBAAgB;IAQxB,OAAO,CAAC,aAAa;IAIrB;;OAEG;YACW,eAAe;IA0B7B,yDAAyD;YAC3C,eAAe;YAaf,cAAc;YAKd,KAAK;CA0CpB"}
+{"version":3,"file":"importer.d.ts","sourceRoot":"","sources":["../src/importer.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,IAAI,iBAAiB,EAC7B,eAAe,EACf,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACb,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAK5C,MAAM,WAAW,kBAAkB;IACjC,oBAAoB;IACpB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,yBAAyB;IACzB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,uEAAuE;IACvE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,oBAAoB;IACpB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,yJAAyJ;IACzJ,+BAA+B,CAAC,EAAE,OAAO,CAAC;CAC3C;AAED,MAAM,WAAW,qBAAsB,SAAQ,eAAe;IAC5D,sBAAsB;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,kBAAkB,CAAC;CACpC;AAWD,qBAAa,QAAS,YAAW,iBAAiB;IAChD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkB;gBAE9B,OAAO,EAAE,qBAAqB;IAa7B,MAAM,CACjB,aAAa,EAAE,YAAY,EAAE,GAC5B,OAAO,CAAC,YAAY,GAAG,gBAAgB,GAAG,YAAY,CAAC;YA+B5C,QAAQ;IAkDtB,OAAO,CAAC,gBAAgB;IAQxB,OAAO,CAAC,aAAa;IAIrB;;OAEG;YACW,eAAe;IA0B7B,yDAAyD;YAC3C,eAAe;YAaf,cAAc;YAKd,KAAK;CA0CpB"}

package/dist/importer.js

@@ -44,7 +44,7 @@ export class Importer {
         return result;
     }
     async doImport(distribution) {
-        const localFile = await this.options.downloader.download(distribution);
+        const { path: localFile, headers } = await this.options.downloader.download(distribution);
         if (await this.isIndexUpToDate(localFile)) {
             const tripleCount = await this.readTripleCount(localFile);
             if (tripleCount === 0) {
@@ -52,7 +52,7 @@ export class Importer {
             }
             return new ImportSuccessful(distribution, undefined, tripleCount);
         }
-        const format = fileFormatFromMimeType(distribution.mimeType);
+        const { format, warning } = fileFormatFor(distribution.mimeType, basename(localFile), headers.get('Content-Type') ?? undefined);
         let logs;
         try {
             logs = await this.index(localFile, format);
@@ -71,7 +71,8 @@ export class Importer {
             return new ImportFailed(distribution, 'Indexed 0 triples from distribution');
         }
         await this.writeCacheInfo(localFile);
-        return new ImportSuccessful(distribution, undefined, tripleCount);
+        const warnings = warning ? [warning] : [];
+        return new ImportSuccessful(distribution, undefined, tripleCount, warnings);
     }
     parseTripleCount(logs) {
         // Extract num-triples.normal from the metadata JSON that the index
@@ -163,10 +164,54 @@ const defaultQleverIndexOptions = {
     'parse-parallel': true,
     'only-pso-and-pos-permutations': false,
 };
-function fileFormatFromMimeType(mimeType) {
-    const format = supportedFormats.get(mimeType);
-    if (format === undefined) {
-        throw new Error(`Unsupported media type: ${mimeType}`);
+const extensionFormats = new Map([
+    ['.nt', 'nt'],
+    ['.nq', 'nq'],
+    ['.ttl', 'ttl'],
+]);
+const compressionTypes = new Set([
+    'application/gzip',
+    'application/x-gzip',
+    'application/octet-stream',
+]);
+/**
+ * Determine the QLever format flag for a distribution.
+ *
+ * Priority:
+ * 1. Server Content-Type (if it maps to a supported RDF format)
+ * 2. File extension (fallback when Content-Type is a compression type or missing)
+ * 3. Declared MIME type from the dataset registry (last resort)
+ */
+function fileFormatFor(declaredMimeType, filename, serverContentType) {
+    const declaredFormat = supportedFormats.get(declaredMimeType);
+    if (declaredFormat === undefined) {
+        throw new Error(`Unsupported media type: ${declaredMimeType}`);
+    }
+    // Try server Content-Type first (strip parameters like "; charset=utf-8").
+    if (serverContentType) {
+        const actualType = serverContentType.split(';')[0].trim();
+        if (!compressionTypes.has(actualType)) {
+            const serverFormat = supportedFormats.get(actualType);
+            if (serverFormat !== undefined && serverFormat !== declaredFormat) {
+                return {
+                    format: serverFormat,
+                    warning: `Server Content-Type ${actualType} does not match declared media type ${declaredMimeType}; using ${serverFormat} format`,
+                };
+            }
+            if (serverFormat !== undefined) {
+                return { format: serverFormat };
+            }
+        }
+    }
+    // Fall back to file extension.
+    const stripped = filename.replace(/\.(gz|bz2|xz|zst)$/i, '');
+    const extension = stripped.slice(stripped.lastIndexOf('.'));
+    const extensionFormat = extensionFormats.get(extension);
+    if (extensionFormat !== undefined && extensionFormat !== declaredFormat) {
+        return {
+            format: extensionFormat,
+            warning: `Declared media type ${declaredMimeType} does not match file extension ${extension}; using ${extensionFormat} format`,
+        };
     }
-    return format;
+    return { format: declaredFormat };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/sparql-qlever",
-  "version": "0.13.0",
+  "version": "0.14.0",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/sparql-qlever"
@@ -25,8 +25,8 @@
   ],
   "dependencies": {
     "@lde/dataset": "0.7.2",
-    "@lde/distribution-downloader": "0.5.3",
-    "@lde/sparql-importer": "0.5.0",
+    "@lde/distribution-downloader": "0.6.0",
+    "@lde/sparql-importer": "0.6.0",
     "@lde/sparql-server": "0.4.10",
     "@lde/task-runner": "0.2.10",
     "@lde/task-runner-docker": "0.2.11",