@lde/pipeline 0.28.1 → 0.28.2

package/README.md

@@ -103,6 +103,15 @@ const executor = new SparqlConstructExecutor({
 });
 ```
 
+When querying endpoints that return line-oriented formats like N-Triples (e.g. QLever), enable `lineBuffer` to work around an [N3.js chunk-splitting bug](https://github.com/rdfjs/N3.js/issues/578) that causes intermittent parse errors on large responses:
+
+```typescript
+const executor = new SparqlConstructExecutor({
+  query: 'CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o }',
+  lineBuffer: true,
+});
+```
+
 `Executor` is an interface, so you can implement your own for logic that's hard to express in pure SPARQL — for example, cleaning up messy date notations or converting locale-specific dates to ISO 8601. The decorator pattern lets you wrap a SPARQL executor and post-process its quad stream in TypeScript:
 
 ```typescript

package/dist/sparql/executor.d.ts

@@ -1,6 +1,7 @@
 import { Dataset, Distribution } from '@lde/dataset';
 import { SparqlEndpointFetcher } from 'fetch-sparql-endpoint';
 import type { NamedNode, Quad } from '@rdfjs/types';
+import { Transform } from 'node:stream';
 /**
  * An executor could not run because the dataset lacks a supported distribution.
  */
@@ -42,6 +43,17 @@ export interface SparqlConstructExecutorOptions {
      * Optional custom SparqlEndpointFetcher instance.
      */
     fetcher?: SparqlEndpointFetcher;
+    /**
+     * Buffer complete lines before passing them to the N3 parser.
+     *
+     * Works around an [N3.js bug](https://github.com/rdfjs/N3.js/issues/578)
+     * where language tags (e.g. `@nl-nl`) split across HTTP chunk boundaries
+     * cause parse errors. Enable this when querying endpoints that return
+     * line-oriented formats such as N-Triples (e.g. QLever).
+     *
+     * @default false
+     */
+    lineBuffer?: boolean;
 }
 /**
  * A streaming SPARQL CONSTRUCT executor.
@@ -75,6 +87,7 @@ export declare class SparqlConstructExecutor implements Executor {
     private readonly preParsed?;
     private readonly fetcher;
     private readonly retries;
+    private readonly lineBuffer;
     private readonly generator;
     constructor(options: SparqlConstructExecutorOptions);
     /**
@@ -86,6 +99,12 @@ export declare class SparqlConstructExecutor implements Executor {
      * @returns AsyncIterable<Quad> stream of results.
      */
     execute(dataset: Dataset, distribution: Distribution, options?: ExecuteOptions): Promise<AsyncIterable<Quad>>;
+    /**
+     * Fetch quads from the endpoint, optionally line-buffering the response
+     * stream before it reaches the N3 parser to work around
+     * {@link https://github.com/rdfjs/N3.js/issues/578 | N3.js#578}.
+     */
+    private fetchQuads;
     /**
      * Create an executor from a query file.
      *
@@ -98,4 +117,29 @@ export declare class SparqlConstructExecutor implements Executor {
  * Read a SPARQL query from a file.
  */
 export declare function readQueryFile(filename: string): Promise<string>;
+/**
+ * Buffers incoming data until complete lines (`\n`-terminated) are available,
+ * then pushes them downstream as a single chunk.
+ *
+ * **Why this exists:** `fetch-sparql-endpoint` pipes the raw HTTP response
+ * stream directly into N3.js's `StreamParser`. N3.js has a bug
+ * ({@link https://github.com/rdfjs/N3.js/issues/578 | N3.js#578}) where
+ * tokens that straddle chunk boundaries — most commonly language tags like
+ * `@nl-nl` — cause spurious `Unexpected "-nl"` parse errors. The error is
+ * non-deterministic and typically surfaces only on responses larger than
+ * ~12 MB, because that is when HTTP chunking starts splitting mid-token.
+ *
+ * By ensuring each chunk passed to the parser ends on a line boundary, we
+ * prevent any N-Triples token from being split. Memory overhead is minimal:
+ * at most one partial line is buffered at a time.
+ *
+ * This transform can be removed once N3.js#578 is fixed upstream.
+ *
+ * @see https://github.com/rdfjs/N3.js/issues/578
+ */
+export declare class LineBufferTransform extends Transform {
+    private remainder;
+    _transform(chunk: Buffer, _encoding: string, callback: () => void): void;
+    _flush(callback: () => void): void;
+}
 //# sourceMappingURL=executor.d.ts.map

package/dist/sparql/executor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/sparql/executor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAiB,MAAM,cAAc,CAAC;AACpE,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAWpD;;GAEG;AACH,qBAAa,YAAY;aACK,OAAO,EAAE,MAAM;gBAAf,OAAO,EAAE,MAAM;CAC5C;AAED,qEAAqE;AACrE,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEzD,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,QAAQ,CAAC,EAAE,gBAAgB,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,QAAQ;IACvB,OAAO,CACL,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,8BAA8B;IAC7C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,qBAAa,uBAAwB,YAAW,QAAQ;IACtD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAiB;IAC5C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;IAChD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;gBAEjC,OAAO,EAAE,8BAA8B;IAmBnD;;;;;;;OAOG;IACG,OAAO,CACX,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IAwC/B;;;;;OAKG;WACiB,QAAQ,CAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,IAAI,CAAC,8BAA8B,EAAE,OAAO,CAAC,GACtD,OAAO,CAAC,uBAAuB,CAAC;CAIpC;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAErE"}
+{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/sparql/executor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAiB,MAAM,cAAc,CAAC;AACpE,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAGpD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAUxC;;GAEG;AACH,qBAAa,YAAY;aACK,OAAO,EAAE,MAAM;gBAAf,OAAO,EAAE,MAAM;CAC5C;AAED,qEAAqE;AACrE,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEzD,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,QAAQ,CAAC,EAAE,gBAAgB,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,QAAQ;IACvB,OAAO,CACL,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,8BAA8B;IAC7C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,OAAO,CAAC,EAAE,qBAAqB,CAAC;IAEhC;;;;;;;;;OASG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,qBAAa,uBAAwB,YAAW,QAAQ;IACtD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAiB;IAC5C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;IAChD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAU;IACrC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;gBAEjC,OAAO,EAAE,8BAA8B;IAoBnD;;;;;;;OAOG;IACG,OAAO,CACX,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IAwC/B;;;;OAIG;YACW,UAAU;IAmBxB;;;;;OAKG;WACiB,QAAQ,CAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,IAAI,CAAC,8BAA8B,EAAE,OAAO,CAAC,GACtD,OAAO,CAAC,uBAAuB,CAAC;CAIpC;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAErE;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,mBAAoB,SAAQ,SAAS;IAChD,OAAO,CAAC,SAAS,CAAM;IAEd,UAAU,CACjB,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,IAAI;IAWb,MAAM,CAAC,QAAQ,EAAE,MAAM,IAAI;CAMrC"}

package/dist/sparql/executor.js

@@ -2,6 +2,8 @@ import { assertSafeIri } from '@lde/dataset';
 import { SparqlEndpointFetcher } from 'fetch-sparql-endpoint';
 import { readFile } from 'node:fs/promises';
 import { resolve } from 'node:path';
+import { Transform } from 'node:stream';
+import { StreamParser } from 'n3';
 import { Parser } from '@traqula/parser-sparql-1-1';
 import { Generator } from '@traqula/generator-sparql-1-1';
 import isNetworkError from 'is-network-error';
@@ -49,10 +51,12 @@ export class SparqlConstructExecutor {
     preParsed;
     fetcher;
     retries;
+    lineBuffer;
     generator = new Generator();
     constructor(options) {
         this.rawQuery = options.query;
         this.retries = options.retries ?? 3;
+        this.lineBuffer = options.lineBuffer ?? false;
         if (!options.query.includes('#subjectFilter#')) {
             const parsed = new Parser().parse(options.query);
             if (parsed.type !== 'query' || parsed.subType !== 'construct') {
@@ -98,11 +102,25 @@ export class SparqlConstructExecutor {
         let query = this.generator.generate(ast);
         assertSafeIri(dataset.iri.toString());
         query = query.replaceAll('?dataset', `<${dataset.iri}>`);
-        return await pRetry(() => this.fetcher.fetchTriples(endpoint.toString(), query), {
+        return await pRetry(() => this.fetchQuads(endpoint.toString(), query), {
             retries: this.retries,
             shouldRetry: ({ error }) => isTransientError(error),
         });
     }
+    /**
+     * Fetch quads from the endpoint, optionally line-buffering the response
+     * stream before it reaches the N3 parser to work around
+     * {@link https://github.com/rdfjs/N3.js/issues/578 | N3.js#578}.
+     */
+    async fetchQuads(endpoint, query) {
+        if (!this.lineBuffer) {
+            return this.fetcher.fetchTriples(endpoint, query);
+        }
+        const [contentType, , responseStream] = await this.fetcher.fetchRawStream(endpoint, query, SparqlEndpointFetcher.CONTENTTYPE_TURTLE);
+        return responseStream
+            .pipe(new LineBufferTransform())
+            .pipe(new StreamParser({ format: contentType }));
+    }
     /**
      * Create an executor from a query file.
      *
@@ -120,6 +138,44 @@ export class SparqlConstructExecutor {
 export async function readQueryFile(filename) {
     return (await readFile(resolve(filename))).toString();
 }
+/**
+ * Buffers incoming data until complete lines (`\n`-terminated) are available,
+ * then pushes them downstream as a single chunk.
+ *
+ * **Why this exists:** `fetch-sparql-endpoint` pipes the raw HTTP response
+ * stream directly into N3.js's `StreamParser`. N3.js has a bug
+ * ({@link https://github.com/rdfjs/N3.js/issues/578 | N3.js#578}) where
+ * tokens that straddle chunk boundaries — most commonly language tags like
+ * `@nl-nl` — cause spurious `Unexpected "-nl"` parse errors. The error is
+ * non-deterministic and typically surfaces only on responses larger than
+ * ~12 MB, because that is when HTTP chunking starts splitting mid-token.
+ *
+ * By ensuring each chunk passed to the parser ends on a line boundary, we
+ * prevent any N-Triples token from being split. Memory overhead is minimal:
+ * at most one partial line is buffered at a time.
+ *
+ * This transform can be removed once N3.js#578 is fixed upstream.
+ *
+ * @see https://github.com/rdfjs/N3.js/issues/578
+ */
+export class LineBufferTransform extends Transform {
+    remainder = '';
+    _transform(chunk, _encoding, callback) {
+        const data = this.remainder + chunk.toString();
+        const lines = data.split('\n');
+        this.remainder = lines.pop() ?? '';
+        if (lines.length > 0) {
+            this.push(lines.join('\n') + '\n');
+        }
+        callback();
+    }
+    _flush(callback) {
+        if (this.remainder.length > 0) {
+            this.push(this.remainder);
+        }
+        callback();
+    }
+}
 const transientStatusPattern = /HTTP status (\d+)/;
 function isTransientError(error) {
     if (isNetworkError(error))

package/dist/sparql/index.d.ts

@@ -1,4 +1,4 @@
-export { SparqlConstructExecutor, NotSupported, readQueryFile, type ExecuteOptions, type Executor, type SparqlConstructExecutorOptions, type VariableBindings, } from './executor.js';
+export { SparqlConstructExecutor, LineBufferTransform, NotSupported, readQueryFile, type ExecuteOptions, type Executor, type SparqlConstructExecutorOptions, type VariableBindings, } from './executor.js';
 export { SparqlItemSelector, type SparqlItemSelectorOptions, } from './selector.js';
 export { injectValues } from './values.js';
 export { withDefaultGraph } from './graph.js';

package/dist/sparql/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/sparql/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,uBAAuB,EACvB,YAAY,EACZ,aAAa,EACb,KAAK,cAAc,EACnB,KAAK,QAAQ,EACb,KAAK,8BAA8B,EACnC,KAAK,gBAAgB,GACtB,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,kBAAkB,EAClB,KAAK,yBAAyB,GAC/B,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/sparql/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,uBAAuB,EACvB,mBAAmB,EACnB,YAAY,EACZ,aAAa,EACb,KAAK,cAAc,EACnB,KAAK,QAAQ,EACb,KAAK,8BAA8B,EACnC,KAAK,gBAAgB,GACtB,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,kBAAkB,EAClB,KAAK,yBAAyB,GAC/B,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC"}

package/dist/sparql/index.js

@@ -1,4 +1,4 @@
-export { SparqlConstructExecutor, NotSupported, readQueryFile, } from './executor.js';
+export { SparqlConstructExecutor, LineBufferTransform, NotSupported, readQueryFile, } from './executor.js';
 export { SparqlItemSelector, } from './selector.js';
 export { injectValues } from './values.js';
 export { withDefaultGraph } from './graph.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.28.1",
+  "version": "0.28.2",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/pipeline"