@lde/pipeline 0.31.1 → 0.31.3

package/README.md

@@ -86,6 +86,12 @@ new Stage({
 
 `maxConcurrency` (default: 10) limits the total number of concurrent SPARQL queries. Within each batch, all executors run in parallel; the number of concurrent batches is automatically reduced to `⌊maxConcurrency / executorCount⌋` so the total query pressure stays within the limit. For example, with `maxConcurrency: 10` and two executors per stage, up to 5 batches run concurrently (10 SPARQL queries total).
 
+#### Expecting output
+
+`expectsOutput` (default: `false`) marks a stage whose query must yield at least one quad. A supported stage that produces none is then treated as a hard failure rather than a legitimately empty result.
+
+Set it for scalar aggregates such as `SELECT (COUNT(*) AS ?n)`, which always return exactly one row — so zero output can only mean the endpoint truncated or aborted the response (e.g. a timeout surfaced as an empty `HTTP 200`). The failure flows through like any other hard stage failure, triggering the [reactive dump fallback](#distribution-resolver) when `strategy: 'sparqlWithImportFallback'` is configured. Leave it `false` for stages that may legitimately be empty, such as class or property partitions of a dataset that lacks that structure.
+
 ### Item Selector
 
 Selects resources from the distribution and fans out executor calls per batch of results. Implements the `ItemSelector` interface:

package/dist/stage.d.ts

@@ -70,6 +70,23 @@ export interface StageOptions {
     maxConcurrency?: number;
     /** Child stages that chain off this stage's output. */
     stages?: Stage[];
+    /**
+     * Treat a supported stage that produces no quads as a hard failure (throws),
+     * rather than a legitimately empty result.
+     *
+     * Set this for stages whose query must yield output — typically a scalar
+     * aggregate such as `SELECT (COUNT(*) AS ?n)`, which always returns exactly
+     * one row, so zero quads can only mean the endpoint truncated or aborted the
+     * response (e.g. a timeout surfaced as an empty `HTTP 200`). The resulting
+     * failure flows through the pipeline like any other hard stage failure,
+     * triggering the reactive dump fallback when one is configured.
+     *
+     * Leave it `false` (default) for stages that may legitimately be empty, such
+     * as class/property partitions of a dataset that lacks that structure.
+     *
+     * @default false
+     */
+    expectsOutput?: boolean;
     /** Optional validation of the combined quads produced by all executors per batch. */
     validation?: {
         validator: Validator;
@@ -95,6 +112,8 @@ export interface SelectOptions {
 export declare class Stage {
     readonly name: string;
     readonly stages: readonly Stage[];
+    /** Whether an empty result is treated as a hard failure. @see {@link StageOptions.expectsOutput} */
+    readonly expectsOutput: boolean;
     private readonly executors;
     private readonly itemSelector?;
     private readonly batchSize;
@@ -104,6 +123,12 @@ export declare class Stage {
     /** The validator for this stage, if configured. */
     get validator(): Validator | undefined;
     run(dataset: Dataset, distribution: Distribution, writer: Writer, options?: RunOptions): Promise<NotSupported | void>;
+    /**
+     * Throw when {@link StageOptions.expectsOutput} is set but the stage produced
+     * no quads — a supported-but-empty result that signals a truncated or aborted
+     * endpoint response rather than a legitimately empty one.
+     */
+    private assertProduced;
     private runWithSelector;
     /**
      * Validate a buffer of quads. Throws on halt, returns the quads to write

package/dist/stage.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE/D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAGjD;;;;;;;GAOG;AACH,MAAM,MAAM,aAAa,CAAC,GAAG,IAAI,CAC/B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,EAC1B,OAAO,EAAE,GAAG,KACT,aAAa,CAAC,IAAI,CAAC,CAAC;AAEzB;;;;;;GAMG;AACH,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,YAAY,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,CAAC,EAAE,aAAa,CAAC,eAAe,CAAC,GAAG,aAAa,CAAC,eAAe,CAAC,EAAE,CAAC;CAC/E;AAED,2EAA2E;AAC3E,MAAM,MAAM,cAAc,GACtB,QAAQ,GACR,gBAAgB,GAChB,CAAC,QAAQ,GAAG,gBAAgB,CAAC,EAAE,CAAC;AAQpC,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,cAAc,CAAC;IAC1B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;;;OAOG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;IACjB,qFAAqF;IACrF,UAAU,CAAC,EAAE;QACX,SAAS,EAAE,SAAS,CAAC;QACrB,iEAAiE;QACjE,SAAS,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;KACvC,CAAC;CACH;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,CAAC,cAAc,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;IACtE;;;;;OAKG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,uDAAuD;AACvD,MAAM,WAAW,aAAa;IAC5B,+BAA+B;IAC/B,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAuB;IACjD,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAA6B;gBAE7C,OAAO,EAAE,YAAY;IAUjC,mDAAmD;IACnD,IAAI,SAAS,IAAI,SAAS,GAAG,SAAS,CAErC;IAEK,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;YAqDjB,eAAe;IA8J7B;;;OAGG;YACW,cAAc;YAqBd,UAAU;IA6BxB;;;;;OAKG;IACH,OAAO,CAAC,eAAe;CAiBxB;AA4BD,6GAA6G;AAC7G,MAAM,WAAW,YAAY;IAC3B,MAAM,CACJ,YAAY,EAAE,YAAY,EAC1B,SAAS,CAAC,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,aAAa,GACtB,aAAa,CAAC,gBAAgB,CAAC,CAAC;CACpC"}
+{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE/D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAGjD;;;;;;;GAOG;AACH,MAAM,MAAM,aAAa,CAAC,GAAG,IAAI,CAC/B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,EAC1B,OAAO,EAAE,GAAG,KACT,aAAa,CAAC,IAAI,CAAC,CAAC;AAEzB;;;;;;GAMG;AACH,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,YAAY,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,CAAC,EAAE,aAAa,CAAC,eAAe,CAAC,GAAG,aAAa,CAAC,eAAe,CAAC,EAAE,CAAC;CAC/E;AAED,2EAA2E;AAC3E,MAAM,MAAM,cAAc,GACtB,QAAQ,GACR,gBAAgB,GAChB,CAAC,QAAQ,GAAG,gBAAgB,CAAC,EAAE,CAAC;AAQpC,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,cAAc,CAAC;IAC1B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;;;OAOG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;IACjB;;;;;;;;;;;;;;;OAeG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,qFAAqF;IACrF,UAAU,CAAC,EAAE;QACX,SAAS,EAAE,SAAS,CAAC;QACrB,iEAAiE;QACjE,SAAS,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;KACvC,CAAC;CACH;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,CAAC,cAAc,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;IACtE;;;;;OAKG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,uDAAuD;AACvD,MAAM,WAAW,aAAa;IAC5B,+BAA+B;IAC/B,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,oGAAoG;IACpG,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAuB;IACjD,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAA6B;gBAE7C,OAAO,EAAE,YAAY;IAWjC,mDAAmD;IACnD,IAAI,SAAS,IAAI,SAAS,GAAG,SAAS,CAErC;IAEK,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;IAqE/B;;;;OAIG;IACH,OAAO,CAAC,cAAc;YAMR,eAAe;IAgK7B;;;OAGG;YACW,cAAc;YAqBd,UAAU;IA6BxB;;;;;OAKG;IACH,OAAO,CAAC,eAAe;CAiBxB;AAiDD,6GAA6G;AAC7G,MAAM,WAAW,YAAY;IAC3B,MAAM,CACJ,YAAY,EAAE,YAAY,EAC1B,SAAS,CAAC,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,aAAa,GACtB,aAAa,CAAC,gBAAgB,CAAC,CAAC;CACpC"}

package/dist/stage.js

@@ -4,6 +4,8 @@ import { AsyncQueue } from './asyncQueue.js';
 export class Stage {
     name;
     stages;
+    /** Whether an empty result is treated as a hard failure. @see {@link StageOptions.expectsOutput} */
+    expectsOutput;
     executors;
     itemSelector;
     batchSize;
@@ -17,6 +19,7 @@ export class Stage {
         this.batchSize = options.batchSize ?? 10;
         this.maxConcurrency = options.maxConcurrency ?? 10;
         this.validation = options.validation;
+        this.expectsOutput = options.expectsOutput ?? false;
     }
     /** The validator for this stage, if configured. */
     get validator() {
@@ -33,6 +36,9 @@ export class Stage {
         if (streams instanceof NotSupported) {
             return streams;
         }
+        // Quads the executors produced (before any validation filtering); used to
+        // enforce `expectsOutput` below.
+        let produced = 0;
         if (this.validation) {
             const buffer = [];
             for (const stream of streams) {
@@ -40,6 +46,7 @@ export class Stage {
                     buffer.push(quad);
                 }
             }
+            produced = buffer.length;
             const onInvalid = this.validation.onInvalid ?? 'write';
             if (onInvalid === 'write') {
                 await Promise.all([
@@ -58,9 +65,27 @@ export class Stage {
                 }
             }
         }
+        else if (this.expectsOutput) {
+            // Only thread the per-quad counter through when the count is actually
+            // needed; the default path stays a plain streaming write with no overhead.
+            await writer.write(dataset, countQuads(mergeStreams(streams), (count) => {
+                produced = count;
+            }));
+        }
         else {
             await writer.write(dataset, mergeStreams(streams));
         }
+        this.assertProduced(produced);
+    }
+    /**
+     * Throw when {@link StageOptions.expectsOutput} is set but the stage produced
+     * no quads — a supported-but-empty result that signals a truncated or aborted
+     * endpoint response rather than a legitimately empty one.
+     */
+    assertProduced(produced) {
+        if (this.expectsOutput && produced === 0) {
+            throw new Error(`Stage '${this.name}' expected output but produced none`);
+        }
     }
     async runWithSelector(selector, dataset, distribution, writer, options) {
         // Peek the first batch to detect an empty selector before starting the
@@ -185,6 +210,7 @@ export class Stage {
         if (!hasResults) {
             return new NotSupported('All executors returned NotSupported');
         }
+        this.assertProduced(quadsGenerated);
     }
     /**
      * Validate a buffer of quads. Throws on halt, returns the quads to write
@@ -260,3 +286,20 @@ async function* mergeStreams(streams) {
         yield* stream;
     }
 }
+/**
+ * Pass a quad stream through unchanged while counting it, reporting the total
+ * via `onCount` once the stream is exhausted. Lets a streaming write enforce
+ * {@link StageOptions.expectsOutput} without buffering.
+ *
+ * `onCount` fires only when the consumer drains the stream — which the pipeline
+ * writers do. A writer that stops early would leave the count short; callers
+ * relying on it for `expectsOutput` must consume the stream fully.
+ */
+async function* countQuads(stream, onCount) {
+    let count = 0;
+    for await (const quad of stream) {
+        count++;
+        yield quad;
+    }
+    onCount(count);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.31.1",
+  "version": "0.31.3",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/pipeline"
@@ -26,8 +26,8 @@
   "dependencies": {
     "@lde/dataset": "0.7.7",
     "@lde/dataset-registry-client": "0.8.4",
-    "@lde/distribution-health": "0.2.0",
-    "@lde/distribution-probe": "0.2.0",
+    "@lde/distribution-health": "0.2.1",
+    "@lde/distribution-probe": "0.2.1",
     "@lde/sparql-importer": "0.6.5",
     "@lde/sparql-server": "0.4.11",
     "@rdfjs/namespace": "^2.0.1",