@lde/pipeline 0.30.4 → 0.30.6

package/README.md

@@ -149,64 +149,48 @@ const executor = new SparqlConstructExecutor({
 
 The dedup set is scoped to each `execute()` call, so memory stays bounded to the number of unique quads per batch. A standalone `deduplicateQuads()` function is also exported for use outside the executor.
 
-`Executor` is an interface, so you can implement your own for logic that's hard to express in pure SPARQL — for example, cleaning up messy date notations or converting locale-specific dates to ISO 8601. The decorator pattern lets you wrap a SPARQL executor and post-process its quad stream in TypeScript:
+### Extending a stage with a quad transform
+
+Some logic is hard to express in pure SPARQL — cleaning up messy date notations, converting locale-specific dates to ISO 8601, or sampling an executor’s output and firing follow-up queries. Rather than subclass `Executor`, attach a `QuadTransform` to it as data: a plain function `(quads, context) => quads` that post-processes one executor’s output before the stage merges it with its siblings. This is extension point 1 of [ADR 2](../../docs/decisions/0002-unify-pipeline-extension-on-quad-transforms.md).
+
+A transform receives an `ExecutorContext` — the `dataset`, the `distribution` (so it can fire its own SPARQL queries), and the `stage` name. It runs once per executor call, so **write it to accept being called more than once**: a global stage calls it once over the executor’s complete output, but a per-class stage with batching enabled calls it once per batch (one class at `batchSize: 1`). Accumulate within an invocation, not across invocations — or keep the transform per-quad, where the number of calls makes no difference.
 
 ```typescript
 import { DataFactory } from 'n3';
-import type { Quad, Literal } from '@rdfjs/types';
-import type { Dataset, Distribution } from '@lde/dataset';
 import {
-  type Executor,
-  type ExecuteOptions,
-  NotSupported,
+  Stage,
+  SparqlConstructExecutor,
+  type QuadTransform,
+  type ExecutorContext,
 } from '@lde/pipeline';
 
-class TransformExecutor implements Executor {
-  constructor(
-    private readonly inner: Executor,
-    private readonly transform: (
-      quads: AsyncIterable<Quad>,
-      dataset: Dataset,
-    ) => AsyncIterable<Quad>,
-  ) {}
-
-  async execute(
-    dataset: Dataset,
-    distribution: Distribution,
-    options?: ExecuteOptions,
-  ): Promise<AsyncIterable<Quad> | NotSupported> {
-    const result = await this.inner.execute(dataset, distribution, options);
-    if (result instanceof NotSupported) return result;
-    return this.transform(result, dataset);
+const cleanDates: QuadTransform<ExecutorContext> = async function* (quads) {
+  for await (const quad of quads) {
+    if (quad.object.termType === 'Literal' && isMessyDate(quad.object)) {
+      yield DataFactory.quad(
+        quad.subject,
+        quad.predicate,
+        DataFactory.literal(
+          parseDutchDate(quad.object.value),
+          DataFactory.namedNode('http://www.w3.org/2001/XMLSchema#date'),
+        ),
+      );
+    } else {
+      yield quad;
+    }
   }
-}
-```
-
-Then use it to wrap any SPARQL executor:
+};
 
-```typescript
 new Stage({
   name: 'dates',
-  executors: new TransformExecutor(
-    await SparqlConstructExecutor.fromFile('dates.rq'),
-    async function* (quads) {
-      for await (const quad of quads) {
-        if (quad.object.termType === 'Literal' && isMessyDate(quad.object)) {
-          const cleaned = DataFactory.literal(
-            parseDutchDate(quad.object.value),
-            DataFactory.namedNode('http://www.w3.org/2001/XMLSchema#date'),
-          );
-          yield DataFactory.quad(quad.subject, quad.predicate, cleaned);
-        } else {
-          yield quad;
-        }
-      }
-    },
-  ),
+  executors: {
+    executor: await SparqlConstructExecutor.fromFile('dates.rq'),
+    transform: cleanDates,
+  },
 });
 ```
 
-This keeps SPARQL doing the heavy lifting while TypeScript handles the edge cases. See [@lde/pipeline-void](../pipeline-void)'s `VocabularyExecutor` for a real-world example of this pattern.
+`transform` accepts a single transform or an array applied in order, so a stage can compose several. This keeps SPARQL doing the heavy lifting while TypeScript handles the edge cases. See [@lde/pipeline-void](../pipeline-void)'s `withVocabularies` for a real-world example of this pattern.
 
 #### Adaptive timeouts
 

package/dist/pipeline.d.ts

@@ -1,3 +1,4 @@
+import { Dataset } from '@lde/dataset';
 import type { DatasetSelector } from './selector.js';
 import { Stage } from './stage.js';
 import type { QuadTransform } from './stage.js';
@@ -9,8 +10,15 @@ import { type TimeoutPolicy } from './sparql/timeoutPolicy.js';
 /** Plugin that hooks into pipeline lifecycle events. */
 export interface PipelinePlugin {
     name: string;
-    /** Transform the quad stream before writing. */
-    beforeStageWrite?: QuadTransform;
+    /**
+     * Transform the merged, post-stage quad stream before writing (extension
+     * point 2: pipeline-wide, post-merge). The home of cross-cutting concerns
+     * – provenance, namespace normalisation – that apply regardless of which
+     * executor produced a quad.
+     */
+    beforeStageWrite?: QuadTransform<{
+        dataset: Dataset;
+    }>;
 }
 export interface PipelineOptions {
     datasetSelector: DatasetSelector;

package/dist/pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAE1B,MAAM,4BAA4B,CAAC;AAQpC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,2BAA2B,CAAC;AAEnC,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,gDAAgD;IAChD,gBAAgB,CAAC,EAAE,aAAa,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,OAAO,CAAC,EAAE,cAAc,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;CAC/B;AAgFD,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAA8B;IACxD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAsB;gBAEzC,OAAO,EAAE,eAAe;IAkC9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAoBZ,cAAc;YAgFd,gBAAgB;IAW9B,OAAO,CAAE,aAAa;IAOtB;;;OAGG;YACW,QAAQ;IA0CtB,2EAA2E;YAC7D,eAAe;YAqBf,QAAQ;YA2DP,SAAS;CAczB"}
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AAGrD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAE1B,MAAM,4BAA4B,CAAC;AAQpC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,2BAA2B,CAAC;AAEnC,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,aAAa,CAAC;QAAE,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;CACxD;AAED,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,OAAO,CAAC,EAAE,cAAc,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;CAC/B;AAgFD,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAA8B;IACxD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAsB;gBAEzC,OAAO,EAAE,eAAe;IAkC9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAoBZ,cAAc;YAgFd,gBAAgB;IAW9B,OAAO,CAAE,aAAa;IAOtB;;;OAGG;YACW,QAAQ;IA0CtB,2EAA2E;YAC7D,eAAe;YAqBf,QAAQ;YA2DP,SAAS;CAczB"}

package/dist/pipeline.js

@@ -66,7 +66,7 @@ class TransformWriter {
         this.transform = transform;
     }
     async write(dataset, quads) {
-        await this.inner.write(dataset, this.transform(quads, dataset));
+        await this.inner.write(dataset, this.transform(quads, { dataset }));
     }
     async flush(dataset) {
         await this.inner.flush?.(dataset);
@@ -96,7 +96,7 @@ export class Pipeline {
             ?.map((p) => p.beforeStageWrite)
             .filter((t) => t !== undefined);
         if (transforms?.length) {
-            const composed = (quads, dataset) => transforms.reduce((q, fn) => fn(q, dataset), quads);
+            const composed = (quads, context) => transforms.reduce((q, fn) => fn(q, context), quads);
             writer = new TransformWriter(writer, composed);
         }
         this.writer = writer;

package/dist/plugin/namespaceNormalization.d.ts

@@ -1,5 +1,6 @@
 import type { QuadTransform } from '../stage.js';
 import type { PipelinePlugin } from '../pipeline.js';
+import type { Dataset } from '@lde/dataset';
 export interface NamespaceNormalizationOptions {
     /** Namespace URI prefix to match (e.g. `http://schema.org/`). */
     from: string;
@@ -14,7 +15,9 @@ export interface NamespaceNormalizationOptions {
  * `void:vocabulary` quads are left unchanged so consumers can see which
  * namespace the source dataset actually uses.
  */
-export declare function namespaceNormalizationTransform(options: NamespaceNormalizationOptions): QuadTransform;
+export declare function namespaceNormalizationTransform(options: NamespaceNormalizationOptions): QuadTransform<{
+    dataset: Dataset;
+}>;
 /**
  * Pipeline plugin that normalizes namespace prefixes in `void:class` and
  * `void:property` quad objects.

package/dist/plugin/namespaceNormalization.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"namespaceNormalization.d.ts","sourceRoot":"","sources":["../../src/plugin/namespaceNormalization.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,aAAa,CAAC;AAC/C,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,gBAAgB,CAAC;AASnD,MAAM,WAAW,6BAA6B;IAC5C,iEAAiE;IACjE,IAAI,EAAE,MAAM,CAAC;IACb,yEAAyE;IACzE,EAAE,EAAE,MAAM,CAAC;CACZ;AAED;;;;;;;GAOG;AACH,wBAAgB,+BAA+B,CAC7C,OAAO,EAAE,6BAA6B,GACrC,aAAa,CAEf;AAED;;;;;;GAMG;AACH,wBAAgB,4BAA4B,CAC1C,OAAO,EAAE,6BAA6B,GACrC,cAAc,CAKhB"}
+{"version":3,"file":"namespaceNormalization.d.ts","sourceRoot":"","sources":["../../src/plugin/namespaceNormalization.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AACrD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAS5C,MAAM,WAAW,6BAA6B;IAC5C,iEAAiE;IACjE,IAAI,EAAE,MAAM,CAAC;IACb,yEAAyE;IACzE,EAAE,EAAE,MAAM,CAAC;CACZ;AAED;;;;;;;GAOG;AACH,wBAAgB,+BAA+B,CAC7C,OAAO,EAAE,6BAA6B,GACrC,aAAa,CAAC;IAAE,OAAO,EAAE,OAAO,CAAA;CAAE,CAAC,CAErC;AAED;;;;;;GAMG;AACH,wBAAgB,4BAA4B,CAC1C,OAAO,EAAE,6BAA6B,GACrC,cAAc,CAKhB"}

package/dist/plugin/provenance.d.ts

@@ -1,7 +1,10 @@
 import type { QuadTransform } from '../stage.js';
 import type { PipelinePlugin } from '../pipeline.js';
+import type { Dataset } from '@lde/dataset';
 /** QuadTransform that appends PROV-O provenance quads. */
-export declare const provenanceTransform: QuadTransform;
+export declare const provenanceTransform: QuadTransform<{
+    dataset: Dataset;
+}>;
 /** Pipeline plugin that appends PROV-O provenance to every stage's output. */
 export declare function provenancePlugin(): PipelinePlugin;
 //# sourceMappingURL=provenance.d.ts.map

package/dist/plugin/provenance.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"provenance.d.ts","sourceRoot":"","sources":["../../src/plugin/provenance.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAkBrD,0DAA0D;AAC1D,eAAO,MAAM,mBAAmB,EAAE,aACgC,CAAC;AAEnE,8EAA8E;AAC9E,wBAAgB,gBAAgB,IAAI,cAAc,CAKjD"}
+{"version":3,"file":"provenance.d.ts","sourceRoot":"","sources":["../../src/plugin/provenance.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AACrD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAkB5C,0DAA0D;AAC1D,eAAO,MAAM,mBAAmB,EAAE,aAAa,CAAC;IAAE,OAAO,EAAE,OAAO,CAAA;CAAE,CAGC,CAAC;AAEtE,8EAA8E;AAC9E,wBAAgB,gBAAgB,IAAI,cAAc,CAKjD"}

package/dist/plugin/provenance.js

@@ -8,7 +8,7 @@ const PROV_STARTED_AT_TIME = namedNode('http://www.w3.org/ns/prov#startedAtTime'
 const PROV_ENDED_AT_TIME = namedNode('http://www.w3.org/ns/prov#endedAtTime');
 const XSD_DATE_TIME = namedNode('http://www.w3.org/2001/XMLSchema#dateTime');
 /** QuadTransform that appends PROV-O provenance quads. */
-export const provenanceTransform = (quads, dataset) => appendProvenanceQuads(quads, dataset.iri.toString(), new Date());
+export const provenanceTransform = (quads, { dataset }) => appendProvenanceQuads(quads, dataset.iri.toString(), new Date());
 /** Pipeline plugin that appends PROV-O provenance to every stage's output. */
 export function provenancePlugin() {
     return {

package/dist/plugin/schemaOrgNormalization.d.ts

@@ -1,11 +1,14 @@
 import type { QuadTransform } from '../stage.js';
 import type { PipelinePlugin } from '../pipeline.js';
+import type { Dataset } from '@lde/dataset';
 export interface SchemaOrgNormalizationOptions {
     /** When true, normalizes `https://schema.org/` to `http://schema.org/` instead. */
     reverse?: boolean;
 }
 /** QuadTransform that normalizes `http://schema.org/` to `https://schema.org/` in `void:class` and `void:property` objects. */
-export declare const schemaOrgNormalizationTransform: QuadTransform;
+export declare const schemaOrgNormalizationTransform: QuadTransform<{
+    dataset: Dataset;
+}>;
 /**
  * Pipeline plugin that normalizes Schema.org namespace prefixes in `void:class`
  * and `void:property` quad objects.

package/dist/plugin/schemaOrgNormalization.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"schemaOrgNormalization.d.ts","sourceRoot":"","sources":["../../src/plugin/schemaOrgNormalization.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,aAAa,CAAC;AAC/C,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,gBAAgB,CAAC;AASnD,MAAM,WAAW,6BAA6B;IAC5C,mFAAmF;IACnF,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,+HAA+H;AAC/H,eAAO,MAAM,+BAA+B,EAAE,aAI1C,CAAC;AAEL;;;;;;;;;GASG;AACH,wBAAgB,4BAA4B,CAC1C,OAAO,CAAC,EAAE,6BAA6B,GACtC,cAAc,CAOhB"}
+{"version":3,"file":"schemaOrgNormalization.d.ts","sourceRoot":"","sources":["../../src/plugin/schemaOrgNormalization.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AACrD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAS5C,MAAM,WAAW,6BAA6B;IAC5C,mFAAmF;IACnF,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,+HAA+H;AAC/H,eAAO,MAAM,+BAA+B,EAAE,aAAa,CAAC;IAC1D,OAAO,EAAE,OAAO,CAAC;CAClB,CAGC,CAAC;AAEH;;;;;;;;;GASG;AACH,wBAAgB,4BAA4B,CAC1C,OAAO,CAAC,EAAE,6BAA6B,GACtC,cAAc,CAOhB"}

package/dist/stage.d.ts

@@ -5,11 +5,49 @@ import { NotSupported } from './sparql/executor.js';
 import type { TimeoutPolicy } from './sparql/timeoutPolicy.js';
 import type { Validator } from './validator.js';
 import type { Writer } from './writer/writer.js';
-/** Transforms a quad stream, optionally using dataset metadata. */
-export type QuadTransform = (quads: AsyncIterable<Quad>, dataset: Dataset) => AsyncIterable<Quad>;
+/**
+ * Transforms a quad stream, given the context of its extension point.
+ *
+ * Every pipeline extension is the same operation – intercept the quad stream,
+ * `AsyncIterable<Quad> → AsyncIterable<Quad>` – differing only in *where* it
+ * runs and the `Ctx` in scope. See
+ * {@link https://github.com/ldelements/lde/blob/main/docs/decisions/0002-unify-pipeline-extension-on-quad-transforms.md | ADR 2}.
+ */
+export type QuadTransform<Ctx> = (quads: AsyncIterable<Quad>, context: Ctx) => AsyncIterable<Quad>;
+/**
+ * Context handed to a {@link QuadTransform} attached to an executor (extension
+ * point 1: per-executor output, pre-merge).
+ *
+ * `distribution` gives the transform endpoint reach – it may fire its own
+ * SPARQL queries – and `stage` carries the stage identity.
+ */
+export interface ExecutorContext {
+    dataset: Dataset;
+    distribution: Distribution;
+    stage: string;
+}
+/**
+ * An {@link Executor} with zero or more {@link QuadTransform}s attached as data.
+ *
+ * The stage runner applies the transform(s) in order to **this executor's
+ * output** before merging it with sibling executors. The window is one
+ * `execute()` call:
+ *
+ * - for a global stage that is the executor's complete output;
+ * - for a per-class stage that is one batch – one class at `batchSize: 1`.
+ *
+ * Decorating an executor is therefore construction-time data, not a wrapping
+ * class: the runner is the only code that delegates to the inner executor.
+ */
+export interface AttachedExecutor {
+    executor: Executor;
+    transform?: QuadTransform<ExecutorContext> | QuadTransform<ExecutorContext>[];
+}
+/** One or more executors, each optionally carrying attached transforms. */
+export type StageExecutors = Executor | AttachedExecutor | (Executor | AttachedExecutor)[];
 export interface StageOptions {
     name: string;
-    executors: Executor | Executor[];
+    executors: StageExecutors;
     itemSelector?: ItemSelector;
     /**
      * Maximum number of bindings per executor call.
@@ -73,6 +111,13 @@ export declare class Stage {
      */
     private validateBuffer;
     private executeAll;
+    /**
+     * Fold an executor's attached transforms over its output stream, in order,
+     * supplying the {@link ExecutorContext}. A transform sees one `execute()`
+     * call's output (see {@link AttachedExecutor}); `NotSupported` is handled by
+     * the caller and never reaches a transform.
+     */
+    private applyTransforms;
 }
 /** Selects items (as variable bindings) for executors to process. Pagination is an implementation detail. */
 export interface ItemSelector {

package/dist/stage.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE/D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAGjD,mEAAmE;AACnE,MAAM,MAAM,aAAa,GAAG,CAC1B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,EAC1B,OAAO,EAAE,OAAO,KACb,aAAa,CAAC,IAAI,CAAC,CAAC;AAEzB,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IACjC,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;;;OAOG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;IACjB,qFAAqF;IACrF,UAAU,CAAC,EAAE;QACX,SAAS,EAAE,SAAS,CAAC;QACrB,iEAAiE;QACjE,SAAS,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;KACvC,CAAC;CACH;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,CAAC,cAAc,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;IACtE;;;;;OAKG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,uDAAuD;AACvD,MAAM,WAAW,aAAa;IAC5B,+BAA+B;IAC/B,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAA6B;gBAE7C,OAAO,EAAE,YAAY;IAYjC,mDAAmD;IACnD,IAAI,SAAS,IAAI,SAAS,GAAG,SAAS,CAErC;IAEK,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;YAqDjB,eAAe;IAwJ7B;;;OAGG;YACW,cAAc;YAqBd,UAAU;CAwBzB;AAUD,6GAA6G;AAC7G,MAAM,WAAW,YAAY;IAC3B,MAAM,CACJ,YAAY,EAAE,YAAY,EAC1B,SAAS,CAAC,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,aAAa,GACtB,aAAa,CAAC,gBAAgB,CAAC,CAAC;CACpC"}
+{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE/D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAGjD;;;;;;;GAOG;AACH,MAAM,MAAM,aAAa,CAAC,GAAG,IAAI,CAC/B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,EAC1B,OAAO,EAAE,GAAG,KACT,aAAa,CAAC,IAAI,CAAC,CAAC;AAEzB;;;;;;GAMG;AACH,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,YAAY,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,CAAC,EAAE,aAAa,CAAC,eAAe,CAAC,GAAG,aAAa,CAAC,eAAe,CAAC,EAAE,CAAC;CAC/E;AAED,2EAA2E;AAC3E,MAAM,MAAM,cAAc,GACtB,QAAQ,GACR,gBAAgB,GAChB,CAAC,QAAQ,GAAG,gBAAgB,CAAC,EAAE,CAAC;AAQpC,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,cAAc,CAAC;IAC1B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;;;OAOG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;IACjB,qFAAqF;IACrF,UAAU,CAAC,EAAE;QACX,SAAS,EAAE,SAAS,CAAC;QACrB,iEAAiE;QACjE,SAAS,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;KACvC,CAAC;CACH;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,CAAC,cAAc,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;IACtE;;;;;OAKG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,uDAAuD;AACvD,MAAM,WAAW,aAAa;IAC5B,+BAA+B;IAC/B,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAuB;IACjD,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAA6B;gBAE7C,OAAO,EAAE,YAAY;IAUjC,mDAAmD;IACnD,IAAI,SAAS,IAAI,SAAS,GAAG,SAAS,CAErC;IAEK,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;YAqDjB,eAAe;IA8J7B;;;OAGG;YACW,cAAc;YAqBd,UAAU;IA6BxB;;;;;OAKG;IACH,OAAO,CAAC,eAAe;CAiBxB;AA4BD,6GAA6G;AAC7G,MAAM,WAAW,YAAY;IAC3B,MAAM,CACJ,YAAY,EAAE,YAAY,EAC1B,SAAS,CAAC,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,aAAa,GACtB,aAAa,CAAC,gBAAgB,CAAC,CAAC;CACpC"}

package/dist/stage.js

@@ -12,9 +12,7 @@ export class Stage {
     constructor(options) {
         this.name = options.name;
         this.stages = options.stages ?? [];
-        this.executors = Array.isArray(options.executors)
-            ? options.executors
-            : [options.executors];
+        this.executors = normalizeExecutors(options.executors);
         this.itemSelector = options.itemSelector;
         this.batchSize = options.batchSize ?? 10;
         this.maxConcurrency = options.maxConcurrency ?? 10;
@@ -118,7 +116,7 @@ export class Stage {
                     }
                     track((async () => {
                         // Run all executors for this batch in parallel.
-                        const executorOutputs = await Promise.all(this.executors.map(async (executor) => {
+                        const executorOutputs = await Promise.all(this.executors.map(async ({ executor, transforms }) => {
                             const result = await executor.execute(dataset, distribution, {
                                 bindings,
                                 timeout: options?.timeout,
@@ -126,8 +124,9 @@ export class Stage {
                             if (result instanceof NotSupported)
                                 return [];
                             hasResults = true;
+                            const stream = this.applyTransforms(transforms, result, dataset, distribution);
                             const quads = [];
-                            for await (const quad of result) {
+                            for await (const quad of stream) {
                                 quads.push(quad);
                             }
                             return quads;
@@ -204,7 +203,14 @@ export class Stage {
         return [];
     }
     async executeAll(dataset, distribution, timeout) {
-        const results = await Promise.all(this.executors.map((executor) => executor.execute(dataset, distribution, { timeout })));
+        const results = await Promise.all(this.executors.map(async ({ executor, transforms }) => {
+            const result = await executor.execute(dataset, distribution, {
+                timeout,
+            });
+            if (result instanceof NotSupported)
+                return result;
+            return this.applyTransforms(transforms, result, dataset, distribution);
+        }));
         const streams = [];
         for (const result of results) {
             if (!(result instanceof NotSupported)) {
@@ -216,6 +222,38 @@ export class Stage {
         }
         return streams;
     }
+    /**
+     * Fold an executor's attached transforms over its output stream, in order,
+     * supplying the {@link ExecutorContext}. A transform sees one `execute()`
+     * call's output (see {@link AttachedExecutor}); `NotSupported` is handled by
+     * the caller and never reaches a transform.
+     */
+    applyTransforms(transforms, stream, dataset, distribution) {
+        if (transforms.length === 0)
+            return stream;
+        const context = {
+            dataset,
+            distribution,
+            stage: this.name,
+        };
+        return transforms.reduce((quads, transform) => transform(quads, context), stream);
+    }
+}
+/** Normalise the {@link StageExecutors} union to executor + transforms pairs. */
+function normalizeExecutors(executors) {
+    const list = Array.isArray(executors) ? executors : [executors];
+    return list.map((entry) => {
+        if ('execute' in entry) {
+            return { executor: entry, transforms: [] };
+        }
+        const { executor, transform } = entry;
+        const transforms = transform === undefined
+            ? []
+            : Array.isArray(transform)
+                ? [...transform]
+                : [transform];
+        return { executor, transforms };
+    });
 }
 async function* mergeStreams(streams) {
     for (const stream of streams) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.30.4",
+  "version": "0.30.6",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/pipeline"
@@ -26,7 +26,7 @@
   "dependencies": {
     "@lde/dataset": "0.7.4",
     "@lde/dataset-registry-client": "0.8.0",
-    "@lde/distribution-probe": "0.1.5",
+    "@lde/distribution-probe": "0.1.6",
     "@lde/sparql-importer": "0.6.2",
     "@lde/sparql-server": "0.4.11",
     "@rdfjs/types": "^2.0.1",