@lde/pipeline 0.34.1 → 0.34.3

package/dist/pipeline.d.ts

@@ -1,4 +1,5 @@
 import { Dataset } from '@lde/dataset';
+import type { Quad } from '@rdfjs/types';
 import type { DatasetSelector } from './selector.js';
 import { Stage } from './stage.js';
 import type { QuadTransform } from './stage.js';
@@ -23,7 +24,7 @@ export interface BeforeStageWriteContext {
 /**
  * Context handed to a {@link PipelinePlugin.beforeDatasetWrite} transform: the
  * `dataset` whose complete, cross-stage output is being written. Unlike
- * {@link BeforeStageWriteContext} there is no `stage` — the transform sees one
+ * {@link BeforeStageWriteContext} there is no `stage` – the transform sees one
  * dataset's quads from every stage as a single stream.
  */
 export interface BeforeDatasetWriteContext {
@@ -36,7 +37,7 @@ export interface PipelinePlugin {
      * Transform a single stage's merged output before writing (extension point
      * 2: per stage, post-merge). Runs once per stage per dataset, carrying the
      * stage identity so it can mint stable per-`(dataset, stage)` IRIs. The home
-     * of stage-scoped concerns — provenance stamping, namespace rewriting of a
+     * of stage-scoped concerns – provenance stamping, namespace rewriting of a
      * stage's own quads.
      */
     beforeStageWrite?: QuadTransform<BeforeStageWriteContext>;
@@ -45,7 +46,7 @@ export interface PipelinePlugin {
      * dataset, cross-stage). Runs once over the concatenation of every stage's
      * output for one dataset, after all stages complete and before the write is
      * flushed. The home of concerns that need to reconcile quads *across* stages
-     * — deduplication, cross-stage roll-ups, or merging partition nodes that
+     * – deduplication, cross-stage roll-ups, or merging partition nodes that
      * different stages contribute to.
      *
      * The transform is a streaming {@link QuadTransform}: the pipeline pipes each
@@ -55,11 +56,19 @@ export interface PipelinePlugin {
      */
     beforeDatasetWrite?: QuadTransform<BeforeDatasetWriteContext>;
 }
-export interface PipelineOptions {
+export interface PipelineOptions<Out = Quad> {
     datasetSelector: DatasetSelector;
-    stages: Stage[];
-    writers: Writer | Writer[];
-    plugins?: PipelinePlugin[];
+    stages: Stage<Out>[];
+    writers: Writer<Out> | Writer<Out>[];
+    /**
+     * Quad-level plugins ({@link PipelinePlugin}). Available only to a plain quad
+     * pipeline: a projecting pipeline's write-side data is already the projected
+     * item, and both plugin points are `QuadTransform`s with no quads to act on,
+     * so the type forbids them (`never`). Reader-attached quad transforms are
+     * unaffected – they run before the projection seam. See
+     * {@link https://github.com/ldelements/lde/blob/main/docs/decisions/0013-project-inside-the-batch-per-root-type.md | ADR 13}.
+     */
+    plugins?: [Out] extends [Quad] ? PipelinePlugin[] : never;
     name?: string;
     distributionResolver?: DistributionResolver;
     chaining?: {
@@ -101,7 +110,7 @@ export interface PipelineOptions {
      */
     timeout?: () => TimeoutPolicy;
 }
-export declare class Pipeline {
+export declare class Pipeline<Out = Quad> {
     private readonly name;
     private readonly datasetSelector;
     private readonly stages;
@@ -114,7 +123,7 @@ export declare class Pipeline {
     private readonly timeoutFactory;
     private readonly provenanceStore?;
     private readonly pipelineVersion?;
-    constructor(options: PipelineOptions);
+    constructor(options: PipelineOptions<Out>);
     run(): Promise<void>;
     private processDataset;
     /** Persist the processing record for a dataset, when a store is configured. */

package/dist/pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AAGrD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD,OAAO,KAAK,EAKV,MAAM,EACP,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACL,KAAK,oBAAoB,EAI1B,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAY7D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,2BAA2B,CAAC;AAEnC;;;;;;;GAOG;AACH,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;GAKG;AACH,MAAM,WAAW,yBAAyB;IACxC,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,aAAa,CAAC,uBAAuB,CAAC,CAAC;IAC1D;;;;;;;;;;;;OAYG;IACH,kBAAkB,CAAC,EAAE,aAAa,CAAC,yBAAyB,CAAC,CAAC;CAC/D;AAED,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,OAAO,CAAC,EAAE,cAAc,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,gBAAgB,GAAG,SAAS,gBAAgB,EAAE,CAAC;IAC1D;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;CAC/B;AA0OD,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAyC;IAC3E,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAA2C;IAC/E,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAA8B;IACxD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAsB;IACrD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAkB;IACnD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAS;gBAE9B,OAAO,EAAE,eAAe;IAyD9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YA8CZ,cAAc;IAgO5B,+EAA+E;YACjE,aAAa;YAmBb,gBAAgB;IAW9B,OAAO,CAAE,aAAa;IAOtB;;;;;OAKG;IACH,OAAO,CAAC,WAAW;IAMnB;;;;;;;;OAQG;IACH,OAAO,CAAC,0BAA0B;IA4BlC;;;;;OAKG;YACW,SAAS;IAuCvB;;;OAGG;YACW,QAAQ;IA0CtB,2EAA2E;YAC7D,eAAe;YAqBf,QAAQ;YAuEP,SAAS;CAczB"}
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD,OAAO,KAAK,EAKV,MAAM,EACP,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACL,KAAK,oBAAoB,EAI1B,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAY7D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,2BAA2B,CAAC;AAEnC;;;;;;;GAOG;AACH,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;GAKG;AACH,MAAM,WAAW,yBAAyB;IACxC,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,aAAa,CAAC,uBAAuB,CAAC,CAAC;IAC1D;;;;;;;;;;;;OAYG;IACH,kBAAkB,CAAC,EAAE,aAAa,CAAC,yBAAyB,CAAC,CAAC;CAC/D;AAED,MAAM,WAAW,eAAe,CAAC,GAAG,GAAG,IAAI;IACzC,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;IACrC;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,cAAc,EAAE,GAAG,KAAK,CAAC;IAC1D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,gBAAgB,GAAG,SAAS,gBAAgB,EAAE,CAAC;IAC1D;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;CAC/B;AA0OD,qBAAa,QAAQ,CAAC,GAAG,GAAG,IAAI;IAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAyC;IAC3E,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAA2C;IAC/E,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmC;IAC7D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAsB;IACrD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAkB;IACnD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAS;gBAE9B,OAAO,EAAE,eAAe,CAAC,GAAG,CAAC;IA6DnC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAoDZ,cAAc;IAgO5B,+EAA+E;YACjE,aAAa;YAmBb,gBAAgB;IAa9B,OAAO,CAAE,aAAa;IAStB;;;;;OAKG;IACH,OAAO,CAAC,WAAW;IAgBnB;;;;;;;;OAQG;IACH,OAAO,CAAC,0BAA0B;IA4BlC;;;;;OAKG;YACW,SAAS;IAuCvB;;;OAGG;YACW,QAAQ;IA0CtB,2EAA2E;YAC7D,eAAe;YAqBf,QAAQ;YA8EP,SAAS;CAczB"}

package/dist/pipeline.js

@@ -60,7 +60,7 @@ class FanOutWriter {
         // Opening a writer's run can have side effects (a cross-pod lock, a fresh
         // collection). If one writer opens but a sibling then fails, the pipeline
         // never gets a RunWriter to abort, so roll the opened ones back here
-        // before rethrowing — otherwise their locks and collections leak.
+        // before rethrowing – otherwise their locks and collections leak.
         const results = await Promise.allSettled(this.writers.map((writer) => writer.openRun(context)));
         const opened = results.flatMap((result) => result.status === 'fulfilled' ? [result.value] : []);
         const failure = results.find((result) => result.status === 'rejected');
@@ -86,8 +86,8 @@ class FanOutRunWriter {
     constructor(runs) {
         this.runs = runs;
     }
-    async write(dataset, quads) {
-        const branches = tee(quads, this.runs.length);
+    async write(dataset, items) {
+        const branches = tee(items, this.runs.length);
         await Promise.all(this.runs.map((run, index) => run.write(dataset, branches[index])));
     }
     async flush(dataset, outcome) {
@@ -241,13 +241,17 @@ export class Pipeline {
         this.writer = Array.isArray(options.writers)
             ? new FanOutWriter(options.writers)
             : options.writers;
-        const transforms = options.plugins
+        // Plugins are quad-typed and only reachable when `Out` is `Quad` (the
+        // `plugins` field is `never` otherwise), but inside the class body `Out` is
+        // opaque, so the conditional-typed field is read as a plain plugin array.
+        const plugins = options.plugins;
+        const transforms = plugins
             ?.map((p) => p.beforeStageWrite)
             .filter((t) => t !== undefined);
         this.beforeStageWrite = transforms?.length
             ? (quads, context) => transforms.reduce((q, fn) => fn(q, context), quads)
             : undefined;
-        const datasetTransforms = options.plugins
+        const datasetTransforms = plugins
             ?.map((p) => p.beforeDatasetWrite)
             .filter((t) => t !== undefined);
         this.beforeDatasetWrite = datasetTransforms?.length
@@ -285,7 +289,10 @@ export class Pipeline {
         };
         const openedWriter = await this.writer.openRun(context);
         // Wrap the run writer so a whole-dataset transform sees every stage's
-        // output for a dataset as one stream before it is written.
+        // output for a dataset as one stream before it is written. The transform is
+        // quad-typed and only present when `Out` is `Quad` (a projecting pipeline
+        // has no plugins), so narrowing the run writer to `RunWriter<Quad>` here is
+        // sound – tsc cannot see that invariant.
         const runWriter = this.beforeDatasetWrite
             ? new DatasetTransformWriter(openedWriter, this.beforeDatasetWrite)
             : openedWriter;
@@ -495,6 +502,8 @@ export class Pipeline {
             this.reporter?.datasetValidated?.(dataset, report);
         }
     }
+    // Walks every stage and sub-stage to collect validators; it reads only
+    // `validator`/`stages`, never output, so it is agnostic to the item type.
     *collectStages(stages) {
         for (const stage of stages) {
             yield stage;
@@ -509,6 +518,9 @@ export class Pipeline {
      * per-`(dataset, stage)` IRIs rather than blank nodes.
      */
     stageWriter(runWriter, stage) {
+        // `beforeStageWrite` is quad-typed and only present when `Out` is `Quad` (a
+        // projecting pipeline has no plugins), so the `TransformWriter` wrap is
+        // reached only in that case; tsc cannot see the invariant, hence the casts.
         return this.beforeStageWrite
             ? new TransformWriter(runWriter, this.beforeStageWrite, stage)
             : runWriter;
@@ -596,10 +608,12 @@ export class Pipeline {
     async runChain(dataset, distribution, stage, runWriter, context, timeout) {
         const { stageOutputResolver, outputDir } = this.chaining;
         const outputFiles = [];
-        // Run one chained stage into its scratch FileWriter and flush it, so the
-        // output file exists on its final path before the resolver reads it. The
-        // scratch run is bracketed like any other: committed on success, aborted
-        // on failure so no half-written temp file is left behind.
+        // Chaining is a quad-only concern: every stage in a chain writes N-Triples
+        // to a scratch FileWriter (a `Writer<Quad>`), and a chained parent cannot
+        // project (a stage with `stages` forbids `project`), so it emits quads. The
+        // requirement "sub-stages imply `Out` is `Quad`" is not expressible – `stages`
+        // and `chaining` are independent options – so it is a runtime contract, and
+        // the scratch machinery treats the parent stage as `Stage<Quad>`.
         const runScratchStage = async (chainedStage, stageDistribution) => {
             const scratchWriter = new FileWriter({
                 outputDir: `${outputDir}/${chainedStage.name}`,
@@ -618,7 +632,7 @@ export class Pipeline {
             return scratchWriter.getOutputPath(dataset);
         };
         try {
-            // 1. Run parent stage → scratch FileWriter.
+            // 1. Run parent stage → scratch FileWriter (quad-only, per above).
             const parentOutput = await runScratchStage(stage, distribution);
             outputFiles.push(parentOutput);
             // 2. Chain through children.
@@ -632,6 +646,8 @@ export class Pipeline {
             }
             // 3. Concatenate all output files → the open run, applying the plugins'
             // beforeStageWrite transforms once for the chain under the parent stage.
+            // The concatenation is quads (per the quad-only contract above), written
+            // to the `Out`-typed run writer – sound because chaining implies `Quad`.
             await this.stageWriter(runWriter, stage.name).write(dataset, this.readFiles(outputFiles));
         }
         finally {

package/dist/stage.d.ts

@@ -26,6 +26,23 @@ export interface ReaderContext {
     distribution: Distribution;
     stage: string;
 }
+/**
+ * The pipeline's one type-changing seam: turn a root-complete batch of quads
+ * into a stage's output items. The only extension point whose output is not
+ * `Quad` – a quad enrichment is a {@link QuadTransform} attached to a reader
+ * (extension point 1), which runs *before* this. See
+ * {@link https://github.com/ldelements/lde/blob/main/docs/decisions/0013-project-inside-the-batch-per-root-type.md | ADR 13}.
+ */
+export type BatchTransform<Out> = (quads: readonly Quad[], context: BatchContext) => Iterable<Out> | AsyncIterable<Out>;
+/**
+ * Context handed to a {@link BatchTransform}: {@link ReaderContext} plus the
+ * batch's selected roots. `bindings` are the item-selector rows the readers were
+ * given as a `VALUES` block, so the batch is root-complete by construction – the
+ * projection frames exactly these roots rather than discovering them.
+ */
+export interface BatchContext extends ReaderContext {
+    bindings: readonly VariableBindings[];
+}
 /**
  * An {@link Reader} with zero or more {@link QuadTransform}s attached as data.
  *
@@ -45,10 +62,27 @@ export interface AttachedReader {
 }
 /** One or more readers, each optionally carrying attached transforms. */
 export type StageReaders = Reader | AttachedReader | (Reader | AttachedReader)[];
-export interface StageOptions {
+export interface StageOptions<Out = Quad> {
     name: string;
     readers: StageReaders;
     itemSelector?: ItemSelector;
+    /**
+     * Turn each root-complete batch into this stage's output items – the one seam
+     * whose output type differs from `Quad` ({@link BatchTransform}). Requires
+     * {@link StageOptions.itemSelector} (a batch only exists under a selector) and
+     * forbids {@link StageOptions.stages} (a chained stage serializes to
+     * N-Triples, which a projected item cannot). Omit for a plain quad stage.
+     */
+    project?: BatchTransform<Out>;
+    /**
+     * Capacity of the bounded queue funnelling this stage's concurrent batches
+     * into the single write. The queue applies backpressure at this many items,
+     * so it bounds memory – set it where the cost per item is large (a projected
+     * document is far heavier than a quad). Only meaningful with an item selector.
+     *
+     * @default 128
+     */
+    queueCapacity?: number;
     /**
      * Maximum number of bindings per reader call.
      *
@@ -74,7 +108,7 @@ export interface StageOptions {
      * Treat a supported stage that produces no quads as a hard failure (throws),
      * rather than a legitimately empty result.
      *
-     * Set this for stages whose query must yield output — typically a scalar
+     * Set this for stages whose query must yield output – typically a scalar
      * aggregate such as `SELECT (COUNT(*) AS ?n)`, which always returns exactly
      * one row, so zero quads can only mean the endpoint truncated or aborted the
      * response (e.g. a timeout surfaced as an empty `HTTP 200`). The resulting
@@ -109,7 +143,7 @@ export interface SelectOptions {
     /** Per-call timeout policy. */
     timeout?: TimeoutPolicy;
 }
-export declare class Stage {
+export declare class Stage<Out = Quad> {
     readonly name: string;
     readonly stages: readonly Stage[];
     /** Whether an empty result is treated as a hard failure. @see {@link StageOptions.expectsOutput} */
@@ -119,13 +153,15 @@ export declare class Stage {
     private readonly batchSize;
     private readonly maxConcurrency;
     private readonly validation?;
-    constructor(options: StageOptions);
+    private readonly project?;
+    private readonly queueCapacity?;
+    constructor(options: StageOptions<Out>);
     /** The validator for this stage, if configured. */
     get validator(): Validator | undefined;
-    run(dataset: Dataset, distribution: Distribution, writer: DatasetWriter, options?: RunOptions): Promise<NotSupported | void>;
+    run(dataset: Dataset, distribution: Distribution, writer: DatasetWriter<Out>, options?: RunOptions): Promise<NotSupported | void>;
     /**
      * Throw when {@link StageOptions.expectsOutput} is set but the stage produced
-     * no quads — a supported-but-empty result that signals a truncated or aborted
+     * no quads – a supported-but-empty result that signals a truncated or aborted
      * endpoint response rather than a legitimately empty one.
      */
     private assertProduced;

package/dist/stage.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE/D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAGxD;;;;;;;GAOG;AACH,MAAM,MAAM,aAAa,CAAC,GAAG,IAAI,CAC/B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,EAC1B,OAAO,EAAE,GAAG,KACT,aAAa,CAAC,IAAI,CAAC,CAAC;AAEzB;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,YAAY,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,aAAa,CAAC,aAAa,CAAC,GAAG,aAAa,CAAC,aAAa,CAAC,EAAE,CAAC;CAC3E;AAED,yEAAyE;AACzE,MAAM,MAAM,YAAY,GACpB,MAAM,GACN,cAAc,GACd,CAAC,MAAM,GAAG,cAAc,CAAC,EAAE,CAAC;AAQhC,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,YAAY,CAAC;IACtB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;;;OAOG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;IACjB;;;;;;;;;;;;;;;OAeG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,mFAAmF;IACnF,UAAU,CAAC,EAAE;QACX,SAAS,EAAE,SAAS,CAAC;QACrB,iEAAiE;QACjE,SAAS,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;KACvC,CAAC;CACH;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,CAAC,cAAc,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;IACtE;;;;;OAKG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,uDAAuD;AACvD,MAAM,WAAW,aAAa;IAC5B,+BAA+B;IAC/B,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,oGAAoG;IACpG,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;IAC7C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAA6B;gBAE7C,OAAO,EAAE,YAAY;IAWjC,mDAAmD;IACnD,IAAI,SAAS,IAAI,SAAS,GAAG,SAAS,CAErC;IAEK,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,aAAa,EACrB,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;IAqE/B;;;;OAIG;IACH,OAAO,CAAC,cAAc;YAMR,eAAe;IAgK7B;;;OAGG;YACW,cAAc;YAqBd,OAAO;IA6BrB;;;;;OAKG;IACH,OAAO,CAAC,eAAe;CAiBxB;AAiDD,2GAA2G;AAC3G,MAAM,WAAW,YAAY;IAC3B,MAAM,CACJ,YAAY,EAAE,YAAY,EAC1B,SAAS,CAAC,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,aAAa,GACtB,aAAa,CAAC,gBAAgB,CAAC,CAAC;CACpC"}
+{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE/D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAGxD;;;;;;;GAOG;AACH,MAAM,MAAM,aAAa,CAAC,GAAG,IAAI,CAC/B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,EAC1B,OAAO,EAAE,GAAG,KACT,aAAa,CAAC,IAAI,CAAC,CAAC;AAEzB;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,YAAY,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;GAMG;AACH,MAAM,MAAM,cAAc,CAAC,GAAG,IAAI,CAChC,KAAK,EAAE,SAAS,IAAI,EAAE,EACtB,OAAO,EAAE,YAAY,KAClB,QAAQ,CAAC,GAAG,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;AAExC;;;;;GAKG;AACH,MAAM,WAAW,YAAa,SAAQ,aAAa;IACjD,QAAQ,EAAE,SAAS,gBAAgB,EAAE,CAAC;CACvC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,aAAa,CAAC,aAAa,CAAC,GAAG,aAAa,CAAC,aAAa,CAAC,EAAE,CAAC;CAC3E;AAED,yEAAyE;AACzE,MAAM,MAAM,YAAY,GACpB,MAAM,GACN,cAAc,GACd,CAAC,MAAM,GAAG,cAAc,CAAC,EAAE,CAAC;AAQhC,MAAM,WAAW,YAAY,CAAC,GAAG,GAAG,IAAI;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,YAAY,CAAC;IACtB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC;IAC9B;;;;;;;OAOG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;;;OAOG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;IACjB;;;;;;;;;;;;;;;OAeG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,mFAAmF;IACnF,UAAU,CAAC,EAAE;QACX,SAAS,EAAE,SAAS,CAAC;QACrB,iEAAiE;QACjE,SAAS,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;KACvC,CAAC;CACH;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,CAAC,cAAc,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;IACtE;;;;;OAKG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,uDAAuD;AACvD,MAAM,WAAW,aAAa;IAC5B,+BAA+B;IAC/B,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,qBAAa,KAAK,CAAC,GAAG,GAAG,IAAI;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,oGAAoG;IACpG,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;IAC7C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAkC;IAC9D,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAS;gBAE5B,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC;IAuBtC,mDAAmD;IACnD,IAAI,SAAS,IAAI,SAAS,GAAG,SAAS,CAErC;IAEK,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,aAAa,CAAC,GAAG,CAAC,EAC1B,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;IA2E/B;;;;OAIG;IACH,OAAO,CAAC,cAAc;YAMR,eAAe;IAiL7B;;;OAGG;YACW,cAAc;YAqBd,OAAO;IA6BrB;;;;;OAKG;IACH,OAAO,CAAC,eAAe;CAiBxB;AAiDD,2GAA2G;AAC3G,MAAM,WAAW,YAAY;IAC3B,MAAM,CACJ,YAAY,EAAE,YAAY,EAC1B,SAAS,CAAC,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,aAAa,GACtB,aAAa,CAAC,gBAAgB,CAAC,CAAC;CACpC"}

package/dist/stage.js

@@ -11,7 +11,15 @@ export class Stage {
     batchSize;
     maxConcurrency;
     validation;
+    project;
+    queueCapacity;
     constructor(options) {
+        if (options.project && !options.itemSelector) {
+            throw new Error(`Stage '${options.name}': 'project' requires an 'itemSelector' – without one there is no batch to project, only the readers' whole output.`);
+        }
+        if (options.project && (options.stages?.length ?? 0) > 0) {
+            throw new Error(`Stage '${options.name}': 'project' cannot combine with chained 'stages' – a chained stage serializes to N-Triples, which a projected item cannot.`);
+        }
         this.name = options.name;
         this.stages = options.stages ?? [];
         this.readers = normalizeReaders(options.readers);
@@ -20,6 +28,8 @@ export class Stage {
         this.maxConcurrency = options.maxConcurrency ?? 10;
         this.validation = options.validation;
         this.expectsOutput = options.expectsOutput ?? false;
+        this.project = options.project;
+        this.queueCapacity = options.queueCapacity;
     }
     /** The validator for this stage, if configured. */
     get validator() {
@@ -36,6 +46,11 @@ export class Stage {
         if (streams instanceof NotSupported) {
             return streams;
         }
+        // The non-selector path has no batch, so `project` is forbidden here (the
+        // constructor rejects `project` without an `itemSelector`). It therefore
+        // only ever writes quads, and `Out` is `Quad` at runtime – tsc cannot see
+        // that invariant, so the writer is narrowed once.
+        const quadWriter = writer;
         // Quads the readers produced (before any validation filtering); used to
         // enforce `expectsOutput` below.
         let produced = 0;
@@ -50,7 +65,7 @@ export class Stage {
             const onInvalid = this.validation.onInvalid ?? 'write';
             if (onInvalid === 'write') {
                 await Promise.all([
-                    writer.write(dataset, (async function* () {
+                    quadWriter.write(dataset, (async function* () {
                         yield* buffer;
                     })()),
                     this.validation.validator.validate(buffer, dataset),
@@ -59,7 +74,7 @@ export class Stage {
             else {
                 const accepted = await this.validateBuffer(buffer, dataset);
                 if (accepted.length > 0) {
-                    await writer.write(dataset, (async function* () {
+                    await quadWriter.write(dataset, (async function* () {
                         yield* accepted;
                     })());
                 }
@@ -68,18 +83,18 @@ export class Stage {
         else if (this.expectsOutput) {
             // Only thread the per-quad counter through when the count is actually
             // needed; the default path stays a plain streaming write with no overhead.
-            await writer.write(dataset, countQuads(mergeStreams(streams), (count) => {
+            await quadWriter.write(dataset, countQuads(mergeStreams(streams), (count) => {
                 produced = count;
             }));
         }
         else {
-            await writer.write(dataset, mergeStreams(streams));
+            await quadWriter.write(dataset, mergeStreams(streams));
         }
         this.assertProduced(produced);
     }
     /**
      * Throw when {@link StageOptions.expectsOutput} is set but the stage produced
-     * no quads — a supported-but-empty result that signals a truncated or aborted
+     * no quads – a supported-but-empty result that signals a truncated or aborted
      * endpoint response rather than a legitimately empty one.
      */
     assertProduced(produced) {
@@ -107,8 +122,11 @@ export class Stage {
                 yield next.value;
             }
         })();
-        const queue = new AsyncQueue();
+        const queue = new AsyncQueue(this.queueCapacity);
         let itemsProcessed = 0;
+        // Output items written this run. Equals quads written for a plain stage;
+        // for a projecting stage it counts projected items (documents), one or more
+        // – or fewer – per root, which is what `expectsOutput`/`onProgress` report.
         let quadsGenerated = 0;
         let hasResults = false;
         const onInvalid = this.validation?.onInvalid ?? 'write';
@@ -157,25 +175,34 @@ export class Stage {
                             return quads;
                         }));
                         const batchQuads = readerOutputs.flat();
+                        // Validation runs on the quads, before projection – validators are
+                        // quad-typed. 'skip'/'halt' must resolve it before writing; 'write'
+                        // validates concurrently, below, without blocking.
+                        let acceptedQuads = batchQuads;
                         if (this.validation &&
                             batchQuads.length > 0 &&
                             onInvalid !== 'write') {
-                            // 'skip' or 'halt': must await validation before deciding to write.
-                            const accepted = await this.validateBuffer(batchQuads, dataset);
-                            for (const quad of accepted) {
-                                await queue.push(quad);
-                                quadsGenerated++;
-                            }
+                            acceptedQuads = await this.validateBuffer(batchQuads, dataset);
                         }
-                        else {
-                            for (const quad of batchQuads) {
-                                await queue.push(quad);
-                                quadsGenerated++;
-                            }
-                            if (this.validation && batchQuads.length > 0) {
-                                // 'write' mode: validate concurrently without blocking the write path.
-                                pendingValidations.push(this.validation.validator.validate(batchQuads, dataset));
-                            }
+                        // The one type-changing seam: project the root-complete batch into
+                        // output items, or pass the quads through for a plain stage (`Out`
+                        // is `Quad` at runtime when there is no projection).
+                        const items = this.project
+                            ? this.project(acceptedQuads, {
+                                dataset,
+                                distribution,
+                                stage: this.name,
+                                bindings,
+                            })
+                            : acceptedQuads;
+                        for await (const item of items) {
+                            await queue.push(item);
+                            quadsGenerated++;
+                        }
+                        if (this.validation &&
+                            batchQuads.length > 0 &&
+                            onInvalid === 'write') {
+                            pendingValidations.push(this.validation.validator.validate(batchQuads, dataset));
                         }
                         itemsProcessed += bindings.length;
                         options?.onProgress?.(itemsProcessed, quadsGenerated);
@@ -291,7 +318,7 @@ async function* mergeStreams(streams) {
  * via `onCount` once the stream is exhausted. Lets a streaming write enforce
  * {@link StageOptions.expectsOutput} without buffering.
  *
- * `onCount` fires only when the consumer drains the stream — which the pipeline
+ * `onCount` fires only when the consumer drains the stream – which the pipeline
  * writers do. A writer that stops early would leave the count short; callers
  * relying on it for `expectsOutput` must consume the stream fully.
  */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.34.1",
+  "version": "0.34.3",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/pipeline"
@@ -26,14 +26,14 @@
   "dependencies": {
     "@lde/dataset": "^0.7.8",
     "@lde/dataset-registry-client": "^0.8.6",
-    "@lde/distribution-health": "^0.2.6",
+    "@lde/distribution-health": "^0.2.7",
     "@lde/distribution-probe": "^0.2.6",
-    "@lde/sparql-importer": "^0.6.6",
+    "@lde/sparql-importer": "^0.6.7",
     "@lde/sparql-server": "^0.4.12",
     "@rdfjs/namespace": "^2.0.1",
     "@rdfjs/types": "^2.0.1",
     "@tpluscode/rdf-ns-builders": "^5.0.0",
-    "@traqula/generator-sparql-1-1": "^1.1.6",
+    "@traqula/generator-sparql-1-1": "^1.1.8",
     "@traqula/parser-sparql-1-1": "^1.1.5",
     "@traqula/rules-sparql-1-1": "^1.1.0",
     "fetch-sparql-endpoint": "^7.1.1",