@lde/pipeline 0.38.1 → 0.39.1

package/dist/combineReporters.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"combineReporters.d.ts","sourceRoot":"","sources":["../src/combineReporters.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAU9D;;;;;;;;;;;;GAYG;AACH,wBAAgB,gBAAgB,CAC9B,SAAS,EAAE,SAAS,gBAAgB,EAAE,GACrC,gBAAgB,CA+ClB"}
+{"version":3,"file":"combineReporters.d.ts","sourceRoot":"","sources":["../src/combineReporters.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAU9D;;;;;;;;;;;;GAYG;AACH,wBAAgB,gBAAgB,CAC9B,SAAS,EAAE,SAAS,gBAAgB,EAAE,GACrC,gBAAgB,CAgDlB"}

package/dist/combineReporters.js

@@ -47,6 +47,7 @@ export function combineReporters(reporters) {
         datasetValidated: (...args) => forward('datasetValidated', ...args),
         datasetComplete: (...args) => forward('datasetComplete', ...args),
         datasetSkipped: (...args) => forward('datasetSkipped', ...args),
+        itemsRejected: (...args) => forward('itemsRejected', ...args),
         pipelineComplete: (...args) => forward('pipelineComplete', ...args),
         timeoutTightened: (...args) => forward('timeoutTightened', ...args),
         timeoutRelaxed: (...args) => forward('timeoutRelaxed', ...args),

package/dist/pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD,OAAO,KAAK,EAKV,MAAM,EACP,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACL,KAAK,oBAAoB,EAI1B,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAY7D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,2BAA2B,CAAC;AAEnC;;;;;;;;;;;GAWG;AACH,qBAAa,cAAe,SAAQ,KAAK;;CAOxC;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;GAKG;AACH,MAAM,WAAW,yBAAyB;IACxC,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,aAAa,CAAC,uBAAuB,CAAC,CAAC;IAC1D;;;;;;;;;;;;OAYG;IACH,kBAAkB,CAAC,EAAE,aAAa,CAAC,yBAAyB,CAAC,CAAC;CAC/D;AAED,MAAM,WAAW,eAAe,CAAC,GAAG,GAAG,IAAI;IACzC,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;IACrC;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,cAAc,EAAE,GAAG,KAAK,CAAC;IAC1D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,gBAAgB,GAAG,SAAS,gBAAgB,EAAE,CAAC;IAC1D;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;CAC/B;AA0OD,qBAAa,QAAQ,CAAC,GAAG,GAAG,IAAI;IAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAyC;IAC3E,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAA2C;IAC/E,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmC;IAC7D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAsB;IACrD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAkB;IACnD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAS;gBAE9B,OAAO,EAAE,eAAe,CAAC,GAAG,CAAC;IA6DnC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAiFZ,cAAc;IA4O5B,+EAA+E;YACjE,aAAa;YAyBb,gBAAgB;IAa9B,OAAO,CAAE,aAAa;IAStB;;;;;OAKG;IACH,OAAO,CAAC,WAAW;IAgBnB;;;;;;;;OAQG;IACH,OAAO,CAAC,0BAA0B;IA0BlC;;;;;OAKG;YACW,SAAS;IAuCvB;;;OAGG;YACW,QAAQ;IA0CtB,2EAA2E;YAC7D,eAAe;YAqBf,QAAQ;YA8EP,SAAS;CAczB"}
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD,OAAO,KAAK,EAKV,MAAM,EACP,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACL,KAAK,oBAAoB,EAI1B,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAY7D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,2BAA2B,CAAC;AAEnC;;;;;;;;;;;GAWG;AACH,qBAAa,cAAe,SAAQ,KAAK;;CAOxC;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;GAKG;AACH,MAAM,WAAW,yBAAyB;IACxC,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,aAAa,CAAC,uBAAuB,CAAC,CAAC;IAC1D;;;;;;;;;;;;OAYG;IACH,kBAAkB,CAAC,EAAE,aAAa,CAAC,yBAAyB,CAAC,CAAC;CAC/D;AAED,MAAM,WAAW,eAAe,CAAC,GAAG,GAAG,IAAI;IACzC,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;IACrC;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,cAAc,EAAE,GAAG,KAAK,CAAC;IAC1D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,gBAAgB,GAAG,SAAS,gBAAgB,EAAE,CAAC;IAC1D;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;CAC/B;AA0OD,qBAAa,QAAQ,CAAC,GAAG,GAAG,IAAI;IAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAyC;IAC3E,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAA2C;IAC/E,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmC;IAC7D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAsB;IACrD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAkB;IACnD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAS;gBAE9B,OAAO,EAAE,eAAe,CAAC,GAAG,CAAC;IA6DnC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAmFZ,cAAc;IA4O5B,+EAA+E;YACjE,aAAa;YAyBb,gBAAgB;IAa9B,OAAO,CAAE,aAAa;IAStB;;;;;OAKG;IACH,OAAO,CAAC,WAAW;IAgBnB;;;;;;;;OAQG;IACH,OAAO,CAAC,0BAA0B;IA0BlC;;;;;OAKG;YACW,SAAS;IAuCvB;;;OAGG;YACW,QAAQ;IA0CtB,2EAA2E;YAC7D,eAAe;YAqBf,QAAQ;YA8EP,SAAS;CAczB"}

package/dist/pipeline.js

@@ -308,6 +308,7 @@ export class Pipeline {
             runId: randomUUID(),
             startedAt: new Date().toISOString(),
             selectedSources: () => selectedSources,
+            itemsRejected: (dataset, rejection) => this.reporter?.itemsRejected?.(dataset, rejection),
             provenance: this.provenanceStore,
         };
         const openedWriter = await this.writer.openRun(context);

package/dist/progressReporter.d.ts

@@ -19,6 +19,31 @@ export interface DistributionAnalysisResult {
      */
     fingerprint?: string | null;
 }
+/**
+ * Items a writer left out of a dataset’s output instead of failing the
+ * dataset – documents a destination refused, say – reported once per
+ * (dataset, target) as the writer finalizes the dataset.
+ */
+export interface ItemsRejected {
+    /** The part of the destination that refused them, e.g. a search index’s
+     *  type name. */
+    readonly target: string;
+    /** Items the dataset wrote to {@link target}, the rejected ones included. */
+    readonly written: number;
+    /** Items left out. */
+    readonly rejected: number;
+    /** Why, most frequent first: each distinct reason, how many items it
+     *  rejected, and the ids of a few of them. Bounded by the writer, so a
+     *  reason that varies per item cannot grow it without limit. */
+    readonly reasons: readonly RejectionReason[];
+}
+/** One reason a writer rejected items, in {@link ItemsRejected.reasons}. */
+export interface RejectionReason {
+    readonly reason: string;
+    readonly count: number;
+    /** Ids of some of the items rejected for this reason. */
+    readonly examples: readonly string[];
+}
 export interface ProgressReporter {
     pipelineStart?(name: string): void;
     datasetsSelected?(count: number, duration: number): void;
@@ -74,6 +99,11 @@ export interface ProgressReporter {
         heapUsedBytes: number;
     }): void;
     datasetSkipped?(dataset: Dataset, reason: string): void;
+    /**
+     * Called when a writer leaves some of a dataset’s items out rather than
+     * failing the dataset ({@link RunContext.itemsRejected}).
+     */
+    itemsRejected?(dataset: Dataset, rejection: ItemsRejected): void;
     pipelineComplete?(result: {
         duration: number;
         memoryUsageBytes: number;

package/dist/progressReporter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"progressReporter.d.ts","sourceRoot":"","sources":["../src/progressReporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACvD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAC;AAExE,MAAM,WAAW,0BAA0B;IACzC,YAAY,EAAE,YAAY,CAAC;IAC3B,IAAI,EAAE,QAAQ,GAAG,WAAW,GAAG,eAAe,CAAC;IAC/C,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B;AAED,MAAM,WAAW,gBAAgB;IAC/B,aAAa,CAAC,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,gBAAgB,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACzD;;;;OAIG;IACH,cAAc,CAAC,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,YAAY,CAAC,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACtC,8DAA8D;IAC9D,kBAAkB,CAAC,CAAC,MAAM,EAAE,0BAA0B,GAAG,IAAI,CAAC;IAC9D,6CAA6C;IAC7C,aAAa,CAAC,IAAI,IAAI,CAAC;IACvB,kDAAkD;IAClD,YAAY,CAAC,CAAC,YAAY,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/D;;;;;OAKG;IACH,qBAAqB,CAAC,CACpB,YAAY,EAAE,YAAY,EAC1B,OAAO,EAAE,eAAe,GACvB,IAAI,CAAC;IACR;+CAC2C;IAC3C,oBAAoB,CAAC,CACnB,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,QAAQ,CAAC,EAAE,aAAa,GACvB,IAAI,CAAC;IACR,UAAU,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,aAAa,CAAC,CAAC,MAAM,EAAE;QACrB,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,MAAM,CAAC;QACvB,gBAAgB,EAAE,MAAM,CAAC;QACzB,aAAa,EAAE,MAAM,CAAC;KACvB,GAAG,IAAI,CAAC;IACT,aAAa,CAAC,CACZ,KAAK,EAAE,MAAM,EACb,MAAM,EAAE;QACN,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,MAAM,CAAC;QACvB,QAAQ,EAAE,MAAM,CAAC;KAClB,GACA,IAAI,CAAC;IACR,WAAW,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC;IAChD,YAAY,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACnD;;;;;;;;OAQG;IACH,gBAAgB,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACpE,eAAe,CAAC,CACd,OAAO,EAAE,OAAO,EAChB,MAAM,EAAE;QAAE,gBAAgB,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,GAC1D,IAAI,CAAC;IACR,cAAc,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACxD,gBAAgB,CAAC,CAAC,MAAM,EAAE;QACxB,QAAQ,EAAE,MAAM,CAAC;QACjB,gBAAgB,EAAE,MAAM,CAAC;QACzB,aAAa,EAAE,MAAM,CAAC;KACvB,GAAG,IAAI,CAAC;IACT;;;;OAIG;IACH,gBAAgB,CAAC,CAAC,KAAK,EAAE,sBAAsB,GAAG,IAAI,CAAC;IACvD;;;OAGG;IACH,cAAc,CAAC,CAAC,KAAK,EAAE,sBAAsB,GAAG,IAAI,CAAC;CACtD"}
+{"version":3,"file":"progressReporter.d.ts","sourceRoot":"","sources":["../src/progressReporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACvD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAC;AAExE,MAAM,WAAW,0BAA0B;IACzC,YAAY,EAAE,YAAY,CAAC;IAC3B,IAAI,EAAE,QAAQ,GAAG,WAAW,GAAG,eAAe,CAAC;IAC/C,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B;AAED;;;;GAIG;AACH,MAAM,WAAW,aAAa;IAC5B;qBACiB;IACjB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,6EAA6E;IAC7E,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,sBAAsB;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B;;oEAEgE;IAChE,QAAQ,CAAC,OAAO,EAAE,SAAS,eAAe,EAAE,CAAC;CAC9C;AAED,4EAA4E;AAC5E,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,yDAAyD;IACzD,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;CACtC;AAED,MAAM,WAAW,gBAAgB;IAC/B,aAAa,CAAC,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,gBAAgB,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACzD;;;;OAIG;IACH,cAAc,CAAC,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,YAAY,CAAC,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACtC,8DAA8D;IAC9D,kBAAkB,CAAC,CAAC,MAAM,EAAE,0BAA0B,GAAG,IAAI,CAAC;IAC9D,6CAA6C;IAC7C,aAAa,CAAC,IAAI,IAAI,CAAC;IACvB,kDAAkD;IAClD,YAAY,CAAC,CAAC,YAAY,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/D;;;;;OAKG;IACH,qBAAqB,CAAC,CACpB,YAAY,EAAE,YAAY,EAC1B,OAAO,EAAE,eAAe,GACvB,IAAI,CAAC;IACR;+CAC2C;IAC3C,oBAAoB,CAAC,CACnB,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,QAAQ,CAAC,EAAE,aAAa,GACvB,IAAI,CAAC;IACR,UAAU,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,aAAa,CAAC,CAAC,MAAM,EAAE;QACrB,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,MAAM,CAAC;QACvB,gBAAgB,EAAE,MAAM,CAAC;QACzB,aAAa,EAAE,MAAM,CAAC;KACvB,GAAG,IAAI,CAAC;IACT,aAAa,CAAC,CACZ,KAAK,EAAE,MAAM,EACb,MAAM,EAAE;QACN,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,MAAM,CAAC;QACvB,QAAQ,EAAE,MAAM,CAAC;KAClB,GACA,IAAI,CAAC;IACR,WAAW,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC;IAChD,YAAY,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACnD;;;;;;;;OAQG;IACH,gBAAgB,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACpE,eAAe,CAAC,CACd,OAAO,EAAE,OAAO,EAChB,MAAM,EAAE;QAAE,gBAAgB,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,GAC1D,IAAI,CAAC;IACR,cAAc,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACxD;;;OAGG;IACH,aAAa,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,aAAa,GAAG,IAAI,CAAC;IACjE,gBAAgB,CAAC,CAAC,MAAM,EAAE;QACxB,QAAQ,EAAE,MAAM,CAAC;QACjB,gBAAgB,EAAE,MAAM,CAAC;QACzB,aAAa,EAAE,MAAM,CAAC;KACvB,GAAG,IAAI,CAAC;IACT;;;;OAIG;IACH,gBAAgB,CAAC,CAAC,KAAK,EAAE,sBAAsB,GAAG,IAAI,CAAC;IACvD;;;OAGG;IACH,cAAc,CAAC,CAAC,KAAK,EAAE,sBAAsB,GAAG,IAAI,CAAC;CACtD"}

package/dist/sparql/values.d.ts

@@ -10,7 +10,10 @@ export declare function findSubSelect(patterns: Pattern[]): QuerySelect | undefi
  *
  * Each row's keys become SPARQL variables; NamedNode values become IRIs in the
  * VALUES block. The VALUES clause is injected into the innermost subquery so
- * that SPARQL engines can constrain scans early.
+ * that SPARQL engines can constrain scans early. When that WHERE clause is a
+ * single UNION whose branches all bind the VALUES variables in a required
+ * triple pattern, the VALUES goes into each branch instead: engines such as
+ * QLever plan and evaluate each branch without the outer VALUES otherwise.
  *
  * The caller owns parsing and stringifying; this function operates on the AST.
  */

package/dist/sparql/values.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"values.d.ts","sourceRoot":"","sources":["../../src/sparql/values.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,OAAO,EAGZ,KAAK,cAAc,EACnB,KAAK,WAAW,EAEjB,MAAM,2BAA2B,CAAC;AACnC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAIpD;;;GAGG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,WAAW,GAAG,SAAS,CAW1E;AA0DD;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAC1B,KAAK,EAAE,cAAc,EACrB,QAAQ,EAAE,gBAAgB,EAAE,GAC3B,cAAc,CAoBhB"}
+{"version":3,"file":"values.d.ts","sourceRoot":"","sources":["../../src/sparql/values.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,OAAO,EAIZ,KAAK,cAAc,EACnB,KAAK,WAAW,EAEjB,MAAM,2BAA2B,CAAC;AACnC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAIpD;;;GAGG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,WAAW,GAAG,SAAS,CAW1E;AA0GD;;;;;;;;;;;GAWG;AACH,wBAAgB,YAAY,CAC1B,KAAK,EAAE,cAAc,EACrB,QAAQ,EAAE,gBAAgB,EAAE,GAC3B,cAAc,CAoBhB"}

package/dist/sparql/values.js

@@ -47,8 +47,9 @@ function mapSubSelect(patterns, replacer) {
  * clause into the innermost WHERE clause. This ensures that SPARQL engines
  * constrain scans at the deepest level rather than only at the outer scope.
  *
- * For flat queries (no SubSelect), the base case injects directly — identical
- * to the previous behavior.
+ * For flat queries (no SubSelect), the base case injects directly: in front of
+ * the WHERE clause, or into each branch when it is a single UNION whose branches
+ * all qualify (see {@link bindsAllVariables}).
  */
 function injectIntoInnermost(where, valuesPattern) {
     const mapped = mapSubSelect(where.patterns, (subSelect) => ({
@@ -56,17 +57,54 @@ function injectIntoInnermost(where, valuesPattern) {
         where: injectIntoInnermost(subSelect.where, valuesPattern),
     }));
     if (!mapped) {
-        // Base case: no SubSelect — inject here.
+        // Base case: no SubSelect – inject here, into each UNION branch if safe.
+        const branches = where.patterns.length === 1 && F.isPatternUnion(where.patterns[0])
+            ? where.patterns[0].patterns
+            : [];
+        if (branches.length > 0 &&
+            branches.every((branch) => bindsAllVariables(branch, valuesPattern))) {
+            return F.patternGroup([
+                F.patternUnion(branches.map((branch) => F.patternGroup([valuesPattern, ...branch.patterns], F.gen())), F.gen()),
+            ], F.gen());
+        }
         return F.patternGroup([valuesPattern, ...where.patterns], F.gen());
     }
     return F.patternGroup(mapped, F.gen());
 }
+/**
+ * Whether a UNION branch binds every VALUES variable in a triple pattern of a
+ * top-level BGP that precedes the branch’s first OPTIONAL, MINUS, BIND or other
+ * non-FILTER pattern. Those apply to the solutions before them, so only then
+ * does joining the VALUES first in the branch return the same solutions as
+ * joining it outside the UNION. FILTERs apply to the whole group, wherever they are.
+ */
+function bindsAllVariables(branch, valuesPattern) {
+    const bound = new Set();
+    for (const pattern of branch.patterns) {
+        if (pattern.subType === 'filter')
+            continue;
+        if (!F.isPatternBgp(pattern))
+            break;
+        for (const triple of pattern.triples) {
+            if (triple.type !== 'triple')
+                continue;
+            for (const term of [triple.subject, triple.predicate, triple.object]) {
+                if (F.isTermVariable(term))
+                    bound.add(term.value);
+            }
+        }
+    }
+    return valuesPattern.variables.every((variable) => bound.has(variable.value));
+}
 /**
  * Inject a VALUES clause into a parsed CONSTRUCT query for the given binding rows.
  *
  * Each row's keys become SPARQL variables; NamedNode values become IRIs in the
  * VALUES block. The VALUES clause is injected into the innermost subquery so
- * that SPARQL engines can constrain scans early.
+ * that SPARQL engines can constrain scans early. When that WHERE clause is a
+ * single UNION whose branches all bind the VALUES variables in a required
+ * triple pattern, the VALUES goes into each branch instead: engines such as
+ * QLever plan and evaluate each branch without the outer VALUES otherwise.
  *
  * The caller owns parsing and stringifying; this function operates on the AST.
  */

package/dist/writer/writer.d.ts

@@ -1,6 +1,7 @@
 import { Dataset } from '@lde/dataset';
 import type { Quad } from '@rdfjs/types';
 import type { ProvenanceStore } from '../provenance/store.js';
+import type { ItemsRejected } from '../progressReporter.js';
 /**
  * Context for one pipeline run, handed to {@link Writer.openRun}.
  */
@@ -23,6 +24,15 @@ export interface RunContext {
      * is when a registry-membership sweep should read it.
      */
     selectedSources(): Iterable<string>;
+    /**
+     * Tell the run’s reporters that this writer left some of a dataset’s items
+     * out rather than failing the dataset – documents the destination refused,
+     * or that miss what the destination requires. Call it once per (dataset,
+     * target) when the dataset is finalized ({@link RunWriter.flush}), so the
+     * counts are the dataset’s whole. Always present when a {@link Pipeline}
+     * drives the run.
+     */
+    itemsRejected?(dataset: Dataset, rejection: ItemsRejected): void;
     /**
      * The pipeline’s per-dataset processing memory, present when skip-unchanged
      * is enabled ({@link PipelineOptions.provenanceStore}).

package/dist/writer/writer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../src/writer/writer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAE9D;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB;;;;OAIG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;OAGG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;;;;;OAKG;IACH,eAAe,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;IACpC;;;OAGG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,eAAe,CAAC;CACvC;AAED;;;;GAIG;AACH,MAAM,WAAW,aAAa,CAAC,IAAI,GAAG,IAAI;IACxC;;;;;;OAMG;IACH,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACpE;AAED;;;;GAIG;AACH,MAAM,MAAM,cAAc,GAAG,SAAS,GAAG,QAAQ,CAAC;AAElD;;;;;;;GAOG;AACH,MAAM,WAAW,SAAS,CAAC,IAAI,GAAG,IAAI,CAAE,SAAQ,aAAa,CAAC,IAAI,CAAC;IACjE;;;;;;;;;;OAUG;IACH,KAAK,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjE;;;;;;;;;OASG;IACH,KAAK,CAAC,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAExC;;;;;;;;;OASG;IACH,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAExB;;;;;;OAMG;IACH,KAAK,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErC;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,OAAO,CAAC,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACzC;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,MAAM,CAAC,IAAI,GAAG,IAAI;IACjC;;;;OAIG;IACH,OAAO,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;CACxD"}
+{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../src/writer/writer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAE5D;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB;;;;OAIG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;OAGG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;;;;;OAKG;IACH,eAAe,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;IACpC;;;;;;;OAOG;IACH,aAAa,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,aAAa,GAAG,IAAI,CAAC;IACjE;;;OAGG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,eAAe,CAAC;CACvC;AAED;;;;GAIG;AACH,MAAM,WAAW,aAAa,CAAC,IAAI,GAAG,IAAI;IACxC;;;;;;OAMG;IACH,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACpE;AAED;;;;GAIG;AACH,MAAM,MAAM,cAAc,GAAG,SAAS,GAAG,QAAQ,CAAC;AAElD;;;;;;;GAOG;AACH,MAAM,WAAW,SAAS,CAAC,IAAI,GAAG,IAAI,CAAE,SAAQ,aAAa,CAAC,IAAI,CAAC;IACjE;;;;;;;;;;OAUG;IACH,KAAK,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjE;;;;;;;;;OASG;IACH,KAAK,CAAC,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAExC;;;;;;;;;OASG;IACH,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAExB;;;;;;OAMG;IACH,KAAK,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErC;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,OAAO,CAAC,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACzC;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,MAAM,CAAC,IAAI,GAAG,IAAI;IACjC;;;;OAIG;IACH,OAAO,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;CACxD"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.38.1",
+  "version": "0.39.1",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/pipeline"