@lde/pipeline 0.49.0 → 0.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/pipeline.d.ts.map +1 -1
- package/dist/pipeline.js +20 -13
- package/dist/stage.d.ts +26 -1
- package/dist/stage.d.ts.map +1 -1
- package/dist/stage.js +12 -4
- package/package.json +1 -1
package/dist/pipeline.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AAMrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AAMrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAc,MAAM,YAAY,CAAC;AAE5D,OAAO,KAAK,EAKV,MAAM,EACP,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACL,KAAK,oBAAoB,EAI1B,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAW7D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AASpE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAG9C,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,2BAA2B,CAAC;AAEnC;;;;;;;;;;;GAWG;AACH,qBAAa,cAAe,SAAQ,KAAK;;CAOxC;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;GAKG;AACH,MAAM,WAAW,yBAAyB;IACxC,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,aAAa,CAAC,uBAAuB,CAAC,CAAC;IAC1D;;;;;;;;;;;;OAYG;IACH,kBAAkB,CAAC,EAAE,aAAa,CAAC,yBAAyB,CAAC,CAAC;CAC/D;AAED,MAAM,WAAW,eAAe,CAAC,GAAG,GAAG,IAAI;IACzC,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;IACrC;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,cAAc,EAAE,GAAG,KAAK,CAAC;IAC1D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,QAAQ,GAAG,SAAS,QAAQ,EAAE,CAAC;IAC1C;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;CAC/B;AA0OD,qBAAa,QAAQ,CAAC,GAAG,GAAG,IAAI;IAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAyC;IAC3E,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAA2C;IAC/E,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmC;IAC7D,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAsB;IAChD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAsB;IACrD;;;;;OAKG;IACH,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAc;IACtC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAkB;IACnD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAS;gBAE9B,OAAO,EAAE,eAAe,CAAC,GAAG,CAAC;IAgEnC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAyFZ,cAAc;IAgS5B,OAAO,CAAC,MAAM;IAyBd,OAAO,CAAC,iBAAiB;IAczB,+EAA+E;YACjE,aAAa;YAuBb,gBAAgB;IAiB9B;;;;;OAKG;IACH,OAAO,CAAC,WAAW;IAgBnB;;;;;;;;OAQG;IACH,OAAO,CAAC,0BAA0B;IA+BlC;;;;;;OAMG;YACW,SAAS;IA0CvB;;;OAGG;YACW,QAAQ;IA+EtB,2EAA2E;YAC7D,eAAe;YAyBf,QAAQ;YAsFP,SAAS;CAczB"}
|
package/dist/pipeline.js
CHANGED
|
@@ -499,7 +499,7 @@ export class Pipeline {
|
|
|
499
499
|
// completes.
|
|
500
500
|
let unread;
|
|
501
501
|
try {
|
|
502
|
-
({ stageFailed, unread } = await this.runStages(dataset, resolved.distribution, timeout, runWriter, context));
|
|
502
|
+
({ stageFailed, unread } = await this.runStages(dataset, resolved.distribution, sourceKindOf(resolved), timeout, runWriter, context));
|
|
503
503
|
// Reactive fallback: an endpoint that passed probing but could not serve
|
|
504
504
|
// the analysis stages is empirically incapable. Switch to the dataset’s
|
|
505
505
|
// data dump and re-run all stages locally, discarding the
|
|
@@ -532,7 +532,7 @@ export class Pipeline {
|
|
|
532
532
|
// Discard the endpoint-sourced partial output before the re-run so
|
|
533
533
|
// the dump-sourced stats replace it rather than appending to it.
|
|
534
534
|
await runWriter.reset?.(dataset);
|
|
535
|
-
({ stageFailed, unread } = await this.runStages(dataset, fallback.distribution, timeout, runWriter, context));
|
|
535
|
+
({ stageFailed, unread } = await this.runStages(dataset, fallback.distribution, sourceKindOf(fallback), timeout, runWriter, context));
|
|
536
536
|
// Adopt the dump’s fingerprint only once the re-run has succeeded.
|
|
537
537
|
// If the reset or the re-run throws, or a stage fails against the
|
|
538
538
|
// dump, the dataset is recorded ‘failed’ under the endpoint’s
|
|
@@ -707,16 +707,16 @@ export class Pipeline {
|
|
|
707
707
|
* reacts to – and what the stages could not read, which belongs to this
|
|
708
708
|
* attempt alone.
|
|
709
709
|
*/
|
|
710
|
-
async runStages(dataset, distribution, timeout, runWriter, context) {
|
|
710
|
+
async runStages(dataset, distribution, source, timeout, runWriter, context) {
|
|
711
711
|
const unread = new UnreadValueTally();
|
|
712
712
|
let stageFailed = false;
|
|
713
713
|
for (const stage of this.stages) {
|
|
714
714
|
try {
|
|
715
715
|
if (stage.stages.length > 0) {
|
|
716
|
-
await this.runChain(dataset, distribution, stage, runWriter, context, timeout);
|
|
716
|
+
await this.runChain(dataset, distribution, source, stage, runWriter, context, timeout);
|
|
717
717
|
}
|
|
718
718
|
else {
|
|
719
|
-
await this.runStage(context.runId, dataset, distribution, stage, this.stageWriter(runWriter, stage.name), timeout, unread);
|
|
719
|
+
await this.runStage(context.runId, dataset, distribution, stage, this.stageWriter(runWriter, stage.name), source, timeout, unread);
|
|
720
720
|
}
|
|
721
721
|
}
|
|
722
722
|
catch (error) {
|
|
@@ -730,13 +730,14 @@ export class Pipeline {
|
|
|
730
730
|
* Run a stage with reporting and return whether it was supported.
|
|
731
731
|
* Returns `true` if the stage produced results, `false` if NotSupported.
|
|
732
732
|
*/
|
|
733
|
-
async runStage(runId, dataset, distribution, stage, writer, timeout, unread) {
|
|
733
|
+
async runStage(runId, dataset, distribution, stage, writer, source, timeout, unread) {
|
|
734
734
|
this.report(runId, { type: 'stageStart', dataset, stage: stage.name });
|
|
735
735
|
const stageStart = Date.now();
|
|
736
736
|
let itemsProcessed = 0;
|
|
737
737
|
let quadsGenerated = 0;
|
|
738
738
|
let summary;
|
|
739
739
|
const result = await stage.run(dataset, distribution, writer, {
|
|
740
|
+
source,
|
|
740
741
|
onProgress: (items, quads) => {
|
|
741
742
|
itemsProcessed = items;
|
|
742
743
|
quadsGenerated = quads;
|
|
@@ -791,13 +792,13 @@ export class Pipeline {
|
|
|
791
792
|
return true;
|
|
792
793
|
}
|
|
793
794
|
/** Run a stage in chained mode, throwing if the stage is not supported. */
|
|
794
|
-
async runChainedStage(runId, dataset, distribution, stage, writer, timeout) {
|
|
795
|
-
const supported = await this.runStage(runId, dataset, distribution, stage, writer, timeout);
|
|
795
|
+
async runChainedStage(runId, dataset, distribution, stage, writer, source, timeout) {
|
|
796
|
+
const supported = await this.runStage(runId, dataset, distribution, stage, writer, source, timeout);
|
|
796
797
|
if (!supported) {
|
|
797
798
|
throw new Error(`Stage '${stage.name}' returned NotSupported in chained mode`);
|
|
798
799
|
}
|
|
799
800
|
}
|
|
800
|
-
async runChain(dataset, distribution, stage, runWriter, context, timeout) {
|
|
801
|
+
async runChain(dataset, distribution, source, stage, runWriter, context, timeout) {
|
|
801
802
|
const { stageOutputResolver, outputDir } = this.chaining;
|
|
802
803
|
const outputFiles = [];
|
|
803
804
|
// Chaining is a quad-only concern: every stage in a chain writes N-Triples
|
|
@@ -806,14 +807,14 @@ export class Pipeline {
|
|
|
806
807
|
// requirement "sub-stages imply `Out` is `Quad`" is not expressible – `stages`
|
|
807
808
|
// and `chaining` are independent options – so it is a runtime contract, and
|
|
808
809
|
// the scratch machinery treats the parent stage as `Stage<Quad>`.
|
|
809
|
-
const runScratchStage = async (chainedStage, stageDistribution) => {
|
|
810
|
+
const runScratchStage = async (chainedStage, stageDistribution, stageSource) => {
|
|
810
811
|
const scratchWriter = new FileWriter({
|
|
811
812
|
outputDir: `${outputDir}/${chainedStage.name}`,
|
|
812
813
|
format: 'n-triples',
|
|
813
814
|
});
|
|
814
815
|
const scratchRun = await scratchWriter.openRun(context);
|
|
815
816
|
try {
|
|
816
|
-
await this.runChainedStage(context.runId, dataset, stageDistribution, chainedStage, scratchRun, timeout);
|
|
817
|
+
await this.runChainedStage(context.runId, dataset, stageDistribution, chainedStage, scratchRun, stageSource, timeout);
|
|
817
818
|
await scratchRun.flush(dataset, 'success');
|
|
818
819
|
await scratchRun.commit();
|
|
819
820
|
}
|
|
@@ -825,12 +826,14 @@ export class Pipeline {
|
|
|
825
826
|
};
|
|
826
827
|
try {
|
|
827
828
|
// 1. Run parent stage → scratch FileWriter (quad-only, per above).
|
|
828
|
-
const parentOutput = await runScratchStage(stage, distribution);
|
|
829
|
+
const parentOutput = await runScratchStage(stage, distribution, source);
|
|
829
830
|
outputFiles.push(parentOutput);
|
|
830
831
|
// 2. Chain through children.
|
|
831
832
|
let currentDistribution = await stageOutputResolver.resolve(parentOutput);
|
|
832
833
|
for (let i = 0; i < stage.stages.length; i++) {
|
|
833
|
-
|
|
834
|
+
// A child reads its parent’s output, which the pipeline serves from
|
|
835
|
+
// a store of its own, as it does an imported dump.
|
|
836
|
+
const childOutput = await runScratchStage(stage.stages[i], currentDistribution, 'imported');
|
|
834
837
|
outputFiles.push(childOutput);
|
|
835
838
|
if (i < stage.stages.length - 1) {
|
|
836
839
|
currentDistribution = await stageOutputResolver.resolve(childOutput);
|
|
@@ -891,3 +894,7 @@ function mapProbeResult(distribution, result) {
|
|
|
891
894
|
fingerprint,
|
|
892
895
|
};
|
|
893
896
|
}
|
|
897
|
+
/** What kind of source a resolved distribution serves the stages. */
|
|
898
|
+
function sourceKindOf(resolved) {
|
|
899
|
+
return resolved.imported === undefined ? 'endpoint' : 'imported';
|
|
900
|
+
}
|
package/dist/stage.d.ts
CHANGED
|
@@ -105,9 +105,19 @@ export interface StageOptions<Out = Quad> {
|
|
|
105
105
|
* fills exactly one batch. A `LIMIT` clause in the selector query
|
|
106
106
|
* overrides this for endpoints with hard result limits.
|
|
107
107
|
*
|
|
108
|
+
* Give a value per {@link SourceKind} where the two kinds of source want
|
|
109
|
+
* different sizes: an engine that imported a dump for this run alone
|
|
110
|
+
* answers a large batch easily, while a live endpoint shared with others
|
|
111
|
+
* may fail on one. The pipeline tells the stage which kind it reads
|
|
112
|
+
* ({@link RunOptions.source}).
|
|
113
|
+
*
|
|
114
|
+
* ```ts
|
|
115
|
+
* batchSize: { endpoint: 10, imported: 100 }
|
|
116
|
+
* ```
|
|
117
|
+
*
|
|
108
118
|
* @default 10
|
|
109
119
|
*/
|
|
110
|
-
batchSize?: number
|
|
120
|
+
batchSize?: number | Readonly<Record<SourceKind, number>>;
|
|
111
121
|
/**
|
|
112
122
|
* Maximum concurrent in-flight SPARQL queries. Within each batch, all
|
|
113
123
|
* readers run in parallel; the number of concurrent batches is
|
|
@@ -172,8 +182,23 @@ export interface StageOptions<Out = Quad> {
|
|
|
172
182
|
onInvalid?: 'write' | 'skip' | 'halt';
|
|
173
183
|
};
|
|
174
184
|
}
|
|
185
|
+
/**
|
|
186
|
+
* What a stage reads: a live SPARQL `endpoint` the dataset publishes, or a
|
|
187
|
+
* data dump the pipeline `imported` into a store of its own for the run.
|
|
188
|
+
*/
|
|
189
|
+
export type SourceKind = 'endpoint' | 'imported';
|
|
175
190
|
export interface RunOptions extends Pick<ReadOptions, 'limiter' | 'onRootSkipped'> {
|
|
176
191
|
onProgress?: (itemsProcessed: number, quadsGenerated: number) => void;
|
|
192
|
+
/**
|
|
193
|
+
* What the distribution handed to {@link Stage.run} serves, which picks the
|
|
194
|
+
* {@link StageOptions.batchSize} of a stage that gives one per kind. A stage
|
|
195
|
+
* whose {@link StageOptions.sourceFor} substitutes another distribution reads
|
|
196
|
+
* a source the pipeline did not resolve, so it reads that as an `endpoint`
|
|
197
|
+
* whatever this says.
|
|
198
|
+
*
|
|
199
|
+
* @default 'endpoint'
|
|
200
|
+
*/
|
|
201
|
+
source?: SourceKind;
|
|
177
202
|
/**
|
|
178
203
|
* Per-dataset {@link TimeoutPolicy} threaded through to readers and
|
|
179
204
|
* item selectors. The Pipeline owns lifecycle (factory invocation per
|
package/dist/stage.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAMrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAmB,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE/D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAExD,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAGpE;;;;;;;GAOG;AACH,MAAM,MAAM,aAAa,CAAC,GAAG,IAAI,CAC/B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,EAC1B,OAAO,EAAE,GAAG,KACT,aAAa,CAAC,IAAI,CAAC,CAAC;AAEzB;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,YAAY,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd;;;;;OAKG;IACH,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5C;AAED;;;;;;GAMG;AACH,MAAM,MAAM,cAAc,CAAC,GAAG,IAAI,CAChC,KAAK,EAAE,SAAS,IAAI,EAAE,EACtB,OAAO,EAAE,YAAY,KAClB,QAAQ,CAAC,GAAG,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;AAExC;;;;;GAKG;AACH,MAAM,WAAW,YAAa,SAAQ,aAAa;IACjD,QAAQ,EAAE,SAAS,gBAAgB,EAAE,CAAC;IACtC;;;;;OAKG;IACH,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC;CACvE;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,aAAa,CAAC,aAAa,CAAC,GAAG,aAAa,CAAC,aAAa,CAAC,EAAE,CAAC;CAC3E;AAED,yEAAyE;AACzE,MAAM,MAAM,YAAY,GACtB,MAAM,GAAG,cAAc,GAAG,CAAC,MAAM,GAAG,cAAc,CAAC,EAAE,CAAC;AAUxD,MAAM,WAAW,YAAY,CAAC,GAAG,GAAG,IAAI;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,YAAY,CAAC;IACtB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC;IAC9B;;;;;;;OAOG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB
|
|
1
|
+
{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAMrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAmB,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE/D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAExD,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAGpE;;;;;;;GAOG;AACH,MAAM,MAAM,aAAa,CAAC,GAAG,IAAI,CAC/B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,EAC1B,OAAO,EAAE,GAAG,KACT,aAAa,CAAC,IAAI,CAAC,CAAC;AAEzB;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,YAAY,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd;;;;;OAKG;IACH,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5C;AAED;;;;;;GAMG;AACH,MAAM,MAAM,cAAc,CAAC,GAAG,IAAI,CAChC,KAAK,EAAE,SAAS,IAAI,EAAE,EACtB,OAAO,EAAE,YAAY,KAClB,QAAQ,CAAC,GAAG,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;AAExC;;;;;GAKG;AACH,MAAM,WAAW,YAAa,SAAQ,aAAa;IACjD,QAAQ,EAAE,SAAS,gBAAgB,EAAE,CAAC;IACtC;;;;;OAKG;IACH,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC;CACvE;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,aAAa,CAAC,aAAa,CAAC,GAAG,aAAa,CAAC,aAAa,CAAC,EAAE,CAAC;CAC3E;AAED,yEAAyE;AACzE,MAAM,MAAM,YAAY,GACtB,MAAM,GAAG,cAAc,GAAG,CAAC,MAAM,GAAG,cAAc,CAAC,EAAE,CAAC;AAUxD,MAAM,WAAW,YAAY,CAAC,GAAG,GAAG,IAAI;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,YAAY,CAAC;IACtB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC;IAC9B;;;;;;;OAOG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;;;;;;;;;;;;;;OAkBG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC;IAC1D;;;;;;;OAOG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,YAAY,KAAK,YAAY,CAAC;IAC3E,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;IACjB;;;;;;;;;;;;;;;OAeG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,mFAAmF;IACnF,UAAU,CAAC,EAAE;QACX,SAAS,EAAE,SAAS,CAAC;QACrB,iEAAiE;QACjE,SAAS,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;KACvC,CAAC;CACH;AAED;;;GAGG;AACH,MAAM,MAAM,UAAU,GAAG,UAAU,GAAG,UAAU,CAAC;AAEjD,MAAM,WAAW,UAAW,SAAQ,IAAI,CACtC,WAAW,EACX,SAAS,GAAG,eAAe,CAC5B;IACC,UAAU,CAAC,EAAE,CAAC,cAAc,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;IACtE;;;;;;;;OAQG;IACH,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB;;;;;OAKG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB;iEAC6D;IAC7D,WAAW,CAAC,EAAE,YAAY,CAAC,aAAa,CAAC,CAAC;IAC1C;oFACgF;IAChF,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,IAAI,CAAC;CAC9C;AAED,uDAAuD;AACvD,MAAM,WAAW,aAAa;IAC5B,+BAA+B;IAC/B,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,qBAAa,KAAK,CAAC,GAAG,GAAG,IAAI;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,oGAAoG;IACpG,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC;qEACiE;IACjE,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,4FAA4F;IAC5F,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;IAC7C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgD;IAC1E,4EAA4E;IAC5E,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAc;IACtC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAkC;IAC9D,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAiC;gBAEhD,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC;IAqCtC,mDAAmD;IACnD,IAAI,SAAS,IAAI,SAAS,GAAG,SAAS,CAErC;IAEK,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,YAAY,EACtB,MAAM,EAAE,aAAa,CAAC,GAAG,CAAC,EAC1B,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;IAuH/B;;;;OAIG;IACH,OAAO,CAAC,cAAc;YAMR,eAAe;IA2L7B;;;OAGG;YACW,cAAc;YAqBd,OAAO;IA0BrB;;;;;OAKG;YACW,IAAI;IA2BlB,8EAA8E;IAC9E,OAAO,KAAK,cAAc,GAOzB;CACF;AA0DD,2GAA2G;AAC3G,MAAM,WAAW,YAAY;IAC3B,MAAM,CACJ,YAAY,EAAE,YAAY,EAC1B,SAAS,CAAC,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,aAAa,GACtB,aAAa,CAAC,gBAAgB,CAAC,CAAC;CACpC"}
|
package/dist/stage.js
CHANGED
|
@@ -70,9 +70,17 @@ export class Stage {
|
|
|
70
70
|
: resolved;
|
|
71
71
|
const tally = new StageTally(this.transformNames);
|
|
72
72
|
if (this.itemSelector) {
|
|
73
|
-
|
|
73
|
+
// A substituted source is one the pipeline did not resolve; one handed
|
|
74
|
+
// back unchanged is still the source the pipeline resolved.
|
|
75
|
+
const source = distribution === resolved
|
|
76
|
+
? (options?.source ?? 'endpoint')
|
|
77
|
+
: 'endpoint';
|
|
78
|
+
const batchSize = typeof this.batchSize === 'number'
|
|
79
|
+
? this.batchSize
|
|
80
|
+
: this.batchSize[source];
|
|
81
|
+
const result = await this.runWithSelector(tally.selected(this.itemSelector.select(distribution, batchSize, {
|
|
74
82
|
timeout,
|
|
75
|
-
})), dataset, distribution, writer, tally, readOptions, options);
|
|
83
|
+
})), batchSize, dataset, distribution, writer, tally, readOptions, options);
|
|
76
84
|
if (!(result instanceof NotSupported)) {
|
|
77
85
|
options?.onComplete?.(tally.summary());
|
|
78
86
|
}
|
|
@@ -144,10 +152,10 @@ export class Stage {
|
|
|
144
152
|
throw new Error(`Stage '${this.name}' expected output but produced none`);
|
|
145
153
|
}
|
|
146
154
|
}
|
|
147
|
-
async runWithSelector(selector, dataset, distribution, writer, tally, readOptions, options) {
|
|
155
|
+
async runWithSelector(selector, batchSize, dataset, distribution, writer, tally, readOptions, options) {
|
|
148
156
|
// Peek the first batch to detect an empty selector before starting the
|
|
149
157
|
// writer (important because e.g. SparqlUpdateWriter does CLEAR GRAPH).
|
|
150
|
-
const batches = batch(selector,
|
|
158
|
+
const batches = batch(selector, batchSize);
|
|
151
159
|
const iter = batches[Symbol.asyncIterator]();
|
|
152
160
|
const first = await iter.next();
|
|
153
161
|
if (first.done) {
|