@lde/pipeline 0.44.0 → 0.46.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/pipeline.d.ts.map +1 -1
- package/dist/pipeline.js +5 -0
- package/dist/pipelineEvent.d.ts +42 -0
- package/dist/pipelineEvent.d.ts.map +1 -1
- package/dist/stage.d.ts +18 -6
- package/dist/stage.d.ts.map +1 -1
- package/dist/stage.js +68 -49
- package/dist/stageTally.d.ts +56 -0
- package/dist/stageTally.d.ts.map +1 -0
- package/dist/stageTally.js +164 -0
- package/package.json +3 -3
package/dist/pipeline.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD,OAAO,KAAK,EAKV,MAAM,EACP,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACL,KAAK,oBAAoB,EAI1B,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAW7D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD,OAAO,KAAK,EAKV,MAAM,EACP,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACL,KAAK,oBAAoB,EAI1B,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAW7D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AASpE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAG9C,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,2BAA2B,CAAC;AAEnC;;;;;;;;;;;GAWG;AACH,qBAAa,cAAe,SAAQ,KAAK;;CAOxC;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;GAKG;AACH,MAAM,WAAW,yBAAyB;IACxC,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,aAAa,CAAC,uBAAuB,CAAC,CAAC;IAC1D;;;;;;;;;;;;OAYG;IACH,kBAAkB,CAAC,EAAE,aAAa,CAAC,yBAAyB,CAAC,CAAC;CAC/D;AAED,MAAM,WAAW,eAAe,CAAC,GAAG,GAAG,IAAI;IACzC,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;IACrC;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,cAAc,EAAE,GAAG,KAAK,CAAC;IAC1D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,QAAQ,GAAG,SAAS,QAAQ,EAAE,CAAC;IAC1C;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;CAC/B;AA0OD,qBAAa,QAAQ,CAAC,GAAG,GAAG,IAAI;IAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAyC;IAC3E,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAA2C;IAC/E,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmC;IAC7D,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAsB;IAChD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAsB;IACrD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAkB;IACnD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAS;gBAE9B,OAAO,EAAE,eAAe,CAAC,GAAG,CAAC;IAyDnC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAyFZ,cAAc;IA8R5B,OAAO,CAAC,MAAM;IAyBd,OAAO,CAAC,iBAAiB;IAczB,+EAA+E;YACjE,aAAa;YAuBb,gBAAgB;IAgB9B,OAAO,CAAE,aAAa;IAStB;;;;;OAKG;IACH,OAAO,CAAC,WAAW;IAgBnB;;;;;;;;OAQG;IACH,OAAO,CAAC,0BAA0B;IA+BlC;;;;;;OAMG;YACW,SAAS;IAuCvB;;;OAGG;YACW,QAAQ;IAoEtB,2EAA2E;YAC7D,eAAe;YAuBf,QAAQ;YA+EP,SAAS;CAczB"}
|
package/dist/pipeline.js
CHANGED
|
@@ -732,6 +732,7 @@ export class Pipeline {
|
|
|
732
732
|
const stageStart = Date.now();
|
|
733
733
|
let itemsProcessed = 0;
|
|
734
734
|
let quadsGenerated = 0;
|
|
735
|
+
let summary;
|
|
735
736
|
const result = await stage.run(dataset, distribution, writer, {
|
|
736
737
|
onProgress: (items, quads) => {
|
|
737
738
|
itemsProcessed = items;
|
|
@@ -749,6 +750,9 @@ export class Pipeline {
|
|
|
749
750
|
},
|
|
750
751
|
timeout,
|
|
751
752
|
valueUnread: unread && ((target, field, value) => unread.add(target, field, value)),
|
|
753
|
+
onComplete: (stageSummary) => {
|
|
754
|
+
summary = stageSummary;
|
|
755
|
+
},
|
|
752
756
|
});
|
|
753
757
|
if (result instanceof NotSupported) {
|
|
754
758
|
this.report(runId, {
|
|
@@ -771,6 +775,7 @@ export class Pipeline {
|
|
|
771
775
|
itemsProcessed,
|
|
772
776
|
quadsGenerated,
|
|
773
777
|
durationMs: Date.now() - stageStart,
|
|
778
|
+
...summary,
|
|
774
779
|
});
|
|
775
780
|
return true;
|
|
776
781
|
}
|
package/dist/pipelineEvent.d.ts
CHANGED
|
@@ -169,6 +169,48 @@ export interface StageCompleteEvent extends DatasetEventEnvelope<'stageComplete'
|
|
|
169
169
|
itemsProcessed: number;
|
|
170
170
|
quadsGenerated: number;
|
|
171
171
|
durationMs: number;
|
|
172
|
+
/** Where the stage spent its time. Always set by the pipeline; optional so
|
|
173
|
+
* code constructing the event need not set it, as is `counts`. */
|
|
174
|
+
phases?: StagePhases;
|
|
175
|
+
/** What the stage’s transforms and projection counted, per name, through
|
|
176
|
+
* `ReaderContext.count`. */
|
|
177
|
+
counts?: Readonly<Record<string, number>>;
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* The time a stage spent in each of its phases, in milliseconds.
|
|
181
|
+
*
|
|
182
|
+
* A stage runs several batches at once, so each phase is added up over all
|
|
183
|
+
* of them: the phases can together exceed the stage’s `durationMs`, and a
|
|
184
|
+
* phase that does is one batches spent their time waiting on. Each phase
|
|
185
|
+
* counts only its own time – a transform waiting on the reader it pulls from
|
|
186
|
+
* is the reader’s time, not the transform’s.
|
|
187
|
+
*/
|
|
188
|
+
export interface StagePhases {
|
|
189
|
+
/** Selecting the stage’s items, page by page. */
|
|
190
|
+
selectMs: number;
|
|
191
|
+
/** Running the readers’ queries and reading their results. */
|
|
192
|
+
readMs: number;
|
|
193
|
+
/** Each attached transform, in the order first attached. A transform
|
|
194
|
+
* attached to several readers is one entry, its times added up. */
|
|
195
|
+
transforms: readonly TransformDuration[];
|
|
196
|
+
/** Validating the readers’ quads. */
|
|
197
|
+
validateMs: number;
|
|
198
|
+
/** Turning batches into the stage’s output items. */
|
|
199
|
+
projectMs: number;
|
|
200
|
+
/** The writer’s own work, not counting the time it waited for items. */
|
|
201
|
+
writeMs: number;
|
|
202
|
+
}
|
|
203
|
+
/** What a completed stage hands back: where its time went, and its counts. */
|
|
204
|
+
export interface StageSummary {
|
|
205
|
+
phases: StagePhases;
|
|
206
|
+
counts: Readonly<Record<string, number>>;
|
|
207
|
+
}
|
|
208
|
+
/** One attached transform’s time, in {@link StagePhases.transforms}. */
|
|
209
|
+
export interface TransformDuration {
|
|
210
|
+
/** The transform function’s name, or `transform N` for one without a name,
|
|
211
|
+
* N counting the stage’s transforms from 1. */
|
|
212
|
+
name: string;
|
|
213
|
+
durationMs: number;
|
|
172
214
|
}
|
|
173
215
|
/**
|
|
174
216
|
* Also reported for per-dataset failures outside any stage, with `stage` set
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipelineEvent.d.ts","sourceRoot":"","sources":["../src/pipelineEvent.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACvD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAC;AAExE;;;GAGG;AACH,MAAM,MAAM,KAAK,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;AAExD;;;;GAIG;AACH,MAAM,WAAW,aAAa,CAAC,IAAI,SAAS,MAAM;IAChD,IAAI,EAAE,IAAI,CAAC;IACX,YAAY,EAAE,KAAK,CAAC;IACpB,SAAS,EAAE,IAAI,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,oEAAoE;AACpE,MAAM,WAAW,oBAAoB,CACnC,IAAI,SAAS,MAAM,CACnB,SAAQ,aAAa,CAAC,IAAI,CAAC;IAC3B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,8FAA8F;AAC9F,MAAM,WAAW,MAAM,CAAC,IAAI,SAAS,MAAM;IACzC,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,0CAA0C;AAC1C,MAAM,MAAM,iBAAiB,GAAG,MAAM,CACpC,WAAW,GAAG,aAAa,GAAG,eAAe,GAAG,yBAAyB,CAC1E,CAAC;AAEF;;;;;GAKG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,CAAC,cAAc,GAAG,iBAAiB,CAAC,CAAC;AAEzE,+CAA+C;AAC/C,MAAM,WAAW,0BAA0B;IACzC,IAAI,EAAE,QAAQ,GAAG,WAAW,GAAG,eAAe,CAAC;IAC/C,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B;AAED;;;;GAIG;AACH,MAAM,WAAW,aAAa;IAC5B;qBACiB;IACjB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,6EAA6E;IAC7E,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,sBAAsB;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B;;oEAEgE;IAChE,QAAQ,CAAC,OAAO,EAAE,SAAS,eAAe,EAAE,CAAC;CAC9C;AAED,4EAA4E;AAC5E,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,yDAAyD;IACzD,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;CACtC;AAED;;;;;GAKG;AACH,MAAM,WAAW,YAAY;IAC3B,4EAA4E;IAC5E,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,qEAAqE;IACrE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,0CAA0C;IAC1C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB;4DACwD;IACxD,QAAQ,CAAC,QAAQ,EAAE,SAAS,WAAW,EAAE,CAAC;CAC3C;AAED,0EAA0E;AAC1E,MAAM,WAAW,WAAW;IAC1B;;8DAE0D;IAC1D,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,yCAAyC;IACzC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,4DAA4D;AAC5D,MAAM,WAAW,WAAW;IAC1B,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,oDAAoD;AACpD,MAAM,WAAW,kBAAmB,SAAQ,aAAa,CAAC,eAAe,CAAC;IACxE,IAAI,EAAE,MAAM,CAAC;CACd;AAED,oFAAoF;AACpF,MAAM,WAAW,qBAAsB,SAAQ,aAAa,CAAC,kBAAkB,CAAC;IAC9E,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;GAGG;AACH,MAAM,WAAW,mBAAoB,SAAQ,aAAa,CAAC,gBAAgB,CAAC;IAC1E,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,sBAAsB;AACtB,MAAM,WAAW,qBACf,SAAQ,aAAa,CAAC,kBAAkB,CAAC,EAAE,WAAW;IACtD,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,iDAAiD;AACjD,MAAM,MAAM,iBAAiB,GAAG,oBAAoB,CAAC,cAAc,CAAC,CAAC;AAErE,mDAAmD;AACnD,MAAM,WAAW,uBAAwB,SAAQ,oBAAoB,CAAC,oBAAoB,CAAC;IACzF,YAAY,EAAE,YAAY,CAAC;IAC3B,MAAM,EAAE,0BAA0B,CAAC;CACpC;AAED,0GAA0G;AAC1G,MAAM,WAAW,0BAA2B,SAAQ,oBAAoB,CAAC,uBAAuB,CAAC;IAC/F,YAAY,EAAE,YAAY,CAAC;IAC3B,OAAO,EAAE,eAAe,CAAC;CAC1B;AAED,mDAAmD;AACnD,MAAM,MAAM,kBAAkB,GAAG,oBAAoB,CAAC,eAAe,CAAC,CAAC;AAEvE,uCAAuC;AACvC,MAAM,WAAW,iBAAkB,SAAQ,oBAAoB,CAAC,cAAc,CAAC;IAC7E,YAAY,EAAE,YAAY,CAAC;IAC3B,KAAK,EAAE,KAAK,CAAC;CACd;AAED,0EAA0E;AAC1E,MAAM,WAAW,yBAA0B,SAAQ,oBAAoB,CAAC,sBAAsB,CAAC;IAC7F,YAAY,EAAE,YAAY,CAAC;IAC3B,QAAQ,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,qCAAqC;AACrC,MAAM,WAAW,eAAgB,SAAQ,oBAAoB,CAAC,YAAY,CAAC;IACzE,KAAK,EAAE,MAAM,CAAC;CACf;AAED,yCAAyC;AACzC,MAAM,WAAW,kBACf,SAAQ,oBAAoB,CAAC,eAAe,CAAC,EAAE,WAAW;IAC1D,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,sCAAsC;AACtC,MAAM,WAAW,kBAAmB,SAAQ,oBAAoB,CAAC,eAAe,CAAC;IAC/E,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAiB,SAAQ,oBAAoB,CAAC,aAAa,CAAC;IAC3E,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,KAAK,CAAC;CACd;AAED,yCAAyC;AACzC,MAAM,WAAW,iBAAkB,SAAQ,oBAAoB,CAAC,cAAc,CAAC;IAC7E,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,eAAe,CAAC;CACzB;AAED;;;GAGG;AACH,MAAM,WAAW,qBAAsB,SAAQ,oBAAoB,CAAC,kBAAkB,CAAC;IACrF,MAAM,EAAE,gBAAgB,CAAC;CAC1B;AAED,kDAAkD;AAClD,MAAM,WAAW,oBACf,SAAQ,oBAAoB,CAAC,iBAAiB,CAAC,EAAE,WAAW;IAC5D,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,uDAAuD;AACvD,MAAM,WAAW,mBAAoB,SAAQ,oBAAoB,CAAC,gBAAgB,CAAC;IACjF,MAAM,EAAE,iBAAiB,CAAC;IAC1B,qEAAqE;IACrE,KAAK,CAAC,EAAE,KAAK,CAAC;CACf;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAmB,SAAQ,oBAAoB,CAAC,eAAe,CAAC;IAC/E,SAAS,EAAE,aAAa,CAAC;CAC1B;AAED;;;;;GAKG;AACH,MAAM,WAAW,iBAAkB,SAAQ,oBAAoB,CAAC,cAAc,CAAC;IAC7E,MAAM,EAAE,YAAY,CAAC;CACtB;AAED,gEAAgE;AAChE,MAAM,WAAW,qBACf,SAAQ,oBAAoB,CAAC,kBAAkB,CAAC,EAAE,sBAAsB;CAAG;AAE7E,8DAA8D;AAC9D,MAAM,WAAW,mBACf,SAAQ,oBAAoB,CAAC,gBAAgB,CAAC,EAAE,sBAAsB;CAAG;AAE3E,kEAAkE;AAClE,MAAM,MAAM,aAAa,GACrB,kBAAkB,GAClB,qBAAqB,GACrB,mBAAmB,GACnB,qBAAqB,GACrB,iBAAiB,GACjB,uBAAuB,GACvB,0BAA0B,GAC1B,kBAAkB,GAClB,iBAAiB,GACjB,yBAAyB,GACzB,eAAe,GACf,kBAAkB,GAClB,kBAAkB,GAClB,gBAAgB,GAChB,iBAAiB,GACjB,qBAAqB,GACrB,oBAAoB,GACpB,mBAAmB,GACnB,kBAAkB,GAClB,iBAAiB,GACjB,qBAAqB,GACrB,mBAAmB,CAAC;AAExB,iFAAiF;AACjF,MAAM,MAAM,oBAAoB,GAAG,aAAa,SAAS,MAAM,KAAK,GAChE,KAAK,SAAS,aAAa,GACzB,IAAI,CAAC,KAAK,EAAE,cAAc,GAAG,WAAW,GAAG,OAAO,CAAC,GACnD,KAAK,GACP,KAAK,CAAC;AAEV,yDAAyD;AACzD,wBAAgB,OAAO,CAAC,KAAK,EAAE,oBAAoB,GAAG,KAAK,CAiC1D"}
|
|
1
|
+
{"version":3,"file":"pipelineEvent.d.ts","sourceRoot":"","sources":["../src/pipelineEvent.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACvD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAC;AAExE;;;GAGG;AACH,MAAM,MAAM,KAAK,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;AAExD;;;;GAIG;AACH,MAAM,WAAW,aAAa,CAAC,IAAI,SAAS,MAAM;IAChD,IAAI,EAAE,IAAI,CAAC;IACX,YAAY,EAAE,KAAK,CAAC;IACpB,SAAS,EAAE,IAAI,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,oEAAoE;AACpE,MAAM,WAAW,oBAAoB,CACnC,IAAI,SAAS,MAAM,CACnB,SAAQ,aAAa,CAAC,IAAI,CAAC;IAC3B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,8FAA8F;AAC9F,MAAM,WAAW,MAAM,CAAC,IAAI,SAAS,MAAM;IACzC,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,0CAA0C;AAC1C,MAAM,MAAM,iBAAiB,GAAG,MAAM,CACpC,WAAW,GAAG,aAAa,GAAG,eAAe,GAAG,yBAAyB,CAC1E,CAAC;AAEF;;;;;GAKG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,CAAC,cAAc,GAAG,iBAAiB,CAAC,CAAC;AAEzE,+CAA+C;AAC/C,MAAM,WAAW,0BAA0B;IACzC,IAAI,EAAE,QAAQ,GAAG,WAAW,GAAG,eAAe,CAAC;IAC/C,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B;AAED;;;;GAIG;AACH,MAAM,WAAW,aAAa;IAC5B;qBACiB;IACjB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,6EAA6E;IAC7E,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,sBAAsB;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B;;oEAEgE;IAChE,QAAQ,CAAC,OAAO,EAAE,SAAS,eAAe,EAAE,CAAC;CAC9C;AAED,4EAA4E;AAC5E,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,yDAAyD;IACzD,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;CACtC;AAED;;;;;GAKG;AACH,MAAM,WAAW,YAAY;IAC3B,4EAA4E;IAC5E,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,qEAAqE;IACrE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,0CAA0C;IAC1C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB;4DACwD;IACxD,QAAQ,CAAC,QAAQ,EAAE,SAAS,WAAW,EAAE,CAAC;CAC3C;AAED,0EAA0E;AAC1E,MAAM,WAAW,WAAW;IAC1B;;8DAE0D;IAC1D,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,yCAAyC;IACzC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,4DAA4D;AAC5D,MAAM,WAAW,WAAW;IAC1B,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,oDAAoD;AACpD,MAAM,WAAW,kBAAmB,SAAQ,aAAa,CAAC,eAAe,CAAC;IACxE,IAAI,EAAE,MAAM,CAAC;CACd;AAED,oFAAoF;AACpF,MAAM,WAAW,qBAAsB,SAAQ,aAAa,CAAC,kBAAkB,CAAC;IAC9E,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;GAGG;AACH,MAAM,WAAW,mBAAoB,SAAQ,aAAa,CAAC,gBAAgB,CAAC;IAC1E,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,sBAAsB;AACtB,MAAM,WAAW,qBACf,SAAQ,aAAa,CAAC,kBAAkB,CAAC,EAAE,WAAW;IACtD,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,iDAAiD;AACjD,MAAM,MAAM,iBAAiB,GAAG,oBAAoB,CAAC,cAAc,CAAC,CAAC;AAErE,mDAAmD;AACnD,MAAM,WAAW,uBAAwB,SAAQ,oBAAoB,CAAC,oBAAoB,CAAC;IACzF,YAAY,EAAE,YAAY,CAAC;IAC3B,MAAM,EAAE,0BAA0B,CAAC;CACpC;AAED,0GAA0G;AAC1G,MAAM,WAAW,0BAA2B,SAAQ,oBAAoB,CAAC,uBAAuB,CAAC;IAC/F,YAAY,EAAE,YAAY,CAAC;IAC3B,OAAO,EAAE,eAAe,CAAC;CAC1B;AAED,mDAAmD;AACnD,MAAM,MAAM,kBAAkB,GAAG,oBAAoB,CAAC,eAAe,CAAC,CAAC;AAEvE,uCAAuC;AACvC,MAAM,WAAW,iBAAkB,SAAQ,oBAAoB,CAAC,cAAc,CAAC;IAC7E,YAAY,EAAE,YAAY,CAAC;IAC3B,KAAK,EAAE,KAAK,CAAC;CACd;AAED,0EAA0E;AAC1E,MAAM,WAAW,yBAA0B,SAAQ,oBAAoB,CAAC,sBAAsB,CAAC;IAC7F,YAAY,EAAE,YAAY,CAAC;IAC3B,QAAQ,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,qCAAqC;AACrC,MAAM,WAAW,eAAgB,SAAQ,oBAAoB,CAAC,YAAY,CAAC;IACzE,KAAK,EAAE,MAAM,CAAC;CACf;AAED,yCAAyC;AACzC,MAAM,WAAW,kBACf,SAAQ,oBAAoB,CAAC,eAAe,CAAC,EAAE,WAAW;IAC1D,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,sCAAsC;AACtC,MAAM,WAAW,kBAAmB,SAAQ,oBAAoB,CAAC,eAAe,CAAC;IAC/E,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB;uEACmE;IACnE,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB;iCAC6B;IAC7B,MAAM,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CAC3C;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,WAAW;IAC1B,iDAAiD;IACjD,QAAQ,EAAE,MAAM,CAAC;IACjB,8DAA8D;IAC9D,MAAM,EAAE,MAAM,CAAC;IACf;wEACoE;IACpE,UAAU,EAAE,SAAS,iBAAiB,EAAE,CAAC;IACzC,qCAAqC;IACrC,UAAU,EAAE,MAAM,CAAC;IACnB,qDAAqD;IACrD,SAAS,EAAE,MAAM,CAAC;IAClB,wEAAwE;IACxE,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,8EAA8E;AAC9E,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,WAAW,CAAC;IACpB,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CAC1C;AAED,wEAAwE;AACxE,MAAM,WAAW,iBAAiB;IAChC;oDACgD;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAiB,SAAQ,oBAAoB,CAAC,aAAa,CAAC;IAC3E,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,KAAK,CAAC;CACd;AAED,yCAAyC;AACzC,MAAM,WAAW,iBAAkB,SAAQ,oBAAoB,CAAC,cAAc,CAAC;IAC7E,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,eAAe,CAAC;CACzB;AAED;;;GAGG;AACH,MAAM,WAAW,qBAAsB,SAAQ,oBAAoB,CAAC,kBAAkB,CAAC;IACrF,MAAM,EAAE,gBAAgB,CAAC;CAC1B;AAED,kDAAkD;AAClD,MAAM,WAAW,oBACf,SAAQ,oBAAoB,CAAC,iBAAiB,CAAC,EAAE,WAAW;IAC5D,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,uDAAuD;AACvD,MAAM,WAAW,mBAAoB,SAAQ,oBAAoB,CAAC,gBAAgB,CAAC;IACjF,MAAM,EAAE,iBAAiB,CAAC;IAC1B,qEAAqE;IACrE,KAAK,CAAC,EAAE,KAAK,CAAC;CACf;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAmB,SAAQ,oBAAoB,CAAC,eAAe,CAAC;IAC/E,SAAS,EAAE,aAAa,CAAC;CAC1B;AAED;;;;;GAKG;AACH,MAAM,WAAW,iBAAkB,SAAQ,oBAAoB,CAAC,cAAc,CAAC;IAC7E,MAAM,EAAE,YAAY,CAAC;CACtB;AAED,gEAAgE;AAChE,MAAM,WAAW,qBACf,SAAQ,oBAAoB,CAAC,kBAAkB,CAAC,EAAE,sBAAsB;CAAG;AAE7E,8DAA8D;AAC9D,MAAM,WAAW,mBACf,SAAQ,oBAAoB,CAAC,gBAAgB,CAAC,EAAE,sBAAsB;CAAG;AAE3E,kEAAkE;AAClE,MAAM,MAAM,aAAa,GACrB,kBAAkB,GAClB,qBAAqB,GACrB,mBAAmB,GACnB,qBAAqB,GACrB,iBAAiB,GACjB,uBAAuB,GACvB,0BAA0B,GAC1B,kBAAkB,GAClB,iBAAiB,GACjB,yBAAyB,GACzB,eAAe,GACf,kBAAkB,GAClB,kBAAkB,GAClB,gBAAgB,GAChB,iBAAiB,GACjB,qBAAqB,GACrB,oBAAoB,GACpB,mBAAmB,GACnB,kBAAkB,GAClB,iBAAiB,GACjB,qBAAqB,GACrB,mBAAmB,CAAC;AAExB,iFAAiF;AACjF,MAAM,MAAM,oBAAoB,GAAG,aAAa,SAAS,MAAM,KAAK,GAChE,KAAK,SAAS,aAAa,GACzB,IAAI,CAAC,KAAK,EAAE,cAAc,GAAG,WAAW,GAAG,OAAO,CAAC,GACnD,KAAK,GACP,KAAK,CAAC;AAEV,yDAAyD;AACzD,wBAAgB,OAAO,CAAC,KAAK,EAAE,oBAAoB,GAAG,KAAK,CAiC1D"}
|
package/dist/stage.d.ts
CHANGED
|
@@ -5,7 +5,7 @@ import { NotSupported } from './sparql/reader.js';
|
|
|
5
5
|
import type { TimeoutPolicy } from './sparql/timeoutPolicy.js';
|
|
6
6
|
import type { Validator } from './validator.js';
|
|
7
7
|
import type { DatasetWriter } from './writer/writer.js';
|
|
8
|
-
import type { UnreadValue } from './pipelineEvent.js';
|
|
8
|
+
import type { StageSummary, UnreadValue } from './pipelineEvent.js';
|
|
9
9
|
/**
|
|
10
10
|
* Transforms a quad stream, given the context of its extension point.
|
|
11
11
|
*
|
|
@@ -26,6 +26,13 @@ export interface ReaderContext {
|
|
|
26
26
|
dataset: Dataset;
|
|
27
27
|
distribution: Distribution;
|
|
28
28
|
stage: string;
|
|
29
|
+
/**
|
|
30
|
+
* Add `amount` (1 when left out) to the count called `name`. The pipeline
|
|
31
|
+
* adds up each name over the stage and reports the totals when it completes
|
|
32
|
+
* (`stageComplete`’s `counts`) – how many images were fetched and how many
|
|
33
|
+
* came from a store, say.
|
|
34
|
+
*/
|
|
35
|
+
count(name: string, amount?: number): void;
|
|
29
36
|
}
|
|
30
37
|
/**
|
|
31
38
|
* The pipeline's one type-changing seam: turn a root-complete batch of quads
|
|
@@ -177,6 +184,9 @@ export interface RunOptions {
|
|
|
177
184
|
/** Receives what a {@link BatchTransform} reports through
|
|
178
185
|
* {@link BatchContext.valueUnread}; dropped when absent. */
|
|
179
186
|
valueUnread?: BatchContext['valueUnread'];
|
|
187
|
+
/** Receives where the stage spent its time and what it counted, once, when
|
|
188
|
+
* it completes. Not called for a stage that is not supported or that fails. */
|
|
189
|
+
onComplete?: (summary: StageSummary) => void;
|
|
180
190
|
}
|
|
181
191
|
/** Options accepted by {@link ItemSelector.select}. */
|
|
182
192
|
export interface SelectOptions {
|
|
@@ -217,12 +227,14 @@ export declare class Stage<Out = Quad> {
|
|
|
217
227
|
private validateBuffer;
|
|
218
228
|
private readAll;
|
|
219
229
|
/**
|
|
220
|
-
*
|
|
221
|
-
* supplying the {@link ReaderContext}
|
|
222
|
-
* call's output (see {@link AttachedReader});
|
|
223
|
-
*
|
|
230
|
+
* Run one reader and fold its attached transforms over its output, in order,
|
|
231
|
+
* supplying the {@link ReaderContext}, each layer timed by `tally`. A
|
|
232
|
+
* transform sees one `read()` call's output (see {@link AttachedReader});
|
|
233
|
+
* `NotSupported` is returned as is and never reaches a transform.
|
|
224
234
|
*/
|
|
225
|
-
private
|
|
235
|
+
private read;
|
|
236
|
+
/** Every attached transform’s name, readers in order, for the phase tally. */
|
|
237
|
+
private get transformNames();
|
|
226
238
|
}
|
|
227
239
|
/** Selects items (as variable bindings) for readers to process. Pagination is an implementation detail. */
|
|
228
240
|
export interface ItemSelector {
|
package/dist/stage.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,EAAmB,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE/D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAExD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,EAAmB,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE/D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAExD,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAGpE;;;;;;;GAOG;AACH,MAAM,MAAM,aAAa,CAAC,GAAG,IAAI,CAC/B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,EAC1B,OAAO,EAAE,GAAG,KACT,aAAa,CAAC,IAAI,CAAC,CAAC;AAEzB;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,YAAY,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd;;;;;OAKG;IACH,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5C;AAED;;;;;;GAMG;AACH,MAAM,MAAM,cAAc,CAAC,GAAG,IAAI,CAChC,KAAK,EAAE,SAAS,IAAI,EAAE,EACtB,OAAO,EAAE,YAAY,KAClB,QAAQ,CAAC,GAAG,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;AAExC;;;;;GAKG;AACH,MAAM,WAAW,YAAa,SAAQ,aAAa;IACjD,QAAQ,EAAE,SAAS,gBAAgB,EAAE,CAAC;IACtC;;;;;OAKG;IACH,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC;CACvE;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,aAAa,CAAC,aAAa,CAAC,GAAG,aAAa,CAAC,aAAa,CAAC,EAAE,CAAC;CAC3E;AAED,yEAAyE;AACzE,MAAM,MAAM,YAAY,GACtB,MAAM,GAAG,cAAc,GAAG,CAAC,MAAM,GAAG,cAAc,CAAC,EAAE,CAAC;AAUxD,MAAM,WAAW,YAAY,CAAC,GAAG,GAAG,IAAI;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,YAAY,CAAC;IACtB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC;IAC9B;;;;;;;OAOG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;;;OAOG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,YAAY,KAAK,YAAY,CAAC;IAC3E,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;IACjB;;;;;;;;;;;;;;;OAeG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,mFAAmF;IACnF,UAAU,CAAC,EAAE;QACX,SAAS,EAAE,SAAS,CAAC;QACrB,iEAAiE;QACjE,SAAS,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;KACvC,CAAC;CACH;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,CAAC,cAAc,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;IACtE;;;;;OAKG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB;iEAC6D;IAC7D,WAAW,CAAC,EAAE,YAAY,CAAC,aAAa,CAAC,CAAC;IAC1C;oFACgF;IAChF,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,IAAI,CAAC;CAC9C;AAED,uDAAuD;AACvD,MAAM,WAAW,aAAa;IAC5B,+BAA+B;IAC/B,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,qBAAa,KAAK,CAAC,GAAG,GAAG,IAAI;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,oGAAoG;IACpG,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC;qEACiE;IACjE,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;IAC7C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAkC;IAC9D,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAiC;gBAEhD,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC;IAmCtC,mDAAmD;IACnD,IAAI,SAAS,IAAI,SAAS,GAAG,SAAS,CAErC;IAEK,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,YAAY,EACtB,MAAM,EAAE,aAAa,CAAC,GAAG,CAAC,EAC1B,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;IAiG/B;;;;OAIG;IACH,OAAO,CAAC,cAAc;YAMR,eAAe;IAyL7B;;;OAGG;YACW,cAAc;YAqBd,OAAO;IA0BrB;;;;;OAKG;YACW,IAAI;IA2BlB,8EAA8E;IAC9E,OAAO,KAAK,cAAc,GAOzB;CACF;AA0DD,2GAA2G;AAC3G,MAAM,WAAW,YAAY;IAC3B,MAAM,CACJ,YAAY,EAAE,YAAY,EAC1B,SAAS,CAAC,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,aAAa,GACtB,aAAa,CAAC,gBAAgB,CAAC,CAAC;CACpC"}
|
package/dist/stage.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { NoItemsSelected, NotSupported } from './sparql/reader.js';
|
|
2
2
|
import { batch } from './batch.js';
|
|
3
3
|
import { AsyncQueue } from './asyncQueue.js';
|
|
4
|
+
import { StageTally } from './stageTally.js';
|
|
4
5
|
export class Stage {
|
|
5
6
|
name;
|
|
6
7
|
stages;
|
|
@@ -56,15 +57,21 @@ export class Stage {
|
|
|
56
57
|
const distribution = this.sourceFor
|
|
57
58
|
? this.sourceFor(dataset, resolved)
|
|
58
59
|
: resolved;
|
|
60
|
+
const tally = new StageTally(this.transformNames);
|
|
59
61
|
if (this.itemSelector) {
|
|
60
|
-
|
|
62
|
+
const result = await this.runWithSelector(tally.selected(this.itemSelector.select(distribution, this.batchSize, {
|
|
61
63
|
timeout,
|
|
62
|
-
}), dataset, distribution, writer, options);
|
|
64
|
+
})), dataset, distribution, writer, tally, options);
|
|
65
|
+
if (!(result instanceof NotSupported)) {
|
|
66
|
+
options?.onComplete?.(tally.summary());
|
|
67
|
+
}
|
|
68
|
+
return result;
|
|
63
69
|
}
|
|
64
|
-
const
|
|
65
|
-
if (
|
|
66
|
-
return
|
|
70
|
+
const chains = await this.readAll(dataset, distribution, timeout, tally);
|
|
71
|
+
if (chains instanceof NotSupported) {
|
|
72
|
+
return chains;
|
|
67
73
|
}
|
|
74
|
+
const streams = chains.map(({ stream }) => stream);
|
|
68
75
|
// The non-selector path has no batch, so `project` is forbidden here (the
|
|
69
76
|
// constructor rejects `project` without an `itemSelector`). It therefore
|
|
70
77
|
// only ever writes quads, and `Out` is `Quad` at runtime – tsc cannot see
|
|
@@ -73,6 +80,7 @@ export class Stage {
|
|
|
73
80
|
// Quads the readers produced (before any validation filtering); used to
|
|
74
81
|
// enforce `expectsOutput` below.
|
|
75
82
|
let produced = 0;
|
|
83
|
+
const write = (quads) => tally.written(quads, (timedQuads) => quadWriter.write(dataset, timedQuads));
|
|
76
84
|
if (this.validation) {
|
|
77
85
|
const buffer = [];
|
|
78
86
|
for (const stream of streams) {
|
|
@@ -84,16 +92,16 @@ export class Stage {
|
|
|
84
92
|
const onInvalid = this.validation.onInvalid ?? 'write';
|
|
85
93
|
if (onInvalid === 'write') {
|
|
86
94
|
await Promise.all([
|
|
87
|
-
|
|
95
|
+
write((async function* () {
|
|
88
96
|
yield* buffer;
|
|
89
97
|
})()),
|
|
90
|
-
this.validation.validator.validate(buffer, dataset),
|
|
98
|
+
tally.validated(this.validation.validator.validate(buffer, dataset)),
|
|
91
99
|
]);
|
|
92
100
|
}
|
|
93
101
|
else {
|
|
94
|
-
const accepted = await this.validateBuffer(buffer, dataset);
|
|
102
|
+
const accepted = await tally.validated(this.validateBuffer(buffer, dataset));
|
|
95
103
|
if (accepted.length > 0) {
|
|
96
|
-
await
|
|
104
|
+
await write((async function* () {
|
|
97
105
|
yield* accepted;
|
|
98
106
|
})());
|
|
99
107
|
}
|
|
@@ -101,15 +109,19 @@ export class Stage {
|
|
|
101
109
|
}
|
|
102
110
|
else if (this.expectsOutput) {
|
|
103
111
|
// Only thread the per-quad counter through when the count is actually
|
|
104
|
-
// needed
|
|
105
|
-
await
|
|
112
|
+
// needed.
|
|
113
|
+
await write(countQuads(mergeStreams(streams), (count) => {
|
|
106
114
|
produced = count;
|
|
107
115
|
}));
|
|
108
116
|
}
|
|
109
117
|
else {
|
|
110
|
-
await
|
|
118
|
+
await write(mergeStreams(streams));
|
|
119
|
+
}
|
|
120
|
+
for (const { settle } of chains) {
|
|
121
|
+
settle();
|
|
111
122
|
}
|
|
112
123
|
this.assertProduced(produced);
|
|
124
|
+
options?.onComplete?.(tally.summary());
|
|
113
125
|
}
|
|
114
126
|
/**
|
|
115
127
|
* Throw when {@link StageOptions.expectsOutput} is set but the stage produced
|
|
@@ -121,7 +133,7 @@ export class Stage {
|
|
|
121
133
|
throw new Error(`Stage '${this.name}' expected output but produced none`);
|
|
122
134
|
}
|
|
123
135
|
}
|
|
124
|
-
async runWithSelector(selector, dataset, distribution, writer, options) {
|
|
136
|
+
async runWithSelector(selector, dataset, distribution, writer, tally, options) {
|
|
125
137
|
// Peek the first batch to detect an empty selector before starting the
|
|
126
138
|
// writer (important because e.g. SparqlUpdateWriter does CLEAR GRAPH).
|
|
127
139
|
const batches = batch(selector, this.batchSize);
|
|
@@ -178,19 +190,16 @@ export class Stage {
|
|
|
178
190
|
}
|
|
179
191
|
track((async () => {
|
|
180
192
|
// Run all readers for this batch in parallel.
|
|
181
|
-
const readerOutputs = await Promise.all(this.readers.map(async (
|
|
182
|
-
const result = await
|
|
183
|
-
bindings,
|
|
184
|
-
timeout: options?.timeout,
|
|
185
|
-
});
|
|
193
|
+
const readerOutputs = await Promise.all(this.readers.map(async (normalized) => {
|
|
194
|
+
const result = await this.read(normalized, dataset, distribution, { bindings, timeout: options?.timeout }, tally);
|
|
186
195
|
if (result instanceof NotSupported)
|
|
187
196
|
return [];
|
|
188
197
|
hasResults = true;
|
|
189
|
-
const stream = this.applyTransforms(transforms, result, dataset, distribution);
|
|
190
198
|
const quads = [];
|
|
191
|
-
for await (const quad of stream) {
|
|
199
|
+
for await (const quad of result.stream) {
|
|
192
200
|
quads.push(quad);
|
|
193
201
|
}
|
|
202
|
+
result.settle();
|
|
194
203
|
return quads;
|
|
195
204
|
}));
|
|
196
205
|
const batchQuads = readerOutputs.flat();
|
|
@@ -201,19 +210,21 @@ export class Stage {
|
|
|
201
210
|
if (this.validation &&
|
|
202
211
|
batchQuads.length > 0 &&
|
|
203
212
|
onInvalid !== 'write') {
|
|
204
|
-
acceptedQuads = await this.validateBuffer(batchQuads, dataset);
|
|
213
|
+
acceptedQuads = await tally.validated(this.validateBuffer(batchQuads, dataset));
|
|
205
214
|
}
|
|
206
215
|
// The one type-changing seam: project the root-complete batch into
|
|
207
216
|
// output items, or pass the quads through for a plain stage (`Out`
|
|
208
217
|
// is `Quad` at runtime when there is no projection).
|
|
209
|
-
const
|
|
210
|
-
|
|
218
|
+
const project = this.project;
|
|
219
|
+
const items = project
|
|
220
|
+
? tally.projected(() => project(acceptedQuads, {
|
|
211
221
|
dataset,
|
|
212
222
|
distribution,
|
|
213
223
|
stage: this.name,
|
|
214
224
|
bindings,
|
|
225
|
+
count: countIn(tally),
|
|
215
226
|
valueUnread: options?.valueUnread ?? ignoreUnreadValue,
|
|
216
|
-
})
|
|
227
|
+
}))
|
|
217
228
|
: acceptedQuads;
|
|
218
229
|
for await (const item of items) {
|
|
219
230
|
await queue.push(item);
|
|
@@ -222,7 +233,7 @@ export class Stage {
|
|
|
222
233
|
if (this.validation &&
|
|
223
234
|
batchQuads.length > 0 &&
|
|
224
235
|
onInvalid === 'write') {
|
|
225
|
-
pendingValidations.push(this.validation.validator.validate(batchQuads, dataset));
|
|
236
|
+
pendingValidations.push(tally.validated(this.validation.validator.validate(batchQuads, dataset)));
|
|
226
237
|
}
|
|
227
238
|
itemsProcessed += bindings.length;
|
|
228
239
|
options?.onProgress?.(itemsProcessed, quadsGenerated);
|
|
@@ -246,7 +257,7 @@ export class Stage {
|
|
|
246
257
|
const dispatchPromise = dispatch();
|
|
247
258
|
const writePromise = (async () => {
|
|
248
259
|
try {
|
|
249
|
-
await writer.write(dataset,
|
|
260
|
+
await tally.written(queue, (items) => writer.write(dataset, items));
|
|
250
261
|
}
|
|
251
262
|
catch (err) {
|
|
252
263
|
queue.abort(err);
|
|
@@ -275,49 +286,51 @@ export class Stage {
|
|
|
275
286
|
// 'skip': discard
|
|
276
287
|
return [];
|
|
277
288
|
}
|
|
278
|
-
async readAll(dataset, distribution, timeout) {
|
|
279
|
-
const results = await Promise.all(this.readers.map(
|
|
280
|
-
|
|
281
|
-
timeout,
|
|
282
|
-
});
|
|
283
|
-
if (result instanceof NotSupported)
|
|
284
|
-
return result;
|
|
285
|
-
return this.applyTransforms(transforms, result, dataset, distribution);
|
|
286
|
-
}));
|
|
287
|
-
const streams = [];
|
|
289
|
+
async readAll(dataset, distribution, timeout, tally) {
|
|
290
|
+
const results = await Promise.all(this.readers.map((normalized) => this.read(normalized, dataset, distribution, { timeout }, tally)));
|
|
291
|
+
const chains = [];
|
|
288
292
|
for (const result of results) {
|
|
289
293
|
if (!(result instanceof NotSupported)) {
|
|
290
|
-
|
|
294
|
+
chains.push(result);
|
|
291
295
|
}
|
|
292
296
|
}
|
|
293
|
-
if (
|
|
297
|
+
if (chains.length === 0) {
|
|
294
298
|
return new NotSupported('All readers returned NotSupported');
|
|
295
299
|
}
|
|
296
|
-
return
|
|
300
|
+
return chains;
|
|
297
301
|
}
|
|
298
302
|
/**
|
|
299
|
-
*
|
|
300
|
-
* supplying the {@link ReaderContext}
|
|
301
|
-
* call's output (see {@link AttachedReader});
|
|
302
|
-
*
|
|
303
|
+
* Run one reader and fold its attached transforms over its output, in order,
|
|
304
|
+
* supplying the {@link ReaderContext}, each layer timed by `tally`. A
|
|
305
|
+
* transform sees one `read()` call's output (see {@link AttachedReader});
|
|
306
|
+
* `NotSupported` is returned as is and never reaches a transform.
|
|
303
307
|
*/
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
308
|
+
async read({ reader, transforms, firstTransform }, dataset, distribution, readOptions, tally) {
|
|
309
|
+
const started = performance.now();
|
|
310
|
+
const result = await reader.read(dataset, distribution, readOptions);
|
|
311
|
+
tally.addRead(performance.now() - started);
|
|
312
|
+
if (result instanceof NotSupported)
|
|
313
|
+
return result;
|
|
307
314
|
const context = {
|
|
308
315
|
dataset,
|
|
309
316
|
distribution,
|
|
310
317
|
stage: this.name,
|
|
318
|
+
count: countIn(tally),
|
|
311
319
|
};
|
|
312
|
-
return transforms.
|
|
320
|
+
return tally.chain(result, transforms.map((transform) => (quads) => transform(quads, context)), firstTransform);
|
|
321
|
+
}
|
|
322
|
+
/** Every attached transform’s name, readers in order, for the phase tally. */
|
|
323
|
+
get transformNames() {
|
|
324
|
+
return this.readers.flatMap(({ transforms, firstTransform }) => transforms.map((transform, index) => transform.name || `transform ${firstTransform + index + 1}`));
|
|
313
325
|
}
|
|
314
326
|
}
|
|
315
327
|
/** Normalise the {@link StageReaders} union to reader + transforms pairs. */
|
|
316
328
|
function normalizeReaders(readers) {
|
|
317
329
|
const list = Array.isArray(readers) ? readers : [readers];
|
|
330
|
+
let firstTransform = 0;
|
|
318
331
|
return list.map((entry) => {
|
|
319
332
|
if ('read' in entry) {
|
|
320
|
-
return { reader: entry, transforms: [] };
|
|
333
|
+
return { reader: entry, transforms: [], firstTransform };
|
|
321
334
|
}
|
|
322
335
|
const { reader, transform } = entry;
|
|
323
336
|
const transforms = transform === undefined
|
|
@@ -325,7 +338,9 @@ function normalizeReaders(readers) {
|
|
|
325
338
|
: Array.isArray(transform)
|
|
326
339
|
? [...transform]
|
|
327
340
|
: [transform];
|
|
328
|
-
|
|
341
|
+
const normalized = { reader, transforms, firstTransform };
|
|
342
|
+
firstTransform += transforms.length;
|
|
343
|
+
return normalized;
|
|
329
344
|
});
|
|
330
345
|
}
|
|
331
346
|
async function* mergeStreams(streams) {
|
|
@@ -350,6 +365,10 @@ async function* countQuads(stream, onCount) {
|
|
|
350
365
|
}
|
|
351
366
|
onCount(count);
|
|
352
367
|
}
|
|
368
|
+
/** The {@link ReaderContext.count} that adds to `tally`. */
|
|
369
|
+
function countIn(tally) {
|
|
370
|
+
return (name, amount = 1) => tally.count(name, amount);
|
|
371
|
+
}
|
|
353
372
|
/** The {@link BatchContext.valueUnread} a stage run without a receiver hands
|
|
354
373
|
* its transform. */
|
|
355
374
|
function ignoreUnreadValue() {
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import type { StageSummary } from './pipelineEvent.js';
|
|
2
|
+
/**
|
|
3
|
+
* Adds up, over every batch one {@link Stage.run} runs, where it spends its
|
|
4
|
+
* time, phase by phase, and what its transforms and projection count.
|
|
5
|
+
*
|
|
6
|
+
* A stage streams: a reader’s quads flow through its transforms into the
|
|
7
|
+
* writer, and every layer pulls from the one before it. Timing a layer from its
|
|
8
|
+
* first quad to its last would count the layers upstream of it too. So the tally
|
|
9
|
+
* times only the wait on each layer’s `next()`, and takes a layer’s own time as
|
|
10
|
+
* its wait minus the wait of the layer it pulls from. That holds as long as a
|
|
11
|
+
* layer pulls from upstream only while its own `next()` is pending, which an
|
|
12
|
+
* async generator does; a transform that prefetches in the background has its
|
|
13
|
+
* time undercounted, never below zero.
|
|
14
|
+
*/
|
|
15
|
+
export declare class StageTally {
|
|
16
|
+
private selectMs;
|
|
17
|
+
private readMs;
|
|
18
|
+
private validateMs;
|
|
19
|
+
private projectMs;
|
|
20
|
+
private writeMs;
|
|
21
|
+
private readonly transforms;
|
|
22
|
+
private readonly counts;
|
|
23
|
+
/** `transformNames` lists every attached transform, readers in order. */
|
|
24
|
+
constructor(transformNames: readonly string[]);
|
|
25
|
+
/** The item selector, timed while it is iterated. */
|
|
26
|
+
selected<T>(selector: AsyncIterable<T>): AsyncIterable<T>;
|
|
27
|
+
/** How long a reader took to answer, before its quads stream. */
|
|
28
|
+
addRead(milliseconds: number): void;
|
|
29
|
+
/**
|
|
30
|
+
* Time one reader’s output and the transforms folded over it. `apply` builds
|
|
31
|
+
* the next layer from the one before; `firstTransform` is the position of the
|
|
32
|
+
* reader’s first transform among all of the stage’s.
|
|
33
|
+
*
|
|
34
|
+
* Returns the timed stream and a `settle` to call once that stream is drained,
|
|
35
|
+
* which adds this chain’s times to the tally.
|
|
36
|
+
*/
|
|
37
|
+
chain<T>(source: AsyncIterable<T>, layers: readonly ((upstream: AsyncIterable<T>) => AsyncIterable<T>)[], firstTransform: number): {
|
|
38
|
+
stream: AsyncIterable<T>;
|
|
39
|
+
settle: () => void;
|
|
40
|
+
};
|
|
41
|
+
/** Time validating one buffer of quads. */
|
|
42
|
+
validated<T>(validation: Promise<T>): Promise<T>;
|
|
43
|
+
/** Time projecting one batch: building the items, then iterating them. */
|
|
44
|
+
projected<T>(project: () => Iterable<T> | AsyncIterable<T>): AsyncIterable<T>;
|
|
45
|
+
/**
|
|
46
|
+
* Time the writer: `write` is handed the items to write and runs to
|
|
47
|
+
* completion. The time it spent waiting on those items is not its own.
|
|
48
|
+
*/
|
|
49
|
+
written<T>(items: AsyncIterable<T>, write: (items: AsyncIterable<T>) => Promise<void>): Promise<void>;
|
|
50
|
+
/** Add `amount` to the count called `name`. */
|
|
51
|
+
count(name: string, amount: number): void;
|
|
52
|
+
/** What the stage counted, and its phases rounded to whole milliseconds. */
|
|
53
|
+
summary(): StageSummary;
|
|
54
|
+
private phases;
|
|
55
|
+
}
|
|
56
|
+
//# sourceMappingURL=stageTally.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stageTally.d.ts","sourceRoot":"","sources":["../src/stageTally.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAEV,YAAY,EAEb,MAAM,oBAAoB,CAAC;AAE5B;;;;;;;;;;;;GAYG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,QAAQ,CAAK;IACrB,OAAO,CAAC,MAAM,CAAK;IACnB,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,OAAO,CAAK;IACpB,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAsB;IACjD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA8B;IAErD,yEAAyE;gBAC7D,cAAc,EAAE,SAAS,MAAM,EAAE;IAI7C,qDAAqD;IACrD,QAAQ,CAAC,CAAC,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC;IAMzD,iEAAiE;IACjE,OAAO,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI;IAInC;;;;;;;OAOG;IACH,KAAK,CAAC,CAAC,EACL,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,EACxB,MAAM,EAAE,SAAS,CAAC,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,KAAK,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,EACrE,cAAc,EAAE,MAAM,GACrB;QAAE,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC;QAAC,MAAM,EAAE,MAAM,IAAI,CAAA;KAAE;IA6BnD,2CAA2C;IACrC,SAAS,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAStD,0EAA0E;IAC1E,SAAS,CAAC,CAAC,EACT,OAAO,EAAE,MAAM,QAAQ,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,GAC5C,aAAa,CAAC,CAAC,CAAC;IASnB;;;OAGG;IACG,OAAO,CAAC,CAAC,EACb,KAAK,EAAE,aAAa,CAAC,CAAC,CAAC,EACvB,KAAK,EAAE,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,GAChD,OAAO,CAAC,IAAI,CAAC;IAchB,+CAA+C;IAC/C,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IAIzC,4EAA4E;IAC5E,OAAO,IAAI,YAAY;IAIvB,OAAO,CAAC,MAAM;CAaf"}
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Adds up, over every batch one {@link Stage.run} runs, where it spends its
|
|
3
|
+
* time, phase by phase, and what its transforms and projection count.
|
|
4
|
+
*
|
|
5
|
+
* A stage streams: a reader’s quads flow through its transforms into the
|
|
6
|
+
* writer, and every layer pulls from the one before it. Timing a layer from its
|
|
7
|
+
* first quad to its last would count the layers upstream of it too. So the tally
|
|
8
|
+
* times only the wait on each layer’s `next()`, and takes a layer’s own time as
|
|
9
|
+
* its wait minus the wait of the layer it pulls from. That holds as long as a
|
|
10
|
+
* layer pulls from upstream only while its own `next()` is pending, which an
|
|
11
|
+
* async generator does; a transform that prefetches in the background has its
|
|
12
|
+
* time undercounted, never below zero.
|
|
13
|
+
*/
|
|
14
|
+
export class StageTally {
|
|
15
|
+
selectMs = 0;
|
|
16
|
+
readMs = 0;
|
|
17
|
+
validateMs = 0;
|
|
18
|
+
projectMs = 0;
|
|
19
|
+
writeMs = 0;
|
|
20
|
+
transforms;
|
|
21
|
+
counts = {};
|
|
22
|
+
/** `transformNames` lists every attached transform, readers in order. */
|
|
23
|
+
constructor(transformNames) {
|
|
24
|
+
this.transforms = transformNames.map((name) => ({ name, durationMs: 0 }));
|
|
25
|
+
}
|
|
26
|
+
/** The item selector, timed while it is iterated. */
|
|
27
|
+
selected(selector) {
|
|
28
|
+
return timed(selector, (milliseconds) => {
|
|
29
|
+
this.selectMs += milliseconds;
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
/** How long a reader took to answer, before its quads stream. */
|
|
33
|
+
addRead(milliseconds) {
|
|
34
|
+
this.readMs += milliseconds;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Time one reader’s output and the transforms folded over it. `apply` builds
|
|
38
|
+
* the next layer from the one before; `firstTransform` is the position of the
|
|
39
|
+
* reader’s first transform among all of the stage’s.
|
|
40
|
+
*
|
|
41
|
+
* Returns the timed stream and a `settle` to call once that stream is drained,
|
|
42
|
+
* which adds this chain’s times to the tally.
|
|
43
|
+
*/
|
|
44
|
+
chain(source, layers, firstTransform) {
|
|
45
|
+
// waits[0] is the reader’s, waits[k] the k-th transform’s, each including
|
|
46
|
+
// the waits before it.
|
|
47
|
+
const waits = layers.map(() => 0).concat(0);
|
|
48
|
+
let stream = timed(source, (milliseconds) => {
|
|
49
|
+
waits[0] += milliseconds;
|
|
50
|
+
});
|
|
51
|
+
// A transform’s own work before it returns its stream is its time alone.
|
|
52
|
+
const built = layers.map(() => 0);
|
|
53
|
+
layers.forEach((layer, index) => {
|
|
54
|
+
const started = performance.now();
|
|
55
|
+
const upstream = layer(stream);
|
|
56
|
+
built[index] = performance.now() - started;
|
|
57
|
+
stream = timed(upstream, (milliseconds) => {
|
|
58
|
+
waits[index + 1] += milliseconds;
|
|
59
|
+
});
|
|
60
|
+
});
|
|
61
|
+
return {
|
|
62
|
+
stream,
|
|
63
|
+
settle: () => {
|
|
64
|
+
this.readMs += waits[0];
|
|
65
|
+
for (let index = 0; index < layers.length; index++) {
|
|
66
|
+
this.transforms[firstTransform + index].durationMs +=
|
|
67
|
+
built[index] + Math.max(0, waits[index + 1] - waits[index]);
|
|
68
|
+
}
|
|
69
|
+
},
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
/** Time validating one buffer of quads. */
|
|
73
|
+
async validated(validation) {
|
|
74
|
+
const started = performance.now();
|
|
75
|
+
try {
|
|
76
|
+
return await validation;
|
|
77
|
+
}
|
|
78
|
+
finally {
|
|
79
|
+
this.validateMs += performance.now() - started;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
/** Time projecting one batch: building the items, then iterating them. */
|
|
83
|
+
projected(project) {
|
|
84
|
+
const started = performance.now();
|
|
85
|
+
const items = project();
|
|
86
|
+
this.projectMs += performance.now() - started;
|
|
87
|
+
return timed(items, (milliseconds) => {
|
|
88
|
+
this.projectMs += milliseconds;
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Time the writer: `write` is handed the items to write and runs to
|
|
93
|
+
* completion. The time it spent waiting on those items is not its own.
|
|
94
|
+
*/
|
|
95
|
+
async written(items, write) {
|
|
96
|
+
let waitedMs = 0;
|
|
97
|
+
const started = performance.now();
|
|
98
|
+
try {
|
|
99
|
+
await write(timed(items, (milliseconds) => {
|
|
100
|
+
waitedMs += milliseconds;
|
|
101
|
+
}));
|
|
102
|
+
}
|
|
103
|
+
finally {
|
|
104
|
+
this.writeMs += Math.max(0, performance.now() - started - waitedMs);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
/** Add `amount` to the count called `name`. */
|
|
108
|
+
count(name, amount) {
|
|
109
|
+
this.counts[name] = (this.counts[name] ?? 0) + amount;
|
|
110
|
+
}
|
|
111
|
+
/** What the stage counted, and its phases rounded to whole milliseconds. */
|
|
112
|
+
summary() {
|
|
113
|
+
return { phases: this.phases(), counts: { ...this.counts } };
|
|
114
|
+
}
|
|
115
|
+
phases() {
|
|
116
|
+
return {
|
|
117
|
+
selectMs: Math.round(this.selectMs),
|
|
118
|
+
readMs: Math.round(this.readMs),
|
|
119
|
+
transforms: [...sumByName(this.transforms)].map(([name, durationMs]) => ({
|
|
120
|
+
name,
|
|
121
|
+
durationMs: Math.round(durationMs),
|
|
122
|
+
})),
|
|
123
|
+
validateMs: Math.round(this.validateMs),
|
|
124
|
+
projectMs: Math.round(this.projectMs),
|
|
125
|
+
writeMs: Math.round(this.writeMs),
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
/** Add up the durations of transforms sharing a name, in order first seen. */
|
|
130
|
+
function sumByName(transforms) {
|
|
131
|
+
const sums = new Map();
|
|
132
|
+
for (const { name, durationMs } of transforms) {
|
|
133
|
+
sums.set(name, (sums.get(name) ?? 0) + durationMs);
|
|
134
|
+
}
|
|
135
|
+
return sums;
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Pass `source` through unchanged, reporting each wait on its `next()` to
|
|
139
|
+
* `onWait`. Accepts a plain iterable too, for a projection that returns an
|
|
140
|
+
* array.
|
|
141
|
+
*/
|
|
142
|
+
function timed(source, onWait) {
|
|
143
|
+
return {
|
|
144
|
+
[Symbol.asyncIterator]() {
|
|
145
|
+
const iterator = Symbol.asyncIterator in source
|
|
146
|
+
? source[Symbol.asyncIterator]()
|
|
147
|
+
: source[Symbol.iterator]();
|
|
148
|
+
return {
|
|
149
|
+
async next() {
|
|
150
|
+
const started = performance.now();
|
|
151
|
+
try {
|
|
152
|
+
return await iterator.next();
|
|
153
|
+
}
|
|
154
|
+
finally {
|
|
155
|
+
onWait(performance.now() - started);
|
|
156
|
+
}
|
|
157
|
+
},
|
|
158
|
+
async return(value) {
|
|
159
|
+
return (await iterator.return?.(value)) ?? { done: true, value };
|
|
160
|
+
},
|
|
161
|
+
};
|
|
162
|
+
},
|
|
163
|
+
};
|
|
164
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lde/pipeline",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.46.0",
|
|
4
4
|
"repository": {
|
|
5
5
|
"url": "git+https://github.com/ldelements/lde.git",
|
|
6
6
|
"directory": "packages/pipeline"
|
|
@@ -26,8 +26,8 @@
|
|
|
26
26
|
"dependencies": {
|
|
27
27
|
"@lde/dataset": "^0.10.1",
|
|
28
28
|
"@lde/dataset-registry-client": "^0.12.1",
|
|
29
|
-
"@lde/distribution-health": "^0.
|
|
30
|
-
"@lde/distribution-probe": "^0.
|
|
29
|
+
"@lde/distribution-health": "^0.8.0",
|
|
30
|
+
"@lde/distribution-probe": "^0.6.0",
|
|
31
31
|
"@lde/sparql-importer": "^0.11.1",
|
|
32
32
|
"@lde/sparql-server": "^0.6.0",
|
|
33
33
|
"@rdfjs/namespace": "^2.0.1",
|