@lde/pipeline 0.43.0 → 0.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/distribution/importResolver.js +4 -4
- package/dist/distribution/resolver.d.ts +2 -2
- package/dist/distribution/resolver.d.ts.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/pipeline.d.ts +6 -9
- package/dist/pipeline.d.ts.map +1 -1
- package/dist/pipeline.js +173 -71
- package/dist/pipelineEvent.d.ts +273 -0
- package/dist/pipelineEvent.d.ts.map +1 -0
- package/dist/pipelineEvent.js +35 -0
- package/dist/reporter.d.ts +14 -0
- package/dist/reporter.d.ts.map +1 -0
- package/dist/sparql/reader.d.ts +16 -7
- package/dist/sparql/reader.d.ts.map +1 -1
- package/dist/sparql/reader.js +16 -5
- package/dist/stage.d.ts +19 -7
- package/dist/stage.d.ts.map +1 -1
- package/dist/stage.js +70 -51
- package/dist/stageTally.d.ts +56 -0
- package/dist/stageTally.d.ts.map +1 -0
- package/dist/stageTally.js +164 -0
- package/dist/unreadValueTally.d.ts +2 -2
- package/dist/unreadValueTally.d.ts.map +1 -1
- package/dist/unreadValueTally.js +1 -1
- package/dist/writer/writer.d.ts +1 -1
- package/dist/writer/writer.d.ts.map +1 -1
- package/package.json +1 -1
- package/dist/combineReporters.d.ts +0 -16
- package/dist/combineReporters.d.ts.map +0 -1
- package/dist/combineReporters.js +0 -57
- package/dist/progressReporter.d.ts +0 -157
- package/dist/progressReporter.d.ts.map +0 -1
- /package/dist/{progressReporter.js → reporter.js} +0 -0
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
import type { Dataset, Distribution } from '@lde/dataset';
|
|
2
|
+
import type { ValidityVerdict } from '@lde/distribution-health';
|
|
3
|
+
import type { ValidationReport } from './validator.js';
|
|
4
|
+
import type { ImportDetails } from './distribution/resolver.js';
|
|
5
|
+
import type { TimeoutTransitionEvent } from './sparql/timeoutPolicy.js';
|
|
6
|
+
/**
|
|
7
|
+
* Deliberately without `fatal`: the pipeline never ends the process; an
|
|
8
|
+
* unrecoverable failure rejects {@link Pipeline.run}.
|
|
9
|
+
*/
|
|
10
|
+
export type Level = 'debug' | 'info' | 'warn' | 'error';
|
|
11
|
+
/**
|
|
12
|
+
* Fields every {@link PipelineEvent} carries. Owning the `type` discriminator
|
|
13
|
+
* here means no event can join the union without one. `defaultLevel` is the
|
|
14
|
+
* pipeline’s default; reporters may remap it.
|
|
15
|
+
*/
|
|
16
|
+
export interface EventEnvelope<Type extends string> {
|
|
17
|
+
type: Type;
|
|
18
|
+
defaultLevel: Level;
|
|
19
|
+
timestamp: Date;
|
|
20
|
+
runId: string;
|
|
21
|
+
}
|
|
22
|
+
/** The envelope of every event that belongs to a single dataset. */
|
|
23
|
+
export interface DatasetEventEnvelope<Type extends string> extends EventEnvelope<Type> {
|
|
24
|
+
dataset: Dataset;
|
|
25
|
+
}
|
|
26
|
+
/** Why something was skipped: a stable `code` to branch on and a human-readable `message`. */
|
|
27
|
+
export interface Reason<Code extends string> {
|
|
28
|
+
code: Code;
|
|
29
|
+
message: string;
|
|
30
|
+
}
|
|
31
|
+
/** Why the pipeline skipped a dataset. */
|
|
32
|
+
export type DatasetSkipReason = Reason<'unchanged' | 'probeFailed' | 'resolveFailed' | 'noDistributionAvailable'>;
|
|
33
|
+
/**
|
|
34
|
+
* Why the pipeline skipped a stage for a dataset: `noItemsSelected` when the
|
|
35
|
+
* stage’s item selector selected nothing, so the stage does not apply to the
|
|
36
|
+
* dataset; `notSupported` when it could not run against the dataset’s
|
|
37
|
+
* distribution.
|
|
38
|
+
*/
|
|
39
|
+
export type StageSkipReason = Reason<'notSupported' | 'noItemsSelected'>;
|
|
40
|
+
/** The outcome of probing one distribution. */
|
|
41
|
+
export interface DistributionAnalysisResult {
|
|
42
|
+
type: 'sparql' | 'data-dump' | 'network-error';
|
|
43
|
+
available: boolean;
|
|
44
|
+
statusCode?: number;
|
|
45
|
+
error?: string;
|
|
46
|
+
warnings: string[];
|
|
47
|
+
/**
|
|
48
|
+
* The source-change fingerprint observed for this distribution, or `null`
|
|
49
|
+
* when none could be established (e.g. a live SPARQL endpoint). The shared
|
|
50
|
+
* key against which a validity verdict's `validatedFingerprint` is matched,
|
|
51
|
+
* so a consumer can record it on the reachability rail. Always populated by
|
|
52
|
+
* the pipeline; optional so consumers constructing the result need not set it.
|
|
53
|
+
*/
|
|
54
|
+
fingerprint?: string | null;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Items a writer left out of a dataset’s output instead of failing the
|
|
58
|
+
* dataset – documents a destination refused, say – reported once per
|
|
59
|
+
* (dataset, target) as the writer finalizes the dataset.
|
|
60
|
+
*/
|
|
61
|
+
export interface ItemsRejected {
|
|
62
|
+
/** The part of the destination that refused them, e.g. a search index’s
|
|
63
|
+
* type name. */
|
|
64
|
+
readonly target: string;
|
|
65
|
+
/** Items the dataset wrote to {@link target}, the rejected ones included. */
|
|
66
|
+
readonly written: number;
|
|
67
|
+
/** Items left out. */
|
|
68
|
+
readonly rejected: number;
|
|
69
|
+
/** Why, most frequent first: each distinct reason, how many items it
|
|
70
|
+
* rejected, and the ids of a few of them. Bounded by the writer, so a
|
|
71
|
+
* reason that varies per item cannot grow it without limit. */
|
|
72
|
+
readonly reasons: readonly RejectionReason[];
|
|
73
|
+
}
|
|
74
|
+
/** One reason a writer rejected items, in {@link ItemsRejected.reasons}. */
|
|
75
|
+
export interface RejectionReason {
|
|
76
|
+
readonly reason: string;
|
|
77
|
+
readonly count: number;
|
|
78
|
+
/** Ids of some of the items rejected for this reason. */
|
|
79
|
+
readonly examples: readonly string[];
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Values a stage read from a dataset but could not interpret – an integer
|
|
83
|
+
* field over `onbekend`, a date over `Z.j` – and so left out of the items it
|
|
84
|
+
* produced, reported once per (dataset, target, field) when the dataset
|
|
85
|
+
* completes. The item itself is still produced; only the value is missing.
|
|
86
|
+
*/
|
|
87
|
+
export interface ValuesUnread {
|
|
88
|
+
/** The kind of item the values were read for, e.g. a search type’s name. */
|
|
89
|
+
readonly target: string;
|
|
90
|
+
/** The field that could not read them; dotted for a nested field. */
|
|
91
|
+
readonly field: string;
|
|
92
|
+
/** How many values the field left out. */
|
|
93
|
+
readonly unread: number;
|
|
94
|
+
/** A few of them, each a distinct value. Bounded, so a field that rejects
|
|
95
|
+
* every value cannot grow the report with the data. */
|
|
96
|
+
readonly examples: readonly UnreadValue[];
|
|
97
|
+
}
|
|
98
|
+
/** One value a stage could not read, in {@link ValuesUnread.examples}. */
|
|
99
|
+
export interface UnreadValue {
|
|
100
|
+
/** What the value was read from, as the stage names it – for a search
|
|
101
|
+
* document, the node it was projected from, which a publisher can look up
|
|
102
|
+
* even where the document is keyed on something else. */
|
|
103
|
+
readonly id: string;
|
|
104
|
+
/** The value as the source stated it. */
|
|
105
|
+
readonly value: string;
|
|
106
|
+
}
|
|
107
|
+
/** The process’s memory use when the event was reported. */
|
|
108
|
+
export interface MemoryUsage {
|
|
109
|
+
memoryUsageBytes: number;
|
|
110
|
+
heapUsedBytes: number;
|
|
111
|
+
}
|
|
112
|
+
/** A run started; `name` is the pipeline’s name. */
|
|
113
|
+
export interface PipelineStartEvent extends EventEnvelope<'pipelineStart'> {
|
|
114
|
+
name: string;
|
|
115
|
+
}
|
|
116
|
+
/** The datasets to process were selected: how many, and how long selecting took. */
|
|
117
|
+
export interface DatasetsSelectedEvent extends EventEnvelope<'datasetsSelected'> {
|
|
118
|
+
count: number;
|
|
119
|
+
durationMs: number;
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* The run was abandoned because the selection was empty: the writers were told
|
|
123
|
+
* to abort, so the destination is unchanged.
|
|
124
|
+
*/
|
|
125
|
+
export interface SelectionEmptyEvent extends EventEnvelope<'selectionEmpty'> {
|
|
126
|
+
message: string;
|
|
127
|
+
}
|
|
128
|
+
/** A run finished. */
|
|
129
|
+
export interface PipelineCompleteEvent extends EventEnvelope<'pipelineComplete'>, MemoryUsage {
|
|
130
|
+
durationMs: number;
|
|
131
|
+
}
|
|
132
|
+
/** The pipeline started processing a dataset. */
|
|
133
|
+
export type DatasetStartEvent = DatasetEventEnvelope<'datasetStart'>;
|
|
134
|
+
/** One of a dataset’s distributions was probed. */
|
|
135
|
+
export interface DistributionProbedEvent extends DatasetEventEnvelope<'distributionProbed'> {
|
|
136
|
+
distribution: Distribution;
|
|
137
|
+
result: DistributionAnalysisResult;
|
|
138
|
+
}
|
|
139
|
+
/** Reported for every distribution the pipeline attempted, even when the dataset is otherwise skipped. */
|
|
140
|
+
export interface DistributionValidatedEvent extends DatasetEventEnvelope<'distributionValidated'> {
|
|
141
|
+
distribution: Distribution;
|
|
142
|
+
verdict: ValidityVerdict;
|
|
143
|
+
}
|
|
144
|
+
/** A data dump started importing for a dataset. */
|
|
145
|
+
export type ImportStartedEvent = DatasetEventEnvelope<'importStarted'>;
|
|
146
|
+
/** Importing a distribution failed. */
|
|
147
|
+
export interface ImportFailedEvent extends DatasetEventEnvelope<'importFailed'> {
|
|
148
|
+
distribution: Distribution;
|
|
149
|
+
error: Error;
|
|
150
|
+
}
|
|
151
|
+
/** `imported` is set when `distribution` serves an imported data dump. */
|
|
152
|
+
export interface DistributionSelectedEvent extends DatasetEventEnvelope<'distributionSelected'> {
|
|
153
|
+
distribution: Distribution;
|
|
154
|
+
imported?: ImportDetails;
|
|
155
|
+
}
|
|
156
|
+
/** A stage started for a dataset. */
|
|
157
|
+
export interface StageStartEvent extends DatasetEventEnvelope<'stageStart'> {
|
|
158
|
+
stage: string;
|
|
159
|
+
}
|
|
160
|
+
/** A running stage’s progress so far. */
|
|
161
|
+
export interface StageProgressEvent extends DatasetEventEnvelope<'stageProgress'>, MemoryUsage {
|
|
162
|
+
stage: string;
|
|
163
|
+
itemsProcessed: number;
|
|
164
|
+
quadsGenerated: number;
|
|
165
|
+
}
|
|
166
|
+
/** A stage finished for a dataset. */
|
|
167
|
+
export interface StageCompleteEvent extends DatasetEventEnvelope<'stageComplete'> {
|
|
168
|
+
stage: string;
|
|
169
|
+
itemsProcessed: number;
|
|
170
|
+
quadsGenerated: number;
|
|
171
|
+
durationMs: number;
|
|
172
|
+
/** Where the stage spent its time. Always set by the pipeline; optional so
|
|
173
|
+
* code constructing the event need not set it, as is `counts`. */
|
|
174
|
+
phases?: StagePhases;
|
|
175
|
+
/** What the stage’s transforms and projection counted, per name, through
|
|
176
|
+
* `ReaderContext.count`. */
|
|
177
|
+
counts?: Readonly<Record<string, number>>;
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* The time a stage spent in each of its phases, in milliseconds.
|
|
181
|
+
*
|
|
182
|
+
* A stage runs several batches at once, so each phase is added up over all
|
|
183
|
+
* of them: the phases can together exceed the stage’s `durationMs`, and a
|
|
184
|
+
* phase that does is one batches spent their time waiting on. Each phase
|
|
185
|
+
* counts only its own time – a transform waiting on the reader it pulls from
|
|
186
|
+
* is the reader’s time, not the transform’s.
|
|
187
|
+
*/
|
|
188
|
+
export interface StagePhases {
|
|
189
|
+
/** Selecting the stage’s items, page by page. */
|
|
190
|
+
selectMs: number;
|
|
191
|
+
/** Running the readers’ queries and reading their results. */
|
|
192
|
+
readMs: number;
|
|
193
|
+
/** Each attached transform, in the order first attached. A transform
|
|
194
|
+
* attached to several readers is one entry, its times added up. */
|
|
195
|
+
transforms: readonly TransformDuration[];
|
|
196
|
+
/** Validating the readers’ quads. */
|
|
197
|
+
validateMs: number;
|
|
198
|
+
/** Turning batches into the stage’s output items. */
|
|
199
|
+
projectMs: number;
|
|
200
|
+
/** The writer’s own work, not counting the time it waited for items. */
|
|
201
|
+
writeMs: number;
|
|
202
|
+
}
|
|
203
|
+
/** What a completed stage hands back: where its time went, and its counts. */
|
|
204
|
+
export interface StageSummary {
|
|
205
|
+
phases: StagePhases;
|
|
206
|
+
counts: Readonly<Record<string, number>>;
|
|
207
|
+
}
|
|
208
|
+
/** One attached transform’s time, in {@link StagePhases.transforms}. */
|
|
209
|
+
export interface TransformDuration {
|
|
210
|
+
/** The transform function’s name, or `transform N` for one without a name,
|
|
211
|
+
* N counting the stage’s transforms from 1. */
|
|
212
|
+
name: string;
|
|
213
|
+
durationMs: number;
|
|
214
|
+
}
|
|
215
|
+
/**
|
|
216
|
+
* Also reported for per-dataset failures outside any stage, with `stage` set
|
|
217
|
+
* to `write`, `provenance` or `reactive-dump-fallback`.
|
|
218
|
+
*/
|
|
219
|
+
export interface StageFailedEvent extends DatasetEventEnvelope<'stageFailed'> {
|
|
220
|
+
stage: string;
|
|
221
|
+
error: Error;
|
|
222
|
+
}
|
|
223
|
+
/** A stage was skipped for a dataset. */
|
|
224
|
+
export interface StageSkippedEvent extends DatasetEventEnvelope<'stageSkipped'> {
|
|
225
|
+
stage: string;
|
|
226
|
+
reason: StageSkipReason;
|
|
227
|
+
}
|
|
228
|
+
/**
|
|
229
|
+
* Reported once per (dataset, validator) pair, even when nothing was validated:
|
|
230
|
+
* read `quadsValidated` to tell ‘not tested’ from ‘tested and passed’.
|
|
231
|
+
*/
|
|
232
|
+
export interface DatasetValidatedEvent extends DatasetEventEnvelope<'datasetValidated'> {
|
|
233
|
+
report: ValidationReport;
|
|
234
|
+
}
|
|
235
|
+
/** The pipeline finished processing a dataset. */
|
|
236
|
+
export interface DatasetCompleteEvent extends DatasetEventEnvelope<'datasetComplete'>, MemoryUsage {
|
|
237
|
+
durationMs: number;
|
|
238
|
+
}
|
|
239
|
+
/** A dataset the pipeline did not process, and why. */
|
|
240
|
+
export interface DatasetSkippedEvent extends DatasetEventEnvelope<'datasetSkipped'> {
|
|
241
|
+
reason: DatasetSkipReason;
|
|
242
|
+
/** The error that caused a `probeFailed` or `resolveFailed` skip. */
|
|
243
|
+
error?: Error;
|
|
244
|
+
}
|
|
245
|
+
/**
|
|
246
|
+
* Reported when a writer leaves some of a dataset’s items out rather than
|
|
247
|
+
* failing the dataset, through `RunContext.itemsRejected`.
|
|
248
|
+
*/
|
|
249
|
+
export interface ItemsRejectedEvent extends DatasetEventEnvelope<'itemsRejected'> {
|
|
250
|
+
rejection: ItemsRejected;
|
|
251
|
+
}
|
|
252
|
+
/**
|
|
253
|
+
* Reported once per (target, field) that left values out of a dataset’s items
|
|
254
|
+
* (`BatchContext.valueUnread`), just before `datasetComplete`. Not reported for
|
|
255
|
+
* a dataset whose stages or write failed: nothing of it is kept, so neither is
|
|
256
|
+
* what it could not read.
|
|
257
|
+
*/
|
|
258
|
+
export interface ValuesUnreadEvent extends DatasetEventEnvelope<'valuesUnread'> {
|
|
259
|
+
unread: ValuesUnread;
|
|
260
|
+
}
|
|
261
|
+
/** The adaptive query timeout for an endpoint was tightened. */
|
|
262
|
+
export interface TimeoutTightenedEvent extends DatasetEventEnvelope<'timeoutTightened'>, TimeoutTransitionEvent {
|
|
263
|
+
}
|
|
264
|
+
/** The adaptive query timeout for an endpoint was relaxed. */
|
|
265
|
+
export interface TimeoutRelaxedEvent extends DatasetEventEnvelope<'timeoutRelaxed'>, TimeoutTransitionEvent {
|
|
266
|
+
}
|
|
267
|
+
/** Everything a pipeline run reports, discriminated on `type`. */
|
|
268
|
+
export type PipelineEvent = PipelineStartEvent | DatasetsSelectedEvent | SelectionEmptyEvent | PipelineCompleteEvent | DatasetStartEvent | DistributionProbedEvent | DistributionValidatedEvent | ImportStartedEvent | ImportFailedEvent | DistributionSelectedEvent | StageStartEvent | StageProgressEvent | StageCompleteEvent | StageFailedEvent | StageSkippedEvent | DatasetValidatedEvent | DatasetCompleteEvent | DatasetSkippedEvent | ItemsRejectedEvent | ValuesUnreadEvent | TimeoutTightenedEvent | TimeoutRelaxedEvent;
|
|
269
|
+
/** A {@link PipelineEvent} without the envelope fields the pipeline fills in. */
|
|
270
|
+
export type PipelineEventPayload = PipelineEvent extends infer Event ? Event extends PipelineEvent ? Omit<Event, 'defaultLevel' | 'timestamp' | 'runId'> : never : never;
|
|
271
|
+
/** The pipeline’s default {@link Level} for an event. */
|
|
272
|
+
export declare function levelOf(event: PipelineEventPayload): Level;
|
|
273
|
+
//# sourceMappingURL=pipelineEvent.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipelineEvent.d.ts","sourceRoot":"","sources":["../src/pipelineEvent.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACvD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAC;AAExE;;;GAGG;AACH,MAAM,MAAM,KAAK,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;AAExD;;;;GAIG;AACH,MAAM,WAAW,aAAa,CAAC,IAAI,SAAS,MAAM;IAChD,IAAI,EAAE,IAAI,CAAC;IACX,YAAY,EAAE,KAAK,CAAC;IACpB,SAAS,EAAE,IAAI,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,oEAAoE;AACpE,MAAM,WAAW,oBAAoB,CACnC,IAAI,SAAS,MAAM,CACnB,SAAQ,aAAa,CAAC,IAAI,CAAC;IAC3B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,8FAA8F;AAC9F,MAAM,WAAW,MAAM,CAAC,IAAI,SAAS,MAAM;IACzC,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,0CAA0C;AAC1C,MAAM,MAAM,iBAAiB,GAAG,MAAM,CACpC,WAAW,GAAG,aAAa,GAAG,eAAe,GAAG,yBAAyB,CAC1E,CAAC;AAEF;;;;;GAKG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,CAAC,cAAc,GAAG,iBAAiB,CAAC,CAAC;AAEzE,+CAA+C;AAC/C,MAAM,WAAW,0BAA0B;IACzC,IAAI,EAAE,QAAQ,GAAG,WAAW,GAAG,eAAe,CAAC;IAC/C,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B;AAED;;;;GAIG;AACH,MAAM,WAAW,aAAa;IAC5B;qBACiB;IACjB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,6EAA6E;IAC7E,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,sBAAsB;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B;;oEAEgE;IAChE,QAAQ,CAAC,OAAO,EAAE,SAAS,eAAe,EAAE,CAAC;CAC9C;AAED,4EAA4E;AAC5E,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,yDAAyD;IACzD,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;CACtC;AAED;;;;;GAKG;AACH,MAAM,WAAW,YAAY;IAC3B,4EAA4E;IAC5E,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,qEAAqE;IACrE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,0CAA0C;IAC1C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB;4DACwD;IACxD,QAAQ,CAAC,QAAQ,EAAE,SAAS,WAAW,EAAE,CAAC;CAC3C;AAED,0EAA0E;AAC1E,MAAM,WAAW,WAAW;IAC1B;;8DAE0D;IAC1D,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,yCAAyC;IACzC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,4DAA4D;AAC5D,MAAM,WAAW,WAAW;IAC1B,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,oDAAoD;AACpD,MAAM,WAAW,kBAAmB,SAAQ,aAAa,CAAC,eAAe,CAAC;IACxE,IAAI,EAAE,MAAM,CAAC;CACd;AAED,oFAAoF;AACpF,MAAM,WAAW,qBAAsB,SAAQ,aAAa,CAAC,kBAAkB,CAAC;IAC9E,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;GAGG;AACH,MAAM,WAAW,mBAAoB,SAAQ,aAAa,CAAC,gBAAgB,CAAC;IAC1E,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,sBAAsB;AACtB,MAAM,WAAW,qBACf,SAAQ,aAAa,CAAC,kBAAkB,CAAC,EAAE,WAAW;IACtD,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,iDAAiD;AACjD,MAAM,MAAM,iBAAiB,GAAG,oBAAoB,CAAC,cAAc,CAAC,CAAC;AAErE,mDAAmD;AACnD,MAAM,WAAW,uBAAwB,SAAQ,oBAAoB,CAAC,oBAAoB,CAAC;IACzF,YAAY,EAAE,YAAY,CAAC;IAC3B,MAAM,EAAE,0BAA0B,CAAC;CACpC;AAED,0GAA0G;AAC1G,MAAM,WAAW,0BAA2B,SAAQ,oBAAoB,CAAC,uBAAuB,CAAC;IAC/F,YAAY,EAAE,YAAY,CAAC;IAC3B,OAAO,EAAE,eAAe,CAAC;CAC1B;AAED,mDAAmD;AACnD,MAAM,MAAM,kBAAkB,GAAG,oBAAoB,CAAC,eAAe,CAAC,CAAC;AAEvE,uCAAuC;AACvC,MAAM,WAAW,iBAAkB,SAAQ,oBAAoB,CAAC,cAAc,CAAC;IAC7E,YAAY,EAAE,YAAY,CAAC;IAC3B,KAAK,EAAE,KAAK,CAAC;CACd;AAED,0EAA0E;AAC1E,MAAM,WAAW,yBAA0B,SAAQ,oBAAoB,CAAC,sBAAsB,CAAC;IAC7F,YAAY,EAAE,YAAY,CAAC;IAC3B,QAAQ,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,qCAAqC;AACrC,MAAM,WAAW,eAAgB,SAAQ,oBAAoB,CAAC,YAAY,CAAC;IACzE,KAAK,EAAE,MAAM,CAAC;CACf;AAED,yCAAyC;AACzC,MAAM,WAAW,kBACf,SAAQ,oBAAoB,CAAC,eAAe,CAAC,EAAE,WAAW;IAC1D,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,sCAAsC;AACtC,MAAM,WAAW,kBAAmB,SAAQ,oBAAoB,CAAC,eAAe,CAAC;IAC/E,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB;uEACmE;IACnE,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB;iCAC6B;IAC7B,MAAM,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CAC3C;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,WAAW;IAC1B,iDAAiD;IACjD,QAAQ,EAAE,MAAM,CAAC;IACjB,8DAA8D;IAC9D,MAAM,EAAE,MAAM,CAAC;IACf;wEACoE;IACpE,UAAU,EAAE,SAAS,iBAAiB,EAAE,CAAC;IACzC,qCAAqC;IACrC,UAAU,EAAE,MAAM,CAAC;IACnB,qDAAqD;IACrD,SAAS,EAAE,MAAM,CAAC;IAClB,wEAAwE;IACxE,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,8EAA8E;AAC9E,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,WAAW,CAAC;IACpB,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CAC1C;AAED,wEAAwE;AACxE,MAAM,WAAW,iBAAiB;IAChC;oDACgD;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAiB,SAAQ,oBAAoB,CAAC,aAAa,CAAC;IAC3E,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,KAAK,CAAC;CACd;AAED,yCAAyC;AACzC,MAAM,WAAW,iBAAkB,SAAQ,oBAAoB,CAAC,cAAc,CAAC;IAC7E,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,eAAe,CAAC;CACzB;AAED;;;GAGG;AACH,MAAM,WAAW,qBAAsB,SAAQ,oBAAoB,CAAC,kBAAkB,CAAC;IACrF,MAAM,EAAE,gBAAgB,CAAC;CAC1B;AAED,kDAAkD;AAClD,MAAM,WAAW,oBACf,SAAQ,oBAAoB,CAAC,iBAAiB,CAAC,EAAE,WAAW;IAC5D,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,uDAAuD;AACvD,MAAM,WAAW,mBAAoB,SAAQ,oBAAoB,CAAC,gBAAgB,CAAC;IACjF,MAAM,EAAE,iBAAiB,CAAC;IAC1B,qEAAqE;IACrE,KAAK,CAAC,EAAE,KAAK,CAAC;CACf;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAmB,SAAQ,oBAAoB,CAAC,eAAe,CAAC;IAC/E,SAAS,EAAE,aAAa,CAAC;CAC1B;AAED;;;;;GAKG;AACH,MAAM,WAAW,iBAAkB,SAAQ,oBAAoB,CAAC,cAAc,CAAC;IAC7E,MAAM,EAAE,YAAY,CAAC;CACtB;AAED,gEAAgE;AAChE,MAAM,WAAW,qBACf,SAAQ,oBAAoB,CAAC,kBAAkB,CAAC,EAAE,sBAAsB;CAAG;AAE7E,8DAA8D;AAC9D,MAAM,WAAW,mBACf,SAAQ,oBAAoB,CAAC,gBAAgB,CAAC,EAAE,sBAAsB;CAAG;AAE3E,kEAAkE;AAClE,MAAM,MAAM,aAAa,GACrB,kBAAkB,GAClB,qBAAqB,GACrB,mBAAmB,GACnB,qBAAqB,GACrB,iBAAiB,GACjB,uBAAuB,GACvB,0BAA0B,GAC1B,kBAAkB,GAClB,iBAAiB,GACjB,yBAAyB,GACzB,eAAe,GACf,kBAAkB,GAClB,kBAAkB,GAClB,gBAAgB,GAChB,iBAAiB,GACjB,qBAAqB,GACrB,oBAAoB,GACpB,mBAAmB,GACnB,kBAAkB,GAClB,iBAAiB,GACjB,qBAAqB,GACrB,mBAAmB,CAAC;AAExB,iFAAiF;AACjF,MAAM,MAAM,oBAAoB,GAAG,aAAa,SAAS,MAAM,KAAK,GAChE,KAAK,SAAS,aAAa,GACzB,IAAI,CAAC,KAAK,EAAE,cAAc,GAAG,WAAW,GAAG,OAAO,CAAC,GACnD,KAAK,GACP,KAAK,CAAC;AAEV,yDAAyD;AACzD,wBAAgB,OAAO,CAAC,KAAK,EAAE,oBAAoB,GAAG,KAAK,CAiC1D"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/** The pipeline’s default {@link Level} for an event. */
|
|
2
|
+
export function levelOf(event) {
|
|
3
|
+
switch (event.type) {
|
|
4
|
+
case 'stageProgress':
|
|
5
|
+
case 'distributionProbed':
|
|
6
|
+
case 'importStarted':
|
|
7
|
+
case 'stageStart':
|
|
8
|
+
return 'debug';
|
|
9
|
+
case 'pipelineStart':
|
|
10
|
+
case 'pipelineComplete':
|
|
11
|
+
case 'datasetsSelected':
|
|
12
|
+
case 'datasetStart':
|
|
13
|
+
case 'datasetComplete':
|
|
14
|
+
case 'distributionSelected':
|
|
15
|
+
case 'stageComplete':
|
|
16
|
+
case 'timeoutRelaxed':
|
|
17
|
+
return 'info';
|
|
18
|
+
case 'datasetSkipped':
|
|
19
|
+
return event.reason.code === 'unchanged' ? 'info' : 'warn';
|
|
20
|
+
case 'distributionValidated':
|
|
21
|
+
return event.verdict.valid ? 'info' : 'warn';
|
|
22
|
+
case 'datasetValidated':
|
|
23
|
+
return event.report.conforms ? 'info' : 'warn';
|
|
24
|
+
case 'stageSkipped':
|
|
25
|
+
return event.reason.code === 'noItemsSelected' ? 'info' : 'warn';
|
|
26
|
+
case 'selectionEmpty':
|
|
27
|
+
case 'importFailed':
|
|
28
|
+
case 'itemsRejected':
|
|
29
|
+
case 'valuesUnread':
|
|
30
|
+
case 'timeoutTightened':
|
|
31
|
+
return 'warn';
|
|
32
|
+
case 'stageFailed':
|
|
33
|
+
return 'error';
|
|
34
|
+
}
|
|
35
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { PipelineEvent } from './pipelineEvent.js';
|
|
2
|
+
/**
|
|
3
|
+
* Observer of a pipeline run. The pipeline hands every {@link PipelineEvent}
|
|
4
|
+
* to {@link onEvent}, synchronously and in order. Each reporter filters by
|
|
5
|
+
* level itself.
|
|
6
|
+
*
|
|
7
|
+
* A reporter cannot break a run: when {@link onEvent} throws, the pipeline
|
|
8
|
+
* still hands the event to the other reporters, carries on, and surfaces the
|
|
9
|
+
* error as a `ReporterWarning` through `process.emitWarning`.
|
|
10
|
+
*/
|
|
11
|
+
export interface Reporter {
|
|
12
|
+
onEvent(event: PipelineEvent): void;
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=reporter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reporter.d.ts","sourceRoot":"","sources":["../src/reporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAExD;;;;;;;;GAQG;AACH,MAAM,WAAW,QAAQ;IACvB,OAAO,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAAC;CACrC"}
|
package/dist/sparql/reader.d.ts
CHANGED
|
@@ -10,6 +10,15 @@ export declare class NotSupported {
|
|
|
10
10
|
readonly message: string;
|
|
11
11
|
constructor(message: string);
|
|
12
12
|
}
|
|
13
|
+
/**
|
|
14
|
+
* A stage’s item selector selected nothing for the dataset, so the stage does
|
|
15
|
+
* not apply to it – the usual outcome for a stage that only some datasets
|
|
16
|
+
* have items for. A {@link NotSupported}, so callers that only check for that
|
|
17
|
+
* keep treating it as a skip.
|
|
18
|
+
*/
|
|
19
|
+
export declare class NoItemsSelected extends NotSupported {
|
|
20
|
+
constructor();
|
|
21
|
+
}
|
|
13
22
|
/** A single row of variable bindings (variable name → NamedNode). */
|
|
14
23
|
export type VariableBindings = Record<string, NamedNode>;
|
|
15
24
|
export interface ReadOptions {
|
|
@@ -49,7 +58,7 @@ export interface SparqlConstructReaderOptions {
|
|
|
49
58
|
* Optional custom SparqlEndpointFetcher instance.
|
|
50
59
|
*
|
|
51
60
|
* When supplied, the reader uses this fetcher as-is for every attempt
|
|
52
|
-
*
|
|
61
|
+
* – the per-attempt timeout from the {@link TimeoutPolicy} is **not**
|
|
53
62
|
* enforced (the supplied fetcher’s own `timeout` governs). Policy
|
|
54
63
|
* `beforeRequest`/`afterRequest` hooks still fire so outcome
|
|
55
64
|
* classification works, but adaptive tightening cannot apply.
|
|
@@ -77,7 +86,7 @@ export interface SparqlConstructReaderOptions {
|
|
|
77
86
|
/**
|
|
78
87
|
* Deduplicate triples in the CONSTRUCT output stream.
|
|
79
88
|
*
|
|
80
|
-
* SPARQL CONSTRUCT queries can produce duplicate triples
|
|
89
|
+
* SPARQL CONSTRUCT queries can produce duplicate triples – for example,
|
|
81
90
|
* constant triples (like `?dataset a edm:ProvidedCHO`) are emitted for
|
|
82
91
|
* every solution row. When enabled, a streaming identity filter removes
|
|
83
92
|
* duplicates inline without buffering.
|
|
@@ -99,9 +108,9 @@ export interface SparqlConstructReaderOptions {
|
|
|
99
108
|
* and parsed at {@link read} time after substitution.
|
|
100
109
|
*
|
|
101
110
|
* Template substitution (applied in order):
|
|
102
|
-
* 1. `#subjectFilter#`
|
|
103
|
-
* 2. `FROM <graph>`
|
|
104
|
-
* 3. `?dataset`
|
|
111
|
+
* 1. `#subjectFilter#` – replaced with `distribution.subjectFilter` (deferred to read)
|
|
112
|
+
* 2. `FROM <graph>` – set via `withDefaultGraph` if the distribution has a named graph
|
|
113
|
+
* 3. `?dataset` – replaced with the dataset IRI (string substitution on the serialised query)
|
|
105
114
|
*
|
|
106
115
|
* @example
|
|
107
116
|
* ```typescript
|
|
@@ -178,8 +187,8 @@ export declare function readQueryFile(filename: string): Promise<string>;
|
|
|
178
187
|
* **Why this exists:** `fetch-sparql-endpoint` pipes the raw HTTP response
|
|
179
188
|
* stream directly into N3.js's `StreamParser`. N3.js has a bug
|
|
180
189
|
* ({@link https://github.com/rdfjs/N3.js/issues/578 | N3.js#578}) where
|
|
181
|
-
* tokens that straddle chunk boundaries
|
|
182
|
-
* `@nl-nl`
|
|
190
|
+
* tokens that straddle chunk boundaries – most commonly language tags like
|
|
191
|
+
* `@nl-nl` – cause spurious `Unexpected "-nl"` parse errors. The error is
|
|
183
192
|
* non-deterministic and typically surfaces only on responses larger than
|
|
184
193
|
* ~12 MB, because that is when HTTP chunking starts splitting mid-token.
|
|
185
194
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reader.d.ts","sourceRoot":"","sources":["../../src/sparql/reader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAiB,MAAM,cAAc,CAAC;AACpE,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAGpD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAUxC,OAAO,EAGL,KAAK,aAAa,EACnB,MAAM,oBAAoB,CAAC;AAS5B;;GAEG;AACH,qBAAa,YAAY;aACK,OAAO,EAAE,MAAM;gBAAf,OAAO,EAAE,MAAM;CAC5C;AAED,qEAAqE;AACrE,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEzD,MAAM,WAAW,WAAW;IAC1B;;;OAGG;IACH,QAAQ,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAE9B;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,MAAM;IACrB,IAAI,CACF,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;;;;;;;;;OAgBG;IACH,OAAO,CAAC,EAAE,qBAAqB,CAAC;IAEhC;;;;;;;;;OASG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB;;;;;;;;;;;;;;OAcG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,qBAAa,qBAAsB,YAAW,MAAM;IAClD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAiB;IAC5C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAwB;IACrD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAU;IACrC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAU;IACtC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;gBAEjC,OAAO,EAAE,4BAA4B;IAiBjD;;;;;;;;OAQG;IACG,IAAI,CACR,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IA4C/B;;;;;OAKG;YACW,oBAAoB;IA+BlC;;;;;;;OAOG;IACH,OAAO,CAAC,iBAAiB;IAKzB;;;;OAIG;YACW,UAAU;IAmBxB;;;;;OAKG;WACiB,QAAQ,CAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,IAAI,CAAC,4BAA4B,EAAE,OAAO,CAAC,GACpD,OAAO,CAAC,qBAAqB,CAAC;CAIlC;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAErE;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,mBAAoB,SAAQ,SAAS;IAChD,OAAO,CAAC,SAAS,CAAM;IAEd,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,IAAI;IAUjE,MAAM,CAAC,QAAQ,EAAE,MAAM,IAAI;CAMrC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAuB,gBAAgB,CACrC,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GACzB,aAAa,CAAC,IAAI,CAAC,CASrB"}
|
|
1
|
+
{"version":3,"file":"reader.d.ts","sourceRoot":"","sources":["../../src/sparql/reader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAiB,MAAM,cAAc,CAAC;AACpE,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAGpD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAUxC,OAAO,EAGL,KAAK,aAAa,EACnB,MAAM,oBAAoB,CAAC;AAS5B;;GAEG;AACH,qBAAa,YAAY;aACK,OAAO,EAAE,MAAM;gBAAf,OAAO,EAAE,MAAM;CAC5C;AAED;;;;;GAKG;AACH,qBAAa,eAAgB,SAAQ,YAAY;;CAIhD;AAED,qEAAqE;AACrE,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEzD,MAAM,WAAW,WAAW;IAC1B;;;OAGG;IACH,QAAQ,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAE9B;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,MAAM;IACrB,IAAI,CACF,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;;;;;;;;;OAgBG;IACH,OAAO,CAAC,EAAE,qBAAqB,CAAC;IAEhC;;;;;;;;;OASG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB;;;;;;;;;;;;;;OAcG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,qBAAa,qBAAsB,YAAW,MAAM;IAClD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAiB;IAC5C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAwB;IACrD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAU;IACrC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAU;IACtC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;gBAEjC,OAAO,EAAE,4BAA4B;IAiBjD;;;;;;;;OAQG;IACG,IAAI,CACR,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IA4C/B;;;;;OAKG;YACW,oBAAoB;IA+BlC;;;;;;;OAOG;IACH,OAAO,CAAC,iBAAiB;IAKzB;;;;OAIG;YACW,UAAU;IAmBxB;;;;;OAKG;WACiB,QAAQ,CAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,IAAI,CAAC,4BAA4B,EAAE,OAAO,CAAC,GACpD,OAAO,CAAC,qBAAqB,CAAC;CAIlC;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAErE;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,mBAAoB,SAAQ,SAAS;IAChD,OAAO,CAAC,SAAS,CAAM;IAEd,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,IAAI;IAUjE,MAAM,CAAC,QAAQ,EAAE,MAAM,IAAI;CAMrC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAuB,gBAAgB,CACrC,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GACzB,aAAa,CAAC,IAAI,CAAC,CASrB"}
|
package/dist/sparql/reader.js
CHANGED
|
@@ -27,6 +27,17 @@ export class NotSupported {
|
|
|
27
27
|
this.message = message;
|
|
28
28
|
}
|
|
29
29
|
}
|
|
30
|
+
/**
|
|
31
|
+
* A stage’s item selector selected nothing for the dataset, so the stage does
|
|
32
|
+
* not apply to it – the usual outcome for a stage that only some datasets
|
|
33
|
+
* have items for. A {@link NotSupported}, so callers that only check for that
|
|
34
|
+
* keep treating it as a skip.
|
|
35
|
+
*/
|
|
36
|
+
export class NoItemsSelected extends NotSupported {
|
|
37
|
+
constructor() {
|
|
38
|
+
super('No items selected');
|
|
39
|
+
}
|
|
40
|
+
}
|
|
30
41
|
/**
|
|
31
42
|
* A streaming SPARQL CONSTRUCT reader.
|
|
32
43
|
*
|
|
@@ -35,9 +46,9 @@ export class NotSupported {
|
|
|
35
46
|
* and parsed at {@link read} time after substitution.
|
|
36
47
|
*
|
|
37
48
|
* Template substitution (applied in order):
|
|
38
|
-
* 1. `#subjectFilter#`
|
|
39
|
-
* 2. `FROM <graph>`
|
|
40
|
-
* 3. `?dataset`
|
|
49
|
+
* 1. `#subjectFilter#` – replaced with `distribution.subjectFilter` (deferred to read)
|
|
50
|
+
* 2. `FROM <graph>` – set via `withDefaultGraph` if the distribution has a named graph
|
|
51
|
+
* 3. `?dataset` – replaced with the dataset IRI (string substitution on the serialised query)
|
|
41
52
|
*
|
|
42
53
|
* @example
|
|
43
54
|
* ```typescript
|
|
@@ -196,8 +207,8 @@ export async function readQueryFile(filename) {
|
|
|
196
207
|
* **Why this exists:** `fetch-sparql-endpoint` pipes the raw HTTP response
|
|
197
208
|
* stream directly into N3.js's `StreamParser`. N3.js has a bug
|
|
198
209
|
* ({@link https://github.com/rdfjs/N3.js/issues/578 | N3.js#578}) where
|
|
199
|
-
* tokens that straddle chunk boundaries
|
|
200
|
-
* `@nl-nl`
|
|
210
|
+
* tokens that straddle chunk boundaries – most commonly language tags like
|
|
211
|
+
* `@nl-nl` – cause spurious `Unexpected "-nl"` parse errors. The error is
|
|
201
212
|
* non-deterministic and typically surfaces only on responses larger than
|
|
202
213
|
* ~12 MB, because that is when HTTP chunking starts splitting mid-token.
|
|
203
214
|
*
|
package/dist/stage.d.ts
CHANGED
|
@@ -5,7 +5,7 @@ import { NotSupported } from './sparql/reader.js';
|
|
|
5
5
|
import type { TimeoutPolicy } from './sparql/timeoutPolicy.js';
|
|
6
6
|
import type { Validator } from './validator.js';
|
|
7
7
|
import type { DatasetWriter } from './writer/writer.js';
|
|
8
|
-
import type { UnreadValue } from './
|
|
8
|
+
import type { StageSummary, UnreadValue } from './pipelineEvent.js';
|
|
9
9
|
/**
|
|
10
10
|
* Transforms a quad stream, given the context of its extension point.
|
|
11
11
|
*
|
|
@@ -26,6 +26,13 @@ export interface ReaderContext {
|
|
|
26
26
|
dataset: Dataset;
|
|
27
27
|
distribution: Distribution;
|
|
28
28
|
stage: string;
|
|
29
|
+
/**
|
|
30
|
+
* Add `amount` (1 when left out) to the count called `name`. The pipeline
|
|
31
|
+
* adds up each name over the stage and reports the totals when it completes
|
|
32
|
+
* (`stageComplete`’s `counts`) – how many images were fetched and how many
|
|
33
|
+
* came from a store, say.
|
|
34
|
+
*/
|
|
35
|
+
count(name: string, amount?: number): void;
|
|
29
36
|
}
|
|
30
37
|
/**
|
|
31
38
|
* The pipeline's one type-changing seam: turn a root-complete batch of quads
|
|
@@ -47,7 +54,7 @@ export interface BatchContext extends ReaderContext {
|
|
|
47
54
|
* Report a value the transform could not read and so left out of the item
|
|
48
55
|
* it produced – an integer field over `onbekend`, say. The pipeline tallies
|
|
49
56
|
* them per (`target`, `field`) and reports them when the dataset completes
|
|
50
|
-
* (
|
|
57
|
+
* (the `valuesUnread` event).
|
|
51
58
|
*/
|
|
52
59
|
valueUnread(target: string, field: string, unread: UnreadValue): void;
|
|
53
60
|
}
|
|
@@ -177,6 +184,9 @@ export interface RunOptions {
|
|
|
177
184
|
/** Receives what a {@link BatchTransform} reports through
|
|
178
185
|
* {@link BatchContext.valueUnread}; dropped when absent. */
|
|
179
186
|
valueUnread?: BatchContext['valueUnread'];
|
|
187
|
+
/** Receives where the stage spent its time and what it counted, once, when
|
|
188
|
+
* it completes. Not called for a stage that is not supported or that fails. */
|
|
189
|
+
onComplete?: (summary: StageSummary) => void;
|
|
180
190
|
}
|
|
181
191
|
/** Options accepted by {@link ItemSelector.select}. */
|
|
182
192
|
export interface SelectOptions {
|
|
@@ -217,12 +227,14 @@ export declare class Stage<Out = Quad> {
|
|
|
217
227
|
private validateBuffer;
|
|
218
228
|
private readAll;
|
|
219
229
|
/**
|
|
220
|
-
*
|
|
221
|
-
* supplying the {@link ReaderContext}
|
|
222
|
-
* call's output (see {@link AttachedReader});
|
|
223
|
-
*
|
|
230
|
+
* Run one reader and fold its attached transforms over its output, in order,
|
|
231
|
+
* supplying the {@link ReaderContext}, each layer timed by `tally`. A
|
|
232
|
+
* transform sees one `read()` call's output (see {@link AttachedReader});
|
|
233
|
+
* `NotSupported` is returned as is and never reaches a transform.
|
|
224
234
|
*/
|
|
225
|
-
private
|
|
235
|
+
private read;
|
|
236
|
+
/** Every attached transform’s name, readers in order, for the phase tally. */
|
|
237
|
+
private get transformNames();
|
|
226
238
|
}
|
|
227
239
|
/** Selects items (as variable bindings) for readers to process. Pagination is an implementation detail. */
|
|
228
240
|
export interface ItemSelector {
|
package/dist/stage.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,
|
|
1
|
+
{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,EAAmB,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE/D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAExD,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAGpE;;;;;;;GAOG;AACH,MAAM,MAAM,aAAa,CAAC,GAAG,IAAI,CAC/B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,EAC1B,OAAO,EAAE,GAAG,KACT,aAAa,CAAC,IAAI,CAAC,CAAC;AAEzB;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,YAAY,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd;;;;;OAKG;IACH,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5C;AAED;;;;;;GAMG;AACH,MAAM,MAAM,cAAc,CAAC,GAAG,IAAI,CAChC,KAAK,EAAE,SAAS,IAAI,EAAE,EACtB,OAAO,EAAE,YAAY,KAClB,QAAQ,CAAC,GAAG,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;AAExC;;;;;GAKG;AACH,MAAM,WAAW,YAAa,SAAQ,aAAa;IACjD,QAAQ,EAAE,SAAS,gBAAgB,EAAE,CAAC;IACtC;;;;;OAKG;IACH,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC;CACvE;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,aAAa,CAAC,aAAa,CAAC,GAAG,aAAa,CAAC,aAAa,CAAC,EAAE,CAAC;CAC3E;AAED,yEAAyE;AACzE,MAAM,MAAM,YAAY,GACtB,MAAM,GAAG,cAAc,GAAG,CAAC,MAAM,GAAG,cAAc,CAAC,EAAE,CAAC;AAUxD,MAAM,WAAW,YAAY,CAAC,GAAG,GAAG,IAAI;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,YAAY,CAAC;IACtB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC;IAC9B;;;;;;;OAOG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;;;OAOG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,YAAY,KAAK,YAAY,CAAC;IAC3E,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;IACjB;;;;;;;;;;;;;;;OAeG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,mFAAmF;IACnF,UAAU,CAAC,EAAE;QACX,SAAS,EAAE,SAAS,CAAC;QACrB,iEAAiE;QACjE,SAAS,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;KACvC,CAAC;CACH;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,CAAC,cAAc,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;IACtE;;;;;OAKG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB;iEAC6D;IAC7D,WAAW,CAAC,EAAE,YAAY,CAAC,aAAa,CAAC,CAAC;IAC1C;oFACgF;IAChF,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,IAAI,CAAC;CAC9C;AAED,uDAAuD;AACvD,MAAM,WAAW,aAAa;IAC5B,+BAA+B;IAC/B,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,qBAAa,KAAK,CAAC,GAAG,GAAG,IAAI;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,oGAAoG;IACpG,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC;qEACiE;IACjE,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;IAC7C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAkC;IAC9D,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAiC;gBAEhD,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC;IAmCtC,mDAAmD;IACnD,IAAI,SAAS,IAAI,SAAS,GAAG,SAAS,CAErC;IAEK,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,YAAY,EACtB,MAAM,EAAE,aAAa,CAAC,GAAG,CAAC,EAC1B,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;IAiG/B;;;;OAIG;IACH,OAAO,CAAC,cAAc;YAMR,eAAe;IAyL7B;;;OAGG;YACW,cAAc;YAqBd,OAAO;IA0BrB;;;;;OAKG;YACW,IAAI;IA2BlB,8EAA8E;IAC9E,OAAO,KAAK,cAAc,GAOzB;CACF;AA0DD,2GAA2G;AAC3G,MAAM,WAAW,YAAY;IAC3B,MAAM,CACJ,YAAY,EAAE,YAAY,EAC1B,SAAS,CAAC,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,aAAa,GACtB,aAAa,CAAC,gBAAgB,CAAC,CAAC;CACpC"}
|