@lde/pipeline 0.43.0 → 0.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/distribution/importResolver.js +4 -4
- package/dist/distribution/resolver.d.ts +2 -2
- package/dist/distribution/resolver.d.ts.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/pipeline.d.ts +6 -9
- package/dist/pipeline.d.ts.map +1 -1
- package/dist/pipeline.js +168 -71
- package/dist/pipelineEvent.d.ts +231 -0
- package/dist/pipelineEvent.d.ts.map +1 -0
- package/dist/pipelineEvent.js +35 -0
- package/dist/reporter.d.ts +14 -0
- package/dist/reporter.d.ts.map +1 -0
- package/dist/sparql/reader.d.ts +16 -7
- package/dist/sparql/reader.d.ts.map +1 -1
- package/dist/sparql/reader.js +16 -5
- package/dist/stage.d.ts +2 -2
- package/dist/stage.d.ts.map +1 -1
- package/dist/stage.js +2 -2
- package/dist/unreadValueTally.d.ts +2 -2
- package/dist/unreadValueTally.d.ts.map +1 -1
- package/dist/unreadValueTally.js +1 -1
- package/dist/writer/writer.d.ts +1 -1
- package/dist/writer/writer.d.ts.map +1 -1
- package/package.json +1 -1
- package/dist/combineReporters.d.ts +0 -16
- package/dist/combineReporters.d.ts.map +0 -1
- package/dist/combineReporters.js +0 -57
- package/dist/progressReporter.d.ts +0 -157
- package/dist/progressReporter.d.ts.map +0 -1
- /package/dist/{progressReporter.js → reporter.js} +0 -0
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
import type { Dataset, Distribution } from '@lde/dataset';
|
|
2
|
+
import type { ValidityVerdict } from '@lde/distribution-health';
|
|
3
|
+
import type { ValidationReport } from './validator.js';
|
|
4
|
+
import type { ImportDetails } from './distribution/resolver.js';
|
|
5
|
+
import type { TimeoutTransitionEvent } from './sparql/timeoutPolicy.js';
|
|
6
|
+
/**
|
|
7
|
+
* Deliberately without `fatal`: the pipeline never ends the process; an
|
|
8
|
+
* unrecoverable failure rejects {@link Pipeline.run}.
|
|
9
|
+
*/
|
|
10
|
+
export type Level = 'debug' | 'info' | 'warn' | 'error';
|
|
11
|
+
/**
|
|
12
|
+
* Fields every {@link PipelineEvent} carries. Owning the `type` discriminator
|
|
13
|
+
* here means no event can join the union without one. `defaultLevel` is the
|
|
14
|
+
* pipeline’s default; reporters may remap it.
|
|
15
|
+
*/
|
|
16
|
+
export interface EventEnvelope<Type extends string> {
|
|
17
|
+
type: Type;
|
|
18
|
+
defaultLevel: Level;
|
|
19
|
+
timestamp: Date;
|
|
20
|
+
runId: string;
|
|
21
|
+
}
|
|
22
|
+
/** The envelope of every event that belongs to a single dataset. */
|
|
23
|
+
export interface DatasetEventEnvelope<Type extends string> extends EventEnvelope<Type> {
|
|
24
|
+
dataset: Dataset;
|
|
25
|
+
}
|
|
26
|
+
/** Why something was skipped: a stable `code` to branch on and a human-readable `message`. */
|
|
27
|
+
export interface Reason<Code extends string> {
|
|
28
|
+
code: Code;
|
|
29
|
+
message: string;
|
|
30
|
+
}
|
|
31
|
+
/** Why the pipeline skipped a dataset. */
|
|
32
|
+
export type DatasetSkipReason = Reason<'unchanged' | 'probeFailed' | 'resolveFailed' | 'noDistributionAvailable'>;
|
|
33
|
+
/**
|
|
34
|
+
* Why the pipeline skipped a stage for a dataset: `noItemsSelected` when the
|
|
35
|
+
* stage’s item selector selected nothing, so the stage does not apply to the
|
|
36
|
+
* dataset; `notSupported` when it could not run against the dataset’s
|
|
37
|
+
* distribution.
|
|
38
|
+
*/
|
|
39
|
+
export type StageSkipReason = Reason<'notSupported' | 'noItemsSelected'>;
|
|
40
|
+
/** The outcome of probing one distribution. */
|
|
41
|
+
export interface DistributionAnalysisResult {
|
|
42
|
+
type: 'sparql' | 'data-dump' | 'network-error';
|
|
43
|
+
available: boolean;
|
|
44
|
+
statusCode?: number;
|
|
45
|
+
error?: string;
|
|
46
|
+
warnings: string[];
|
|
47
|
+
/**
|
|
48
|
+
* The source-change fingerprint observed for this distribution, or `null`
|
|
49
|
+
* when none could be established (e.g. a live SPARQL endpoint). The shared
|
|
50
|
+
* key against which a validity verdict's `validatedFingerprint` is matched,
|
|
51
|
+
* so a consumer can record it on the reachability rail. Always populated by
|
|
52
|
+
* the pipeline; optional so consumers constructing the result need not set it.
|
|
53
|
+
*/
|
|
54
|
+
fingerprint?: string | null;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Items a writer left out of a dataset’s output instead of failing the
|
|
58
|
+
* dataset – documents a destination refused, say – reported once per
|
|
59
|
+
* (dataset, target) as the writer finalizes the dataset.
|
|
60
|
+
*/
|
|
61
|
+
export interface ItemsRejected {
|
|
62
|
+
/** The part of the destination that refused them, e.g. a search index’s
|
|
63
|
+
* type name. */
|
|
64
|
+
readonly target: string;
|
|
65
|
+
/** Items the dataset wrote to {@link target}, the rejected ones included. */
|
|
66
|
+
readonly written: number;
|
|
67
|
+
/** Items left out. */
|
|
68
|
+
readonly rejected: number;
|
|
69
|
+
/** Why, most frequent first: each distinct reason, how many items it
|
|
70
|
+
* rejected, and the ids of a few of them. Bounded by the writer, so a
|
|
71
|
+
* reason that varies per item cannot grow it without limit. */
|
|
72
|
+
readonly reasons: readonly RejectionReason[];
|
|
73
|
+
}
|
|
74
|
+
/** One reason a writer rejected items, in {@link ItemsRejected.reasons}. */
|
|
75
|
+
export interface RejectionReason {
|
|
76
|
+
readonly reason: string;
|
|
77
|
+
readonly count: number;
|
|
78
|
+
/** Ids of some of the items rejected for this reason. */
|
|
79
|
+
readonly examples: readonly string[];
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Values a stage read from a dataset but could not interpret – an integer
|
|
83
|
+
* field over `onbekend`, a date over `Z.j` – and so left out of the items it
|
|
84
|
+
* produced, reported once per (dataset, target, field) when the dataset
|
|
85
|
+
* completes. The item itself is still produced; only the value is missing.
|
|
86
|
+
*/
|
|
87
|
+
export interface ValuesUnread {
|
|
88
|
+
/** The kind of item the values were read for, e.g. a search type’s name. */
|
|
89
|
+
readonly target: string;
|
|
90
|
+
/** The field that could not read them; dotted for a nested field. */
|
|
91
|
+
readonly field: string;
|
|
92
|
+
/** How many values the field left out. */
|
|
93
|
+
readonly unread: number;
|
|
94
|
+
/** A few of them, each a distinct value. Bounded, so a field that rejects
|
|
95
|
+
* every value cannot grow the report with the data. */
|
|
96
|
+
readonly examples: readonly UnreadValue[];
|
|
97
|
+
}
|
|
98
|
+
/** One value a stage could not read, in {@link ValuesUnread.examples}. */
|
|
99
|
+
export interface UnreadValue {
|
|
100
|
+
/** What the value was read from, as the stage names it – for a search
|
|
101
|
+
* document, the node it was projected from, which a publisher can look up
|
|
102
|
+
* even where the document is keyed on something else. */
|
|
103
|
+
readonly id: string;
|
|
104
|
+
/** The value as the source stated it. */
|
|
105
|
+
readonly value: string;
|
|
106
|
+
}
|
|
107
|
+
/** The process’s memory use when the event was reported. */
|
|
108
|
+
export interface MemoryUsage {
|
|
109
|
+
memoryUsageBytes: number;
|
|
110
|
+
heapUsedBytes: number;
|
|
111
|
+
}
|
|
112
|
+
/** A run started; `name` is the pipeline’s name. */
|
|
113
|
+
export interface PipelineStartEvent extends EventEnvelope<'pipelineStart'> {
|
|
114
|
+
name: string;
|
|
115
|
+
}
|
|
116
|
+
/** The datasets to process were selected: how many, and how long selecting took. */
|
|
117
|
+
export interface DatasetsSelectedEvent extends EventEnvelope<'datasetsSelected'> {
|
|
118
|
+
count: number;
|
|
119
|
+
durationMs: number;
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* The run was abandoned because the selection was empty: the writers were told
|
|
123
|
+
* to abort, so the destination is unchanged.
|
|
124
|
+
*/
|
|
125
|
+
export interface SelectionEmptyEvent extends EventEnvelope<'selectionEmpty'> {
|
|
126
|
+
message: string;
|
|
127
|
+
}
|
|
128
|
+
/** A run finished. */
|
|
129
|
+
export interface PipelineCompleteEvent extends EventEnvelope<'pipelineComplete'>, MemoryUsage {
|
|
130
|
+
durationMs: number;
|
|
131
|
+
}
|
|
132
|
+
/** The pipeline started processing a dataset. */
|
|
133
|
+
export type DatasetStartEvent = DatasetEventEnvelope<'datasetStart'>;
|
|
134
|
+
/** One of a dataset’s distributions was probed. */
|
|
135
|
+
export interface DistributionProbedEvent extends DatasetEventEnvelope<'distributionProbed'> {
|
|
136
|
+
distribution: Distribution;
|
|
137
|
+
result: DistributionAnalysisResult;
|
|
138
|
+
}
|
|
139
|
+
/** Reported for every distribution the pipeline attempted, even when the dataset is otherwise skipped. */
|
|
140
|
+
export interface DistributionValidatedEvent extends DatasetEventEnvelope<'distributionValidated'> {
|
|
141
|
+
distribution: Distribution;
|
|
142
|
+
verdict: ValidityVerdict;
|
|
143
|
+
}
|
|
144
|
+
/** A data dump started importing for a dataset. */
|
|
145
|
+
export type ImportStartedEvent = DatasetEventEnvelope<'importStarted'>;
|
|
146
|
+
/** Importing a distribution failed. */
|
|
147
|
+
export interface ImportFailedEvent extends DatasetEventEnvelope<'importFailed'> {
|
|
148
|
+
distribution: Distribution;
|
|
149
|
+
error: Error;
|
|
150
|
+
}
|
|
151
|
+
/** `imported` is set when `distribution` serves an imported data dump. */
|
|
152
|
+
export interface DistributionSelectedEvent extends DatasetEventEnvelope<'distributionSelected'> {
|
|
153
|
+
distribution: Distribution;
|
|
154
|
+
imported?: ImportDetails;
|
|
155
|
+
}
|
|
156
|
+
/** A stage started for a dataset. */
|
|
157
|
+
export interface StageStartEvent extends DatasetEventEnvelope<'stageStart'> {
|
|
158
|
+
stage: string;
|
|
159
|
+
}
|
|
160
|
+
/** A running stage’s progress so far. */
|
|
161
|
+
export interface StageProgressEvent extends DatasetEventEnvelope<'stageProgress'>, MemoryUsage {
|
|
162
|
+
stage: string;
|
|
163
|
+
itemsProcessed: number;
|
|
164
|
+
quadsGenerated: number;
|
|
165
|
+
}
|
|
166
|
+
/** A stage finished for a dataset. */
|
|
167
|
+
export interface StageCompleteEvent extends DatasetEventEnvelope<'stageComplete'> {
|
|
168
|
+
stage: string;
|
|
169
|
+
itemsProcessed: number;
|
|
170
|
+
quadsGenerated: number;
|
|
171
|
+
durationMs: number;
|
|
172
|
+
}
|
|
173
|
+
/**
|
|
174
|
+
* Also reported for per-dataset failures outside any stage, with `stage` set
|
|
175
|
+
* to `write`, `provenance` or `reactive-dump-fallback`.
|
|
176
|
+
*/
|
|
177
|
+
export interface StageFailedEvent extends DatasetEventEnvelope<'stageFailed'> {
|
|
178
|
+
stage: string;
|
|
179
|
+
error: Error;
|
|
180
|
+
}
|
|
181
|
+
/** A stage was skipped for a dataset. */
|
|
182
|
+
export interface StageSkippedEvent extends DatasetEventEnvelope<'stageSkipped'> {
|
|
183
|
+
stage: string;
|
|
184
|
+
reason: StageSkipReason;
|
|
185
|
+
}
|
|
186
|
+
/**
|
|
187
|
+
* Reported once per (dataset, validator) pair, even when nothing was validated:
|
|
188
|
+
* read `quadsValidated` to tell ‘not tested’ from ‘tested and passed’.
|
|
189
|
+
*/
|
|
190
|
+
export interface DatasetValidatedEvent extends DatasetEventEnvelope<'datasetValidated'> {
|
|
191
|
+
report: ValidationReport;
|
|
192
|
+
}
|
|
193
|
+
/** The pipeline finished processing a dataset. */
|
|
194
|
+
export interface DatasetCompleteEvent extends DatasetEventEnvelope<'datasetComplete'>, MemoryUsage {
|
|
195
|
+
durationMs: number;
|
|
196
|
+
}
|
|
197
|
+
/** A dataset the pipeline did not process, and why. */
|
|
198
|
+
export interface DatasetSkippedEvent extends DatasetEventEnvelope<'datasetSkipped'> {
|
|
199
|
+
reason: DatasetSkipReason;
|
|
200
|
+
/** The error that caused a `probeFailed` or `resolveFailed` skip. */
|
|
201
|
+
error?: Error;
|
|
202
|
+
}
|
|
203
|
+
/**
|
|
204
|
+
* Reported when a writer leaves some of a dataset’s items out rather than
|
|
205
|
+
* failing the dataset, through `RunContext.itemsRejected`.
|
|
206
|
+
*/
|
|
207
|
+
export interface ItemsRejectedEvent extends DatasetEventEnvelope<'itemsRejected'> {
|
|
208
|
+
rejection: ItemsRejected;
|
|
209
|
+
}
|
|
210
|
+
/**
|
|
211
|
+
* Reported once per (target, field) that left values out of a dataset’s items
|
|
212
|
+
* (`BatchContext.valueUnread`), just before `datasetComplete`. Not reported for
|
|
213
|
+
* a dataset whose stages or write failed: nothing of it is kept, so neither is
|
|
214
|
+
* what it could not read.
|
|
215
|
+
*/
|
|
216
|
+
export interface ValuesUnreadEvent extends DatasetEventEnvelope<'valuesUnread'> {
|
|
217
|
+
unread: ValuesUnread;
|
|
218
|
+
}
|
|
219
|
+
/** The adaptive query timeout for an endpoint was tightened. */
|
|
220
|
+
export interface TimeoutTightenedEvent extends DatasetEventEnvelope<'timeoutTightened'>, TimeoutTransitionEvent {
|
|
221
|
+
}
|
|
222
|
+
/** The adaptive query timeout for an endpoint was relaxed. */
|
|
223
|
+
export interface TimeoutRelaxedEvent extends DatasetEventEnvelope<'timeoutRelaxed'>, TimeoutTransitionEvent {
|
|
224
|
+
}
|
|
225
|
+
/** Everything a pipeline run reports, discriminated on `type`. */
|
|
226
|
+
export type PipelineEvent = PipelineStartEvent | DatasetsSelectedEvent | SelectionEmptyEvent | PipelineCompleteEvent | DatasetStartEvent | DistributionProbedEvent | DistributionValidatedEvent | ImportStartedEvent | ImportFailedEvent | DistributionSelectedEvent | StageStartEvent | StageProgressEvent | StageCompleteEvent | StageFailedEvent | StageSkippedEvent | DatasetValidatedEvent | DatasetCompleteEvent | DatasetSkippedEvent | ItemsRejectedEvent | ValuesUnreadEvent | TimeoutTightenedEvent | TimeoutRelaxedEvent;
|
|
227
|
+
/** A {@link PipelineEvent} without the envelope fields the pipeline fills in. */
|
|
228
|
+
export type PipelineEventPayload = PipelineEvent extends infer Event ? Event extends PipelineEvent ? Omit<Event, 'defaultLevel' | 'timestamp' | 'runId'> : never : never;
|
|
229
|
+
/** The pipeline’s default {@link Level} for an event. */
|
|
230
|
+
export declare function levelOf(event: PipelineEventPayload): Level;
|
|
231
|
+
//# sourceMappingURL=pipelineEvent.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipelineEvent.d.ts","sourceRoot":"","sources":["../src/pipelineEvent.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACvD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAC;AAExE;;;GAGG;AACH,MAAM,MAAM,KAAK,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;AAExD;;;;GAIG;AACH,MAAM,WAAW,aAAa,CAAC,IAAI,SAAS,MAAM;IAChD,IAAI,EAAE,IAAI,CAAC;IACX,YAAY,EAAE,KAAK,CAAC;IACpB,SAAS,EAAE,IAAI,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,oEAAoE;AACpE,MAAM,WAAW,oBAAoB,CACnC,IAAI,SAAS,MAAM,CACnB,SAAQ,aAAa,CAAC,IAAI,CAAC;IAC3B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,8FAA8F;AAC9F,MAAM,WAAW,MAAM,CAAC,IAAI,SAAS,MAAM;IACzC,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,0CAA0C;AAC1C,MAAM,MAAM,iBAAiB,GAAG,MAAM,CACpC,WAAW,GAAG,aAAa,GAAG,eAAe,GAAG,yBAAyB,CAC1E,CAAC;AAEF;;;;;GAKG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,CAAC,cAAc,GAAG,iBAAiB,CAAC,CAAC;AAEzE,+CAA+C;AAC/C,MAAM,WAAW,0BAA0B;IACzC,IAAI,EAAE,QAAQ,GAAG,WAAW,GAAG,eAAe,CAAC;IAC/C,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B;AAED;;;;GAIG;AACH,MAAM,WAAW,aAAa;IAC5B;qBACiB;IACjB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,6EAA6E;IAC7E,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,sBAAsB;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B;;oEAEgE;IAChE,QAAQ,CAAC,OAAO,EAAE,SAAS,eAAe,EAAE,CAAC;CAC9C;AAED,4EAA4E;AAC5E,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,yDAAyD;IACzD,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;CACtC;AAED;;;;;GAKG;AACH,MAAM,WAAW,YAAY;IAC3B,4EAA4E;IAC5E,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,qEAAqE;IACrE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,0CAA0C;IAC1C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB;4DACwD;IACxD,QAAQ,CAAC,QAAQ,EAAE,SAAS,WAAW,EAAE,CAAC;CAC3C;AAED,0EAA0E;AAC1E,MAAM,WAAW,WAAW;IAC1B;;8DAE0D;IAC1D,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,yCAAyC;IACzC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,4DAA4D;AAC5D,MAAM,WAAW,WAAW;IAC1B,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,oDAAoD;AACpD,MAAM,WAAW,kBAAmB,SAAQ,aAAa,CAAC,eAAe,CAAC;IACxE,IAAI,EAAE,MAAM,CAAC;CACd;AAED,oFAAoF;AACpF,MAAM,WAAW,qBAAsB,SAAQ,aAAa,CAAC,kBAAkB,CAAC;IAC9E,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;GAGG;AACH,MAAM,WAAW,mBAAoB,SAAQ,aAAa,CAAC,gBAAgB,CAAC;IAC1E,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,sBAAsB;AACtB,MAAM,WAAW,qBACf,SAAQ,aAAa,CAAC,kBAAkB,CAAC,EAAE,WAAW;IACtD,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,iDAAiD;AACjD,MAAM,MAAM,iBAAiB,GAAG,oBAAoB,CAAC,cAAc,CAAC,CAAC;AAErE,mDAAmD;AACnD,MAAM,WAAW,uBAAwB,SAAQ,oBAAoB,CAAC,oBAAoB,CAAC;IACzF,YAAY,EAAE,YAAY,CAAC;IAC3B,MAAM,EAAE,0BAA0B,CAAC;CACpC;AAED,0GAA0G;AAC1G,MAAM,WAAW,0BAA2B,SAAQ,oBAAoB,CAAC,uBAAuB,CAAC;IAC/F,YAAY,EAAE,YAAY,CAAC;IAC3B,OAAO,EAAE,eAAe,CAAC;CAC1B;AAED,mDAAmD;AACnD,MAAM,MAAM,kBAAkB,GAAG,oBAAoB,CAAC,eAAe,CAAC,CAAC;AAEvE,uCAAuC;AACvC,MAAM,WAAW,iBAAkB,SAAQ,oBAAoB,CAAC,cAAc,CAAC;IAC7E,YAAY,EAAE,YAAY,CAAC;IAC3B,KAAK,EAAE,KAAK,CAAC;CACd;AAED,0EAA0E;AAC1E,MAAM,WAAW,yBAA0B,SAAQ,oBAAoB,CAAC,sBAAsB,CAAC;IAC7F,YAAY,EAAE,YAAY,CAAC;IAC3B,QAAQ,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,qCAAqC;AACrC,MAAM,WAAW,eAAgB,SAAQ,oBAAoB,CAAC,YAAY,CAAC;IACzE,KAAK,EAAE,MAAM,CAAC;CACf;AAED,yCAAyC;AACzC,MAAM,WAAW,kBACf,SAAQ,oBAAoB,CAAC,eAAe,CAAC,EAAE,WAAW;IAC1D,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,sCAAsC;AACtC,MAAM,WAAW,kBAAmB,SAAQ,oBAAoB,CAAC,eAAe,CAAC;IAC/E,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAiB,SAAQ,oBAAoB,CAAC,aAAa,CAAC;IAC3E,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,KAAK,CAAC;CACd;AAED,yCAAyC;AACzC,MAAM,WAAW,iBAAkB,SAAQ,oBAAoB,CAAC,cAAc,CAAC;IAC7E,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,eAAe,CAAC;CACzB;AAED;;;GAGG;AACH,MAAM,WAAW,qBAAsB,SAAQ,oBAAoB,CAAC,kBAAkB,CAAC;IACrF,MAAM,EAAE,gBAAgB,CAAC;CAC1B;AAED,kDAAkD;AAClD,MAAM,WAAW,oBACf,SAAQ,oBAAoB,CAAC,iBAAiB,CAAC,EAAE,WAAW;IAC5D,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,uDAAuD;AACvD,MAAM,WAAW,mBAAoB,SAAQ,oBAAoB,CAAC,gBAAgB,CAAC;IACjF,MAAM,EAAE,iBAAiB,CAAC;IAC1B,qEAAqE;IACrE,KAAK,CAAC,EAAE,KAAK,CAAC;CACf;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAmB,SAAQ,oBAAoB,CAAC,eAAe,CAAC;IAC/E,SAAS,EAAE,aAAa,CAAC;CAC1B;AAED;;;;;GAKG;AACH,MAAM,WAAW,iBAAkB,SAAQ,oBAAoB,CAAC,cAAc,CAAC;IAC7E,MAAM,EAAE,YAAY,CAAC;CACtB;AAED,gEAAgE;AAChE,MAAM,WAAW,qBACf,SAAQ,oBAAoB,CAAC,kBAAkB,CAAC,EAAE,sBAAsB;CAAG;AAE7E,8DAA8D;AAC9D,MAAM,WAAW,mBACf,SAAQ,oBAAoB,CAAC,gBAAgB,CAAC,EAAE,sBAAsB;CAAG;AAE3E,kEAAkE;AAClE,MAAM,MAAM,aAAa,GACrB,kBAAkB,GAClB,qBAAqB,GACrB,mBAAmB,GACnB,qBAAqB,GACrB,iBAAiB,GACjB,uBAAuB,GACvB,0BAA0B,GAC1B,kBAAkB,GAClB,iBAAiB,GACjB,yBAAyB,GACzB,eAAe,GACf,kBAAkB,GAClB,kBAAkB,GAClB,gBAAgB,GAChB,iBAAiB,GACjB,qBAAqB,GACrB,oBAAoB,GACpB,mBAAmB,GACnB,kBAAkB,GAClB,iBAAiB,GACjB,qBAAqB,GACrB,mBAAmB,CAAC;AAExB,iFAAiF;AACjF,MAAM,MAAM,oBAAoB,GAAG,aAAa,SAAS,MAAM,KAAK,GAChE,KAAK,SAAS,aAAa,GACzB,IAAI,CAAC,KAAK,EAAE,cAAc,GAAG,WAAW,GAAG,OAAO,CAAC,GACnD,KAAK,GACP,KAAK,CAAC;AAEV,yDAAyD;AACzD,wBAAgB,OAAO,CAAC,KAAK,EAAE,oBAAoB,GAAG,KAAK,CAiC1D"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/** The pipeline’s default {@link Level} for an event. */
|
|
2
|
+
export function levelOf(event) {
|
|
3
|
+
switch (event.type) {
|
|
4
|
+
case 'stageProgress':
|
|
5
|
+
case 'distributionProbed':
|
|
6
|
+
case 'importStarted':
|
|
7
|
+
case 'stageStart':
|
|
8
|
+
return 'debug';
|
|
9
|
+
case 'pipelineStart':
|
|
10
|
+
case 'pipelineComplete':
|
|
11
|
+
case 'datasetsSelected':
|
|
12
|
+
case 'datasetStart':
|
|
13
|
+
case 'datasetComplete':
|
|
14
|
+
case 'distributionSelected':
|
|
15
|
+
case 'stageComplete':
|
|
16
|
+
case 'timeoutRelaxed':
|
|
17
|
+
return 'info';
|
|
18
|
+
case 'datasetSkipped':
|
|
19
|
+
return event.reason.code === 'unchanged' ? 'info' : 'warn';
|
|
20
|
+
case 'distributionValidated':
|
|
21
|
+
return event.verdict.valid ? 'info' : 'warn';
|
|
22
|
+
case 'datasetValidated':
|
|
23
|
+
return event.report.conforms ? 'info' : 'warn';
|
|
24
|
+
case 'stageSkipped':
|
|
25
|
+
return event.reason.code === 'noItemsSelected' ? 'info' : 'warn';
|
|
26
|
+
case 'selectionEmpty':
|
|
27
|
+
case 'importFailed':
|
|
28
|
+
case 'itemsRejected':
|
|
29
|
+
case 'valuesUnread':
|
|
30
|
+
case 'timeoutTightened':
|
|
31
|
+
return 'warn';
|
|
32
|
+
case 'stageFailed':
|
|
33
|
+
return 'error';
|
|
34
|
+
}
|
|
35
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { PipelineEvent } from './pipelineEvent.js';
|
|
2
|
+
/**
|
|
3
|
+
* Observer of a pipeline run. The pipeline hands every {@link PipelineEvent}
|
|
4
|
+
* to {@link onEvent}, synchronously and in order. Each reporter filters by
|
|
5
|
+
* level itself.
|
|
6
|
+
*
|
|
7
|
+
* A reporter cannot break a run: when {@link onEvent} throws, the pipeline
|
|
8
|
+
* still hands the event to the other reporters, carries on, and surfaces the
|
|
9
|
+
* error as a `ReporterWarning` through `process.emitWarning`.
|
|
10
|
+
*/
|
|
11
|
+
export interface Reporter {
|
|
12
|
+
onEvent(event: PipelineEvent): void;
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=reporter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reporter.d.ts","sourceRoot":"","sources":["../src/reporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAExD;;;;;;;;GAQG;AACH,MAAM,WAAW,QAAQ;IACvB,OAAO,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAAC;CACrC"}
|
package/dist/sparql/reader.d.ts
CHANGED
|
@@ -10,6 +10,15 @@ export declare class NotSupported {
|
|
|
10
10
|
readonly message: string;
|
|
11
11
|
constructor(message: string);
|
|
12
12
|
}
|
|
13
|
+
/**
|
|
14
|
+
* A stage’s item selector selected nothing for the dataset, so the stage does
|
|
15
|
+
* not apply to it – the usual outcome for a stage that only some datasets
|
|
16
|
+
* have items for. A {@link NotSupported}, so callers that only check for that
|
|
17
|
+
* keep treating it as a skip.
|
|
18
|
+
*/
|
|
19
|
+
export declare class NoItemsSelected extends NotSupported {
|
|
20
|
+
constructor();
|
|
21
|
+
}
|
|
13
22
|
/** A single row of variable bindings (variable name → NamedNode). */
|
|
14
23
|
export type VariableBindings = Record<string, NamedNode>;
|
|
15
24
|
export interface ReadOptions {
|
|
@@ -49,7 +58,7 @@ export interface SparqlConstructReaderOptions {
|
|
|
49
58
|
* Optional custom SparqlEndpointFetcher instance.
|
|
50
59
|
*
|
|
51
60
|
* When supplied, the reader uses this fetcher as-is for every attempt
|
|
52
|
-
*
|
|
61
|
+
* – the per-attempt timeout from the {@link TimeoutPolicy} is **not**
|
|
53
62
|
* enforced (the supplied fetcher’s own `timeout` governs). Policy
|
|
54
63
|
* `beforeRequest`/`afterRequest` hooks still fire so outcome
|
|
55
64
|
* classification works, but adaptive tightening cannot apply.
|
|
@@ -77,7 +86,7 @@ export interface SparqlConstructReaderOptions {
|
|
|
77
86
|
/**
|
|
78
87
|
* Deduplicate triples in the CONSTRUCT output stream.
|
|
79
88
|
*
|
|
80
|
-
* SPARQL CONSTRUCT queries can produce duplicate triples
|
|
89
|
+
* SPARQL CONSTRUCT queries can produce duplicate triples – for example,
|
|
81
90
|
* constant triples (like `?dataset a edm:ProvidedCHO`) are emitted for
|
|
82
91
|
* every solution row. When enabled, a streaming identity filter removes
|
|
83
92
|
* duplicates inline without buffering.
|
|
@@ -99,9 +108,9 @@ export interface SparqlConstructReaderOptions {
|
|
|
99
108
|
* and parsed at {@link read} time after substitution.
|
|
100
109
|
*
|
|
101
110
|
* Template substitution (applied in order):
|
|
102
|
-
* 1. `#subjectFilter#`
|
|
103
|
-
* 2. `FROM <graph>`
|
|
104
|
-
* 3. `?dataset`
|
|
111
|
+
* 1. `#subjectFilter#` – replaced with `distribution.subjectFilter` (deferred to read)
|
|
112
|
+
* 2. `FROM <graph>` – set via `withDefaultGraph` if the distribution has a named graph
|
|
113
|
+
* 3. `?dataset` – replaced with the dataset IRI (string substitution on the serialised query)
|
|
105
114
|
*
|
|
106
115
|
* @example
|
|
107
116
|
* ```typescript
|
|
@@ -178,8 +187,8 @@ export declare function readQueryFile(filename: string): Promise<string>;
|
|
|
178
187
|
* **Why this exists:** `fetch-sparql-endpoint` pipes the raw HTTP response
|
|
179
188
|
* stream directly into N3.js's `StreamParser`. N3.js has a bug
|
|
180
189
|
* ({@link https://github.com/rdfjs/N3.js/issues/578 | N3.js#578}) where
|
|
181
|
-
* tokens that straddle chunk boundaries
|
|
182
|
-
* `@nl-nl`
|
|
190
|
+
* tokens that straddle chunk boundaries – most commonly language tags like
|
|
191
|
+
* `@nl-nl` – cause spurious `Unexpected "-nl"` parse errors. The error is
|
|
183
192
|
* non-deterministic and typically surfaces only on responses larger than
|
|
184
193
|
* ~12 MB, because that is when HTTP chunking starts splitting mid-token.
|
|
185
194
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reader.d.ts","sourceRoot":"","sources":["../../src/sparql/reader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAiB,MAAM,cAAc,CAAC;AACpE,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAGpD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAUxC,OAAO,EAGL,KAAK,aAAa,EACnB,MAAM,oBAAoB,CAAC;AAS5B;;GAEG;AACH,qBAAa,YAAY;aACK,OAAO,EAAE,MAAM;gBAAf,OAAO,EAAE,MAAM;CAC5C;AAED,qEAAqE;AACrE,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEzD,MAAM,WAAW,WAAW;IAC1B;;;OAGG;IACH,QAAQ,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAE9B;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,MAAM;IACrB,IAAI,CACF,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;;;;;;;;;OAgBG;IACH,OAAO,CAAC,EAAE,qBAAqB,CAAC;IAEhC;;;;;;;;;OASG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB;;;;;;;;;;;;;;OAcG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,qBAAa,qBAAsB,YAAW,MAAM;IAClD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAiB;IAC5C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAwB;IACrD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAU;IACrC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAU;IACtC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;gBAEjC,OAAO,EAAE,4BAA4B;IAiBjD;;;;;;;;OAQG;IACG,IAAI,CACR,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IA4C/B;;;;;OAKG;YACW,oBAAoB;IA+BlC;;;;;;;OAOG;IACH,OAAO,CAAC,iBAAiB;IAKzB;;;;OAIG;YACW,UAAU;IAmBxB;;;;;OAKG;WACiB,QAAQ,CAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,IAAI,CAAC,4BAA4B,EAAE,OAAO,CAAC,GACpD,OAAO,CAAC,qBAAqB,CAAC;CAIlC;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAErE;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,mBAAoB,SAAQ,SAAS;IAChD,OAAO,CAAC,SAAS,CAAM;IAEd,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,IAAI;IAUjE,MAAM,CAAC,QAAQ,EAAE,MAAM,IAAI;CAMrC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAuB,gBAAgB,CACrC,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GACzB,aAAa,CAAC,IAAI,CAAC,CASrB"}
|
|
1
|
+
{"version":3,"file":"reader.d.ts","sourceRoot":"","sources":["../../src/sparql/reader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAiB,MAAM,cAAc,CAAC;AACpE,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAGpD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAUxC,OAAO,EAGL,KAAK,aAAa,EACnB,MAAM,oBAAoB,CAAC;AAS5B;;GAEG;AACH,qBAAa,YAAY;aACK,OAAO,EAAE,MAAM;gBAAf,OAAO,EAAE,MAAM;CAC5C;AAED;;;;;GAKG;AACH,qBAAa,eAAgB,SAAQ,YAAY;;CAIhD;AAED,qEAAqE;AACrE,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEzD,MAAM,WAAW,WAAW;IAC1B;;;OAGG;IACH,QAAQ,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAE9B;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,MAAM;IACrB,IAAI,CACF,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;;;;;;;;;OAgBG;IACH,OAAO,CAAC,EAAE,qBAAqB,CAAC;IAEhC;;;;;;;;;OASG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB;;;;;;;;;;;;;;OAcG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,qBAAa,qBAAsB,YAAW,MAAM;IAClD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAiB;IAC5C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAwB;IACrD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAU;IACrC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAU;IACtC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;gBAEjC,OAAO,EAAE,4BAA4B;IAiBjD;;;;;;;;OAQG;IACG,IAAI,CACR,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IA4C/B;;;;;OAKG;YACW,oBAAoB;IA+BlC;;;;;;;OAOG;IACH,OAAO,CAAC,iBAAiB;IAKzB;;;;OAIG;YACW,UAAU;IAmBxB;;;;;OAKG;WACiB,QAAQ,CAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,IAAI,CAAC,4BAA4B,EAAE,OAAO,CAAC,GACpD,OAAO,CAAC,qBAAqB,CAAC;CAIlC;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAErE;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,mBAAoB,SAAQ,SAAS;IAChD,OAAO,CAAC,SAAS,CAAM;IAEd,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,IAAI;IAUjE,MAAM,CAAC,QAAQ,EAAE,MAAM,IAAI;CAMrC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAuB,gBAAgB,CACrC,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GACzB,aAAa,CAAC,IAAI,CAAC,CASrB"}
|
package/dist/sparql/reader.js
CHANGED
|
@@ -27,6 +27,17 @@ export class NotSupported {
|
|
|
27
27
|
this.message = message;
|
|
28
28
|
}
|
|
29
29
|
}
|
|
30
|
+
/**
|
|
31
|
+
* A stage’s item selector selected nothing for the dataset, so the stage does
|
|
32
|
+
* not apply to it – the usual outcome for a stage that only some datasets
|
|
33
|
+
* have items for. A {@link NotSupported}, so callers that only check for that
|
|
34
|
+
* keep treating it as a skip.
|
|
35
|
+
*/
|
|
36
|
+
export class NoItemsSelected extends NotSupported {
|
|
37
|
+
constructor() {
|
|
38
|
+
super('No items selected');
|
|
39
|
+
}
|
|
40
|
+
}
|
|
30
41
|
/**
|
|
31
42
|
* A streaming SPARQL CONSTRUCT reader.
|
|
32
43
|
*
|
|
@@ -35,9 +46,9 @@ export class NotSupported {
|
|
|
35
46
|
* and parsed at {@link read} time after substitution.
|
|
36
47
|
*
|
|
37
48
|
* Template substitution (applied in order):
|
|
38
|
-
* 1. `#subjectFilter#`
|
|
39
|
-
* 2. `FROM <graph>`
|
|
40
|
-
* 3. `?dataset`
|
|
49
|
+
* 1. `#subjectFilter#` – replaced with `distribution.subjectFilter` (deferred to read)
|
|
50
|
+
* 2. `FROM <graph>` – set via `withDefaultGraph` if the distribution has a named graph
|
|
51
|
+
* 3. `?dataset` – replaced with the dataset IRI (string substitution on the serialised query)
|
|
41
52
|
*
|
|
42
53
|
* @example
|
|
43
54
|
* ```typescript
|
|
@@ -196,8 +207,8 @@ export async function readQueryFile(filename) {
|
|
|
196
207
|
* **Why this exists:** `fetch-sparql-endpoint` pipes the raw HTTP response
|
|
197
208
|
* stream directly into N3.js's `StreamParser`. N3.js has a bug
|
|
198
209
|
* ({@link https://github.com/rdfjs/N3.js/issues/578 | N3.js#578}) where
|
|
199
|
-
* tokens that straddle chunk boundaries
|
|
200
|
-
* `@nl-nl`
|
|
210
|
+
* tokens that straddle chunk boundaries – most commonly language tags like
|
|
211
|
+
* `@nl-nl` – cause spurious `Unexpected "-nl"` parse errors. The error is
|
|
201
212
|
* non-deterministic and typically surfaces only on responses larger than
|
|
202
213
|
* ~12 MB, because that is when HTTP chunking starts splitting mid-token.
|
|
203
214
|
*
|
package/dist/stage.d.ts
CHANGED
|
@@ -5,7 +5,7 @@ import { NotSupported } from './sparql/reader.js';
|
|
|
5
5
|
import type { TimeoutPolicy } from './sparql/timeoutPolicy.js';
|
|
6
6
|
import type { Validator } from './validator.js';
|
|
7
7
|
import type { DatasetWriter } from './writer/writer.js';
|
|
8
|
-
import type { UnreadValue } from './
|
|
8
|
+
import type { UnreadValue } from './pipelineEvent.js';
|
|
9
9
|
/**
|
|
10
10
|
* Transforms a quad stream, given the context of its extension point.
|
|
11
11
|
*
|
|
@@ -47,7 +47,7 @@ export interface BatchContext extends ReaderContext {
|
|
|
47
47
|
* Report a value the transform could not read and so left out of the item
|
|
48
48
|
* it produced – an integer field over `onbekend`, say. The pipeline tallies
|
|
49
49
|
* them per (`target`, `field`) and reports them when the dataset completes
|
|
50
|
-
* (
|
|
50
|
+
* (the `valuesUnread` event).
|
|
51
51
|
*/
|
|
52
52
|
valueUnread(target: string, field: string, unread: UnreadValue): void;
|
|
53
53
|
}
|
package/dist/stage.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,
|
|
1
|
+
{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,EAAmB,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE/D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAExD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEtD;;;;;;;GAOG;AACH,MAAM,MAAM,aAAa,CAAC,GAAG,IAAI,CAC/B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,EAC1B,OAAO,EAAE,GAAG,KACT,aAAa,CAAC,IAAI,CAAC,CAAC;AAEzB;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,YAAY,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;GAMG;AACH,MAAM,MAAM,cAAc,CAAC,GAAG,IAAI,CAChC,KAAK,EAAE,SAAS,IAAI,EAAE,EACtB,OAAO,EAAE,YAAY,KAClB,QAAQ,CAAC,GAAG,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;AAExC;;;;;GAKG;AACH,MAAM,WAAW,YAAa,SAAQ,aAAa;IACjD,QAAQ,EAAE,SAAS,gBAAgB,EAAE,CAAC;IACtC;;;;;OAKG;IACH,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC;CACvE;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,aAAa,CAAC,aAAa,CAAC,GAAG,aAAa,CAAC,aAAa,CAAC,EAAE,CAAC;CAC3E;AAED,yEAAyE;AACzE,MAAM,MAAM,YAAY,GACtB,MAAM,GAAG,cAAc,GAAG,CAAC,MAAM,GAAG,cAAc,CAAC,EAAE,CAAC;AAQxD,MAAM,WAAW,YAAY,CAAC,GAAG,GAAG,IAAI;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,YAAY,CAAC;IACtB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC;IAC9B;;;;;;;OAOG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;;;OAOG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,YAAY,KAAK,YAAY,CAAC;IAC3E,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;IACjB;;;;;;;;;;;;;;;OAeG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,mFAAmF;IACnF,UAAU,CAAC,EAAE;QACX,SAAS,EAAE,SAAS,CAAC;QACrB,iEAAiE;QACjE,SAAS,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;KACvC,CAAC;CACH;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,CAAC,cAAc,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;IACtE;;;;;OAKG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB;iEAC6D;IAC7D,WAAW,CAAC,EAAE,YAAY,CAAC,aAAa,CAAC,CAAC;CAC3C;AAED,uDAAuD;AACvD,MAAM,WAAW,aAAa;IAC5B,+BAA+B;IAC/B,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,qBAAa,KAAK,CAAC,GAAG,GAAG,IAAI;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,oGAAoG;IACpG,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC;qEACiE;IACjE,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;IAC7C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAkC;IAC9D,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAiC;gBAEhD,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC;IAmCtC,mDAAmD;IACnD,IAAI,SAAS,IAAI,SAAS,GAAG,SAAS,CAErC;IAEK,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,YAAY,EACtB,MAAM,EAAE,aAAa,CAAC,GAAG,CAAC,EAC1B,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;IAgF/B;;;;OAIG;IACH,OAAO,CAAC,cAAc;YAMR,eAAe;IAkL7B;;;OAGG;YACW,cAAc;YAqBd,OAAO;IA6BrB;;;;;OAKG;IACH,OAAO,CAAC,eAAe;CAiBxB;AAiDD,2GAA2G;AAC3G,MAAM,WAAW,YAAY;IAC3B,MAAM,CACJ,YAAY,EAAE,YAAY,EAC1B,SAAS,CAAC,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,aAAa,GACtB,aAAa,CAAC,gBAAgB,CAAC,CAAC;CACpC"}
|
package/dist/stage.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { NotSupported } from './sparql/reader.js';
|
|
1
|
+
import { NoItemsSelected, NotSupported } from './sparql/reader.js';
|
|
2
2
|
import { batch } from './batch.js';
|
|
3
3
|
import { AsyncQueue } from './asyncQueue.js';
|
|
4
4
|
export class Stage {
|
|
@@ -128,7 +128,7 @@ export class Stage {
|
|
|
128
128
|
const iter = batches[Symbol.asyncIterator]();
|
|
129
129
|
const first = await iter.next();
|
|
130
130
|
if (first.done) {
|
|
131
|
-
return new
|
|
131
|
+
return new NoItemsSelected();
|
|
132
132
|
}
|
|
133
133
|
// Reconstruct a full iterable including the peeked first batch.
|
|
134
134
|
const allBatches = (async function* () {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import type { UnreadValue, ValuesUnread } from './
|
|
1
|
+
import type { UnreadValue, ValuesUnread } from './pipelineEvent.js';
|
|
2
2
|
/**
|
|
3
3
|
* The values a dataset’s stages could not read, tallied per (target, field)
|
|
4
|
-
* for
|
|
4
|
+
* for the `valuesUnread` event. Bounded by the schema, not the
|
|
5
5
|
* data: one entry per field that rejected anything, each holding a count and
|
|
6
6
|
* at most {@link MAX_EXAMPLES} distinct examples.
|
|
7
7
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"unreadValueTally.d.ts","sourceRoot":"","sources":["../src/unreadValueTally.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"unreadValueTally.d.ts","sourceRoot":"","sources":["../src/unreadValueTally.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAKpE;;;;;;;GAOG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAGpB;IAEJ,0DAA0D;IAC1D,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,GAAG,IAAI;IAiB7D,mEAAmE;IACnE,OAAO,IAAI,SAAS,YAAY,EAAE;CAGnC"}
|
package/dist/unreadValueTally.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
const MAX_EXAMPLES = 5;
|
|
3
3
|
/**
|
|
4
4
|
* The values a dataset’s stages could not read, tallied per (target, field)
|
|
5
|
-
* for
|
|
5
|
+
* for the `valuesUnread` event. Bounded by the schema, not the
|
|
6
6
|
* data: one entry per field that rejected anything, each holding a count and
|
|
7
7
|
* at most {@link MAX_EXAMPLES} distinct examples.
|
|
8
8
|
*
|
package/dist/writer/writer.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { Dataset } from '@lde/dataset';
|
|
2
2
|
import type { Quad } from '@rdfjs/types';
|
|
3
3
|
import type { ProvenanceStore } from '../provenance/store.js';
|
|
4
|
-
import type { ItemsRejected } from '../
|
|
4
|
+
import type { ItemsRejected } from '../pipelineEvent.js';
|
|
5
5
|
/**
|
|
6
6
|
* Context for one pipeline run, handed to {@link Writer.openRun}.
|
|
7
7
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../src/writer/writer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../src/writer/writer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEzD;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB;;;;OAIG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;OAGG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;;;;;OAKG;IACH,eAAe,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;IACpC;;;;;;;OAOG;IACH,aAAa,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,aAAa,GAAG,IAAI,CAAC;IACjE;;;OAGG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,eAAe,CAAC;CACvC;AAED;;;;GAIG;AACH,MAAM,WAAW,aAAa,CAAC,IAAI,GAAG,IAAI;IACxC;;;;;;OAMG;IACH,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACpE;AAED;;;;GAIG;AACH,MAAM,MAAM,cAAc,GAAG,SAAS,GAAG,QAAQ,CAAC;AAElD;;;;;;;GAOG;AACH,MAAM,WAAW,SAAS,CAAC,IAAI,GAAG,IAAI,CAAE,SAAQ,aAAa,CAAC,IAAI,CAAC;IACjE;;;;;;;;;;OAUG;IACH,KAAK,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjE;;;;;;;;;OASG;IACH,KAAK,CAAC,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAExC;;;;;;;;;OASG;IACH,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAExB;;;;;;OAMG;IACH,KAAK,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErC;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,OAAO,CAAC,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACzC;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,MAAM,CAAC,IAAI,GAAG,IAAI;IACjC;;;;OAIG;IACH,OAAO,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;CACxD"}
|
package/package.json
CHANGED
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
import type { ProgressReporter } from './progressReporter.js';
|
|
2
|
-
/**
|
|
3
|
-
* Combine several {@link ProgressReporter}s into one that forwards every
|
|
4
|
-
* lifecycle call to each child that implements it. Lets a single run be
|
|
5
|
-
* observed by more than one reporter – e.g. a console reporter alongside a
|
|
6
|
-
* verdict-collecting one.
|
|
7
|
-
*
|
|
8
|
-
* Each method is dispatched to the children in array order; a child that does
|
|
9
|
-
* not implement a given (optional) method is skipped for that call.
|
|
10
|
-
*
|
|
11
|
-
* Internal to the package: not re-exported from `index.ts`. {@link Pipeline}
|
|
12
|
-
* uses it to normalise a `reporter` array into the single reporter its call
|
|
13
|
-
* sites expect, so the broader API need not grow a new public symbol.
|
|
14
|
-
*/
|
|
15
|
-
export declare function combineReporters(reporters: readonly ProgressReporter[]): ProgressReporter;
|
|
16
|
-
//# sourceMappingURL=combineReporters.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"combineReporters.d.ts","sourceRoot":"","sources":["../src/combineReporters.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAU9D;;;;;;;;;;;;GAYG;AACH,wBAAgB,gBAAgB,CAC9B,SAAS,EAAE,SAAS,gBAAgB,EAAE,GACrC,gBAAgB,CAiDlB"}
|
package/dist/combineReporters.js
DELETED
|
@@ -1,57 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Combine several {@link ProgressReporter}s into one that forwards every
|
|
3
|
-
* lifecycle call to each child that implements it. Lets a single run be
|
|
4
|
-
* observed by more than one reporter – e.g. a console reporter alongside a
|
|
5
|
-
* verdict-collecting one.
|
|
6
|
-
*
|
|
7
|
-
* Each method is dispatched to the children in array order; a child that does
|
|
8
|
-
* not implement a given (optional) method is skipped for that call.
|
|
9
|
-
*
|
|
10
|
-
* Internal to the package: not re-exported from `index.ts`. {@link Pipeline}
|
|
11
|
-
* uses it to normalise a `reporter` array into the single reporter its call
|
|
12
|
-
* sites expect, so the broader API need not grow a new public symbol.
|
|
13
|
-
*/
|
|
14
|
-
export function combineReporters(reporters) {
|
|
15
|
-
const forward = (method, ...args) => {
|
|
16
|
-
for (const reporter of reporters) {
|
|
17
|
-
// Cast to the concrete signature for `method`: indexing by a generic key
|
|
18
|
-
// yields a union of method types TS won't call directly, even though the
|
|
19
|
-
// arguments are correlated.
|
|
20
|
-
const handler = reporter[method];
|
|
21
|
-
// Every method is optional; notify only the children that implement it.
|
|
22
|
-
// Invoke through `call` so a class-based reporter keeps its `this`: a bare
|
|
23
|
-
// `handler?.(...)` would drop the receiver and break methods that touch
|
|
24
|
-
// instance state (e.g. ConsoleReporter’s spinner).
|
|
25
|
-
handler?.call(reporter, ...args);
|
|
26
|
-
}
|
|
27
|
-
};
|
|
28
|
-
// Listing every method explicitly (rather than a Proxy) keeps the forwarding
|
|
29
|
-
// type-safe: typing the result as `Required<ProgressReporter>` forces a new
|
|
30
|
-
// entry here whenever the interface grows, so a forgotten method fails to
|
|
31
|
-
// compile instead of silently going unforwarded.
|
|
32
|
-
const combined = {
|
|
33
|
-
pipelineStart: (...args) => forward('pipelineStart', ...args),
|
|
34
|
-
datasetsSelected: (...args) => forward('datasetsSelected', ...args),
|
|
35
|
-
selectionEmpty: (...args) => forward('selectionEmpty', ...args),
|
|
36
|
-
datasetStart: (...args) => forward('datasetStart', ...args),
|
|
37
|
-
distributionProbed: (...args) => forward('distributionProbed', ...args),
|
|
38
|
-
importStarted: (...args) => forward('importStarted', ...args),
|
|
39
|
-
importFailed: (...args) => forward('importFailed', ...args),
|
|
40
|
-
distributionValidated: (...args) => forward('distributionValidated', ...args),
|
|
41
|
-
distributionSelected: (...args) => forward('distributionSelected', ...args),
|
|
42
|
-
stageStart: (...args) => forward('stageStart', ...args),
|
|
43
|
-
stageProgress: (...args) => forward('stageProgress', ...args),
|
|
44
|
-
stageComplete: (...args) => forward('stageComplete', ...args),
|
|
45
|
-
stageFailed: (...args) => forward('stageFailed', ...args),
|
|
46
|
-
stageSkipped: (...args) => forward('stageSkipped', ...args),
|
|
47
|
-
datasetValidated: (...args) => forward('datasetValidated', ...args),
|
|
48
|
-
datasetComplete: (...args) => forward('datasetComplete', ...args),
|
|
49
|
-
datasetSkipped: (...args) => forward('datasetSkipped', ...args),
|
|
50
|
-
itemsRejected: (...args) => forward('itemsRejected', ...args),
|
|
51
|
-
valuesUnread: (...args) => forward('valuesUnread', ...args),
|
|
52
|
-
pipelineComplete: (...args) => forward('pipelineComplete', ...args),
|
|
53
|
-
timeoutTightened: (...args) => forward('timeoutTightened', ...args),
|
|
54
|
-
timeoutRelaxed: (...args) => forward('timeoutRelaxed', ...args),
|
|
55
|
-
};
|
|
56
|
-
return combined;
|
|
57
|
-
}
|