@lde/pipeline 0.31.5 → 0.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -29
- package/dist/pipeline.d.ts +2 -2
- package/dist/pipeline.d.ts.map +1 -1
- package/dist/pipeline.js +114 -41
- package/dist/provenance/fileLoadedSparqlProvenanceStore.d.ts.map +1 -1
- package/dist/provenance/fileLoadedSparqlProvenanceStore.js +5 -2
- package/dist/sparql/index.d.ts +1 -1
- package/dist/sparql/index.d.ts.map +1 -1
- package/dist/sparql/index.js +1 -1
- package/dist/sparql/{executor.d.ts → reader.d.ts} +28 -27
- package/dist/sparql/reader.d.ts.map +1 -0
- package/dist/sparql/{executor.js → reader.js} +18 -17
- package/dist/sparql/selector.d.ts +1 -1
- package/dist/sparql/selector.d.ts.map +1 -1
- package/dist/sparql/values.d.ts +1 -1
- package/dist/sparql/values.d.ts.map +1 -1
- package/dist/stage.d.ts +31 -31
- package/dist/stage.d.ts.map +1 -1
- package/dist/stage.js +27 -27
- package/dist/writer/fileWriter.d.ts +34 -7
- package/dist/writer/fileWriter.d.ts.map +1 -1
- package/dist/writer/fileWriter.js +71 -39
- package/dist/writer/sparqlUpdateWriter.d.ts +19 -8
- package/dist/writer/sparqlUpdateWriter.d.ts.map +1 -1
- package/dist/writer/sparqlUpdateWriter.js +25 -12
- package/dist/writer/writer.d.ts +103 -16
- package/dist/writer/writer.d.ts.map +1 -1
- package/package.json +7 -7
- package/dist/sparql/executor.d.ts.map +0 -1
|
@@ -4,7 +4,7 @@ import type { NamedNode, Quad } from '@rdfjs/types';
|
|
|
4
4
|
import { Transform } from 'node:stream';
|
|
5
5
|
import { type TimeoutPolicy } from './timeoutPolicy.js';
|
|
6
6
|
/**
|
|
7
|
-
*
|
|
7
|
+
* A reader could not run because the dataset lacks a supported distribution.
|
|
8
8
|
*/
|
|
9
9
|
export declare class NotSupported {
|
|
10
10
|
readonly message: string;
|
|
@@ -12,30 +12,30 @@ export declare class NotSupported {
|
|
|
12
12
|
}
|
|
13
13
|
/** A single row of variable bindings (variable name → NamedNode). */
|
|
14
14
|
export type VariableBindings = Record<string, NamedNode>;
|
|
15
|
-
export interface
|
|
15
|
+
export interface ReadOptions {
|
|
16
16
|
/**
|
|
17
17
|
* Variable bindings to inject as a VALUES clause into the query.
|
|
18
18
|
* When non-empty, a VALUES block is prepended to the WHERE clause.
|
|
19
19
|
*/
|
|
20
20
|
bindings?: VariableBindings[];
|
|
21
21
|
/**
|
|
22
|
-
* Per-call {@link TimeoutPolicy}. When supplied, the
|
|
22
|
+
* Per-call {@link TimeoutPolicy}. When supplied, the reader calls
|
|
23
23
|
* {@link TimeoutPolicy.beforeRequest} once per attempt (including
|
|
24
24
|
* retries), installs an {@link AbortSignal} with the returned budget,
|
|
25
25
|
* and reports the outcome via {@link TimeoutPolicy.afterRequest}.
|
|
26
26
|
*
|
|
27
|
-
* Overrides the
|
|
27
|
+
* Overrides the reader-level policy passed at construction time.
|
|
28
28
|
* Pipeline runners use this to thread the per-dataset policy through.
|
|
29
29
|
*/
|
|
30
30
|
timeout?: TimeoutPolicy;
|
|
31
31
|
}
|
|
32
|
-
export interface
|
|
33
|
-
|
|
32
|
+
export interface Reader {
|
|
33
|
+
read(dataset: Dataset, distribution: Distribution, options?: ReadOptions): Promise<AsyncIterable<Quad> | NotSupported>;
|
|
34
34
|
}
|
|
35
35
|
/**
|
|
36
|
-
* Options for
|
|
36
|
+
* Options for SparqlConstructReader.
|
|
37
37
|
*/
|
|
38
|
-
export interface
|
|
38
|
+
export interface SparqlConstructReaderOptions {
|
|
39
39
|
/**
|
|
40
40
|
* SPARQL CONSTRUCT query to execute.
|
|
41
41
|
*/
|
|
@@ -48,13 +48,13 @@ export interface SparqlConstructExecutorOptions {
|
|
|
48
48
|
/**
|
|
49
49
|
* Optional custom SparqlEndpointFetcher instance.
|
|
50
50
|
*
|
|
51
|
-
* When supplied, the
|
|
51
|
+
* When supplied, the reader uses this fetcher as-is for every attempt
|
|
52
52
|
* — the per-attempt timeout from the {@link TimeoutPolicy} is **not**
|
|
53
53
|
* enforced (the supplied fetcher’s own `timeout` governs). Policy
|
|
54
54
|
* `beforeRequest`/`afterRequest` hooks still fire so outcome
|
|
55
55
|
* classification works, but adaptive tightening cannot apply.
|
|
56
56
|
*
|
|
57
|
-
* When omitted, the
|
|
57
|
+
* When omitted, the reader builds a fresh
|
|
58
58
|
* {@link SparqlEndpointFetcher} per attempt with the per-attempt timeout
|
|
59
59
|
* baked in.
|
|
60
60
|
*
|
|
@@ -82,7 +82,7 @@ export interface SparqlConstructExecutorOptions {
|
|
|
82
82
|
* every solution row. When enabled, a streaming identity filter removes
|
|
83
83
|
* duplicates inline without buffering.
|
|
84
84
|
*
|
|
85
|
-
* The dedup set is scoped to each {@link
|
|
85
|
+
* The dedup set is scoped to each {@link read} call, so memory stays
|
|
86
86
|
* bounded to the number of unique quads per call (typically one batch).
|
|
87
87
|
*
|
|
88
88
|
* Inspired by Comunica's `distinctConstruct` context option.
|
|
@@ -92,23 +92,23 @@ export interface SparqlConstructExecutorOptions {
|
|
|
92
92
|
deduplicate?: boolean;
|
|
93
93
|
}
|
|
94
94
|
/**
|
|
95
|
-
* A streaming SPARQL CONSTRUCT
|
|
95
|
+
* A streaming SPARQL CONSTRUCT reader.
|
|
96
96
|
*
|
|
97
97
|
* Queries **without** `#subjectFilter#` are parsed once in the constructor
|
|
98
98
|
* (fast path). Queries that contain the template are stored as raw strings
|
|
99
|
-
* and parsed at {@link
|
|
99
|
+
* and parsed at {@link read} time after substitution.
|
|
100
100
|
*
|
|
101
101
|
* Template substitution (applied in order):
|
|
102
|
-
* 1. `#subjectFilter#` — replaced with `distribution.subjectFilter` (deferred to
|
|
102
|
+
* 1. `#subjectFilter#` — replaced with `distribution.subjectFilter` (deferred to read)
|
|
103
103
|
* 2. `FROM <graph>` — set via `withDefaultGraph` if the distribution has a named graph
|
|
104
104
|
* 3. `?dataset` — replaced with the dataset IRI (string substitution on the serialised query)
|
|
105
105
|
*
|
|
106
106
|
* @example
|
|
107
107
|
* ```typescript
|
|
108
|
-
* const
|
|
108
|
+
* const reader = new SparqlConstructReader({
|
|
109
109
|
* query: 'CONSTRUCT { ?dataset ?p ?o } WHERE { ?s ?p ?o }',
|
|
110
110
|
* });
|
|
111
|
-
* const result = await
|
|
111
|
+
* const result = await reader.read(dataset, distribution);
|
|
112
112
|
* if (result instanceof NotSupported) {
|
|
113
113
|
* console.log(result.message);
|
|
114
114
|
* } else {
|
|
@@ -118,7 +118,7 @@ export interface SparqlConstructExecutorOptions {
|
|
|
118
118
|
* }
|
|
119
119
|
* ```
|
|
120
120
|
*/
|
|
121
|
-
export declare class
|
|
121
|
+
export declare class SparqlConstructReader implements Reader {
|
|
122
122
|
private readonly rawQuery;
|
|
123
123
|
private readonly preParsed?;
|
|
124
124
|
private readonly userFetcher?;
|
|
@@ -126,16 +126,17 @@ export declare class SparqlConstructExecutor implements Executor {
|
|
|
126
126
|
private readonly lineBuffer;
|
|
127
127
|
private readonly deduplicate;
|
|
128
128
|
private readonly generator;
|
|
129
|
-
constructor(options:
|
|
129
|
+
constructor(options: SparqlConstructReaderOptions);
|
|
130
130
|
/**
|
|
131
|
-
*
|
|
131
|
+
* Read quads by executing the SPARQL CONSTRUCT query against the
|
|
132
|
+
* distribution's endpoint.
|
|
132
133
|
*
|
|
133
|
-
* @param dataset The dataset to
|
|
134
|
+
* @param dataset The dataset to read from.
|
|
134
135
|
* @param distribution The distribution providing the SPARQL endpoint.
|
|
135
|
-
* @param options Optional
|
|
136
|
+
* @param options Optional read options (bindings).
|
|
136
137
|
* @returns AsyncIterable<Quad> stream of results.
|
|
137
138
|
*/
|
|
138
|
-
|
|
139
|
+
read(dataset: Dataset, distribution: Distribution, options?: ReadOptions): Promise<AsyncIterable<Quad>>;
|
|
139
140
|
/**
|
|
140
141
|
* Run a single attempt against the endpoint with a per-call abort
|
|
141
142
|
* signal derived from {@link TimeoutPolicy.beforeRequest}. Reports the
|
|
@@ -147,7 +148,7 @@ export declare class SparqlConstructExecutor implements Executor {
|
|
|
147
148
|
* Pick the fetcher to use for a single attempt. A user-supplied fetcher
|
|
148
149
|
* is used as-is and its own timeout governs the request; the per-attempt
|
|
149
150
|
* policy budget is bypassed in that case (see the JSDoc on
|
|
150
|
-
* {@link
|
|
151
|
+
* {@link SparqlConstructReaderOptions.fetcher}). Otherwise a fresh
|
|
151
152
|
* {@link SparqlEndpointFetcher} is constructed per attempt with the
|
|
152
153
|
* policy-supplied timeout baked in.
|
|
153
154
|
*/
|
|
@@ -159,12 +160,12 @@ export declare class SparqlConstructExecutor implements Executor {
|
|
|
159
160
|
*/
|
|
160
161
|
private fetchQuads;
|
|
161
162
|
/**
|
|
162
|
-
* Create
|
|
163
|
+
* Create a reader from a query file.
|
|
163
164
|
*
|
|
164
165
|
* @param filename Path to the query file.
|
|
165
|
-
* @param options Optional
|
|
166
|
+
* @param options Optional reader options (timeout, fetcher).
|
|
166
167
|
*/
|
|
167
|
-
static fromFile(filename: string, options?: Omit<
|
|
168
|
+
static fromFile(filename: string, options?: Omit<SparqlConstructReaderOptions, 'query'>): Promise<SparqlConstructReader>;
|
|
168
169
|
}
|
|
169
170
|
/**
|
|
170
171
|
* Read a SPARQL query from a file.
|
|
@@ -213,4 +214,4 @@ export declare class LineBufferTransform extends Transform {
|
|
|
213
214
|
* ```
|
|
214
215
|
*/
|
|
215
216
|
export declare function deduplicateQuads(quads: AsyncIterable<Quad>): AsyncIterable<Quad>;
|
|
216
|
-
//# sourceMappingURL=
|
|
217
|
+
//# sourceMappingURL=reader.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reader.d.ts","sourceRoot":"","sources":["../../src/sparql/reader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAiB,MAAM,cAAc,CAAC;AACpE,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAGpD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAUxC,OAAO,EAGL,KAAK,aAAa,EACnB,MAAM,oBAAoB,CAAC;AAS5B;;GAEG;AACH,qBAAa,YAAY;aACK,OAAO,EAAE,MAAM;gBAAf,OAAO,EAAE,MAAM;CAC5C;AAED,qEAAqE;AACrE,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEzD,MAAM,WAAW,WAAW;IAC1B;;;OAGG;IACH,QAAQ,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAE9B;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,MAAM;IACrB,IAAI,CACF,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;;;;;;;;;OAgBG;IACH,OAAO,CAAC,EAAE,qBAAqB,CAAC;IAEhC;;;;;;;;;OASG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB;;;;;;;;;;;;;;OAcG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,qBAAa,qBAAsB,YAAW,MAAM;IAClD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAiB;IAC5C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAwB;IACrD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAU;IACrC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAU;IACtC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;gBAEjC,OAAO,EAAE,4BAA4B;IAiBjD;;;;;;;;OAQG;IACG,IAAI,CACR,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IA4C/B;;;;;OAKG;YACW,oBAAoB;IA+BlC;;;;;;;OAOG;IACH,OAAO,CAAC,iBAAiB;IAKzB;;;;OAIG;YACW,UAAU;IAmBxB;;;;;OAKG;WACiB,QAAQ,CAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,IAAI,CAAC,4BAA4B,EAAE,OAAO,CAAC,GACpD,OAAO,CAAC,qBAAqB,CAAC;CAIlC;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAErE;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,mBAAoB,SAAQ,SAAS;IAChD,OAAO,CAAC,SAAS,CAAM;IAEd,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,IAAI;IAUjE,MAAM,CAAC,QAAQ,EAAE,MAAM,IAAI;CAMrC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAuB,gBAAgB,CACrC,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GACzB,aAAa,CAAC,IAAI,CAAC,CASrB"}
|
|
@@ -14,12 +14,12 @@ import { injectValues } from './values.js';
|
|
|
14
14
|
import { ConstantTimeoutPolicy, } from './timeoutPolicy.js';
|
|
15
15
|
/**
|
|
16
16
|
* Fallback policy when no per-call `TimeoutPolicy` is supplied via
|
|
17
|
-
* {@link
|
|
18
|
-
* matters when the
|
|
17
|
+
* {@link ReadOptions.timeout}. Pipeline always supplies one, so this only
|
|
18
|
+
* matters when the reader is driven directly (without a Pipeline).
|
|
19
19
|
*/
|
|
20
20
|
const defaultTimeoutPolicy = new ConstantTimeoutPolicy(300_000);
|
|
21
21
|
/**
|
|
22
|
-
*
|
|
22
|
+
* A reader could not run because the dataset lacks a supported distribution.
|
|
23
23
|
*/
|
|
24
24
|
export class NotSupported {
|
|
25
25
|
message;
|
|
@@ -28,23 +28,23 @@ export class NotSupported {
|
|
|
28
28
|
}
|
|
29
29
|
}
|
|
30
30
|
/**
|
|
31
|
-
* A streaming SPARQL CONSTRUCT
|
|
31
|
+
* A streaming SPARQL CONSTRUCT reader.
|
|
32
32
|
*
|
|
33
33
|
* Queries **without** `#subjectFilter#` are parsed once in the constructor
|
|
34
34
|
* (fast path). Queries that contain the template are stored as raw strings
|
|
35
|
-
* and parsed at {@link
|
|
35
|
+
* and parsed at {@link read} time after substitution.
|
|
36
36
|
*
|
|
37
37
|
* Template substitution (applied in order):
|
|
38
|
-
* 1. `#subjectFilter#` — replaced with `distribution.subjectFilter` (deferred to
|
|
38
|
+
* 1. `#subjectFilter#` — replaced with `distribution.subjectFilter` (deferred to read)
|
|
39
39
|
* 2. `FROM <graph>` — set via `withDefaultGraph` if the distribution has a named graph
|
|
40
40
|
* 3. `?dataset` — replaced with the dataset IRI (string substitution on the serialised query)
|
|
41
41
|
*
|
|
42
42
|
* @example
|
|
43
43
|
* ```typescript
|
|
44
|
-
* const
|
|
44
|
+
* const reader = new SparqlConstructReader({
|
|
45
45
|
* query: 'CONSTRUCT { ?dataset ?p ?o } WHERE { ?s ?p ?o }',
|
|
46
46
|
* });
|
|
47
|
-
* const result = await
|
|
47
|
+
* const result = await reader.read(dataset, distribution);
|
|
48
48
|
* if (result instanceof NotSupported) {
|
|
49
49
|
* console.log(result.message);
|
|
50
50
|
* } else {
|
|
@@ -54,7 +54,7 @@ export class NotSupported {
|
|
|
54
54
|
* }
|
|
55
55
|
* ```
|
|
56
56
|
*/
|
|
57
|
-
export class
|
|
57
|
+
export class SparqlConstructReader {
|
|
58
58
|
rawQuery;
|
|
59
59
|
preParsed;
|
|
60
60
|
userFetcher;
|
|
@@ -77,14 +77,15 @@ export class SparqlConstructExecutor {
|
|
|
77
77
|
this.userFetcher = options.fetcher;
|
|
78
78
|
}
|
|
79
79
|
/**
|
|
80
|
-
*
|
|
80
|
+
* Read quads by executing the SPARQL CONSTRUCT query against the
|
|
81
|
+
* distribution's endpoint.
|
|
81
82
|
*
|
|
82
|
-
* @param dataset The dataset to
|
|
83
|
+
* @param dataset The dataset to read from.
|
|
83
84
|
* @param distribution The distribution providing the SPARQL endpoint.
|
|
84
|
-
* @param options Optional
|
|
85
|
+
* @param options Optional read options (bindings).
|
|
85
86
|
* @returns AsyncIterable<Quad> stream of results.
|
|
86
87
|
*/
|
|
87
|
-
async
|
|
88
|
+
async read(dataset, distribution, options) {
|
|
88
89
|
const endpoint = distribution.accessUrl;
|
|
89
90
|
let ast;
|
|
90
91
|
if (this.preParsed) {
|
|
@@ -148,7 +149,7 @@ export class SparqlConstructExecutor {
|
|
|
148
149
|
* Pick the fetcher to use for a single attempt. A user-supplied fetcher
|
|
149
150
|
* is used as-is and its own timeout governs the request; the per-attempt
|
|
150
151
|
* policy budget is bypassed in that case (see the JSDoc on
|
|
151
|
-
* {@link
|
|
152
|
+
* {@link SparqlConstructReaderOptions.fetcher}). Otherwise a fresh
|
|
152
153
|
* {@link SparqlEndpointFetcher} is constructed per attempt with the
|
|
153
154
|
* policy-supplied timeout baked in.
|
|
154
155
|
*/
|
|
@@ -172,14 +173,14 @@ export class SparqlConstructExecutor {
|
|
|
172
173
|
.pipe(new StreamParser({ format: contentType }));
|
|
173
174
|
}
|
|
174
175
|
/**
|
|
175
|
-
* Create
|
|
176
|
+
* Create a reader from a query file.
|
|
176
177
|
*
|
|
177
178
|
* @param filename Path to the query file.
|
|
178
|
-
* @param options Optional
|
|
179
|
+
* @param options Optional reader options (timeout, fetcher).
|
|
179
180
|
*/
|
|
180
181
|
static async fromFile(filename, options) {
|
|
181
182
|
const query = await readQueryFile(filename);
|
|
182
|
-
return new
|
|
183
|
+
return new SparqlConstructReader({ ...options, query });
|
|
183
184
|
}
|
|
184
185
|
}
|
|
185
186
|
/**
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { Distribution } from '@lde/dataset';
|
|
2
2
|
import { SparqlEndpointFetcher } from 'fetch-sparql-endpoint';
|
|
3
3
|
import type { ItemSelector, SelectOptions } from '../stage.js';
|
|
4
|
-
import type { VariableBindings } from './
|
|
4
|
+
import type { VariableBindings } from './reader.js';
|
|
5
5
|
export interface SparqlItemSelectorOptions {
|
|
6
6
|
/**
|
|
7
7
|
* SELECT query projecting at least one named variable.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"selector.d.ts","sourceRoot":"","sources":["../../src/sparql/selector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAEjD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAQ9D,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC/D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"selector.d.ts","sourceRoot":"","sources":["../../src/sparql/selector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAEjD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAQ9D,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC/D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAoBpD,MAAM,WAAW,yBAAyB;IACxC;;;;;;;;OAQG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;;;;;OAMG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+BAA+B;IAC/B,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,kBAAmB,YAAW,YAAY;IACrD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAwB;gBAEzC,OAAO,EAAE,yBAAyB;IAmBvC,MAAM,CACX,YAAY,EAAE,YAAY,EAC1B,SAAS,CAAC,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,aAAa,GACtB,qBAAqB,CAAC,gBAAgB,CAAC;IA4D1C;;;;OAIG;YACW,uBAAuB;CA8BtC"}
|
package/dist/sparql/values.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { type Pattern, type QueryConstruct, type QuerySelect } from '@traqula/rules-sparql-1-1';
|
|
2
|
-
import type { VariableBindings } from './
|
|
2
|
+
import type { VariableBindings } from './reader.js';
|
|
3
3
|
/**
|
|
4
4
|
* Find the first SubSelect within a list of patterns, looking through
|
|
5
5
|
* intermediate group patterns (the parser wraps `{ SELECT }` in a group).
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"values.d.ts","sourceRoot":"","sources":["../../src/sparql/values.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,OAAO,EAGZ,KAAK,cAAc,EACnB,KAAK,WAAW,EAEjB,MAAM,2BAA2B,CAAC;AACnC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"values.d.ts","sourceRoot":"","sources":["../../src/sparql/values.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,OAAO,EAGZ,KAAK,cAAc,EACnB,KAAK,WAAW,EAEjB,MAAM,2BAA2B,CAAC;AACnC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAIpD;;;GAGG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,WAAW,GAAG,SAAS,CAW1E;AA0DD;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAC1B,KAAK,EAAE,cAAc,EACrB,QAAQ,EAAE,gBAAgB,EAAE,GAC3B,cAAc,CAoBhB"}
|
package/dist/stage.d.ts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { Dataset, Distribution } from '@lde/dataset';
|
|
2
2
|
import type { Quad } from '@rdfjs/types';
|
|
3
|
-
import type {
|
|
4
|
-
import { NotSupported } from './sparql/
|
|
3
|
+
import type { Reader, VariableBindings } from './sparql/reader.js';
|
|
4
|
+
import { NotSupported } from './sparql/reader.js';
|
|
5
5
|
import type { TimeoutPolicy } from './sparql/timeoutPolicy.js';
|
|
6
6
|
import type { Validator } from './validator.js';
|
|
7
|
-
import type {
|
|
7
|
+
import type { DatasetWriter } from './writer/writer.js';
|
|
8
8
|
/**
|
|
9
9
|
* Transforms a quad stream, given the context of its extension point.
|
|
10
10
|
*
|
|
@@ -15,42 +15,42 @@ import type { Writer } from './writer/writer.js';
|
|
|
15
15
|
*/
|
|
16
16
|
export type QuadTransform<Ctx> = (quads: AsyncIterable<Quad>, context: Ctx) => AsyncIterable<Quad>;
|
|
17
17
|
/**
|
|
18
|
-
* Context handed to a {@link QuadTransform} attached to
|
|
19
|
-
* point 1: per-
|
|
18
|
+
* Context handed to a {@link QuadTransform} attached to a reader (extension
|
|
19
|
+
* point 1: per-reader output, pre-merge).
|
|
20
20
|
*
|
|
21
21
|
* `distribution` gives the transform endpoint reach – it may fire its own
|
|
22
22
|
* SPARQL queries – and `stage` carries the stage identity.
|
|
23
23
|
*/
|
|
24
|
-
export interface
|
|
24
|
+
export interface ReaderContext {
|
|
25
25
|
dataset: Dataset;
|
|
26
26
|
distribution: Distribution;
|
|
27
27
|
stage: string;
|
|
28
28
|
}
|
|
29
29
|
/**
|
|
30
|
-
* An {@link
|
|
30
|
+
* An {@link Reader} with zero or more {@link QuadTransform}s attached as data.
|
|
31
31
|
*
|
|
32
|
-
* The stage runner applies the transform(s) in order to **this
|
|
33
|
-
* output** before merging it with sibling
|
|
34
|
-
* `
|
|
32
|
+
* The stage runner applies the transform(s) in order to **this reader's
|
|
33
|
+
* output** before merging it with sibling readers. The window is one
|
|
34
|
+
* `read()` call:
|
|
35
35
|
*
|
|
36
|
-
* - for a global stage that is the
|
|
36
|
+
* - for a global stage that is the reader's complete output;
|
|
37
37
|
* - for a per-class stage that is one batch – one class at `batchSize: 1`.
|
|
38
38
|
*
|
|
39
|
-
* Decorating
|
|
40
|
-
* class: the runner is the only code that delegates to the inner
|
|
39
|
+
* Decorating a reader is therefore construction-time data, not a wrapping
|
|
40
|
+
* class: the runner is the only code that delegates to the inner reader.
|
|
41
41
|
*/
|
|
42
|
-
export interface
|
|
43
|
-
|
|
44
|
-
transform?: QuadTransform<
|
|
42
|
+
export interface AttachedReader {
|
|
43
|
+
reader: Reader;
|
|
44
|
+
transform?: QuadTransform<ReaderContext> | QuadTransform<ReaderContext>[];
|
|
45
45
|
}
|
|
46
|
-
/** One or more
|
|
47
|
-
export type
|
|
46
|
+
/** One or more readers, each optionally carrying attached transforms. */
|
|
47
|
+
export type StageReaders = Reader | AttachedReader | (Reader | AttachedReader)[];
|
|
48
48
|
export interface StageOptions {
|
|
49
49
|
name: string;
|
|
50
|
-
|
|
50
|
+
readers: StageReaders;
|
|
51
51
|
itemSelector?: ItemSelector;
|
|
52
52
|
/**
|
|
53
|
-
* Maximum number of bindings per
|
|
53
|
+
* Maximum number of bindings per reader call.
|
|
54
54
|
*
|
|
55
55
|
* Also used as the selector's page size so that each paginated request
|
|
56
56
|
* fills exactly one batch. A `LIMIT` clause in the selector query
|
|
@@ -61,8 +61,8 @@ export interface StageOptions {
|
|
|
61
61
|
batchSize?: number;
|
|
62
62
|
/**
|
|
63
63
|
* Maximum concurrent in-flight SPARQL queries. Within each batch, all
|
|
64
|
-
*
|
|
65
|
-
* automatically reduced to `⌊maxConcurrency /
|
|
64
|
+
* readers run in parallel; the number of concurrent batches is
|
|
65
|
+
* automatically reduced to `⌊maxConcurrency / readerCount⌋` so the
|
|
66
66
|
* total query pressure stays within this limit.
|
|
67
67
|
*
|
|
68
68
|
* @default 10
|
|
@@ -87,7 +87,7 @@ export interface StageOptions {
|
|
|
87
87
|
* @default false
|
|
88
88
|
*/
|
|
89
89
|
expectsOutput?: boolean;
|
|
90
|
-
/** Optional validation of the combined quads produced by all
|
|
90
|
+
/** Optional validation of the combined quads produced by all readers per batch. */
|
|
91
91
|
validation?: {
|
|
92
92
|
validator: Validator;
|
|
93
93
|
/** What to do when a batch fails validation. @default 'write' */
|
|
@@ -97,7 +97,7 @@ export interface StageOptions {
|
|
|
97
97
|
export interface RunOptions {
|
|
98
98
|
onProgress?: (itemsProcessed: number, quadsGenerated: number) => void;
|
|
99
99
|
/**
|
|
100
|
-
* Per-dataset {@link TimeoutPolicy} threaded through to
|
|
100
|
+
* Per-dataset {@link TimeoutPolicy} threaded through to readers and
|
|
101
101
|
* item selectors. The Pipeline owns lifecycle (factory invocation per
|
|
102
102
|
* dataset), so a single policy instance covers all stages and child
|
|
103
103
|
* stages within one dataset.
|
|
@@ -114,7 +114,7 @@ export declare class Stage {
|
|
|
114
114
|
readonly stages: readonly Stage[];
|
|
115
115
|
/** Whether an empty result is treated as a hard failure. @see {@link StageOptions.expectsOutput} */
|
|
116
116
|
readonly expectsOutput: boolean;
|
|
117
|
-
private readonly
|
|
117
|
+
private readonly readers;
|
|
118
118
|
private readonly itemSelector?;
|
|
119
119
|
private readonly batchSize;
|
|
120
120
|
private readonly maxConcurrency;
|
|
@@ -122,7 +122,7 @@ export declare class Stage {
|
|
|
122
122
|
constructor(options: StageOptions);
|
|
123
123
|
/** The validator for this stage, if configured. */
|
|
124
124
|
get validator(): Validator | undefined;
|
|
125
|
-
run(dataset: Dataset, distribution: Distribution, writer:
|
|
125
|
+
run(dataset: Dataset, distribution: Distribution, writer: DatasetWriter, options?: RunOptions): Promise<NotSupported | void>;
|
|
126
126
|
/**
|
|
127
127
|
* Throw when {@link StageOptions.expectsOutput} is set but the stage produced
|
|
128
128
|
* no quads — a supported-but-empty result that signals a truncated or aborted
|
|
@@ -135,16 +135,16 @@ export declare class Stage {
|
|
|
135
135
|
* (empty array when skipping invalid batches).
|
|
136
136
|
*/
|
|
137
137
|
private validateBuffer;
|
|
138
|
-
private
|
|
138
|
+
private readAll;
|
|
139
139
|
/**
|
|
140
|
-
* Fold
|
|
141
|
-
* supplying the {@link
|
|
142
|
-
* call's output (see {@link
|
|
140
|
+
* Fold a reader's attached transforms over its output stream, in order,
|
|
141
|
+
* supplying the {@link ReaderContext}. A transform sees one `read()`
|
|
142
|
+
* call's output (see {@link AttachedReader}); `NotSupported` is handled by
|
|
143
143
|
* the caller and never reaches a transform.
|
|
144
144
|
*/
|
|
145
145
|
private applyTransforms;
|
|
146
146
|
}
|
|
147
|
-
/** Selects items (as variable bindings) for
|
|
147
|
+
/** Selects items (as variable bindings) for readers to process. Pagination is an implementation detail. */
|
|
148
148
|
export interface ItemSelector {
|
|
149
149
|
select(distribution: Distribution, batchSize?: number, options?: SelectOptions): AsyncIterable<VariableBindings>;
|
|
150
150
|
}
|
package/dist/stage.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE/D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAGxD;;;;;;;GAOG;AACH,MAAM,MAAM,aAAa,CAAC,GAAG,IAAI,CAC/B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,EAC1B,OAAO,EAAE,GAAG,KACT,aAAa,CAAC,IAAI,CAAC,CAAC;AAEzB;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,YAAY,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,aAAa,CAAC,aAAa,CAAC,GAAG,aAAa,CAAC,aAAa,CAAC,EAAE,CAAC;CAC3E;AAED,yEAAyE;AACzE,MAAM,MAAM,YAAY,GACpB,MAAM,GACN,cAAc,GACd,CAAC,MAAM,GAAG,cAAc,CAAC,EAAE,CAAC;AAQhC,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,YAAY,CAAC;IACtB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;;;OAOG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;IACjB;;;;;;;;;;;;;;;OAeG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,mFAAmF;IACnF,UAAU,CAAC,EAAE;QACX,SAAS,EAAE,SAAS,CAAC;QACrB,iEAAiE;QACjE,SAAS,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;KACvC,CAAC;CACH;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,CAAC,cAAc,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;IACtE;;;;;OAKG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,uDAAuD;AACvD,MAAM,WAAW,aAAa;IAC5B,+BAA+B;IAC/B,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,oGAAoG;IACpG,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;IAC7C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAA6B;gBAE7C,OAAO,EAAE,YAAY;IAWjC,mDAAmD;IACnD,IAAI,SAAS,IAAI,SAAS,GAAG,SAAS,CAErC;IAEK,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,aAAa,EACrB,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;IAqE/B;;;;OAIG;IACH,OAAO,CAAC,cAAc;YAMR,eAAe;IAgK7B;;;OAGG;YACW,cAAc;YAqBd,OAAO;IA6BrB;;;;;OAKG;IACH,OAAO,CAAC,eAAe;CAiBxB;AAiDD,2GAA2G;AAC3G,MAAM,WAAW,YAAY;IAC3B,MAAM,CACJ,YAAY,EAAE,YAAY,EAC1B,SAAS,CAAC,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,aAAa,GACtB,aAAa,CAAC,gBAAgB,CAAC,CAAC;CACpC"}
|
package/dist/stage.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { NotSupported } from './sparql/
|
|
1
|
+
import { NotSupported } from './sparql/reader.js';
|
|
2
2
|
import { batch } from './batch.js';
|
|
3
3
|
import { AsyncQueue } from './asyncQueue.js';
|
|
4
4
|
export class Stage {
|
|
@@ -6,7 +6,7 @@ export class Stage {
|
|
|
6
6
|
stages;
|
|
7
7
|
/** Whether an empty result is treated as a hard failure. @see {@link StageOptions.expectsOutput} */
|
|
8
8
|
expectsOutput;
|
|
9
|
-
|
|
9
|
+
readers;
|
|
10
10
|
itemSelector;
|
|
11
11
|
batchSize;
|
|
12
12
|
maxConcurrency;
|
|
@@ -14,7 +14,7 @@ export class Stage {
|
|
|
14
14
|
constructor(options) {
|
|
15
15
|
this.name = options.name;
|
|
16
16
|
this.stages = options.stages ?? [];
|
|
17
|
-
this.
|
|
17
|
+
this.readers = normalizeReaders(options.readers);
|
|
18
18
|
this.itemSelector = options.itemSelector;
|
|
19
19
|
this.batchSize = options.batchSize ?? 10;
|
|
20
20
|
this.maxConcurrency = options.maxConcurrency ?? 10;
|
|
@@ -32,11 +32,11 @@ export class Stage {
|
|
|
32
32
|
timeout,
|
|
33
33
|
}), dataset, distribution, writer, options);
|
|
34
34
|
}
|
|
35
|
-
const streams = await this.
|
|
35
|
+
const streams = await this.readAll(dataset, distribution, timeout);
|
|
36
36
|
if (streams instanceof NotSupported) {
|
|
37
37
|
return streams;
|
|
38
38
|
}
|
|
39
|
-
// Quads the
|
|
39
|
+
// Quads the readers produced (before any validation filtering); used to
|
|
40
40
|
// enforce `expectsOutput` below.
|
|
41
41
|
let produced = 0;
|
|
42
42
|
if (this.validation) {
|
|
@@ -116,10 +116,10 @@ export class Stage {
|
|
|
116
116
|
const dispatch = async () => {
|
|
117
117
|
const inFlight = new Set();
|
|
118
118
|
let firstError;
|
|
119
|
-
// Divide maxConcurrency by
|
|
119
|
+
// Divide maxConcurrency by reader count so the total concurrent
|
|
120
120
|
// SPARQL queries stays at maxConcurrency (each batch runs all
|
|
121
|
-
//
|
|
122
|
-
const maxConcurrentBatches = Math.max(1, Math.floor(this.maxConcurrency / this.
|
|
121
|
+
// readers in parallel).
|
|
122
|
+
const maxConcurrentBatches = Math.max(1, Math.floor(this.maxConcurrency / this.readers.length));
|
|
123
123
|
const track = (promise) => {
|
|
124
124
|
const p = promise.then(() => {
|
|
125
125
|
inFlight.delete(p);
|
|
@@ -140,9 +140,9 @@ export class Stage {
|
|
|
140
140
|
break;
|
|
141
141
|
}
|
|
142
142
|
track((async () => {
|
|
143
|
-
// Run all
|
|
144
|
-
const
|
|
145
|
-
const result = await
|
|
143
|
+
// Run all readers for this batch in parallel.
|
|
144
|
+
const readerOutputs = await Promise.all(this.readers.map(async ({ reader, transforms }) => {
|
|
145
|
+
const result = await reader.read(dataset, distribution, {
|
|
146
146
|
bindings,
|
|
147
147
|
timeout: options?.timeout,
|
|
148
148
|
});
|
|
@@ -156,7 +156,7 @@ export class Stage {
|
|
|
156
156
|
}
|
|
157
157
|
return quads;
|
|
158
158
|
}));
|
|
159
|
-
const batchQuads =
|
|
159
|
+
const batchQuads = readerOutputs.flat();
|
|
160
160
|
if (this.validation &&
|
|
161
161
|
batchQuads.length > 0 &&
|
|
162
162
|
onInvalid !== 'write') {
|
|
@@ -208,7 +208,7 @@ export class Stage {
|
|
|
208
208
|
})();
|
|
209
209
|
await Promise.all([dispatchPromise, writePromise]);
|
|
210
210
|
if (!hasResults) {
|
|
211
|
-
return new NotSupported('All
|
|
211
|
+
return new NotSupported('All readers returned NotSupported');
|
|
212
212
|
}
|
|
213
213
|
this.assertProduced(quadsGenerated);
|
|
214
214
|
}
|
|
@@ -228,9 +228,9 @@ export class Stage {
|
|
|
228
228
|
// 'skip': discard
|
|
229
229
|
return [];
|
|
230
230
|
}
|
|
231
|
-
async
|
|
232
|
-
const results = await Promise.all(this.
|
|
233
|
-
const result = await
|
|
231
|
+
async readAll(dataset, distribution, timeout) {
|
|
232
|
+
const results = await Promise.all(this.readers.map(async ({ reader, transforms }) => {
|
|
233
|
+
const result = await reader.read(dataset, distribution, {
|
|
234
234
|
timeout,
|
|
235
235
|
});
|
|
236
236
|
if (result instanceof NotSupported)
|
|
@@ -244,14 +244,14 @@ export class Stage {
|
|
|
244
244
|
}
|
|
245
245
|
}
|
|
246
246
|
if (streams.length === 0) {
|
|
247
|
-
return new NotSupported('All
|
|
247
|
+
return new NotSupported('All readers returned NotSupported');
|
|
248
248
|
}
|
|
249
249
|
return streams;
|
|
250
250
|
}
|
|
251
251
|
/**
|
|
252
|
-
* Fold
|
|
253
|
-
* supplying the {@link
|
|
254
|
-
* call's output (see {@link
|
|
252
|
+
* Fold a reader's attached transforms over its output stream, in order,
|
|
253
|
+
* supplying the {@link ReaderContext}. A transform sees one `read()`
|
|
254
|
+
* call's output (see {@link AttachedReader}); `NotSupported` is handled by
|
|
255
255
|
* the caller and never reaches a transform.
|
|
256
256
|
*/
|
|
257
257
|
applyTransforms(transforms, stream, dataset, distribution) {
|
|
@@ -265,20 +265,20 @@ export class Stage {
|
|
|
265
265
|
return transforms.reduce((quads, transform) => transform(quads, context), stream);
|
|
266
266
|
}
|
|
267
267
|
}
|
|
268
|
-
/** Normalise the {@link
|
|
269
|
-
function
|
|
270
|
-
const list = Array.isArray(
|
|
268
|
+
/** Normalise the {@link StageReaders} union to reader + transforms pairs. */
|
|
269
|
+
function normalizeReaders(readers) {
|
|
270
|
+
const list = Array.isArray(readers) ? readers : [readers];
|
|
271
271
|
return list.map((entry) => {
|
|
272
|
-
if ('
|
|
273
|
-
return {
|
|
272
|
+
if ('read' in entry) {
|
|
273
|
+
return { reader: entry, transforms: [] };
|
|
274
274
|
}
|
|
275
|
-
const {
|
|
275
|
+
const { reader, transform } = entry;
|
|
276
276
|
const transforms = transform === undefined
|
|
277
277
|
? []
|
|
278
278
|
: Array.isArray(transform)
|
|
279
279
|
? [...transform]
|
|
280
280
|
: [transform];
|
|
281
|
-
return {
|
|
281
|
+
return { reader, transforms };
|
|
282
282
|
});
|
|
283
283
|
}
|
|
284
284
|
async function* mergeStreams(streams) {
|