@lde/pipeline 0.6.13 → 0.6.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/distribution/analyzer.d.ts +5 -8
- package/dist/distribution/analyzer.d.ts.map +1 -1
- package/dist/distribution/analyzer.js +25 -14
- package/dist/distribution/index.d.ts +2 -1
- package/dist/distribution/index.d.ts.map +1 -1
- package/dist/distribution/index.js +2 -1
- package/dist/distribution/probe.d.ts +1 -1
- package/dist/distribution/probe.js +3 -9
- package/dist/distribution/resolver.d.ts +35 -0
- package/dist/distribution/resolver.d.ts.map +1 -0
- package/dist/distribution/resolver.js +57 -0
- package/dist/index.js +1 -0
- package/dist/pipeline.d.ts.map +1 -1
- package/dist/pipeline.js +2 -1
- package/dist/sparql/executor.d.ts +16 -32
- package/dist/sparql/executor.d.ts.map +1 -1
- package/dist/sparql/executor.js +10 -17
- package/dist/sparql/index.d.ts +1 -1
- package/dist/sparql/index.d.ts.map +1 -1
- package/dist/stage.d.ts +3 -2
- package/dist/stage.d.ts.map +1 -1
- package/dist/stage.js +2 -2
- package/dist/step/sparqlQuery.d.ts +3 -3
- package/dist/step/sparqlQuery.d.ts.map +1 -1
- package/dist/step/sparqlQuery.js +2 -7
- package/dist/step.d.ts +2 -2
- package/dist/step.d.ts.map +1 -1
- package/package.json +5 -5
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { Dataset } from '@lde/dataset';
|
|
2
2
|
import { Importer, ImportFailed, ImportSuccessful, NotSupported } from '@lde/sparql-importer';
|
|
3
3
|
import { Store } from 'n3';
|
|
4
|
+
import { NoDistributionAvailable } from './resolver.js';
|
|
4
5
|
export type { Importer };
|
|
5
6
|
export { ImportFailed, ImportSuccessful, NotSupported };
|
|
6
7
|
/**
|
|
@@ -20,20 +21,12 @@ export interface DistributionAnalyzerOptions {
|
|
|
20
21
|
*/
|
|
21
22
|
timeout?: number;
|
|
22
23
|
}
|
|
23
|
-
/**
|
|
24
|
-
* Result indicating the analyzer could not find a usable distribution.
|
|
25
|
-
*/
|
|
26
|
-
export declare class NoDistributionAvailable {
|
|
27
|
-
readonly message: string;
|
|
28
|
-
constructor(message: string);
|
|
29
|
-
}
|
|
30
24
|
/**
|
|
31
25
|
* Analyzes dataset distributions by probing their availability.
|
|
32
26
|
*
|
|
33
27
|
* - Probes SPARQL endpoints with a simple SELECT query
|
|
34
28
|
* - Probes data dumps with HEAD/GET requests
|
|
35
29
|
* - Records probe results as RDF (schema:Action)
|
|
36
|
-
* - Updates distribution metadata (isValid, lastModified, byteSize)
|
|
37
30
|
* - Optionally imports data dumps if no SPARQL endpoint is available
|
|
38
31
|
*/
|
|
39
32
|
export declare class DistributionAnalyzer {
|
|
@@ -51,6 +44,10 @@ export declare class DistributionAnalyzer {
|
|
|
51
44
|
* Cleanup resources (e.g., importer connections).
|
|
52
45
|
*/
|
|
53
46
|
finish(): Promise<void>;
|
|
47
|
+
/**
|
|
48
|
+
* Apply probe results to distributions so `getSparqlDistribution()` works.
|
|
49
|
+
*/
|
|
50
|
+
private applyProbeResults;
|
|
54
51
|
private buildProbeResultsRdf;
|
|
55
52
|
private addSuccessTriples;
|
|
56
53
|
private addImportError;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"analyzer.d.ts","sourceRoot":"","sources":["../../src/distribution/analyzer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AACrD,OAAO,EACL,QAAQ,EACR,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACb,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAe,KAAK,EAAE,MAAM,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"analyzer.d.ts","sourceRoot":"","sources":["../../src/distribution/analyzer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AACrD,OAAO,EACL,QAAQ,EACR,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACb,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAe,KAAK,EAAE,MAAM,IAAI,CAAC;AAQxC,OAAO,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AAExD,YAAY,EAAE,QAAQ,EAAE,CAAC;AACzB,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,YAAY,EAAE,CAAC;AAWxD;;GAEG;AACH,MAAM,WAAW,kBAAmB,SAAQ,QAAQ;IAClD,MAAM,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,MAAM,WAAW,2BAA2B;IAC1C;;OAEG;IACH,QAAQ,CAAC,EAAE,kBAAkB,CAAC;IAE9B;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;GAOG;AACH,qBAAa,oBAAoB;IAC/B,SAAgB,IAAI,kBAAkB;IACtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAqB;IAC/C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;gBAErB,OAAO,CAAC,EAAE,2BAA2B;IAKjD;;;;OAIG;IACG,OAAO,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,KAAK,GAAG,uBAAuB,CAAC;IAuCzE;;OAEG;IACG,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAI7B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAsBzB,OAAO,CAAC,oBAAoB;IAwC5B,OAAO,CAAC,iBAAiB;IAiDzB,OAAO,CAAC,cAAc;CAevB"}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { Distribution } from '@lde/dataset';
|
|
2
2
|
import { ImportFailed, ImportSuccessful, NotSupported, } from '@lde/sparql-importer';
|
|
3
3
|
import { DataFactory, Store } from 'n3';
|
|
4
|
-
import { probe, NetworkError, SparqlProbeResult, } from './probe.js';
|
|
4
|
+
import { probe, NetworkError, SparqlProbeResult, DataDumpProbeResult, } from './probe.js';
|
|
5
|
+
import { NoDistributionAvailable } from './resolver.js';
|
|
5
6
|
export { ImportFailed, ImportSuccessful, NotSupported };
|
|
6
7
|
const { quad, namedNode, blankNode, literal } = DataFactory;
|
|
7
8
|
// Namespace prefixes
|
|
@@ -10,22 +11,12 @@ const SCHEMA = 'https://schema.org/';
|
|
|
10
11
|
const VOID = 'http://rdfs.org/ns/void#';
|
|
11
12
|
const XSD = 'http://www.w3.org/2001/XMLSchema#';
|
|
12
13
|
const HTTP_STATUS = 'https://www.w3.org/2011/http-statusCodes#';
|
|
13
|
-
/**
|
|
14
|
-
* Result indicating the analyzer could not find a usable distribution.
|
|
15
|
-
*/
|
|
16
|
-
export class NoDistributionAvailable {
|
|
17
|
-
message;
|
|
18
|
-
constructor(message) {
|
|
19
|
-
this.message = message;
|
|
20
|
-
}
|
|
21
|
-
}
|
|
22
14
|
/**
|
|
23
15
|
* Analyzes dataset distributions by probing their availability.
|
|
24
16
|
*
|
|
25
17
|
* - Probes SPARQL endpoints with a simple SELECT query
|
|
26
18
|
* - Probes data dumps with HEAD/GET requests
|
|
27
19
|
* - Records probe results as RDF (schema:Action)
|
|
28
|
-
* - Updates distribution metadata (isValid, lastModified, byteSize)
|
|
29
20
|
* - Optionally imports data dumps if no SPARQL endpoint is available
|
|
30
21
|
*/
|
|
31
22
|
export class DistributionAnalyzer {
|
|
@@ -42,8 +33,10 @@ export class DistributionAnalyzer {
|
|
|
42
33
|
* @returns Store with probe results as RDF, or NoDistributionAvailable if no usable distribution found
|
|
43
34
|
*/
|
|
44
35
|
async execute(dataset) {
|
|
45
|
-
const
|
|
46
|
-
|
|
36
|
+
const probeResults = await Promise.all(dataset.distributions.map((distribution) => probe(distribution, this.timeout)));
|
|
37
|
+
// Apply probe results to distributions (isValid, lastModified, byteSize).
|
|
38
|
+
this.applyProbeResults(dataset.distributions, probeResults);
|
|
39
|
+
const store = this.buildProbeResultsRdf(probeResults, dataset);
|
|
47
40
|
// If no SPARQL endpoint available, try to import a data dump
|
|
48
41
|
if (dataset.getSparqlDistribution() === null && this.importer) {
|
|
49
42
|
const importResult = await this.importer.import(dataset);
|
|
@@ -58,7 +51,7 @@ export class DistributionAnalyzer {
|
|
|
58
51
|
}
|
|
59
52
|
}
|
|
60
53
|
if (dataset.getSparqlDistribution() === null) {
|
|
61
|
-
return new NoDistributionAvailable('No SPARQL endpoint or importable data dump available');
|
|
54
|
+
return new NoDistributionAvailable(dataset, 'No SPARQL endpoint or importable data dump available');
|
|
62
55
|
}
|
|
63
56
|
return store;
|
|
64
57
|
}
|
|
@@ -68,6 +61,24 @@ export class DistributionAnalyzer {
|
|
|
68
61
|
async finish() {
|
|
69
62
|
await this.importer?.finish?.();
|
|
70
63
|
}
|
|
64
|
+
/**
|
|
65
|
+
* Apply probe results to distributions so `getSparqlDistribution()` works.
|
|
66
|
+
*/
|
|
67
|
+
applyProbeResults(distributions, results) {
|
|
68
|
+
for (let i = 0; i < distributions.length; i++) {
|
|
69
|
+
const distribution = distributions[i];
|
|
70
|
+
const result = results[i];
|
|
71
|
+
if (result instanceof NetworkError) {
|
|
72
|
+
distribution.isValid = false;
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
distribution.isValid = result.isSuccess();
|
|
76
|
+
if (result instanceof DataDumpProbeResult) {
|
|
77
|
+
distribution.lastModified ??= result.lastModified ?? undefined;
|
|
78
|
+
distribution.byteSize ??= result.contentSize ?? undefined;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
71
82
|
buildProbeResultsRdf(results, dataset) {
|
|
72
83
|
const store = new Store();
|
|
73
84
|
for (const result of results) {
|
|
@@ -1,3 +1,4 @@
|
|
|
1
1
|
export { probe, NetworkError, SparqlProbeResult, DataDumpProbeResult, type ProbeResultType, } from './probe.js';
|
|
2
|
-
export { DistributionAnalyzer, ImportSuccessful, ImportFailed,
|
|
2
|
+
export { DistributionAnalyzer, ImportSuccessful, ImportFailed, type Importer, type DistributionAnalyzerOptions, } from './analyzer.js';
|
|
3
|
+
export { ResolvedDistribution, NoDistributionAvailable, SparqlDistributionResolver, type DistributionResolver, type SparqlDistributionResolverOptions, } from './resolver.js';
|
|
3
4
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/distribution/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,EACL,YAAY,EACZ,iBAAiB,EACjB,mBAAmB,EACnB,KAAK,eAAe,GACrB,MAAM,YAAY,CAAC;AAEpB,OAAO,EACL,oBAAoB,EACpB,gBAAgB,EAChB,YAAY,EACZ,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/distribution/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,EACL,YAAY,EACZ,iBAAiB,EACjB,mBAAmB,EACnB,KAAK,eAAe,GACrB,MAAM,YAAY,CAAC;AAEpB,OAAO,EACL,oBAAoB,EACpB,gBAAgB,EAChB,YAAY,EACZ,KAAK,QAAQ,EACb,KAAK,2BAA2B,GACjC,MAAM,eAAe,CAAC;AAEvB,OAAO,EACL,oBAAoB,EACpB,uBAAuB,EACvB,0BAA0B,EAC1B,KAAK,oBAAoB,EACzB,KAAK,iCAAiC,GACvC,MAAM,eAAe,CAAC"}
|
|
@@ -1,2 +1,3 @@
|
|
|
1
1
|
export { probe, NetworkError, SparqlProbeResult, DataDumpProbeResult, } from './probe.js';
|
|
2
|
-
export { DistributionAnalyzer, ImportSuccessful, ImportFailed,
|
|
2
|
+
export { DistributionAnalyzer, ImportSuccessful, ImportFailed, } from './analyzer.js';
|
|
3
|
+
export { ResolvedDistribution, NoDistributionAvailable, SparqlDistributionResolver, } from './resolver.js';
|
|
@@ -40,7 +40,7 @@ export type ProbeResultType = SparqlProbeResult | DataDumpProbeResult | NetworkE
|
|
|
40
40
|
* For SPARQL endpoints, sends a simple SELECT query.
|
|
41
41
|
* For data dumps, sends HEAD (or GET if HEAD returns no Content-Length).
|
|
42
42
|
*
|
|
43
|
-
*
|
|
43
|
+
* Returns pure probe results without mutating the distribution.
|
|
44
44
|
*/
|
|
45
45
|
export declare function probe(distribution: Distribution, timeout?: number): Promise<ProbeResultType>;
|
|
46
46
|
export {};
|
|
@@ -61,7 +61,7 @@ export class DataDumpProbeResult extends ProbeResult {
|
|
|
61
61
|
* For SPARQL endpoints, sends a simple SELECT query.
|
|
62
62
|
* For data dumps, sends HEAD (or GET if HEAD returns no Content-Length).
|
|
63
63
|
*
|
|
64
|
-
*
|
|
64
|
+
* Returns pure probe results without mutating the distribution.
|
|
65
65
|
*/
|
|
66
66
|
export async function probe(distribution, timeout = 5000) {
|
|
67
67
|
try {
|
|
@@ -85,9 +85,7 @@ async function probeSparqlEndpoint(distribution, timeout) {
|
|
|
85
85
|
},
|
|
86
86
|
body: `query=${encodeURIComponent('SELECT * { ?s ?p ?o } LIMIT 1')}`,
|
|
87
87
|
});
|
|
88
|
-
|
|
89
|
-
distribution.isValid = result.isSuccess();
|
|
90
|
-
return result;
|
|
88
|
+
return new SparqlProbeResult(url, response);
|
|
91
89
|
}
|
|
92
90
|
async function probeDataDump(distribution, timeout) {
|
|
93
91
|
const url = distribution.accessUrl.toString();
|
|
@@ -112,9 +110,5 @@ async function probeDataDump(distribution, timeout) {
|
|
|
112
110
|
...requestOptions,
|
|
113
111
|
});
|
|
114
112
|
}
|
|
115
|
-
|
|
116
|
-
distribution.isValid = result.isSuccess();
|
|
117
|
-
distribution.lastModified ??= result.lastModified ?? undefined;
|
|
118
|
-
distribution.byteSize ??= result.contentSize ?? undefined;
|
|
119
|
-
return result;
|
|
113
|
+
return new DataDumpProbeResult(url, response);
|
|
120
114
|
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { Dataset, Distribution } from '@lde/dataset';
|
|
2
|
+
import type { Importer } from '@lde/sparql-importer';
|
|
3
|
+
export declare class ResolvedDistribution {
|
|
4
|
+
readonly distribution: Distribution;
|
|
5
|
+
constructor(distribution: Distribution);
|
|
6
|
+
}
|
|
7
|
+
export declare class NoDistributionAvailable {
|
|
8
|
+
readonly dataset: Dataset;
|
|
9
|
+
readonly message: string;
|
|
10
|
+
constructor(dataset: Dataset, message: string);
|
|
11
|
+
}
|
|
12
|
+
export interface DistributionResolver {
|
|
13
|
+
resolve(dataset: Dataset): Promise<ResolvedDistribution | NoDistributionAvailable>;
|
|
14
|
+
}
|
|
15
|
+
export interface SparqlDistributionResolverOptions {
|
|
16
|
+
importer?: Importer;
|
|
17
|
+
timeout?: number;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Resolves a dataset to a usable SPARQL distribution by probing its distributions.
|
|
21
|
+
*
|
|
22
|
+
* 1. Probes all distributions in parallel.
|
|
23
|
+
* 2. Returns the first valid SPARQL endpoint as a `ResolvedDistribution`.
|
|
24
|
+
* 3. If none: tries the importer (if provided) and returns the imported distribution.
|
|
25
|
+
* 4. If nothing works: returns `NoDistributionAvailable`.
|
|
26
|
+
*
|
|
27
|
+
* Does not mutate `dataset.distributions`.
|
|
28
|
+
*/
|
|
29
|
+
export declare class SparqlDistributionResolver implements DistributionResolver {
|
|
30
|
+
private readonly importer?;
|
|
31
|
+
private readonly timeout;
|
|
32
|
+
constructor(options?: SparqlDistributionResolverOptions);
|
|
33
|
+
resolve(dataset: Dataset): Promise<ResolvedDistribution | NoDistributionAvailable>;
|
|
34
|
+
}
|
|
35
|
+
//# sourceMappingURL=resolver.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"resolver.d.ts","sourceRoot":"","sources":["../../src/distribution/resolver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAIrD,qBAAa,oBAAoB;IACnB,QAAQ,CAAC,YAAY,EAAE,YAAY;gBAA1B,YAAY,EAAE,YAAY;CAChD;AAED,qBAAa,uBAAuB;IACtB,QAAQ,CAAC,OAAO,EAAE,OAAO;IAAE,QAAQ,CAAC,OAAO,EAAE,MAAM;gBAA1C,OAAO,EAAE,OAAO,EAAW,OAAO,EAAE,MAAM;CAChE;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,CACL,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC,CAAC;CAC5D;AAED,MAAM,WAAW,iCAAiC;IAChD,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;;;GASG;AACH,qBAAa,0BAA2B,YAAW,oBAAoB;IACrE,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAW;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;gBAErB,OAAO,CAAC,EAAE,iCAAiC;IAKjD,OAAO,CACX,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC;CAsC3D"}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { Distribution } from '@lde/dataset';
|
|
2
|
+
import { ImportSuccessful } from '@lde/sparql-importer';
|
|
3
|
+
import { probe, SparqlProbeResult } from './probe.js';
|
|
4
|
+
export class ResolvedDistribution {
|
|
5
|
+
distribution;
|
|
6
|
+
constructor(distribution) {
|
|
7
|
+
this.distribution = distribution;
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
export class NoDistributionAvailable {
|
|
11
|
+
dataset;
|
|
12
|
+
message;
|
|
13
|
+
constructor(dataset, message) {
|
|
14
|
+
this.dataset = dataset;
|
|
15
|
+
this.message = message;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Resolves a dataset to a usable SPARQL distribution by probing its distributions.
|
|
20
|
+
*
|
|
21
|
+
* 1. Probes all distributions in parallel.
|
|
22
|
+
* 2. Returns the first valid SPARQL endpoint as a `ResolvedDistribution`.
|
|
23
|
+
* 3. If none: tries the importer (if provided) and returns the imported distribution.
|
|
24
|
+
* 4. If nothing works: returns `NoDistributionAvailable`.
|
|
25
|
+
*
|
|
26
|
+
* Does not mutate `dataset.distributions`.
|
|
27
|
+
*/
|
|
28
|
+
export class SparqlDistributionResolver {
|
|
29
|
+
importer;
|
|
30
|
+
timeout;
|
|
31
|
+
constructor(options) {
|
|
32
|
+
this.importer = options?.importer;
|
|
33
|
+
this.timeout = options?.timeout ?? 5000;
|
|
34
|
+
}
|
|
35
|
+
async resolve(dataset) {
|
|
36
|
+
const results = await Promise.all(dataset.distributions.map((distribution) => probe(distribution, this.timeout)));
|
|
37
|
+
// Find first valid SPARQL endpoint.
|
|
38
|
+
for (let i = 0; i < dataset.distributions.length; i++) {
|
|
39
|
+
const distribution = dataset.distributions[i];
|
|
40
|
+
const result = results[i];
|
|
41
|
+
if (distribution.isSparql() &&
|
|
42
|
+
result instanceof SparqlProbeResult &&
|
|
43
|
+
result.isSuccess()) {
|
|
44
|
+
return new ResolvedDistribution(distribution);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
// No SPARQL endpoint; try importer if available.
|
|
48
|
+
if (this.importer) {
|
|
49
|
+
const importResult = await this.importer.import(dataset);
|
|
50
|
+
if (importResult instanceof ImportSuccessful) {
|
|
51
|
+
const distribution = Distribution.sparql(importResult.distribution.accessUrl, importResult.identifier);
|
|
52
|
+
return new ResolvedDistribution(distribution);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return new NoDistributionAvailable(dataset, 'No SPARQL endpoint or importable data dump available');
|
|
56
|
+
}
|
|
57
|
+
}
|
package/dist/index.js
CHANGED
package/dist/pipeline.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAc,IAAI,EAAE,MAAM,WAAW,CAAC;AAI7C,qBAAa,QAAQ;IACP,OAAO,CAAC,QAAQ,CAAC,MAAM;gBAAN,MAAM,EAAE;QAAE,QAAQ,EAAE,QAAQ,CAAC;QAAC,KAAK,EAAE,IAAI,EAAE,CAAA;KAAE;IAE7D,GAAG;
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAc,IAAI,EAAE,MAAM,WAAW,CAAC;AAI7C,qBAAa,QAAQ;IACP,OAAO,CAAC,QAAQ,CAAC,MAAM;gBAAN,MAAM,EAAE;QAAE,QAAQ,EAAE,QAAQ,CAAC;QAAC,KAAK,EAAE,IAAI,EAAE,CAAA;KAAE;IAE7D,GAAG;CAiCjB"}
|
package/dist/pipeline.js
CHANGED
|
@@ -8,8 +8,9 @@ export class Pipeline {
|
|
|
8
8
|
async run() {
|
|
9
9
|
const datasets = await this.config.selector.select();
|
|
10
10
|
for await (const dataset of datasets) {
|
|
11
|
+
const distribution = dataset.getSparqlDistribution() ?? undefined;
|
|
11
12
|
for (const step of this.config.steps) {
|
|
12
|
-
const result = await step.execute(dataset);
|
|
13
|
+
const result = await step.execute(dataset, distribution);
|
|
13
14
|
if (result instanceof NotSupported) {
|
|
14
15
|
console.error(result);
|
|
15
16
|
}
|
|
@@ -6,24 +6,21 @@ import { NotSupported } from '../step.js';
|
|
|
6
6
|
export { NotSupported } from '../step.js';
|
|
7
7
|
/** A single row of variable bindings (variable name → NamedNode). */
|
|
8
8
|
export type VariableBindings = Record<string, NamedNode>;
|
|
9
|
+
export interface ExecuteOptions {
|
|
10
|
+
/**
|
|
11
|
+
* Variable bindings to inject as a VALUES clause into the query.
|
|
12
|
+
* When non-empty, a VALUES block is prepended to the WHERE clause.
|
|
13
|
+
*/
|
|
14
|
+
bindings?: VariableBindings[];
|
|
15
|
+
}
|
|
9
16
|
export interface Executor {
|
|
10
|
-
execute(dataset:
|
|
17
|
+
execute(dataset: Dataset, distribution: Distribution, options?: ExecuteOptions): Promise<AsyncIterable<Quad> | NotSupported>;
|
|
11
18
|
}
|
|
12
19
|
/**
|
|
13
20
|
* A quad stream that is both an RDFJS Stream and Node.js Readable (async iterable).
|
|
14
21
|
* This is the actual return type from SparqlEndpointFetcher.fetchTriples().
|
|
15
22
|
*/
|
|
16
23
|
export type QuadStream = Readable & Stream<Quad>;
|
|
17
|
-
/**
|
|
18
|
-
* Extended dataset with optional SPARQL filtering options.
|
|
19
|
-
*/
|
|
20
|
-
export interface ExecutableDataset extends Dataset {
|
|
21
|
-
/**
|
|
22
|
-
* Optional SPARQL filter clause to restrict analysis to a subset of the data.
|
|
23
|
-
* This is substituted for `#subjectFilter#` in queries.
|
|
24
|
-
*/
|
|
25
|
-
subjectFilter?: string;
|
|
26
|
-
}
|
|
27
24
|
/**
|
|
28
25
|
* Options for SparqlConstructExecutor.
|
|
29
26
|
*/
|
|
@@ -42,20 +39,6 @@ export interface SparqlConstructExecutorOptions {
|
|
|
42
39
|
*/
|
|
43
40
|
fetcher?: SparqlEndpointFetcher;
|
|
44
41
|
}
|
|
45
|
-
/**
|
|
46
|
-
* Options for `execute()`.
|
|
47
|
-
*/
|
|
48
|
-
export interface SparqlConstructExecuteOptions {
|
|
49
|
-
/**
|
|
50
|
-
* Explicit SPARQL endpoint URL. If not provided, uses the dataset's SPARQL distribution.
|
|
51
|
-
*/
|
|
52
|
-
endpoint?: URL;
|
|
53
|
-
/**
|
|
54
|
-
* Variable bindings to inject as a VALUES clause into the query.
|
|
55
|
-
* When non-empty, a VALUES block is prepended to the WHERE clause.
|
|
56
|
-
*/
|
|
57
|
-
bindings?: VariableBindings[];
|
|
58
|
-
}
|
|
59
42
|
/**
|
|
60
43
|
* A streaming SPARQL CONSTRUCT executor that parses the query once (in the
|
|
61
44
|
* constructor) and operates on the AST for graph and VALUES injection.
|
|
@@ -69,7 +52,7 @@ export interface SparqlConstructExecuteOptions {
|
|
|
69
52
|
* const executor = new SparqlConstructExecutor({
|
|
70
53
|
* query: 'CONSTRUCT { ?dataset ?p ?o } WHERE { ?s ?p ?o }',
|
|
71
54
|
* });
|
|
72
|
-
* const result = await executor.execute(dataset);
|
|
55
|
+
* const result = await executor.execute(dataset, distribution);
|
|
73
56
|
* if (result instanceof NotSupported) {
|
|
74
57
|
* console.log(result.message);
|
|
75
58
|
* } else {
|
|
@@ -85,13 +68,14 @@ export declare class SparqlConstructExecutor implements Executor {
|
|
|
85
68
|
private readonly generator;
|
|
86
69
|
constructor(options: SparqlConstructExecutorOptions);
|
|
87
70
|
/**
|
|
88
|
-
* Execute the SPARQL CONSTRUCT query against the
|
|
71
|
+
* Execute the SPARQL CONSTRUCT query against the distribution's endpoint.
|
|
89
72
|
*
|
|
90
73
|
* @param dataset The dataset to execute against.
|
|
91
|
-
* @param
|
|
92
|
-
* @
|
|
74
|
+
* @param distribution The distribution providing the SPARQL endpoint.
|
|
75
|
+
* @param options Optional execution options (bindings).
|
|
76
|
+
* @returns AsyncIterable<Quad> stream of results.
|
|
93
77
|
*/
|
|
94
|
-
execute(dataset:
|
|
78
|
+
execute(dataset: Dataset, distribution: Distribution, options?: ExecuteOptions): Promise<QuadStream>;
|
|
95
79
|
/**
|
|
96
80
|
* Create an executor from a query file.
|
|
97
81
|
*
|
|
@@ -103,11 +87,11 @@ export declare class SparqlConstructExecutor implements Executor {
|
|
|
103
87
|
/**
|
|
104
88
|
* Substitute template variables in a SPARQL query.
|
|
105
89
|
*
|
|
106
|
-
* - `#subjectFilter#` — replaced with the distribution's
|
|
90
|
+
* - `#subjectFilter#` — replaced with the distribution's subject filter
|
|
107
91
|
* - `#namedGraph#` — replaced with `FROM <graph>` clause if the distribution has a named graph
|
|
108
92
|
* - `?dataset` — replaced with the dataset IRI
|
|
109
93
|
*/
|
|
110
|
-
export declare function substituteQueryTemplates(query: string, distribution: Distribution | null, dataset:
|
|
94
|
+
export declare function substituteQueryTemplates(query: string, distribution: Distribution | null, dataset: Dataset): string;
|
|
111
95
|
/**
|
|
112
96
|
* Read a SPARQL query from a file.
|
|
113
97
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/sparql/executor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAC5D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAI5C,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAK1C,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE1C,qEAAqE;AACrE,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEzD,MAAM,WAAW,QAAQ;IACvB,OAAO,CACL,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/sparql/executor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAC5D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAI5C,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAK1C,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE1C,qEAAqE;AACrE,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEzD,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,QAAQ,CAAC,EAAE,gBAAgB,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,QAAQ;IACvB,OAAO,CACL,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;CAChD;AAED;;;GAGG;AACH,MAAM,MAAM,UAAU,GAAG,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;AAEjD;;GAEG;AACH,MAAM,WAAW,8BAA8B;IAC7C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,qBAAa,uBAAwB,YAAW,QAAQ;IACtD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiB;IACvC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;IAChD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;gBAEjC,OAAO,EAAE,8BAA8B;IAcnD;;;;;;;OAOG;IACG,OAAO,CACX,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,UAAU,CAAC;IAoBtB;;;;;OAKG;WACiB,QAAQ,CAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,IAAI,CAAC,8BAA8B,EAAE,OAAO,CAAC,GACtD,OAAO,CAAC,uBAAuB,CAAC;CAIpC;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,MAAM,EACb,YAAY,EAAE,YAAY,GAAG,IAAI,EACjC,OAAO,EAAE,OAAO,GACf,MAAM,CAWR;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAErE"}
|
package/dist/sparql/executor.js
CHANGED
|
@@ -2,7 +2,6 @@ import { SparqlEndpointFetcher } from 'fetch-sparql-endpoint';
|
|
|
2
2
|
import { readFile } from 'node:fs/promises';
|
|
3
3
|
import { resolve } from 'node:path';
|
|
4
4
|
import { Generator, Parser } from 'sparqljs';
|
|
5
|
-
import { NotSupported } from '../step.js';
|
|
6
5
|
import { withDefaultGraph } from './graph.js';
|
|
7
6
|
import { injectValues } from './values.js';
|
|
8
7
|
// Re-export for convenience
|
|
@@ -20,7 +19,7 @@ export { NotSupported } from '../step.js';
|
|
|
20
19
|
* const executor = new SparqlConstructExecutor({
|
|
21
20
|
* query: 'CONSTRUCT { ?dataset ?p ?o } WHERE { ?s ?p ?o }',
|
|
22
21
|
* });
|
|
23
|
-
* const result = await executor.execute(dataset);
|
|
22
|
+
* const result = await executor.execute(dataset, distribution);
|
|
24
23
|
* if (result instanceof NotSupported) {
|
|
25
24
|
* console.log(result.message);
|
|
26
25
|
* } else {
|
|
@@ -48,23 +47,17 @@ export class SparqlConstructExecutor {
|
|
|
48
47
|
});
|
|
49
48
|
}
|
|
50
49
|
/**
|
|
51
|
-
* Execute the SPARQL CONSTRUCT query against the
|
|
50
|
+
* Execute the SPARQL CONSTRUCT query against the distribution's endpoint.
|
|
52
51
|
*
|
|
53
52
|
* @param dataset The dataset to execute against.
|
|
54
|
-
* @param
|
|
55
|
-
* @
|
|
53
|
+
* @param distribution The distribution providing the SPARQL endpoint.
|
|
54
|
+
* @param options Optional execution options (bindings).
|
|
55
|
+
* @returns AsyncIterable<Quad> stream of results.
|
|
56
56
|
*/
|
|
57
|
-
async execute(dataset, options) {
|
|
58
|
-
const
|
|
59
|
-
let endpoint = options?.endpoint;
|
|
60
|
-
if (endpoint === undefined) {
|
|
61
|
-
if (distribution === null || !distribution.isValid) {
|
|
62
|
-
return new NotSupported('No SPARQL distribution available');
|
|
63
|
-
}
|
|
64
|
-
endpoint = distribution.accessUrl;
|
|
65
|
-
}
|
|
57
|
+
async execute(dataset, distribution, options) {
|
|
58
|
+
const endpoint = distribution.accessUrl;
|
|
66
59
|
let ast = structuredClone(this.query);
|
|
67
|
-
if (distribution
|
|
60
|
+
if (distribution.namedGraph) {
|
|
68
61
|
withDefaultGraph(ast, distribution.namedGraph);
|
|
69
62
|
}
|
|
70
63
|
const bindings = options?.bindings;
|
|
@@ -89,12 +82,12 @@ export class SparqlConstructExecutor {
|
|
|
89
82
|
/**
|
|
90
83
|
* Substitute template variables in a SPARQL query.
|
|
91
84
|
*
|
|
92
|
-
* - `#subjectFilter#` — replaced with the distribution's
|
|
85
|
+
* - `#subjectFilter#` — replaced with the distribution's subject filter
|
|
93
86
|
* - `#namedGraph#` — replaced with `FROM <graph>` clause if the distribution has a named graph
|
|
94
87
|
* - `?dataset` — replaced with the dataset IRI
|
|
95
88
|
*/
|
|
96
89
|
export function substituteQueryTemplates(query, distribution, dataset) {
|
|
97
|
-
const subjectFilter = distribution?.subjectFilter ??
|
|
90
|
+
const subjectFilter = distribution?.subjectFilter ?? '';
|
|
98
91
|
const namedGraph = distribution?.namedGraph
|
|
99
92
|
? `FROM <${distribution.namedGraph}>`
|
|
100
93
|
: '';
|
package/dist/sparql/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { SparqlConstructExecutor, substituteQueryTemplates, NotSupported, readQueryFile, type
|
|
1
|
+
export { SparqlConstructExecutor, substituteQueryTemplates, NotSupported, readQueryFile, type ExecuteOptions, type Executor, type SparqlConstructExecutorOptions, type QuadStream, type VariableBindings, } from './executor.js';
|
|
2
2
|
export { collect } from './collect.js';
|
|
3
3
|
export { SparqlSelector, type SparqlSelectorOptions } from './selector.js';
|
|
4
4
|
export { injectValues } from './values.js';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/sparql/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,uBAAuB,EACvB,wBAAwB,EACxB,YAAY,EACZ,aAAa,EACb,KAAK,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/sparql/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,uBAAuB,EACvB,wBAAwB,EACxB,YAAY,EACZ,aAAa,EACb,KAAK,cAAc,EACnB,KAAK,QAAQ,EACb,KAAK,8BAA8B,EACnC,KAAK,UAAU,EACf,KAAK,gBAAgB,GACtB,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,OAAO,EAAE,cAAc,EAAE,KAAK,qBAAqB,EAAE,MAAM,eAAe,CAAC;AAE3E,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC"}
|
package/dist/stage.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
+
import { Dataset, Distribution } from '@lde/dataset';
|
|
1
2
|
import type { Quad } from '@rdfjs/types';
|
|
2
|
-
import type {
|
|
3
|
+
import type { Executor, VariableBindings } from './sparql/executor.js';
|
|
3
4
|
import { NotSupported } from './sparql/executor.js';
|
|
4
5
|
export interface StageOptions {
|
|
5
6
|
name: string;
|
|
@@ -11,7 +12,7 @@ export declare class Stage {
|
|
|
11
12
|
private readonly executors;
|
|
12
13
|
private readonly selector?;
|
|
13
14
|
constructor(options: StageOptions);
|
|
14
|
-
run(dataset:
|
|
15
|
+
run(dataset: Dataset, distribution: Distribution): Promise<AsyncIterable<Quad> | NotSupported>;
|
|
15
16
|
private collectBindings;
|
|
16
17
|
}
|
|
17
18
|
/** Stage-level selector that yields variable bindings for use in executor queries. Pagination is an implementation detail. */
|
package/dist/stage.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EACV,
|
|
1
|
+
{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EACV,QAAQ,EAER,gBAAgB,EACjB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IACjC,QAAQ,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAgB;gBAE9B,OAAO,EAAE,YAAY;IAQ3B,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,GACzB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC;YAyBhC,eAAe;CAW9B;AAUD,8HAA8H;AAE9H,MAAM,WAAW,aAAc,SAAQ,aAAa,CAAC,gBAAgB,CAAC;CAAG"}
|
package/dist/stage.js
CHANGED
|
@@ -10,12 +10,12 @@ export class Stage {
|
|
|
10
10
|
: [options.executors];
|
|
11
11
|
this.selector = options.selector;
|
|
12
12
|
}
|
|
13
|
-
async run(dataset) {
|
|
13
|
+
async run(dataset, distribution) {
|
|
14
14
|
const bindings = await this.collectBindings();
|
|
15
15
|
const executeOptions = bindings.length > 0 ? { bindings } : undefined;
|
|
16
16
|
const streams = [];
|
|
17
17
|
for (const executor of this.executors) {
|
|
18
|
-
const result = await executor.execute(dataset, executeOptions);
|
|
18
|
+
const result = await executor.execute(dataset, distribution, executeOptions);
|
|
19
19
|
if (!(result instanceof NotSupported)) {
|
|
20
20
|
streams.push(result);
|
|
21
21
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { DataEmittingStep
|
|
2
|
-
import { Dataset } from '@lde/dataset';
|
|
1
|
+
import { DataEmittingStep } from './../step.js';
|
|
2
|
+
import { Dataset, Distribution } from '@lde/dataset';
|
|
3
3
|
import { SparqlEndpointFetcher } from 'fetch-sparql-endpoint';
|
|
4
4
|
/**
|
|
5
5
|
* Arguments for the SparqlQuery step.
|
|
@@ -25,7 +25,7 @@ export declare class SparqlQuery implements DataEmittingStep {
|
|
|
25
25
|
private readonly query;
|
|
26
26
|
private readonly fetcher?;
|
|
27
27
|
constructor({ identifier, query, fetcher }: Args);
|
|
28
|
-
execute(dataset: Dataset): Promise<
|
|
28
|
+
execute(dataset: Dataset, distribution: Distribution): Promise<import("../sparql/executor.js").QuadStream>;
|
|
29
29
|
static fromFile(filename: string): Promise<SparqlQuery>;
|
|
30
30
|
}
|
|
31
31
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sparqlQuery.d.ts","sourceRoot":"","sources":["../../src/step/sparqlQuery.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,
|
|
1
|
+
{"version":3,"file":"sparqlQuery.d.ts","sourceRoot":"","sources":["../../src/step/sparqlQuery.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAO9D;;;;;;GAMG;AACH,MAAM,WAAW,IAAI;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AAED;;;;;;GAMG;AACH,qBAAa,WAAY,YAAW,gBAAgB;IAClD,SAAgB,UAAU,SAAC;IAC3B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAwB;gBAErC,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,IAAI;IAM1C,OAAO,CAAC,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,YAAY;WAatC,QAAQ,CAAC,QAAQ,EAAE,MAAM;CAM9C;AAED;;GAEG;AACH,wBAAsB,QAAQ,CAAC,QAAQ,EAAE,MAAM,mBAE9C"}
|
package/dist/step/sparqlQuery.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { NotSupported } from './../step.js';
|
|
2
1
|
import { SparqlConstructExecutor, substituteQueryTemplates, readQueryFile, } from '../sparql/index.js';
|
|
3
2
|
/**
|
|
4
3
|
* Executes a SPARQL CONSTRUCT query and emits the resulting quads.
|
|
@@ -16,17 +15,13 @@ export class SparqlQuery {
|
|
|
16
15
|
this.query = query;
|
|
17
16
|
this.fetcher = fetcher;
|
|
18
17
|
}
|
|
19
|
-
async execute(dataset) {
|
|
20
|
-
const distribution = dataset.getSparqlDistribution();
|
|
21
|
-
if (distribution === null || !distribution.isValid) {
|
|
22
|
-
return new NotSupported('No SPARQL distribution available');
|
|
23
|
-
}
|
|
18
|
+
async execute(dataset, distribution) {
|
|
24
19
|
const substituted = substituteQueryTemplates(this.query, distribution, dataset);
|
|
25
20
|
const executor = new SparqlConstructExecutor({
|
|
26
21
|
query: substituted,
|
|
27
22
|
fetcher: this.fetcher,
|
|
28
23
|
});
|
|
29
|
-
return await executor.execute(dataset);
|
|
24
|
+
return await executor.execute(dataset, distribution);
|
|
30
25
|
}
|
|
31
26
|
static async fromFile(filename) {
|
|
32
27
|
return new this({
|
package/dist/step.d.ts
CHANGED
|
@@ -9,13 +9,13 @@ export type Step = DataEmittingStep | SingleStep;
|
|
|
9
9
|
* Failure is expressed by emitting an error event; success by the end event.
|
|
10
10
|
*/
|
|
11
11
|
export interface DataEmittingStep extends AbstractStep {
|
|
12
|
-
execute(dataset: Dataset): Promise<Stream | NotSupported>;
|
|
12
|
+
execute(dataset: Dataset, distribution: Distribution): Promise<Stream | NotSupported>;
|
|
13
13
|
}
|
|
14
14
|
/**
|
|
15
15
|
* A pipeline step that executes an operation without emitting data.
|
|
16
16
|
*/
|
|
17
17
|
export interface SingleStep extends AbstractStep {
|
|
18
|
-
execute(dataset: Dataset): Promise<NotSupported | Failure | Success>;
|
|
18
|
+
execute(dataset: Dataset, distribution?: Distribution): Promise<NotSupported | Failure | Success>;
|
|
19
19
|
}
|
|
20
20
|
export interface Finishable {
|
|
21
21
|
finish(): Promise<void>;
|
package/dist/step.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"step.d.ts","sourceRoot":"","sources":["../src/step.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAE3C,UAAU,YAAY;IACpB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,MAAM,IAAI,GAAG,gBAAgB,GAAG,UAAU,CAAC;AAEjD;;;GAGG;AACH,MAAM,WAAW,gBAAiB,SAAQ,YAAY;IACpD,OAAO,
|
|
1
|
+
{"version":3,"file":"step.d.ts","sourceRoot":"","sources":["../src/step.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAE3C,UAAU,YAAY;IACpB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,MAAM,IAAI,GAAG,gBAAgB,GAAG,UAAU,CAAC;AAEjD;;;GAGG;AACH,MAAM,WAAW,gBAAiB,SAAQ,YAAY;IACpD,OAAO,CACL,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,GACzB,OAAO,CAAC,MAAM,GAAG,YAAY,CAAC,CAAC;CACnC;AAED;;GAEG;AACH,MAAM,WAAW,UAAW,SAAQ,YAAY;IAC9C,OAAO,CACL,OAAO,EAAE,OAAO,EAChB,YAAY,CAAC,EAAE,YAAY,GAC1B,OAAO,CAAC,YAAY,GAAG,OAAO,GAAG,OAAO,CAAC,CAAC;CAC9C;AAED,MAAM,WAAW,UAAU;IACzB,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACzB;AAED;;;;;GAKG;AACH,qBAAa,OAAO;aAEA,YAAY,EAAE,YAAY;aAC1B,OAAO,CAAC,EAAE,MAAM;gBADhB,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,MAAM,YAAA;CAEnC;AAED;;;;;GAKG;AACH,qBAAa,OAAO;aAEA,OAAO,EAAE,OAAO;aAChB,YAAY,EAAE,YAAY;gBAD1B,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY;CAE7C;AAED;;;;GAIG;AACH,qBAAa,YAAY;aACK,OAAO,EAAE,MAAM;gBAAf,OAAO,EAAE,MAAM;CAC5C"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lde/pipeline",
|
|
3
|
-
"version": "0.6.
|
|
3
|
+
"version": "0.6.15",
|
|
4
4
|
"repository": {
|
|
5
5
|
"url": "https://github.com/ldengine/lde",
|
|
6
6
|
"directory": "packages/pipeline"
|
|
@@ -35,10 +35,10 @@
|
|
|
35
35
|
"!**/*.tsbuildinfo"
|
|
36
36
|
],
|
|
37
37
|
"dependencies": {
|
|
38
|
-
"@lde/dataset": "0.6.
|
|
39
|
-
"@lde/dataset-registry-client": "0.6.
|
|
40
|
-
"@lde/sparql-importer": "0.2.
|
|
41
|
-
"@lde/sparql-server": "0.4.
|
|
38
|
+
"@lde/dataset": "0.6.8",
|
|
39
|
+
"@lde/dataset-registry-client": "0.6.14",
|
|
40
|
+
"@lde/sparql-importer": "0.2.8",
|
|
41
|
+
"@lde/sparql-server": "0.4.8",
|
|
42
42
|
"@rdfjs/types": "^2.0.1",
|
|
43
43
|
"c12": "^3.0.2",
|
|
44
44
|
"fetch-sparql-endpoint": "^6.0.0",
|