@lde/pipeline 0.6.15 → 0.6.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/distribution/index.d.ts +1 -1
- package/dist/distribution/index.d.ts.map +1 -1
- package/dist/distribution/index.js +1 -1
- package/dist/distribution/report.d.ts +15 -0
- package/dist/distribution/report.d.ts.map +1 -0
- package/dist/distribution/report.js +64 -0
- package/dist/sparql/index.d.ts +0 -1
- package/dist/sparql/index.d.ts.map +1 -1
- package/dist/sparql/index.js +0 -1
- package/package.json +1 -1
- package/dist/analyzer.d.ts +0 -36
- package/dist/analyzer.d.ts.map +0 -1
- package/dist/analyzer.js +0 -29
- package/dist/distribution/analyzer.d.ts +0 -55
- package/dist/distribution/analyzer.d.ts.map +0 -1
- package/dist/distribution/analyzer.js +0 -131
- package/dist/sparql/collect.d.ts +0 -19
- package/dist/sparql/collect.d.ts.map +0 -1
- package/dist/sparql/collect.js +0 -23
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
export { probe, NetworkError, SparqlProbeResult, DataDumpProbeResult, type ProbeResultType, } from './probe.js';
|
|
2
|
-
export {
|
|
2
|
+
export { probeResultsToQuads } from './report.js';
|
|
3
3
|
export { ResolvedDistribution, NoDistributionAvailable, SparqlDistributionResolver, type DistributionResolver, type SparqlDistributionResolverOptions, } from './resolver.js';
|
|
4
4
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/distribution/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,EACL,YAAY,EACZ,iBAAiB,EACjB,mBAAmB,EACnB,KAAK,eAAe,GACrB,MAAM,YAAY,CAAC;AAEpB,OAAO,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/distribution/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,EACL,YAAY,EACZ,iBAAiB,EACjB,mBAAmB,EACnB,KAAK,eAAe,GACrB,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAElD,OAAO,EACL,oBAAoB,EACpB,uBAAuB,EACvB,0BAA0B,EAC1B,KAAK,oBAAoB,EACzB,KAAK,iCAAiC,GACvC,MAAM,eAAe,CAAC"}
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
export { probe, NetworkError, SparqlProbeResult, DataDumpProbeResult, } from './probe.js';
|
|
2
|
-
export {
|
|
2
|
+
export { probeResultsToQuads } from './report.js';
|
|
3
3
|
export { ResolvedDistribution, NoDistributionAvailable, SparqlDistributionResolver, } from './resolver.js';
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { ImportFailed } from '@lde/sparql-importer';
|
|
2
|
+
import { type Quad } from 'n3';
|
|
3
|
+
import { type ProbeResultType } from './probe.js';
|
|
4
|
+
/**
|
|
5
|
+
* Convert probe results into RDF quads describing each probe as a `schema:Action`.
|
|
6
|
+
*
|
|
7
|
+
* Successful SPARQL probes emit `void:sparqlEndpoint`;
|
|
8
|
+
* successful data-dump probes emit `void:dataDump` with optional metadata.
|
|
9
|
+
* Failed probes emit `schema:error`.
|
|
10
|
+
*
|
|
11
|
+
* When an {@link ImportFailed} is provided its error is attached to the action
|
|
12
|
+
* whose `schema:target` matches the failed distribution's access URL.
|
|
13
|
+
*/
|
|
14
|
+
export declare function probeResultsToQuads(probeResults: ProbeResultType[], datasetIri: string, importResult?: ImportFailed): AsyncIterable<Quad>;
|
|
15
|
+
//# sourceMappingURL=report.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/distribution/report.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAe,KAAK,IAAI,EAAE,MAAM,IAAI,CAAC;AAC5C,OAAO,EAIL,KAAK,eAAe,EACrB,MAAM,YAAY,CAAC;AAUpB;;;;;;;;;GASG;AACH,wBAAuB,mBAAmB,CACxC,YAAY,EAAE,eAAe,EAAE,EAC/B,UAAU,EAAE,MAAM,EAClB,YAAY,CAAC,EAAE,YAAY,GAC1B,aAAa,CAAC,IAAI,CAAC,CAkCrB"}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { DataFactory } from 'n3';
|
|
2
|
+
import { NetworkError, SparqlProbeResult, } from './probe.js';
|
|
3
|
+
const { quad, namedNode, blankNode, literal } = DataFactory;
|
|
4
|
+
const RDF = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#';
|
|
5
|
+
const SCHEMA = 'https://schema.org/';
|
|
6
|
+
const VOID = 'http://rdfs.org/ns/void#';
|
|
7
|
+
const XSD = 'http://www.w3.org/2001/XMLSchema#';
|
|
8
|
+
const HTTP_STATUS = 'https://www.w3.org/2011/http-statusCodes#';
|
|
9
|
+
/**
|
|
10
|
+
* Convert probe results into RDF quads describing each probe as a `schema:Action`.
|
|
11
|
+
*
|
|
12
|
+
* Successful SPARQL probes emit `void:sparqlEndpoint`;
|
|
13
|
+
* successful data-dump probes emit `void:dataDump` with optional metadata.
|
|
14
|
+
* Failed probes emit `schema:error`.
|
|
15
|
+
*
|
|
16
|
+
* When an {@link ImportFailed} is provided its error is attached to the action
|
|
17
|
+
* whose `schema:target` matches the failed distribution's access URL.
|
|
18
|
+
*/
|
|
19
|
+
export async function* probeResultsToQuads(probeResults, datasetIri, importResult) {
|
|
20
|
+
// Track blank nodes per URL so import errors can reference the right action.
|
|
21
|
+
const actionsByUrl = new Map();
|
|
22
|
+
for (const result of probeResults) {
|
|
23
|
+
const action = blankNode();
|
|
24
|
+
actionsByUrl.set(result.url, action);
|
|
25
|
+
yield quad(action, namedNode(`${RDF}type`), namedNode(`${SCHEMA}Action`));
|
|
26
|
+
yield quad(action, namedNode(`${SCHEMA}target`), namedNode(result.url));
|
|
27
|
+
if (result instanceof NetworkError) {
|
|
28
|
+
yield quad(action, namedNode(`${SCHEMA}error`), literal(result.message));
|
|
29
|
+
}
|
|
30
|
+
else if (result.isSuccess()) {
|
|
31
|
+
yield* successQuads(action, result, datasetIri);
|
|
32
|
+
}
|
|
33
|
+
else {
|
|
34
|
+
// HTTP error
|
|
35
|
+
const statusUri = `${HTTP_STATUS}${result.statusText.replace(/ /g, '')}`;
|
|
36
|
+
yield quad(action, namedNode(`${SCHEMA}error`), namedNode(statusUri));
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
if (importResult) {
|
|
40
|
+
const action = actionsByUrl.get(importResult.distribution.accessUrl.toString());
|
|
41
|
+
if (action) {
|
|
42
|
+
yield quad(action, namedNode(`${SCHEMA}error`), literal(importResult.error));
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
function* successQuads(action, result, datasetIri) {
|
|
47
|
+
const distributionUrl = namedNode(result.url);
|
|
48
|
+
yield quad(action, namedNode(`${SCHEMA}result`), distributionUrl);
|
|
49
|
+
if (result.lastModified) {
|
|
50
|
+
yield quad(distributionUrl, namedNode(`${SCHEMA}dateModified`), literal(result.lastModified.toISOString(), namedNode(`${XSD}dateTime`)));
|
|
51
|
+
}
|
|
52
|
+
if (result instanceof SparqlProbeResult) {
|
|
53
|
+
yield quad(namedNode(datasetIri), namedNode(`${VOID}sparqlEndpoint`), distributionUrl);
|
|
54
|
+
}
|
|
55
|
+
else {
|
|
56
|
+
yield quad(namedNode(datasetIri), namedNode(`${VOID}dataDump`), distributionUrl);
|
|
57
|
+
if (result.contentSize) {
|
|
58
|
+
yield quad(distributionUrl, namedNode(`${SCHEMA}contentSize`), literal(result.contentSize.toString()));
|
|
59
|
+
}
|
|
60
|
+
if (result.contentType) {
|
|
61
|
+
yield quad(distributionUrl, namedNode(`${SCHEMA}encodingFormat`), literal(result.contentType));
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
package/dist/sparql/index.d.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
export { SparqlConstructExecutor, substituteQueryTemplates, NotSupported, readQueryFile, type ExecuteOptions, type Executor, type SparqlConstructExecutorOptions, type QuadStream, type VariableBindings, } from './executor.js';
|
|
2
|
-
export { collect } from './collect.js';
|
|
3
2
|
export { SparqlSelector, type SparqlSelectorOptions } from './selector.js';
|
|
4
3
|
export { injectValues } from './values.js';
|
|
5
4
|
export { withDefaultGraph } from './graph.js';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/sparql/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,uBAAuB,EACvB,wBAAwB,EACxB,YAAY,EACZ,aAAa,EACb,KAAK,cAAc,EACnB,KAAK,QAAQ,EACb,KAAK,8BAA8B,EACnC,KAAK,UAAU,EACf,KAAK,gBAAgB,GACtB,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/sparql/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,uBAAuB,EACvB,wBAAwB,EACxB,YAAY,EACZ,aAAa,EACb,KAAK,cAAc,EACnB,KAAK,QAAQ,EACb,KAAK,8BAA8B,EACnC,KAAK,UAAU,EACf,KAAK,gBAAgB,GACtB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,cAAc,EAAE,KAAK,qBAAqB,EAAE,MAAM,eAAe,CAAC;AAE3E,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC"}
|
package/dist/sparql/index.js
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
export { SparqlConstructExecutor, substituteQueryTemplates, NotSupported, readQueryFile, } from './executor.js';
|
|
2
|
-
export { collect } from './collect.js';
|
|
3
2
|
export { SparqlSelector } from './selector.js';
|
|
4
3
|
export { injectValues } from './values.js';
|
|
5
4
|
export { withDefaultGraph } from './graph.js';
|
package/package.json
CHANGED
package/dist/analyzer.d.ts
DELETED
|
@@ -1,36 +0,0 @@
|
|
|
1
|
-
import { Dataset } from '@lde/dataset';
|
|
2
|
-
import type { DatasetCore } from '@rdfjs/types';
|
|
3
|
-
import { NotSupported } from './step.js';
|
|
4
|
-
export { NotSupported } from './step.js';
|
|
5
|
-
/**
|
|
6
|
-
* Result of a successful analysis.
|
|
7
|
-
*/
|
|
8
|
-
export declare class Success {
|
|
9
|
-
readonly data: DatasetCore;
|
|
10
|
-
constructor(data: DatasetCore);
|
|
11
|
-
}
|
|
12
|
-
/**
|
|
13
|
-
* Analysis failed.
|
|
14
|
-
*/
|
|
15
|
-
export declare class Failure {
|
|
16
|
-
readonly endpoint: URL;
|
|
17
|
-
readonly message?: string | undefined;
|
|
18
|
-
constructor(endpoint: URL, message?: string | undefined);
|
|
19
|
-
}
|
|
20
|
-
/**
|
|
21
|
-
* Interface for analyzers.
|
|
22
|
-
*/
|
|
23
|
-
export interface Analyzer {
|
|
24
|
-
readonly name: string;
|
|
25
|
-
execute(dataset: Dataset): Promise<Success | Failure | NotSupported>;
|
|
26
|
-
finish?(): Promise<void>;
|
|
27
|
-
}
|
|
28
|
-
/**
|
|
29
|
-
* Base class for analyzers with default implementations.
|
|
30
|
-
*/
|
|
31
|
-
export declare abstract class BaseAnalyzer implements Analyzer {
|
|
32
|
-
abstract readonly name: string;
|
|
33
|
-
abstract execute(dataset: Dataset): Promise<Success | Failure | NotSupported>;
|
|
34
|
-
finish(): Promise<void>;
|
|
35
|
-
}
|
|
36
|
-
//# sourceMappingURL=analyzer.d.ts.map
|
package/dist/analyzer.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"analyzer.d.ts","sourceRoot":"","sources":["../src/analyzer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAEzC,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAEzC;;GAEG;AACH,qBAAa,OAAO;aACU,IAAI,EAAE,WAAW;gBAAjB,IAAI,EAAE,WAAW;CAC9C;AAED;;GAEG;AACH,qBAAa,OAAO;aAEA,QAAQ,EAAE,GAAG;aACb,OAAO,CAAC,EAAE,MAAM;gBADhB,QAAQ,EAAE,GAAG,EACb,OAAO,CAAC,EAAE,MAAM,YAAA;CAEnC;AAED;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,OAAO,GAAG,OAAO,GAAG,YAAY,CAAC,CAAC;IACrE,MAAM,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED;;GAEG;AACH,8BAAsB,YAAa,YAAW,QAAQ;IACpD,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,OAAO,GAAG,OAAO,GAAG,YAAY,CAAC;IAEvE,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;CAG9B"}
|
package/dist/analyzer.js
DELETED
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
export { NotSupported } from './step.js';
|
|
2
|
-
/**
|
|
3
|
-
* Result of a successful analysis.
|
|
4
|
-
*/
|
|
5
|
-
export class Success {
|
|
6
|
-
data;
|
|
7
|
-
constructor(data) {
|
|
8
|
-
this.data = data;
|
|
9
|
-
}
|
|
10
|
-
}
|
|
11
|
-
/**
|
|
12
|
-
* Analysis failed.
|
|
13
|
-
*/
|
|
14
|
-
export class Failure {
|
|
15
|
-
endpoint;
|
|
16
|
-
message;
|
|
17
|
-
constructor(endpoint, message) {
|
|
18
|
-
this.endpoint = endpoint;
|
|
19
|
-
this.message = message;
|
|
20
|
-
}
|
|
21
|
-
}
|
|
22
|
-
/**
|
|
23
|
-
* Base class for analyzers with default implementations.
|
|
24
|
-
*/
|
|
25
|
-
export class BaseAnalyzer {
|
|
26
|
-
async finish() {
|
|
27
|
-
// Default no-op implementation.
|
|
28
|
-
}
|
|
29
|
-
}
|
|
@@ -1,55 +0,0 @@
|
|
|
1
|
-
import { Dataset } from '@lde/dataset';
|
|
2
|
-
import { Importer, ImportFailed, ImportSuccessful, NotSupported } from '@lde/sparql-importer';
|
|
3
|
-
import { Store } from 'n3';
|
|
4
|
-
import { NoDistributionAvailable } from './resolver.js';
|
|
5
|
-
export type { Importer };
|
|
6
|
-
export { ImportFailed, ImportSuccessful, NotSupported };
|
|
7
|
-
/**
|
|
8
|
-
* Extended importer interface with optional cleanup method.
|
|
9
|
-
*/
|
|
10
|
-
export interface ImporterWithFinish extends Importer {
|
|
11
|
-
finish?(): Promise<void>;
|
|
12
|
-
}
|
|
13
|
-
export interface DistributionAnalyzerOptions {
|
|
14
|
-
/**
|
|
15
|
-
* Optional importer for loading data dumps when no SPARQL endpoint is available.
|
|
16
|
-
*/
|
|
17
|
-
importer?: ImporterWithFinish;
|
|
18
|
-
/**
|
|
19
|
-
* Timeout for probe requests in milliseconds.
|
|
20
|
-
* @default 5000
|
|
21
|
-
*/
|
|
22
|
-
timeout?: number;
|
|
23
|
-
}
|
|
24
|
-
/**
|
|
25
|
-
* Analyzes dataset distributions by probing their availability.
|
|
26
|
-
*
|
|
27
|
-
* - Probes SPARQL endpoints with a simple SELECT query
|
|
28
|
-
* - Probes data dumps with HEAD/GET requests
|
|
29
|
-
* - Records probe results as RDF (schema:Action)
|
|
30
|
-
* - Optionally imports data dumps if no SPARQL endpoint is available
|
|
31
|
-
*/
|
|
32
|
-
export declare class DistributionAnalyzer {
|
|
33
|
-
readonly name = "distribution";
|
|
34
|
-
private readonly importer?;
|
|
35
|
-
private readonly timeout;
|
|
36
|
-
constructor(options?: DistributionAnalyzerOptions);
|
|
37
|
-
/**
|
|
38
|
-
* Analyze all distributions of a dataset.
|
|
39
|
-
*
|
|
40
|
-
* @returns Store with probe results as RDF, or NoDistributionAvailable if no usable distribution found
|
|
41
|
-
*/
|
|
42
|
-
execute(dataset: Dataset): Promise<Store | NoDistributionAvailable>;
|
|
43
|
-
/**
|
|
44
|
-
* Cleanup resources (e.g., importer connections).
|
|
45
|
-
*/
|
|
46
|
-
finish(): Promise<void>;
|
|
47
|
-
/**
|
|
48
|
-
* Apply probe results to distributions so `getSparqlDistribution()` works.
|
|
49
|
-
*/
|
|
50
|
-
private applyProbeResults;
|
|
51
|
-
private buildProbeResultsRdf;
|
|
52
|
-
private addSuccessTriples;
|
|
53
|
-
private addImportError;
|
|
54
|
-
}
|
|
55
|
-
//# sourceMappingURL=analyzer.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"analyzer.d.ts","sourceRoot":"","sources":["../../src/distribution/analyzer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AACrD,OAAO,EACL,QAAQ,EACR,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACb,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAe,KAAK,EAAE,MAAM,IAAI,CAAC;AAQxC,OAAO,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AAExD,YAAY,EAAE,QAAQ,EAAE,CAAC;AACzB,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,YAAY,EAAE,CAAC;AAWxD;;GAEG;AACH,MAAM,WAAW,kBAAmB,SAAQ,QAAQ;IAClD,MAAM,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,MAAM,WAAW,2BAA2B;IAC1C;;OAEG;IACH,QAAQ,CAAC,EAAE,kBAAkB,CAAC;IAE9B;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;GAOG;AACH,qBAAa,oBAAoB;IAC/B,SAAgB,IAAI,kBAAkB;IACtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAqB;IAC/C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;gBAErB,OAAO,CAAC,EAAE,2BAA2B;IAKjD;;;;OAIG;IACG,OAAO,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,KAAK,GAAG,uBAAuB,CAAC;IAuCzE;;OAEG;IACG,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAI7B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAsBzB,OAAO,CAAC,oBAAoB;IAwC5B,OAAO,CAAC,iBAAiB;IAiDzB,OAAO,CAAC,cAAc;CAevB"}
|
|
@@ -1,131 +0,0 @@
|
|
|
1
|
-
import { Distribution } from '@lde/dataset';
|
|
2
|
-
import { ImportFailed, ImportSuccessful, NotSupported, } from '@lde/sparql-importer';
|
|
3
|
-
import { DataFactory, Store } from 'n3';
|
|
4
|
-
import { probe, NetworkError, SparqlProbeResult, DataDumpProbeResult, } from './probe.js';
|
|
5
|
-
import { NoDistributionAvailable } from './resolver.js';
|
|
6
|
-
export { ImportFailed, ImportSuccessful, NotSupported };
|
|
7
|
-
const { quad, namedNode, blankNode, literal } = DataFactory;
|
|
8
|
-
// Namespace prefixes
|
|
9
|
-
const RDF = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#';
|
|
10
|
-
const SCHEMA = 'https://schema.org/';
|
|
11
|
-
const VOID = 'http://rdfs.org/ns/void#';
|
|
12
|
-
const XSD = 'http://www.w3.org/2001/XMLSchema#';
|
|
13
|
-
const HTTP_STATUS = 'https://www.w3.org/2011/http-statusCodes#';
|
|
14
|
-
/**
|
|
15
|
-
* Analyzes dataset distributions by probing their availability.
|
|
16
|
-
*
|
|
17
|
-
* - Probes SPARQL endpoints with a simple SELECT query
|
|
18
|
-
* - Probes data dumps with HEAD/GET requests
|
|
19
|
-
* - Records probe results as RDF (schema:Action)
|
|
20
|
-
* - Optionally imports data dumps if no SPARQL endpoint is available
|
|
21
|
-
*/
|
|
22
|
-
export class DistributionAnalyzer {
|
|
23
|
-
name = 'distribution';
|
|
24
|
-
importer;
|
|
25
|
-
timeout;
|
|
26
|
-
constructor(options) {
|
|
27
|
-
this.importer = options?.importer;
|
|
28
|
-
this.timeout = options?.timeout ?? 5000;
|
|
29
|
-
}
|
|
30
|
-
/**
|
|
31
|
-
* Analyze all distributions of a dataset.
|
|
32
|
-
*
|
|
33
|
-
* @returns Store with probe results as RDF, or NoDistributionAvailable if no usable distribution found
|
|
34
|
-
*/
|
|
35
|
-
async execute(dataset) {
|
|
36
|
-
const probeResults = await Promise.all(dataset.distributions.map((distribution) => probe(distribution, this.timeout)));
|
|
37
|
-
// Apply probe results to distributions (isValid, lastModified, byteSize).
|
|
38
|
-
this.applyProbeResults(dataset.distributions, probeResults);
|
|
39
|
-
const store = this.buildProbeResultsRdf(probeResults, dataset);
|
|
40
|
-
// If no SPARQL endpoint available, try to import a data dump
|
|
41
|
-
if (dataset.getSparqlDistribution() === null && this.importer) {
|
|
42
|
-
const importResult = await this.importer.import(dataset);
|
|
43
|
-
if (importResult instanceof ImportSuccessful) {
|
|
44
|
-
// Add imported SPARQL distribution to dataset so subsequent steps can use it
|
|
45
|
-
const distribution = Distribution.sparql(importResult.distribution.accessUrl, importResult.identifier);
|
|
46
|
-
dataset.distributions.push(distribution);
|
|
47
|
-
}
|
|
48
|
-
else if (importResult instanceof ImportFailed) {
|
|
49
|
-
// Record import error in the store
|
|
50
|
-
this.addImportError(store, importResult);
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
|
-
if (dataset.getSparqlDistribution() === null) {
|
|
54
|
-
return new NoDistributionAvailable(dataset, 'No SPARQL endpoint or importable data dump available');
|
|
55
|
-
}
|
|
56
|
-
return store;
|
|
57
|
-
}
|
|
58
|
-
/**
|
|
59
|
-
* Cleanup resources (e.g., importer connections).
|
|
60
|
-
*/
|
|
61
|
-
async finish() {
|
|
62
|
-
await this.importer?.finish?.();
|
|
63
|
-
}
|
|
64
|
-
/**
|
|
65
|
-
* Apply probe results to distributions so `getSparqlDistribution()` works.
|
|
66
|
-
*/
|
|
67
|
-
applyProbeResults(distributions, results) {
|
|
68
|
-
for (let i = 0; i < distributions.length; i++) {
|
|
69
|
-
const distribution = distributions[i];
|
|
70
|
-
const result = results[i];
|
|
71
|
-
if (result instanceof NetworkError) {
|
|
72
|
-
distribution.isValid = false;
|
|
73
|
-
continue;
|
|
74
|
-
}
|
|
75
|
-
distribution.isValid = result.isSuccess();
|
|
76
|
-
if (result instanceof DataDumpProbeResult) {
|
|
77
|
-
distribution.lastModified ??= result.lastModified ?? undefined;
|
|
78
|
-
distribution.byteSize ??= result.contentSize ?? undefined;
|
|
79
|
-
}
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
buildProbeResultsRdf(results, dataset) {
|
|
83
|
-
const store = new Store();
|
|
84
|
-
for (const result of results) {
|
|
85
|
-
const action = blankNode();
|
|
86
|
-
// Base action triples
|
|
87
|
-
store.addQuads([
|
|
88
|
-
quad(action, namedNode(`${RDF}type`), namedNode(`${SCHEMA}Action`)),
|
|
89
|
-
quad(action, namedNode(`${SCHEMA}target`), namedNode(result.url)),
|
|
90
|
-
]);
|
|
91
|
-
if (result instanceof NetworkError) {
|
|
92
|
-
store.addQuad(action, namedNode(`${SCHEMA}error`), literal(result.message));
|
|
93
|
-
}
|
|
94
|
-
else if (result.isSuccess()) {
|
|
95
|
-
this.addSuccessTriples(store, action, result, dataset);
|
|
96
|
-
}
|
|
97
|
-
else {
|
|
98
|
-
// HTTP error
|
|
99
|
-
const statusUri = `${HTTP_STATUS}${result.statusText.replace(/ /g, '')}`;
|
|
100
|
-
store.addQuad(action, namedNode(`${SCHEMA}error`), namedNode(statusUri));
|
|
101
|
-
}
|
|
102
|
-
}
|
|
103
|
-
return store;
|
|
104
|
-
}
|
|
105
|
-
addSuccessTriples(store, action, result, dataset) {
|
|
106
|
-
const distributionUrl = namedNode(result.url);
|
|
107
|
-
store.addQuad(action, namedNode(`${SCHEMA}result`), distributionUrl);
|
|
108
|
-
if (result.lastModified) {
|
|
109
|
-
store.addQuad(distributionUrl, namedNode(`${SCHEMA}dateModified`), literal(result.lastModified.toISOString(), namedNode(`${XSD}dateTime`)));
|
|
110
|
-
}
|
|
111
|
-
if (result instanceof SparqlProbeResult) {
|
|
112
|
-
store.addQuad(namedNode(dataset.iri.toString()), namedNode(`${VOID}sparqlEndpoint`), distributionUrl);
|
|
113
|
-
}
|
|
114
|
-
else {
|
|
115
|
-
store.addQuad(namedNode(dataset.iri.toString()), namedNode(`${VOID}dataDump`), distributionUrl);
|
|
116
|
-
if (result.contentSize) {
|
|
117
|
-
store.addQuad(distributionUrl, namedNode(`${SCHEMA}contentSize`), literal(result.contentSize));
|
|
118
|
-
}
|
|
119
|
-
if (result.contentType) {
|
|
120
|
-
store.addQuad(distributionUrl, namedNode(`${SCHEMA}encodingFormat`), literal(result.contentType));
|
|
121
|
-
}
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
addImportError(store, importResult) {
|
|
125
|
-
// Find the action for this download URL and add the error
|
|
126
|
-
const matches = store.match(null, namedNode(`${SCHEMA}target`), namedNode(importResult.distribution.accessUrl.toString()));
|
|
127
|
-
for (const match of matches) {
|
|
128
|
-
store.addQuad(match.subject, namedNode(`${SCHEMA}error`), literal(importResult.error));
|
|
129
|
-
}
|
|
130
|
-
}
|
|
131
|
-
}
|
package/dist/sparql/collect.d.ts
DELETED
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
import { Store } from 'n3';
|
|
2
|
-
import type { QuadStream } from './executor.js';
|
|
3
|
-
/**
|
|
4
|
-
* Collect all quads from a stream into an N3 Store.
|
|
5
|
-
*
|
|
6
|
-
* @param stream The quad stream to collect from.
|
|
7
|
-
* @returns Promise that resolves to a Store containing all quads.
|
|
8
|
-
*
|
|
9
|
-
* @example
|
|
10
|
-
* ```typescript
|
|
11
|
-
* const result = await executor.execute(dataset);
|
|
12
|
-
* if (!(result instanceof NotSupported)) {
|
|
13
|
-
* const store = await collect(result);
|
|
14
|
-
* console.log(`Collected ${store.size} quads`);
|
|
15
|
-
* }
|
|
16
|
-
* ```
|
|
17
|
-
*/
|
|
18
|
-
export declare function collect(stream: QuadStream): Promise<Store>;
|
|
19
|
-
//# sourceMappingURL=collect.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"collect.d.ts","sourceRoot":"","sources":["../../src/sparql/collect.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,IAAI,CAAC;AAC3B,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAEhD;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,OAAO,CAAC,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,CAMhE"}
|
package/dist/sparql/collect.js
DELETED
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
import { Store } from 'n3';
|
|
2
|
-
/**
|
|
3
|
-
* Collect all quads from a stream into an N3 Store.
|
|
4
|
-
*
|
|
5
|
-
* @param stream The quad stream to collect from.
|
|
6
|
-
* @returns Promise that resolves to a Store containing all quads.
|
|
7
|
-
*
|
|
8
|
-
* @example
|
|
9
|
-
* ```typescript
|
|
10
|
-
* const result = await executor.execute(dataset);
|
|
11
|
-
* if (!(result instanceof NotSupported)) {
|
|
12
|
-
* const store = await collect(result);
|
|
13
|
-
* console.log(`Collected ${store.size} quads`);
|
|
14
|
-
* }
|
|
15
|
-
* ```
|
|
16
|
-
*/
|
|
17
|
-
export async function collect(stream) {
|
|
18
|
-
const store = new Store();
|
|
19
|
-
for await (const quad of stream) {
|
|
20
|
-
store.addQuad(quad);
|
|
21
|
-
}
|
|
22
|
-
return store;
|
|
23
|
-
}
|