@lde/pipeline 0.37.0 → 0.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/distribution/importResolver.d.ts +3 -3
- package/dist/distribution/importResolver.d.ts.map +1 -1
- package/dist/distribution/importResolver.js +6 -1
- package/dist/distribution/index.d.ts +1 -1
- package/dist/distribution/index.d.ts.map +1 -1
- package/dist/distribution/resolver.d.ts +17 -5
- package/dist/distribution/resolver.d.ts.map +1 -1
- package/dist/distribution/resolver.js +5 -7
- package/dist/pipeline.d.ts.map +1 -1
- package/dist/pipeline.js +9 -8
- package/dist/progressReporter.d.ts +4 -1
- package/dist/progressReporter.d.ts.map +1 -1
- package/package.json +3 -3
|
@@ -9,14 +9,14 @@ export interface ImportResolverOptions {
|
|
|
9
9
|
* Controls how a dataset's distribution is selected, ordered by how eagerly a
|
|
10
10
|
* data dump is imported:
|
|
11
11
|
*
|
|
12
|
-
* - `'sparql'` (default)
|
|
12
|
+
* - `'sparql'` (default) – use a dataset's own SPARQL endpoint when one is
|
|
13
13
|
* available; import a data dump only when no endpoint responds.
|
|
14
|
-
* - `'sparqlWithImportFallback'`
|
|
14
|
+
* - `'sparqlWithImportFallback'` – like `'sparql'`, but additionally fall
|
|
15
15
|
* back to importing the data dump when the endpoint passes probing yet fails
|
|
16
16
|
* to serve an analysis stage at runtime. The pipeline then re-runs all
|
|
17
17
|
* stages locally against the import (see
|
|
18
18
|
* {@link ImportResolver.resolveFallback}).
|
|
19
|
-
* - `'import'`
|
|
19
|
+
* - `'import'` – always import a data dump into a local SPARQL server,
|
|
20
20
|
* even when the dataset advertises a working SPARQL endpoint. Useful when
|
|
21
21
|
* the remote endpoint is too slow or unreliable.
|
|
22
22
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"importResolver.d.ts","sourceRoot":"","sources":["../../src/distribution/importResolver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,OAAO,EAAgB,MAAM,cAAc,CAAC;AAC1D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAMrD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EACL,KAAK,oBAAoB,EAEzB,KAAK,gBAAgB,EACrB,uBAAuB,EACvB,mBAAmB,EACnB,oBAAoB,EACrB,MAAM,eAAe,CAAC;AAGvB,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,EAAE,YAAY,CAAC;IACrB;;;;;;;;;;;;;;;;;OAiBG;IACH,QAAQ,CAAC,EAAE,QAAQ,GAAG,0BAA0B,GAAG,QAAQ,CAAC;CAC7D;AAED;;;;;;;;;;;;GAYG;AACH,qBAAa,cAAe,YAAW,oBAAoB;IAEvD,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,OAAO;gBADP,KAAK,EAAE,oBAAoB,EAC3B,OAAO,EAAE,qBAAqB;IAG3C,KAAK,CACT,OAAO,EAAE,OAAO,EAChB,SAAS,CAAC,EAAE,gBAAgB,GAC3B,OAAO,CAAC,mBAAmB,CAAC;IAgBzB,OAAO,CACX,MAAM,EAAE,mBAAmB,EAC3B,SAAS,CAAC,EAAE,gBAAgB,GAC3B,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC;IAoBpD,eAAe,CACnB,MAAM,EAAE,mBAAmB,EAC3B,SAAS,CAAC,EAAE,gBAAgB,GAC3B,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC;IAc1D;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAY7B;;;OAGG;IACH,OAAO,CAAC,gBAAgB;YAeV,aAAa;
|
|
1
|
+
{"version":3,"file":"importResolver.d.ts","sourceRoot":"","sources":["../../src/distribution/importResolver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,OAAO,EAAgB,MAAM,cAAc,CAAC;AAC1D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAMrD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EACL,KAAK,oBAAoB,EAEzB,KAAK,gBAAgB,EACrB,uBAAuB,EACvB,mBAAmB,EACnB,oBAAoB,EACrB,MAAM,eAAe,CAAC;AAGvB,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,EAAE,YAAY,CAAC;IACrB;;;;;;;;;;;;;;;;;OAiBG;IACH,QAAQ,CAAC,EAAE,QAAQ,GAAG,0BAA0B,GAAG,QAAQ,CAAC;CAC7D;AAED;;;;;;;;;;;;GAYG;AACH,qBAAa,cAAe,YAAW,oBAAoB;IAEvD,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,OAAO;gBADP,KAAK,EAAE,oBAAoB,EAC3B,OAAO,EAAE,qBAAqB;IAG3C,KAAK,CACT,OAAO,EAAE,OAAO,EAChB,SAAS,CAAC,EAAE,gBAAgB,GAC3B,OAAO,CAAC,mBAAmB,CAAC;IAgBzB,OAAO,CACX,MAAM,EAAE,mBAAmB,EAC3B,SAAS,CAAC,EAAE,gBAAgB,GAC3B,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC;IAoBpD,eAAe,CACnB,MAAM,EAAE,mBAAmB,EAC3B,SAAS,CAAC,EAAE,gBAAgB,GAC3B,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC;IAc1D;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAY7B;;;OAGG;IACH,OAAO,CAAC,gBAAgB;YAeV,aAAa;IAkGrB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAG/B"}
|
|
@@ -111,7 +111,12 @@ export class ImportResolver {
|
|
|
111
111
|
}
|
|
112
112
|
const distribution = Distribution.sparql(this.options.server.queryEndpoint, importResult.identifier);
|
|
113
113
|
distribution.subjectFilter = importResult.distribution.subjectFilter;
|
|
114
|
-
return new ResolvedDistribution(distribution, probeResults,
|
|
114
|
+
return new ResolvedDistribution(distribution, probeResults, {
|
|
115
|
+
from: importResult.distribution,
|
|
116
|
+
duration: Date.now() - importStart,
|
|
117
|
+
tripleCount: importResult.tripleCount,
|
|
118
|
+
engine: importResult.engine,
|
|
119
|
+
});
|
|
115
120
|
}
|
|
116
121
|
if (importResult instanceof ImportFailed) {
|
|
117
122
|
callbacks?.onImportFailed?.(importResult.distribution, importResult.error);
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export { probe, NetworkError, SparqlProbeResult, DataDumpProbeResult, type ProbeResultType, } from '@lde/distribution-probe';
|
|
2
2
|
export { probeResultsToQuads } from './report.js';
|
|
3
3
|
export { ImportResolver, type ImportResolverOptions, } from './importResolver.js';
|
|
4
|
-
export { ResolvedDistribution, NoDistributionAvailable, ProbedDistributions, SparqlDistributionResolver, type DistributionResolver, type ProbedSource, type ResolveCallbacks, type SparqlDistributionResolverOptions, } from './resolver.js';
|
|
4
|
+
export { ResolvedDistribution, NoDistributionAvailable, ProbedDistributions, SparqlDistributionResolver, type DistributionResolver, type ImportDetails, type ProbedSource, type ResolveCallbacks, type SparqlDistributionResolverOptions, } from './resolver.js';
|
|
5
5
|
export { resolveDistributions, type DistributionStageResult, } from './resolveDistributions.js';
|
|
6
6
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/distribution/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,EACL,YAAY,EACZ,iBAAiB,EACjB,mBAAmB,EACnB,KAAK,eAAe,GACrB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAElD,OAAO,EACL,cAAc,EACd,KAAK,qBAAqB,GAC3B,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EACL,oBAAoB,EACpB,uBAAuB,EACvB,mBAAmB,EACnB,0BAA0B,EAC1B,KAAK,oBAAoB,EACzB,KAAK,YAAY,EACjB,KAAK,gBAAgB,EACrB,KAAK,iCAAiC,GACvC,MAAM,eAAe,CAAC;AAEvB,OAAO,EACL,oBAAoB,EACpB,KAAK,uBAAuB,GAC7B,MAAM,2BAA2B,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/distribution/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,EACL,YAAY,EACZ,iBAAiB,EACjB,mBAAmB,EACnB,KAAK,eAAe,GACrB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAElD,OAAO,EACL,cAAc,EACd,KAAK,qBAAqB,GAC3B,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EACL,oBAAoB,EACpB,uBAAuB,EACvB,mBAAmB,EACnB,0BAA0B,EAC1B,KAAK,oBAAoB,EACzB,KAAK,aAAa,EAClB,KAAK,YAAY,EACjB,KAAK,gBAAgB,EACrB,KAAK,iCAAiC,GACvC,MAAM,eAAe,CAAC;AAEvB,OAAO,EACL,oBAAoB,EACpB,KAAK,uBAAuB,GAC7B,MAAM,2BAA2B,CAAC"}
|
|
@@ -1,13 +1,25 @@
|
|
|
1
1
|
import { Dataset, Distribution } from '@lde/dataset';
|
|
2
|
-
import type { ImportFailed } from '@lde/sparql-importer';
|
|
2
|
+
import type { Engine, ImportFailed } from '@lde/sparql-importer';
|
|
3
3
|
import { type ProbeResultType } from '@lde/distribution-probe';
|
|
4
4
|
export declare class ResolvedDistribution {
|
|
5
5
|
readonly distribution: Distribution;
|
|
6
6
|
readonly probeResults: ProbeResultType[];
|
|
7
|
-
|
|
8
|
-
readonly
|
|
9
|
-
|
|
10
|
-
|
|
7
|
+
/** Set when {@link distribution} serves an imported data dump. */
|
|
8
|
+
readonly imported?: ImportDetails | undefined;
|
|
9
|
+
constructor(distribution: Distribution, probeResults: ProbeResultType[],
|
|
10
|
+
/** Set when {@link distribution} serves an imported data dump. */
|
|
11
|
+
imported?: ImportDetails | undefined);
|
|
12
|
+
}
|
|
13
|
+
/** How a data dump was imported into the store a dataset is read from. */
|
|
14
|
+
export interface ImportDetails {
|
|
15
|
+
/** The data dump that was imported. */
|
|
16
|
+
readonly from: Distribution;
|
|
17
|
+
/** Milliseconds the import took. */
|
|
18
|
+
readonly duration: number;
|
|
19
|
+
/** Triples in the imported data, when the importer can tell. */
|
|
20
|
+
readonly tripleCount?: number;
|
|
21
|
+
/** The engine that built the imported data, when the importer can tell. */
|
|
22
|
+
readonly engine?: Engine;
|
|
11
23
|
}
|
|
12
24
|
export declare class NoDistributionAvailable {
|
|
13
25
|
readonly dataset: Dataset;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"resolver.d.ts","sourceRoot":"","sources":["../../src/distribution/resolver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"resolver.d.ts","sourceRoot":"","sources":["../../src/distribution/resolver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACjE,OAAO,EAGL,KAAK,eAAe,EACrB,MAAM,yBAAyB,CAAC;AAEjC,qBAAa,oBAAoB;IAE7B,QAAQ,CAAC,YAAY,EAAE,YAAY;IACnC,QAAQ,CAAC,YAAY,EAAE,eAAe,EAAE;IACxC,kEAAkE;IAClE,QAAQ,CAAC,QAAQ,CAAC,EAAE,aAAa;gBAHxB,YAAY,EAAE,YAAY,EAC1B,YAAY,EAAE,eAAe,EAAE;IACxC,kEAAkE;IACzD,QAAQ,CAAC,EAAE,aAAa,YAAA;CAEpC;AAED,0EAA0E;AAC1E,MAAM,WAAW,aAAa;IAC5B,uCAAuC;IACvC,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;IAC5B,oCAAoC;IACpC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,gEAAgE;IAChE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,2EAA2E;IAC3E,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,qBAAa,uBAAuB;IAEhC,QAAQ,CAAC,OAAO,EAAE,OAAO;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM;IACxB,QAAQ,CAAC,YAAY,EAAE,eAAe,EAAE;IACxC,QAAQ,CAAC,YAAY,CAAC,EAAE,YAAY;gBAH3B,OAAO,EAAE,OAAO,EAChB,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,eAAe,EAAE,EAC/B,YAAY,CAAC,EAAE,YAAY,YAAA;CAEvC;AAED;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,YAAY,CAAC;IAC3B,WAAW,EAAE,eAAe,CAAC;CAC9B;AAED;;;;;GAKG;AACH,qBAAa,mBAAmB;IAE5B,QAAQ,CAAC,OAAO,EAAE,OAAO;IACzB,QAAQ,CAAC,YAAY,EAAE,eAAe,EAAE;IACxC,QAAQ,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI;gBAF3B,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,eAAe,EAAE,EAC/B,MAAM,EAAE,YAAY,GAAG,IAAI;CAEvC;AAED,kEAAkE;AAClE,MAAM,WAAW,gBAAgB;IAC/B,4EAA4E;IAC5E,OAAO,CAAC,EAAE,CAAC,YAAY,EAAE,YAAY,EAAE,MAAM,EAAE,eAAe,KAAK,IAAI,CAAC;IACxE,6DAA6D;IAC7D,aAAa,CAAC,EAAE,MAAM,IAAI,CAAC;IAC3B,kEAAkE;IAClE,cAAc,CAAC,EAAE,CAAC,YAAY,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CACtE;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,CACH,OAAO,EAAE,OAAO,EAChB,SAAS,CAAC,EAAE,gBAAgB,GAC3B,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAChC,OAAO,CACL,MAAM,EAAE,mBAAmB,EAC3B,SAAS,CAAC,EAAE,gBAAgB,GAC3B,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC,CAAC;IAC3D;;;;;;;;;;OAUG;IACH,eAAe,CAAC,CACd,MAAM,EAAE,mBAAmB,EAC3B,SAAS,CAAC,EAAE,gBAAgB,GAC3B,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC,CAAC;IAC3D,OAAO,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAED,MAAM,WAAW,iCAAiC;IAChD,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;;;;GAUG;AACH,qBAAa,0BAA2B,YAAW,oBAAoB;IACrE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;gBAErB,OAAO,CAAC,EAAE,iCAAiC;IAIjD,KAAK,CACT,OAAO,EAAE,OAAO,EAChB,SAAS,CAAC,EAAE,gBAAgB,GAC3B,OAAO,CAAC,mBAAmB,CAAC;IA4BzB,OAAO,CACX,MAAM,EAAE,mBAAmB,GAC1B,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC;CAc3D"}
|
|
@@ -2,15 +2,13 @@ import { probe, SparqlProbeResult, } from '@lde/distribution-probe';
|
|
|
2
2
|
export class ResolvedDistribution {
|
|
3
3
|
distribution;
|
|
4
4
|
probeResults;
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
5
|
+
imported;
|
|
6
|
+
constructor(distribution, probeResults,
|
|
7
|
+
/** Set when {@link distribution} serves an imported data dump. */
|
|
8
|
+
imported) {
|
|
9
9
|
this.distribution = distribution;
|
|
10
10
|
this.probeResults = probeResults;
|
|
11
|
-
this.
|
|
12
|
-
this.importDuration = importDuration;
|
|
13
|
-
this.tripleCount = tripleCount;
|
|
11
|
+
this.imported = imported;
|
|
14
12
|
}
|
|
15
13
|
}
|
|
16
14
|
export class NoDistributionAvailable {
|
package/dist/pipeline.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD,OAAO,KAAK,EAKV,MAAM,EACP,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACL,KAAK,oBAAoB,EAI1B,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAY7D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,2BAA2B,CAAC;AAEnC;;;;;;;;;;;GAWG;AACH,qBAAa,cAAe,SAAQ,KAAK;;CAOxC;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;GAKG;AACH,MAAM,WAAW,yBAAyB;IACxC,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,aAAa,CAAC,uBAAuB,CAAC,CAAC;IAC1D;;;;;;;;;;;;OAYG;IACH,kBAAkB,CAAC,EAAE,aAAa,CAAC,yBAAyB,CAAC,CAAC;CAC/D;AAED,MAAM,WAAW,eAAe,CAAC,GAAG,GAAG,IAAI;IACzC,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;IACrC;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,cAAc,EAAE,GAAG,KAAK,CAAC;IAC1D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,gBAAgB,GAAG,SAAS,gBAAgB,EAAE,CAAC;IAC1D;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;CAC/B;AA0OD,qBAAa,QAAQ,CAAC,GAAG,GAAG,IAAI;IAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAyC;IAC3E,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAA2C;IAC/E,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmC;IAC7D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAsB;IACrD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAkB;IACnD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAS;gBAE9B,OAAO,EAAE,eAAe,CAAC,GAAG,CAAC;IA6DnC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAiFZ,cAAc;
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD,OAAO,KAAK,EAKV,MAAM,EACP,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACL,KAAK,oBAAoB,EAI1B,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAY7D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,2BAA2B,CAAC;AAEnC;;;;;;;;;;;GAWG;AACH,qBAAa,cAAe,SAAQ,KAAK;;CAOxC;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;GAKG;AACH,MAAM,WAAW,yBAAyB;IACxC,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,aAAa,CAAC,uBAAuB,CAAC,CAAC;IAC1D;;;;;;;;;;;;OAYG;IACH,kBAAkB,CAAC,EAAE,aAAa,CAAC,yBAAyB,CAAC,CAAC;CAC/D;AAED,MAAM,WAAW,eAAe,CAAC,GAAG,GAAG,IAAI;IACzC,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;IACrC;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,cAAc,EAAE,GAAG,KAAK,CAAC;IAC1D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,gBAAgB,GAAG,SAAS,gBAAgB,EAAE,CAAC;IAC1D;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;CAC/B;AA0OD,qBAAa,QAAQ,CAAC,GAAG,GAAG,IAAI;IAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAyC;IAC3E,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAA2C;IAC/E,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmC;IAC7D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAsB;IACrD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAkB;IACnD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAS;gBAE9B,OAAO,EAAE,eAAe,CAAC,GAAG,CAAC;IA6DnC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAiFZ,cAAc;IA4O5B,+EAA+E;YACjE,aAAa;YAyBb,gBAAgB;IAa9B,OAAO,CAAE,aAAa;IAStB;;;;;OAKG;IACH,OAAO,CAAC,WAAW;IAgBnB;;;;;;;;OAQG;IACH,OAAO,CAAC,0BAA0B;IA0BlC;;;;;OAKG;YACW,SAAS;IAuCvB;;;OAGG;YACW,QAAQ;IA0CtB,2EAA2E;YAC7D,eAAe;YAqBf,QAAQ;YA8EP,SAAS;CAczB"}
|
package/dist/pipeline.js
CHANGED
|
@@ -453,10 +453,10 @@ export class Pipeline {
|
|
|
453
453
|
// the analysis stages is empirically incapable. Switch to the dataset’s
|
|
454
454
|
// data dump and re-run all stages locally, discarding the
|
|
455
455
|
// endpoint-sourced partial results. Only a live endpoint
|
|
456
|
-
// (`
|
|
456
|
+
// (`imported === undefined`) can fall back – a run already on an
|
|
457
457
|
// imported dump has nowhere further to go.
|
|
458
458
|
if (stageFailed &&
|
|
459
|
-
resolved.
|
|
459
|
+
resolved.imported === undefined &&
|
|
460
460
|
this.distributionResolver.resolveFallback) {
|
|
461
461
|
// A failing fallback import must abort only this dataset, never the
|
|
462
462
|
// whole run – matching the per-dataset isolation of the primary resolve
|
|
@@ -473,10 +473,11 @@ export class Pipeline {
|
|
|
473
473
|
// run can skip an unchanged dump (the endpoint's fingerprint is
|
|
474
474
|
// null, which would force a re-import every run).
|
|
475
475
|
let fallbackFingerprint = fingerprint;
|
|
476
|
-
if (fallback.
|
|
477
|
-
const
|
|
476
|
+
if (fallback.imported) {
|
|
477
|
+
const dump = fallback.imported.from;
|
|
478
|
+
const dumpProbeResult = probed.probeResults.find((result) => result.url === dump.accessUrl.toString());
|
|
478
479
|
if (dumpProbeResult) {
|
|
479
|
-
fallbackFingerprint = sourceFingerprint(
|
|
480
|
+
fallbackFingerprint = sourceFingerprint(dump, dumpProbeResult);
|
|
480
481
|
}
|
|
481
482
|
}
|
|
482
483
|
this.reportSelectedDistribution(dataset, fallback, fallbackFingerprint);
|
|
@@ -594,9 +595,9 @@ export class Pipeline {
|
|
|
594
595
|
* deep verdict.
|
|
595
596
|
*/
|
|
596
597
|
reportSelectedDistribution(dataset, resolved, fingerprint) {
|
|
597
|
-
this.reporter?.distributionSelected?.(dataset, resolved.distribution, resolved.
|
|
598
|
-
if (resolved.
|
|
599
|
-
this.reporter?.distributionValidated?.(resolved.
|
|
598
|
+
this.reporter?.distributionSelected?.(dataset, resolved.distribution, resolved.imported);
|
|
599
|
+
if (resolved.imported) {
|
|
600
|
+
this.reporter?.distributionValidated?.(resolved.imported.from, importOutcomeToVerdict(new ImportSuccessful(resolved.imported.from, undefined, resolved.imported.tripleCount), fingerprint));
|
|
600
601
|
}
|
|
601
602
|
}
|
|
602
603
|
/**
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { Dataset, Distribution } from '@lde/dataset';
|
|
2
2
|
import type { ValidityVerdict } from '@lde/distribution-health';
|
|
3
3
|
import type { ValidationReport } from './validator.js';
|
|
4
|
+
import type { ImportDetails } from './distribution/resolver.js';
|
|
4
5
|
import type { TimeoutTransitionEvent } from './sparql/timeoutPolicy.js';
|
|
5
6
|
export interface DistributionAnalysisResult {
|
|
6
7
|
distribution: Distribution;
|
|
@@ -41,7 +42,9 @@ export interface ProgressReporter {
|
|
|
41
42
|
* TypeScript value; consumers decide how (and whether) to record it as RDF.
|
|
42
43
|
*/
|
|
43
44
|
distributionValidated?(distribution: Distribution, verdict: ValidityVerdict): void;
|
|
44
|
-
|
|
45
|
+
/** Called when a dataset’s source is resolved; `imported` is set when the
|
|
46
|
+
* source serves an imported data dump. */
|
|
47
|
+
distributionSelected?(dataset: Dataset, distribution: Distribution, imported?: ImportDetails): void;
|
|
45
48
|
stageStart?(stage: string): void;
|
|
46
49
|
stageProgress?(update: {
|
|
47
50
|
itemsProcessed: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"progressReporter.d.ts","sourceRoot":"","sources":["../src/progressReporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACvD,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAC;AAExE,MAAM,WAAW,0BAA0B;IACzC,YAAY,EAAE,YAAY,CAAC;IAC3B,IAAI,EAAE,QAAQ,GAAG,WAAW,GAAG,eAAe,CAAC;IAC/C,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B;AAED,MAAM,WAAW,gBAAgB;IAC/B,aAAa,CAAC,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,gBAAgB,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACzD;;;;OAIG;IACH,cAAc,CAAC,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,YAAY,CAAC,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACtC,8DAA8D;IAC9D,kBAAkB,CAAC,CAAC,MAAM,EAAE,0BAA0B,GAAG,IAAI,CAAC;IAC9D,6CAA6C;IAC7C,aAAa,CAAC,IAAI,IAAI,CAAC;IACvB,kDAAkD;IAClD,YAAY,CAAC,CAAC,YAAY,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/D;;;;;OAKG;IACH,qBAAqB,CAAC,CACpB,YAAY,EAAE,YAAY,EAC1B,OAAO,EAAE,eAAe,GACvB,IAAI,CAAC;IACR,oBAAoB,CAAC,CACnB,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,
|
|
1
|
+
{"version":3,"file":"progressReporter.d.ts","sourceRoot":"","sources":["../src/progressReporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACvD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAC;AAExE,MAAM,WAAW,0BAA0B;IACzC,YAAY,EAAE,YAAY,CAAC;IAC3B,IAAI,EAAE,QAAQ,GAAG,WAAW,GAAG,eAAe,CAAC;IAC/C,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B;AAED,MAAM,WAAW,gBAAgB;IAC/B,aAAa,CAAC,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,gBAAgB,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACzD;;;;OAIG;IACH,cAAc,CAAC,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,YAAY,CAAC,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACtC,8DAA8D;IAC9D,kBAAkB,CAAC,CAAC,MAAM,EAAE,0BAA0B,GAAG,IAAI,CAAC;IAC9D,6CAA6C;IAC7C,aAAa,CAAC,IAAI,IAAI,CAAC;IACvB,kDAAkD;IAClD,YAAY,CAAC,CAAC,YAAY,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/D;;;;;OAKG;IACH,qBAAqB,CAAC,CACpB,YAAY,EAAE,YAAY,EAC1B,OAAO,EAAE,eAAe,GACvB,IAAI,CAAC;IACR;+CAC2C;IAC3C,oBAAoB,CAAC,CACnB,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,QAAQ,CAAC,EAAE,aAAa,GACvB,IAAI,CAAC;IACR,UAAU,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,aAAa,CAAC,CAAC,MAAM,EAAE;QACrB,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,MAAM,CAAC;QACvB,gBAAgB,EAAE,MAAM,CAAC;QACzB,aAAa,EAAE,MAAM,CAAC;KACvB,GAAG,IAAI,CAAC;IACT,aAAa,CAAC,CACZ,KAAK,EAAE,MAAM,EACb,MAAM,EAAE;QACN,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,MAAM,CAAC;QACvB,QAAQ,EAAE,MAAM,CAAC;KAClB,GACA,IAAI,CAAC;IACR,WAAW,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC;IAChD,YAAY,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACnD;;;;;;;;OAQG;IACH,gBAAgB,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACpE,eAAe,CAAC,CACd,OAAO,EAAE,OAAO,EAChB,MAAM,EAAE;QAAE,gBAAgB,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,GAC1D,IAAI,CAAC;IACR,cAAc,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACxD,gBAAgB,CAAC,CAAC,MAAM,EAAE;QACxB,QAAQ,EAAE,MAAM,CAAC;QACjB,gBAAgB,EAAE,MAAM,CAAC;QACzB,aAAa,EAAE,MAAM,CAAC;KACvB,GAAG,IAAI,CAAC;IACT;;;;OAIG;IACH,gBAAgB,CAAC,CAAC,KAAK,EAAE,sBAAsB,GAAG,IAAI,CAAC;IACvD;;;OAGG;IACH,cAAc,CAAC,CAAC,KAAK,EAAE,sBAAsB,GAAG,IAAI,CAAC;CACtD"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lde/pipeline",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.38.0",
|
|
4
4
|
"repository": {
|
|
5
5
|
"url": "git+https://github.com/ldelements/lde.git",
|
|
6
6
|
"directory": "packages/pipeline"
|
|
@@ -26,9 +26,9 @@
|
|
|
26
26
|
"dependencies": {
|
|
27
27
|
"@lde/dataset": "^0.8.0",
|
|
28
28
|
"@lde/dataset-registry-client": "^0.10.0",
|
|
29
|
-
"@lde/distribution-health": "^0.
|
|
29
|
+
"@lde/distribution-health": "^0.4.0",
|
|
30
30
|
"@lde/distribution-probe": "^0.3.0",
|
|
31
|
-
"@lde/sparql-importer": "^0.
|
|
31
|
+
"@lde/sparql-importer": "^0.8.0",
|
|
32
32
|
"@lde/sparql-server": "^0.5.0",
|
|
33
33
|
"@rdfjs/namespace": "^2.0.1",
|
|
34
34
|
"@rdfjs/types": "^2.0.1",
|