@lde/pipeline 0.15.2 → 0.17.0

package/dist/distribution/importResolver.d.ts

@@ -4,19 +4,35 @@ import { type DistributionResolver, NoDistributionAvailable, ResolvedDistributio
 export interface ImportResolverOptions {
     importer: Importer;
     server: SparqlServer;
+    /**
+     * Controls how a dataset's distribution is selected.
+     *
+     * - `'sparql'` (default) — use a dataset's own SPARQL endpoint when one is
+     *   available; fall back to importing a data dump only when no endpoint
+     *   responds.
+     * - `'import'` — always import a data dump into a local SPARQL server,
+     *   even when the dataset advertises a working SPARQL endpoint. Useful when
+     *   the remote endpoint is too slow or unreliable.
+     *
+     * In both modes the inner resolver still runs so that probe results are
+     * collected for reporting and the dataset knowledge graph.
+     */
+    strategy?: 'sparql' | 'import';
 }
 /**
- * A {@link DistributionResolver} decorator that adds import-as-fallback logic.
+ * A {@link DistributionResolver} decorator that adds data-dump import logic.
  *
- * Delegates to an inner resolver first. If the inner resolver returns
- * {@link NoDistributionAvailable}, tries importing the dataset and optionally
- * starts a SPARQL server.
+ * Wraps an inner resolver (typically {@link SparqlDistributionResolver}) and
+ * adds the ability to import a data dump into a local SPARQL server. The
+ * {@link ImportResolverOptions.strategy | strategy} option controls whether the
+ * inner resolver's SPARQL endpoint is preferred or bypassed.
  */
 export declare class ImportResolver implements DistributionResolver {
     private readonly inner;
     private readonly options;
     constructor(inner: DistributionResolver, options: ImportResolverOptions);
     resolve(...args: Parameters<DistributionResolver['resolve']>): Promise<ResolvedDistribution | NoDistributionAvailable>;
+    private importDataset;
     cleanup(): Promise<void>;
 }
 //# sourceMappingURL=importResolver.d.ts.map

package/dist/distribution/importResolver.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"importResolver.d.ts","sourceRoot":"","sources":["../../src/distribution/importResolver.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EACL,KAAK,oBAAoB,EACzB,uBAAuB,EACvB,oBAAoB,EACrB,MAAM,eAAe,CAAC;AAEvB,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,EAAE,YAAY,CAAC;CACtB;AAED;;;;;;GAMG;AACH,qBAAa,cAAe,YAAW,oBAAoB;IAEvD,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,OAAO;gBADP,KAAK,EAAE,oBAAoB,EAC3B,OAAO,EAAE,qBAAqB;IAG3C,OAAO,CACX,GAAG,IAAI,EAAE,UAAU,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC,GACnD,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC;IAiCpD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAG/B"}
+{"version":3,"file":"importResolver.d.ts","sourceRoot":"","sources":["../../src/distribution/importResolver.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EACL,KAAK,oBAAoB,EACzB,uBAAuB,EACvB,oBAAoB,EACrB,MAAM,eAAe,CAAC;AAEvB,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,EAAE,YAAY,CAAC;IACrB;;;;;;;;;;;;OAYG;IACH,QAAQ,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAAC;CAChC;AAED;;;;;;;GAOG;AACH,qBAAa,cAAe,YAAW,oBAAoB;IAEvD,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,OAAO;gBADP,KAAK,EAAE,oBAAoB,EAC3B,OAAO,EAAE,qBAAqB;IAG3C,OAAO,CACX,GAAG,IAAI,EAAE,UAAU,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC,GACnD,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC;YAgB5C,aAAa;IAgCrB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAG/B"}

package/dist/distribution/importResolver.js

@@ -2,11 +2,12 @@ import { Distribution } from '@lde/dataset';
 import { ImportFailed, ImportSuccessful } from '@lde/sparql-importer';
 import { NoDistributionAvailable, ResolvedDistribution, } from './resolver.js';
 /**
- * A {@link DistributionResolver} decorator that adds import-as-fallback logic.
+ * A {@link DistributionResolver} decorator that adds data-dump import logic.
  *
- * Delegates to an inner resolver first. If the inner resolver returns
- * {@link NoDistributionAvailable}, tries importing the dataset and optionally
- * starts a SPARQL server.
+ * Wraps an inner resolver (typically {@link SparqlDistributionResolver}) and
+ * adds the ability to import a data dump into a local SPARQL server. The
+ * {@link ImportResolverOptions.strategy | strategy} option controls whether the
+ * inner resolver's SPARQL endpoint is preferred or bypassed.
  */
 export class ImportResolver {
     inner;
@@ -16,19 +17,26 @@ export class ImportResolver {
         this.options = options;
     }
     async resolve(...args) {
+        const [dataset] = args;
         const result = await this.inner.resolve(...args);
-        if (result instanceof ResolvedDistribution)
+        // 'sparql' strategy (default): use SPARQL endpoint if inner found one.
+        if (this.options.strategy !== 'import' &&
+            result instanceof ResolvedDistribution) {
             return result;
-        const [dataset] = args;
+        }
+        // Either 'import' strategy or inner found nothing: import a data dump.
+        return this.importDataset(dataset, result.probeResults);
+    }
+    async importDataset(dataset, probeResults) {
         const importStart = Date.now();
         const importResult = await this.options.importer.import(dataset);
         if (importResult instanceof ImportSuccessful) {
             await this.options.server.start();
             const distribution = Distribution.sparql(this.options.server.queryEndpoint, importResult.identifier);
             distribution.subjectFilter = importResult.distribution.subjectFilter;
-            return new ResolvedDistribution(distribution, result.probeResults, importResult.distribution, Date.now() - importStart);
+            return new ResolvedDistribution(distribution, probeResults, importResult.distribution, Date.now() - importStart);
         }
-        return new NoDistributionAvailable(dataset, 'No SPARQL endpoint or importable data dump available', result.probeResults, importResult instanceof ImportFailed ? importResult : undefined);
+        return new NoDistributionAvailable(dataset, 'No SPARQL endpoint or importable data dump available', probeResults, importResult instanceof ImportFailed ? importResult : undefined);
     }
     async cleanup() {
         await this.options.server.stop();

package/dist/pipeline.js

@@ -115,13 +115,13 @@ export class Pipeline {
     async runStage(dataset, distribution, stage) {
         this.reporter?.stageStart?.(stage.name);
         const stageStart = Date.now();
-        let elementsProcessed = 0;
+        let itemsProcessed = 0;
         let quadsGenerated = 0;
         const result = await stage.run(dataset, distribution, this.writer, {
-            onProgress: (elements, quads) => {
-                elementsProcessed = elements;
+            onProgress: (items, quads) => {
+                itemsProcessed = items;
                 quadsGenerated = quads;
-                this.reporter?.stageProgress?.({ elementsProcessed, quadsGenerated });
+                this.reporter?.stageProgress?.({ itemsProcessed, quadsGenerated });
             },
         });
         if (result instanceof NotSupported) {
@@ -129,7 +129,7 @@ export class Pipeline {
         }
         else {
             this.reporter?.stageComplete?.(stage.name, {
-                elementsProcessed,
+                itemsProcessed,
                 quadsGenerated,
                 duration: Date.now() - stageStart,
             });
@@ -170,13 +170,13 @@ export class Pipeline {
     async runChainedStage(dataset, distribution, stage, stageWriter) {
         this.reporter?.stageStart?.(stage.name);
         const stageStart = Date.now();
-        let elementsProcessed = 0;
+        let itemsProcessed = 0;
         let quadsGenerated = 0;
         const result = await stage.run(dataset, distribution, stageWriter, {
-            onProgress: (elements, quads) => {
-                elementsProcessed = elements;
+            onProgress: (items, quads) => {
+                itemsProcessed = items;
                 quadsGenerated = quads;
-                this.reporter?.stageProgress?.({ elementsProcessed, quadsGenerated });
+                this.reporter?.stageProgress?.({ itemsProcessed, quadsGenerated });
             },
         });
         if (result instanceof NotSupported) {
@@ -184,7 +184,7 @@ export class Pipeline {
             throw new Error(`Stage '${stage.name}' returned NotSupported in chained mode`);
         }
         this.reporter?.stageComplete?.(stage.name, {
-            elementsProcessed,
+            itemsProcessed,
             quadsGenerated,
             duration: Date.now() - stageStart,
         });

package/dist/progressReporter.d.ts

@@ -14,11 +14,11 @@ export interface ProgressReporter {
     distributionSelected?(dataset: Dataset, distribution: Distribution, importedFrom?: Distribution, importDuration?: number): void;
     stageStart?(stage: string): void;
     stageProgress?(update: {
-        elementsProcessed: number;
+        itemsProcessed: number;
         quadsGenerated: number;
     }): void;
     stageComplete?(stage: string, result: {
-        elementsProcessed: number;
+        itemsProcessed: number;
         quadsGenerated: number;
         duration: number;
     }): void;

package/dist/progressReporter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"progressReporter.d.ts","sourceRoot":"","sources":["../src/progressReporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE1D,MAAM,WAAW,0BAA0B;IACzC,YAAY,EAAE,YAAY,CAAC;IAC3B,IAAI,EAAE,QAAQ,GAAG,WAAW,GAAG,eAAe,CAAC;IAC/C,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,gBAAgB;IAC/B,aAAa,CAAC,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,gBAAgB,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACzD,YAAY,CAAC,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACtC,qBAAqB,CAAC,CACpB,OAAO,EAAE,OAAO,EAChB,OAAO,EAAE,0BAA0B,EAAE,GACpC,IAAI,CAAC;IACR,oBAAoB,CAAC,CACnB,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,YAAY,CAAC,EAAE,YAAY,EAC3B,cAAc,CAAC,EAAE,MAAM,GACtB,IAAI,CAAC;IACR,UAAU,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,aAAa,CAAC,CAAC,MAAM,EAAE;QACrB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,cAAc,EAAE,MAAM,CAAC;KACxB,GAAG,IAAI,CAAC;IACT,aAAa,CAAC,CACZ,KAAK,EAAE,MAAM,EACb,MAAM,EAAE;QACN,iBAAiB,EAAE,MAAM,CAAC;QAC1B,cAAc,EAAE,MAAM,CAAC;QACvB,QAAQ,EAAE,MAAM,CAAC;KAClB,GACA,IAAI,CAAC;IACR,WAAW,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC;IAChD,YAAY,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACnD,eAAe,CAAC,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACzC,cAAc,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACxD,gBAAgB,CAAC,CAAC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;CACvD"}
+{"version":3,"file":"progressReporter.d.ts","sourceRoot":"","sources":["../src/progressReporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE1D,MAAM,WAAW,0BAA0B;IACzC,YAAY,EAAE,YAAY,CAAC;IAC3B,IAAI,EAAE,QAAQ,GAAG,WAAW,GAAG,eAAe,CAAC;IAC/C,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,gBAAgB;IAC/B,aAAa,CAAC,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,gBAAgB,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACzD,YAAY,CAAC,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACtC,qBAAqB,CAAC,CACpB,OAAO,EAAE,OAAO,EAChB,OAAO,EAAE,0BAA0B,EAAE,GACpC,IAAI,CAAC;IACR,oBAAoB,CAAC,CACnB,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,YAAY,CAAC,EAAE,YAAY,EAC3B,cAAc,CAAC,EAAE,MAAM,GACtB,IAAI,CAAC;IACR,UAAU,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,aAAa,CAAC,CAAC,MAAM,EAAE;QACrB,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,MAAM,CAAC;KACxB,GAAG,IAAI,CAAC;IACT,aAAa,CAAC,CACZ,KAAK,EAAE,MAAM,EACb,MAAM,EAAE;QACN,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,MAAM,CAAC;QACvB,QAAQ,EAAE,MAAM,CAAC;KAClB,GACA,IAAI,CAAC;IACR,WAAW,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC;IAChD,YAAY,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACnD,eAAe,CAAC,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACzC,cAAc,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACxD,gBAAgB,CAAC,CAAC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;CACvD"}

package/dist/stage.d.ts

@@ -17,7 +17,7 @@ export interface StageOptions {
     stages?: Stage[];
 }
 export interface RunOptions {
-    onProgress?: (elementsProcessed: number, quadsGenerated: number) => void;
+    onProgress?: (itemsProcessed: number, quadsGenerated: number) => void;
 }
 export declare class Stage {
     readonly name: string;

package/dist/stage.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAGjD,mEAAmE;AACnE,MAAM,MAAM,aAAa,GAAG,CAC1B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,EAC1B,OAAO,EAAE,OAAO,KACb,aAAa,CAAC,IAAI,CAAC,CAAC;AAEzB,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IACjC,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,CAAC,iBAAiB,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;CAC1E;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;gBAE5B,OAAO,EAAE,YAAY;IAW3B,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;YAmBjB,eAAe;YA+Gf,UAAU;CAqBzB;AAUD,6GAA6G;AAC7G,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,YAAY,EAAE,YAAY,GAAG,aAAa,CAAC,gBAAgB,CAAC,CAAC;CACrE"}
+{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAGjD,mEAAmE;AACnE,MAAM,MAAM,aAAa,GAAG,CAC1B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,EAC1B,OAAO,EAAE,OAAO,KACb,aAAa,CAAC,IAAI,CAAC,CAAC;AAEzB,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IACjC,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,CAAC,cAAc,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;CACvE;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;gBAE5B,OAAO,EAAE,YAAY;IAW3B,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;YAmBjB,eAAe;YA+Gf,UAAU;CAqBzB;AAUD,6GAA6G;AAC7G,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,YAAY,EAAE,YAAY,GAAG,aAAa,CAAC,gBAAgB,CAAC,CAAC;CACrE"}

package/dist/stage.js

@@ -49,7 +49,7 @@ export class Stage {
             }
         })();
         const queue = new AsyncQueue();
-        let elementsProcessed = 0;
+        let itemsProcessed = 0;
         let quadsGenerated = 0;
         let hasResults = false;
         const dispatch = async () => {
@@ -88,8 +88,8 @@ export class Stage {
                                     quadsGenerated++;
                                 }
                             }
-                            elementsProcessed += bindings.length;
-                            options?.onProgress?.(elementsProcessed, quadsGenerated);
+                            itemsProcessed += bindings.length;
+                            options?.onProgress?.(itemsProcessed, quadsGenerated);
                         })());
                     }
                 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.15.2",
+  "version": "0.17.0",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/pipeline"