@lde/pipeline 0.6.17 → 0.6.19

package/dist/batch.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Groups items from an async iterable into arrays of at most `size` items.
+ * Yields partial final batches.
+ */
+export declare function batch<T>(iterable: AsyncIterable<T>, size: number): AsyncIterable<T[]>;
+//# sourceMappingURL=batch.d.ts.map

package/dist/batch.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"batch.d.ts","sourceRoot":"","sources":["../src/batch.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,wBAAuB,KAAK,CAAC,CAAC,EAC5B,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,EAC1B,IAAI,EAAE,MAAM,GACX,aAAa,CAAC,CAAC,EAAE,CAAC,CAYpB"}

package/dist/batch.js

@@ -0,0 +1,17 @@
+/**
+ * Groups items from an async iterable into arrays of at most `size` items.
+ * Yields partial final batches.
+ */
+export async function* batch(iterable, size) {
+    let buffer = [];
+    for await (const item of iterable) {
+        buffer.push(item);
+        if (buffer.length === size) {
+            yield buffer;
+            buffer = [];
+        }
+    }
+    if (buffer.length > 0) {
+        yield buffer;
+    }
+}

package/dist/distribution/index.d.ts

@@ -1,4 +1,5 @@
 export { probe, NetworkError, SparqlProbeResult, DataDumpProbeResult, type ProbeResultType, } from './probe.js';
 export { probeResultsToQuads } from './report.js';
 export { ResolvedDistribution, NoDistributionAvailable, SparqlDistributionResolver, type DistributionResolver, type SparqlDistributionResolverOptions, } from './resolver.js';
+export { resolveDistributions, type DistributionStageResult, } from './resolveDistributions.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/distribution/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/distribution/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,EACL,YAAY,EACZ,iBAAiB,EACjB,mBAAmB,EACnB,KAAK,eAAe,GACrB,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAElD,OAAO,EACL,oBAAoB,EACpB,uBAAuB,EACvB,0BAA0B,EAC1B,KAAK,oBAAoB,EACzB,KAAK,iCAAiC,GACvC,MAAM,eAAe,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/distribution/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,EACL,YAAY,EACZ,iBAAiB,EACjB,mBAAmB,EACnB,KAAK,eAAe,GACrB,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAElD,OAAO,EACL,oBAAoB,EACpB,uBAAuB,EACvB,0BAA0B,EAC1B,KAAK,oBAAoB,EACzB,KAAK,iCAAiC,GACvC,MAAM,eAAe,CAAC;AAEvB,OAAO,EACL,oBAAoB,EACpB,KAAK,uBAAuB,GAC7B,MAAM,2BAA2B,CAAC"}

package/dist/distribution/index.js

@@ -1,3 +1,4 @@
 export { probe, NetworkError, SparqlProbeResult, DataDumpProbeResult, } from './probe.js';
 export { probeResultsToQuads } from './report.js';
 export { ResolvedDistribution, NoDistributionAvailable, SparqlDistributionResolver, } from './resolver.js';
+export { resolveDistributions, } from './resolveDistributions.js';

package/dist/distribution/resolveDistributions.d.ts

@@ -0,0 +1,11 @@
+import { type Distribution, type Dataset } from '@lde/dataset';
+import type { Quad } from 'n3';
+import type { ProbeResultType } from './probe.js';
+import { type DistributionResolver } from './resolver.js';
+export interface DistributionStageResult {
+    distribution: Distribution | null;
+    probeResults: ProbeResultType[];
+    quads: AsyncIterable<Quad>;
+}
+export declare function resolveDistributions(dataset: Dataset, resolver: DistributionResolver): Promise<DistributionStageResult>;
+//# sourceMappingURL=resolveDistributions.d.ts.map

package/dist/distribution/resolveDistributions.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"resolveDistributions.d.ts","sourceRoot":"","sources":["../../src/distribution/resolveDistributions.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,YAAY,EAAE,KAAK,OAAO,EAAE,MAAM,cAAc,CAAC;AAC/D,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC/B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAElD,OAAO,EACL,KAAK,oBAAoB,EAE1B,MAAM,eAAe,CAAC;AAEvB,MAAM,WAAW,uBAAuB;IACtC,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAClC,YAAY,EAAE,eAAe,EAAE,CAAC;IAChC,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC;CAC5B;AAED,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,oBAAoB,GAC7B,OAAO,CAAC,uBAAuB,CAAC,CAoBlC"}

package/dist/distribution/resolveDistributions.js

@@ -0,0 +1,17 @@
+import { probeResultsToQuads } from './report.js';
+import { NoDistributionAvailable, } from './resolver.js';
+export async function resolveDistributions(dataset, resolver) {
+    const result = await resolver.resolve(dataset);
+    if (result instanceof NoDistributionAvailable) {
+        return {
+            distribution: null,
+            probeResults: result.probeResults,
+            quads: probeResultsToQuads(result.probeResults, dataset.iri.toString(), result.importFailed),
+        };
+    }
+    return {
+        distribution: result.distribution,
+        probeResults: result.probeResults,
+        quads: probeResultsToQuads(result.probeResults, dataset.iri.toString()),
+    };
+}

package/dist/distribution/resolver.d.ts

@@ -1,13 +1,18 @@
 import { Dataset, Distribution } from '@lde/dataset';
 import type { Importer } from '@lde/sparql-importer';
+import { ImportFailed } from '@lde/sparql-importer';
+import { type ProbeResultType } from './probe.js';
 export declare class ResolvedDistribution {
     readonly distribution: Distribution;
-    constructor(distribution: Distribution);
+    readonly probeResults: ProbeResultType[];
+    constructor(distribution: Distribution, probeResults: ProbeResultType[]);
 }
 export declare class NoDistributionAvailable {
     readonly dataset: Dataset;
     readonly message: string;
-    constructor(dataset: Dataset, message: string);
+    readonly probeResults: ProbeResultType[];
+    readonly importFailed?: ImportFailed | undefined;
+    constructor(dataset: Dataset, message: string, probeResults: ProbeResultType[], importFailed?: ImportFailed | undefined);
 }
 export interface DistributionResolver {
     resolve(dataset: Dataset): Promise<ResolvedDistribution | NoDistributionAvailable>;

package/dist/distribution/resolver.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"resolver.d.ts","sourceRoot":"","sources":["../../src/distribution/resolver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAIrD,qBAAa,oBAAoB;IACnB,QAAQ,CAAC,YAAY,EAAE,YAAY;gBAA1B,YAAY,EAAE,YAAY;CAChD;AAED,qBAAa,uBAAuB;IACtB,QAAQ,CAAC,OAAO,EAAE,OAAO;IAAE,QAAQ,CAAC,OAAO,EAAE,MAAM;gBAA1C,OAAO,EAAE,OAAO,EAAW,OAAO,EAAE,MAAM;CAChE;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,CACL,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC,CAAC;CAC5D;AAED,MAAM,WAAW,iCAAiC;IAChD,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;;;GASG;AACH,qBAAa,0BAA2B,YAAW,oBAAoB;IACrE,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAW;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;gBAErB,OAAO,CAAC,EAAE,iCAAiC;IAKjD,OAAO,CACX,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC;CAsC3D"}
+{"version":3,"file":"resolver.d.ts","sourceRoot":"","sources":["../../src/distribution/resolver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,YAAY,EAAoB,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAA4B,KAAK,eAAe,EAAE,MAAM,YAAY,CAAC;AAE5E,qBAAa,oBAAoB;IAE7B,QAAQ,CAAC,YAAY,EAAE,YAAY;IACnC,QAAQ,CAAC,YAAY,EAAE,eAAe,EAAE;gBAD/B,YAAY,EAAE,YAAY,EAC1B,YAAY,EAAE,eAAe,EAAE;CAE3C;AAED,qBAAa,uBAAuB;IAEhC,QAAQ,CAAC,OAAO,EAAE,OAAO;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM;IACxB,QAAQ,CAAC,YAAY,EAAE,eAAe,EAAE;IACxC,QAAQ,CAAC,YAAY,CAAC,EAAE,YAAY;gBAH3B,OAAO,EAAE,OAAO,EAChB,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,eAAe,EAAE,EAC/B,YAAY,CAAC,EAAE,YAAY,YAAA;CAEvC;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,CACL,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC,CAAC;CAC5D;AAED,MAAM,WAAW,iCAAiC;IAChD,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;;;GASG;AACH,qBAAa,0BAA2B,YAAW,oBAAoB;IACrE,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAW;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;gBAErB,OAAO,CAAC,EAAE,iCAAiC;IAKjD,OAAO,CACX,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC;CA+C3D"}

package/dist/distribution/resolver.js

@@ -1,18 +1,24 @@
 import { Distribution } from '@lde/dataset';
-import { ImportSuccessful } from '@lde/sparql-importer';
+import { ImportFailed, ImportSuccessful } from '@lde/sparql-importer';
 import { probe, SparqlProbeResult } from './probe.js';
 export class ResolvedDistribution {
     distribution;
-    constructor(distribution) {
+    probeResults;
+    constructor(distribution, probeResults) {
         this.distribution = distribution;
+        this.probeResults = probeResults;
     }
 }
 export class NoDistributionAvailable {
     dataset;
     message;
-    constructor(dataset, message) {
+    probeResults;
+    importFailed;
+    constructor(dataset, message, probeResults, importFailed) {
         this.dataset = dataset;
         this.message = message;
+        this.probeResults = probeResults;
+        this.importFailed = importFailed;
     }
 }
 /**
@@ -41,7 +47,7 @@ export class SparqlDistributionResolver {
             if (distribution.isSparql() &&
                 result instanceof SparqlProbeResult &&
                 result.isSuccess()) {
-                return new ResolvedDistribution(distribution);
+                return new ResolvedDistribution(distribution, results);
             }
         }
         // No SPARQL endpoint; try importer if available.
@@ -49,9 +55,12 @@ export class SparqlDistributionResolver {
             const importResult = await this.importer.import(dataset);
             if (importResult instanceof ImportSuccessful) {
                 const distribution = Distribution.sparql(importResult.distribution.accessUrl, importResult.identifier);
-                return new ResolvedDistribution(distribution);
+                return new ResolvedDistribution(distribution, results);
+            }
+            if (importResult instanceof ImportFailed) {
+                return new NoDistributionAvailable(dataset, 'No SPARQL endpoint or importable data dump available', results, importResult);
             }
         }
-        return new NoDistributionAvailable(dataset, 'No SPARQL endpoint or importable data dump available');
+        return new NoDistributionAvailable(dataset, 'No SPARQL endpoint or importable data dump available', results);
     }
 }

package/dist/index.d.ts

@@ -1,3 +1,4 @@
+export * from './batch.js';
 export * from './pipeline.js';
 export * from './selector.js';
 export * from './stage.js';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAC;AAC9B,cAAc,eAAe,CAAC;AAC9B,cAAc,YAAY,CAAC;AAC3B,cAAc,WAAW,CAAC;AAC1B,cAAc,uBAAuB,CAAC;AACtC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,mBAAmB,CAAC;AAClC,cAAc,yBAAyB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,eAAe,CAAC;AAC9B,cAAc,YAAY,CAAC;AAC3B,cAAc,WAAW,CAAC;AAC1B,cAAc,uBAAuB,CAAC;AACtC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,mBAAmB,CAAC;AAClC,cAAc,yBAAyB,CAAC"}

package/dist/index.js

@@ -1,3 +1,4 @@
+export * from './batch.js';
 export * from './pipeline.js';
 export * from './selector.js';
 export * from './stage.js';

package/dist/stage.d.ts

@@ -6,14 +6,17 @@ export interface StageOptions {
     name: string;
     executors: Executor | Executor[];
     selector?: StageSelector;
+    /** Maximum number of bindings per executor call. @default 10 */
+    batchSize?: number;
 }
 export declare class Stage {
     readonly name: string;
     private readonly executors;
     private readonly selector?;
+    private readonly batchSize;
     constructor(options: StageOptions);
     run(dataset: Dataset, distribution: Distribution): Promise<AsyncIterable<Quad> | NotSupported>;
-    private collectBindings;
+    private executeAll;
 }
 /** Stage-level selector that yields variable bindings for use in executor queries. Pagination is an implementation detail. */
 export interface StageSelector extends AsyncIterable<VariableBindings> {

package/dist/stage.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EACV,QAAQ,EAER,gBAAgB,EACjB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IACjC,QAAQ,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAgB;gBAE9B,OAAO,EAAE,YAAY;IAQ3B,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,GACzB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC;YAyBhC,eAAe;CAW9B;AAUD,8HAA8H;AAE9H,MAAM,WAAW,aAAc,SAAQ,aAAa,CAAC,gBAAgB,CAAC;CAAG"}
+{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAGpD,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IACjC,QAAQ,CAAC,EAAE,aAAa,CAAC;IACzB,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAgB;IAC1C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAEvB,OAAO,EAAE,YAAY;IAS3B,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,GACzB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC;YAwBhC,UAAU;CAkBzB;AAUD,8HAA8H;AAE9H,MAAM,WAAW,aAAc,SAAQ,aAAa,CAAC,gBAAgB,CAAC;CAAG"}

package/dist/stage.js

@@ -1,23 +1,31 @@
 import { NotSupported } from './sparql/executor.js';
+import { batch } from './batch.js';
 export class Stage {
     name;
     executors;
     selector;
+    batchSize;
     constructor(options) {
         this.name = options.name;
         this.executors = Array.isArray(options.executors)
             ? options.executors
             : [options.executors];
         this.selector = options.selector;
+        this.batchSize = options.batchSize ?? 10;
     }
     async run(dataset, distribution) {
-        const bindings = await this.collectBindings();
-        const executeOptions = bindings.length > 0 ? { bindings } : undefined;
+        if (!this.selector) {
+            return this.executeAll(dataset, distribution);
+        }
         const streams = [];
-        for (const executor of this.executors) {
-            const result = await executor.execute(dataset, distribution, executeOptions);
-            if (!(result instanceof NotSupported)) {
-                streams.push(result);
+        for await (const bindings of batch(this.selector, this.batchSize)) {
+            for (const executor of this.executors) {
+                const result = await executor.execute(dataset, distribution, {
+                    bindings,
+                });
+                if (!(result instanceof NotSupported)) {
+                    streams.push(result);
+                }
             }
         }
         if (streams.length === 0) {
@@ -25,15 +33,18 @@ export class Stage {
         }
         return mergeStreams(streams);
     }
-    async collectBindings() {
-        if (this.selector === undefined) {
-            return [];
+    async executeAll(dataset, distribution) {
+        const streams = [];
+        for (const executor of this.executors) {
+            const result = await executor.execute(dataset, distribution);
+            if (!(result instanceof NotSupported)) {
+                streams.push(result);
+            }
         }
-        const bindings = [];
-        for await (const row of this.selector) {
-            bindings.push(row);
+        if (streams.length === 0) {
+            return new NotSupported('All executors returned NotSupported');
         }
-        return bindings;
+        return mergeStreams(streams);
     }
 }
 async function* mergeStreams(streams) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.6.17",
+  "version": "0.6.19",
   "repository": {
     "url": "https://github.com/ldengine/lde",
     "directory": "packages/pipeline"