@lde/pipeline 0.6.18 → 0.6.19

package/dist/batch.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Groups items from an async iterable into arrays of at most `size` items.
+ * Yields partial final batches.
+ */
+export declare function batch<T>(iterable: AsyncIterable<T>, size: number): AsyncIterable<T[]>;
+//# sourceMappingURL=batch.d.ts.map

package/dist/batch.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"batch.d.ts","sourceRoot":"","sources":["../src/batch.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,wBAAuB,KAAK,CAAC,CAAC,EAC5B,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,EAC1B,IAAI,EAAE,MAAM,GACX,aAAa,CAAC,CAAC,EAAE,CAAC,CAYpB"}

package/dist/batch.js

@@ -0,0 +1,17 @@
+/**
+ * Groups items from an async iterable into arrays of at most `size` items.
+ * Yields partial final batches.
+ */
+export async function* batch(iterable, size) {
+    let buffer = [];
+    for await (const item of iterable) {
+        buffer.push(item);
+        if (buffer.length === size) {
+            yield buffer;
+            buffer = [];
+        }
+    }
+    if (buffer.length > 0) {
+        yield buffer;
+    }
+}

package/dist/index.d.ts

@@ -1,3 +1,4 @@
+export * from './batch.js';
 export * from './pipeline.js';
 export * from './selector.js';
 export * from './stage.js';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAC;AAC9B,cAAc,eAAe,CAAC;AAC9B,cAAc,YAAY,CAAC;AAC3B,cAAc,WAAW,CAAC;AAC1B,cAAc,uBAAuB,CAAC;AACtC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,mBAAmB,CAAC;AAClC,cAAc,yBAAyB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,eAAe,CAAC;AAC9B,cAAc,YAAY,CAAC;AAC3B,cAAc,WAAW,CAAC;AAC1B,cAAc,uBAAuB,CAAC;AACtC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,mBAAmB,CAAC;AAClC,cAAc,yBAAyB,CAAC"}

package/dist/index.js

@@ -1,3 +1,4 @@
+export * from './batch.js';
 export * from './pipeline.js';
 export * from './selector.js';
 export * from './stage.js';

package/dist/stage.d.ts

@@ -6,14 +6,17 @@ export interface StageOptions {
     name: string;
     executors: Executor | Executor[];
     selector?: StageSelector;
+    /** Maximum number of bindings per executor call. @default 10 */
+    batchSize?: number;
 }
 export declare class Stage {
     readonly name: string;
     private readonly executors;
     private readonly selector?;
+    private readonly batchSize;
     constructor(options: StageOptions);
     run(dataset: Dataset, distribution: Distribution): Promise<AsyncIterable<Quad> | NotSupported>;
-    private collectBindings;
+    private executeAll;
 }
 /** Stage-level selector that yields variable bindings for use in executor queries. Pagination is an implementation detail. */
 export interface StageSelector extends AsyncIterable<VariableBindings> {

package/dist/stage.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EACV,QAAQ,EAER,gBAAgB,EACjB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IACjC,QAAQ,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAgB;gBAE9B,OAAO,EAAE,YAAY;IAQ3B,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,GACzB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC;YAyBhC,eAAe;CAW9B;AAUD,8HAA8H;AAE9H,MAAM,WAAW,aAAc,SAAQ,aAAa,CAAC,gBAAgB,CAAC;CAAG"}
+{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAGpD,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IACjC,QAAQ,CAAC,EAAE,aAAa,CAAC;IACzB,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAgB;IAC1C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAEvB,OAAO,EAAE,YAAY;IAS3B,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,GACzB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC;YAwBhC,UAAU;CAkBzB;AAUD,8HAA8H;AAE9H,MAAM,WAAW,aAAc,SAAQ,aAAa,CAAC,gBAAgB,CAAC;CAAG"}

package/dist/stage.js

@@ -1,23 +1,31 @@
 import { NotSupported } from './sparql/executor.js';
+import { batch } from './batch.js';
 export class Stage {
     name;
     executors;
     selector;
+    batchSize;
     constructor(options) {
         this.name = options.name;
         this.executors = Array.isArray(options.executors)
             ? options.executors
             : [options.executors];
         this.selector = options.selector;
+        this.batchSize = options.batchSize ?? 10;
     }
     async run(dataset, distribution) {
-        const bindings = await this.collectBindings();
-        const executeOptions = bindings.length > 0 ? { bindings } : undefined;
+        if (!this.selector) {
+            return this.executeAll(dataset, distribution);
+        }
         const streams = [];
-        for (const executor of this.executors) {
-            const result = await executor.execute(dataset, distribution, executeOptions);
-            if (!(result instanceof NotSupported)) {
-                streams.push(result);
+        for await (const bindings of batch(this.selector, this.batchSize)) {
+            for (const executor of this.executors) {
+                const result = await executor.execute(dataset, distribution, {
+                    bindings,
+                });
+                if (!(result instanceof NotSupported)) {
+                    streams.push(result);
+                }
             }
         }
         if (streams.length === 0) {
@@ -25,15 +33,18 @@ export class Stage {
         }
         return mergeStreams(streams);
     }
-    async collectBindings() {
-        if (this.selector === undefined) {
-            return [];
+    async executeAll(dataset, distribution) {
+        const streams = [];
+        for (const executor of this.executors) {
+            const result = await executor.execute(dataset, distribution);
+            if (!(result instanceof NotSupported)) {
+                streams.push(result);
+            }
         }
-        const bindings = [];
-        for await (const row of this.selector) {
-            bindings.push(row);
+        if (streams.length === 0) {
+            return new NotSupported('All executors returned NotSupported');
         }
-        return bindings;
+        return mergeStreams(streams);
     }
 }
 async function* mergeStreams(streams) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.6.18",
+  "version": "0.6.19",
   "repository": {
     "url": "https://github.com/ldengine/lde",
     "directory": "packages/pipeline"