@lde/pipeline 0.6.28 → 0.6.29

package/README.md

@@ -86,9 +86,7 @@ import {
 } from '@lde/pipeline';
 
 const pipeline = new Pipeline({
-  name: 'example',
   datasetSelector: new ManualDatasetSelection([dataset]),
-  distributionResolver: new SparqlDistributionResolver(),
   stages: [
     new Stage({
       name: 'per-class',
@@ -101,7 +99,7 @@ const pipeline = new Pipeline({
       }),
     }),
   ],
-  writer: new SparqlUpdateWriter({
+  writers: new SparqlUpdateWriter({
     endpoint: new URL('http://localhost:7200/repositories/lde/statements'),
   }),
 });

package/dist/pipeline.d.ts

@@ -5,18 +5,26 @@ import { type DistributionResolver } from './distribution/resolver.js';
 import type { StageOutputResolver } from './stageOutputResolver.js';
 import type { ProgressReporter } from './progressReporter.js';
 export interface PipelineOptions {
-    name: string;
     datasetSelector: DatasetSelector;
     stages: Stage[];
-    writer: Writer;
-    distributionResolver: DistributionResolver;
-    stageOutputResolver?: StageOutputResolver;
-    outputDir?: string;
-    outputFormat?: 'turtle' | 'n-triples' | 'n-quads';
+    writers: Writer | Writer[];
+    name?: string;
+    distributionResolver?: DistributionResolver;
+    chaining?: {
+        stageOutputResolver: StageOutputResolver;
+        outputDir: string;
+        outputFormat?: 'turtle' | 'n-triples' | 'n-quads';
+    };
     reporter?: ProgressReporter;
 }
 export declare class Pipeline {
-    private readonly options;
+    private readonly name;
+    private readonly datasetSelector;
+    private readonly stages;
+    private readonly writer;
+    private readonly distributionResolver;
+    private readonly chaining?;
+    private readonly reporter?;
     constructor(options: PipelineOptions);
     run(): Promise<void>;
     private processDataset;

package/dist/pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAE1B,MAAM,4BAA4B,CAAC;AAEpC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAE9D,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,oBAAoB,EAAE,oBAAoB,CAAC;IAC3C,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAC1C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;IAClD,QAAQ,CAAC,EAAE,gBAAgB,CAAC;CAC7B;AAED,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkB;gBAE9B,OAAO,EAAE,eAAe;IAe9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAcZ,cAAc;YA2Bd,QAAQ;YAgCR,QAAQ;YAoDR,eAAe;YAoCd,SAAS;CAUzB"}
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAE1B,MAAM,4BAA4B,CAAC;AAGpC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAE9D,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;KACnD,CAAC;IACF,QAAQ,CAAC,EAAE,gBAAgB,CAAC;CAC7B;AAmBD,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAA8B;IACxD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;gBAEjC,OAAO,EAAE,eAAe;IAoB9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAaZ,cAAc;YA0Bd,QAAQ;YA8BR,QAAQ;YAmDR,eAAe;YAkCd,SAAS;CAUzB"}

package/dist/pipeline.js

@@ -2,40 +2,67 @@ import { createReadStream } from 'node:fs';
 import { StreamParser } from 'n3';
 import { FileWriter } from './writer/fileWriter.js';
 import { NoDistributionAvailable, } from './distribution/resolver.js';
+import { SparqlDistributionResolver } from './distribution/index.js';
 import { NotSupported } from './sparql/executor.js';
+class FanOutWriter {
+    writers;
+    constructor(writers) {
+        this.writers = writers;
+    }
+    async write(dataset, quads) {
+        const collected = [];
+        for await (const quad of quads)
+            collected.push(quad);
+        for (const w of this.writers) {
+            await w.write(dataset, (async function* () {
+                yield* collected;
+            })());
+        }
+    }
+}
 export class Pipeline {
-    options;
+    name;
+    datasetSelector;
+    stages;
+    writer;
+    distributionResolver;
+    chaining;
+    reporter;
     constructor(options) {
         const hasSubStages = options.stages.some((stage) => stage.stages.length > 0);
-        if (hasSubStages && !options.stageOutputResolver) {
-            throw new Error('stageOutputResolver is required when any stage has sub-stages');
-        }
-        if (hasSubStages && !options.outputDir) {
-            throw new Error('outputDir is required when any stage has sub-stages');
+        if (hasSubStages && !options.chaining) {
+            throw new Error('chaining is required when any stage has sub-stages');
         }
-        this.options = options;
+        this.name = options.name ?? '';
+        this.datasetSelector = options.datasetSelector;
+        this.stages = options.stages;
+        this.writer = Array.isArray(options.writers)
+            ? new FanOutWriter(options.writers)
+            : options.writers;
+        this.distributionResolver =
+            options.distributionResolver ?? new SparqlDistributionResolver();
+        this.chaining = options.chaining;
+        this.reporter = options.reporter;
     }
     async run() {
-        const { datasetSelector, reporter, name } = this.options;
         const start = Date.now();
-        reporter?.pipelineStart(name);
-        const datasets = await datasetSelector.select();
+        this.reporter?.pipelineStart(this.name);
+        const datasets = await this.datasetSelector.select();
         for await (const dataset of datasets) {
             await this.processDataset(dataset);
         }
-        reporter?.pipelineComplete({ duration: Date.now() - start });
+        this.reporter?.pipelineComplete({ duration: Date.now() - start });
     }
     async processDataset(dataset) {
-        const { distributionResolver, reporter } = this.options;
         const datasetIri = dataset.iri.toString();
-        reporter?.datasetStart(datasetIri);
-        const resolved = await distributionResolver.resolve(dataset);
+        this.reporter?.datasetStart(datasetIri);
+        const resolved = await this.distributionResolver.resolve(dataset);
         if (resolved instanceof NoDistributionAvailable) {
-            reporter?.datasetSkipped(datasetIri, resolved.message);
+            this.reporter?.datasetSkipped(datasetIri, resolved.message);
             return;
         }
         try {
-            for (const stage of this.options.stages) {
+            for (const stage of this.stages) {
                 if (stage.stages.length > 0) {
                     await this.runChain(dataset, resolved.distribution, stage);
                 }
@@ -47,26 +74,25 @@ export class Pipeline {
         catch {
             // Stage error for this dataset; continue to next dataset.
         }
-        reporter?.datasetComplete(datasetIri);
+        this.reporter?.datasetComplete(datasetIri);
     }
     async runStage(dataset, distribution, stage) {
-        const { writer, reporter } = this.options;
-        reporter?.stageStart(stage.name);
+        this.reporter?.stageStart(stage.name);
         const stageStart = Date.now();
         let elementsProcessed = 0;
         let quadsGenerated = 0;
-        const result = await stage.run(dataset, distribution, writer, {
+        const result = await stage.run(dataset, distribution, this.writer, {
             onProgress: (elements, quads) => {
                 elementsProcessed = elements;
                 quadsGenerated = quads;
-                reporter?.stageProgress({ elementsProcessed, quadsGenerated });
+                this.reporter?.stageProgress({ elementsProcessed, quadsGenerated });
             },
         });
         if (result instanceof NotSupported) {
-            reporter?.stageSkipped(stage.name, result.message);
+            this.reporter?.stageSkipped(stage.name, result.message);
         }
         else {
-            reporter?.stageComplete(stage.name, {
+            this.reporter?.stageComplete(stage.name, {
                 elementsProcessed,
                 quadsGenerated,
                 duration: Date.now() - stageStart,
@@ -74,7 +100,7 @@ export class Pipeline {
         }
     }
     async runChain(dataset, distribution, stage) {
-        const { writer, stageOutputResolver, outputDir, outputFormat } = this.options;
+        const { stageOutputResolver, outputDir, outputFormat } = this.chaining;
         const outputFiles = [];
         try {
             // 1. Run parent stage → FileWriter.
@@ -99,15 +125,14 @@ export class Pipeline {
                 }
             }
             // 3. Concatenate all output files → user writer.
-            await writer.write(dataset, this.readFiles(outputFiles));
+            await this.writer.write(dataset, this.readFiles(outputFiles));
         }
         finally {
             await stageOutputResolver.cleanup();
         }
     }
     async runChainedStage(dataset, distribution, stage, stageWriter) {
-        const { reporter } = this.options;
-        reporter?.stageStart(stage.name);
+        this.reporter?.stageStart(stage.name);
         const stageStart = Date.now();
         let elementsProcessed = 0;
         let quadsGenerated = 0;
@@ -115,14 +140,14 @@ export class Pipeline {
             onProgress: (elements, quads) => {
                 elementsProcessed = elements;
                 quadsGenerated = quads;
-                reporter?.stageProgress({ elementsProcessed, quadsGenerated });
+                this.reporter?.stageProgress({ elementsProcessed, quadsGenerated });
             },
         });
         if (result instanceof NotSupported) {
-            reporter?.stageSkipped(stage.name, result.message);
+            this.reporter?.stageSkipped(stage.name, result.message);
             throw new Error(`Stage '${stage.name}' returned NotSupported in chained mode`);
         }
-        reporter?.stageComplete(stage.name, {
+        this.reporter?.stageComplete(stage.name, {
             elementsProcessed,
             quadsGenerated,
             duration: Date.now() - stageStart,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.6.28",
+  "version": "0.6.29",
   "repository": {
     "url": "https://github.com/ldengine/lde",
     "directory": "packages/pipeline"