@lde/pipeline 0.6.32 → 0.7.0

package/README.md

@@ -1,6 +1,10 @@
 # Pipeline
 
-Framework for building RDF data processing pipelines with SPARQL.
+A framework for transforming large RDF datasets using pure [SPARQL](https://www.w3.org/TR/sparql11-query/) queries.
+
+- **SPARQL-native.** Data transformations are plain SPARQL query files — portable, transparent, testable and version-controlled.
+- **Composable.** Decorators wrap executors and resolvers to add behaviour (provenance, vocabulary detection, data import) without subclassing.
+- **Extensible.** A plugin system lets packages like [@lde/pipeline-void](../pipeline-void) (or your own plugins) hook into the pipeline lifecycle.
 
 ## Components
 
@@ -82,7 +86,6 @@ import {
   SparqlItemSelector,
   SparqlUpdateWriter,
   ManualDatasetSelection,
-  SparqlDistributionResolver,
 } from '@lde/pipeline';
 
 const pipeline = new Pipeline({
@@ -106,9 +109,3 @@ const pipeline = new Pipeline({
 
 await pipeline.run();
 ```
-
-## Validation
-
-```sh
-npx nx run-many -t lint test typecheck build --projects=@lde/pipeline
-```

package/dist/distribution/importResolver.d.ts

@@ -0,0 +1,22 @@
+import type { Importer } from '@lde/sparql-importer';
+import type { SparqlServer } from '@lde/sparql-server';
+import { type DistributionResolver, NoDistributionAvailable, ResolvedDistribution } from './resolver.js';
+export interface ImportResolverOptions {
+    importer: Importer;
+    server?: SparqlServer;
+}
+/**
+ * A {@link DistributionResolver} decorator that adds import-as-fallback logic.
+ *
+ * Delegates to an inner resolver first. If the inner resolver returns
+ * {@link NoDistributionAvailable}, tries importing the dataset and optionally
+ * starts a SPARQL server.
+ */
+export declare class ImportResolver implements DistributionResolver {
+    private readonly inner;
+    private readonly options;
+    constructor(inner: DistributionResolver, options: ImportResolverOptions);
+    resolve(...args: Parameters<DistributionResolver['resolve']>): Promise<ResolvedDistribution | NoDistributionAvailable>;
+    cleanup(): Promise<void>;
+}
+//# sourceMappingURL=importResolver.d.ts.map

package/dist/distribution/importResolver.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"importResolver.d.ts","sourceRoot":"","sources":["../../src/distribution/importResolver.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EACL,KAAK,oBAAoB,EACzB,uBAAuB,EACvB,oBAAoB,EACrB,MAAM,eAAe,CAAC;AAEvB,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB;AAED;;;;;;GAMG;AACH,qBAAa,cAAe,YAAW,oBAAoB;IAEvD,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,OAAO;gBADP,KAAK,EAAE,oBAAoB,EAC3B,OAAO,EAAE,qBAAqB;IAG3C,OAAO,CACX,GAAG,IAAI,EAAE,UAAU,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC,GACnD,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC;IAoCpD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAG/B"}

package/dist/distribution/importResolver.js

@@ -0,0 +1,36 @@
+import { Distribution } from '@lde/dataset';
+import { ImportFailed, ImportSuccessful } from '@lde/sparql-importer';
+import { NoDistributionAvailable, ResolvedDistribution, } from './resolver.js';
+/**
+ * A {@link DistributionResolver} decorator that adds import-as-fallback logic.
+ *
+ * Delegates to an inner resolver first. If the inner resolver returns
+ * {@link NoDistributionAvailable}, tries importing the dataset and optionally
+ * starts a SPARQL server.
+ */
+export class ImportResolver {
+    inner;
+    options;
+    constructor(inner, options) {
+        this.inner = inner;
+        this.options = options;
+    }
+    async resolve(...args) {
+        const result = await this.inner.resolve(...args);
+        if (result instanceof ResolvedDistribution)
+            return result;
+        const [dataset] = args;
+        const importResult = await this.options.importer.import(dataset);
+        if (importResult instanceof ImportSuccessful) {
+            if (this.options.server) {
+                await this.options.server.start();
+                return new ResolvedDistribution(Distribution.sparql(this.options.server.queryEndpoint, importResult.identifier), result.probeResults);
+            }
+            return new ResolvedDistribution(Distribution.sparql(importResult.distribution.accessUrl, importResult.identifier), result.probeResults);
+        }
+        return new NoDistributionAvailable(dataset, 'No SPARQL endpoint or importable data dump available', result.probeResults, importResult instanceof ImportFailed ? importResult : undefined);
+    }
+    async cleanup() {
+        await this.options.server?.stop();
+    }
+}

package/dist/distribution/index.d.ts

@@ -1,5 +1,6 @@
 export { probe, NetworkError, SparqlProbeResult, DataDumpProbeResult, type ProbeResultType, } from './probe.js';
 export { probeResultsToQuads } from './report.js';
+export { ImportResolver, type ImportResolverOptions, } from './importResolver.js';
 export { ResolvedDistribution, NoDistributionAvailable, SparqlDistributionResolver, type DistributionResolver, type SparqlDistributionResolverOptions, } from './resolver.js';
 export { resolveDistributions, type DistributionStageResult, } from './resolveDistributions.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/distribution/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/distribution/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,EACL,YAAY,EACZ,iBAAiB,EACjB,mBAAmB,EACnB,KAAK,eAAe,GACrB,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAElD,OAAO,EACL,oBAAoB,EACpB,uBAAuB,EACvB,0BAA0B,EAC1B,KAAK,oBAAoB,EACzB,KAAK,iCAAiC,GACvC,MAAM,eAAe,CAAC;AAEvB,OAAO,EACL,oBAAoB,EACpB,KAAK,uBAAuB,GAC7B,MAAM,2BAA2B,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/distribution/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,EACL,YAAY,EACZ,iBAAiB,EACjB,mBAAmB,EACnB,KAAK,eAAe,GACrB,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAElD,OAAO,EACL,cAAc,EACd,KAAK,qBAAqB,GAC3B,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EACL,oBAAoB,EACpB,uBAAuB,EACvB,0BAA0B,EAC1B,KAAK,oBAAoB,EACzB,KAAK,iCAAiC,GACvC,MAAM,eAAe,CAAC;AAEvB,OAAO,EACL,oBAAoB,EACpB,KAAK,uBAAuB,GAC7B,MAAM,2BAA2B,CAAC"}

package/dist/distribution/index.js

@@ -1,4 +1,5 @@
 export { probe, NetworkError, SparqlProbeResult, DataDumpProbeResult, } from './probe.js';
 export { probeResultsToQuads } from './report.js';
+export { ImportResolver, } from './importResolver.js';
 export { ResolvedDistribution, NoDistributionAvailable, SparqlDistributionResolver, } from './resolver.js';
 export { resolveDistributions, } from './resolveDistributions.js';

package/dist/distribution/resolver.d.ts

@@ -1,7 +1,5 @@
 import { Dataset, Distribution } from '@lde/dataset';
-import type { Importer } from '@lde/sparql-importer';
-import { ImportFailed } from '@lde/sparql-importer';
-import type { SparqlServer } from '@lde/sparql-server';
+import type { ImportFailed } from '@lde/sparql-importer';
 import { type ProbeResultType } from './probe.js';
 export declare class ResolvedDistribution {
     readonly distribution: Distribution;
@@ -20,8 +18,6 @@ export interface DistributionResolver {
     cleanup?(): Promise<void>;
 }
 export interface SparqlDistributionResolverOptions {
-    importer?: Importer;
-    server?: SparqlServer;
     timeout?: number;
 }
 /**
@@ -29,17 +25,13 @@ export interface SparqlDistributionResolverOptions {
  *
  * 1. Probes all distributions in parallel.
  * 2. Returns the first valid SPARQL endpoint as a `ResolvedDistribution`.
- * 3. If none: tries the importer (if provided) and returns the imported distribution.
- * 4. If nothing works: returns `NoDistributionAvailable`.
+ * 3. If none: returns `NoDistributionAvailable`.
  *
  * Does not mutate `dataset.distributions`.
  */
 export declare class SparqlDistributionResolver implements DistributionResolver {
-    private readonly importer?;
-    private readonly server?;
     private readonly timeout;
     constructor(options?: SparqlDistributionResolverOptions);
     resolve(dataset: Dataset): Promise<ResolvedDistribution | NoDistributionAvailable>;
-    cleanup(): Promise<void>;
 }
 //# sourceMappingURL=resolver.d.ts.map

package/dist/distribution/resolver.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"resolver.d.ts","sourceRoot":"","sources":["../../src/distribution/resolver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,YAAY,EAAoB,MAAM,sBAAsB,CAAC;AACtE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAA4B,KAAK,eAAe,EAAE,MAAM,YAAY,CAAC;AAE5E,qBAAa,oBAAoB;IAE7B,QAAQ,CAAC,YAAY,EAAE,YAAY;IACnC,QAAQ,CAAC,YAAY,EAAE,eAAe,EAAE;gBAD/B,YAAY,EAAE,YAAY,EAC1B,YAAY,EAAE,eAAe,EAAE;CAE3C;AAED,qBAAa,uBAAuB;IAEhC,QAAQ,CAAC,OAAO,EAAE,OAAO;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM;IACxB,QAAQ,CAAC,YAAY,EAAE,eAAe,EAAE;IACxC,QAAQ,CAAC,YAAY,CAAC,EAAE,YAAY;gBAH3B,OAAO,EAAE,OAAO,EAChB,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,eAAe,EAAE,EAC/B,YAAY,CAAC,EAAE,YAAY,YAAA;CAEvC;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,CACL,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC,CAAC;IAC3D,OAAO,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAED,MAAM,WAAW,iCAAiC;IAChD,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;;;GASG;AACH,qBAAa,0BAA2B,YAAW,oBAAoB;IACrE,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAW;IACrC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe;IACvC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;gBAErB,OAAO,CAAC,EAAE,iCAAiC;IAMjD,OAAO,CACX,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC;IA0DpD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAG/B"}
+{"version":3,"file":"resolver.d.ts","sourceRoot":"","sources":["../../src/distribution/resolver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAA4B,KAAK,eAAe,EAAE,MAAM,YAAY,CAAC;AAE5E,qBAAa,oBAAoB;IAE7B,QAAQ,CAAC,YAAY,EAAE,YAAY;IACnC,QAAQ,CAAC,YAAY,EAAE,eAAe,EAAE;gBAD/B,YAAY,EAAE,YAAY,EAC1B,YAAY,EAAE,eAAe,EAAE;CAE3C;AAED,qBAAa,uBAAuB;IAEhC,QAAQ,CAAC,OAAO,EAAE,OAAO;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM;IACxB,QAAQ,CAAC,YAAY,EAAE,eAAe,EAAE;IACxC,QAAQ,CAAC,YAAY,CAAC,EAAE,YAAY;gBAH3B,OAAO,EAAE,OAAO,EAChB,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,eAAe,EAAE,EAC/B,YAAY,CAAC,EAAE,YAAY,YAAA;CAEvC;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,CACL,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC,CAAC;IAC3D,OAAO,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAED,MAAM,WAAW,iCAAiC;IAChD,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;;GAQG;AACH,qBAAa,0BAA2B,YAAW,oBAAoB;IACrE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;gBAErB,OAAO,CAAC,EAAE,iCAAiC;IAIjD,OAAO,CACX,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC;CA2B3D"}

package/dist/distribution/resolver.js

@@ -1,5 +1,3 @@
-import { Distribution } from '@lde/dataset';
-import { ImportFailed, ImportSuccessful } from '@lde/sparql-importer';
 import { probe, SparqlProbeResult } from './probe.js';
 export class ResolvedDistribution {
     distribution;
@@ -26,18 +24,13 @@ export class NoDistributionAvailable {
  *
  * 1. Probes all distributions in parallel.
  * 2. Returns the first valid SPARQL endpoint as a `ResolvedDistribution`.
- * 3. If none: tries the importer (if provided) and returns the imported distribution.
- * 4. If nothing works: returns `NoDistributionAvailable`.
+ * 3. If none: returns `NoDistributionAvailable`.
  *
  * Does not mutate `dataset.distributions`.
  */
 export class SparqlDistributionResolver {
-    importer;
-    server;
     timeout;
     constructor(options) {
-        this.importer = options?.importer;
-        this.server = options?.server;
         this.timeout = options?.timeout ?? 5000;
     }
     async resolve(dataset) {
@@ -52,26 +45,6 @@ export class SparqlDistributionResolver {
                 return new ResolvedDistribution(distribution, results);
             }
         }
-        // No SPARQL endpoint; try importer if available.
-        if (this.importer) {
-            const importResult = await this.importer.import(dataset);
-            if (importResult instanceof ImportSuccessful) {
-                // Start server if provided, using its query endpoint.
-                if (this.server) {
-                    await this.server.start();
-                    const distribution = Distribution.sparql(this.server.queryEndpoint, importResult.identifier);
-                    return new ResolvedDistribution(distribution, results);
-                }
-                const distribution = Distribution.sparql(importResult.distribution.accessUrl, importResult.identifier);
-                return new ResolvedDistribution(distribution, results);
-            }
-            if (importResult instanceof ImportFailed) {
-                return new NoDistributionAvailable(dataset, 'No SPARQL endpoint or importable data dump available', results, importResult);
-            }
-        }
-        return new NoDistributionAvailable(dataset, 'No SPARQL endpoint or importable data dump available', results);
-    }
-    async cleanup() {
-        await this.server?.stop();
+        return new NoDistributionAvailable(dataset, 'No SPARQL endpoint available', results);
     }
 }

package/dist/index.d.ts

@@ -8,4 +8,5 @@ export * from './stageOutputResolver.js';
 export * from './sparql/index.js';
 export * from './distribution/index.js';
 export * from './writer/index.js';
+export * from './provenance.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC;AAChC,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,YAAY,CAAC;AAC3B,cAAc,0BAA0B,CAAC;AACzC,cAAc,mBAAmB,CAAC;AAClC,cAAc,yBAAyB,CAAC;AACxC,cAAc,mBAAmB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC;AAChC,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,YAAY,CAAC;AAC3B,cAAc,0BAA0B,CAAC;AACzC,cAAc,mBAAmB,CAAC;AAClC,cAAc,yBAAyB,CAAC;AACxC,cAAc,mBAAmB,CAAC;AAClC,cAAc,iBAAiB,CAAC"}

package/dist/index.js

@@ -8,3 +8,4 @@ export * from './stageOutputResolver.js';
 export * from './sparql/index.js';
 export * from './distribution/index.js';
 export * from './writer/index.js';
+export * from './provenance.js';

package/dist/pipeline.d.ts

@@ -1,19 +1,26 @@
 import type { DatasetSelector } from './selector.js';
 import { Stage } from './stage.js';
+import type { QuadTransform } from './stage.js';
 import type { Writer } from './writer/writer.js';
 import { type DistributionResolver } from './distribution/resolver.js';
 import type { StageOutputResolver } from './stageOutputResolver.js';
 import type { ProgressReporter } from './progressReporter.js';
+/** Plugin that hooks into pipeline lifecycle events. */
+export interface PipelinePlugin {
+    name: string;
+    /** Transform the quad stream before writing. */
+    beforeStageWrite?: QuadTransform;
+}
 export interface PipelineOptions {
     datasetSelector: DatasetSelector;
     stages: Stage[];
     writers: Writer | Writer[];
+    plugins?: PipelinePlugin[];
     name?: string;
     distributionResolver?: DistributionResolver;
     chaining?: {
         stageOutputResolver: StageOutputResolver;
         outputDir: string;
-        outputFormat?: 'turtle' | 'n-triples' | 'n-quads';
     };
     reporter?: ProgressReporter;
 }

package/dist/pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAE1B,MAAM,4BAA4B,CAAC;AAGpC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAE9D,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;KACnD,CAAC;IACF,QAAQ,CAAC,EAAE,gBAAgB,CAAC;CAC7B;AAmBD,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAA8B;IACxD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;gBAEjC,OAAO,EAAE,eAAe;IAoB9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAaZ,cAAc;YA4Bd,QAAQ;YA8BR,QAAQ;YAmDR,eAAe;YAkCd,SAAS;CAUzB"}
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAE1B,MAAM,4BAA4B,CAAC;AAGpC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAE9D,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,gDAAgD;IAChD,gBAAgB,CAAC,EAAE,aAAa,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,OAAO,CAAC,EAAE,cAAc,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,QAAQ,CAAC,EAAE,gBAAgB,CAAC;CAC7B;AA8BD,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAA8B;IACxD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;gBAEjC,OAAO,EAAE,eAAe;IAgC9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAaZ,cAAc;YA4Bd,QAAQ;YA8BR,QAAQ;YAmDR,eAAe;YAkCd,SAAS;CAUzB"}

package/dist/pipeline.js

@@ -20,6 +20,17 @@ class FanOutWriter {
         }
     }
 }
+class TransformWriter {
+    inner;
+    transform;
+    constructor(inner, transform) {
+        this.inner = inner;
+        this.transform = transform;
+    }
+    async write(dataset, quads) {
+        await this.inner.write(dataset, this.transform(quads, dataset));
+    }
+}
 export class Pipeline {
     name;
     datasetSelector;
@@ -36,9 +47,17 @@ export class Pipeline {
         this.name = options.name ?? '';
         this.datasetSelector = options.datasetSelector;
         this.stages = options.stages;
-        this.writer = Array.isArray(options.writers)
+        let writer = Array.isArray(options.writers)
             ? new FanOutWriter(options.writers)
             : options.writers;
+        const transforms = options.plugins
+            ?.map((p) => p.beforeStageWrite)
+            .filter((t) => t !== undefined);
+        if (transforms?.length) {
+            const composed = (quads, dataset) => transforms.reduce((q, fn) => fn(q, dataset), quads);
+            writer = new TransformWriter(writer, composed);
+        }
+        this.writer = writer;
         this.distributionResolver =
             options.distributionResolver ?? new SparqlDistributionResolver();
         this.chaining = options.chaining;
@@ -103,13 +122,13 @@ export class Pipeline {
         }
     }
     async runChain(dataset, distribution, stage) {
-        const { stageOutputResolver, outputDir, outputFormat } = this.chaining;
+        const { stageOutputResolver, outputDir } = this.chaining;
         const outputFiles = [];
         try {
             // 1. Run parent stage → FileWriter.
             const parentWriter = new FileWriter({
                 outputDir: `${outputDir}/${stage.name}`,
-                format: outputFormat,
+                format: 'n-triples',
             });
             await this.runChainedStage(dataset, distribution, stage, parentWriter);
             outputFiles.push(parentWriter.getOutputPath(dataset));
@@ -119,7 +138,7 @@ export class Pipeline {
                 const child = stage.stages[i];
                 const childWriter = new FileWriter({
                     outputDir: `${outputDir}/${child.name}`,
-                    format: outputFormat,
+                    format: 'n-triples',
                 });
                 await this.runChainedStage(dataset, currentDistribution, child, childWriter);
                 outputFiles.push(childWriter.getOutputPath(dataset));

package/dist/provenance.d.ts

@@ -0,0 +1,7 @@
+import type { QuadTransform } from './stage.js';
+import type { PipelinePlugin } from './pipeline.js';
+/** QuadTransform that appends PROV-O provenance quads. */
+export declare const provenanceTransform: QuadTransform;
+/** Pipeline plugin that appends PROV-O provenance to every stage's output. */
+export declare function provenancePlugin(): PipelinePlugin;
+//# sourceMappingURL=provenance.d.ts.map

package/dist/provenance.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"provenance.d.ts","sourceRoot":"","sources":["../src/provenance.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAkBpD,0DAA0D;AAC1D,eAAO,MAAM,mBAAmB,EAAE,aACgC,CAAC;AAEnE,8EAA8E;AAC9E,wBAAgB,gBAAgB,IAAI,cAAc,CAKjD"}

package/dist/provenance.js

@@ -0,0 +1,31 @@
+import { DataFactory } from 'n3';
+const { namedNode, literal, blankNode, quad } = DataFactory;
+const RDF_TYPE = namedNode('http://www.w3.org/1999/02/22-rdf-syntax-ns#type');
+const PROV_ENTITY = namedNode('http://www.w3.org/ns/prov#Entity');
+const PROV_ACTIVITY = namedNode('http://www.w3.org/ns/prov#Activity');
+const PROV_WAS_GENERATED_BY = namedNode('http://www.w3.org/ns/prov#wasGeneratedBy');
+const PROV_STARTED_AT_TIME = namedNode('http://www.w3.org/ns/prov#startedAtTime');
+const PROV_ENDED_AT_TIME = namedNode('http://www.w3.org/ns/prov#endedAtTime');
+const XSD_DATE_TIME = namedNode('http://www.w3.org/2001/XMLSchema#dateTime');
+/** QuadTransform that appends PROV-O provenance quads. */
+export const provenanceTransform = (quads, dataset) => appendProvenanceQuads(quads, dataset.iri.toString(), new Date());
+/** Pipeline plugin that appends PROV-O provenance to every stage's output. */
+export function provenancePlugin() {
+    return {
+        name: 'provenance',
+        beforeStageWrite: provenanceTransform,
+    };
+}
+async function* appendProvenanceQuads(quads, iri, startedAt) {
+    for await (const q of quads) {
+        yield q;
+    }
+    const endedAt = new Date();
+    const subject = namedNode(iri);
+    const activity = blankNode();
+    yield quad(subject, RDF_TYPE, PROV_ENTITY);
+    yield quad(subject, PROV_WAS_GENERATED_BY, activity);
+    yield quad(activity, RDF_TYPE, PROV_ACTIVITY);
+    yield quad(activity, PROV_STARTED_AT_TIME, literal(startedAt.toISOString(), XSD_DATE_TIME));
+    yield quad(activity, PROV_ENDED_AT_TIME, literal(endedAt.toISOString(), XSD_DATE_TIME));
+}

package/dist/stage.d.ts

@@ -1,7 +1,10 @@
 import { Dataset, Distribution } from '@lde/dataset';
+import type { Quad } from '@rdfjs/types';
 import type { Executor, VariableBindings } from './sparql/executor.js';
 import { NotSupported } from './sparql/executor.js';
 import type { Writer } from './writer/writer.js';
+/** Transforms a quad stream, optionally using dataset metadata. */
+export type QuadTransform = (quads: AsyncIterable<Quad>, dataset: Dataset) => AsyncIterable<Quad>;
 export interface StageOptions {
     name: string;
     executors: Executor | Executor[];

package/dist/stage.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAErD,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAGjD,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IACjC,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,CAAC,iBAAiB,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;CAC1E;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;gBAE5B,OAAO,EAAE,YAAY;IAW3B,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;YAmBjB,eAAe;YA+Gf,UAAU;CAqBzB;AAUD,6GAA6G;AAC7G,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,YAAY,EAAE,YAAY,GAAG,aAAa,CAAC,gBAAgB,CAAC,CAAC;CACrE"}
+{"version":3,"file":"stage.d.ts","sourceRoot":"","sources":["../src/stage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAGjD,mEAAmE;AACnE,MAAM,MAAM,aAAa,GAAG,CAC1B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,EAC1B,OAAO,EAAE,OAAO,KACb,aAAa,CAAC,IAAI,CAAC,CAAC;AAEzB,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,CAAC;IACjC,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,CAAC,iBAAiB,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;CAC1E;AAED,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;gBAE5B,OAAO,EAAE,YAAY;IAW3B,GAAG,CACP,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;YAmBjB,eAAe;YA+Gf,UAAU;CAqBzB;AAUD,6GAA6G;AAC7G,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,YAAY,EAAE,YAAY,GAAG,aAAa,CAAC,gBAAgB,CAAC,CAAC;CACrE"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.6.32",
+  "version": "0.7.0",
   "repository": {
     "url": "https://github.com/ldengine/lde",
     "directory": "packages/pipeline"