@lde/pipeline 0.12.0 → 0.14.0

package/dist/distribution/importResolver.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"importResolver.d.ts","sourceRoot":"","sources":["../../src/distribution/importResolver.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EACL,KAAK,oBAAoB,EACzB,uBAAuB,EACvB,oBAAoB,EACrB,MAAM,eAAe,CAAC;AAEvB,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,EAAE,YAAY,CAAC;CACtB;AAED;;;;;;GAMG;AACH,qBAAa,cAAe,YAAW,oBAAoB;IAEvD,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,OAAO;gBADP,KAAK,EAAE,oBAAoB,EAC3B,OAAO,EAAE,qBAAqB;IAG3C,OAAO,CACX,GAAG,IAAI,EAAE,UAAU,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC,GACnD,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC;IA+BpD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAG/B"}
+{"version":3,"file":"importResolver.d.ts","sourceRoot":"","sources":["../../src/distribution/importResolver.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EACL,KAAK,oBAAoB,EACzB,uBAAuB,EACvB,oBAAoB,EACrB,MAAM,eAAe,CAAC;AAEvB,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,EAAE,YAAY,CAAC;CACtB;AAED;;;;;;GAMG;AACH,qBAAa,cAAe,YAAW,oBAAoB;IAEvD,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,OAAO;gBADP,KAAK,EAAE,oBAAoB,EAC3B,OAAO,EAAE,qBAAqB;IAG3C,OAAO,CACX,GAAG,IAAI,EAAE,UAAU,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC,GACnD,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC;IAiCpD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAG/B"}

package/dist/distribution/importResolver.js

@@ -20,12 +20,13 @@ export class ImportResolver {
         if (result instanceof ResolvedDistribution)
             return result;
         const [dataset] = args;
+        const importStart = Date.now();
         const importResult = await this.options.importer.import(dataset);
         if (importResult instanceof ImportSuccessful) {
             await this.options.server.start();
             const distribution = Distribution.sparql(this.options.server.queryEndpoint, importResult.identifier);
             distribution.subjectFilter = importResult.distribution.subjectFilter;
-            return new ResolvedDistribution(distribution, result.probeResults, importResult.distribution);
+            return new ResolvedDistribution(distribution, result.probeResults, importResult.distribution, Date.now() - importStart);
         }
         return new NoDistributionAvailable(dataset, 'No SPARQL endpoint or importable data dump available', result.probeResults, importResult instanceof ImportFailed ? importResult : undefined);
     }

package/dist/distribution/resolver.d.ts

@@ -5,7 +5,8 @@ export declare class ResolvedDistribution {
     readonly distribution: Distribution;
     readonly probeResults: ProbeResultType[];
     readonly importedFrom?: Distribution | undefined;
-    constructor(distribution: Distribution, probeResults: ProbeResultType[], importedFrom?: Distribution | undefined);
+    readonly importDuration?: number | undefined;
+    constructor(distribution: Distribution, probeResults: ProbeResultType[], importedFrom?: Distribution | undefined, importDuration?: number | undefined);
 }
 export declare class NoDistributionAvailable {
     readonly dataset: Dataset;

package/dist/distribution/resolver.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"resolver.d.ts","sourceRoot":"","sources":["../../src/distribution/resolver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAA4B,KAAK,eAAe,EAAE,MAAM,YAAY,CAAC;AAE5E,qBAAa,oBAAoB;IAE7B,QAAQ,CAAC,YAAY,EAAE,YAAY;IACnC,QAAQ,CAAC,YAAY,EAAE,eAAe,EAAE;IACxC,QAAQ,CAAC,YAAY,CAAC,EAAE,YAAY;gBAF3B,YAAY,EAAE,YAAY,EAC1B,YAAY,EAAE,eAAe,EAAE,EAC/B,YAAY,CAAC,EAAE,YAAY,YAAA;CAEvC;AAED,qBAAa,uBAAuB;IAEhC,QAAQ,CAAC,OAAO,EAAE,OAAO;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM;IACxB,QAAQ,CAAC,YAAY,EAAE,eAAe,EAAE;IACxC,QAAQ,CAAC,YAAY,CAAC,EAAE,YAAY;gBAH3B,OAAO,EAAE,OAAO,EAChB,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,eAAe,EAAE,EAC/B,YAAY,CAAC,EAAE,YAAY,YAAA;CAEvC;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,CACL,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC,CAAC;IAC3D,OAAO,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAED,MAAM,WAAW,iCAAiC;IAChD,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;;GAQG;AACH,qBAAa,0BAA2B,YAAW,oBAAoB;IACrE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;gBAErB,OAAO,CAAC,EAAE,iCAAiC;IAIjD,OAAO,CACX,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC;CA2B3D"}
+{"version":3,"file":"resolver.d.ts","sourceRoot":"","sources":["../../src/distribution/resolver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAA4B,KAAK,eAAe,EAAE,MAAM,YAAY,CAAC;AAE5E,qBAAa,oBAAoB;IAE7B,QAAQ,CAAC,YAAY,EAAE,YAAY;IACnC,QAAQ,CAAC,YAAY,EAAE,eAAe,EAAE;IACxC,QAAQ,CAAC,YAAY,CAAC,EAAE,YAAY;IACpC,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM;gBAHvB,YAAY,EAAE,YAAY,EAC1B,YAAY,EAAE,eAAe,EAAE,EAC/B,YAAY,CAAC,EAAE,YAAY,YAAA,EAC3B,cAAc,CAAC,EAAE,MAAM,YAAA;CAEnC;AAED,qBAAa,uBAAuB;IAEhC,QAAQ,CAAC,OAAO,EAAE,OAAO;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM;IACxB,QAAQ,CAAC,YAAY,EAAE,eAAe,EAAE;IACxC,QAAQ,CAAC,YAAY,CAAC,EAAE,YAAY;gBAH3B,OAAO,EAAE,OAAO,EAChB,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,eAAe,EAAE,EAC/B,YAAY,CAAC,EAAE,YAAY,YAAA;CAEvC;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,CACL,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC,CAAC;IAC3D,OAAO,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAED,MAAM,WAAW,iCAAiC;IAChD,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;;GAQG;AACH,qBAAa,0BAA2B,YAAW,oBAAoB;IACrE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;gBAErB,OAAO,CAAC,EAAE,iCAAiC;IAIjD,OAAO,CACX,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC;CA2B3D"}

package/dist/distribution/resolver.js

@@ -3,10 +3,12 @@ export class ResolvedDistribution {
     distribution;
     probeResults;
     importedFrom;
-    constructor(distribution, probeResults, importedFrom) {
+    importDuration;
+    constructor(distribution, probeResults, importedFrom, importDuration) {
         this.distribution = distribution;
         this.probeResults = probeResults;
         this.importedFrom = importedFrom;
+        this.importDuration = importDuration;
     }
 }
 export class NoDistributionAvailable {

package/dist/pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAE1B,MAAM,4BAA4B,CAAC;AAQpC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAE9D,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,gDAAgD;IAChD,gBAAgB,CAAC,EAAE,aAAa,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,OAAO,CAAC,EAAE,cAAc,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,QAAQ,CAAC,EAAE,gBAAgB,CAAC;CAC7B;AA8BD,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAA8B;IACxD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;gBAEjC,OAAO,EAAE,eAAe;IAgC9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAaZ,cAAc;YA6Cd,QAAQ;YA8BR,QAAQ;YAmDR,eAAe;YAkCd,SAAS;CAUzB"}
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAE1B,MAAM,4BAA4B,CAAC;AAQpC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAE/B,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,gDAAgD;IAChD,gBAAgB,CAAC,EAAE,aAAa,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,OAAO,CAAC,EAAE,cAAc,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,QAAQ,CAAC,EAAE,gBAAgB,CAAC;CAC7B;AAsCD,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAA8B;IACxD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;gBAEjC,OAAO,EAAE,eAAe;IAgC9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAaZ,cAAc;YA6Cd,QAAQ;YA8BR,QAAQ;YAmDR,eAAe;YAkCd,SAAS;CAUzB"}

package/dist/pipeline.js

@@ -20,6 +20,10 @@ class FanOutWriter {
             })());
         }
     }
+    async flush(dataset) {
+        for (const w of this.writers)
+            await w.flush?.(dataset);
+    }
 }
 class TransformWriter {
     inner;
@@ -31,6 +35,9 @@ class TransformWriter {
     async write(dataset, quads) {
         await this.inner.write(dataset, this.transform(quads, dataset));
     }
+    async flush(dataset) {
+        await this.inner.flush?.(dataset);
+    }
 }
 export class Pipeline {
     name;
@@ -66,27 +73,22 @@ export class Pipeline {
     }
     async run() {
         const start = Date.now();
-        this.reporter?.pipelineStart(this.name);
+        this.reporter?.pipelineStart?.(this.name);
         const datasets = await this.datasetSelector.select();
         for await (const dataset of datasets) {
             await this.processDataset(dataset);
         }
-        this.reporter?.pipelineComplete({ duration: Date.now() - start });
+        this.reporter?.pipelineComplete?.({ duration: Date.now() - start });
     }
     async processDataset(dataset) {
-        const datasetIri = dataset.iri.toString();
-        this.reporter?.datasetStart(datasetIri);
+        this.reporter?.datasetStart?.(dataset);
         const resolved = await this.distributionResolver.resolve(dataset);
-        this.reporter?.distributionsAnalyzed(datasetIri, mapProbeResults(resolved.probeResults));
+        this.reporter?.distributionsAnalyzed?.(dataset, mapProbeResults(dataset, resolved.probeResults));
         if (resolved instanceof NoDistributionAvailable) {
-            this.reporter?.datasetSkipped(datasetIri, resolved.message);
+            this.reporter?.datasetSkipped?.(dataset, resolved.message);
             return;
         }
-        this.reporter?.distributionSelected(datasetIri, {
-            accessUrl: resolved.distribution.accessUrl.toString(),
-            namedGraph: resolved.distribution.namedGraph,
-            importedFrom: resolved.importedFrom?.accessUrl?.toString(),
-        });
+        this.reporter?.distributionSelected?.(dataset, resolved.distribution, resolved.importedFrom, resolved.importDuration);
         try {
             for (const stage of this.stages) {
                 try {
@@ -98,17 +100,18 @@ export class Pipeline {
                     }
                 }
                 catch (error) {
-                    this.reporter?.stageFailed(stage.name, error instanceof Error ? error : new Error(String(error)));
+                    this.reporter?.stageFailed?.(stage.name, error instanceof Error ? error : new Error(String(error)));
                 }
             }
         }
         finally {
             await this.distributionResolver.cleanup?.();
         }
-        this.reporter?.datasetComplete(datasetIri);
+        await this.writer.flush?.(dataset);
+        this.reporter?.datasetComplete?.(dataset);
     }
     async runStage(dataset, distribution, stage) {
-        this.reporter?.stageStart(stage.name);
+        this.reporter?.stageStart?.(stage.name);
         const stageStart = Date.now();
         let elementsProcessed = 0;
         let quadsGenerated = 0;
@@ -116,14 +119,14 @@ export class Pipeline {
             onProgress: (elements, quads) => {
                 elementsProcessed = elements;
                 quadsGenerated = quads;
-                this.reporter?.stageProgress({ elementsProcessed, quadsGenerated });
+                this.reporter?.stageProgress?.({ elementsProcessed, quadsGenerated });
             },
         });
         if (result instanceof NotSupported) {
-            this.reporter?.stageSkipped(stage.name, result.message);
+            this.reporter?.stageSkipped?.(stage.name, result.message);
         }
         else {
-            this.reporter?.stageComplete(stage.name, {
+            this.reporter?.stageComplete?.(stage.name, {
                 elementsProcessed,
                 quadsGenerated,
                 duration: Date.now() - stageStart,
@@ -163,7 +166,7 @@ export class Pipeline {
         }
     }
     async runChainedStage(dataset, distribution, stage, stageWriter) {
-        this.reporter?.stageStart(stage.name);
+        this.reporter?.stageStart?.(stage.name);
         const stageStart = Date.now();
         let elementsProcessed = 0;
         let quadsGenerated = 0;
@@ -171,14 +174,14 @@ export class Pipeline {
             onProgress: (elements, quads) => {
                 elementsProcessed = elements;
                 quadsGenerated = quads;
-                this.reporter?.stageProgress({ elementsProcessed, quadsGenerated });
+                this.reporter?.stageProgress?.({ elementsProcessed, quadsGenerated });
             },
         });
         if (result instanceof NotSupported) {
-            this.reporter?.stageSkipped(stage.name, result.message);
+            this.reporter?.stageSkipped?.(stage.name, result.message);
             throw new Error(`Stage '${stage.name}' returned NotSupported in chained mode`);
         }
-        this.reporter?.stageComplete(stage.name, {
+        this.reporter?.stageComplete?.(stage.name, {
             elementsProcessed,
             quadsGenerated,
             duration: Date.now() - stageStart,
@@ -195,18 +198,18 @@ export class Pipeline {
         }
     }
 }
-function mapProbeResults(probeResults) {
-    return probeResults.map((result) => {
+function mapProbeResults(dataset, probeResults) {
+    return probeResults.map((result, index) => {
         if (result instanceof NetworkError) {
             return {
-                accessUrl: result.url,
+                distribution: dataset.distributions[index],
                 type: 'network-error',
                 available: false,
                 error: result.message,
             };
         }
         return {
-            accessUrl: result.url,
+            distribution: dataset.distributions[index],
             type: result instanceof SparqlProbeResult
                 ? 'sparql'
                 : 'data-dump',

package/dist/progressReporter.d.ts

@@ -1,33 +1,32 @@
+import type { Dataset, Distribution } from '@lde/dataset';
+export interface DistributionAnalysisResult {
+    distribution: Distribution;
+    type: 'sparql' | 'data-dump' | 'network-error';
+    available: boolean;
+    statusCode?: number;
+    error?: string;
+}
 export interface ProgressReporter {
-    pipelineStart(name: string): void;
-    datasetStart(dataset: string): void;
-    distributionsAnalyzed(dataset: string, results: Array<{
-        accessUrl: string;
-        type: 'sparql' | 'data-dump' | 'network-error';
-        available: boolean;
-        statusCode?: number;
-        error?: string;
-    }>): void;
-    distributionSelected(dataset: string, distribution: {
-        accessUrl: string;
-        namedGraph?: string;
-        importedFrom?: string;
-    }): void;
-    stageStart(stage: string): void;
-    stageProgress(update: {
+    pipelineStart?(name: string): void;
+    datasetsSelected?(count: number): void;
+    datasetStart?(dataset: Dataset): void;
+    distributionsAnalyzed?(dataset: Dataset, results: DistributionAnalysisResult[]): void;
+    distributionSelected?(dataset: Dataset, distribution: Distribution, importedFrom?: Distribution, importDuration?: number): void;
+    stageStart?(stage: string): void;
+    stageProgress?(update: {
         elementsProcessed: number;
         quadsGenerated: number;
     }): void;
-    stageComplete(stage: string, result: {
+    stageComplete?(stage: string, result: {
         elementsProcessed: number;
         quadsGenerated: number;
         duration: number;
     }): void;
-    stageFailed(stage: string, error: Error): void;
-    stageSkipped(stage: string, reason: string): void;
-    datasetComplete(dataset: string): void;
-    datasetSkipped(dataset: string, reason: string): void;
-    pipelineComplete(result: {
+    stageFailed?(stage: string, error: Error): void;
+    stageSkipped?(stage: string, reason: string): void;
+    datasetComplete?(dataset: Dataset): void;
+    datasetSkipped?(dataset: Dataset, reason: string): void;
+    pipelineComplete?(result: {
         duration: number;
     }): void;
 }

package/dist/progressReporter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"progressReporter.d.ts","sourceRoot":"","sources":["../src/progressReporter.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,gBAAgB;IAC/B,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,qBAAqB,CACnB,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,KAAK,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;QAClB,IAAI,EAAE,QAAQ,GAAG,WAAW,GAAG,eAAe,CAAC;QAC/C,SAAS,EAAE,OAAO,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC,GACD,IAAI,CAAC;IACR,oBAAoB,CAClB,OAAO,EAAE,MAAM,EACf,YAAY,EAAE;QACZ,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,GACA,IAAI,CAAC;IACR,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,aAAa,CAAC,MAAM,EAAE;QACpB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,cAAc,EAAE,MAAM,CAAC;KACxB,GAAG,IAAI,CAAC;IACT,aAAa,CACX,KAAK,EAAE,MAAM,EACb,MAAM,EAAE;QACN,iBAAiB,EAAE,MAAM,CAAC;QAC1B,cAAc,EAAE,MAAM,CAAC;QACvB,QAAQ,EAAE,MAAM,CAAC;KAClB,GACA,IAAI,CAAC;IACR,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC;IAC/C,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAClD,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtD,gBAAgB,CAAC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;CACtD"}
+{"version":3,"file":"progressReporter.d.ts","sourceRoot":"","sources":["../src/progressReporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE1D,MAAM,WAAW,0BAA0B;IACzC,YAAY,EAAE,YAAY,CAAC;IAC3B,IAAI,EAAE,QAAQ,GAAG,WAAW,GAAG,eAAe,CAAC;IAC/C,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,gBAAgB;IAC/B,aAAa,CAAC,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,gBAAgB,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,YAAY,CAAC,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACtC,qBAAqB,CAAC,CACpB,OAAO,EAAE,OAAO,EAChB,OAAO,EAAE,0BAA0B,EAAE,GACpC,IAAI,CAAC;IACR,oBAAoB,CAAC,CACnB,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,YAAY,CAAC,EAAE,YAAY,EAC3B,cAAc,CAAC,EAAE,MAAM,GACtB,IAAI,CAAC;IACR,UAAU,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,aAAa,CAAC,CAAC,MAAM,EAAE;QACrB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,cAAc,EAAE,MAAM,CAAC;KACxB,GAAG,IAAI,CAAC;IACT,aAAa,CAAC,CACZ,KAAK,EAAE,MAAM,EACb,MAAM,EAAE;QACN,iBAAiB,EAAE,MAAM,CAAC;QAC1B,cAAc,EAAE,MAAM,CAAC;QACvB,QAAQ,EAAE,MAAM,CAAC;KAClB,GACA,IAAI,CAAC;IACR,WAAW,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC;IAChD,YAAY,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACnD,eAAe,CAAC,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACzC,cAAc,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACxD,gBAAgB,CAAC,CAAC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;CACvD"}

package/dist/writer/fileWriter.d.ts

@@ -16,16 +16,25 @@ export interface FileWriterOptions {
      * @default '-'
      */
     replacementCharacter?: string;
+    /**
+     * Turtle prefix declarations. Keys are prefix names, values are namespace IRIs.
+     * Only used when format is 'turtle'.
+     */
+    prefixes?: Record<string, string>;
 }
 export declare class FileWriter implements Writer {
     private readonly outputDir;
     readonly format: 'turtle' | 'n-triples' | 'n-quads';
     private readonly replacementCharacter;
-    private readonly writtenFiles;
+    private readonly prefixes?;
+    private readonly activeWriters;
     constructor(options: FileWriterOptions);
     write(dataset: Dataset, quads: AsyncIterable<Quad>): Promise<void>;
+    flush(dataset: Dataset): Promise<void>;
     getOutputPath(dataset: Dataset): string;
     getFilename(dataset: Dataset): string;
+    private getFilePath;
+    private getOrCreateWriter;
     private getExtension;
 }
 //# sourceMappingURL=fileWriter.d.ts.map

package/dist/writer/fileWriter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fileWriter.d.ts","sourceRoot":"","sources":["../../src/writer/fileWriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAMzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,MAAM,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;IAC5C;;;OAGG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;CAC/B;AAqBD,qBAAa,UAAW,YAAW,MAAM;IACvC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,QAAQ,CAAC,MAAM,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;IACpD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAS;IAC9C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAqB;gBAEtC,OAAO,EAAE,iBAAiB;IAMhC,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA4BxE,aAAa,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAIvC,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAQrC,OAAO,CAAC,YAAY;CAUrB"}
+{"version":3,"file":"fileWriter.d.ts","sourceRoot":"","sources":["../../src/writer/fileWriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAMzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,MAAM,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;IAC5C;;;OAGG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACnC;AAiBD,qBAAa,UAAW,YAAW,MAAM;IACvC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,QAAQ,CAAC,MAAM,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;IACpD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAS;IAC9C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAyB;IACnD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAG1B;gBAEQ,OAAO,EAAE,iBAAiB;IAOhC,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAclE,KAAK,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAc5C,aAAa,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAIvC,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAQrC,OAAO,CAAC,WAAW;YAIL,iBAAiB;IAoB/B,OAAO,CAAC,YAAY;CAUrB"}

package/dist/writer/fileWriter.js

@@ -8,13 +8,9 @@ import { Writer as N3Writer } from 'n3';
  *
  * Files are named based on the dataset IRI using filenamify-url.
  *
- * The first {@link write} call for a given dataset creates (or overwrites) the file.
- * Subsequent calls for the same dataset append to it, so that multiple pipeline stages
- * can each contribute quads to a single output file.
- *
- * **Note:** With `format: 'turtle'` each append will repeat the prefix declarations
- * at the start of each chunk. The default `format: 'n-triples'` produces clean
- * line-oriented output without repeated headers.
+ * A single N3Writer is kept open per dataset across all {@link write} calls,
+ * so Turtle prefix declarations are written once and triples can be grouped
+ * by subject. Call {@link flush} after all stages complete to finalize the file.
  */
 const formatMap = {
     turtle: 'Turtle',
@@ -25,11 +21,13 @@ export class FileWriter {
     outputDir;
     format;
     replacementCharacter;
-    writtenFiles = new Set();
+    prefixes;
+    activeWriters = new Map();
     constructor(options) {
         this.outputDir = options.outputDir;
         this.format = options.format ?? 'n-triples';
         this.replacementCharacter = options.replacementCharacter ?? '-';
+        this.prefixes = options.prefixes;
     }
     async write(dataset, quads) {
         // Peek at the first quad to avoid creating empty files.
@@ -37,18 +35,20 @@ export class FileWriter {
         const first = await iterator.next();
         if (first.done)
             return;
-        const filePath = join(this.outputDir, this.getFilename(dataset));
-        await mkdir(dirname(filePath), { recursive: true });
-        const flags = this.writtenFiles.has(filePath) ? 'a' : 'w';
-        this.writtenFiles.add(filePath);
-        const stream = createWriteStream(filePath, { flags });
-        const writer = new N3Writer(stream, { format: formatMap[this.format] });
-        writer.addQuad(first.value);
+        const { n3Writer } = await this.getOrCreateWriter(dataset);
+        n3Writer.addQuad(first.value);
         for await (const quad of { [Symbol.asyncIterator]: () => iterator }) {
-            writer.addQuad(quad);
+            n3Writer.addQuad(quad);
         }
+    }
+    async flush(dataset) {
+        const key = this.getFilePath(dataset);
+        const entry = this.activeWriters.get(key);
+        if (!entry)
+            return;
+        this.activeWriters.delete(key);
         await new Promise((resolve, reject) => {
-            writer.end((error) => {
+            entry.n3Writer.end((error) => {
                 if (error)
                     reject(error);
                 else
@@ -57,7 +57,7 @@ export class FileWriter {
         });
     }
     getOutputPath(dataset) {
-        return join(this.outputDir, this.getFilename(dataset));
+        return this.getFilePath(dataset);
     }
     getFilename(dataset) {
         const extension = this.getExtension();
@@ -66,6 +66,24 @@ export class FileWriter {
         });
         return `${baseName}.${extension}`;
     }
+    getFilePath(dataset) {
+        return join(this.outputDir, this.getFilename(dataset));
+    }
+    async getOrCreateWriter(dataset) {
+        const key = this.getFilePath(dataset);
+        const existing = this.activeWriters.get(key);
+        if (existing)
+            return existing;
+        await mkdir(dirname(key), { recursive: true });
+        const stream = createWriteStream(key, { flags: 'w' });
+        const n3Writer = new N3Writer(stream, {
+            format: formatMap[this.format],
+            prefixes: this.prefixes,
+        });
+        const entry = { n3Writer, stream };
+        this.activeWriters.set(key, entry);
+        return entry;
+    }
     getExtension() {
         switch (this.format) {
             case 'turtle':

package/dist/writer/writer.d.ts

@@ -11,5 +11,13 @@ export interface Writer {
      * @param quads The RDF quads to write
      */
     write(dataset: Dataset, quads: AsyncIterable<Quad>): Promise<void>;
+    /**
+     * Finalize writing for a dataset. Called after all stages complete.
+     *
+     * Writers that buffer output across multiple {@link write} calls (e.g. to
+     * share Turtle prefix declarations) should implement this to flush remaining
+     * data and release resources.
+     */
+    flush?(dataset: Dataset): Promise<void>;
 }
 //# sourceMappingURL=writer.d.ts.map

package/dist/writer/writer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../src/writer/writer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC;;GAEG;AACH,MAAM,WAAW,MAAM;IACrB;;;;;OAKG;IACH,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACpE"}
+{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../src/writer/writer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC;;GAEG;AACH,MAAM,WAAW,MAAM;IACrB;;;;;OAKG;IACH,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEnE;;;;;;OAMG;IACH,KAAK,CAAC,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACzC"}

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.12.0",
+  "version": "0.14.0",
   "repository": {
-    "url": "https://github.com/ldengine/lde",
+    "url": "git+https://github.com/ldengine/lde.git",
     "directory": "packages/pipeline"
   },
   "type": "module",