@lde/pipeline 0.6.28 → 0.6.30

package/README.md

@@ -86,9 +86,7 @@ import {
 } from '@lde/pipeline';
 
 const pipeline = new Pipeline({
-  name: 'example',
   datasetSelector: new ManualDatasetSelection([dataset]),
-  distributionResolver: new SparqlDistributionResolver(),
   stages: [
     new Stage({
       name: 'per-class',
@@ -101,7 +99,7 @@ const pipeline = new Pipeline({
       }),
     }),
   ],
-  writer: new SparqlUpdateWriter({
+  writers: new SparqlUpdateWriter({
     endpoint: new URL('http://localhost:7200/repositories/lde/statements'),
   }),
 });

package/dist/distribution/probe.d.ts

@@ -31,7 +31,9 @@ export declare class SparqlProbeResult extends ProbeResult {
  */
 export declare class DataDumpProbeResult extends ProbeResult {
     readonly contentSize: number | null;
-    constructor(url: string, response: Response);
+    readonly failureReason: string | null;
+    constructor(url: string, response: Response, failureReason?: string | null);
+    isSuccess(): boolean;
 }
 export type ProbeResultType = SparqlProbeResult | DataDumpProbeResult | NetworkError;
 /**

package/dist/distribution/probe.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"probe.d.ts","sourceRoot":"","sources":["../../src/distribution/probe.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C;;GAEG;AACH,qBAAa,YAAY;aACK,GAAG,EAAE,MAAM;aAAkB,OAAO,EAAE,MAAM;gBAA5C,GAAG,EAAE,MAAM,EAAkB,OAAO,EAAE,MAAM;CACzE;AAED;;GAEG;AACH,uBAAe,WAAW;aAMI,GAAG,EAAE,MAAM;IALvC,SAAgB,UAAU,EAAE,MAAM,CAAC;IACnC,SAAgB,UAAU,EAAE,MAAM,CAAC;IACnC,SAAgB,YAAY,EAAE,IAAI,GAAG,IAAI,CAAQ;IACjD,SAAgB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;gBAEf,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ;IAUpD,SAAS,IAAI,OAAO;CAG5B;AAED;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,WAAW;IAChD,SAAgB,mBAAmB,qCAAqC;IAE/D,SAAS,IAAI,OAAO;CAM9B;AAED;;GAEG;AACH,qBAAa,mBAAoB,SAAQ,WAAW;IAClD,SAAgB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAQ;gBAEtC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ;CAO5C;AAED,MAAM,MAAM,eAAe,GACvB,iBAAiB,GACjB,mBAAmB,GACnB,YAAY,CAAC;AAEjB;;;;;;;GAOG;AACH,wBAAsB,KAAK,CACzB,YAAY,EAAE,YAAY,EAC1B,OAAO,SAAO,GACb,OAAO,CAAC,eAAe,CAAC,CAY1B"}
+{"version":3,"file":"probe.d.ts","sourceRoot":"","sources":["../../src/distribution/probe.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAG5C;;GAEG;AACH,qBAAa,YAAY;aAEL,GAAG,EAAE,MAAM;aACX,OAAO,EAAE,MAAM;gBADf,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,MAAM;CAElC;AAED;;GAEG;AACH,uBAAe,WAAW;aAON,GAAG,EAAE,MAAM;IAN7B,SAAgB,UAAU,EAAE,MAAM,CAAC;IACnC,SAAgB,UAAU,EAAE,MAAM,CAAC;IACnC,SAAgB,YAAY,EAAE,IAAI,GAAG,IAAI,CAAQ;IACjD,SAAgB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;gBAGzB,GAAG,EAAE,MAAM,EAC3B,QAAQ,EAAE,QAAQ;IAWb,SAAS,IAAI,OAAO;CAG5B;AAED;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,WAAW;IAChD,SAAgB,mBAAmB,qCAAqC;IAE/D,SAAS,IAAI,OAAO;CAM9B;AAED;;GAEG;AACH,qBAAa,mBAAoB,SAAQ,WAAW;IAClD,SAAgB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAQ;IAClD,SAAgB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;gBAG3C,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,QAAQ,EAClB,aAAa,GAAE,MAAM,GAAG,IAAW;IAU5B,SAAS,IAAI,OAAO;CAG9B;AAED,MAAM,MAAM,eAAe,GACvB,iBAAiB,GACjB,mBAAmB,GACnB,YAAY,CAAC;AAEjB;;;;;;;GAOG;AACH,wBAAsB,KAAK,CACzB,YAAY,EAAE,YAAY,EAC1B,OAAO,SAAO,GACb,OAAO,CAAC,eAAe,CAAC,CAY1B"}

package/dist/distribution/probe.js

@@ -1,3 +1,4 @@
+import { Parser } from 'n3';
 /**
  * Result of a network error during probing.
  */
@@ -47,13 +48,18 @@ export class SparqlProbeResult extends ProbeResult {
  */
 export class DataDumpProbeResult extends ProbeResult {
     contentSize = null;
-    constructor(url, response) {
+    failureReason;
+    constructor(url, response, failureReason = null) {
         super(url, response);
+        this.failureReason = failureReason;
         const contentLengthHeader = response.headers.get('Content-Length');
         if (contentLengthHeader) {
             this.contentSize = parseInt(contentLengthHeader);
         }
     }
+    isSuccess() {
+        return super.isSuccess() && this.failureReason === null;
+    }
 }
 /**
  * Probe a distribution to check availability and gather metadata.
@@ -96,19 +102,48 @@ async function probeDataDump(distribution, timeout) {
             'Accept-Encoding': 'identity', // Return uncompressed responses.
         },
     };
-    let response = await fetch(url, {
+    const headResponse = await fetch(url, {
         method: 'HEAD',
         ...requestOptions,
     });
-    const contentLength = response.headers.get('Content-Length');
-    if (contentLength === null || contentLength === '0') {
-        // Retry as GET request for servers incorrectly returning HEAD request Content-Length,
-        // which *should* be the size of the response body when issuing a GET, not that of
-        // the response to a HEAD request, which is intentionally 0.
-        response = await fetch(url, {
+    const contentLength = headResponse.headers.get('Content-Length');
+    const contentLengthBytes = contentLength ? parseInt(contentLength) : 0;
+    // For small or unknown-size files, do a GET to validate body content.
+    // This also handles servers that incorrectly return 0 Content-Length for HEAD.
+    if (contentLengthBytes <= 10_240) {
+        const getResponse = await fetch(url, {
             method: 'GET',
             ...requestOptions,
         });
+        const body = await getResponse.text();
+        const isHttpSuccess = getResponse.status >= 200 && getResponse.status < 400;
+        const failureReason = isHttpSuccess
+            ? validateBody(body, getResponse.headers.get('Content-Type'))
+            : null;
+        return new DataDumpProbeResult(url, getResponse, failureReason);
+    }
+    return new DataDumpProbeResult(url, headResponse);
+}
+const rdfContentTypes = [
+    'text/turtle',
+    'application/n-triples',
+    'application/n-quads',
+];
+function validateBody(body, contentType) {
+    if (body.length === 0) {
+        return 'Distribution is empty';
+    }
+    if (contentType && rdfContentTypes.some((t) => contentType.startsWith(t))) {
+        try {
+            const parser = new Parser();
+            const quads = parser.parse(body);
+            if (quads.length === 0) {
+                return 'Distribution contains no RDF triples';
+            }
+        }
+        catch (e) {
+            return e instanceof Error ? e.message : String(e);
+        }
     }
-    return new DataDumpProbeResult(url, response);
+    return null;
 }

package/dist/distribution/report.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/distribution/report.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAe,KAAK,IAAI,EAAE,MAAM,IAAI,CAAC;AAC5C,OAAO,EAIL,KAAK,eAAe,EACrB,MAAM,YAAY,CAAC;AAUpB;;;;;;;;;GASG;AACH,wBAAuB,mBAAmB,CACxC,YAAY,EAAE,eAAe,EAAE,EAC/B,UAAU,EAAE,MAAM,EAClB,YAAY,CAAC,EAAE,YAAY,GAC1B,aAAa,CAAC,IAAI,CAAC,CAkCrB"}
+{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/distribution/report.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAe,KAAK,IAAI,EAAE,MAAM,IAAI,CAAC;AAC5C,OAAO,EAIL,KAAK,eAAe,EACrB,MAAM,YAAY,CAAC;AAUpB;;;;;;;;;GASG;AACH,wBAAuB,mBAAmB,CACxC,YAAY,EAAE,eAAe,EAAE,EAC/B,UAAU,EAAE,MAAM,EAClB,YAAY,CAAC,EAAE,YAAY,GAC1B,aAAa,CAAC,IAAI,CAAC,CAwCrB"}

package/dist/distribution/report.js

@@ -1,5 +1,5 @@
 import { DataFactory } from 'n3';
-import { NetworkError, SparqlProbeResult, } from './probe.js';
+import { NetworkError, SparqlProbeResult, DataDumpProbeResult, } from './probe.js';
 const { quad, namedNode, blankNode, literal } = DataFactory;
 const RDF = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#';
 const SCHEMA = 'https://schema.org/';
@@ -30,6 +30,9 @@ export async function* probeResultsToQuads(probeResults, datasetIri, importResul
         else if (result.isSuccess()) {
             yield* successQuads(action, result, datasetIri);
         }
+        else if (result instanceof DataDumpProbeResult && result.failureReason) {
+            yield quad(action, namedNode(`${SCHEMA}error`), literal(result.failureReason));
+        }
         else {
             // HTTP error
             const statusUri = `${HTTP_STATUS}${result.statusText.replace(/ /g, '')}`;

package/dist/pipeline.d.ts

@@ -5,18 +5,26 @@ import { type DistributionResolver } from './distribution/resolver.js';
 import type { StageOutputResolver } from './stageOutputResolver.js';
 import type { ProgressReporter } from './progressReporter.js';
 export interface PipelineOptions {
-    name: string;
     datasetSelector: DatasetSelector;
     stages: Stage[];
-    writer: Writer;
-    distributionResolver: DistributionResolver;
-    stageOutputResolver?: StageOutputResolver;
-    outputDir?: string;
-    outputFormat?: 'turtle' | 'n-triples' | 'n-quads';
+    writers: Writer | Writer[];
+    name?: string;
+    distributionResolver?: DistributionResolver;
+    chaining?: {
+        stageOutputResolver: StageOutputResolver;
+        outputDir: string;
+        outputFormat?: 'turtle' | 'n-triples' | 'n-quads';
+    };
     reporter?: ProgressReporter;
 }
 export declare class Pipeline {
-    private readonly options;
+    private readonly name;
+    private readonly datasetSelector;
+    private readonly stages;
+    private readonly writer;
+    private readonly distributionResolver;
+    private readonly chaining?;
+    private readonly reporter?;
     constructor(options: PipelineOptions);
     run(): Promise<void>;
     private processDataset;

package/dist/pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAE1B,MAAM,4BAA4B,CAAC;AAEpC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAE9D,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,oBAAoB,EAAE,oBAAoB,CAAC;IAC3C,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAC1C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;IAClD,QAAQ,CAAC,EAAE,gBAAgB,CAAC;CAC7B;AAED,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkB;gBAE9B,OAAO,EAAE,eAAe;IAe9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAcZ,cAAc;YA2Bd,QAAQ;YAgCR,QAAQ;YAoDR,eAAe;YAoCd,SAAS;CAUzB"}
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAE1B,MAAM,4BAA4B,CAAC;AAGpC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAE9D,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;KACnD,CAAC;IACF,QAAQ,CAAC,EAAE,gBAAgB,CAAC;CAC7B;AAmBD,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAA8B;IACxD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;gBAEjC,OAAO,EAAE,eAAe;IAoB9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAaZ,cAAc;YA0Bd,QAAQ;YA8BR,QAAQ;YAmDR,eAAe;YAkCd,SAAS;CAUzB"}

package/dist/pipeline.js

@@ -2,40 +2,67 @@ import { createReadStream } from 'node:fs';
 import { StreamParser } from 'n3';
 import { FileWriter } from './writer/fileWriter.js';
 import { NoDistributionAvailable, } from './distribution/resolver.js';
+import { SparqlDistributionResolver } from './distribution/index.js';
 import { NotSupported } from './sparql/executor.js';
+class FanOutWriter {
+    writers;
+    constructor(writers) {
+        this.writers = writers;
+    }
+    async write(dataset, quads) {
+        const collected = [];
+        for await (const quad of quads)
+            collected.push(quad);
+        for (const w of this.writers) {
+            await w.write(dataset, (async function* () {
+                yield* collected;
+            })());
+        }
+    }
+}
 export class Pipeline {
-    options;
+    name;
+    datasetSelector;
+    stages;
+    writer;
+    distributionResolver;
+    chaining;
+    reporter;
     constructor(options) {
         const hasSubStages = options.stages.some((stage) => stage.stages.length > 0);
-        if (hasSubStages && !options.stageOutputResolver) {
-            throw new Error('stageOutputResolver is required when any stage has sub-stages');
-        }
-        if (hasSubStages && !options.outputDir) {
-            throw new Error('outputDir is required when any stage has sub-stages');
+        if (hasSubStages && !options.chaining) {
+            throw new Error('chaining is required when any stage has sub-stages');
         }
-        this.options = options;
+        this.name = options.name ?? '';
+        this.datasetSelector = options.datasetSelector;
+        this.stages = options.stages;
+        this.writer = Array.isArray(options.writers)
+            ? new FanOutWriter(options.writers)
+            : options.writers;
+        this.distributionResolver =
+            options.distributionResolver ?? new SparqlDistributionResolver();
+        this.chaining = options.chaining;
+        this.reporter = options.reporter;
     }
     async run() {
-        const { datasetSelector, reporter, name } = this.options;
         const start = Date.now();
-        reporter?.pipelineStart(name);
-        const datasets = await datasetSelector.select();
+        this.reporter?.pipelineStart(this.name);
+        const datasets = await this.datasetSelector.select();
         for await (const dataset of datasets) {
             await this.processDataset(dataset);
         }
-        reporter?.pipelineComplete({ duration: Date.now() - start });
+        this.reporter?.pipelineComplete({ duration: Date.now() - start });
     }
     async processDataset(dataset) {
-        const { distributionResolver, reporter } = this.options;
         const datasetIri = dataset.iri.toString();
-        reporter?.datasetStart(datasetIri);
-        const resolved = await distributionResolver.resolve(dataset);
+        this.reporter?.datasetStart(datasetIri);
+        const resolved = await this.distributionResolver.resolve(dataset);
         if (resolved instanceof NoDistributionAvailable) {
-            reporter?.datasetSkipped(datasetIri, resolved.message);
+            this.reporter?.datasetSkipped(datasetIri, resolved.message);
             return;
         }
         try {
-            for (const stage of this.options.stages) {
+            for (const stage of this.stages) {
                 if (stage.stages.length > 0) {
                     await this.runChain(dataset, resolved.distribution, stage);
                 }
@@ -47,26 +74,25 @@ export class Pipeline {
         catch {
             // Stage error for this dataset; continue to next dataset.
         }
-        reporter?.datasetComplete(datasetIri);
+        this.reporter?.datasetComplete(datasetIri);
     }
     async runStage(dataset, distribution, stage) {
-        const { writer, reporter } = this.options;
-        reporter?.stageStart(stage.name);
+        this.reporter?.stageStart(stage.name);
         const stageStart = Date.now();
         let elementsProcessed = 0;
         let quadsGenerated = 0;
-        const result = await stage.run(dataset, distribution, writer, {
+        const result = await stage.run(dataset, distribution, this.writer, {
             onProgress: (elements, quads) => {
                 elementsProcessed = elements;
                 quadsGenerated = quads;
-                reporter?.stageProgress({ elementsProcessed, quadsGenerated });
+                this.reporter?.stageProgress({ elementsProcessed, quadsGenerated });
             },
         });
         if (result instanceof NotSupported) {
-            reporter?.stageSkipped(stage.name, result.message);
+            this.reporter?.stageSkipped(stage.name, result.message);
         }
         else {
-            reporter?.stageComplete(stage.name, {
+            this.reporter?.stageComplete(stage.name, {
                 elementsProcessed,
                 quadsGenerated,
                 duration: Date.now() - stageStart,
@@ -74,7 +100,7 @@ export class Pipeline {
         }
     }
     async runChain(dataset, distribution, stage) {
-        const { writer, stageOutputResolver, outputDir, outputFormat } = this.options;
+        const { stageOutputResolver, outputDir, outputFormat } = this.chaining;
         const outputFiles = [];
         try {
             // 1. Run parent stage → FileWriter.
@@ -99,15 +125,14 @@ export class Pipeline {
                 }
             }
             // 3. Concatenate all output files → user writer.
-            await writer.write(dataset, this.readFiles(outputFiles));
+            await this.writer.write(dataset, this.readFiles(outputFiles));
         }
         finally {
             await stageOutputResolver.cleanup();
         }
     }
     async runChainedStage(dataset, distribution, stage, stageWriter) {
-        const { reporter } = this.options;
-        reporter?.stageStart(stage.name);
+        this.reporter?.stageStart(stage.name);
         const stageStart = Date.now();
         let elementsProcessed = 0;
         let quadsGenerated = 0;
@@ -115,14 +140,14 @@ export class Pipeline {
             onProgress: (elements, quads) => {
                 elementsProcessed = elements;
                 quadsGenerated = quads;
-                reporter?.stageProgress({ elementsProcessed, quadsGenerated });
+                this.reporter?.stageProgress({ elementsProcessed, quadsGenerated });
             },
         });
         if (result instanceof NotSupported) {
-            reporter?.stageSkipped(stage.name, result.message);
+            this.reporter?.stageSkipped(stage.name, result.message);
             throw new Error(`Stage '${stage.name}' returned NotSupported in chained mode`);
         }
-        reporter?.stageComplete(stage.name, {
+        this.reporter?.stageComplete(stage.name, {
             elementsProcessed,
             quadsGenerated,
             duration: Date.now() - stageStart,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.6.28",
+  "version": "0.6.30",
   "repository": {
     "url": "https://github.com/ldengine/lde",
     "directory": "packages/pipeline"