@lde/pipeline 0.30.11 → 0.30.13

package/README.md

@@ -276,6 +276,63 @@ Writes generated quads to a destination:
 - `SparqlUpdateWriter` — writes to a SPARQL endpoint via UPDATE queries
 - `FileWriter` — writes to local files
 
+### Provenance store
+
+A `ProvenanceStore` gives the pipeline a small per-dataset memory, so a future run can skip datasets that are genuinely unchanged. It is purely a storage seam: the framework owns the skip decision (see [`sourceFingerprint`](#source-change-fingerprint) and `shouldReprocess`), the store owns only how each record is persisted.
+
+```typescript
+interface ProvenanceStore {
+  get(datasetUri: URL): Promise<ProcessingRecord | null>;
+  set(datasetUri: URL, record: ProcessingRecord): Promise<void>;
+}
+```
+
+A `ProcessingRecord` holds the two opaque change fields — `sourceFingerprint` (derived automatically from source metadata) and `pipelineVersion` (consumer-declared) — plus `generatedAt` and a `status` of `'success'` or `'failed'`. The two change fields are compared only for equality, never parsed or ordered.
+
+#### `FileLoadedSparqlProvenanceStore`
+
+The reference implementation targets a triplestore that is served read-only and rebuilt by bulk-loading files (e.g. [QLever](https://github.com/ad-freiburg/qlever)). It reads through SPARQL queries against the live endpoint, and writes records as files for the next bulk-load — because the endpoint accepts no SPARQL UPDATE.
+
+```typescript
+import { FileLoadedSparqlProvenanceStore } from '@lde/pipeline';
+
+const store = new FileLoadedSparqlProvenanceStore({
+  queryEndpoint: new URL('http://localhost:7001/sparql'),
+  pipelineIri: new URL('https://example.org/pipelines/dkg'),
+  outputDir: './provenance',
+});
+```
+
+- `get` runs a named-graph-scoped SPARQL `SELECT` against `queryEndpoint`, reading the records a previous run loaded.
+- `set` writes one flat [PROV-O](https://www.w3.org/TR/prov-o/) N-Quads file per dataset into `outputDir`, in the pipeline-scoped named graph, to be bulk-loaded after the run.
+
+Each record is stored as flat PROV-O on the dataset entity — `prov:generatedAtTime` plus `sourceFingerprint`, `pipelineVersion` and `status` under the `https://w3id.org/lde/provenance#` namespace. Scoping every record by `pipelineIri` (used as the named graph) lets multiple pipelines share one triplestore without colliding.
+
+#### Enabling skipping
+
+Skipping is opt-in. Pass a `provenanceStore` and a `pipelineVersion` to the `Pipeline`:
+
+```typescript
+new Pipeline({
+  // …
+  provenanceStore: store,
+  pipelineVersion: 'v3', // rotate only on releases that change output
+});
+```
+
+For each dataset the pipeline probes its distributions, derives the source-change fingerprint, reads the stored record, and **skips before importing** when both change fields match:
+
+```
+skip iff  recorded.sourceFingerprint === current.sourceFingerprint
+     AND  recorded.pipelineVersion   === current.pipelineVersion
+```
+
+Otherwise it imports (if needed), runs the stages, and writes an updated record. `pipelineVersion` is consumer-owned and opaque: rotate it only on releases that change output, and every dataset reprocesses on the next run. It is **required** when a `provenanceStore` is configured (a skip-enabled pipeline with no version would silently freeze); when no store is configured, every dataset is reprocessed — today’s behaviour. A dataset that failed but whose source is unchanged is recorded as `'failed'` and skipped on later runs until its source changes or the version rotates, so a deterministically failing import is not retried every run.
+
+### Source-change fingerprint
+
+`sourceFingerprint(distribution, probeResult)` derives a cheap, opaque change signal for a distribution from metadata the probe already collected — no body download. For a data dump it combines the most recent of the register’s `dct:modified` and the artifact’s HTTP `Last-Modified` with the byte size (the probe’s `Content-Length`, falling back to the declared `dcat:byteSize`). It returns `null` for a live SPARQL endpoint, or when no date and no size can be established — a `null` fingerprint never compares equal, so such a distribution is always reprocessed.
+
 ### Plugins
 
 Plugins hook into the pipeline lifecycle via the `PipelinePlugin` interface. Register them in the `plugins` array when constructing a `Pipeline`.

package/dist/distribution/importResolver.d.ts

@@ -1,6 +1,7 @@
+import { type Dataset } from '@lde/dataset';
 import type { Importer } from '@lde/sparql-importer';
 import type { SparqlServer } from '@lde/sparql-server';
-import { type DistributionResolver, NoDistributionAvailable, ResolvedDistribution } from './resolver.js';
+import { type DistributionResolver, type ResolveCallbacks, NoDistributionAvailable, ProbedDistributions, ResolvedDistribution } from './resolver.js';
 export interface ImportResolverOptions {
     importer: Importer;
     server: SparqlServer;
@@ -26,12 +27,28 @@ export interface ImportResolverOptions {
  * adds the ability to import a data dump into a local SPARQL server. The
  * {@link ImportResolverOptions.strategy | strategy} option controls whether the
  * inner resolver's SPARQL endpoint is preferred or bypassed.
+ *
+ * The split is preserved across both phases: {@link probe} chooses the
+ * {@link ProbedSource} (the inner SPARQL endpoint, or the preferred importable
+ * data dump) without importing; {@link resolve} performs the import only when
+ * that source is a data dump.
  */
 export declare class ImportResolver implements DistributionResolver {
     private readonly inner;
     private readonly options;
     constructor(inner: DistributionResolver, options: ImportResolverOptions);
-    resolve(...args: Parameters<DistributionResolver['resolve']>): Promise<ResolvedDistribution | NoDistributionAvailable>;
+    probe(dataset: Dataset, callbacks?: ResolveCallbacks): Promise<ProbedDistributions>;
+    resolve(probed: ProbedDistributions, callbacks?: ResolveCallbacks): Promise<ResolvedDistribution | NoDistributionAvailable>;
+    /**
+     * The preferred importable data dump and its probe result, or `null` if no
+     * downloadable distribution passed probing.
+     */
+    private selectImportCandidate;
+    /**
+     * Downloadable distributions whose access URL passed probing, in preference
+     * order (compressed first, see {@link Dataset.getDownloadDistributions}).
+     */
+    private importCandidates;
     private importDataset;
     cleanup(): Promise<void>;
 }

package/dist/distribution/importResolver.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"importResolver.d.ts","sourceRoot":"","sources":["../../src/distribution/importResolver.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAMrD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EACL,KAAK,oBAAoB,EAEzB,uBAAuB,EACvB,oBAAoB,EACrB,MAAM,eAAe,CAAC;AAGvB,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,EAAE,YAAY,CAAC;IACrB;;;;;;;;;;;;OAYG;IACH,QAAQ,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAAC;CAChC;AAED;;;;;;;GAOG;AACH,qBAAa,cAAe,YAAW,oBAAoB;IAEvD,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,OAAO;gBADP,KAAK,EAAE,oBAAoB,EAC3B,OAAO,EAAE,qBAAqB;IAG3C,OAAO,CACX,GAAG,IAAI,EAAE,UAAU,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC,GACnD,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC;YAgB5C,aAAa;IA2GrB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAG/B"}
+{"version":3,"file":"importResolver.d.ts","sourceRoot":"","sources":["../../src/distribution/importResolver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,OAAO,EAAgB,MAAM,cAAc,CAAC;AAC1D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAMrD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EACL,KAAK,oBAAoB,EAEzB,KAAK,gBAAgB,EACrB,uBAAuB,EACvB,mBAAmB,EACnB,oBAAoB,EACrB,MAAM,eAAe,CAAC;AAGvB,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,EAAE,YAAY,CAAC;IACrB;;;;;;;;;;;;OAYG;IACH,QAAQ,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAAC;CAChC;AAED;;;;;;;;;;;;GAYG;AACH,qBAAa,cAAe,YAAW,oBAAoB;IAEvD,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,OAAO;gBADP,KAAK,EAAE,oBAAoB,EAC3B,OAAO,EAAE,qBAAqB;IAG3C,KAAK,CACT,OAAO,EAAE,OAAO,EAChB,SAAS,CAAC,EAAE,gBAAgB,GAC3B,OAAO,CAAC,mBAAmB,CAAC;IAgBzB,OAAO,CACX,MAAM,EAAE,mBAAmB,EAC3B,SAAS,CAAC,EAAE,gBAAgB,GAC3B,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC;IAoB1D;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAY7B;;;OAGG;IACH,OAAO,CAAC,gBAAgB;YAeV,aAAa;IAmGrB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAG/B"}

package/dist/distribution/importResolver.js

@@ -1,6 +1,6 @@
 import { Distribution } from '@lde/dataset';
 import { ImportFailed, ImportSuccessful, NotSupported, } from '@lde/sparql-importer';
-import { NoDistributionAvailable, ResolvedDistribution, } from './resolver.js';
+import { NoDistributionAvailable, ProbedDistributions, ResolvedDistribution, } from './resolver.js';
 import { NetworkError } from '@lde/distribution-probe';
 /**
  * A {@link DistributionResolver} decorator that adds data-dump import logic.
@@ -9,6 +9,11 @@ import { NetworkError } from '@lde/distribution-probe';
  * adds the ability to import a data dump into a local SPARQL server. The
  * {@link ImportResolverOptions.strategy | strategy} option controls whether the
  * inner resolver's SPARQL endpoint is preferred or bypassed.
+ *
+ * The split is preserved across both phases: {@link probe} chooses the
+ * {@link ProbedSource} (the inner SPARQL endpoint, or the preferred importable
+ * data dump) without importing; {@link resolve} performs the import only when
+ * that source is a data dump.
  */
 export class ImportResolver {
     inner;
@@ -17,24 +22,54 @@ export class ImportResolver {
         this.inner = inner;
         this.options = options;
     }
-    async resolve(...args) {
-        const [dataset, callbacks] = args;
-        const result = await this.inner.resolve(...args);
-        // 'sparql' strategy (default): use SPARQL endpoint if inner found one.
-        if (this.options.strategy !== 'import' &&
-            result instanceof ResolvedDistribution) {
-            return result;
+    async probe(dataset, callbacks) {
+        const probed = await this.inner.probe(dataset, callbacks);
+        // 'sparql' strategy (default): keep the inner SPARQL endpoint if found.
+        if (this.options.strategy !== 'import' && probed.source) {
+            return probed;
         }
-        // Either 'import' strategy or inner found nothing: import a data dump.
-        return this.importDataset(dataset, result.probeResults, callbacks);
+        // Either 'import' strategy or no SPARQL endpoint: select a data dump to
+        // import. Choosing the candidate here (not in resolve) keeps the import
+        // cost out of the probe phase while still letting the pipeline fingerprint
+        // the dump it would import.
+        const source = this.selectImportCandidate(dataset, probed.probeResults);
+        return new ProbedDistributions(dataset, probed.probeResults, source);
     }
-    async importDataset(dataset, probeResults, callbacks) {
+    async resolve(probed, callbacks) {
+        if (!probed.source) {
+            return new NoDistributionAvailable(probed.dataset, 'No importable distributions passed probing', probed.probeResults);
+        }
+        // A SPARQL endpoint source needs no import.
+        if (probed.source.distribution.isSparql()) {
+            return new ResolvedDistribution(probed.source.distribution, probed.probeResults);
+        }
+        return this.importDataset(probed.dataset, probed.probeResults, callbacks);
+    }
+    /**
+     * The preferred importable data dump and its probe result, or `null` if no
+     * downloadable distribution passed probing.
+     */
+    selectImportCandidate(dataset, probeResults) {
+        const candidate = this.importCandidates(dataset, probeResults)[0];
+        if (!candidate)
+            return null;
+        const probeResult = probeResults.find((result) => result.url === candidate.accessUrl.toString());
+        return probeResult ? { distribution: candidate, probeResult } : null;
+    }
+    /**
+     * Downloadable distributions whose access URL passed probing, in preference
+     * order (compressed first, see {@link Dataset.getDownloadDistributions}).
+     */
+    importCandidates(dataset, probeResults) {
         const successfulUrls = new Set(probeResults
             .filter((r) => !(r instanceof NetworkError) && r.isSuccess())
             .map((r) => r.url));
-        const candidates = dataset
+        return dataset
             .getDownloadDistributions()
             .filter((d) => d.accessUrl && successfulUrls.has(d.accessUrl.toString()));
+    }
+    async importDataset(dataset, probeResults, callbacks) {
+        const candidates = this.importCandidates(dataset, probeResults);
         // Establish a trustworthy change signal for the downloader so it can skip
         // redundant downloads (and preserve the QLever index cache). For a data
         // dump the authoritative date is the most recent of the register’s declared

package/dist/distribution/index.d.ts

@@ -1,6 +1,6 @@
 export { probe, NetworkError, SparqlProbeResult, DataDumpProbeResult, type ProbeResultType, } from '@lde/distribution-probe';
 export { probeResultsToQuads } from './report.js';
 export { ImportResolver, type ImportResolverOptions, } from './importResolver.js';
-export { ResolvedDistribution, NoDistributionAvailable, SparqlDistributionResolver, type DistributionResolver, type ResolveCallbacks, type SparqlDistributionResolverOptions, } from './resolver.js';
+export { ResolvedDistribution, NoDistributionAvailable, ProbedDistributions, SparqlDistributionResolver, type DistributionResolver, type ProbedSource, type ResolveCallbacks, type SparqlDistributionResolverOptions, } from './resolver.js';
 export { resolveDistributions, type DistributionStageResult, } from './resolveDistributions.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/distribution/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/distribution/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,EACL,YAAY,EACZ,iBAAiB,EACjB,mBAAmB,EACnB,KAAK,eAAe,GACrB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAElD,OAAO,EACL,cAAc,EACd,KAAK,qBAAqB,GAC3B,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EACL,oBAAoB,EACpB,uBAAuB,EACvB,0BAA0B,EAC1B,KAAK,oBAAoB,EACzB,KAAK,gBAAgB,EACrB,KAAK,iCAAiC,GACvC,MAAM,eAAe,CAAC;AAEvB,OAAO,EACL,oBAAoB,EACpB,KAAK,uBAAuB,GAC7B,MAAM,2BAA2B,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/distribution/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,EACL,YAAY,EACZ,iBAAiB,EACjB,mBAAmB,EACnB,KAAK,eAAe,GACrB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAElD,OAAO,EACL,cAAc,EACd,KAAK,qBAAqB,GAC3B,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EACL,oBAAoB,EACpB,uBAAuB,EACvB,mBAAmB,EACnB,0BAA0B,EAC1B,KAAK,oBAAoB,EACzB,KAAK,YAAY,EACjB,KAAK,gBAAgB,EACrB,KAAK,iCAAiC,GACvC,MAAM,eAAe,CAAC;AAEvB,OAAO,EACL,oBAAoB,EACpB,KAAK,uBAAuB,GAC7B,MAAM,2BAA2B,CAAC"}

package/dist/distribution/index.js

@@ -1,5 +1,5 @@
 export { probe, NetworkError, SparqlProbeResult, DataDumpProbeResult, } from '@lde/distribution-probe';
 export { probeResultsToQuads } from './report.js';
 export { ImportResolver, } from './importResolver.js';
-export { ResolvedDistribution, NoDistributionAvailable, SparqlDistributionResolver, } from './resolver.js';
+export { ResolvedDistribution, NoDistributionAvailable, ProbedDistributions, SparqlDistributionResolver, } from './resolver.js';
 export { resolveDistributions, } from './resolveDistributions.js';

package/dist/distribution/resolveDistributions.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"resolveDistributions.d.ts","sourceRoot":"","sources":["../../src/distribution/resolveDistributions.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,YAAY,EAAE,KAAK,OAAO,EAAE,MAAM,cAAc,CAAC;AAC/D,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC/B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAE/D,OAAO,EACL,KAAK,oBAAoB,EAE1B,MAAM,eAAe,CAAC;AAEvB,MAAM,WAAW,uBAAuB;IACtC,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAClC,YAAY,EAAE,eAAe,EAAE,CAAC;IAChC,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC;CAC5B;AAED,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,oBAAoB,GAC7B,OAAO,CAAC,uBAAuB,CAAC,CAoBlC"}
+{"version":3,"file":"resolveDistributions.d.ts","sourceRoot":"","sources":["../../src/distribution/resolveDistributions.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,YAAY,EAAE,KAAK,OAAO,EAAE,MAAM,cAAc,CAAC;AAC/D,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC/B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAE/D,OAAO,EACL,KAAK,oBAAoB,EAE1B,MAAM,eAAe,CAAC;AAEvB,MAAM,WAAW,uBAAuB;IACtC,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAClC,YAAY,EAAE,eAAe,EAAE,CAAC;IAChC,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC;CAC5B;AAED,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,oBAAoB,GAC7B,OAAO,CAAC,uBAAuB,CAAC,CAqBlC"}

package/dist/distribution/resolveDistributions.js

@@ -1,7 +1,8 @@
 import { probeResultsToQuads } from './report.js';
 import { NoDistributionAvailable, } from './resolver.js';
 export async function resolveDistributions(dataset, resolver) {
-    const result = await resolver.resolve(dataset);
+    const probed = await resolver.probe(dataset);
+    const result = await resolver.resolve(probed);
     if (result instanceof NoDistributionAvailable) {
         return {
             distribution: null,

package/dist/distribution/resolver.d.ts

@@ -16,34 +16,68 @@ export declare class NoDistributionAvailable {
     readonly importFailed?: ImportFailed | undefined;
     constructor(dataset: Dataset, message: string, probeResults: ProbeResultType[], importFailed?: ImportFailed | undefined);
 }
-/** Callbacks fired during distribution resolution. */
+/**
+ * The distribution a dataset will be processed from, paired with its probe
+ * result. Drives the source-change fingerprint: a live SPARQL endpoint yields
+ * `null` (always reprocess), a data dump yields its change fingerprint.
+ */
+export interface ProbedSource {
+    distribution: Distribution;
+    probeResult: ProbeResultType;
+}
+/**
+ * The outcome of the probe phase: every distribution’s probe result, plus the
+ * {@link ProbedSource} that will be used to process the dataset (or `null` if
+ * none is available). Determined without importing, so the pipeline can decide
+ * to skip a dataset before paying the import cost.
+ */
+export declare class ProbedDistributions {
+    readonly dataset: Dataset;
+    readonly probeResults: ProbeResultType[];
+    readonly source: ProbedSource | null;
+    constructor(dataset: Dataset, probeResults: ProbeResultType[], source: ProbedSource | null);
+}
+/** Callbacks fired during distribution probing and resolution. */
 export interface ResolveCallbacks {
-    /** Called each time a single distribution probe completes. */
+    /** Called each time a single distribution probe completes (probe phase). */
     onProbe?: (distribution: Distribution, result: ProbeResultType) => void;
-    /** Called when a data-dump import begins. */
+    /** Called when a data-dump import begins (resolve phase). */
     onImportStart?: () => void;
-    /** Called when importing a distribution fails. */
+    /** Called when importing a distribution fails (resolve phase). */
     onImportFailed?: (distribution: Distribution, error: string) => void;
 }
+/**
+ * Resolves a dataset to a usable distribution in two phases so the pipeline can
+ * gate on a dataset’s source-change fingerprint before paying any import cost:
+ *
+ * 1. {@link probe} probes every distribution and selects the source-to-be,
+ *    without importing.
+ * 2. {@link resolve} turns that probed source into a usable SPARQL endpoint,
+ *    importing a data dump only when the source is one.
+ */
 export interface DistributionResolver {
-    resolve(dataset: Dataset, callbacks?: ResolveCallbacks): Promise<ResolvedDistribution | NoDistributionAvailable>;
+    probe(dataset: Dataset, callbacks?: ResolveCallbacks): Promise<ProbedDistributions>;
+    resolve(probed: ProbedDistributions, callbacks?: ResolveCallbacks): Promise<ResolvedDistribution | NoDistributionAvailable>;
     cleanup?(): Promise<void>;
 }
 export interface SparqlDistributionResolverOptions {
     timeout?: number;
 }
 /**
- * Resolves a dataset to a usable SPARQL distribution by probing its distributions.
+ * Resolves a dataset to its own SPARQL endpoint by probing its distributions.
  *
- * 1. Probes all distributions in parallel.
- * 2. Returns the first valid SPARQL endpoint as a `ResolvedDistribution`.
- * 3. If none: returns `NoDistributionAvailable`.
+ * {@link probe} returns the first valid SPARQL endpoint as the
+ * {@link ProbedSource}; {@link resolve} returns it as a
+ * {@link ResolvedDistribution}, or {@link NoDistributionAvailable} when none
+ * responded. Never imports a data dump – wrap with {@link ImportResolver} for
+ * that.
  *
  * Does not mutate `dataset.distributions`.
  */
 export declare class SparqlDistributionResolver implements DistributionResolver {
     private readonly timeout;
     constructor(options?: SparqlDistributionResolverOptions);
-    resolve(dataset: Dataset, callbacks?: ResolveCallbacks): Promise<ResolvedDistribution | NoDistributionAvailable>;
+    probe(dataset: Dataset, callbacks?: ResolveCallbacks): Promise<ProbedDistributions>;
+    resolve(probed: ProbedDistributions): Promise<ResolvedDistribution | NoDistributionAvailable>;
 }
 //# sourceMappingURL=resolver.d.ts.map

package/dist/distribution/resolver.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"resolver.d.ts","sourceRoot":"","sources":["../../src/distribution/resolver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAGL,KAAK,eAAe,EACrB,MAAM,yBAAyB,CAAC;AAEjC,qBAAa,oBAAoB;IAE7B,QAAQ,CAAC,YAAY,EAAE,YAAY;IACnC,QAAQ,CAAC,YAAY,EAAE,eAAe,EAAE;IACxC,QAAQ,CAAC,YAAY,CAAC,EAAE,YAAY;IACpC,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM;IAChC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM;gBAJpB,YAAY,EAAE,YAAY,EAC1B,YAAY,EAAE,eAAe,EAAE,EAC/B,YAAY,CAAC,EAAE,YAAY,YAAA,EAC3B,cAAc,CAAC,EAAE,MAAM,YAAA,EACvB,WAAW,CAAC,EAAE,MAAM,YAAA;CAEhC;AAED,qBAAa,uBAAuB;IAEhC,QAAQ,CAAC,OAAO,EAAE,OAAO;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM;IACxB,QAAQ,CAAC,YAAY,EAAE,eAAe,EAAE;IACxC,QAAQ,CAAC,YAAY,CAAC,EAAE,YAAY;gBAH3B,OAAO,EAAE,OAAO,EAChB,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,eAAe,EAAE,EAC/B,YAAY,CAAC,EAAE,YAAY,YAAA;CAEvC;AAED,sDAAsD;AACtD,MAAM,WAAW,gBAAgB;IAC/B,8DAA8D;IAC9D,OAAO,CAAC,EAAE,CAAC,YAAY,EAAE,YAAY,EAAE,MAAM,EAAE,eAAe,KAAK,IAAI,CAAC;IACxE,6CAA6C;IAC7C,aAAa,CAAC,EAAE,MAAM,IAAI,CAAC;IAC3B,kDAAkD;IAClD,cAAc,CAAC,EAAE,CAAC,YAAY,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CACtE;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,CACL,OAAO,EAAE,OAAO,EAChB,SAAS,CAAC,EAAE,gBAAgB,GAC3B,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC,CAAC;IAC3D,OAAO,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAED,MAAM,WAAW,iCAAiC;IAChD,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;;GAQG;AACH,qBAAa,0BAA2B,YAAW,oBAAoB;IACrE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;gBAErB,OAAO,CAAC,EAAE,iCAAiC;IAIjD,OAAO,CACX,OAAO,EAAE,OAAO,EAChB,SAAS,CAAC,EAAE,gBAAgB,GAC3B,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC;CA6B3D"}
+{"version":3,"file":"resolver.d.ts","sourceRoot":"","sources":["../../src/distribution/resolver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAGL,KAAK,eAAe,EACrB,MAAM,yBAAyB,CAAC;AAEjC,qBAAa,oBAAoB;IAE7B,QAAQ,CAAC,YAAY,EAAE,YAAY;IACnC,QAAQ,CAAC,YAAY,EAAE,eAAe,EAAE;IACxC,QAAQ,CAAC,YAAY,CAAC,EAAE,YAAY;IACpC,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM;IAChC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM;gBAJpB,YAAY,EAAE,YAAY,EAC1B,YAAY,EAAE,eAAe,EAAE,EAC/B,YAAY,CAAC,EAAE,YAAY,YAAA,EAC3B,cAAc,CAAC,EAAE,MAAM,YAAA,EACvB,WAAW,CAAC,EAAE,MAAM,YAAA;CAEhC;AAED,qBAAa,uBAAuB;IAEhC,QAAQ,CAAC,OAAO,EAAE,OAAO;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM;IACxB,QAAQ,CAAC,YAAY,EAAE,eAAe,EAAE;IACxC,QAAQ,CAAC,YAAY,CAAC,EAAE,YAAY;gBAH3B,OAAO,EAAE,OAAO,EAChB,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,eAAe,EAAE,EAC/B,YAAY,CAAC,EAAE,YAAY,YAAA;CAEvC;AAED;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,YAAY,CAAC;IAC3B,WAAW,EAAE,eAAe,CAAC;CAC9B;AAED;;;;;GAKG;AACH,qBAAa,mBAAmB;IAE5B,QAAQ,CAAC,OAAO,EAAE,OAAO;IACzB,QAAQ,CAAC,YAAY,EAAE,eAAe,EAAE;IACxC,QAAQ,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI;gBAF3B,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,eAAe,EAAE,EAC/B,MAAM,EAAE,YAAY,GAAG,IAAI;CAEvC;AAED,kEAAkE;AAClE,MAAM,WAAW,gBAAgB;IAC/B,4EAA4E;IAC5E,OAAO,CAAC,EAAE,CAAC,YAAY,EAAE,YAAY,EAAE,MAAM,EAAE,eAAe,KAAK,IAAI,CAAC;IACxE,6DAA6D;IAC7D,aAAa,CAAC,EAAE,MAAM,IAAI,CAAC;IAC3B,kEAAkE;IAClE,cAAc,CAAC,EAAE,CAAC,YAAY,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CACtE;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,CACH,OAAO,EAAE,OAAO,EAChB,SAAS,CAAC,EAAE,gBAAgB,GAC3B,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAChC,OAAO,CACL,MAAM,EAAE,mBAAmB,EAC3B,SAAS,CAAC,EAAE,gBAAgB,GAC3B,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC,CAAC;IAC3D,OAAO,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAED,MAAM,WAAW,iCAAiC;IAChD,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;;;;GAUG;AACH,qBAAa,0BAA2B,YAAW,oBAAoB;IACrE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;gBAErB,OAAO,CAAC,EAAE,iCAAiC;IAIjD,KAAK,CACT,OAAO,EAAE,OAAO,EAChB,SAAS,CAAC,EAAE,gBAAgB,GAC3B,OAAO,CAAC,mBAAmB,CAAC;IA4BzB,OAAO,CACX,MAAM,EAAE,mBAAmB,GAC1B,OAAO,CAAC,oBAAoB,GAAG,uBAAuB,CAAC;CAc3D"}

package/dist/distribution/resolver.js

@@ -26,11 +26,29 @@ export class NoDistributionAvailable {
     }
 }
 /**
- * Resolves a dataset to a usable SPARQL distribution by probing its distributions.
+ * The outcome of the probe phase: every distribution’s probe result, plus the
+ * {@link ProbedSource} that will be used to process the dataset (or `null` if
+ * none is available). Determined without importing, so the pipeline can decide
+ * to skip a dataset before paying the import cost.
+ */
+export class ProbedDistributions {
+    dataset;
+    probeResults;
+    source;
+    constructor(dataset, probeResults, source) {
+        this.dataset = dataset;
+        this.probeResults = probeResults;
+        this.source = source;
+    }
+}
+/**
+ * Resolves a dataset to its own SPARQL endpoint by probing its distributions.
  *
- * 1. Probes all distributions in parallel.
- * 2. Returns the first valid SPARQL endpoint as a `ResolvedDistribution`.
- * 3. If none: returns `NoDistributionAvailable`.
+ * {@link probe} returns the first valid SPARQL endpoint as the
+ * {@link ProbedSource}; {@link resolve} returns it as a
+ * {@link ResolvedDistribution}, or {@link NoDistributionAvailable} when none
+ * responded. Never imports a data dump – wrap with {@link ImportResolver} for
+ * that.
  *
  * Does not mutate `dataset.distributions`.
  */
@@ -39,22 +57,30 @@ export class SparqlDistributionResolver {
     constructor(options) {
         this.timeout = options?.timeout ?? 5000;
     }
-    async resolve(dataset, callbacks) {
+    async probe(dataset, callbacks) {
         const results = await Promise.all(dataset.distributions.map(async (distribution) => {
             const result = await probe(distribution, { timeoutMs: this.timeout });
             callbacks?.onProbe?.(distribution, result);
             return result;
         }));
         // Find first valid SPARQL endpoint.
+        let source = null;
         for (let i = 0; i < dataset.distributions.length; i++) {
             const distribution = dataset.distributions[i];
             const result = results[i];
             if (distribution.isSparql() &&
                 result instanceof SparqlProbeResult &&
                 result.isSuccess()) {
-                return new ResolvedDistribution(distribution, results);
+                source = { distribution, probeResult: result };
+                break;
             }
         }
-        return new NoDistributionAvailable(dataset, 'No SPARQL endpoint available', results);
+        return new ProbedDistributions(dataset, results, source);
+    }
+    async resolve(probed) {
+        if (probed.source && probed.source.distribution.isSparql()) {
+            return new ResolvedDistribution(probed.source.distribution, probed.probeResults);
+        }
+        return new NoDistributionAvailable(probed.dataset, 'No SPARQL endpoint available', probed.probeResults);
     }
 }

package/dist/pipeline.d.ts

@@ -4,6 +4,7 @@ import { Stage } from './stage.js';
 import type { QuadTransform } from './stage.js';
 import type { Writer } from './writer/writer.js';
 import { type DistributionResolver } from './distribution/resolver.js';
+import type { ProvenanceStore } from './provenance/store.js';
 import type { StageOutputResolver } from './stageOutputResolver.js';
 import type { ProgressReporter } from './progressReporter.js';
 import { type TimeoutPolicy } from './sparql/timeoutPolicy.js';
@@ -32,6 +33,22 @@ export interface PipelineOptions {
         outputDir: string;
     };
     reporter?: ProgressReporter;
+    /**
+     * Optional per-dataset processing memory. When set, the pipeline skips a
+     * dataset whose source-change fingerprint and {@link pipelineVersion} both
+     * match the stored record – before paying the import cost – and writes an
+     * updated record after processing. When omitted, every dataset is
+     * reprocessed (today’s behaviour).
+     */
+    provenanceStore?: ProvenanceStore;
+    /**
+     * Opaque, consumer-declared version of the pipeline’s output-affecting
+     * logic, rotated only on releases that change output. Compared for equality,
+     * never parsed or ordered. Required when {@link provenanceStore} is set (a
+     * skip-enabled pipeline with no version would silently freeze); ignored
+     * otherwise.
+     */
+    pipelineVersion?: string;
     /**
      * Factory producing a fresh {@link TimeoutPolicy} per dataset. Defaults
      * to {@link constantTimeoutPolicy}`(300_000)` so existing call sites
@@ -53,9 +70,13 @@ export declare class Pipeline {
     private readonly chaining?;
     private readonly reporter?;
     private readonly timeoutFactory;
+    private readonly provenanceStore?;
+    private readonly pipelineVersion?;
     constructor(options: PipelineOptions);
     run(): Promise<void>;
     private processDataset;
+    /** Persist the processing record for a dataset, when a store is configured. */
+    private recordOutcome;
     private reportValidators;
     private collectStages;
     /**

package/dist/pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AAGrD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAE1B,MAAM,4BAA4B,CAAC;AAQpC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,2BAA2B,CAAC;AAEnC,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,aAAa,CAAC;QAAE,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;CACxD;AAED,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,OAAO,CAAC,EAAE,cAAc,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;CAC/B;AAgFD,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAA8B;IACxD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAsB;gBAEzC,OAAO,EAAE,eAAe;IAkC9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAoBZ,cAAc;YAgFd,gBAAgB;IAW9B,OAAO,CAAE,aAAa;IAOtB;;;OAGG;YACW,QAAQ;IA0CtB,2EAA2E;YAC7D,eAAe;YAqBf,QAAQ;YA2DP,SAAS;CAczB"}
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AAGrD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EACL,KAAK,oBAAoB,EAG1B,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAO7D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,2BAA2B,CAAC;AAEnC,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,aAAa,CAAC;QAAE,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;CACxD;AAED,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,OAAO,CAAC,EAAE,cAAc,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;CAC/B;AAgFD,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAA8B;IACxD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAsB;IACrD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAkB;IACnD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAS;gBAE9B,OAAO,EAAE,eAAe;IA0C9B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAoBZ,cAAc;IA0I5B,+EAA+E;YACjE,aAAa;YAmBb,gBAAgB;IAW9B,OAAO,CAAE,aAAa;IAOtB;;;OAGG;YACW,QAAQ;IA0CtB,2EAA2E;YAC7D,eAAe;YAqBf,QAAQ;YA2DP,SAAS;CAczB"}

package/dist/pipeline.js

@@ -3,6 +3,8 @@ import { StreamParser } from 'n3';
 import { FileWriter } from './writer/fileWriter.js';
 import { NoDistributionAvailable, } from './distribution/resolver.js';
 import { SparqlDistributionResolver } from './distribution/index.js';
+import { sourceFingerprint } from './provenance/sourceFingerprint.js';
+import { shouldReprocess } from './provenance/reprocessDecision.js';
 import { NetworkError, SparqlProbeResult, } from '@lde/distribution-probe';
 import { NotSupported } from './sparql/executor.js';
 import { ConstantTimeoutPolicy, } from './sparql/timeoutPolicy.js';
@@ -81,11 +83,16 @@ export class Pipeline {
     chaining;
     reporter;
     timeoutFactory;
+    provenanceStore;
+    pipelineVersion;
     constructor(options) {
         const hasSubStages = options.stages.some((stage) => stage.stages.length > 0);
         if (hasSubStages && !options.chaining) {
             throw new Error('chaining is required when any stage has sub-stages');
         }
+        if (options.provenanceStore && options.pipelineVersion === undefined) {
+            throw new Error('pipelineVersion is required when a provenanceStore is configured');
+        }
         this.name = options.name ?? '';
         this.datasetSelector = options.datasetSelector;
         this.stages = options.stages;
@@ -106,6 +113,8 @@ export class Pipeline {
         this.reporter = options.reporter;
         this.timeoutFactory =
             options.timeout ?? (() => new ConstantTimeoutPolicy(300_000));
+        this.provenanceStore = options.provenanceStore;
+        this.pipelineVersion = options.pipelineVersion;
     }
     async run() {
         const start = Date.now();
@@ -125,17 +134,48 @@ export class Pipeline {
     }
     async processDataset(dataset) {
         this.reporter?.datasetStart?.(dataset);
-        const timeout = this.timeoutFactory();
-        const unsubscribe = timeout.subscribe?.({
-            onTighten: (event) => this.reporter?.timeoutTightened?.(event),
-            onRelax: (event) => this.reporter?.timeoutRelaxed?.(event),
-        });
-        let resolved;
+        // Probe phase: gather probe results and the source-to-be, without importing.
+        let probed;
         try {
-            resolved = await this.distributionResolver.resolve(dataset, {
+            probed = await this.distributionResolver.probe(dataset, {
                 onProbe: (distribution, result) => {
                     this.reporter?.distributionProbed?.(mapProbeResult(distribution, result));
                 },
+            });
+        }
+        catch (error) {
+            this.reporter?.datasetSkipped?.(dataset, `Distribution probing failed: ${error instanceof Error ? error.message : String(error)}`);
+            return;
+        }
+        // Derive the source-change fingerprint from the probed source: null for a
+        // live SPARQL endpoint (always reprocess) or when no source is available.
+        const fingerprint = probed.source
+            ? sourceFingerprint(probed.source.distribution, probed.source.probeResult)
+            : null;
+        // Gate: skip an unchanged dataset before paying any import cost.
+        if (this.provenanceStore) {
+            let stored = null;
+            try {
+                stored = await this.provenanceStore.get(dataset.iri);
+            }
+            catch {
+                // An unreadable record must not abort the whole run, nor wrongly skip:
+                // treat it as ‘never processed’ so this dataset reprocesses. The
+                // periodic full reprocess is the backstop.
+                stored = null;
+            }
+            if (!shouldReprocess({
+                sourceFingerprint: fingerprint,
+                pipelineVersion: this.pipelineVersion,
+            }, stored)) {
+                this.reporter?.datasetSkipped?.(dataset, 'Unchanged since last run');
+                return;
+            }
+        }
+        // Resolve phase: import a data dump only when the source is one.
+        let resolved;
+        try {
+            resolved = await this.distributionResolver.resolve(probed, {
                 onImportStart: () => {
                     this.reporter?.importStarted?.();
                 },
@@ -149,10 +189,20 @@ export class Pipeline {
             return;
         }
         if (resolved instanceof NoDistributionAvailable) {
+            // Record the failure so a dataset whose source is unchanged is not
+            // re-imported every run; it is retried at the next fingerprint change or
+            // version rotation.
+            await this.recordOutcome(dataset, fingerprint, 'failed');
             this.reporter?.datasetSkipped?.(dataset, resolved.message);
             return;
         }
         this.reporter?.distributionSelected?.(dataset, resolved.distribution, resolved.importedFrom, resolved.importDuration, resolved.tripleCount);
+        const timeout = this.timeoutFactory();
+        const unsubscribe = timeout.subscribe?.({
+            onTighten: (event) => this.reporter?.timeoutTightened?.(event),
+            onRelax: (event) => this.reporter?.timeoutRelaxed?.(event),
+        });
+        let stageFailed = false;
         try {
             for (const stage of this.stages) {
                 try {
@@ -164,6 +214,7 @@ export class Pipeline {
                     }
                 }
                 catch (error) {
+                    stageFailed = true;
                     this.reporter?.stageFailed?.(stage.name, error instanceof Error ? error : new Error(String(error)));
                 }
             }
@@ -174,12 +225,32 @@ export class Pipeline {
         }
         await this.writer.flush?.(dataset);
         await this.reportValidators(dataset);
+        // A dataset whose stages threw produced incomplete output; record it as
+        // ‘failed’ rather than freezing a broken result under a ‘success’ record.
+        await this.recordOutcome(dataset, fingerprint, stageFailed ? 'failed' : 'success');
         const datasetMemory = process.memoryUsage();
         this.reporter?.datasetComplete?.(dataset, {
             memoryUsageBytes: datasetMemory.rss,
             heapUsedBytes: datasetMemory.heapUsed,
         });
     }
+    /** Persist the processing record for a dataset, when a store is configured. */
+    async recordOutcome(dataset, fingerprint, status) {
+        if (!this.provenanceStore)
+            return;
+        try {
+            await this.provenanceStore.set(dataset.iri, {
+                sourceFingerprint: fingerprint,
+                pipelineVersion: this.pipelineVersion,
+                generatedAt: new Date().toISOString(),
+                status,
+            });
+        }
+        catch {
+            // A failed write must not abort the run; the dataset simply reprocesses
+            // next run, its record not yet updated.
+        }
+    }
     async reportValidators(dataset) {
         const validators = new Set();
         for (const stage of this.collectStages(this.stages)) {

package/dist/provenance/fileLoadedSparqlProvenanceStore.d.ts

@@ -0,0 +1,44 @@
+import { SparqlEndpointFetcher } from 'fetch-sparql-endpoint';
+import type { ProcessingRecord } from './record.js';
+import type { ProvenanceStore } from './store.js';
+export interface FileLoadedSparqlProvenanceStoreOptions {
+    /** Read-only SPARQL endpoint to query for previously-loaded records. */
+    queryEndpoint: URL;
+    /**
+     * The pipeline’s IRI, used as the named graph that scopes this pipeline’s
+     * records so multiple pipelines sharing one triplestore do not collide.
+     */
+    pipelineIri: URL;
+    /**
+     * Directory the records are written to as files, to be bulk-loaded into the
+     * read-only triplestore after the run. Kept separate from the data output
+     * directory so filenames (keyed by dataset URI) never collide.
+     */
+    outputDir: string;
+    /**
+     * Optional {@link SparqlEndpointFetcher} for the query side, intended for
+     * tests. Defaults to a fresh instance.
+     */
+    fetcher?: SparqlEndpointFetcher;
+}
+/**
+ * A {@link ProvenanceStore} for a triplestore that is served read-only and
+ * rebuilt by bulk-loading files (e.g. QLever).
+ *
+ * Reads through SPARQL queries against the live endpoint (records loaded from
+ * a previous run); writes the records as files for the next bulk-load, since
+ * the endpoint accepts no SPARQL UPDATE. Records are flat PROV-O keyed by the
+ * dataset URI, written into the pipeline-scoped provenance named graph.
+ */
+export declare class FileLoadedSparqlProvenanceStore implements ProvenanceStore {
+    private readonly queryEndpoint;
+    private readonly pipelineIri;
+    private readonly writer;
+    private readonly fetcher;
+    constructor(options: FileLoadedSparqlProvenanceStoreOptions);
+    get(datasetUri: URL): Promise<ProcessingRecord | null>;
+    private selectQuery;
+    set(datasetUri: URL, record: ProcessingRecord): Promise<void>;
+    private toQuads;
+}
+//# sourceMappingURL=fileLoadedSparqlProvenanceStore.d.ts.map

package/dist/provenance/fileLoadedSparqlProvenanceStore.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fileLoadedSparqlProvenanceStore.d.ts","sourceRoot":"","sources":["../../src/provenance/fileLoadedSparqlProvenanceStore.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAE9D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AASlD,MAAM,WAAW,sCAAsC;IACrD,wEAAwE;IACxE,aAAa,EAAE,GAAG,CAAC;IACnB;;;OAGG;IACH,WAAW,EAAE,GAAG,CAAC;IACjB;;;;OAIG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,OAAO,CAAC,EAAE,qBAAqB,CAAC;CACjC;AAED;;;;;;;;GAQG;AACH,qBAAa,+BAAgC,YAAW,eAAe;IACrE,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAM;IACpC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAM;IAClC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAa;IACpC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;gBAEpC,OAAO,EAAE,sCAAsC;IAWrD,GAAG,CAAC,UAAU,EAAE,GAAG,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAoB5D,OAAO,CAAC,WAAW;IAmBb,GAAG,CAAC,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;YAMpD,OAAO;CA0BvB"}

package/dist/provenance/fileLoadedSparqlProvenanceStore.js

@@ -0,0 +1,81 @@
+import { Dataset, assertSafeIri } from '@lde/dataset';
+import { DataFactory } from 'n3';
+import { SparqlEndpointFetcher } from 'fetch-sparql-endpoint';
+import { FileWriter } from '../writer/fileWriter.js';
+const { namedNode, literal, quad } = DataFactory;
+const RDF_TYPE = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type';
+const PROV = 'http://www.w3.org/ns/prov#';
+const LDE = 'https://w3id.org/lde/provenance#';
+const XSD_DATE_TIME = 'http://www.w3.org/2001/XMLSchema#dateTime';
+/**
+ * A {@link ProvenanceStore} for a triplestore that is served read-only and
+ * rebuilt by bulk-loading files (e.g. QLever).
+ *
+ * Reads through SPARQL queries against the live endpoint (records loaded from
+ * a previous run); writes the records as files for the next bulk-load, since
+ * the endpoint accepts no SPARQL UPDATE. Records are flat PROV-O keyed by the
+ * dataset URI, written into the pipeline-scoped provenance named graph.
+ */
+export class FileLoadedSparqlProvenanceStore {
+    queryEndpoint;
+    pipelineIri;
+    writer;
+    fetcher;
+    constructor(options) {
+        this.queryEndpoint = options.queryEndpoint;
+        this.pipelineIri = options.pipelineIri;
+        this.writer = new FileWriter({
+            outputDir: options.outputDir,
+            format: 'n-quads',
+            graphIri: () => this.pipelineIri,
+        });
+        this.fetcher = options.fetcher ?? new SparqlEndpointFetcher();
+    }
+    async get(datasetUri) {
+        const stream = (await this.fetcher.fetchBindings(this.queryEndpoint.toString(), this.selectQuery(datasetUri)));
+        for await (const binding of stream) {
+            // A record exists iff the mandatory fields bound; the fingerprint is
+            // optional and absent for a run with no establishable signal.
+            return {
+                sourceFingerprint: binding.fingerprint?.value ?? null,
+                pipelineVersion: binding.version.value,
+                generatedAt: binding.generatedAt.value,
+                status: binding.status.value,
+            };
+        }
+        return null;
+    }
+    selectQuery(datasetUri) {
+        // Guard before interpolating into a SPARQL `<…>` reference, mirroring
+        // SparqlUpdateWriter. URL normalisation already encodes unsafe characters,
+        // so this is defence-in-depth against a non-normalised IRI reaching here.
+        const datasetIri = datasetUri.toString();
+        const pipelineIri = this.pipelineIri.toString();
+        assertSafeIri(datasetIri);
+        assertSafeIri(pipelineIri);
+        const dataset = `<${datasetIri}>`;
+        return `SELECT ?fingerprint ?version ?status ?generatedAt WHERE {
+      GRAPH <${pipelineIri}> {
+        ${dataset} <${LDE}pipelineVersion> ?version ;
+                   <${LDE}status> ?status ;
+                   <${PROV}generatedAtTime> ?generatedAt .
+        OPTIONAL { ${dataset} <${LDE}sourceFingerprint> ?fingerprint }
+      }
+    } LIMIT 1`;
+    }
+    async set(datasetUri, record) {
+        const dataset = new Dataset({ iri: datasetUri, distributions: [] });
+        await this.writer.write(dataset, this.toQuads(datasetUri, record));
+        await this.writer.flush(dataset);
+    }
+    async *toQuads(datasetUri, record) {
+        const subject = namedNode(datasetUri.toString());
+        yield quad(subject, namedNode(RDF_TYPE), namedNode(`${PROV}Entity`));
+        yield quad(subject, namedNode(`${PROV}generatedAtTime`), literal(record.generatedAt, namedNode(XSD_DATE_TIME)));
+        if (record.sourceFingerprint !== null) {
+            yield quad(subject, namedNode(`${LDE}sourceFingerprint`), literal(record.sourceFingerprint));
+        }
+        yield quad(subject, namedNode(`${LDE}pipelineVersion`), literal(record.pipelineVersion));
+        yield quad(subject, namedNode(`${LDE}status`), literal(record.status));
+    }
+}

package/dist/provenance/index.d.ts

@@ -1,4 +1,6 @@
 export { sourceFingerprint } from './sourceFingerprint.js';
 export { shouldReprocess } from './reprocessDecision.js';
 export type { ProcessingRecord, ChangeKey } from './record.js';
+export type { ProvenanceStore } from './store.js';
+export { FileLoadedSparqlProvenanceStore, type FileLoadedSparqlProvenanceStoreOptions, } from './fileLoadedSparqlProvenanceStore.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/provenance/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/provenance/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,YAAY,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/provenance/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,YAAY,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAC/D,YAAY,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAClD,OAAO,EACL,+BAA+B,EAC/B,KAAK,sCAAsC,GAC5C,MAAM,sCAAsC,CAAC"}

package/dist/provenance/index.js

@@ -1,2 +1,3 @@
 export { sourceFingerprint } from './sourceFingerprint.js';
 export { shouldReprocess } from './reprocessDecision.js';
+export { FileLoadedSparqlProvenanceStore, } from './fileLoadedSparqlProvenanceStore.js';

package/dist/provenance/store.d.ts

@@ -0,0 +1,20 @@
+import type { ProcessingRecord } from './record.js';
+/**
+ * The pipeline’s per-dataset processing memory.
+ *
+ * The framework owns the skip semantics (see `shouldReprocess`); a
+ * `ProvenanceStore` owns only the physical storage of {@link ProcessingRecord}s,
+ * keyed by dataset URI. Implementations are free to back this with a
+ * triplestore, files, or anything else.
+ */
+export interface ProvenanceStore {
+    /**
+     * The record from the dataset’s last processing, or `null` if it has never
+     * been processed (or the store was wiped). A `null` result drives a
+     * reprocess.
+     */
+    get(datasetUri: URL): Promise<ProcessingRecord | null>;
+    /** Persist the record for a dataset, replacing any previous one. */
+    set(datasetUri: URL, record: ProcessingRecord): Promise<void>;
+}
+//# sourceMappingURL=store.d.ts.map

package/dist/provenance/store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/provenance/store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAEpD;;;;;;;GAOG;AACH,MAAM,WAAW,eAAe;IAC9B;;;;OAIG;IACH,GAAG,CAAC,UAAU,EAAE,GAAG,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAAC;IACvD,oEAAoE;IACpE,GAAG,CAAC,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC/D"}

package/dist/provenance/store.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.30.11",
+  "version": "0.30.13",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/pipeline"
@@ -24,14 +24,14 @@
     "!**/*.tsbuildinfo"
   ],
   "dependencies": {
-    "@lde/dataset": "0.7.5",
-    "@lde/dataset-registry-client": "0.8.1",
-    "@lde/distribution-probe": "0.1.8",
-    "@lde/sparql-importer": "0.6.3",
+    "@lde/dataset": "0.7.6",
+    "@lde/dataset-registry-client": "0.8.2",
+    "@lde/distribution-probe": "0.1.9",
+    "@lde/sparql-importer": "0.6.4",
     "@lde/sparql-server": "0.4.11",
     "@rdfjs/types": "^2.0.1",
-    "@traqula/generator-sparql-1-1": "^1.1.1",
-    "@traqula/parser-sparql-1-1": "^1.1.1",
+    "@traqula/generator-sparql-1-1": "^1.1.4",
+    "@traqula/parser-sparql-1-1": "^1.1.4",
     "@traqula/rules-sparql-1-1": "^1.1.0",
     "fetch-sparql-endpoint": "^7.1.0",
     "filenamify-url": "^4.0.0",