@lde/pipeline 0.35.2 → 0.35.4

package/README.md

@@ -339,11 +339,14 @@ A `ProvenanceStore` gives the pipeline a small per-dataset memory, so a future r
 
 ```typescript
 interface ProvenanceStore {
+  check?(): Promise<void>;
   get(datasetUri: URL): Promise<ProcessingRecord | null>;
   set(datasetUri: URL, record: ProcessingRecord): Promise<void>;
 }
 ```
 
+The optional `check` verifies the store can persist records; the pipeline calls it once at the start of a run, so a store that would fail every write fails the run immediately instead of silently disabling skip-unchanged. A record write that fails mid-run is reported through the reporter (`stageFailed` with stage `'provenance'`) and does not abort the run.
+
 A `ProcessingRecord` holds the two opaque change fields – `sourceFingerprint` (derived automatically from source metadata) and `pipelineVersion` (consumer-declared) – plus `generatedAt` and a `status` of `'success'` or `'failed'`. The two change fields are compared only for equality, never parsed or ordered.
 
 #### `FileLoadedSparqlProvenanceStore`
@@ -375,7 +378,7 @@ import { FileProvenanceStore } from '@lde/pipeline';
 const store = new FileProvenanceStore({ path: './state/provenance.json' });
 ```
 
-Writes are atomic (temp file + rename), so a run killed mid-write cannot corrupt the next run’s skip decisions. Safe for a single writer: concurrent pipeline processes writing the same file lose each other’s updates – use the SPARQL-backed store (or file locking) instead. The file must sit on a durable volume to survive across runs, the same requirement the SPARQL-backed store already has.
+Writes are atomic (temp file + rename), so a run killed mid-write cannot corrupt the next run’s skip decisions. Safe for a single writer: concurrent pipeline processes writing the same file lose each other’s updates – use the SPARQL-backed store (or file locking) instead. The file must sit on a durable volume to survive across runs, the same requirement the SPARQL-backed store already has, and its directory must be writable by the user the pipeline runs as – `check` probes this with a real write at the start of the run, so a read-only or root-owned mount fails loudly instead of silently disabling skipping.
 
 #### Enabling skipping
 

package/dist/pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD,OAAO,KAAK,EAKV,MAAM,EACP,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACL,KAAK,oBAAoB,EAI1B,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAY7D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,2BAA2B,CAAC;AAEnC;;;;;;;GAOG;AACH,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;GAKG;AACH,MAAM,WAAW,yBAAyB;IACxC,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,aAAa,CAAC,uBAAuB,CAAC,CAAC;IAC1D;;;;;;;;;;;;OAYG;IACH,kBAAkB,CAAC,EAAE,aAAa,CAAC,yBAAyB,CAAC,CAAC;CAC/D;AAED,MAAM,WAAW,eAAe,CAAC,GAAG,GAAG,IAAI;IACzC,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;IACrC;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,cAAc,EAAE,GAAG,KAAK,CAAC;IAC1D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,gBAAgB,GAAG,SAAS,gBAAgB,EAAE,CAAC;IAC1D;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;CAC/B;AA0OD,qBAAa,QAAQ,CAAC,GAAG,GAAG,IAAI;IAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAyC;IAC3E,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAA2C;IAC/E,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmC;IAC7D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAsB;IACrD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAkB;IACnD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAS;gBAE9B,OAAO,EAAE,eAAe,CAAC,GAAG,CAAC;IA6DnC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAoDZ,cAAc;IAgO5B,+EAA+E;YACjE,aAAa;YAmBb,gBAAgB;IAa9B,OAAO,CAAE,aAAa;IAStB;;;;;OAKG;IACH,OAAO,CAAC,WAAW;IAgBnB;;;;;;;;OAQG;IACH,OAAO,CAAC,0BAA0B;IA4BlC;;;;;OAKG;YACW,SAAS;IAuCvB;;;OAGG;YACW,QAAQ;IA0CtB,2EAA2E;YAC7D,eAAe;YAqBf,QAAQ;YA8EP,SAAS;CAczB"}
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAgB,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD,OAAO,KAAK,EAKV,MAAM,EACP,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACL,KAAK,oBAAoB,EAI1B,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAY7D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,2BAA2B,CAAC;AAEnC;;;;;;;GAOG;AACH,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;GAKG;AACH,MAAM,WAAW,yBAAyB;IACxC,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,aAAa,CAAC,uBAAuB,CAAC,CAAC;IAC1D;;;;;;;;;;;;OAYG;IACH,kBAAkB,CAAC,EAAE,aAAa,CAAC,yBAAyB,CAAC,CAAC;CAC/D;AAED,MAAM,WAAW,eAAe,CAAC,GAAG,GAAG,IAAI;IACzC,eAAe,EAAE,eAAe,CAAC;IACjC,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;IACrC;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,cAAc,EAAE,GAAG,KAAK,CAAC;IAC1D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,QAAQ,CAAC,EAAE;QACT,mBAAmB,EAAE,mBAAmB,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,gBAAgB,GAAG,SAAS,gBAAgB,EAAE,CAAC;IAC1D;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;CAC/B;AA0OD,qBAAa,QAAQ,CAAC,GAAG,GAAG,IAAI;IAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAyC;IAC3E,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAA2C;IAC/E,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAuB;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmC;IAC7D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAsB;IACrD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAkB;IACnD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAS;gBAE9B,OAAO,EAAE,eAAe,CAAC,GAAG,CAAC;IA6DnC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YA0DZ,cAAc;IAgO5B,+EAA+E;YACjE,aAAa;YAyBb,gBAAgB;IAa9B,OAAO,CAAE,aAAa;IAStB;;;;;OAKG;IACH,OAAO,CAAC,WAAW;IAgBnB;;;;;;;;OAQG;IACH,OAAO,CAAC,0BAA0B;IA4BlC;;;;;OAKG;YACW,SAAS;IAuCvB;;;OAGG;YACW,QAAQ;IA0CtB,2EAA2E;YAC7D,eAAe;YAqBf,QAAQ;YA8EP,SAAS;CAczB"}

package/dist/pipeline.js

@@ -273,6 +273,11 @@ export class Pipeline {
     async run() {
         const start = Date.now();
         this.reporter?.pipelineStart?.(this.name);
+        // Fail fast on a store that cannot persist records (e.g. a provenance
+        // file on a mount the runtime user cannot write to): every dataset would
+        // process fine, but the records would be lost and skip-unchanged would
+        // silently never engage on any later run.
+        await this.provenanceStore?.check?.();
         const selectStart = Date.now();
         const datasets = await this.datasetSelector.select();
         this.reporter?.datasetsSelected?.(datasets.total, Date.now() - selectStart);
@@ -486,9 +491,12 @@ export class Pipeline {
                 status,
             });
         }
-        catch {
-            // A failed write must not abort the run; the dataset simply reprocesses
-            // next run, its record not yet updated.
+        catch (error) {
+            // A failed write must not abort the run – the dataset simply
+            // reprocesses next run, its record not yet updated – but it must be
+            // surfaced: a store that fails every write silently disables
+            // skip-unchanged.
+            this.reporter?.stageFailed?.('provenance', error instanceof Error ? error : new Error(String(error)));
         }
     }
     async reportValidators(dataset) {

package/dist/provenance/fileProvenanceStore.d.ts

@@ -26,6 +26,13 @@ export interface FileProvenanceStoreOptions {
 export declare class FileProvenanceStore implements ProvenanceStore {
     private readonly path;
     constructor(options: FileProvenanceStoreOptions);
+    /**
+     * Verify the file can be written by actually creating (and removing) a
+     * probe file next to it. Catches the mount that a non-root runtime user
+     * cannot write to (e.g. a root-owned Docker volume, see issue #661) at the
+     * start of the run, before writes would silently fail per dataset.
+     */
+    check(): Promise<void>;
     get(datasetUri: URL): Promise<ProcessingRecord | null>;
     set(datasetUri: URL, record: ProcessingRecord): Promise<void>;
     private readAll;

package/dist/provenance/fileProvenanceStore.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fileProvenanceStore.d.ts","sourceRoot":"","sources":["../../src/provenance/fileProvenanceStore.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAElD,MAAM,WAAW,0BAA0B;IACzC;;;;OAIG;IACH,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,mBAAoB,YAAW,eAAe;IACzD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;gBAElB,OAAO,EAAE,0BAA0B;IAIzC,GAAG,CAAC,UAAU,EAAE,GAAG,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAKtD,GAAG,CAAC,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;YAMrD,OAAO;YAcP,QAAQ;CAQvB"}
+{"version":3,"file":"fileProvenanceStore.d.ts","sourceRoot":"","sources":["../../src/provenance/fileProvenanceStore.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAElD,MAAM,WAAW,0BAA0B;IACzC;;;;OAIG;IACH,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,mBAAoB,YAAW,eAAe;IACzD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;gBAElB,OAAO,EAAE,0BAA0B;IAI/C;;;;;OAKG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IActB,GAAG,CAAC,UAAU,EAAE,GAAG,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAKtD,GAAG,CAAC,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;YAMrD,OAAO;YAcP,QAAQ;CAQvB"}

package/dist/provenance/fileProvenanceStore.js

@@ -1,4 +1,4 @@
-import { mkdir, readFile, rename, writeFile } from 'node:fs/promises';
+import { mkdir, readFile, rename, unlink, writeFile } from 'node:fs/promises';
 import { dirname } from 'node:path';
 /**
  * A {@link ProvenanceStore} that persists all records to a single JSON file,
@@ -20,6 +20,23 @@ export class FileProvenanceStore {
     constructor(options) {
         this.path = options.path;
     }
+    /**
+     * Verify the file can be written by actually creating (and removing) a
+     * probe file next to it. Catches the mount that a non-root runtime user
+     * cannot write to (e.g. a root-owned Docker volume, see issue #661) at the
+     * start of the run, before writes would silently fail per dataset.
+     */
+    async check() {
+        try {
+            await mkdir(dirname(this.path), { recursive: true });
+            const probePath = `${this.path}.${process.pid}.check`;
+            await writeFile(probePath, '');
+            await unlink(probePath);
+        }
+        catch (error) {
+            throw new Error(`Provenance file ${this.path} is not writable; make its directory writable by the user the pipeline runs as: ${String(error)}`, { cause: error });
+        }
+    }
     async get(datasetUri) {
         const records = await this.readAll();
         return records[datasetUri.toString()] ?? null;

package/dist/provenance/store.d.ts

@@ -8,6 +8,14 @@ import type { ProcessingRecord } from './record.js';
  * triplestore, files, or anything else.
  */
 export interface ProvenanceStore {
+    /**
+     * Verify the store can persist records, throwing when it cannot. The
+     * pipeline calls this once at the start of a run, so a store that would
+     * fail every write – e.g. a provenance file on a mount the runtime user
+     * cannot write to – fails the run immediately instead of silently
+     * disabling skip-unchanged.
+     */
+    check?(): Promise<void>;
     /**
      * The record from the dataset’s last processing, or `null` if it has never
      * been processed (or the store was wiped). A `null` result drives a

package/dist/provenance/store.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/provenance/store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAEpD;;;;;;;GAOG;AACH,MAAM,WAAW,eAAe;IAC9B;;;;OAIG;IACH,GAAG,CAAC,UAAU,EAAE,GAAG,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAAC;IACvD,oEAAoE;IACpE,GAAG,CAAC,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC/D"}
+{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/provenance/store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAEpD;;;;;;;GAOG;AACH,MAAM,WAAW,eAAe;IAC9B;;;;;;OAMG;IACH,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB;;;;OAIG;IACH,GAAG,CAAC,UAAU,EAAE,GAAG,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAAC;IACvD,oEAAoE;IACpE,GAAG,CAAC,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC/D"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.35.2",
+  "version": "0.35.4",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/pipeline"
@@ -24,12 +24,12 @@
     "!**/*.tsbuildinfo"
   ],
   "dependencies": {
-    "@lde/dataset": "^0.7.8",
-    "@lde/dataset-registry-client": "^0.9.0",
-    "@lde/distribution-health": "^0.2.8",
-    "@lde/distribution-probe": "^0.2.6",
-    "@lde/sparql-importer": "^0.6.8",
-    "@lde/sparql-server": "^0.4.13",
+    "@lde/dataset": "^0.7.9",
+    "@lde/dataset-registry-client": "^0.9.1",
+    "@lde/distribution-health": "^0.2.9",
+    "@lde/distribution-probe": "^0.2.7",
+    "@lde/sparql-importer": "^0.6.9",
+    "@lde/sparql-server": "^0.4.14",
     "@rdfjs/namespace": "^2.0.1",
     "@rdfjs/types": "^2.0.1",
     "@tpluscode/rdf-ns-builders": "^5.0.0",