@lde/pipeline 0.30.10 → 0.30.11

package/dist/index.d.ts

@@ -8,6 +8,7 @@ export * from './stage.js';
 export * from './stageOutputResolver.js';
 export * from './sparql/index.js';
 export * from './distribution/index.js';
+export * from './provenance/index.js';
 export * from './writer/index.js';
 export * from './plugin/namespaceNormalization.js';
 export * from './plugin/provenance.js';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,YAAY,CAAC;AAC3B,cAAc,0BAA0B,CAAC;AACzC,cAAc,mBAAmB,CAAC;AAClC,cAAc,yBAAyB,CAAC;AACxC,cAAc,mBAAmB,CAAC;AAClC,cAAc,oCAAoC,CAAC;AACnD,cAAc,wBAAwB,CAAC;AACvC,cAAc,oCAAoC,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,YAAY,CAAC;AAC3B,cAAc,0BAA0B,CAAC;AACzC,cAAc,mBAAmB,CAAC;AAClC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,mBAAmB,CAAC;AAClC,cAAc,oCAAoC,CAAC;AACnD,cAAc,wBAAwB,CAAC;AACvC,cAAc,oCAAoC,CAAC"}

package/dist/index.js

@@ -8,6 +8,7 @@ export * from './stage.js';
 export * from './stageOutputResolver.js';
 export * from './sparql/index.js';
 export * from './distribution/index.js';
+export * from './provenance/index.js';
 export * from './writer/index.js';
 export * from './plugin/namespaceNormalization.js';
 export * from './plugin/provenance.js';

package/dist/provenance/index.d.ts

@@ -0,0 +1,4 @@
+export { sourceFingerprint } from './sourceFingerprint.js';
+export { shouldReprocess } from './reprocessDecision.js';
+export type { ProcessingRecord, ChangeKey } from './record.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/provenance/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/provenance/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,YAAY,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC"}

package/dist/provenance/index.js

@@ -0,0 +1,2 @@
+export { sourceFingerprint } from './sourceFingerprint.js';
+export { shouldReprocess } from './reprocessDecision.js';

package/dist/provenance/record.d.ts

@@ -0,0 +1,35 @@
+/**
+ * The per-dataset processing memory the pipeline keeps to decide whether a
+ * dataset can be skipped on the next run.
+ *
+ * Both change fields ({@link sourceFingerprint} and {@link pipelineVersion})
+ * are opaque strings, compared only for equality – never parsed or ordered.
+ */
+export interface ProcessingRecord {
+    /**
+     * The source-change fingerprint at the time of processing (see
+     * `sourceFingerprint`), or `null` when none could be established (e.g. a live
+     * SPARQL endpoint). Derived automatically from observed source metadata, not
+     * a declared version. A `null` fingerprint never compares equal, so the
+     * dataset is always reprocessed.
+     */
+    sourceFingerprint: string | null;
+    /**
+     * The consumer-declared pipeline version under which the dataset was
+     * processed. Kept separate from {@link sourceFingerprint}, never combined
+     * into a single fingerprint: the data side is observed, the logic side is
+     * intentionally declared.
+     */
+    pipelineVersion: string;
+    /** ISO timestamp of when the record was written. */
+    generatedAt: string;
+    /**
+     * Whether processing succeeded. Recorded so a dataset that failed but whose
+     * source is unchanged is skipped on subsequent runs rather than re-imported
+     * every run; it is retried at the next source change or version rotation.
+     */
+    status: 'success' | 'failed';
+}
+/** The two fields the skip rule compares for equality. */
+export type ChangeKey = Pick<ProcessingRecord, 'sourceFingerprint' | 'pipelineVersion'>;
+//# sourceMappingURL=record.d.ts.map

package/dist/provenance/record.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"record.d.ts","sourceRoot":"","sources":["../../src/provenance/record.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;;;;OAMG;IACH,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC;;;;;OAKG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB,oDAAoD;IACpD,WAAW,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,MAAM,EAAE,SAAS,GAAG,QAAQ,CAAC;CAC9B;AAED,0DAA0D;AAC1D,MAAM,MAAM,SAAS,GAAG,IAAI,CAC1B,gBAAgB,EAChB,mBAAmB,GAAG,iBAAiB,CACxC,CAAC"}

package/dist/provenance/record.js

@@ -0,0 +1 @@
+export {};

package/dist/provenance/reprocessDecision.d.ts

@@ -0,0 +1,21 @@
+import type { ChangeKey, ProcessingRecord } from './record.js';
+/**
+ * Decide whether a dataset must be reprocessed, given its current change
+ * fields and the record from the last run (or `null` if it has never been
+ * processed).
+ *
+ * The rule is pure equality on the two change fields:
+ *
+ * ```
+ * skip iff  stored !== null
+ *      AND  current.sourceFingerprint === stored.sourceFingerprint
+ *      AND  current.pipelineVersion === stored.pipelineVersion
+ * ```
+ *
+ * Equality, never ordering – any opaque version representation works, a
+ * rollback to identical logic correctly skips, and a partial run resumes
+ * cleanly. A `null` source fingerprint never compares equal, so a dataset with
+ * no establishable fingerprint is always reprocessed.
+ */
+export declare function shouldReprocess(current: ChangeKey, stored: ProcessingRecord | null): boolean;
+//# sourceMappingURL=reprocessDecision.d.ts.map

package/dist/provenance/reprocessDecision.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reprocessDecision.d.ts","sourceRoot":"","sources":["../../src/provenance/reprocessDecision.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAE/D;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,eAAe,CAC7B,OAAO,EAAE,SAAS,EAClB,MAAM,EAAE,gBAAgB,GAAG,IAAI,GAC9B,OAAO,CAOT"}

package/dist/provenance/reprocessDecision.js

@@ -0,0 +1,30 @@
+/**
+ * Decide whether a dataset must be reprocessed, given its current change
+ * fields and the record from the last run (or `null` if it has never been
+ * processed).
+ *
+ * The rule is pure equality on the two change fields:
+ *
+ * ```
+ * skip iff  stored !== null
+ *      AND  current.sourceFingerprint === stored.sourceFingerprint
+ *      AND  current.pipelineVersion === stored.pipelineVersion
+ * ```
+ *
+ * Equality, never ordering – any opaque version representation works, a
+ * rollback to identical logic correctly skips, and a partial run resumes
+ * cleanly. A `null` source fingerprint never compares equal, so a dataset with
+ * no establishable fingerprint is always reprocessed.
+ */
+export function shouldReprocess(current, stored) {
+    if (stored === null)
+        return true;
+    // A null source fingerprint never compares equal, even to a stored null.
+    if (current.sourceFingerprint === null)
+        return true;
+    if (current.sourceFingerprint !== stored.sourceFingerprint)
+        return true;
+    if (current.pipelineVersion !== stored.pipelineVersion)
+        return true;
+    return false;
+}

package/dist/provenance/sourceFingerprint.d.ts

@@ -0,0 +1,29 @@
+import type { Distribution } from '@lde/dataset';
+import { type ProbeResultType } from '@lde/distribution-probe';
+/**
+ * Derive a cheap source-change fingerprint for a distribution from metadata the
+ * probe already collected – no body download.
+ *
+ * For a data dump the fingerprint combines the most recent of the register’s
+ * declared `dct:modified` and the artifact’s HTTP `Last-Modified` with the
+ * artifact’s byte size (the probe’s `Content-Length`, falling back to the
+ * register’s declared `dcat:byteSize`). Taking the maximum date errs toward
+ * reprocessing rather than serving stale output, and mirrors the change signal
+ * {@link ImportResolver} computes for the downloader so the skip layer and the
+ * download/import layer agree.
+ *
+ * The returned string is opaque: it is only ever compared for equality, never
+ * parsed or ordered.
+ *
+ * Returns `null` when no fingerprint can be established – a live SPARQL
+ * endpoint (which exposes none), or a distribution whose probe yielded neither
+ * a usable date nor a byte size. A `null` fingerprint never compares equal, so
+ * those distributions are always reprocessed.
+ *
+ * Robust against malformed third-party metadata: an unparseable HTTP
+ * `Last-Modified` or `dct:modified` (an Invalid Date) and a non-numeric
+ * `Content-Length` (`NaN`) are both treated as absent rather than producing a
+ * throw or an unstable fingerprint.
+ */
+export declare function sourceFingerprint(distribution: Distribution, probeResult: ProbeResultType): string | null;
+//# sourceMappingURL=sourceFingerprint.d.ts.map

package/dist/provenance/sourceFingerprint.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sourceFingerprint.d.ts","sourceRoot":"","sources":["../../src/provenance/sourceFingerprint.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACjD,OAAO,EAEL,KAAK,eAAe,EACrB,MAAM,yBAAyB,CAAC;AAEjC;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAgB,iBAAiB,CAC/B,YAAY,EAAE,YAAY,EAC1B,WAAW,EAAE,eAAe,GAC3B,MAAM,GAAG,IAAI,CAwBf"}

package/dist/provenance/sourceFingerprint.js

@@ -0,0 +1,57 @@
+import { DataDumpProbeResult, } from '@lde/distribution-probe';
+/**
+ * Derive a cheap source-change fingerprint for a distribution from metadata the
+ * probe already collected – no body download.
+ *
+ * For a data dump the fingerprint combines the most recent of the register’s
+ * declared `dct:modified` and the artifact’s HTTP `Last-Modified` with the
+ * artifact’s byte size (the probe’s `Content-Length`, falling back to the
+ * register’s declared `dcat:byteSize`). Taking the maximum date errs toward
+ * reprocessing rather than serving stale output, and mirrors the change signal
+ * {@link ImportResolver} computes for the downloader so the skip layer and the
+ * download/import layer agree.
+ *
+ * The returned string is opaque: it is only ever compared for equality, never
+ * parsed or ordered.
+ *
+ * Returns `null` when no fingerprint can be established – a live SPARQL
+ * endpoint (which exposes none), or a distribution whose probe yielded neither
+ * a usable date nor a byte size. A `null` fingerprint never compares equal, so
+ * those distributions are always reprocessed.
+ *
+ * Robust against malformed third-party metadata: an unparseable HTTP
+ * `Last-Modified` or `dct:modified` (an Invalid Date) and a non-numeric
+ * `Content-Length` (`NaN`) are both treated as absent rather than producing a
+ * throw or an unstable fingerprint.
+ */
+export function sourceFingerprint(distribution, probeResult) {
+    if (distribution.isSparql()) {
+        return null;
+    }
+    const modifiedDate = mostRecent(distribution.lastModified, probeResult instanceof DataDumpProbeResult
+        ? (probeResult.lastModified ?? undefined)
+        : undefined);
+    const probeSize = probeResult instanceof DataDumpProbeResult ? probeResult.contentSize : null;
+    const byteSize = probeSize !== null && !Number.isNaN(probeSize)
+        ? probeSize
+        : distribution.byteSize;
+    if (modifiedDate === undefined && byteSize === undefined) {
+        return null;
+    }
+    return `${modifiedDate?.toISOString() ?? ''}|${byteSize ?? ''}`;
+}
+/**
+ * The most recent of the given dates, ignoring `undefined` and Invalid Dates.
+ * Filtering invalid dates keeps a malformed metadata value from being selected
+ * (which would make `toISOString` throw) and from sticking ahead of a valid
+ * date – `validDate > invalidDate` is `number > NaN`, i.e. always `false`.
+ */
+function mostRecent(...dates) {
+    return dates.reduce((latest, date) => {
+        if (date === undefined || Number.isNaN(date.valueOf()))
+            return latest;
+        if (latest === undefined || date > latest)
+            return date;
+        return latest;
+    }, undefined);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lde/pipeline",
-  "version": "0.30.10",
+  "version": "0.30.11",
   "repository": {
     "url": "git+https://github.com/ldelements/lde.git",
     "directory": "packages/pipeline"