@directory-builder/core 0.1.4 → 0.1.5

package/README.md

@@ -16,6 +16,11 @@ sources/<name>/
   fetch.js              # how to fetch this source
   clean.sparql          # how to clean its lifted RDF
   static/               # the data itself, for static-file sources
+registry/
+  identity.ttl          # engine-maintained: minted entity IRIs and their
+                        # source members — accumulated state, commit it
+  history.ttl           # engine-maintained: append-only log of identity
+                        # events (mint, member joined)
 webapp/
   content/about.md      # optional: the webapp's About page prose
   exporters/<name>.js   # optional: output adapters the webapp loads at runtime
@@ -78,6 +83,19 @@ aren't available yet.
 Engines journal their executed steps as p-plan RDF (`data/ingest/ingest-log.ttl`,
 `data/pipeline/federate-log.ttl`) — evidence of what ran, not a plan.
 
+Minting is write-once: the match step keeps an identity registry
+(`registry/identity.ttl`, created on the first run) assigning each source
+record to its minted entity IRI. A cluster with a known member reuses the
+registered IRI, so identities survive re-harvests however membership evolves;
+only unseen entities mint fresh. Alongside it, `registry/history.ttl` is an
+append-only log of identity events (mint, member joined) grouped under a
+timestamped `:Revision` node per changing run — the registry's provenance,
+where the snapshot in `identity.ttl` came from. Both are written only when
+something changes, so a no-op harvest leaves them — and their git diff —
+untouched, and the revision counter only advances when identity actually moves.
+Unlike `data/`, the registry is accumulated state, not derived output — commit
+it, and review its diff after each harvest.
+
 ## Run the webapp
 
 The webapp ships with the package; it fetches an instance's `config/` +

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@directory-builder/core",
-  "version": "0.1.4",
+  "version": "0.1.5",
   "description": "Use-case-agnostic engine for config-driven federation pipelines",
   "author": "Civic Data Lab",
   "repository": "github:foederierter-datenpool/directory-builder-core",

package/src/pipeline/federate.js

@@ -58,7 +58,7 @@ export async function federate(root = process.cwd()) {
         await writeTurtleFile(abs(PATHS.mapped), mappedQuads, { ...COMMON_PREFIXES, cdp: CDP })
         console.log(`map: wrote ${mappedQuads.length} triples → ${PATHS.mapped}`)
     })
-    const matchStep   = await journal.step("match",   { after: [mapStep] },   () => runMatch(ctx, PATHS.matches))
+    const matchStep   = await journal.step("match",   { after: [mapStep] },   () => runMatch(ctx, PATHS.matches, PATHS.registry, PATHS.registryHistory))
     const mergeStep   = await journal.step("merge",   { after: [matchStep] }, () => runMerge(ctx, PATHS.merged, PATHS.provenance))
     await journal.step("resolve", { after: [mergeStep] }, () => runResolve(ctx, PATHS.final))
 

package/src/pipeline/steps/match.js

@@ -1,10 +1,11 @@
 import { sparqlSelect } from "@foerderfunke/sem-ops-utils"
 import { COMMON_PREFIXES, writeTurtleFile } from "../write-turtle.js"
 import { MAPPED_GRAPH } from "./map.js"
-import { CDP } from "../../utils.js"
+import { CDP, parseTtl, shrink } from "../../utils.js"
 import { token_set_ratio } from "fuzzball"
 import { DataFactory } from "n3"
 import { createHash } from "crypto"
+import fs from "fs"
 
 const df = DataFactory
 
@@ -22,7 +23,28 @@ const MATCH_CLUSTER = df.namedNode(CDP + "MatchCluster")
 const SIMILARITY_ALGORITHM = "token_set_ratio"
 const similarity = (a, b) => token_set_ratio(a ?? "", b ?? "") / 100
 
-export const runMatch = async ({ store, defStore, abs }, outPath) => {
+export const runMatch = async ({ store, defStore, abs }, outPath, registryPath, historyPath) => {
+    // The identity registry (minted IRI :hasMember source IRI, one assignment
+    // per member) makes minting write-once: an entity's IRI is computed at
+    // most once — at first sight — recorded here, and afterwards only looked
+    // up, so membership can change without identity churn. Instance state to
+    // commit, neither config nor regenerable data; empty on a fresh instance.
+    const registry = new Map() // member source IRI → minted IRI
+    if (fs.existsSync(abs(registryPath))) {
+        for (const q of parseTtl(fs.readFileSync(abs(registryPath), "utf8"))) {
+            if (q.predicate.value === HAS_MEMBER.value) registry.set(q.object.value, q.subject.value)
+        }
+    }
+    const reserved = new Set(registry.values()) // every IRI ever minted — never mint one again
+    const known    = new Set(registry.keys())   // members assigned in a prior run
+    const taken = new Set()                     // minted IRIs claimed by a cluster this run
+    let reusedCount = 0, mintedCount = 0
+    // Identity events this run, appended to history.ttl (the registry's
+    // provenance): when each entity was first minted, gained a member, or
+    // absorbed/split off another. Append-only and written only when non-empty,
+    // so a no-change harvest leaves the file — and its git diff — untouched.
+    const events = []
+
     // One match rule per target schema; each rule scores its own fields, mints
     // with its own prefix, and clusters only subjects of its :targetClass.
     const rules = await sparqlSelect(`
@@ -169,8 +191,39 @@ export const runMatch = async ({ store, defStore, abs }, outPath) => {
         let multiSource = 0
         const clusterIriByRoot = new Map()
         for (const members of clusterMembers) {
-            const id = createHash("sha1").update(members.join("|")).digest("hex").slice(0, 12)
-            const minted = df.namedNode(namespace + mintedPrefix + id)
+            // Reconcile against the registry: any member already known → its
+            // entity exists, reuse the IRI (clusters come largest-first, so on
+            // a split the larger fragment keeps the identity). Only unseen
+            // entities mint, seeded by their smallest member at mint time — a
+            // one-time uniqueness seed, not a content address: the registry
+            // pins the IRI afterwards, however membership evolves.
+            const prior = [...new Set(members.map(m => registry.get(m)).filter(Boolean))].sort()
+            const free = prior.filter(iri => !taken.has(iri))
+            let minted
+            // TODO: merge and split (prior carrying ≥2 IRIs in the reuse branch,
+            // or any prior in the mint branch) are reconciled correctly — a
+            // survivor keeps the IRI — but their history events (:Merged /
+            // :Split) and the tombstone they imply (the retired IRI preserved
+            // with :isReplacedBy, rather than silently vanishing from
+            // identity.ttl) are their own rung. For now they only warn.
+            if (free.length) {
+                minted = df.namedNode(free[0])
+                reusedCount++
+                const joined = members.filter(m => !known.has(m))
+                if (joined.length) events.push({ type: "MemberJoined", entity: free[0], member: joined })
+                if (prior.length > 1) console.warn(`match: clusters merged (${prior.join(" + ")}) — keeping ${free[0]}`)
+            } else {
+                if (prior.length) console.warn(`match: cluster split off ${prior.join(", ")} — minting fresh`)
+                let id = createHash("sha1").update(members[0]).digest("hex").slice(0, 12)
+                // Seed collision (e.g. a split remainder re-hashing its old anchor): re-hash until free.
+                while (taken.has(namespace + mintedPrefix + id) || reserved.has(namespace + mintedPrefix + id))
+                    id = createHash("sha1").update(id).digest("hex").slice(0, 12)
+                minted = df.namedNode(namespace + mintedPrefix + id)
+                mintedCount++
+                if (!prior.length) events.push({ type: "Minted", entity: minted.value, member: members })
+            }
+            taken.add(minted.value)
+            for (const m of members) registry.set(m, minted.value)
             clusterIriByRoot.set(find(members[0]), minted)
             if (members.length > 1) multiSource++
             store.addQuad(df.quad(minted, RDF_TYPE, MATCH_CLUSTER, MATCH_GRAPH))
@@ -209,4 +262,42 @@ export const runMatch = async ({ store, defStore, abs }, outPath) => {
     const matchQuads = store.getQuads(null, null, null, MATCH_GRAPH)
     await writeTurtleFile(abs(outPath), matchQuads, { cdp: CDP, cdf: rules[0].ns, ...COMMON_PREFIXES })
     console.log(`match: wrote cluster log → ${outPath}`)
+
+    await writeTurtleFile(abs(registryPath), [...registry].map(([member, minted]) =>
+        df.quad(df.namedNode(minted), HAS_MEMBER, df.namedNode(member))), { cdp: CDP, cdf: rules[0].ns })
+    console.log(`match: identity registry ${reusedCount} reused, ${mintedCount} minted → ${registryPath}`)
+
+    // Append this run's events to the history (the registry's provenance) as
+    // one :Revision node carrying the timestamp, with each event hung off it as
+    // a nested [entity ; members] binding under a type predicate (cdp:minted /
+    // cdp:memberJoined). Revisions count only changing runs — a no-op harvest
+    // appends nothing — so the next number is one past the highest on file. The
+    // whole block is one append, so the named :Revision and its fresh blank
+    // nodes never collide with earlier revisions when the file is re-parsed.
+    if (events.length) {
+        const prefixes = { cdp: CDP, cdf: rules[0].ns }
+        const sh   = (iri) => shrink(iri, prefixes)
+        const list = (arr) => arr.map(sh).join(", ")
+        const existing = fs.existsSync(abs(historyPath)) ? fs.readFileSync(abs(historyPath), "utf8") : ""
+        const rev = Math.max(0, ...[...existing.matchAll(/revision-(\d+)/g)].map(m => +m[1])) + 1
+
+        const byPredicate = new Map() // cdp:minted / cdp:memberJoined → binding strings
+        for (const e of events) {
+            const pred = "cdp:" + e.type[0].toLowerCase() + e.type.slice(1)
+            if (!byPredicate.has(pred)) byPredicate.set(pred, [])
+            byPredicate.get(pred).push(`[ cdp:entity ${sh(e.entity)} ; cdp:member ${list(e.member)} ]`)
+        }
+        const props = [...byPredicate].map(([pred, bindings]) =>
+            `    ${pred}\n        ${bindings.join(" ,\n        ")}`).join(" ;\n")
+        const block = `cdp:revision-${rev} a cdp:Revision ; prov:atTime "${new Date().toISOString()}"^^xsd:dateTime ;\n${props} .\n`
+
+        const header = `@prefix cdp:  <${CDP}> .
+@prefix cdf:  <${rules[0].ns}> .
+@prefix prov: <http://www.w3.org/ns/prov#> .
+@prefix xsd:  <http://www.w3.org/2001/XMLSchema#> .
+
+`
+        fs.appendFileSync(abs(historyPath), (existing ? "\n" : header) + block)
+        console.log(`match: revision ${rev} — ${events.length} identity event(s) → ${historyPath}`)
+    }
 }

package/src/utils.js

@@ -56,6 +56,8 @@ export const LIFTED_FORMAT = "http://publications.europa.eu/resource/authority/f
 export const PATHS = {
     federation:     "config/federation.ttl",
     matchKnowledge: "config/match-knowledge.ttl",
+    registry:       "registry/identity.ttl",
+    registryHistory: "registry/history.ttl",
     about:          "webapp/content/about.md",
     query:          "webapp/content/query.sparql",
     fetchScript: (name) => `sources/${name}/fetch.js`,

package/test/pipeline.test.js

@@ -1,4 +1,4 @@
-import { parseTtl, PATHS } from "@directory-builder/core/utils"
+import { CDP, parseTtl, PATHS } from "@directory-builder/core/utils"
 import { Pipeline, validate } from "@directory-builder/core"
 import { makeInstance } from "./helpers/instance.js"
 import assert from "node:assert/strict"
@@ -6,10 +6,11 @@ import { test } from "node:test"
 import path from "path"
 import fs from "fs"
 
-// The ultra-minimal instance: federation.ttl + two static JSON sources,
-// nothing else — fetch, clean and resolve all run on engine defaults. The
-// sources share one record by name ("Entry One"), so the pipeline should
-// merge a1+b1 and leave a2 and b2 as their own entities.
+// ---- Shared fixture: the ultra-minimal instance both tests run on ----------
+// federation.ttl + two static JSON sources, nothing else — fetch, clean and
+// resolve all run on engine defaults. The sources share one record by name
+// ("Entry One"), so the pipeline should merge a1+b1 and leave a2 and b2 as
+// their own entities.
 
 const federation = `
 @prefix :       <https://civic-data.de/pipeline#> .
@@ -56,8 +57,7 @@ const beta = [
     { id: "b2", label: "Entry Three" },
 ]
 
-const root = makeInstance("tiny", { federation, sources: { alpha, beta } })
-
+// The consumer-facing artifact the shared fixture resolves to (both tests).
 const expectedFinal = `@prefix schema: <http://schema.org/>.
 @prefix foaf: <http://xmlns.com/foaf/0.1/>.
 @prefix dct: <http://purl.org/dc/terms/>.
@@ -65,13 +65,16 @@ const expectedFinal = `@prefix schema: <http://schema.org/>.
 
 cdf:thing-5a45645edb31 a schema:Thing;
     schema:name "Entry Two".
-cdf:thing-616feb993283 a schema:Thing;
-    schema:name "Entry One".
 cdf:thing-d1583c098826 a schema:Thing;
     schema:name "Entry Three".
+cdf:thing-e427416d02ac a schema:Thing;
+    schema:name "Entry One".
 `
 
+// ---- Test 1: the whole pipeline on defaults --------------------------------
+
 test("the tiny fixture validates and runs the whole pipeline on defaults", async () => {
+    const root = makeInstance("tiny", { federation, sources: { alpha, beta } })
     // the fixture satisfies the instance contract (folders, derivable defaults, shape)
     assert.deepEqual(await validate(root), [])
     await new Pipeline({ root }).run()
@@ -88,3 +91,92 @@ test("the tiny fixture validates and runs the whole pipeline on defaults", async
     // and the consumer-facing artifact as a whole
     assert.equal(finalTtl, expectedFinal)
 })
+
+// ---- Test 2: periodic harvesting & the identity registry -------------------
+
+// The identity registry the first harvest writes: each minted IRI's source
+// members, the write-once record later runs reconcile against.
+const expectedRegistry = `@prefix cdp: <https://civic-data.de/pipeline#>.
+@prefix cdf: <urn:test:>.
+
+cdf:thing-5a45645edb31 cdp:hasMember cdp:alpha-a2.
+cdf:thing-d1583c098826 cdp:hasMember cdp:beta-b2.
+cdf:thing-e427416d02ac cdp:hasMember cdp:alpha-a1, cdp:beta-b1.
+`
+
+// history.ttl events as {type, entity, member[], revision}: each event is a
+// nested [entity ; members] binding hung off its :Revision node under a type
+// predicate (cdp:minted / cdp:memberJoined). Timestamps vary per run, so the
+// test asserts structure, not bytes.
+const RDF_TYPE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"
+const EVENT_PREDS = { minted: "Minted", memberJoined: "MemberJoined" }
+const parseEvents = (ttl) => {
+    const quads = parseTtl(ttl)
+    const events = []
+    for (const [local, type] of Object.entries(EVENT_PREDS)) {
+        for (const q of quads.filter((x) => x.predicate.value === CDP + local)) {
+            const node = q.object.value // the [entity ; members] binding's blank node
+            events.push({
+                type,
+                entity: quads.find((x) => x.subject.value === node && x.predicate.value === CDP + "entity")?.object.value,
+                member: quads.filter((x) => x.subject.value === node && x.predicate.value === CDP + "member")
+                    .map((x) => x.object.value).toSorted(),
+                revision: q.subject.value, // the :Revision node the binding hangs off
+            })
+        }
+    }
+    return events
+}
+const revisionNodes = (ttl) => parseTtl(ttl)
+    .filter((q) => q.predicate.value === RDF_TYPE && q.object.value === CDP + "Revision")
+    .map((q) => q.subject.value).toSorted()
+
+test("harvest rounds keep minted IRIs stable (write-once identity registry)", async () => {
+    const root = makeInstance("harvest", { federation, sources: { alpha, beta } })
+    const pipeline = new Pipeline({ root })
+    const writeSource = (name, records) =>
+        fs.writeFileSync(path.join(root, PATHS.staticDir(name), "data.json"), JSON.stringify(records, null, 4))
+    const artifact = (p) => fs.readFileSync(path.join(root, p), "utf8")
+    const id = (local) => "urn:test:" + local
+    const src = (local) => CDP + local
+
+    // round 1 — the first harvest mints the three identities into the registry,
+    // and opens the history with one Minted event apiece (the genesis record).
+    await pipeline.run()
+    assert.equal(artifact(PATHS.final), expectedFinal)
+    assert.equal(artifact(PATHS.registry), expectedRegistry)
+    const history1 = artifact(PATHS.registryHistory)
+    assert.deepEqual(parseEvents(history1).toSorted((a, b) => a.entity.localeCompare(b.entity)), [
+        { type: "Minted", entity: id("thing-5a45645edb31"), member: [src("alpha-a2")], revision: src("revision-1") },
+        { type: "Minted", entity: id("thing-d1583c098826"), member: [src("beta-b2")], revision: src("revision-1") },
+        { type: "Minted", entity: id("thing-e427416d02ac"), member: [src("alpha-a1"), src("beta-b1")], revision: src("revision-1") },
+    ])
+    assert.deepEqual(revisionNodes(history1), [src("revision-1")], "genesis opens revision 1")
+
+    // round 2 — harmless upstream edit: b2 renames to "Entry Drei", membership
+    // unchanged. The directory carries the new name under the same IRI, and both
+    // registry and history stay byte-identical (a no-change harvest, clean diff).
+    writeSource("beta", [beta[0], { id: "b2", label: "Entry Drei" }])
+    await pipeline.run()
+    const expectedRenamed = expectedFinal.replace(`"Entry Three"`, `"Entry Drei"`)
+    assert.equal(artifact(PATHS.final), expectedRenamed)
+    assert.equal(artifact(PATHS.registry), expectedRegistry)
+    assert.equal(artifact(PATHS.registryHistory), history1, "no event appended for a no-op harvest")
+
+    // round 3 — a new alpha record joins b2's cluster. alpha-a3 sorts before
+    // beta-b2, so a stateless smallest-member seed would re-mint here — only the
+    // registry lookup preserves the identity: the directory is unchanged, the
+    // entity just gained its second member, and history records exactly that.
+    writeSource("alpha", [...alpha, { id: "a3", name: "Entry Drei" }])
+    await pipeline.run()
+    assert.equal(artifact(PATHS.final), expectedRenamed)
+    assert.equal(artifact(PATHS.registry),
+        expectedRegistry.replace("cdp:beta-b2.", "cdp:beta-b2, cdp:alpha-a3."))
+    assert.ok(artifact(PATHS.registryHistory).startsWith(history1), "history only appends, never rewrites")
+    const inRev2 = parseEvents(artifact(PATHS.registryHistory)).filter((e) => e.revision === src("revision-2"))
+    assert.deepEqual(inRev2, [
+        { type: "MemberJoined", entity: id("thing-d1583c098826"), member: [src("alpha-a3")], revision: src("revision-2") },
+    ])
+    assert.deepEqual(revisionNodes(artifact(PATHS.registryHistory)), [src("revision-1"), src("revision-2")],
+        "the changing harvest opens revision 2; the no-op round 2 added none")
+})