@directory-builder/core 0.1.3 → 0.1.5

package/README.md

@@ -11,11 +11,16 @@ artefacts — no engine code:
 config/
   federation.ttl        # the decisions: sources + facts, target schemas,
                         # field mappings, match/merge/resolve rules
-  match-knowledge.ttl   # curated owl:sameAs pairs
+  match-knowledge.ttl   # optional: curated owl:sameAs pairs
 sources/<name>/
   fetch.js              # how to fetch this source
   clean.sparql          # how to clean its lifted RDF
   static/               # the data itself, for static-file sources
+registry/
+  identity.ttl          # engine-maintained: minted entity IRIs and their
+                        # source members — accumulated state, commit it
+  history.ttl           # engine-maintained: append-only log of identity
+                        # events (mint, member joined)
 webapp/
   content/about.md      # optional: the webapp's About page prose
   exporters/<name>.js   # optional: output adapters the webapp loads at runtime
@@ -64,7 +69,12 @@ fixture.
 
 Each source's `fetch.js` is invoked as `node fetch.js <outDir> <fetchUrl-or-staticDir>
 <runParamsJson>` — the JSON holds all `:hasRunParam` values grouped by name;
-each fetcher picks the parameters it needs.
+each fetcher picks the parameters it needs. For static-file sources `fetch.js`
+is optional: without one, the default fetch copies `sources/<name>/static/`
+verbatim. `clean.sparql` is likewise optional when the source maps a field to
+`schema:identifier`: the engine derives a default clean from that mapping —
+skolemise on the identifier field, copy the scalar fields — and puts the
+resolved query on record under `data/pipeline/default-clean-queries/`.
 
 A source declared with `:enabled false` stays in the config but is skipped by
 the engines and hidden from the webapp's Sources page — e.g. while its files
@@ -73,6 +83,19 @@ aren't available yet.
 Engines journal their executed steps as p-plan RDF (`data/ingest/ingest-log.ttl`,
 `data/pipeline/federate-log.ttl`) — evidence of what ran, not a plan.
 
+Minting is write-once: the match step keeps an identity registry
+(`registry/identity.ttl`, created on the first run) assigning each source
+record to its minted entity IRI. A cluster with a known member reuses the
+registered IRI, so identities survive re-harvests however membership evolves;
+only unseen entities mint fresh. Alongside it, `registry/history.ttl` is an
+append-only log of identity events (mint, member joined) grouped under a
+timestamped `:Revision` node per changing run — the registry's provenance,
+where the snapshot in `identity.ttl` came from. Both are written only when
+something changes, so a no-op harvest leaves them — and their git diff —
+untouched, and the revision counter only advances when identity actually moves.
+Unlike `data/`, the registry is accumulated state, not derived output — commit
+it, and review its diff after each harvest.
+
 ## Run the webapp
 
 The webapp ships with the package; it fetches an instance's `config/` +

package/example/config/federation.ttl

@@ -2,13 +2,12 @@
 @prefix rdfs:   <http://www.w3.org/2000/01/rdf-schema#> .
 @prefix schema: <http://schema.org/> .
 @prefix foaf:   <http://xmlns.com/foaf/0.1/> .
-@prefix prov:   <http://www.w3.org/ns/prov#> .
 @prefix skos:   <http://www.w3.org/2004/02/skos/core#> .
 @prefix ft:     <http://publications.europa.eu/resource/authority/file-type/> .
 
 # A minimal single-entity federation: two directories of libraries, merged into
 # one set of schema:Organization records. Config declares decisions only —
-# sources and their facts, target schema, mappings, match/merge/resolve rules.
+# sources and their facts, target schema, mappings, match/resolve rules.
 # The engines own the step shape (fetch → lift, clean → map → match → merge →
 # resolve) and resolve all file paths by convention from the source names.
 # Add target schemas / mappings / match rules to model more entity types
@@ -18,7 +17,6 @@
     :hasSource       :cityopenSource, :civichubSource ;
     :hasTargetSchema :organisationSchema ;
     :hasMatchRule    :organisationMatch ;
-    :hasMergeRule    :merge ;
     :hasResolveRule  :resolve .
 
 # ---- Target schema ------------------------------------------------------
@@ -87,7 +85,6 @@
 :cityopen-mapping a :Mapping ;
     :fromSource  :cityopenSource ;
     :toTarget    :organisationSchema ;
-    :sourceGraph <urn:source:cityopen> ;
     :hasFieldMapping
         [ :from :co-id         ; :to :t-identifier    ] ,
         [ :from :co-name       ; :to :t-name          ] ,
@@ -101,7 +98,6 @@
 :civichub-mapping a :Mapping ;
     :fromSource  :civichubSource ;
     :toTarget    :organisationSchema ;
-    :sourceGraph <urn:source:civichub> ;
     :hasFieldMapping
         [ :from :ch-uid         ; :to :t-identifier    ] ,
         [ :from :ch-bezeichnung ; :to :t-name          ] ,
@@ -124,11 +120,6 @@
     :minScore             0.5 ;
     :hasWeightedCriterion [ :on schema:name ; :weight 1.0 ] .
 
-# ---- Merge --------------------------------------------------------------
-
-:merge a :MergeRule ;
-    :originPredicate prov:wasDerivedFrom .
-
 # ---- Resolve ------------------------------------------------------------
 # One value per predicate per merged record; alphabeticFirst is deterministic.
 

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "@directory-builder/core",
-  "version": "0.1.3",
+  "version": "0.1.5",
   "description": "Use-case-agnostic engine for config-driven federation pipelines",
   "author": "Civic Data Lab",
   "repository": "github:foederierter-datenpool/directory-builder-core",
   "license": "MIT",
   "type": "module",
   "scripts": {
-    "test": "node --test",
+    "test": "node --test 'test/*.test.js'",
     "example": "cd example && node ../bin/cli.js",
     "webapp": "vite webapp",
     "webapp:build": "vite build webapp"

package/src/clean/default.sparql

@@ -0,0 +1,19 @@
+# Default clean, applied when a source ships no clean.sparql: skolemise each
+# record from its identifier field (the source field mapped to the target
+# schema's schema:identifier) into a stable cdp:__name__-<id> IRI, copy its
+# scalar fields verbatim, and tag the source. The engine fills __source__,
+# __name__ and __idPath__ from federation.ttl and puts the resolved query on
+# record under data/pipeline/default-clean-queries/.
+
+PREFIX xyz: <http://sparql.xyz/facade-x/data/>
+PREFIX cdp: <https://civic-data.de/pipeline#>
+
+CONSTRUCT {
+    ?record cdp:fromSource __source__ ;
+            ?p ?o .
+} WHERE {
+    ?node xyz:__idPath__ ?id ;
+          ?p ?o .
+    FILTER(isLiteral(?o))
+    BIND(IRI(CONCAT(STR(cdp:), "__name__-", STR(?id))) AS ?record)
+}

package/src/pipeline/federate.js

@@ -25,18 +25,20 @@ const df = DataFactory
 export async function federate(root = process.cwd()) {
     const abs = (p) => path.join(root, p)
     const federationTtl = fs.readFileSync(abs(PATHS.federation), "utf8")
-    const defStore = storeFromTurtles([federationTtl, fs.readFileSync(abs(PATHS.matchKnowledge), "utf8")])
-    const sources = enabledSources(parseTtl(federationTtl))
+    // match-knowledge.ttl (curated owl:sameAs pairs) is optional — no file, no manual matches.
+    const matchKnowledge = fs.existsSync(abs(PATHS.matchKnowledge)) ? [fs.readFileSync(abs(PATHS.matchKnowledge), "utf8")] : []
+    const defStore = storeFromTurtles([federationTtl, ...matchKnowledge])
+    const federationQuads = parseTtl(federationTtl)
+    const sources = enabledSources(federationQuads)
 
     const store = newStore()
     const journal = stepJournal()
-    const ctx = { store, defStore, abs }
+    const ctx = { store, defStore, abs, quads: federationQuads }
 
     const cleanSteps = []
     for (const src of sources) {
-        const name = sourceName(src)
-        cleanSteps.push(await journal.step("clean", { source: src, after: [stepIri("lift", name)] },
-            () => runClean(ctx, name)))
+        cleanSteps.push(await journal.step("clean", { source: src, after: [stepIri("lift", sourceName(src))] },
+            () => runClean(ctx, src)))
     }
 
     // Load each source's cleaned TTL into its own graph — plain mechanics, not a
@@ -56,7 +58,7 @@ export async function federate(root = process.cwd()) {
         await writeTurtleFile(abs(PATHS.mapped), mappedQuads, { ...COMMON_PREFIXES, cdp: CDP })
         console.log(`map: wrote ${mappedQuads.length} triples → ${PATHS.mapped}`)
     })
-    const matchStep   = await journal.step("match",   { after: [mapStep] },   () => runMatch(ctx, PATHS.matches))
+    const matchStep   = await journal.step("match",   { after: [mapStep] },   () => runMatch(ctx, PATHS.matches, PATHS.registry, PATHS.registryHistory))
     const mergeStep   = await journal.step("merge",   { after: [matchStep] }, () => runMerge(ctx, PATHS.merged, PATHS.provenance))
     await journal.step("resolve", { after: [mergeStep] }, () => runResolve(ctx, PATHS.final))
 

package/src/pipeline/steps/clean.js

@@ -1,13 +1,21 @@
 import { sparqlConstruct, storeFromTurtles } from "@foerderfunke/sem-ops-utils"
+import { CDP, identifierFieldPath, PATHS, sourceName } from "../../utils.js"
 import { writeTurtleFile } from "../write-turtle.js"
-import { CDP, PATHS } from "../../utils.js"
 import path from "path"
 import fs from "fs"
 
+// The default clean ships with the engine, like the lift queries.
+const DEFAULT_CLEAN = path.join(import.meta.dirname, "../../clean/default.sparql")
+
 // Clean step: the source's clean.sparql reshapes its lifted RDF into
 // federation subjects (xyz:/cdp: vocabulary only — schema: enters at map).
-export const runClean = async ({ abs }, name) => {
-    const cleanQuery = fs.readFileSync(abs(PATHS.cleanQuery(name)), "utf8")
+// clean.sparql is optional when the source maps a field to schema:identifier:
+// the engine then derives the default clean from that mapping.
+export const runClean = async ({ abs, quads }, sourceIri) => {
+    const name = sourceName(sourceIri)
+    const cleanQuery = fs.existsSync(abs(PATHS.cleanQuery(name)))
+        ? fs.readFileSync(abs(PATHS.cleanQuery(name)), "utf8")
+        : defaultClean({ abs, quads }, sourceIri, name)
     const inDir = PATHS.lifted(name)
     const outPath = PATHS.cleaned(name)
     // Run CONSTRUCT per file so each lifted TTL stays isolated in its
@@ -25,3 +33,18 @@ export const runClean = async ({ abs }, name) => {
         cdp: CDP,
     })
 }
+
+// No clean.sparql given: resolve the engine's default template with the
+// source's identifier field as skolem key, and put the applied query on
+// record under data/ — no silent fallbacks.
+const defaultClean = ({ abs, quads }, sourceIri, name) => {
+    const idPath = identifierFieldPath(quads, sourceIri)
+    if (!idPath) throw new Error(`${PATHS.cleanQuery(name)} missing and no schema:identifier mapping to derive the default clean from`)
+    const query = fs.readFileSync(DEFAULT_CLEAN, "utf8")
+        .replaceAll("__source__", `<${sourceIri}>`).replaceAll("__name__", name).replaceAll("__idPath__", idPath)
+    const outPath = abs(PATHS.defaultCleanQuery(name))
+    fs.mkdirSync(path.dirname(outPath), { recursive: true })
+    fs.writeFileSync(outPath, query)
+    console.log(`clean  ${name} default (id field: ${idPath}) → ${PATHS.defaultCleanQuery(name)}`)
+    return query
+}

package/src/pipeline/steps/fetch.js

@@ -6,7 +6,8 @@ import fs from "fs"
 // Fetch step: run the source's fetch.js. Live sources pass their :fetchUrl;
 // static-file sources pass the absolute static dir instead — the script gets
 // whichever applies, plus the federation's run params as one JSON argument.
-// Returns the harvest record for the ingest log.
+// fetch.js is optional for static sources: without it, the default fetch
+// copies static/ verbatim. Returns the harvest record for the ingest log.
 export const runFetch = ({ abs, root }, { name, fetchUrl, paramsJson }) => {
     const outDir = PATHS.raw(name)
     const origin = fetchUrl ?? abs(PATHS.staticDir(name))
@@ -14,7 +15,9 @@ export const runFetch = ({ abs, root }, { name, fetchUrl, paramsJson }) => {
     // Clear any prior output first, so changed run params (or changed records) can't leave stale files behind
     fs.rmSync(abs(outDir), { recursive: true, force: true })
     fs.mkdirSync(abs(outDir), { recursive: true })
-    run("node", [abs(PATHS.fetchScript(name)), abs(outDir), origin, paramsJson])
+    const script = abs(PATHS.fetchScript(name))
+    if (fs.existsSync(script)) run("node", [script, abs(outDir), origin, paramsJson])
+    else localCopyFallback({ name, fetchUrl, origin, outDir: abs(outDir) })
     const harvest = { time: new Date().toISOString() }
     // Static sources have no live harvest — record the files' git commit
     // time instead (the freshness the Sources page shows for them).
@@ -24,3 +27,10 @@ export const runFetch = ({ abs, root }, { name, fetchUrl, paramsJson }) => {
     } catch { /* not committed yet / no git → omit */ }
     return harvest
 }
+
+// Fallback when a source ships no dedicated fetch.js: static sources get
+// their static/ dir copied verbatim; live sources have no fallback yet.
+const localCopyFallback = ({ name, fetchUrl, origin, outDir }) => {
+    if (fetchUrl) throw new Error(`${PATHS.fetchScript(name)} missing (no default fetch for live sources yet)`)
+    fs.cpSync(origin, outDir, { recursive: true })
+}

package/src/pipeline/steps/map.js

@@ -1,5 +1,5 @@
 import { sparqlInsertDelete, sparqlSelect } from "@foerderfunke/sem-ops-utils"
-import { buildPrefixBlock, CDP, PATHS, shrink, sourceName } from "../../utils.js"
+import { buildPrefixBlock, CDP, PATHS, shrink, sourceGraph, sourceName } from "../../utils.js"
 import { DataFactory } from "n3"
 import path from "path"
 import fs from "fs"
@@ -88,10 +88,9 @@ ${insertBlock}
 export const runMap = async ({ store, defStore, abs }, queriesDir) => {
     const mappings = await sparqlSelect(`
         PREFIX : <${CDP}>
-        SELECT ?mapping ?source ?sourceGraph ?target ?targetClass WHERE {
+        SELECT ?mapping ?source ?target ?targetClass WHERE {
             ?mapping a :Mapping ;
                 :fromSource ?source .
-            OPTIONAL { ?mapping :sourceGraph ?sourceGraph }
             OPTIONAL { ?mapping :toTarget ?target }
             OPTIONAL { ?mapping :toTarget/:targetClass ?targetClass }
         } ORDER BY ?mapping`, [defStore])
@@ -108,9 +107,11 @@ export const runMap = async ({ store, defStore, abs }, queriesDir) => {
                 OPTIONAL { ?parent :hasSubField ?src . ?parent :fieldPath ?parentPath }
             }`, [defStore])
 
-        if (directRows.length && m.sourceGraph) {
+        if (directRows.length) {
             const localName = m.mapping.split("#").pop()
-            const query = buildDirectInsert(m, directRows)
+            // The mapping's source graph follows by convention from :fromSource —
+            // the load step names it the same way.
+            const query = buildDirectInsert({ ...m, sourceGraph: sourceGraph(sourceName(m.source)) }, directRows)
             const queryPath = abs(path.join(queriesDir, `${localName}.sparql`))
             fs.mkdirSync(path.dirname(queryPath), { recursive: true })
             fs.writeFileSync(queryPath, query)
@@ -139,9 +140,9 @@ export const runMap = async ({ store, defStore, abs }, queriesDir) => {
     // the merge step rewrites them to the minted cluster IRIs.
     const linkRows = await sparqlSelect(`
         PREFIX : <${CDP}>
-        SELECT ?mapping ?sourceGraph ?fromSchema ?sourcePredicate ?targetPredicate ?toSchema WHERE {
+        SELECT ?mapping ?source ?fromSchema ?sourcePredicate ?targetPredicate ?toSchema WHERE {
             ?mapping a :Mapping ;
-                :sourceGraph     ?sourceGraph ;
+                :fromSource      ?source ;
                 :toTarget        ?fromSchema ;
                 :hasRelationship ?rel .
             ?rel :sourcePredicate ?sourcePredicate ;
@@ -160,7 +161,7 @@ INSERT {
         ?from ${short(rel.targetPredicate)} ?to .
     }
 } WHERE {
-    GRAPH <${rel.sourceGraph}> {
+    GRAPH <${sourceGraph(sourceName(rel.source))}> {
         ?from ${short(rel.sourcePredicate)} ?to ;
               cdp:targetSchema ${short(rel.fromSchema)} .
         ?to cdp:targetSchema ${short(rel.toSchema)} .

package/src/pipeline/steps/match.js

@@ -1,10 +1,11 @@
 import { sparqlSelect } from "@foerderfunke/sem-ops-utils"
 import { COMMON_PREFIXES, writeTurtleFile } from "../write-turtle.js"
 import { MAPPED_GRAPH } from "./map.js"
-import { CDP } from "../../utils.js"
+import { CDP, parseTtl, shrink } from "../../utils.js"
 import { token_set_ratio } from "fuzzball"
 import { DataFactory } from "n3"
 import { createHash } from "crypto"
+import fs from "fs"
 
 const df = DataFactory
 
@@ -22,7 +23,28 @@ const MATCH_CLUSTER = df.namedNode(CDP + "MatchCluster")
 const SIMILARITY_ALGORITHM = "token_set_ratio"
 const similarity = (a, b) => token_set_ratio(a ?? "", b ?? "") / 100
 
-export const runMatch = async ({ store, defStore, abs }, outPath) => {
+export const runMatch = async ({ store, defStore, abs }, outPath, registryPath, historyPath) => {
+    // The identity registry (minted IRI :hasMember source IRI, one assignment
+    // per member) makes minting write-once: an entity's IRI is computed at
+    // most once — at first sight — recorded here, and afterwards only looked
+    // up, so membership can change without identity churn. Instance state to
+    // commit, neither config nor regenerable data; empty on a fresh instance.
+    const registry = new Map() // member source IRI → minted IRI
+    if (fs.existsSync(abs(registryPath))) {
+        for (const q of parseTtl(fs.readFileSync(abs(registryPath), "utf8"))) {
+            if (q.predicate.value === HAS_MEMBER.value) registry.set(q.object.value, q.subject.value)
+        }
+    }
+    const reserved = new Set(registry.values()) // every IRI ever minted — never mint one again
+    const known    = new Set(registry.keys())   // members assigned in a prior run
+    const taken = new Set()                     // minted IRIs claimed by a cluster this run
+    let reusedCount = 0, mintedCount = 0
+    // Identity events this run, appended to history.ttl (the registry's
+    // provenance): when each entity was first minted, gained a member, or
+    // absorbed/split off another. Append-only and written only when non-empty,
+    // so a no-change harvest leaves the file — and its git diff — untouched.
+    const events = []
+
     // One match rule per target schema; each rule scores its own fields, mints
     // with its own prefix, and clusters only subjects of its :targetClass.
     const rules = await sparqlSelect(`
@@ -169,8 +191,39 @@ export const runMatch = async ({ store, defStore, abs }, outPath) => {
         let multiSource = 0
         const clusterIriByRoot = new Map()
         for (const members of clusterMembers) {
-            const id = createHash("sha1").update(members.join("|")).digest("hex").slice(0, 12)
-            const minted = df.namedNode(namespace + mintedPrefix + id)
+            // Reconcile against the registry: any member already known → its
+            // entity exists, reuse the IRI (clusters come largest-first, so on
+            // a split the larger fragment keeps the identity). Only unseen
+            // entities mint, seeded by their smallest member at mint time — a
+            // one-time uniqueness seed, not a content address: the registry
+            // pins the IRI afterwards, however membership evolves.
+            const prior = [...new Set(members.map(m => registry.get(m)).filter(Boolean))].sort()
+            const free = prior.filter(iri => !taken.has(iri))
+            let minted
+            // TODO: merge and split (prior carrying ≥2 IRIs in the reuse branch,
+            // or any prior in the mint branch) are reconciled correctly — a
+            // survivor keeps the IRI — but their history events (:Merged /
+            // :Split) and the tombstone they imply (the retired IRI preserved
+            // with :isReplacedBy, rather than silently vanishing from
+            // identity.ttl) are their own rung. For now they only warn.
+            if (free.length) {
+                minted = df.namedNode(free[0])
+                reusedCount++
+                const joined = members.filter(m => !known.has(m))
+                if (joined.length) events.push({ type: "MemberJoined", entity: free[0], member: joined })
+                if (prior.length > 1) console.warn(`match: clusters merged (${prior.join(" + ")}) — keeping ${free[0]}`)
+            } else {
+                if (prior.length) console.warn(`match: cluster split off ${prior.join(", ")} — minting fresh`)
+                let id = createHash("sha1").update(members[0]).digest("hex").slice(0, 12)
+                // Seed collision (e.g. a split remainder re-hashing its old anchor): re-hash until free.
+                while (taken.has(namespace + mintedPrefix + id) || reserved.has(namespace + mintedPrefix + id))
+                    id = createHash("sha1").update(id).digest("hex").slice(0, 12)
+                minted = df.namedNode(namespace + mintedPrefix + id)
+                mintedCount++
+                if (!prior.length) events.push({ type: "Minted", entity: minted.value, member: members })
+            }
+            taken.add(minted.value)
+            for (const m of members) registry.set(m, minted.value)
             clusterIriByRoot.set(find(members[0]), minted)
             if (members.length > 1) multiSource++
             store.addQuad(df.quad(minted, RDF_TYPE, MATCH_CLUSTER, MATCH_GRAPH))
@@ -209,4 +262,42 @@ export const runMatch = async ({ store, defStore, abs }, outPath) => {
     const matchQuads = store.getQuads(null, null, null, MATCH_GRAPH)
     await writeTurtleFile(abs(outPath), matchQuads, { cdp: CDP, cdf: rules[0].ns, ...COMMON_PREFIXES })
     console.log(`match: wrote cluster log → ${outPath}`)
+
+    await writeTurtleFile(abs(registryPath), [...registry].map(([member, minted]) =>
+        df.quad(df.namedNode(minted), HAS_MEMBER, df.namedNode(member))), { cdp: CDP, cdf: rules[0].ns })
+    console.log(`match: identity registry ${reusedCount} reused, ${mintedCount} minted → ${registryPath}`)
+
+    // Append this run's events to the history (the registry's provenance) as
+    // one :Revision node carrying the timestamp, with each event hung off it as
+    // a nested [entity ; members] binding under a type predicate (cdp:minted /
+    // cdp:memberJoined). Revisions count only changing runs — a no-op harvest
+    // appends nothing — so the next number is one past the highest on file. The
+    // whole block is one append, so the named :Revision and its fresh blank
+    // nodes never collide with earlier revisions when the file is re-parsed.
+    if (events.length) {
+        const prefixes = { cdp: CDP, cdf: rules[0].ns }
+        const sh   = (iri) => shrink(iri, prefixes)
+        const list = (arr) => arr.map(sh).join(", ")
+        const existing = fs.existsSync(abs(historyPath)) ? fs.readFileSync(abs(historyPath), "utf8") : ""
+        const rev = Math.max(0, ...[...existing.matchAll(/revision-(\d+)/g)].map(m => +m[1])) + 1
+
+        const byPredicate = new Map() // cdp:minted / cdp:memberJoined → binding strings
+        for (const e of events) {
+            const pred = "cdp:" + e.type[0].toLowerCase() + e.type.slice(1)
+            if (!byPredicate.has(pred)) byPredicate.set(pred, [])
+            byPredicate.get(pred).push(`[ cdp:entity ${sh(e.entity)} ; cdp:member ${list(e.member)} ]`)
+        }
+        const props = [...byPredicate].map(([pred, bindings]) =>
+            `    ${pred}\n        ${bindings.join(" ,\n        ")}`).join(" ;\n")
+        const block = `cdp:revision-${rev} a cdp:Revision ; prov:atTime "${new Date().toISOString()}"^^xsd:dateTime ;\n${props} .\n`
+
+        const header = `@prefix cdp:  <${CDP}> .
+@prefix cdf:  <${rules[0].ns}> .
+@prefix prov: <http://www.w3.org/ns/prov#> .
+@prefix xsd:  <http://www.w3.org/2001/XMLSchema#> .
+
+`
+        fs.appendFileSync(abs(historyPath), (existing ? "\n" : header) + block)
+        console.log(`match: revision ${rev} — ${events.length} identity event(s) → ${historyPath}`)
+    }
 }

package/src/pipeline/steps/merge.js

@@ -11,22 +11,22 @@ export const MERGED_GRAPH = df.namedNode("urn:merged")
 
 const RDF_REIFIES = df.namedNode("http://www.w3.org/1999/02/22-rdf-syntax-ns#reifies")
 
+// Engine invariant, mirrored by the webapp's loadMerge: each derivation's
+// origin hangs off its reifier via prov:wasDerivedFrom.
+const PROV_DERIVED_FROM = df.namedNode("http://www.w3.org/ns/prov#wasDerivedFrom")
+
 export const runMerge = async ({ store, defStore, abs }, outPath, provOutPath) => {
     const [cfg] = await sparqlSelect(`
         PREFIX : <${CDP}>
-        SELECT ?ns ?originPred WHERE {
-            ?match a :MatchRule ; :targetNamespace ?ns .
-            ?merge a :MergeRule ; :originPredicate ?originPred .
-        }`, [defStore])
-    if (!cfg) throw new Error(":MergeRule / :MatchRule config missing in federation.ttl")
-    const { ns: namespace, originPred } = cfg
+        SELECT ?ns WHERE { ?match a :MatchRule ; :targetNamespace ?ns . }`, [defStore])
+    if (!cfg) throw new Error(":MatchRule config missing in federation.ttl")
+    const namespace = cfg.ns
 
     const memberQuads = store.getQuads(null, HAS_MEMBER, null, MATCH_GRAPH)
     const mintedFor = new Map()
     for (const mq of memberQuads) mintedFor.set(mq.object.value, mq.subject)
 
     const fedQuads = store.getQuads(null, null, null, MAPPED_GRAPH)
-    const originPredNode = df.namedNode(originPred)
     const provQuads = []
     for (const qu of fedQuads) {
         const minted = mintedFor.get(qu.subject.value)
@@ -43,7 +43,7 @@ export const runMerge = async ({ store, defStore, abs }, outPath, provOutPath) =
         // per-derivation metadata (time, confidence) has a home when needed.
         const reifier = df.blankNode()
         provQuads.push(df.quad(reifier, RDF_REIFIES, df.quad(minted, qu.predicate, object)))
-        provQuads.push(df.quad(reifier, originPredNode, qu.subject))
+        provQuads.push(df.quad(reifier, PROV_DERIVED_FROM, qu.subject))
     }
 
     const mergedQuads = store.getQuads(null, null, null, MERGED_GRAPH)

package/src/pipeline/steps/resolve.js

@@ -26,11 +26,12 @@ export const runResolve = async ({ store, defStore, abs }, outPath) => {
     const [cfg] = await sparqlSelect(`
         PREFIX : <${CDP}>
         SELECT ?strategy ?ns WHERE {
-            ?resolve a :ResolveRule ; :defaultStrategy ?strategy .
-            ?match   a :MatchRule   ; :targetNamespace ?ns .
+            ?match a :MatchRule ; :targetNamespace ?ns .
+            OPTIONAL { ?resolve a :ResolveRule ; :defaultStrategy ?strategy }
         }`, [defStore])
-    if (!cfg) throw new Error(":ResolveRule config missing in federation.ttl")
-    const defaultPick = lookupStrategy(cfg.strategy)
+    if (!cfg) throw new Error(":MatchRule config missing in federation.ttl")
+    // No :ResolveRule (or none with a :defaultStrategy) → alphabeticFirst.
+    const defaultPick = lookupStrategy(cfg.strategy ?? `${CDP}alphabeticFirst`)
 
     const overrideRows = await sparqlSelect(`
         PREFIX : <${CDP}>

package/src/utils.js

@@ -56,6 +56,8 @@ export const LIFTED_FORMAT = "http://publications.europa.eu/resource/authority/f
 export const PATHS = {
     federation:     "config/federation.ttl",
     matchKnowledge: "config/match-knowledge.ttl",
+    registry:       "registry/identity.ttl",
+    registryHistory: "registry/history.ttl",
     about:          "webapp/content/about.md",
     query:          "webapp/content/query.sparql",
     fetchScript: (name) => `sources/${name}/fetch.js`,
@@ -69,6 +71,7 @@ export const PATHS = {
     ingestLog:   "data/ingest/ingest-log.ttl",
     federateLog: "data/pipeline/federate-log.ttl",
     mappingQueries: "data/pipeline/direct-mapping-queries/",
+    defaultCleanQuery: (name) => `data/pipeline/default-clean-queries/${name}.sparql`,
     mapped:      "data/pipeline/mapped.ttl",
     matches:     "data/pipeline/matches.ttl",
     merged:      "data/pipeline/merged.ttl",
@@ -115,6 +118,18 @@ export const enabledSources = (quads) => {
     return objectsOf(quads, `${CDP}hasSource`).filter((iri) => !disabled.has(iri))
 }
 
+// The source's skolem key for the default clean: the :fieldPath of the source
+// field whose mapping points at the target field with :targetPredicate
+// schema:identifier. Undefined when the source declares no such mapping.
+export const identifierFieldPath = (quads, sourceIri) => {
+    const o = (s, p) => quads.filter((q) => q.subject.value === s && q.predicate.value === `${CDP}${p}`).map((q) => q.object.value)
+    for (const m of quads.filter((q) => q.predicate.value === `${CDP}fromSource` && q.object.value === sourceIri).map((q) => q.subject.value)) {
+        for (const fm of o(m, "hasFieldMapping")) {
+            if (o(o(fm, "to")[0], "targetPredicate")[0] === "http://schema.org/identifier") return o(o(fm, "from")[0], "fieldPath")[0]
+        }
+    }
+}
+
 // Set of subjects typed `rdf:type typeIri`. Iteration order = encounter order.
 export function subjectsOfType(quads, typeIri) {
     const out = new Set()

package/src/validate.js

@@ -1,5 +1,5 @@
 import { buildValidator, turtleToDataset } from "@foerderfunke/sem-ops-utils"
-import { CDP, objectsOf, parseTtl, PATHS, shrink, sourceName } from "./utils.js"
+import { CDP, identifierFieldPath, objectsOf, parseTtl, PATHS, shrink, sourceName } from "./utils.js"
 import path from "path"
 import fs from "fs"
 
@@ -19,23 +19,27 @@ export async function validate(root = process.cwd()) {
     return (await Promise.all(checks.map((check) => check(ctx)))).flat()
 }
 
-// Every :hasSource in federation.ttl has its sources/<name>/ folder with
-// fetch.js + clean.sparql - and no folder exists that the federation doesn't
-// declare. Checks all declared sources, enabled or not: folder presence is a
-// repo-layout contract.
+// Every :hasSource in federation.ttl has what its engine steps need: a
+// fetch.js or static/ to default to, a clean.sparql or a schema:identifier
+// mapping to derive the default clean from - and no sources/ folder exists
+// that the federation doesn't declare. Checks all declared sources, enabled
+// or not: folder presence is a repo-layout contract.
 function sourcesFoldersInSync({ abs, quads }) {
-    const declared = objectsOf(quads, `${CDP}hasSource`).map(sourceName)
+    const declared = objectsOf(quads, `${CDP}hasSource`)
     const problems = []
-    for (const name of declared) {
-        for (const file of [PATHS.fetchScript(name), PATHS.cleanQuery(name)]) {
-            if (!fs.existsSync(abs(file))) problems.push(`${file} missing`)
-        }
+    for (const iri of declared) {
+        const name = sourceName(iri)
+        if (![PATHS.fetchScript(name), PATHS.staticDir(name)].some((f) => fs.existsSync(abs(f))))
+            problems.push(`${PATHS.fetchScript(name)} missing and no ${PATHS.staticDir(name)} to default to`)
+        if (!fs.existsSync(abs(PATHS.cleanQuery(name))) && !identifierFieldPath(quads, iri))
+            problems.push(`${PATHS.cleanQuery(name)} missing and no schema:identifier mapping to derive the default clean from`)
     }
+    const declaredNames = declared.map(sourceName)
     const folders = fs.existsSync(abs("sources"))
         ? fs.readdirSync(abs("sources"), { withFileTypes: true }).filter((d) => d.isDirectory()).map((d) => d.name)
         : []
     for (const name of folders) {
-        if (!declared.includes(name)) problems.push(`sources/${name}/ has no :hasSource declaration in ${PATHS.federation}`)
+        if (!declaredNames.includes(name)) problems.push(`sources/${name}/ has no :hasSource declaration in ${PATHS.federation}`)
     }
     return problems
 }

package/test/helpers/instance.js

@@ -0,0 +1,24 @@
+import { PATHS } from "@directory-builder/core/utils"
+import path from "path"
+import fs from "fs"
+
+// SPARQL Anything cache shared with example/ — a fixture's tools/ symlinks
+// here, so test runs never re-download the jar.
+const TOOLS_CACHE = path.join(import.meta.dirname, "../../example/tools")
+
+// Materialize an in-test instance definition (federation.ttl string + records
+// per source) into test/tmp/<name>/ — a real instance folder the engines run
+// against, wiped at setup and left in place afterwards for inspection.
+export const makeInstance = (name, { federation, sources }) => {
+    const root = path.join(import.meta.dirname, "../tmp", name)
+    fs.rmSync(root, { recursive: true, force: true })
+    fs.mkdirSync(path.join(root, "config"), { recursive: true })
+    fs.writeFileSync(path.join(root, PATHS.federation), federation)
+    for (const [source, records] of Object.entries(sources)) {
+        fs.mkdirSync(path.join(root, PATHS.staticDir(source)), { recursive: true })
+        fs.writeFileSync(path.join(root, PATHS.staticDir(source), "data.json"), JSON.stringify(records, null, 4))
+    }
+    fs.mkdirSync(TOOLS_CACHE, { recursive: true })
+    fs.symlinkSync(TOOLS_CACHE, path.join(root, "tools"))
+    return root
+}

package/test/pipeline.test.js

@@ -0,0 +1,182 @@
+import { CDP, parseTtl, PATHS } from "@directory-builder/core/utils"
+import { Pipeline, validate } from "@directory-builder/core"
+import { makeInstance } from "./helpers/instance.js"
+import assert from "node:assert/strict"
+import { test } from "node:test"
+import path from "path"
+import fs from "fs"
+
+// ---- Shared fixture: the ultra-minimal instance both tests run on ----------
+// federation.ttl + two static JSON sources, nothing else — fetch, clean and
+// resolve all run on engine defaults. The sources share one record by name
+// ("Entry One"), so the pipeline should merge a1+b1 and leave a2 and b2 as
+// their own entities.
+
+const federation = `
+@prefix :       <https://civic-data.de/pipeline#> .
+@prefix schema: <http://schema.org/> .
+@prefix ft:     <http://publications.europa.eu/resource/authority/file-type/> .
+
+:federation a :Federation ;
+    :hasSource :alphaSource, :betaSource .
+
+:thingSchema a :TargetSchema ;
+    :targetClass schema:Thing .
+
+:t-id   a :TargetField ; :targetPredicate schema:identifier .
+:t-name a :TargetField ; :targetPredicate schema:name .
+
+:alphaSource a :Source ; :format ft:JSON .
+:betaSource  a :Source ; :format ft:JSON .
+
+:alpha-id   a :SourceField ; :fieldPath "id" .
+:alpha-name a :SourceField ; :fieldPath "name" .
+:beta-id    a :SourceField ; :fieldPath "id" .
+:beta-label a :SourceField ; :fieldPath "label" .
+
+:alpha-mapping a :Mapping ; :fromSource :alphaSource ; :toTarget :thingSchema ;
+    :hasFieldMapping [ :from :alpha-id ; :to :t-id ] , [ :from :alpha-name ; :to :t-name ] .
+
+:beta-mapping a :Mapping ; :fromSource :betaSource ; :toTarget :thingSchema ;
+    :hasFieldMapping [ :from :beta-id ; :to :t-id ] , [ :from :beta-label ; :to :t-name ] .
+
+:match a :MatchRule ;
+    :forTarget           :thingSchema ;
+    :targetNamespace     "urn:test:" ;
+    :mintedSubjectPrefix "thing-" ;
+    :minScore             1.0 ;
+    :hasWeightedCriterion [ :on schema:name ; :weight 1.0 ] .
+`
+
+const alpha = [
+    { id: "a1", name: "Entry One" },
+    { id: "a2", name: "Entry Two" },
+]
+const beta = [
+    { id: "b1", label: "Entry One" },
+    { id: "b2", label: "Entry Three" },
+]
+
+// The consumer-facing artifact the shared fixture resolves to (both tests).
+const expectedFinal = `@prefix schema: <http://schema.org/>.
+@prefix foaf: <http://xmlns.com/foaf/0.1/>.
+@prefix dct: <http://purl.org/dc/terms/>.
+@prefix cdf: <urn:test:>.
+
+cdf:thing-5a45645edb31 a schema:Thing;
+    schema:name "Entry Two".
+cdf:thing-d1583c098826 a schema:Thing;
+    schema:name "Entry Three".
+cdf:thing-e427416d02ac a schema:Thing;
+    schema:name "Entry One".
+`
+
+// ---- Test 1: the whole pipeline on defaults --------------------------------
+
+test("the tiny fixture validates and runs the whole pipeline on defaults", async () => {
+    const root = makeInstance("tiny", { federation, sources: { alpha, beta } })
+    // the fixture satisfies the instance contract (folders, derivable defaults, shape)
+    assert.deepEqual(await validate(root), [])
+    await new Pipeline({ root }).run()
+    const finalTtl = fs.readFileSync(path.join(root, PATHS.final), "utf8")
+    const final = parseTtl(finalTtl)
+    // match merged a1+b1 on their identical name; a2 and b2 stay their own entities
+    const subjects = new Set(final.map((q) => q.subject.value))
+    assert.equal(subjects.size, 3, "a1+b1 merge, a2 and b2 stay alone")
+    // entity IRIs are minted from the match rule's :targetNamespace + :mintedSubjectPrefix
+    for (const s of subjects) assert.match(s, /^urn:test:thing-/)
+    // map carried both sources' name fields through, resolve kept one value per entity
+    const names = final.filter((q) => q.predicate.value === "http://schema.org/name").map((q) => q.object.value)
+    assert.deepEqual(names.toSorted(), ["Entry One", "Entry Three", "Entry Two"])
+    // and the consumer-facing artifact as a whole
+    assert.equal(finalTtl, expectedFinal)
+})
+
+// ---- Test 2: periodic harvesting & the identity registry -------------------
+
+// The identity registry the first harvest writes: each minted IRI's source
+// members, the write-once record later runs reconcile against.
+const expectedRegistry = `@prefix cdp: <https://civic-data.de/pipeline#>.
+@prefix cdf: <urn:test:>.
+
+cdf:thing-5a45645edb31 cdp:hasMember cdp:alpha-a2.
+cdf:thing-d1583c098826 cdp:hasMember cdp:beta-b2.
+cdf:thing-e427416d02ac cdp:hasMember cdp:alpha-a1, cdp:beta-b1.
+`
+
+// history.ttl events as {type, entity, member[], revision}: each event is a
+// nested [entity ; members] binding hung off its :Revision node under a type
+// predicate (cdp:minted / cdp:memberJoined). Timestamps vary per run, so the
+// test asserts structure, not bytes.
+const RDF_TYPE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"
+const EVENT_PREDS = { minted: "Minted", memberJoined: "MemberJoined" }
+const parseEvents = (ttl) => {
+    const quads = parseTtl(ttl)
+    const events = []
+    for (const [local, type] of Object.entries(EVENT_PREDS)) {
+        for (const q of quads.filter((x) => x.predicate.value === CDP + local)) {
+            const node = q.object.value // the [entity ; members] binding's blank node
+            events.push({
+                type,
+                entity: quads.find((x) => x.subject.value === node && x.predicate.value === CDP + "entity")?.object.value,
+                member: quads.filter((x) => x.subject.value === node && x.predicate.value === CDP + "member")
+                    .map((x) => x.object.value).toSorted(),
+                revision: q.subject.value, // the :Revision node the binding hangs off
+            })
+        }
+    }
+    return events
+}
+const revisionNodes = (ttl) => parseTtl(ttl)
+    .filter((q) => q.predicate.value === RDF_TYPE && q.object.value === CDP + "Revision")
+    .map((q) => q.subject.value).toSorted()
+
+test("harvest rounds keep minted IRIs stable (write-once identity registry)", async () => {
+    const root = makeInstance("harvest", { federation, sources: { alpha, beta } })
+    const pipeline = new Pipeline({ root })
+    const writeSource = (name, records) =>
+        fs.writeFileSync(path.join(root, PATHS.staticDir(name), "data.json"), JSON.stringify(records, null, 4))
+    const artifact = (p) => fs.readFileSync(path.join(root, p), "utf8")
+    const id = (local) => "urn:test:" + local
+    const src = (local) => CDP + local
+
+    // round 1 — the first harvest mints the three identities into the registry,
+    // and opens the history with one Minted event apiece (the genesis record).
+    await pipeline.run()
+    assert.equal(artifact(PATHS.final), expectedFinal)
+    assert.equal(artifact(PATHS.registry), expectedRegistry)
+    const history1 = artifact(PATHS.registryHistory)
+    assert.deepEqual(parseEvents(history1).toSorted((a, b) => a.entity.localeCompare(b.entity)), [
+        { type: "Minted", entity: id("thing-5a45645edb31"), member: [src("alpha-a2")], revision: src("revision-1") },
+        { type: "Minted", entity: id("thing-d1583c098826"), member: [src("beta-b2")], revision: src("revision-1") },
+        { type: "Minted", entity: id("thing-e427416d02ac"), member: [src("alpha-a1"), src("beta-b1")], revision: src("revision-1") },
+    ])
+    assert.deepEqual(revisionNodes(history1), [src("revision-1")], "genesis opens revision 1")
+
+    // round 2 — harmless upstream edit: b2 renames to "Entry Drei", membership
+    // unchanged. The directory carries the new name under the same IRI, and both
+    // registry and history stay byte-identical (a no-change harvest, clean diff).
+    writeSource("beta", [beta[0], { id: "b2", label: "Entry Drei" }])
+    await pipeline.run()
+    const expectedRenamed = expectedFinal.replace(`"Entry Three"`, `"Entry Drei"`)
+    assert.equal(artifact(PATHS.final), expectedRenamed)
+    assert.equal(artifact(PATHS.registry), expectedRegistry)
+    assert.equal(artifact(PATHS.registryHistory), history1, "no event appended for a no-op harvest")
+
+    // round 3 — a new alpha record joins b2's cluster. alpha-a3 sorts before
+    // beta-b2, so a stateless smallest-member seed would re-mint here — only the
+    // registry lookup preserves the identity: the directory is unchanged, the
+    // entity just gained its second member, and history records exactly that.
+    writeSource("alpha", [...alpha, { id: "a3", name: "Entry Drei" }])
+    await pipeline.run()
+    assert.equal(artifact(PATHS.final), expectedRenamed)
+    assert.equal(artifact(PATHS.registry),
+        expectedRegistry.replace("cdp:beta-b2.", "cdp:beta-b2, cdp:alpha-a3."))
+    assert.ok(artifact(PATHS.registryHistory).startsWith(history1), "history only appends, never rewrites")
+    const inRev2 = parseEvents(artifact(PATHS.registryHistory)).filter((e) => e.revision === src("revision-2"))
+    assert.deepEqual(inRev2, [
+        { type: "MemberJoined", entity: id("thing-d1583c098826"), member: [src("alpha-a3")], revision: src("revision-2") },
+    ])
+    assert.deepEqual(revisionNodes(artifact(PATHS.registryHistory)), [src("revision-1"), src("revision-2")],
+        "the changing harvest opens revision 2; the no-op round 2 added none")
+})

package/example/sources/cityopen/fetch.js

@@ -1,14 +0,0 @@
-import path from "path"
-import fs from "fs"
-
-// Static-file source: copy the committed JSON straight into the ingest area.
-// A live source would instead call an API here and write the responses out.
-// argv: [outDir, sourceDir, runParamsJson] — params unused for this static example.
-const OUT_DIR = process.argv[2]
-const SRC_DIR = process.argv[3]
-
-fs.mkdirSync(OUT_DIR, { recursive: true })
-for (const f of fs.readdirSync(SRC_DIR).filter((f) => f.endsWith(".json"))) {
-    fs.copyFileSync(path.join(SRC_DIR, f), path.join(OUT_DIR, f))
-    console.log(`  ${f} → ${OUT_DIR}`)
-}

package/example/sources/civichub/fetch.js

@@ -1,14 +0,0 @@
-import path from "path"
-import fs from "fs"
-
-// Static-file source: copy the committed JSON straight into the ingest area.
-// A live source would instead call an API here and write the responses out.
-// argv: [outDir, sourceDir, runParamsJson] — params unused for this static example.
-const OUT_DIR = process.argv[2]
-const SRC_DIR = process.argv[3]
-
-fs.mkdirSync(OUT_DIR, { recursive: true })
-for (const f of fs.readdirSync(SRC_DIR).filter((f) => f.endsWith(".json"))) {
-    fs.copyFileSync(path.join(SRC_DIR, f), path.join(OUT_DIR, f))
-    console.log(`  ${f} → ${OUT_DIR}`)
-}