@directory-builder/core 0.1.3 → 0.1.4

package/README.md

@@ -11,7 +11,7 @@ artefacts — no engine code:
 config/
   federation.ttl        # the decisions: sources + facts, target schemas,
                         # field mappings, match/merge/resolve rules
-  match-knowledge.ttl   # curated owl:sameAs pairs
+  match-knowledge.ttl   # optional: curated owl:sameAs pairs
 sources/<name>/
   fetch.js              # how to fetch this source
   clean.sparql          # how to clean its lifted RDF
@@ -64,7 +64,12 @@ fixture.
 
 Each source's `fetch.js` is invoked as `node fetch.js <outDir> <fetchUrl-or-staticDir>
 <runParamsJson>` — the JSON holds all `:hasRunParam` values grouped by name;
-each fetcher picks the parameters it needs.
+each fetcher picks the parameters it needs. For static-file sources `fetch.js`
+is optional: without one, the default fetch copies `sources/<name>/static/`
+verbatim. `clean.sparql` is likewise optional when the source maps a field to
+`schema:identifier`: the engine derives a default clean from that mapping —
+skolemise on the identifier field, copy the scalar fields — and puts the
+resolved query on record under `data/pipeline/default-clean-queries/`.
 
 A source declared with `:enabled false` stays in the config but is skipped by
 the engines and hidden from the webapp's Sources page — e.g. while its files

package/example/config/federation.ttl

@@ -2,13 +2,12 @@
 @prefix rdfs:   <http://www.w3.org/2000/01/rdf-schema#> .
 @prefix schema: <http://schema.org/> .
 @prefix foaf:   <http://xmlns.com/foaf/0.1/> .
-@prefix prov:   <http://www.w3.org/ns/prov#> .
 @prefix skos:   <http://www.w3.org/2004/02/skos/core#> .
 @prefix ft:     <http://publications.europa.eu/resource/authority/file-type/> .
 
 # A minimal single-entity federation: two directories of libraries, merged into
 # one set of schema:Organization records. Config declares decisions only —
-# sources and their facts, target schema, mappings, match/merge/resolve rules.
+# sources and their facts, target schema, mappings, match/resolve rules.
 # The engines own the step shape (fetch → lift, clean → map → match → merge →
 # resolve) and resolve all file paths by convention from the source names.
 # Add target schemas / mappings / match rules to model more entity types
@@ -18,7 +17,6 @@
     :hasSource       :cityopenSource, :civichubSource ;
     :hasTargetSchema :organisationSchema ;
     :hasMatchRule    :organisationMatch ;
-    :hasMergeRule    :merge ;
     :hasResolveRule  :resolve .
 
 # ---- Target schema ------------------------------------------------------
@@ -87,7 +85,6 @@
 :cityopen-mapping a :Mapping ;
     :fromSource  :cityopenSource ;
     :toTarget    :organisationSchema ;
-    :sourceGraph <urn:source:cityopen> ;
     :hasFieldMapping
         [ :from :co-id         ; :to :t-identifier    ] ,
         [ :from :co-name       ; :to :t-name          ] ,
@@ -101,7 +98,6 @@
 :civichub-mapping a :Mapping ;
     :fromSource  :civichubSource ;
     :toTarget    :organisationSchema ;
-    :sourceGraph <urn:source:civichub> ;
     :hasFieldMapping
         [ :from :ch-uid         ; :to :t-identifier    ] ,
         [ :from :ch-bezeichnung ; :to :t-name          ] ,
@@ -124,11 +120,6 @@
     :minScore             0.5 ;
     :hasWeightedCriterion [ :on schema:name ; :weight 1.0 ] .
 
-# ---- Merge --------------------------------------------------------------
-
-:merge a :MergeRule ;
-    :originPredicate prov:wasDerivedFrom .
-
 # ---- Resolve ------------------------------------------------------------
 # One value per predicate per merged record; alphabeticFirst is deterministic.
 

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "@directory-builder/core",
-  "version": "0.1.3",
+  "version": "0.1.4",
   "description": "Use-case-agnostic engine for config-driven federation pipelines",
   "author": "Civic Data Lab",
   "repository": "github:foederierter-datenpool/directory-builder-core",
   "license": "MIT",
   "type": "module",
   "scripts": {
-    "test": "node --test",
+    "test": "node --test 'test/*.test.js'",
     "example": "cd example && node ../bin/cli.js",
     "webapp": "vite webapp",
     "webapp:build": "vite build webapp"

package/src/clean/default.sparql

@@ -0,0 +1,19 @@
+# Default clean, applied when a source ships no clean.sparql: skolemise each
+# record from its identifier field (the source field mapped to the target
+# schema's schema:identifier) into a stable cdp:__name__-<id> IRI, copy its
+# scalar fields verbatim, and tag the source. The engine fills __source__,
+# __name__ and __idPath__ from federation.ttl and puts the resolved query on
+# record under data/pipeline/default-clean-queries/.
+
+PREFIX xyz: <http://sparql.xyz/facade-x/data/>
+PREFIX cdp: <https://civic-data.de/pipeline#>
+
+CONSTRUCT {
+    ?record cdp:fromSource __source__ ;
+            ?p ?o .
+} WHERE {
+    ?node xyz:__idPath__ ?id ;
+          ?p ?o .
+    FILTER(isLiteral(?o))
+    BIND(IRI(CONCAT(STR(cdp:), "__name__-", STR(?id))) AS ?record)
+}

package/src/pipeline/federate.js

@@ -25,18 +25,20 @@ const df = DataFactory
 export async function federate(root = process.cwd()) {
     const abs = (p) => path.join(root, p)
     const federationTtl = fs.readFileSync(abs(PATHS.federation), "utf8")
-    const defStore = storeFromTurtles([federationTtl, fs.readFileSync(abs(PATHS.matchKnowledge), "utf8")])
-    const sources = enabledSources(parseTtl(federationTtl))
+    // match-knowledge.ttl (curated owl:sameAs pairs) is optional — no file, no manual matches.
+    const matchKnowledge = fs.existsSync(abs(PATHS.matchKnowledge)) ? [fs.readFileSync(abs(PATHS.matchKnowledge), "utf8")] : []
+    const defStore = storeFromTurtles([federationTtl, ...matchKnowledge])
+    const federationQuads = parseTtl(federationTtl)
+    const sources = enabledSources(federationQuads)
 
     const store = newStore()
     const journal = stepJournal()
-    const ctx = { store, defStore, abs }
+    const ctx = { store, defStore, abs, quads: federationQuads }
 
     const cleanSteps = []
     for (const src of sources) {
-        const name = sourceName(src)
-        cleanSteps.push(await journal.step("clean", { source: src, after: [stepIri("lift", name)] },
-            () => runClean(ctx, name)))
+        cleanSteps.push(await journal.step("clean", { source: src, after: [stepIri("lift", sourceName(src))] },
+            () => runClean(ctx, src)))
     }
 
     // Load each source's cleaned TTL into its own graph — plain mechanics, not a

package/src/pipeline/steps/clean.js

@@ -1,13 +1,21 @@
 import { sparqlConstruct, storeFromTurtles } from "@foerderfunke/sem-ops-utils"
+import { CDP, identifierFieldPath, PATHS, sourceName } from "../../utils.js"
 import { writeTurtleFile } from "../write-turtle.js"
-import { CDP, PATHS } from "../../utils.js"
 import path from "path"
 import fs from "fs"
 
+// The default clean ships with the engine, like the lift queries.
+const DEFAULT_CLEAN = path.join(import.meta.dirname, "../../clean/default.sparql")
+
 // Clean step: the source's clean.sparql reshapes its lifted RDF into
 // federation subjects (xyz:/cdp: vocabulary only — schema: enters at map).
-export const runClean = async ({ abs }, name) => {
-    const cleanQuery = fs.readFileSync(abs(PATHS.cleanQuery(name)), "utf8")
+// clean.sparql is optional when the source maps a field to schema:identifier:
+// the engine then derives the default clean from that mapping.
+export const runClean = async ({ abs, quads }, sourceIri) => {
+    const name = sourceName(sourceIri)
+    const cleanQuery = fs.existsSync(abs(PATHS.cleanQuery(name)))
+        ? fs.readFileSync(abs(PATHS.cleanQuery(name)), "utf8")
+        : defaultClean({ abs, quads }, sourceIri, name)
     const inDir = PATHS.lifted(name)
     const outPath = PATHS.cleaned(name)
     // Run CONSTRUCT per file so each lifted TTL stays isolated in its
@@ -25,3 +33,18 @@ export const runClean = async ({ abs }, name) => {
         cdp: CDP,
     })
 }
+
+// No clean.sparql given: resolve the engine's default template with the
+// source's identifier field as skolem key, and put the applied query on
+// record under data/ — no silent fallbacks.
+const defaultClean = ({ abs, quads }, sourceIri, name) => {
+    const idPath = identifierFieldPath(quads, sourceIri)
+    if (!idPath) throw new Error(`${PATHS.cleanQuery(name)} missing and no schema:identifier mapping to derive the default clean from`)
+    const query = fs.readFileSync(DEFAULT_CLEAN, "utf8")
+        .replaceAll("__source__", `<${sourceIri}>`).replaceAll("__name__", name).replaceAll("__idPath__", idPath)
+    const outPath = abs(PATHS.defaultCleanQuery(name))
+    fs.mkdirSync(path.dirname(outPath), { recursive: true })
+    fs.writeFileSync(outPath, query)
+    console.log(`clean  ${name} default (id field: ${idPath}) → ${PATHS.defaultCleanQuery(name)}`)
+    return query
+}

package/src/pipeline/steps/fetch.js

@@ -6,7 +6,8 @@ import fs from "fs"
 // Fetch step: run the source's fetch.js. Live sources pass their :fetchUrl;
 // static-file sources pass the absolute static dir instead — the script gets
 // whichever applies, plus the federation's run params as one JSON argument.
-// Returns the harvest record for the ingest log.
+// fetch.js is optional for static sources: without it, the default fetch
+// copies static/ verbatim. Returns the harvest record for the ingest log.
 export const runFetch = ({ abs, root }, { name, fetchUrl, paramsJson }) => {
     const outDir = PATHS.raw(name)
     const origin = fetchUrl ?? abs(PATHS.staticDir(name))
@@ -14,7 +15,9 @@ export const runFetch = ({ abs, root }, { name, fetchUrl, paramsJson }) => {
     // Clear any prior output first, so changed run params (or changed records) can't leave stale files behind
     fs.rmSync(abs(outDir), { recursive: true, force: true })
     fs.mkdirSync(abs(outDir), { recursive: true })
-    run("node", [abs(PATHS.fetchScript(name)), abs(outDir), origin, paramsJson])
+    const script = abs(PATHS.fetchScript(name))
+    if (fs.existsSync(script)) run("node", [script, abs(outDir), origin, paramsJson])
+    else localCopyFallback({ name, fetchUrl, origin, outDir: abs(outDir) })
     const harvest = { time: new Date().toISOString() }
     // Static sources have no live harvest — record the files' git commit
     // time instead (the freshness the Sources page shows for them).
@@ -24,3 +27,10 @@ export const runFetch = ({ abs, root }, { name, fetchUrl, paramsJson }) => {
     } catch { /* not committed yet / no git → omit */ }
     return harvest
 }
+
+// Fallback when a source ships no dedicated fetch.js: static sources get
+// their static/ dir copied verbatim; live sources have no fallback yet.
+const localCopyFallback = ({ name, fetchUrl, origin, outDir }) => {
+    if (fetchUrl) throw new Error(`${PATHS.fetchScript(name)} missing (no default fetch for live sources yet)`)
+    fs.cpSync(origin, outDir, { recursive: true })
+}

package/src/pipeline/steps/map.js

@@ -1,5 +1,5 @@
 import { sparqlInsertDelete, sparqlSelect } from "@foerderfunke/sem-ops-utils"
-import { buildPrefixBlock, CDP, PATHS, shrink, sourceName } from "../../utils.js"
+import { buildPrefixBlock, CDP, PATHS, shrink, sourceGraph, sourceName } from "../../utils.js"
 import { DataFactory } from "n3"
 import path from "path"
 import fs from "fs"
@@ -88,10 +88,9 @@ ${insertBlock}
 export const runMap = async ({ store, defStore, abs }, queriesDir) => {
     const mappings = await sparqlSelect(`
         PREFIX : <${CDP}>
-        SELECT ?mapping ?source ?sourceGraph ?target ?targetClass WHERE {
+        SELECT ?mapping ?source ?target ?targetClass WHERE {
             ?mapping a :Mapping ;
                 :fromSource ?source .
-            OPTIONAL { ?mapping :sourceGraph ?sourceGraph }
             OPTIONAL { ?mapping :toTarget ?target }
             OPTIONAL { ?mapping :toTarget/:targetClass ?targetClass }
         } ORDER BY ?mapping`, [defStore])
@@ -108,9 +107,11 @@ export const runMap = async ({ store, defStore, abs }, queriesDir) => {
                 OPTIONAL { ?parent :hasSubField ?src . ?parent :fieldPath ?parentPath }
             }`, [defStore])
 
-        if (directRows.length && m.sourceGraph) {
+        if (directRows.length) {
             const localName = m.mapping.split("#").pop()
-            const query = buildDirectInsert(m, directRows)
+            // The mapping's source graph follows by convention from :fromSource —
+            // the load step names it the same way.
+            const query = buildDirectInsert({ ...m, sourceGraph: sourceGraph(sourceName(m.source)) }, directRows)
             const queryPath = abs(path.join(queriesDir, `${localName}.sparql`))
             fs.mkdirSync(path.dirname(queryPath), { recursive: true })
             fs.writeFileSync(queryPath, query)
@@ -139,9 +140,9 @@ export const runMap = async ({ store, defStore, abs }, queriesDir) => {
     // the merge step rewrites them to the minted cluster IRIs.
     const linkRows = await sparqlSelect(`
         PREFIX : <${CDP}>
-        SELECT ?mapping ?sourceGraph ?fromSchema ?sourcePredicate ?targetPredicate ?toSchema WHERE {
+        SELECT ?mapping ?source ?fromSchema ?sourcePredicate ?targetPredicate ?toSchema WHERE {
             ?mapping a :Mapping ;
-                :sourceGraph     ?sourceGraph ;
+                :fromSource      ?source ;
                 :toTarget        ?fromSchema ;
                 :hasRelationship ?rel .
             ?rel :sourcePredicate ?sourcePredicate ;
@@ -160,7 +161,7 @@ INSERT {
         ?from ${short(rel.targetPredicate)} ?to .
     }
 } WHERE {
-    GRAPH <${rel.sourceGraph}> {
+    GRAPH <${sourceGraph(sourceName(rel.source))}> {
         ?from ${short(rel.sourcePredicate)} ?to ;
               cdp:targetSchema ${short(rel.fromSchema)} .
         ?to cdp:targetSchema ${short(rel.toSchema)} .

package/src/pipeline/steps/merge.js

@@ -11,22 +11,22 @@ export const MERGED_GRAPH = df.namedNode("urn:merged")
 
 const RDF_REIFIES = df.namedNode("http://www.w3.org/1999/02/22-rdf-syntax-ns#reifies")
 
+// Engine invariant, mirrored by the webapp's loadMerge: each derivation's
+// origin hangs off its reifier via prov:wasDerivedFrom.
+const PROV_DERIVED_FROM = df.namedNode("http://www.w3.org/ns/prov#wasDerivedFrom")
+
 export const runMerge = async ({ store, defStore, abs }, outPath, provOutPath) => {
     const [cfg] = await sparqlSelect(`
         PREFIX : <${CDP}>
-        SELECT ?ns ?originPred WHERE {
-            ?match a :MatchRule ; :targetNamespace ?ns .
-            ?merge a :MergeRule ; :originPredicate ?originPred .
-        }`, [defStore])
-    if (!cfg) throw new Error(":MergeRule / :MatchRule config missing in federation.ttl")
-    const { ns: namespace, originPred } = cfg
+        SELECT ?ns WHERE { ?match a :MatchRule ; :targetNamespace ?ns . }`, [defStore])
+    if (!cfg) throw new Error(":MatchRule config missing in federation.ttl")
+    const namespace = cfg.ns
 
     const memberQuads = store.getQuads(null, HAS_MEMBER, null, MATCH_GRAPH)
     const mintedFor = new Map()
     for (const mq of memberQuads) mintedFor.set(mq.object.value, mq.subject)
 
     const fedQuads = store.getQuads(null, null, null, MAPPED_GRAPH)
-    const originPredNode = df.namedNode(originPred)
     const provQuads = []
     for (const qu of fedQuads) {
         const minted = mintedFor.get(qu.subject.value)
@@ -43,7 +43,7 @@ export const runMerge = async ({ store, defStore, abs }, outPath, provOutPath) =
         // per-derivation metadata (time, confidence) has a home when needed.
         const reifier = df.blankNode()
         provQuads.push(df.quad(reifier, RDF_REIFIES, df.quad(minted, qu.predicate, object)))
-        provQuads.push(df.quad(reifier, originPredNode, qu.subject))
+        provQuads.push(df.quad(reifier, PROV_DERIVED_FROM, qu.subject))
     }
 
     const mergedQuads = store.getQuads(null, null, null, MERGED_GRAPH)

package/src/pipeline/steps/resolve.js

@@ -26,11 +26,12 @@ export const runResolve = async ({ store, defStore, abs }, outPath) => {
     const [cfg] = await sparqlSelect(`
         PREFIX : <${CDP}>
         SELECT ?strategy ?ns WHERE {
-            ?resolve a :ResolveRule ; :defaultStrategy ?strategy .
-            ?match   a :MatchRule   ; :targetNamespace ?ns .
+            ?match a :MatchRule ; :targetNamespace ?ns .
+            OPTIONAL { ?resolve a :ResolveRule ; :defaultStrategy ?strategy }
         }`, [defStore])
-    if (!cfg) throw new Error(":ResolveRule config missing in federation.ttl")
-    const defaultPick = lookupStrategy(cfg.strategy)
+    if (!cfg) throw new Error(":MatchRule config missing in federation.ttl")
+    // No :ResolveRule (or none with a :defaultStrategy) → alphabeticFirst.
+    const defaultPick = lookupStrategy(cfg.strategy ?? `${CDP}alphabeticFirst`)
 
     const overrideRows = await sparqlSelect(`
         PREFIX : <${CDP}>

package/src/utils.js

@@ -69,6 +69,7 @@ export const PATHS = {
     ingestLog:   "data/ingest/ingest-log.ttl",
     federateLog: "data/pipeline/federate-log.ttl",
     mappingQueries: "data/pipeline/direct-mapping-queries/",
+    defaultCleanQuery: (name) => `data/pipeline/default-clean-queries/${name}.sparql`,
     mapped:      "data/pipeline/mapped.ttl",
     matches:     "data/pipeline/matches.ttl",
     merged:      "data/pipeline/merged.ttl",
@@ -115,6 +116,18 @@ export const enabledSources = (quads) => {
     return objectsOf(quads, `${CDP}hasSource`).filter((iri) => !disabled.has(iri))
 }
 
+// The source's skolem key for the default clean: the :fieldPath of the source
+// field whose mapping points at the target field with :targetPredicate
+// schema:identifier. Undefined when the source declares no such mapping.
+export const identifierFieldPath = (quads, sourceIri) => {
+    const o = (s, p) => quads.filter((q) => q.subject.value === s && q.predicate.value === `${CDP}${p}`).map((q) => q.object.value)
+    for (const m of quads.filter((q) => q.predicate.value === `${CDP}fromSource` && q.object.value === sourceIri).map((q) => q.subject.value)) {
+        for (const fm of o(m, "hasFieldMapping")) {
+            if (o(o(fm, "to")[0], "targetPredicate")[0] === "http://schema.org/identifier") return o(o(fm, "from")[0], "fieldPath")[0]
+        }
+    }
+}
+
 // Set of subjects typed `rdf:type typeIri`. Iteration order = encounter order.
 export function subjectsOfType(quads, typeIri) {
     const out = new Set()

package/src/validate.js

@@ -1,5 +1,5 @@
 import { buildValidator, turtleToDataset } from "@foerderfunke/sem-ops-utils"
-import { CDP, objectsOf, parseTtl, PATHS, shrink, sourceName } from "./utils.js"
+import { CDP, identifierFieldPath, objectsOf, parseTtl, PATHS, shrink, sourceName } from "./utils.js"
 import path from "path"
 import fs from "fs"
 
@@ -19,23 +19,27 @@ export async function validate(root = process.cwd()) {
     return (await Promise.all(checks.map((check) => check(ctx)))).flat()
 }
 
-// Every :hasSource in federation.ttl has its sources/<name>/ folder with
-// fetch.js + clean.sparql - and no folder exists that the federation doesn't
-// declare. Checks all declared sources, enabled or not: folder presence is a
-// repo-layout contract.
+// Every :hasSource in federation.ttl has what its engine steps need: a
+// fetch.js or static/ to default to, a clean.sparql or a schema:identifier
+// mapping to derive the default clean from - and no sources/ folder exists
+// that the federation doesn't declare. Checks all declared sources, enabled
+// or not: folder presence is a repo-layout contract.
 function sourcesFoldersInSync({ abs, quads }) {
-    const declared = objectsOf(quads, `${CDP}hasSource`).map(sourceName)
+    const declared = objectsOf(quads, `${CDP}hasSource`)
     const problems = []
-    for (const name of declared) {
-        for (const file of [PATHS.fetchScript(name), PATHS.cleanQuery(name)]) {
-            if (!fs.existsSync(abs(file))) problems.push(`${file} missing`)
-        }
+    for (const iri of declared) {
+        const name = sourceName(iri)
+        if (![PATHS.fetchScript(name), PATHS.staticDir(name)].some((f) => fs.existsSync(abs(f))))
+            problems.push(`${PATHS.fetchScript(name)} missing and no ${PATHS.staticDir(name)} to default to`)
+        if (!fs.existsSync(abs(PATHS.cleanQuery(name))) && !identifierFieldPath(quads, iri))
+            problems.push(`${PATHS.cleanQuery(name)} missing and no schema:identifier mapping to derive the default clean from`)
     }
+    const declaredNames = declared.map(sourceName)
     const folders = fs.existsSync(abs("sources"))
         ? fs.readdirSync(abs("sources"), { withFileTypes: true }).filter((d) => d.isDirectory()).map((d) => d.name)
         : []
     for (const name of folders) {
-        if (!declared.includes(name)) problems.push(`sources/${name}/ has no :hasSource declaration in ${PATHS.federation}`)
+        if (!declaredNames.includes(name)) problems.push(`sources/${name}/ has no :hasSource declaration in ${PATHS.federation}`)
     }
     return problems
 }

package/test/helpers/instance.js

@@ -0,0 +1,24 @@
+import { PATHS } from "@directory-builder/core/utils"
+import path from "path"
+import fs from "fs"
+
+// SPARQL Anything cache shared with example/ — a fixture's tools/ symlinks
+// here, so test runs never re-download the jar.
+const TOOLS_CACHE = path.join(import.meta.dirname, "../../example/tools")
+
+// Materialize an in-test instance definition (federation.ttl string + records
+// per source) into test/tmp/<name>/ — a real instance folder the engines run
+// against, wiped at setup and left in place afterwards for inspection.
+export const makeInstance = (name, { federation, sources }) => {
+    const root = path.join(import.meta.dirname, "../tmp", name)
+    fs.rmSync(root, { recursive: true, force: true })
+    fs.mkdirSync(path.join(root, "config"), { recursive: true })
+    fs.writeFileSync(path.join(root, PATHS.federation), federation)
+    for (const [source, records] of Object.entries(sources)) {
+        fs.mkdirSync(path.join(root, PATHS.staticDir(source)), { recursive: true })
+        fs.writeFileSync(path.join(root, PATHS.staticDir(source), "data.json"), JSON.stringify(records, null, 4))
+    }
+    fs.mkdirSync(TOOLS_CACHE, { recursive: true })
+    fs.symlinkSync(TOOLS_CACHE, path.join(root, "tools"))
+    return root
+}

package/test/pipeline.test.js

@@ -0,0 +1,90 @@
+import { parseTtl, PATHS } from "@directory-builder/core/utils"
+import { Pipeline, validate } from "@directory-builder/core"
+import { makeInstance } from "./helpers/instance.js"
+import assert from "node:assert/strict"
+import { test } from "node:test"
+import path from "path"
+import fs from "fs"
+
+// The ultra-minimal instance: federation.ttl + two static JSON sources,
+// nothing else — fetch, clean and resolve all run on engine defaults. The
+// sources share one record by name ("Entry One"), so the pipeline should
+// merge a1+b1 and leave a2 and b2 as their own entities.
+
+const federation = `
+@prefix :       <https://civic-data.de/pipeline#> .
+@prefix schema: <http://schema.org/> .
+@prefix ft:     <http://publications.europa.eu/resource/authority/file-type/> .
+
+:federation a :Federation ;
+    :hasSource :alphaSource, :betaSource .
+
+:thingSchema a :TargetSchema ;
+    :targetClass schema:Thing .
+
+:t-id   a :TargetField ; :targetPredicate schema:identifier .
+:t-name a :TargetField ; :targetPredicate schema:name .
+
+:alphaSource a :Source ; :format ft:JSON .
+:betaSource  a :Source ; :format ft:JSON .
+
+:alpha-id   a :SourceField ; :fieldPath "id" .
+:alpha-name a :SourceField ; :fieldPath "name" .
+:beta-id    a :SourceField ; :fieldPath "id" .
+:beta-label a :SourceField ; :fieldPath "label" .
+
+:alpha-mapping a :Mapping ; :fromSource :alphaSource ; :toTarget :thingSchema ;
+    :hasFieldMapping [ :from :alpha-id ; :to :t-id ] , [ :from :alpha-name ; :to :t-name ] .
+
+:beta-mapping a :Mapping ; :fromSource :betaSource ; :toTarget :thingSchema ;
+    :hasFieldMapping [ :from :beta-id ; :to :t-id ] , [ :from :beta-label ; :to :t-name ] .
+
+:match a :MatchRule ;
+    :forTarget           :thingSchema ;
+    :targetNamespace     "urn:test:" ;
+    :mintedSubjectPrefix "thing-" ;
+    :minScore             1.0 ;
+    :hasWeightedCriterion [ :on schema:name ; :weight 1.0 ] .
+`
+
+const alpha = [
+    { id: "a1", name: "Entry One" },
+    { id: "a2", name: "Entry Two" },
+]
+const beta = [
+    { id: "b1", label: "Entry One" },
+    { id: "b2", label: "Entry Three" },
+]
+
+const root = makeInstance("tiny", { federation, sources: { alpha, beta } })
+
+const expectedFinal = `@prefix schema: <http://schema.org/>.
+@prefix foaf: <http://xmlns.com/foaf/0.1/>.
+@prefix dct: <http://purl.org/dc/terms/>.
+@prefix cdf: <urn:test:>.
+
+cdf:thing-5a45645edb31 a schema:Thing;
+    schema:name "Entry Two".
+cdf:thing-616feb993283 a schema:Thing;
+    schema:name "Entry One".
+cdf:thing-d1583c098826 a schema:Thing;
+    schema:name "Entry Three".
+`
+
+test("the tiny fixture validates and runs the whole pipeline on defaults", async () => {
+    // the fixture satisfies the instance contract (folders, derivable defaults, shape)
+    assert.deepEqual(await validate(root), [])
+    await new Pipeline({ root }).run()
+    const finalTtl = fs.readFileSync(path.join(root, PATHS.final), "utf8")
+    const final = parseTtl(finalTtl)
+    // match merged a1+b1 on their identical name; a2 and b2 stay their own entities
+    const subjects = new Set(final.map((q) => q.subject.value))
+    assert.equal(subjects.size, 3, "a1+b1 merge, a2 and b2 stay alone")
+    // entity IRIs are minted from the match rule's :targetNamespace + :mintedSubjectPrefix
+    for (const s of subjects) assert.match(s, /^urn:test:thing-/)
+    // map carried both sources' name fields through, resolve kept one value per entity
+    const names = final.filter((q) => q.predicate.value === "http://schema.org/name").map((q) => q.object.value)
+    assert.deepEqual(names.toSorted(), ["Entry One", "Entry Three", "Entry Two"])
+    // and the consumer-facing artifact as a whole
+    assert.equal(finalTtl, expectedFinal)
+})

package/example/sources/cityopen/fetch.js

@@ -1,14 +0,0 @@
-import path from "path"
-import fs from "fs"
-
-// Static-file source: copy the committed JSON straight into the ingest area.
-// A live source would instead call an API here and write the responses out.
-// argv: [outDir, sourceDir, runParamsJson] — params unused for this static example.
-const OUT_DIR = process.argv[2]
-const SRC_DIR = process.argv[3]
-
-fs.mkdirSync(OUT_DIR, { recursive: true })
-for (const f of fs.readdirSync(SRC_DIR).filter((f) => f.endsWith(".json"))) {
-    fs.copyFileSync(path.join(SRC_DIR, f), path.join(OUT_DIR, f))
-    console.log(`  ${f} → ${OUT_DIR}`)
-}

package/example/sources/civichub/fetch.js

@@ -1,14 +0,0 @@
-import path from "path"
-import fs from "fs"
-
-// Static-file source: copy the committed JSON straight into the ingest area.
-// A live source would instead call an API here and write the responses out.
-// argv: [outDir, sourceDir, runParamsJson] — params unused for this static example.
-const OUT_DIR = process.argv[2]
-const SRC_DIR = process.argv[3]
-
-fs.mkdirSync(OUT_DIR, { recursive: true })
-for (const f of fs.readdirSync(SRC_DIR).filter((f) => f.endsWith(".json"))) {
-    fs.copyFileSync(path.join(SRC_DIR, f), path.join(OUT_DIR, f))
-    console.log(`  ${f} → ${OUT_DIR}`)
-}