@directory-builder/core 0.1.0

package/webapp/src/loadMatch.js

@@ -0,0 +1,228 @@
+// Match "lanes" view, derived entirely from federation.ttl — no assumptions about
+// how many entity types there are or how they relate. One lane per :TargetSchema
+// (ordered by the relationship hierarchy, roots left), each preceded by a tinted
+// "source duplications" column; cross-lane edges come from every :hasRelationship.
+// When the relationships form a tree the layout groups each subtree vertically
+// (parent centred on its children); otherwise it just lays out gracefully.
+// Reads:  federation.ttl (schemas, classes, labels, relationships),
+//         matches.ttl (clusters + hasMember), merged.ttl (rdf:type, name, links)
+// Does:   returns everything <ColumnGraph> needs + a per-lane nodeY layout.
+
+import { CDP as NS, localName, parseTtl, prefixesOf, shrink, subjectsOfType } from "@directory-builder/core/utils"
+
+const CDF = "https://civic-data.de/federated-directory#"
+const S   = "http://schema.org/"
+const RDF_TYPE   = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"
+const RDFS_LABEL = "http://www.w3.org/2000/01/rdf-schema#label"
+const HAS_TARGET_SCHEMA = `${NS}hasTargetSchema`
+const TARGET_CLASS      = `${NS}targetClass`
+const TO_TARGET         = `${NS}toTarget`
+const HAS_RELATIONSHIP  = `${NS}hasRelationship`
+const TO_TARGET_SCHEMA  = `${NS}toTargetSchema`
+const TO_TARGET_FIELD   = `${NS}toTargetField`
+const TARGET_PREDICATE  = `${NS}targetPredicate`
+const MATCH_CLUSTER = `${NS}MatchCluster`
+const HAS_MEMBER    = `${NS}hasMember`
+const NAME = `${S}name`
+const CATEGORY = `${S}category`   // label fallback for entities with no name (e.g. AWO services)
+
+// Lane colours, assigned by hierarchy position; cycles if there are more lanes.
+const PALETTE = ["#cdddff", "#f7d2e3", "#cfe9d4", "#ffe2b8", "#e3d4f7", "#cfeef0", "#f3d9c0"]
+const SRC_COLOR = "#e9e9ee"
+const GAP = 84          // vertical spacing between single-member leaves
+const SRC_GAP = 56      // spacing of a cluster's stacked source members
+const NODE_H = 48       // approx node height — keeps source stacks from colliding
+const MARGIN = GAP - NODE_H   // inter-cluster gap; keeps single-member spacing == GAP
+
+const cap = (s) => s.charAt(0).toUpperCase() + s.slice(1)
+// Mix a hex colour toward white by t∈[0,1] — the pale entity-column band tint.
+const lighten = (hex, t) => {
+    const n = parseInt(hex.slice(1), 16)
+    const c = (sh) => { const v = (n >> sh) & 255; return Math.round(v + (255 - v) * t) }
+    return `#${((c(16) << 16) | (c(8) << 8) | c(0)).toString(16).padStart(6, "0")}`
+}
+
+// ---- federation.ttl → schema model -------------------------------------
+
+function readSchemas(federationTtl) {
+    const q = parseTtl(federationTtl)
+    const classPrefixes = prefixesOf(federationTtl)
+    const order = []                  // schema IRIs in document order
+    const targetClass = new Map()
+    const label = new Map()           // any subject → its rdfs:label
+    const toTarget = new Map()        // mapping → its :toTarget schema
+    const relMapping = new Map()      // rel bnode → its mapping
+    const relToSchema = new Map()     // rel bnode → :toTargetSchema
+    const relToField = new Map()      // rel bnode → :toTargetField
+    const fieldPred = new Map()       // target field → :targetPredicate
+    for (const { subject: s, predicate: p, object: o } of q) {
+        switch (p.value) {
+            case HAS_TARGET_SCHEMA: order.push(o.value); break
+            case TARGET_CLASS:      targetClass.set(s.value, o.value); break
+            case RDFS_LABEL:        if (!label.has(s.value)) label.set(s.value, o.value); break
+            case TO_TARGET:         toTarget.set(s.value, o.value); break
+            case HAS_RELATIONSHIP:  relMapping.set(o.value, s.value); break
+            case TO_TARGET_SCHEMA:  relToSchema.set(s.value, o.value); break
+            case TO_TARGET_FIELD:   relToField.set(s.value, o.value); break
+            case TARGET_PREDICATE:  fieldPred.set(s.value, o.value); break
+        }
+    }
+
+    // Schema-level relationships: from = the mapping's :toTarget, to = :toTargetSchema,
+    // predicate = the target field's :targetPredicate. Drives both the cross-lane
+    // edges (by predicate) and the lane ordering (by the from→to graph).
+    const relPreds = new Set()
+    const out = new Map()             // schema → Set(schema it points at)
+    for (const [rel, mapping] of relMapping) {
+        const from = toTarget.get(mapping), to = relToSchema.get(rel)
+        const pred = fieldPred.get(relToField.get(rel))
+        if (!from || !to || !pred) continue
+        relPreds.add(pred)
+        if (!out.has(from)) out.set(from, new Set())
+        out.get(from).add(to)
+    }
+
+    // Lane order: a schema sits left of anything that relates to it (a parent is
+    // left of its children). level = longest chain of out-edges; roots (sinks) = 0.
+    const docIdx = new Map(order.map((s, i) => [s, i]))
+    const memo = new Map()
+    const levelOf = (s, stack = new Set()) => {
+        if (memo.has(s)) return memo.get(s)
+        if (stack.has(s)) return 0                       // cycle guard
+        stack.add(s)
+        let lvl = 0
+        for (const t of out.get(s) ?? []) lvl = Math.max(lvl, 1 + levelOf(t, stack))
+        stack.delete(s)
+        memo.set(s, lvl)
+        return lvl
+    }
+    const ordered = [...order].sort((a, b) => levelOf(a) - levelOf(b) || docIdx.get(a) - docIdx.get(b))
+
+    const lanes = ordered.map((schema, i) => {
+        const cls = targetClass.get(schema)
+        const name = label.get(schema) ?? (cls && label.get(cls)) ?? cap(localName(schema).replace(/Schema$/, ""))
+        return {
+            schema, cls,
+            key: localName(schema).replace(/Schema$/, ""),
+            label: name,
+            title: `${name}\n${cls ? shrink(cls, classPrefixes) : ""}`,
+            color: PALETTE[i % PALETTE.length],
+        }
+    })
+    return { lanes, relPreds }
+}
+
+// ---- main ---------------------------------------------------------------
+
+export function loadMatch(federationTtl, matchesTtl, mergedTtl, { showDuplications = false, show1to1 = false } = {}) {
+    const { lanes, relPreds } = readSchemas(federationTtl)
+    const keyOfClass = new Map(lanes.filter((l) => l.cls).map((l) => [l.cls, l.key]))
+    const laneIdx = new Map(lanes.map((l, i) => [l.key, i]))
+
+    const columns = lanes.flatMap((l) => [`${l.key}Src`, l.key])
+    const colors = {}, columnTitles = {}, columnBands = {}, columnHeaderStyle = {}
+    for (const l of lanes) {
+        colors[l.key] = l.color;        colors[`${l.key}Src`] = SRC_COLOR
+        columnTitles[l.key] = l.title;  columnTitles[`${l.key}Src`] = "source duplications"
+        columnBands[l.key] = lighten(l.color, 0.6)                          // entity column gets a brighter tint of its nodes
+        columnHeaderStyle[`${l.key}Src`] = { fontSize: 10, color: "#aaa" }  // de-emphasise the source-column labels
+    }
+
+    const merged = parseTtl(mergedTtl)
+    const tierOf = new Map()           // entity → lane key, via its rdf:type
+    const nameOf = new Map(), catOf = new Map()
+    for (const q of merged) {
+        if (q.predicate.value === RDF_TYPE && keyOfClass.has(q.object.value)) tierOf.set(q.subject.value, keyOfClass.get(q.object.value))
+        else if (q.predicate.value === NAME && !nameOf.has(q.subject.value)) nameOf.set(q.subject.value, q.object.value)
+        else if (q.predicate.value === CATEGORY && !catOf.has(q.subject.value)) catOf.set(q.subject.value, q.object.value)
+    }
+
+    const quads = parseTtl(matchesTtl)
+    const clusters = subjectsOfType(quads, MATCH_CLUSTER)
+    const members = new Map()
+    for (const q of quads) if (q.predicate.value === HAS_MEMBER) {
+        if (!members.has(q.subject.value)) members.set(q.subject.value, [])
+        members.get(q.subject.value).push(q.object.value)
+    }
+
+    const nodes = []
+    const edges = []
+    const nodeIds = new Set()
+    for (const c of clusters) {
+        const tier = tierOf.get(c)
+        if (!tier) continue
+        nodes.push({ id: c, type: tier, label: nameOf.get(c) ?? catOf.get(c) ?? localName(c), isCluster: true })
+        nodeIds.add(c)
+        const ms = members.get(c) ?? []
+        if (!showDuplications || (!show1to1 && ms.length <= 1)) continue   // master off → no source cols; hide 1:1 unless "show 1:1"
+        for (const src of ms) {
+            nodes.push({ id: src, type: `${tier}Src`, label: localName(src) })
+            nodeIds.add(src)
+            edges.push({ from: src, to: c })           // dedup (hasMember) edge
+        }
+    }
+
+    // Cross-lane links: any merged triple whose predicate is a declared relationship
+    // and whose ends are both placed entities. Stored object→subject so the parent
+    // (object) sits left of the child (subject) and edges flow toward the root.
+    for (const q of merged) {
+        if (relPreds.has(q.predicate.value) && q.object.termType === "NamedNode"
+            && tierOf.has(q.subject.value) && tierOf.has(q.object.value)) {
+            edges.push({ from: q.object.value, to: q.subject.value, rel: true })
+        }
+    }
+
+    return { nodes, edges, members, lanes, columns, colors, columnTitles, columnBands, columnHeaderStyle,
+             nodeY: layout(nodes, edges, members, nodeIds, laneIdx) }
+}
+
+// Tidy-tree vertical layout: place leaves on a running cursor, centre each parent
+// on its children. Roots (entities with no parent) are ordered by lane, so the
+// upper lanes' subtrees group at the top. Graceful on non-tree graphs: a node is
+// placed once (first visit), so multiple parents / cycles can't loop or duplicate.
+function layout(nodes, edges, members, nodeIds, laneIdx) {
+    const childrenOf = new Map()
+    const hasParent = new Set()
+    for (const e of edges) if (e.rel) {
+        if (!childrenOf.has(e.from)) childrenOf.set(e.from, [])
+        childrenOf.get(e.from).push(e.to)
+        hasParent.add(e.to)
+    }
+
+    const y = new Map()
+    let cursor = 0
+    const mean = (a) => a.reduce((s, v) => s + v, 0) / a.length
+    const place = (id) => {
+        if (y.has(id)) return
+        const kids = childrenOf.get(id) ?? []
+        if (kids.length) { kids.forEach(place); y.set(id, mean(kids.map((k) => y.get(k)))) }
+        else { y.set(id, cursor); cursor += GAP }
+    }
+    const clusters = nodes.filter((n) => n.isCluster)
+    clusters.filter((n) => !hasParent.has(n.id)).sort((a, b) => laneIdx.get(a.type) - laneIdx.get(b.type)).forEach((n) => place(n.id))
+    clusters.filter((n) => !y.has(n.id)).forEach((n) => place(n.id))   // safety net
+
+    // When source columns are shown, push clusters apart within each lane so a
+    // cluster's stacked source members never collide with its neighbours'.
+    const stackHalf = (c) => (((members.get(c) ?? []).filter((m) => nodeIds.has(m)).length || 1) - 1) * SRC_GAP / 2 + NODE_H / 2
+    if (nodes.some((n) => !n.isCluster)) {
+        const byLane = new Map()
+        for (const n of clusters) { if (!byLane.has(n.type)) byLane.set(n.type, []); byLane.get(n.type).push(n.id) }
+        for (const ids of byLane.values()) {
+            ids.sort((a, b) => y.get(a) - y.get(b))
+            let prevBottom = -Infinity
+            for (const id of ids) {
+                const h = stackHalf(id)
+                const cy = Math.max(y.get(id), prevBottom + MARGIN + h)
+                y.set(id, cy); prevBottom = cy + h
+            }
+        }
+    }
+
+    for (const [c, ms] of members) {
+        if (!y.has(c)) continue
+        const shown = ms.filter((m) => nodeIds.has(m))
+        shown.forEach((m, i) => y.set(m, y.get(c) + (i - (shown.length - 1) / 2) * SRC_GAP))
+    }
+    return y
+}

package/webapp/src/loadMerge.js

@@ -0,0 +1,93 @@
+// Parses merged + provenance TTL into org objects: each field's values and the
+// :Source(s) that contributed them, ordered by config. Pure (ttl in → data out).
+// Reads:  TTL strings passed by mergeOrgs.js; resolves sources via sourceMeta.js
+// Does:   returns org[] (each {iri, label, type, fields[], sources[]})
+
+import { CDP as NS, parseTtl, parseTtlStar, prefixesOf, shrink } from "@directory-builder/core/utils"
+import { compareSources, loadSourceMeta } from "./sourceMeta.js"
+
+const PROV_DERIVED_FROM = "http://www.w3.org/ns/prov#wasDerivedFrom"
+const RDF_TYPE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"
+const RDF_REIFIES = "http://www.w3.org/1999/02/22-rdf-syntax-ns#reifies"
+const FROM_SOURCE = `${NS}fromSource`
+
+export function loadMerge(mergedTtl, provTtl, federationTtl = "") {
+    // IRIs render shortened against the federation's own @prefix declarations.
+    const prefixes = { cdp: NS, ...prefixesOf(federationTtl) }
+    const prefixedIri = (iri) => shrink(iri, prefixes)
+    const mergedQuads = parseTtl(mergedTtl)
+    const provQuads = parseTtlStar(provTtl)
+    const sourceMeta = federationTtl ? loadSourceMeta(federationTtl) : new Map()
+
+    // Each prov:wasDerivedFrom in provenance.ttl annotates a merged triple
+    // `<<s p o>>` with the source record IRI it came from. n3.js exposes the
+    // quoted-triple subject either directly as a Quad term, or via an
+    // auto-generated reifier bnode + rdf:reifies triple — accept both shapes.
+    const reifies = new Map()
+    for (const q of provQuads) {
+        if (q.predicate.value === RDF_REIFIES && q.object.termType === "Quad") reifies.set(q.subject.value, q.object)
+    }
+    const annotations = []
+    for (const q of provQuads) {
+        if (q.predicate.value !== PROV_DERIVED_FROM) continue
+        const t = q.subject.termType === "Quad" ? q.subject : reifies.get(q.subject.value)
+        if (t) annotations.push({ s: t.subject.value, p: t.predicate.value, o: t.object.value, rec: q.object.value })
+    }
+    // Resolve each record to its :Source via cdp:fromSource (reified in
+    // provenance) so downstream code deals only in Source IRIs, not record IRIs.
+    const sourceOfRecord = new Map()
+    for (const { p, o, rec } of annotations) if (p === FROM_SOURCE) sourceOfRecord.set(rec, o)
+    const toSources = (records) => [...new Set([...records].map((r) => sourceOfRecord.get(r)))]
+
+    const provIndex = new Map()
+    const tripleKey = (s, p, o) => `${s}\t${p}\t${o}`
+    for (const { s, p, o, rec } of annotations) {
+        const key = tripleKey(s, p, o)
+        if (!provIndex.has(key)) provIndex.set(key, new Set())
+        provIndex.get(key).add(rec)
+    }
+
+    // Walk merged.ttl in parse order so card order = pipeline order.
+    const orgs = []
+    const orgIndex = new Map()
+    const fieldIndexByOrg = new Map()
+    for (const q of mergedQuads) {
+        const orgIri = q.subject.value
+        const predIri = q.predicate.value
+        const value = q.object.value
+
+        if (!orgIndex.has(orgIri)) {
+            orgIndex.set(orgIri, orgs.length)
+            fieldIndexByOrg.set(orgIri, new Map())
+            orgs.push({ iri: orgIri, label: prefixedIri(orgIri), fields: [] })
+        }
+        const org = orgs[orgIndex.get(orgIri)]
+        const fieldIndex = fieldIndexByOrg.get(orgIri)
+
+        // rdf:type carries the entity class — surface it in the card header
+        // (see OrgCard), not as a field row.
+        if (predIri === RDF_TYPE) { org.type = prefixedIri(value); continue }
+
+        if (!fieldIndex.has(predIri)) {
+            fieldIndex.set(predIri, org.fields.length)
+            org.fields.push({ predicate: predIri, predLabel: prefixedIri(predIri), values: [] })
+        }
+        const field = org.fields[fieldIndex.get(predIri)]
+        const records = [...(provIndex.get(tripleKey(orgIri, predIri, value)) ?? [])]
+        const sources = toSources(records)
+        const displayValue = q.object.termType === "NamedNode" ? prefixedIri(value) : value
+        field.values.push({ value: displayValue, raw: value, sources, records })
+    }
+
+    // Per-field: sort values by source-count desc so the most-supported one is index 0.
+    // Per-org: one column per contributing record (two records from the same source
+    // get two columns), ordered by source then record IRI.
+    for (const org of orgs) {
+        for (const f of org.fields) f.values.sort((a, b) => b.sources.length - a.sources.length)
+        const all = new Set()
+        for (const f of org.fields) for (const v of f.values) for (const r of v.records) all.add(r)
+        org.columns = [...all].map((r) => ({ record: r, source: sourceOfRecord.get(r) }))
+            .sort((a, b) => compareSources(a.source, b.source, sourceMeta) || a.record.localeCompare(b.record))
+    }
+    return orgs
+}

package/webapp/src/loadPipeline.js

@@ -0,0 +1,130 @@
+// Helper for the Pipeline view: turn the engines' step journals into a graph.
+// Reads:  the step-journal TTL strings (ingest-log.ttl + federate-log.ttl —
+//         evidence of what actually ran) and the federation TTL, passed by
+//         Pipeline.jsx
+// Does:   returns { nodes, edges } — Source lane-header nodes (transparent
+//         fill, light-gray border) above each Fetch step, step nodes labelled
+//         by their type (fetch/lift/clean/map/match/merge/resolve), and an
+//         End sink so resolve's output is shown on a visible edge, plus a
+//         boundary node feeding the Match step with the conventional
+//         match-knowledge file. Edge labels come from federation.ttl —
+//         a source's :format (uppercased) and :retrieval — or from the
+//         conventions: Lift emits Turtle (LIFTED_FORMAT), other steps their
+//         output file(s) per PATHS, resolved per source for Clean steps.
+//         Multiple outputs (merge's provenance) stack as newlines.
+
+import { CDP as NS, formatFamily, LIFTED_FORMAT, localName, parseTtl, PATHS, sourceName } from "@directory-builder/core/utils"
+
+const PPLAN_STEP = "http://purl.org/net/p-plan#Step"
+const PPLAN_IS_PRECEDED_BY = "http://purl.org/net/p-plan#isPrecededBy"
+const RDF_TYPE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"
+const RDFS_LABEL = "http://www.w3.org/2000/01/rdf-schema#label"
+const FROM_SOURCE = `${NS}fromSource`
+const RETRIEVAL = `${NS}retrieval`
+const FORMAT = `${NS}format`
+const LANE_BORDER = "#bbb"
+
+const basename = (path) => path.replace(/^.*\//, "")
+
+// Output file(s) per step type, by the PATHS conventions (name = source name).
+const STEP_OUTPUTS = {
+    Clean:   (name) => [PATHS.cleaned(name)],
+    Map:     () => [PATHS.mapped],
+    Match:   () => [PATHS.matches],
+    Merge:   () => [PATHS.merged, PATHS.provenance],
+    Resolve: () => [PATHS.final],
+}
+
+export function loadPipeline(stepTtls, federationTtl) {
+    const quads = stepTtls.flatMap((ttl) => ttl ? parseTtl(ttl) : [])
+    const fedQuads = federationTtl ? parseTtl(federationTtl) : []
+
+    // A step is whatever the journals typed p-plan:Step; its display type is
+    // the co-declared pipeline-NS class (:Fetch, :Lift, …) — no fixed list.
+    const isStep = new Set()
+    const nsTypeOf = new Map()
+    const rawEdges = []
+    const sourceOfStep = new Map()
+    const formatBySubject = new Map()
+    const retrievalBySubject = new Map()
+    for (const q of [...quads, ...fedQuads]) {
+        const p = q.predicate.value
+        if (p === RDF_TYPE) {
+            if (q.object.value === PPLAN_STEP) isStep.add(q.subject.value)
+            else if (q.object.value.startsWith(NS)) nsTypeOf.set(q.subject.value, q.object.value.slice(NS.length))
+        } else if (p === PPLAN_IS_PRECEDED_BY) rawEdges.push({ from: q.object.value, to: q.subject.value })
+        else if (p === FROM_SOURCE)    sourceOfStep.set(q.subject.value, q.object.value)
+        else if (p === RETRIEVAL)      retrievalBySubject.set(q.subject.value, q.object.value)
+        else if (p === FORMAT)         formatBySubject.set(q.subject.value, q.object.value)
+    }
+    const stepType = new Map([...isStep].map((iri) => [iri, nsTypeOf.get(iri)]))
+
+    const fileLabel = (iri) => {
+        const src = sourceOfStep.get(iri)
+        const outs = (STEP_OUTPUTS[stepType.get(iri)] ?? (() => []))(src && sourceName(src)).map(basename)
+        return outs.length ? outs.join("\n") : null
+    }
+    // A Fetch step emits its source's :format from federation.ttl; a Lift
+    // step always emits Turtle (engine invariant, see LIFTED_FORMAT).
+    const formatOf = (iri) => ({
+        Fetch: formatBySubject.get(sourceOfStep.get(iri) ?? ""),
+        Lift:  LIFTED_FORMAT,
+    })[stepType.get(iri)]
+    // Edge label = the format the step emits (its file-type IRI's short label),
+    // else its conventional output file(s); nothing hardcoded per source.
+    const edgeLabel = (fromIri) => {
+        const fmt = formatOf(fromIri)
+        return fmt ? formatFamily(fmt) : fileLabel(fromIri)
+    }
+
+    const stepEdges = rawEdges.map((e) => ({ ...e, value: edgeLabel(e.from) ?? undefined, centered: true }))
+
+    const sourceLabel = new Map()
+    for (const q of fedQuads) {
+        if (q.predicate.value === RDFS_LABEL) sourceLabel.set(q.subject.value, q.object.value)
+    }
+
+    const stepNodes = [...stepType].map(([iri, type]) => ({ id: iri, label: type.toLowerCase(), type }))
+
+    const laneNodes = []
+    const laneEdges = []
+    for (const [iri, type] of stepType) {
+        if (type !== "Fetch") continue
+        const sourceIri = sourceOfStep.get(iri)
+        if (!sourceIri) continue
+        const laneId = `lane:${sourceIri}`
+        laneNodes.push({
+            id: laneId,
+            label: sourceLabel.get(sourceIri) ?? localName(sourceIri),
+            type: "Source",
+            color: "transparent",
+            borderColor: LANE_BORDER,
+        })
+        laneEdges.push({ from: laneId, to: iri, value: retrievalBySubject.get(sourceIri), centered: true })
+    }
+
+    // End sink so resolve's output (final.ttl) is shown on a visible edge.
+    const resolveIri = [...stepType].find(([, t]) => t === "Resolve")?.[0]
+    const endNodes = []
+    const endEdges = []
+    if (resolveIri) {
+        endNodes.push({ id: "end", label: "end", type: "End", color: "transparent", borderColor: LANE_BORDER })
+        endEdges.push({ from: resolveIri, to: "end", value: edgeLabel(resolveIri) ?? undefined, centered: true })
+    }
+
+    // Side input: the Match step consumes the conventional match-knowledge
+    // file — a boundary node labelled with the file basename.
+    const matchIri = [...stepType].find(([, t]) => t === "Match")?.[0]
+    const inputNodes = []
+    const inputEdges = []
+    if (matchIri) {
+        const inId = `input:${PATHS.matchKnowledge}`
+        inputNodes.push({ id: inId, label: "input", type: "Input", color: "transparent", borderColor: LANE_BORDER })
+        inputEdges.push({ from: inId, to: matchIri, value: basename(PATHS.matchKnowledge), centered: true, sideInput: true })
+    }
+
+    return {
+        nodes: [...laneNodes, ...inputNodes, ...stepNodes, ...endNodes],
+        edges: [...laneEdges, ...inputEdges, ...stepEdges, ...endEdges],
+    }
+}

package/webapp/src/loadSources.js

@@ -0,0 +1,102 @@
+// Helper for the Sources view: aggregate per-:Source facts (label, URL, format,
+// field counts, record count, freshness) across config + pipeline data.
+// Reads:  federation, mapped, ingest-log TTL strings passed by Sources.jsx
+// Does:   returns source[] ({iri, label, format, totalFields, mappedFields, records, …})
+
+import { CDP as NS, formatFamily, parseTtl, PATHS, sourceName, subjectsOfType } from "@directory-builder/core/utils"
+
+const PROV_AT_TIME = "http://www.w3.org/ns/prov#atTime"
+const RDFS_LABEL = "http://www.w3.org/2000/01/rdf-schema#label"
+
+const setAdd = (map, key, val) => {
+    if (!map.has(key)) map.set(key, new Set())
+    map.get(key).add(val)
+}
+
+export function loadSources(federationTtl, mappedTtl, ingestLogTtl) {
+    const fedQuads = parseTtl(federationTtl)
+    const mappedQuads = mappedTtl ? parseTtl(mappedTtl) : []
+    const logQuads = ingestLogTtl ? parseTtl(ingestLogTtl) : []
+
+    const sourceIris = subjectsOfType(fedQuads, `${NS}Source`)
+
+    const props = new Map()
+    const get = (iri) => {
+        if (!props.has(iri)) props.set(iri, { iri })
+        return props.get(iri)
+    }
+
+    // Source-level: label, top-level fields, sub-fields, mappings.
+    const topFieldsOf = new Map()    // sourceIri -> Set<fieldIri>
+    const subFieldsOf = new Map()    // fieldIri  -> Set<subFieldIri>
+    const mappingSource = new Map()  // mappingIri -> sourceIri
+    const fmsOfMapping = new Map()   // mappingIri -> Set<fieldMappingBnode>
+    const fromsOfFm = new Map()      // bnode      -> Set<fieldIri>
+
+    for (const q of fedQuads) {
+        const p = q.predicate.value
+        if (p === RDFS_LABEL && sourceIris.has(q.subject.value))   get(q.subject.value).label = q.object.value
+        else if (p === `${NS}fetchUrl` && sourceIris.has(q.subject.value))
+                                               get(q.subject.value).fetchUrl = q.object.value
+        else if (p === `${NS}format` && sourceIris.has(q.subject.value))
+                                               get(q.subject.value).format = formatFamily(q.object.value)
+        else if (p === `${NS}hasField`)        setAdd(topFieldsOf, q.subject.value, q.object.value)
+        else if (p === `${NS}hasSubField`)     setAdd(subFieldsOf, q.subject.value, q.object.value)
+        else if (p === `${NS}fromSource`)      mappingSource.set(q.subject.value, q.object.value)
+        else if (p === `${NS}hasFieldMapping`) setAdd(fmsOfMapping, q.subject.value, q.object.value)
+        else if (p === `${NS}from`)            setAdd(fromsOfFm, q.subject.value, q.object.value)
+    }
+
+    for (const sourceIri of sourceIris) {
+        const top = topFieldsOf.get(sourceIri) ?? new Set()
+        const all = new Set(top)
+        for (const tf of top) for (const sf of subFieldsOf.get(tf) ?? []) all.add(sf)
+        get(sourceIri).totalFields = all.size
+
+        const mapped = new Set()
+        for (const [mappingIri, srcIri] of mappingSource) {
+            if (srcIri !== sourceIri) continue
+            for (const fm of fmsOfMapping.get(mappingIri) ?? []) {
+                for (const f of fromsOfFm.get(fm) ?? []) mapped.add(f)
+            }
+        }
+        get(sourceIri).mappedFields = mapped.size
+    }
+
+    // Static-file sources (no :fetchUrl) read from the conventional static dir.
+    for (const sourceIri of sourceIris) {
+        if (!get(sourceIri).fetchUrl) get(sourceIri).staticSource = PATHS.staticDir(sourceName(sourceIri))
+    }
+
+    // Records: count distinct orgs in mapped.ttl per source via cdp:fromSource.
+    const FROM_SOURCE = `${NS}fromSource`
+    const subjectsBySource = new Map()
+    for (const q of mappedQuads) {
+        if (q.predicate.value === FROM_SOURCE) setAdd(subjectsBySource, q.object.value, q.subject.value)
+    }
+    for (const sourceIri of sourceIris) {
+        get(sourceIri).records = subjectsBySource.get(sourceIri)?.size ?? 0
+    }
+
+    // Latest harvest timestamp per source from ingest-log.ttl. Each :harvested
+    // bnode carries (:ofSource ?source, prov:atTime ?time) and, for static-file
+    // sources, the files' git commit time (:staticCommittedAt); find the max time.
+    const harvestBnode = new Map()
+    const harvest = (bnode) => {
+        if (!harvestBnode.has(bnode)) harvestBnode.set(bnode, {})
+        return harvestBnode.get(bnode)
+    }
+    for (const q of logQuads) {
+        if (q.predicate.value === `${NS}ofSource`)                 harvest(q.subject.value).source = q.object.value
+        else if (q.predicate.value === PROV_AT_TIME)               harvest(q.subject.value).time = q.object.value
+        else if (q.predicate.value === `${NS}staticCommittedAt`)   harvest(q.subject.value).committedAt = q.object.value
+    }
+    for (const { source, time, committedAt } of harvestBnode.values()) {
+        if (!source || !time || !sourceIris.has(source)) continue
+        const cur = get(source).lastHarvestedAt
+        if (!cur || time > cur) get(source).lastHarvestedAt = time
+        if (committedAt) get(source).staticCommittedAt = committedAt
+    }
+
+    return [...sourceIris].map((iri) => get(iri))
+}

package/webapp/src/main.jsx

@@ -0,0 +1,9 @@
+// Browser entry point: mounts the React app into the page.
+// Reads:  App.jsx
+// Does:   renders <App> into the #root element
+
+import { createRoot } from "react-dom/client"
+import App from "./App.jsx"
+import React from "react"
+
+createRoot(document.getElementById("root")).render(<App />)

package/webapp/src/mergeOrgs.js

@@ -0,0 +1,15 @@
+// Builds the org lists for the Merge and Directory views, in one shared order.
+// Reads:  data/pipeline/{merged,provenance,final}.ttl, config/federation.ttl (via loadMerge.js)
+// Does:   exports mergedOrgs and finalOrgs (consumed by MergeTables, Directory)
+
+import { loadMerge } from "./loadMerge.js"
+import { isConflict } from "./OrgCard.jsx"
+import { federationTtl, provenanceTtl as provTtl, mergedTtl, finalTtl } from "./instanceData.js"
+
+const conflictCount = (org) => org.fields.reduce((n, f) => n + (isConflict(f) ? 1 : 0), 0)
+
+// Merge view sorts by conflict count desc; the directory mirrors that order
+// so the same org sits in the same visual slot across pages.
+export const mergedOrgs = loadMerge(mergedTtl, provTtl, federationTtl).sort((a, b) => conflictCount(b) - conflictCount(a) || a.iri.localeCompare(b.iri))
+const orderIndex = new Map(mergedOrgs.map((o, i) => [o.iri, i]))
+export const finalOrgs = loadMerge(finalTtl, "", federationTtl).sort((a, b) => (orderIndex.get(a.iri) ?? Infinity) - (orderIndex.get(b.iri) ?? Infinity))

package/webapp/src/sourceMeta.js

@@ -0,0 +1,81 @@
+// Source identity lives in config: federation.ttl declares each :Source (label,
+// skos:notation, order); its cleaned-data file follows from the source name by
+// the PATHS conventions. JS never hardcodes a source name — it resolves records
+// to a :Source via cdp:fromSource.
+// Reads:  TTL strings passed in (federation, mapped, ingest-log)
+// Does:   returns lookup maps + helpers (used by loadMerge, OrgCard, MapGraph, MatchGraph)
+
+import { CDP as NS, parseTtl, PATHS, sourceName } from "@directory-builder/core/utils"
+
+const RDFS_LABEL    = "http://www.w3.org/2000/01/rdf-schema#label"
+const SKOS_NOTATION = "http://www.w3.org/2004/02/skos/core#notation"
+const PROV_AT_TIME  = "http://www.w3.org/ns/prov#atTime"
+const HAS_SOURCE    = `${NS}hasSource`
+const FROM_SOURCE   = `${NS}fromSource`
+const OF_SOURCE     = `${NS}ofSource`
+
+// Map<SourceIRI, {iri, label, notation, order}> from federation.ttl; order
+// follows the :hasSource list. Assumes each :Source has a label and notation.
+export function loadSourceMeta(federationTtl) {
+    const order = new Map()
+    const labelOf = new Map()
+    const notationOf = new Map()
+    let n = 0
+    for (const q of parseTtl(federationTtl)) {
+        const p = q.predicate.value
+        if      (p === HAS_SOURCE && !order.has(q.object.value)) order.set(q.object.value, n++)
+        else if (p === RDFS_LABEL)    labelOf.set(q.subject.value, q.object.value)
+        else if (p === SKOS_NOTATION) notationOf.set(q.subject.value, q.object.value)
+    }
+    const meta = new Map()
+    for (const iri of order.keys()) {
+        meta.set(iri, { iri, label: labelOf.get(iri), notation: notationOf.get(iri), order: order.get(iri) })
+    }
+    return meta
+}
+
+// Order two Source IRIs by their federation declaration order, then IRI.
+export function compareSources(a, b, meta) {
+    const oa = meta.get(a).order
+    const ob = meta.get(b).order
+    return oa !== ob ? oa - ob : a.localeCompare(b)
+}
+
+// Map<recordIri, SourceIRI> from plain cdp:fromSource triples (mapped.ttl).
+export function loadSourceOfRecord(ttl) {
+    const out = new Map()
+    for (const q of parseTtl(ttl)) if (q.predicate.value === FROM_SOURCE) out.set(q.subject.value, q.object.value)
+    return out
+}
+
+// Map<SourceIRI, latest ISO timestamp> from the ingest log's harvest entries.
+export function loadHarvestBySource(logTtl) {
+    const source = new Map()
+    const time = new Map()
+    for (const q of parseTtl(logTtl)) {
+        if      (q.predicate.value === OF_SOURCE)    source.set(q.subject.value, q.object.value)
+        else if (q.predicate.value === PROV_AT_TIME) time.set(q.subject.value, q.object.value)
+    }
+    const out = new Map()
+    for (const [bnode, src] of source) {
+        const t = time.get(bnode)
+        if (t && (!out.has(src) || t > out.get(src))) out.set(src, t)
+    }
+    return out
+}
+
+// Map<SourceIRI, cleaned-TTL raw string> for every source a :Mapping draws
+// from (:fromSource); the file is the conventional cleaned path's basename.
+// `rawByPath` comes from import.meta.glob(".../cleaned/*.ttl", ...).
+export function loadCleanedBySource(federationTtl, rawByPath) {
+    const basename = (p) => p.split("/").pop()
+    const rawByBase = new Map(Object.entries(rawByPath).map(([path, raw]) => [basename(path), raw]))
+
+    const out = new Map()
+    for (const q of parseTtl(federationTtl)) {
+        if (q.predicate.value !== FROM_SOURCE) continue
+        const raw = rawByBase.get(basename(PATHS.cleaned(sourceName(q.object.value))))
+        if (raw) out.set(q.object.value, raw)
+    }
+    return out
+}