@directory-builder/core 0.1.1 → 0.1.3

package/README.md

@@ -66,6 +66,10 @@ Each source's `fetch.js` is invoked as `node fetch.js <outDir> <fetchUrl-or-stat
 <runParamsJson>` — the JSON holds all `:hasRunParam` values grouped by name;
 each fetcher picks the parameters it needs.
 
+A source declared with `:enabled false` stays in the config but is skipped by
+the engines and hidden from the webapp's Sources page — e.g. while its files
+aren't available yet.
+
 Engines journal their executed steps as p-plan RDF (`data/ingest/ingest-log.ttl`,
 `data/pipeline/federate-log.ttl`) — evidence of what ran, not a plan.
 
@@ -84,6 +88,11 @@ npx directory-builder webapp build --base /repo/     # production build → weba
 `webapp/{content,exporters}/` into `webapp/dist/` next to the bundle —
 `webapp/dist/` is the complete site, ready to publish as-is.
 
+The two are independent: the dev server never needs a prior build — `webapp
+build` exists only to produce the deployable. Both show whatever `data/` the
+pipeline last produced, so run the pipeline first (and rebuild before
+publishing, or `dist/` keeps the stale snapshot).
+
 For webapp development in this repo:
 
 ```sh

package/bin/cli.js

@@ -5,11 +5,13 @@
 //   directory-builder                          run the full pipeline (ingest + federate)
 //   directory-builder ingest                   fetch + lift only
 //   directory-builder federate                 clean → map → match → merge → resolve only
+//   directory-builder validate                 check the instance's config ↔ sources/ integrity
 //   directory-builder webapp                   dev server for the instance's webapp
 //   directory-builder webapp build [--base /x/]  build the webapp → <instance>/webapp/dist/
 
 import { webappBuild, webappDev } from "../src/webapp.js"
 import { Pipeline } from "../src/pipeline.js"
+import { validate } from "../src/validate.js"
 
 const [cmd = "run", ...rest] = process.argv.slice(2)
 const flag = (name) => {
@@ -22,6 +24,11 @@ const commands = {
     run:      () => pipeline.run(),
     ingest:   () => pipeline.ingest(),
     federate: () => pipeline.federate(),
+    validate: async () => {
+        const problems = await validate()
+        if (problems.length) { console.error(problems.join("\n")); process.exit(1) }
+        console.log("instance valid")
+    },
     webapp:   () => {
         if (rest[0] && rest[0] !== "build") {
             console.error(`Unknown webapp subcommand "${rest[0]}" — expected "build" or nothing (dev server)`)

package/package.json

@@ -1,12 +1,13 @@
 {
   "name": "@directory-builder/core",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "description": "Use-case-agnostic engine for config-driven federation pipelines",
   "author": "Civic Data Lab",
   "repository": "github:foederierter-datenpool/directory-builder-core",
   "license": "MIT",
   "type": "module",
   "scripts": {
+    "test": "node --test",
     "example": "cd example && node ../bin/cli.js",
     "webapp": "vite webapp",
     "webapp:build": "vite build webapp"

package/src/index.js

@@ -4,3 +4,4 @@
 export { Pipeline } from "./pipeline.js"
 export { ingest } from "./pipeline/ingest.js"
 export { federate } from "./pipeline/federate.js"
+export { validate } from "./validate.js"

package/src/lift/xml.sparql

@@ -0,0 +1,12 @@
+PREFIX xyz: <http://sparql.xyz/facade-x/data/>
+PREFIX fx: <http://sparql.xyz/facade-x/ns/>
+PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+CONSTRUCT {
+    ?s ?p ?o
+} WHERE {
+    SERVICE <x-sparql-anything:> {
+        fx:properties fx:location ?_location ;
+            fx:media-type "application/xml" .
+        ?s ?p ?o .
+    }
+}

package/src/pipeline/federate.js

@@ -1,5 +1,5 @@
 import { newStore, parser as n3Parser, storeFromTurtles } from "@foerderfunke/sem-ops-utils"
-import { CDP, objectsOf, parseTtl, PATHS, sourceGraph, sourceName, stepIri, stepJournal } from "../utils.js"
+import { CDP, enabledSources, parseTtl, PATHS, sourceGraph, sourceName, stepIri, stepJournal } from "../utils.js"
 import { COMMON_PREFIXES, writeTurtleFile } from "./write-turtle.js"
 import { MAPPED_GRAPH, runMap } from "./steps/map.js"
 import { runClean } from "./steps/clean.js"
@@ -26,7 +26,7 @@ export async function federate(root = process.cwd()) {
     const abs = (p) => path.join(root, p)
     const federationTtl = fs.readFileSync(abs(PATHS.federation), "utf8")
     const defStore = storeFromTurtles([federationTtl, fs.readFileSync(abs(PATHS.matchKnowledge), "utf8")])
-    const sources = objectsOf(parseTtl(federationTtl), `${CDP}hasSource`)
+    const sources = enabledSources(parseTtl(federationTtl))
 
     const store = newStore()
     const journal = stepJournal()

package/src/pipeline/ingest.js

@@ -1,5 +1,5 @@
 import { sparqlSelect, storeFromTurtles } from "@foerderfunke/sem-ops-utils"
-import { CDP, objectsOf, parseTtl, PATHS, sourceName, stepJournal } from "../utils.js"
+import { CDP, enabledSources, parseTtl, PATHS, sourceName, stepJournal } from "../utils.js"
 import { ensureJar, runLift } from "./steps/lift.js"
 import { runFetch } from "./steps/fetch.js"
 import path from "path"
@@ -30,7 +30,7 @@ export async function ingest(root = process.cwd()) {
         if (!facts.has(r.source)) facts.set(r.source, { fetchUrl: r.fetchUrl, format: r.format, params: [] })
         if (r.paramName) facts.get(r.source).params.push([r.paramName, r.paramValue])
     }
-    const sources = new Map(objectsOf(parseTtl(federationTtl), `${CDP}hasSource`).map((iri) => [iri, facts.get(iri)]))
+    const sources = new Map(enabledSources(parseTtl(federationTtl)).map((iri) => [iri, facts.get(iri)]))
     for (const [iri, s] of sources) {
         if (!s.format) throw new Error(`${iri} declares no :format (needed to pick the lift query)`)
     }

package/src/pipeline/steps/fetch.js

@@ -11,6 +11,8 @@ export const runFetch = ({ abs, root }, { name, fetchUrl, paramsJson }) => {
     const outDir = PATHS.raw(name)
     const origin = fetchUrl ?? abs(PATHS.staticDir(name))
     console.log(`fetch  ${fetchUrl ?? PATHS.staticDir(name)} (params ${paramsJson}) → ${outDir}`)
+    // Clear any prior output first, so changed run params (or changed records) can't leave stale files behind
+    fs.rmSync(abs(outDir), { recursive: true, force: true })
     fs.mkdirSync(abs(outDir), { recursive: true })
     run("node", [abs(PATHS.fetchScript(name)), abs(outDir), origin, paramsJson])
     const harvest = { time: new Date().toISOString() }

package/src/pipeline/steps/lift.js

@@ -49,6 +49,8 @@ export const runLift = ({ abs }, { jar, name, format, params }) => {
     const inAbs = abs(PATHS.raw(name))
     const outAbs = abs(PATHS.lifted(name))
     const files = fs.readdirSync(inAbs).filter(f => !f.startsWith(".")).sort()
+    // Clear stale lifted files first — the clean step reads every .ttl here.
+    fs.rmSync(outAbs, { recursive: true, force: true })
     fs.mkdirSync(outAbs, { recursive: true })
     console.log(`lift   ${PATHS.raw(name)} (${files.length} files) → ${PATHS.lifted(name)}`)
     for (const f of files) {

package/src/pipeline.js

@@ -1,5 +1,6 @@
 import { ingest } from "./pipeline/ingest.js"
 import { federate } from "./pipeline/federate.js"
+import { validate } from "./validate.js"
 
 // Programmatic entry: hold the instance root once, run the engines against it.
 // The CLI (bin/cli.js) is this same class with defaults — root = cwd.
@@ -7,8 +8,12 @@ export class Pipeline {
     constructor({ root = process.cwd() } = {}) {
         this.root = root
     }
-    ingest()   { return ingest(this.root) }
-    federate() { return federate(this.root) }
+    async validate() {
+        const problems = await validate(this.root)
+        if (problems.length) throw new Error(`invalid instance at ${this.root}:\n  ${problems.join("\n  ")}`)
+    }
+    async ingest()   { await this.validate(); return ingest(this.root) }
+    async federate() { await this.validate(); return federate(this.root) }
     async run() {
         await this.ingest()
         await this.federate()

package/src/utils.js

@@ -108,6 +108,13 @@ export const shrink = (iri, prefixMap) => {
 export const objectsOf = (quads, predIri) =>
     [...new Set(quads.filter((q) => q.predicate.value === predIri).map((q) => q.object.value))]
 
+// The federation's sources minus any switched off with `:enabled false`, in
+// :hasSource declaration order — the source list engines and webapp run on.
+export const enabledSources = (quads) => {
+    const disabled = new Set(quads.filter((q) => q.predicate.value === `${CDP}enabled` && q.object.value === "false").map((q) => q.subject.value))
+    return objectsOf(quads, `${CDP}hasSource`).filter((iri) => !disabled.has(iri))
+}
+
 // Set of subjects typed `rdf:type typeIri`. Iteration order = encounter order.
 export function subjectsOfType(quads, typeIri) {
     const out = new Set()

package/src/validate/federation.shacl.ttl

@@ -0,0 +1,26 @@
+@prefix :   <https://civic-data.de/pipeline#> .
+@prefix sh: <http://www.w3.org/ns/shacl#> .
+
+# The engine's contract for a well-formed federation.ttl
+# expressed as SHACL and enforced by ../validate.js
+
+:federationShape a sh:NodeShape ;
+    sh:targetNode :federation ;
+    sh:property [
+        sh:path :hasSource ;
+        sh:minCount 1 ;
+        sh:nodeKind sh:IRI ;
+        sh:message "needs at least one :hasSource"
+    ] .
+
+:sourceShape a sh:NodeShape ;
+    sh:targetObjectsOf :hasSource ;
+    sh:property [
+        sh:path :format ;
+        sh:minCount 1 ;
+        sh:maxCount 1 ;
+        sh:nodeKind sh:IRI ;
+        sh:message "needs exactly one :format"
+    ] .
+
+# TODO: add the rest

package/src/validate.js

@@ -0,0 +1,52 @@
+import { buildValidator, turtleToDataset } from "@foerderfunke/sem-ops-utils"
+import { CDP, objectsOf, parseTtl, PATHS, shrink, sourceName } from "./utils.js"
+import path from "path"
+import fs from "fs"
+
+// Instance integrity checks. Each check takes { abs, ttl, quads } (path
+// resolver rooted at the instance, federation.ttl raw + parsed) and returns
+// problem strings. validate() runs them all; empty result = valid. Runs
+// automatically before the engines; `directory-builder validate` triggers it
+// on its own.
+
+const checks = [sourcesFoldersInSync, federationConformsToShape]
+
+export async function validate(root = process.cwd()) {
+    const abs = (p) => path.join(root, p)
+    if (!fs.existsSync(abs(PATHS.federation))) return [`${PATHS.federation} missing`]
+    const ttl = fs.readFileSync(abs(PATHS.federation), "utf8")
+    const ctx = { abs, ttl, quads: parseTtl(ttl) }
+    return (await Promise.all(checks.map((check) => check(ctx)))).flat()
+}
+
+// Every :hasSource in federation.ttl has its sources/<name>/ folder with
+// fetch.js + clean.sparql - and no folder exists that the federation doesn't
+// declare. Checks all declared sources, enabled or not: folder presence is a
+// repo-layout contract.
+function sourcesFoldersInSync({ abs, quads }) {
+    const declared = objectsOf(quads, `${CDP}hasSource`).map(sourceName)
+    const problems = []
+    for (const name of declared) {
+        for (const file of [PATHS.fetchScript(name), PATHS.cleanQuery(name)]) {
+            if (!fs.existsSync(abs(file))) problems.push(`${file} missing`)
+        }
+    }
+    const folders = fs.existsSync(abs("sources"))
+        ? fs.readdirSync(abs("sources"), { withFileTypes: true }).filter((d) => d.isDirectory()).map((d) => d.name)
+        : []
+    for (const name of folders) {
+        if (!declared.includes(name)) problems.push(`sources/${name}/ has no :hasSource declaration in ${PATHS.federation}`)
+    }
+    return problems
+}
+
+// federation.ttl conforms to the engine's config contract, expressed as SHACL
+// in federation.shacl.ttl next to this file - the shape ships with the
+// package, instances never carry it.
+const validator = buildValidator(fs.readFileSync(path.join(import.meta.dirname, "validate/federation.shacl.ttl"), "utf8"))
+
+async function federationConformsToShape({ ttl }) {
+    const report = await validator.validate({ dataset: turtleToDataset(ttl) })
+    return report.results.map((r) =>
+        `${PATHS.federation}: ${shrink(r.focusNode.value, { "": CDP })} ${r.message.map((m) => m.value).join("; ")}`)
+}

package/test/validate.test.js

@@ -0,0 +1,14 @@
+import { validate } from "@directory-builder/core"
+import assert from "node:assert/strict"
+import { test } from "node:test"
+import path from "path"
+
+const INSTANCE_ROOT = path.join(import.meta.dirname, "../example")
+
+// The example instance satisfies the contract validate() enforces: every
+// :hasSource in federation.ttl has its sources/<name>/ folder with fetch.js
+// + clean.sparql, no folder exists that the federation doesn't declare, and
+// federation.ttl conforms to the engine's SHACL shape.
+test("validate() finds no problems in the example instance", async () => {
+    assert.deepEqual(await validate(INSTANCE_ROOT), [])
+})

package/webapp/src/instanceData.js

@@ -6,7 +6,7 @@
 // artifact resolves to "" (pages render empty). Top-level await — importing
 // modules stay synchronous.
 
-import { CDP, objectsOf, parseTtl, PATHS, prefixesOf, sourceName } from "@directory-builder/core/utils"
+import { CDP, enabledSources, objectsOf, parseTtl, PATHS, prefixesOf, sourceName } from "@directory-builder/core/utils"
 
 const fetchText = async (path) => {
     const res = await fetch(`${import.meta.env.BASE_URL}${path}`).catch(() => null)
@@ -16,7 +16,7 @@ const fetchText = async (path) => {
 export const federationTtl = await fetchText(PATHS.federation)
 
 const fedQuads = parseTtl(federationTtl)
-const cleanedPaths = objectsOf(fedQuads, `${CDP}hasSource`).map((iri) => PATHS.cleaned(sourceName(iri)))
+const cleanedPaths = enabledSources(fedQuads).map((iri) => PATHS.cleaned(sourceName(iri)))
 // The instance's repo URL (:federation :repository …) — undefined when not
 // declared; pages hide their GitHub links then.
 export const repositoryUrl = objectsOf(fedQuads, `${CDP}repository`)[0]

package/webapp/src/loadSources.js

@@ -3,7 +3,7 @@
 // Reads:  federation, mapped, ingest-log TTL strings passed by Sources.jsx
 // Does:   returns source[] ({iri, label, format, totalFields, mappedFields, records, …})
 
-import { CDP as NS, formatFamily, parseTtl, PATHS, sourceName, subjectsOfType } from "@directory-builder/core/utils"
+import { CDP as NS, enabledSources, formatFamily, parseTtl, PATHS, sourceName } from "@directory-builder/core/utils"
 
 const PROV_AT_TIME = "http://www.w3.org/ns/prov#atTime"
 const RDFS_LABEL = "http://www.w3.org/2000/01/rdf-schema#label"
@@ -18,7 +18,7 @@ export function loadSources(federationTtl, mappedTtl, ingestLogTtl) {
     const mappedQuads = mappedTtl ? parseTtl(mappedTtl) : []
     const logQuads = ingestLogTtl ? parseTtl(ingestLogTtl) : []
 
-    const sourceIris = subjectsOfType(fedQuads, `${NS}Source`)
+    const sourceIris = new Set(enabledSources(fedQuads))
 
     const props = new Map()
     const get = (iri) => {