@directory-builder/core 0.1.2 → 0.1.3

package/README.md

@@ -88,6 +88,11 @@ npx directory-builder webapp build --base /repo/     # production build → weba
 `webapp/{content,exporters}/` into `webapp/dist/` next to the bundle —
 `webapp/dist/` is the complete site, ready to publish as-is.
 
+The two are independent: the dev server never needs a prior build — `webapp
+build` exists only to produce the deployable. Both show whatever `data/` the
+pipeline last produced, so run the pipeline first (and rebuild before
+publishing, or `dist/` keeps the stale snapshot).
+
 For webapp development in this repo:
 
 ```sh

package/bin/cli.js

@@ -5,11 +5,13 @@
 //   directory-builder                          run the full pipeline (ingest + federate)
 //   directory-builder ingest                   fetch + lift only
 //   directory-builder federate                 clean → map → match → merge → resolve only
+//   directory-builder validate                 check the instance's config ↔ sources/ integrity
 //   directory-builder webapp                   dev server for the instance's webapp
 //   directory-builder webapp build [--base /x/]  build the webapp → <instance>/webapp/dist/
 
 import { webappBuild, webappDev } from "../src/webapp.js"
 import { Pipeline } from "../src/pipeline.js"
+import { validate } from "../src/validate.js"
 
 const [cmd = "run", ...rest] = process.argv.slice(2)
 const flag = (name) => {
@@ -22,6 +24,11 @@ const commands = {
     run:      () => pipeline.run(),
     ingest:   () => pipeline.ingest(),
     federate: () => pipeline.federate(),
+    validate: async () => {
+        const problems = await validate()
+        if (problems.length) { console.error(problems.join("\n")); process.exit(1) }
+        console.log("instance valid")
+    },
     webapp:   () => {
         if (rest[0] && rest[0] !== "build") {
             console.error(`Unknown webapp subcommand "${rest[0]}" — expected "build" or nothing (dev server)`)

package/package.json

@@ -1,12 +1,13 @@
 {
   "name": "@directory-builder/core",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "description": "Use-case-agnostic engine for config-driven federation pipelines",
   "author": "Civic Data Lab",
   "repository": "github:foederierter-datenpool/directory-builder-core",
   "license": "MIT",
   "type": "module",
   "scripts": {
+    "test": "node --test",
     "example": "cd example && node ../bin/cli.js",
     "webapp": "vite webapp",
     "webapp:build": "vite build webapp"

package/src/index.js

@@ -4,3 +4,4 @@
 export { Pipeline } from "./pipeline.js"
 export { ingest } from "./pipeline/ingest.js"
 export { federate } from "./pipeline/federate.js"
+export { validate } from "./validate.js"

package/src/pipeline.js

@@ -1,5 +1,6 @@
 import { ingest } from "./pipeline/ingest.js"
 import { federate } from "./pipeline/federate.js"
+import { validate } from "./validate.js"
 
 // Programmatic entry: hold the instance root once, run the engines against it.
 // The CLI (bin/cli.js) is this same class with defaults — root = cwd.
@@ -7,8 +8,12 @@ export class Pipeline {
     constructor({ root = process.cwd() } = {}) {
         this.root = root
     }
-    ingest()   { return ingest(this.root) }
-    federate() { return federate(this.root) }
+    async validate() {
+        const problems = await validate(this.root)
+        if (problems.length) throw new Error(`invalid instance at ${this.root}:\n  ${problems.join("\n  ")}`)
+    }
+    async ingest()   { await this.validate(); return ingest(this.root) }
+    async federate() { await this.validate(); return federate(this.root) }
     async run() {
         await this.ingest()
         await this.federate()

package/src/validate/federation.shacl.ttl

@@ -0,0 +1,26 @@
+@prefix :   <https://civic-data.de/pipeline#> .
+@prefix sh: <http://www.w3.org/ns/shacl#> .
+
+# The engine's contract for a well-formed federation.ttl
+# expressed as SHACL and enforced by ../validate.js
+
+:federationShape a sh:NodeShape ;
+    sh:targetNode :federation ;
+    sh:property [
+        sh:path :hasSource ;
+        sh:minCount 1 ;
+        sh:nodeKind sh:IRI ;
+        sh:message "needs at least one :hasSource"
+    ] .
+
+:sourceShape a sh:NodeShape ;
+    sh:targetObjectsOf :hasSource ;
+    sh:property [
+        sh:path :format ;
+        sh:minCount 1 ;
+        sh:maxCount 1 ;
+        sh:nodeKind sh:IRI ;
+        sh:message "needs exactly one :format"
+    ] .
+
+# TODO: add the rest

package/src/validate.js

@@ -0,0 +1,52 @@
+import { buildValidator, turtleToDataset } from "@foerderfunke/sem-ops-utils"
+import { CDP, objectsOf, parseTtl, PATHS, shrink, sourceName } from "./utils.js"
+import path from "path"
+import fs from "fs"
+
+// Instance integrity checks. Each check takes { abs, ttl, quads } (path
+// resolver rooted at the instance, federation.ttl raw + parsed) and returns
+// problem strings. validate() runs them all; empty result = valid. Runs
+// automatically before the engines; `directory-builder validate` triggers it
+// on its own.
+
+const checks = [sourcesFoldersInSync, federationConformsToShape]
+
+export async function validate(root = process.cwd()) {
+    const abs = (p) => path.join(root, p)
+    if (!fs.existsSync(abs(PATHS.federation))) return [`${PATHS.federation} missing`]
+    const ttl = fs.readFileSync(abs(PATHS.federation), "utf8")
+    const ctx = { abs, ttl, quads: parseTtl(ttl) }
+    return (await Promise.all(checks.map((check) => check(ctx)))).flat()
+}
+
+// Every :hasSource in federation.ttl has its sources/<name>/ folder with
+// fetch.js + clean.sparql - and no folder exists that the federation doesn't
+// declare. Checks all declared sources, enabled or not: folder presence is a
+// repo-layout contract.
+function sourcesFoldersInSync({ abs, quads }) {
+    const declared = objectsOf(quads, `${CDP}hasSource`).map(sourceName)
+    const problems = []
+    for (const name of declared) {
+        for (const file of [PATHS.fetchScript(name), PATHS.cleanQuery(name)]) {
+            if (!fs.existsSync(abs(file))) problems.push(`${file} missing`)
+        }
+    }
+    const folders = fs.existsSync(abs("sources"))
+        ? fs.readdirSync(abs("sources"), { withFileTypes: true }).filter((d) => d.isDirectory()).map((d) => d.name)
+        : []
+    for (const name of folders) {
+        if (!declared.includes(name)) problems.push(`sources/${name}/ has no :hasSource declaration in ${PATHS.federation}`)
+    }
+    return problems
+}
+
+// federation.ttl conforms to the engine's config contract, expressed as SHACL
+// in federation.shacl.ttl next to this file - the shape ships with the
+// package, instances never carry it.
+const validator = buildValidator(fs.readFileSync(path.join(import.meta.dirname, "validate/federation.shacl.ttl"), "utf8"))
+
+async function federationConformsToShape({ ttl }) {
+    const report = await validator.validate({ dataset: turtleToDataset(ttl) })
+    return report.results.map((r) =>
+        `${PATHS.federation}: ${shrink(r.focusNode.value, { "": CDP })} ${r.message.map((m) => m.value).join("; ")}`)
+}

package/test/validate.test.js

@@ -0,0 +1,14 @@
+import { validate } from "@directory-builder/core"
+import assert from "node:assert/strict"
+import { test } from "node:test"
+import path from "path"
+
+const INSTANCE_ROOT = path.join(import.meta.dirname, "../example")
+
+// The example instance satisfies the contract validate() enforces: every
+// :hasSource in federation.ttl has its sources/<name>/ folder with fetch.js
+// + clean.sparql, no folder exists that the federation doesn't declare, and
+// federation.ttl conforms to the engine's SHACL shape.
+test("validate() finds no problems in the example instance", async () => {
+    assert.deepEqual(await validate(INSTANCE_ROOT), [])
+})