@directory-builder/core 0.1.1 → 0.1.2

package/README.md

@@ -66,6 +66,10 @@ Each source's `fetch.js` is invoked as `node fetch.js <outDir> <fetchUrl-or-stat
 <runParamsJson>` — the JSON holds all `:hasRunParam` values grouped by name;
 each fetcher picks the parameters it needs.
 
+A source declared with `:enabled false` stays in the config but is skipped by
+the engines and hidden from the webapp's Sources page — e.g. while its files
+aren't available yet.
+
 Engines journal their executed steps as p-plan RDF (`data/ingest/ingest-log.ttl`,
 `data/pipeline/federate-log.ttl`) — evidence of what ran, not a plan.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@directory-builder/core",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "description": "Use-case-agnostic engine for config-driven federation pipelines",
   "author": "Civic Data Lab",
   "repository": "github:foederierter-datenpool/directory-builder-core",

package/src/lift/xml.sparql

@@ -0,0 +1,12 @@
+PREFIX xyz: <http://sparql.xyz/facade-x/data/>
+PREFIX fx: <http://sparql.xyz/facade-x/ns/>
+PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+CONSTRUCT {
+    ?s ?p ?o
+} WHERE {
+    SERVICE <x-sparql-anything:> {
+        fx:properties fx:location ?_location ;
+            fx:media-type "application/xml" .
+        ?s ?p ?o .
+    }
+}

package/src/pipeline/federate.js

@@ -1,5 +1,5 @@
 import { newStore, parser as n3Parser, storeFromTurtles } from "@foerderfunke/sem-ops-utils"
-import { CDP, objectsOf, parseTtl, PATHS, sourceGraph, sourceName, stepIri, stepJournal } from "../utils.js"
+import { CDP, enabledSources, parseTtl, PATHS, sourceGraph, sourceName, stepIri, stepJournal } from "../utils.js"
 import { COMMON_PREFIXES, writeTurtleFile } from "./write-turtle.js"
 import { MAPPED_GRAPH, runMap } from "./steps/map.js"
 import { runClean } from "./steps/clean.js"
@@ -26,7 +26,7 @@ export async function federate(root = process.cwd()) {
     const abs = (p) => path.join(root, p)
     const federationTtl = fs.readFileSync(abs(PATHS.federation), "utf8")
     const defStore = storeFromTurtles([federationTtl, fs.readFileSync(abs(PATHS.matchKnowledge), "utf8")])
-    const sources = objectsOf(parseTtl(federationTtl), `${CDP}hasSource`)
+    const sources = enabledSources(parseTtl(federationTtl))
 
     const store = newStore()
     const journal = stepJournal()

package/src/pipeline/ingest.js

@@ -1,5 +1,5 @@
 import { sparqlSelect, storeFromTurtles } from "@foerderfunke/sem-ops-utils"
-import { CDP, objectsOf, parseTtl, PATHS, sourceName, stepJournal } from "../utils.js"
+import { CDP, enabledSources, parseTtl, PATHS, sourceName, stepJournal } from "../utils.js"
 import { ensureJar, runLift } from "./steps/lift.js"
 import { runFetch } from "./steps/fetch.js"
 import path from "path"
@@ -30,7 +30,7 @@ export async function ingest(root = process.cwd()) {
         if (!facts.has(r.source)) facts.set(r.source, { fetchUrl: r.fetchUrl, format: r.format, params: [] })
         if (r.paramName) facts.get(r.source).params.push([r.paramName, r.paramValue])
     }
-    const sources = new Map(objectsOf(parseTtl(federationTtl), `${CDP}hasSource`).map((iri) => [iri, facts.get(iri)]))
+    const sources = new Map(enabledSources(parseTtl(federationTtl)).map((iri) => [iri, facts.get(iri)]))
     for (const [iri, s] of sources) {
         if (!s.format) throw new Error(`${iri} declares no :format (needed to pick the lift query)`)
     }

package/src/pipeline/steps/fetch.js

@@ -11,6 +11,8 @@ export const runFetch = ({ abs, root }, { name, fetchUrl, paramsJson }) => {
     const outDir = PATHS.raw(name)
     const origin = fetchUrl ?? abs(PATHS.staticDir(name))
     console.log(`fetch  ${fetchUrl ?? PATHS.staticDir(name)} (params ${paramsJson}) → ${outDir}`)
+    // Clear any prior output first, so changed run params (or changed records) can't leave stale files behind
+    fs.rmSync(abs(outDir), { recursive: true, force: true })
     fs.mkdirSync(abs(outDir), { recursive: true })
     run("node", [abs(PATHS.fetchScript(name)), abs(outDir), origin, paramsJson])
     const harvest = { time: new Date().toISOString() }

package/src/pipeline/steps/lift.js

@@ -49,6 +49,8 @@ export const runLift = ({ abs }, { jar, name, format, params }) => {
     const inAbs = abs(PATHS.raw(name))
     const outAbs = abs(PATHS.lifted(name))
     const files = fs.readdirSync(inAbs).filter(f => !f.startsWith(".")).sort()
+    // Clear stale lifted files first — the clean step reads every .ttl here.
+    fs.rmSync(outAbs, { recursive: true, force: true })
     fs.mkdirSync(outAbs, { recursive: true })
     console.log(`lift   ${PATHS.raw(name)} (${files.length} files) → ${PATHS.lifted(name)}`)
     for (const f of files) {

package/src/utils.js

@@ -108,6 +108,13 @@ export const shrink = (iri, prefixMap) => {
 export const objectsOf = (quads, predIri) =>
     [...new Set(quads.filter((q) => q.predicate.value === predIri).map((q) => q.object.value))]
 
+// The federation's sources minus any switched off with `:enabled false`, in
+// :hasSource declaration order — the source list engines and webapp run on.
+export const enabledSources = (quads) => {
+    const disabled = new Set(quads.filter((q) => q.predicate.value === `${CDP}enabled` && q.object.value === "false").map((q) => q.subject.value))
+    return objectsOf(quads, `${CDP}hasSource`).filter((iri) => !disabled.has(iri))
+}
+
 // Set of subjects typed `rdf:type typeIri`. Iteration order = encounter order.
 export function subjectsOfType(quads, typeIri) {
     const out = new Set()

package/webapp/src/instanceData.js

@@ -6,7 +6,7 @@
 // artifact resolves to "" (pages render empty). Top-level await — importing
 // modules stay synchronous.
 
-import { CDP, objectsOf, parseTtl, PATHS, prefixesOf, sourceName } from "@directory-builder/core/utils"
+import { CDP, enabledSources, objectsOf, parseTtl, PATHS, prefixesOf, sourceName } from "@directory-builder/core/utils"
 
 const fetchText = async (path) => {
     const res = await fetch(`${import.meta.env.BASE_URL}${path}`).catch(() => null)
@@ -16,7 +16,7 @@ const fetchText = async (path) => {
 export const federationTtl = await fetchText(PATHS.federation)
 
 const fedQuads = parseTtl(federationTtl)
-const cleanedPaths = objectsOf(fedQuads, `${CDP}hasSource`).map((iri) => PATHS.cleaned(sourceName(iri)))
+const cleanedPaths = enabledSources(fedQuads).map((iri) => PATHS.cleaned(sourceName(iri)))
 // The instance's repo URL (:federation :repository …) — undefined when not
 // declared; pages hide their GitHub links then.
 export const repositoryUrl = objectsOf(fedQuads, `${CDP}repository`)[0]

package/webapp/src/loadSources.js

@@ -3,7 +3,7 @@
 // Reads:  federation, mapped, ingest-log TTL strings passed by Sources.jsx
 // Does:   returns source[] ({iri, label, format, totalFields, mappedFields, records, …})
 
-import { CDP as NS, formatFamily, parseTtl, PATHS, sourceName, subjectsOfType } from "@directory-builder/core/utils"
+import { CDP as NS, enabledSources, formatFamily, parseTtl, PATHS, sourceName } from "@directory-builder/core/utils"
 
 const PROV_AT_TIME = "http://www.w3.org/ns/prov#atTime"
 const RDFS_LABEL = "http://www.w3.org/2000/01/rdf-schema#label"
@@ -18,7 +18,7 @@ export function loadSources(federationTtl, mappedTtl, ingestLogTtl) {
     const mappedQuads = mappedTtl ? parseTtl(mappedTtl) : []
     const logQuads = ingestLogTtl ? parseTtl(ingestLogTtl) : []
 
-    const sourceIris = subjectsOfType(fedQuads, `${NS}Source`)
+    const sourceIris = new Set(enabledSources(fedQuads))
 
     const props = new Map()
     const get = (iri) => {