@bpmn-ld/bpmn-ld 0.0.0-stage → 0.1.0

package/dist/coverage.js

@@ -0,0 +1,152 @@
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { BPMN_NAMESPACE, BPMNDI_NAMESPACE } from "./namespace.js";
+import { declaredClasses, declaredProperties, domainAlternatives, loadOntology, superClasses } from "./ontology.js";
+import { loadXsdTypes } from "./coverage-xsd.js";
+import { REAL_SCOPE } from "./coverage-scope.js";
+import { BUILT_IN_PROPERTIES, CLASS_ALIASES, PROPERTY_ALIASES } from "./coverage-mappings.js";
+const PREFIXES = [
+    [BPMN_NAMESPACE, "bpmn:"],
+    [BPMNDI_NAMESPACE, "bpmndi:"],
+];
+const compact = (iri) => {
+    for (const [namespace, prefix] of PREFIXES)
+        if (iri.startsWith(namespace))
+            return prefix + iri.slice(namespace.length);
+    return iri;
+};
+const semantic = (type) => type.file === "Semantic.xsd";
+const stripPrefix = (name) => name.replace(/^t(?=[A-Z])/, "");
+const idName = (type) => (semantic(type) ? stripPrefix(type.name) : type.name);
+const localClassName = (type) => CLASS_ALIASES[type.name] ?? idName(type);
+const namespaceOf = (type) => (semantic(type) ? BPMN_NAMESPACE : BPMNDI_NAMESPACE);
+const groupOf = (groups, name) => Object.entries(groups).find(([, members]) => members.includes(name))?.[0];
+const emptyCounts = () => ({ covered: 0, gap: 0, "out-of-scope": 0 });
+const propertyCovered = (store, classIri, property) => {
+    const ancestors = new Set([classIri, ...superClasses(store, classIri)]);
+    const statements = domainAlternatives(store, property);
+    return statements.length === 0 || statements.some((alternatives) => alternatives.some((c) => ancestors.has(c)));
+};
+const reasonOf = (scope, group) => {
+    const reason = scope.outOfScopeReasons?.[group];
+    return reason ? { reason } : {};
+};
+export const checkCoverage = (store, types, scope) => {
+    const classes = declaredClasses(store);
+    const properties = declaredProperties(store);
+    const items = [];
+    const unclassified = [];
+    for (const type of types) {
+        const local = idName(type);
+        const classIri = namespaceOf(type) + localClassName(type);
+        const xsd = `${type.file}#${type.name}`;
+        if (!scope.inScope.includes(type.name)) {
+            const group = groupOf(scope.outOfScopeTypes, type.name);
+            if (group)
+                items.push({ id: local, kind: "class", xsd, ontology: null, status: "out-of-scope", group, ...reasonOf(scope, group) });
+            else
+                unclassified.push(xsd);
+            continue;
+        }
+        const covered = classes.has(classIri);
+        items.push({ id: local, kind: "class", xsd, ontology: compact(classIri), status: covered ? "covered" : "gap" });
+        for (const member of type.members) {
+            const memberXsd = `${xsd}/${member.kind === "attribute" ? "@" : ""}${member.name}`;
+            const id = `${local}.${member.name}`;
+            const builtIn = BUILT_IN_PROPERTIES[member.name];
+            if (builtIn) {
+                items.push({ id, kind: "property", xsd: memberXsd, ontology: builtIn, status: "covered" });
+                continue;
+            }
+            const alias = PROPERTY_ALIASES[`${local}.${member.name}`] ?? PROPERTY_ALIASES[member.name] ?? member.name;
+            const candidates = [BPMN_NAMESPACE + alias, BPMNDI_NAMESPACE + alias].filter((iri) => properties.has(iri));
+            const hit = candidates.find((iri) => propertyCovered(store, classIri, iri));
+            if (hit) {
+                items.push({ id, kind: "property", xsd: memberXsd, ontology: compact(hit), status: "covered" });
+                continue;
+            }
+            const group = groupOf(scope.outOfScopeProperties, member.name);
+            if (group)
+                items.push({ id, kind: "property", xsd: memberXsd, ontology: null, status: "out-of-scope", group, ...reasonOf(scope, group) });
+            else
+                items.push({ id, kind: "property", xsd: memberXsd, ontology: null, status: "gap" });
+        }
+    }
+    items.sort((a, b) => (a.kind === b.kind ? a.id.localeCompare(b.id) : a.kind === "class" ? -1 : 1));
+    unclassified.sort();
+    const summary = { classes: emptyCounts(), properties: emptyCounts(), total: emptyCounts() };
+    for (const item of items) {
+        summary[item.kind === "class" ? "classes" : "properties"][item.status]++;
+        summary.total[item.status]++;
+    }
+    return { summary, unclassified, items };
+};
+const table = (rows, columns, cell) => rows.length
+    ? [
+        `| ${columns.join(" | ")} |`,
+        `| ${columns.map(() => "---").join(" | ")} |`,
+        ...rows.map((item) => `| ${cell(item).join(" | ")} |`),
+    ].join("\n")
+    : "_None._";
+export const renderMarkdown = (report) => {
+    const { summary, items } = report;
+    const inScope = items.filter((i) => i.status !== "out-of-scope");
+    const gaps = items.filter((i) => i.status === "gap");
+    const covered = items.filter((i) => i.status === "covered");
+    const out = items.filter((i) => i.status === "out-of-scope");
+    const groups = [...new Set(out.map((i) => i.group))].sort();
+    const lines = [
+        "# BPMN coverage report",
+        "",
+        "Generated by `npm run coverage` from the normative BPMN 2.0.2 XSDs in `reference/bpmn-xsd/` and the ontology in `ontology/`. Do not edit by hand.",
+        "",
+        "## Summary",
+        "",
+        "| Kind | Covered | Gaps | Out of scope |",
+        "| --- | ---: | ---: | ---: |",
+        ...["classes", "properties", "total"].map((k) => `| ${k} | ${summary[k].covered} | ${summary[k].gap} | ${summary[k]["out-of-scope"]} |`),
+        "",
+        `In scope: ${inScope.length} items, ${summary.total.covered} covered and ${summary.total.gap} gaps.`,
+        "",
+        `## Gaps (${gaps.length})`,
+        "",
+        table(gaps, ["Kind", "Item", "XSD"], (i) => [i.kind, `\`${i.id}\``, `\`${i.xsd}\``]),
+        "",
+        `## Covered (${covered.length})`,
+        "",
+        table(covered, ["Kind", "Item", "XSD", "Ontology"], (i) => [i.kind, `\`${i.id}\``, `\`${i.xsd}\``, `\`${i.ontology}\``]),
+        "",
+        `## Out of scope (${out.length})`,
+        "",
+        ...groups.flatMap((group) => {
+            const members = out.filter((i) => i.group === group);
+            const reason = members.find((i) => i.reason)?.reason;
+            return [
+                `### ${group} (${members.length})`,
+                "",
+                ...(reason ? [reason, ""] : []),
+                members.map((i) => `\`${i.kind === "class" ? i.xsd : i.id}\``).join(", "),
+                "",
+            ];
+        }),
+    ];
+    if (report.unclassified.length) {
+        lines.push(`## Unclassified (${report.unclassified.length})`, "", report.unclassified.map((u) => `\`${u}\``).join(", "), "");
+    }
+    return lines.join("\n").replace(/\n+$/, "\n");
+};
+export const renderJson = (report) => `${JSON.stringify(report, null, 2)}\n`;
+export const generateReports = (root) => {
+    const report = checkCoverage(loadOntology(), loadXsdTypes(root), REAL_SCOPE);
+    return { report, markdown: renderMarkdown(report), json: renderJson(report) };
+};
+export const REPORT_MARKDOWN = "reports/coverage.md";
+export const REPORT_JSON = "reports/coverage.json";
+export const writeReports = (root) => {
+    const { report, markdown, json } = generateReports(root);
+    mkdirSync(join(root, "reports"), { recursive: true });
+    writeFileSync(join(root, REPORT_MARKDOWN), markdown);
+    writeFileSync(join(root, REPORT_JSON), json);
+    return report;
+};
+export const readReport = (root, file) => readFileSync(join(root, file), "utf8");

package/dist/docs-cli.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/docs-cli.js

@@ -0,0 +1,18 @@
+import { checkVocabulary, writeVocabulary } from "./docs.js";
+const args = process.argv.slice(2);
+const check = args.includes("--check");
+const path = args.find((arg) => !arg.startsWith("--"));
+if (check) {
+    const result = checkVocabulary(path);
+    if (result.ok) {
+        console.log(result.message);
+    }
+    else {
+        console.error(result.message);
+        process.exitCode = 1;
+    }
+}
+else {
+    writeVocabulary(path);
+    console.log(`wrote ${path ?? "docs/reference/vocabulary.md"}`);
+}

package/dist/docs.d.ts

@@ -0,0 +1,8 @@
+import type { Store } from "n3";
+export declare const VOCABULARY_PATH = "docs/reference/vocabulary.md";
+export declare const renderVocabulary: (store?: Store) => string;
+export declare const writeVocabulary: (path?: string, store?: Store) => void;
+export declare const checkVocabulary: (path?: string, store?: Store) => {
+    ok: boolean;
+    message: string;
+};

package/dist/docs.js

@@ -0,0 +1,136 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { BPMN_NAMESPACE, BPMNDI_NAMESPACE } from "./namespace.js";
+import { ROOT } from "./paths.js";
+import { rangeAlternatives, declaredClasses, declaredIndividuals, declaredProperties, domainAlternatives, isSubClassOf, loadOntology, typesOf, } from "./ontology.js";
+export const VOCABULARY_PATH = "docs/reference/vocabulary.md";
+const RDFS = "http://www.w3.org/2000/01/rdf-schema#";
+const OWL_OBJECT_PROPERTY = "http://www.w3.org/2002/07/owl#ObjectProperty";
+const XSD = "http://www.w3.org/2001/XMLSchema#";
+const MODULES = [
+    { title: "Core", prefix: "bpmn", namespace: BPMN_NAMESPACE },
+    { title: "Diagram interchange", prefix: "bpmndi", namespace: BPMNDI_NAMESPACE },
+];
+const compact = (iri) => {
+    const module = MODULES.find((m) => iri.startsWith(m.namespace));
+    if (module)
+        return `${module.prefix}:${iri.slice(module.namespace.length)}`;
+    return iri.startsWith(XSD) ? `xsd:${iri.slice(XSD.length)}` : iri;
+};
+const byCompact = (a, b) => compact(a).localeCompare(compact(b));
+const cell = (text) => text.replace(/\|/g, "\\|").replace(/\s+/g, " ").trim();
+const comment = (store, iri) => store.getObjects(iri, `${RDFS}comment`, null).map((o) => o.value).sort().join(" ") || "No definition.";
+const code = (iri) => `\`${compact(iri)}\``;
+const list = (iris) => (iris.length ? [...iris].sort(byCompact).map(code).join(", ") : "none");
+const directSuperClasses = (store, cls) => store.getObjects(cls, `${RDFS}subClassOf`, null).filter((o) => o.termType === "NamedNode").map((o) => o.value);
+const ancestorsOf = (store, cls) => {
+    const result = new Set();
+    const walk = (current) => {
+        for (const parent of directSuperClasses(store, current)) {
+            if (!result.has(parent)) {
+                result.add(parent);
+                walk(parent);
+            }
+        }
+    };
+    walk(cls);
+    return [...result];
+};
+const domainText = (store, property) => {
+    const alternatives = domainAlternatives(store, property);
+    return alternatives.length
+        ? alternatives.map((members) => [...members].sort(byCompact).map(code).join(" or ")).join("; and ")
+        : "unspecified";
+};
+const rangeText = (store, property) => {
+    const ranges = rangeAlternatives(store, property);
+    return ranges.length ? ranges.map((members) => [...members].sort(byCompact).map(code).join(" or ")).sort().join(", ") : "unspecified";
+};
+const appliesTo = (store, property, cls) => domainAlternatives(store, property).some((members) => members.some((member) => isSubClassOf(store, cls, member)));
+const renderClass = (store, cls, properties) => {
+    const subClasses = [...declaredClasses(store)].filter((c) => directSuperClasses(store, c).includes(cls));
+    const applicable = properties.filter((p) => appliesTo(store, p, cls));
+    const lines = [
+        `#### ${code(cls)}`,
+        "",
+        comment(store, cls),
+        "",
+        `- Superclasses: ${list(directSuperClasses(store, cls))}`,
+        `- All ancestors: ${list(ancestorsOf(store, cls))}`,
+        `- Subclasses: ${list(subClasses)}`,
+        "",
+    ];
+    if (applicable.length) {
+        lines.push("| Property | Range | Definition |", "| --- | --- | --- |");
+        for (const property of applicable) {
+            lines.push(`| ${code(property)} | ${rangeText(store, property)} | ${cell(comment(store, property))} |`);
+        }
+        lines.push("");
+    }
+    else {
+        lines.push("No properties apply.", "");
+    }
+    return lines;
+};
+const renderProperty = (store, property) => {
+    const kind = typesOf(store, property).includes(OWL_OBJECT_PROPERTY) ? "object" : "datatype";
+    return [
+        `#### ${code(property)}`,
+        "",
+        comment(store, property),
+        "",
+        `- Kind: ${kind}`,
+        `- Domain: ${domainText(store, property)}`,
+        `- Range: ${rangeText(store, property)}`,
+        "",
+    ];
+};
+const renderIndividual = (store, individual) => {
+    const classes = typesOf(store, individual).filter((t) => declaredClasses(store).has(t));
+    return [`#### ${code(individual)}`, "", comment(store, individual), "", `- Type: ${list(classes)}`, ""];
+};
+export const renderVocabulary = (store = loadOntology()) => {
+    const classes = declaredClasses(store);
+    const properties = declaredProperties(store);
+    const individuals = declaredIndividuals(store);
+    const lines = [
+        "# Vocabulary reference",
+        "",
+        "Generated from the ontology by `npm run docs`. Do not edit by hand.",
+        "",
+        ...MODULES.map((m) => `- \`${m.prefix}:\` expands to \`${m.namespace}\``),
+        "",
+    ];
+    for (const module of MODULES) {
+        const inModule = (iri) => iri.startsWith(module.namespace);
+        const moduleClasses = [...classes].filter(inModule).sort(byCompact);
+        const moduleProperties = [...properties].filter(inModule).sort(byCompact);
+        const moduleIndividuals = [...individuals].filter(inModule).sort(byCompact);
+        const applicableProperties = [...properties].sort(byCompact);
+        lines.push(`## ${module.title} module (\`${module.prefix}:\`)`, "");
+        lines.push("### Classes", "");
+        for (const cls of moduleClasses)
+            lines.push(...renderClass(store, cls, applicableProperties));
+        lines.push("### Properties", "");
+        for (const property of moduleProperties)
+            lines.push(...renderProperty(store, property));
+        if (moduleIndividuals.length) {
+            lines.push("### Named individuals", "");
+            for (const individual of moduleIndividuals)
+                lines.push(...renderIndividual(store, individual));
+        }
+    }
+    return `${lines.join("\n").replace(/\n+$/, "")}\n`;
+};
+export const writeVocabulary = (path = join(ROOT, VOCABULARY_PATH), store) => {
+    mkdirSync(dirname(path), { recursive: true });
+    writeFileSync(path, renderVocabulary(store));
+};
+export const checkVocabulary = (path = join(ROOT, VOCABULARY_PATH), store) => {
+    const expected = renderVocabulary(store);
+    const actual = existsSync(path) ? readFileSync(path, "utf8") : null;
+    if (actual === expected)
+        return { ok: true, message: `${path} is up to date` };
+    const reason = actual === null ? "is missing" : "differs from the ontology";
+    return { ok: false, message: `${path} ${reason}; run npm run docs and commit the result` };
+};

package/dist/eval/checks.d.ts

@@ -0,0 +1,19 @@
+export interface Check {
+    name: string;
+    run: (file: string) => Promise<string[]>;
+}
+export interface Failure {
+    check: string;
+    message: string;
+}
+export interface FileResult {
+    file: string;
+    failures: Failure[];
+}
+export declare const CHECKS: Check[];
+export declare const EXPLANATION_CHECKS: Check[];
+export declare const MODIFICATION_CHECKS: Check[];
+export declare const evaluateFile: (file: string, checks?: Check[]) => Promise<FileResult>;
+export declare const evaluateOutputs: (dir: string, checks?: Check[]) => Promise<FileResult[]>;
+export declare const evaluateExplanations: (dir: string, checks?: Check[]) => Promise<FileResult[]>;
+export declare const evaluateModifications: (dir: string, checks?: Check[]) => Promise<FileResult[]>;

package/dist/eval/checks.js

@@ -0,0 +1,252 @@
+import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { basename, dirname, extname, join, resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import { convert } from "../convert/convert.js";
+import { analyze } from "../analyze/analyze.js";
+import { checkXml } from "../check-xml/check.js";
+import { layoutAndExport } from "../export/layout-export.js";
+import { ExportInputError } from "../export/export.js";
+import { layoutDocument, LayoutInputError } from "../layout/layout.js";
+import { compileSchema, validateWithSchema } from "../schema-validate.js";
+import { validateShacl } from "../shacl.js";
+import { validateExample } from "../validate.js";
+const WARNING_PREFIX = "warning ";
+const severityPrefix = (severity) => (severity === "error" ? "" : WARNING_PREFIX);
+const validateSchema = compileSchema();
+const structural = {
+    name: "structural",
+    run: async (file) => (await validateExample(file)).errors,
+};
+const schema = {
+    name: "schema",
+    run: async (file) => validateWithSchema(JSON.parse(readFileSync(file, "utf8")), validateSchema),
+};
+const shacl = {
+    name: "shacl",
+    run: async (file) => {
+        const result = await validateShacl(file);
+        const messages = result.violations.map((v) => `${v.focusNode} violates ${v.sourceShape}`);
+        if (!result.conforms && messages.length === 0)
+            messages.push("SHACL validation failed");
+        return messages;
+    },
+};
+const analyzer = {
+    name: "analyzer",
+    run: async (file) => {
+        const findings = await analyze(JSON.parse(readFileSync(file, "utf8")), pathToFileURL(resolve(file)).href);
+        return findings.map((f) => `${severityPrefix(f.severity)}${f.rule} ${f.node ?? "-"}: ${f.message}`);
+    },
+};
+const layout = {
+    name: "layout",
+    run: async (file) => {
+        try {
+            layoutDocument(JSON.parse(readFileSync(file, "utf8")));
+            return [];
+        }
+        catch (error) {
+            if (error instanceof LayoutInputError)
+                return error.errors;
+            throw error;
+        }
+    },
+};
+const exportXml = async (file) => (await layoutAndExport(JSON.parse(readFileSync(file, "utf8")), { documentUrl: pathToFileURL(resolve(file)).href })).xml;
+const exported = {
+    name: "export",
+    run: async (file) => {
+        try {
+            await exportXml(file);
+            return [];
+        }
+        catch (error) {
+            if (error instanceof ExportInputError)
+                return error.errors;
+            throw error;
+        }
+    },
+};
+const reports = new Map();
+const externalReport = (file) => {
+    const key = `${resolve(file)}\n${readFileSync(file, "utf8")}`;
+    let report = reports.get(key);
+    if (!report) {
+        report = exportXml(file).then(checkXml);
+        reports.set(key, report);
+    }
+    return report;
+};
+const external = (name) => ({
+    name,
+    run: async (file) => {
+        const result = (await externalReport(file)).checks[name];
+        return result.issues.map((issue) => `${severityPrefix(issue.severity)}${issue.rule ? `${issue.rule}: ` : ""}${issue.message}`);
+    },
+});
+export const CHECKS = [structural, schema, shacl, analyzer, layout, exported, external("xsd"), external("moddle"), external("lint")];
+const MODEL_LINE = /^Model:\s*(\S+)\s*$/m;
+const collectNames = (value, names) => {
+    if (Array.isArray(value))
+        value.forEach((item) => collectNames(item, names));
+    else if (value && typeof value === "object")
+        for (const [key, child] of Object.entries(value)) {
+            if (key === "name" && typeof child === "string")
+                names.add(child);
+            else
+                collectNames(child, names);
+        }
+};
+const modelNames = (modelFile) => {
+    const text = readFileSync(modelFile, "utf8");
+    const document = extname(modelFile) === ".bpmn" ? convert(text, { fileName: modelFile }).document : JSON.parse(text);
+    const names = new Set();
+    collectNames(document, names);
+    return [...names];
+};
+const normalize = (text) => text.replace(/\s+/g, " ").toLowerCase();
+const mentionsElements = {
+    name: "mentions-elements",
+    run: async (file) => {
+        const evalsDir = dirname(dirname(file));
+        const prompt = readFileSync(join(evalsDir, "prompts", `${basename(file, ".md")}.md`), "utf8");
+        const model = prompt.match(MODEL_LINE)?.[1];
+        if (!model)
+            return ["prompt has no Model line"];
+        const explanation = normalize(readFileSync(file, "utf8"));
+        return modelNames(resolve(dirname(evalsDir), model))
+            .filter((name) => !explanation.includes(normalize(name)))
+            .map((name) => `does not mention "${name}"`);
+    },
+};
+export const EXPLANATION_CHECKS = [mentionsElements];
+const ALLOWED_LINE = /^Allowed-changes:[ \t]*(.*)$/m;
+const DI_TYPES = new Set(["BPMNDiagram", "BPMNPlane", "BPMNShape", "BPMNEdge", "BPMNLabel", "Bounds", "Point"]);
+const loadDocument = (modelFile) => {
+    const text = readFileSync(modelFile, "utf8");
+    return extname(modelFile) === ".bpmn" ? convert(text, { fileName: modelFile }).document : JSON.parse(text);
+};
+const isNode = (value) => !!value && typeof value === "object" && !Array.isArray(value);
+const isDi = (node) => DI_TYPES.has(String(node.type));
+const indexNodes = (value, elements, di) => {
+    if (Array.isArray(value))
+        value.forEach((item) => indexNodes(item, elements, di));
+    else if (isNode(value)) {
+        if (typeof value.id === "string")
+            (isDi(value) ? di : elements).set(value.id, value);
+        for (const [key, child] of Object.entries(value))
+            if (key !== "@context")
+                indexNodes(child, elements, di);
+    }
+};
+const ownProperties = (node) => {
+    const strip = (value) => {
+        if (Array.isArray(value))
+            return value.map(strip).filter((item) => item !== undefined);
+        if (isNode(value))
+            return typeof value.id === "string" ? undefined : Object.fromEntries(Object.entries(value).map(([k, v]) => [k, strip(v)]));
+        return value;
+    };
+    return JSON.stringify(Object.fromEntries(Object.entries(node).filter(([key]) => key !== "id").map(([key, value]) => [key, strip(value)]).sort(([a], [b]) => a.localeCompare(b))));
+};
+const modificationContext = (file) => {
+    const evalsDir = dirname(dirname(file));
+    const prompt = readFileSync(join(evalsDir, "prompts", `${basename(file, ".jsonld")}.md`), "utf8");
+    const model = prompt.match(MODEL_LINE)?.[1];
+    if (!model)
+        throw new Error("prompt has no Model line");
+    const allowedText = prompt.match(ALLOWED_LINE)?.[1];
+    if (allowedText === undefined)
+        throw new Error("prompt has no Allowed-changes line");
+    const allowed = new Set(allowedText.split(",").map((id) => id.trim()).filter(Boolean));
+    const before = { elements: new Map(), di: new Map() };
+    const after = { elements: new Map(), di: new Map() };
+    indexNodes(loadDocument(resolve(dirname(evalsDir), model)), before.elements, before.di);
+    indexNodes(JSON.parse(readFileSync(file, "utf8")), after.elements, after.di);
+    return { allowed, before, after };
+};
+const unchangedElements = {
+    name: "unchanged-elements",
+    run: async (file) => {
+        const { allowed, before, after } = modificationContext(file);
+        const failures = [];
+        for (const [id, node] of before.elements) {
+            if (allowed.has(id))
+                continue;
+            const next = after.elements.get(id);
+            if (!next)
+                failures.push(`element "${id}" was removed`);
+            else if (ownProperties(node) !== ownProperties(next))
+                failures.push(`element "${id}" changed`);
+        }
+        for (const id of after.elements.keys())
+            if (!before.elements.has(id) && !allowed.has(id))
+                failures.push(`element "${id}" was added`);
+        return failures;
+    },
+};
+const unchangedDi = {
+    name: "unchanged-di",
+    run: async (file) => {
+        const { allowed, before, after } = modificationContext(file);
+        const failures = [];
+        for (const [id, node] of before.di) {
+            if (node.type !== "BPMNShape" && node.type !== "BPMNEdge")
+                continue;
+            const next = after.di.get(id);
+            const element = String(node.bpmnElement);
+            if (!next) {
+                if (!allowed.has(element))
+                    failures.push(`diagram element "${id}" was removed`);
+                continue;
+            }
+            const key = node.type === "BPMNShape" ? "bounds" : "waypoints";
+            if (JSON.stringify(node[key]) !== JSON.stringify(next[key]))
+                failures.push(`diagram element "${id}" changed its ${key}`);
+        }
+        return failures;
+    },
+};
+export const MODIFICATION_CHECKS = [unchangedElements, unchangedDi];
+const loadExpected = (file) => {
+    const expectFile = join(dirname(file), `${basename(file, extname(file))}.expect.json`);
+    return existsSync(expectFile) ? JSON.parse(readFileSync(expectFile, "utf8")) : {};
+};
+export const evaluateFile = async (file, checks = CHECKS) => {
+    const failures = [];
+    const expected = loadExpected(file);
+    for (const [name, messages] of Object.entries(expected))
+        for (const message of messages)
+            if (!message.startsWith(WARNING_PREFIX))
+                failures.push({ check: name, message: `error findings cannot be recorded as expected: ${message}` });
+    for (const check of checks) {
+        try {
+            const remaining = [...(expected[check.name] ?? [])];
+            for (const message of await check.run(file)) {
+                const index = remaining.indexOf(message);
+                if (index >= 0)
+                    remaining.splice(index, 1);
+                else
+                    failures.push({ check: check.name, message });
+            }
+            for (const message of remaining)
+                failures.push({ check: check.name, message: `expected finding missing: ${message}` });
+        }
+        catch (error) {
+            failures.push({ check: check.name, message: error.message });
+        }
+    }
+    return { file, failures };
+};
+export const evaluateOutputs = async (dir, checks = CHECKS) => Promise.all(readdirSync(dir)
+    .filter((f) => f.endsWith(".jsonld"))
+    .sort()
+    .map((f) => evaluateFile(join(dir, f), checks)));
+export const evaluateExplanations = async (dir, checks = EXPLANATION_CHECKS) => Promise.all(readdirSync(dir)
+    .filter((f) => f.startsWith("explain-") && f.endsWith(".md"))
+    .sort()
+    .map((f) => evaluateFile(join(dir, f), checks)));
+export const evaluateModifications = async (dir, checks = MODIFICATION_CHECKS) => Promise.all(readdirSync(dir)
+    .filter((f) => (f.startsWith("modify-") || f.startsWith("refine-")) && f.endsWith(".jsonld"))
+    .sort()
+    .map((f) => evaluateFile(join(dir, f), checks)));

package/dist/eval/regenerate.d.ts

@@ -0,0 +1,10 @@
+export declare const EXPLAIN_PREFIX = "explain-";
+export declare const outputExtension: (name: string) => string;
+export type Generator = (prompt: string) => Promise<string>;
+export interface RegenerationEntry {
+    name: string;
+    status: "unchanged" | "changed" | "new";
+    diff: string;
+}
+export declare const lineDiff: (before: string, after: string) => string;
+export declare const regenerate: (dir: string, generate: Generator) => Promise<RegenerationEntry[]>;

package/dist/eval/regenerate.js

@@ -0,0 +1,48 @@
+import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+export const EXPLAIN_PREFIX = "explain-";
+export const outputExtension = (name) => (name.startsWith(EXPLAIN_PREFIX) ? ".md" : ".jsonld");
+export const lineDiff = (before, after) => {
+    const a = before.split("\n");
+    const b = after.split("\n");
+    const lcs = Array.from({ length: a.length + 1 }, () => new Array(b.length + 1).fill(0));
+    for (let i = a.length - 1; i >= 0; i--)
+        for (let j = b.length - 1; j >= 0; j--)
+            lcs[i][j] = a[i] === b[j] ? lcs[i + 1][j + 1] + 1 : Math.max(lcs[i + 1][j], lcs[i][j + 1]);
+    const lines = [];
+    let i = 0;
+    let j = 0;
+    while (i < a.length && j < b.length) {
+        if (a[i] === b[j])
+            (i++, j++);
+        else if (lcs[i + 1][j] >= lcs[i][j + 1])
+            lines.push(`-${a[i++]}`);
+        else
+            lines.push(`+${b[j++]}`);
+    }
+    while (i < a.length)
+        lines.push(`-${a[i++]}`);
+    while (j < b.length)
+        lines.push(`+${b[j++]}`);
+    return lines.join("\n");
+};
+export const regenerate = async (dir, generate) => {
+    const outputs = join(dir, "outputs");
+    mkdirSync(outputs, { recursive: true });
+    const report = [];
+    for (const file of readdirSync(join(dir, "prompts")).filter((f) => f.endsWith(".md")).sort()) {
+        const name = file.replace(/\.md$/, "");
+        const target = join(outputs, `${name}${outputExtension(name)}`);
+        const generated = await generate(readFileSync(join(dir, "prompts", file), "utf8"));
+        const existed = existsSync(target);
+        const previous = existed ? readFileSync(target, "utf8") : "";
+        writeFileSync(target, generated);
+        if (!existed)
+            report.push({ name, status: "new", diff: lineDiff("", generated) });
+        else if (previous === generated)
+            report.push({ name, status: "unchanged", diff: "" });
+        else
+            report.push({ name, status: "changed", diff: lineDiff(previous, generated) });
+    }
+    return report;
+};

package/dist/eval-regenerate-cli.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/eval-regenerate-cli.js

@@ -0,0 +1,16 @@
+import { spawnSync } from "node:child_process";
+import { regenerate } from "./eval/regenerate.js";
+import { ROOT } from "./paths.js";
+const dir = process.argv[2] ?? `${ROOT}skills/bpmn-ld/evals`;
+const command = process.env.BPMN_LD_EVAL_GENERATOR ?? "claude -p";
+const generate = async (prompt) => {
+    const run = spawnSync(command, { input: prompt, encoding: "utf8", shell: true, maxBuffer: 64 * 1024 * 1024 });
+    if (run.status !== 0)
+        throw new Error(`generator "${command}" failed: ${run.stderr}`);
+    return run.stdout;
+};
+for (const entry of await regenerate(dir, generate)) {
+    console.log(`${entry.status.padEnd(9)}${entry.name}`);
+    if (entry.diff)
+        console.log(entry.diff);
+}