@m6d/cortex-cli 0.1.0

package/src/graph/expand-domains.ts

@@ -0,0 +1,288 @@
+import type {
+    ConceptDef,
+    DomainDef,
+    EndpointDef,
+    RuleDef,
+    ServiceDef,
+} from "@cortex/contracts/graph";
+
+type DomainCollections = {
+    concepts: ConceptDef[];
+    endpoints: EndpointDef[];
+    services: ServiceDef[];
+    rules: RuleDef[];
+};
+
+type OwnershipMaps = {
+    concepts: WeakMap<ConceptDef, string>;
+    endpoints: WeakMap<EndpointDef, string>;
+    services: WeakMap<ServiceDef, string>;
+    rules: WeakMap<RuleDef, string>;
+};
+
+export function expandDomains(domains: Record<string, DomainDef>) {
+    const ownership = createOwnershipMaps(domains);
+    const expanded: Record<string, DomainDef> = {};
+
+    for (const [domainKey, domain] of Object.entries(domains)) {
+        const collections = cloneCollections(domain);
+        const pendingConcepts = [...collections.concepts];
+        const pendingEndpoints = [...collections.endpoints];
+        const pendingServices = [...collections.services];
+
+        while (
+            pendingConcepts.length > 0 ||
+            pendingEndpoints.length > 0 ||
+            pendingServices.length > 0
+        ) {
+            const concept = pendingConcepts.pop();
+            if (concept) {
+                includeConcept({
+                    domainKey,
+                    concept,
+                    ownership,
+                    collections,
+                    pendingConcepts,
+                    pendingEndpoints,
+                });
+            }
+
+            const endpoint = pendingEndpoints.pop();
+            if (endpoint) {
+                includeEndpoint({
+                    domainKey,
+                    endpoint,
+                    ownership,
+                    collections,
+                    pendingConcepts,
+                    pendingEndpoints,
+                });
+            }
+
+            const service = pendingServices.pop();
+            if (service) {
+                includeService({
+                    domainKey,
+                    service,
+                    ownership,
+                    collections,
+                    pendingConcepts,
+                });
+            }
+        }
+
+        expanded[domainKey] = {
+            ...domain,
+            concepts: collections.concepts,
+            endpoints: collections.endpoints,
+            services: collections.services,
+            rules: collections.rules,
+        };
+    }
+
+    return expanded;
+}
+
+function createOwnershipMaps(domains: Record<string, DomainDef>) {
+    const ownership: OwnershipMaps = {
+        concepts: new WeakMap<ConceptDef, string>(),
+        endpoints: new WeakMap<EndpointDef, string>(),
+        services: new WeakMap<ServiceDef, string>(),
+        rules: new WeakMap<RuleDef, string>(),
+    };
+
+    for (const [domainKey, domain] of Object.entries(domains)) {
+        for (const concept of domain.concepts ?? []) {
+            assignOwner(ownership.concepts, concept, domainKey);
+        }
+
+        for (const endpoint of domain.endpoints ?? []) {
+            assignOwner(ownership.endpoints, endpoint, domainKey);
+        }
+
+        for (const service of domain.services ?? []) {
+            assignOwner(ownership.services, service, domainKey);
+        }
+
+        for (const rule of domain.rules ?? []) {
+            assignOwner(ownership.rules, rule, domainKey);
+        }
+    }
+
+    return ownership;
+}
+
+function assignOwner<T extends object>(owners: WeakMap<T, string>, value: T, domainKey: string) {
+    if (!owners.has(value)) {
+        owners.set(value, domainKey);
+    }
+}
+
+function cloneCollections(domain: DomainDef) {
+    return {
+        concepts: [...(domain.concepts ?? [])],
+        endpoints: [...(domain.endpoints ?? [])],
+        services: [...(domain.services ?? [])],
+        rules: [...(domain.rules ?? [])],
+    } satisfies DomainCollections;
+}
+
+function includeConcept(options: {
+    domainKey: string;
+    concept: ConceptDef;
+    ownership: OwnershipMaps;
+    collections: DomainCollections;
+    pendingConcepts: ConceptDef[];
+    pendingEndpoints: EndpointDef[];
+}) {
+    const { domainKey, concept, ownership, collections, pendingConcepts, pendingEndpoints } =
+        options;
+
+    includeRule(domainKey, concept.governedBy ?? [], ownership, collections);
+
+    if (shouldAttach(ownership.concepts, concept, domainKey)) {
+        pushUnique(collections.concepts, concept);
+    }
+
+    if (
+        concept.parentConcept &&
+        shouldAttach(ownership.concepts, concept.parentConcept, domainKey)
+    ) {
+        if (pushUnique(collections.concepts, concept.parentConcept)) {
+            pendingConcepts.push(concept.parentConcept);
+        }
+    }
+
+    for (const endpoint of collections.endpoints) {
+        if (referencesConcept(endpoint, concept)) {
+            includeEndpoint({
+                domainKey,
+                endpoint,
+                ownership,
+                collections,
+                pendingConcepts,
+                pendingEndpoints,
+            });
+        }
+    }
+}
+
+function includeEndpoint(options: {
+    domainKey: string;
+    endpoint: EndpointDef;
+    ownership: OwnershipMaps;
+    collections: DomainCollections;
+    pendingConcepts: ConceptDef[];
+    pendingEndpoints: EndpointDef[];
+}) {
+    const { domainKey, endpoint, ownership, collections, pendingConcepts, pendingEndpoints } =
+        options;
+
+    if (shouldAttach(ownership.endpoints, endpoint, domainKey)) {
+        pushUnique(collections.endpoints, endpoint);
+    }
+
+    includeRule(domainKey, endpoint.governedBy ?? [], ownership, collections);
+
+    // An endpoint reaches concepts three ways and other endpoints one way, but the
+    // handling is identical: take what this domain owns, and queue whatever is new.
+    collectOwned({
+        items: [
+            ...(endpoint.queries ?? []),
+            ...(endpoint.mutates ?? []),
+            ...(endpoint.returns ?? []).map((returned) => returned.concept),
+        ],
+        owners: ownership.concepts,
+        collection: collections.concepts,
+        pending: pendingConcepts,
+        domainKey,
+    });
+
+    collectOwned({
+        items: (endpoint.dependsOn ?? []).map((dependency) => dependency.endpoint),
+        owners: ownership.endpoints,
+        collection: collections.endpoints,
+        pending: pendingEndpoints,
+        domainKey,
+    });
+}
+
+/**
+ * Only what this domain owns is attached, and only the first sighting is queued —
+ * re-queueing something already collected would walk the graph forever.
+ */
+function collectOwned<T extends object>(options: {
+    items: T[];
+    owners: WeakMap<T, string>;
+    collection: T[];
+    pending: T[];
+    domainKey: string;
+}) {
+    const { items, owners, collection, pending, domainKey } = options;
+
+    for (const item of items) {
+        if (shouldAttach(owners, item, domainKey) && pushUnique(collection, item)) {
+            pending.push(item);
+        }
+    }
+}
+
+function includeService(options: {
+    domainKey: string;
+    service: ServiceDef;
+    ownership: OwnershipMaps;
+    collections: DomainCollections;
+    pendingConcepts: ConceptDef[];
+}) {
+    const { domainKey, service, ownership, collections, pendingConcepts } = options;
+
+    if (shouldAttach(ownership.services, service, domainKey)) {
+        pushUnique(collections.services, service);
+    }
+
+    includeRule(domainKey, service.governedBy ?? [], ownership, collections);
+
+    if (
+        shouldAttach(ownership.concepts, service.belongsTo, domainKey) &&
+        pushUnique(collections.concepts, service.belongsTo)
+    ) {
+        pendingConcepts.push(service.belongsTo);
+    }
+}
+
+function includeRule(
+    domainKey: string,
+    rules: readonly RuleDef[],
+    ownership: OwnershipMaps,
+    collections: DomainCollections,
+) {
+    for (const rule of rules) {
+        if (shouldAttach(ownership.rules, rule, domainKey)) {
+            pushUnique(collections.rules, rule);
+        }
+    }
+}
+
+function shouldAttach<T extends object>(owners: WeakMap<T, string>, value: T, domainKey: string) {
+    const owner = owners.get(value);
+    return owner == null || owner === domainKey;
+}
+
+function pushUnique<T extends object>(values: T[], value: T) {
+    if (values.includes(value)) {
+        return false;
+    }
+
+    values.push(value);
+    return true;
+}
+
+function referencesConcept(endpoint: EndpointDef, concept: ConceptDef) {
+    return (
+        endpoint.queries?.includes(concept) ||
+        endpoint.mutates?.includes(concept) ||
+        endpoint.returns?.some(function (returned) {
+            return returned.concept === concept;
+        })
+    );
+}

package/src/graph/generate-cypher.ts

@@ -0,0 +1,201 @@
+/**
+ * Transforms declarative DomainDef into idempotent MERGE-based Cypher.
+ *
+ * Returns two phases:
+ *   - `nodes`  -- MERGE statements that create/update Domain, Concept, Endpoint,
+ *                 Service, and Rule nodes.
+ *   - `edges`  -- MATCH+MERGE statements that wire up relationships between nodes.
+ *
+ * The caller MUST run all `nodes` from every domain before running any `edges`,
+ * because edges may reference nodes defined in a different domain (e.g. a Rule
+ * in the leaves domain that GOVERNS a Concept created by the servicing domain).
+ *
+ * Every label, relationship type and property name comes from the `GraphSchema`
+ * the caller passes in — the project's own copy, so what this writes is by
+ * construction what that project's resolver reads back.
+ */
+
+import type {
+    ConceptDef,
+    DomainDef,
+    EndpointDef,
+    GraphSchema,
+    RuleDef,
+    ServiceDef,
+} from "@cortex/contracts/graph";
+
+export type GeneratedCypher = {
+    nodes: string[];
+    edges: string[];
+};
+
+export function generateCypher(data: DomainDef, s: GraphSchema) {
+    const nodes: string[] = [domainNode(data, s)];
+    const edges: string[] = [];
+
+    for (const concept of data.concepts ?? []) {
+        nodes.push(conceptNode(concept, s));
+        edges.push(...conceptEdges(concept, data.name, s));
+    }
+
+    for (const endpoint of data.endpoints ?? []) {
+        nodes.push(endpointNode(endpoint, s));
+        edges.push(...endpointEdges(endpoint, s));
+    }
+
+    for (const service of data.services ?? []) {
+        nodes.push(serviceNode(service, s));
+        edges.push(...serviceEdges(service, s));
+    }
+
+    for (const rule of data.rules ?? []) {
+        nodes.push(ruleNode(rule, s));
+    }
+
+    return { nodes, edges } satisfies GeneratedCypher;
+}
+
+function domainNode(data: DomainDef, s: GraphSchema) {
+    return `MERGE (d:${s.label.domain} {${s.prop.name}: '${esc(data.name)}'})
+        SET d.${s.prop.description} = '${esc(data.description)}'${metadataClause("d", data.metadata, s)}`;
+}
+
+function conceptNode(concept: ConceptDef, s: GraphSchema) {
+    const aliases =
+        concept.aliases && concept.aliases.length > 0
+            ? `, c.${s.prop.aliases} = [${concept.aliases.map((alias) => `'${esc(alias)}'`).join(", ")}]`
+            : "";
+
+    return `MERGE (c:${s.label.concept} {${s.prop.name}: '${esc(concept.name)}'})
+            SET c.${s.prop.description} = '${esc(concept.description)}'${aliases}${metadataClause("c", concept.metadata, s)}`;
+}
+
+function conceptEdges(concept: ConceptDef, domainName: string, s: GraphSchema) {
+    const edges = [
+        `MATCH (c:${conceptRef(concept, s)})
+            MATCH (d:${s.label.domain} {${s.prop.name}: '${esc(domainName)}'})
+            MERGE (c)-[:${s.rel.belongsTo}]->(d)`,
+    ];
+
+    if (concept.parentConcept) {
+        edges.push(
+            `MATCH (child:${conceptRef(concept, s)})
+                MATCH (parent:${conceptRef(concept.parentConcept, s)})
+                MERGE (child)-[:${s.rel.specializes}]->(parent)`,
+        );
+    }
+
+    edges.push(...governedByEdges(concept.governedBy, conceptRef(concept, s), s));
+
+    return edges;
+}
+
+function endpointNode(endpoint: EndpointDef, s: GraphSchema) {
+    return `MERGE (e:${endpointRef(endpoint, s)})
+            SET e.${s.prop.name} = '${esc(endpoint.name)}',
+                e.${s.prop.description} = '${esc(endpoint.description)}',
+                e.${s.prop.params} = '${jsonStr(endpoint.params)}',
+                e.${s.prop.body} = '${jsonStr(endpoint.body)}',
+                e.${s.prop.response} = '${jsonStr(endpoint.response)}',
+                e.${s.prop.propertiesDescriptions} = '${jsonStr(endpoint.propertiesDescriptions)}',
+                e.${s.prop.successStatus} = ${endpoint.successStatus},
+                e.${s.prop.errorStatuses} = [${endpoint.errorStatuses.join(", ")}]${metadataClause("e", endpoint.metadata, s)}`;
+}
+
+function endpointEdges(endpoint: EndpointDef, s: GraphSchema) {
+    const edges: string[] = [];
+    const self = endpointRef(endpoint, s);
+
+    for (const [concepts, relationship] of [
+        [endpoint.queries, s.rel.queriedVia],
+        [endpoint.mutates, s.rel.mutatedVia],
+    ] as const) {
+        for (const concept of concepts ?? []) {
+            edges.push(
+                `MATCH (c:${conceptRef(concept, s)})
+                MATCH (e:${self})
+                MERGE (c)-[:${relationship}]->(e)`,
+            );
+        }
+    }
+
+    for (const returned of endpoint.returns ?? []) {
+        // The field says which part of the response carries the concept; without
+        // one the whole body is the concept.
+        const field = returned.field ? ` SET r.${s.prop.field} = '${esc(returned.field)}'` : "";
+        edges.push(
+            `MATCH (e:${self})
+                MATCH (c:${conceptRef(returned.concept, s)})
+                MERGE (e)-[r:${s.rel.returns}]->(c)${field}`,
+        );
+    }
+
+    edges.push(...governedByEdges(endpoint.governedBy, self, s));
+
+    for (const dependency of endpoint.dependsOn ?? []) {
+        edges.push(
+            `MATCH (e:${self})
+                MATCH (dep:${endpointRef(dependency.endpoint, s)})
+                MERGE (e)-[d:${s.rel.dependsOn}]->(dep)
+                SET d.${s.prop.paramName} = '${esc(dependency.paramName)}', d.${s.prop.fromField} = '${esc(dependency.fromField)}'`,
+        );
+    }
+
+    return edges;
+}
+
+function serviceNode(service: ServiceDef, s: GraphSchema) {
+    return `MERGE (s:${serviceRef(service, s)})
+            SET s.${s.prop.name} = '${esc(service.name)}',
+            s.${s.prop.description} = '${esc(service.description)}'${metadataClause("s", service.metadata, s)}`;
+}
+
+function serviceEdges(service: ServiceDef, s: GraphSchema) {
+    return [
+        `MATCH (c:${conceptRef(service.belongsTo, s)})
+            MATCH (s:${serviceRef(service, s)})
+            MERGE (c)-[:${s.rel.requestedVia}]->(s)`,
+        ...governedByEdges(service.governedBy, serviceRef(service, s), s),
+    ];
+}
+
+function ruleNode(rule: RuleDef, s: GraphSchema) {
+    return `MERGE (r:${s.label.rule} {${s.prop.name}: '${esc(rule.name)}'})
+            SET r.${s.prop.description} = '${esc(rule.description)}'${metadataClause("r", rule.metadata, s)}`;
+}
+
+/** Rules govern concepts, endpoints and services identically; only the target differs. */
+function governedByEdges(rules: RuleDef[] | undefined, targetRef: string, s: GraphSchema) {
+    return (rules ?? []).map(
+        (rule) => `MATCH (r:${s.label.rule} {${s.prop.name}: '${esc(rule.name)}'})
+                MATCH (t:${targetRef})
+                MERGE (r)-[:${s.rel.governs}]->(t)`,
+    );
+}
+
+/** Endpoints are keyed on path plus method, so both must appear in every match. */
+function endpointRef(endpoint: Pick<EndpointDef, "path" | "method">, s: GraphSchema) {
+    return `${s.label.endpoint} {${s.prop.path}: '${esc(endpoint.path)}', ${s.prop.method}: '${endpoint.method}'}`;
+}
+
+function conceptRef(concept: Pick<ConceptDef, "name">, s: GraphSchema) {
+    return `${s.label.concept} {${s.prop.name}: '${esc(concept.name)}'}`;
+}
+
+function serviceRef(service: Pick<ServiceDef, "builtInId">, s: GraphSchema) {
+    return `${s.label.service} {${s.prop.builtInId}: '${esc(service.builtInId)}'}`;
+}
+
+/** Omitted entirely when absent — an empty assignment is not valid Cypher. */
+function metadataClause(alias: string, metadata: unknown, s: GraphSchema) {
+    return metadata ? `, ${alias}.${s.prop.metadata} = '${jsonStr(metadata)}'` : "";
+}
+
+/** Backslashes first, so escaping a quote cannot itself be escaped away. */
+function esc(value: string) {
+    return value.replace(/\\/g, "\\\\").replace(/'/g, "\\'").replace(/\n/g, "\\n");
+}
+
+function jsonStr(value: unknown) {
+    return esc(JSON.stringify(value));
+}

package/src/graph/seed.ts

@@ -0,0 +1,207 @@
+/**
+ * Seed orchestrator for the knowledge graph: validates domains, generates
+ * Cypher, executes it against Neo4j, and embeds every concept.
+ */
+
+import {
+    Neo4jQueryError,
+    type DomainDef,
+    type EmbedFn,
+    type GraphSchema,
+    type Neo4jClient,
+} from "@cortex/contracts/graph";
+
+import { expandDomains } from "./expand-domains";
+import { generateCypher, type GeneratedCypher } from "./generate-cypher";
+import { validateDomain } from "./validate";
+
+type SeedEmbedding = {
+    embed: EmbedFn;
+    dimension: number;
+};
+
+type SeedGraphConfig = {
+    neo4j: Neo4jClient;
+    embedding: SeedEmbedding;
+    /** `GRAPH_SCHEMA`, passed rather than imported so tests can drive a fake one. */
+    schema: GraphSchema;
+    /** Stamped onto the graph, so the server can refuse a shape it cannot read. */
+    schemaVersion: number;
+};
+
+type StatementFailure = {
+    /** The Neo4j error code where there was one, its message otherwise. */
+    error: string;
+    /** The statement as generated, untruncated — the caller decides how much to show. */
+    statement: string;
+};
+
+type DomainSeedResult = {
+    nodes: number;
+    edges: number;
+    failed: number;
+    failures: StatementFailure[];
+};
+
+type SeedGraphResult = {
+    nodes: number;
+    edges: number;
+    failed: number;
+    /** Keyed by the same names the config declares under `knowledge.domains`. */
+    domains: Record<string, DomainSeedResult>;
+    embeddings: { concepts: number; dimension: number };
+};
+
+// Nodes for every domain run before any edge does, so a statement carries the
+// domain it came from rather than being attributed by the loop that runs it.
+type TaggedStatement = { domain: string; statement: string };
+
+/** Runs one statement; the failure string is Neo4j's error code when it has one. */
+async function runStatement(client: Neo4jClient, statement: string) {
+    try {
+        await client.query(statement);
+        return undefined;
+    } catch (error) {
+        if (error instanceof Neo4jQueryError)
+            return /^\d+$/.test(error.code) ? error.message : error.code;
+        return error instanceof Error ? error.message : String(error);
+    }
+}
+
+// Maps, not objects: a domain is named by the user, and `constructor` is a legal
+// domain name that on a plain record reads back `Object.prototype.constructor` —
+// a function, so `?? 0` never fires and `.push` throws.
+async function runPhase(client: Neo4jClient, statements: TaggedStatement[]) {
+    const succeeded = new Map<string, number>();
+    const failures = new Map<string, StatementFailure[]>();
+
+    for (const { domain, statement } of statements) {
+        const error = await runStatement(client, statement);
+        if (error === undefined) {
+            succeeded.set(domain, (succeeded.get(domain) ?? 0) + 1);
+            continue;
+        }
+
+        const collected = failures.get(domain) ?? [];
+        collected.push({ error, statement });
+        failures.set(domain, collected);
+    }
+
+    return { succeeded, failures };
+}
+
+async function generateEmbeddings(
+    client: Neo4jClient,
+    embeddingConfig: SeedEmbedding,
+    s: GraphSchema,
+) {
+    // 1. Create the vector index (idempotent)
+    await client.query(
+        `CREATE VECTOR INDEX ${s.conceptEmbeddingsIndex} IF NOT EXISTS
+         FOR (c:${s.label.concept}) ON (c.${s.prop.embedding})
+         OPTIONS {indexConfig: {
+             \`vector.dimensions\`: ${embeddingConfig.dimension},
+             \`vector.similarity_function\`: 'cosine'
+         }}`,
+    );
+
+    // 2. Fetch all concepts
+    const concepts = (await client.query(
+        `MATCH (c:${s.label.concept}) RETURN c.${s.prop.name} AS name, c.${s.prop.description} AS description`,
+    )) as { name: string; description: string }[];
+
+    if (concepts.length === 0) {
+        return { concepts: 0, dimension: embeddingConfig.dimension };
+    }
+
+    // 3. Generate embeddings for every concept in one request
+    const embeddings = await embeddingConfig.embed(concepts.map((c) => c.description));
+
+    // 4. Store the embeddings in the graph
+    await Promise.all(
+        concepts.map((concept, idx) =>
+            client.query(
+                `MATCH (c:${s.label.concept} {${s.prop.name}: $name})
+                 SET c.${s.prop.embedding} = $embedding`,
+                { name: concept.name, embedding: embeddings[idx]! },
+            ),
+        ),
+    );
+
+    return { concepts: concepts.length, dimension: embeddingConfig.dimension };
+}
+
+export async function seedGraph(config: SeedGraphConfig, domains: Record<string, DomainDef>) {
+    const expandedDomains = expandDomains(domains);
+
+    // Validate all domains
+    const validationErrors = Object.entries(expandedDomains).flatMap(([name, domain]) =>
+        validateDomain(domain).map((err) => `[${name}] ${err}`),
+    );
+
+    if (validationErrors.length) {
+        throw new Error(
+            `Validation failed with ${validationErrors.length} error(s):\n${validationErrors.join("\n")}`,
+        );
+    }
+
+    const client = config.neo4j;
+    const targets = Object.entries(expandedDomains);
+
+    // Generate Cypher for all target domains
+    const allCypher: [string, GeneratedCypher][] = targets.map(([name, data]) => [
+        name,
+        generateCypher(data, config.schema),
+    ]);
+
+    // Stamped before anything else is written, so an interrupted seed still
+    // leaves the graph labelled with the schema it was written against.
+    await client.query(
+        `MERGE (g:${config.schema.label.marker})
+         SET g.${config.schema.prop.version} = ${config.schemaVersion}`,
+    );
+
+    // Nodes across every domain first, so cross-domain references resolve.
+    const allNodes = allCypher.flatMap(([domain, c]) => tag(domain, c.nodes));
+    const nodes = await runPhase(client, allNodes);
+
+    // Edges once every node exists.
+    const allEdges = allCypher.flatMap(([domain, c]) => tag(domain, c.edges));
+    const edges = await runPhase(client, allEdges);
+
+    // Embeddings for every concept.
+    const embeddings = await generateEmbeddings(client, config.embedding, config.schema);
+
+    const domainResults: Record<string, DomainSeedResult> = Object.fromEntries(
+        targets.map(([name]) => {
+            const failures = [
+                ...(nodes.failures.get(name) ?? []),
+                ...(edges.failures.get(name) ?? []),
+            ];
+            return [
+                name,
+                {
+                    nodes: nodes.succeeded.get(name) ?? 0,
+                    edges: edges.succeeded.get(name) ?? 0,
+                    failed: failures.length,
+                    failures,
+                },
+            ];
+        }),
+    );
+
+    // The aggregates stay the headline number; per-domain is the breakdown of it.
+    return Object.values(domainResults).reduce<SeedGraphResult>(
+        (total, domain) => ({
+            ...total,
+            nodes: total.nodes + domain.nodes,
+            edges: total.edges + domain.edges,
+            failed: total.failed + domain.failed,
+        }),
+        { nodes: 0, edges: 0, failed: 0, domains: domainResults, embeddings },
+    );
+}
+
+function tag(domain: string, statements: string[]) {
+    return statements.map((statement) => ({ domain, statement }) satisfies TaggedStatement);
+}