@rubytech/create-maxy 1.0.705 → 1.0.707

package/payload/platform/plugins/memory/mcp/dist/tools/memory-ingest.js

@@ -1,90 +1,143 @@
 import { getSession } from "../lib/neo4j.js";
-import { embedBatch } from "../lib/embeddings.js";
+import { embed, embedBatch } from "../lib/embeddings.js";
 import { extractCache } from "./memory-ingest-extract.js";
 import { deleteDocumentChildren } from "../lib/document-hierarchy.js";
 import { restoreNode } from "../../../../../lib/graph-trash/dist/index.js";
+// ---------------------------------------------------------------------------
+// Typed-node document ingestion (Task 737).
+//
+// CACHE LOOKUP --> MERGE/REVIVE DOC --> CLEANUP CHILDREN --> EMBED --> WRITE
+//   (by attachmentId)  KnowledgeDocument    delete prior         batch    typed nodes
+//                      (idempotent)         Section/Chunk/typed  Ollama   + anchor edges
+//                                                                         + REFERENCES
+//
+// Each classified section becomes either:
+//   - a typed graph node (Position, Service, Credential, etc.) anchored to
+//     UserProfile / LocalBusiness / Person / Organization via the natural
+//     ontology edge, plus a (KnowledgeDocument)-[:REFERENCES]->(typed) link
+//     for retrieval, plus optional related-entity nodes (e.g. Position's
+//     employer Organization, MERGEd by name).
+//   - a generic :Section node (UNMAPPED fallback) hanging off the document
+//     via the legacy (KnowledgeDocument)-[:HAS_SECTION] edge, with optional
+//     :Chunk overflow when the body exceeds MAX_CHUNK_SIZE.
+//
+// Provenance properties (createdByAgent, createdBySession, source,
+// sourceDocumentId) stamp every node and edge the skill creates so the
+// re-ingest cleanup can find and replace them deterministically.
+// ---------------------------------------------------------------------------
+const MAX_CHUNK_SIZE = 2000;
+const PREVIEW_LENGTH = 150;
+const UNMAPPED = "UNMAPPED";
+const PROVENANCE_AGENT = "document-ingest";
+const PROVENANCE_SOURCE = "document";
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+function normaliseKeywords(arr) {
+    if (!arr || arr.length === 0)
+        return [];
+    return arr.map((k) => k.toLowerCase().trim()).filter(Boolean);
+}
+function chunkBody(body) {
+    if (body.length <= MAX_CHUNK_SIZE)
+        return [body];
+    const chunks = [];
+    for (let i = 0; i < body.length; i += MAX_CHUNK_SIZE) {
+        chunks.push(body.slice(i, i + MAX_CHUNK_SIZE));
+    }
+    return chunks;
+}
+/**
+ * Identifying property for MERGE on a related node by kind.
+ * Choose a stable, human-recognisable key per label so the same real-world
+ * entity collapses to one node across documents.
+ */
+function mergeKeyFor(kind, properties) {
+    switch (kind) {
+        case "Organization":
+            return typeof properties.name === "string" && properties.name.trim()
+                ? { key: "name", value: properties.name.trim() }
+                : null;
+        case "Person":
+            if (typeof properties.email === "string" && properties.email.trim()) {
+                return { key: "email", value: properties.email.trim() };
+            }
+            if (typeof properties.telephone === "string" && properties.telephone.trim()) {
+                return { key: "telephone", value: properties.telephone.trim() };
+            }
+            return null;
+        case "DefinedTerm":
+            return typeof properties.name === "string" && properties.name.trim()
+                ? { key: "name", value: properties.name.trim() }
+                : null;
+        case "Credential":
+            return typeof properties.name === "string" && properties.name.trim()
+                ? { key: "name", value: properties.name.trim() }
+                : null;
+        default:
+            return null;
+    }
+}
+// ---------------------------------------------------------------------------
+// Main entry point
+// ---------------------------------------------------------------------------
 export async function memoryIngest(params) {
-    const { accountId, attachmentId, documentSummary, sections, scope, entities, sourceUrl, sourceType, keywords: rawKeywords, userKeywords: rawUserKeywords, } = params;
+    const { accountId, attachmentId, documentSummary, anchorNodeId, anchorLabel, sections, scope, sourceUrl, sourceType, documentKeywords: rawDocKeywords, userKeywords: rawUserKeywords, sessionId, } = params;
     if (!scope) {
         throw new Error("scope is required — valid values: 'public', 'shared', 'admin', 'user:{identifier}'");
     }
-    // Normalize and merge keywords: user-supplied ∪ LLM-extracted, deduplicated.
-    // User keywords appear first (cosmetic — both are equal after merge).
-    const normalizeArray = (arr) => arr.map((k) => k.toLowerCase().trim()).filter(Boolean);
-    const hasUserKeywords = rawUserKeywords && rawUserKeywords.length > 0;
-    const hasKeywords = rawKeywords && rawKeywords.length > 0;
-    const keywords = hasUserKeywords || hasKeywords
-        ? [...new Set([
-                ...normalizeArray(rawUserKeywords ?? []),
-                ...normalizeArray(rawKeywords ?? []),
-            ])]
-        : undefined;
+    if (!anchorNodeId) {
+        throw new Error("anchorNodeId is required — the document subject's element ID (UserProfile/LocalBusiness/Person/Organization)");
+    }
+    if (!anchorLabel) {
+        throw new Error("anchorLabel is required — the anchor node's primary label");
+    }
+    const keywords = (() => {
+        const u = normaliseKeywords(rawUserKeywords);
+        const d = normaliseKeywords(rawDocKeywords);
+        if (u.length === 0 && d.length === 0)
+            return undefined;
+        return [...new Set([...u, ...d])];
+    })();
     const t0 = Date.now();
     const log = (stage, detail) => console.error(`[memory-ingest] [${attachmentId.slice(0, 8)}] ${stage}${detail ? ` — ${detail}` : ""} (${Date.now() - t0}ms)`);
-    log("start", `${sections.length} sections, scope=${scope}`);
-    // 1. Retrieve cached content from memory-ingest-extract
+    log("start", `${sections.length} sections, scope=${scope}, anchor=${anchorLabel}`);
     const cached = extractCache.get(attachmentId);
     if (!cached) {
         throw new Error(`No cached extract found for attachment "${attachmentId}". ` +
-            `Call memory-ingest-extract first to extract and chunk the document.`);
+            `Call memory-ingest-extract first.`);
     }
     log("cache-hit", cached.filename);
-    // Validate section count matches
-    if (sections.length !== cached.sections.length) {
-        throw new Error(`Section count mismatch: model provided ${sections.length} sections ` +
-            `but the extracted document has ${cached.sections.length} sections. ` +
-            `Provide exactly one summary per section returned by memory-ingest-extract.`);
-    }
-    // Validate chunk summary counts per section
-    for (let i = 0; i < sections.length; i++) {
-        const expectedChunks = cached.sections[i].chunks.length;
-        const providedSummaries = sections[i].chunkSummaries.length;
-        if (providedSummaries !== expectedChunks) {
-            throw new Error(`Chunk count mismatch in section "${sections[i].title}": ` +
-                `model provided ${providedSummaries} chunk summaries ` +
-                `but the section has ${expectedChunks} chunks.`);
-        }
-    }
     const { filename, mimeType } = cached;
     const now = new Date().toISOString();
-    // 2. Collect all texts that need embedding in a flat array.
-    const textsToEmbed = [];
-    // Document-level summary
-    textsToEmbed.push(documentSummary);
-    const docEmbedIdx = 0;
-    // Section-level summaries
-    const sectionEmbedIndices = [];
-    for (const section of sections) {
-        sectionEmbedIndices.push(textsToEmbed.length);
-        textsToEmbed.push(section.summary);
-    }
-    // Chunk-level summaries
-    const chunkEmbedIndices = [];
-    for (let si = 0; si < sections.length; si++) {
-        for (let ci = 0; ci < sections[si].chunkSummaries.length; ci++) {
-            chunkEmbedIndices.push({
-                sectionIdx: si,
-                chunkIdx: ci,
-                embedIdx: textsToEmbed.length,
-            });
-            textsToEmbed.push(sections[si].chunkSummaries[ci]);
-        }
-    }
-    // 3. Batch embed all summaries in a single Ollama call.
+    // 1. Embed document summary + every section body in one batch.
+    const textsToEmbed = [documentSummary, ...sections.map((s) => s.body)];
     log("embedding", `${textsToEmbed.length} texts`);
     const embeddings = await embedBatch(textsToEmbed);
     log("embedded", `${embeddings.length} vectors`);
-    // 4. Write nodes to Neo4j.
-    log("neo4j-write", "starting");
+    const docEmbedding = embeddings[0];
     const session = getSession();
     let documentNodeId = "";
-    let totalChunks = 0;
+    let typedCount = 0;
+    let unmappedCount = 0;
+    let chunkCount = 0;
     let entityLinks = 0;
     try {
-        // 4a. Create KnowledgeDocument node
-        // Build optional SET clauses for web-sourced properties.
-        // When sourceUrl/sourceType/keywords are undefined (file uploads),
-        // the corresponding SET lines are omitted — existing values preserved on re-ingest.
+        // 2. Trash-revival (Task 576) — restore a soft-deleted KnowledgeDocument
+        //    with this attachmentId so MERGE finds the existing node.
+        const trashedDoc = await session.run(`MATCH (d:KnowledgeDocument:Trashed)
+       WHERE d.accountId = $accountId
+         AND d._trashedKeys IS NOT NULL
+         AND d._trashedKeys CONTAINS $attachmentId
+       RETURN elementId(d) AS eid LIMIT 1`, { accountId, attachmentId });
+        if (trashedDoc.records.length > 0) {
+            const eid = trashedDoc.records[0].get("eid");
+            await restoreNode({ session, accountId, elementId: eid });
+            log("revived", `restored trashed KnowledgeDocument elementId=${eid}`);
+        }
+        // 3. MERGE the KnowledgeDocument parent. Optional fields (sourceUrl,
+        //    sourceType, keywords) are SET only when present so re-ingest of a
+        //    file upload doesn't null out a previously-set web-source URL.
         const optionalSets = [];
         const optionalParams = {};
         if (sourceUrl !== undefined) {
@@ -99,35 +152,19 @@ export async function memoryIngest(params) {
             optionalSets.push("d.keywords = $keywords");
             optionalParams.keywords = keywords;
         }
-        const optionalSetClause = optionalSets.length > 0
-            ? ", " + optionalSets.join(", ")
-            : "";
-        // Trash recovery: a prior memory-delete on this attachmentId nulls the
-        // live attachmentId (Task 576 unique-key handling). MERGE on
-        // attachmentId would then create a NEW node, orphaning the trashed copy
-        // — and 30 days later memory-empty-trash would purge the disk dir the
-        // new node depends on. Restore first so MERGE finds the existing node.
-        const trashedDoc = await session.run(`MATCH (d:KnowledgeDocument:Trashed)
-       WHERE d.accountId = $accountId
-         AND d._trashedKeys IS NOT NULL
-         AND d._trashedKeys CONTAINS $attachmentId
-       RETURN elementId(d) AS eid LIMIT 1`, { accountId, attachmentId });
-        if (trashedDoc.records.length > 0) {
-            const eid = trashedDoc.records[0].get("eid");
-            await restoreNode({ session, accountId, elementId: eid });
-            log("revived", `restored trashed KnowledgeDocument elementId=${eid}`);
-        }
+        const optionalSetClause = optionalSets.length > 0 ? ", " + optionalSets.join(", ") : "";
         const docResult = await session.run(`MERGE (d:KnowledgeDocument { attachmentId: $attachmentId })
        SET d.accountId      = $accountId,
            d.name           = $filename,
-           d.encodingFormat  = $mimeType,
-           d.summary         = $documentSummary,
-           d.sectionCount    = $sectionCount,
-           d.scope           = $scope,
-           d.embedding       = $embedding,
-           d.createdAt       = $createdAt,
-           d.updatedAt       = $updatedAt
+           d.encodingFormat = $mimeType,
+           d.summary        = $documentSummary,
+           d.sectionCount   = $sectionCount,
+           d.scope          = $scope,
+           d.embedding      = $embedding,
+           d.createdAt      = coalesce(d.createdAt, $createdAt),
+           d.updatedAt      = $updatedAt
            ${optionalSetClause}
+       REMOVE d.deletedAt
        RETURN elementId(d) AS nodeId`, {
             attachmentId,
             accountId,
@@ -136,114 +173,323 @@ export async function memoryIngest(params) {
             documentSummary,
             sectionCount: sections.length,
             scope,
-            embedding: embeddings[docEmbedIdx],
+            embedding: docEmbedding,
             createdAt: now,
             updatedAt: now,
             ...optionalParams,
         });
         documentNodeId = docResult.records[0].get("nodeId");
-        // 4a-cleanup. Remove stale children before re-creating.
-        // On first ingestion the queries return nothing and the DELETEs are no-ops.
-        // Also clear any soft-delete marker on the document itself (re-ingestion revives).
+        // 4. Cleanup prior children (idempotent re-ingest). Removes generic
+        //    :Section/Chunk fallbacks and typed nodes that originated from this
+        //    document. MERGEd related nodes (Organizations, Persons) are spared.
         const cleanup = await deleteDocumentChildren(attachmentId, session);
-        if (cleanup.sections > 0 || cleanup.chunks > 0 || cleanup.references > 0) {
-            log("cleanup", `deleted ${cleanup.sections} sections, ${cleanup.chunks} chunks, ${cleanup.references} references`);
+        if (cleanup.sections > 0 ||
+            cleanup.chunks > 0 ||
+            cleanup.typed > 0 ||
+            cleanup.references > 0) {
+            log("cleanup", `deleted ${cleanup.sections} sections, ${cleanup.chunks} chunks, ${cleanup.typed} typed, ${cleanup.references} references`);
         }
-        // Clear deletedAt if the document was previously soft-deleted
-        await session.run(`MATCH (d:KnowledgeDocument { attachmentId: $attachmentId })
-       WHERE d.deletedAt IS NOT NULL
-       REMOVE d.deletedAt`, { attachmentId });
-        // 4b. Create Section nodes and link HAS_SECTION
-        const sectionNodeIds = [];
-        for (let si = 0; si < sections.length; si++) {
-            const section = sections[si];
-            const sectionResult = await session.run(`CREATE (s:Section {
-           accountId:  $accountId,
-           title:      $title,
-           summary:    $summary,
-           position:   $position,
-           scope:      $scope,
-           embedding:  $embedding,
-           createdAt:  $createdAt,
-           updatedAt:  $updatedAt
-         })
-         RETURN elementId(s) AS nodeId`, {
+        // 5. Per-section writes.
+        for (let i = 0; i < sections.length; i++) {
+            const section = sections[i];
+            const bodyEmbedding = embeddings[i + 1]; // +1 because index 0 is docEmbedding
+            const sectionParams = {
                 accountId,
                 title: section.title,
-                summary: section.summary,
-                position: si,
+                body: section.body,
+                bodyPreview: section.body.slice(0, PREVIEW_LENGTH),
+                position: i,
                 scope,
-                embedding: embeddings[sectionEmbedIndices[si]],
+                embedding: bodyEmbedding,
                 createdAt: now,
                 updatedAt: now,
-            });
-            const sectionNodeId = sectionResult.records[0].get("nodeId");
-            sectionNodeIds.push(sectionNodeId);
-            // Link KnowledgeDocument -[HAS_SECTION]-> Section
-            await session.run(`MATCH (d), (s)
-         WHERE elementId(d) = $docId AND elementId(s) = $sectionId
-         CREATE (d)-[:HAS_SECTION]->(s)`, { docId: documentNodeId, sectionId: sectionNodeId });
-        }
-        // 4c. Create Chunk nodes and link HAS_CHUNK — content from cache
-        for (const { sectionIdx, chunkIdx, embedIdx } of chunkEmbedIndices) {
-            const cachedChunk = cached.sections[sectionIdx].chunks[chunkIdx];
-            const chunkSummary = sections[sectionIdx].chunkSummaries[chunkIdx];
-            const chunkResult = await session.run(`CREATE (c:Chunk {
-           accountId:  $accountId,
-           summary:    $summary,
-           content:    $content,
-           position:   $position,
-           scope:      $scope,
-           embedding:  $embedding,
-           createdAt:  $createdAt,
-           updatedAt:  $updatedAt
-         })
-         RETURN elementId(c) AS nodeId`, {
+                createdByAgent: PROVENANCE_AGENT,
+                createdBySource: PROVENANCE_AGENT,
+                createdBySession: sessionId ?? "",
+                source: PROVENANCE_SOURCE,
+                sourceDocumentId: attachmentId,
+            };
+            if (section.kind === UNMAPPED) {
+                // 5a. UNMAPPED: generic :Section + (KnowledgeDocument)-[:HAS_SECTION]
+                const sectionResult = await session.run(`CREATE (s:Section {
+             accountId:        $accountId,
+             title:            $title,
+             body:             $body,
+             summary:          $bodyPreview,
+             position:         $position,
+             scope:            $scope,
+             embedding:        $embedding,
+             createdAt:        $createdAt,
+             updatedAt:        $updatedAt,
+             createdByAgent:   $createdByAgent,
+             createdBySource:  $createdBySource,
+             createdBySession: $createdBySession,
+             source:           $source,
+             sourceDocumentId: $sourceDocumentId
+           })
+           WITH s
+           MATCH (d:KnowledgeDocument) WHERE elementId(d) = $docId
+           CREATE (d)-[:HAS_SECTION]->(s)
+           RETURN elementId(s) AS nodeId`, { ...sectionParams, docId: documentNodeId });
+                const sectionId = sectionResult.records[0].get("nodeId");
+                // Body overflow → :Chunk children
+                if (section.body.length > MAX_CHUNK_SIZE) {
+                    const overflowChunks = chunkBody(section.body);
+                    const overflowEmbeddings = await embedBatch(overflowChunks);
+                    for (let ci = 0; ci < overflowChunks.length; ci++) {
+                        await session.run(`MATCH (s:Section) WHERE elementId(s) = $sectionId
+               CREATE (s)-[:HAS_CHUNK]->(c:Chunk {
+                 accountId:        $accountId,
+                 content:          $content,
+                 summary:          $summary,
+                 position:         $position,
+                 scope:            $scope,
+                 embedding:        $embedding,
+                 createdAt:        $createdAt,
+                 updatedAt:        $updatedAt,
+                 createdByAgent:   $createdByAgent,
+                 createdBySource:  $createdBySource,
+                 createdBySession: $createdBySession,
+                 source:           $source,
+                 sourceDocumentId: $sourceDocumentId
+               })`, {
+                            sectionId,
+                            accountId,
+                            content: overflowChunks[ci],
+                            summary: overflowChunks[ci].slice(0, PREVIEW_LENGTH),
+                            position: ci,
+                            scope,
+                            embedding: overflowEmbeddings[ci],
+                            createdAt: now,
+                            updatedAt: now,
+                            createdByAgent: PROVENANCE_AGENT,
+                            createdBySource: PROVENANCE_AGENT,
+                            createdBySession: sessionId ?? "",
+                            source: PROVENANCE_SOURCE,
+                            sourceDocumentId: attachmentId,
+                        });
+                        chunkCount += 1;
+                    }
+                }
+                unmappedCount += 1;
+                console.error(`[document-ingest] unmapped-section title="${section.title}" chars=${section.body.length}`);
+                continue;
+            }
+            // 5b. Typed kind — write the typed node, anchor edge, related nodes,
+            //     and the document REFERENCES link.
+            // Build the typed-node properties: classifier-supplied properties
+            // overlaid on top of the system fields, with system fields winning.
+            const typedProps = {
+                ...section.properties,
                 accountId,
-                summary: chunkSummary,
-                content: cachedChunk.content,
-                position: chunkIdx,
+                title: section.title,
+                body: section.body,
                 scope,
-                embedding: embeddings[embedIdx],
+                embedding: bodyEmbedding,
                 createdAt: now,
                 updatedAt: now,
-            });
-            const chunkNodeId = chunkResult.records[0].get("nodeId");
-            totalChunks++;
-            // Link Section -[HAS_CHUNK]-> Chunk
-            await session.run(`MATCH (s), (c)
-         WHERE elementId(s) = $sectionId AND elementId(c) = $chunkId
-         CREATE (s)-[:HAS_CHUNK]->(c)`, { sectionId: sectionNodeIds[sectionIdx], chunkId: chunkNodeId });
-        }
-        // 4d. Create REFERENCES links to entities
-        if (entities && entities.length > 0) {
-            for (const entity of entities) {
-                try {
-                    await session.run(`MATCH (d), (e)
-             WHERE elementId(d) = $docId AND elementId(e) = $entityId
-             MERGE (d)-[:REFERENCES]->(e)`, { docId: documentNodeId, entityId: entity.nodeId });
-                    entityLinks++;
+                createdByAgent: PROVENANCE_AGENT,
+                createdBySource: PROVENANCE_AGENT,
+                createdBySession: sessionId ?? "",
+                source: PROVENANCE_SOURCE,
+                sourceDocumentId: attachmentId,
+            };
+            // CREATE the typed node. Label is interpolated (validated against the
+            // ontology label set by the classifier) — Cypher does not allow
+            // parameterising labels.
+            const typedResult = await session.run(`CREATE (t:\`${section.kind}\`)
+         SET t = $props
+         RETURN elementId(t) AS nodeId`, { props: typedProps });
+            const typedNodeId = typedResult.records[0].get("nodeId");
+            // Body overflow → :Chunk children attached directly to the typed node
+            // (mirroring :Section overflow). Most typed nodes won't trip this.
+            if (section.body.length > MAX_CHUNK_SIZE) {
+                const overflowChunks = chunkBody(section.body);
+                const overflowEmbeddings = await embedBatch(overflowChunks);
+                for (let ci = 0; ci < overflowChunks.length; ci++) {
+                    await session.run(`MATCH (t) WHERE elementId(t) = $typedId
+             CREATE (t)-[:HAS_CHUNK]->(c:Chunk {
+               accountId:        $accountId,
+               content:          $content,
+               summary:          $summary,
+               position:         $position,
+               scope:            $scope,
+               embedding:        $embedding,
+               createdAt:        $createdAt,
+               updatedAt:        $updatedAt,
+               createdByAgent:   $createdByAgent,
+               createdBySource:  $createdBySource,
+               createdBySession: $createdBySession,
+               source:           $source,
+               sourceDocumentId: $sourceDocumentId
+             })`, {
+                        typedId: typedNodeId,
+                        accountId,
+                        content: overflowChunks[ci],
+                        summary: overflowChunks[ci].slice(0, PREVIEW_LENGTH),
+                        position: ci,
+                        scope,
+                        embedding: overflowEmbeddings[ci],
+                        createdAt: now,
+                        updatedAt: now,
+                        createdByAgent: PROVENANCE_AGENT,
+                        createdBySource: PROVENANCE_AGENT,
+                        createdBySession: sessionId ?? "",
+                        source: PROVENANCE_SOURCE,
+                        sourceDocumentId: attachmentId,
+                    });
+                    chunkCount += 1;
                 }
-                catch {
-                    // Entity node may have been deleted — skip silently, log via caller
+            }
+            // Related nodes (e.g. Position's employer Organization). MERGE when
+            // the related kind has a stable identifying property; CREATE when not.
+            // Provenance is stamped only on creation (ON CREATE SET) so shared
+            // entities don't get rewritten by subsequent ingestions.
+            if (section.related && section.related.length > 0) {
+                for (const related of section.related) {
+                    const relatedNodeId = await writeRelatedNode({
+                        session,
+                        accountId,
+                        related,
+                        now,
+                        sessionId,
+                    });
+                    // Edge from typed node to related node
+                    await session.run(`MATCH (t) WHERE elementId(t) = $typedId
+             MATCH (r) WHERE elementId(r) = $relatedId
+             CREATE (t)-[edge:\`${related.edge.type}\`]->(r)
+             SET edge += $edgeProps`, {
+                        typedId: related.edge.direction === "outgoing" ? typedNodeId : relatedNodeId,
+                        relatedId: related.edge.direction === "outgoing" ? relatedNodeId : typedNodeId,
+                        edgeProps: {
+                            ...(related.edge.properties ?? {}),
+                            createdByAgent: PROVENANCE_AGENT,
+                            createdBySession: sessionId ?? "",
+                            source: PROVENANCE_SOURCE,
+                            sourceDocumentId: attachmentId,
+                            createdAt: now,
+                        },
+                    });
+                }
+            }
+            // Anchor edge (anchor → typed or typed → anchor)
+            if (section.anchorEdge) {
+                const direction = section.anchorEdge.direction;
+                const edgeProps = {
+                    ...(section.anchorEdge.properties ?? {}),
+                    createdByAgent: PROVENANCE_AGENT,
+                    createdBySession: sessionId ?? "",
+                    source: PROVENANCE_SOURCE,
+                    sourceDocumentId: attachmentId,
+                    createdAt: now,
+                };
+                if (direction === "from-anchor") {
+                    await session.run(`MATCH (a:\`${anchorLabel}\`) WHERE elementId(a) = $anchorId AND a.accountId = $accountId
+             MATCH (t) WHERE elementId(t) = $typedId
+             CREATE (a)-[edge:\`${section.anchorEdge.type}\`]->(t)
+             SET edge += $edgeProps`, { anchorId: anchorNodeId, typedId: typedNodeId, accountId, edgeProps });
+                }
+                else {
+                    await session.run(`MATCH (a:\`${anchorLabel}\`) WHERE elementId(a) = $anchorId AND a.accountId = $accountId
+             MATCH (t) WHERE elementId(t) = $typedId
+             CREATE (t)-[edge:\`${section.anchorEdge.type}\`]->(a)
+             SET edge += $edgeProps`, { anchorId: anchorNodeId, typedId: typedNodeId, accountId, edgeProps });
                 }
             }
+            // (KnowledgeDocument)-[:REFERENCES]->(typed) for retrieval.
+            await session.run(`MATCH (d:KnowledgeDocument) WHERE elementId(d) = $docId
+         MATCH (t) WHERE elementId(t) = $typedId
+         CREATE (d)-[r:REFERENCES]->(t)
+         SET r.createdByAgent   = $createdByAgent,
+             r.createdBySession = $createdBySession,
+             r.source           = $source,
+             r.sourceDocumentId = $sourceDocumentId,
+             r.createdAt        = $createdAt`, {
+                docId: documentNodeId,
+                typedId: typedNodeId,
+                createdByAgent: PROVENANCE_AGENT,
+                createdBySession: sessionId ?? "",
+                source: PROVENANCE_SOURCE,
+                sourceDocumentId: attachmentId,
+                createdAt: now,
+            });
+            entityLinks += 1;
+            typedCount += 1;
+            console.error(`[document-ingest] section kind=${section.kind} title="${section.title}" chars=${section.body.length}`);
         }
-        log("neo4j-done", `doc=${documentNodeId.slice(0, 12)} sections=${sections.length} chunks=${totalChunks} entityLinks=${entityLinks}`);
+        log("neo4j-done", `doc=${documentNodeId.slice(0, 12)} sections=${sections.length} typed=${typedCount} unmapped=${unmappedCount} chunks=${chunkCount} entityLinks=${entityLinks}`);
+        console.error(`[document-ingest] done sections=${sections.length} typed=${typedCount} unmapped=${unmappedCount} ms=${Date.now() - t0}`);
         return {
             documentNodeId,
             sectionCount: sections.length,
-            chunkCount: totalChunks,
+            typedCount,
+            unmappedCount,
+            chunkCount,
             entityLinks,
             documentSummary,
             keywords,
         };
     }
     finally {
-        // Evict cache entry — content is now persisted in Neo4j
         extractCache.delete(attachmentId);
         await session.close();
         log("complete");
     }
 }
+async function writeRelatedNode(opts) {
+    const { session, accountId, related, now, sessionId } = opts;
+    // Compute embedding from a representative property string so the related
+    // node is searchable. Falls back to label+JSON if no `name` field exists.
+    const embedSource = typeof related.properties.name === "string"
+        ? related.properties.name
+        : `${related.kind} ${JSON.stringify(related.properties)}`;
+    const relatedEmbedding = await embed(embedSource);
+    const mergeKey = related.merge !== false ? mergeKeyFor(related.kind, related.properties) : null;
+    if (mergeKey) {
+        // MERGE on the identifying property + accountId. ON CREATE stamps
+        // provenance; ON MATCH leaves provenance intact (so re-ingest of the
+        // same Organization across docs doesn't overwrite the original
+        // provenance — first-write wins for shared entities).
+        const result = await session.run(`MERGE (r:\`${related.kind}\` { accountId: $accountId, \`${mergeKey.key}\`: $mergeValue })
+       ON CREATE SET r += $createProps,
+                     r.embedding        = $embedding,
+                     r.createdAt        = $createdAt,
+                     r.updatedAt        = $createdAt,
+                     r.createdByAgent   = $createdByAgent,
+                     r.createdBySource  = $createdByAgent,
+                     r.createdBySession = $createdBySession,
+                     r.source           = $source
+       ON MATCH  SET r.updatedAt = $createdAt
+       RETURN elementId(r) AS nodeId`, {
+            accountId,
+            mergeValue: mergeKey.value,
+            createProps: { ...related.properties, accountId, scope: "shared" },
+            embedding: relatedEmbedding,
+            createdAt: now,
+            createdByAgent: PROVENANCE_AGENT,
+            createdBySession: sessionId ?? "",
+            source: PROVENANCE_SOURCE,
+        });
+        return result.records[0].get("nodeId");
+    }
+    // CREATE — no stable identifying property, treat as one-of-a-kind for
+    // this document. Track sourceDocumentId so re-ingest cleanup catches it.
+    const result = await session.run(`CREATE (r:\`${related.kind}\`)
+     SET r = $props
+     RETURN elementId(r) AS nodeId`, {
+        props: {
+            ...related.properties,
+            accountId,
+            scope: "shared",
+            embedding: relatedEmbedding,
+            createdAt: now,
+            updatedAt: now,
+            createdByAgent: PROVENANCE_AGENT,
+            createdBySource: PROVENANCE_AGENT,
+            createdBySession: sessionId ?? "",
+            source: PROVENANCE_SOURCE,
+        },
+    });
+    return result.records[0].get("nodeId");
+}
 //# sourceMappingURL=memory-ingest.js.map