@indigoai-us/hq-cli 5.288.0 → 5.290.0

package/dist/lib/checkpoint-ontology/merge-copies.js

@@ -26,7 +26,11 @@
  * merge cannot rewrite (a copy, a knowledge file, a signal) is held.
  *
  * A signal whose `entity_refs` names a copy holds that copy: signals are
- * server-owned, so the ref cannot be re-pointed from here.
+ * server-owned, so the ref cannot be re-pointed from here. With `rewriteRefs`
+ * (flag `checkpoint.ontology.merge-rewrite-refs`) signal refs, invoice and
+ * bill links, source and knowledge links are re-pointed at the survivor, the
+ * survivor receives the copy's `external_ids` and `web_domains`, and only a
+ * link from another copy or an external id conflict holds a pair.
  *
  * With `pairKind: "variants"` (flag `checkpoint.ontology.merge-variants`) the
  * pairs are two-file duplicate groups whose canonical names differ only in
@@ -39,7 +43,7 @@ import * as path from "node:path";
 import { isDeepStrictEqual } from "node:util";
 import { freshAliases, ownNames } from "./aliases.js";
 import { entitiesRoot, safeWriteFile } from "./company.js";
-import { appendBackRefs, OWNED_KEYS, parseEntityFile, patchEntityFile } from "./entity-file.js";
+import { appendBackRefs, OWNED_KEYS, parseEntityFile, patchEntityFile, setIdentityKeys } from "./entity-file.js";
 import { isWritableEntityType, normalizeEntityName } from "./identity.js";
 import { findDuplicateGroups, readBackRefs, recordOutbox } from "./multi-writer.js";
 import { buildEntityIndex, listSignalFiles, readSignal } from "./tree.js";
@@ -49,6 +53,12 @@ export const MERGE_COPIES_FLAG = "checkpoint.ontology.merge-copies";
  * names differ only in spacing, case or punctuation (CO-65).
  */
 export const MERGE_VARIANTS_FLAG = "checkpoint.ontology.merge-variants";
+/**
+ * `merge-copies --rewrite-refs`: re-point signal `entity_refs`, invoice and
+ * bill links, source and knowledge links from a copy to its survivor, and
+ * carry the copy's `external_ids` and `web_domains` to the survivor.
+ */
+export const MERGE_REWRITE_REFS_FLAG = "checkpoint.ontology.merge-rewrite-refs";
 /** `<slug> <n>.md`, the name a file manager gives a second copy. */
 const COPY_NAME = /^(.+) (\d+)\.md$/;
 /** A relationship list item with a link, as the gardener and checkpoint write them. */
@@ -65,8 +75,17 @@ function splitBody(content) {
     const match = /^(---\r?\n[\s\S]*?\r?\n---\r?\n?)([\s\S]*)$/.exec(content);
     return match ? { frontmatter: match[1], body: match[2] } : null;
 }
-/** Relationship lines of the copy that the original lacks (back-references excluded). */
-function missingRelationshipLines(originalContent, copyContent) {
+/**
+ * A line of the copy whose every link points at the copy itself or at its
+ * original. Moved to the original it would link the survivor to itself or to
+ * the file about to be deleted, so it is neither moved nor unmerged content.
+ */
+function isPairSelfLine(line, pair) {
+    const targets = [...line.matchAll(MD_LINK)].map((match) => resolveLink(pair.copy, match[1]));
+    return targets.length > 0 && targets.every((target) => target === pair.copy || target === pair.original);
+}
+/** Relationship lines of the copy that the original lacks (back-references and self links excluded). */
+function missingRelationshipLines(originalContent, copyContent, pair) {
     const original = splitBody(originalContent);
     const copy = splitBody(copyContent);
     if (!original || !copy)
@@ -74,7 +93,7 @@ function missingRelationshipLines(originalContent, copyContent) {
     const have = new Set(original.body.split(/\r?\n/).map((line) => line.trimEnd()));
     const out = [];
     for (const line of sectionLines(copy.body)) {
-        if (!LINK_ITEM.test(line) || BACKREF_ITEM.test(line) || have.has(line))
+        if (!LINK_ITEM.test(line) || BACKREF_ITEM.test(line) || have.has(line) || isPairSelfLine(line, pair))
             continue;
         have.add(line);
         out.push(line);
@@ -284,7 +303,7 @@ function scanOtherLinks(company, copies) {
         for (const match of content.matchAll(/\[\[(ontology\/entities\/[^\]|\n]+?)(?:\|[^\]\n]*)?\]\]/g)) {
             add(match[1].endsWith(".md") ? match[1] : `${match[1]}.md`, rel);
         }
-        for (const match of content.matchAll(/^\s*path:\s*["']?(ontology\/entities\/[^"'\n]+?\.md)["']?\s*$/gm))
+        for (const match of content.matchAll(/^\s*(?:- )?path:\s*["']?(ontology\/entities\/[^"'\n]+?\.md)["']?\s*$/gm))
             add(match[1], rel);
     }
     return out;
@@ -321,7 +340,8 @@ function scanSignalRefs(company, copies) {
     }
     for (const rel of listSignalFiles(company.dir)) {
         for (const ref of readSignal(company.dir, rel)?.refs ?? []) {
-            const copy = byRef.get(ref);
+            // Signals write refs both as `type/slug` and as `type/slug.md`.
+            const copy = byRef.get(ref.endsWith(".md") ? ref.slice(0, -3) : ref);
             if (copy)
                 out.set(copy, [...(out.get(copy) ?? []), rel]);
         }
@@ -342,6 +362,350 @@ function relink(fromRel, content, copy, original) {
         return `](${target.slice(0, cut + 1)}${encoded}${fragment})`;
     });
 }
+const FRONT_RE = /^---\r?\n([\s\S]*?)\r?\n---(\r?\n|$)/;
+const REL_REF = /^(\s*(?:- )?[A-Za-z_][A-Za-z0-9_]*:\s*)(["']?)(\.\.\/[^"'\n]+?\.md)\2\s*$/;
+const WIKI_LINK = /\[\[(ontology\/entities\/[^\]|\n]+?)(\|[^\]\n]*)?\]\]/g;
+const PATH_LINE = /^(\s*(?:- )?path:\s*)(["']?)(ontology\/entities\/[^"'\n]+?\.md)\2\s*$/;
+function frontLines(content) {
+    const match = FRONT_RE.exec(content);
+    if (!match)
+        return null;
+    return { start: match.index + match[0].indexOf(match[1]), lines: match[1].split(/\r?\n/) };
+}
+/** Put rewritten front-matter lines back; every byte outside them stays. */
+function withFrontLines(content, front, lines, newline) {
+    const originalLength = front.lines.join(newline).length;
+    return `${content.slice(0, front.start)}${lines.join(newline)}${content.slice(front.start + originalLength)}`;
+}
+function fileNewline(content) {
+    return content.includes("\r\n") ? "\r\n" : "\n";
+}
+/** In the front matter only, re-point a `../…md` value that resolves to `copy`. */
+function rewriteRelRefs(fromRel, content, copy, original) {
+    const front = frontLines(content);
+    if (!front)
+        return content;
+    const name = path.posix.basename(original);
+    let changed = false;
+    const lines = front.lines.map((line) => {
+        const match = REL_REF.exec(line);
+        if (!match || resolveLink(fromRel, match[3]) !== copy)
+            return line;
+        const value = match[3];
+        const cut = value.lastIndexOf("/");
+        changed = true;
+        return `${match[1]}${match[2]}${value.slice(0, cut + 1)}${name}${match[2]}`;
+    });
+    if (!changed)
+        return content;
+    return withFrontLines(content, front, lines, fileNewline(content));
+}
+/** Entity files whose front matter points at a copy through a relative ref. */
+function scanRelRefs(company, copies) {
+    const out = new Map();
+    if (copies.size === 0)
+        return out;
+    for (const rel of listEntityFiles(company)) {
+        const front = frontLines(read(company, rel));
+        if (!front)
+            continue;
+        for (const line of front.lines) {
+            const match = REL_REF.exec(line);
+            if (!match)
+                continue;
+            const target = resolveLink(rel, match[3]);
+            if (!target || !copies.has(target))
+                continue;
+            let set = out.get(target);
+            if (!set)
+                out.set(target, (set = new Set()));
+            set.add(rel);
+        }
+    }
+    return out;
+}
+function listItemBounds(lines, pathIndex) {
+    let start = pathIndex;
+    let dashIndent = -1;
+    while (start >= 0) {
+        const dash = /^(\s*)- /.exec(lines[start]);
+        if (dash) {
+            dashIndent = dash[1].length;
+            break;
+        }
+        start -= 1;
+    }
+    if (dashIndent < 0)
+        return null;
+    let keyIndent = -1;
+    for (let i = start + 1; i < lines.length; i += 1) {
+        if (lines[i].trim() === "")
+            continue;
+        const indent = /^(\s*)/.exec(lines[i])[1].length;
+        if (indent <= dashIndent)
+            break;
+        keyIndent = indent;
+        break;
+    }
+    if (keyIndent < 0)
+        keyIndent = dashIndent + 1;
+    let end = start + 1;
+    while (end < lines.length) {
+        const line = lines[end];
+        const dash = /^(\s*)- /.exec(line);
+        if (dash && dash[1].length === dashIndent)
+            break;
+        if (line.trim() !== "" && /^(\s*)/.exec(line)[1].length < keyIndent)
+            break;
+        end += 1;
+    }
+    return { start, end };
+}
+function rewriteWikiLinks(content, copy, original) {
+    WIKI_LINK.lastIndex = 0;
+    return content.replace(WIKI_LINK, (whole, target, label) => {
+        const resolved = target.endsWith(".md") ? target : `${target}.md`;
+        if (resolved !== copy)
+            return whole;
+        const next = target.endsWith(".md") ? original : original.slice(0, -3);
+        return `[[${next}${label ?? ""}]]`;
+    });
+}
+/**
+ * Re-point markdown links, wiki-links, annotation `path:` lines and, when both
+ * ids are known, the `id:` of that same list item.
+ */
+function rewriteOtherFile(fromRel, content, copy, original, copyId, originalId) {
+    const linked = rewriteWikiLinks(relink(fromRel, content, copy, original), copy, original);
+    const newline = fileNewline(content);
+    const lines = linked.split(/\r?\n/);
+    for (let i = 0; i < lines.length; i += 1) {
+        const pathMatch = PATH_LINE.exec(lines[i]);
+        if (!pathMatch || pathMatch[3] !== copy)
+            continue;
+        lines[i] = `${pathMatch[1]}${pathMatch[2]}${original}${pathMatch[2]}`;
+        if (typeof copyId !== "string" || typeof originalId !== "string")
+            continue;
+        const bounds = listItemBounds(lines, i);
+        if (!bounds)
+            continue;
+        for (let j = bounds.start; j < bounds.end; j += 1) {
+            const idMatch = /^(\s*id:\s*)(["']?)([^"'\n]*?)\2(\s*)$/.exec(lines[j]);
+            if (idMatch && idMatch[3] === copyId)
+                lines[j] = `${idMatch[1]}${idMatch[2]}${originalId}${idMatch[2]}${idMatch[4]}`;
+        }
+    }
+    return lines.join(newline);
+}
+/** True when `content` still names `copy` by a link, wiki-link, path or front-matter ref. */
+function referencesCopy(fromRel, content, copy) {
+    MD_LINK.lastIndex = 0;
+    for (const match of content.matchAll(MD_LINK)) {
+        if (resolveLink(fromRel, match[1]) === copy)
+            return true;
+    }
+    WIKI_LINK.lastIndex = 0;
+    for (const match of content.matchAll(WIKI_LINK)) {
+        const target = match[1].endsWith(".md") ? match[1] : `${match[1]}.md`;
+        if (target === copy)
+            return true;
+    }
+    for (const line of content.split(/\r?\n/)) {
+        const pathMatch = PATH_LINE.exec(line);
+        if (pathMatch && pathMatch[3] === copy)
+            return true;
+    }
+    const front = frontLines(content);
+    if (front) {
+        for (const line of front.lines) {
+            const rel = REL_REF.exec(line);
+            if (rel && resolveLink(fromRel, rel[3]) === copy)
+                return true;
+        }
+    }
+    return false;
+}
+function entityRefOf(rel) {
+    const [, , type, file] = rel.split("/");
+    return `${type}/${file.slice(0, -3)}`;
+}
+/** Re-point block-style `entity_refs` items from a copy to its survivor. */
+function rewriteSignalRefs(content, copyRef, originalRef) {
+    const front = frontLines(content);
+    if (!front)
+        return content;
+    const index = front.lines.findIndex((line) => /^entity_refs:\s*$/.test(line));
+    if (index === -1) {
+        const inline = front.lines.some((line) => /^entity_refs:\s*\S/.test(line));
+        if (inline && content.includes(copyRef))
+            return "signal_ref_format";
+        return content;
+    }
+    let end = index + 1;
+    // A block sequence may sit at the key's own indent: `entity_refs:\n- company/x`.
+    while (end < front.lines.length && (/^\s/.test(front.lines[end]) || /^- /.test(front.lines[end])))
+        end += 1;
+    const itemRe = /^(\s*)- (["']?)([^"'\n]+?)\2\s*$/;
+    const refs = [];
+    for (let i = index + 1; i < end; i += 1) {
+        const match = itemRe.exec(front.lines[i]);
+        if (!match)
+            continue;
+        const hadMd = match[3].endsWith(".md");
+        refs.push({
+            index: i,
+            value: hadMd ? match[3].slice(0, -3) : match[3],
+            hadMd,
+            indent: match[1],
+            quote: match[2],
+        });
+    }
+    const copyItems = refs.filter((item) => item.value === copyRef);
+    if (copyItems.length === 0)
+        return content;
+    const lines = [...front.lines];
+    const remove = new Set();
+    let seenOriginal = refs.some((item) => item.value === originalRef);
+    for (const item of copyItems) {
+        if (seenOriginal) {
+            remove.add(item.index);
+        }
+        else {
+            const value = item.hadMd ? `${originalRef}.md` : originalRef;
+            lines[item.index] = `${item.indent}- ${item.quote}${value}${item.quote}`;
+            seenOriginal = true;
+        }
+    }
+    const next = withFrontLines(content, front, lines.filter((_, i) => !remove.has(i)), fileNewline(content));
+    const oldParsed = parseEntityFile(content);
+    const newParsed = parseEntityFile(next);
+    if (!oldParsed || !newParsed || oldParsed.body !== newParsed.body)
+        return "signal_ref_format";
+    const keys = new Set([...Object.keys(oldParsed.frontmatter), ...Object.keys(newParsed.frontmatter)]);
+    for (const key of keys) {
+        if (key === "entity_refs")
+            continue;
+        if (!isDeepStrictEqual(oldParsed.frontmatter[key], newParsed.frontmatter[key]))
+            return "signal_ref_format";
+    }
+    const newRefs = newParsed.frontmatter.entity_refs;
+    if (!Array.isArray(newRefs) || !newRefs.every((ref) => typeof ref === "string"))
+        return "signal_ref_format";
+    const normalized = newRefs.map((ref) => (ref.endsWith(".md") ? ref.slice(0, -3) : ref));
+    if (!normalized.includes(originalRef) || normalized.includes(copyRef))
+        return "signal_ref_format";
+    return next;
+}
+const REF_KEYS = new Set(["system", "namespace", "objectType", "recordId"]);
+function refKey(ref) {
+    return `${ref.system}:${ref.namespace}:${ref.recordId}`.toLowerCase();
+}
+/** An `external_refs` entry, or null when it has another shape (so it is never rewritten). */
+function readRef(raw) {
+    if (raw === null || typeof raw !== "object" || Array.isArray(raw))
+        return null;
+    const record = raw;
+    if (!Object.keys(record).every((key) => REF_KEYS.has(key)))
+        return null;
+    if (!Object.values(record).every((value) => typeof value === "string" && value.length > 0))
+        return null;
+    if (record.system === undefined || record.namespace === undefined || record.recordId === undefined)
+        return null;
+    return record;
+}
+/** `external_ids` and `web_domains` of a front matter, or "identity_shape". */
+function readIdentity(frontmatter) {
+    let ids = {};
+    if (frontmatter.external_ids !== undefined) {
+        const raw = frontmatter.external_ids;
+        if (raw === null || typeof raw !== "object" || Array.isArray(raw))
+            return "identity_shape";
+        ids = {};
+        for (const [key, value] of Object.entries(raw)) {
+            if (typeof value !== "string")
+                return "identity_shape";
+            ids[key] = value;
+        }
+    }
+    if (frontmatter.web_domains !== undefined) {
+        if (!Array.isArray(frontmatter.web_domains) || !frontmatter.web_domains.every((domain) => typeof domain === "string")) {
+            return "identity_shape";
+        }
+    }
+    const domains = frontmatter.web_domains === undefined ? [] : frontmatter.web_domains;
+    const refs = [];
+    if (frontmatter.external_refs !== undefined) {
+        if (!Array.isArray(frontmatter.external_refs))
+            return "identity_shape";
+        for (const raw of frontmatter.external_refs) {
+            const ref = readRef(raw);
+            if (!ref)
+                return "identity_shape";
+            refs.push(ref);
+        }
+    }
+    return { ids, domains, refs };
+}
+/**
+ * The survivor's identity after taking in the copy's. Two different ids for
+ * one provider are a conflict unless `resolution` names one of them as
+ * primary; the other is then kept in `external_refs`.
+ */
+function unionIdentity(original, copy, resolution) {
+    const ids = { ...original.ids };
+    const addIds = [];
+    const conflicts = [];
+    const refs = [...original.refs];
+    const addRefs = [];
+    const seenRefs = new Set(refs.map(refKey));
+    const addRef = (ref) => {
+        const key = refKey(ref);
+        if (seenRefs.has(key))
+            return;
+        seenRefs.add(key);
+        refs.push(ref);
+        addRefs.push(key);
+    };
+    for (const ref of copy.refs)
+        addRef(ref);
+    for (const [key, value] of Object.entries(copy.ids)) {
+        if (!(key in ids)) {
+            ids[key] = value;
+            addIds.push(key);
+        }
+        else if (ids[key] !== value) {
+            const settle = Object.hasOwn(resolution ?? {}, key) ? resolution[key] : undefined;
+            if (!settle || (settle.primary !== ids[key] && settle.primary !== value)) {
+                conflicts.push(key);
+                continue;
+            }
+            const secondary = settle.primary === ids[key] ? value : ids[key];
+            ids[key] = settle.primary;
+            addRef({
+                system: key,
+                namespace: settle.namespace,
+                ...(settle.object_type !== undefined ? { objectType: settle.object_type } : {}),
+                recordId: secondary,
+            });
+        }
+    }
+    const domains = [...original.domains];
+    const addDomains = [];
+    const seen = new Set(original.domains.map((domain) => domain.toLowerCase().trim()));
+    for (const domain of copy.domains) {
+        const key = domain.toLowerCase().trim();
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        domains.push(domain);
+        addDomains.push(domain);
+    }
+    return { ids, domains, refs, addIds, addDomains, addRefs, conflicts };
+}
+function frontMatterId(content) {
+    const id = parseEntityFile(content)?.frontmatter.id;
+    return typeof id === "string" ? id : null;
+}
 /**
  * The original's content with the copy merged in. Never removes anything from
  * the original. `siblings` are all copies of the same original; none of them
@@ -374,7 +738,7 @@ function mergeCopy(index, pair, siblings, originalContent, copyContent, now) {
         return normalized !== "" && !own.has(normalized) && !moved.has(normalized);
     });
     const withRefs = appendBackRefs(originalContent, readBackRefs(copy.body));
-    const lines = missingRelationshipLines(withRefs.content, copyContent);
+    const lines = missingRelationshipLines(withRefs.content, copyContent, pair);
     const body = appendRelationshipLines(withRefs.content, lines);
     const raise = copyEntity.signal_count > originalEntity.signal_count ? copyEntity.signal_count : undefined;
     const patched = patchEntityFile(body, {
@@ -386,9 +750,16 @@ function mergeCopy(index, pair, siblings, originalContent, copyContent, now) {
     if (!patched.ok)
         return patched.reason;
     const have = new Set(splitBody(patched.content)?.body.split(/\r?\n/).map((line) => line.trim()) ?? []);
+    // The copy's `# Name` heading is not content: its name moves to an alias.
+    // The `## Relationships` heading is not content either: its lines count one by one.
+    const heading = `# ${copyEntity.canonical_name}`;
     const leftover = (splitBody(copyContent)?.body.split(/\r?\n/) ?? [])
         .map((line) => line.trim())
-        .filter((line) => line !== "" && !have.has(line)).length;
+        .filter((line) => line !== "" &&
+        line !== heading &&
+        !/^##\s+Relationships$/.test(line) &&
+        !have.has(line) &&
+        !(LINK_ITEM.test(line) && !BACKREF_ITEM.test(line) && isPairSelfLine(line, pair))).length;
     const keys = [...new Set([...Object.keys(original.frontmatter), ...Object.keys(copy.frontmatter)])].sort();
     const kept = keys.filter((key) => !OWNED_KEYS.has(key) && !IGNORED_KEYS.has(key) && !isDeepStrictEqual(original.frontmatter[key], copy.frontmatter[key]));
     return {
@@ -423,10 +794,19 @@ function briefLinks(company, copies) {
     }
     return [...out].sort();
 }
-function plan(company, index, pairs, allPairs, now) {
+function plan(company, index, pairs, allPairs, now, rewriteRefs, resolutions = new Map()) {
     const links = scanLinks(company);
     const copies = new Set(allPairs.map((pair) => pair.copy));
     const other = scanOtherLinks(company, copies);
+    if (rewriteRefs) {
+        for (const [copy, rels] of scanRelRefs(company, copies)) {
+            let set = other.get(copy);
+            if (!set)
+                other.set(copy, (set = new Set()));
+            for (const rel of rels)
+                set.add(rel);
+        }
+    }
     const signalRefs = scanSignalRefs(company, copies);
     const siblingsOf = new Map();
     for (const pair of allPairs)
@@ -438,7 +818,39 @@ function plan(company, index, pairs, allPairs, now) {
         const siblings = siblingsOf.get(pair.original) ?? [pair.copy];
         const copyContent = read(company, pair.copy);
         const onDisk = read(company, pair.original);
-        const merged = mergeCopy(index, pair, siblings, out.finals.get(pair.original) ?? onDisk, copyContent, now);
+        let merged = mergeCopy(index, pair, siblings, out.finals.get(pair.original) ?? onDisk, copyContent, now);
+        let identity = null;
+        const resolution = resolutions.get(pair.copy);
+        if (rewriteRefs && typeof merged !== "string") {
+            const mergedFront = parseEntityFile(merged.content)?.frontmatter;
+            const copyFront = parseEntityFile(copyContent)?.frontmatter;
+            const mergedIdentity = mergedFront ? readIdentity(mergedFront) : "identity_shape";
+            const copyIdentity = copyFront ? readIdentity(copyFront) : "identity_shape";
+            if (!mergedFront || !copyFront || mergedIdentity === "identity_shape" || copyIdentity === "identity_shape") {
+                merged = "identity_shape";
+            }
+            else {
+                const union = unionIdentity(mergedIdentity, copyIdentity, resolution);
+                if (union.conflicts.length > 0) {
+                    identity = union;
+                }
+                else {
+                    const ids = setIdentityKeys(merged.content, {
+                        externalIds: union.ids,
+                        webDomains: union.domains,
+                        ...(union.addRefs.length > 0 ? { externalRefs: union.refs } : {}),
+                        lastUpdated: now.toISOString(),
+                    });
+                    if (!ids.ok) {
+                        merged = ids.reason;
+                    }
+                    else {
+                        merged = { ...merged, content: ids.content, changed: merged.changed || ids.changed };
+                        identity = union;
+                    }
+                }
+            }
+        }
         const entry = {
             ...pair,
             add_aliases: typeof merged === "string" ? [] : merged.aliases,
@@ -451,12 +863,23 @@ function plan(company, index, pairs, allPairs, now) {
             signal_refs: refs,
             unmerged_body_lines: typeof merged === "string" ? 0 : merged.leftover,
         };
-        const unwritable = inbound.filter((rel) => !rel.startsWith("ontology/entities/") || !isWritableEntityType(rel.split("/")[2]) || copies.has(rel));
+        if (rewriteRefs && identity && identity.conflicts.length === 0 && typeof merged !== "string") {
+            entry.add_external_ids = identity.addIds;
+            entry.add_web_domains = identity.addDomains;
+            entry.add_external_refs = identity.addRefs;
+        }
+        const unwritable = rewriteRefs
+            ? inbound.filter((rel) => copies.has(rel))
+            : inbound.filter((rel) => !rel.startsWith("ontology/entities/") || !isWritableEntityType(rel.split("/")[2]) || copies.has(rel));
         if (typeof merged === "string") {
             entry.held = "merge_invalid";
             entry.detail = merged;
         }
-        else if (refs.length > 0) {
+        else if (identity && identity.conflicts.length > 0) {
+            entry.held = "external_id_conflict";
+            entry.detail = identity.conflicts.join(", ");
+        }
+        else if (!rewriteRefs && refs.length > 0) {
             entry.held = "signal_refs";
         }
         else if (unwritable.length > 0) {
@@ -468,15 +891,53 @@ function plan(company, index, pairs, allPairs, now) {
             out.finals.set(pair.original, merged.content);
             // Deletion is judged against the original as it is on disk now.
             const current = mergeCopy(index, pair, siblings, onDisk, copyContent, now);
-            const reason = typeof current === "string"
-                ? "merge_invalid"
-                : current.changed
-                    ? "not_merged"
-                    : current.leftover > 0
-                        ? "unmerged_body"
-                        : inbound.length > 0
-                            ? "linked"
-                            : null;
+            let reason;
+            if (!rewriteRefs) {
+                reason =
+                    typeof current === "string"
+                        ? "merge_invalid"
+                        : current.changed
+                            ? "not_merged"
+                            : current.leftover > 0
+                                ? "unmerged_body"
+                                : inbound.length > 0
+                                    ? "linked"
+                                    : null;
+            }
+            else if (typeof current === "string") {
+                reason = "merge_invalid";
+            }
+            else {
+                const diskFront = parseEntityFile(onDisk)?.frontmatter;
+                const copyFront = parseEntityFile(copyContent)?.frontmatter;
+                const diskIdentity = diskFront ? readIdentity(diskFront) : "identity_shape";
+                const copyIdentity = copyFront ? readIdentity(copyFront) : "identity_shape";
+                if (!diskFront || !copyFront || diskIdentity === "identity_shape" || copyIdentity === "identity_shape") {
+                    reason = "merge_invalid";
+                }
+                else {
+                    const union = unionIdentity(diskIdentity, copyIdentity, resolution);
+                    const patched = setIdentityKeys(current.content, {
+                        externalIds: union.ids,
+                        webDomains: union.domains,
+                        ...(union.addRefs.length > 0 ? { externalRefs: union.refs } : {}),
+                        lastUpdated: now.toISOString(),
+                    });
+                    reason = union.conflicts.length > 0
+                        ? "external_id_conflict"
+                        : !patched.ok
+                            ? "merge_invalid"
+                            : current.changed || patched.changed
+                                ? "not_merged"
+                                : current.leftover > 0
+                                    ? "unmerged_body"
+                                    : inbound.length > 0
+                                        ? "linked"
+                                        : refs.length > 0
+                                            ? "signal_refs"
+                                            : null;
+                }
+            }
             if (reason)
                 out.notReady.set(pair.copy, reason);
         }
@@ -505,7 +966,13 @@ export async function mergeCopies(options) {
         : found.pairs;
     const pairs = options.limit !== undefined ? listed.slice(0, Math.max(0, options.limit)) : listed;
     const before = scanLinks(company).broken;
-    const { plans, merges, finals, notReady } = plan(company, index, pairs, found.pairs, now);
+    const rewriteRefs = options.rewriteRefs === true;
+    const resolutions = new Map();
+    for (const entry of options.only ?? []) {
+        if (entry.resolve_external_ids)
+            resolutions.set(entry.copy, entry.resolve_external_ids);
+    }
+    const { plans, merges, finals, notReady } = plan(company, index, pairs, found.pairs, now, rewriteRefs, resolutions);
     const report = {
         company: company.slug,
         mode,
@@ -529,6 +996,7 @@ export async function mergeCopies(options) {
             : {}),
         merged: [],
         relinked: [],
+        ...(rewriteRefs ? { rewritten: [] } : {}),
         deleted: [],
         vault_already_absent: [],
         not_ready: [],
@@ -554,15 +1022,61 @@ export async function mergeCopies(options) {
         for (const entry of plans) {
             if (!merges.has(entry.copy))
                 continue;
+            const copyId = rewriteRefs ? frontMatterId(read(company, entry.copy)) : null;
+            const originalId = rewriteRefs ? frontMatterId(read(company, entry.original)) : null;
             for (const rel of entry.inbound_links) {
                 try {
                     const content = read(company, rel);
-                    const next = relink(rel, content, entry.copy, entry.original);
+                    if (!rewriteRefs) {
+                        const next = relink(rel, content, entry.copy, entry.original);
+                        if (next === content)
+                            continue;
+                        writeEntity(hqRoot, company, rel, next, now);
+                        if (!report.relinked.includes(rel))
+                            report.relinked.push(rel);
+                        continue;
+                    }
+                    const next = rel.startsWith("ontology/entities/")
+                        ? rewriteRelRefs(rel, relink(rel, content, entry.copy, entry.original), entry.copy, entry.original)
+                        : rewriteOtherFile(rel, content, entry.copy, entry.original, copyId, originalId);
+                    if (next === content)
+                        continue;
+                    if (referencesCopy(rel, next, entry.copy)) {
+                        report.errors.push({ path: rel, error: "link_not_rewritten" });
+                        continue;
+                    }
+                    if (rel.startsWith("ontology/entities/")) {
+                        writeEntity(hqRoot, company, rel, next, now);
+                        if (!report.relinked.includes(rel))
+                            report.relinked.push(rel);
+                    }
+                    else {
+                        safeWriteFile(hqRoot, [company.dir], path.join(company.dir, rel), next);
+                        if (!report.rewritten.includes(rel))
+                            report.rewritten.push(rel);
+                    }
+                }
+                catch (error) {
+                    report.errors.push({ path: rel, error: error.name });
+                }
+            }
+            if (!rewriteRefs)
+                continue;
+            const copyRef = entityRefOf(entry.copy);
+            const originalRef = entityRefOf(entry.original);
+            for (const rel of entry.signal_refs) {
+                try {
+                    const content = read(company, rel);
+                    const next = rewriteSignalRefs(content, copyRef, originalRef);
+                    if (next === "signal_ref_format") {
+                        report.errors.push({ path: rel, error: "signal_ref_format" });
+                        continue;
+                    }
                     if (next === content)
                         continue;
-                    writeEntity(hqRoot, company, rel, next, now);
-                    if (!report.relinked.includes(rel))
-                        report.relinked.push(rel);
+                    safeWriteFile(hqRoot, [company.dir], path.join(company.dir, rel), next);
+                    if (!report.rewritten.includes(rel))
+                        report.rewritten.push(rel);
                 }
                 catch (error) {
                     report.errors.push({ path: rel, error: error.name });