@indigoai-us/hq-cli 5.270.0 → 5.272.0

package/dist/lib/checkpoint-ontology/merge-copies.js

@@ -0,0 +1,521 @@
+/**
+ * Merge numbered entity copies (`<slug> 2.md`) into their original file and
+ * delete them (CO-54, owner decision CO-D-007 option 2).
+ *
+ * Gated by `checkpoint.ontology.merge-copies`. A copy is a file named
+ * `<slug> <n>.md` next to `<slug>.md` of the same type, with the same `id`,
+ * in one duplicate group. Other members of a group are never touched.
+ *
+ * Two steps, run separately so sync can push the merged original before its
+ * copy goes away:
+ *
+ * - merge: the copy's fresh aliases, back-references and relationship lines
+ *   are added to the original, `signal_count` takes the higher value, and
+ *   entity files that link to the copy are re-pointed at the original. Every
+ *   rewritten file is recorded in the outbox before it is written, so sync
+ *   pushes it. Keys the checkpoint does not own keep the original's value.
+ * - delete: a copy is deleted from the vault, by exact key, only when merging
+ *   it again would change nothing and no entity file or signal links to it.
+ *   The vault keeps every old version. The local file goes after the vault
+ *   delete succeeds, or when the vault no longer has the key.
+ *
+ * Copies of one original merge in path order, each into the result of the
+ * previous one, and none of them counts as holding a name the others move.
+ * A copy whose body has lines the merged original lacks (prose the merge does
+ * not carry) is merged but never deleted. A copy linked from any file the
+ * merge cannot rewrite (a copy, a knowledge file, a signal) is held.
+ *
+ * A signal whose `entity_refs` names a copy holds that copy: signals are
+ * server-owned, so the ref cannot be re-pointed from here.
+ */
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { isDeepStrictEqual } from "node:util";
+import { freshAliases, ownNames } from "./aliases.js";
+import { entitiesRoot, safeWriteFile } from "./company.js";
+import { appendBackRefs, OWNED_KEYS, parseEntityFile, patchEntityFile } from "./entity-file.js";
+import { isWritableEntityType, normalizeEntityName } from "./identity.js";
+import { findDuplicateGroups, readBackRefs, recordOutbox } from "./multi-writer.js";
+import { buildEntityIndex, listSignalFiles, readSignal } from "./tree.js";
+export const MERGE_COPIES_FLAG = "checkpoint.ontology.merge-copies";
+/** `<slug> <n>.md`, the name a file manager gives a second copy. */
+const COPY_NAME = /^(.+) (\d+)\.md$/;
+/** A relationship list item with a link, as the gardener and checkpoint write them. */
+const LINK_ITEM = /^- .*\]\([^)\n]+\)\s*$/;
+const BACKREF_ITEM = /^- Referenced in \[/;
+const MD_LINK = /\]\(([^)\n]+)\)/g;
+/** Keys that differ between copies for reasons that say nothing about content. */
+const IGNORED_KEYS = new Set(["first_seen", "confidence"]);
+function sectionLines(body) {
+    const match = /(^|\n)##\s+Relationships\s*\n([\s\S]*?)(?=\n##\s|$)/.exec(body);
+    return match ? match[2].split(/\r?\n/).map((line) => line.trimEnd()) : [];
+}
+function splitBody(content) {
+    const match = /^(---\r?\n[\s\S]*?\r?\n---\r?\n?)([\s\S]*)$/.exec(content);
+    return match ? { frontmatter: match[1], body: match[2] } : null;
+}
+/** Relationship lines of the copy that the original lacks (back-references excluded). */
+function missingRelationshipLines(originalContent, copyContent) {
+    const original = splitBody(originalContent);
+    const copy = splitBody(copyContent);
+    if (!original || !copy)
+        return [];
+    const have = new Set(original.body.split(/\r?\n/).map((line) => line.trimEnd()));
+    const out = [];
+    for (const line of sectionLines(copy.body)) {
+        if (!LINK_ITEM.test(line) || BACKREF_ITEM.test(line) || have.has(line))
+            continue;
+        have.add(line);
+        out.push(line);
+    }
+    return out;
+}
+/** Append lines to `## Relationships`, creating the section when it is missing. */
+function appendRelationshipLines(content, lines) {
+    if (lines.length === 0)
+        return content;
+    const parts = splitBody(content);
+    const body = parts.body;
+    const section = /(^|\n)(##\s+Relationships\s*\n)([\s\S]*?)(?=\n##\s|\s*$)/.exec(body);
+    let next;
+    if (section) {
+        const before = body.slice(0, section.index + section[1].length);
+        const kept = section[3].replace(/\s+$/, "");
+        const after = body.slice(section.index + section[0].length);
+        next = `${before}${section[2]}${kept ? `${kept}\n${lines.join("\n")}` : lines.join("\n")}\n${after}`;
+    }
+    else {
+        const trimmed = body.replace(/\s+$/, "");
+        next = `${trimmed.length > 0 ? `${trimmed}\n\n` : "\n"}## Relationships\n\n${lines.join("\n")}\n`;
+    }
+    return `${parts.frontmatter}${next}`;
+}
+/** Remove one entity from the name indexes so its names stop counting as taken. */
+function unindex(index, rel) {
+    index.byRel.delete(rel);
+    for (const map of [index.canonical, index.alias, index.squashed]) {
+        for (const [key, rels] of map) {
+            if (!rels.delete(rel))
+                continue;
+            if (rels.size === 0)
+                map.delete(key);
+        }
+    }
+}
+/** Copies in scope, and the ones skipped with a reason. Read-only. */
+export function findCopyPairs(index) {
+    const groupOf = new Map();
+    findDuplicateGroups(index).forEach((group, i) => {
+        for (const rel of group.paths)
+            groupOf.set(rel, i);
+    });
+    const pairs = [];
+    const skipped = [];
+    const candidates = [...index.byRel.values()]
+        .filter((entity) => isWritableEntityType(entity.type) && COPY_NAME.test(path.posix.basename(entity.rel)))
+        .sort((a, b) => a.rel.localeCompare(b.rel));
+    for (const copy of candidates) {
+        const base = COPY_NAME.exec(path.posix.basename(copy.rel))[1];
+        const original = `${path.posix.dirname(copy.rel)}/${base}.md`;
+        const target = index.byRel.get(original);
+        if (!target) {
+            skipped.push({ copy: copy.rel, reason: index.invalid.has(original) ? "original_invalid" : "no_original" });
+        }
+        else if (target.id !== copy.id || target.type !== copy.type) {
+            skipped.push({ copy: copy.rel, reason: "identity_differs" });
+        }
+        else if (groupOf.get(copy.rel) === undefined || groupOf.get(copy.rel) !== groupOf.get(original)) {
+            skipped.push({ copy: copy.rel, reason: "not_in_duplicate_group" });
+        }
+        else {
+            pairs.push({ type: copy.type, copy: copy.rel, original });
+        }
+    }
+    return { pairs, skipped };
+}
+/** Resolve a relative link in `fromRel` to a company-relative path, or null. */
+function resolveLink(fromRel, href) {
+    if (/^[a-z][a-z0-9+.-]*:/i.test(href) || href.startsWith("#") || href.startsWith("/"))
+        return null;
+    let decoded;
+    try {
+        decoded = decodeURI(href.split("#")[0]);
+    }
+    catch {
+        return null;
+    }
+    const resolved = path.posix.normalize(path.posix.join(path.posix.dirname(fromRel), decoded));
+    return resolved.startsWith("..") ? null : resolved;
+}
+function listEntityFiles(company) {
+    const root = entitiesRoot(company);
+    const out = [];
+    let types;
+    try {
+        types = fs.readdirSync(root, { withFileTypes: true });
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return out;
+        throw error;
+    }
+    for (const type of types.filter((entry) => entry.isDirectory())) {
+        for (const file of fs.readdirSync(path.join(root, type.name), { withFileTypes: true })) {
+            if (file.isFile() && file.name.endsWith(".md"))
+                out.push(`ontology/entities/${type.name}/${file.name}`);
+        }
+    }
+    return out.sort();
+}
+/** Company-relative `.md` files outside the entity tree and the rendered brief. */
+function listOtherFiles(company) {
+    const out = [];
+    const walk = (rel) => {
+        for (const entry of fs.readdirSync(path.join(company.dir, rel), { withFileTypes: true })) {
+            if (entry.name.startsWith(".") || entry.name === "node_modules")
+                continue;
+            const child = rel ? `${rel}/${entry.name}` : entry.name;
+            if (entry.isDirectory()) {
+                if (child !== "ontology/entities")
+                    walk(child);
+            }
+            else if (entry.isFile() && entry.name.endsWith(".md") && child !== "company-brief.md") {
+                out.push(child);
+            }
+        }
+    };
+    walk("");
+    return out.sort();
+}
+/**
+ * Files outside the entity tree that link to a copy, by markdown link,
+ * `[[ontology/entities/...]]` annotation or a front-matter `path:` entry.
+ * These files are not the merge's to rewrite, so a copy they name is held.
+ */
+function scanOtherLinks(company, copies) {
+    const out = new Map();
+    if (copies.size === 0)
+        return out;
+    const names = [...new Set([...copies].map((copy) => path.posix.basename(copy, ".md")))];
+    const add = (target, rel) => {
+        if (!copies.has(target))
+            return;
+        let set = out.get(target);
+        if (!set)
+            out.set(target, (set = new Set()));
+        set.add(rel);
+    };
+    for (const rel of listOtherFiles(company)) {
+        const content = read(company, rel);
+        if (!names.some((name) => content.includes(name) || content.includes(encodeURI(name))))
+            continue;
+        for (const match of content.matchAll(MD_LINK)) {
+            const target = resolveLink(rel, match[1]);
+            if (target)
+                add(target, rel);
+        }
+        for (const match of content.matchAll(/\[\[(ontology\/entities\/[^\]|\n]+?)(?:\|[^\]\n]*)?\]\]/g)) {
+            add(match[1].endsWith(".md") ? match[1] : `${match[1]}.md`, rel);
+        }
+        for (const match of content.matchAll(/^\s*path:\s*["']?(ontology\/entities\/[^"'\n]+?\.md)["']?\s*$/gm))
+            add(match[1], rel);
+    }
+    return out;
+}
+function scanLinks(company) {
+    const scan = { inbound: new Map(), broken: 0 };
+    for (const rel of listEntityFiles(company)) {
+        const content = fs.readFileSync(path.join(company.dir, rel), "utf8");
+        for (const match of content.matchAll(MD_LINK)) {
+            const target = resolveLink(rel, match[1]);
+            if (!target)
+                continue;
+            if (!fs.existsSync(path.join(company.dir, target)))
+                scan.broken += 1;
+            if (target === rel)
+                continue;
+            let set = scan.inbound.get(target);
+            if (!set)
+                scan.inbound.set(target, (set = new Set()));
+            set.add(rel);
+        }
+    }
+    return scan;
+}
+/** Signal files whose `entity_refs` name a copy, keyed by the copy's path. */
+function scanSignalRefs(company, copies) {
+    const out = new Map();
+    if (copies.size === 0)
+        return out;
+    const byRef = new Map();
+    for (const copy of copies) {
+        const [, , type, file] = copy.split("/");
+        byRef.set(`${type}/${file.slice(0, -3)}`, copy);
+    }
+    for (const rel of listSignalFiles(company.dir)) {
+        for (const ref of readSignal(company.dir, rel)?.refs ?? []) {
+            const copy = byRef.get(ref);
+            if (copy)
+                out.set(copy, [...(out.get(copy) ?? []), rel]);
+        }
+    }
+    return out;
+}
+/** Re-point every link in `content` that resolves to `copy` at `original`. */
+function relink(fromRel, content, copy, original) {
+    return content.replace(MD_LINK, (whole, href) => {
+        if (resolveLink(fromRel, href) !== copy)
+            return whole;
+        const hash = href.indexOf("#");
+        const target = hash === -1 ? href : href.slice(0, hash);
+        const fragment = hash === -1 ? "" : href.slice(hash);
+        const cut = target.lastIndexOf("/");
+        const name = path.posix.basename(original);
+        const encoded = target.slice(cut + 1).includes("%20") ? encodeURI(name) : name;
+        return `](${target.slice(0, cut + 1)}${encoded}${fragment})`;
+    });
+}
+/**
+ * The original's content with the copy merged in. Never removes anything from
+ * the original. `siblings` are all copies of the same original; none of them
+ * counts as holding a name.
+ */
+function mergeCopy(index, pair, siblings, originalContent, copyContent, now) {
+    const original = parseEntityFile(originalContent);
+    const copy = parseEntityFile(copyContent);
+    if (!original || !copy)
+        return "unparseable";
+    const copyEntity = index.byRel.get(pair.copy);
+    const originalEntity = index.byRel.get(pair.original);
+    if (!copyEntity || !originalEntity)
+        return "not_indexed";
+    const scratch = {
+        ...index,
+        byRel: new Map(index.byRel),
+        canonical: new Map([...index.canonical].map(([k, v]) => [k, new Set(v)])),
+        alias: new Map([...index.alias].map(([k, v]) => [k, new Set(v)])),
+        squashed: new Map([...index.squashed].map(([k, v]) => [k, new Set(v)])),
+    };
+    for (const sibling of siblings)
+        unindex(scratch, sibling);
+    const own = ownNames(scratch, pair.original, []);
+    const names = [copyEntity.canonical_name, ...copyEntity.aliases];
+    const aliases = freshAliases(scratch, pair.type, pair.original, own, names);
+    const moved = new Set(aliases.map((alias) => normalizeEntityName(alias, pair.type)));
+    const dropped = names.filter((name) => {
+        const normalized = normalizeEntityName(name, pair.type);
+        return normalized !== "" && !own.has(normalized) && !moved.has(normalized);
+    });
+    const withRefs = appendBackRefs(originalContent, readBackRefs(copy.body));
+    const lines = missingRelationshipLines(withRefs.content, copyContent);
+    const body = appendRelationshipLines(withRefs.content, lines);
+    const raise = copyEntity.signal_count > originalEntity.signal_count ? copyEntity.signal_count : undefined;
+    const patched = patchEntityFile(body, {
+        addAliases: aliases,
+        signalCount: copyEntity.signal_count,
+        lastUpdated: now.toISOString(),
+        bodyChanged: withRefs.added > 0 || lines.length > 0,
+    });
+    if (!patched.ok)
+        return patched.reason;
+    const have = new Set(splitBody(patched.content)?.body.split(/\r?\n/).map((line) => line.trim()) ?? []);
+    const leftover = (splitBody(copyContent)?.body.split(/\r?\n/) ?? [])
+        .map((line) => line.trim())
+        .filter((line) => line !== "" && !have.has(line)).length;
+    const keys = [...new Set([...Object.keys(original.frontmatter), ...Object.keys(copy.frontmatter)])].sort();
+    const kept = keys.filter((key) => !OWNED_KEYS.has(key) && !IGNORED_KEYS.has(key) && !isDeepStrictEqual(original.frontmatter[key], copy.frontmatter[key]));
+    return {
+        content: patched.content,
+        changed: patched.changed,
+        aliases,
+        dropped,
+        backrefs: withRefs.added,
+        lines: lines.length,
+        ...(raise !== undefined ? { raise } : {}),
+        kept,
+        leftover,
+    };
+}
+function read(company, rel) {
+    return fs.readFileSync(path.join(company.dir, rel), "utf8");
+}
+/** Links to copies in the company brief (rendered, so reported and not rewritten). */
+function briefLinks(company, copies) {
+    let content;
+    try {
+        content = fs.readFileSync(path.join(company.dir, "company-brief.md"), "utf8");
+    }
+    catch {
+        return [];
+    }
+    const out = new Set();
+    for (const match of content.matchAll(MD_LINK)) {
+        const target = resolveLink("company-brief.md", match[1]);
+        if (target && copies.has(target))
+            out.add(target);
+    }
+    return [...out].sort();
+}
+function plan(company, index, pairs, allPairs, now) {
+    const links = scanLinks(company);
+    const copies = new Set(allPairs.map((pair) => pair.copy));
+    const other = scanOtherLinks(company, copies);
+    const signalRefs = scanSignalRefs(company, copies);
+    const siblingsOf = new Map();
+    for (const pair of allPairs)
+        siblingsOf.set(pair.original, [...(siblingsOf.get(pair.original) ?? []), pair.copy]);
+    const out = { plans: [], merges: new Map(), finals: new Map(), notReady: new Map() };
+    for (const pair of pairs) {
+        const inbound = [...new Set([...(links.inbound.get(pair.copy) ?? []), ...(other.get(pair.copy) ?? [])])].sort();
+        const refs = (signalRefs.get(pair.copy) ?? []).sort();
+        const siblings = siblingsOf.get(pair.original) ?? [pair.copy];
+        const copyContent = read(company, pair.copy);
+        const onDisk = read(company, pair.original);
+        const merged = mergeCopy(index, pair, siblings, out.finals.get(pair.original) ?? onDisk, copyContent, now);
+        const entry = {
+            ...pair,
+            add_aliases: typeof merged === "string" ? [] : merged.aliases,
+            dropped_aliases: typeof merged === "string" ? [] : merged.dropped,
+            add_backrefs: typeof merged === "string" ? 0 : merged.backrefs,
+            add_relationship_lines: typeof merged === "string" ? 0 : merged.lines,
+            ...(typeof merged !== "string" && merged.raise !== undefined ? { raise_signal_count: merged.raise } : {}),
+            kept_original_keys: typeof merged === "string" ? [] : merged.kept,
+            inbound_links: inbound,
+            signal_refs: refs,
+            unmerged_body_lines: typeof merged === "string" ? 0 : merged.leftover,
+        };
+        const unwritable = inbound.filter((rel) => !rel.startsWith("ontology/entities/") || !isWritableEntityType(rel.split("/")[2]) || copies.has(rel));
+        if (typeof merged === "string") {
+            entry.held = "merge_invalid";
+            entry.detail = merged;
+        }
+        else if (refs.length > 0) {
+            entry.held = "signal_refs";
+        }
+        else if (unwritable.length > 0) {
+            entry.held = "linked_from_unwritable";
+            entry.detail = unwritable.join(", ");
+        }
+        else {
+            out.merges.set(pair.copy, merged);
+            out.finals.set(pair.original, merged.content);
+            // Deletion is judged against the original as it is on disk now.
+            const current = mergeCopy(index, pair, siblings, onDisk, copyContent, now);
+            const reason = typeof current === "string"
+                ? "merge_invalid"
+                : current.changed
+                    ? "not_merged"
+                    : current.leftover > 0
+                        ? "unmerged_body"
+                        : inbound.length > 0
+                            ? "linked"
+                            : null;
+            if (reason)
+                out.notReady.set(pair.copy, reason);
+        }
+        out.plans.push(entry);
+    }
+    return out;
+}
+function writeEntity(hqRoot, company, rel, content, now) {
+    // Outbox first: sync pushes an entity file only when its bytes match.
+    const write = { rel, content };
+    recordOutbox(hqRoot, company, [write], now);
+    safeWriteFile(hqRoot, [entitiesRoot(company)], path.join(company.dir, rel), content);
+}
+/**
+ * Plan, merge or delete numbered copies for one company. The dry run writes
+ * nothing; merge writes entity files only; delete removes vault keys and
+ * local copy files only.
+ */
+export async function mergeCopies(options) {
+    const { hqRoot, company, mode, now } = options;
+    const index = buildEntityIndex(company.dir);
+    const found = findCopyPairs(index);
+    const pairs = options.limit !== undefined ? found.pairs.slice(0, Math.max(0, options.limit)) : found.pairs;
+    const before = scanLinks(company).broken;
+    const { plans, merges, finals, notReady } = plan(company, index, pairs, found.pairs, now);
+    const report = {
+        company: company.slug,
+        mode,
+        duplicate_groups_before: findDuplicateGroups(index).length,
+        duplicate_groups_after: 0,
+        copies_found: found.pairs.length,
+        plans,
+        skipped: found.skipped,
+        merged: [],
+        relinked: [],
+        deleted: [],
+        vault_already_absent: [],
+        not_ready: [],
+        errors: [],
+        broken_links_before: before,
+        broken_links_after: before,
+        brief_links_to_copies: briefLinks(company, new Set(pairs.map((pair) => pair.copy))),
+    };
+    if (mode === "merge") {
+        // Originals first, once each with every copy merged in; then links, read
+        // back from disk so a relinked original keeps its merge.
+        for (const [original, content] of finals) {
+            try {
+                if (content === read(company, original))
+                    continue;
+                writeEntity(hqRoot, company, original, content, now);
+                report.merged.push(original);
+            }
+            catch (error) {
+                report.errors.push({ path: original, error: error.name });
+            }
+        }
+        for (const entry of plans) {
+            if (!merges.has(entry.copy))
+                continue;
+            for (const rel of entry.inbound_links) {
+                try {
+                    const content = read(company, rel);
+                    const next = relink(rel, content, entry.copy, entry.original);
+                    if (next === content)
+                        continue;
+                    writeEntity(hqRoot, company, rel, next, now);
+                    if (!report.relinked.includes(rel))
+                        report.relinked.push(rel);
+                }
+                catch (error) {
+                    report.errors.push({ path: rel, error: error.name });
+                }
+            }
+        }
+    }
+    if (mode === "delete") {
+        if (!options.deleteVaultKey)
+            throw new Error("merge-copies: delete needs a vault client");
+        for (const entry of plans) {
+            const reason = entry.held ?? notReady.get(entry.copy) ?? null;
+            if (reason) {
+                report.not_ready.push({ copy: entry.copy, reason });
+                continue;
+            }
+            try {
+                const deleted = await options.deleteVaultKey(entry.copy);
+                // 0 means the exact key is already gone (a run that stopped before the
+                // local removal, or a copy that never synced): finish the local side.
+                if (deleted !== 1 && deleted !== 0) {
+                    report.errors.push({ path: entry.copy, error: `vault_deleted_${deleted}` });
+                    continue;
+                }
+                fs.rmSync(path.join(company.dir, entry.copy), { force: true });
+                report.deleted.push(entry.copy);
+                if (deleted === 0)
+                    report.vault_already_absent.push(entry.copy);
+            }
+            catch (error) {
+                report.errors.push({ path: entry.copy, error: error.message || error.name });
+            }
+        }
+    }
+    const after = buildEntityIndex(company.dir);
+    report.duplicate_groups_after = findDuplicateGroups(after).length;
+    report.broken_links_after = scanLinks(company).broken;
+    return report;
+}
+//# sourceMappingURL=merge-copies.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@indigoai-us/hq-cli",
-  "version": "5.270.0",
+  "version": "5.272.0",
   "description": "HQ by Indigo management CLI — modules and cloud sync",
   "main": "dist/index.js",
   "bin": {