@indigoai-us/hq-cli 5.221.0 → 5.223.0

package/dist/lib/checkpoint-ontology/apply.d.ts

@@ -5,6 +5,7 @@
  * sections 4.3, 5 and 14). The model output is untrusted input: a malformed
  * candidate is rejected and counted, never repaired.
  */
+import { type DuplicateReport, type MergeBackOutcome } from "./multi-writer.js";
 import { type CompanyBriefResult } from "./company-brief.js";
 import { type WritableEntityType } from "./identity.js";
 import { type EntityIndex } from "./tree.js";
@@ -76,6 +77,15 @@ export interface ApplyReport {
     }>;
     /** Present only when the plan ran the company brief and this apply created or updated entities. */
     company_brief?: CompanyBriefResult;
+    /** Present only when `checkpoint.ontology.multi-writer` was on for the plan (design section 20). */
+    multi_writer?: {
+        conflicts: MergeBackOutcome[];
+        /** Entity files this apply wrote and recorded in the outbox. */
+        outbox_recorded: number;
+        outbox_reset?: true;
+        error?: string;
+        duplicates?: DuplicateReport;
+    };
     duration_ms: number;
 }
 export interface ApplyOptions {

package/dist/lib/checkpoint-ontology/apply.js

@@ -8,6 +8,8 @@
 import { createHash } from "node:crypto";
 import * as fs from "node:fs";
 import * as path from "node:path";
+import { entityKey, freshAliases, MAX_NAME_LENGTH, ownNames } from "./aliases.js";
+import { recordOutbox, scanDuplicates } from "./multi-writer.js";
 import { writeCompanyBrief } from "./company-brief.js";
 import { entitiesRoot, resolveOntologyCompany, safeWriteFile } from "./company.js";
 import { appendBackRefs, parseEntityFile, patchEntityFile, previewSignalContent, renderNewEntity, validateEntityFrontmatter, } from "./entity-file.js";
@@ -20,7 +22,7 @@ export const CAPS = {
     bytes: 200 * 1024,
     newAliasesPerEntity: 3,
     aliasesPerCandidate: 5,
-    nameLength: 200,
+    nameLength: MAX_NAME_LENGTH,
 };
 export const MIN_CREATE_CONFIDENCE = 0.7;
 export class ApplyError extends Error {
@@ -60,21 +62,18 @@ export function parseCandidate(raw) {
         evidence: typeof c.evidence === "string" ? c.evidence : "",
     };
 }
-function key(type, normalized) {
-    return `${type}\0${normalized}`;
-}
 /** Design section 5.3, tiers 1 to 4. */
 export function resolveCandidate(index, type, canonical) {
     const normalized = normalizeEntityName(canonical, type);
-    const byCanonical = [...(index.canonical.get(key(type, normalized)) ?? [])].sort();
+    const byCanonical = [...(index.canonical.get(entityKey(type, normalized)) ?? [])].sort();
     if (byCanonical.length > 0)
         return { tier: 1, target: byCanonical[0], duplicates: byCanonical.slice(1) };
-    const byAlias = [...(index.alias.get(key(type, normalized)) ?? [])].sort();
+    const byAlias = [...(index.alias.get(entityKey(type, normalized)) ?? [])].sort();
     if (byAlias.length === 1)
         return { tier: 2, target: byAlias[0] };
     if (byAlias.length > 1)
         return { tier: "ambiguous_alias", paths: byAlias };
-    const squashed = [...(index.squashed.get(key(type, squash(normalized))) ?? [])].sort();
+    const squashed = [...(index.squashed.get(entityKey(type, squash(normalized))) ?? [])].sort();
     if (squashed.length > 0)
         return { tier: "suspect_duplicate", paths: squashed };
     const otherTypes = [...(index.types.get(normalized) ?? [])].filter((other) => other !== type).sort();
@@ -95,14 +94,14 @@ export function overlappingEntities(index, type, canonical, aliases) {
         // A generic alias still counts when an existing entity already holds it.
         if (!normalized)
             continue;
-        for (const rel of index.canonical.get(key(type, normalized)) ?? [])
+        for (const rel of index.canonical.get(entityKey(type, normalized)) ?? [])
             paths.add(rel);
-        for (const rel of index.alias.get(key(type, normalized)) ?? [])
+        for (const rel of index.alias.get(entityKey(type, normalized)) ?? [])
             paths.add(rel);
     }
     if (type === "person") {
         const words = new Set(normalizeEntityName(canonical, type).split(" "));
-        const prefix = key(type, "");
+        const prefix = entityKey(type, "");
         for (const [name, rels] of index.canonical) {
             if (!name.startsWith(prefix))
                 continue;
@@ -114,39 +113,6 @@ export function overlappingEntities(index, type, canonical, aliases) {
     }
     return [...paths].sort();
 }
-/**
- * Aliases worth adding to the entity at `self`: new to it, and held by no
- * other entity of any type, so an alias can never make a later match
- * ambiguous. `own` holds the normalised names the entity already answers to.
- */
-function freshAliases(index, type, self, own, names) {
-    const seen = new Set(own);
-    const out = [];
-    for (const raw of names) {
-        const alias = canonicalizeName(raw);
-        if (!alias || alias.length > CAPS.nameLength)
-            continue;
-        const normalized = normalizeEntityName(alias, type);
-        if (!normalized || seen.has(normalized) || isRejectedName(normalized))
-            continue;
-        const holders = [
-            ...(index.canonical.get(key(type, normalized)) ?? []),
-            ...(index.alias.get(key(type, normalized)) ?? []),
-        ];
-        if (holders.some((holder) => holder !== self))
-            continue;
-        if ([...(index.types.get(normalized) ?? [])].some((other) => other !== type))
-            continue;
-        seen.add(normalized);
-        out.push(alias);
-    }
-    return out;
-}
-/** The normalised names an indexed entity answers to, plus aliases queued this run. */
-function ownNames(index, rel, queued) {
-    const entity = index.byRel.get(rel);
-    return new Set([entity.canonical_name, ...entity.aliases, ...queued].map((name) => normalizeEntityName(name, entity.type)));
-}
 function readJson(file) {
     return JSON.parse(fs.readFileSync(file, "utf8"));
 }
@@ -211,7 +177,33 @@ export function applyOntologyRun(options) {
         write_errors: [],
         duration_ms: 0,
     };
+    const outboxRels = new Set();
+    if (plan.multi_writer) {
+        report.multi_writer = {
+            conflicts: plan.multi_writer.merge_back ?? [],
+            outbox_recorded: 0,
+            ...(plan.multi_writer.error ? { error: plan.multi_writer.error } : {}),
+        };
+    }
+    // Design 20.4: the outbox entry is written before the entity file, so sync
+    // never sees a checkpoint write without one. A failed outbox write stops
+    // the entity write and leaves its inputs pending for the next run.
+    const recordBeforeWrite = (rel, content) => {
+        if (!report.multi_writer)
+            return null;
+        try {
+            if (recordOutbox(options.hqRoot, company, [{ rel, content }], options.now ?? new Date()).reset) {
+                report.multi_writer.outbox_reset = true;
+            }
+            return null;
+        }
+        catch (error) {
+            return `outbox:${error.name}`;
+        }
+    };
     const finish = () => {
+        if (report.multi_writer)
+            report.multi_writer.outbox_recorded = outboxRels.size;
         report.duration_ms = Date.now() - started;
         fs.writeFileSync(path.join(options.runDir, REPORT_FILE), `${JSON.stringify(report, null, 2)}\n`);
         return report;
@@ -377,6 +369,12 @@ export function applyOntologyRun(options) {
             blocked.add(candidate.input);
             continue;
         }
+        const createOutboxError = recordBeforeWrite(rel, content);
+        if (createOutboxError) {
+            report.write_errors.push({ path: rel, reason: createOutboxError });
+            blocked.add(candidate.input);
+            continue;
+        }
         try {
             safeWriteFile(options.hqRoot, [entityRoot], target, content);
         }
@@ -386,9 +384,11 @@ export function applyOntologyRun(options) {
             continue;
         }
         report.bytes_written += bytes;
+        if (report.multi_writer)
+            outboxRels.add(rel);
         report.created.push(rel);
         report.links.push({ input: candidate.input, action: "create", type: candidate.type, canonical_name: canonical, path: rel });
-        indexEntity(index, { ...fields, rel, slug, last_updated: now });
+        indexEntity(index, { ...fields, rel, slug, last_updated: now, first_seen: now });
     }
     applySignalRefs(company, plan, inputs, index, update);
     for (const rel of [...updates.keys()].sort()) {
@@ -423,6 +423,13 @@ export function applyOntologyRun(options) {
             report.deferred.push({ input: [...pending.inputs][0] ?? "", type: entity.type, name: entity.canonical_name, reason });
             continue;
         }
+        const patchOutboxError = recordBeforeWrite(rel, patched.content);
+        if (patchOutboxError) {
+            report.write_errors.push({ path: rel, reason: patchOutboxError });
+            for (const id of pending.inputs)
+                blocked.add(id);
+            continue;
+        }
         try {
             safeWriteFile(options.hqRoot, [entityRoot], target, patched.content);
         }
@@ -434,6 +441,8 @@ export function applyOntologyRun(options) {
         }
         report.bytes_written += bytes;
         report.backrefs_added += withRefs.added;
+        if (report.multi_writer)
+            outboxRels.add(rel);
         // An entity created earlier in this run is reported once, as created.
         if (!report.created.includes(rel))
             report.updated.push(rel);
@@ -443,6 +452,14 @@ export function applyOntologyRun(options) {
     if (plan.company_brief && report.created.length + report.updated.length > 0) {
         report.company_brief = writeCompanyBrief(options.hqRoot, company, options.now ?? new Date());
     }
+    if (report.multi_writer) {
+        try {
+            report.multi_writer.duplicates = scanDuplicates(company);
+        }
+        catch (error) {
+            report.multi_writer.error = `duplicates:${error.name}`;
+        }
+    }
     return finish();
 }
 let signalRefIndex = null;

package/dist/lib/checkpoint-ontology/entity-file.d.ts

@@ -43,6 +43,8 @@ export interface NewEntity {
 }
 /** A new entity file in the cloud schema, with a `# Name` heading. */
 export declare function renderNewEntity(entity: NewEntity): string;
+/** The frontmatter keys a checkpoint may change in an existing file. */
+export declare const OWNED_KEYS: ReadonlySet<string>;
 export interface EntityPatch {
     addAliases?: string[];
     signalCount?: number;

package/dist/lib/checkpoint-ontology/entity-file.js

@@ -111,7 +111,8 @@ export function renderNewEntity(entity) {
         "",
     ].join("\n");
 }
-const OWNED_KEYS = new Set(["aliases", "signal_count", "last_updated"]);
+/** The frontmatter keys a checkpoint may change in an existing file. */
+export const OWNED_KEYS = new Set(["aliases", "signal_count", "last_updated"]);
 /**
  * Replace one top-level key in a raw frontmatter block. The key's value may
  * span several lines (a block sequence); continuation lines are the ones that

package/dist/lib/checkpoint-ontology/multi-writer.d.ts

@@ -0,0 +1,106 @@
+/**
+ * Several checkpoint writers in one company vault (design section 20).
+ *
+ * Gated by `checkpoint.ontology.multi-writer`. Three parts:
+ *
+ * - The outbox records the exact bytes this machine's checkpoint wrote to
+ *   each entity file. Sync pushes an entity file only when it still matches
+ *   its outbox entry, so an old pulled copy is never uploaded (C1, C2).
+ * - Merge-back re-applies the monotone part of an entity version that sync
+ *   set aside after losing a push race (C3, C4). A version whose entity was
+ *   deleted stays aside (C6); one that differs in keys the checkpoint does
+ *   not own is merged for the owned keys and also stays aside (C8).
+ * - The duplicate scan reports entities of one type whose names collide, with
+ *   a proposed survivor. Nothing is deleted (C5, C7).
+ */
+import { type OntologyCompany } from "./company.js";
+import { type SignalBackRef } from "./entity-file.js";
+import { type EntityIndex } from "./tree.js";
+export declare const OUTBOX_SCHEMA = "hq-ontology-outbox.v1";
+export declare const OUTBOX_DIR = ".hq/ontology-outbox";
+export declare const OUTBOX_RETENTION_MS: number;
+/** Where sync parks a losing local body (hq-cloud `CONFLICT_BACKUPS_DIR`). */
+export declare const CONFLICT_BACKUPS_DIR = ".hq/conflict-backups";
+export declare const MERGED_BACKUPS_DIR = ".hq/conflict-backups/ontology-merged";
+export declare const STALE_BACKUP_MS: number;
+export declare const DUPLICATE_REPORT_LIMIT = 20;
+export interface OutboxEntry {
+    sha256: string;
+    written_at: string;
+}
+export interface Outbox {
+    schema: typeof OUTBOX_SCHEMA;
+    company: string;
+    entries: Record<string, OutboxEntry>;
+}
+export interface OutboxWrite {
+    /** Company-relative vault key, `ontology/entities/<type>/<file>.md`. */
+    rel: string;
+    content: string;
+}
+export declare function outboxPath(hqRoot: string, company: OntologyCompany): string;
+/** The company's outbox. A missing file is an empty outbox; an unreadable one throws. */
+export declare function readOutbox(hqRoot: string, company: OntologyCompany): Outbox;
+/**
+ * Record entity bytes this checkpoint wrote, and drop entries older than the
+ * retention window. An unreadable outbox is replaced: a lost entry only makes
+ * sync hold that file back, which is the safe direction.
+ */
+export declare function recordOutbox(hqRoot: string, company: OntologyCompany, writes: OutboxWrite[], now: Date): {
+    reset: boolean;
+};
+export type MergeBackResult = "merged" | "kept_both" | "stale" | "deleted" | "identity_conflict" | "invalid" | "write_error";
+export interface MergeBackOutcome {
+    /** HQ-relative path of the set-aside version. */
+    backup: string;
+    /** Company-relative entity file it belongs to. */
+    target: string;
+    result: MergeBackResult;
+    /** For `kept_both`, the keys that differ (`body` for prose); for `invalid` and `write_error`, the reason. */
+    detail?: string;
+}
+/** Back-reference lines in a body, as refs `appendBackRefs` can re-add. */
+export declare function readBackRefs(body: string): SignalBackRef[];
+export interface MergeBackRun {
+    outcomes: MergeBackOutcome[];
+    /** Entity files rewritten; each was recorded in the outbox before it was written. */
+    writes: OutboxWrite[];
+    outbox_reset?: true;
+}
+/**
+ * Re-apply set-aside entity versions of this company to the current files
+ * (design 20.4). Never deletes an entity file and never re-creates one.
+ */
+export declare function mergeBackConflicts(hqRoot: string, company: OntologyCompany, now: Date): MergeBackRun;
+export interface DuplicateGroup {
+    type: string;
+    /** `same_name`: equal canonical names, or one's canonical name is the other's unshared alias. `similar_name`: canonical names match once spaces are removed. */
+    reason: "same_name" | "similar_name";
+    paths: string[];
+    /** Highest `signal_count`, then earliest `first_seen`, then path. */
+    survivor: string;
+}
+export interface DuplicateReport {
+    groups: number;
+    shown: DuplicateGroup[];
+}
+/** Groups of writable entities of one type whose names collide. Read-only. */
+export declare function findDuplicateGroups(index: EntityIndex): DuplicateGroup[];
+/** The duplicate scan for a company, capped for the apply report. */
+export declare function scanDuplicates(company: OntologyCompany, limit?: number): DuplicateReport;
+export interface MultiWriterStart {
+    merge_back: MergeBackOutcome[];
+    /** Why merge-back or the outbox write stopped, when it did. */
+    error?: string;
+    outbox_reset?: true;
+}
+/**
+ * The start of a checkpoint run: merge back set-aside versions (each merged
+ * file is recorded in the outbox before it is written). Runs in `plan`, so a
+ * run with no changed inputs still merges. Errors are returned, not thrown,
+ * so the rest of the checkpoint goes on.
+ */
+export declare function startMultiWriterRun(hqRoot: string, company: OntologyCompany, now: Date): MultiWriterStart;
+/** Count merge-back outcomes by result, for one-line summaries. */
+export declare function summarizeMergeBack(outcomes: MergeBackOutcome[]): string;
+//# sourceMappingURL=multi-writer.d.ts.map