@indigoai-us/hq-cli 5.193.1 → 5.194.1

package/dist/lib/checkpoint-ontology/plan.js

@@ -0,0 +1,284 @@
+/**
+ * `hq core ontology plan`: pick this run's inputs.
+ *
+ * Runs in the detached checkpoint sibling. It resolves the flag, finds the
+ * company's changed knowledge and signal files with a content-hash ledger,
+ * applies the per-run budget, and writes `<runDir>/ontology-plan.json` for the
+ * backend model to read. The model only fills a candidates file; everything
+ * that affects identity, scope and cost stays in code.
+ */
+import { createHash } from "node:crypto";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { resolveOntologyCompany, safeWriteFile, stateRoot } from "./company.js";
+import { buildEntityIndex, listKnowledgeFiles, listSignalFiles, readSignal } from "./tree.js";
+export const ONTOLOGY_FLAG = "checkpoint.ontology";
+export const SHARED_INPUTS_FLAG = "checkpoint.ontology.shared-inputs";
+export const PLAN_FILE = "ontology-plan.json";
+export const CANDIDATES_FILE = "ontology-candidates.json";
+export const REPORT_FILE = "ontology-report.json";
+export const PLAN_SCHEMA = "hq-checkpoint-ontology-plan.v1";
+export const CANDIDATES_SCHEMA = "hq-checkpoint-ontology-candidates.v1";
+export const LEDGER_SCHEMA = "hq-checkpoint-ontology-ledger.v1";
+export const BUDGET = {
+    maxInputs: 20,
+    maxTokens: 60_000,
+    chunkTokens: 12_000,
+};
+const CANDIDATE_INSTRUCTIONS = [
+    `Write ${CANDIDATES_FILE} in the run directory as {"schema":"${CANDIDATES_SCHEMA}","candidates":[...]} and nothing else.`,
+    "Each candidate: {input, type, name, aliases, confidence, evidence}. input is one of this plan's input ids; type is person, project, company or concept; name is the entity's full proper name; aliases are other names the input uses for it (at most 5); confidence is 0 to 1; evidence is a short quote from the input.",
+    "Read only the files and line ranges this plan lists. For a signal, read its body and name the entities in missing_refs if the signal supports them.",
+    "Name the specific people, projects, products, companies and named concepts that the input says something about. That includes the company that owns this knowledge and its own products when the input describes them. Name a third-party tool, vendor or service only when a section of the input is about it; skip ones that appear in a list, a stack summary or a passing mention. Skip generic words, file names and headings.",
+    "For the session input (its text is session_text in this plan), name only entities that already exist in the company and that the session did work on. Session candidates are recorded as links, never created.",
+    "Do not create, edit, move or delete anything under ontology/, signals/ or sources/ yourself.",
+];
+function sha256File(file) {
+    return createHash("sha256").update(fs.readFileSync(file)).digest("hex");
+}
+export function ledgerPath(hqRoot, company) {
+    return path.join(stateRoot(hqRoot, company), "ledger.json");
+}
+export function readLedger(hqRoot, company) {
+    let raw;
+    try {
+        raw = fs.readFileSync(ledgerPath(hqRoot, company), "utf8");
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return null;
+        throw error;
+    }
+    const parsed = JSON.parse(raw);
+    if (parsed.schema !== LEDGER_SCHEMA || parsed.company !== company.slug || typeof parsed.entries !== "object") {
+        throw new Error(`ontology ledger for ${company.slug} has an unexpected schema`);
+    }
+    return parsed;
+}
+export function writeLedger(hqRoot, company, ledger) {
+    const target = ledgerPath(hqRoot, company);
+    safeWriteFile(hqRoot, [stateRoot(hqRoot, company)], target, `${JSON.stringify(ledger, null, 2)}\n`);
+}
+/** Company-relative paths of the payload's touched files that are inside this company. */
+export function sessionTouched(hqRoot, company, files) {
+    const out = new Set();
+    if (!Array.isArray(files))
+        return out;
+    for (const file of files) {
+        if (typeof file !== "string" || !file.trim())
+            continue;
+        const absolute = path.resolve(hqRoot, file.trim());
+        const rel = path.relative(company.dir, absolute);
+        if (!rel || rel.startsWith("..") || path.isAbsolute(rel))
+            continue;
+        out.add(rel.split(path.sep).join("/"));
+    }
+    return out;
+}
+/** 1-based line ranges, each at most `chunkTokens` tokens (bytes / 4). */
+export function chunkLines(content, chunkTokens) {
+    const lines = content.split("\n");
+    const maxBytes = chunkTokens * 4;
+    const chunks = [];
+    let start = 1;
+    let bytes = 0;
+    lines.forEach((line, i) => {
+        const size = Buffer.byteLength(line, "utf8") + 1;
+        if (bytes > 0 && bytes + size > maxBytes) {
+            chunks.push([start, i]);
+            start = i + 1;
+            bytes = 0;
+        }
+        bytes += size;
+    });
+    chunks.push([start, lines.length]);
+    return chunks;
+}
+function estTokens(bytes) {
+    return Math.ceil(bytes / 4);
+}
+/** Latest `last_updated` over the company's writable entity types (design 14.3). */
+function entityBoundary(index) {
+    let best = null;
+    let bestMs = -Infinity;
+    for (const entity of index.byRel.values()) {
+        if (!["person", "project", "company", "concept"].includes(entity.type) || !entity.last_updated)
+            continue;
+        const ms = Date.parse(entity.last_updated);
+        if (Number.isFinite(ms) && ms > bestMs) {
+            bestMs = ms;
+            best = entity.last_updated;
+        }
+    }
+    return best;
+}
+function missingRefs(companyDir, rel) {
+    const signal = readSignal(companyDir, rel);
+    if (!signal)
+        return [];
+    return signal.refs.filter((ref) => !fs.existsSync(path.join(companyDir, "ontology", "entities", `${ref}.md`)));
+}
+/**
+ * Update the ledger with the current tree and return it. New or changed files
+ * become pending when this machine should process them (design 14.1) and
+ * seen otherwise. The first run seeds from the entity boundary (design 14.3).
+ */
+export function refreshLedger(company, existing, touched, sharedInputs) {
+    const firstRun = existing === null;
+    const index = firstRun ? buildEntityIndex(company.dir) : null;
+    const boundary = existing ? existing.boundary : entityBoundary(index);
+    const boundaryMs = boundary ? Date.parse(boundary) : Number.NaN;
+    const entries = {};
+    const files = [...listKnowledgeFiles(company.dir), ...listSignalFiles(company.dir)];
+    for (const rel of files) {
+        const absolute = path.join(company.dir, rel);
+        const stat = fs.statSync(absolute);
+        const previous = existing?.entries[rel];
+        const unchangedStat = previous && previous.size === stat.size && previous.mtime_ms === stat.mtimeMs;
+        const sha256 = unchangedStat ? previous.sha256 : sha256File(absolute);
+        const base = { sha256, size: stat.size, mtime_ms: stat.mtimeMs };
+        if (previous && previous.sha256 === sha256) {
+            // Same content. A pending entry stays pending until apply finishes it.
+            // A file another run recorded as seen is still this session's input.
+            entries[rel] = previous.state === "seen" && touched.has(rel)
+                ? { ...base, state: "pending", reason: "session-touched" }
+                : { ...previous, ...base };
+            continue;
+        }
+        let reason = null;
+        if (touched.has(rel))
+            reason = "session-touched";
+        else if (sharedInputs && firstRun) {
+            if (Number.isFinite(boundaryMs) && stat.mtimeMs > boundaryMs)
+                reason = "first-run-gap";
+            else if (rel.startsWith("signals/") && missingRefs(company.dir, rel).length > 0)
+                reason = "missing-refs";
+        }
+        else if (sharedInputs)
+            reason = "shared";
+        entries[rel] = reason ? { ...base, state: "pending", reason } : { ...base, state: "seen" };
+    }
+    return { ledger: { schema: LEDGER_SCHEMA, company: company.slug, boundary, entries }, firstRun };
+}
+function reasonRank(reason) {
+    return reason === "session-touched" ? 0 : 1;
+}
+/** Plan this run. Never throws for a skip; returns the reason instead. */
+export async function planOntologyRun(options) {
+    const { hqRoot, runDir, payload } = options;
+    const resolution = resolveOntologyCompany(hqRoot, typeof payload.company === "string" ? payload.company : null);
+    if (!resolution.ok)
+        return { status: "skipped", reason: resolution.reason };
+    const company = resolution.company;
+    if (!(await options.readFlag(ONTOLOGY_FLAG, company)))
+        return { status: "skipped", reason: "flag off" };
+    const sharedInputs = await options.readFlag(SHARED_INPUTS_FLAG, company);
+    const touched = sessionTouched(hqRoot, company, payload.files);
+    const { ledger, firstRun } = refreshLedger(company, readLedger(hqRoot, company), touched, sharedInputs);
+    const pending = Object.entries(ledger.entries)
+        .filter(([, entry]) => entry.state === "pending" && (sharedInputs || entry.reason === "session-touched"))
+        .sort(([aRel, a], [bRel, b]) => reasonRank(a.reason) - reasonRank(b.reason) || a.mtime_ms - b.mtime_ms || aRel.localeCompare(bRel));
+    const inputs = [];
+    let tokens = 0;
+    let remaining = 0;
+    for (const [rel, entry] of pending) {
+        if (inputs.length >= BUDGET.maxInputs || tokens >= BUDGET.maxTokens) {
+            remaining += 1;
+            continue;
+        }
+        const lines = fs.readFileSync(path.join(company.dir, rel), "utf8").split("\n");
+        const chunks = chunkLines(lines.join("\n"), BUDGET.chunkTokens);
+        const kind = rel.startsWith("signals/") ? "signal" : "knowledge";
+        let index = entry.next_chunk ?? 0;
+        for (; index < chunks.length; index += 1) {
+            const [start, end] = chunks[index];
+            const cost = estTokens(Buffer.byteLength(lines.slice(start - 1, end).join("\n"), "utf8"));
+            if (inputs.length >= BUDGET.maxInputs || tokens + cost > BUDGET.maxTokens)
+                break;
+            inputs.push({
+                id: chunks.length > 1 ? `${rel}#${index}` : rel,
+                kind,
+                path: rel,
+                audience: "company",
+                reason: entry.reason ?? "shared",
+                sha256: entry.sha256,
+                est_tokens: cost,
+                lines: chunks.length > 1 ? [start, end] : null,
+                chunk: chunks.length > 1 ? { index, count: chunks.length } : null,
+                ...(kind === "signal" ? { missing_refs: missingRefs(company.dir, rel) } : {}),
+            });
+            tokens += cost;
+        }
+        if (index < chunks.length)
+            remaining += 1;
+    }
+    writeLedger(hqRoot, company, ledger);
+    if (inputs.length === 0) {
+        return { status: "skipped", reason: pending.length > 0 ? "over budget" : "nothing changed" };
+    }
+    // Session content is scoped to the session's user (design section 6, D2).
+    // v1 records the existing entities it names as links in the report and
+    // writes nothing from it, so it rides along only when files changed.
+    const sessionText = sessionContent(payload);
+    if (sessionText) {
+        const cost = estTokens(Buffer.byteLength(sessionText, "utf8"));
+        inputs.unshift({
+            id: "session",
+            kind: "session",
+            path: null,
+            audience: "session",
+            reason: "session-touched",
+            sha256: null,
+            est_tokens: cost,
+            lines: null,
+            chunk: null,
+        });
+        tokens += cost;
+    }
+    const plan = {
+        schema: PLAN_SCHEMA,
+        company: company.slug,
+        run_id: path.basename(runDir),
+        created_at: (options.now ?? new Date()).toISOString(),
+        first_run: firstRun,
+        boundary: ledger.boundary,
+        shared_inputs: sharedInputs,
+        inputs,
+        session_text: sessionText,
+        pending_after_run: remaining,
+        budget: {
+            max_inputs: BUDGET.maxInputs,
+            max_tokens: BUDGET.maxTokens,
+            chunk_tokens: BUDGET.chunkTokens,
+            planned_tokens: tokens,
+        },
+        candidates_file: CANDIDATES_FILE,
+        instructions: CANDIDATE_INSTRUCTIONS,
+    };
+    fs.mkdirSync(runDir, { recursive: true });
+    const planPath = path.join(runDir, PLAN_FILE);
+    fs.writeFileSync(planPath, `${JSON.stringify(plan, null, 2)}\n`);
+    return { status: "planned", plan, planPath };
+}
+const SESSION_TEXT_MAX_BYTES = 8_000;
+/** The session summary, decisions and learnings from the payload, capped. */
+function sessionContent(payload) {
+    const parts = [];
+    if (typeof payload.summary === "string" && payload.summary.trim())
+        parts.push(payload.summary.trim());
+    for (const key of ["decisions", "learnings", "insights"]) {
+        const items = payload[key];
+        if (!Array.isArray(items))
+            continue;
+        for (const item of items)
+            if (typeof item === "string" && item.trim())
+                parts.push(`- ${item.trim()}`);
+    }
+    if (parts.length === 0)
+        return null;
+    const text = parts.join("\n");
+    return Buffer.byteLength(text, "utf8") > SESSION_TEXT_MAX_BYTES
+        ? Buffer.from(text, "utf8").subarray(0, SESSION_TEXT_MAX_BYTES).toString("utf8").replace(/\uFFFD$/, "")
+        : text;
+}
+//# sourceMappingURL=plan.js.map

package/dist/lib/checkpoint-ontology/tree.d.ts

@@ -0,0 +1,51 @@
+/**
+ * In-memory indexes over a company's ontology and signals trees.
+ */
+import { type EntityFields } from "./entity-file.js";
+/** The eight company-audience signal types (`signals/{type}/*.md`). */
+export declare const SIGNAL_TYPES: readonly ["action_item", "commitment", "decision", "risk", "question", "key_point", "participant_contribution", "summary"];
+export interface IndexedEntity extends EntityFields {
+    /** Company-relative path, e.g. `ontology/entities/person/jane-doe.md`. */
+    rel: string;
+    slug: string;
+    last_updated: string | null;
+}
+export interface EntityIndex {
+    byRel: Map<string, IndexedEntity>;
+    /** `<type>\0<normalised canonical name>` → entity paths. */
+    canonical: Map<string, Set<string>>;
+    /** `<type>\0<normalised alias>` → entity paths. */
+    alias: Map<string, Set<string>>;
+    /** `<type>\0<squashed name or alias>` → entity paths. */
+    squashed: Map<string, Set<string>>;
+    /** Normalised name or alias → the types that hold it. */
+    types: Map<string, Set<string>>;
+    /** Files under `ontology/entities/` that fail validation. Never written. */
+    invalid: Set<string>;
+}
+export declare function emptyIndex(): EntityIndex;
+/** Add one parsed entity to the index (also used for entities created mid-run). */
+export declare function indexEntity(index: EntityIndex, entity: IndexedEntity): void;
+/** Record that the entity at `rel` answers to `alias`. */
+export declare function indexAlias(index: EntityIndex, type: string, rel: string, alias: string): void;
+/**
+ * Index every entity file of every type under `ontology/entities/`. Other
+ * types (CRM, accounting, legacy `tool`) are indexed so a name they hold is a
+ * type conflict, but they are never written.
+ */
+export declare function buildEntityIndex(companyDir: string): EntityIndex;
+export interface SignalRecord {
+    /** Company-relative path, `signals/<type>/<file>.md`. */
+    rel: string;
+    type: string;
+    canonicalContent: string;
+    /** `entity_refs` items such as `person/jane-doe`. */
+    refs: string[];
+}
+/** Parse the frontmatter of one company-audience signal file. */
+export declare function readSignal(companyDir: string, rel: string): SignalRecord | null;
+/** Company-relative paths of every root signal file, sorted. */
+export declare function listSignalFiles(companyDir: string): string[];
+/** Company-relative paths of every knowledge markdown file, sorted. */
+export declare function listKnowledgeFiles(companyDir: string): string[];
+//# sourceMappingURL=tree.d.ts.map

package/dist/lib/checkpoint-ontology/tree.js

@@ -0,0 +1,163 @@
+/**
+ * In-memory indexes over a company's ontology and signals trees.
+ */
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { normalizeEntityName, squash } from "./identity.js";
+import { parseEntityFile, validateEntityFrontmatter } from "./entity-file.js";
+/** The eight company-audience signal types (`signals/{type}/*.md`). */
+export const SIGNAL_TYPES = [
+    "action_item", "commitment", "decision", "risk", "question", "key_point",
+    "participant_contribution", "summary",
+];
+function add(map, key, value) {
+    let set = map.get(key);
+    if (!set) {
+        set = new Set();
+        map.set(key, set);
+    }
+    set.add(value);
+}
+export function emptyIndex() {
+    return {
+        byRel: new Map(),
+        canonical: new Map(),
+        alias: new Map(),
+        squashed: new Map(),
+        types: new Map(),
+        invalid: new Set(),
+    };
+}
+/** Add one parsed entity to the index (also used for entities created mid-run). */
+export function indexEntity(index, entity) {
+    index.byRel.set(entity.rel, entity);
+    const canonical = normalizeEntityName(entity.canonical_name, entity.type);
+    if (canonical) {
+        add(index.canonical, `${entity.type}\0${canonical}`, entity.rel);
+        add(index.squashed, `${entity.type}\0${squash(canonical)}`, entity.rel);
+        add(index.types, canonical, entity.type);
+    }
+    for (const alias of entity.aliases)
+        indexAlias(index, entity.type, entity.rel, alias);
+}
+/** Record that the entity at `rel` answers to `alias`. */
+export function indexAlias(index, type, rel, alias) {
+    const normalized = normalizeEntityName(alias, type);
+    if (!normalized)
+        return;
+    add(index.alias, `${type}\0${normalized}`, rel);
+    add(index.squashed, `${type}\0${squash(normalized)}`, rel);
+    add(index.types, normalized, type);
+}
+/**
+ * Index every entity file of every type under `ontology/entities/`. Other
+ * types (CRM, accounting, legacy `tool`) are indexed so a name they hold is a
+ * type conflict, but they are never written.
+ */
+export function buildEntityIndex(companyDir) {
+    const index = emptyIndex();
+    const root = path.join(companyDir, "ontology", "entities");
+    let types;
+    try {
+        types = fs.readdirSync(root, { withFileTypes: true });
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return index;
+        throw error;
+    }
+    for (const typeDir of types.filter((entry) => entry.isDirectory()).sort((a, b) => a.name.localeCompare(b.name))) {
+        const dir = path.join(root, typeDir.name);
+        for (const file of fs.readdirSync(dir, { withFileTypes: true })) {
+            if (!file.isFile() || !file.name.endsWith(".md"))
+                continue;
+            const rel = `ontology/entities/${typeDir.name}/${file.name}`;
+            const parsed = parseEntityFile(fs.readFileSync(path.join(dir, file.name), "utf8"));
+            const fields = parsed ? validateEntityFrontmatter(parsed.frontmatter) : "unparseable";
+            if (typeof fields === "string" || !parsed) {
+                index.invalid.add(rel);
+                continue;
+            }
+            const lastUpdated = parsed.frontmatter.last_updated;
+            indexEntity(index, {
+                ...fields,
+                rel,
+                slug: file.name.slice(0, -3),
+                last_updated: typeof lastUpdated === "string" ? lastUpdated : null,
+            });
+        }
+    }
+    return index;
+}
+/** Parse the frontmatter of one company-audience signal file. */
+export function readSignal(companyDir, rel) {
+    let content;
+    try {
+        content = fs.readFileSync(path.join(companyDir, rel), "utf8");
+    }
+    catch {
+        return null;
+    }
+    const parsed = parseEntityFile(content);
+    if (!parsed)
+        return null;
+    const fm = parsed.frontmatter;
+    const refs = Array.isArray(fm.entity_refs)
+        ? fm.entity_refs.filter((ref) => typeof ref === "string" && /^[a-z_-]+\/[^/]+$/.test(ref))
+        : [];
+    return {
+        rel,
+        type: typeof fm.type === "string" ? fm.type : rel.split("/")[1],
+        canonicalContent: typeof fm.canonical_content === "string" ? fm.canonical_content : "",
+        refs,
+    };
+}
+/** Company-relative paths of every root signal file, sorted. */
+export function listSignalFiles(companyDir) {
+    const out = [];
+    for (const type of SIGNAL_TYPES) {
+        let files;
+        try {
+            files = fs.readdirSync(path.join(companyDir, "signals", type), { withFileTypes: true });
+        }
+        catch (error) {
+            if (error.code === "ENOENT")
+                continue;
+            throw error;
+        }
+        for (const file of files) {
+            if (file.isFile() && file.name.endsWith(".md"))
+                out.push(`signals/${type}/${file.name}`);
+        }
+    }
+    return out.sort();
+}
+/** Company-relative paths of every knowledge markdown file, sorted. */
+export function listKnowledgeFiles(companyDir) {
+    const out = [];
+    const walk = (rel) => {
+        let entries;
+        try {
+            entries = fs.readdirSync(path.join(companyDir, rel), { withFileTypes: true });
+        }
+        catch (error) {
+            if (error.code === "ENOENT")
+                return;
+            throw error;
+        }
+        for (const entry of entries) {
+            // Dot files are local state; `@{key}` folders are scoped audiences.
+            if (entry.name.startsWith(".") || entry.name.startsWith("@"))
+                continue;
+            const child = `${rel}/${entry.name}`;
+            // Symlinks are not followed: a link could point into another company.
+            if (entry.isDirectory())
+                walk(child);
+            else if (entry.isFile() && entry.name.endsWith(".md"))
+                out.push(child);
+        }
+    };
+    walk("knowledge");
+    return out.sort();
+}
+//# sourceMappingURL=tree.js.map

package/dist/lib/lanes/reconcile.js

@@ -569,7 +569,7 @@ async function withReconcileLock(hqRoot, fn) {
         }
     }
 }
-function classifyDeadLocalLane(hqRoot, laneId) {
+function classifyDeadLocalLane(hqRoot, laneId, now) {
     const runDir = laneRunDir(hqRoot, laneId);
     const manifest = readSpawnManifest(runDir);
     const minEnvelopeName = manifest.envelope_path ? path.basename(manifest.envelope_path) : undefined;
@@ -586,7 +586,6 @@ function classifyDeadLocalLane(hqRoot, laneId) {
     // the quota signal, any other becomes provider_rejected, and only a stream
     // with no failure object falls back to the quota reader.
     const workerOutOffset = readSpawnManifest(runDir).worker_out_offset ?? 0;
-    const now = new Date();
     const routed = envelope
         ? { quota: null, rejection: null }
         : routeProviderFailure(readProviderRejection(runDir, lane.provider, workerOutOffset), () => readQuotaSignal(runDir, lane.model, now, workerOutOffset), lane.model, now);
@@ -894,7 +893,7 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
                 continue;
             }
             try {
-                const { classification, latest, quota, providerRejection, lane: classified } = classifyDeadLocalLane(hqRoot, lane.lane_id);
+                const { classification, latest, quota, providerRejection, lane: classified } = classifyDeadLocalLane(hqRoot, lane.lane_id, now);
                 // Match the watcher's one-repair decision before classifying an
                 // unparseable envelope after both processes have died. As in the
                 // watcher, a provider rejection or a quota signal skips the repair.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@indigoai-us/hq-cli",
-  "version": "5.193.1",
+  "version": "5.194.1",
   "description": "HQ by Indigo management CLI — modules and cloud sync",
   "main": "dist/index.js",
   "bin": {