@indigoai-us/hq-cli 5.345.191 → 5.345.193

package/dist/lib/signals/steps/garden.d.ts

@@ -0,0 +1,24 @@
+import type { SignalsStep, StepResult } from "./types.js";
+interface GardenReport {
+    candidates_processed?: number;
+    entities_created?: number;
+    entities_updated?: number;
+    signals_promoted?: number;
+    facts_written?: number;
+}
+/** Book manuscript paths the gardener must never touch. */
+export declare const BYA_BOOK_PATTERNS: RegExp[];
+/** itemsOut = promoted entities + promoted signals + facts written. */
+export declare function mapGardenReport(r: GardenReport): Pick<StepResult, "itemsIn" | "itemsOut">;
+export declare function isByaBook(company: string, ref: string): boolean;
+type Fm = Record<string, string | string[]>;
+/** Same flat key: value reader as ontology-garden.mjs. */
+export declare function readFrontmatter(text: string): {
+    fm: Fm;
+    body: string;
+};
+/** Pending candidate files (relative to hqRoot) under one _candidates dir. */
+export declare function listPending(hqRoot: string, candRel: string): string[];
+export declare const gardenStep: SignalsStep;
+export {};
+//# sourceMappingURL=garden.d.ts.map

package/dist/lib/signals/steps/garden.js

@@ -0,0 +1,250 @@
+/**
+ * garden step: runs core/scripts/ontology-garden.mjs for one company.
+ *
+ * Around the script it:
+ *   - counts pending candidates (itemsIn) before the run,
+ *   - holds back bya-book candidates (policy indigo-bya-book-skip-ontology-gardening)
+ *     so they are not promoted, then puts them back where they were (skipped),
+ *   - moves entity candidates whose name is 0.8..0.95 similar to an existing
+ *     entity of the same type, or to an earlier pending candidate of the same
+ *     type that this run keeps, to ontology/_candidates/_review/{date}/ ({date}
+ *     is the engine's local date) and appends a line to
+ *     ontology/review/merges.jsonl (review),
+ *   - first puts back any candidates an earlier crashed run left in _held/.
+ * Only exact slug or alias matches are merged, by ontology-garden.mjs itself;
+ * this step adds no merging of its own for the >= 0.95 band.
+ */
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { localDate } from "../local-date.js";
+import { assertAllowedPath, assertNoEscapingLinks, guardedRemove, guardedRmdir } from "../write-guard.js";
+import { nameSimilarity, similarityBand } from "./entity-similarity.js";
+/** Book manuscript paths the gardener must never touch. */
+export const BYA_BOOK_PATTERNS = [
+    /(^|[\s/:"'])companies\/indigo\/knowledge\/book(\/|$)/,
+    /(^|[\s:"'])knowledge\/book\/build-your-own-agi\.md/,
+];
+const HOLD_DIR = "_held";
+const REVIEW_DIR = "_review";
+const SKIP_DIRS = new Set(["_done", HOLD_DIR, REVIEW_DIR]);
+/** itemsOut = promoted entities + promoted signals + facts written. */
+export function mapGardenReport(r) {
+    const n = (v) => (typeof v === "number" && Number.isFinite(v) ? v : 0);
+    return {
+        itemsIn: n(r.candidates_processed),
+        itemsOut: n(r.entities_created) + n(r.signals_promoted) + n(r.facts_written),
+    };
+}
+export function isByaBook(company, ref) {
+    if (!ref)
+        return false;
+    if (BYA_BOOK_PATTERNS.some((re) => re.test(ref)))
+        return true;
+    return company === "indigo" && /^["']?knowledge\/book(\/|$)/.test(ref);
+}
+/** Same flat key: value reader as ontology-garden.mjs. */
+export function readFrontmatter(text) {
+    const m = text.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
+    if (!m)
+        return { fm: {}, body: text.trim() };
+    const fm = {};
+    for (const line of m[1].split("\n")) {
+        const mm = line.match(/^([A-Za-z_]+):\s*(.*)$/);
+        if (!mm)
+            continue;
+        const v = mm[2].trim();
+        fm[mm[1]] = v.startsWith("[") && v.endsWith("]")
+            ? v.slice(1, -1).split(",").map((s) => s.trim().replace(/^["']|["']$/g, "")).filter(Boolean)
+            : v.replace(/^["']|["']$/g, "");
+    }
+    return { fm, body: m[2].trim() };
+}
+const str = (v) => (Array.isArray(v) ? v.join(",") : v ?? "");
+/** Pending candidate files (relative to hqRoot) under one _candidates dir. */
+export function listPending(hqRoot, candRel) {
+    const dir = path.join(hqRoot, candRel);
+    if (!fs.existsSync(dir))
+        return [];
+    const out = [];
+    for (const d of fs.readdirSync(dir).sort()) {
+        if (SKIP_DIRS.has(d))
+            continue;
+        const full = path.join(dir, d);
+        if (!fs.statSync(full).isDirectory())
+            continue;
+        for (const f of fs.readdirSync(full).sort())
+            if (f.endsWith(".md"))
+                out.push(`${candRel}/${d}/${f}`);
+    }
+    return out;
+}
+/**
+ * Move a file through the guarded writer, then remove the source through the
+ * guard. The source is checked first, scoped to this company, so a link into
+ * another company's folder is refused before anything is read or written.
+ */
+function guardedMove(ctx, fromRel, toRel) {
+    const scope = { company: ctx.company };
+    const fromAbs = assertAllowedPath(ctx.hqRoot, fromRel, scope);
+    const data = fs.readFileSync(fromAbs, "utf8");
+    ctx.write(toRel, data);
+    guardedRemove(ctx.hqRoot, fromRel, scope);
+}
+/** Remove rel and its subdirs when they hold no files. Links are neither followed nor removed. */
+function removeEmptyDirs(ctx, rel) {
+    const abs = path.join(ctx.hqRoot, rel);
+    let st;
+    try {
+        st = fs.lstatSync(abs);
+    }
+    catch {
+        return;
+    }
+    if (!st.isDirectory())
+        return;
+    for (const e of fs.readdirSync(abs))
+        removeEmptyDirs(ctx, `${rel}/${e}`);
+    if (fs.readdirSync(abs).length === 0)
+        guardedRmdir(ctx.hqRoot, rel, { company: ctx.company });
+}
+/**
+ * Move files left under {candRoot}/_held/ back to {candRoot}/, unless a file
+ * already sits at the original path (then the held copy is left for a person).
+ */
+function restoreStranded(ctx, candRoot) {
+    const holdRoot = `${candRoot}/${HOLD_DIR}`;
+    const abs = path.join(ctx.hqRoot, holdRoot);
+    if (!fs.existsSync(abs) || !fs.statSync(abs).isDirectory())
+        return;
+    for (const d of fs.readdirSync(abs).sort()) {
+        const dir = path.join(abs, d);
+        if (!fs.statSync(dir).isDirectory())
+            continue;
+        for (const f of fs.readdirSync(dir).sort()) {
+            const back = `${candRoot}/${d}/${f}`;
+            if (!fs.existsSync(path.join(ctx.hqRoot, back)))
+                guardedMove(ctx, `${holdRoot}/${d}/${f}`, back);
+        }
+    }
+    removeEmptyDirs(ctx, holdRoot);
+}
+function loadEntities(hqRoot, ontRel) {
+    const dir = path.join(hqRoot, ontRel, "entities");
+    if (!fs.existsSync(dir))
+        return [];
+    const out = [];
+    for (const t of fs.readdirSync(dir)) {
+        const td = path.join(dir, t);
+        if (!fs.statSync(td).isDirectory())
+            continue;
+        for (const f of fs.readdirSync(td)) {
+            if (!f.endsWith(".md"))
+                continue;
+            const { fm } = readFrontmatter(fs.readFileSync(path.join(td, f), "utf8"));
+            const aliases = Array.isArray(fm.aliases) ? fm.aliases : [];
+            const names = [str(fm.canonical_name), ...aliases].filter(Boolean);
+            out.push({ type: str(fm.type) || t, names });
+        }
+    }
+    return out;
+}
+export const gardenStep = {
+    name: "garden",
+    async run(ctx) {
+        const script = path.join(ctx.hqRoot, "core", "scripts", "ontology-garden.mjs");
+        if (!fs.existsSync(script)) {
+            return { status: "skipped", itemsIn: 0, itemsOut: 0, skippedReason: "garden-script-missing" };
+        }
+        const coRel = `companies/${ctx.company}`;
+        const ontRel = `${coRel}/ontology`;
+        const entCand = `${ontRel}/_candidates`;
+        const sigCand = `${coRel}/signals/_candidates`;
+        const date = localDate(ctx.now());
+        // A run that crashed between holding and restoring left candidates in
+        // _held/. Put them back first; the scan below holds them again if needed.
+        // Fail closed before reading anything: a link in either candidate tree
+        // that leaves this company's folder stops the step.
+        for (const root of [entCand, sigCand])
+            assertNoEscapingLinks(ctx.hqRoot, root, { company: ctx.company });
+        for (const root of [entCand, sigCand])
+            restoreStranded(ctx, root);
+        const pending = [...listPending(ctx.hqRoot, entCand), ...listPending(ctx.hqRoot, sigCand)];
+        const itemsIn = pending.length;
+        const held = [];
+        // Names each candidate is compared against: existing entities, plus every
+        // entity candidate this scan keeps, so two pending near-duplicates of each
+        // other are caught too (the later one is queued, the earlier one goes on).
+        const known = loadEntities(ctx.hqRoot, ontRel).map((e) => ({ ...e, against: "entity" }));
+        const mergeLines = [];
+        for (const rel of pending) {
+            const { fm, body } = readFrontmatter(fs.readFileSync(path.join(ctx.hqRoot, rel), "utf8"));
+            if (isByaBook(ctx.company, str(fm.source_ref)) || isByaBook(ctx.company, str(fm.path))) {
+                const candRoot = rel.startsWith(`${entCand}/`) ? entCand : sigCand;
+                held.push({ from: rel, to: `${candRoot}/${HOLD_DIR}/${rel.slice(candRoot.length + 1)}` });
+                continue;
+            }
+            if (!rel.startsWith(`${entCand}/`))
+                continue;
+            const name = body.split("\n")[0].trim();
+            const type = str(fm.type);
+            if (!name || !type)
+                continue;
+            let best = null;
+            for (const e of known) {
+                if (e.type !== type)
+                    continue;
+                for (const n of e.names) {
+                    const sim = nameSimilarity(name, n);
+                    if (!best || sim > best.sim)
+                        best = { sim, existing: n, against: e.against };
+                }
+            }
+            if (best && similarityBand(best.sim) === "uncertain") {
+                mergeLines.push(JSON.stringify({
+                    candidate: name,
+                    existing: best.existing,
+                    similarity: Math.round(best.sim * 1000) / 1000,
+                    type,
+                    against: best.against,
+                    at: ctx.now().toISOString(),
+                }));
+                guardedMove(ctx, rel, `${entCand}/${REVIEW_DIR}/${date}/${path.basename(rel)}`);
+                continue;
+            }
+            known.push({ type, names: [name], against: "candidate" });
+        }
+        if (mergeLines.length) {
+            const mRel = `${ontRel}/review/merges.jsonl`;
+            const mAbs = path.join(ctx.hqRoot, mRel);
+            const prev = fs.existsSync(mAbs) ? fs.readFileSync(mAbs, "utf8") : "";
+            const sep = prev && !prev.endsWith("\n") ? "\n" : "";
+            ctx.write(mRel, `${prev}${sep}${mergeLines.join("\n")}\n`);
+        }
+        const base = { itemsIn, skipped: held.length, review: mergeLines.length };
+        // Hold bya-book candidates outside the script's view, restore afterwards.
+        for (const h of held)
+            guardedMove(ctx, h.from, h.to);
+        try {
+            const res = ctx.spawn(process.execPath, [script, "--company", ctx.company, "--hq-root", ctx.hqRoot, "--json"], { cwd: ctx.hqRoot });
+            if (res.error || res.status !== 0) {
+                const msg = res.error?.message ?? (res.stderr.trim() || `exit ${String(res.status)}`);
+                return { status: "failed", ...base, itemsOut: 0, error: msg.slice(0, 500), errorKind: "other" };
+            }
+            const line = res.stdout.trim().split("\n").filter(Boolean).pop() ?? "{}";
+            try {
+                return { status: "ok", ...base, itemsOut: mapGardenReport(JSON.parse(line)).itemsOut };
+            }
+            catch {
+                return { status: "failed", ...base, itemsOut: 0, error: "garden output was not JSON", errorKind: "other" };
+            }
+        }
+        finally {
+            for (const h of held)
+                if (fs.existsSync(path.join(ctx.hqRoot, h.to)))
+                    guardedMove(ctx, h.to, h.from);
+            for (const root of [entCand, sigCand])
+                removeEmptyDirs(ctx, `${root}/${HOLD_DIR}`);
+        }
+    },
+};
+//# sourceMappingURL=garden.js.map

package/dist/lib/signals/steps/types.d.ts

@@ -0,0 +1,112 @@
+/** Pluggable step interface for the `hq signals run` engine. */
+import type { OnDemandSyncReport } from "../../hot-channels/run.js";
+import type { RunTrigger } from "../../hot-channels/ledger.js";
+import type { UploadDeps } from "./upload.js";
+import type { AdapterFlagReader } from "../adapters-catalog.js";
+/** Same input shape as runCompanyHotChannelSync in src/commands/sources.ts. */
+export type HotChannelSyncFn = (input: {
+    hqRoot: string;
+    company: string;
+    /** Run only this adapter (catalog provider or adapter id); all when unset. */
+    adapter?: string;
+    /** Run only these registered adapter ids; ids the catalog lacks are left out, not errors. */
+    onlyAdapters?: readonly string[];
+    /** Company flag reader for the adapter gate; the sync defaults to the hq-flags evaluator. */
+    isFlagOn?: AdapterFlagReader;
+    now?: () => Date;
+    trigger?: RunTrigger;
+    catchUp?: boolean;
+}) => Promise<OnDemandSyncReport>;
+export type StepName = "garden" | "adapters" | "extract" | "upload";
+export type StepStatus = "ok" | "skipped" | "failed" | "partial";
+export type ErrorKind = "reconnect" | "signed-out" | "transient" | "config" | "other";
+export interface SpawnResult {
+    status: number | null;
+    stdout: string;
+    stderr: string;
+    error?: Error;
+}
+export type SpawnSyncFn = (cmd: string, args: string[], opts: {
+    cwd?: string;
+    env?: NodeJS.ProcessEnv;
+    timeoutMs?: number;
+}) => SpawnResult;
+export interface StepContext {
+    hqRoot: string;
+    company: string;
+    dryRun: boolean;
+    trigger: string;
+    catchUp: boolean;
+    maxTurns: number;
+    maxBudgetUsd: number;
+    now: () => Date;
+    spawn: SpawnSyncFn;
+    /** Guarded writer bound to hqRoot. Every engine write goes through it. */
+    write: (relPath: string, data: string) => void;
+    /** Hot-channel sync used by the adapters step; defaults to runCompanyHotChannelSync. */
+    hotChannelSync?: HotChannelSyncFn;
+    /** Overrides for the upload step's session, fetch and vault reader (tests). */
+    uploadDeps?: Partial<UploadDeps>;
+    /** Registered hot-channel adapter ids; defaults to the builtin registry. */
+    registeredAdapterIds?: readonly string[];
+    /**
+     * Company flag reader for the adapter gate (hq-flags keys and their
+     * registered defaults). When set, the step gates on it and hands it to the
+     * sync. When unset, the sync reads the flags through the shared hq-flags
+     * evaluator with the signed-in session.
+     */
+    isFlagOn?: AdapterFlagReader;
+}
+export interface StepResult {
+    status: StepStatus;
+    itemsIn: number;
+    itemsOut: number;
+    skippedReason?: string;
+    error?: string;
+    modelTurns?: number;
+    estimatedSpendUsd?: number;
+    byAdapter?: Record<string, number>;
+    /** Adapter id -> items the adapter read this run (landed + duplicates). */
+    itemsInByAdapter?: Record<string, number>;
+    /** Adapter id -> token level used (`user` or `workspace`); recorded for attio and gmail. */
+    tokenLevels?: Record<string, string>;
+    /** Adapter id -> reason it did not run (`flag-off`). */
+    adapterSkips?: Record<string, string>;
+    uploaded?: {
+        count: number;
+        bytes: number;
+    };
+    /** Dry-run only: human-readable lines describing what the step would do. */
+    plan?: string[];
+    /** Step-specific detail kept in the run record (for example projection state). */
+    detail?: Record<string, unknown>;
+    errorKind?: ErrorKind;
+    /** Candidates left in place on purpose (garden: bya-book). */
+    skipped?: number;
+    /** Candidates queued for human review (garden: uncertain merges). */
+    review?: number;
+    /** Items whose step work failed; they are left unprocessed for the next run. */
+    failures?: StepFailure[];
+    /** Items the source worklist skipped (for example no-audience). */
+    skippedItems?: {
+        file: string;
+        reason: string;
+    }[];
+    /** Candidates deleted after a turn because they lacked source_ref/audience_key or had a bad type. */
+    rejectedCandidates?: number;
+}
+export interface StepFailure {
+    file: string;
+    error: string;
+}
+export interface SignalsStep {
+    name: StepName;
+    run(ctx: StepContext): Promise<StepResult>;
+    /**
+     * Optional read-only planner called instead of run() under --dry-run. It must
+     * not write files or make network calls that change state.
+     */
+    dryRun?(ctx: StepContext): Promise<StepResult>;
+}
+export declare function notImplemented(): StepResult;
+//# sourceMappingURL=types.d.ts.map

package/dist/lib/signals/steps/types.js

@@ -0,0 +1,4 @@
+export function notImplemented() {
+    return { status: "skipped", itemsIn: 0, itemsOut: 0, skippedReason: "not-implemented" };
+}
+//# sourceMappingURL=types.js.map

package/dist/lib/signals/steps/upload.d.ts

@@ -0,0 +1,43 @@
+import type { SignalsStep } from "./types.js";
+export interface UploadSession {
+    token: string;
+    memberUid: string;
+    companyUid: string;
+    /** Control-plane origin, e.g. DEFAULT_VAULT_API_URL. */
+    baseUrl: string;
+}
+export type FetchFn = (url: string, init: {
+    method: string;
+    headers: Record<string, string>;
+    body?: string;
+}) => Promise<{
+    status: number;
+    text(): Promise<string>;
+}>;
+export interface UploadDeps {
+    /** Resolves the signed-in member for this company; null when there is no HQ login. */
+    getSession(company: string): Promise<UploadSession | null>;
+    fetch: FetchFn;
+    /** Reads one bucket-relative key from the company vault; null when absent. */
+    readVaultObject(session: UploadSession, company: string, key: string): Promise<string | null>;
+}
+export interface UploadWatermark {
+    lastUploadedAt: string | null;
+    uploaded: string[];
+    rejected: Record<string, string>;
+}
+export declare function contributionRoute(company: string): string;
+export declare function watermarkRel(company: string, memberUid: string): string;
+export declare function readWatermark(hqRoot: string, company: string, memberUid: string): UploadWatermark;
+export interface SignalCandidate {
+    /** Company-relative, `signals/...`. */
+    rel: string;
+    sha: string;
+    mtimeMs: number;
+}
+/** Promoted signals for a company not yet uploaded or rejected, newer than the watermark. */
+export declare function listSignalCandidates(hqRoot: string, company: string, wm: UploadWatermark): SignalCandidate[];
+export declare const defaultUploadDeps: UploadDeps;
+export declare function createUploadStep(overrides?: Partial<UploadDeps>): SignalsStep;
+export declare const uploadStep: SignalsStep;
+//# sourceMappingURL=upload.d.ts.map