@indigoai-us/hq-cli 5.312.2-beta.0 → 5.314.0

package/dist/lib/search-index/load-gate.js

@@ -1,270 +0,0 @@
-import * as fs from 'node:fs';
-import * as os from 'node:os';
-import * as path from 'node:path';
-import { lockRoot } from './embed-lock.js';
-export const DEFAULT_MAX_LOAD_PERCENT = 50;
-/**
- * Twelve requests is enough to avoid quietly starving a frequently requested
- * index, while still leaving several opportunities for a busy machine to cool
- * down before we spend CPU on an embed. The thirteenth request is forced.
- */
-export const MAX_CONSECUTIVE_DEFERRED_EMBEDS = 12;
-/** Six hours bounds deferral even when indexing requests are infrequent. */
-export const MAX_EMBED_DEFERRAL_SECONDS = 6 * 60 * 60;
-const LOAD_GATE_STATE_NAME = 'qmd-embed-load-state';
-const PROC_STAT = '/proc/stat';
-const PROC_MEMINFO = '/proc/meminfo';
-export class LoadGateConfigurationError extends Error {
-    constructor(value) {
-        super(`HQ_INDEX_MAX_LOAD_PERCENT must be an integer from 1 to 100, 0, or off; received ${JSON.stringify(value)}.`);
-        this.name = 'LoadGateConfigurationError';
-    }
-}
-function synchronousSleep(milliseconds) {
-    const slot = new Int32Array(new SharedArrayBuffer(4));
-    Atomics.wait(slot, 0, 0, milliseconds);
-}
-export function defaultLoadGateDependencies(env = process.env) {
-    return {
-        env,
-        now: () => Math.floor(Date.now() / 1_000),
-        readFile: (file) => fs.readFileSync(file, 'utf8'),
-        writeFile: (file, contents) => fs.writeFileSync(file, contents),
-        rename: (source, destination) => fs.renameSync(source, destination),
-        mkdir: (directory) => fs.mkdirSync(directory, { recursive: true }),
-        sleep: synchronousSleep,
-        cpus: () => os.cpus(),
-        // os.freemem() reports MemFree on Linux, which excludes reclaimable cache.
-        // Use it only where /proc/meminfo does not exist.
-        freemem: () => os.freemem(),
-        totalmem: () => os.totalmem(),
-    };
-}
-function errorCode(error) {
-    return error?.code;
-}
-function missing(error) {
-    return errorCode(error) === 'ENOENT';
-}
-function parseProcStatAggregate(contents) {
-    const line = contents.split('\n').find((candidate) => /^cpu\s+/.test(candidate));
-    if (!line)
-        return undefined;
-    const values = line.trim().split(/\s+/).slice(1).map(Number);
-    if (values.length < 5 || values.some((value) => !Number.isFinite(value) || value < 0))
-        return undefined;
-    // Linux reports guest and guest_nice inside user and nice respectively.
-    const total = values.slice(0, 8).reduce((sum, value) => sum + value, 0);
-    const idle = values[3] + values[4];
-    return total > 0 && Number.isFinite(idle) ? { total, idle } : undefined;
-}
-function cpuBusyPercent(before, after) {
-    const totalDelta = after.total - before.total;
-    const idleDelta = after.idle - before.idle;
-    if (totalDelta <= 0 || idleDelta < 0)
-        return undefined;
-    return Math.max(0, Math.min(100, (1 - idleDelta / totalDelta) * 100));
-}
-function cpuCountersFromCpus(cpus) {
-    let total = 0;
-    let idle = 0;
-    if (cpus.length === 0)
-        return undefined;
-    for (const cpu of cpus) {
-        const { user, nice, sys, idle: cpuIdle, irq } = cpu.times;
-        const values = [user, nice, sys, cpuIdle, irq];
-        if (values.some((value) => !Number.isFinite(value) || value < 0))
-            return undefined;
-        total += values.reduce((sum, value) => sum + value, 0);
-        idle += cpuIdle;
-    }
-    return total > 0 ? { total, idle } : undefined;
-}
-function fallbackCpuPercent(dependencies) {
-    const before = cpuCountersFromCpus(dependencies.cpus());
-    dependencies.sleep(200);
-    const after = cpuCountersFromCpus(dependencies.cpus());
-    const busy = before && after ? cpuBusyPercent(before, after) : undefined;
-    if (busy === undefined)
-        throw new Error('Cannot determine CPU busy time from os.cpus().');
-    return busy;
-}
-/** Sample aggregate CPU busy time over 200 ms, falling back only without /proc/stat. */
-export function measureCpuPercent(dependencies) {
-    let first;
-    try {
-        first = dependencies.readFile(PROC_STAT);
-    }
-    catch (error) {
-        if (missing(error))
-            return fallbackCpuPercent(dependencies);
-        throw error;
-    }
-    dependencies.sleep(200);
-    let second;
-    try {
-        second = dependencies.readFile(PROC_STAT);
-    }
-    catch (error) {
-        if (missing(error))
-            return fallbackCpuPercent(dependencies);
-        throw error;
-    }
-    const before = parseProcStatAggregate(first);
-    const after = parseProcStatAggregate(second);
-    const busy = before && after ? cpuBusyPercent(before, after) : undefined;
-    if (busy === undefined)
-        throw new Error('Cannot determine CPU busy time from /proc/stat.');
-    return busy;
-}
-function memAvailableFraction(contents) {
-    const fields = new Map();
-    for (const line of contents.split('\n')) {
-        const match = /^(MemTotal|MemAvailable):\s+(\d+)\s+kB$/.exec(line.trim());
-        if (match)
-            fields.set(match[1], Number(match[2]));
-    }
-    const total = fields.get('MemTotal');
-    const available = fields.get('MemAvailable');
-    if (total === undefined || available === undefined || total <= 0 || available < 0 || available > total)
-        return undefined;
-    return 1 - available / total;
-}
-/** Prefer Linux MemAvailable, because MemFree alone treats reclaimable cache as used memory. */
-export function measureMemoryPercent(dependencies) {
-    try {
-        const fraction = memAvailableFraction(dependencies.readFile(PROC_MEMINFO));
-        if (fraction === undefined)
-            throw new Error('Cannot determine memory use from /proc/meminfo.');
-        return fraction * 100;
-    }
-    catch (error) {
-        if (!missing(error))
-            throw error;
-    }
-    const total = dependencies.totalmem();
-    const free = dependencies.freemem();
-    if (total <= 0 || free < 0 || free > total)
-        throw new Error('Cannot determine memory use.');
-    return (1 - free / total) * 100;
-}
-export function parseLoadThreshold(env) {
-    const value = env.HQ_INDEX_MAX_LOAD_PERCENT?.trim() || String(DEFAULT_MAX_LOAD_PERCENT);
-    if (value === '0' || value.toLowerCase() === 'off')
-        return undefined;
-    if (!/^[1-9]\d{0,2}$/.test(value) || Number(value) > 100)
-        throw new LoadGateConfigurationError(value);
-    return Number(value);
-}
-export function loadGateStatePath(home) {
-    return path.join(lockRoot(home), LOAD_GATE_STATE_NAME);
-}
-function readFields(file, dependencies) {
-    try {
-        const fields = {};
-        for (const line of dependencies.readFile(file).split('\n')) {
-            if (line === '')
-                continue;
-            const index = line.indexOf('=');
-            const key = line.slice(0, index);
-            if (index < 1 || !['skips', 'firstDeferredAt', 'lastSuccessfulEmbedAt'].includes(key) || fields[key] !== undefined)
-                return undefined;
-            fields[key] = line.slice(index + 1);
-        }
-        return fields;
-    }
-    catch (error) {
-        if (missing(error))
-            return undefined;
-        throw error;
-    }
-}
-function nonnegativeInteger(value) {
-    return value !== undefined && /^\d+$/.test(value) ? Number(value) : undefined;
-}
-function readState(home, dependencies) {
-    const fields = readFields(loadGateStatePath(home), dependencies);
-    if (!fields)
-        return { skips: 0 };
-    const skips = nonnegativeInteger(fields.skips);
-    const firstDeferredAt = nonnegativeInteger(fields.firstDeferredAt);
-    const lastSuccessfulEmbedAt = nonnegativeInteger(fields.lastSuccessfulEmbedAt);
-    if (skips === undefined
-        || (fields.firstDeferredAt !== undefined && firstDeferredAt === undefined)
-        || (fields.lastSuccessfulEmbedAt !== undefined && lastSuccessfulEmbedAt === undefined)
-        || (skips > 0 && firstDeferredAt === undefined)
-        || (skips === 0 && firstDeferredAt !== undefined))
-        return { skips: 0 };
-    return {
-        skips,
-        ...(firstDeferredAt === undefined ? {} : { firstDeferredAt }),
-        ...(lastSuccessfulEmbedAt === undefined ? {} : { lastSuccessfulEmbedAt }),
-    };
-}
-function writeState(home, state, dependencies) {
-    dependencies.mkdir(lockRoot(home));
-    const statePath = loadGateStatePath(home);
-    const temporaryPath = `${statePath}.${process.pid}.${Math.random().toString(36).slice(2)}.tmp`;
-    dependencies.writeFile(temporaryPath, [
-        `skips=${state.skips}`,
-        ...(state.firstDeferredAt === undefined ? [] : [`firstDeferredAt=${state.firstDeferredAt}`]),
-        ...(state.lastSuccessfulEmbedAt === undefined ? [] : [`lastSuccessfulEmbedAt=${state.lastSuccessfulEmbedAt}`]),
-        '',
-    ].join('\n'));
-    dependencies.rename(temporaryPath, statePath);
-}
-function rounded(percent) {
-    return Math.round(percent);
-}
-function trippedSignal(cpuPercent, memoryPercent, limit) {
-    if (cpuPercent >= limit)
-        return { signal: 'cpu', percent: rounded(cpuPercent) };
-    if (memoryPercent >= limit)
-        return { signal: 'memory', percent: rounded(memoryPercent) };
-    return undefined;
-}
-/**
- * Decide immediately before qmd embed whether this host can absorb it. The
- * caller must already own the shared embed lock, so state updates are atomic
- * with respect to hq-cli's other embed entry points.
- */
-export function checkEmbedLoad(home, dependencies) {
-    const limit = parseLoadThreshold(dependencies.env);
-    if (limit === undefined)
-        return { action: 'embed', forced: false };
-    const cpuPercent = measureCpuPercent(dependencies);
-    const memoryPercent = measureMemoryPercent(dependencies);
-    const tripped = trippedSignal(cpuPercent, memoryPercent, limit);
-    if (!tripped)
-        return { action: 'embed', forced: false };
-    const now = dependencies.now();
-    const state = readState(home, dependencies);
-    const skips = state.skips + 1;
-    const deferredSince = state.firstDeferredAt ?? now;
-    const exceededSkipBound = skips > MAX_CONSECUTIVE_DEFERRED_EMBEDS;
-    const exceededTimeBound = now - deferredSince >= MAX_EMBED_DEFERRAL_SECONDS;
-    if (exceededSkipBound || exceededTimeBound) {
-        return {
-            action: 'embed',
-            forced: true,
-            notice: `hq: proceeding with qmd embed despite ${tripped.signal} ${tripped.percent}% (limit ${limit}%) because embeddings have been deferred too long.\n`,
-        };
-    }
-    writeState(home, {
-        skips,
-        firstDeferredAt: state.firstDeferredAt ?? now,
-        ...(state.lastSuccessfulEmbedAt === undefined ? {} : { lastSuccessfulEmbedAt: state.lastSuccessfulEmbedAt }),
-    }, dependencies);
-    return {
-        action: 'skip',
-        signal: tripped.signal,
-        percent: tripped.percent,
-        limit,
-        notice: `hq: skipping qmd embed — ${tripped.signal} ${tripped.percent}% (limit ${limit}%).\n`,
-    };
-}
-/** Reset the starvation counter only after qmd embed has actually returned successfully. */
-export function recordSuccessfulEmbed(home, dependencies) {
-    writeState(home, { skips: 0, lastSuccessfulEmbedAt: dependencies.now() }, dependencies);
-}
-//# sourceMappingURL=load-gate.js.map

package/dist/lib/search-index/max-doc-bytes.d.ts

@@ -1,219 +0,0 @@
-/**
- * Document size cap for the local HQ search index.
- *
- * WHY THIS EXISTS
- *
- * qmd indexes a collection by globbing its mask and reading every match — there
- * is no size check anywhere on that path (`reindexCollection` in @tobilu/qmd
- * 2.5.3 goes straight from the glob to `readFileSync`). HQ's project
- * collections are registered with a `md`/`json` mask, which sweeps in build
- * artifacts nobody would ever search as prose: browser trace dumps, perf
- * captures, scraped datasets, lockfiles.
- *
- * Embedding is the consumer that makes this expensive. `qmd embed` runs under a
- * hard 30-minute session cap; once the corpus stops fitting inside that budget
- * the run aborts and discards its work, so an oversized corpus does not merely
- * cost CPU — it stops the index ever converging.
- *
- * HOW IT WORKS
- *
- * qmd has no size option, but it does honour a per-collection `ignore` list of
- * globs, applied at glob time, so an ignored file is never read, never chunked
- * and never embedded. That list is not reachable through `qmd collection add`
- * (there is no flag) and qmd's own YAML writer drops the key — but qmd's READER
- * passes the parsed YAML collection object straight through to the store, and
- * `syncConfigToDb` treats external config as authoritative ("External config
- * always wins"). Writing an `ignore:` key into qmd's `index.yml` is therefore a
- * durable way to express the cap, and it survives qmd's config resync.
- *
- * So: resolve the configured cap, walk each collection the way qmd would, and
- * rewrite its `ignore` list to the files currently over the cap. The list is
- * recomputed every run, so a file that shrinks or is deleted stops being
- * ignored with no manual cleanup.
- *
- * OWNERSHIP
- *
- * The `ignore` list is shared with whatever an operator hand-wrote into
- * `index.yml`. We therefore never treat "looks like our output" as proof of
- * provenance: entries this module INTRODUCED are recorded in
- * `<hqRoot>/.hq/index-size-ignores.json`, and only entries named in that record
- * are removed on a later run. Anything else in `ignore` is left untouched.
- *
- * "Introduced" is doing real work in that sentence. An operator who already
- * wrote `big.md` by hand keeps owning it even when our scan independently finds
- * the same file oversized, so lowering the cap later can never delete their
- * line. The record is also written BEFORE the config: an entry whose provenance
- * failed to persist would read as foreign forever, which is to say permanently
- * un-removable, and skipping the change is the cheaper mistake.
- */
-import * as fs from "node:fs";
-/**
- * Default cap: 100 KB. Chosen against a real HQ corpus where the median indexed
- * document is ~1.2 KB — a 100 KB document is three orders of magnitude off the
- * median and is, in practice, always generated output rather than prose.
- */
-export declare const DEFAULT_INDEX_MAX_DOC_BYTES: number;
-/**
- * qmd's config file, resolved the way qmd's own CLI resolves it.
- *
- * Mirrors `dist/cli/qmd.js`: with no `--index` flag, a project-local
- * `.qmd/index.yaml` (then `.qmd/index.yml`), found by walking up from the
- * directory qmd runs in, overrides the global config entirely. Otherwise the
- * global ladder applies: `QMD_CONFIG_DIR`, then `XDG_CONFIG_HOME/qmd`, then
- * `$HOME/.config/qmd`.
- *
- * Getting this wrong fails silently in the worst way: we would write a correct
- * `ignore` list into a file qmd never reads, every test asserting the key would
- * pass, and nothing would actually be skipped. A `$HOME`-only version of this
- * function shipped to CI and did exactly that, because GitHub runners set
- * `XDG_CONFIG_HOME` — the e2e positive control is what caught it.
- *
- * Returns undefined when no rung resolves, which callers treat as "no config to
- * update" rather than inventing a path.
- */
-export declare function resolveQmdConfigPath(opts: {
-    env: NodeJS.ProcessEnv;
-    /** Directory qmd will run in; hq-cli always invokes it with cwd = hqRoot. */
-    startDir?: string;
-    existsFile?: (p: string) => boolean;
-}): string | undefined;
-/**
- * Resolve the per-document size cap in bytes. Precedence, first match wins:
- *
- *   1. `HQ_INDEX_MAX_DOC_BYTES` env var (integer bytes),
- *   2. `<hqRoot>/.hq/config.json` → `indexMaxDocBytes` (integer bytes),
- *   3. `DEFAULT_INDEX_MAX_DOC_BYTES` (100 KB).
- *
- * `0` disables the cap — and is honoured rather than ignored, so turning the
- * setting off also clears any ignore entries a previous run wrote.
- *
- * A negative, non-integer or otherwise unparseable override falls through to
- * the next source rather than throwing: this is operator-facing config and a
- * typo must not break an index run.
- */
-export declare function resolveIndexMaxDocBytes(opts?: {
-    hqRoot?: string;
-    /** Test seam — defaults to `process.env.HQ_INDEX_MAX_DOC_BYTES`. */
-    envValue?: string | undefined;
-    readFile?: (p: string) => string;
-    existsFile?: (p: string) => boolean;
-}): number;
-/**
- * Extensions a qmd mask selects, lowercased and without the dot.
- *
- * HQ registers exactly two mask shapes today (see `deriveCollections`), but an
- * operator's own collection may carry anything. An unrecognised mask returns
- * `null`, meaning "match every file" — the cap then applies to the whole
- * collection, which is the safe direction: we may ignore a file qmd would not
- * have indexed (a no-op) rather than miss one it would.
- */
-export declare function maskExtensions(mask: string): Set<string> | null;
-/** Injected filesystem seam, so the walk is testable without a real tree. */
-export type OversizedScanIo = {
-    readdirSync: (directory: string) => fs.Dirent[];
-    statSync: (file: string) => {
-        size: number;
-    };
-};
-/**
- * Escape a relative path so qmd's glob matcher treats it as a literal filename.
- *
- * Without this, a file genuinely named `reports/[final].json` becomes a
- * character class: it fails to ignore itself (so the oversized document is
- * still indexed) while matching unrelated paths like `reports/f.json` (so a
- * small document silently disappears from search). Both halves of that are
- * worse than not having the cap.
- */
-export declare function escapeGlobLiteral(relativePath: string): string;
-/** Outcome of walking one collection for oversized files. */
-export type OversizedScan = {
-    /** Escaped, sorted glob literals for files over the cap. */
-    oversized: string[];
-    /**
-     * Escaped literals for every file the walk POSITIVELY evaluated, over the cap
-     * or under it. An owned ignore entry missing from this set was not observed,
-     * which is only proof the file is gone when `complete` is true.
-     */
-    evaluated: Set<string>;
-    /** False when any directory or file could not be read, so the walk has gaps. */
-    complete: boolean;
-};
-/**
- * Walk `collectionPath` for files the mask selects whose size exceeds `maxBytes`.
- *
- * Mirrors qmd's own walk: excluded directories are skipped, dot-prefixed files
- * and directories are skipped at every level, and symlinks are never followed.
- *
- * `prunePrefixes` carries directory prefixes already excluded by the
- * collection's own ignore globs. Descending into them would burn exactly the
- * CPU this cap exists to save, and qmd is not going to read them anyway.
- *
- * A failure to read a directory or stat a file sets `complete = false` rather
- * than being silently treated as "not oversized". The difference matters: an
- * unreadable file that is still on disk must keep its existing ignore entry,
- * because dropping it would let the next update read the very document the cap
- * was protecting against.
- */
-export declare function findOversizedFiles(collectionPath: string, mask: string, maxBytes: number, io?: OversizedScanIo, prunePrefixes?: readonly string[]): OversizedScan;
-/**
- * Directory prefixes an operator's own ignore globs already exclude.
- *
- * Only the `dir/**` and `dir/**\/*` shapes are recognised. Any other glob stays
- * unpruned, which costs a walk we did not need but never changes the result —
- * the conservative direction for a heuristic whose only job is saving work.
- */
-export declare function prunablePrefixes(patterns: readonly string[]): string[];
-/** What a single collection ended up ignoring on account of the size cap. */
-type CappedCollection = {
-    name: string;
-    ignored: string[];
-};
-export type IndexSizeLimitResult = {
-    /** The cap that was applied. `0` means the cap is disabled. */
-    maxBytes: number;
-    /** qmd's config file, or undefined when there was none to update. */
-    configPath?: string;
-    /** Per-collection oversized files, only for collections that have any. */
-    collections: CappedCollection[];
-    /** True when qmd's config was actually rewritten. */
-    changed: boolean;
-};
-export type ApplyIndexSizeLimitOptions = {
-    /** Test seam — defaults to `process.env`. */
-    env?: NodeJS.ProcessEnv;
-    maxBytes?: number;
-    scanIo?: OversizedScanIo;
-    readFile?: (p: string) => string;
-    writeFile?: (p: string, contents: string) => void;
-    existsFile?: (p: string) => boolean;
-    mkdir?: (p: string) => void;
-};
-/**
- * Apply the size cap to every collection in qmd's config.
- *
- * Runs BEFORE `qmd update`, so the update that follows never reads an oversized
- * file. Rewrites the config only when the resulting ignore lists differ from
- * what is already on disk, so a steady-state HQ does no config churn.
- *
- * ## Concurrency
- *
- * qmd's config is shared with `qmd collection add` and with any other HQ
- * process running an index pass. A naive read, long scan, then full-file dump
- * would erase a collection registered while the scan was running.
- *
- * The order here is read, scan, RE-READ, then write. That re-read is the part
- * that matters: the scan is slow (it walks every collection), so the config can
- * easily move while it runs, and comparing the fresh bytes against the ones the
- * merge was computed from catches exactly that. On a mismatch the whole merge is
- * recomputed against the newer config, up to {@link CONFIG_WRITE_ATTEMPTS}
- * times, after which we decline to write rather than guess. The write itself is
- * a temp-file rename, so no reader ever sees a half-written config.
- *
- * This narrows the window to the microseconds between the re-read and the
- * rename; it is not a mutex. A writer that lands inside that gap still loses.
- * A lockfile would not close it either, because qmd's own writers would not
- * take the lock, so the residual race is documented rather than papered over.
- */
-export declare function applyIndexSizeLimit(hqRoot: string, options?: ApplyIndexSizeLimitOptions): IndexSizeLimitResult;
-export {};
-//# sourceMappingURL=max-doc-bytes.d.ts.map