@indigoai-us/hq-cli 5.115.3 → 5.115.5

package/dist/lib/search-index/load-gate.js

@@ -0,0 +1,270 @@
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import { lockRoot } from './embed-lock.js';
+export const DEFAULT_MAX_LOAD_PERCENT = 50;
+/**
+ * Twelve requests is enough to avoid quietly starving a frequently requested
+ * index, while still leaving several opportunities for a busy machine to cool
+ * down before we spend CPU on an embed. The thirteenth request is forced.
+ */
+export const MAX_CONSECUTIVE_DEFERRED_EMBEDS = 12;
+/** Six hours bounds deferral even when indexing requests are infrequent. */
+export const MAX_EMBED_DEFERRAL_SECONDS = 6 * 60 * 60;
+const LOAD_GATE_STATE_NAME = 'qmd-embed-load-state';
+const PROC_STAT = '/proc/stat';
+const PROC_MEMINFO = '/proc/meminfo';
+export class LoadGateConfigurationError extends Error {
+    constructor(value) {
+        super(`HQ_INDEX_MAX_LOAD_PERCENT must be an integer from 1 to 100, 0, or off; received ${JSON.stringify(value)}.`);
+        this.name = 'LoadGateConfigurationError';
+    }
+}
+function synchronousSleep(milliseconds) {
+    const slot = new Int32Array(new SharedArrayBuffer(4));
+    Atomics.wait(slot, 0, 0, milliseconds);
+}
+export function defaultLoadGateDependencies(env = process.env) {
+    return {
+        env,
+        now: () => Math.floor(Date.now() / 1_000),
+        readFile: (file) => fs.readFileSync(file, 'utf8'),
+        writeFile: (file, contents) => fs.writeFileSync(file, contents),
+        rename: (source, destination) => fs.renameSync(source, destination),
+        mkdir: (directory) => fs.mkdirSync(directory, { recursive: true }),
+        sleep: synchronousSleep,
+        cpus: () => os.cpus(),
+        // os.freemem() reports MemFree on Linux, which excludes reclaimable cache.
+        // Use it only where /proc/meminfo does not exist.
+        freemem: () => os.freemem(),
+        totalmem: () => os.totalmem(),
+    };
+}
+function errorCode(error) {
+    return error?.code;
+}
+function missing(error) {
+    return errorCode(error) === 'ENOENT';
+}
+export function parseProcStatAggregate(contents) {
+    const line = contents.split('\n').find((candidate) => /^cpu\s+/.test(candidate));
+    if (!line)
+        return undefined;
+    const values = line.trim().split(/\s+/).slice(1).map(Number);
+    if (values.length < 5 || values.some((value) => !Number.isFinite(value) || value < 0))
+        return undefined;
+    // Linux reports guest and guest_nice inside user and nice respectively.
+    const total = values.slice(0, 8).reduce((sum, value) => sum + value, 0);
+    const idle = values[3] + values[4];
+    return total > 0 && Number.isFinite(idle) ? { total, idle } : undefined;
+}
+export function cpuBusyPercent(before, after) {
+    const totalDelta = after.total - before.total;
+    const idleDelta = after.idle - before.idle;
+    if (totalDelta <= 0 || idleDelta < 0)
+        return undefined;
+    return Math.max(0, Math.min(100, (1 - idleDelta / totalDelta) * 100));
+}
+function cpuCountersFromCpus(cpus) {
+    let total = 0;
+    let idle = 0;
+    if (cpus.length === 0)
+        return undefined;
+    for (const cpu of cpus) {
+        const { user, nice, sys, idle: cpuIdle, irq } = cpu.times;
+        const values = [user, nice, sys, cpuIdle, irq];
+        if (values.some((value) => !Number.isFinite(value) || value < 0))
+            return undefined;
+        total += values.reduce((sum, value) => sum + value, 0);
+        idle += cpuIdle;
+    }
+    return total > 0 ? { total, idle } : undefined;
+}
+function fallbackCpuPercent(dependencies) {
+    const before = cpuCountersFromCpus(dependencies.cpus());
+    dependencies.sleep(200);
+    const after = cpuCountersFromCpus(dependencies.cpus());
+    const busy = before && after ? cpuBusyPercent(before, after) : undefined;
+    if (busy === undefined)
+        throw new Error('Cannot determine CPU busy time from os.cpus().');
+    return busy;
+}
+/** Sample aggregate CPU busy time over 200 ms, falling back only without /proc/stat. */
+export function measureCpuPercent(dependencies) {
+    let first;
+    try {
+        first = dependencies.readFile(PROC_STAT);
+    }
+    catch (error) {
+        if (missing(error))
+            return fallbackCpuPercent(dependencies);
+        throw error;
+    }
+    dependencies.sleep(200);
+    let second;
+    try {
+        second = dependencies.readFile(PROC_STAT);
+    }
+    catch (error) {
+        if (missing(error))
+            return fallbackCpuPercent(dependencies);
+        throw error;
+    }
+    const before = parseProcStatAggregate(first);
+    const after = parseProcStatAggregate(second);
+    const busy = before && after ? cpuBusyPercent(before, after) : undefined;
+    if (busy === undefined)
+        throw new Error('Cannot determine CPU busy time from /proc/stat.');
+    return busy;
+}
+export function memAvailableFraction(contents) {
+    const fields = new Map();
+    for (const line of contents.split('\n')) {
+        const match = /^(MemTotal|MemAvailable):\s+(\d+)\s+kB$/.exec(line.trim());
+        if (match)
+            fields.set(match[1], Number(match[2]));
+    }
+    const total = fields.get('MemTotal');
+    const available = fields.get('MemAvailable');
+    if (total === undefined || available === undefined || total <= 0 || available < 0 || available > total)
+        return undefined;
+    return 1 - available / total;
+}
+/** Prefer Linux MemAvailable, because MemFree alone treats reclaimable cache as used memory. */
+export function measureMemoryPercent(dependencies) {
+    try {
+        const fraction = memAvailableFraction(dependencies.readFile(PROC_MEMINFO));
+        if (fraction === undefined)
+            throw new Error('Cannot determine memory use from /proc/meminfo.');
+        return fraction * 100;
+    }
+    catch (error) {
+        if (!missing(error))
+            throw error;
+    }
+    const total = dependencies.totalmem();
+    const free = dependencies.freemem();
+    if (total <= 0 || free < 0 || free > total)
+        throw new Error('Cannot determine memory use.');
+    return (1 - free / total) * 100;
+}
+export function parseLoadThreshold(env) {
+    const value = env.HQ_INDEX_MAX_LOAD_PERCENT?.trim() || String(DEFAULT_MAX_LOAD_PERCENT);
+    if (value === '0' || value.toLowerCase() === 'off')
+        return undefined;
+    if (!/^[1-9]\d{0,2}$/.test(value) || Number(value) > 100)
+        throw new LoadGateConfigurationError(value);
+    return Number(value);
+}
+export function loadGateStatePath(home) {
+    return path.join(lockRoot(home), LOAD_GATE_STATE_NAME);
+}
+function readFields(file, dependencies) {
+    try {
+        const fields = {};
+        for (const line of dependencies.readFile(file).split('\n')) {
+            if (line === '')
+                continue;
+            const index = line.indexOf('=');
+            const key = line.slice(0, index);
+            if (index < 1 || !['skips', 'firstDeferredAt', 'lastSuccessfulEmbedAt'].includes(key) || fields[key] !== undefined)
+                return undefined;
+            fields[key] = line.slice(index + 1);
+        }
+        return fields;
+    }
+    catch (error) {
+        if (missing(error))
+            return undefined;
+        throw error;
+    }
+}
+function nonnegativeInteger(value) {
+    return value !== undefined && /^\d+$/.test(value) ? Number(value) : undefined;
+}
+function readState(home, dependencies) {
+    const fields = readFields(loadGateStatePath(home), dependencies);
+    if (!fields)
+        return { skips: 0 };
+    const skips = nonnegativeInteger(fields.skips);
+    const firstDeferredAt = nonnegativeInteger(fields.firstDeferredAt);
+    const lastSuccessfulEmbedAt = nonnegativeInteger(fields.lastSuccessfulEmbedAt);
+    if (skips === undefined
+        || (fields.firstDeferredAt !== undefined && firstDeferredAt === undefined)
+        || (fields.lastSuccessfulEmbedAt !== undefined && lastSuccessfulEmbedAt === undefined)
+        || (skips > 0 && firstDeferredAt === undefined)
+        || (skips === 0 && firstDeferredAt !== undefined))
+        return { skips: 0 };
+    return {
+        skips,
+        ...(firstDeferredAt === undefined ? {} : { firstDeferredAt }),
+        ...(lastSuccessfulEmbedAt === undefined ? {} : { lastSuccessfulEmbedAt }),
+    };
+}
+function writeState(home, state, dependencies) {
+    dependencies.mkdir(lockRoot(home));
+    const statePath = loadGateStatePath(home);
+    const temporaryPath = `${statePath}.${process.pid}.${Math.random().toString(36).slice(2)}.tmp`;
+    dependencies.writeFile(temporaryPath, [
+        `skips=${state.skips}`,
+        ...(state.firstDeferredAt === undefined ? [] : [`firstDeferredAt=${state.firstDeferredAt}`]),
+        ...(state.lastSuccessfulEmbedAt === undefined ? [] : [`lastSuccessfulEmbedAt=${state.lastSuccessfulEmbedAt}`]),
+        '',
+    ].join('\n'));
+    dependencies.rename(temporaryPath, statePath);
+}
+function rounded(percent) {
+    return Math.round(percent);
+}
+function trippedSignal(cpuPercent, memoryPercent, limit) {
+    if (cpuPercent >= limit)
+        return { signal: 'cpu', percent: rounded(cpuPercent) };
+    if (memoryPercent >= limit)
+        return { signal: 'memory', percent: rounded(memoryPercent) };
+    return undefined;
+}
+/**
+ * Decide immediately before qmd embed whether this host can absorb it. The
+ * caller must already own the shared embed lock, so state updates are atomic
+ * with respect to hq-cli's other embed entry points.
+ */
+export function checkEmbedLoad(home, dependencies) {
+    const limit = parseLoadThreshold(dependencies.env);
+    if (limit === undefined)
+        return { action: 'embed', forced: false };
+    const cpuPercent = measureCpuPercent(dependencies);
+    const memoryPercent = measureMemoryPercent(dependencies);
+    const tripped = trippedSignal(cpuPercent, memoryPercent, limit);
+    if (!tripped)
+        return { action: 'embed', forced: false };
+    const now = dependencies.now();
+    const state = readState(home, dependencies);
+    const skips = state.skips + 1;
+    const deferredSince = state.firstDeferredAt ?? now;
+    const exceededSkipBound = skips > MAX_CONSECUTIVE_DEFERRED_EMBEDS;
+    const exceededTimeBound = now - deferredSince >= MAX_EMBED_DEFERRAL_SECONDS;
+    if (exceededSkipBound || exceededTimeBound) {
+        return {
+            action: 'embed',
+            forced: true,
+            notice: `hq: proceeding with qmd embed despite ${tripped.signal} ${tripped.percent}% (limit ${limit}%) because embeddings have been deferred too long.\n`,
+        };
+    }
+    writeState(home, {
+        skips,
+        firstDeferredAt: state.firstDeferredAt ?? now,
+        ...(state.lastSuccessfulEmbedAt === undefined ? {} : { lastSuccessfulEmbedAt: state.lastSuccessfulEmbedAt }),
+    }, dependencies);
+    return {
+        action: 'skip',
+        signal: tripped.signal,
+        percent: tripped.percent,
+        limit,
+        notice: `hq: skipping qmd embed — ${tripped.signal} ${tripped.percent}% (limit ${limit}%).\n`,
+    };
+}
+/** Reset the starvation counter only after qmd embed has actually returned successfully. */
+export function recordSuccessfulEmbed(home, dependencies) {
+    writeState(home, { skips: 0, lastSuccessfulEmbedAt: dependencies.now() }, dependencies);
+}
+//# sourceMappingURL=load-gate.js.map

package/dist/lib/workers-registry/index.js

@@ -18,10 +18,26 @@ function isFile(target) {
         return false;
     }
 }
-/** A sorted `find -L … -name worker.yaml -type f` equivalent, including symlinked pack workers. */
+/**
+ * A sorted `find -L … -name worker.yaml -type f` equivalent, including
+ * symlinked pack workers.
+ *
+ * Excluded subtrees are pruned at the descent, not filtered from the result.
+ * That distinction is the whole cost of this scan: a tenant's `repos/` holds
+ * source checkouts, so filtering afterwards means reading every `node_modules`
+ * and `.git` directory inside them — tens of thousands of directories on a real
+ * root — to produce matches that are discarded a moment later.
+ *
+ * Directory entries carry their own type, so the common case costs one
+ * `readdir` per directory instead of an extra `stat` per entry. Symlinks are
+ * the exception: their dirent type describes the link, so resolving one still
+ * needs a `stat`, which is what keeps this faithful to `find -L`.
+ */
 function workerYamlFiles(root, relativeRoot) {
     const output = [];
     const walk = (relative, ancestors) => {
+        if (isExcludedWorkerDirectory(relative))
+            return;
         const absolute = path.join(root, relative);
         if (!isDirectory(absolute))
             return;
@@ -35,20 +51,23 @@ function workerYamlFiles(root, relativeRoot) {
         if (ancestors.has(physical))
             return; // `find -L` reports loops; a bounded walk is safer here.
         const nextAncestors = new Set(ancestors).add(physical);
-        let names;
+        let entries;
         try {
-            names = fs.readdirSync(absolute).sort();
+            entries = fs.readdirSync(absolute, { withFileTypes: true });
         }
         catch {
             return;
         }
-        for (const name of names) {
-            const childRelative = path.join(relative, name);
+        entries.sort((left, right) => (left.name < right.name ? -1 : left.name > right.name ? 1 : 0));
+        for (const entry of entries) {
+            const childRelative = path.join(relative, entry.name);
             const child = path.join(root, childRelative);
-            if (name === 'worker.yaml' && isFile(child))
+            if (entry.name === 'worker.yaml' && (entry.isFile() || isFile(child))) {
                 output.push(childRelative);
-            else if (isDirectory(child))
+            }
+            else if (entry.isDirectory() || (entry.isSymbolicLink() && isDirectory(child))) {
                 walk(childRelative, nextAncestors);
+            }
         }
     };
     walk(relativeRoot, new Set());
@@ -139,8 +158,8 @@ function withoutTimestamp(content) {
  * both `core` and `personal` — the checkout wins. The operator's actual worker
  * then silently vanishes from the registry every skill reads.
  */
-function isExcludedWorkerPath(relativeFile) {
-    const normalized = relativeFile.replaceAll('\\', '/');
+function isExcludedWorkerDirectory(relativeDirectory) {
+    const normalized = relativeDirectory.replaceAll('\\', '/');
     const [root, tenant, nested] = normalized.split('/');
     if (root !== 'companies')
         return /(?:^|\/)_overrides(?:\/|$)/.test(normalized);
@@ -150,6 +169,15 @@ function isExcludedWorkerPath(relativeFile) {
         return true;
     return /(?:^|\/)_overrides(?:\/|$)/.test(normalized);
 }
+/**
+ * Every exclusion is a property of the directory holding the `worker.yaml`, so
+ * the file rule is the directory rule applied to its parent. Deriving it keeps
+ * the pruned descent and this final filter from ever disagreeing — a drift that
+ * would either resurrect a shadowing copy or silently drop a real worker.
+ */
+function isExcludedWorkerPath(relativeFile) {
+    return isExcludedWorkerDirectory(path.dirname(relativeFile));
+}
 /**
  * Generate core/workers/registry.yaml from worker.yaml files. Invalid workers
  * are quarantined but do not prevent all valid workers from being registered.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@indigoai-us/hq-cli",
-  "version": "5.115.3",
+  "version": "5.115.5",
   "description": "HQ by Indigo management CLI \u2014 modules and cloud sync",
   "main": "dist/index.js",
   "bin": {