@indigoai-us/hq-cli 5.125.0 → 5.127.0

package/dist/lib/lanes/throttle.d.ts

@@ -0,0 +1,89 @@
+/**
+ * CPU busy-fraction and memory admission gate (spec §5).
+ *
+ * Algorithm is the workflow-runner sampler (two os.cpus() readings
+ * CPU_SAMPLE_MS apart, 1 - idleDelta/totalDelta, clamped 0..1). The sampler
+ * is reimplemented here because hq-cli cannot import core/scripts and the
+ * shared @indigoai-us/hq-cloud export listed in spec §13 does not exist yet.
+ *
+ * Overrides HQ_LANES_CPU_BUSY_OVERRIDE and HQ_LANES_CPU_CHECK are deliberately
+ * distinct from the runner's HQ_WORKFLOW_* pair — this module never reads those.
+ */
+import * as fs from "node:fs";
+import * as os from "node:os";
+import type { ThrottleConfig } from "./types.js";
+export declare const CPU_SAMPLE_MS = 200;
+export declare const STARVATION_MAX_DEFERRALS = 10;
+export declare const STARVATION_MAX_MS: number;
+export declare const PROC_MEMINFO = "/proc/meminfo";
+export declare const LANES_CPU_BUSY_OVERRIDE = "HQ_LANES_CPU_BUSY_OVERRIDE";
+export declare const LANES_CPU_CHECK = "HQ_LANES_CPU_CHECK";
+export interface CpuTimes {
+    idle: number;
+    total: number;
+}
+export interface ThrottleDeps {
+    env: NodeJS.ProcessEnv;
+    now: () => number;
+    cpus: () => os.CpuInfo[];
+    sleep: (ms: number) => Promise<void>;
+    readFileSync: (file: string, encoding: "utf8") => string;
+    /**
+     * Fallback only where MemAvailable is absent from /proc/meminfo (or the
+     * file itself is missing). Do not use this as the Linux primary reading.
+     */
+    freemem: () => number;
+    existsSync: (file: string) => boolean;
+    mkdirSync: typeof fs.mkdirSync;
+    writeFileSync: typeof fs.writeFileSync;
+    renameSync: typeof fs.renameSync;
+    unlinkSync: typeof fs.unlinkSync;
+}
+export declare function defaultThrottleDeps(env?: NodeJS.ProcessEnv): ThrottleDeps;
+export declare function cpuTimesSnapshot(cpus: os.CpuInfo[]): CpuTimes;
+export declare function busyFractionFromDeltas(before: CpuTimes, after: CpuTimes): number | null;
+export declare function sampleCpuBusyFraction(deps?: ThrottleDeps, sampleMs?: number): Promise<number | null>;
+export declare function isCpuCheckEnabled(env?: NodeJS.ProcessEnv): boolean;
+export declare function resolveCpuBusyFraction(deps?: ThrottleDeps): Promise<number | null>;
+/** Parse MemAvailable kB from /proc/meminfo. */
+export declare function parseMemAvailableKb(contents: string): number | null;
+/**
+ * Available memory in MiB. Prefers kernel MemAvailable; falls back to
+ * os.freemem() only where MemAvailable is absent.
+ */
+export declare function readAvailableMemMb(deps?: ThrottleDeps): number | null;
+export interface ThrottleState {
+    consecutiveDeferrals: number;
+    lastAdmissionAt: string | null;
+    cpuPaused: boolean;
+}
+export type ThrottleAdmitReason = "ok" | "starvation_bound" | "unreadable" | "cpu_check_disabled";
+export type ThrottleDeferReason = "cpu" | "memory";
+export type ThrottleDecision = {
+    admit: true;
+    reason: ThrottleAdmitReason;
+} | {
+    admit: false;
+    reason: ThrottleDeferReason;
+    busy?: number;
+    availableMemMb?: number;
+};
+export declare function emptyThrottleState(): ThrottleState;
+export declare function starvationBoundReached(state: ThrottleState, nowMs: number): boolean;
+export declare function evaluateThrottle(input: {
+    busy: number | null;
+    availableMemMb: number | null;
+    cpuCheckEnabled: boolean;
+    state: ThrottleState | "unreadable";
+    throttle: ThrottleConfig;
+    nowMs: number;
+}): ThrottleDecision;
+export declare function applyThrottleDecision(state: ThrottleState, decision: ThrottleDecision, nowMs: number): ThrottleState;
+export declare function throttleStatePath(hqRoot: string): string;
+export declare function readThrottleState(filePath: string, deps?: ThrottleDeps): ThrottleState | "unreadable" | "missing";
+export declare function writeThrottleState(filePath: string, state: ThrottleState, deps?: ThrottleDeps): void;
+export declare function decideAdmission(hqRoot: string, throttle?: ThrottleConfig, deps?: ThrottleDeps): Promise<{
+    decision: ThrottleDecision;
+    state: ThrottleState;
+}>;
+//# sourceMappingURL=throttle.d.ts.map

package/dist/lib/lanes/throttle.js

@@ -0,0 +1,297 @@
+/**
+ * CPU busy-fraction and memory admission gate (spec §5).
+ *
+ * Algorithm is the workflow-runner sampler (two os.cpus() readings
+ * CPU_SAMPLE_MS apart, 1 - idleDelta/totalDelta, clamped 0..1). The sampler
+ * is reimplemented here because hq-cli cannot import core/scripts and the
+ * shared @indigoai-us/hq-cloud export listed in spec §13 does not exist yet.
+ *
+ * Overrides HQ_LANES_CPU_BUSY_OVERRIDE and HQ_LANES_CPU_CHECK are deliberately
+ * distinct from the runner's HQ_WORKFLOW_* pair — this module never reads those.
+ */
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+import { DEFAULT_CPU_BUSY_PAUSE, DEFAULT_CPU_BUSY_RESUME, DEFAULT_MIN_AVAILABLE_MEM_MB, } from "./config.js";
+export const CPU_SAMPLE_MS = 200;
+export const STARVATION_MAX_DEFERRALS = 10;
+export const STARVATION_MAX_MS = 30 * 60 * 1000;
+export const PROC_MEMINFO = "/proc/meminfo";
+export const LANES_CPU_BUSY_OVERRIDE = "HQ_LANES_CPU_BUSY_OVERRIDE";
+export const LANES_CPU_CHECK = "HQ_LANES_CPU_CHECK";
+function defaultSleep(ms) {
+    return new Promise((resolve) => {
+        setTimeout(resolve, ms);
+    });
+}
+export function defaultThrottleDeps(env = process.env) {
+    return {
+        env,
+        now: () => Date.now(),
+        cpus: () => os.cpus() ?? [],
+        sleep: defaultSleep,
+        readFileSync: (file, encoding) => fs.readFileSync(file, encoding),
+        // os.freemem() reports MemFree on Linux (excludes page cache). Used only
+        // where MemAvailable is absent — see readAvailableMemMb.
+        freemem: () => os.freemem(),
+        existsSync: (file) => fs.existsSync(file),
+        mkdirSync: fs.mkdirSync.bind(fs),
+        writeFileSync: fs.writeFileSync.bind(fs),
+        renameSync: fs.renameSync.bind(fs),
+        unlinkSync: fs.unlinkSync.bind(fs),
+    };
+}
+export function cpuTimesSnapshot(cpus) {
+    let idle = 0;
+    let total = 0;
+    for (const cpu of cpus) {
+        const t = cpu.times;
+        idle += t.idle;
+        total += t.user + t.nice + t.sys + t.idle + t.irq;
+    }
+    return { idle, total };
+}
+export function busyFractionFromDeltas(before, after) {
+    const idleDelta = after.idle - before.idle;
+    const totalDelta = after.total - before.total;
+    if (!(totalDelta > 0))
+        return null;
+    return Math.max(0, Math.min(1, 1 - idleDelta / totalDelta));
+}
+export async function sampleCpuBusyFraction(deps = defaultThrottleDeps(), sampleMs = CPU_SAMPLE_MS) {
+    try {
+        const a = cpuTimesSnapshot(deps.cpus());
+        await deps.sleep(sampleMs);
+        const b = cpuTimesSnapshot(deps.cpus());
+        return busyFractionFromDeltas(a, b);
+    }
+    catch (err) {
+        process.stderr.write(`lanes throttle: CPU sample failed: ${err instanceof Error ? err.message : String(err)}\n`);
+        return null;
+    }
+}
+export function isCpuCheckEnabled(env = process.env) {
+    return !/^(0|false|off|no)$/i.test(env[LANES_CPU_CHECK] || "");
+}
+export async function resolveCpuBusyFraction(deps = defaultThrottleDeps()) {
+    const raw = deps.env[LANES_CPU_BUSY_OVERRIDE];
+    if (raw !== undefined && raw !== "") {
+        const v = Number(raw);
+        if (Number.isFinite(v))
+            return Math.max(0, Math.min(1, v));
+        process.stderr.write(`lanes throttle: ${LANES_CPU_BUSY_OVERRIDE} is not a finite number (${JSON.stringify(raw)}); sampling instead\n`);
+    }
+    return sampleCpuBusyFraction(deps);
+}
+/** Parse MemAvailable kB from /proc/meminfo. */
+export function parseMemAvailableKb(contents) {
+    for (const line of contents.split("\n")) {
+        const match = /^MemAvailable:\s+(\d+)\s+kB\s*$/.exec(line.trimEnd());
+        if (match) {
+            const kb = Number(match[1]);
+            return Number.isFinite(kb) && kb >= 0 ? kb : null;
+        }
+    }
+    return null;
+}
+/**
+ * Available memory in MiB. Prefers kernel MemAvailable; falls back to
+ * os.freemem() only where MemAvailable is absent.
+ */
+export function readAvailableMemMb(deps = defaultThrottleDeps()) {
+    try {
+        const contents = deps.readFileSync(PROC_MEMINFO, "utf8");
+        const kb = parseMemAvailableKb(contents);
+        if (kb !== null)
+            return kb / 1024;
+        // /proc/meminfo exists but has no MemAvailable (unusual). Fall back.
+        process.stderr.write("lanes throttle: MemAvailable missing from /proc/meminfo; falling back to os.freemem()\n");
+    }
+    catch (err) {
+        const code = err.code;
+        if (code !== "ENOENT") {
+            process.stderr.write(`lanes throttle: failed to read ${PROC_MEMINFO}: ${err instanceof Error ? err.message : String(err)}\n`);
+            return null;
+        }
+        // Platform without /proc/meminfo — fall back to free memory.
+    }
+    try {
+        const bytes = deps.freemem();
+        if (!Number.isFinite(bytes) || bytes < 0)
+            return null;
+        return bytes / (1024 * 1024);
+    }
+    catch (err) {
+        process.stderr.write(`lanes throttle: freemem fallback failed: ${err instanceof Error ? err.message : String(err)}\n`);
+        return null;
+    }
+}
+export function emptyThrottleState() {
+    return {
+        consecutiveDeferrals: 0,
+        lastAdmissionAt: null,
+        cpuPaused: false,
+    };
+}
+export function starvationBoundReached(state, nowMs) {
+    if (state.consecutiveDeferrals >= STARVATION_MAX_DEFERRALS)
+        return true;
+    if (state.lastAdmissionAt === null)
+        return false;
+    const last = Date.parse(state.lastAdmissionAt);
+    if (!Number.isFinite(last))
+        return false;
+    return nowMs - last >= STARVATION_MAX_MS;
+}
+export function evaluateThrottle(input) {
+    if (input.state === "unreadable") {
+        return { admit: true, reason: "unreadable" };
+    }
+    const pause = input.throttle.cpu_busy_pause;
+    const resume = input.throttle.cpu_busy_resume;
+    const minMem = input.throttle.min_available_mem_mb;
+    let cpuBlocks = false;
+    if (input.cpuCheckEnabled && input.busy !== null) {
+        if (input.state.cpuPaused) {
+            cpuBlocks = input.busy > resume;
+        }
+        else {
+            cpuBlocks = input.busy >= pause;
+        }
+    }
+    let memBlocks = false;
+    if (input.availableMemMb !== null) {
+        memBlocks = input.availableMemMb < minMem;
+    }
+    if (!cpuBlocks && !memBlocks) {
+        return {
+            admit: true,
+            reason: input.cpuCheckEnabled ? "ok" : "cpu_check_disabled",
+        };
+    }
+    if (starvationBoundReached(input.state, input.nowMs)) {
+        return { admit: true, reason: "starvation_bound" };
+    }
+    if (cpuBlocks) {
+        return { admit: false, reason: "cpu", busy: input.busy ?? undefined };
+    }
+    return {
+        admit: false,
+        reason: "memory",
+        availableMemMb: input.availableMemMb ?? undefined,
+    };
+}
+export function applyThrottleDecision(state, decision, nowMs) {
+    if (decision.admit) {
+        return {
+            consecutiveDeferrals: 0,
+            lastAdmissionAt: new Date(nowMs).toISOString(),
+            cpuPaused: false,
+        };
+    }
+    return {
+        consecutiveDeferrals: state.consecutiveDeferrals + 1,
+        lastAdmissionAt: state.lastAdmissionAt,
+        cpuPaused: decision.reason === "cpu" ? true : state.cpuPaused,
+    };
+}
+export function throttleStatePath(hqRoot) {
+    return path.join(hqRoot, "workspace", "lanes-runs", "throttle-state.json");
+}
+export function readThrottleState(filePath, deps = defaultThrottleDeps()) {
+    if (!deps.existsSync(filePath))
+        return "missing";
+    let raw;
+    try {
+        raw = deps.readFileSync(filePath, "utf8");
+    }
+    catch (err) {
+        process.stderr.write(`lanes throttle: failed to read state ${filePath}: ${err instanceof Error ? err.message : String(err)}\n`);
+        return "unreadable";
+    }
+    try {
+        const parsed = JSON.parse(raw);
+        if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
+            process.stderr.write(`lanes throttle: state ${filePath} is not an object\n`);
+            return "unreadable";
+        }
+        const obj = parsed;
+        if (typeof obj.consecutiveDeferrals !== "number" ||
+            !Number.isInteger(obj.consecutiveDeferrals) ||
+            obj.consecutiveDeferrals < 0) {
+            process.stderr.write(`lanes throttle: state ${filePath} has invalid consecutiveDeferrals\n`);
+            return "unreadable";
+        }
+        if (obj.lastAdmissionAt !== null &&
+            typeof obj.lastAdmissionAt !== "string") {
+            process.stderr.write(`lanes throttle: state ${filePath} has invalid lastAdmissionAt\n`);
+            return "unreadable";
+        }
+        if (typeof obj.cpuPaused !== "boolean") {
+            process.stderr.write(`lanes throttle: state ${filePath} has invalid cpuPaused\n`);
+            return "unreadable";
+        }
+        return {
+            consecutiveDeferrals: obj.consecutiveDeferrals,
+            lastAdmissionAt: obj.lastAdmissionAt,
+            cpuPaused: obj.cpuPaused,
+        };
+    }
+    catch (err) {
+        process.stderr.write(`lanes throttle: state ${filePath} is not valid JSON: ${err instanceof Error ? err.message : String(err)}\n`);
+        return "unreadable";
+    }
+}
+export function writeThrottleState(filePath, state, deps = defaultThrottleDeps()) {
+    const dir = path.dirname(filePath);
+    deps.mkdirSync(dir, { recursive: true });
+    const tmp = `${filePath}.${process.pid}.${Date.now()}.tmp`;
+    try {
+        deps.writeFileSync(tmp, `${JSON.stringify(state, null, 2)}\n`, {
+            encoding: "utf8",
+        });
+        deps.renameSync(tmp, filePath);
+    }
+    catch (err) {
+        try {
+            deps.unlinkSync(tmp);
+        }
+        catch (unlinkErr) {
+            const code = unlinkErr.code;
+            if (code !== "ENOENT") {
+                process.stderr.write(`lanes throttle: unlink tmp failed for ${tmp}: ${unlinkErr instanceof Error ? unlinkErr.message : String(unlinkErr)}\n`);
+            }
+        }
+        throw err;
+    }
+}
+export async function decideAdmission(hqRoot, throttle = {
+    cpu_busy_pause: DEFAULT_CPU_BUSY_PAUSE,
+    cpu_busy_resume: DEFAULT_CPU_BUSY_RESUME,
+    min_available_mem_mb: DEFAULT_MIN_AVAILABLE_MEM_MB,
+}, deps = defaultThrottleDeps()) {
+    const filePath = throttleStatePath(hqRoot);
+    const loaded = readThrottleState(filePath, deps);
+    if (loaded === "unreadable") {
+        return {
+            decision: { admit: true, reason: "unreadable" },
+            state: emptyThrottleState(),
+        };
+    }
+    const state = loaded === "missing" ? emptyThrottleState() : loaded;
+    const busy = await resolveCpuBusyFraction(deps);
+    const availableMemMb = readAvailableMemMb(deps);
+    const nowMs = deps.now();
+    const decision = evaluateThrottle({
+        busy,
+        availableMemMb,
+        cpuCheckEnabled: isCpuCheckEnabled(deps.env),
+        state,
+        throttle,
+        nowMs,
+    });
+    const next = applyThrottleDecision(state, decision, nowMs);
+    writeThrottleState(filePath, next, deps);
+    return { decision, state: next };
+}
+//# sourceMappingURL=throttle.js.map