@indigoai-us/hq-cli 5.314.0 → 5.316.0

package/dist/lib/lanes/quota.js

@@ -0,0 +1,471 @@
+/**
+ * Machine-written provider quota, keyed by account and model.
+ *
+ * `enabled` and `disabled_reason` live in personal/lanes/config.json and are
+ * the operator's hand-written switch. `quota_reset_at` and `needs_credits`
+ * are written here, in personal/lanes/quota.json, from the provider's own
+ * 429. They are separate fields because a quota hold expires on its own:
+ * copying it into `disabled_reason` or flipping `enabled` would erase the
+ * operator's reason, and would leave the account switched off after the
+ * provider reset. The balancer reads this file and does not rewrite the
+ * operator fields.
+ *
+ * A weekly-limit 429 names a reset and applies to every model on that
+ * account (measured: account b, every model, until the reset). A credits
+ * 429 names one model and no reset (measured: account d refuses
+ * claude-fable-5-1 and still runs claude-opus-5). The record is therefore
+ * per (account, model), plus an account-wide reset when the text is a
+ * usage limit rather than a switch-model refusal.
+ */
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { withLaneRecordFileLock } from "./store.js";
+export const QUOTA_STATE_REL = path.join("personal", "lanes", "quota.json");
+export const QUOTA_SCHEMA_VERSION = "hq-lanes-quota.v1";
+/**
+ * A 429 whose message names no reset time and no credits wording would
+ * otherwise record no hold at all, and the next lane on the account would
+ * hit the same 429. Give such a signal the provider's configured default
+ * hold (`quota_default_hold_ms`, fifteen minutes unless set) as its reset,
+ * marked `default_hold`; a signal that already carries a reset keeps it,
+ * marked `provider`. A per-model credits refusal is untouched.
+ */
+export function applyDefaultQuotaHold(signal, defaultHoldMs, now) {
+    if (signal.reason !== "account_quota_exhausted" || signal.needs_credits)
+        return signal;
+    if (signal.quota_reset_at)
+        return { ...signal, reset_source: "provider" };
+    return {
+        ...signal,
+        quota_reset_at: new Date(now.getTime() + defaultHoldMs).toISOString(),
+        account_wide: true,
+        reset_source: "default_hold",
+    };
+}
+const MONTHS = {
+    jan: 0,
+    feb: 1,
+    mar: 2,
+    apr: 3,
+    may: 4,
+    jun: 5,
+    jul: 6,
+    aug: 7,
+    sep: 8,
+    oct: 9,
+    nov: 10,
+    dec: 11,
+};
+function emptyState() {
+    return { schema_version: QUOTA_SCHEMA_VERSION, providers: {} };
+}
+function isPlainObject(value) {
+    return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+export function quotaStatePath(hqRoot) {
+    return path.join(hqRoot, QUOTA_STATE_REL);
+}
+export function loadQuotaState(hqRoot) {
+    const filePath = quotaStatePath(hqRoot);
+    let text;
+    try {
+        text = fs.readFileSync(filePath, "utf8");
+    }
+    catch (err) {
+        const code = err.code;
+        if (code === "ENOENT")
+            return emptyState();
+        throw err;
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(text);
+    }
+    catch {
+        return emptyState();
+    }
+    if (!isPlainObject(parsed))
+        return emptyState();
+    const providers = {};
+    const rawProviders = isPlainObject(parsed.providers) ? parsed.providers : {};
+    for (const provider of ["claude", "codex", "grok"]) {
+        const rawAccounts = rawProviders[provider];
+        if (!isPlainObject(rawAccounts))
+            continue;
+        const accounts = {};
+        for (const [id, raw] of Object.entries(rawAccounts)) {
+            if (!isPlainObject(raw))
+                continue;
+            const models = {};
+            if (isPlainObject(raw.models)) {
+                for (const [model, entry] of Object.entries(raw.models)) {
+                    if (!isPlainObject(entry))
+                        continue;
+                    const quota = {};
+                    if (typeof entry.quota_reset_at === "string" && entry.quota_reset_at.length > 0) {
+                        quota.quota_reset_at = entry.quota_reset_at;
+                    }
+                    if (entry.needs_credits === true)
+                        quota.needs_credits = true;
+                    if (typeof entry.marker === "string" && entry.marker.length > 0) {
+                        quota.marker = entry.marker;
+                    }
+                    if (quota.quota_reset_at || quota.needs_credits || quota.marker) {
+                        models[model] = quota;
+                    }
+                }
+            }
+            const account = { models };
+            if (typeof raw.all_models_quota_reset_at === "string" &&
+                raw.all_models_quota_reset_at.length > 0) {
+                account.all_models_quota_reset_at = raw.all_models_quota_reset_at;
+                if (raw.reset_source === "provider" || raw.reset_source === "default_hold") {
+                    account.reset_source = raw.reset_source;
+                }
+            }
+            accounts[id] = account;
+        }
+        providers[provider] = accounts;
+    }
+    return { schema_version: QUOTA_SCHEMA_VERSION, providers };
+}
+function writeQuotaState(hqRoot, state) {
+    const filePath = quotaStatePath(hqRoot);
+    fs.mkdirSync(path.dirname(filePath), { recursive: true });
+    const tmp = `${filePath}.${process.pid}.tmp`;
+    fs.writeFileSync(tmp, `${JSON.stringify(state, null, 2)}\n`, "utf8");
+    fs.renameSync(tmp, filePath);
+}
+/**
+ * Parse provider reset times such as "resets Sep 24, 2am (UTC)" and Codex's
+ * "try again at Sep 14th, 2026 9:24 PM" into an ISO timestamp. Codex's
+ * unzoned form is displayed in the provider CLI's local timezone, while the
+ * provider `resets` form is UTC when it names `(UTC)`. The year is the current
+ * UTC year for an unyear-tagged reset, rolled forward when it is already past.
+ * An explicit provider year is kept as stated.
+ */
+export function parseProviderResetTime(text, now) {
+    const resetMatch = /\bresets\s+([A-Za-z]+)\s+(\d{1,2})(?:st|nd|rd|th)?(?:,\s*(?:(\d{4})\s+)?(\d{1,2})(?::(\d{2}))?\s*(am|pm))?(?:\s*\((UTC)\))?/i.exec(text);
+    const codexMatch = /\btry\s+again\s+at\s+([A-Za-z]+)\s+(\d{1,2})(?:st|nd|rd|th)?,\s*(\d{4})\s+(\d{1,2})(?::(\d{2}))?\s*(am|pm)(?:\s*\((UTC)\))?/i.exec(text);
+    const match = resetMatch ?? codexMatch;
+    if (!match)
+        return null;
+    const codexLocalTime = codexMatch !== null && (match[7] ?? "").toLowerCase() !== "utc";
+    const month = MONTHS[(match[1] ?? "").slice(0, 3).toLowerCase()];
+    if (month === undefined)
+        return null;
+    const day = Number(match[2]);
+    if (!Number.isInteger(day) || day < 1 || day > 31)
+        return null;
+    const explicitYear = match[3] !== undefined;
+    const parsedYear = explicitYear ? Number(match[3]) : now.getUTCFullYear();
+    if (!Number.isInteger(parsedYear) || parsedYear < 1970)
+        return null;
+    let hour = 0;
+    let minute = 0;
+    if (match[4] !== undefined) {
+        hour = Number(match[4]);
+        minute = match[5] !== undefined ? Number(match[5]) : 0;
+        const ap = (match[6] ?? "").toLowerCase();
+        if (ap === "pm" && hour < 12)
+            hour += 12;
+        if (ap === "am" && hour === 12)
+            hour = 0;
+        if (hour > 23 || minute > 59)
+            return null;
+    }
+    const timestampForYear = (targetYear) => {
+        const candidate = codexLocalTime
+            ? new Date(targetYear, month, day, hour, minute, 0, 0)
+            : new Date(Date.UTC(targetYear, month, day, hour, minute, 0, 0));
+        const valid = codexLocalTime
+            ? candidate.getFullYear() === targetYear &&
+                candidate.getMonth() === month &&
+                candidate.getDate() === day &&
+                candidate.getHours() === hour &&
+                candidate.getMinutes() === minute
+            : candidate.getUTCFullYear() === targetYear &&
+                candidate.getUTCMonth() === month &&
+                candidate.getUTCDate() === day &&
+                candidate.getUTCHours() === hour &&
+                candidate.getUTCMinutes() === minute;
+        return valid ? candidate.getTime() : null;
+    };
+    let year = parsedYear;
+    let ts = timestampForYear(year);
+    if (ts === null)
+        return null;
+    if (!explicitYear && ts < now.getTime() - 36 * 60 * 60 * 1000) {
+        year += 1;
+        ts = timestampForYear(year);
+        if (ts === null)
+            return null;
+    }
+    return new Date(ts).toISOString();
+}
+function isCreditsRefusal(text) {
+    return /requires usage credits/i.test(text) || /switch to another model/i.test(text);
+}
+function isUsageLimitShape(text) {
+    return /weekly limit|usage limit|rate limit|quota|usage credits|hit your .+ limit/i.test(text);
+}
+/**
+ * Read only `api_error_status`, `is_error`, and `result` off the last
+ * `type: "result"` object. Nothing else from the payload is returned, and
+ * this function does not log.
+ */
+export function quotaFieldsFromOutput(text) {
+    const lines = text.split("\n");
+    for (let i = lines.length - 1; i >= 0; i -= 1) {
+        const line = (lines[i] ?? "").trim();
+        if (!line.startsWith("{"))
+            continue;
+        let parsed;
+        try {
+            parsed = JSON.parse(line);
+        }
+        catch {
+            continue;
+        }
+        if (!isPlainObject(parsed) || parsed.type !== "result")
+            continue;
+        return {
+            api_error_status: parsed.api_error_status,
+            is_error: parsed.is_error,
+            result: parsed.result,
+        };
+    }
+    return null;
+}
+export function quotaSignalFromFields(fields, model, now) {
+    const text = typeof fields.result === "string" ? fields.result : "";
+    const is429 = fields.api_error_status === 429;
+    const usageShaped = fields.is_error === true && isUsageLimitShape(text);
+    if (!is429 && !usageShaped)
+        return null;
+    if (isCreditsRefusal(text) && !/resets\s+/i.test(text)) {
+        const marker = model ? `needs credits for ${model}` : "needs credits";
+        return {
+            reason: "account_model_unavailable",
+            model,
+            needs_credits: true,
+            marker,
+        };
+    }
+    const quota_reset_at = text.length > 0 ? parseProviderResetTime(text, now) ?? undefined : undefined;
+    return {
+        reason: "account_quota_exhausted",
+        quota_reset_at,
+        model,
+        account_wide: true,
+    };
+}
+export function readQuotaSignal(runDir, model, now, workerOutOffset = 0) {
+    const outPath = path.join(runDir, "logs", "worker.out");
+    let text;
+    try {
+        const bytes = fs.readFileSync(outPath);
+        text = bytes.subarray(Math.min(workerOutOffset, bytes.length)).toString("utf8");
+    }
+    catch (err) {
+        const code = err.code;
+        if (code === "ENOENT")
+            return null;
+        throw err;
+    }
+    const fields = quotaFieldsFromOutput(text);
+    if (!fields)
+        return null;
+    return quotaSignalFromFields(fields, model, now);
+}
+export function recordAccountQuota(hqRoot, provider, accountId, signal) {
+    return withLaneRecordFileLock(quotaStatePath(hqRoot), () => {
+        const state = loadQuotaState(hqRoot);
+        const accounts = state.providers[provider] ?? {};
+        const current = accounts[accountId] ?? { models: {} };
+        const models = { ...current.models };
+        if (signal.model) {
+            const prior = models[signal.model] ?? {};
+            const next = { ...prior };
+            if (signal.needs_credits) {
+                next.needs_credits = true;
+                next.marker = signal.marker ?? `needs credits for ${signal.model}`;
+                delete next.quota_reset_at;
+            }
+            if (signal.quota_reset_at &&
+                !signal.needs_credits &&
+                shouldReplaceReset(next.quota_reset_at, signal.quota_reset_at)) {
+                next.quota_reset_at = signal.quota_reset_at;
+            }
+            models[signal.model] = next;
+        }
+        const account = { ...current, models };
+        if (signal.account_wide && signal.quota_reset_at) {
+            if (shouldReplaceReset(current.all_models_quota_reset_at, signal.quota_reset_at)) {
+                account.all_models_quota_reset_at = signal.quota_reset_at;
+                account.reset_source = signal.reset_source ?? "provider";
+            }
+            else if (current.all_models_quota_reset_at) {
+                account.all_models_quota_reset_at = current.all_models_quota_reset_at;
+                account.reset_source = current.reset_source ?? "provider";
+            }
+        }
+        accounts[accountId] = account;
+        state.providers[provider] = accounts;
+        writeQuotaState(hqRoot, state);
+        return state;
+    });
+}
+/** Clear quota reset marks and persist them with the quota file's atomic writer. */
+export function clearAccountQuota(hqRoot, provider, accountId, options = {}) {
+    return withLaneRecordFileLock(quotaStatePath(hqRoot), () => {
+        const cleared = {
+            all_models_quota_reset_at: null,
+            model_quota_resets: [],
+        };
+        const state = loadQuotaState(hqRoot);
+        const account = state.providers[provider]?.[accountId];
+        if (!account)
+            return cleared;
+        const now = options.now ?? new Date();
+        const shouldClear = (reset) => reset !== undefined &&
+            (!options.futureOnly || resetStillAhead(reset, now));
+        let changed = false;
+        const accountWideReset = account.all_models_quota_reset_at;
+        if (shouldClear(accountWideReset)) {
+            cleared.all_models_quota_reset_at = accountWideReset;
+            delete account.all_models_quota_reset_at;
+            delete account.reset_source;
+            changed = true;
+        }
+        const modelNames = options.model === undefined
+            ? Object.keys(account.models).sort()
+            : [options.model];
+        for (const model of modelNames) {
+            const entry = account.models[model];
+            const modelReset = entry?.quota_reset_at;
+            if (!entry || !shouldClear(modelReset))
+                continue;
+            cleared.model_quota_resets.push({
+                model,
+                quota_reset_at: modelReset,
+            });
+            delete entry.quota_reset_at;
+            if (Object.keys(entry).length === 0)
+                delete account.models[model];
+            changed = true;
+        }
+        if (!changed)
+            return cleared;
+        if (account.all_models_quota_reset_at === undefined) {
+            delete account.reset_source;
+        }
+        const accounts = state.providers[provider];
+        if (account.all_models_quota_reset_at === undefined &&
+            Object.keys(account.models).length === 0) {
+            delete accounts[accountId];
+            if (Object.keys(accounts).length === 0)
+                delete state.providers[provider];
+        }
+        else {
+            accounts[accountId] = account;
+        }
+        writeQuotaState(hqRoot, state);
+        return cleared;
+    });
+}
+export function clearAccountQuotaAfterSuccessfulTurn(hqRoot, provider, accountId, model, now) {
+    return clearAccountQuota(hqRoot, provider, accountId, {
+        model,
+        futureOnly: true,
+        now,
+    });
+}
+export function clearAccountQuotaAfterSuccessfulTurnBestEffort(hqRoot, provider, accountId, model, now) {
+    try {
+        clearAccountQuotaAfterSuccessfulTurn(hqRoot, provider, accountId, model, now);
+    }
+    catch (err) {
+        const detail = err instanceof Error ? err.message : String(err);
+        process.stderr.write(`hq lanes: quota cleanup after successful turn failed for provider=${provider} account=${accountId} model=${model}: ${detail}\n`);
+    }
+}
+function shouldReplaceReset(current, incoming) {
+    if (!current)
+        return true;
+    const currentMs = Date.parse(current);
+    const incomingMs = Date.parse(incoming);
+    if (!Number.isFinite(currentMs))
+        return Number.isFinite(incomingMs);
+    if (!Number.isFinite(incomingMs))
+        return false;
+    return incomingMs > currentMs;
+}
+function resetStillAhead(iso, now) {
+    if (!iso)
+        return false;
+    const ts = Date.parse(iso);
+    return Number.isFinite(ts) && ts > now.getTime();
+}
+/**
+ * Skip an account only for the models it cannot run. An account-wide reset
+ * blocks every model. A needs-credits marker blocks only that model, with
+ * no expiry. A past reset does not block.
+ */
+export function quotaBlockFor(state, provider, accountId, model, now) {
+    const account = state.providers[provider]?.[accountId];
+    if (!account)
+        return { blocked: false };
+    if (resetStillAhead(account.all_models_quota_reset_at, now)) {
+        return {
+            blocked: true,
+            reason: "account_quota_exhausted",
+            quota_reset_at: account.all_models_quota_reset_at,
+            model,
+        };
+    }
+    if (!model)
+        return { blocked: false };
+    const entry = account.models[model];
+    if (!entry)
+        return { blocked: false };
+    if (entry.needs_credits) {
+        return {
+            blocked: true,
+            reason: "account_model_unavailable",
+            marker: entry.marker ?? `needs credits for ${model}`,
+            model,
+        };
+    }
+    if (resetStillAhead(entry.quota_reset_at, now)) {
+        return {
+            blocked: true,
+            reason: "account_quota_exhausted",
+            quota_reset_at: entry.quota_reset_at,
+            model,
+        };
+    }
+    return { blocked: false };
+}
+export function accountQuotaView(state, provider, accountId) {
+    const account = state.providers[provider]?.[accountId];
+    if (!account)
+        return { quota_reset_at: null, model_quota: [] };
+    const model_quota = Object.entries(account.models)
+        .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
+        .map(([model, entry]) => {
+        const row = { model };
+        if (entry.quota_reset_at)
+            row.quota_reset_at = entry.quota_reset_at;
+        if (entry.needs_credits)
+            row.needs_credits = true;
+        if (entry.marker)
+            row.marker = entry.marker;
+        return row;
+    });
+    return {
+        quota_reset_at: account.all_models_quota_reset_at ?? null,
+        model_quota,
+    };
+}
+//# sourceMappingURL=quota.js.map

package/dist/lib/lanes/reconcile-install.d.ts

@@ -0,0 +1,90 @@
+/**
+ * systemd user timer install for the lanes reconcile pass.
+ *
+ * Policy: hq-infra-watchdogs-systemd-not-session. The pass is a oneshot
+ * invoked by a timer, never a session-bound background job. Install writes
+ * unit files under a systemd user dir and is idempotent and reversible.
+ * Enabling the timer is optional — the CLI pass works without it.
+ */
+export declare const RECONCILE_TIMER_INTERVAL = "1min";
+export interface ProviderPathEnv {
+    /**
+     * For each resolved provider: the directory where the command name was
+     * found on the installing shell's PATH, then the realpath target's
+     * directory when it differs. Provider order, deduplicated.
+     */
+    resolved_dirs: string[];
+    /** Providers whose binary resolved on the installing shell's PATH. */
+    resolved: string[];
+    /** Providers whose binary did not resolve. Reported, never fatal on their own. */
+    unresolved: string[];
+}
+/**
+ * Resolve provider commands with the installing shell's PATH. The resulting
+ * directories are carried into the systemd unit, whose default PATH is too
+ * narrow for per-user provider installations.
+ *
+ * A default config enables one account per provider, so `providers` usually
+ * names every provider even on a box with one installed. A provider that does
+ * not resolve is reported in `unresolved`; the install only refuses when none
+ * of the requested providers resolves, because then the timer has nothing to
+ * run.
+ */
+export declare function providerPathEnv(providers: readonly string[], pathEnv?: string | undefined): ProviderPathEnv;
+/**
+ * The unit's PATH: each resolved provider binary directory, the directory of
+ * the hq binary itself, the directory of the node binary that runs it (a
+ * provider installed as a `#!/usr/bin/env node` script needs `node` on the
+ * unit PATH, and nvm/asdf installs keep it outside the baseline), then the
+ * systemd baseline, deduplicated. Nothing
+ * from the installing shell's PATH is carried over: on a box whose profile
+ * is synced from another machine that PATH holds directories from the other
+ * machine (measured 2026-09-22: `/Users/…` and `/opt/homebrew/…` entries on
+ * a Linux outpost), and none of them belong in a systemd unit.
+ */
+export declare function unitPathEnv(input: {
+    providerDirs: readonly string[];
+    hqBinary: string;
+    nodeBinary: string;
+}): string;
+export interface RunCommandResult {
+    status: number;
+    stdout?: string;
+    stderr?: string;
+}
+export type RunCommand = (command: string, args: string[]) => RunCommandResult;
+export interface ReconcileInstallOpts {
+    systemdRoot: string;
+    hqRoot: string;
+    nodeBinary?: string;
+    hqBinary?: string;
+    /** Resolved provider binary directories (providerPathEnv().resolved_dirs). */
+    providerDirs?: readonly string[];
+    home?: string;
+    enable?: boolean;
+    runCommand?: RunCommand;
+}
+export interface ReconcileInstallResult {
+    ok: true;
+    action: "reconcile.install" | "reconcile.uninstall";
+    systemd_root: string;
+    unit: string;
+    timer: string;
+    written?: boolean;
+    enabled?: boolean;
+    removed?: boolean;
+}
+export declare function renderReconcileService(opts: {
+    nodeBinary: string;
+    hqBinary: string;
+    hqRoot: string;
+    home?: string;
+    pathEnv?: string;
+}): string;
+export declare function renderReconcileTimer(): string;
+export declare function defaultSystemdUserDir(home?: string): string;
+export declare function installReconcileTimer(opts: ReconcileInstallOpts): ReconcileInstallResult;
+export declare function uninstallReconcileTimer(opts: Pick<ReconcileInstallOpts, "systemdRoot" | "runCommand"> & {
+    disable?: boolean;
+}): ReconcileInstallResult;
+//# sourceMappingURL=reconcile-install.d.ts.map