@botbotgo/agent-harness 0.0.75 → 0.0.77

package/dist/runtime/health-monitor.js

@@ -0,0 +1,448 @@
+import path from "node:path";
+import { readdir, stat } from "node:fs/promises";
+import { getRuntimeDefaults } from "../workspace/support/workspace-ref-utils.js";
+import { discoverCheckpointMaintenanceTargets } from "./checkpoint-maintenance.js";
+const DEFAULT_HEALTH_CONFIG = {
+    enabled: false,
+    evaluateIntervalSeconds: 30,
+    emitEvents: true,
+    thresholds: {
+        llmErrorRate: {
+            windowSeconds: 60,
+            degradedAbove: 0.15,
+            unhealthyAbove: 0.4,
+        },
+        llmP95LatencyMs: {
+            windowSeconds: 300,
+            degradedAbove: 8_000,
+            unhealthyAbove: 20_000,
+        },
+        pendingApprovals: {
+            degradedAbove: 20,
+            unhealthyAbove: 100,
+        },
+        stuckRunSeconds: {
+            degradedAbove: 300,
+            unhealthyAbove: 900,
+        },
+        checkpointBytes: {
+            degradedAbove: 512 * 1024 * 1024,
+            unhealthyAbove: 2 * 1024 * 1024 * 1024,
+        },
+        runtimeDbBytes: {
+            degradedAbove: 256 * 1024 * 1024,
+            unhealthyAbove: 1024 * 1024 * 1024,
+        },
+        artifactBytes: {
+            degradedAbove: 512 * 1024 * 1024,
+            unhealthyAbove: 2 * 1024 * 1024 * 1024,
+        },
+    },
+};
+function asObject(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value) ? value : undefined;
+}
+function readPositiveNumber(value, fallback) {
+    return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : fallback;
+}
+function readNonNegativeNumber(value, fallback) {
+    return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : fallback;
+}
+function maxStatus(left, right) {
+    const rank = { healthy: 0, degraded: 1, unhealthy: 2 };
+    return rank[left] >= rank[right] ? left : right;
+}
+function compareStatus(left, right) {
+    return left === right;
+}
+function computeP95(values) {
+    if (values.length === 0) {
+        return undefined;
+    }
+    const sorted = [...values].sort((a, b) => a - b);
+    const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil(sorted.length * 0.95) - 1));
+    return sorted[index];
+}
+function isoFromMs(value) {
+    return new Date(value).toISOString();
+}
+function buildCheck(status, updatedAt, reason) {
+    return reason ? { status, updatedAt, reason } : { status, updatedAt };
+}
+function describeThresholdStatus(value, thresholds) {
+    if (value >= thresholds.unhealthyAbove) {
+        return "unhealthy";
+    }
+    if (value >= thresholds.degradedAbove) {
+        return "degraded";
+    }
+    return "healthy";
+}
+export function readHealthMonitorConfig(workspace) {
+    const runtimeDefaults = getRuntimeDefaults(workspace.refs);
+    const observability = asObject(runtimeDefaults?.observability);
+    const health = asObject(observability?.health);
+    const thresholds = asObject(health?.thresholds);
+    const llmErrorRate = asObject(thresholds?.llmErrorRate);
+    const llmP95LatencyMs = asObject(thresholds?.llmP95LatencyMs);
+    const pendingApprovals = asObject(thresholds?.pendingApprovals);
+    const stuckRunSeconds = asObject(thresholds?.stuckRunSeconds);
+    const checkpointBytes = asObject(thresholds?.checkpointBytes);
+    const runtimeDbBytes = asObject(thresholds?.runtimeDbBytes);
+    const artifactBytes = asObject(thresholds?.artifactBytes);
+    return {
+        enabled: health?.enabled === true,
+        evaluateIntervalSeconds: readPositiveNumber(health?.evaluateIntervalSeconds, DEFAULT_HEALTH_CONFIG.evaluateIntervalSeconds),
+        emitEvents: health?.emitEvents !== false,
+        thresholds: {
+            llmErrorRate: {
+                windowSeconds: readPositiveNumber(llmErrorRate?.windowSeconds, DEFAULT_HEALTH_CONFIG.thresholds.llmErrorRate.windowSeconds),
+                degradedAbove: readNonNegativeNumber(llmErrorRate?.degradedAbove, DEFAULT_HEALTH_CONFIG.thresholds.llmErrorRate.degradedAbove),
+                unhealthyAbove: readNonNegativeNumber(llmErrorRate?.unhealthyAbove, DEFAULT_HEALTH_CONFIG.thresholds.llmErrorRate.unhealthyAbove),
+            },
+            llmP95LatencyMs: {
+                windowSeconds: readPositiveNumber(llmP95LatencyMs?.windowSeconds, DEFAULT_HEALTH_CONFIG.thresholds.llmP95LatencyMs.windowSeconds),
+                degradedAbove: readNonNegativeNumber(llmP95LatencyMs?.degradedAbove, DEFAULT_HEALTH_CONFIG.thresholds.llmP95LatencyMs.degradedAbove),
+                unhealthyAbove: readNonNegativeNumber(llmP95LatencyMs?.unhealthyAbove, DEFAULT_HEALTH_CONFIG.thresholds.llmP95LatencyMs.unhealthyAbove),
+            },
+            pendingApprovals: {
+                degradedAbove: readNonNegativeNumber(pendingApprovals?.degradedAbove, DEFAULT_HEALTH_CONFIG.thresholds.pendingApprovals.degradedAbove),
+                unhealthyAbove: readNonNegativeNumber(pendingApprovals?.unhealthyAbove, DEFAULT_HEALTH_CONFIG.thresholds.pendingApprovals.unhealthyAbove),
+            },
+            stuckRunSeconds: {
+                degradedAbove: readNonNegativeNumber(stuckRunSeconds?.degradedAbove, DEFAULT_HEALTH_CONFIG.thresholds.stuckRunSeconds.degradedAbove),
+                unhealthyAbove: readNonNegativeNumber(stuckRunSeconds?.unhealthyAbove, DEFAULT_HEALTH_CONFIG.thresholds.stuckRunSeconds.unhealthyAbove),
+            },
+            checkpointBytes: {
+                degradedAbove: readNonNegativeNumber(checkpointBytes?.degradedAbove, DEFAULT_HEALTH_CONFIG.thresholds.checkpointBytes.degradedAbove),
+                unhealthyAbove: readNonNegativeNumber(checkpointBytes?.unhealthyAbove, DEFAULT_HEALTH_CONFIG.thresholds.checkpointBytes.unhealthyAbove),
+            },
+            runtimeDbBytes: {
+                degradedAbove: readNonNegativeNumber(runtimeDbBytes?.degradedAbove, DEFAULT_HEALTH_CONFIG.thresholds.runtimeDbBytes.degradedAbove),
+                unhealthyAbove: readNonNegativeNumber(runtimeDbBytes?.unhealthyAbove, DEFAULT_HEALTH_CONFIG.thresholds.runtimeDbBytes.unhealthyAbove),
+            },
+            artifactBytes: {
+                degradedAbove: readNonNegativeNumber(artifactBytes?.degradedAbove, DEFAULT_HEALTH_CONFIG.thresholds.artifactBytes.degradedAbove),
+                unhealthyAbove: readNonNegativeNumber(artifactBytes?.unhealthyAbove, DEFAULT_HEALTH_CONFIG.thresholds.artifactBytes.unhealthyAbove),
+            },
+        },
+    };
+}
+async function safeFileSize(filePath) {
+    try {
+        const stats = await stat(filePath);
+        return stats.size;
+    }
+    catch {
+        return 0;
+    }
+}
+async function directorySize(root) {
+    try {
+        const entries = await readdir(root, { withFileTypes: true });
+        let total = 0;
+        for (const entry of entries) {
+            const fullPath = path.join(root, entry.name);
+            if (entry.isDirectory()) {
+                total += await directorySize(fullPath);
+                continue;
+            }
+            if (entry.isFile()) {
+                total += await safeFileSize(fullPath);
+            }
+        }
+        return total;
+    }
+    catch {
+        return 0;
+    }
+}
+function evaluateMaintenanceStatus(label, status, updatedAt) {
+    if (!status) {
+        return {
+            check: buildCheck("healthy", updatedAt, `${label} disabled`),
+            symptoms: [],
+        };
+    }
+    if (status.consecutiveFailures > 0) {
+        return {
+            check: buildCheck("degraded", updatedAt, `${label} has ${status.consecutiveFailures} consecutive failure(s)`),
+            symptoms: [{
+                    code: `${label}.loop.failure`,
+                    severity: "warn",
+                    message: status.lastError ?? `${label} maintenance loop failed`,
+                    firstSeenAt: status.lastFailedAt ?? updatedAt,
+                    lastSeenAt: status.lastFailedAt ?? updatedAt,
+                }],
+        };
+    }
+    if (status.lastCompletedAt) {
+        return {
+            check: buildCheck("healthy", updatedAt, `${label} completed successfully`),
+            symptoms: [],
+        };
+    }
+    return {
+        check: buildCheck("healthy", updatedAt, `${label} idle`),
+        symptoms: [],
+    };
+}
+function normalizeSeverity(status) {
+    if (status === "unhealthy") {
+        return "error";
+    }
+    if (status === "degraded") {
+        return "warn";
+    }
+    return "info";
+}
+function createSymptom(code, status, message, timestamp) {
+    return {
+        code,
+        severity: normalizeSeverity(status),
+        message,
+        firstSeenAt: timestamp,
+        lastSeenAt: timestamp,
+    };
+}
+export class HealthMonitor {
+    options;
+    config;
+    runRoots;
+    checkpointDbPaths;
+    llmSamples = [];
+    timer = null;
+    latestSnapshot = null;
+    runtimeEventSequence = 0;
+    constructor(options) {
+        this.options = options;
+        this.config = readHealthMonitorConfig(options.workspace);
+        this.runRoots = Array.from(new Set(Array.from(options.workspace.bindings.values()).map((binding) => binding.harnessRuntime.runRoot)));
+        this.checkpointDbPaths = Array.from(new Set(discoverCheckpointMaintenanceTargets(options.workspace).map((target) => target.dbPath)));
+    }
+    recordLlmSuccess(latencyMs, nowMs = Date.now()) {
+        this.recordLlmSample({ timestampMs: nowMs, latencyMs, success: true });
+    }
+    recordLlmFailure(latencyMs, nowMs = Date.now()) {
+        this.recordLlmSample({ timestampMs: nowMs, latencyMs, success: false });
+    }
+    recordLlmSample(sample) {
+        this.llmSamples.push(sample);
+        const keepAfter = sample.timestampMs - Math.max(this.config.thresholds.llmErrorRate.windowSeconds, this.config.thresholds.llmP95LatencyMs.windowSeconds) * 1000;
+        while (this.llmSamples.length > 0 && this.llmSamples[0].timestampMs < keepAfter) {
+            this.llmSamples.shift();
+        }
+    }
+    async start() {
+        if (!this.config.enabled || this.timer) {
+            return;
+        }
+        await this.evaluate();
+        this.timer = setInterval(() => {
+            void this.evaluate();
+        }, this.config.evaluateIntervalSeconds * 1000);
+        this.timer.unref?.();
+    }
+    async stop() {
+        if (this.timer) {
+            clearInterval(this.timer);
+            this.timer = null;
+        }
+    }
+    async getSnapshot() {
+        return this.evaluate();
+    }
+    async evaluate(nowMs = Date.now()) {
+        const updatedAt = isoFromMs(nowMs);
+        const [runs, approvals, runtimeDbBytes, checkpointBytes, artifactBytes] = await Promise.all([
+            this.options.persistence.listRuns(),
+            this.options.persistence.listApprovals(),
+            this.sumRuntimeDbBytes(),
+            this.sumCheckpointDbBytes(),
+            this.sumArtifactBytes(),
+        ]);
+        const pendingApprovals = approvals.filter((approval) => approval.status === "pending").length;
+        const stuckRuns = this.countStuckRuns(runs, nowMs);
+        const runtimeMaintenance = evaluateMaintenanceStatus("checkpoint", this.options.getCheckpointMaintenanceStatus(), updatedAt);
+        const recordMaintenance = evaluateMaintenanceStatus("records", this.options.getRuntimeRecordMaintenanceStatus(), updatedAt);
+        let runtimeCheck = buildCheck("healthy", updatedAt, "runtime loops healthy");
+        const runtimeSymptoms = [...runtimeMaintenance.symptoms, ...recordMaintenance.symptoms];
+        runtimeCheck = buildCheck(maxStatus(runtimeMaintenance.check.status, recordMaintenance.check.status), updatedAt, runtimeSymptoms.length > 0 ? runtimeSymptoms.map((symptom) => symptom.message).join("; ") : "runtime loops healthy");
+        const llmCheck = this.evaluateLlmCheck(updatedAt, nowMs);
+        const persistenceCheck = this.evaluatePersistenceCheck(updatedAt, runtimeDbBytes, runtimeSymptoms);
+        const capacityCheck = this.evaluateCapacityCheck(updatedAt, checkpointBytes, runtimeDbBytes, artifactBytes);
+        const workloadCheck = this.evaluateWorkloadCheck(updatedAt, pendingApprovals, stuckRuns);
+        const symptoms = [
+            ...runtimeSymptoms,
+            ...this.capacitySymptoms(capacityCheck, checkpointBytes, runtimeDbBytes, artifactBytes, updatedAt),
+            ...this.workloadSymptoms(workloadCheck, pendingApprovals, stuckRuns, updatedAt),
+            ...this.llmSymptoms(llmCheck, updatedAt),
+            ...(persistenceCheck.status === "healthy"
+                ? []
+                : [createSymptom("persistence.runtime-db", persistenceCheck.status, persistenceCheck.reason ?? "runtime persistence degraded", updatedAt)]),
+        ];
+        const checks = {
+            runtime: runtimeCheck,
+            llm: llmCheck,
+            persistence: persistenceCheck,
+            capacity: capacityCheck,
+            workload: workloadCheck,
+        };
+        let overallStatus = "healthy";
+        for (const check of Object.values(checks)) {
+            overallStatus = maxStatus(overallStatus, check.status);
+        }
+        const snapshot = {
+            status: overallStatus,
+            updatedAt,
+            checks,
+            symptoms,
+            stats: {
+                activeRunSlots: this.options.getActiveRunSlots(),
+                pendingRunSlots: this.options.getPendingRunSlots(),
+                pendingApprovals,
+                stuckRuns,
+                checkpointBytes,
+                runtimeDbBytes,
+                artifactBytes,
+                ...this.llmStats(nowMs),
+            },
+        };
+        const previous = this.latestSnapshot;
+        this.latestSnapshot = snapshot;
+        if (this.config.enabled &&
+            this.config.emitEvents &&
+            this.options.publishEvent &&
+            previous &&
+            !compareStatus(previous.status, snapshot.status)) {
+            await this.options.publishEvent({
+                previousStatus: previous.status,
+                status: snapshot.status,
+                checks: snapshot.checks,
+                stats: snapshot.stats,
+            });
+        }
+        return snapshot;
+    }
+    llmStats(nowMs) {
+        const oneMinuteWindowMs = 60 * 1000;
+        const recent = this.llmSamples.filter((sample) => nowMs - sample.timestampMs <= oneMinuteWindowMs);
+        const successes = recent.filter((sample) => sample.success).length;
+        const latencies = recent.map((sample) => sample.latencyMs);
+        return {
+            llmSuccessRate1m: recent.length > 0 ? successes / recent.length : undefined,
+            llmP95LatencyMs1m: computeP95(latencies),
+        };
+    }
+    evaluateLlmCheck(updatedAt, nowMs) {
+        const errorWindow = this.config.thresholds.llmErrorRate.windowSeconds * 1000;
+        const latencyWindow = this.config.thresholds.llmP95LatencyMs.windowSeconds * 1000;
+        const errorSamples = this.llmSamples.filter((sample) => nowMs - sample.timestampMs <= errorWindow);
+        const latencySamples = this.llmSamples.filter((sample) => nowMs - sample.timestampMs <= latencyWindow);
+        if (errorSamples.length === 0 && latencySamples.length === 0) {
+            return buildCheck("healthy", updatedAt, "no recent llm invocations");
+        }
+        const errors = errorSamples.filter((sample) => !sample.success).length;
+        const errorRate = errorSamples.length > 0 ? errors / errorSamples.length : 0;
+        const p95Latency = computeP95(latencySamples.map((sample) => sample.latencyMs));
+        let status = "healthy";
+        if (errorRate >= this.config.thresholds.llmErrorRate.unhealthyAbove) {
+            status = "unhealthy";
+        }
+        else if (errorRate >= this.config.thresholds.llmErrorRate.degradedAbove) {
+            status = "degraded";
+        }
+        if (p95Latency !== undefined) {
+            const latencyStatus = describeThresholdStatus(p95Latency, this.config.thresholds.llmP95LatencyMs);
+            status = maxStatus(status, latencyStatus);
+        }
+        return buildCheck(status, updatedAt, `errorRate=${errorRate.toFixed(2)}${p95Latency !== undefined ? ` p95LatencyMs=${Math.round(p95Latency)}` : ""}`);
+    }
+    llmSymptoms(check, updatedAt) {
+        if (check.status === "healthy") {
+            return [];
+        }
+        return [createSymptom("llm.invocations", check.status, check.reason ?? "llm health degraded", updatedAt)];
+    }
+    evaluatePersistenceCheck(updatedAt, runtimeDbBytes, runtimeSymptoms) {
+        if (runtimeSymptoms.length > 0) {
+            return buildCheck("degraded", updatedAt, runtimeSymptoms.map((symptom) => symptom.message).join("; "));
+        }
+        if (runtimeDbBytes === 0) {
+            return buildCheck("healthy", updatedAt, "runtime sqlite not materialized yet");
+        }
+        return buildCheck("healthy", updatedAt, "runtime sqlite accessible");
+    }
+    evaluateCapacityCheck(updatedAt, checkpointBytes, runtimeDbBytes, artifactBytes) {
+        let status = "healthy";
+        status = maxStatus(status, describeThresholdStatus(checkpointBytes, this.config.thresholds.checkpointBytes));
+        status = maxStatus(status, describeThresholdStatus(runtimeDbBytes, this.config.thresholds.runtimeDbBytes));
+        status = maxStatus(status, describeThresholdStatus(artifactBytes, this.config.thresholds.artifactBytes));
+        return buildCheck(status, updatedAt, `checkpointBytes=${checkpointBytes} runtimeDbBytes=${runtimeDbBytes} artifactBytes=${artifactBytes}`);
+    }
+    capacitySymptoms(check, checkpointBytes, runtimeDbBytes, artifactBytes, updatedAt) {
+        if (check.status === "healthy") {
+            return [];
+        }
+        const symptoms = [];
+        if (describeThresholdStatus(checkpointBytes, this.config.thresholds.checkpointBytes) !== "healthy") {
+            symptoms.push(createSymptom("capacity.checkpoints", describeThresholdStatus(checkpointBytes, this.config.thresholds.checkpointBytes), `checkpointBytes=${checkpointBytes}`, updatedAt));
+        }
+        if (describeThresholdStatus(runtimeDbBytes, this.config.thresholds.runtimeDbBytes) !== "healthy") {
+            symptoms.push(createSymptom("capacity.runtime-db", describeThresholdStatus(runtimeDbBytes, this.config.thresholds.runtimeDbBytes), `runtimeDbBytes=${runtimeDbBytes}`, updatedAt));
+        }
+        if (describeThresholdStatus(artifactBytes, this.config.thresholds.artifactBytes) !== "healthy") {
+            symptoms.push(createSymptom("capacity.artifacts", describeThresholdStatus(artifactBytes, this.config.thresholds.artifactBytes), `artifactBytes=${artifactBytes}`, updatedAt));
+        }
+        return symptoms;
+    }
+    evaluateWorkloadCheck(updatedAt, pendingApprovals, stuckRuns) {
+        let status = describeThresholdStatus(pendingApprovals, this.config.thresholds.pendingApprovals);
+        status = maxStatus(status, describeThresholdStatus(stuckRuns, {
+            degradedAbove: this.config.thresholds.stuckRunSeconds.degradedAbove > 0 ? 1 : Number.MAX_SAFE_INTEGER,
+            unhealthyAbove: this.config.thresholds.stuckRunSeconds.unhealthyAbove > 0 ? 1 : Number.MAX_SAFE_INTEGER,
+        }));
+        return buildCheck(status, updatedAt, `pendingApprovals=${pendingApprovals} stuckRuns=${stuckRuns}`);
+    }
+    workloadSymptoms(check, pendingApprovals, stuckRuns, updatedAt) {
+        if (check.status === "healthy") {
+            return [];
+        }
+        const symptoms = [];
+        if (describeThresholdStatus(pendingApprovals, this.config.thresholds.pendingApprovals) !== "healthy") {
+            symptoms.push(createSymptom("workload.pending-approvals", describeThresholdStatus(pendingApprovals, this.config.thresholds.pendingApprovals), `pendingApprovals=${pendingApprovals}`, updatedAt));
+        }
+        if (stuckRuns > 0) {
+            const stuckStatus = stuckRuns >= 1 && this.config.thresholds.stuckRunSeconds.unhealthyAbove <= this.config.thresholds.stuckRunSeconds.degradedAbove
+                ? "unhealthy"
+                : "degraded";
+            symptoms.push(createSymptom("workload.stuck-runs", stuckStatus, `stuckRuns=${stuckRuns}`, updatedAt));
+        }
+        return symptoms;
+    }
+    countStuckRuns(runs, nowMs) {
+        return runs.filter((run) => {
+            if (!["running", "resuming", "queued"].includes(run.state)) {
+                return false;
+            }
+            const updatedAtMs = Date.parse(run.updatedAt);
+            if (!Number.isFinite(updatedAtMs)) {
+                return false;
+            }
+            const ageSeconds = (nowMs - updatedAtMs) / 1000;
+            return ageSeconds >= this.config.thresholds.stuckRunSeconds.degradedAbove;
+        }).length;
+    }
+    async sumRuntimeDbBytes() {
+        const sizes = await Promise.all(this.runRoots.map((runRoot) => safeFileSize(path.join(runRoot, "runtime.sqlite"))));
+        return sizes.reduce((sum, value) => sum + value, 0);
+    }
+    async sumCheckpointDbBytes() {
+        const sizes = await Promise.all(this.checkpointDbPaths.map((dbPath) => safeFileSize(dbPath)));
+        return sizes.reduce((sum, value) => sum + value, 0);
+    }
+    async sumArtifactBytes() {
+        const sizes = await Promise.all(this.runRoots.map((runRoot) => directorySize(path.join(runRoot, "threads"))));
+        return sizes.reduce((sum, value) => sum + value, 0);
+    }
+}

package/dist/runtime/runtime-record-maintenance.d.ts

@@ -0,0 +1,43 @@
+import type { WorkspaceBundle } from "../contracts/types.js";
+type RuntimeRecordMaintenanceConfig = {
+    enabled: boolean;
+    schedule: {
+        intervalSeconds: number;
+        runOnStartup: boolean;
+    };
+    policies: {
+        maxAgeSeconds?: number;
+    };
+    sqlite: {
+        sweepBatchSize: number;
+        vacuum: boolean;
+    };
+};
+type RuntimeRecordMaintenanceTarget = {
+    dbPath: string;
+};
+export type RuntimeRecordMaintenanceLoopStatus = {
+    lastStartedAt?: string;
+    lastCompletedAt?: string;
+    lastFailedAt?: string;
+    consecutiveFailures: number;
+    lastError?: string;
+};
+export declare function readRuntimeRecordMaintenanceConfig(workspace: WorkspaceBundle): RuntimeRecordMaintenanceConfig | null;
+export declare function discoverRuntimeRecordMaintenanceTargets(workspace: WorkspaceBundle): RuntimeRecordMaintenanceTarget[];
+export declare function maintainSqliteRuntimeRecords(dbPath: string, config: RuntimeRecordMaintenanceConfig, nowMs?: number): Promise<{
+    deletedThreadCount: number;
+}>;
+export declare class RuntimeRecordMaintenanceLoop {
+    private readonly targets;
+    private readonly config;
+    private timer;
+    private running;
+    private status;
+    constructor(targets: RuntimeRecordMaintenanceTarget[], config: RuntimeRecordMaintenanceConfig);
+    runOnce(nowMs?: number): Promise<void>;
+    getStatus(): RuntimeRecordMaintenanceLoopStatus;
+    start(): Promise<void>;
+    stop(): Promise<void>;
+}
+export {};

package/dist/runtime/runtime-record-maintenance.js

@@ -0,0 +1,169 @@
+import path from "node:path";
+import { rm } from "node:fs/promises";
+import { createClient } from "@libsql/client";
+import { fileExists } from "../utils/fs.js";
+import { getRuntimeDefaults } from "../workspace/support/workspace-ref-utils.js";
+function asObject(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value) ? value : undefined;
+}
+function readPositiveNumber(value, label, allowUndefined = true) {
+    if (value === undefined && allowUndefined) {
+        return undefined;
+    }
+    if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) {
+        throw new Error(`${label} must be a positive number`);
+    }
+    return value;
+}
+export function readRuntimeRecordMaintenanceConfig(workspace) {
+    const runtimeDefaults = getRuntimeDefaults(workspace.refs);
+    const maintenance = asObject(runtimeDefaults?.maintenance);
+    const records = asObject(maintenance?.records);
+    if (!records || records.enabled !== true) {
+        return null;
+    }
+    const schedule = asObject(records.schedule);
+    const policies = asObject(records.policies);
+    const sqlite = asObject(records.sqlite);
+    const config = {
+        enabled: true,
+        schedule: {
+            intervalSeconds: readPositiveNumber(schedule?.intervalSeconds, "runtime.maintenance.records.schedule.intervalSeconds") ?? 3600,
+            runOnStartup: schedule?.runOnStartup !== false,
+        },
+        policies: {
+            maxAgeSeconds: readPositiveNumber(policies?.maxAgeSeconds, "runtime.maintenance.records.policies.maxAgeSeconds"),
+        },
+        sqlite: {
+            sweepBatchSize: readPositiveNumber(sqlite?.sweepBatchSize, "runtime.maintenance.records.sqlite.sweepBatchSize") ?? 100,
+            vacuum: sqlite?.vacuum === true,
+        },
+    };
+    if (config.policies.maxAgeSeconds === undefined) {
+        throw new Error("runtime.maintenance.records.enabled requires at least one cleanup policy");
+    }
+    return config;
+}
+export function discoverRuntimeRecordMaintenanceTargets(workspace) {
+    const runRoots = new Set();
+    for (const binding of workspace.bindings.values()) {
+        runRoots.add(binding.harnessRuntime.runRoot);
+    }
+    return Array.from(runRoots.values()).map((runRoot) => ({
+        dbPath: path.join(runRoot, "runtime.sqlite"),
+    }));
+}
+export async function maintainSqliteRuntimeRecords(dbPath, config, nowMs = Date.now()) {
+    if (!(await fileExists(dbPath))) {
+        return { deletedThreadCount: 0 };
+    }
+    const client = createClient({ url: `file:${dbPath}` });
+    const cutoffIso = new Date(nowMs - (config.policies.maxAgeSeconds ?? 0) * 1000).toISOString();
+    const runRoot = path.dirname(dbPath);
+    const result = await client.execute({
+        sql: `SELECT thread_id
+          FROM threads
+          WHERE status IN ('completed', 'failed')
+            AND updated_at <= ?
+          ORDER BY updated_at ASC, thread_id ASC
+          LIMIT ?`,
+        args: [cutoffIso, config.sqlite.sweepBatchSize],
+    });
+    const rows = result.rows.map((row) => row);
+    if (rows.length === 0) {
+        return { deletedThreadCount: 0 };
+    }
+    let deletedThreadCount = 0;
+    for (const row of rows) {
+        const threadId = row.thread_id;
+        const artifacts = await client.execute({
+            sql: `SELECT run_id, path FROM artifacts WHERE thread_id = ?`,
+            args: [threadId],
+        });
+        for (const artifact of artifacts.rows) {
+            const runId = String(artifact.run_id ?? "");
+            const artifactPath = String(artifact.path ?? "");
+            if (runId && artifactPath) {
+                await rm(path.join(runRoot, "threads", threadId, "runs", runId, artifactPath), { force: true });
+            }
+        }
+        await client.batch([
+            "DELETE FROM artifacts WHERE thread_id = ?",
+            "DELETE FROM approvals WHERE thread_id = ?",
+            "DELETE FROM events WHERE thread_id = ?",
+            "DELETE FROM run_requests WHERE thread_id = ?",
+            "DELETE FROM recovery_intents WHERE thread_id = ?",
+            "DELETE FROM thread_messages WHERE thread_id = ?",
+            "DELETE FROM runs WHERE thread_id = ?",
+            "DELETE FROM threads WHERE thread_id = ?",
+        ].map((sql) => ({ sql, args: [threadId] })), "write");
+        await rm(path.join(runRoot, "threads", threadId), { recursive: true, force: true });
+        deletedThreadCount += 1;
+    }
+    if (deletedThreadCount > 0 && config.sqlite.vacuum) {
+        await client.execute("VACUUM");
+    }
+    return { deletedThreadCount };
+}
+export class RuntimeRecordMaintenanceLoop {
+    targets;
+    config;
+    timer = null;
+    running = false;
+    status = {
+        consecutiveFailures: 0,
+    };
+    constructor(targets, config) {
+        this.targets = targets;
+        this.config = config;
+    }
+    async runOnce(nowMs = Date.now()) {
+        this.status = {
+            ...this.status,
+            lastStartedAt: new Date(nowMs).toISOString(),
+        };
+        try {
+            for (const target of this.targets) {
+                await maintainSqliteRuntimeRecords(target.dbPath, this.config, nowMs);
+            }
+            this.status = {
+                ...this.status,
+                lastCompletedAt: new Date(nowMs).toISOString(),
+                consecutiveFailures: 0,
+                lastError: undefined,
+            };
+        }
+        catch (error) {
+            this.status = {
+                ...this.status,
+                lastFailedAt: new Date(nowMs).toISOString(),
+                consecutiveFailures: this.status.consecutiveFailures + 1,
+                lastError: error instanceof Error ? error.message : String(error),
+            };
+            throw error;
+        }
+    }
+    getStatus() {
+        return { ...this.status };
+    }
+    async start() {
+        if (this.running) {
+            return;
+        }
+        this.running = true;
+        if (this.config.schedule.runOnStartup) {
+            await this.runOnce();
+        }
+        this.timer = setInterval(() => {
+            void this.runOnce();
+        }, this.config.schedule.intervalSeconds * 1000);
+        this.timer.unref?.();
+    }
+    async stop() {
+        if (this.timer) {
+            clearInterval(this.timer);
+            this.timer = null;
+        }
+        this.running = false;
+    }
+}

package/dist/runtime/store.d.ts

@@ -24,7 +24,9 @@ export declare class FileBackedStore {
     private readonly filePath;
     private readonly delegate;
     private loaded;
+    private operationChain;
     constructor(filePath: string);
+    private runSerialized;
     private ensureLoaded;
     private persist;
     batch(operations: Parameters<InMemoryStore["batch"]>[0]): Promise<readonly unknown[]>;