arisa 5.1.68 → 5.2.17

package/test/weighted-resource-governor.test.js

@@ -12,11 +12,24 @@ function deferred() {
   return { promise, resolve };
 }
 
+const safeMemoryPressure = async () => ({
+  availableBytes: 512 * 1024 * 1024,
+  totalBytes: 4 * 1024 * 1024 * 1024,
+  workerRssBytes: 100 * 1024 * 1024,
+  swapTotalBytes: 1024,
+  swapUsedPercent: 50
+});
+
+const settle = () => new Promise((resolve) => setImmediate(resolve));
+
 test("normalizes manifest weights and configurable class capacities", () => {
   assert.deepEqual(normalizeToolExecution({ resourceClass: "browser", weight: 2 }), {
     resourceClass: "browser",
     weight: 2,
-    deduplicateConcurrent: false
+    deduplicateConcurrent: false,
+    maxHeapMb: 4096,
+    maxMemoryMb: 16_384,
+    maxOutputBytes: 1_048_576
   });
   assert.equal(normalizeToolExecution({
     resourceClass: "orchestrator",
@@ -33,6 +46,16 @@ test("normalizes manifest weights and configurable class capacities", () => {
   }), {
     defaultCapacity: 3,
     maxQueuedPerClass: 12,
+    maxWorkerRssMb: 384,
+    maxSwapUsedPercent: 95,
+    initialToolMemoryMb: 384,
+    minimumToolMemoryMb: 128,
+    maximumToolMemoryMb: 4096,
+    systemReserveMb: 128,
+    coreReserveMb: 384,
+    toolHeapPercent: 65,
+    toolMemoryHighPercent: 85,
+    toolSwapMaxMb: 128,
     capacities: { browser: 4 }
   });
 });
@@ -42,13 +65,15 @@ test("queues weighted work fairly within one resource class", async () => {
   const governor = new WeightedResourceGovernor({
     policy: { defaultCapacity: 2 },
     now: () => time,
-    memoryUsage: () => ({ rss: 100 * 1024 * 1024 })
+    memoryUsage: () => ({ rss: 100 * 1024 * 1024 }),
+    memoryPressure: safeMemoryPressure
   });
   const execution = { resourceClass: "browser", weight: 1 };
   const first = await governor.acquire(execution, "first");
   const second = await governor.acquire(execution, "second");
   const thirdLease = governor.acquire(execution, "third");
   const fourthLease = governor.acquire(execution, "fourth");
+  await settle();
 
   assert.deepEqual(governor.snapshot().resources.browser, {
     capacity: 2,
@@ -72,26 +97,80 @@ test("queues weighted work fairly within one resource class", async () => {
   assert.equal(governor.snapshot().resources.browser.activeWeight, 0);
 });
 
-test("undeclared lightweight work bypasses constrained resource queues", async () => {
-  const governor = new WeightedResourceGovernor({ policy: { defaultCapacity: 1 } });
+test("undeclared work bypasses the optional resource governor", async () => {
+  const governor = new WeightedResourceGovernor({ policy: { defaultCapacity: 1 }, memoryPressure: safeMemoryPressure });
   const heavy = await governor.acquire({ resourceClass: "browser", weight: 1 }, "heavy");
   const queued = governor.acquire({ resourceClass: "browser", weight: 1 }, "queued");
+  await settle();
   const light = await governor.acquire(null, "light");
   assert.equal(light.waitedMs, 0);
+  assert.equal(light.memoryLimitMb, undefined);
+  assert.equal(light.heapLimitMb, undefined);
   assert.equal(governor.snapshot().resources.browser.queued, 1);
   light.release();
   heavy.release();
   (await queued).release();
 });
 
+test("rejects declared heavy tools before spawn when memory pressure is unsafe", async () => {
+  const logs = [];
+  const governor = new WeightedResourceGovernor({
+    policy: { maxWorkerRssMb: 384, maxSwapUsedPercent: 95 },
+    memoryPressure: async () => ({
+      availableBytes: 80 * 1024 * 1024,
+      workerRssBytes: 400 * 1024 * 1024,
+      swapTotalBytes: 1024,
+      swapUsedPercent: 50
+    }),
+    logger: { log: (...parts) => logs.push(parts.join(" ")) }
+  });
+
+  await assert.rejects(
+    () => governor.acquire({ resourceClass: "browser", weight: 1 }, "web-browser"),
+    (error) => error.code === "TOOL_RESOURCE_PRESSURE" && /was not started/.test(error.message)
+  );
+  assert.match(logs.join("\n"), /web-browser rejected for browser/);
+  assert.equal(governor.snapshot().resources.browser.activeWeight, 0);
+});
+
+test("ready daemon jobs bypass only worker RSS spawn admission", async () => {
+  const governor = new WeightedResourceGovernor({
+    policy: { maxWorkerRssMb: 384, maxSwapUsedPercent: 95 },
+    memoryPressure: async () => ({
+      availableBytes: 512 * 1024 * 1024,
+      totalBytes: 4 * 1024 * 1024 * 1024,
+      workerRssBytes: 450 * 1024 * 1024,
+      swapTotalBytes: 100,
+      swapUsedPercent: 50
+    })
+  });
+  const lease = await governor.acquire({ resourceClass: "browser", weight: 1 }, "ready-daemon", { ignoreWorkerRss: true });
+  assert.equal(lease.memoryLimitMb, 384);
+  lease.release({ success: true });
+
+  governor.memoryPressure = async () => ({
+    availableBytes: 512 * 1024 * 1024,
+    totalBytes: 4 * 1024 * 1024 * 1024,
+    workerRssBytes: 450 * 1024 * 1024,
+    swapTotalBytes: 100,
+    swapUsedPercent: 96
+  });
+  await assert.rejects(
+    () => governor.acquire({ resourceClass: "browser", weight: 1 }, "ready-daemon", { ignoreWorkerRss: true }),
+    /swap use/
+  );
+});
+
 test("larger weights consume shared capacity and worker RSS peaks are retained", async () => {
   let rss = 120 * 1024 * 1024;
   const governor = new WeightedResourceGovernor({
     policy: { defaultCapacity: 3 },
-    memoryUsage: () => ({ rss })
+    memoryUsage: () => ({ rss }),
+    memoryPressure: safeMemoryPressure
   });
   const large = await governor.acquire({ resourceClass: "browser", weight: 2 }, "large");
   const waiting = governor.acquire({ resourceClass: "browser", weight: 2 }, "waiting");
+  await settle();
   assert.equal(governor.snapshot().resources.browser.queued, 1);
   rss = 180 * 1024 * 1024;
   large.release();
@@ -99,3 +178,44 @@ test("larger weights consume shared capacity and worker RSS peaks are retained",
   assert.equal(governor.snapshot().peakRssBytes, rss);
   next.release();
 });
+
+test("shares one host memory budget across independent resource classes", async () => {
+  const pressure = async () => ({
+    availableBytes: 512 * 1024 * 1024,
+    totalBytes: 1024 * 1024 * 1024,
+    workerRssBytes: 100 * 1024 * 1024,
+    swapTotalBytes: 0,
+    swapUsedPercent: 0
+  });
+  const governor = new WeightedResourceGovernor({
+    policy: { systemReserveMb: 128, coreReserveMb: 384, initialToolMemoryMb: 384 },
+    memoryPressure: pressure
+  });
+  const browser = await governor.acquire({ resourceClass: "browser" }, "browser");
+  const queued = governor.acquire({ resourceClass: "orchestrator" }, "orchestrator");
+  await settle();
+
+  assert.equal(governor.snapshot().memory.budgetMb, 512);
+  assert.equal(governor.snapshot().memory.activeMb, 384);
+  assert.equal(governor.snapshot().resources.orchestrator.queued, 1);
+
+  browser.release({ success: true });
+  const orchestrator = await queued;
+  assert.equal(orchestrator.memoryLimitMb, 384);
+  orchestrator.release({ success: true });
+});
+
+test("raises a tool memory recommendation after an isolated limit failure", async () => {
+  const governor = new WeightedResourceGovernor({
+    policy: { initialToolMemoryMb: 256 },
+    memoryPressure: safeMemoryPressure
+  });
+  const first = await governor.acquire({}, "growing-tool");
+  assert.equal(first.memoryLimitMb, 256);
+  first.release({ memoryLimited: true });
+
+  const second = await governor.acquire({}, "growing-tool");
+  assert.equal(second.memoryLimitMb, 384);
+  assert.equal(governor.snapshot().memory.profiles["growing-tool"].memoryLimitFailures, 1);
+  second.release({ success: true });
+});

package/test/worker-heap-circuit-breaker.test.js

@@ -0,0 +1,79 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import {
+  readWorkerHeapPressure,
+  WorkerHeapCircuitBreaker,
+  WorkerHeapPressureError
+} from "../src/core/agent/worker-heap-circuit-breaker.js";
+
+function pressure(percent) {
+  return { heapUsed: percent, heapLimit: 100, percent };
+}
+
+test("reports bounded worker heap pressure", () => {
+  assert.deepEqual(readWorkerHeapPressure({
+    memoryUsage: () => ({ heapUsed: 40 }),
+    heapStatistics: () => ({ heap_size_limit: 100 })
+  }), pressure(40));
+});
+
+test("evicts inactive sessions under soft pressure and admits recovered work", async () => {
+  const samples = [pressure(75), pressure(60)];
+  let evictions = 0;
+  const breaker = new WorkerHeapCircuitBreaker({
+    lifecycle: { async evictInactive() { evictions += 1; return [{ sessionKey: "idle" }]; } },
+    measure: () => samples.shift() || pressure(60)
+  });
+
+  assert.deepEqual(await breaker.admit(), pressure(60));
+  assert.equal(evictions, 1);
+  assert.deepEqual(breaker.getDiagnostic(), {
+    enabled: true,
+    softPercent: 70,
+    criticalPercent: 82,
+    waitMs: 15_000,
+    pollMs: 500,
+    heapUsed: 60,
+    heapLimit: 100,
+    currentHeapPercent: 60,
+    pressureEvents: 1,
+    evictedSessions: 1,
+    delayedAdmissions: 0,
+    rejectedAdmissions: 0,
+    peakHeapPercent: 75
+  });
+});
+
+test("delays critical admissions until active work releases memory", async () => {
+  const samples = [pressure(90), pressure(89), pressure(78)];
+  let now = 0;
+  const breaker = new WorkerHeapCircuitBreaker({
+    lifecycle: { async evictInactive() { return []; } },
+    config: { waitMs: 1_000, pollMs: 100 },
+    measure: () => samples.shift() || pressure(78),
+    now: () => now,
+    sleep: async (ms) => { now += ms; }
+  });
+
+  assert.deepEqual(await breaker.admit(), pressure(78));
+  assert.equal(breaker.getDiagnostic().delayedAdmissions, 1);
+});
+
+test("rejects persistent critical pressure with a retryable error", async () => {
+  let now = 0;
+  const breaker = new WorkerHeapCircuitBreaker({
+    lifecycle: { async evictInactive() { return []; } },
+    config: { waitMs: 200, pollMs: 100 },
+    measure: () => pressure(90),
+    now: () => now,
+    sleep: async (ms) => { now += ms; }
+  });
+
+  await assert.rejects(
+    breaker.admit(),
+    (error) => error instanceof WorkerHeapPressureError
+      && error.code === "WORKER_HEAP_PRESSURE"
+      && error.retryable === true
+  );
+  assert.equal(breaker.getDiagnostic().rejectedAdmissions, 1);
+});

package/test/worker-recovery-report.test.js

@@ -0,0 +1,69 @@
+import assert from "node:assert/strict";
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+import {
+  consumeWorkerRecoveryReport,
+  loadWorkerRecoveryReport,
+  recordUnexpectedWorkerExit,
+  summarizeRecoveryEvidence
+} from "../src/runtime/worker-recovery-report.js";
+
+function localLogTimestamp(date) {
+  const two = (value) => String(value).padStart(2, "0");
+  return `${date.getFullYear()}-${two(date.getMonth() + 1)}-${two(date.getDate())} ${two(date.getHours())}:${two(date.getMinutes())}:${two(date.getSeconds())}`;
+}
+
+test("summarizes only bounded crash evidence", () => {
+  const evidence = summarizeRecoveryEvidence([
+    "FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory",
+    "[agent] run_tool web-browser",
+    "[agent] run_tool web-browser",
+    "[agent] run_tool campaign-draft-runner"
+  ], { occurredAt: "invalid", signal: "SIGABRT" });
+
+  assert.equal(evidence.cause, "JavaScript heap out of memory");
+  assert.deepEqual(evidence.tools, [["web-browser", 2], ["campaign-draft-runner", 1]]);
+});
+
+test("persists, formats, and consumes one automatic recovery report", async () => {
+  const directory = await mkdtemp(path.join(os.tmpdir(), "arisa-worker-recovery-"));
+  const reportFile = path.join(directory, "report.json");
+  const taskFile = path.join(directory, "tasks.json");
+  const occurredAt = new Date();
+  const report = await recordUnexpectedWorkerExit({
+    occurredAt: occurredAt.toISOString(),
+    runtimeMs: 10_000,
+    restartDelayMs: 2_000,
+    consecutiveFailures: 1,
+    code: null,
+    signal: "SIGABRT",
+    detail: "signal=SIGABRT"
+  }, { reportFile });
+  await writeFile(taskFile, `${JSON.stringify({ tasks: [{
+    updatedAt: occurredAt.toISOString(),
+    lastOutcome: "outcome_uncertain",
+    lastError: "execution interrupted before confirmation"
+  }] })}\n`);
+  const timestamp = localLogTimestamp(occurredAt);
+  const recovery = await loadWorkerRecoveryReport({
+    reportFile,
+    taskFile,
+    readLines: async () => ({ text: [
+      `[${timestamp}] [agent] run_tool web-browser`,
+      `[${timestamp}] [agent] run_tool web-browser`,
+      "FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory",
+      `[${timestamp}] [service] worker exited unexpectedly (signal=SIGABRT)`
+    ].join("\n") }),
+    getVersion: async () => "5.1.70"
+  });
+
+  assert.match(recovery.text, /Cause: JavaScript heap out of memory/);
+  assert.match(recovery.text, /web-browser ×2/);
+  assert.match(recovery.text, /1 scheduled execution was marked outcome-uncertain and not replayed/);
+  assert.match(recovery.text, /restarted after 2s; Arisa 5\.1\.70 is running/);
+  assert.equal(await consumeWorkerRecoveryReport(report.id, { reportFile }), true);
+  await assert.rejects(() => readFile(reportFile), /ENOENT/);
+  await rm(directory, { recursive: true, force: true });
+});

package/test/worker-tool-fanout.test.js

@@ -0,0 +1,79 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { WorkerToolFanoutController } from "../src/core/agent/worker-tool-fanout.js";
+
+function deferred() {
+  let resolve;
+  const promise = new Promise((done) => { resolve = done; });
+  return { promise, resolve };
+}
+
+function breaker(percent = 20) {
+  return {
+    sample: () => ({ heapUsed: percent, heapLimit: 100, percent }),
+    admit: async () => ({ heapUsed: percent, heapLimit: 100, percent })
+  };
+}
+
+test("admits low-pressure tool calls in pairs", async () => {
+  const controller = new WorkerToolFanoutController({ heapCircuitBreaker: breaker() });
+  const gates = Array.from({ length: 4 }, deferred);
+  let active = 0;
+  let peak = 0;
+  const runs = gates.map((gate) => controller.run(async () => {
+    active += 1;
+    peak = Math.max(peak, active);
+    await gate.promise;
+    active -= 1;
+  }));
+
+  await new Promise((resolve) => setImmediate(resolve));
+  assert.equal(active, 2);
+  gates[0].resolve();
+  gates[1].resolve();
+  await new Promise((resolve) => setImmediate(resolve));
+  assert.equal(active, 2);
+  gates[2].resolve();
+  gates[3].resolve();
+  await Promise.all(runs);
+
+  assert.equal(peak, 2);
+  assert.equal(controller.getDiagnostic().peakQueued, 3);
+});
+
+test("serializes tool calls when the worker heap is elevated", async () => {
+  const controller = new WorkerToolFanoutController({ heapCircuitBreaker: breaker(70) });
+  const gates = Array.from({ length: 3 }, deferred);
+  let active = 0;
+  let peak = 0;
+  const runs = gates.map((gate) => controller.run(async () => {
+    active += 1;
+    peak = Math.max(peak, active);
+    await gate.promise;
+    active -= 1;
+  }));
+
+  await new Promise((resolve) => setImmediate(resolve));
+  assert.equal(active, 1);
+  for (const gate of gates) {
+    gate.resolve();
+    await new Promise((resolve) => setImmediate(resolve));
+  }
+  await Promise.all(runs);
+
+  assert.equal(peak, 1);
+  assert.ok(controller.getDiagnostic().pressureSerializations > 0);
+});
+
+test("rejects a queued tool call when heap admission remains critical", async () => {
+  const expected = Object.assign(new Error("critical"), { code: "WORKER_HEAP_PRESSURE" });
+  const controller = new WorkerToolFanoutController({
+    heapCircuitBreaker: {
+      sample: () => ({ heapUsed: 90, heapLimit: 100, percent: 90 }),
+      admit: async () => { throw expected; }
+    }
+  });
+
+  await assert.rejects(controller.run(async () => "unreachable"), expected);
+  assert.equal(controller.getDiagnostic().rejectedAdmissions, 1);
+});

package/test-fixtures/fake-daemon.js

@@ -43,6 +43,11 @@ await runtime.workLoop({
   recover,
   processJob: async (payload, execution) => {
     if (payload.action === "fail") throw new Error("synthetic job failure");
+    if (payload.action === "hang-until-cancelled") {
+      await new Promise((_, reject) => execution.signal.addEventListener("abort", () => {
+        reject(Object.assign(new Error("synthetic job cancelled"), { code: "DAEMON_JOB_CANCELLED" }));
+      }, { once: true }));
+    }
     if (payload.action === "stream") {
       await execution.emit("progress", { percent: 50 });
       await execution.emit("chunk", { text: "partial" });