arisa 5.1.66 → 5.2.7

package/test/weighted-resource-governor.test.js

@@ -12,11 +12,30 @@ function deferred() {
   return { promise, resolve };
 }
 
+const safeMemoryPressure = async () => ({
+  availableBytes: 512 * 1024 * 1024,
+  totalBytes: 4 * 1024 * 1024 * 1024,
+  workerRssBytes: 100 * 1024 * 1024,
+  swapTotalBytes: 1024,
+  swapUsedPercent: 50
+});
+
+const settle = () => new Promise((resolve) => setImmediate(resolve));
+
 test("normalizes manifest weights and configurable class capacities", () => {
   assert.deepEqual(normalizeToolExecution({ resourceClass: "browser", weight: 2 }), {
     resourceClass: "browser",
-    weight: 2
+    weight: 2,
+    deduplicateConcurrent: false,
+    maxHeapMb: 4096,
+    maxMemoryMb: 16_384,
+    maxOutputBytes: 1_048_576
   });
+  assert.equal(normalizeToolExecution({
+    resourceClass: "orchestrator",
+    weight: 1,
+    deduplicateConcurrent: true
+  }).deduplicateConcurrent, true);
   assert.equal(normalizeToolExecution(undefined), null);
   assert.throws(() => normalizeToolExecution({ resourceClass: "Browser!", weight: 1 }), /Invalid tool execution resource class/);
   assert.throws(() => normalizeToolExecution({ resourceClass: "browser", weight: 0 }), /positive integer/);
@@ -27,6 +46,16 @@ test("normalizes manifest weights and configurable class capacities", () => {
   }), {
     defaultCapacity: 3,
     maxQueuedPerClass: 12,
+    maxWorkerRssMb: 384,
+    maxSwapUsedPercent: 95,
+    initialToolMemoryMb: 384,
+    minimumToolMemoryMb: 128,
+    maximumToolMemoryMb: 4096,
+    systemReserveMb: 128,
+    coreReserveMb: 384,
+    toolHeapPercent: 65,
+    toolMemoryHighPercent: 85,
+    toolSwapMaxMb: 128,
     capacities: { browser: 4 }
   });
 });
@@ -36,13 +65,15 @@ test("queues weighted work fairly within one resource class", async () => {
   const governor = new WeightedResourceGovernor({
     policy: { defaultCapacity: 2 },
     now: () => time,
-    memoryUsage: () => ({ rss: 100 * 1024 * 1024 })
+    memoryUsage: () => ({ rss: 100 * 1024 * 1024 }),
+    memoryPressure: safeMemoryPressure
   });
   const execution = { resourceClass: "browser", weight: 1 };
   const first = await governor.acquire(execution, "first");
   const second = await governor.acquire(execution, "second");
   const thirdLease = governor.acquire(execution, "third");
   const fourthLease = governor.acquire(execution, "fourth");
+  await settle();
 
   assert.deepEqual(governor.snapshot().resources.browser, {
     capacity: 2,
@@ -66,26 +97,52 @@ test("queues weighted work fairly within one resource class", async () => {
   assert.equal(governor.snapshot().resources.browser.activeWeight, 0);
 });
 
-test("undeclared lightweight work bypasses constrained resource queues", async () => {
-  const governor = new WeightedResourceGovernor({ policy: { defaultCapacity: 1 } });
+test("undeclared work bypasses the optional resource governor", async () => {
+  const governor = new WeightedResourceGovernor({ policy: { defaultCapacity: 1 }, memoryPressure: safeMemoryPressure });
   const heavy = await governor.acquire({ resourceClass: "browser", weight: 1 }, "heavy");
   const queued = governor.acquire({ resourceClass: "browser", weight: 1 }, "queued");
+  await settle();
   const light = await governor.acquire(null, "light");
   assert.equal(light.waitedMs, 0);
+  assert.equal(light.memoryLimitMb, undefined);
+  assert.equal(light.heapLimitMb, undefined);
   assert.equal(governor.snapshot().resources.browser.queued, 1);
   light.release();
   heavy.release();
   (await queued).release();
 });
 
+test("rejects declared heavy tools before spawn when memory pressure is unsafe", async () => {
+  const logs = [];
+  const governor = new WeightedResourceGovernor({
+    policy: { maxWorkerRssMb: 384, maxSwapUsedPercent: 95 },
+    memoryPressure: async () => ({
+      availableBytes: 80 * 1024 * 1024,
+      workerRssBytes: 400 * 1024 * 1024,
+      swapTotalBytes: 1024,
+      swapUsedPercent: 50
+    }),
+    logger: { log: (...parts) => logs.push(parts.join(" ")) }
+  });
+
+  await assert.rejects(
+    () => governor.acquire({ resourceClass: "browser", weight: 1 }, "web-browser"),
+    (error) => error.code === "TOOL_RESOURCE_PRESSURE" && /was not started/.test(error.message)
+  );
+  assert.match(logs.join("\n"), /web-browser rejected for browser/);
+  assert.equal(governor.snapshot().resources.browser.activeWeight, 0);
+});
+
 test("larger weights consume shared capacity and worker RSS peaks are retained", async () => {
   let rss = 120 * 1024 * 1024;
   const governor = new WeightedResourceGovernor({
     policy: { defaultCapacity: 3 },
-    memoryUsage: () => ({ rss })
+    memoryUsage: () => ({ rss }),
+    memoryPressure: safeMemoryPressure
   });
   const large = await governor.acquire({ resourceClass: "browser", weight: 2 }, "large");
   const waiting = governor.acquire({ resourceClass: "browser", weight: 2 }, "waiting");
+  await settle();
   assert.equal(governor.snapshot().resources.browser.queued, 1);
   rss = 180 * 1024 * 1024;
   large.release();
@@ -93,3 +150,44 @@ test("larger weights consume shared capacity and worker RSS peaks are retained",
   assert.equal(governor.snapshot().peakRssBytes, rss);
   next.release();
 });
+
+test("shares one host memory budget across independent resource classes", async () => {
+  const pressure = async () => ({
+    availableBytes: 512 * 1024 * 1024,
+    totalBytes: 1024 * 1024 * 1024,
+    workerRssBytes: 100 * 1024 * 1024,
+    swapTotalBytes: 0,
+    swapUsedPercent: 0
+  });
+  const governor = new WeightedResourceGovernor({
+    policy: { systemReserveMb: 128, coreReserveMb: 384, initialToolMemoryMb: 384 },
+    memoryPressure: pressure
+  });
+  const browser = await governor.acquire({ resourceClass: "browser" }, "browser");
+  const queued = governor.acquire({ resourceClass: "orchestrator" }, "orchestrator");
+  await settle();
+
+  assert.equal(governor.snapshot().memory.budgetMb, 512);
+  assert.equal(governor.snapshot().memory.activeMb, 384);
+  assert.equal(governor.snapshot().resources.orchestrator.queued, 1);
+
+  browser.release({ success: true });
+  const orchestrator = await queued;
+  assert.equal(orchestrator.memoryLimitMb, 384);
+  orchestrator.release({ success: true });
+});
+
+test("raises a tool memory recommendation after an isolated limit failure", async () => {
+  const governor = new WeightedResourceGovernor({
+    policy: { initialToolMemoryMb: 256 },
+    memoryPressure: safeMemoryPressure
+  });
+  const first = await governor.acquire({}, "growing-tool");
+  assert.equal(first.memoryLimitMb, 256);
+  first.release({ memoryLimited: true });
+
+  const second = await governor.acquire({}, "growing-tool");
+  assert.equal(second.memoryLimitMb, 384);
+  assert.equal(governor.snapshot().memory.profiles["growing-tool"].memoryLimitFailures, 1);
+  second.release({ success: true });
+});

package/test/worker-heap-circuit-breaker.test.js

@@ -0,0 +1,79 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import {
+  readWorkerHeapPressure,
+  WorkerHeapCircuitBreaker,
+  WorkerHeapPressureError
+} from "../src/core/agent/worker-heap-circuit-breaker.js";
+
+function pressure(percent) {
+  return { heapUsed: percent, heapLimit: 100, percent };
+}
+
+test("reports bounded worker heap pressure", () => {
+  assert.deepEqual(readWorkerHeapPressure({
+    memoryUsage: () => ({ heapUsed: 40 }),
+    heapStatistics: () => ({ heap_size_limit: 100 })
+  }), pressure(40));
+});
+
+test("evicts inactive sessions under soft pressure and admits recovered work", async () => {
+  const samples = [pressure(75), pressure(60)];
+  let evictions = 0;
+  const breaker = new WorkerHeapCircuitBreaker({
+    lifecycle: { async evictInactive() { evictions += 1; return [{ sessionKey: "idle" }]; } },
+    measure: () => samples.shift() || pressure(60)
+  });
+
+  assert.deepEqual(await breaker.admit(), pressure(60));
+  assert.equal(evictions, 1);
+  assert.deepEqual(breaker.getDiagnostic(), {
+    enabled: true,
+    softPercent: 70,
+    criticalPercent: 82,
+    waitMs: 15_000,
+    pollMs: 500,
+    heapUsed: 60,
+    heapLimit: 100,
+    currentHeapPercent: 60,
+    pressureEvents: 1,
+    evictedSessions: 1,
+    delayedAdmissions: 0,
+    rejectedAdmissions: 0,
+    peakHeapPercent: 75
+  });
+});
+
+test("delays critical admissions until active work releases memory", async () => {
+  const samples = [pressure(90), pressure(89), pressure(78)];
+  let now = 0;
+  const breaker = new WorkerHeapCircuitBreaker({
+    lifecycle: { async evictInactive() { return []; } },
+    config: { waitMs: 1_000, pollMs: 100 },
+    measure: () => samples.shift() || pressure(78),
+    now: () => now,
+    sleep: async (ms) => { now += ms; }
+  });
+
+  assert.deepEqual(await breaker.admit(), pressure(78));
+  assert.equal(breaker.getDiagnostic().delayedAdmissions, 1);
+});
+
+test("rejects persistent critical pressure with a retryable error", async () => {
+  let now = 0;
+  const breaker = new WorkerHeapCircuitBreaker({
+    lifecycle: { async evictInactive() { return []; } },
+    config: { waitMs: 200, pollMs: 100 },
+    measure: () => pressure(90),
+    now: () => now,
+    sleep: async (ms) => { now += ms; }
+  });
+
+  await assert.rejects(
+    breaker.admit(),
+    (error) => error instanceof WorkerHeapPressureError
+      && error.code === "WORKER_HEAP_PRESSURE"
+      && error.retryable === true
+  );
+  assert.equal(breaker.getDiagnostic().rejectedAdmissions, 1);
+});

package/test/worker-recovery-report.test.js

@@ -0,0 +1,69 @@
+import assert from "node:assert/strict";
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+import {
+  consumeWorkerRecoveryReport,
+  loadWorkerRecoveryReport,
+  recordUnexpectedWorkerExit,
+  summarizeRecoveryEvidence
+} from "../src/runtime/worker-recovery-report.js";
+
+function localLogTimestamp(date) {
+  const two = (value) => String(value).padStart(2, "0");
+  return `${date.getFullYear()}-${two(date.getMonth() + 1)}-${two(date.getDate())} ${two(date.getHours())}:${two(date.getMinutes())}:${two(date.getSeconds())}`;
+}
+
+test("summarizes only bounded crash evidence", () => {
+  const evidence = summarizeRecoveryEvidence([
+    "FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory",
+    "[agent] run_tool web-browser",
+    "[agent] run_tool web-browser",
+    "[agent] run_tool campaign-draft-runner"
+  ], { occurredAt: "invalid", signal: "SIGABRT" });
+
+  assert.equal(evidence.cause, "JavaScript heap out of memory");
+  assert.deepEqual(evidence.tools, [["web-browser", 2], ["campaign-draft-runner", 1]]);
+});
+
+test("persists, formats, and consumes one automatic recovery report", async () => {
+  const directory = await mkdtemp(path.join(os.tmpdir(), "arisa-worker-recovery-"));
+  const reportFile = path.join(directory, "report.json");
+  const taskFile = path.join(directory, "tasks.json");
+  const occurredAt = new Date();
+  const report = await recordUnexpectedWorkerExit({
+    occurredAt: occurredAt.toISOString(),
+    runtimeMs: 10_000,
+    restartDelayMs: 2_000,
+    consecutiveFailures: 1,
+    code: null,
+    signal: "SIGABRT",
+    detail: "signal=SIGABRT"
+  }, { reportFile });
+  await writeFile(taskFile, `${JSON.stringify({ tasks: [{
+    updatedAt: occurredAt.toISOString(),
+    lastOutcome: "outcome_uncertain",
+    lastError: "execution interrupted before confirmation"
+  }] })}\n`);
+  const timestamp = localLogTimestamp(occurredAt);
+  const recovery = await loadWorkerRecoveryReport({
+    reportFile,
+    taskFile,
+    readLines: async () => ({ text: [
+      `[${timestamp}] [agent] run_tool web-browser`,
+      `[${timestamp}] [agent] run_tool web-browser`,
+      "FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory",
+      `[${timestamp}] [service] worker exited unexpectedly (signal=SIGABRT)`
+    ].join("\n") }),
+    getVersion: async () => "5.1.70"
+  });
+
+  assert.match(recovery.text, /Cause: JavaScript heap out of memory/);
+  assert.match(recovery.text, /web-browser ×2/);
+  assert.match(recovery.text, /1 scheduled execution was marked outcome-uncertain and not replayed/);
+  assert.match(recovery.text, /restarted after 2s; Arisa 5\.1\.70 is running/);
+  assert.equal(await consumeWorkerRecoveryReport(report.id, { reportFile }), true);
+  await assert.rejects(() => readFile(reportFile), /ENOENT/);
+  await rm(directory, { recursive: true, force: true });
+});

package/test-fixtures/fake-daemon.js

@@ -43,6 +43,11 @@ await runtime.workLoop({
   recover,
   processJob: async (payload, execution) => {
     if (payload.action === "fail") throw new Error("synthetic job failure");
+    if (payload.action === "hang-until-cancelled") {
+      await new Promise((_, reject) => execution.signal.addEventListener("abort", () => {
+        reject(Object.assign(new Error("synthetic job cancelled"), { code: "DAEMON_JOB_CANCELLED" }));
+      }, { once: true }));
+    }
     if (payload.action === "stream") {
       await execution.emit("progress", { percent: 50 });
       await execution.emit("chunk", { text: "partial" });