@pasko70/pibo 1.11.1 → 1.11.2

package/dist/core/gateway-resource-guard.js

@@ -4,11 +4,13 @@ import { promisify } from "node:util";
 import { getHeapStatistics } from "node:v8";
 const execFileAsync = promisify(execFile);
 const DEFAULT_POLICY = Object.freeze({
-    mode: "warn",
+    mode: "block",
     minFreeMemoryBytes: 256 * 1024 * 1024,
     minHeapAvailableBytes: 64 * 1024 * 1024,
     maxRssBytes: 1536 * 1024 * 1024,
     knownDaemonWarningRssBytes: 2 * 1024 * 1024 * 1024,
+    maxConcurrentYieldedRuns: 1,
+    yieldedRunMemoryReservationBytes: 2 * 1024 * 1024 * 1024,
 });
 export function resolveGatewayResourceGuardPolicy(env = process.env) {
     return {
@@ -17,6 +19,8 @@ export function resolveGatewayResourceGuardPolicy(env = process.env) {
         minHeapAvailableBytes: parseByteThreshold(env.PIBO_GATEWAY_MIN_HEAP_AVAILABLE_BYTES, DEFAULT_POLICY.minHeapAvailableBytes),
         maxRssBytes: parseByteThreshold(env.PIBO_GATEWAY_MAX_RSS_BYTES, DEFAULT_POLICY.maxRssBytes),
         knownDaemonWarningRssBytes: parseByteThreshold(env.PIBO_GATEWAY_KNOWN_DAEMON_WARNING_RSS_BYTES, DEFAULT_POLICY.knownDaemonWarningRssBytes),
+        maxConcurrentYieldedRuns: parsePositiveInteger(env.PIBO_GATEWAY_MAX_CONCURRENT_YIELDED_RUNS, DEFAULT_POLICY.maxConcurrentYieldedRuns),
+        yieldedRunMemoryReservationBytes: parseByteThreshold(env.PIBO_GATEWAY_YIELDED_RUN_MEMORY_RESERVATION_BYTES, DEFAULT_POLICY.yieldedRunMemoryReservationBytes),
     };
 }
 export function collectGatewayProcessMemory() {
@@ -74,8 +78,39 @@ export function assertGatewayResourceAvailableForWork(workLabel, env = process.e
     const snapshot = buildGatewayResourceSnapshot({ env, includeProcesses: false });
     if (snapshot.guardAction !== "block")
         return;
-    const reasons = snapshot.checks.filter((check) => check.severity === "critical").map((check) => check.message).join("; ");
-    throw new Error(`Gateway resource guard blocked ${workLabel} before starting: ${reasons}`);
+    throwGatewayResourceBlock(workLabel, snapshot.checks.filter((check) => check.severity === "critical").map((check) => check.message));
+}
+export class GatewayWorkAdmissionController {
+    activeReservations = new Set();
+    reserve(workLabel, env = process.env) {
+        const snapshot = buildGatewayResourceSnapshot({ env, includeProcesses: false });
+        const policy = snapshot.policy;
+        if (snapshot.guardAction === "block") {
+            throwGatewayResourceBlock(workLabel, snapshot.checks.filter((check) => check.severity === "critical").map((check) => check.message));
+        }
+        if (policy.mode === "block" && this.activeReservations.size >= policy.maxConcurrentYieldedRuns) {
+            throwGatewayResourceBlock(workLabel, [
+                `Active yielded runs ${this.activeReservations.size} reached the configured limit ${policy.maxConcurrentYieldedRuns}. Wait for an active run to settle or raise PIBO_GATEWAY_MAX_CONCURRENT_YIELDED_RUNS explicitly.`,
+            ]);
+        }
+        if (policy.mode === "block" &&
+            snapshot.host.freeBytes < policy.minFreeMemoryBytes + policy.yieldedRunMemoryReservationBytes) {
+            throwGatewayResourceBlock(workLabel, [
+                `Host free memory ${snapshot.host.freeBytes} cannot preserve reserve ${policy.minFreeMemoryBytes} after the yielded-run reservation ${policy.yieldedRunMemoryReservationBytes}.`,
+            ]);
+        }
+        const reservation = Symbol(workLabel);
+        this.activeReservations.add(reservation);
+        let released = false;
+        return {
+            release: () => {
+                if (released)
+                    return;
+                released = true;
+                this.activeReservations.delete(reservation);
+            },
+        };
+    }
 }
 export function parseHostProcessResourceList(output, gatewayPid, policy = DEFAULT_POLICY) {
     const rows = [];
@@ -106,7 +141,7 @@ export function renderGatewayResourceSnapshotText(snapshot) {
     lines.push(`Gateway PID: ${snapshot.gateway.pid}`);
     lines.push(`Gateway memory: rss=${snapshot.gateway.rssBytes} heapUsed=${snapshot.gateway.heapUsedBytes} heapAvailable=${snapshot.gateway.heapAvailableBytes} heapLimit=${snapshot.gateway.heapLimitBytes}`);
     lines.push(`Host memory: free=${snapshot.host.freeBytes} total=${snapshot.host.totalBytes}`);
-    lines.push(`Thresholds: minFree=${snapshot.policy.minFreeMemoryBytes} minHeapAvailable=${snapshot.policy.minHeapAvailableBytes} maxRss=${snapshot.policy.maxRssBytes} daemonWarnRss=${snapshot.policy.knownDaemonWarningRssBytes}`);
+    lines.push(`Thresholds: minFree=${snapshot.policy.minFreeMemoryBytes} minHeapAvailable=${snapshot.policy.minHeapAvailableBytes} maxRss=${snapshot.policy.maxRssBytes} daemonWarnRss=${snapshot.policy.knownDaemonWarningRssBytes} maxYieldedRuns=${snapshot.policy.maxConcurrentYieldedRuns} yieldedRunReservation=${snapshot.policy.yieldedRunMemoryReservationBytes}`);
     lines.push(`Related processes: children=${snapshot.processes.children.length} knownDaemons=${snapshot.processes.knownDaemons.length} processList=${snapshot.processes.available ? "available" : "unavailable"}`);
     if (snapshot.processes.error)
         lines.push(`Process list error: ${snapshot.processes.error}`);
@@ -160,11 +195,13 @@ function processResultFromOptions(gatewayPid, options, policy) {
 }
 function parseMode(value, fallback) {
     const normalized = value?.trim().toLowerCase();
+    if (normalized === undefined || normalized === "")
+        return fallback;
     if (normalized === "off" || normalized === "0" || normalized === "false")
         return "off";
     if (normalized === "block" || normalized === "strict")
         return "block";
-    if (normalized === "warn" || normalized === "1" || normalized === "true" || normalized === undefined || normalized === "")
+    if (normalized === "warn" || normalized === "1" || normalized === "true")
         return "warn";
     return fallback;
 }
@@ -174,6 +211,15 @@ function parseByteThreshold(value, fallback) {
     const parsed = Number(value);
     return Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : fallback;
 }
+function parsePositiveInteger(value, fallback) {
+    if (value === undefined || value.trim() === "")
+        return fallback;
+    const parsed = Number(value);
+    return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
+}
+function throwGatewayResourceBlock(workLabel, reasons) {
+    throw new Error(`Gateway resource guard blocked ${workLabel} before starting: ${reasons.join("; ")}`);
+}
 function knownDaemonLabel(commandName, args) {
     const combined = `${commandName} ${args}`;
     if (/comfyui|main\.py.*--port\s+8188/i.test(combined))

package/dist/core/session-router.js

@@ -16,7 +16,7 @@ import { loadPiboUserSettings } from "./user-settings.js";
 import { resolvePiboSessionActiveModel } from "./session-model.js";
 import { isPiboThinkingLevel } from "./thinking.js";
 import { RuntimeSessionRegistry } from "../tools/runtime/registry.js";
-import { assertGatewayResourceAvailableForWork } from "./gateway-resource-guard.js";
+import { GatewayWorkAdmissionController } from "./gateway-resource-guard.js";
 import { withWorkflowSessionKind } from "../sessions/workflow-session-kind.js";
 import { PiboRuntimeTelemetryRecorder } from "./runtime-telemetry.js";
 import { createPiboProviderTelemetryExtension } from "./provider-telemetry.js";
@@ -143,6 +143,7 @@ export class PiboSessionRouter {
     pendingSessions = new Map();
     listeners = new Set();
     runRegistry;
+    gatewayWorkAdmission = new GatewayWorkAdmissionController();
     signalRegistry;
     runtimeRegistry;
     scheduledRunReminders = new Map();
@@ -697,17 +698,24 @@ export class PiboSessionRouter {
     createRunToolController(parentPiboSessionId) {
         return {
             startToolRun: ({ toolName, params, completionPolicy, retryable, maxAttempts, timeoutMs, serviceWarning, execute }) => {
-                assertGatewayResourceAvailableForWork(`yielded run ${toolName}`);
-                const run = this.runRegistry.startToolRun({
-                    controllerPiboSessionId: parentPiboSessionId,
-                    toolName,
-                    params,
-                    completionPolicy,
-                    retryable,
-                    maxAttempts,
-                    timeoutMs,
-                    serviceWarning,
-                });
+                const admission = this.gatewayWorkAdmission.reserve(`yielded run ${toolName}`);
+                let run;
+                try {
+                    run = this.runRegistry.startToolRun({
+                        controllerPiboSessionId: parentPiboSessionId,
+                        toolName,
+                        params,
+                        completionPolicy,
+                        retryable,
+                        maxAttempts,
+                        timeoutMs,
+                        serviceWarning,
+                    });
+                }
+                catch (error) {
+                    admission.release();
+                    throw error;
+                }
                 void (async () => {
                     try {
                         const result = await execute();
@@ -723,6 +731,9 @@ export class PiboSessionRouter {
                         if (terminalRun)
                             this.scheduleRunReminder(parentPiboSessionId, false);
                     }
+                    finally {
+                        admission.release();
+                    }
                 })();
                 return run;
             },

package/dist/debug/index.js

@@ -1058,11 +1058,13 @@ Reports:
   Gateway RSS/heap headroom, host free-memory reserve, direct child processes, and known heavy local daemons such as ComfyUI or Unity when process listing is available.
 
 Environment:
-  PIBO_GATEWAY_RESOURCE_GUARD=warn|block|off
+  PIBO_GATEWAY_RESOURCE_GUARD=block|warn|off (default: block)
   PIBO_GATEWAY_MIN_FREE_MEMORY_BYTES=<bytes>
   PIBO_GATEWAY_MIN_HEAP_AVAILABLE_BYTES=<bytes>
   PIBO_GATEWAY_MAX_RSS_BYTES=<bytes>
   PIBO_GATEWAY_KNOWN_DAEMON_WARNING_RSS_BYTES=<bytes>
+  PIBO_GATEWAY_MAX_CONCURRENT_YIELDED_RUNS=<count> (default: 1)
+  PIBO_GATEWAY_YIELDED_RUN_MEMORY_RESERVATION_BYTES=<bytes> (default: 2147483648)
 
 Next:
   pibo debug resources --json

package/dist/reliability/store.js

@@ -500,7 +500,7 @@ export class PiboReliabilityStore {
 				) VALUES (?, 'tool', ?, 'running', ?, 0, ?, ?, NULL, NULL, NULL, NULL, ?, ?, NULL, ?, ?, ?, ?, ?, NULL, ?)
 			`)
             .run(runId, input.controllerPiboSessionId, input.completionPolicy, input.toolName, `${input.toolName} run is running.`, timestamp, timestamp, job.jobId, input.retryable ? 1 : 0, maxAttempts, input.timeoutMs ?? null, timeoutAt ?? null, input.serviceWarning ?? null);
-        this.claimJob(job.jobId, `run-registry:${process.pid}`, 24 * 60 * 60 * 1000);
+        this.claimJob(job.jobId, input.workerId ?? `run-registry:${process.pid}`, 24 * 60 * 60 * 1000);
         return this.requireRun(runId);
     }
     updateRun(runId, patch) {
@@ -572,14 +572,27 @@ export class PiboReliabilityStore {
         const result = this.db.prepare(`DELETE FROM pibo_runs WHERE run_id IN (${placeholders})`).run(...ids);
         return Number(result.changes ?? 0);
     }
-    recoverInterruptedRuns() {
+    recoverInterruptedRuns(workerId = `run-registry:${process.pid}`) {
         const rows = this.db.prepare("SELECT * FROM pibo_runs WHERE status = 'running'").all();
         const recovered = [];
         const timestamp = now();
         for (const row of rows) {
-            if (row.job_id && this.hasUnexpiredJobClaim(row.job_id, timestamp))
+            if (row.job_id && this.hasUnexpiredJobClaim(row.job_id, timestamp, workerId))
                 continue;
             const run = runFromRow(row);
+            if (run.timeoutAt && run.timeoutAt <= timestamp) {
+                const error = `Run deadline ${run.timeoutAt} elapsed before the interrupted runtime recovered.`;
+                if (run.jobId)
+                    this.moveLiveJobToDead(run.jobId, error, "timeout", timestamp);
+                recovered.push(this.updateRun(run.runId, {
+                    status: "timed_out",
+                    error,
+                    timeoutPhase: "lifetime",
+                    summary: `${run.toolName} run started successfully, then reached its configured timeout.`,
+                    completedAt: timestamp,
+                }) ?? run);
+                continue;
+            }
             if (run.retryable && run.maxAttempts > 1) {
                 if (run.jobId)
                     this.releaseJobForRetry(run.jobId, timestamp);
@@ -653,16 +666,17 @@ export class PiboReliabilityStore {
         for (const row of expired)
             this.moveJobToDead(row, row.last_error ?? "Job expired.", "expired", timestamp);
     }
-    hasUnexpiredJobClaim(jobId, timestamp) {
+    hasUnexpiredJobClaim(jobId, timestamp, workerId) {
         const row = this.db
             .prepare(`
 				SELECT job_id FROM pibo_jobs
 				WHERE job_id = ?
 					AND state = 'running'
+					AND worker_id = ?
 					AND claim_expires_at IS NOT NULL
 					AND claim_expires_at > ?
 			`)
-            .get(jobId, timestamp);
+            .get(jobId, workerId, timestamp);
         return row !== undefined;
     }
     releaseJobForRetry(jobId, timestamp) {

package/dist/runs/registry.js

@@ -46,14 +46,16 @@ export class PiboRunRegistry {
     runs = new Map();
     waiters = new Map();
     listeners = new Set();
+    workerId;
     subscribe(listener) {
         this.listeners.add(listener);
         return () => this.listeners.delete(listener);
     }
     constructor(options = {}) {
         this.options = options;
+        this.workerId = options.workerId ?? `run-registry:${process.pid}:${randomUUID()}`;
         if (this.options.store) {
-            this.options.store.recoverInterruptedRuns();
+            this.options.store.recoverInterruptedRuns(this.workerId);
             for (const record of this.options.store.listRuns({ includeConsumed: true, includeDetached: true })) {
                 this.runs.set(record.runId, recordFromStored(record));
             }
@@ -71,6 +73,7 @@ export class PiboRunRegistry {
                 maxAttempts: input.maxAttempts ?? 1,
                 timeoutMs: input.timeoutMs,
                 serviceWarning: input.serviceWarning,
+                workerId: this.workerId,
             });
             const record = recordFromStored(stored);
             this.runs.set(record.runId, record);
@@ -112,7 +115,7 @@ export class PiboRunRegistry {
         this.finish(record);
         this.options.store?.updateRun(runId, record);
         if (record.jobId)
-            this.options.store?.ack(record.jobId, `run-registry:${process.pid}`);
+            this.options.store?.ack(record.jobId, this.workerId);
         const output = snapshot(record);
         this.notify({ type: "run_changed", run: output, previousStatus });
         return output;
@@ -128,7 +131,7 @@ export class PiboRunRegistry {
         this.finish(record);
         this.options.store?.updateRun(runId, record);
         if (record.jobId)
-            this.options.store?.fail(record.jobId, `run-registry:${process.pid}`, error);
+            this.options.store?.fail(record.jobId, this.workerId, error);
         const output = snapshot(record);
         this.notify({ type: "run_changed", run: output, previousStatus, reason: error });
         return output;
@@ -147,7 +150,7 @@ export class PiboRunRegistry {
         this.finish(record);
         this.options.store?.updateRun(runId, record);
         if (record.jobId)
-            this.options.store?.fail(record.jobId, `run-registry:${process.pid}`, error);
+            this.options.store?.fail(record.jobId, this.workerId, error);
         const output = snapshot(record);
         this.notify({ type: "run_changed", run: output, previousStatus, reason: error });
         return output;
@@ -227,7 +230,7 @@ export class PiboRunRegistry {
             record.summary = `${record.toolName} run cancelled.`;
             this.finish(record);
             if (record.jobId)
-                this.options.store?.fail(record.jobId, `run-registry:${process.pid}`, "Run was cancelled.");
+                this.options.store?.fail(record.jobId, this.workerId, "Run was cancelled.");
         }
         record.consumed = true;
         record.updatedAt = now();
@@ -292,7 +295,7 @@ export class PiboRunRegistry {
             this.finish(record);
             this.options.store?.updateRun(record.runId, record);
             if (record.jobId)
-                this.options.store?.fail(record.jobId, `run-registry:${process.pid}`, reason);
+                this.options.store?.fail(record.jobId, this.workerId, reason);
             const output = snapshot(record);
             this.notify({ type: "run_changed", run: output, previousStatus: "running", reason });
             cancelled.push(output);
@@ -311,7 +314,7 @@ export class PiboRunRegistry {
             this.finish(record);
             this.options.store?.updateRun(record.runId, record);
             if (record.jobId)
-                this.options.store?.fail(record.jobId, `run-registry:${process.pid}`, reason);
+                this.options.store?.fail(record.jobId, this.workerId, reason);
             const output = snapshot(record);
             this.notify({ type: "run_changed", run: output, previousStatus: "running", reason });
             cancelled.push(output);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pasko70/pibo",
-  "version": "1.11.1",
+  "version": "1.11.2",
   "type": "module",
   "imports": {
     "vscode": "./src/apps/chat-vscode/extension/src/vscode-shim.js"