@botiverse/raft-daemon 0.68.0 → 0.69.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -3288,6 +3288,7 @@ var AGENT_ACTIVITY_DETAIL_KINDS = [
|
|
|
3288
3288
|
"compaction_stale",
|
|
3289
3289
|
"reviewing_changes",
|
|
3290
3290
|
"review_finished",
|
|
3291
|
+
"review_stale",
|
|
3291
3292
|
"runtime_reconnecting",
|
|
3292
3293
|
"runtime_error",
|
|
3293
3294
|
"runtime_crashed",
|
|
@@ -13560,11 +13561,12 @@ var RUNTIME_ERROR_DELIVERY_BACKOFF_MAX_MS = 5 * 6e4;
|
|
|
13560
13561
|
var RUNTIME_ERROR_FINGERPRINT_FENCE_THRESHOLD = 3;
|
|
13561
13562
|
var SPAWN_FAIL_BACKOFF_BASE_MS = 1e3;
|
|
13562
13563
|
var SPAWN_FAIL_BACKOFF_MAX_MS = 3e4;
|
|
13563
|
-
var SPAWN_FAIL_BACKOFF_THRESHOLD =
|
|
13564
|
+
var SPAWN_FAIL_BACKOFF_THRESHOLD = 0;
|
|
13564
13565
|
var RUNNER_CREDENTIAL_MINT_BACKOFF_BASE_MS = 6e4;
|
|
13565
13566
|
var RUNNER_CREDENTIAL_MINT_BACKOFF_MAX_MS = 10 * 6e4;
|
|
13566
13567
|
var RUNNER_CREDENTIAL_MINT_BACKOFF_THRESHOLD = 0;
|
|
13567
13568
|
var COMPACTION_STALE_MS = 5 * 6e4;
|
|
13569
|
+
var REVIEW_STALE_MS = 10 * 6e4;
|
|
13568
13570
|
var RUNTIME_PROGRESS_STALE_MS = 15 * 6e4;
|
|
13569
13571
|
var DEFAULT_RUNTIME_START_TIMEOUT_MS = 2 * 6e4;
|
|
13570
13572
|
var DEFAULT_STALLED_RECOVERY_SIGTERM_TIMEOUT_MS = 1e4;
|
|
@@ -14653,10 +14655,10 @@ var AgentProcessManager = class _AgentProcessManager {
|
|
|
14653
14655
|
}
|
|
14654
14656
|
// ----- SPAWN-FAIL BACKOFF (per-agent) ----------------------------------
|
|
14655
14657
|
// Anchored at auto_restart_from_idle (apm:3596) + runtime_profile_auto_restart (apm:4137).
|
|
14656
|
-
//
|
|
14657
|
-
//
|
|
14658
|
-
// inner credential-mint loop exhausted
|
|
14659
|
-
//
|
|
14658
|
+
// One outer spawn failure enters cooldown immediately so repeated wakes cannot burn one
|
|
14659
|
+
// full spawn attempt per delivery. Runner credential mint failures keep a longer cooldown
|
|
14660
|
+
// because one outer failure already means the inner credential-mint loop exhausted
|
|
14661
|
+
// RUNNER_CREDENTIAL_MINT_MAX_ATTEMPTS.
|
|
14660
14662
|
// Successful spawn resets state. State lives outside AgentProcess because spawn fails BEFORE
|
|
14661
14663
|
// ap exists and the rate-window must persist across stop/respawn (else stop/start churn
|
|
14662
14664
|
// bypasses the cap — CC1 lifecycle pattern). Never advances any consume cursor or model-seen
|
|
@@ -14809,14 +14811,22 @@ var AgentProcessManager = class _AgentProcessManager {
|
|
|
14809
14811
|
this.sendStdinNotification(agentId);
|
|
14810
14812
|
}, delayMs);
|
|
14811
14813
|
}
|
|
14812
|
-
|
|
14814
|
+
scheduleIdleInboxDeliveryRetry(agentId, ap, notificationCount, source, traceName) {
|
|
14815
|
+
if (notificationCount <= 0 || !ap.driver.supportsStdinNotification || !ap.sessionId) return false;
|
|
14816
|
+
ap.notifications.add(notificationCount);
|
|
14817
|
+
ap.notifications.clearTimer();
|
|
14818
|
+
return ap.notifications.schedule(() => {
|
|
14819
|
+
this.flushIdleInboxDeliveryRetry(agentId, source, traceName);
|
|
14820
|
+
}, this.stdinNotificationRetryMs);
|
|
14821
|
+
}
|
|
14822
|
+
flushIdleInboxDeliveryRetry(agentId, source, traceName) {
|
|
14813
14823
|
const ap = this.agents.get(agentId);
|
|
14814
14824
|
if (!ap) return false;
|
|
14815
14825
|
const count = ap.notifications.takePendingAndClearTimer();
|
|
14816
14826
|
if (count === 0) return false;
|
|
14817
14827
|
if (!ap.driver.supportsStdinNotification || !ap.sessionId || ap.inbox.length === 0) {
|
|
14818
14828
|
ap.notifications.add(count);
|
|
14819
|
-
this.recordDaemonTrace(
|
|
14829
|
+
this.recordDaemonTrace(traceName, {
|
|
14820
14830
|
agentId,
|
|
14821
14831
|
runtime: ap.config.runtime,
|
|
14822
14832
|
model: ap.config.model,
|
|
@@ -14837,7 +14847,7 @@ var AgentProcessManager = class _AgentProcessManager {
|
|
|
14837
14847
|
ap.notifications.pruneContributedToPending(ap.inbox, ap.sessionId);
|
|
14838
14848
|
const messages = ap.notifications.filterUncontributedMessages(ap.inbox, ap.sessionId);
|
|
14839
14849
|
if (messages.length === 0) {
|
|
14840
|
-
this.recordDaemonTrace(
|
|
14850
|
+
this.recordDaemonTrace(traceName, {
|
|
14841
14851
|
agentId,
|
|
14842
14852
|
runtime: ap.config.runtime,
|
|
14843
14853
|
model: ap.config.model,
|
|
@@ -14851,16 +14861,16 @@ var AgentProcessManager = class _AgentProcessManager {
|
|
|
14851
14861
|
return false;
|
|
14852
14862
|
}
|
|
14853
14863
|
this.commitApmIdleState(agentId, ap, false);
|
|
14854
|
-
this.startRuntimeTrace(agentId, ap, "
|
|
14864
|
+
this.startRuntimeTrace(agentId, ap, source.replaceAll("_", "-"), messages);
|
|
14855
14865
|
this.broadcastMessageReceivedActivity(agentId);
|
|
14856
14866
|
const accepted = this.deliverInboxUpdateViaStdin(
|
|
14857
14867
|
agentId,
|
|
14858
14868
|
ap,
|
|
14859
14869
|
messages,
|
|
14860
14870
|
"idle",
|
|
14861
|
-
|
|
14871
|
+
source
|
|
14862
14872
|
);
|
|
14863
|
-
this.recordDaemonTrace(
|
|
14873
|
+
this.recordDaemonTrace(traceName, {
|
|
14864
14874
|
agentId,
|
|
14865
14875
|
runtime: ap.config.runtime,
|
|
14866
14876
|
model: ap.config.model,
|
|
@@ -14873,6 +14883,13 @@ var AgentProcessManager = class _AgentProcessManager {
|
|
|
14873
14883
|
});
|
|
14874
14884
|
return accepted;
|
|
14875
14885
|
}
|
|
14886
|
+
flushAsyncRejectedIdleDelivery(agentId) {
|
|
14887
|
+
return this.flushIdleInboxDeliveryRetry(
|
|
14888
|
+
agentId,
|
|
14889
|
+
"async_rejected_idle_delivery_retry",
|
|
14890
|
+
"daemon.agent.stdin_delivery.async_rejected.retry"
|
|
14891
|
+
);
|
|
14892
|
+
}
|
|
14876
14893
|
isApmIdle(ap) {
|
|
14877
14894
|
return ap.gatedSteering.isIdle;
|
|
14878
14895
|
}
|
|
@@ -14936,6 +14953,7 @@ var AgentProcessManager = class _AgentProcessManager {
|
|
|
14936
14953
|
if (options.includeCompactionWatchdog ?? true) {
|
|
14937
14954
|
this.clearCompactionWatchdog(ap);
|
|
14938
14955
|
}
|
|
14956
|
+
this.clearReviewWatchdog(ap);
|
|
14939
14957
|
this.clearStalledRecoverySigtermWatchdog(ap);
|
|
14940
14958
|
}
|
|
14941
14959
|
clearRuntimeErrorDeliveryBackoffWithTrace(agentId, ap, resetSource) {
|
|
@@ -16112,6 +16130,7 @@ Use \`raft message read\` to catch up on the channels listed above, then stop. R
|
|
|
16112
16130
|
readinessTransition: null,
|
|
16113
16131
|
activation: { kind: "idle" },
|
|
16114
16132
|
compaction: { kind: "none" },
|
|
16133
|
+
review: { kind: "none" },
|
|
16115
16134
|
runtimeProgress: new RuntimeProgressState(Date.now()),
|
|
16116
16135
|
runtimeTraceSpan: null,
|
|
16117
16136
|
runtimeTraceCounters: createRuntimeTraceCounters(),
|
|
@@ -18489,6 +18508,29 @@ Use \`raft message read\` to catch up on the channels listed above, then stop. R
|
|
|
18489
18508
|
ap.compaction = { ...ap.compaction, watchdog: null };
|
|
18490
18509
|
this.broadcastActivity(agentId, "working", "Context compaction still running; no finish event observed", [], void 0, "compaction_stale");
|
|
18491
18510
|
}
|
|
18511
|
+
startReviewWatchdog(agentId, ap) {
|
|
18512
|
+
this.clearReviewWatchdog(ap);
|
|
18513
|
+
const startedAt = Date.now();
|
|
18514
|
+
const watchdog = setTimeout(() => {
|
|
18515
|
+
this.markReviewStale(agentId, startedAt);
|
|
18516
|
+
}, REVIEW_STALE_MS);
|
|
18517
|
+
ap.review = { kind: "active", startedAt, watchdog };
|
|
18518
|
+
}
|
|
18519
|
+
clearReviewWatchdog(ap) {
|
|
18520
|
+
if (ap.review.kind === "active" && ap.review.watchdog) {
|
|
18521
|
+
clearTimeout(ap.review.watchdog);
|
|
18522
|
+
}
|
|
18523
|
+
ap.review = { kind: "none" };
|
|
18524
|
+
}
|
|
18525
|
+
markReviewStale(agentId, startedAt) {
|
|
18526
|
+
const ap = this.agents.get(agentId);
|
|
18527
|
+
if (!ap || ap.review.kind !== "active" || ap.review.startedAt !== startedAt) return;
|
|
18528
|
+
ap.review = { ...ap.review, watchdog: null };
|
|
18529
|
+
this.broadcastActivity(agentId, "working", "Review mode still active; no finish event observed", [], void 0, "review_stale");
|
|
18530
|
+
const reduction = reduceApmGatedReview(ap.gatedSteering, { kind: "review_finished" });
|
|
18531
|
+
this.commitGatedSteeringDecisionState(agentId, ap, reduction.nextState, { event: "review_finished", inferred: true });
|
|
18532
|
+
this.flushReviewBoundaryMessages(agentId, ap);
|
|
18533
|
+
}
|
|
18492
18534
|
completeCompactionIfActive(agentId, detail = "Context compaction finished", options = {}) {
|
|
18493
18535
|
const ap = this.agents.get(agentId);
|
|
18494
18536
|
if (!ap || ap.compaction.kind !== "active") return;
|
|
@@ -19428,11 +19470,15 @@ Use \`raft message read\` to catch up on the channels listed above, then stop. R
|
|
|
19428
19470
|
if (ap) {
|
|
19429
19471
|
const reduction = reduceApmGatedReview(ap.gatedSteering, { kind: "review_started" });
|
|
19430
19472
|
this.commitGatedSteeringDecisionState(agentId, ap, reduction.nextState, { event: "review_started" });
|
|
19473
|
+
this.startReviewWatchdog(agentId, ap);
|
|
19431
19474
|
}
|
|
19432
19475
|
break;
|
|
19433
19476
|
case "review_finished":
|
|
19434
19477
|
this.flushPendingTrajectory(agentId);
|
|
19435
|
-
if (ap)
|
|
19478
|
+
if (ap) {
|
|
19479
|
+
this.clearReviewWatchdog(ap);
|
|
19480
|
+
this.recordRuntimeTraceEvent(agentId, ap, "runtime.review_mode.finished");
|
|
19481
|
+
}
|
|
19436
19482
|
this.broadcastActivity(agentId, "working", "Review finished", [], void 0, "review_finished");
|
|
19437
19483
|
if (ap) {
|
|
19438
19484
|
const reduction = reduceApmGatedReview(ap.gatedSteering, { kind: "review_finished" });
|
|
@@ -19953,6 +19999,15 @@ ${RESPONSE_TARGET_HINT}`;
|
|
|
19953
19999
|
`${mode} inbox update`
|
|
19954
20000
|
);
|
|
19955
20001
|
if (!sendResult.ok) {
|
|
20002
|
+
const retryNotificationCount = mode === "idle" && ap.driver.supportsStdinNotification && ap.sessionId ? messages.length : 0;
|
|
20003
|
+
const retrySource = source.endsWith("_retry") ? source : `${source}_retry`;
|
|
20004
|
+
const retryScheduled = mode === "idle" ? this.scheduleIdleInboxDeliveryRetry(
|
|
20005
|
+
agentId,
|
|
20006
|
+
ap,
|
|
20007
|
+
retryNotificationCount,
|
|
20008
|
+
retrySource,
|
|
20009
|
+
"daemon.agent.stdin_delivery.idle_retry"
|
|
20010
|
+
) : false;
|
|
19956
20011
|
if (mode === "idle") {
|
|
19957
20012
|
this.commitApmIdleState(agentId, ap, true);
|
|
19958
20013
|
}
|
|
@@ -19977,7 +20032,9 @@ ${RESPONSE_TARGET_HINT}`;
|
|
|
19977
20032
|
outcome: runtimeSendFailureOutcome(sendResult),
|
|
19978
20033
|
failure_reason: sendResult.reason,
|
|
19979
20034
|
failure_error: sendResult.error,
|
|
19980
|
-
requeued_messages_count:
|
|
20035
|
+
requeued_messages_count: retryNotificationCount,
|
|
20036
|
+
retry_scheduled: retryScheduled,
|
|
20037
|
+
notification_timer_present: ap.notifications.hasTimer,
|
|
19981
20038
|
cursors_advanced: "none"
|
|
19982
20039
|
}, "error");
|
|
19983
20040
|
return false;
|
|
@@ -20162,6 +20219,14 @@ var systemClock = {
|
|
|
20162
20219
|
clearTimeout: (timer) => clearTimeout(timer)
|
|
20163
20220
|
};
|
|
20164
20221
|
var INBOUND_WATCHDOG_MS = 7e4;
|
|
20222
|
+
function normalizeHandshakeReason(value) {
|
|
20223
|
+
const raw = Array.isArray(value) ? value[0] : value;
|
|
20224
|
+
if (typeof raw !== "string") return null;
|
|
20225
|
+
const trimmed = raw.trim();
|
|
20226
|
+
if (!trimmed) return null;
|
|
20227
|
+
if (!/^[a-z0-9_:-]{1,80}$/i.test(trimmed)) return "invalid_header";
|
|
20228
|
+
return trimmed;
|
|
20229
|
+
}
|
|
20165
20230
|
function durationMsBucket(ms) {
|
|
20166
20231
|
if (ms == null || !Number.isFinite(ms) || ms < 0) return "unknown";
|
|
20167
20232
|
if (ms === 0) return "0";
|
|
@@ -20254,6 +20319,7 @@ var DaemonConnection = class {
|
|
|
20254
20319
|
});
|
|
20255
20320
|
const ws = this.options.wsFactory ? this.options.wsFactory(wsUrl, wsOptions) : new WebSocket(wsUrl, wsOptions);
|
|
20256
20321
|
this.ws = ws;
|
|
20322
|
+
let handshakeRejected = false;
|
|
20257
20323
|
ws.on("open", () => {
|
|
20258
20324
|
if (this.ws !== ws) return;
|
|
20259
20325
|
if (!this.shouldConnect) return;
|
|
@@ -20314,8 +20380,27 @@ var DaemonConnection = class {
|
|
|
20314
20380
|
this.options.onDisconnect();
|
|
20315
20381
|
this.scheduleReconnect();
|
|
20316
20382
|
});
|
|
20383
|
+
ws.on("unexpected-response", (_request, response) => {
|
|
20384
|
+
if (this.ws !== ws) return;
|
|
20385
|
+
handshakeRejected = true;
|
|
20386
|
+
const reason = normalizeHandshakeReason(response.headers["slock-reason"]);
|
|
20387
|
+
const statusCode = response.statusCode ?? 0;
|
|
20388
|
+
const reasonText = reason ? `, slock_reason=${reason}` : "";
|
|
20389
|
+
logger.error(`[Daemon] WebSocket handshake rejected (status=${statusCode}${reasonText})`);
|
|
20390
|
+
this.trace("daemon.connection.handshake_rejected", {
|
|
20391
|
+
status_code: statusCode,
|
|
20392
|
+
slock_reason_present: Boolean(reason),
|
|
20393
|
+
slock_reason: reason
|
|
20394
|
+
}, "error");
|
|
20395
|
+
response.resume();
|
|
20396
|
+
try {
|
|
20397
|
+
ws.terminate();
|
|
20398
|
+
} catch {
|
|
20399
|
+
}
|
|
20400
|
+
});
|
|
20317
20401
|
ws.on("error", (err) => {
|
|
20318
20402
|
if (this.ws !== ws) return;
|
|
20403
|
+
if (handshakeRejected) return;
|
|
20319
20404
|
logger.error(`[Daemon] WebSocket error: ${err.message}`);
|
|
20320
20405
|
this.trace("daemon.connection.error", {
|
|
20321
20406
|
error_class: err.name || "Error"
|
|
@@ -21408,7 +21493,7 @@ function resolveSlockCliPathOrEmpty(moduleUrl = import.meta.url) {
|
|
|
21408
21493
|
}
|
|
21409
21494
|
async function runBundledSlockCli(argv) {
|
|
21410
21495
|
process.argv = [process.execPath, "slock", ...argv];
|
|
21411
|
-
await import("./dist-
|
|
21496
|
+
await import("./dist-P3SAWND7.js");
|
|
21412
21497
|
}
|
|
21413
21498
|
function detectRuntimes(tracer = noopTracer) {
|
|
21414
21499
|
const ids = [];
|
package/dist/cli/index.js
CHANGED
|
@@ -44853,6 +44853,7 @@ var AGENT_ACTIVITY_DETAIL_KINDS = [
|
|
|
44853
44853
|
"compaction_stale",
|
|
44854
44854
|
"reviewing_changes",
|
|
44855
44855
|
"review_finished",
|
|
44856
|
+
"review_stale",
|
|
44856
44857
|
"runtime_reconnecting",
|
|
44857
44858
|
"runtime_error",
|
|
44858
44859
|
"runtime_crashed",
|
|
@@ -51041,6 +51042,9 @@ function formatClaimResults(channel, data) {
|
|
|
51041
51042
|
const msgShort = r.messageId ? r.messageId.slice(0, 8) : "";
|
|
51042
51043
|
return `${label} (msg:${msgShort}): claimed`;
|
|
51043
51044
|
}
|
|
51045
|
+
if (r.reason === "already claimed by you") {
|
|
51046
|
+
return `${label}: already claimed by you.`;
|
|
51047
|
+
}
|
|
51044
51048
|
return `${label}: FAILED \u2014 ${r.reason || "already claimed"}. Do not work on this task unless an owner/admin explicitly redirects it to you.`;
|
|
51045
51049
|
});
|
|
51046
51050
|
const succeeded = data.results.filter((r) => r.success).length;
|
package/dist/core.js
CHANGED
|
@@ -44382,6 +44382,7 @@ var AGENT_ACTIVITY_DETAIL_KINDS = [
|
|
|
44382
44382
|
"compaction_stale",
|
|
44383
44383
|
"reviewing_changes",
|
|
44384
44384
|
"review_finished",
|
|
44385
|
+
"review_stale",
|
|
44385
44386
|
"runtime_reconnecting",
|
|
44386
44387
|
"runtime_error",
|
|
44387
44388
|
"runtime_crashed",
|
|
@@ -50421,6 +50422,9 @@ function formatClaimResults(channel, data) {
|
|
|
50421
50422
|
const msgShort = r.messageId ? r.messageId.slice(0, 8) : "";
|
|
50422
50423
|
return `${label} (msg:${msgShort}): claimed`;
|
|
50423
50424
|
}
|
|
50425
|
+
if (r.reason === "already claimed by you") {
|
|
50426
|
+
return `${label}: already claimed by you.`;
|
|
50427
|
+
}
|
|
50424
50428
|
return `${label}: FAILED \u2014 ${r.reason || "already claimed"}. Do not work on this task unless an owner/admin explicitly redirects it to you.`;
|
|
50425
50429
|
});
|
|
50426
50430
|
const succeeded = data.results.filter((r) => r.success).length;
|
package/dist/index.js
CHANGED