@botiverse/raft-daemon 0.68.0 → 0.69.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3288,6 +3288,7 @@ var AGENT_ACTIVITY_DETAIL_KINDS = [
3288
3288
  "compaction_stale",
3289
3289
  "reviewing_changes",
3290
3290
  "review_finished",
3291
+ "review_stale",
3291
3292
  "runtime_reconnecting",
3292
3293
  "runtime_error",
3293
3294
  "runtime_crashed",
@@ -13560,11 +13561,12 @@ var RUNTIME_ERROR_DELIVERY_BACKOFF_MAX_MS = 5 * 6e4;
13560
13561
  var RUNTIME_ERROR_FINGERPRINT_FENCE_THRESHOLD = 3;
13561
13562
  var SPAWN_FAIL_BACKOFF_BASE_MS = 1e3;
13562
13563
  var SPAWN_FAIL_BACKOFF_MAX_MS = 3e4;
13563
- var SPAWN_FAIL_BACKOFF_THRESHOLD = 3;
13564
+ var SPAWN_FAIL_BACKOFF_THRESHOLD = 0;
13564
13565
  var RUNNER_CREDENTIAL_MINT_BACKOFF_BASE_MS = 6e4;
13565
13566
  var RUNNER_CREDENTIAL_MINT_BACKOFF_MAX_MS = 10 * 6e4;
13566
13567
  var RUNNER_CREDENTIAL_MINT_BACKOFF_THRESHOLD = 0;
13567
13568
  var COMPACTION_STALE_MS = 5 * 6e4;
13569
+ var REVIEW_STALE_MS = 10 * 6e4;
13568
13570
  var RUNTIME_PROGRESS_STALE_MS = 15 * 6e4;
13569
13571
  var DEFAULT_RUNTIME_START_TIMEOUT_MS = 2 * 6e4;
13570
13572
  var DEFAULT_STALLED_RECOVERY_SIGTERM_TIMEOUT_MS = 1e4;
@@ -14653,10 +14655,10 @@ var AgentProcessManager = class _AgentProcessManager {
14653
14655
  }
14654
14656
  // ----- SPAWN-FAIL BACKOFF (per-agent) ----------------------------------
14655
14657
  // Anchored at auto_restart_from_idle (apm:3596) + runtime_profile_auto_restart (apm:4137).
14656
- // Generic spawn errors keep a threshold > 1 so single-transient hiccups behave as today.
14657
- // Runner credential mint failures are different: one outer spawn failure already means the
14658
- // inner credential-mint loop exhausted RUNNER_CREDENTIAL_MINT_MAX_ATTEMPTS. Those enter a
14659
- // longer cooldown immediately so repeated wakes don't keep burning three network fetches.
14658
+ // One outer spawn failure enters cooldown immediately so repeated wakes cannot burn one
14659
+ // full spawn attempt per delivery. Runner credential mint failures keep a longer cooldown
14660
+ // because one outer failure already means the inner credential-mint loop exhausted
14661
+ // RUNNER_CREDENTIAL_MINT_MAX_ATTEMPTS.
14660
14662
  // Successful spawn resets state. State lives outside AgentProcess because spawn fails BEFORE
14661
14663
  // ap exists and the rate-window must persist across stop/respawn (else stop/start churn
14662
14664
  // bypasses the cap — CC1 lifecycle pattern). Never advances any consume cursor or model-seen
@@ -14809,14 +14811,22 @@ var AgentProcessManager = class _AgentProcessManager {
14809
14811
  this.sendStdinNotification(agentId);
14810
14812
  }, delayMs);
14811
14813
  }
14812
- flushAsyncRejectedIdleDelivery(agentId) {
14814
+ scheduleIdleInboxDeliveryRetry(agentId, ap, notificationCount, source, traceName) {
14815
+ if (notificationCount <= 0 || !ap.driver.supportsStdinNotification || !ap.sessionId) return false;
14816
+ ap.notifications.add(notificationCount);
14817
+ ap.notifications.clearTimer();
14818
+ return ap.notifications.schedule(() => {
14819
+ this.flushIdleInboxDeliveryRetry(agentId, source, traceName);
14820
+ }, this.stdinNotificationRetryMs);
14821
+ }
14822
+ flushIdleInboxDeliveryRetry(agentId, source, traceName) {
14813
14823
  const ap = this.agents.get(agentId);
14814
14824
  if (!ap) return false;
14815
14825
  const count = ap.notifications.takePendingAndClearTimer();
14816
14826
  if (count === 0) return false;
14817
14827
  if (!ap.driver.supportsStdinNotification || !ap.sessionId || ap.inbox.length === 0) {
14818
14828
  ap.notifications.add(count);
14819
- this.recordDaemonTrace("daemon.agent.stdin_delivery.async_rejected.retry", {
14829
+ this.recordDaemonTrace(traceName, {
14820
14830
  agentId,
14821
14831
  runtime: ap.config.runtime,
14822
14832
  model: ap.config.model,
@@ -14837,7 +14847,7 @@ var AgentProcessManager = class _AgentProcessManager {
14837
14847
  ap.notifications.pruneContributedToPending(ap.inbox, ap.sessionId);
14838
14848
  const messages = ap.notifications.filterUncontributedMessages(ap.inbox, ap.sessionId);
14839
14849
  if (messages.length === 0) {
14840
- this.recordDaemonTrace("daemon.agent.stdin_delivery.async_rejected.retry", {
14850
+ this.recordDaemonTrace(traceName, {
14841
14851
  agentId,
14842
14852
  runtime: ap.config.runtime,
14843
14853
  model: ap.config.model,
@@ -14851,16 +14861,16 @@ var AgentProcessManager = class _AgentProcessManager {
14851
14861
  return false;
14852
14862
  }
14853
14863
  this.commitApmIdleState(agentId, ap, false);
14854
- this.startRuntimeTrace(agentId, ap, "async-rejected-idle-delivery-retry", messages);
14864
+ this.startRuntimeTrace(agentId, ap, source.replaceAll("_", "-"), messages);
14855
14865
  this.broadcastMessageReceivedActivity(agentId);
14856
14866
  const accepted = this.deliverInboxUpdateViaStdin(
14857
14867
  agentId,
14858
14868
  ap,
14859
14869
  messages,
14860
14870
  "idle",
14861
- "async_rejected_idle_delivery_retry"
14871
+ source
14862
14872
  );
14863
- this.recordDaemonTrace("daemon.agent.stdin_delivery.async_rejected.retry", {
14873
+ this.recordDaemonTrace(traceName, {
14864
14874
  agentId,
14865
14875
  runtime: ap.config.runtime,
14866
14876
  model: ap.config.model,
@@ -14873,6 +14883,13 @@ var AgentProcessManager = class _AgentProcessManager {
14873
14883
  });
14874
14884
  return accepted;
14875
14885
  }
14886
+ flushAsyncRejectedIdleDelivery(agentId) {
14887
+ return this.flushIdleInboxDeliveryRetry(
14888
+ agentId,
14889
+ "async_rejected_idle_delivery_retry",
14890
+ "daemon.agent.stdin_delivery.async_rejected.retry"
14891
+ );
14892
+ }
14876
14893
  isApmIdle(ap) {
14877
14894
  return ap.gatedSteering.isIdle;
14878
14895
  }
@@ -14936,6 +14953,7 @@ var AgentProcessManager = class _AgentProcessManager {
14936
14953
  if (options.includeCompactionWatchdog ?? true) {
14937
14954
  this.clearCompactionWatchdog(ap);
14938
14955
  }
14956
+ this.clearReviewWatchdog(ap);
14939
14957
  this.clearStalledRecoverySigtermWatchdog(ap);
14940
14958
  }
14941
14959
  clearRuntimeErrorDeliveryBackoffWithTrace(agentId, ap, resetSource) {
@@ -16112,6 +16130,7 @@ Use \`raft message read\` to catch up on the channels listed above, then stop. R
16112
16130
  readinessTransition: null,
16113
16131
  activation: { kind: "idle" },
16114
16132
  compaction: { kind: "none" },
16133
+ review: { kind: "none" },
16115
16134
  runtimeProgress: new RuntimeProgressState(Date.now()),
16116
16135
  runtimeTraceSpan: null,
16117
16136
  runtimeTraceCounters: createRuntimeTraceCounters(),
@@ -18489,6 +18508,29 @@ Use \`raft message read\` to catch up on the channels listed above, then stop. R
18489
18508
  ap.compaction = { ...ap.compaction, watchdog: null };
18490
18509
  this.broadcastActivity(agentId, "working", "Context compaction still running; no finish event observed", [], void 0, "compaction_stale");
18491
18510
  }
18511
+ startReviewWatchdog(agentId, ap) {
18512
+ this.clearReviewWatchdog(ap);
18513
+ const startedAt = Date.now();
18514
+ const watchdog = setTimeout(() => {
18515
+ this.markReviewStale(agentId, startedAt);
18516
+ }, REVIEW_STALE_MS);
18517
+ ap.review = { kind: "active", startedAt, watchdog };
18518
+ }
18519
+ clearReviewWatchdog(ap) {
18520
+ if (ap.review.kind === "active" && ap.review.watchdog) {
18521
+ clearTimeout(ap.review.watchdog);
18522
+ }
18523
+ ap.review = { kind: "none" };
18524
+ }
18525
+ markReviewStale(agentId, startedAt) {
18526
+ const ap = this.agents.get(agentId);
18527
+ if (!ap || ap.review.kind !== "active" || ap.review.startedAt !== startedAt) return;
18528
+ ap.review = { ...ap.review, watchdog: null };
18529
+ this.broadcastActivity(agentId, "working", "Review mode still active; no finish event observed", [], void 0, "review_stale");
18530
+ const reduction = reduceApmGatedReview(ap.gatedSteering, { kind: "review_finished" });
18531
+ this.commitGatedSteeringDecisionState(agentId, ap, reduction.nextState, { event: "review_finished", inferred: true });
18532
+ this.flushReviewBoundaryMessages(agentId, ap);
18533
+ }
18492
18534
  completeCompactionIfActive(agentId, detail = "Context compaction finished", options = {}) {
18493
18535
  const ap = this.agents.get(agentId);
18494
18536
  if (!ap || ap.compaction.kind !== "active") return;
@@ -19428,11 +19470,15 @@ Use \`raft message read\` to catch up on the channels listed above, then stop. R
19428
19470
  if (ap) {
19429
19471
  const reduction = reduceApmGatedReview(ap.gatedSteering, { kind: "review_started" });
19430
19472
  this.commitGatedSteeringDecisionState(agentId, ap, reduction.nextState, { event: "review_started" });
19473
+ this.startReviewWatchdog(agentId, ap);
19431
19474
  }
19432
19475
  break;
19433
19476
  case "review_finished":
19434
19477
  this.flushPendingTrajectory(agentId);
19435
- if (ap) this.recordRuntimeTraceEvent(agentId, ap, "runtime.review_mode.finished");
19478
+ if (ap) {
19479
+ this.clearReviewWatchdog(ap);
19480
+ this.recordRuntimeTraceEvent(agentId, ap, "runtime.review_mode.finished");
19481
+ }
19436
19482
  this.broadcastActivity(agentId, "working", "Review finished", [], void 0, "review_finished");
19437
19483
  if (ap) {
19438
19484
  const reduction = reduceApmGatedReview(ap.gatedSteering, { kind: "review_finished" });
@@ -19953,6 +19999,15 @@ ${RESPONSE_TARGET_HINT}`;
19953
19999
  `${mode} inbox update`
19954
20000
  );
19955
20001
  if (!sendResult.ok) {
20002
+ const retryNotificationCount = mode === "idle" && ap.driver.supportsStdinNotification && ap.sessionId ? messages.length : 0;
20003
+ const retrySource = source.endsWith("_retry") ? source : `${source}_retry`;
20004
+ const retryScheduled = mode === "idle" ? this.scheduleIdleInboxDeliveryRetry(
20005
+ agentId,
20006
+ ap,
20007
+ retryNotificationCount,
20008
+ retrySource,
20009
+ "daemon.agent.stdin_delivery.idle_retry"
20010
+ ) : false;
19956
20011
  if (mode === "idle") {
19957
20012
  this.commitApmIdleState(agentId, ap, true);
19958
20013
  }
@@ -19977,7 +20032,9 @@ ${RESPONSE_TARGET_HINT}`;
19977
20032
  outcome: runtimeSendFailureOutcome(sendResult),
19978
20033
  failure_reason: sendResult.reason,
19979
20034
  failure_error: sendResult.error,
19980
- requeued_messages_count: 0,
20035
+ requeued_messages_count: retryNotificationCount,
20036
+ retry_scheduled: retryScheduled,
20037
+ notification_timer_present: ap.notifications.hasTimer,
19981
20038
  cursors_advanced: "none"
19982
20039
  }, "error");
19983
20040
  return false;
@@ -20162,6 +20219,14 @@ var systemClock = {
20162
20219
  clearTimeout: (timer) => clearTimeout(timer)
20163
20220
  };
20164
20221
  var INBOUND_WATCHDOG_MS = 7e4;
20222
+ function normalizeHandshakeReason(value) {
20223
+ const raw = Array.isArray(value) ? value[0] : value;
20224
+ if (typeof raw !== "string") return null;
20225
+ const trimmed = raw.trim();
20226
+ if (!trimmed) return null;
20227
+ if (!/^[a-z0-9_:-]{1,80}$/i.test(trimmed)) return "invalid_header";
20228
+ return trimmed;
20229
+ }
20165
20230
  function durationMsBucket(ms) {
20166
20231
  if (ms == null || !Number.isFinite(ms) || ms < 0) return "unknown";
20167
20232
  if (ms === 0) return "0";
@@ -20254,6 +20319,7 @@ var DaemonConnection = class {
20254
20319
  });
20255
20320
  const ws = this.options.wsFactory ? this.options.wsFactory(wsUrl, wsOptions) : new WebSocket(wsUrl, wsOptions);
20256
20321
  this.ws = ws;
20322
+ let handshakeRejected = false;
20257
20323
  ws.on("open", () => {
20258
20324
  if (this.ws !== ws) return;
20259
20325
  if (!this.shouldConnect) return;
@@ -20314,8 +20380,27 @@ var DaemonConnection = class {
20314
20380
  this.options.onDisconnect();
20315
20381
  this.scheduleReconnect();
20316
20382
  });
20383
+ ws.on("unexpected-response", (_request, response) => {
20384
+ if (this.ws !== ws) return;
20385
+ handshakeRejected = true;
20386
+ const reason = normalizeHandshakeReason(response.headers["slock-reason"]);
20387
+ const statusCode = response.statusCode ?? 0;
20388
+ const reasonText = reason ? `, slock_reason=${reason}` : "";
20389
+ logger.error(`[Daemon] WebSocket handshake rejected (status=${statusCode}${reasonText})`);
20390
+ this.trace("daemon.connection.handshake_rejected", {
20391
+ status_code: statusCode,
20392
+ slock_reason_present: Boolean(reason),
20393
+ slock_reason: reason
20394
+ }, "error");
20395
+ response.resume();
20396
+ try {
20397
+ ws.terminate();
20398
+ } catch {
20399
+ }
20400
+ });
20317
20401
  ws.on("error", (err) => {
20318
20402
  if (this.ws !== ws) return;
20403
+ if (handshakeRejected) return;
20319
20404
  logger.error(`[Daemon] WebSocket error: ${err.message}`);
20320
20405
  this.trace("daemon.connection.error", {
20321
20406
  error_class: err.name || "Error"
@@ -21408,7 +21493,7 @@ function resolveSlockCliPathOrEmpty(moduleUrl = import.meta.url) {
21408
21493
  }
21409
21494
  async function runBundledSlockCli(argv) {
21410
21495
  process.argv = [process.execPath, "slock", ...argv];
21411
- await import("./dist-Y3VRAU5E.js");
21496
+ await import("./dist-P3SAWND7.js");
21412
21497
  }
21413
21498
  function detectRuntimes(tracer = noopTracer) {
21414
21499
  const ids = [];
package/dist/cli/index.js CHANGED
@@ -44853,6 +44853,7 @@ var AGENT_ACTIVITY_DETAIL_KINDS = [
44853
44853
  "compaction_stale",
44854
44854
  "reviewing_changes",
44855
44855
  "review_finished",
44856
+ "review_stale",
44856
44857
  "runtime_reconnecting",
44857
44858
  "runtime_error",
44858
44859
  "runtime_crashed",
@@ -51041,6 +51042,9 @@ function formatClaimResults(channel, data) {
51041
51042
  const msgShort = r.messageId ? r.messageId.slice(0, 8) : "";
51042
51043
  return `${label} (msg:${msgShort}): claimed`;
51043
51044
  }
51045
+ if (r.reason === "already claimed by you") {
51046
+ return `${label}: already claimed by you.`;
51047
+ }
51044
51048
  return `${label}: FAILED \u2014 ${r.reason || "already claimed"}. Do not work on this task unless an owner/admin explicitly redirects it to you.`;
51045
51049
  });
51046
51050
  const succeeded = data.results.filter((r) => r.success).length;
package/dist/core.js CHANGED
@@ -12,7 +12,7 @@ import {
12
12
  runBundledSlockCli,
13
13
  scanWorkspaceDirectories,
14
14
  subscribeDaemonLogs
15
- } from "./chunk-RXK2KSYW.js";
15
+ } from "./chunk-URPIDKXK.js";
16
16
  export {
17
17
  DAEMON_CLI_USAGE,
18
18
  DAEMON_CORE_TRACE_ATTR_CONTRACTS,
@@ -44382,6 +44382,7 @@ var AGENT_ACTIVITY_DETAIL_KINDS = [
44382
44382
  "compaction_stale",
44383
44383
  "reviewing_changes",
44384
44384
  "review_finished",
44385
+ "review_stale",
44385
44386
  "runtime_reconnecting",
44386
44387
  "runtime_error",
44387
44388
  "runtime_crashed",
@@ -50421,6 +50422,9 @@ function formatClaimResults(channel, data) {
50421
50422
  const msgShort = r.messageId ? r.messageId.slice(0, 8) : "";
50422
50423
  return `${label} (msg:${msgShort}): claimed`;
50423
50424
  }
50425
+ if (r.reason === "already claimed by you") {
50426
+ return `${label}: already claimed by you.`;
50427
+ }
50424
50428
  return `${label}: FAILED \u2014 ${r.reason || "already claimed"}. Do not work on this task unless an owner/admin explicitly redirects it to you.`;
50425
50429
  });
50426
50430
  const succeeded = data.results.filter((r) => r.success).length;
package/dist/index.js CHANGED
@@ -3,7 +3,7 @@ import {
3
3
  DAEMON_CLI_USAGE,
4
4
  DaemonCore,
5
5
  parseDaemonCliArgs
6
- } from "./chunk-RXK2KSYW.js";
6
+ } from "./chunk-URPIDKXK.js";
7
7
 
8
8
  // src/index.ts
9
9
  var parsedArgs = parseDaemonCliArgs(process.argv.slice(2));
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@botiverse/raft-daemon",
3
- "version": "0.68.0",
3
+ "version": "0.69.0",
4
4
  "type": "module",
5
5
  "bin": {
6
6
  "raft-daemon": "dist/raft-daemon.js",