@staix/agent-hub 0.27.0 → 0.29.0

package/docs/verification/2026-11-03-control-write-stall-windows.json

@@ -0,0 +1,241 @@
+{
+  "schema": "agent-hub.stall-window-correlation/v1",
+  "clock": "recorded UTC wall time, not a monotonic lag measurement",
+  "windows": [
+    {
+      "window": "proposes",
+      "start": "2026-11-02T20:55:10Z",
+      "end": "2026-11-02T20:57:00Z",
+      "event_counts": {
+        "quota": 5,
+        "attach": 4,
+        "agent_cli": 1,
+        "task": 6,
+        "envelope": 3,
+        "state": 1,
+        "turn_start": 1,
+        "conduct": 1
+      },
+      "largest_event_gap": {
+        "seconds": 60.237,
+        "from": "2026-11-02T20:55:33.923Z",
+        "to": "2026-11-02T20:56:34.160Z",
+        "from_line": 572984,
+        "to_line": 572985,
+        "from_kind": "envelope",
+        "to_kind": "envelope"
+      },
+      "periodic_quota_count": 5,
+      "largest_periodic_quota_gap_seconds": 65.133,
+      "task_events": [
+        {
+          "at": "2026-11-02T20:55:33.866Z",
+          "task": 667,
+          "event": "proposed",
+          "source_line": 572981
+        },
+        {
+          "at": "2026-11-02T20:55:33.895Z",
+          "task": 667,
+          "event": "lifecycle linked",
+          "source_line": 572982
+        },
+        {
+          "at": "2026-11-02T20:55:33.916Z",
+          "task": 667,
+          "event": "assigned",
+          "source_line": 572983
+        },
+        {
+          "at": "2026-11-02T20:56:34.414Z",
+          "task": 668,
+          "event": "proposed",
+          "source_line": 572990
+        },
+        {
+          "at": "2026-11-02T20:56:34.445Z",
+          "task": 668,
+          "event": "lifecycle linked",
+          "source_line": 572991
+        },
+        {
+          "at": "2026-11-02T20:56:34.466Z",
+          "task": 668,
+          "event": "assigned",
+          "source_line": 572992
+        }
+      ]
+    },
+    {
+      "window": "late_propose",
+      "start": "2026-11-02T21:00:45Z",
+      "end": "2026-11-02T21:02:30Z",
+      "event_counts": {
+        "quota": 42,
+        "tokens": 2,
+        "usage": 2,
+        "progress": 1
+      },
+      "largest_event_gap": {
+        "seconds": 5.004,
+        "from": "2026-11-02T21:02:01.805Z",
+        "to": "2026-11-02T21:02:06.809Z",
+        "from_line": 573128,
+        "to_line": 573129,
+        "from_kind": "quota",
+        "to_kind": "quota"
+      },
+      "periodic_quota_count": 21,
+      "largest_periodic_quota_gap_seconds": 5.014,
+      "task_events": []
+    },
+    {
+      "window": "review_667",
+      "start": "2026-11-02T21:31:30Z",
+      "end": "2026-11-02T21:33:00Z",
+      "event_counts": {
+        "quota": 30,
+        "tokens": 6,
+        "usage": 8,
+        "progress": 3,
+        "task": 6,
+        "envelope": 12,
+        "attach": 12,
+        "state": 5,
+        "turn_start": 3,
+        "turn_end": 2,
+        "agent_cli": 1,
+        "cohort": 1,
+        "split": 1,
+        "overlap": 1
+      },
+      "largest_event_gap": {
+        "seconds": 4.818,
+        "from": "2026-11-02T21:31:42.684Z",
+        "to": "2026-11-02T21:31:47.502Z",
+        "from_line": 573825,
+        "to_line": 573826,
+        "from_kind": "quota",
+        "to_kind": "progress"
+      },
+      "periodic_quota_count": 14,
+      "largest_periodic_quota_gap_seconds": 5.225,
+      "task_events": [
+        {
+          "at": "2026-11-02T21:31:47.586Z",
+          "task": 668,
+          "event": "accepted",
+          "source_line": 573827
+        },
+        {
+          "at": "2026-11-02T21:32:03.597Z",
+          "task": 667,
+          "event": "done",
+          "source_line": 573849
+        },
+        {
+          "at": "2026-11-02T21:32:03.721Z",
+          "task": 667,
+          "event": "memory reference started",
+          "source_line": 573851
+        },
+        {
+          "at": "2026-11-02T21:32:42.925Z",
+          "task": 667,
+          "event": "approved",
+          "source_line": 573897
+        },
+        {
+          "at": "2026-11-02T21:32:43.509Z",
+          "task": 670,
+          "event": "ready",
+          "source_line": 573904
+        },
+        {
+          "at": "2026-11-02T21:32:43.542Z",
+          "task": 670,
+          "event": "assigned",
+          "source_line": 573905
+        }
+      ]
+    },
+    {
+      "window": "review_668",
+      "start": "2026-11-02T21:49:30Z",
+      "end": "2026-11-02T21:51:00Z",
+      "event_counts": {
+        "quota": 22,
+        "progress": 6,
+        "tokens": 9,
+        "usage": 9,
+        "task": 5,
+        "envelope": 11,
+        "attach": 4,
+        "state": 5,
+        "turn_start": 3,
+        "turn_end": 2,
+        "stale": 1,
+        "cohort": 1,
+        "split": 1,
+        "overlap": 1
+      },
+      "largest_event_gap": {
+        "seconds": 5.015,
+        "from": "2026-11-02T21:50:01.150Z",
+        "to": "2026-11-02T21:50:06.165Z",
+        "from_line": 574463,
+        "to_line": 574464,
+        "from_kind": "quota",
+        "to_kind": "quota"
+      },
+      "periodic_quota_count": 13,
+      "largest_periodic_quota_gap_seconds": 5.336,
+      "task_events": [
+        {
+          "at": "2026-11-02T21:50:12.802Z",
+          "task": 668,
+          "event": "done",
+          "source_line": 574466
+        },
+        {
+          "at": "2026-11-02T21:50:12.936Z",
+          "task": 668,
+          "event": "memory reference started",
+          "source_line": 574468
+        },
+        {
+          "at": "2026-11-02T21:50:33.167Z",
+          "task": 668,
+          "event": "approved",
+          "source_line": 574500
+        },
+        {
+          "at": "2026-11-02T21:50:33.760Z",
+          "task": 669,
+          "event": "ready",
+          "source_line": 574507
+        },
+        {
+          "at": "2026-11-02T21:50:33.782Z",
+          "task": 669,
+          "event": "assigned",
+          "source_line": 574508
+        }
+      ]
+    }
+  ],
+  "nearby_proposal_context": {
+    "gap_seconds": 68.894,
+    "from": "2026-11-02T20:56:34.474Z",
+    "to": "2026-11-02T20:57:43.368Z",
+    "from_line": 572993,
+    "to_line": 572994,
+    "end_kind": "memory_brief",
+    "task": 668,
+    "native_items": 4,
+    "fail_items": 4,
+    "legacy_items": 0,
+    "following_offer_at": "2026-11-02T20:57:43.369Z",
+    "interpretation": "A nearby gap ending at a brief record, not a measured memory-call duration or proven cause."
+  }
+}

package/docs/verification/2026-11-03-control-write-stall-windows.md

@@ -0,0 +1,20 @@
+# Timed control-write stall windows, 2026-11-03
+
+Task #671, issue #1126, Refs #1078. Read-only correlation of the running hub's hub.log and events.jsonl, all timestamps UTC on 2026-11-02. No request was sent to the daemon and its database/configuration were not opened or changed. Only metadata, counts and timestamps are retained here and in the [aggregate JSON](2026-11-03-control-write-stall-windows.json).
+
+The records are wall-clock observations, not an event-loop-delay measurement. A long gap is consistent with an unavailable loop, process suspension or blocked logging; it does not identify a blocking function. Conversely, periodic callbacks continuing to record rules out a single uninterrupted block spanning that entire interval, without ruling out short blocks or asynchronous waits.
+
+| Reported stall | Recorded activity | Interpretation and limit |
+| --- | --- | --- |
+| Proposes around 20:56:04-20:56:34 | Task 667 proposed at 20:55:33.866, linked at .895, assigned at .916. Its assigned-status envelope was recorded at .923. The next event, its Jcode task-offer envelope, is at 20:56:34.160, a 60.237 s gap (events lines 572984-572985). Jcode busy at .282 and turn_start at .339; conductor proposal audit at .368; task 668 proposed at .414, linked at .445 and assigned at .466. | The gap begins before the supplied 30-second window and lies between assignment and offer publication. No event of any kind was recorded inside it. The periodic Claude quota stream also has a 65.133 s gap. This is evidence to investigate that delivery/publication boundary, not proof of a specific snapshot, Git, memory or native-adapter cause. |
+| Propose around 21:01-21:02 | No propose is recorded in 21:00:45-21:02:30. The nearest earlier one is task 670 at 21:00:17.480, linked .524 and blocked .555; conductor audit .572. Claude native turn end/turn restart occur at 21:00:24, then another end at 21:00:30. At 21:01:33.464, hub.log reports memory sync refused by the configured privacy policy for 20 long-term entries. | The supplied proposal cannot be matched in this window from these records. Quota callbacks continue about every 5 s, with maximum Claude interval 5.014 s and maximum all-event gap 5.004 s in the selected range. The memory-sync refusal is concurrent evidence only; no duration or request correlation establishes it as the cause. |
+| Review of task 667 around 21:32 | Review-stage SDLC advice at 21:31:59.453; Claude busy/turn_start at 21:32:02.845/.900. Task done at 03.597, memory-reference bookkeeping at 03.721. Jcode turn ends at 10.639, restarts at 10.734 and ends at 14.939. Approval-stage SDLC advice at 42.894; approved event at 42.925, followed by notices and release/assignment of task 670. A memory-sync privacy refusal was recorded earlier at 21:31:33.465. | Quota and other events continue during the wait: maximum selected all-event gap 4.818 s; maximum Claude quota interval 5.225 s. A continuous 30-second blocked loop is not shown. External metadata/readiness waits remain plausible, but no per-call start/end span identifies them. The memory-reference event is board bookkeeping, not a measured capture duration. |
+| Review of task 668 around 21:50 | Done at 21:50:12.802, memory-reference bookkeeping at .936, review envelope at 13.424, Claude busy/turn_start at 13.669/.734. Codex turn ends at 16.972, restarts at 17.603 and ends at 26.495. At 17.602, hub.log reports snapshot burst budget exhausted and boundary evidence unknown. Approval-stage SDLC advice at 33.149; approved at 33.167, followed by release/assignment of task 669. | Maximum selected all-event gap 5.015 s; maximum Claude quota interval 5.336 s. The snapshot refusal is observable, but the log has no duration and cannot account for a 30-second wait by itself. Periodic activity continues. |
+
+Wider proposal context adds a second gap: task 668's assigned-status envelope at 20:56:34.474 is followed by no event until a memory_brief record at 20:57:43.368 (68.894 s, lines 572993-572994). That record reports four native items, all fail notes, without their identities or content. Its offer follows at .369 and the conductor audit at .563. This is a candidate memory-brief boundary, not a measured 68.894-second memory call. The existing plan-fail lookup is evaluated before Briefs.forTask, so the new instrumentation wraps that lookup as well as the brief itself; its existing reads and limits are unchanged.
+
+No check start/end, digest/inference duration, or Git start/end record in these selected windows identifies a slow operation. That is missing instrumentation, not proof those operations did not run. The first proposal and the two reviews exhibit different recorded behavior and should not be assigned one cause merely because callers saw the same timeout.
+
+Source inspection found no event-loop lag probe or correlated control receive/reply spans in the 0.27.0 path. The change for #1126 adds a bounded monotonic probe, metadata-only request and phase spans, and a slow-request hub.log line. It must retain recently completed intervals: a synchronous blocking span can finish before the delayed timer callback runs. Still-open asynchronous spans are reported as overlapping waits, not as proven CPU causes. Unknown/untracked occupancy remains unknown. Raw request, task prose, paths, actor/session/rid identifiers and external output stay out of telemetry.
+
+The new instrumentation has not been activated on the live daemon by this task. Its purpose is to name the observed boundary on the next naturally occurring stall; it does not retroactively prove the cause of these four reports or change a timeout to hide one.