@staix/agent-hub 0.27.0 → 0.29.0

package/docs/verification/2026-11-03-live-board-latency.json

@@ -0,0 +1,556 @@
+{
+  "schema": "agent-hub.private-board-profile/v1",
+  "result": "not_reproduced",
+  "source_heads": {
+    "current": "e2ff29696d6a5ddb7a9232c4aadeb7fc97995794",
+    "v027": "1d0557d6ee7f6e1b48416e5905865704b35c9dbe"
+  },
+  "snapshot": {
+    "captured_at_utc": "2026-11-02T21:52:22.818871+00:00",
+    "bytes": 33292288,
+    "tasks": 670,
+    "history_bytes": 4719605,
+    "deliveries_by_state": {
+      "accepted": {
+        "count": 7,
+        "payload_bytes": 77970
+      },
+      "completed": {
+        "count": 1801,
+        "payload_bytes": 8697266
+      },
+      "discarded": {
+        "count": 111,
+        "payload_bytes": 121132
+      },
+      "failed": {
+        "count": 136,
+        "payload_bytes": 492788
+      }
+    }
+  },
+  "read_only_current": {
+    "tasks": 670,
+    "board_list_ms": 100.6,
+    "tasks_constructor_ms": 132.49,
+    "public_view_all_ms": 142.67,
+    "status_tasks_ms": 0.44,
+    "stalled_ms": 0.68,
+    "pending_offers_ms": 0.18
+  },
+  "runs": {
+    "current_core": {
+      "host": {
+        "platform": "darwin",
+        "arch": "arm64",
+        "bun": "1.4.3"
+      },
+      "startup": {
+        "totalMs": 330.14,
+        "includesStartupEventLogReads": true
+      },
+      "iterations_per_phase": 3,
+      "phases": [
+        {
+          "name": "review_approved",
+          "count": 3,
+          "p50Ms": 67.7,
+          "p95Ms": 78.34,
+          "outcomes": {
+            "ok": 3
+          },
+          "states": {
+            "approved": 3
+          },
+          "board": {
+            "listCalls": 21,
+            "listRows": 8103,
+            "listMs": 82.45,
+            "privacyProjectionCalls": 6,
+            "privacyProjectionRows": 84,
+            "privacyProjectionMs": 0.65,
+            "taskWriteCalls": 3,
+            "taskWriteMs": 108.65,
+            "deliveryCompleteCalls": 0,
+            "deliveryCompleteMs": 0
+          }
+        },
+        {
+          "name": "proposal_backlog",
+          "count": 3,
+          "p50Ms": 31.8,
+          "p95Ms": 32.22,
+          "outcomes": {
+            "ok": 3
+          },
+          "states": {
+            "backlog": 3
+          },
+          "board": {
+            "listCalls": 6,
+            "listRows": 2028,
+            "listMs": 19.74,
+            "privacyProjectionCalls": 6,
+            "privacyProjectionRows": 84,
+            "privacyProjectionMs": 0.85,
+            "taskWriteCalls": 3,
+            "taskWriteMs": 69.4,
+            "deliveryCompleteCalls": 0,
+            "deliveryCompleteMs": 0
+          }
+        },
+        {
+          "name": "proposal_claim",
+          "count": 3,
+          "p50Ms": 86.74,
+          "p95Ms": 87.95,
+          "outcomes": {
+            "ok": 3
+          },
+          "states": {
+            "in_progress": 3
+          },
+          "board": {
+            "listCalls": 33,
+            "listRows": 16254,
+            "listMs": 159.76,
+            "privacyProjectionCalls": 15,
+            "privacyProjectionRows": 5364,
+            "privacyProjectionMs": 3.62,
+            "taskWriteCalls": 9,
+            "taskWriteMs": 157.19,
+            "deliveryCompleteCalls": 0,
+            "deliveryCompleteMs": 0
+          }
+        },
+        {
+          "name": "delivery_complete",
+          "count": 3,
+          "p50Ms": 18,
+          "p95Ms": 18.62,
+          "outcomes": {
+            "completed": 3
+          },
+          "states": {
+            "completed": 3
+          },
+          "board": {
+            "listCalls": 0,
+            "listRows": 0,
+            "listMs": 0,
+            "privacyProjectionCalls": 0,
+            "privacyProjectionRows": 0,
+            "privacyProjectionMs": 0,
+            "taskWriteCalls": 0,
+            "taskWriteMs": 0,
+            "deliveryCompleteCalls": 3,
+            "deliveryCompleteMs": 51.38
+          }
+        }
+      ],
+      "harness_sha256": "6433ad66b7a696c0bfd6d176a72f6edf28b2446617d7b08467efd9fd7933dcec"
+    },
+    "v027_core": {
+      "host": {
+        "platform": "darwin",
+        "arch": "arm64",
+        "bun": "1.4.3"
+      },
+      "startup": {
+        "totalMs": 350.95,
+        "includesStartupEventLogReads": true
+      },
+      "iterations_per_phase": 3,
+      "phases": [
+        {
+          "name": "review_approved",
+          "count": 3,
+          "p50Ms": 89.69,
+          "p95Ms": 103.84,
+          "outcomes": {
+            "ok": 3
+          },
+          "states": {
+            "approved": 3
+          },
+          "board": {
+            "listCalls": 27,
+            "listRows": 12141,
+            "listMs": 131.62,
+            "privacyProjectionCalls": 0,
+            "privacyProjectionRows": 0,
+            "privacyProjectionMs": 0,
+            "taskWriteCalls": 3,
+            "taskWriteMs": 122.15,
+            "deliveryCompleteCalls": 0,
+            "deliveryCompleteMs": 0
+          }
+        },
+        {
+          "name": "proposal_backlog",
+          "count": 3,
+          "p50Ms": 28.07,
+          "p95Ms": 28.43,
+          "outcomes": {
+            "ok": 3
+          },
+          "states": {
+            "backlog": 3
+          },
+          "board": {
+            "listCalls": 6,
+            "listRows": 2028,
+            "listMs": 20.38,
+            "privacyProjectionCalls": 0,
+            "privacyProjectionRows": 0,
+            "privacyProjectionMs": 0,
+            "taskWriteCalls": 3,
+            "taskWriteMs": 77.6,
+            "deliveryCompleteCalls": 0,
+            "deliveryCompleteMs": 0
+          }
+        },
+        {
+          "name": "proposal_claim",
+          "count": 3,
+          "p50Ms": 81.62,
+          "p95Ms": 89.19,
+          "outcomes": {
+            "ok": 3
+          },
+          "states": {
+            "in_progress": 3
+          },
+          "board": {
+            "listCalls": 33,
+            "listRows": 16254,
+            "listMs": 175.25,
+            "privacyProjectionCalls": 0,
+            "privacyProjectionRows": 0,
+            "privacyProjectionMs": 0,
+            "taskWriteCalls": 9,
+            "taskWriteMs": 177.28,
+            "deliveryCompleteCalls": 0,
+            "deliveryCompleteMs": 0
+          }
+        },
+        {
+          "name": "delivery_complete",
+          "count": 3,
+          "p50Ms": 20.43,
+          "p95Ms": 20.68,
+          "outcomes": {
+            "completed": 3
+          },
+          "states": {
+            "completed": 3
+          },
+          "board": {
+            "listCalls": 0,
+            "listRows": 0,
+            "listMs": 0,
+            "privacyProjectionCalls": 0,
+            "privacyProjectionRows": 0,
+            "privacyProjectionMs": 0,
+            "taskWriteCalls": 0,
+            "taskWriteMs": 0,
+            "deliveryCompleteCalls": 3,
+            "deliveryCompleteMs": 56.9
+          }
+        }
+      ],
+      "harness_sha256": "94801693c03d665ee9e044222edb2383a79eac058e35b56b60aab0161de8ebb0"
+    },
+    "v027_matched_isolated": {
+      "host": {
+        "platform": "darwin",
+        "arch": "arm64",
+        "bun": "1.4.3"
+      },
+      "startup": {
+        "totalMs": 1525.36,
+        "includesStartupEventLogReads": true
+      },
+      "iterations_per_phase": 3,
+      "phases": [
+        {
+          "name": "review_approved",
+          "count": 3,
+          "p50Ms": 137.69,
+          "p95Ms": 146.96,
+          "outcomes": {
+            "ok": 3
+          },
+          "states": {
+            "approved": 3
+          },
+          "board": {
+            "listCalls": 42,
+            "listRows": 20217,
+            "listMs": 203.23,
+            "privacyProjectionCalls": 0,
+            "privacyProjectionRows": 0,
+            "privacyProjectionMs": 0,
+            "taskWriteCalls": 6,
+            "taskWriteMs": 181.76,
+            "deliveryCompleteCalls": 0,
+            "deliveryCompleteMs": 0
+          }
+        },
+        {
+          "name": "proposal_backlog",
+          "count": 3,
+          "p50Ms": 34.71,
+          "p95Ms": 35.46,
+          "outcomes": {
+            "ok": 3
+          },
+          "states": {
+            "backlog": 3
+          },
+          "board": {
+            "listCalls": 9,
+            "listRows": 4056,
+            "listMs": 36.75,
+            "privacyProjectionCalls": 0,
+            "privacyProjectionRows": 0,
+            "privacyProjectionMs": 0,
+            "taskWriteCalls": 3,
+            "taskWriteMs": 98.63,
+            "deliveryCompleteCalls": 0,
+            "deliveryCompleteMs": 0
+          }
+        },
+        {
+          "name": "proposal_claim",
+          "count": 3,
+          "p50Ms": 144.18,
+          "p95Ms": 145.91,
+          "outcomes": {
+            "ok": 3
+          },
+          "states": {
+            "in_progress": 3
+          },
+          "board": {
+            "listCalls": 43,
+            "listRows": 23026,
+            "listMs": 219.58,
+            "privacyProjectionCalls": 0,
+            "privacyProjectionRows": 0,
+            "privacyProjectionMs": 0,
+            "taskWriteCalls": 9,
+            "taskWriteMs": 230.26,
+            "deliveryCompleteCalls": 0,
+            "deliveryCompleteMs": 0
+          }
+        },
+        {
+          "name": "delivery_complete",
+          "count": 3,
+          "p50Ms": 19.38,
+          "p95Ms": 20.48,
+          "outcomes": {
+            "completed": 3
+          },
+          "states": {
+            "completed": 3
+          },
+          "board": {
+            "listCalls": 0,
+            "listRows": 0,
+            "listMs": 0,
+            "privacyProjectionCalls": 0,
+            "privacyProjectionRows": 0,
+            "privacyProjectionMs": 0,
+            "taskWriteCalls": 0,
+            "taskWriteMs": 0,
+            "deliveryCompleteCalls": 3,
+            "deliveryCompleteMs": 56.87
+          }
+        }
+      ],
+      "harness_sha256": "af94616627702b8aa1f783478a68f6aeaaddeedca65680361d1954f5c3481b32"
+    },
+    "v027_matched_hooks": {
+      "host": {
+        "platform": "darwin",
+        "arch": "arm64",
+        "bun": "1.4.3"
+      },
+      "startup": {
+        "totalMs": 1662.28,
+        "includesStartupEventLogReads": true
+      },
+      "iterations_per_phase": 3,
+      "phases": [
+        {
+          "name": "review_approved",
+          "count": 3,
+          "p50Ms": 134.3,
+          "p95Ms": 144.84,
+          "outcomes": {
+            "ok": 3
+          },
+          "states": {
+            "approved": 3
+          },
+          "board": {
+            "queueHookMs": 90.21,
+            "queueHookCalls": 18,
+            "boardHookMs": 179.69,
+            "boardHookCalls": 6,
+            "listCalls": 42,
+            "listRows": 20217,
+            "listMs": 199.85,
+            "privacyProjectionCalls": 0,
+            "privacyProjectionRows": 0,
+            "privacyProjectionMs": 0,
+            "taskWriteCalls": 6,
+            "taskWriteMs": 180.41,
+            "deliveryCompleteCalls": 0,
+            "deliveryCompleteMs": 0
+          }
+        },
+        {
+          "name": "proposal_backlog",
+          "count": 3,
+          "p50Ms": 33.77,
+          "p95Ms": 34.75,
+          "outcomes": {
+            "ok": 3
+          },
+          "states": {
+            "backlog": 3
+          },
+          "board": {
+            "queueHookMs": 0,
+            "queueHookCalls": 0,
+            "boardHookMs": 91.97,
+            "boardHookCalls": 3,
+            "listCalls": 9,
+            "listRows": 4056,
+            "listMs": 39.33,
+            "privacyProjectionCalls": 0,
+            "privacyProjectionRows": 0,
+            "privacyProjectionMs": 0,
+            "taskWriteCalls": 3,
+            "taskWriteMs": 98.96,
+            "deliveryCompleteCalls": 0,
+            "deliveryCompleteMs": 0
+          }
+        },
+        {
+          "name": "proposal_claim",
+          "count": 3,
+          "p50Ms": 141.73,
+          "p95Ms": 146.71,
+          "outcomes": {
+            "ok": 3
+          },
+          "states": {
+            "in_progress": 3
+          },
+          "board": {
+            "queueHookMs": 0,
+            "queueHookCalls": 0,
+            "boardHookMs": 223.18,
+            "boardHookCalls": 9,
+            "listCalls": 43,
+            "listRows": 23026,
+            "listMs": 218.43,
+            "privacyProjectionCalls": 0,
+            "privacyProjectionRows": 0,
+            "privacyProjectionMs": 0,
+            "taskWriteCalls": 9,
+            "taskWriteMs": 228.89,
+            "deliveryCompleteCalls": 0,
+            "deliveryCompleteMs": 0
+          }
+        },
+        {
+          "name": "task_assign",
+          "count": 3,
+          "p50Ms": 186.26,
+          "p95Ms": 192.51,
+          "outcomes": {
+            "ok": 3
+          },
+          "states": {
+            "proposed": 3
+          },
+          "board": {
+            "queueHookMs": 77.33,
+            "queueHookCalls": 15,
+            "boardHookMs": 116.67,
+            "boardHookCalls": 3,
+            "listCalls": 57,
+            "listRows": 34743,
+            "listMs": 327.33,
+            "privacyProjectionCalls": 0,
+            "privacyProjectionRows": 0,
+            "privacyProjectionMs": 0,
+            "taskWriteCalls": 3,
+            "taskWriteMs": 116.94,
+            "deliveryCompleteCalls": 0,
+            "deliveryCompleteMs": 0
+          }
+        },
+        {
+          "name": "delivery_complete",
+          "count": 3,
+          "p50Ms": 21.35,
+          "p95Ms": 21.94,
+          "outcomes": {
+            "completed": 3
+          },
+          "states": {
+            "completed": 3
+          },
+          "board": {
+            "queueHookMs": 58.45,
+            "queueHookCalls": 6,
+            "boardHookMs": 0,
+            "boardHookCalls": 0,
+            "listCalls": 0,
+            "listRows": 0,
+            "listMs": 0,
+            "privacyProjectionCalls": 0,
+            "privacyProjectionRows": 0,
+            "privacyProjectionMs": 0,
+            "taskWriteCalls": 0,
+            "taskWriteMs": 0,
+            "deliveryCompleteCalls": 3,
+            "deliveryCompleteMs": 61.09
+          }
+        }
+      ],
+      "harness_sha256": "3aa30a564680a95c12f00b3432b1c36a34c9e976757df06e57e5185a6b0646d0"
+    }
+  },
+  "limitations": [
+    "Three samples per phase; p95 is the maximum sample, not a stable production percentile.",
+    "Private consistent database backup, not original WAL files, lock holders, memory repository, sessions or scheduler state.",
+    "Models, native agents and outbound fetch were not exercised. Matched runs use fresh isolated memory, disabled inference/MLX/native capture/task sweep and WIP 32.",
+    "MCP stdio wrapper, live socket backlog and live memory capture contention are not measured.",
+    "macOS Bun 1.4.3 diagnostic; not a pinned-Bun gate or exact live-process binary reproduction."
+  ],
+  "replay_audit": {
+    "current_registry": [
+      "hub_task_propose",
+      "hub_review"
+    ],
+    "natural_journal_idempotence": [
+      "delivery_complete"
+    ],
+    "not_in_current_write_operation": [
+      "hub_task_accept",
+      "task_assign",
+      "hub_task_done",
+      "hub_task_decline",
+      "hub_task_cancel"
+    ],
+    "observed_live_replay_feature": false,
+    "accept_history_note": "The four accepts on task 668 were plan revisions; they are not a reproduced timeout retry."
+  }
+}

package/docs/verification/2026-11-03-live-board-latency.md

@@ -0,0 +1,64 @@
+# Live-board-copy write latency, 2026-11-03
+
+Task #669, Refs #1078. **The reported 30-second stall was not reproduced and its cause remains unknown.** These measurements do not establish an environmental cause or a latency fix. No production code, timeout, retry bound, live configuration or running daemon was changed.
+
+## Input and isolation
+
+A read-only SQLite connection backed up the project's live database into an owner-only scratch directory. SQLite backup includes committed WAL state without copying an inconsistent database/WAL pair. Profiling then used that snapshot and disposable copies only. The snapshot contains 670 tasks, 4,719,605 bytes of history JSON and a 33,292,288-byte database. Its 2,055 deliveries comprise 1,801 completed, 136 failed, 111 discarded and 7 accepted rows. Counts differ from the assignment's earlier observation because the board continued operating before backup.
+
+Matching config.json, config.local.json, routing.toml and sdlc.toml were copied privately. No task text, config contents, database, account path or native/model output is in this receipt or the aggregate. The original snapshot was never used as a writable daemon database. Each control experiment first copied it to a new scratch project's database, appended three synthetic review tasks and used scripted peers. Private HOME, AGENTHUB_HOME, AMEM_HOME and agent homes prevented access to account or memory state. No native agent or real model was started.
+
+Read-only current-source probes used a readonly Bun SQLite handle with query_only and Board's read methods. Control probes adapted the existing scripts/benchmarks/control-write-latency.ts runner: replace synthetic board seeding with the snapshot copy plus synthetic review rows, preserve the production startDaemon/Tasks/Bus/ControlClient path, and emit aggregate timings only. The copied delivery journal retained its historical rows. Synthetic event-log bulk seeding was disabled; the person's event log was not replayed. Startup is timed separately.
+
+The matched-config runs loaded the copied configuration and routing, with explicit isolation differences: fresh empty memory store enabled; native memory capture, inference, triage, MLX and task sweep disabled; WIP limits set to 32 for the scripted batch; batching set to zero; PII screening scripted clear; outbound fetch refused. Research and snapshot settings otherwise followed the loaded configuration. This is not a claim to reproduce the live memory repository, external services, native session state or concurrent activity.
+
+## Source and measurements
+
+All diagnostics ran on the same Mac, darwin/arm64, Bun 1.4.3. They are bounded private-copy profiling, not a test gate or a pinned-Bun live-binary reproduction. The 0.27.0 source is tag v0.27.0, commit 1d0557d6ee7f6e1b48416e5905865704b35c9dbe, using the installed runtime's exact agent-mem 0.18.0 dependency through an own-worktree symlink. The current source is e2ff29696d6a5ddb7a9232c4aadeb7fc97995794 with agent-mem 0.19.0. No dependency was installed or changed on the Mac.
+
+A token-free status copy confirmed the running hub was 0.27.0, protocol 16, without tool_write_idempotency. The released-source comparisons below are separate from the running process; it received no profiling requests.
+
+Read-only current-source observation: Board.list 100.60 ms, Tasks construction 132.49 ms, all 670 publicView projections 142.67 ms, statusTasks 0.44 ms, stalled 0.68 ms and pendingOffers 0.18 ms. An earlier independent SQLite select/decode observation was 4.09/12.47 ms. These are single observations, not latency distributions or proof that repeated scans can never accumulate.
+
+Each control phase has only three successful samples. The reported p95 is nearest-rank, therefore the maximum of those three samples. It is not a stable production percentile. Values are request round trips in milliseconds:
+
+| Source / fixture | Review p50 / p95 | Backlog propose p50 / p95 | Claimed propose p50 / p95 | Delivery complete p50 / p95 |
+| --- | --- | --- | --- | --- |
+| Current, isolated core | 67.70 / 78.34 | 31.80 / 32.22 | 86.74 / 87.95 | 18.00 / 18.62 |
+| 0.27.0, isolated core | 89.69 / 103.84 | 28.07 / 28.43 | 81.62 / 89.19 | 20.43 / 20.68 |
+| 0.27.0, matched config with isolation | 137.69 / 146.96 | 34.71 / 35.46 | 144.18 / 145.91 | 19.38 / 20.48 |
+| 0.27.0, same config with hook spans | 134.30 / 144.84 | 33.77 / 34.75 | 141.73 / 146.71 | 21.35 / 21.94 |
+
+The last run also exercised the console's real task_assign operation: 186.26 / 192.51 ms, three successful assignments. Assigned tasks remain proposed until their owners accept, as expected. Startup ranged from 330.14 ms to 1,662.28 ms and is excluded from the table. No speedup claim is made from comparisons with different versions, configuration or instrumentation.
+
+## Shared hook observations
+
+Both board writes and delivery completion synchronously refresh queue/status state in daemon.ts. Board.onChange calls settleSupersededDeliveries and writeStatus; bus.onQueues also refreshes holds and status. In addition, Bus.refreshRecoveryHold reads journal.list(peer), which includes historical completed deliveries. These are plausible accumulation points absent from the earlier zero-delivery synthetic fixture, but the copy measurements did not show the reported stall.
+
+The last run wrapped the real board.onChange and bus.onQueues callbacks, retaining their behavior. Across three samples per phase, aggregate board-hook / queue-hook times were:
+
+| Phase | Board hook total ms / calls | Queue hook total ms / calls |
+| --- | --- | --- |
+| Review | 179.69 / 6 | 90.21 / 18 |
+| Backlog propose | 91.97 / 3 | 0 / 0 |
+| Claimed propose | 223.18 / 9 | 0 / 0 |
+| Assignment | 116.67 / 3 | 77.33 / 15 |
+| Delivery complete | 0 / 0 | 58.45 / 6 |
+
+These spans nest inside writes and sometimes inside one another; do not add them to round-trip times. They do not separate SQLite fsync/checkpoint time from the rest of a write. The original active WAL, lock holders and their timing are not retained by an online backup, so transient disk/lock behavior remains unverified.
+
+On the snapshot, tasks 667 and 668 went from proposed to lifecycle linked to assigned within 52 and 54 ms respectively. Their first proposed timestamps are 2026-11-02T20:55:33.856Z and 20:56:34.404Z. This bounds those recorded transitions only: no request-start or reply timestamp is present, so it cannot establish their complete tool-call latency.
+
+## Replay coverage finding
+
+Source audit at e2ff2969 confirms toolWriteOperation recognizes task propose and review, plus delivery_complete for unknown-outcome handling. Delivery completion uses the journal's natural idempotence rather than registry capacity. Task accept, task_assign, done, decline and cancel are outside that operation set; transport uncertainty/replay protection must not be described as covering every write. No timeout retry was injected in this investigation.
+
+The four accept entries on task 668 were deliberate plan revisions as the implementation scope grew, not evidence of a timeout retry. A separate controlled test is needed to demonstrate a lost-response retry for accept or assignment before expanding their idempotency contract. The running 0.27.0 process also lacks the feature shipped in the later source; a source merge is not live activation.
+
+## Result and next step
+
+No causal production fix is justified by these observations. The copy rules out a deterministic 30-second stall for these particular sampled inputs and isolated configurations; it does not exonerate every local path. Live memory-store contention, MCP stdio handling, queued socket work, concurrent background passes, external metadata calls and intermittent filesystem delays remain unmeasured.
+
+Next, in a separately approved instrumentation change, record bounded metadata-only monotonic spans at client send/receive, daemon receive/reply, and the board/queue/status/memory/external-call boundaries. Include event-loop delay so queued admission can be distinguished from post-write work. Use counts, operation names and opaque correlation IDs, never payloads or task text. Deploy through the normal reviewed release/upgrade process and capture one naturally occurring stall before choosing a fix. Do not raise the 30-second timeout or WAL/resource thresholds. This task did not deploy, restart, upgrade, inject a failure into, or attach a profiler to the live daemon.
+
+The [aggregate JSON](2026-11-03-live-board-latency.json) contains counts and timings only, SHA-256 e44ef37a822b350673064f2d2640c372bb51725bd8ad5152d3d8c38e7f8cc65d. Harness hashes identify the private adapters of the existing runner; raw inputs and account/environment state are intentionally not published. This is a findings receipt, not an exact reproduction package or before/after fix receipt.