@cohortapp/agent-sdk 2.3.1 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/maestro.mjs +37 -50
- package/framework-features.json +30 -0
- package/lib/backlog.mjs +136 -0
- package/lib/cadences.mjs +63 -2
- package/lib/cadences.test.mjs +105 -0
- package/lib/capability/inventory.mjs +542 -0
- package/lib/capability/inventory.test.mjs +232 -0
- package/lib/capability/probe.mjs +255 -0
- package/lib/channels/contract.mjs +37 -1
- package/lib/channels/contract.test.mjs +25 -1
- package/lib/channels/inbox-item.mjs +20 -0
- package/lib/claude-bin.mjs +37 -3
- package/lib/claude-bin.test.mjs +42 -8
- package/lib/execution/disposition.mjs +501 -0
- package/lib/execution/disposition.test.mjs +482 -0
- package/lib/execution/drive.mjs +352 -0
- package/lib/execution/drive.test.mjs +270 -0
- package/lib/execution/effects.mjs +340 -0
- package/lib/execution/effects.test.mjs +193 -0
- package/lib/execution/index.mjs +152 -0
- package/lib/execution/intake.mjs +581 -0
- package/lib/execution/intake.test.mjs +343 -0
- package/lib/execution/journal.mjs +374 -0
- package/lib/execution/journal.test.mjs +261 -0
- package/lib/execution/match.mjs +331 -0
- package/lib/execution/match.test.mjs +235 -0
- package/lib/execution/pipeline.mjs +341 -0
- package/lib/execution/pipeline.test.mjs +389 -0
- package/lib/execution/route.mjs +332 -0
- package/lib/execution/route.test.mjs +186 -0
- package/lib/execution/surface-policy.mjs +446 -0
- package/lib/execution/surface-policy.test.mjs +162 -0
- package/lib/goals/admission.mjs +209 -0
- package/lib/goals/admission.test.mjs +139 -0
- package/lib/goals/classify.mjs +206 -0
- package/lib/goals/classify.test.mjs +109 -0
- package/lib/goals/collaborate.mjs +415 -0
- package/lib/goals/collaborate.test.mjs +324 -0
- package/lib/goals/gaps.mjs +111 -0
- package/lib/goals/gaps.test.mjs +284 -0
- package/lib/goals/loop.mjs +537 -0
- package/lib/goals/loop.test.mjs +719 -0
- package/lib/identity/persona.mjs +247 -0
- package/lib/identity/persona.test.mjs +117 -0
- package/lib/kpi.mjs +469 -0
- package/lib/kpi.test.mjs +244 -0
- package/lib/mandate/audit.mjs +168 -0
- package/lib/mandate/audit.test.mjs +195 -0
- package/lib/mandate/cache.mjs +162 -0
- package/lib/mandate/derive.mjs +317 -0
- package/lib/mandate/derive.test.mjs +224 -0
- package/lib/mandate/model.mjs +352 -0
- package/lib/mandate/model.test.mjs +145 -0
- package/lib/mandate/refresh.mjs +187 -0
- package/lib/mandate/refresh.test.mjs +293 -0
- package/lib/mcp/server.test.mjs +4 -4
- package/lib/org/approvals.mjs +14 -2
- package/lib/org/client.mjs +79 -25
- package/lib/org/client.test.mjs +54 -1
- package/lib/org/doctor.mjs +64 -0
- package/lib/org/doctor.test.mjs +31 -2
- package/lib/org/inbound/directedness.mjs +720 -0
- package/lib/org/inbound/directedness.test.mjs +543 -0
- package/lib/org/inbound/facts.mjs +501 -0
- package/lib/org/inbound/facts.test.mjs +375 -0
- package/lib/org/inbound/hydrate.mjs +535 -0
- package/lib/org/inbound/hydrate.test.mjs +326 -0
- package/lib/org/inbound/index.mjs +233 -0
- package/lib/org/inbound/index.test.mjs +324 -0
- package/lib/org/inbound/io.mjs +141 -0
- package/lib/org/inbound/project.mjs +201 -0
- package/lib/org/inbound/project.test.mjs +287 -0
- package/lib/org/inbound/surfaces.mjs +257 -0
- package/lib/org/knowledge.mjs +10 -1
- package/lib/org/knowledge.test.mjs +8 -1
- package/lib/org/leases.mjs +5 -0
- package/lib/org/mesh.mjs +45 -2
- package/lib/org/mesh.test.mjs +55 -0
- package/lib/org/messaging.mjs +180 -15
- package/lib/org/messaging.test.mjs +117 -0
- package/lib/org/param-contract.mjs +694 -0
- package/lib/org/param-contract.test.mjs +451 -0
- package/lib/org/protocol.checksum +1 -1
- package/lib/org/protocol.mjs +8 -0
- package/lib/org/protocol.test.mjs +5 -1
- package/lib/org/push.mjs +1025 -0
- package/lib/org/push.test.mjs +690 -0
- package/lib/org/tool-surface.mjs +138 -38
- package/lib/org/tool-surface.test.mjs +13 -8
- package/lib/org/typing.mjs +341 -0
- package/lib/org/typing.test.mjs +291 -0
- package/lib/plan/compile.mjs +510 -0
- package/lib/plan/compile.test.mjs +286 -0
- package/lib/plan/emit.mjs +256 -0
- package/lib/plan/emit.test.mjs +246 -0
- package/lib/plan/explain.mjs +226 -0
- package/lib/plan/explain.test.mjs +188 -0
- package/lib/plan/schema.mjs +140 -0
- package/lib/resource-governor.mjs +47 -1
- package/lib/resource-governor.test.mjs +21 -1
- package/lib/setup/enroll-from-cohort.mjs +84 -16
- package/lib/setup/enroll-from-cohort.test.mjs +43 -1
- package/lib/setup/sections/identity.mjs +15 -4
- package/lib/setup/sections/identity.test.mjs +94 -0
- package/lib/setup/sections/inventory.mjs +178 -0
- package/lib/setup/sections/inventory.test.mjs +198 -0
- package/lib/setup/sections/mandate.mjs +392 -0
- package/lib/setup/sections/mandate.test.mjs +373 -0
- package/lib/setup/sections/subagents.mjs +427 -0
- package/lib/setup/sections/subagents.test.mjs +429 -0
- package/lib/setup/sections/verify.mjs +121 -0
- package/lib/setup/sections/verify.test.mjs +175 -0
- package/lib/setup/sot.mjs +2 -0
- package/lib/subagents/cli.mjs +463 -0
- package/lib/subagents/cli.test.mjs +389 -0
- package/lib/subagents/client.mjs +373 -0
- package/lib/subagents/client.test.mjs +309 -0
- package/lib/subagents/gap.mjs +268 -0
- package/lib/subagents/gap.test.mjs +234 -0
- package/lib/subagents/lock.mjs +296 -0
- package/lib/subagents/lock.test.mjs +248 -0
- package/lib/subagents/manifest.mjs +224 -0
- package/lib/subagents/manifest.test.mjs +175 -0
- package/lib/subagents/refs.mjs +274 -0
- package/lib/subagents/refs.test.mjs +204 -0
- package/lib/subagents/resolve.mjs +455 -0
- package/lib/subagents/resolve.test.mjs +422 -0
- package/lib/subagents/schema.mjs +467 -0
- package/lib/subagents/schema.test.mjs +306 -0
- package/package.json +9 -4
- package/plugins/maestro-skills/.claude-plugin/marketplace.json +16 -0
- package/policies/ai-disclosure.yaml +42 -2
- package/scaffold/CLAUDE.md +16 -2
- package/schedules/triggers/goal-steward.md +79 -0
- package/scripts/ci/conformance-org-api.mjs +792 -0
- package/scripts/ci/conformance-org-api.test.mjs +417 -0
- package/scripts/daemon/agent-daemon.mjs +70 -11
- package/scripts/daemon/cadence-handlers.mjs +187 -5
- package/scripts/daemon/goal-steward-cadence.test.mjs +243 -0
- package/scripts/daemon/inbox-deferral.mjs +45 -2
- package/scripts/daemon/inbox-deferral.test.mjs +56 -0
- package/scripts/daemon/inbox-wake.mjs +282 -0
- package/scripts/daemon/inbox-wake.test.mjs +199 -0
- package/scripts/daemon/maestro-daemon.mjs +23 -0
- package/scripts/daemon/prompt-builder.mjs +41 -1
- package/scripts/daemon/responder.mjs +56 -0
- package/scripts/daemon/typing-registry.mjs +55 -2
- package/scripts/daemon/typing-registry.test.mjs +25 -0
- package/scripts/local-triggers/generate-plists.test.mjs +5 -5
- package/scripts/poller/inbox-scan-poller.mjs +26 -1
- package/scripts/poller/inbox-scan-poller.test.mjs +64 -0
- package/scripts/poller/slack-cloud-relay-client.mjs +5 -0
- package/scripts/poller/slack-poller.mjs +32 -0
- package/scripts/poller/slack-socket-mode.mjs +27 -1
- package/scripts/poller/slack-socket-mode.test.mjs +52 -0
- package/scripts/poller/utils.mjs +47 -0
- package/scripts/setup/gen-subagent-manifest.mjs +95 -0
- package/scripts/setup/gen-subagent-manifest.test.mjs +124 -0
- package/scripts/setup/generate-plan.mjs +108 -0
- package/scripts/setup/init-capability-manifest.mjs +70 -0
- package/scripts/setup/init-skill-marketplace.mjs +155 -0
- package/scripts/setup/init-skill-marketplace.test.mjs +193 -0
package/lib/org/push.mjs
ADDED
|
@@ -0,0 +1,1025 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* lib/org/push.mjs — the agent's PUSH channel onto the org event ledger.
|
|
3
|
+
*
|
|
4
|
+
* THE GAP this closes: an agent currently learns about org activity on a POLL.
|
|
5
|
+
* The `messaging-inbound` cadence runs every 45s (lib/cadences.mjs:66) and the
|
|
6
|
+
* orgmail adapter polls every 45s (lib/channels/orgmail/adapter.mjs:61), so a
|
|
7
|
+
* human who @mentions an AI colleague in the Cohort app waits up to three
|
|
8
|
+
* quarters of a minute for the agent to even NOTICE. hq already knows the agent
|
|
9
|
+
* is alive — it suppresses its own in-process LLM responder for any member whose
|
|
10
|
+
* daemon beat within 100s (hq src/server/llm-responder/sdk-driven.ts) — so the
|
|
11
|
+
* hand-off is real: hq stays quiet, and the daemon owes the org a fast answer.
|
|
12
|
+
* This module is the delivery half of that hand-off.
|
|
13
|
+
*
|
|
14
|
+
* ============================================================================
|
|
15
|
+
* THE LADDER (this is the whole design; the rest is mechanics)
|
|
16
|
+
* ============================================================================
|
|
17
|
+
*
|
|
18
|
+
* 1. STREAM — one held HTTP connection, server pushes frames as they commit.
|
|
19
|
+
* Preferred: zero idle requests, sub-second wake.
|
|
20
|
+
* 2. LONGPOLL — a bounded `agent.wait` request (the `approval.wait` precedent,
|
|
21
|
+
* hq src/server/rpc/reads.ts:532 — proven to survive a ~55s hold
|
|
22
|
+
* behind Railway's proxy). Used when the stream cannot be
|
|
23
|
+
* established or keeps dying.
|
|
24
|
+
* 3. CADENCE — nothing. We stop talking and the EXISTING 45s cadences carry
|
|
25
|
+
* the agent, exactly as they do today.
|
|
26
|
+
*
|
|
27
|
+
* Rung 3 is not a failure mode we tolerate reluctantly — it is the safety net the
|
|
28
|
+
* whole design rests on. **The cadences are never disabled by this module.** They
|
|
29
|
+
* keep running underneath the push channel at all times, which is what makes
|
|
30
|
+
* every hard decision below cheap:
|
|
31
|
+
*
|
|
32
|
+
* - a dropped frame is not lost work (the cadence re-discovers it ≤45s later),
|
|
33
|
+
* - a duplicate frame is harmless (the cadence path is already dedup-guarded:
|
|
34
|
+
* `pullInbound` has its own cursor and `writeInboxItem` refuses re-writes),
|
|
35
|
+
* - and a push channel that dies entirely degrades the agent to TODAY'S
|
|
36
|
+
* behaviour rather than making it deaf.
|
|
37
|
+
*
|
|
38
|
+
* That is the house rule from lib/org/mesh.mjs, applied to a live connection:
|
|
39
|
+
* fail-open, loudly, never take the daemon down.
|
|
40
|
+
*
|
|
41
|
+
* ============================================================================
|
|
42
|
+
* WHY THIS IS A *WAKE* CHANNEL, NOT A DELIVERY CHANNEL
|
|
43
|
+
* ============================================================================
|
|
44
|
+
*
|
|
45
|
+
* hq's event frames are REDACTED by design. `messaging.send` appends only
|
|
46
|
+
* `{actor, channelId, channelKind, recipientCount, mentionCount, ...}` — counts,
|
|
47
|
+
* never recipient ids (hq src/server/methods/messaging/send.ts:124-139); a call
|
|
48
|
+
* invite carries `inviteCount`, not invitees. **A frame is structurally incapable
|
|
49
|
+
* of telling us "this one is for you."** Answering that question means joining
|
|
50
|
+
* Mention / ChannelMember / Task.assigneeId / CallParticipant / Mailbox — i.e.
|
|
51
|
+
* exactly what `lib/org/messaging.pullInbound` and `email.inbox` already do,
|
|
52
|
+
* server-side, correctly, with their own cursors.
|
|
53
|
+
*
|
|
54
|
+
* So we do NOT try to re-derive recipient-ness on the client. A pushed frame is
|
|
55
|
+
* treated as a HINT: "something happened in family X at seq N — go look." The
|
|
56
|
+
* wake handler kicks the existing authoritative fetch (by default: enqueue a
|
|
57
|
+
* `messaging-inbound` tick onto the cadence bus, which the consumer drains within
|
|
58
|
+
* ~2s and which routes real directed items into the inbox pipeline). The push
|
|
59
|
+
* channel therefore changes ONE number — time-to-notice, 45s → ~1s — and changes
|
|
60
|
+
* nothing else about how work is fetched, deduped, or dispatched. Small blast
|
|
61
|
+
* radius, and the 45s path stays warm and exercised.
|
|
62
|
+
*
|
|
63
|
+
* ============================================================================
|
|
64
|
+
* THE WIRE CONTRACT (what this expects of hq)
|
|
65
|
+
* ============================================================================
|
|
66
|
+
*
|
|
67
|
+
* Rung 1 — GET `{base}/api/v1/agent.stream?cursor=<seq>`
|
|
68
|
+
* Accept: `text/event-stream, application/x-ndjson`. Held open; each frame is
|
|
69
|
+
* one line. BOTH framings are accepted by the parser below:
|
|
70
|
+
* - SSE: `data: {...}\n\n`, with `:` comment lines as keepalives
|
|
71
|
+
* - NDJSON: `{...}\n` (hq's house streaming format — see its AI gateway,
|
|
72
|
+
* which returns `application/x-ndjson`; there is no SSE precedent
|
|
73
|
+
* in that repo, so committing to only one framing would be a bet)
|
|
74
|
+
* Frame bodies:
|
|
75
|
+
* `{seq, family, kind, entityId, actor, at, payload}` — a committed event
|
|
76
|
+
* `{type:"heartbeat"}` — liveness only
|
|
77
|
+
* `{type:"ready", cursor}` — optional preamble
|
|
78
|
+
* Anything unrecognised is ignored (forward-compatible).
|
|
79
|
+
*
|
|
80
|
+
* Rung 2 — GET `{base}/api/v1/agent.wait?cursor=<seq>&timeoutMs=55000`
|
|
81
|
+
* Returns the BARE payload (hq's GET lane never wraps reads in an ok-frame):
|
|
82
|
+
* `{ events: [ {seq, family, kind, ...}, ... ], cursor: <newHead> }`
|
|
83
|
+
* Empty `events` after the timeout is a normal, successful response.
|
|
84
|
+
*
|
|
85
|
+
* Both lanes key off `CohortEvent.seq` — the per-org monotonic, DB-unique,
|
|
86
|
+
* gap-free ledger cursor hq's own poll route uses. Not the 30s ephemeral ring,
|
|
87
|
+
* not the 500-frame Redis tail: those lose data across a deploy, and a cursor
|
|
88
|
+
* that survives a restart is the entire point. A 404/405/501 on either path means
|
|
89
|
+
* "this hq doesn't have that endpoint yet" and demotes INSTANTLY without retries —
|
|
90
|
+
* an SDK that ships ahead of the server must not hammer it.
|
|
91
|
+
*
|
|
92
|
+
* ============================================================================
|
|
93
|
+
* Node builtins + existing maestro libs only. ESM. Every seam — clock, timers,
|
|
94
|
+
* fetch, RNG, the org client, the cadence enqueuer — is injectable, so the whole
|
|
95
|
+
* ladder is testable with no sockets and no real time.
|
|
96
|
+
*
|
|
97
|
+
* @module lib/org/push
|
|
98
|
+
*/
|
|
99
|
+
|
|
100
|
+
"use strict";
|
|
101
|
+
|
|
102
|
+
import { join, resolve, dirname } from "node:path";
|
|
103
|
+
import { mkdirSync, writeFileSync, readFileSync, existsSync, renameSync } from "node:fs";
|
|
104
|
+
|
|
105
|
+
import * as defaultClient from "./client.mjs";
|
|
106
|
+
import { PROTOCOL_VERSION } from "./protocol.mjs";
|
|
107
|
+
import { decorrelatedBackoff } from "../util/reconnect.mjs";
|
|
108
|
+
|
|
109
|
+
// ---------------------------------------------------------------------------
|
|
110
|
+
// Contract constants
|
|
111
|
+
// ---------------------------------------------------------------------------
|
|
112
|
+
|
|
113
|
+
/** The three rungs of the ladder. Frozen — this is a contract, not a hint. */
|
|
114
|
+
export const PUSH_RUNG = Object.freeze({
|
|
115
|
+
STREAM: "stream",
|
|
116
|
+
LONGPOLL: "longpoll",
|
|
117
|
+
CADENCE: "cadence",
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
/** Read path for the held stream (rung 1). */
|
|
121
|
+
export const DEFAULT_STREAM_PATH = "agent.stream";
|
|
122
|
+
/** Read path for the bounded long-poll (rung 2). */
|
|
123
|
+
export const DEFAULT_WAIT_PATH = "agent.wait";
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Long-poll hold, ms. 55s is not arbitrary: it is the bound hq's `approval.wait`
|
|
127
|
+
* clamps to and the only hold length PROVEN to survive Railway's proxy in front
|
|
128
|
+
* of `next start`. Do not raise it without measuring against the deployed app.
|
|
129
|
+
*/
|
|
130
|
+
export const DEFAULT_WAIT_MS = 55_000;
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Recycle a held stream after this long even when it looks healthy (ms). A
|
|
134
|
+
* half-open connection through a proxy is indistinguishable from an idle one
|
|
135
|
+
* until you try to write; periodically re-establishing turns "silently deaf
|
|
136
|
+
* forever" into "at most a 15-minute deafness, and we'd have noticed via the
|
|
137
|
+
* idle watchdog first anyway".
|
|
138
|
+
*/
|
|
139
|
+
export const DEFAULT_STREAM_MAX_MS = 15 * 60_000;
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* Abort a stream that has produced NO bytes at all (not even a heartbeat) for
|
|
143
|
+
* this long (ms). hq is expected to heartbeat; absent bytes means the connection
|
|
144
|
+
* is dead or the proxy is buffering us into uselessness — either way, drop it and
|
|
145
|
+
* let the ladder decide.
|
|
146
|
+
*/
|
|
147
|
+
export const DEFAULT_IDLE_MS = 90_000;
|
|
148
|
+
|
|
149
|
+
/** Consecutive failures on a rung before demoting to the next one down. */
|
|
150
|
+
export const DEFAULT_DEMOTE_AFTER = 3;
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* How long to sit on a lower rung before probing the one above (ms). Applies to
|
|
154
|
+
* both LONGPOLL→STREAM and CADENCE→STREAM: a transient hq deploy must not
|
|
155
|
+
* permanently strand the agent on the slow lane.
|
|
156
|
+
*/
|
|
157
|
+
export const DEFAULT_PROMOTE_AFTER_MS = 5 * 60_000;
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* Wake debounce (ms). A 40-message burst in a busy channel is ONE thing to go
|
|
161
|
+
* look at, not 40. Leading-edge fire (first event wakes instantly — that is the
|
|
162
|
+
* product promise) + trailing-edge fire (anything that arrived during the window
|
|
163
|
+
* still gets a wake, so the tail of a burst is never stranded until the 45s
|
|
164
|
+
* cadence).
|
|
165
|
+
*/
|
|
166
|
+
export const DEFAULT_WAKE_COALESCE_MS = 1_500;
|
|
167
|
+
|
|
168
|
+
/** Cursor + status files, relative to the agent root. */
|
|
169
|
+
export const CURSOR_RELATIVE = "state/org/push-cursor.json";
|
|
170
|
+
export const STATUS_RELATIVE = "state/org/push-status.json";
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* family → wake target. The target NAMES a fetch path; it does not perform one.
|
|
174
|
+
* Handlers live in `wake` (see DEFAULT_WAKE_HANDLERS) so an agent repo can add,
|
|
175
|
+
* replace, or disable any of them without touching this module.
|
|
176
|
+
*/
|
|
177
|
+
export const FAMILY_WAKE_TARGET = Object.freeze({
|
|
178
|
+
messaging: "org-messaging",
|
|
179
|
+
calling: "org-messaging",
|
|
180
|
+
email: "orgmail",
|
|
181
|
+
board: "board",
|
|
182
|
+
work: "board",
|
|
183
|
+
branding: "brand",
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Cadence a target's DEFAULT handler ticks. Only `org-messaging` is enabled by
|
|
188
|
+
* default, deliberately:
|
|
189
|
+
*
|
|
190
|
+
* - `messaging-inbound` is GUARDED and routes inline — it pulls directed items
|
|
191
|
+
* and writes inbox files. No LLM session is spawned by the tick itself, so a
|
|
192
|
+
* chatty channel costs I/O, not money.
|
|
193
|
+
* - `backlog-executor` / `brand-steward` ESCALATE (they spawn a session when
|
|
194
|
+
* they find work). Waking those on every board or branding event would let a
|
|
195
|
+
* third party drive this agent's spend. Opt in explicitly if you want it:
|
|
196
|
+
* wake: { board: () => enqueueTick({ cadence: "backlog-executor", ... }) }
|
|
197
|
+
* - `orgmail` has no cadence at all — its fetcher is the channel adapter's own
|
|
198
|
+
* poll loop, which the daemon owns. The wiring can pass a handler that pokes
|
|
199
|
+
* that adapter; absent one, email degrades to the adapter's 45s poll, which
|
|
200
|
+
* is exactly today's behaviour.
|
|
201
|
+
*/
|
|
202
|
+
export const TARGET_CADENCE = Object.freeze({
|
|
203
|
+
"org-messaging": "messaging-inbound",
|
|
204
|
+
});
|
|
205
|
+
|
|
206
|
+
// ---------------------------------------------------------------------------
|
|
207
|
+
// Logging — never throws, always loud enough to be found
|
|
208
|
+
// ---------------------------------------------------------------------------
|
|
209
|
+
|
|
210
|
+
function logInfo(msg) { try { console.log(`[org-push] ${msg}`); } catch { /* never throw from logging */ } }
|
|
211
|
+
function logWarn(msg) { try { console.warn(`[org-push] ${msg}`); } catch { /* never throw from logging */ } }
|
|
212
|
+
|
|
213
|
+
// ---------------------------------------------------------------------------
|
|
214
|
+
// Cursor persistence — the one piece of durable state
|
|
215
|
+
// ---------------------------------------------------------------------------
|
|
216
|
+
|
|
217
|
+
/**
|
|
218
|
+
* Read the persisted ledger cursor. Returns `null` when we have never run (or
|
|
219
|
+
* the file is unreadable/corrupt).
|
|
220
|
+
*
|
|
221
|
+
* `null` is MEANINGFULLY DIFFERENT FROM 0 and the distinction is load-bearing:
|
|
222
|
+
* hq treats an ABSENT cursor as "bootstrap at head, replay nothing", and treats
|
|
223
|
+
* `cursor=0` as "replay the entire org ledger from the beginning". A fresh agent
|
|
224
|
+
* that sent 0 would wake on every event in the workspace's history. So: no file →
|
|
225
|
+
* send no cursor at all. Anything older than boot is the cadences' problem, and
|
|
226
|
+
* they have their own cursors for it.
|
|
227
|
+
*
|
|
228
|
+
* @param {string} agentRoot
|
|
229
|
+
* @returns {number|null}
|
|
230
|
+
*/
|
|
231
|
+
export function readCursor(agentRoot) {
|
|
232
|
+
try {
|
|
233
|
+
const p = join(agentRoot, CURSOR_RELATIVE);
|
|
234
|
+
if (!existsSync(p)) return null;
|
|
235
|
+
const raw = JSON.parse(readFileSync(p, "utf8"));
|
|
236
|
+
const n = Number(raw && raw.cursor);
|
|
237
|
+
// A non-positive / non-finite cursor is corrupt, not "start from zero" —
|
|
238
|
+
// treat it as absent (bootstrap at head) rather than replaying all history.
|
|
239
|
+
return Number.isFinite(n) && n > 0 ? Math.floor(n) : null;
|
|
240
|
+
} catch {
|
|
241
|
+
return null; // fail-open: a corrupt cursor bootstraps at head
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* Persist the ledger cursor (tmp + rename, so a crash mid-write can never leave a
|
|
247
|
+
* truncated file that reads as "bootstrap"). Fail-open; never throws.
|
|
248
|
+
* @param {string} agentRoot
|
|
249
|
+
* @param {number} cursor
|
|
250
|
+
* @param {Function} [now]
|
|
251
|
+
*/
|
|
252
|
+
export function writeCursor(agentRoot, cursor, now = Date.now) {
|
|
253
|
+
const n = Number(cursor);
|
|
254
|
+
if (!Number.isFinite(n) || n <= 0) return;
|
|
255
|
+
try {
|
|
256
|
+
const p = join(agentRoot, CURSOR_RELATIVE);
|
|
257
|
+
mkdirSync(dirname(p), { recursive: true });
|
|
258
|
+
const tmp = `${p}.${process.pid}.${Date.now()}.tmp`;
|
|
259
|
+
writeFileSync(tmp, `${JSON.stringify({ cursor: Math.floor(n), updatedAt: new Date(now()).toISOString() }, null, 2)}\n`);
|
|
260
|
+
renameSync(tmp, p);
|
|
261
|
+
} catch { /* fail-open */ }
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
/**
|
|
265
|
+
* Publish the channel's current rung to disk so `doctor`/an operator can see
|
|
266
|
+
* WHICH lane an agent is on without reading logs. A push channel that quietly
|
|
267
|
+
* degraded to the 45s cadence and told nobody is the silent failure this whole
|
|
268
|
+
* codebase is allergic to. Fail-open.
|
|
269
|
+
*/
|
|
270
|
+
function writeStatus(ctx, extra = {}) {
|
|
271
|
+
try {
|
|
272
|
+
const p = join(ctx.agentRoot, STATUS_RELATIVE);
|
|
273
|
+
mkdirSync(dirname(p), { recursive: true });
|
|
274
|
+
const body = {
|
|
275
|
+
rung: ctx.rung,
|
|
276
|
+
since: new Date(ctx.rungSinceMs).toISOString(),
|
|
277
|
+
cursor: ctx.cursor,
|
|
278
|
+
reason: ctx.rungReason || null,
|
|
279
|
+
cadenceFallbackActive: true, // ALWAYS true — the cadences never stop.
|
|
280
|
+
updatedAt: new Date(ctx.now()).toISOString(),
|
|
281
|
+
...extra,
|
|
282
|
+
};
|
|
283
|
+
const tmp = `${p}.${process.pid}.${Date.now()}.tmp`;
|
|
284
|
+
writeFileSync(tmp, `${JSON.stringify(body, null, 2)}\n`);
|
|
285
|
+
renameSync(tmp, p);
|
|
286
|
+
} catch { /* fail-open */ }
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
// ---------------------------------------------------------------------------
|
|
290
|
+
// Frame decoding — tolerant on purpose
|
|
291
|
+
// ---------------------------------------------------------------------------
|
|
292
|
+
|
|
293
|
+
/**
|
|
294
|
+
* Decode one wire line into a typed outcome. Understands SSE framing and bare
|
|
295
|
+
* NDJSON, because hq's house streaming format is NDJSON while the requested rung
|
|
296
|
+
* is "SSE" — accepting both costs ~10 lines and removes a deployment coin-flip.
|
|
297
|
+
*
|
|
298
|
+
* Never throws. An unparseable or unrecognised line is `{kind:"malformed"}` /
|
|
299
|
+
* `{kind:"ignore"}`, and the caller keeps reading the stream: one bad byte must
|
|
300
|
+
* not cost us the connection, let alone the daemon.
|
|
301
|
+
*
|
|
302
|
+
* @param {string} line
|
|
303
|
+
* @returns {{kind:"event"|"heartbeat"|"ready"|"ignore"|"malformed", event?:object, cursor?:number}}
|
|
304
|
+
*/
|
|
305
|
+
export function decodeFrameLine(line) {
|
|
306
|
+
const s = typeof line === "string" ? line.trim() : "";
|
|
307
|
+
if (!s) return { kind: "ignore" };
|
|
308
|
+
// SSE comment — the canonical keepalive. Counts as liveness, carries nothing.
|
|
309
|
+
if (s.startsWith(":")) return { kind: "heartbeat" };
|
|
310
|
+
|
|
311
|
+
let json = s;
|
|
312
|
+
if (s.startsWith("data:")) {
|
|
313
|
+
json = s.slice(5).trim();
|
|
314
|
+
if (!json) return { kind: "ignore" };
|
|
315
|
+
} else if (/^(event|id|retry):/i.test(s)) {
|
|
316
|
+
// Other SSE field lines. We key off the ledger seq inside `data:`, never off
|
|
317
|
+
// SSE's own `id:`/`event:` fields, so these are pure noise to us.
|
|
318
|
+
return { kind: "ignore" };
|
|
319
|
+
} else if (!(s.startsWith("{") || s.startsWith("["))) {
|
|
320
|
+
// Not JSON and not an SSE field we know → ignore rather than cry malformed
|
|
321
|
+
// (proxies and frameworks inject blank/padding lines).
|
|
322
|
+
return { kind: "ignore" };
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
let obj;
|
|
326
|
+
try { obj = JSON.parse(json); } catch { return { kind: "malformed" }; }
|
|
327
|
+
if (!obj || typeof obj !== "object" || Array.isArray(obj)) return { kind: "malformed" };
|
|
328
|
+
|
|
329
|
+
const type = typeof obj.type === "string" ? obj.type : "";
|
|
330
|
+
if (type === "heartbeat" || type === "ping" || type === "keepalive") return { kind: "heartbeat" };
|
|
331
|
+
if (type === "ready" || type === "hello") {
|
|
332
|
+
const c = Number(obj.cursor);
|
|
333
|
+
return { kind: "ready", cursor: Number.isFinite(c) ? c : undefined };
|
|
334
|
+
}
|
|
335
|
+
if (type === "error") return { kind: "malformed" }; // a server-side error frame ends the read
|
|
336
|
+
|
|
337
|
+
// An event either says so, or is recognised structurally: hq's own fan-out
|
|
338
|
+
// frame is {family, kind, entityId, actor, at, seq, payload} with NO `type`.
|
|
339
|
+
const seq = Number(obj.seq);
|
|
340
|
+
const hasShape = Number.isFinite(seq) && typeof obj.family === "string" && obj.family !== "";
|
|
341
|
+
if (type === "event" || hasShape) {
|
|
342
|
+
if (!hasShape) return { kind: "malformed" }; // claimed to be an event, isn't one
|
|
343
|
+
return { kind: "event", event: obj };
|
|
344
|
+
}
|
|
345
|
+
return { kind: "ignore" }; // forward-compatible: unknown frame types are fine
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
// ---------------------------------------------------------------------------
|
|
349
|
+
// Relevance — a coarse filter, never a verdict
|
|
350
|
+
// ---------------------------------------------------------------------------
|
|
351
|
+
|
|
352
|
+
/**
|
|
353
|
+
* Should this event wake anything, and which target? Returns null to skip.
|
|
354
|
+
*
|
|
355
|
+
* This is DELIBERATELY coarse. The frame cannot prove an event is ours (see the
|
|
356
|
+
* module header), so the only safe direction is to wake and let the authoritative
|
|
357
|
+
* fetch decide. The single exception hq gives us is `email.received`, whose
|
|
358
|
+
* payload really does name the owning seat (`memberId`) — and even there we only
|
|
359
|
+
* skip when the caller supplied ids to compare against AND the payload names a
|
|
360
|
+
* different one. When we cannot prove it isn't ours, we wake.
|
|
361
|
+
*
|
|
362
|
+
* @param {object} ctx
|
|
363
|
+
* @param {object} ev
|
|
364
|
+
* @returns {string|null} wake target name, or null
|
|
365
|
+
*/
|
|
366
|
+
function wakeTargetFor(ctx, ev) {
|
|
367
|
+
const family = typeof ev.family === "string" ? ev.family : "";
|
|
368
|
+
const target = FAMILY_WAKE_TARGET[family];
|
|
369
|
+
if (!target) return null;
|
|
370
|
+
|
|
371
|
+
// Our own committed action echoing back is not news. Only applies when the
|
|
372
|
+
// caller told us which ids are "us" — otherwise we cannot know, so we wake.
|
|
373
|
+
if (ctx.selfIds.size > 0 && ev.actor && ctx.selfIds.has(String(ev.actor))) return null;
|
|
374
|
+
|
|
375
|
+
// The one payload that names a seat. Skip only on a POSITIVE mismatch.
|
|
376
|
+
const memberId = ev.payload && ev.payload.memberId;
|
|
377
|
+
if (ctx.selfIds.size > 0 && memberId && !ctx.selfIds.has(String(memberId))) return null;
|
|
378
|
+
|
|
379
|
+
return target;
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
// ---------------------------------------------------------------------------
|
|
383
|
+
// Wake dispatch — leading + trailing debounce
|
|
384
|
+
// ---------------------------------------------------------------------------
|
|
385
|
+
|
|
386
|
+
/**
|
|
387
|
+
* Default handler for the `org-messaging` target: enqueue a high-priority
|
|
388
|
+
* `messaging-inbound` tick. The cadence consumer drains the bus every ~2s and
|
|
389
|
+
* `guardMessagingInbound` then does the real, authoritative pull from ITS own
|
|
390
|
+
* cursor — so this is a nudge onto an existing, already-correct path, not a
|
|
391
|
+
* second ingestion route.
|
|
392
|
+
*/
|
|
393
|
+
async function defaultCadenceWake(ctx, target, meta) {
|
|
394
|
+
const cadence = TARGET_CADENCE[target];
|
|
395
|
+
if (!cadence) {
|
|
396
|
+
// No default fetcher for this target (e.g. orgmail, whose fetcher is the
|
|
397
|
+
// channel adapter the daemon owns). Degrading to that adapter's own poll is
|
|
398
|
+
// correct — it is today's behaviour — but say so once, loudly enough to find.
|
|
399
|
+
if (!ctx.warnedTargets.has(target)) {
|
|
400
|
+
ctx.warnedTargets.add(target);
|
|
401
|
+
logInfo(`no wake handler for target "${target}" — that lane keeps its own poll cadence (no regression, just no speed-up)`);
|
|
402
|
+
}
|
|
403
|
+
return false;
|
|
404
|
+
}
|
|
405
|
+
const enqueue = ctx.enqueueTick || (await import("../cadence-bus.mjs")).enqueueTick;
|
|
406
|
+
enqueue({
|
|
407
|
+
cadence,
|
|
408
|
+
type: "cadence_tick",
|
|
409
|
+
source: "org-push",
|
|
410
|
+
priority: "high",
|
|
411
|
+
agentRoot: ctx.agentRoot,
|
|
412
|
+
metadata: { pushSeq: meta.seq, family: meta.family, kind: meta.kind, rung: ctx.rung },
|
|
413
|
+
});
|
|
414
|
+
return true;
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
/**
|
|
418
|
+
* Fire a wake for `target`, debounced. Leading edge fires immediately (the whole
|
|
419
|
+
* point is speed); anything arriving inside the window sets a trailing wake so
|
|
420
|
+
* the tail of a burst is never stranded waiting for the 45s cadence.
|
|
421
|
+
*
|
|
422
|
+
* Fully fail-open: a throwing handler is logged and swallowed. A wake that fails
|
|
423
|
+
* costs latency, never correctness — the cadence still runs.
|
|
424
|
+
*/
|
|
425
|
+
function scheduleWake(ctx, target, meta) {
|
|
426
|
+
const nowMs = ctx.now();
|
|
427
|
+
const state = ctx.wakeState.get(target) || { cooldownUntil: 0, trailing: null, timer: null };
|
|
428
|
+
ctx.wakeState.set(target, state);
|
|
429
|
+
|
|
430
|
+
const fire = (m) => {
|
|
431
|
+
state.cooldownUntil = ctx.now() + ctx.wakeCoalesceMs;
|
|
432
|
+
Promise.resolve()
|
|
433
|
+
.then(() => {
|
|
434
|
+
const handler = ctx.wake[target];
|
|
435
|
+
if (handler === null || handler === false) return false; // explicitly disabled
|
|
436
|
+
if (typeof handler === "function") return handler(m, ctx);
|
|
437
|
+
return defaultCadenceWake(ctx, target, m);
|
|
438
|
+
})
|
|
439
|
+
.catch((err) => logWarn(`wake handler "${target}" failed (${err && err.message}) — the ${ctx.cadenceNote} still covers this`));
|
|
440
|
+
};
|
|
441
|
+
|
|
442
|
+
if (nowMs >= state.cooldownUntil) { fire(meta); return; }
|
|
443
|
+
|
|
444
|
+
// Inside the debounce window → remember the newest event and make sure exactly
|
|
445
|
+
// one trailing wake is scheduled for when the window closes.
|
|
446
|
+
state.trailing = meta;
|
|
447
|
+
if (state.timer == null) {
|
|
448
|
+
const delay = Math.max(1, state.cooldownUntil - nowMs);
|
|
449
|
+
state.timer = ctx.setTimeoutFn(() => {
|
|
450
|
+
state.timer = null;
|
|
451
|
+
const m = state.trailing;
|
|
452
|
+
state.trailing = null;
|
|
453
|
+
if (m && !ctx.stopped) fire(m);
|
|
454
|
+
}, delay);
|
|
455
|
+
if (state.timer && typeof state.timer.unref === "function") { try { state.timer.unref(); } catch { /* */ } }
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
// ---------------------------------------------------------------------------
|
|
460
|
+
// Event dispatch — the watermark
|
|
461
|
+
// ---------------------------------------------------------------------------
|
|
462
|
+
|
|
463
|
+
/**
|
|
464
|
+
* Consume one event frame: dedup against the watermark, wake the right target,
|
|
465
|
+
* then advance + persist the cursor.
|
|
466
|
+
*
|
|
467
|
+
* ORDERING (copied from hq's own connection-manager, which solved this): the
|
|
468
|
+
* watermark is exclusive-lower-bound, so a frame at or below the cursor is a
|
|
469
|
+
* duplicate and is dropped. Duplicates genuinely happen — a long-poll response
|
|
470
|
+
* and a re-established stream overlap by design, and overlapping is much safer
|
|
471
|
+
* than gapping.
|
|
472
|
+
*
|
|
473
|
+
* The cursor advances even for events we ignored (wrong family, someone else's
|
|
474
|
+
* mailbox): they ARE consumed, and not advancing would re-deliver them forever.
|
|
475
|
+
*
|
|
476
|
+
* The cursor also advances when the wake handler fails. That is a deliberate
|
|
477
|
+
* trade: this channel carries HINTS, and the authoritative fetch behind the 45s
|
|
478
|
+
* cadence will find the work regardless. Refusing to advance would rebuild the
|
|
479
|
+
* whole poison-message problem on a lane that has a backstop by construction.
|
|
480
|
+
*
|
|
481
|
+
* @returns {"delivered"|"skipped"|"duplicate"|"malformed"}
|
|
482
|
+
*/
|
|
483
|
+
function dispatchEvent(ctx, ev) {
|
|
484
|
+
const seq = Number(ev && ev.seq);
|
|
485
|
+
if (!Number.isFinite(seq) || seq <= 0) { ctx.stats.malformed += 1; return "malformed"; }
|
|
486
|
+
if (ctx.cursor != null && seq <= ctx.cursor) { ctx.stats.duplicate += 1; return "duplicate"; }
|
|
487
|
+
|
|
488
|
+
let outcome = "skipped";
|
|
489
|
+
try {
|
|
490
|
+
const target = wakeTargetFor(ctx, ev);
|
|
491
|
+
if (target) {
|
|
492
|
+
scheduleWake(ctx, target, { seq, family: ev.family, kind: ev.kind, entityId: ev.entityId, at: ev.at });
|
|
493
|
+
ctx.stats.woke += 1;
|
|
494
|
+
outcome = "delivered";
|
|
495
|
+
}
|
|
496
|
+
if (typeof ctx.onEvent === "function") {
|
|
497
|
+
// Observer seam — additive, never load-bearing, independently fail-open.
|
|
498
|
+
try { ctx.onEvent({ ...ev, seq, rung: ctx.rung }); } catch (err) { logWarn(`onEvent observer threw (${err && err.message})`); }
|
|
499
|
+
}
|
|
500
|
+
} catch (err) {
|
|
501
|
+
logWarn(`event dispatch failed at seq ${seq} (${err && err.message}) — advancing anyway; the ${ctx.cadenceNote} covers it`);
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
ctx.cursor = seq;
|
|
505
|
+
ctx.stats.lastSeqAt = ctx.now();
|
|
506
|
+
writeCursor(ctx.agentRoot, seq, ctx.now);
|
|
507
|
+
return outcome;
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
// ---------------------------------------------------------------------------
|
|
511
|
+
// URLs + headers
|
|
512
|
+
// ---------------------------------------------------------------------------
|
|
513
|
+
|
|
514
|
+
/**
|
|
515
|
+
* Build a `/v1` URL. Mirrors lib/org/client.mjs's private `v1Url` — that helper
|
|
516
|
+
* is module-private and client.mjs is owned elsewhere, so we re-derive the same
|
|
517
|
+
* two rules here rather than reach into it: an already-`/v1`-suffixed base is
|
|
518
|
+
* used verbatim, anything else gets `/api/v1/`.
|
|
519
|
+
*/
|
|
520
|
+
function v1Url(base, path) {
|
|
521
|
+
const b = String(base || "").replace(/\/+$/, "");
|
|
522
|
+
if (/\/(?:api\/)?v1$/.test(b)) return `${b}/${path}`;
|
|
523
|
+
return `${b}/api/v1/${path}`;
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
/** Auth headers for a GET on the /v1 binding. Same shape client.mjs sends. */
|
|
527
|
+
function streamHeaders(ctx) {
|
|
528
|
+
const h = {
|
|
529
|
+
accept: "text/event-stream, application/x-ndjson;q=0.9, application/json;q=0.8",
|
|
530
|
+
"x-org-protocol": String(PROTOCOL_VERSION),
|
|
531
|
+
// Best-effort hint to proxies that buffer: we want bytes forwarded as they
|
|
532
|
+
// arrive. Harmless where unsupported.
|
|
533
|
+
"cache-control": "no-cache",
|
|
534
|
+
};
|
|
535
|
+
if (ctx.token) h.authorization = `Bearer ${ctx.token}`;
|
|
536
|
+
if (ctx.orgId) h["x-org-id"] = String(ctx.orgId);
|
|
537
|
+
return h;
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
/** `?cursor=` only when we HAVE one — see readCursor for why 0 is not a cursor. */
|
|
541
|
+
function cursorQuery(ctx) {
|
|
542
|
+
return ctx.cursor != null ? `?cursor=${encodeURIComponent(ctx.cursor)}` : "";
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
// ---------------------------------------------------------------------------
|
|
546
|
+
// Rung 1 — the held stream
|
|
547
|
+
// ---------------------------------------------------------------------------
|
|
548
|
+
|
|
549
|
+
/**
|
|
550
|
+
* Iterate a fetch response body as chunks, tolerating both shapes Node gives us:
|
|
551
|
+
* an async-iterable (undici's default) and a WHATWG reader. Tests inject plain
|
|
552
|
+
* async generators, which hit the first branch.
|
|
553
|
+
*/
|
|
554
|
+
async function* iterateBody(body) {
|
|
555
|
+
if (!body) return;
|
|
556
|
+
if (typeof body[Symbol.asyncIterator] === "function") {
|
|
557
|
+
for await (const chunk of body) yield chunk;
|
|
558
|
+
return;
|
|
559
|
+
}
|
|
560
|
+
if (typeof body.getReader === "function") {
|
|
561
|
+
const reader = body.getReader();
|
|
562
|
+
try {
|
|
563
|
+
for (;;) {
|
|
564
|
+
const { value, done } = await reader.read();
|
|
565
|
+
if (done) return;
|
|
566
|
+
if (value) yield value;
|
|
567
|
+
}
|
|
568
|
+
} finally {
|
|
569
|
+
try { reader.releaseLock(); } catch { /* fail-open */ }
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
/** HTTP statuses that mean "this hq does not have this endpoint". */
|
|
575
|
+
function isUnsupportedStatus(status) {
|
|
576
|
+
return status === 404 || status === 405 || status === 501;
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
/**
|
|
580
|
+
* Open the stream and consume it until it ends, errors, goes idle, or hits the
|
|
581
|
+
* max hold. Returns a typed outcome; NEVER throws.
|
|
582
|
+
*
|
|
583
|
+
* @returns {Promise<{ok:boolean, unsupported?:boolean, reason:string, delivered:number, lived:number}>}
|
|
584
|
+
*/
|
|
585
|
+
async function consumeStream(ctx) {
|
|
586
|
+
const startedAt = ctx.now();
|
|
587
|
+
const f = ctx.fetchImpl || globalThis.fetch;
|
|
588
|
+
if (typeof f !== "function") return { ok: false, unsupported: true, reason: "no fetch implementation", delivered: 0, lived: 0 };
|
|
589
|
+
|
|
590
|
+
const url = v1Url(ctx.base, `${ctx.streamPath}${cursorQuery(ctx)}`);
|
|
591
|
+
const ac = typeof AbortController === "function" ? new AbortController() : null;
|
|
592
|
+
ctx.abort = ac;
|
|
593
|
+
|
|
594
|
+
// Two watchdogs on one abort: idle (no bytes at all) and max-hold (recycle).
|
|
595
|
+
let idleTimer = null;
|
|
596
|
+
const armIdle = () => {
|
|
597
|
+
if (idleTimer != null) { try { ctx.clearTimeoutFn(idleTimer); } catch { /* */ } }
|
|
598
|
+
idleTimer = ctx.setTimeoutFn(() => {
|
|
599
|
+
ctx.streamAbortReason = "idle";
|
|
600
|
+
try { ac && ac.abort(); } catch { /* */ }
|
|
601
|
+
}, ctx.idleMs);
|
|
602
|
+
if (idleTimer && typeof idleTimer.unref === "function") { try { idleTimer.unref(); } catch { /* */ } }
|
|
603
|
+
};
|
|
604
|
+
const maxTimer = ctx.setTimeoutFn(() => {
|
|
605
|
+
ctx.streamAbortReason = "max-hold";
|
|
606
|
+
try { ac && ac.abort(); } catch { /* */ }
|
|
607
|
+
}, ctx.streamMaxMs);
|
|
608
|
+
if (maxTimer && typeof maxTimer.unref === "function") { try { maxTimer.unref(); } catch { /* */ } }
|
|
609
|
+
const disarm = () => {
|
|
610
|
+
if (idleTimer != null) { try { ctx.clearTimeoutFn(idleTimer); } catch { /* */ } idleTimer = null; }
|
|
611
|
+
try { ctx.clearTimeoutFn(maxTimer); } catch { /* */ }
|
|
612
|
+
};
|
|
613
|
+
|
|
614
|
+
ctx.streamAbortReason = null;
|
|
615
|
+
let delivered = 0;
|
|
616
|
+
try {
|
|
617
|
+
armIdle();
|
|
618
|
+
let res;
|
|
619
|
+
try {
|
|
620
|
+
res = await f(url, { method: "GET", headers: streamHeaders(ctx), signal: ac ? ac.signal : undefined });
|
|
621
|
+
} catch (err) {
|
|
622
|
+
return { ok: false, reason: `connect failed: ${err && err.message}`, delivered, lived: ctx.now() - startedAt };
|
|
623
|
+
}
|
|
624
|
+
const status = Number(res && res.status) || 0;
|
|
625
|
+
if (isUnsupportedStatus(status)) {
|
|
626
|
+
return { ok: false, unsupported: true, reason: `http ${status} (endpoint absent)`, delivered, lived: ctx.now() - startedAt };
|
|
627
|
+
}
|
|
628
|
+
if (status !== 200) {
|
|
629
|
+
// 401/403 is a credential problem, not a transport one — say so plainly;
|
|
630
|
+
// it is the exact failure that made mesh beats look healthy for hours.
|
|
631
|
+
const hint = status === 401 || status === 403 ? " — check COHORT_API_KEY is present and paired" : "";
|
|
632
|
+
return { ok: false, reason: `http ${status}${hint}`, delivered, lived: ctx.now() - startedAt };
|
|
633
|
+
}
|
|
634
|
+
if (!res.body) {
|
|
635
|
+
return { ok: false, unsupported: true, reason: "no response body (not a stream)", delivered, lived: ctx.now() - startedAt };
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
const decoder = new TextDecoder();
|
|
639
|
+
let buf = "";
|
|
640
|
+
for await (const chunk of iterateBody(res.body)) {
|
|
641
|
+
if (ctx.stopped) break;
|
|
642
|
+
armIdle(); // ANY byte, including a heartbeat comment, proves liveness
|
|
643
|
+
buf += typeof chunk === "string" ? chunk : decoder.decode(chunk, { stream: true });
|
|
644
|
+
let nl;
|
|
645
|
+
while ((nl = buf.indexOf("\n")) >= 0) {
|
|
646
|
+
const line = buf.slice(0, nl);
|
|
647
|
+
buf = buf.slice(nl + 1);
|
|
648
|
+
const frame = decodeFrameLine(line);
|
|
649
|
+
if (frame.kind === "event") {
|
|
650
|
+
if (dispatchEvent(ctx, frame.event) === "delivered") delivered += 1;
|
|
651
|
+
} else if (frame.kind === "malformed") {
|
|
652
|
+
// ONE bad frame must not cost the connection: count it, keep reading.
|
|
653
|
+
ctx.stats.malformed += 1;
|
|
654
|
+
if (ctx.stats.malformed <= 5) logWarn(`ignoring malformed stream frame (total ${ctx.stats.malformed})`);
|
|
655
|
+
}
|
|
656
|
+
// heartbeat / ready / ignore: liveness already recorded above.
|
|
657
|
+
}
|
|
658
|
+
if (buf.length > 1_000_000) buf = ""; // a frame this large is a bug, not a frame
|
|
659
|
+
}
|
|
660
|
+
const reason = ctx.streamAbortReason || (ctx.stopped ? "stopped" : "eof");
|
|
661
|
+
// An idle abort is a FAILED stream (the server went quiet on us); a clean EOF
|
|
662
|
+
// or a scheduled recycle is a healthy end — reconnect immediately.
|
|
663
|
+
const ok = reason !== "idle";
|
|
664
|
+
return { ok, reason, delivered, lived: ctx.now() - startedAt };
|
|
665
|
+
} catch (err) {
|
|
666
|
+
// An abort surfaces HERE, as a throw out of the body iterator — which means
|
|
667
|
+
// this catch has to tell a deliberate teardown apart from a real failure.
|
|
668
|
+
// A scheduled recycle (`max-hold`) and our own `stop()` are successes: the
|
|
669
|
+
// connection did its job and we ended it on purpose, so they must not spend
|
|
670
|
+
// the rung's failure budget and must not trigger a demotion. Only an idle
|
|
671
|
+
// timeout or a genuine transport error counts against us.
|
|
672
|
+
const reason = ctx.streamAbortReason || (ctx.stopped ? "stopped" : `read failed: ${err && err.message}`);
|
|
673
|
+
const ok = reason === "max-hold" || reason === "stopped";
|
|
674
|
+
return { ok, reason, delivered, lived: ctx.now() - startedAt };
|
|
675
|
+
} finally {
|
|
676
|
+
disarm();
|
|
677
|
+
ctx.abort = null;
|
|
678
|
+
}
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
// ---------------------------------------------------------------------------
|
|
682
|
+
// Rung 2 — the bounded long-poll
|
|
683
|
+
// ---------------------------------------------------------------------------
|
|
684
|
+
|
|
685
|
+
/**
|
|
686
|
+
* One `agent.wait` round-trip. Follows the `approval.wait` precedent exactly:
|
|
687
|
+
* a GET on the read lane returning the BARE payload, with the client abort
|
|
688
|
+
* budgeted to outlast the server's own bound so the server gets to answer rather
|
|
689
|
+
* than us aborting first. Never throws.
|
|
690
|
+
*
|
|
691
|
+
* @returns {Promise<{ok:boolean, unsupported?:boolean, reason:string, delivered:number}>}
|
|
692
|
+
*/
|
|
693
|
+
async function waitOnce(ctx) {
|
|
694
|
+
const startedAt = ctx.now();
|
|
695
|
+
const q = ctx.cursor != null ? `?cursor=${encodeURIComponent(ctx.cursor)}&` : "?";
|
|
696
|
+
const path = `${ctx.waitPath}${q}timeoutMs=${encodeURIComponent(ctx.waitMs)}`;
|
|
697
|
+
let r;
|
|
698
|
+
try {
|
|
699
|
+
r = await ctx.client.read(path, {
|
|
700
|
+
base: ctx.base,
|
|
701
|
+
token: ctx.token,
|
|
702
|
+
orgId: ctx.orgId,
|
|
703
|
+
fetchImpl: ctx.fetchImpl,
|
|
704
|
+
timeoutMs: ctx.waitMs + 5_000,
|
|
705
|
+
});
|
|
706
|
+
} catch (err) {
|
|
707
|
+
// client.read is documented fail-open, but a caller must never assume.
|
|
708
|
+
return { ok: false, reason: `wait threw: ${err && err.message}`, delivered: 0, lived: ctx.now() - startedAt };
|
|
709
|
+
}
|
|
710
|
+
if (!r || !r.ok) {
|
|
711
|
+
const status = Number(r && r.status) || 0;
|
|
712
|
+
if (isUnsupportedStatus(status)) return { ok: false, unsupported: true, reason: `http ${status} (endpoint absent)`, delivered: 0, lived: ctx.now() - startedAt };
|
|
713
|
+
const code = (r && r.error && r.error.code) || "UNKNOWN";
|
|
714
|
+
return { ok: false, reason: `${code}${status ? ` (http ${status})` : ""}`, delivered: 0, lived: ctx.now() - startedAt };
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
const payload = r.payload || {};
|
|
718
|
+
const events = Array.isArray(payload.events) ? payload.events : Array.isArray(payload) ? payload : [];
|
|
719
|
+
let delivered = 0;
|
|
720
|
+
for (const ev of events) {
|
|
721
|
+
if (ctx.stopped) break;
|
|
722
|
+
const outcome = dispatchEvent(ctx, ev);
|
|
723
|
+
if (outcome === "delivered") delivered += 1;
|
|
724
|
+
}
|
|
725
|
+
// hq may report a head beyond the last event it sent (e.g. everything in the
|
|
726
|
+
// slice was filtered server-side). Trust it forward-only — never rewind.
|
|
727
|
+
const head = Number(payload.cursor);
|
|
728
|
+
if (Number.isFinite(head) && head > 0 && (ctx.cursor == null || head > ctx.cursor)) {
|
|
729
|
+
ctx.cursor = head;
|
|
730
|
+
writeCursor(ctx.agentRoot, head, ctx.now);
|
|
731
|
+
}
|
|
732
|
+
return {
|
|
733
|
+
ok: true,
|
|
734
|
+
reason: events.length ? `${events.length} event(s)` : "timeout (no events)",
|
|
735
|
+
delivered,
|
|
736
|
+
events: events.length,
|
|
737
|
+
lived: ctx.now() - startedAt,
|
|
738
|
+
};
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
// ---------------------------------------------------------------------------
|
|
742
|
+
// The ladder
|
|
743
|
+
// ---------------------------------------------------------------------------
|
|
744
|
+
|
|
745
|
+
/**
|
|
746
|
+
* Move to a rung and say so. Every transition is logged AND written to
|
|
747
|
+
* state/org/push-status.json — the channel is never allowed to change lanes in
|
|
748
|
+
* silence.
|
|
749
|
+
*/
|
|
750
|
+
function setRung(ctx, rung, reason) {
|
|
751
|
+
if (ctx.rung === rung) return;
|
|
752
|
+
const from = ctx.rung;
|
|
753
|
+
ctx.rung = rung;
|
|
754
|
+
ctx.rungSinceMs = ctx.now();
|
|
755
|
+
ctx.rungReason = reason;
|
|
756
|
+
ctx.failures = 0;
|
|
757
|
+
ctx.backoffMs = 0;
|
|
758
|
+
const line = `ladder ${from} → ${rung} (${reason})`;
|
|
759
|
+
if (rung === PUSH_RUNG.CADENCE) {
|
|
760
|
+
logWarn(`${line} — push is OFF; the agent falls back to the ${ctx.cadenceNote}, i.e. today's behaviour. Will re-probe in ${Math.round(ctx.promoteAfterMs / 1000)}s.`);
|
|
761
|
+
} else if (rung === PUSH_RUNG.LONGPOLL) {
|
|
762
|
+
logWarn(`${line} — degraded but still push: bounded ${Math.round(ctx.waitMs / 1000)}s waits.`);
|
|
763
|
+
} else {
|
|
764
|
+
logInfo(line);
|
|
765
|
+
}
|
|
766
|
+
writeStatus(ctx);
|
|
767
|
+
}
|
|
768
|
+
|
|
769
|
+
/** Demote one rung. STREAM → LONGPOLL → CADENCE; CADENCE is the floor. */
|
|
770
|
+
function demote(ctx, reason) {
|
|
771
|
+
if (ctx.rung === PUSH_RUNG.STREAM) return setRung(ctx, PUSH_RUNG.LONGPOLL, reason);
|
|
772
|
+
if (ctx.rung === PUSH_RUNG.LONGPOLL) return setRung(ctx, PUSH_RUNG.CADENCE, reason);
|
|
773
|
+
// Already on the floor: refresh the dwell clock so we don't probe in a tight
|
|
774
|
+
// loop against an hq that is plainly down.
|
|
775
|
+
ctx.rungSinceMs = ctx.now();
|
|
776
|
+
ctx.rungReason = reason;
|
|
777
|
+
writeStatus(ctx);
|
|
778
|
+
}
|
|
779
|
+
|
|
780
|
+
/**
|
|
781
|
+
* Run exactly ONE step of the ladder and return how long to wait before the next
|
|
782
|
+
* one. Exported through `_internals` so tests drive the state machine without
|
|
783
|
+
* real timers — the same discipline mesh.mjs uses with `beatOnce`.
|
|
784
|
+
*
|
|
785
|
+
* @returns {Promise<number>} delay in ms before the next step
|
|
786
|
+
*/
|
|
787
|
+
async function ladderStep(ctx) {
|
|
788
|
+
if (ctx.stopped) return 0;
|
|
789
|
+
|
|
790
|
+
// Promotion check first: any rung below STREAM is a temporary state, and a
|
|
791
|
+
// transient hq deploy must never strand an agent on the slow lane forever.
|
|
792
|
+
if (ctx.rung !== PUSH_RUNG.STREAM) {
|
|
793
|
+
const dwell = ctx.now() - ctx.rungSinceMs;
|
|
794
|
+
if (dwell >= ctx.promoteAfterMs) {
|
|
795
|
+
setRung(ctx, PUSH_RUNG.STREAM, "probing the preferred rung after a dwell period");
|
|
796
|
+
} else if (ctx.rung === PUSH_RUNG.CADENCE) {
|
|
797
|
+
// Nothing to do on the floor: the cadences are carrying the agent. Sleep
|
|
798
|
+
// out the remaining dwell rather than burning requests.
|
|
799
|
+
return Math.max(1_000, ctx.promoteAfterMs - dwell);
|
|
800
|
+
}
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
const rung = ctx.rung;
|
|
804
|
+
const res = rung === PUSH_RUNG.STREAM ? await consumeStream(ctx) : await waitOnce(ctx);
|
|
805
|
+
ctx.stats.attempts += 1;
|
|
806
|
+
if (ctx.stopped) return 0;
|
|
807
|
+
|
|
808
|
+
if (res.unsupported) {
|
|
809
|
+
// "This hq doesn't have that endpoint." Retrying is pointless and rude —
|
|
810
|
+
// demote immediately, no backoff spent, no failure budget consumed.
|
|
811
|
+
demote(ctx, `${rung} unsupported: ${res.reason}`);
|
|
812
|
+
return 0;
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
if (res.ok) {
|
|
816
|
+
ctx.failures = 0;
|
|
817
|
+
ctx.backoffMs = 0;
|
|
818
|
+
ctx.stats.ok += 1;
|
|
819
|
+
if (rung === PUSH_RUNG.STREAM) {
|
|
820
|
+
// A clean EOF/recycle is normal for a held stream: reconnect at once, with
|
|
821
|
+
// a small floor so a server that instantly closes cannot spin us hot.
|
|
822
|
+
const lived = Number(res.lived) || 0;
|
|
823
|
+
return lived < ctx.minHealthyMs ? ctx.baseBackoffMs : 0;
|
|
824
|
+
}
|
|
825
|
+
// Long-poll: immediately re-arm — the re-arm IS the mechanism. The one guard
|
|
826
|
+
// is against an `agent.wait` that answers instantly instead of holding (a
|
|
827
|
+
// half-implemented endpoint, or a proxy that terminates the hold): an empty
|
|
828
|
+
// answer that came back far faster than the hold we asked for would otherwise
|
|
829
|
+
// spin this loop at wire speed against hq. Floor it at the base backoff.
|
|
830
|
+
const livedWait = Number(res.lived) || 0;
|
|
831
|
+
if (!res.events && livedWait < ctx.minHealthyMs) return ctx.baseBackoffMs;
|
|
832
|
+
return 0;
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
ctx.failures += 1;
|
|
836
|
+
ctx.stats.failures += 1;
|
|
837
|
+
logWarn(`${rung} attempt failed (${res.reason}) [${ctx.failures}/${ctx.demoteAfter}] — the ${ctx.cadenceNote} is still covering this agent`);
|
|
838
|
+
if (ctx.failures >= ctx.demoteAfter) {
|
|
839
|
+
demote(ctx, `${ctx.demoteAfter} consecutive failures (last: ${res.reason})`);
|
|
840
|
+
return 0;
|
|
841
|
+
}
|
|
842
|
+
// Jittered decorrelated backoff — the same algorithm as lib/util/reconnect and
|
|
843
|
+
// lib/rate-guard, so the whole fleet backs off a struggling hq identically and
|
|
844
|
+
// never in lockstep.
|
|
845
|
+
ctx.backoffMs = decorrelatedBackoff(ctx.backoffMs, {
|
|
846
|
+
base: ctx.baseBackoffMs,
|
|
847
|
+
max: ctx.maxBackoffMs,
|
|
848
|
+
factor: ctx.backoffFactor,
|
|
849
|
+
rng: ctx.rng,
|
|
850
|
+
});
|
|
851
|
+
return ctx.backoffMs;
|
|
852
|
+
}
|
|
853
|
+
|
|
854
|
+
// ---------------------------------------------------------------------------
|
|
855
|
+
// Public entry point
|
|
856
|
+
// ---------------------------------------------------------------------------
|
|
857
|
+
|
|
858
|
+
/**
|
|
859
|
+
* Open the push channel. Returns a handle with `stop()`.
|
|
860
|
+
*
|
|
861
|
+
* A no-op (inert handle) when the org integration is disabled or the push channel
|
|
862
|
+
* is switched off, so the caller's shutdown path is always safe to call. NEVER
|
|
863
|
+
* throws: a push problem must not take the daemon down, and the 45s cadences keep
|
|
864
|
+
* running underneath regardless of what happens here.
|
|
865
|
+
*
|
|
866
|
+
* @param {object} o
|
|
867
|
+
* @param {string} o.agentRoot the agent repo root
|
|
868
|
+
* @param {object} o.cfg org config (lib/org/client.loadOrgConfig)
|
|
869
|
+
* @param {object} [o.client] injected org client (defaults to lib/org/client)
|
|
870
|
+
* @param {Function} [o.fetchImpl] injected fetch
|
|
871
|
+
* @param {Function} [o.now] injected clock () => ms
|
|
872
|
+
* @param {Function} [o.setTimeout] injected timer
|
|
873
|
+
* @param {Function} [o.clearTimeout] injected timer clearer
|
|
874
|
+
* @param {Function} [o.rng] injected uniform [0,1) for jitter
|
|
875
|
+
* @param {Function} [o.onEvent] observer, called per accepted event
|
|
876
|
+
* @param {object} [o.wake] target → handler (fn | null to disable)
|
|
877
|
+
* @param {Function} [o.enqueueTick] injected cadence-bus enqueuer
|
|
878
|
+
* @param {string[]} [o.selfIds] ids that mean "this seat" (actor/mailbox filter)
|
|
879
|
+
* @param {boolean} [o.enabled] force off (default: on when the org is enabled)
|
|
880
|
+
* @param {number} [o.waitMs] [o.idleMs] [o.streamMaxMs] [o.demoteAfter]
|
|
881
|
+
* @param {number} [o.promoteAfterMs] [o.wakeCoalesceMs]
|
|
882
|
+
* @param {number} [o.baseBackoffMs] [o.maxBackoffMs] [o.backoffFactor]
|
|
883
|
+
* @param {string} [o.startRung] start on a specific rung (tests/ops)
|
|
884
|
+
* @returns {Promise<{stop:Function, isEnabled:boolean, rung:Function, _ctx?:object}>}
|
|
885
|
+
*/
|
|
886
|
+
export async function connectOrgPush(o = {}) {
|
|
887
|
+
const client = o.client || defaultClient;
|
|
888
|
+
const cfg = o.cfg || {};
|
|
889
|
+
|
|
890
|
+
// Same gate as the mesh: disabled org → inert handle, zero network.
|
|
891
|
+
let enabled = false;
|
|
892
|
+
try { enabled = !!client.isEnabled(cfg); } catch { enabled = false; }
|
|
893
|
+
// Explicit kill switch for operators who want the pure-cadence world back.
|
|
894
|
+
if (o.enabled === false || process.env.COHORT_PUSH_DISABLED === "1") enabled = false;
|
|
895
|
+
if (!enabled) {
|
|
896
|
+
return { stop() {}, isEnabled: false, rung: () => PUSH_RUNG.CADENCE };
|
|
897
|
+
}
|
|
898
|
+
|
|
899
|
+
const agentRoot = resolve(o.agentRoot || process.env.AGENT_ROOT || process.env.AGENT_DIR || process.cwd());
|
|
900
|
+
let conn = {};
|
|
901
|
+
try { conn = client.configFromAgent(cfg) || {}; } catch { conn = {}; }
|
|
902
|
+
|
|
903
|
+
const num = (v, d) => (Number.isFinite(Number(v)) && Number(v) > 0 ? Number(v) : d);
|
|
904
|
+
// Some knobs are legitimately ZERO ("never coalesce", "trust any stream"), so
|
|
905
|
+
// they need a variant that treats 0 as a value rather than as "unset".
|
|
906
|
+
const num0 = (v, d) => (Number.isFinite(Number(v)) && Number(v) >= 0 ? Number(v) : d);
|
|
907
|
+
|
|
908
|
+
const ctx = {
|
|
909
|
+
agentRoot,
|
|
910
|
+
client,
|
|
911
|
+
fetchImpl: o.fetchImpl,
|
|
912
|
+
base: conn.base,
|
|
913
|
+
token: conn.token,
|
|
914
|
+
orgId: conn.orgId,
|
|
915
|
+
|
|
916
|
+
now: typeof o.now === "function" ? o.now : Date.now,
|
|
917
|
+
setTimeoutFn: typeof o.setTimeout === "function" ? o.setTimeout : setTimeout,
|
|
918
|
+
clearTimeoutFn: typeof o.clearTimeout === "function" ? o.clearTimeout : clearTimeout,
|
|
919
|
+
rng: typeof o.rng === "function" ? o.rng : undefined,
|
|
920
|
+
|
|
921
|
+
streamPath: o.streamPath || DEFAULT_STREAM_PATH,
|
|
922
|
+
waitPath: o.waitPath || DEFAULT_WAIT_PATH,
|
|
923
|
+
waitMs: Math.min(55_000, num(o.waitMs, DEFAULT_WAIT_MS)),
|
|
924
|
+
idleMs: num(o.idleMs, DEFAULT_IDLE_MS),
|
|
925
|
+
streamMaxMs: num(o.streamMaxMs, DEFAULT_STREAM_MAX_MS),
|
|
926
|
+
minHealthyMs: num0(o.minHealthyMs, 2_000),
|
|
927
|
+
demoteAfter: num(o.demoteAfter, DEFAULT_DEMOTE_AFTER),
|
|
928
|
+
promoteAfterMs: num(o.promoteAfterMs, DEFAULT_PROMOTE_AFTER_MS),
|
|
929
|
+
wakeCoalesceMs: num0(o.wakeCoalesceMs, DEFAULT_WAKE_COALESCE_MS),
|
|
930
|
+
baseBackoffMs: num(o.baseBackoffMs, 1_000),
|
|
931
|
+
maxBackoffMs: num(o.maxBackoffMs, 60_000),
|
|
932
|
+
backoffFactor: num(o.backoffFactor, 3),
|
|
933
|
+
|
|
934
|
+
onEvent: typeof o.onEvent === "function" ? o.onEvent : null,
|
|
935
|
+
wake: { ...(o.wake || {}) },
|
|
936
|
+
enqueueTick: typeof o.enqueueTick === "function" ? o.enqueueTick : null,
|
|
937
|
+
selfIds: new Set((Array.isArray(o.selfIds) ? o.selfIds : []).filter(Boolean).map(String)),
|
|
938
|
+
|
|
939
|
+
// The phrase we use everywhere we mention the backstop, so an operator reading
|
|
940
|
+
// logs always sees the same words for the same thing.
|
|
941
|
+
cadenceNote: "45s messaging-inbound cadence",
|
|
942
|
+
|
|
943
|
+
cursor: readCursor(agentRoot),
|
|
944
|
+
rung: PUSH_RUNG.STREAM,
|
|
945
|
+
rungSinceMs: 0,
|
|
946
|
+
rungReason: "boot",
|
|
947
|
+
failures: 0,
|
|
948
|
+
backoffMs: 0,
|
|
949
|
+
stopped: false,
|
|
950
|
+
timer: null,
|
|
951
|
+
abort: null,
|
|
952
|
+
streamAbortReason: null,
|
|
953
|
+
wakeState: new Map(),
|
|
954
|
+
warnedTargets: new Set(),
|
|
955
|
+
stats: { attempts: 0, ok: 0, failures: 0, malformed: 0, duplicate: 0, woke: 0, lastSeqAt: 0 },
|
|
956
|
+
};
|
|
957
|
+
ctx.rungSinceMs = ctx.now();
|
|
958
|
+
if (o.startRung && Object.values(PUSH_RUNG).includes(o.startRung)) ctx.rung = o.startRung;
|
|
959
|
+
|
|
960
|
+
const stop = function stop() {
|
|
961
|
+
if (ctx.stopped) return;
|
|
962
|
+
ctx.stopped = true;
|
|
963
|
+
if (ctx.timer != null) { try { ctx.clearTimeoutFn(ctx.timer); } catch { /* */ } ctx.timer = null; }
|
|
964
|
+
for (const state of ctx.wakeState.values()) {
|
|
965
|
+
if (state.timer != null) { try { ctx.clearTimeoutFn(state.timer); } catch { /* */ } state.timer = null; }
|
|
966
|
+
}
|
|
967
|
+
try { ctx.abort && ctx.abort.abort(); } catch { /* */ }
|
|
968
|
+
writeStatus(ctx, { stopped: true });
|
|
969
|
+
};
|
|
970
|
+
ctx.stop = stop;
|
|
971
|
+
|
|
972
|
+
// The run loop: one ladder step, then sleep for whatever the step asked for.
|
|
973
|
+
// Recursive scheduling (not setInterval) because each rung has its own natural
|
|
974
|
+
// rhythm and a step must never overlap itself.
|
|
975
|
+
const pump = async () => {
|
|
976
|
+
if (ctx.stopped) return;
|
|
977
|
+
let delay = ctx.baseBackoffMs;
|
|
978
|
+
try {
|
|
979
|
+
delay = await ladderStep(ctx);
|
|
980
|
+
} catch (err) {
|
|
981
|
+
// Belt and braces: ladderStep is already fail-open, but this loop is the
|
|
982
|
+
// last thing standing between a push bug and a dead daemon.
|
|
983
|
+
logWarn(`ladder step threw (${err && err.message}) — backing off; the ${ctx.cadenceNote} still covers this agent`);
|
|
984
|
+
delay = ctx.maxBackoffMs;
|
|
985
|
+
}
|
|
986
|
+
if (ctx.stopped) return;
|
|
987
|
+
ctx.timer = ctx.setTimeoutFn(() => { void pump(); }, Math.max(0, delay));
|
|
988
|
+
if (ctx.timer && typeof ctx.timer.unref === "function") { try { ctx.timer.unref(); } catch { /* */ } }
|
|
989
|
+
};
|
|
990
|
+
|
|
991
|
+
logInfo(
|
|
992
|
+
`push channel starting on rung "${ctx.rung}" (cursor ${ctx.cursor == null ? "head (bootstrap, no replay)" : ctx.cursor}) — ` +
|
|
993
|
+
`the ${ctx.cadenceNote} remains active as the safety net`,
|
|
994
|
+
);
|
|
995
|
+
writeStatus(ctx);
|
|
996
|
+
// The first step is SCHEDULED, never run inline: boot must not block on a
|
|
997
|
+
// round-trip, and it means an injected timer gives a test full control of the
|
|
998
|
+
// loop (the same discipline mesh.mjs uses for its beat interval).
|
|
999
|
+
ctx.timer = ctx.setTimeoutFn(() => { void pump(); }, 0);
|
|
1000
|
+
if (ctx.timer && typeof ctx.timer.unref === "function") { try { ctx.timer.unref(); } catch { /* */ } }
|
|
1001
|
+
|
|
1002
|
+
return {
|
|
1003
|
+
stop,
|
|
1004
|
+
isEnabled: true,
|
|
1005
|
+
rung: () => ctx.rung,
|
|
1006
|
+
stats: () => ({ ...ctx.stats, rung: ctx.rung, cursor: ctx.cursor }),
|
|
1007
|
+
_ctx: ctx,
|
|
1008
|
+
};
|
|
1009
|
+
}
|
|
1010
|
+
|
|
1011
|
+
export const _internals = {
|
|
1012
|
+
ladderStep,
|
|
1013
|
+
consumeStream,
|
|
1014
|
+
waitOnce,
|
|
1015
|
+
dispatchEvent,
|
|
1016
|
+
scheduleWake,
|
|
1017
|
+
wakeTargetFor,
|
|
1018
|
+
setRung,
|
|
1019
|
+
demote,
|
|
1020
|
+
writeStatus,
|
|
1021
|
+
v1Url,
|
|
1022
|
+
iterateBody,
|
|
1023
|
+
};
|
|
1024
|
+
|
|
1025
|
+
export default { connectOrgPush, PUSH_RUNG, readCursor, writeCursor, decodeFrameLine, _internals };
|