@cotal-ai/connector-core 0.48.2 → 0.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent.d.ts +91 -2
- package/dist/agent.d.ts.map +1 -1
- package/dist/agent.js +173 -7
- package/dist/agent.js.map +1 -1
- package/dist/agui.d.ts +85 -4
- package/dist/agui.d.ts.map +1 -1
- package/dist/agui.js +265 -3
- package/dist/agui.js.map +1 -1
- package/dist/config.d.ts +3 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +4 -0
- package/dist/config.js.map +1 -1
- package/dist/control.d.ts +10 -0
- package/dist/control.d.ts.map +1 -1
- package/dist/control.js +11 -11
- package/dist/control.js.map +1 -1
- package/dist/docs-bundle.generated.d.ts.map +1 -1
- package/dist/docs-bundle.generated.js +30 -23
- package/dist/docs-bundle.generated.js.map +1 -1
- package/dist/framing.d.ts +87 -0
- package/dist/framing.d.ts.map +1 -0
- package/dist/framing.js +101 -0
- package/dist/framing.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/runtime.d.ts +0 -19
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +27 -0
- package/dist/runtime.js.map +1 -1
- package/dist/tool-specs.d.ts +2 -2
- package/dist/tool-specs.d.ts.map +1 -1
- package/dist/tool-specs.js +71 -75
- package/dist/tool-specs.js.map +1 -1
- package/package.json +2 -2
package/dist/agent.d.ts
CHANGED
|
@@ -85,7 +85,17 @@ export interface ExactDrainResult {
|
|
|
85
85
|
* NOT the same as `disconnected`, and it is not a stopped session either. `stopped` is terminal and
|
|
86
86
|
* is not a fault at all.
|
|
87
87
|
*/
|
|
88
|
-
export type ConnectionState = "ready" | "degraded" | "connecting" | "disconnected" | "stopped";
|
|
88
|
+
export type ConnectionState = "ready" | "stalled" | "degraded" | "connecting" | "disconnected" | "stopped";
|
|
89
|
+
/**
|
|
90
|
+
* How long a non-empty automatic queue may make no progress before this session stops calling
|
|
91
|
+
* itself `ready` (#1233).
|
|
92
|
+
*
|
|
93
|
+
* Generous on purpose. A healthy seat commits automatic deliveries within seconds of the turn that
|
|
94
|
+
* carries them, and the incident this bound exists for ran for HOURS: 27 deliveries held 13.8h
|
|
95
|
+
* behind timed-out soft interrupts, reported `ready` throughout. Ten minutes is far outside normal
|
|
96
|
+
* turn length and far inside the window in which an operator still has a healthy seat to save.
|
|
97
|
+
*/
|
|
98
|
+
export declare const AUTOMATIC_QUEUE_STALL_MS = 600000;
|
|
89
99
|
export declare class MeshAgent extends EventEmitter {
|
|
90
100
|
readonly ep: CotalEndpoint;
|
|
91
101
|
readonly config: AgentConfig;
|
|
@@ -123,6 +133,22 @@ export declare class MeshAgent extends EventEmitter {
|
|
|
123
133
|
* This is measured only after the backing acknowledgements succeed, never inferred from a read
|
|
124
134
|
* attempt or from an empty inbox. */
|
|
125
135
|
private _lastInboxDrainedAt?;
|
|
136
|
+
/** Wall-clock time of the latest drain that committed at least one AUTOMATIC delivery.
|
|
137
|
+
*
|
|
138
|
+
* Deliberately separate from {@link _lastInboxDrainedAt}, and #1233 is why: on a host-mode
|
|
139
|
+
* connector the tool inbox and the connector-managed automatic queue are different queues, and
|
|
140
|
+
* the measured seat was draining the first every few minutes while the second had not moved in
|
|
141
|
+
* 13.8 hours. A progress mark that counts a pull-only drain as progress would have reported that
|
|
142
|
+
* seat as making progress, which is the exact lie this field exists to refuse. */
|
|
143
|
+
private _lastAutomaticDrainedAt?;
|
|
144
|
+
/** The latest commit that took the OLDEST automatic delivery then queued (#1526).
|
|
145
|
+
*
|
|
146
|
+
* Distinct from {@link _lastAutomaticDrainedAt}, and the distinction is the defect: any automatic
|
|
147
|
+
* commit used to count, so a seat that kept committing fresh arrivals over a head it could not
|
|
148
|
+
* deliver reset its own stall clock on every turn and reported `ready` indefinitely, while the
|
|
149
|
+
* entries at the front of its queue aged past two hours. A queue drains when its head moves; a
|
|
150
|
+
* queue that serves only its newest arrivals is not draining, it is skipping. */
|
|
151
|
+
private _lastAutomaticHeadDrainedAt?;
|
|
126
152
|
/** Latest connection failure, retained until the endpoint binds so a bounded readiness gate can
|
|
127
153
|
* explain why an otherwise healthy host never joined the mesh. */
|
|
128
154
|
private lastConnectionError?;
|
|
@@ -172,15 +198,73 @@ export declare class MeshAgent extends EventEmitter {
|
|
|
172
198
|
get transportConnected(): boolean;
|
|
173
199
|
/** Latest pre-bind failure. A successful bind clears it; stop preserves it for post-mortem diagnosis. */
|
|
174
200
|
get connectionIssue(): string | undefined;
|
|
201
|
+
/** #1356: the space's presence bucket has been refusing writes since this time, or `undefined`.
|
|
202
|
+
*
|
|
203
|
+
* Deliberately NOT folded into {@link connectionIssue}. That field has a defined relationship to
|
|
204
|
+
* readiness which existing callers already read, and a bound endpoint surviving refused heartbeats
|
|
205
|
+
* IS still ready — the neighbouring decision at the `warning` listener is correct and stands. This
|
|
206
|
+
* is a distinct fact that can be true at the same time as `ready`, which is the actual shape of the
|
|
207
|
+
* incident: connected, serving, and silently unable to publish presence.
|
|
208
|
+
*
|
|
209
|
+
* Presence writes are the CANARY, not the scope — see CotalEndpoint.presenceWriteFailure. */
|
|
210
|
+
get presenceWriteFailure(): {
|
|
211
|
+
since: number;
|
|
212
|
+
forMs: number;
|
|
213
|
+
error?: string;
|
|
214
|
+
bucket: string;
|
|
215
|
+
} | undefined;
|
|
175
216
|
/** The latest successful, non-empty inbox drain in this session. */
|
|
176
217
|
get lastInboxDrainedAt(): number | undefined;
|
|
218
|
+
/** The latest drain that committed at least one AUTOMATIC (connector-managed) delivery. */
|
|
219
|
+
get lastAutomaticDrainedAt(): number | undefined;
|
|
220
|
+
/** The latest drain that committed the automatic delivery that was then at the FRONT of the queue
|
|
221
|
+
* (#1526). This is the fact {@link automaticQueueStalledForMs} measures progress by; the looser
|
|
222
|
+
* {@link lastAutomaticDrainedAt} is still reported beside it, because the two disagreeing is
|
|
223
|
+
* precisely the shape of a wedged head being served around. */
|
|
224
|
+
get lastAutomaticHeadDrainedAt(): number | undefined;
|
|
225
|
+
/** The receive key of the oldest still-queued automatic delivery: the queue's head, and the entry
|
|
226
|
+
* whose commit counts as progress. */
|
|
227
|
+
private oldestAutomaticKey;
|
|
228
|
+
/** Record head progress when this batch takes the current head of the automatic queue. Call
|
|
229
|
+
* BEFORE the batch leaves {@link inbox}, since the head is read from the live queue. */
|
|
230
|
+
private noteAutomaticHeadProgress;
|
|
231
|
+
/**
|
|
232
|
+
* How long a non-empty automatic queue has been making no progress, or `undefined`.
|
|
233
|
+
*
|
|
234
|
+
* PROGRESS AT THE HEAD, not depth and not throughput. The queue is not stalled because it is deep;
|
|
235
|
+
* it is stalled because the entries at the front of it are not coming off. The clock therefore
|
|
236
|
+
* starts at the later of the arrival of the oldest still-queued delivery and the last commit that
|
|
237
|
+
* took the queue's head, so a seat that is steadily draining keeps resetting it however busy it
|
|
238
|
+
* is, and a seat that has not moved its head since that queue formed accrues from the moment it
|
|
239
|
+
* formed.
|
|
240
|
+
*
|
|
241
|
+
* Keyed on the HEAD rather than on any automatic commit (#1526). The reported session held 96
|
|
242
|
+
* deliveries with the oldest over two hours old and reported `ready` throughout: it was committing
|
|
243
|
+
* fresh arrivals, and under a looser measure each of those reset the clock for the backlog behind
|
|
244
|
+
* them. A seat that serves only its newest traffic is skipping its queue, not draining it, and the
|
|
245
|
+
* messages that are actually undelivered are the ones the measure must speak for.
|
|
246
|
+
*
|
|
247
|
+
* Measured from the oldest ARRIVAL rather than from session start so a session that never drained
|
|
248
|
+
* anything is still measurable — that is precisely the shape of a seat whose first soft interrupt
|
|
249
|
+
* timed out.
|
|
250
|
+
*/
|
|
251
|
+
automaticQueueStalledForMs(now?: number): number | undefined;
|
|
177
252
|
/** Whether {@link stop} has been called. Terminal, and never cleared: a stopped session does not
|
|
178
253
|
* serve again. This is the ONLY way to tell a deliberate shutdown from a lost connection, because
|
|
179
254
|
* `stop()` clears readiness and transport together, so those two read identically in both cases. */
|
|
180
255
|
get stopping(): boolean;
|
|
181
256
|
/** The three liveness facts combined, in one place. Every combination maps, so a caller never has
|
|
182
257
|
* to guess what an unlisted pair means, and a caller that disagrees with this reading can still
|
|
183
|
-
* read {@link connected}, {@link transportConnected} and {@link stopping} directly.
|
|
258
|
+
* read {@link connected}, {@link transportConnected} and {@link stopping} directly.
|
|
259
|
+
*
|
|
260
|
+
* `stalled` is the fourth fact and it is not about the connection at all — see
|
|
261
|
+
* {@link automaticQueueStalledForMs}. It is reported HERE, ahead of `ready`, because #1233 was
|
|
262
|
+
* not a caller misreading the facts: a seat holding 27 undeliverable messages for 13.8 hours
|
|
263
|
+
* answered this question with `ready`, and every operator who asked it was told the seat was
|
|
264
|
+
* fine. A status that reads healthy while the thing it describes is not happening is the defect,
|
|
265
|
+
* so the one word a caller acts on has to move. It is ordered below `degraded` deliberately: a
|
|
266
|
+
* dead socket is the more specific and more actionable fault, and reporting the queue symptom
|
|
267
|
+
* over its own cause would hide it. */
|
|
184
268
|
get connectionState(): ConnectionState;
|
|
185
269
|
/** Wait for the endpoint's real post-bind connection signal. `start()` deliberately stays
|
|
186
270
|
* background for connectors whose MCP surface must boot while the broker is absent; a host that
|
|
@@ -552,6 +636,11 @@ export declare class MeshAgent extends EventEmitter {
|
|
|
552
636
|
name: string;
|
|
553
637
|
prompt: string;
|
|
554
638
|
model?: string;
|
|
639
|
+
role?: string;
|
|
640
|
+
agent?: string;
|
|
641
|
+
subscribe?: string[];
|
|
642
|
+
allowSubscribe?: string[];
|
|
643
|
+
allowPublish?: string[];
|
|
555
644
|
announce?: string;
|
|
556
645
|
}): Promise<ControlReply & {
|
|
557
646
|
announceError?: string;
|
package/dist/agent.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"agent.d.ts","sourceRoot":"","sources":["../src/agent.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,EAQL,aAAa,EAQb,KAAK,YAAY,EAIjB,KAAK,QAAQ,EACb,KAAK,cAAc,EAEnB,KAAK,aAAa,EAClB,KAAK,WAAW,EAChB,KAAK,YAAY,EAClB,MAAM,gBAAgB,CAAC;AACxB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAI/C,YAAY,EAAE,aAAa,EAAE,WAAW,EAAE,CAAC;AAE3C;;;6DAG6D;AAC7D,eAAO,MAAM,gBAAgB,QAAS,CAAC;AAEvC;;;;;;8EAM8E;AAC9E,eAAO,MAAM,gBAAgB,QAAS,CAAC;AAEvC;;8BAE8B;AAC9B,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAItD;AAgCD,2EAA2E;AAC3E,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX;;;;;;6EAMyE;IACzE,OAAO,EAAE,MAAM,CAAC;IAChB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,SAAS,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,mCAAmC;IACnC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wDAAwD;IACxD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wEAAwE;IACxE,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,kGAAkG;IAClG,UAAU,EAAE,OAAO,CAAC;IACpB,+FAA+F;IAC/F,UAAU,EAAE,OAAO,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAaD,0GAA0G;AAC1G,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,EAAE,EAAE,MAAM,CAAC;CACZ;AAED,0GAA0G;AAC1G,wBAAgB,eAAe,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,UAAU,GAAG,OAAO,CASrE;AAmCD,MAAM,MAAM,UAAU,GAAG,KAAK,GAAG,WAAW,GAAG,WAAW,CAAC;AAE3D,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,SAAS,EAAE,CAAC;IACnB,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB;AAiBD;;;;;;;;;;;;;;;GAeG;AACH;;;;;;;GAOG;AACH,MAAM,MAAM,eAAe,GAAG,OAAO,GAAG,UAAU,GAAG,YAAY,GAAG,cAAc,GAAG,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"agent.d.ts","sourceRoot":"","sources":["../src/agent.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,EAQL,aAAa,EAQb,KAAK,YAAY,EAIjB,KAAK,QAAQ,EACb,KAAK,cAAc,EAEnB,KAAK,aAAa,EAClB,KAAK,WAAW,EAChB,KAAK,YAAY,EAClB,MAAM,gBAAgB,CAAC;AACxB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAI/C,YAAY,EAAE,aAAa,EAAE,WAAW,EAAE,CAAC;AAE3C;;;6DAG6D;AAC7D,eAAO,MAAM,gBAAgB,QAAS,CAAC;AAEvC;;;;;;8EAM8E;AAC9E,eAAO,MAAM,gBAAgB,QAAS,CAAC;AAEvC;;8BAE8B;AAC9B,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAItD;AAgCD,2EAA2E;AAC3E,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX;;;;;;6EAMyE;IACzE,OAAO,EAAE,MAAM,CAAC;IAChB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,SAAS,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,mCAAmC;IACnC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wDAAwD;IACxD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wEAAwE;IACxE,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,kGAAkG;IAClG,UAAU,EAAE,OAAO,CAAC;IACpB,+FAA+F;IAC/F,UAAU,EAAE,OAAO,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAaD,0GAA0G;AAC1G,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,EAAE,EAAE,MAAM,CAAC;CACZ;AAED,0GAA0G;AAC1G,wBAAgB,eAAe,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,UAAU,GAAG,OAAO,CASrE;AAmCD,MAAM,MAAM,UAAU,GAAG,KAAK,GAAG,WAAW,GAAG,WAAW,CAAC;AAE3D,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,SAAS,EAAE,CAAC;IACnB,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB;AAiBD;;;;;;;;;;;;;;;GAeG;AACH;;;;;;;GAOG;AACH,MAAM,MAAM,eAAe,GAAG,OAAO,GAAG,SAAS,GAAG,UAAU,GAAG,YAAY,GAAG,cAAc,GAAG,SAAS,CAAC;AAE3G;;;;;;;;GAQG;AACH,eAAO,MAAM,wBAAwB,SAAU,CAAC;AAmChD,qBAAa,SAAU,SAAQ,YAAY;IACzC,QAAQ,CAAC,EAAE,EAAE,aAAa,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAE7B,OAAO,CAAC,KAAK,CAAiB;IAC9B;;;;;;oGAMgG;IAChG,OAAO,CAAC,UAAU,CAAqB;IACvC,OAAO,CAAC,cAAc,CAAqB;IAC3C;;8DAE0D;IAC1D,OAAO,CAAC,sBAAsB,CAA6D;IAC3F;wGACoG;IACpG,OAAO,CAAC,iBAAiB,CAA6B;IACtD;oDACgD;IAChD,OAAO,CAAC,WAAW,CAA6B;IAChD,OAAO,CAAC,oBAAoB,CAAS;IACrC;;6EAEyE;IACzE,OAAO,CAAC,oBAAoB,CAAqB;IACjD,OAAO,CAAC,gBAAgB,CAAqB;IAC7C,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,UAAU,CAAS;IAC3B,+FAA+F;IAC/F,OAAO,CAAC,mBAAmB,CAAS;IACpC;;0CAEsC;IACtC,OAAO,CAAC,mBAAmB,CAAC,CAAS;IACrC;;;;;;uFAMmF;IACnF,OAAO,CAAC,uBAAuB,CAAC,CAAS;IACzC;;;;;;sFAMkF;IAClF,OAAO,CAAC,2BAA2B,CAAC,CAAS;IAC7C;sEACkE;IAClE,OAAO,CAAC,mBAAmB,CAAC,CAAS;IACrC,OAAO,CAAC,iBAAiB,CAAmE;IAC5F,OAAO,CAAC,OAAO,CAA0B;IACzC,OAAO,CAAC,UAAU,CAAyB;IAC3C,OAAO,CAAC,aAAa,CAAiC;IACtD;;kEAE8D;IAC9D,OAAO,CAAC,cAAc,CAAqB;IAC3C;;;8EAG0E;IAC1E,OAAO,CAAC,YAAY,CAAkC;IACtD;;;;mFAI+E;IAC/E,OAAO,CAAC,WAAW,CAAiC;IACpD,OAAO,CAAC,aAAa,CAAC,CAAiC;IACvD,wFAAwF;IACxF,OAAO,CAAC,WAAW,CAAC,CAAS;IAC7B,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,UAAU,CAAqB;IACvC;qFACiF;IACjF,OAAO,CAAC,UAAU,CAAC,CAAS;IAC5B,OAAO,CAAC,aAAa,CAAS;IAC9B;;;;;mGAK+F;IAC/F,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAkC;IAChE,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,gBAAgB,CAA6B;IACrD,OAAO,CAAC,yBAAyB,CAAqB;IACtD,OAAO,CAAC,SAAS,CAAS;gBAEd,MAAM,EAAE,WAAW;IAwE/B,IAAI,EAAE,IAAI,MAAM,CAEf;IAED,IAAI,SAAS,IAAI,OAAO,CAEvB;IAED,qGAAqG;IACrG,IAAI,kBAAkB,IAAI,OAAO,CAEhC;IAED,yGAAyG;IACzG,IAAI,eAAe,IAAI,MAAM,GAAG,SAAS,CAExC;IAED;;;;;;;;kGAQ8F;IAC9F,IAAI,oBAAoB,IAAI;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAEvG;IAED,oEAAoE;IACpE,IAAI,kBAAkB,IAAI,MAAM,GAAG,SAAS,CAE3C;IAED,2FAA2F;IAC3F,IAAI,sBAAsB,IAAI,MAAM,GAAG,SAAS,CAE/C;IAED;;;oEAGgE;IAChE,IAAI,0BAA0B,IAAI,MAAM,GAAG,SAAS,CAEnD;IAED;2CACuC;IACvC,OAAO,CAAC,kBAAkB;IAS1B;6FACyF;IACzF,OAAO,CAAC,yBAAyB;IAMjC;;;;;;;;;;;;;;;;;;;OAmBG;IACH,0BAA0B,CAAC,GAAG,SAAa,GAAG,MAAM,GAAG,SAAS;IAQhE;;yGAEqG;IACrG,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED;;;;;;;;;;;4CAWwC;IACxC,IAAI,eAAe,IAAI,eAAe,CAQrC;IAED;;oFAEgF;IAC1E,kBAAkB,CAAC,SAAS,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IA0B3D,+EAA+E;IAC/E,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI;IAKjD,4FAA4F;IAC5F,KAAK,CAAC,OAAO,SAAO,GAAG,OAAO,CAAC,IAAI,CAAC;YAItB,WAAW;IAyCnB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAmB3B;;;;iGAI6F;IACvF,SAAS,IAAI,OAAO,CAAC;QAAE,EAAE,EAAE,OAAO,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IA+B5D,OAAO,CAAC,MAAM;IA+Fd,OAAO,CAAC,MAAM;IA8Ed,OAAO,CAAC,eAAe;IAYvB,OAAO,CAAC,gBAAgB;IAYxB,OAAO,CAAC,kBAAkB;IAa1B;;;;gDAI4C;IAC5C,OAAO,CAAC,WAAW;IAwBnB;sDACkD;IAClD,SAAS,CAAC,KAAK,GAAE,UAAkB,GAAG,SAAS,EAAE;IAIjD;;;;;;;;;;;;;kEAa8D;IAC9D,YAAY,CAAC,GAAG,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO;IAa7C;;;sGAGkG;IAClG,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAI/B;mGAC+F;IAC/F,eAAe,CAAC,GAAG,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI;IAS7C,8FAA8F;IAC9F,UAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,KAAK,GAAE,UAAkB,GAAG,SAAS,EAAE;IAsBlE;;;;;;iDAM6C;IAC7C,oBAAoB,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,gBAAgB;IAiC/D,OAAO,CAAC,aAAa;IASrB,OAAO,CAAC,OAAO;IAIf;;gGAE4F;IAC5F,OAAO,CAAC,WAAW;IAUnB,UAAU,CAAC,KAAK,GAAE,UAAkB,GAAG,MAAM;IAI7C,kFAAkF;IAClF,yBAAyB,IAAI,MAAM,GAAG,SAAS;IAS/C;;;;;;;;;;;;;;;;;;;OAmBG;IACH,IAAI,YAAY,IAAI,UAAU,CAE7B;IAED;;;;;;;OAOG;IACH,YAAY,CAAC,IAAI,EAAE,UAAU,GAAG,IAAI;IAIpC;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,WAAW,CAAC,IAAI,EAAE;QAAE,EAAE,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO;IAI1C,oFAAoF;IACpF,eAAe,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAIpC;;;;;;;;OAQG;IACH,eAAe,IAAI,MAAM;IAIzB,kFAAkF;IAClF,iBAAiB,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI;IAInC;;;;;;;;;OASG;IACH,OAAO,CAAC,eAAe;IAKvB;gCAC4B;IAC5B,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC,GAAG,SAAS;IAK/D;;;0EAGsE;IACtE,oBAAoB,IAAI,MAAM;IAI9B;;;;;;0GAMsG;IACtG,WAAW,IAAI,MAAM;IAYrB;;;wFAGoF;IACpF,WAAW,IAAI,IAAI;IAMnB;qFACiF;IACjF,IAAI,SAAS,IAAI,aAAa,CAE7B;IAED,4FAA4F;IAC5F,WAAW,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS;IAItD,4EAA4E;IAC5E,kBAAkB,IAAI,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC;IAIjD;;;;;;;;iDAQ6C;IACvC,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,GAAG,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAUlF;;;;;;kCAM8B;IACxB,YAAY,CAAC,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IA6BtD;;;;;;;;;iBASa;IACP,aAAa,IAAI,OAAO,CAAC;QAAE,KAAK,EAAE,SAAS,EAAE,CAAC;QAAC,eAAe,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IA0B3E,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,CAAC;IAOtF;;;;;OAKG;IACG,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAe3D;;;;;2DAKuD;IACvD,OAAO,CAAC,mBAAmB;IASrB,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAKhE;;wEAEoE;IACpE,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,QAAQ,GAAG,SAAS;IAI3C,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,GAAG,EAAE,YAAY,CAAC;QAAC,IAAI,EAAE,QAAQ,CAAA;KAAE,CAAC;IAUtF;;;;;;;;;;;kFAW8E;IACxE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,YAAY,CAAC;IAiDpM;6FACyF;IACnF,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,EAAE,EAAE,IAAI,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG;QAAE,EAAE,EAAE,KAAK,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAStG;;;;;;sEAMkE;YACpD,aAAa;IAyC3B;;;;;kGAK8F;YAChF,gBAAgB;IAS9B;;;;;;;4FAOwF;IAClF,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,OAAO,CAAC,YAAY,CAAC;IAYlF;;;;4CAIwC;IAClC,GAAG,CAAC,IAAI,EAAE,OAAO,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC;IAOtH,OAAO,CAAC,cAAc;IAStB;;;;sGAIkG;YACpF,SAAS;IAsBvB;;;;;;;;;OASG;IACH,OAAO,CAAC,eAAe;IASvB;;;;;mGAK+F;IAC/F,gBAAgB,IAAI;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAA;KAAE,GAAG,SAAS;IAqBnE;;uFAEmF;IACnF,mBAAmB,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI;IAOrD;;;oFAGgF;IAC1E,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,SAAS,EAAE,IAAI,GAAE;QAAE,EAAE,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG,OAAO,CAAC,YAAY,CAAC;IAgBxI;uGACmG;IAC7F,YAAY,CAAC,IAAI,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,OAAO,CAAC,YAAY,CAAC;IAM1E;;;;;;;;;;;;;;;;;;;+FAmB2F;IACrF,aAAa,CAAC,GAAG,EAAE;QACvB,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;QACrB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;QAC1B,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;QACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,GAAG,OAAO,CAAC,YAAY,GAAG;QAAE,aAAa,CAAC,EAAE,MAAM,CAAC;QAAC,eAAe,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAA;KAAE,CAAC;IAwE9F;6CACyC;IACnC,YAAY,IAAI,OAAO,CAAC,YAAY,CAAC;IAK3C,8FAA8F;IACxF,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAOtD,4CAA4C;IAC5C,MAAM,IAAI,QAAQ,EAAE;IAIpB,8CAA8C;IAC9C,IAAI,MAAM,IAAI,cAAc,CAE3B;IAEK,SAAS,CAAC,MAAM,EAAE,cAAc,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBzE;;;;;;;;;OASG;IACG,WAAW,CAAC,MAAM,EAAE,cAAc,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;YAS7D,aAAa;IAK3B;;;4FAGwF;IACxF,OAAO,CAAC,cAAc;IAUtB;;;8EAG0E;IACpE,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAO9D;;;;qFAIiF;IACjF,eAAe,IAAI,MAAM,GAAG,SAAS;IAUrC;oFACgF;IAChF,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAC;QAAC,UAAU,EAAE,OAAO,CAAA;KAAE;IAUnH,2EAA2E;IAC3E,cAAc,IAAI,MAAM,EAAE;IAI1B;;iEAE6D;IACvD,YAAY,IAAI,OAAO,CAC3B;QACE,OAAO,EAAE,MAAM,CAAC;QAChB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,EAAE,OAAO,CAAC;QAChB,MAAM,EAAE,OAAO,CAAC;QAChB,eAAe,EAAE,OAAO,CAAC;QACzB,cAAc,CAAC,EAAE,QAAQ,GAAG,UAAU,CAAC;QACvC,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,WAAW,GAAG,QAAQ,CAAC;KAC9B,EAAE,CACJ;IA6DD;;;;;2BAKuB;IACjB,WAAW,CACf,OAAO,EAAE,MAAM,GACd,OAAO,CAAC;QAAE,MAAM,EAAE,OAAO,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAKtF,mEAAmE;IAC7D,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,OAAO,CAAA;KAAE,CAAC;IAO/D,OAAO,CAAC,GAAG;IAIX;;wEAEoE;YACtD,gBAAgB;IAI9B;;;;+EAI2E;IACrE,aAAa,CAAC,SAAS,GAAE,MAAyB,GAAG,OAAO,CAAC,IAAI,CAAC;IAKxE;;;6DAGyD;IACzD,OAAO,CAAC,eAAe;IAmBvB;;qEAEiE;IACjE,OAAO,CAAC,mBAAmB;IAO3B;;uFAEmF;IACnF,OAAO,CAAC,mBAAmB;IAQ3B,OAAO,CAAC,qBAAqB;IAI7B,OAAO,CAAC,iBAAiB;IAezB,OAAO,CAAC,GAAG;CAGZ;AAED,kGAAkG;AAClG,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,CAOrF"}
|
package/dist/agent.js
CHANGED
|
@@ -100,6 +100,16 @@ function sleep(ms) {
|
|
|
100
100
|
function ingestDedupKey(id) {
|
|
101
101
|
return id === "" ? undefined : id;
|
|
102
102
|
}
|
|
103
|
+
/**
|
|
104
|
+
* How long a non-empty automatic queue may make no progress before this session stops calling
|
|
105
|
+
* itself `ready` (#1233).
|
|
106
|
+
*
|
|
107
|
+
* Generous on purpose. A healthy seat commits automatic deliveries within seconds of the turn that
|
|
108
|
+
* carries them, and the incident this bound exists for ran for HOURS: 27 deliveries held 13.8h
|
|
109
|
+
* behind timed-out soft interrupts, reported `ready` throughout. Ten minutes is far outside normal
|
|
110
|
+
* turn length and far inside the window in which an operator still has a healthy seat to save.
|
|
111
|
+
*/
|
|
112
|
+
export const AUTOMATIC_QUEUE_STALL_MS = 600_000;
|
|
103
113
|
/** An `ask` relayed as a turn: the record the run needs, the attempt it is on, the previous
|
|
104
114
|
* refusal when there was one, and the command that answers it (`cotal run answer`, the same door
|
|
105
115
|
* a checkpoint is answered through). Empty when the payload carries no ask. */
|
|
@@ -171,6 +181,22 @@ export class MeshAgent extends EventEmitter {
|
|
|
171
181
|
* This is measured only after the backing acknowledgements succeed, never inferred from a read
|
|
172
182
|
* attempt or from an empty inbox. */
|
|
173
183
|
_lastInboxDrainedAt;
|
|
184
|
+
/** Wall-clock time of the latest drain that committed at least one AUTOMATIC delivery.
|
|
185
|
+
*
|
|
186
|
+
* Deliberately separate from {@link _lastInboxDrainedAt}, and #1233 is why: on a host-mode
|
|
187
|
+
* connector the tool inbox and the connector-managed automatic queue are different queues, and
|
|
188
|
+
* the measured seat was draining the first every few minutes while the second had not moved in
|
|
189
|
+
* 13.8 hours. A progress mark that counts a pull-only drain as progress would have reported that
|
|
190
|
+
* seat as making progress, which is the exact lie this field exists to refuse. */
|
|
191
|
+
_lastAutomaticDrainedAt;
|
|
192
|
+
/** The latest commit that took the OLDEST automatic delivery then queued (#1526).
|
|
193
|
+
*
|
|
194
|
+
* Distinct from {@link _lastAutomaticDrainedAt}, and the distinction is the defect: any automatic
|
|
195
|
+
* commit used to count, so a seat that kept committing fresh arrivals over a head it could not
|
|
196
|
+
* deliver reset its own stall clock on every turn and reported `ready` indefinitely, while the
|
|
197
|
+
* entries at the front of its queue aged past two hours. A queue drains when its head moves; a
|
|
198
|
+
* queue that serves only its newest arrivals is not draining, it is skipping. */
|
|
199
|
+
_lastAutomaticHeadDrainedAt;
|
|
174
200
|
/** Latest connection failure, retained until the endpoint binds so a bounded readiness gate can
|
|
175
201
|
* explain why an otherwise healthy host never joined the mesh. */
|
|
176
202
|
lastConnectionError;
|
|
@@ -230,6 +256,7 @@ export class MeshAgent extends EventEmitter {
|
|
|
230
256
|
pass: config.pass,
|
|
231
257
|
creds: config.creds,
|
|
232
258
|
lifecycleUid: config.lifecycleUid,
|
|
259
|
+
acceptedToken: config.acceptedToken,
|
|
233
260
|
// USER MODE: the endpoint execs the spawner-provided argv per bearer refresh — the exchange
|
|
234
261
|
// protocol lives entirely behind that command, this runtime just runs it and reads a line.
|
|
235
262
|
bearer: config.userAuth ? () => execBearerCmd(config.userAuth.bearerCmd) : undefined,
|
|
@@ -302,10 +329,82 @@ export class MeshAgent extends EventEmitter {
|
|
|
302
329
|
get connectionIssue() {
|
|
303
330
|
return this.lastConnectionError;
|
|
304
331
|
}
|
|
332
|
+
/** #1356: the space's presence bucket has been refusing writes since this time, or `undefined`.
|
|
333
|
+
*
|
|
334
|
+
* Deliberately NOT folded into {@link connectionIssue}. That field has a defined relationship to
|
|
335
|
+
* readiness which existing callers already read, and a bound endpoint surviving refused heartbeats
|
|
336
|
+
* IS still ready — the neighbouring decision at the `warning` listener is correct and stands. This
|
|
337
|
+
* is a distinct fact that can be true at the same time as `ready`, which is the actual shape of the
|
|
338
|
+
* incident: connected, serving, and silently unable to publish presence.
|
|
339
|
+
*
|
|
340
|
+
* Presence writes are the CANARY, not the scope — see CotalEndpoint.presenceWriteFailure. */
|
|
341
|
+
get presenceWriteFailure() {
|
|
342
|
+
return this.ep.presenceWriteFailure();
|
|
343
|
+
}
|
|
305
344
|
/** The latest successful, non-empty inbox drain in this session. */
|
|
306
345
|
get lastInboxDrainedAt() {
|
|
307
346
|
return this._lastInboxDrainedAt;
|
|
308
347
|
}
|
|
348
|
+
/** The latest drain that committed at least one AUTOMATIC (connector-managed) delivery. */
|
|
349
|
+
get lastAutomaticDrainedAt() {
|
|
350
|
+
return this._lastAutomaticDrainedAt;
|
|
351
|
+
}
|
|
352
|
+
/** The latest drain that committed the automatic delivery that was then at the FRONT of the queue
|
|
353
|
+
* (#1526). This is the fact {@link automaticQueueStalledForMs} measures progress by; the looser
|
|
354
|
+
* {@link lastAutomaticDrainedAt} is still reported beside it, because the two disagreeing is
|
|
355
|
+
* precisely the shape of a wedged head being served around. */
|
|
356
|
+
get lastAutomaticHeadDrainedAt() {
|
|
357
|
+
return this._lastAutomaticHeadDrainedAt;
|
|
358
|
+
}
|
|
359
|
+
/** The receive key of the oldest still-queued automatic delivery: the queue's head, and the entry
|
|
360
|
+
* whose commit counts as progress. */
|
|
361
|
+
oldestAutomaticKey() {
|
|
362
|
+
let head;
|
|
363
|
+
for (const pending of this.inbox) {
|
|
364
|
+
if (pending.pullOnly)
|
|
365
|
+
continue;
|
|
366
|
+
if (head === undefined || pending.receivedAt < head.receivedAt)
|
|
367
|
+
head = pending;
|
|
368
|
+
}
|
|
369
|
+
return head?.item.recvKey;
|
|
370
|
+
}
|
|
371
|
+
/** Record head progress when this batch takes the current head of the automatic queue. Call
|
|
372
|
+
* BEFORE the batch leaves {@link inbox}, since the head is read from the live queue. */
|
|
373
|
+
noteAutomaticHeadProgress(selected) {
|
|
374
|
+
const head = this.oldestAutomaticKey();
|
|
375
|
+
if (head === undefined)
|
|
376
|
+
return;
|
|
377
|
+
if (selected.some((p) => !p.pullOnly && p.item.recvKey === head))
|
|
378
|
+
this._lastAutomaticHeadDrainedAt = Date.now();
|
|
379
|
+
}
|
|
380
|
+
/**
|
|
381
|
+
* How long a non-empty automatic queue has been making no progress, or `undefined`.
|
|
382
|
+
*
|
|
383
|
+
* PROGRESS AT THE HEAD, not depth and not throughput. The queue is not stalled because it is deep;
|
|
384
|
+
* it is stalled because the entries at the front of it are not coming off. The clock therefore
|
|
385
|
+
* starts at the later of the arrival of the oldest still-queued delivery and the last commit that
|
|
386
|
+
* took the queue's head, so a seat that is steadily draining keeps resetting it however busy it
|
|
387
|
+
* is, and a seat that has not moved its head since that queue formed accrues from the moment it
|
|
388
|
+
* formed.
|
|
389
|
+
*
|
|
390
|
+
* Keyed on the HEAD rather than on any automatic commit (#1526). The reported session held 96
|
|
391
|
+
* deliveries with the oldest over two hours old and reported `ready` throughout: it was committing
|
|
392
|
+
* fresh arrivals, and under a looser measure each of those reset the clock for the backlog behind
|
|
393
|
+
* them. A seat that serves only its newest traffic is skipping its queue, not draining it, and the
|
|
394
|
+
* messages that are actually undelivered are the ones the measure must speak for.
|
|
395
|
+
*
|
|
396
|
+
* Measured from the oldest ARRIVAL rather than from session start so a session that never drained
|
|
397
|
+
* anything is still measurable — that is precisely the shape of a seat whose first soft interrupt
|
|
398
|
+
* timed out.
|
|
399
|
+
*/
|
|
400
|
+
automaticQueueStalledForMs(now = Date.now()) {
|
|
401
|
+
const oldest = this.oldestAutomaticReceivedAt();
|
|
402
|
+
if (oldest === undefined)
|
|
403
|
+
return undefined;
|
|
404
|
+
const since = Math.max(oldest, this._lastAutomaticHeadDrainedAt ?? 0);
|
|
405
|
+
const held = now - since;
|
|
406
|
+
return held > 0 ? held : 0;
|
|
407
|
+
}
|
|
309
408
|
/** Whether {@link stop} has been called. Terminal, and never cleared: a stopped session does not
|
|
310
409
|
* serve again. This is the ONLY way to tell a deliberate shutdown from a lost connection, because
|
|
311
410
|
* `stop()` clears readiness and transport together, so those two read identically in both cases. */
|
|
@@ -314,12 +413,25 @@ export class MeshAgent extends EventEmitter {
|
|
|
314
413
|
}
|
|
315
414
|
/** The three liveness facts combined, in one place. Every combination maps, so a caller never has
|
|
316
415
|
* to guess what an unlisted pair means, and a caller that disagrees with this reading can still
|
|
317
|
-
* read {@link connected}, {@link transportConnected} and {@link stopping} directly.
|
|
416
|
+
* read {@link connected}, {@link transportConnected} and {@link stopping} directly.
|
|
417
|
+
*
|
|
418
|
+
* `stalled` is the fourth fact and it is not about the connection at all — see
|
|
419
|
+
* {@link automaticQueueStalledForMs}. It is reported HERE, ahead of `ready`, because #1233 was
|
|
420
|
+
* not a caller misreading the facts: a seat holding 27 undeliverable messages for 13.8 hours
|
|
421
|
+
* answered this question with `ready`, and every operator who asked it was told the seat was
|
|
422
|
+
* fine. A status that reads healthy while the thing it describes is not happening is the defect,
|
|
423
|
+
* so the one word a caller acts on has to move. It is ordered below `degraded` deliberately: a
|
|
424
|
+
* dead socket is the more specific and more actionable fault, and reporting the queue symptom
|
|
425
|
+
* over its own cause would hide it. */
|
|
318
426
|
get connectionState() {
|
|
319
427
|
if (this._stopping)
|
|
320
428
|
return "stopped";
|
|
321
|
-
if (this._connected)
|
|
322
|
-
|
|
429
|
+
if (this._connected) {
|
|
430
|
+
if (!this._transportConnected)
|
|
431
|
+
return "degraded";
|
|
432
|
+
const held = this.automaticQueueStalledForMs();
|
|
433
|
+
return held !== undefined && held >= AUTOMATIC_QUEUE_STALL_MS ? "stalled" : "ready";
|
|
434
|
+
}
|
|
323
435
|
return this._transportConnected ? "connecting" : "disconnected";
|
|
324
436
|
}
|
|
325
437
|
/** Wait for the endpoint's real post-bind connection signal. `start()` deliberately stays
|
|
@@ -382,7 +494,26 @@ export class MeshAgent extends EventEmitter {
|
|
|
382
494
|
if (this._stopping)
|
|
383
495
|
return;
|
|
384
496
|
this.lastConnectionError = error.message;
|
|
385
|
-
|
|
497
|
+
// #1356: "mesh unreachable" was asserted unconditionally, including while this same object
|
|
498
|
+
// held `transportConnected: true` — measured against a presence bucket whose writes the
|
|
499
|
+
// broker refuses: broker up, TCP connected, every sibling stream writable, and the operator
|
|
500
|
+
// told the mesh was down. An anonymous error makes an operator look; a wrong one makes them
|
|
501
|
+
// look in the wrong place. Both facts are already here, so report the one that is true.
|
|
502
|
+
// The presence sentence CLAIMS the transport is fine, so it is reachable only while a live
|
|
503
|
+
// transport observation says so. The endpoint clears the record whenever a connection is torn
|
|
504
|
+
// down, which is the primary guard; this is the second one, for a transport that drops without
|
|
505
|
+
// a teardown running. Belt and braces, because the failure this replaced was a confident wrong
|
|
506
|
+
// answer and the cost of one redundant check is lower than the cost of another.
|
|
507
|
+
const presence = this._transportConnected ? this.ep.presenceWriteFailure() : undefined;
|
|
508
|
+
// "for 0s" on the first retry read like a broken template. Report a sub-second age as "<1s"
|
|
509
|
+
// rather than rounding it up to a duration that has not elapsed yet.
|
|
510
|
+
const forS = presence === undefined ? "" : presence.forMs < 1000 ? "<1" : String(Math.round(presence.forMs / 1000));
|
|
511
|
+
const diagnosis = presence
|
|
512
|
+
? `connected, but this space's presence bucket "${presence.bucket}" has refused every write for ${forS}s (${presence.error ?? error.message}) - the transport is fine and the fault is not the network`
|
|
513
|
+
: this._transportConnected
|
|
514
|
+
? `transport is connected but the mesh bind did not complete (${error.message})`
|
|
515
|
+
: `mesh unreachable (${error.message})`;
|
|
516
|
+
this.log(`${diagnosis}; retrying in ${retryMs}ms`);
|
|
386
517
|
await sleep(retryMs);
|
|
387
518
|
}
|
|
388
519
|
}
|
|
@@ -420,7 +551,19 @@ export class MeshAgent extends EventEmitter {
|
|
|
420
551
|
try {
|
|
421
552
|
await this.ep.reconnect();
|
|
422
553
|
// _connected is set by the endpoint's "connection" event on the successful rebind, not here.
|
|
423
|
-
|
|
554
|
+
//
|
|
555
|
+
// #1233: a rebuilt connection is NOT a served queue. The measured seat's transport was up the
|
|
556
|
+
// whole time, so this call rebound a connection that was never broken and answered
|
|
557
|
+
// "Reconnected ✓" over 27 deliveries it had not touched — a recovery path reporting success
|
|
558
|
+
// while doing nothing, which sent the operator looking for a different remedy. The rebind is
|
|
559
|
+
// still reported honestly as a success, because it succeeded; what is added is the fact that
|
|
560
|
+
// makes the reply actionable, from the same measurement the state uses.
|
|
561
|
+
const held = this.automaticQueueStalledForMs();
|
|
562
|
+
const queued = this.inboxCount("automatic");
|
|
563
|
+
const stalled = held !== undefined && held >= AUTOMATIC_QUEUE_STALL_MS && queued > 0
|
|
564
|
+
? ` Note: this session's ${queued} queued automatic ${queued === 1 ? "delivery has" : "deliveries have"} made no progress for ${Math.round(held / 60_000)} minutes. Reconnecting did not deliver ${queued === 1 ? "it" : "them"}: the connection was not what was holding ${queued === 1 ? "it" : "them"} up.`
|
|
565
|
+
: "";
|
|
566
|
+
return { ok: true, message: `Reconnected ✓ (${this.config.name}@${this.config.space})${stalled}` };
|
|
424
567
|
}
|
|
425
568
|
catch (e) {
|
|
426
569
|
return { ok: false, message: `Reconnect failed: ${e.message}. Still retrying automatically — or run /reconnect to retry now.` };
|
|
@@ -738,14 +881,23 @@ export class MeshAgent extends EventEmitter {
|
|
|
738
881
|
const eligible = this.inbox.filter((p) => this.inScope(p, scope));
|
|
739
882
|
const n = limit && limit > 0 ? Math.min(limit, eligible.length) : eligible.length;
|
|
740
883
|
const selected = eligible.slice(0, n);
|
|
884
|
+
// Head progress is read from the LIVE queue, so it must be recorded while the batch is still in
|
|
885
|
+
// it (#1526).
|
|
886
|
+
this.noteAutomaticHeadProgress(selected);
|
|
741
887
|
// Remove by OBJECT IDENTITY, not by a set of wire ids (#624): the id-less entries share "", so
|
|
742
888
|
// an id set would remove every id-less neighbor beyond the limit and outside the scope while
|
|
743
889
|
// acking only the selected — silent loss by selection. Identity removes exactly what was taken.
|
|
744
890
|
const taken = new Set(selected);
|
|
745
891
|
this.inbox = this.inbox.filter((p) => !taken.has(p));
|
|
892
|
+
const automatic = selected.some((p) => !p.pullOnly);
|
|
746
893
|
const items = this.commitPending(selected);
|
|
747
894
|
if (items.length)
|
|
748
895
|
this._lastInboxDrainedAt = Date.now();
|
|
896
|
+
// Automatic progress is recorded from the SELECTED entries' own classification, not from the
|
|
897
|
+
// returned items: `commitPending` returns `InboxItem`s, which do not carry `pullOnly`, and
|
|
898
|
+
// counting a pull-only drain as automatic progress is the #1233 lie in miniature.
|
|
899
|
+
if (items.length && automatic)
|
|
900
|
+
this._lastAutomaticDrainedAt = Date.now();
|
|
749
901
|
return items;
|
|
750
902
|
}
|
|
751
903
|
/** Ack exact surfaced deliveries without assuming they still form the physical inbox prefix.
|
|
@@ -759,6 +911,7 @@ export class MeshAgent extends EventEmitter {
|
|
|
759
911
|
const requested = [...new Set(keys)];
|
|
760
912
|
const wanted = new Set(requested);
|
|
761
913
|
const selected = this.inbox.filter((p) => wanted.has(p.item.recvKey));
|
|
914
|
+
this.noteAutomaticHeadProgress(selected);
|
|
762
915
|
const present = new Set(selected.map((p) => p.item.recvKey));
|
|
763
916
|
const pullOnly = new Map(selected.map((p) => [p.item.recvKey, p.pullOnly]));
|
|
764
917
|
for (const id of requested) {
|
|
@@ -767,9 +920,12 @@ export class MeshAgent extends EventEmitter {
|
|
|
767
920
|
pullOnly.set(id, remembered.pullOnly);
|
|
768
921
|
}
|
|
769
922
|
this.inbox = this.inbox.filter((p) => !present.has(p.item.recvKey));
|
|
923
|
+
const automatic = selected.some((p) => !p.pullOnly);
|
|
770
924
|
const items = this.commitPending(selected);
|
|
771
925
|
if (items.length)
|
|
772
926
|
this._lastInboxDrainedAt = Date.now();
|
|
927
|
+
if (items.length && automatic)
|
|
928
|
+
this._lastAutomaticDrainedAt = Date.now();
|
|
773
929
|
for (const id of requested) {
|
|
774
930
|
// A MINTED key (an id-less delivery) is never handled-authority: its wire id is "", which
|
|
775
931
|
// markHandled already refuses, so skipping it here is the same at-least-once stance rather
|
|
@@ -1468,8 +1624,18 @@ export class MeshAgent extends EventEmitter {
|
|
|
1468
1624
|
if (!isConcreteChannel(def.announce))
|
|
1469
1625
|
throw new Error(`announce: "${def.announce}" is a wildcard — announce to one concrete channel`);
|
|
1470
1626
|
}
|
|
1471
|
-
// role
|
|
1472
|
-
|
|
1627
|
+
// Explicit role / agent / grants ride with the prompt. A leading frontmatter block in `prompt`
|
|
1628
|
+
// is merged by the manager (this layer forwards the fields; it does not wrap).
|
|
1629
|
+
const args = {
|
|
1630
|
+
name: def.name,
|
|
1631
|
+
model: def.model,
|
|
1632
|
+
persona: def.prompt,
|
|
1633
|
+
role: def.role,
|
|
1634
|
+
agent: def.agent,
|
|
1635
|
+
subscribe: def.subscribe,
|
|
1636
|
+
allowSubscribe: def.allowSubscribe,
|
|
1637
|
+
allowPublish: def.allowPublish,
|
|
1638
|
+
};
|
|
1473
1639
|
const reply = await this.managerInvoke("define-persona", args);
|
|
1474
1640
|
if (!reply.ok || !def.announce)
|
|
1475
1641
|
return reply;
|