agentchatme 1.4.0 → 1.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +124 -0
- package/README.md +27 -6
- package/dist/index.cjs +1123 -100
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +723 -43
- package/dist/index.d.ts +723 -43
- package/dist/index.js +1123 -100
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -47,6 +47,11 @@ var ErrorCode = {
|
|
|
47
47
|
RESTRICTED: "RESTRICTED",
|
|
48
48
|
CONVERSATION_NOT_FOUND: "CONVERSATION_NOT_FOUND",
|
|
49
49
|
MESSAGE_NOT_FOUND: "MESSAGE_NOT_FOUND",
|
|
50
|
+
/**
|
|
51
|
+
* 410 when you download the attachment of a message hidden from you
|
|
52
|
+
* because your machine could not process it (`Message.could_not_process`).
|
|
53
|
+
*/
|
|
54
|
+
COULD_NOT_PROCESS: "COULD_NOT_PROCESS",
|
|
50
55
|
GROUP_DELETED: "GROUP_DELETED",
|
|
51
56
|
RATE_LIMITED: "RATE_LIMITED",
|
|
52
57
|
RECIPIENT_BACKLOGGED: "RECIPIENT_BACKLOGGED",
|
|
@@ -308,7 +313,7 @@ function createAgentChatError(body, status, headers) {
|
|
|
308
313
|
}
|
|
309
314
|
|
|
310
315
|
// src/version.ts
|
|
311
|
-
var VERSION = "1.
|
|
316
|
+
var VERSION = "1.5.1" ;
|
|
312
317
|
|
|
313
318
|
// src/runtime.ts
|
|
314
319
|
function detectRuntime() {
|
|
@@ -674,6 +679,7 @@ function parseBacklogWarning(header2) {
|
|
|
674
679
|
if (!Number.isFinite(undeliveredCount) || !Number.isInteger(undeliveredCount)) return null;
|
|
675
680
|
return { recipientHandle, undeliveredCount };
|
|
676
681
|
}
|
|
682
|
+
var PROCESSING_FAILURE_ACTIONS = /* @__PURE__ */ new Set(["retry", "hidden", "paused", "none"]);
|
|
677
683
|
function intHeader(headers, name) {
|
|
678
684
|
const raw = headers?.get?.(name);
|
|
679
685
|
if (typeof raw !== "string" || !/^\d+$/.test(raw.trim())) return null;
|
|
@@ -765,7 +771,8 @@ var AgentChatClient = class _AgentChatClient {
|
|
|
765
771
|
record: (conversationId, opts) => this.recordDecision(conversationId, opts)
|
|
766
772
|
};
|
|
767
773
|
this.conversations = {
|
|
768
|
-
markRead: (conversationId, throughMessageId, opts) => this.markConversationRead(conversationId, throughMessageId, opts)
|
|
774
|
+
markRead: (conversationId, throughMessageId, opts) => this.markConversationRead(conversationId, throughMessageId, opts),
|
|
775
|
+
reportProcessingFailure: (conversationId, opts) => this.reportProcessingFailure(conversationId, opts)
|
|
769
776
|
};
|
|
770
777
|
this.texts = {
|
|
771
778
|
get: (opts) => this.getTexts(opts)
|
|
@@ -837,6 +844,36 @@ var AgentChatClient = class _AgentChatClient {
|
|
|
837
844
|
options
|
|
838
845
|
);
|
|
839
846
|
}
|
|
847
|
+
async reportProcessingFailure(conversationId, options) {
|
|
848
|
+
const body = {
|
|
849
|
+
through_message_id: options.throughMessageId,
|
|
850
|
+
outcome: options.outcome,
|
|
851
|
+
evidence: options.evidence
|
|
852
|
+
};
|
|
853
|
+
let res;
|
|
854
|
+
try {
|
|
855
|
+
res = await this.post(
|
|
856
|
+
`/v1/conversations/${encodeURIComponent(conversationId)}/processing-failures`,
|
|
857
|
+
body,
|
|
858
|
+
// One report, one count: a retried request carries the same key.
|
|
859
|
+
{ ...options, idempotencyKey: options.idempotencyKey ?? generateClientMsgId() }
|
|
860
|
+
);
|
|
861
|
+
} catch (err) {
|
|
862
|
+
if (err instanceof AgentChatError && err.status === 404 && err.code !== "CONVERSATION_NOT_FOUND" && err.code !== "MESSAGE_NOT_FOUND") {
|
|
863
|
+
return { action: "none", attempts: 0, hidden_message_ids: [], paused_until: null };
|
|
864
|
+
}
|
|
865
|
+
throw err;
|
|
866
|
+
}
|
|
867
|
+
const r = res !== null && typeof res === "object" ? res : {};
|
|
868
|
+
const action = PROCESSING_FAILURE_ACTIONS.has(r.action) ? r.action : "none";
|
|
869
|
+
return {
|
|
870
|
+
action,
|
|
871
|
+
attempts: typeof r.attempts === "number" && Number.isFinite(r.attempts) ? r.attempts : 0,
|
|
872
|
+
hidden_message_ids: Array.isArray(r.hidden_message_ids) ? r.hidden_message_ids.filter((id) => typeof id === "string") : [],
|
|
873
|
+
...typeof r.hidden_count === "number" && Number.isFinite(r.hidden_count) ? { hidden_count: r.hidden_count } : {},
|
|
874
|
+
paused_until: typeof r.paused_until === "string" ? r.paused_until : null
|
|
875
|
+
};
|
|
876
|
+
}
|
|
840
877
|
getTexts(options) {
|
|
841
878
|
const params = new URLSearchParams({ runtime: options.runtime });
|
|
842
879
|
return this.get(`/v1/texts?${params.toString()}`, options);
|
|
@@ -1101,24 +1138,48 @@ var AgentChatClient = class _AgentChatClient {
|
|
|
1101
1138
|
* most one:
|
|
1102
1139
|
* - `beforeSeq` — backwards scrollback (rows with seq < N, newest first)
|
|
1103
1140
|
* - `afterSeq` — forwards gap-fill (rows with seq > N, oldest first)
|
|
1141
|
+
* - `afterMessageId` — forwards from a message you hold (instead of `afterSeq`)
|
|
1104
1142
|
* - `aroundMessageId` — backwards window ending at that exact message
|
|
1105
1143
|
*
|
|
1106
1144
|
* `afterSeq` is the path `RealtimeClient` uses for in-order recovery
|
|
1107
1145
|
* when a per-conversation seq gap is detected. Application code usually
|
|
1108
1146
|
* only needs `beforeSeq` for normal pagination.
|
|
1147
|
+
*
|
|
1148
|
+
* With `waitMs` > 0 (and `afterSeq` or `afterMessageId`) this is the
|
|
1149
|
+
* scoped wait: the server holds the request until a message from someone
|
|
1150
|
+
* else arrives after the cursor, or answers `[]` after `waitMs`. One
|
|
1151
|
+
* conversation only, no sync ack. `getMessagesPage` also returns how long
|
|
1152
|
+
* the server held it.
|
|
1109
1153
|
*/
|
|
1110
|
-
getMessages(conversationId, options) {
|
|
1154
|
+
async getMessages(conversationId, options) {
|
|
1155
|
+
return (await this.getMessagesPage(conversationId, options)).messages;
|
|
1156
|
+
}
|
|
1157
|
+
/** `getMessages()` with the response facts a waiter needs: how long the server held a scoped wait (`waitedMs`). */
|
|
1158
|
+
async getMessagesPage(conversationId, options) {
|
|
1111
1159
|
const params = new URLSearchParams();
|
|
1112
1160
|
params.set("limit", String(options?.limit ?? 50));
|
|
1113
1161
|
if (options?.beforeSeq !== void 0) params.set("before_seq", String(options.beforeSeq));
|
|
1114
1162
|
if (options?.afterSeq !== void 0) params.set("after_seq", String(options.afterSeq));
|
|
1163
|
+
if (options?.afterMessageId !== void 0) params.set("after_message_id", options.afterMessageId);
|
|
1115
1164
|
if (options?.aroundMessageId !== void 0) {
|
|
1116
1165
|
params.set("around_message_id", options.aroundMessageId);
|
|
1117
1166
|
}
|
|
1118
|
-
|
|
1167
|
+
const waitMs = Math.floor(options?.waitMs ?? 0);
|
|
1168
|
+
if (waitMs > 0) params.set("wait_ms", String(waitMs));
|
|
1169
|
+
const timeoutMs = options?.timeoutMs !== void 0 ? options.timeoutMs : waitMs > 0 ? waitMs + INBOX_WAIT_TIMEOUT_GRACE_MS : void 0;
|
|
1170
|
+
const res = await this.http.request(
|
|
1171
|
+
"GET",
|
|
1119
1172
|
`/v1/messages/${encodeURIComponent(conversationId)}?${params.toString()}`,
|
|
1120
|
-
options
|
|
1173
|
+
this.toRequestOpts({ ...options, timeoutMs })
|
|
1121
1174
|
);
|
|
1175
|
+
const waitedMs = intHeader(res.headers, "x-agentchat-waited-ms");
|
|
1176
|
+
return {
|
|
1177
|
+
messages: res.data,
|
|
1178
|
+
waitedMs,
|
|
1179
|
+
scopedWaitSupported: waitMs > 0 ? waitedMs !== null : null,
|
|
1180
|
+
requestId: res.requestId,
|
|
1181
|
+
headers: res.headers
|
|
1182
|
+
};
|
|
1122
1183
|
}
|
|
1123
1184
|
/**
|
|
1124
1185
|
* Hide a message from your own view (hide-for-me). Either side of the
|
|
@@ -1410,11 +1471,13 @@ var AgentChatClient = class _AgentChatClient {
|
|
|
1410
1471
|
}
|
|
1411
1472
|
// ─── Mutes ────────────────────────────────────────────────────────────────
|
|
1412
1473
|
//
|
|
1413
|
-
// Mute
|
|
1414
|
-
//
|
|
1415
|
-
//
|
|
1416
|
-
//
|
|
1417
|
-
//
|
|
1474
|
+
// Mute stops the wake for a specific agent or conversation without
|
|
1475
|
+
// blocking or leaving: no real-time WebSocket push and no wake webhook.
|
|
1476
|
+
// Nothing is hidden — the conversation stays on the inbox tray
|
|
1477
|
+
// (`muted: true`), envelopes still land in `/v1/messages/sync`, and the
|
|
1478
|
+
// unread counter still bumps. A muted message reaches the machine at its
|
|
1479
|
+
// next sync (every reconnect) or when the agent opens the conversation;
|
|
1480
|
+
// until then the sender sees it as `stored`, not `delivered`.
|
|
1418
1481
|
//
|
|
1419
1482
|
// All mute APIs are idempotent:
|
|
1420
1483
|
// - Re-muting with a different `mutedUntil` refreshes the expiry.
|
|
@@ -3562,6 +3625,7 @@ __export(integration_exports, {
|
|
|
3562
3625
|
anchor: () => anchor_exports,
|
|
3563
3626
|
applyInviteOutcome: () => applyInviteOutcome,
|
|
3564
3627
|
applyOutcome: () => applyOutcome,
|
|
3628
|
+
conversationRunPrompt: () => conversationRunPrompt,
|
|
3565
3629
|
hostedMcpUrl: () => hostedMcpUrl,
|
|
3566
3630
|
inviteOutcome: () => inviteOutcome,
|
|
3567
3631
|
isTexts: () => isTexts,
|
|
@@ -4797,6 +4861,7 @@ function readResult(text2) {
|
|
|
4797
4861
|
// src/integration/receive-loop.ts
|
|
4798
4862
|
var TRAY_LIMIT = 50;
|
|
4799
4863
|
var SILENT_MEMORY = 1e3;
|
|
4864
|
+
var CONVERSATION_KEY = "conversation:";
|
|
4800
4865
|
var PACE_MS = 6e4;
|
|
4801
4866
|
var TURN_OVERDUE_MS = 30 * 6e4;
|
|
4802
4867
|
var NO_PROGRESS_BACKOFF_MS = 1e3;
|
|
@@ -4806,6 +4871,11 @@ var CYCLE_TIMEOUT_MS = 10 * 6e4;
|
|
|
4806
4871
|
var GATE_CHUNK_SIZE = 10;
|
|
4807
4872
|
var REPLY_TURN_PLACEHOLDER = "<conversation_id>";
|
|
4808
4873
|
var RECENT_CYCLES = 20;
|
|
4874
|
+
var MAX_RERUNS_PER_CYCLE = 3;
|
|
4875
|
+
var SPLIT_STOP_AFTER_FAILURES = 2;
|
|
4876
|
+
var RUNTIME_CHECK_TIMEOUT_MS = 6e4;
|
|
4877
|
+
var AWAITING_EVIDENCE_MS = 10 * 6e4;
|
|
4878
|
+
var EVIDENCE_MEMORY = 100;
|
|
4809
4879
|
var Pacer = class {
|
|
4810
4880
|
constructor(paceMs, now2) {
|
|
4811
4881
|
this.paceMs = paceMs;
|
|
@@ -4814,30 +4884,121 @@ var Pacer = class {
|
|
|
4814
4884
|
paceMs;
|
|
4815
4885
|
now;
|
|
4816
4886
|
until = /* @__PURE__ */ new Map();
|
|
4887
|
+
/** Orders holds and tray reads: a read lifts only holds older than its start. */
|
|
4888
|
+
seq = 0;
|
|
4889
|
+
/** A D5 pace. It never replaces a could-not-process hold at the same anchor (that input stays held). */
|
|
4817
4890
|
hold(key, anchor) {
|
|
4818
|
-
this.until.
|
|
4819
|
-
|
|
4820
|
-
|
|
4821
|
-
|
|
4822
|
-
|
|
4823
|
-
|
|
4824
|
-
|
|
4825
|
-
|
|
4826
|
-
|
|
4891
|
+
const prior = this.until.get(key);
|
|
4892
|
+
if (prior?.failure !== void 0 && prior.anchor === anchor) return;
|
|
4893
|
+
this.set(key, { anchor, until: this.now() + this.paceMs });
|
|
4894
|
+
}
|
|
4895
|
+
/** Could not process: hold the key at this anchor until a newer message moves it (held again at it: since the first time). */
|
|
4896
|
+
holdFailed(key, anchor, reason) {
|
|
4897
|
+
const prior = this.until.get(key);
|
|
4898
|
+
const since = prior?.failure !== void 0 && prior.anchor === anchor ? prior.failure.since : this.now();
|
|
4899
|
+
this.set(key, { anchor, until: Number.POSITIVE_INFINITY, failure: { reason, since, seq: ++this.seq } });
|
|
4900
|
+
}
|
|
4901
|
+
/** What holds the key at this anchor now: a pace, a could-not-process hold, or nothing (a stale entry goes). */
|
|
4902
|
+
check(key, anchor) {
|
|
4827
4903
|
const entry = this.until.get(key);
|
|
4828
|
-
if (!entry) return
|
|
4904
|
+
if (!entry) return null;
|
|
4829
4905
|
if (entry.anchor !== anchor || this.now() >= entry.until) {
|
|
4830
4906
|
this.until.delete(key);
|
|
4831
|
-
return
|
|
4907
|
+
return null;
|
|
4832
4908
|
}
|
|
4833
|
-
return
|
|
4909
|
+
return entry.failure ? "failed" : "paced";
|
|
4910
|
+
}
|
|
4911
|
+
held(key, anchor) {
|
|
4912
|
+
return this.check(key, anchor) !== null;
|
|
4834
4913
|
}
|
|
4914
|
+
/** A could-not-process hold on the key at exactly this anchor; changes nothing (a queued run's anchor may be older than the card's). */
|
|
4915
|
+
heldFailedAt(key, anchor) {
|
|
4916
|
+
const entry = this.until.get(key);
|
|
4917
|
+
return entry?.failure !== void 0 && entry.anchor === anchor;
|
|
4918
|
+
}
|
|
4919
|
+
/** Lift whatever holds the key (a pace or a could-not-process hold). */
|
|
4835
4920
|
release(key) {
|
|
4836
4921
|
this.until.delete(key);
|
|
4837
4922
|
}
|
|
4923
|
+
/** Lift a pace; a could-not-process hold stays. */
|
|
4924
|
+
releasePace(key) {
|
|
4925
|
+
if (this.until.get(key)?.failure === void 0) this.until.delete(key);
|
|
4926
|
+
}
|
|
4838
4927
|
releaseInvites() {
|
|
4839
4928
|
for (const key of [...this.until.keys()]) if (key.startsWith("invite:")) this.until.delete(key);
|
|
4840
4929
|
}
|
|
4930
|
+
/** Taken before a tray read: `keepFailedOn` lifts only holds made before it. */
|
|
4931
|
+
mark() {
|
|
4932
|
+
return this.seq;
|
|
4933
|
+
}
|
|
4934
|
+
/**
|
|
4935
|
+
* After a tray read from `mark` (`anchors`: each card's key and anchor):
|
|
4936
|
+
* a hold whose card shows another anchor is over (a newer message came),
|
|
4937
|
+
* and, on a complete tray (`complete`: not truncated), so is one whose
|
|
4938
|
+
* card is gone (read meanwhile: a verdict, a mark-read). Only holds made
|
|
4939
|
+
* before the read started: a read that began earlier says nothing of a
|
|
4940
|
+
* later hold.
|
|
4941
|
+
*/
|
|
4942
|
+
keepFailedOn(anchors, mark, complete) {
|
|
4943
|
+
for (const [key, entry] of this.until) {
|
|
4944
|
+
if (entry.failure === void 0 || entry.failure.seq > mark) continue;
|
|
4945
|
+
const shown = anchors.get(key);
|
|
4946
|
+
if (shown === void 0 ? complete : shown !== entry.anchor) this.until.delete(key);
|
|
4947
|
+
}
|
|
4948
|
+
}
|
|
4949
|
+
/** The could-not-process holds, oldest first (by `since`). */
|
|
4950
|
+
failed() {
|
|
4951
|
+
const out = [];
|
|
4952
|
+
for (const [key, entry] of this.until) {
|
|
4953
|
+
if (entry.failure !== void 0 && entry.anchor !== null) {
|
|
4954
|
+
out.push({ key, anchor: entry.anchor, reason: entry.failure.reason, since: entry.failure.since });
|
|
4955
|
+
}
|
|
4956
|
+
}
|
|
4957
|
+
return out.sort((a, b) => a.since - b.since);
|
|
4958
|
+
}
|
|
4959
|
+
set(key, entry) {
|
|
4960
|
+
this.until.delete(key);
|
|
4961
|
+
this.until.set(key, entry);
|
|
4962
|
+
let paces = 0;
|
|
4963
|
+
let holds = 0;
|
|
4964
|
+
for (const held of this.until.values()) {
|
|
4965
|
+
if (held.failure === void 0) paces += 1;
|
|
4966
|
+
else holds += 1;
|
|
4967
|
+
}
|
|
4968
|
+
for (const [key2, held] of this.until) {
|
|
4969
|
+
if (paces <= SILENT_MEMORY && holds <= SILENT_MEMORY) break;
|
|
4970
|
+
if (held.failure === void 0 && paces > SILENT_MEMORY) {
|
|
4971
|
+
this.until.delete(key2);
|
|
4972
|
+
paces -= 1;
|
|
4973
|
+
} else if (held.failure !== void 0 && holds > SILENT_MEMORY) {
|
|
4974
|
+
this.until.delete(key2);
|
|
4975
|
+
holds -= 1;
|
|
4976
|
+
}
|
|
4977
|
+
}
|
|
4978
|
+
}
|
|
4979
|
+
};
|
|
4980
|
+
function clears(completion, failure) {
|
|
4981
|
+
return completion.kind === failure.source && completion.conversations.has(failure.conversationId);
|
|
4982
|
+
}
|
|
4983
|
+
function evidences(completion, failure) {
|
|
4984
|
+
return completion.kind === failure.source && completion.at > failure.startedAt && completion.conversations.size > 0 && !completion.conversations.has(failure.conversationId);
|
|
4985
|
+
}
|
|
4986
|
+
var EvidenceClock = class {
|
|
4987
|
+
last = 0;
|
|
4988
|
+
completions = [];
|
|
4989
|
+
start() {
|
|
4990
|
+
return ++this.last;
|
|
4991
|
+
}
|
|
4992
|
+
complete(kind, conversations) {
|
|
4993
|
+
const entry = { at: ++this.last, kind, conversations: new Set(conversations) };
|
|
4994
|
+
this.completions.push(entry);
|
|
4995
|
+
if (this.completions.length > EVIDENCE_MEMORY) this.completions.shift();
|
|
4996
|
+
return entry;
|
|
4997
|
+
}
|
|
4998
|
+
/** A completion since the failed attempt started that is evidence for it (`evidences`). */
|
|
4999
|
+
since(failure) {
|
|
5000
|
+
return this.completions.some((c) => evidences(c, failure));
|
|
5001
|
+
}
|
|
4841
5002
|
};
|
|
4842
5003
|
var ReceiveLoop = class {
|
|
4843
5004
|
options;
|
|
@@ -4856,15 +5017,68 @@ var ReceiveLoop = class {
|
|
|
4856
5017
|
/** When a single-run invite's silent run was counted (D5 pacing in single-run mode). */
|
|
4857
5018
|
silentAt = /* @__PURE__ */ new Map();
|
|
4858
5019
|
paceMs;
|
|
5020
|
+
/** Turns and runs waiting for a slot; `drop` ends what else the entry holds (an invite run's other invites). */
|
|
4859
5021
|
waiting = [];
|
|
5022
|
+
/** Conversation and invite runs started and not yet concluded: `idle()` and `stop()` wait for them. */
|
|
5023
|
+
runWork = /* @__PURE__ */ new Set();
|
|
5024
|
+
/** Could not process: the order of attempts and completions. */
|
|
5025
|
+
evidence = new EvidenceClock();
|
|
5026
|
+
/** Could not process: failures waiting for a completion (the evidence a report needs), by conversation. */
|
|
5027
|
+
awaitingEvidence = /* @__PURE__ */ new Map();
|
|
5028
|
+
/** Waiting failures the runtime check has already been asked about (once each). */
|
|
5029
|
+
checkedFailures = /* @__PURE__ */ new WeakSet();
|
|
5030
|
+
/**
|
|
5031
|
+
* Per kind of work, the conversations whose attempt failed since that kind
|
|
5032
|
+
* last completed. Two or more: several failing together is the runtime, so
|
|
5033
|
+
* a passing `checkRuntime()` is no evidence for any of them.
|
|
5034
|
+
*/
|
|
5035
|
+
failingSince = { gate: /* @__PURE__ */ new Set(), turn: /* @__PURE__ */ new Set(), run: /* @__PURE__ */ new Set() };
|
|
5036
|
+
/** gateCall: conversations whose last gate call failed; gated alone, after the others. */
|
|
5037
|
+
gateAlone = /* @__PURE__ */ new Set();
|
|
5038
|
+
/**
|
|
5039
|
+
* gateCall: the running cycle's gate: its cards (`members`) and its
|
|
5040
|
+
* single-card queue. A `retry` answer for one of its cards joins the queue;
|
|
5041
|
+
* a `hidden` one skips the card here (its card, read before the hide, is
|
|
5042
|
+
* stale) and gates it in the next cycle.
|
|
5043
|
+
*/
|
|
5044
|
+
activeSplit = null;
|
|
5045
|
+
/**
|
|
5046
|
+
* gateCall: reply turns to start again on a `retry` / `hidden` answer that
|
|
5047
|
+
* came while a gate was deciding: they start after that cycle's own turns
|
|
5048
|
+
* (no turn starts while cards of a cycle still wait for their gate call).
|
|
5049
|
+
*/
|
|
5050
|
+
turnRerunsAfterGate = [];
|
|
5051
|
+
lastRuntimeCheck = null;
|
|
5052
|
+
failureCounts = {
|
|
5053
|
+
reported: 0,
|
|
5054
|
+
retry: 0,
|
|
5055
|
+
hidden: 0,
|
|
5056
|
+
paused: 0,
|
|
5057
|
+
none: 0,
|
|
5058
|
+
reportErrors: 0,
|
|
5059
|
+
reruns: 0,
|
|
5060
|
+
rerunsCapped: 0,
|
|
5061
|
+
runtimeChecksOk: 0,
|
|
5062
|
+
runtimeChecksFailed: 0,
|
|
5063
|
+
splitsStopped: 0,
|
|
5064
|
+
startsHeld: 0
|
|
5065
|
+
};
|
|
4860
5066
|
/** Conversations whose verdict a gate call is recording right now (an abandoned cycle's included). */
|
|
4861
5067
|
recordsInFlight = /* @__PURE__ */ new Set();
|
|
5068
|
+
/**
|
|
5069
|
+
* Could not process: conversations whose report is out on another's
|
|
5070
|
+
* completion (`takeDue`): held out of the gate and of new runs, and a run
|
|
5071
|
+
* or turn of theirs waiting for a slot waits, until the server answers.
|
|
5072
|
+
*/
|
|
5073
|
+
reporting = /* @__PURE__ */ new Map();
|
|
4862
5074
|
/** Turns holding a slot (admitted, not yet ended or released). */
|
|
4863
5075
|
slotted = /* @__PURE__ */ new Set();
|
|
4864
5076
|
handle;
|
|
4865
5077
|
started = false;
|
|
4866
5078
|
stopped = false;
|
|
4867
5079
|
heldWhileRunning = false;
|
|
5080
|
+
/** A cycle left a card out because its could-not-process report was out: `letGo` reads the tray again. */
|
|
5081
|
+
heldWhileReporting = false;
|
|
4868
5082
|
activeTurns = 0;
|
|
4869
5083
|
noProgressStreak = 0;
|
|
4870
5084
|
moreTimer = null;
|
|
@@ -4891,6 +5105,8 @@ var ReceiveLoop = class {
|
|
|
4891
5105
|
runsNotStarted: 0,
|
|
4892
5106
|
chunksOk: 0,
|
|
4893
5107
|
chunksFailed: 0,
|
|
5108
|
+
splitCallsOk: 0,
|
|
5109
|
+
splitCallsFailed: 0,
|
|
4894
5110
|
outcomeRecordFailures: 0
|
|
4895
5111
|
};
|
|
4896
5112
|
outcomes = {};
|
|
@@ -4988,7 +5204,10 @@ var ReceiveLoop = class {
|
|
|
4988
5204
|
this.clearMoreTimer();
|
|
4989
5205
|
this.abort.abort();
|
|
4990
5206
|
this.cycleAbort?.abort();
|
|
4991
|
-
for (const { handle } of this.waiting.splice(0))
|
|
5207
|
+
for (const { handle, drop } of this.waiting.splice(0)) {
|
|
5208
|
+
this.tracker.end(handle);
|
|
5209
|
+
drop?.();
|
|
5210
|
+
}
|
|
4992
5211
|
for (const off of this.unsubscribe.splice(0)) {
|
|
4993
5212
|
try {
|
|
4994
5213
|
off();
|
|
@@ -5000,11 +5219,21 @@ var ReceiveLoop = class {
|
|
|
5000
5219
|
} catch {
|
|
5001
5220
|
}
|
|
5002
5221
|
await this.coalescer.idle();
|
|
5003
|
-
await Promise.allSettled([...this.pendingWork]);
|
|
5222
|
+
await Promise.allSettled([...this.pendingWork, ...this.runWork]);
|
|
5004
5223
|
}
|
|
5005
|
-
/**
|
|
5006
|
-
|
|
5007
|
-
|
|
5224
|
+
/**
|
|
5225
|
+
* Resolves when the running cycle has finished (tests, graceful stops) and,
|
|
5226
|
+
* in single-run mode, no run is running or waiting for a slot (conversation
|
|
5227
|
+
* runs go on after the cycle that started them). Gate-call turns are not
|
|
5228
|
+
* waited for.
|
|
5229
|
+
*/
|
|
5230
|
+
async idle() {
|
|
5231
|
+
for (; ; ) {
|
|
5232
|
+
await this.coalescer.idle();
|
|
5233
|
+
const runs = [...this.runWork];
|
|
5234
|
+
if (runs.length === 0) return;
|
|
5235
|
+
await Promise.allSettled(runs);
|
|
5236
|
+
}
|
|
5008
5237
|
}
|
|
5009
5238
|
/** Let go of a turn whose end the runtime will never confirm (an operator's call). Nothing is recorded for it. */
|
|
5010
5239
|
releaseTurn(key) {
|
|
@@ -5044,6 +5273,17 @@ var ReceiveLoop = class {
|
|
|
5044
5273
|
turnsWaiting: this.waiting.length,
|
|
5045
5274
|
outcomes: { ...this.outcomes }
|
|
5046
5275
|
},
|
|
5276
|
+
processingFailures: {
|
|
5277
|
+
...this.failureCounts,
|
|
5278
|
+
awaitingEvidence: this.awaitingEvidence.size,
|
|
5279
|
+
lastRuntimeCheck: this.lastRuntimeCheck === null ? null : { ...this.lastRuntimeCheck },
|
|
5280
|
+
held: this.pacer.failed().map(({ key, anchor, reason, since }) => ({
|
|
5281
|
+
conversationId: key.slice(CONVERSATION_KEY.length),
|
|
5282
|
+
anchor,
|
|
5283
|
+
since: new Date(since).toISOString(),
|
|
5284
|
+
reason
|
|
5285
|
+
}))
|
|
5286
|
+
},
|
|
5047
5287
|
lastError: this.lastError
|
|
5048
5288
|
};
|
|
5049
5289
|
}
|
|
@@ -5072,7 +5312,7 @@ var ReceiveLoop = class {
|
|
|
5072
5312
|
const id = typeof payload.conversation_id === "string" ? payload.conversation_id : void 0;
|
|
5073
5313
|
const named = Array.isArray(payload.conversation_ids) ? payload.conversation_ids.filter((x) => typeof x === "string") : [];
|
|
5074
5314
|
if (reason === "unanswered") {
|
|
5075
|
-
for (const conversationId of named) this.pacer.
|
|
5315
|
+
for (const conversationId of named) this.pacer.releasePace(`${CONVERSATION_KEY}${conversationId}`);
|
|
5076
5316
|
this.pacer.releaseInvites();
|
|
5077
5317
|
}
|
|
5078
5318
|
this.wakeFrom("wake_frame", id, {
|
|
@@ -5115,6 +5355,8 @@ var ReceiveLoop = class {
|
|
|
5115
5355
|
const work = (async () => {
|
|
5116
5356
|
const summary = this.options.mode === "gateCall" ? await this.gateCallCycle(run3) : await this.singleRunCycle(run3);
|
|
5117
5357
|
if (!run3.abandoned()) this.finishCycle(summary);
|
|
5358
|
+
this.flushTurnReruns();
|
|
5359
|
+
void this.track(this.checkParked(this.options.mode === "gateCall" ? "turn" : "run"));
|
|
5118
5360
|
})();
|
|
5119
5361
|
let timer = null;
|
|
5120
5362
|
const overdue = new Promise((resolve) => {
|
|
@@ -5182,12 +5424,13 @@ var ReceiveLoop = class {
|
|
|
5182
5424
|
* cycle that is making progress is not abandoned for it. With the deadline
|
|
5183
5425
|
* off the watchdog is the calls' only bound, so they count.
|
|
5184
5426
|
*/
|
|
5185
|
-
boundedModelCall(run3) {
|
|
5186
|
-
const call2 = this.deadlineModelCall(run3);
|
|
5427
|
+
boundedModelCall(run3, timedOut) {
|
|
5428
|
+
const call2 = this.deadlineModelCall(run3, timedOut);
|
|
5187
5429
|
if (!Number.isFinite(this.gateCallTimeoutMs)) return call2;
|
|
5188
5430
|
return (prompt, options) => run3.exclude(() => call2(prompt, options));
|
|
5189
5431
|
}
|
|
5190
|
-
|
|
5432
|
+
/** `timedOut` is set when a call runs past its deadline (a could-not-process report then says `timed_out`). */
|
|
5433
|
+
deadlineModelCall(run3, timedOut) {
|
|
5191
5434
|
return (prompt) => new Promise((resolve, reject) => {
|
|
5192
5435
|
const controller = new AbortController();
|
|
5193
5436
|
const abort = () => {
|
|
@@ -5201,6 +5444,7 @@ var ReceiveLoop = class {
|
|
|
5201
5444
|
run3.signal.addEventListener("abort", abort, { once: true });
|
|
5202
5445
|
const timer = Number.isFinite(this.gateCallTimeoutMs) ? this._setTimeout(() => {
|
|
5203
5446
|
run3.signal.removeEventListener("abort", abort);
|
|
5447
|
+
if (timedOut) timedOut.value = true;
|
|
5204
5448
|
controller.abort();
|
|
5205
5449
|
reject(new Error(`no answer within ${this.gateCallTimeoutMs} ms`));
|
|
5206
5450
|
}, this.gateCallTimeoutMs) : null;
|
|
@@ -5248,20 +5492,22 @@ var ReceiveLoop = class {
|
|
|
5248
5492
|
}
|
|
5249
5493
|
}
|
|
5250
5494
|
async gateCallCycle(run3) {
|
|
5495
|
+
const mark = this.pacer.mark();
|
|
5251
5496
|
const tray = await this.readTray();
|
|
5252
5497
|
if (run3.stale()) return this.summary("abandoned");
|
|
5253
5498
|
if (tray === null) return this.summary("tray_failed", { failure: this.lastError });
|
|
5254
5499
|
const cards = cardsOf(tray);
|
|
5500
|
+
this.forgetCleared(tray, mark);
|
|
5255
5501
|
const invites = trayInvites(tray);
|
|
5256
5502
|
const unheld = this.unheld(cards);
|
|
5257
5503
|
const heldByTurns = cards.length - unheld.length;
|
|
5258
5504
|
if (heldByTurns > 0) this.heldWhileRunning = true;
|
|
5259
|
-
const gateCards
|
|
5505
|
+
const { open: gateCards, paced: pacedCards, heldAfterFailure } = this.paceCards(unheld);
|
|
5260
5506
|
const openInvites = invites.filter((invite) => !this.tracker.inviteRunning(invite.invite_id));
|
|
5261
5507
|
const owedInvites = openInvites.filter((invite) => !this.pacer.held(`invite:${invite.invite_id}`, null));
|
|
5262
|
-
const paced =
|
|
5508
|
+
const paced = pacedCards + (openInvites.length - owedInvites.length);
|
|
5263
5509
|
const owedCards = gateCards.filter((card) => card.muted !== true);
|
|
5264
|
-
const base = { cards: cards.length, invites: invites.length, heldByTurns, paced, truncated: tray.truncated === true };
|
|
5510
|
+
const base = { cards: cards.length, invites: invites.length, heldByTurns, paced, heldAfterFailure, truncated: tray.truncated === true };
|
|
5265
5511
|
if (cards.length === 0 && invites.length === 0) return this.summary("empty", base);
|
|
5266
5512
|
if (owedCards.length === 0 && owedInvites.length === 0) return this.summary("nothing_owed", base);
|
|
5267
5513
|
const texts = await this.options.texts.ensure();
|
|
@@ -5282,14 +5528,19 @@ var ReceiveLoop = class {
|
|
|
5282
5528
|
turnsRefused: 0,
|
|
5283
5529
|
turnsQueued: 0,
|
|
5284
5530
|
chunksOk: 0,
|
|
5285
|
-
chunksFailed: 0
|
|
5531
|
+
chunksFailed: 0,
|
|
5532
|
+
splitCallsOk: 0,
|
|
5533
|
+
splitCallsFailed: 0,
|
|
5534
|
+
heldAfterFailure
|
|
5286
5535
|
};
|
|
5287
5536
|
let heldDuringGate = 0;
|
|
5537
|
+
let splitStopped = false;
|
|
5288
5538
|
let outcome = "invites_only";
|
|
5289
5539
|
let failure = null;
|
|
5290
5540
|
if (owedCards.length > 0) {
|
|
5291
5541
|
const gate = await this.gateInChunks(run3, tray, gateCards, texts);
|
|
5292
5542
|
heldDuringGate = gate.held;
|
|
5543
|
+
tally.heldAfterFailure += gate.heldAfterFailure;
|
|
5293
5544
|
tally.gated = gate.gated;
|
|
5294
5545
|
tally.reply = gate.verdicts.reply;
|
|
5295
5546
|
tally.dismiss = gate.verdicts.dismiss;
|
|
@@ -5299,24 +5550,27 @@ var ReceiveLoop = class {
|
|
|
5299
5550
|
tally.recordFailures = gate.recordFailures;
|
|
5300
5551
|
tally.chunksOk = gate.chunks.filter((c) => c.outcome === "decided").length;
|
|
5301
5552
|
tally.chunksFailed = gate.chunks.length - tally.chunksOk;
|
|
5302
|
-
|
|
5303
|
-
|
|
5553
|
+
tally.splitCallsOk = gate.split.filter((c) => c.outcome === "decided").length;
|
|
5554
|
+
tally.splitCallsFailed = gate.split.length - tally.splitCallsOk;
|
|
5555
|
+
splitStopped = gate.splitStopped;
|
|
5556
|
+
failure = gateFailureText(gate.chunks, gate.split);
|
|
5557
|
+
if (tally.chunksOk + tally.splitCallsOk > 0) {
|
|
5304
5558
|
outcome = "decided";
|
|
5305
5559
|
for (const reply of gate.reply) {
|
|
5306
5560
|
if (run3.stale()) break;
|
|
5307
5561
|
tally[launchKey(await run3.exclude(() => this.startReplyTurn(reply, texts)))] += 1;
|
|
5308
5562
|
}
|
|
5309
|
-
} else if (tally.chunksFailed > 0) {
|
|
5563
|
+
} else if (tally.chunksFailed + tally.splitCallsFailed > 0) {
|
|
5310
5564
|
outcome = "fail_closed";
|
|
5311
5565
|
} else if (owedInvites.length === 0 && !run3.stale()) {
|
|
5312
|
-
return this.summary("nothing_owed", { ...base, heldByTurns: heldByTurns + heldDuringGate });
|
|
5566
|
+
return this.summary("nothing_owed", { ...base, heldByTurns: heldByTurns + heldDuringGate, heldAfterFailure: tally.heldAfterFailure });
|
|
5313
5567
|
}
|
|
5314
5568
|
}
|
|
5315
5569
|
for (const invite of owedInvites) {
|
|
5316
5570
|
if (run3.stale()) break;
|
|
5317
5571
|
tally[launchKey(await run3.exclude(() => this.startInviteTurn(invite, texts)))] += 1;
|
|
5318
5572
|
}
|
|
5319
|
-
const parts = { ...base, heldByTurns: heldByTurns + heldDuringGate, ...tally };
|
|
5573
|
+
const parts = { ...base, heldByTurns: heldByTurns + heldDuringGate, ...tally, splitStopped };
|
|
5320
5574
|
if (run3.stale()) return this.summary("abandoned", parts);
|
|
5321
5575
|
if (tray.truncated === true && outcome === "decided") {
|
|
5322
5576
|
this.readTheRest(tally.recorded > 0 || tally.turnsStarted > 0);
|
|
@@ -5326,10 +5580,54 @@ var ReceiveLoop = class {
|
|
|
5326
5580
|
/**
|
|
5327
5581
|
* Held out of the gate: a running or waiting turn, or a verdict still being
|
|
5328
5582
|
* recorded (an abandoned cycle's last call): gating it again could record
|
|
5329
|
-
* the same verdict twice (a second defer climbs the ladder).
|
|
5583
|
+
* the same verdict twice (a second defer climbs the ladder). Also a
|
|
5584
|
+
* conversation whose could-not-process report is out (`reporting`): its
|
|
5585
|
+
* answer may hold it.
|
|
5330
5586
|
*/
|
|
5331
5587
|
unheld(cards) {
|
|
5332
|
-
return this.tracker.withoutRunning(cards).filter((card) =>
|
|
5588
|
+
return this.tracker.withoutRunning(cards).filter((card) => {
|
|
5589
|
+
if (this.recordsInFlight.has(card.conversation_id)) return false;
|
|
5590
|
+
if (!this.reporting.has(card.conversation_id)) return true;
|
|
5591
|
+
this.heldWhileReporting = true;
|
|
5592
|
+
return false;
|
|
5593
|
+
});
|
|
5594
|
+
}
|
|
5595
|
+
/**
|
|
5596
|
+
* The cards neither paced (D5) nor held after a could-not-process answer
|
|
5597
|
+
* (`none` / `paused` through the anchor they still show), and how many of
|
|
5598
|
+
* each were left out. A card whose anchor moved is no longer held.
|
|
5599
|
+
*/
|
|
5600
|
+
paceCards(cards) {
|
|
5601
|
+
const open = [];
|
|
5602
|
+
let paced = 0;
|
|
5603
|
+
let heldAfterFailure = 0;
|
|
5604
|
+
for (const card of cards) {
|
|
5605
|
+
const state = this.pacer.check(`${CONVERSATION_KEY}${card.conversation_id}`, anchorFrom(card));
|
|
5606
|
+
if (state === "paced") paced += 1;
|
|
5607
|
+
else if (state === "failed") heldAfterFailure += 1;
|
|
5608
|
+
else open.push(card);
|
|
5609
|
+
}
|
|
5610
|
+
return { open, paced, heldAfterFailure };
|
|
5611
|
+
}
|
|
5612
|
+
/** Held after a could-not-process answer, at the anchor this card still shows. */
|
|
5613
|
+
heldAfterFailure(card) {
|
|
5614
|
+
return this.pacer.check(`${CONVERSATION_KEY}${card.conversation_id}`, anchorFrom(card)) === "failed";
|
|
5615
|
+
}
|
|
5616
|
+
/**
|
|
5617
|
+
* A tray read after `mark`, every card on it (a turn's included): a held
|
|
5618
|
+
* conversation whose card shows a newer anchor is no longer held, and on
|
|
5619
|
+
* a complete tray (not truncated) neither is one whose card is gone (read
|
|
5620
|
+
* meanwhile: a verdict, a mark-read). A truncated tray proves nothing
|
|
5621
|
+
* about the cards it does not show.
|
|
5622
|
+
*/
|
|
5623
|
+
forgetCleared(tray, mark) {
|
|
5624
|
+
const anchors = new Map(cardsOf(tray).map((card) => [`${CONVERSATION_KEY}${card.conversation_id}`, anchorFrom(card)]));
|
|
5625
|
+
this.pacer.keepFailedOn(anchors, mark, tray.truncated !== true);
|
|
5626
|
+
}
|
|
5627
|
+
/** The start of a run or reply turn: held after a could-not-process answer at the anchor it was started for. */
|
|
5628
|
+
failureHeld(handle) {
|
|
5629
|
+
const turn = handle.turn;
|
|
5630
|
+
return turn.kind === "reply" && this.pacer.heldFailedAt(`${CONVERSATION_KEY}${turn.conversationId}`, turn.anchor);
|
|
5333
5631
|
}
|
|
5334
5632
|
/**
|
|
5335
5633
|
* The gate in chunks (contract v1 §3.6): the cards, in tray order, split
|
|
@@ -5344,62 +5642,192 @@ var ReceiveLoop = class {
|
|
|
5344
5642
|
* cards only (or none) is not called. Stops once the cycle is stale; a call
|
|
5345
5643
|
* a stop cut short is not counted as a failed chunk, and a cycle the
|
|
5346
5644
|
* watchdog abandoned reports and counts no chunk after its summary.
|
|
5645
|
+
*
|
|
5646
|
+
* The split (could not process): every owed card of a chunk of more than
|
|
5647
|
+
* one card that failed closed is then gated again alone, one call per card,
|
|
5648
|
+
* in tray order, before any turn starts, so one card the model cannot
|
|
5649
|
+
* handle fails only its own call. A card whose last gate call failed is
|
|
5650
|
+
* kept out of the chunks and gated alone after them. A single-card call that
|
|
5651
|
+
* fails is the card's failure (reported with evidence; a `retry` answer
|
|
5652
|
+
* gates it once more at the end of the queue). When the queue is empty, a
|
|
5653
|
+
* failure still waiting gets the runtime check. After
|
|
5654
|
+
* `SPLIT_STOP_AFTER_FAILURES` single-card calls in a row failed with nothing
|
|
5655
|
+
* decided in the cycle, the split goes on only if `checkRuntime()` passes;
|
|
5656
|
+
* otherwise it stops (`split_stopped`).
|
|
5347
5657
|
*/
|
|
5348
5658
|
async gateInChunks(run3, tray, cards, texts) {
|
|
5349
5659
|
const size = this.gateChunkSize;
|
|
5350
|
-
const of = Math.ceil(cards.length / size);
|
|
5351
|
-
const callModel = this.boundedModelCall(run3);
|
|
5352
5660
|
const client = this.recordingClient(run3);
|
|
5353
5661
|
const out = {
|
|
5354
5662
|
gated: 0,
|
|
5355
5663
|
held: 0,
|
|
5664
|
+
heldAfterFailure: 0,
|
|
5356
5665
|
reply: [],
|
|
5357
5666
|
verdicts: { reply: 0, dismiss: 0, defer: 0 },
|
|
5358
5667
|
undecided: 0,
|
|
5359
5668
|
recorded: 0,
|
|
5360
5669
|
recordFailures: 0,
|
|
5361
|
-
chunks: []
|
|
5670
|
+
chunks: [],
|
|
5671
|
+
split: [],
|
|
5672
|
+
splitStopped: false
|
|
5362
5673
|
};
|
|
5363
|
-
|
|
5364
|
-
|
|
5365
|
-
|
|
5366
|
-
const
|
|
5367
|
-
out.held += planned.length - chunk.length;
|
|
5368
|
-
if (chunk.length < planned.length) this.heldWhileRunning = true;
|
|
5369
|
-
if (!chunk.some((card) => card.muted !== true)) continue;
|
|
5370
|
-
const result = await run({
|
|
5371
|
-
tray: { ...tray, conversations: chunk },
|
|
5372
|
-
callModel,
|
|
5373
|
-
client,
|
|
5374
|
-
texts,
|
|
5375
|
-
signal: run3.signal,
|
|
5376
|
-
...this.options.toolNamer ? { toolNamer: this.options.toolNamer } : {}
|
|
5377
|
-
});
|
|
5378
|
-
if (run3.abandoned()) break;
|
|
5379
|
-
if (run3.stale() && result.outcome !== "decided") break;
|
|
5380
|
-
out.gated += chunk.length;
|
|
5674
|
+
const gated = /* @__PURE__ */ new Set();
|
|
5675
|
+
const decided = /* @__PURE__ */ new Set();
|
|
5676
|
+
const take = (result, sent) => {
|
|
5677
|
+
for (const card of sent) gated.add(card.conversation_id);
|
|
5381
5678
|
const undecided = new Set(result.undecided);
|
|
5382
5679
|
for (const verdict of result.verdicts) {
|
|
5383
|
-
if (
|
|
5680
|
+
if (undecided.has(verdict.conversation_id) || decided.has(verdict.conversation_id)) continue;
|
|
5681
|
+
decided.add(verdict.conversation_id);
|
|
5682
|
+
out.verdicts[verdict.decision] += 1;
|
|
5384
5683
|
}
|
|
5385
5684
|
out.reply.push(...result.reply);
|
|
5386
|
-
out.undecided += result.undecided.length;
|
|
5387
5685
|
out.recorded += result.recorded;
|
|
5388
5686
|
out.recordFailures += result.recordFailures.length;
|
|
5389
|
-
|
|
5390
|
-
|
|
5391
|
-
|
|
5392
|
-
|
|
5393
|
-
|
|
5394
|
-
|
|
5395
|
-
|
|
5396
|
-
|
|
5397
|
-
|
|
5398
|
-
|
|
5399
|
-
|
|
5400
|
-
|
|
5401
|
-
|
|
5687
|
+
};
|
|
5688
|
+
const gateOne = (card, timedOut) => run({
|
|
5689
|
+
tray: { ...tray, conversations: [card] },
|
|
5690
|
+
callModel: this.boundedModelCall(run3, timedOut),
|
|
5691
|
+
client,
|
|
5692
|
+
texts,
|
|
5693
|
+
signal: run3.signal,
|
|
5694
|
+
...this.options.toolNamer ? { toolNamer: this.options.toolNamer } : {}
|
|
5695
|
+
});
|
|
5696
|
+
const queue = [];
|
|
5697
|
+
const alone = cards.filter((card) => card.muted !== true && this.gateAlone.has(card.conversation_id));
|
|
5698
|
+
const chunked = alone.length === 0 ? cards : cards.filter((card) => !alone.includes(card));
|
|
5699
|
+
const split = {
|
|
5700
|
+
token: run3.token,
|
|
5701
|
+
members: new Map(cards.map((card) => [card.conversation_id, card])),
|
|
5702
|
+
queue,
|
|
5703
|
+
skip: /* @__PURE__ */ new Set()
|
|
5704
|
+
};
|
|
5705
|
+
this.activeSplit = split;
|
|
5706
|
+
try {
|
|
5707
|
+
const of = Math.ceil(chunked.length / size);
|
|
5708
|
+
for (let index2 = 1; index2 <= of; index2++) {
|
|
5709
|
+
if (run3.stale()) break;
|
|
5710
|
+
const slice = chunked.slice((index2 - 1) * size, index2 * size);
|
|
5711
|
+
const planned = slice.filter((card) => !this.heldAfterFailure(card));
|
|
5712
|
+
out.heldAfterFailure += slice.length - planned.length;
|
|
5713
|
+
const chunk = this.unheld(planned);
|
|
5714
|
+
out.held += planned.length - chunk.length;
|
|
5715
|
+
if (chunk.length < planned.length) this.heldWhileRunning = true;
|
|
5716
|
+
if (!chunk.some((card) => card.muted !== true)) continue;
|
|
5717
|
+
const startedAt = this.evidence.start();
|
|
5718
|
+
const timedOut = { value: false };
|
|
5719
|
+
const result = await run({
|
|
5720
|
+
tray: { ...tray, conversations: chunk },
|
|
5721
|
+
callModel: this.boundedModelCall(run3, timedOut),
|
|
5722
|
+
client,
|
|
5723
|
+
texts,
|
|
5724
|
+
signal: run3.signal,
|
|
5725
|
+
...this.options.toolNamer ? { toolNamer: this.options.toolNamer } : {}
|
|
5726
|
+
});
|
|
5727
|
+
if (run3.abandoned()) break;
|
|
5728
|
+
if (run3.stale() && result.outcome !== "decided") break;
|
|
5729
|
+
take(result, chunk);
|
|
5730
|
+
const report = {
|
|
5731
|
+
cycle: run3.token,
|
|
5732
|
+
index: index2,
|
|
5733
|
+
of,
|
|
5734
|
+
cards: chunk.length,
|
|
5735
|
+
outcome: result.outcome === "decided" ? "decided" : "fail_closed",
|
|
5736
|
+
undecided: result.undecided.length,
|
|
5737
|
+
failure: result.failure
|
|
5738
|
+
};
|
|
5739
|
+
out.chunks.push(report);
|
|
5740
|
+
if (report.outcome === "decided") this.counts.chunksOk += 1;
|
|
5741
|
+
else this.counts.chunksFailed += 1;
|
|
5742
|
+
this.emit({ type: "gate_chunk", ...report });
|
|
5743
|
+
if (report.outcome === "decided") {
|
|
5744
|
+
const through = decidedBy(result);
|
|
5745
|
+
for (const id of through) this.gateAlone.delete(id);
|
|
5746
|
+
await run3.exclude(() => this.reportDue(this.takeDue(this.completed("gate", through))));
|
|
5747
|
+
continue;
|
|
5748
|
+
}
|
|
5749
|
+
this.failedAttempt("gate", chunk.filter((card) => card.muted !== true).map((card) => card.conversation_id));
|
|
5750
|
+
if (chunk.length === 1) {
|
|
5751
|
+
this.rememberGateAlone(chunk[0].conversation_id);
|
|
5752
|
+
await run3.exclude(() => this.failed(gateFailure(chunk[0], startedAt, 0, timedOut.value, this.now())));
|
|
5753
|
+
} else {
|
|
5754
|
+
for (const card of chunk) if (card.muted !== true) queue.push({ card, reruns: 0 });
|
|
5755
|
+
}
|
|
5756
|
+
}
|
|
5757
|
+
for (const card of alone) {
|
|
5758
|
+
if (!queue.some((item) => item.card.conversation_id === card.conversation_id)) queue.push({ card, reruns: 0 });
|
|
5759
|
+
}
|
|
5760
|
+
let index = 0;
|
|
5761
|
+
let failuresInARow = 0;
|
|
5762
|
+
splitting: for (; ; ) {
|
|
5763
|
+
while (queue.length > 0) {
|
|
5764
|
+
if (run3.stale()) break splitting;
|
|
5765
|
+
const { card, reruns } = queue.shift();
|
|
5766
|
+
const id = card.conversation_id;
|
|
5767
|
+
if (decided.has(id) || split.skip.has(id) || this.heldAfterFailure(card)) continue;
|
|
5768
|
+
if (this.unheld([card]).length === 0) {
|
|
5769
|
+
if (!gated.has(id)) out.held += 1;
|
|
5770
|
+
this.heldWhileRunning = true;
|
|
5771
|
+
continue;
|
|
5772
|
+
}
|
|
5773
|
+
index += 1;
|
|
5774
|
+
const startedAt = this.evidence.start();
|
|
5775
|
+
const timedOut = { value: false };
|
|
5776
|
+
const result = await gateOne(card, timedOut);
|
|
5777
|
+
if (run3.abandoned()) break splitting;
|
|
5778
|
+
if (run3.stale() && result.outcome !== "decided") break splitting;
|
|
5779
|
+
take(result, [card]);
|
|
5780
|
+
const report = {
|
|
5781
|
+
cycle: run3.token,
|
|
5782
|
+
index,
|
|
5783
|
+
of: index + queue.length,
|
|
5784
|
+
cards: 1,
|
|
5785
|
+
outcome: result.outcome === "decided" ? "decided" : "fail_closed",
|
|
5786
|
+
undecided: result.undecided.length,
|
|
5787
|
+
failure: result.failure,
|
|
5788
|
+
split: true,
|
|
5789
|
+
conversationId: id
|
|
5790
|
+
};
|
|
5791
|
+
out.split.push(report);
|
|
5792
|
+
if (report.outcome === "decided") this.counts.splitCallsOk += 1;
|
|
5793
|
+
else this.counts.splitCallsFailed += 1;
|
|
5794
|
+
this.emit({ type: "gate_chunk", ...report });
|
|
5795
|
+
if (report.outcome === "decided") {
|
|
5796
|
+
failuresInARow = 0;
|
|
5797
|
+
this.gateAlone.delete(id);
|
|
5798
|
+
await run3.exclude(() => this.reportDue(this.takeDue(this.completed("gate", decidedBy(result)))));
|
|
5799
|
+
continue;
|
|
5800
|
+
}
|
|
5801
|
+
this.rememberGateAlone(id);
|
|
5802
|
+
this.failedAttempt("gate", [id]);
|
|
5803
|
+
await run3.exclude(() => this.failed(gateFailure(card, startedAt, reruns, timedOut.value, this.now())));
|
|
5804
|
+
failuresInARow += 1;
|
|
5805
|
+
if (failuresInARow >= SPLIT_STOP_AFTER_FAILURES && decided.size === 0) {
|
|
5806
|
+
const ok = this.options.checkRuntime ? await run3.exclude(() => this.runtimeCheck()) : false;
|
|
5807
|
+
if (run3.stale()) break splitting;
|
|
5808
|
+
if (!ok) {
|
|
5809
|
+
out.splitStopped = true;
|
|
5810
|
+
this.failureCounts.splitsStopped += 1;
|
|
5811
|
+
this.emit({
|
|
5812
|
+
type: "split_stopped",
|
|
5813
|
+
cycle: run3.token,
|
|
5814
|
+
remaining: queue.filter((item) => !decided.has(item.card.conversation_id)).length,
|
|
5815
|
+
reason: this.options.checkRuntime ? "runtime_check_failed" : "no_runtime_check"
|
|
5816
|
+
});
|
|
5817
|
+
break splitting;
|
|
5818
|
+
}
|
|
5819
|
+
failuresInARow = 0;
|
|
5820
|
+
}
|
|
5821
|
+
}
|
|
5822
|
+
if (run3.stale()) break;
|
|
5823
|
+
await run3.exclude(() => this.checkParked("gate"));
|
|
5824
|
+
if (queue.length === 0) break;
|
|
5825
|
+
}
|
|
5826
|
+
} finally {
|
|
5827
|
+
if (this.activeSplit === split) this.activeSplit = null;
|
|
5402
5828
|
}
|
|
5829
|
+
out.gated = gated.size;
|
|
5830
|
+
out.undecided = gated.size - decided.size;
|
|
5403
5831
|
return out;
|
|
5404
5832
|
}
|
|
5405
5833
|
/**
|
|
@@ -5432,12 +5860,14 @@ var ReceiveLoop = class {
|
|
|
5432
5860
|
}
|
|
5433
5861
|
}
|
|
5434
5862
|
async singleRunCycle(run3) {
|
|
5863
|
+
const mark = this.pacer.mark();
|
|
5435
5864
|
const tray = await this.readTray();
|
|
5436
5865
|
if (run3.stale()) return this.summary("abandoned");
|
|
5437
5866
|
let cards = null;
|
|
5438
5867
|
let invites = null;
|
|
5439
5868
|
let base = { cards: 0, invites: 0, heldByTurns: 0, truncated: false };
|
|
5440
5869
|
if (tray !== null) {
|
|
5870
|
+
this.forgetCleared(tray, mark);
|
|
5441
5871
|
cards = cardsOf(tray).filter((card) => card.muted !== true).length;
|
|
5442
5872
|
invites = trayInvites(tray).length;
|
|
5443
5873
|
base = { cards: cardsOf(tray).length, invites, heldByTurns: 0, truncated: tray.truncated === true };
|
|
@@ -5447,27 +5877,202 @@ var ReceiveLoop = class {
|
|
|
5447
5877
|
const texts = await this.options.texts.ensure();
|
|
5448
5878
|
if (run3.stale()) return this.summary("abandoned", base);
|
|
5449
5879
|
if (texts === null) return this.summary("texts_missing", { ...base, failure: "the served texts are not available" });
|
|
5880
|
+
if (this.options.conversationRuns !== false && conversationRunTemplate(texts) !== null) {
|
|
5881
|
+
if (tray !== null) return this.conversationRunCycle(run3, tray, texts);
|
|
5882
|
+
if (this.runWork.size > 0 || this.waiting.length > 0) this.heldWhileRunning = true;
|
|
5883
|
+
return this.summary("tray_failed", { ...base, failure: this.lastError });
|
|
5884
|
+
}
|
|
5450
5885
|
const unready = await this.checkReady();
|
|
5451
5886
|
if (run3.stale()) return this.summary("abandoned", base);
|
|
5452
5887
|
if (unready !== null) return this.summary("tools_unavailable", { ...base, failure: unready });
|
|
5453
5888
|
const prompt = cards === 0 && (invites ?? 0) > 0 ? texts.invite_prompt.trim() : (invites ?? 0) > 0 ? `${texts.wake_instruction.trim()}
|
|
5454
5889
|
|
|
5455
5890
|
${texts.invite_prompt.trim()}` : texts.wake_instruction.trim();
|
|
5891
|
+
const kind = cards === 0 && (invites ?? 0) > 0 ? "invite" : "tray";
|
|
5456
5892
|
this.counts.runsStarted += 1;
|
|
5457
|
-
|
|
5893
|
+
const end = await run3.exclude(() => this.callRun({ prompt, texts, cards, invites, signal: this.abort.signal, kind }));
|
|
5894
|
+
this.countRunEnd(end);
|
|
5895
|
+
this.emit({ type: "run", end, kind });
|
|
5896
|
+
const startedWith = tray === null || this.options.dismissIgnoredInvites === false ? [] : trayInvites(tray);
|
|
5897
|
+
if (startedWith.length > 0) await this.track(this.afterRunInvites(startedWith, end));
|
|
5898
|
+
const outcome = end.status === "completed" ? "run_completed" : end.status === "failed" ? "run_failed" : "run_not_started";
|
|
5899
|
+
return this.summary(outcome, { ...base, failure: end.status === "completed" ? null : end.detail ?? null });
|
|
5900
|
+
}
|
|
5901
|
+
/**
|
|
5902
|
+
* Conversation runs (single-run mode, texts with `conversation_run_template`):
|
|
5903
|
+
* one run per owed card (unmuted, with a preview: a notices-only card is not
|
|
5904
|
+
* owed), prompted for that one conversation, and one invite run for the
|
|
5905
|
+
* pending invites. A conversation whose run is running or
|
|
5906
|
+
* waiting for a slot is held out (the tracker), and so is one paced after a
|
|
5907
|
+
* completed run that left its card at the same anchor (D5). Runs take a slot
|
|
5908
|
+
* each (`maxConcurrentTurns`) in the order queued: the cards whose last run
|
|
5909
|
+
* failed go last, after the invite run, so they never hold up the others.
|
|
5910
|
+
* The loop records no verdict for a card: the run decides.
|
|
5911
|
+
*/
|
|
5912
|
+
async conversationRunCycle(run3, tray, texts) {
|
|
5913
|
+
const cards = cardsOf(tray);
|
|
5914
|
+
const invites = trayInvites(tray);
|
|
5915
|
+
const unheld = this.unheld(cards);
|
|
5916
|
+
const heldByTurns = cards.length - unheld.length;
|
|
5917
|
+
if (heldByTurns > 0) this.heldWhileRunning = true;
|
|
5918
|
+
const { open: runCards, paced: pacedCards, heldAfterFailure } = this.paceCards(unheld);
|
|
5919
|
+
const openInvites = invites.filter((invite) => !this.tracker.inviteRunning(invite.invite_id));
|
|
5920
|
+
const owedInvites = openInvites.filter((invite) => !this.pacer.held(`invite:${invite.invite_id}`, null));
|
|
5921
|
+
const paced = pacedCards + (openInvites.length - owedInvites.length);
|
|
5922
|
+
const owedCards = runCards.filter((card) => card.muted !== true && card.preview != null);
|
|
5923
|
+
const base = { cards: cards.length, invites: invites.length, heldByTurns, paced, heldAfterFailure, truncated: tray.truncated === true };
|
|
5924
|
+
if (owedCards.length === 0 && owedInvites.length === 0) return this.summary("nothing_owed", base);
|
|
5925
|
+
const unready = await this.checkReady();
|
|
5926
|
+
if (run3.stale()) return this.summary("abandoned", base);
|
|
5927
|
+
if (unready !== null) return this.summary("tools_unavailable", { ...base, failure: unready });
|
|
5928
|
+
const tally = { turnsStarted: 0, turnsRefused: 0, turnsQueued: 0, heldAfterFailure };
|
|
5929
|
+
const failedLast = owedCards.filter((card) => this.awaitingEvidence.has(card.conversation_id));
|
|
5930
|
+
for (const card of owedCards.filter((card2) => !failedLast.includes(card2))) {
|
|
5931
|
+
if (run3.stale()) break;
|
|
5932
|
+
tally[launchKey(await this.startConversationRun(card, texts))] += 1;
|
|
5933
|
+
}
|
|
5934
|
+
if (owedInvites.length > 0 && !run3.stale()) tally[launchKey(await this.startInviteRun(owedInvites, texts))] += 1;
|
|
5935
|
+
for (const card of failedLast) {
|
|
5936
|
+
if (run3.stale()) break;
|
|
5937
|
+
tally[launchKey(await this.startConversationRun(card, texts))] += 1;
|
|
5938
|
+
}
|
|
5939
|
+
const parts = { ...base, ...tally };
|
|
5940
|
+
if (run3.stale()) return this.summary("abandoned", parts);
|
|
5941
|
+
const launched = tally.turnsStarted + tally.turnsQueued;
|
|
5942
|
+
if (tray.truncated === true) this.readTheRest(launched > 0);
|
|
5943
|
+
return this.summary(launched > 0 ? "runs_started" : "nothing_owed", parts);
|
|
5944
|
+
}
|
|
5945
|
+
/** One conversation run: a slot (or a place in the queue), the run, then what its end says. */
|
|
5946
|
+
async startConversationRun(card, texts, reruns = 0) {
|
|
5947
|
+
const conversationId = card.conversation_id;
|
|
5948
|
+
const anchor = anchorFrom(card);
|
|
5949
|
+
const handle = this.tracker.begin({ kind: "reply", conversationId, anchor });
|
|
5950
|
+
if (handle === null) return "refused";
|
|
5951
|
+
const request = {
|
|
5952
|
+
kind: "conversation",
|
|
5953
|
+
conversationId,
|
|
5954
|
+
prompt: conversationRunPrompt(texts, conversationId),
|
|
5955
|
+
texts,
|
|
5956
|
+
cards: 1,
|
|
5957
|
+
invites: 0,
|
|
5958
|
+
signal: this.abort.signal
|
|
5959
|
+
};
|
|
5960
|
+
return this.launch(handle, async () => {
|
|
5961
|
+
const attempt = { conversationId, source: "run", card, anchor, startedAt: this.evidence.start(), reruns };
|
|
5962
|
+
this.counts.runsStarted += 1;
|
|
5963
|
+
this.trackRun(this.afterConversationRun(handle, attempt, this.callRun(request)));
|
|
5964
|
+
return true;
|
|
5965
|
+
});
|
|
5966
|
+
}
|
|
5967
|
+
async afterConversationRun(handle, attempt, done) {
|
|
5968
|
+
const end = await done;
|
|
5969
|
+
const held = this.tracker.holds(handle);
|
|
5970
|
+
let completion = null;
|
|
5971
|
+
let failure = null;
|
|
5972
|
+
let due = [];
|
|
5973
|
+
let ended = false;
|
|
5458
5974
|
try {
|
|
5459
|
-
|
|
5460
|
-
|
|
5461
|
-
|
|
5975
|
+
this.countRunEnd(end);
|
|
5976
|
+
this.emit({ type: "run", end, kind: "conversation", conversationId: attempt.conversationId });
|
|
5977
|
+
if (!held) return;
|
|
5978
|
+
if (end.status === "completed") {
|
|
5979
|
+
completion = this.completed("run", [attempt.conversationId]);
|
|
5980
|
+
this.pacer.hold(`conversation:${attempt.conversationId}`, attempt.anchor);
|
|
5981
|
+
} else if (end.status === "failed" && !this.stopped) {
|
|
5982
|
+
this.failedAttempt("run", [attempt.conversationId]);
|
|
5983
|
+
if (untouched(attempt.card, await this.readTray())) {
|
|
5984
|
+
failure = { ...attempt, outcome: end.timedOut === true ? "timed_out" : "failed", failedAt: this.now(), holding: handle };
|
|
5985
|
+
}
|
|
5986
|
+
}
|
|
5987
|
+
ended = true;
|
|
5988
|
+
} finally {
|
|
5989
|
+
if (failure === null) this.tracker.end(handle);
|
|
5990
|
+
if (ended && !this.stopped && completion !== null) due = this.takeDue(completion);
|
|
5991
|
+
this.releaseSlot(handle);
|
|
5992
|
+
if (failure === null) this.afterTurnEnded();
|
|
5993
|
+
}
|
|
5994
|
+
if (failure !== null) {
|
|
5995
|
+
try {
|
|
5996
|
+
if (!this.stopped) await this.failed(failure);
|
|
5997
|
+
} finally {
|
|
5998
|
+
this.tracker.end(handle);
|
|
5999
|
+
this.afterTurnEnded();
|
|
6000
|
+
}
|
|
5462
6001
|
}
|
|
6002
|
+
await this.reportDue(due);
|
|
6003
|
+
await this.checkParked("run");
|
|
6004
|
+
}
|
|
6005
|
+
/** The invite run: one run for the pending invites, holding each of them until it ends. */
|
|
6006
|
+
async startInviteRun(invites, texts) {
|
|
6007
|
+
const handles = [];
|
|
6008
|
+
const held = [];
|
|
6009
|
+
for (const invite of invites) {
|
|
6010
|
+
const handle = this.tracker.begin({ kind: "invite", inviteId: invite.invite_id });
|
|
6011
|
+
if (handle === null) continue;
|
|
6012
|
+
handles.push(handle);
|
|
6013
|
+
held.push(invite);
|
|
6014
|
+
}
|
|
6015
|
+
const slot = handles[0];
|
|
6016
|
+
if (slot === void 0) return "refused";
|
|
6017
|
+
const endAll = () => {
|
|
6018
|
+
for (const handle of handles) this.tracker.end(handle);
|
|
6019
|
+
};
|
|
6020
|
+
const request = {
|
|
6021
|
+
kind: "invite",
|
|
6022
|
+
prompt: texts.invite_prompt.trim(),
|
|
6023
|
+
texts,
|
|
6024
|
+
cards: 0,
|
|
6025
|
+
invites: held.length,
|
|
6026
|
+
signal: this.abort.signal
|
|
6027
|
+
};
|
|
6028
|
+
return this.launch(
|
|
6029
|
+
slot,
|
|
6030
|
+
async () => {
|
|
6031
|
+
this.counts.runsStarted += 1;
|
|
6032
|
+
this.trackRun(this.afterInviteRun(slot, held, endAll, this.callRun(request)));
|
|
6033
|
+
return true;
|
|
6034
|
+
},
|
|
6035
|
+
endAll
|
|
6036
|
+
);
|
|
6037
|
+
}
|
|
6038
|
+
async afterInviteRun(slot, invites, endAll, done) {
|
|
6039
|
+
const end = await done;
|
|
6040
|
+
const held = this.tracker.holds(slot);
|
|
6041
|
+
let completion = null;
|
|
6042
|
+
let due = [];
|
|
6043
|
+
let ended = false;
|
|
6044
|
+
try {
|
|
6045
|
+
this.countRunEnd(end);
|
|
6046
|
+
this.emit({ type: "run", end, kind: "invite" });
|
|
6047
|
+
if (!held) return;
|
|
6048
|
+
if (end.status === "completed") {
|
|
6049
|
+
completion = this.completed("invite", []);
|
|
6050
|
+
for (const invite of invites) this.pacer.hold(`invite:${invite.invite_id}`, null);
|
|
6051
|
+
}
|
|
6052
|
+
if (this.options.dismissIgnoredInvites !== false) await this.afterRunInvites(invites, end);
|
|
6053
|
+
ended = true;
|
|
6054
|
+
} finally {
|
|
6055
|
+
endAll();
|
|
6056
|
+
if (ended && !this.stopped && completion !== null) due = this.takeDue(completion);
|
|
6057
|
+
this.releaseSlot(slot);
|
|
6058
|
+
this.afterTurnEnded();
|
|
6059
|
+
}
|
|
6060
|
+
await this.reportDue(due);
|
|
6061
|
+
await this.checkParked("run");
|
|
6062
|
+
}
|
|
6063
|
+
/** The runtime's `startRun`, never rejecting: a throw is a run that did not start. */
|
|
6064
|
+
callRun(request) {
|
|
6065
|
+
return Promise.resolve().then(() => this.options.startRun(request)).catch((err) => ({ status: "not_started", detail: err instanceof Error ? err.message : String(err) }));
|
|
6066
|
+
}
|
|
6067
|
+
countRunEnd(end) {
|
|
5463
6068
|
if (end.status === "completed") this.counts.runsCompleted += 1;
|
|
5464
6069
|
else if (end.status === "failed") this.counts.runsFailed += 1;
|
|
5465
6070
|
else this.counts.runsNotStarted += 1;
|
|
5466
|
-
|
|
5467
|
-
|
|
5468
|
-
|
|
5469
|
-
const
|
|
5470
|
-
|
|
6071
|
+
}
|
|
6072
|
+
/** A run's whole course after its start: `idle()` and `stop()` wait for it. */
|
|
6073
|
+
trackRun(work) {
|
|
6074
|
+
const tracked = work.catch((err) => this.fail(err)).finally(() => this.runWork.delete(tracked));
|
|
6075
|
+
this.runWork.add(tracked);
|
|
5471
6076
|
}
|
|
5472
6077
|
async afterRunInvites(invites, end) {
|
|
5473
6078
|
const completed = end.status === "completed";
|
|
@@ -5478,7 +6083,7 @@ ${texts.invite_prompt.trim()}` : texts.wake_instruction.trim();
|
|
|
5478
6083
|
}
|
|
5479
6084
|
}
|
|
5480
6085
|
// ─── turns (gate-call mode) ─────────────────────────────────────────────
|
|
5481
|
-
async startReplyTurn(reply, texts) {
|
|
6086
|
+
async startReplyTurn(reply, texts, reruns = 0) {
|
|
5482
6087
|
const anchor = anchorFrom(reply.card);
|
|
5483
6088
|
const handle = this.tracker.begin({ kind: "reply", conversationId: reply.conversationId, anchor });
|
|
5484
6089
|
if (handle === null) return "refused";
|
|
@@ -5493,6 +6098,7 @@ ${texts.invite_prompt.trim()}` : texts.wake_instruction.trim();
|
|
|
5493
6098
|
signal: this.abort.signal
|
|
5494
6099
|
};
|
|
5495
6100
|
return this.launch(handle, async () => {
|
|
6101
|
+
const startedAt = this.evidence.start();
|
|
5496
6102
|
const admission = await this.admit(this.options.startTurn, request);
|
|
5497
6103
|
if (!admission.started) {
|
|
5498
6104
|
this.tracker.end(handle);
|
|
@@ -5502,7 +6108,8 @@ ${texts.invite_prompt.trim()}` : texts.wake_instruction.trim();
|
|
|
5502
6108
|
}
|
|
5503
6109
|
this.counts.turnsStarted += 1;
|
|
5504
6110
|
this.emit({ type: "turn_started", request: withoutRuntime(request) });
|
|
5505
|
-
|
|
6111
|
+
const attempt = { conversationId: reply.conversationId, source: "turn", card: reply.card, anchor, startedAt, reruns, reply };
|
|
6112
|
+
void this.afterReplyTurn(handle, attempt, admission.done);
|
|
5506
6113
|
return true;
|
|
5507
6114
|
});
|
|
5508
6115
|
}
|
|
@@ -5542,9 +6149,15 @@ invite_id: ${invite.invite_id}`,
|
|
|
5542
6149
|
* queues behind them instead of taking the slot. A refused admission here
|
|
5543
6150
|
* hands its slot on the same way.
|
|
5544
6151
|
*/
|
|
5545
|
-
async launch(handle, admit) {
|
|
5546
|
-
if (this.
|
|
5547
|
-
this.
|
|
6152
|
+
async launch(handle, admit, drop) {
|
|
6153
|
+
if (this.failureHeld(handle)) {
|
|
6154
|
+
this.tracker.end(handle);
|
|
6155
|
+
drop?.();
|
|
6156
|
+
this.failureCounts.startsHeld += 1;
|
|
6157
|
+
return "held";
|
|
6158
|
+
}
|
|
6159
|
+
if (this.waiting.length > 0 || this.activeTurns >= this.maxTurns || this.awaitsAnswer(handle)) {
|
|
6160
|
+
this.waiting.push({ handle, admit, ...drop ? { drop } : {} });
|
|
5548
6161
|
this.counts.turnsQueued += 1;
|
|
5549
6162
|
if (this.activeTurns < this.maxTurns) void this.pump();
|
|
5550
6163
|
return "queued";
|
|
@@ -5564,30 +6177,63 @@ invite_id: ${invite.invite_id}`,
|
|
|
5564
6177
|
releaseSlot(handle, options = {}) {
|
|
5565
6178
|
if (this.slotted.delete(handle)) this.activeTurns = Math.max(0, this.activeTurns - 1);
|
|
5566
6179
|
const queued = this.waiting.findIndex((w) => w.handle === handle);
|
|
5567
|
-
if (queued >= 0) this.waiting.splice(queued, 1);
|
|
6180
|
+
if (queued >= 0) this.waiting.splice(queued, 1)[0].drop?.();
|
|
5568
6181
|
if (options.pump !== false) void this.pump();
|
|
5569
6182
|
}
|
|
5570
6183
|
async pump() {
|
|
5571
|
-
while (this.
|
|
5572
|
-
const
|
|
6184
|
+
while (this.activeTurns < this.maxTurns && !this.stopped) {
|
|
6185
|
+
const index = this.waiting.findIndex((w) => !this.awaitsAnswer(w.handle));
|
|
6186
|
+
if (index < 0) break;
|
|
6187
|
+
const next = this.waiting.splice(index, 1)[0];
|
|
5573
6188
|
if (!this.tracker.holds(next.handle)) continue;
|
|
6189
|
+
if (this.failureHeld(next.handle)) {
|
|
6190
|
+
this.tracker.end(next.handle);
|
|
6191
|
+
next.drop?.();
|
|
6192
|
+
this.failureCounts.startsHeld += 1;
|
|
6193
|
+
continue;
|
|
6194
|
+
}
|
|
5574
6195
|
await this.admitInSlot(next.handle, next.admit);
|
|
5575
6196
|
}
|
|
5576
6197
|
}
|
|
5577
|
-
|
|
6198
|
+
awaitsAnswer(handle) {
|
|
6199
|
+
return handle.turn.kind === "reply" && this.reporting.has(handle.turn.conversationId);
|
|
6200
|
+
}
|
|
6201
|
+
async afterReplyTurn(handle, attempt, done) {
|
|
6202
|
+
const reply = attempt.reply;
|
|
6203
|
+
let completion = null;
|
|
6204
|
+
let failure = null;
|
|
6205
|
+
let due = [];
|
|
6206
|
+
let ended = false;
|
|
5578
6207
|
try {
|
|
5579
6208
|
const end = await this.tracker.confirmedEnd(handle, done);
|
|
5580
6209
|
if (end === null || !this.tracker.holds(handle)) return;
|
|
5581
|
-
|
|
6210
|
+
if (end.status === "completed") completion = this.completed("turn", [reply.conversationId]);
|
|
6211
|
+
const recorded = await this.track(this.recordReplyOutcome(handle, reply, end));
|
|
6212
|
+
if (end.status === "failed" && !this.stopped) this.failedAttempt("turn", [reply.conversationId]);
|
|
6213
|
+
if (end.status === "failed" && recorded?.freshRead === true && recorded.outcome.reason === "turn_failed") {
|
|
6214
|
+
failure = { ...attempt, outcome: end.timedOut === true ? "timed_out" : "failed", failedAt: this.now(), holding: handle };
|
|
6215
|
+
}
|
|
6216
|
+
ended = true;
|
|
5582
6217
|
} finally {
|
|
5583
|
-
this.tracker.end(handle);
|
|
6218
|
+
if (failure === null) this.tracker.end(handle);
|
|
6219
|
+
if (ended && !this.stopped && completion !== null) due = this.takeDue(completion);
|
|
5584
6220
|
this.releaseSlot(handle);
|
|
5585
|
-
this.afterTurnEnded();
|
|
6221
|
+
if (failure === null) this.afterTurnEnded();
|
|
6222
|
+
}
|
|
6223
|
+
if (failure !== null) {
|
|
6224
|
+
try {
|
|
6225
|
+
if (!this.stopped) await this.track(this.failed(failure));
|
|
6226
|
+
} finally {
|
|
6227
|
+
this.tracker.end(handle);
|
|
6228
|
+
this.afterTurnEnded();
|
|
6229
|
+
}
|
|
5586
6230
|
}
|
|
6231
|
+
await this.track(this.reportDue(due));
|
|
6232
|
+
await this.track(this.checkParked("turn"));
|
|
5587
6233
|
}
|
|
5588
6234
|
async recordReplyOutcome(handle, reply, end) {
|
|
5589
6235
|
const fresh = await this.readTray();
|
|
5590
|
-
if (!this.tracker.holds(handle)) return;
|
|
6236
|
+
if (!this.tracker.holds(handle)) return null;
|
|
5591
6237
|
const anchor = anchorFrom(reply.card);
|
|
5592
6238
|
const key = `${reply.conversationId}|${anchor ?? ""}`;
|
|
5593
6239
|
const silentBefore = this.silent.get(key) ?? 0;
|
|
@@ -5617,6 +6263,7 @@ invite_id: ${invite.invite_id}`,
|
|
|
5617
6263
|
}
|
|
5618
6264
|
}
|
|
5619
6265
|
this.emit({ type: "turn_outcome", outcome, recorded });
|
|
6266
|
+
return { outcome, freshRead: fresh !== null };
|
|
5620
6267
|
}
|
|
5621
6268
|
async afterInviteTurn(handle, invite, done) {
|
|
5622
6269
|
try {
|
|
@@ -5686,6 +6333,332 @@ invite_id: ${invite.invite_id}`,
|
|
|
5686
6333
|
this.wakeFrom("more");
|
|
5687
6334
|
}
|
|
5688
6335
|
}
|
|
6336
|
+
// ─── could not process: failures, evidence, the server's answer ─────────
|
|
6337
|
+
/**
|
|
6338
|
+
* A run, turn or gate call that completed, on the evidence clock. A failure
|
|
6339
|
+
* it clears (the same kind of attempt for the same conversation went
|
|
6340
|
+
* through since) waits no more. Call it the moment the work ended, before
|
|
6341
|
+
* its slot frees, so the next attempt starts after it.
|
|
6342
|
+
*/
|
|
6343
|
+
completed(kind, conversations) {
|
|
6344
|
+
const completion = this.evidence.complete(kind, conversations);
|
|
6345
|
+
if (kind !== "invite") this.failingSince[kind].clear();
|
|
6346
|
+
for (const id of conversations) {
|
|
6347
|
+
const failure = this.awaitingEvidence.get(id);
|
|
6348
|
+
if (failure !== void 0 && clears(completion, failure)) this.awaitingEvidence.delete(id);
|
|
6349
|
+
}
|
|
6350
|
+
return completion;
|
|
6351
|
+
}
|
|
6352
|
+
/**
|
|
6353
|
+
* The failures this completion is the evidence for (each once), taken off
|
|
6354
|
+
* the waiting list. Their conversations are held from now until the
|
|
6355
|
+
* server has answered (`reporting`): no gate, and a run or turn of theirs
|
|
6356
|
+
* waiting for a slot waits for the answer, so it never starts on the input
|
|
6357
|
+
* the answer is about. Take them before the completed work's slot frees.
|
|
6358
|
+
* One that waited past `AWAITING_EVIDENCE_MS` is dropped instead: a
|
|
6359
|
+
* completion that late says little about the runtime when it failed.
|
|
6360
|
+
*/
|
|
6361
|
+
takeDue(completion) {
|
|
6362
|
+
const now2 = this.now();
|
|
6363
|
+
const due = [];
|
|
6364
|
+
for (const failure of [...this.awaitingEvidence.values()]) {
|
|
6365
|
+
if (now2 - failure.failedAt > AWAITING_EVIDENCE_MS) {
|
|
6366
|
+
this.awaitingEvidence.delete(failure.conversationId);
|
|
6367
|
+
continue;
|
|
6368
|
+
}
|
|
6369
|
+
if (evidences(completion, failure)) {
|
|
6370
|
+
this.awaitingEvidence.delete(failure.conversationId);
|
|
6371
|
+
this.holdForReport(failure);
|
|
6372
|
+
due.push(failure);
|
|
6373
|
+
}
|
|
6374
|
+
}
|
|
6375
|
+
return due;
|
|
6376
|
+
}
|
|
6377
|
+
/** Report what `takeDue` took, letting go of each conversation once its answer is in: what waits for it goes on (or was dropped). */
|
|
6378
|
+
async reportDue(due) {
|
|
6379
|
+
const held = [...due];
|
|
6380
|
+
try {
|
|
6381
|
+
while (held.length > 0) {
|
|
6382
|
+
const failure = held[0];
|
|
6383
|
+
await this.reportFailure(failure, "other_run_completed");
|
|
6384
|
+
held.shift();
|
|
6385
|
+
this.letGo([failure]);
|
|
6386
|
+
}
|
|
6387
|
+
} finally {
|
|
6388
|
+
this.letGo(held);
|
|
6389
|
+
}
|
|
6390
|
+
}
|
|
6391
|
+
/** A report for this failure is due: its conversation is held until `letGo` (counted: two reports out hold it until both are answered). */
|
|
6392
|
+
holdForReport(failure) {
|
|
6393
|
+
this.reporting.set(failure.conversationId, (this.reporting.get(failure.conversationId) ?? 0) + 1);
|
|
6394
|
+
}
|
|
6395
|
+
/**
|
|
6396
|
+
* The answers are in (or the report is off): those conversations are no
|
|
6397
|
+
* longer held for it, what waits for them goes on, and a card a cycle left
|
|
6398
|
+
* out meanwhile for a report is read again (only then: a wake for
|
|
6399
|
+
* anything else could spin while a turn runs).
|
|
6400
|
+
*/
|
|
6401
|
+
letGo(failures) {
|
|
6402
|
+
if (failures.length === 0) return;
|
|
6403
|
+
for (const { conversationId } of failures) {
|
|
6404
|
+
const left = (this.reporting.get(conversationId) ?? 0) - 1;
|
|
6405
|
+
if (left > 0) this.reporting.set(conversationId, left);
|
|
6406
|
+
else this.reporting.delete(conversationId);
|
|
6407
|
+
}
|
|
6408
|
+
if (this.stopped) return;
|
|
6409
|
+
if (this.waiting.length > 0) void this.pump();
|
|
6410
|
+
if (this.heldWhileReporting) {
|
|
6411
|
+
this.heldWhileReporting = false;
|
|
6412
|
+
this.wakeFrom("more");
|
|
6413
|
+
}
|
|
6414
|
+
}
|
|
6415
|
+
/** An attempt of this kind failed for these conversations (whatever comes of it): see `failingSince`. */
|
|
6416
|
+
failedAttempt(kind, conversations) {
|
|
6417
|
+
for (const id of conversations) this.failingSince[kind].add(id);
|
|
6418
|
+
}
|
|
6419
|
+
/**
|
|
6420
|
+
* A conversation's run, reply turn or single-card gate call failed. It is
|
|
6421
|
+
* reported only with evidence that the runtime otherwise works:
|
|
6422
|
+
* (a) a later completion of the same kind of work for another conversation
|
|
6423
|
+
* (`evidences`), or (b) `checkRuntime()` passing while no other
|
|
6424
|
+
* conversation of that kind has failed since that kind last completed.
|
|
6425
|
+
* Without (a) it waits: the work already queued runs first (cheap
|
|
6426
|
+
* evidence), and the runtime check is asked only once nothing of that kind
|
|
6427
|
+
* is queued (`checkParked`). It waits at most `AWAITING_EVIDENCE_MS`.
|
|
6428
|
+
* Nothing is reported after `stop()`: a run the stop cut short did not
|
|
6429
|
+
* fail on its conversation.
|
|
6430
|
+
*/
|
|
6431
|
+
async failed(failure) {
|
|
6432
|
+
if (this.stopped) return;
|
|
6433
|
+
this.awaitingEvidence.delete(failure.conversationId);
|
|
6434
|
+
if (this.evidence.since(failure)) {
|
|
6435
|
+
await this.reportFailure(failure, "other_run_completed");
|
|
6436
|
+
return;
|
|
6437
|
+
}
|
|
6438
|
+
if (!this.canReport(failure)) return;
|
|
6439
|
+
this.awaitingEvidence.set(failure.conversationId, failure);
|
|
6440
|
+
while (this.awaitingEvidence.size > SILENT_MEMORY) {
|
|
6441
|
+
const oldest = this.awaitingEvidence.keys().next().value;
|
|
6442
|
+
if (oldest === void 0) break;
|
|
6443
|
+
this.awaitingEvidence.delete(oldest);
|
|
6444
|
+
}
|
|
6445
|
+
if (failure.source !== "gate") await this.checkParked(failure.source);
|
|
6446
|
+
}
|
|
6447
|
+
/**
|
|
6448
|
+
* Evidence (b), once nothing of this kind is queued: ask `checkRuntime()`
|
|
6449
|
+
* about the failure still waiting for evidence, if it is the only
|
|
6450
|
+
* conversation of that kind failing since that kind last completed. Several
|
|
6451
|
+
* failing together is the runtime: none of them is reported on (b), and
|
|
6452
|
+
* they keep waiting for (a). A `timed_out` failure is never reported on
|
|
6453
|
+
* (b): a timeout is ambiguous (a brownout, a hung hook) and the check does
|
|
6454
|
+
* not exercise those paths; it waits for (a). Each failure is asked about
|
|
6455
|
+
* once. A check that fails drops it (a broken runtime reports nothing).
|
|
6456
|
+
*/
|
|
6457
|
+
async checkParked(kind) {
|
|
6458
|
+
if (!this.options.checkRuntime || this.stopped || !this.idleFor(kind)) return;
|
|
6459
|
+
const failing = this.failingSince[kind];
|
|
6460
|
+
const due = [...this.awaitingEvidence.values()].filter(
|
|
6461
|
+
(failure) => failure.source === kind && // A timeout is ambiguous (a brownout, a hung hook) and the runtime
|
|
6462
|
+
// check does not exercise those paths: only (a) can report it.
|
|
6463
|
+
failure.outcome !== "timed_out" && !this.checkedFailures.has(failure) && this.now() - failure.failedAt <= AWAITING_EVIDENCE_MS && [...failing].every((id) => id === failure.conversationId)
|
|
6464
|
+
);
|
|
6465
|
+
if (due.length === 0) return;
|
|
6466
|
+
for (const failure of due) {
|
|
6467
|
+
this.checkedFailures.add(failure);
|
|
6468
|
+
this.holdForReport(failure);
|
|
6469
|
+
}
|
|
6470
|
+
try {
|
|
6471
|
+
const ok = await this.runtimeCheck();
|
|
6472
|
+
for (const failure of due) {
|
|
6473
|
+
if (this.stopped || this.awaitingEvidence.get(failure.conversationId) !== failure) continue;
|
|
6474
|
+
if (!ok) {
|
|
6475
|
+
this.awaitingEvidence.delete(failure.conversationId);
|
|
6476
|
+
continue;
|
|
6477
|
+
}
|
|
6478
|
+
if (![...failing].every((id) => id === failure.conversationId)) continue;
|
|
6479
|
+
this.awaitingEvidence.delete(failure.conversationId);
|
|
6480
|
+
await this.reportFailure(failure, "runtime_check");
|
|
6481
|
+
}
|
|
6482
|
+
} finally {
|
|
6483
|
+
this.letGo(due);
|
|
6484
|
+
}
|
|
6485
|
+
}
|
|
6486
|
+
/** Nothing of this kind is running or queued: the runtime check may be asked now. */
|
|
6487
|
+
idleFor(kind) {
|
|
6488
|
+
if (kind === "gate") return this.activeSplit === null || this.activeSplit.queue.length === 0;
|
|
6489
|
+
if (kind === "turn" && this.activeSplit !== null) return false;
|
|
6490
|
+
return this.activeTurns === 0 && this.waiting.length === 0;
|
|
6491
|
+
}
|
|
6492
|
+
canReport(failure) {
|
|
6493
|
+
return failure.anchor !== null && typeof this.client.conversations?.reportProcessingFailure === "function";
|
|
6494
|
+
}
|
|
6495
|
+
/** `checkRuntime()`, bounded by `RUNTIME_CHECK_TIMEOUT_MS`; a throw or a timeout is not ok. */
|
|
6496
|
+
async runtimeCheck() {
|
|
6497
|
+
const check = this.options.checkRuntime;
|
|
6498
|
+
let timer = null;
|
|
6499
|
+
let answer;
|
|
6500
|
+
try {
|
|
6501
|
+
answer = await Promise.race([
|
|
6502
|
+
Promise.resolve().then(() => check()),
|
|
6503
|
+
new Promise((resolve) => {
|
|
6504
|
+
timer = this._setTimeout(
|
|
6505
|
+
() => resolve({ ok: false, detail: `no answer within ${RUNTIME_CHECK_TIMEOUT_MS} ms` }),
|
|
6506
|
+
RUNTIME_CHECK_TIMEOUT_MS
|
|
6507
|
+
);
|
|
6508
|
+
timer.unref?.();
|
|
6509
|
+
})
|
|
6510
|
+
]);
|
|
6511
|
+
} catch (err) {
|
|
6512
|
+
answer = { ok: false, detail: `the check threw: ${err instanceof Error ? err.message : String(err)}` };
|
|
6513
|
+
} finally {
|
|
6514
|
+
if (timer !== null) this._clearTimeout(timer);
|
|
6515
|
+
}
|
|
6516
|
+
const ok = answer?.ok === true;
|
|
6517
|
+
if (ok) this.failureCounts.runtimeChecksOk += 1;
|
|
6518
|
+
else this.failureCounts.runtimeChecksFailed += 1;
|
|
6519
|
+
this.lastRuntimeCheck = { at: new Date(this.now()).toISOString(), ok, detail: typeof answer?.detail === "string" ? answer.detail : null };
|
|
6520
|
+
return ok;
|
|
6521
|
+
}
|
|
6522
|
+
/** Tell the server, and do what it answers: run the conversation again (`retry`, `hidden`) or stop (`paused`, `none`). */
|
|
6523
|
+
async reportFailure(failure, evidence) {
|
|
6524
|
+
const conversations = this.client.conversations;
|
|
6525
|
+
if (!this.canReport(failure) || conversations === void 0 || this.stopped) return;
|
|
6526
|
+
const base = {
|
|
6527
|
+
conversationId: failure.conversationId,
|
|
6528
|
+
source: failure.source,
|
|
6529
|
+
throughMessageId: failure.anchor,
|
|
6530
|
+
outcome: failure.outcome,
|
|
6531
|
+
evidence
|
|
6532
|
+
};
|
|
6533
|
+
let result;
|
|
6534
|
+
try {
|
|
6535
|
+
result = await conversations.reportProcessingFailure(failure.conversationId, {
|
|
6536
|
+
throughMessageId: failure.anchor,
|
|
6537
|
+
outcome: failure.outcome,
|
|
6538
|
+
evidence
|
|
6539
|
+
});
|
|
6540
|
+
} catch (err) {
|
|
6541
|
+
this.failureCounts.reportErrors += 1;
|
|
6542
|
+
this.fail(err);
|
|
6543
|
+
this.emit({ type: "processing_failure", ...base, result: null, error: err instanceof Error ? err.message : String(err), next: "stop" });
|
|
6544
|
+
return;
|
|
6545
|
+
}
|
|
6546
|
+
this.failureCounts.reported += 1;
|
|
6547
|
+
this.failureCounts[result.action] += 1;
|
|
6548
|
+
let next = "stop";
|
|
6549
|
+
if (result.action === "retry" || result.action === "hidden") {
|
|
6550
|
+
if (failure.reruns >= MAX_RERUNS_PER_CYCLE) {
|
|
6551
|
+
next = "rerun_capped";
|
|
6552
|
+
this.failureCounts.rerunsCapped += 1;
|
|
6553
|
+
} else {
|
|
6554
|
+
next = this.rerun(failure, result.action === "hidden");
|
|
6555
|
+
}
|
|
6556
|
+
}
|
|
6557
|
+
if (result.action === "none" || result.action === "paused") this.holdFailed(failure, result.action);
|
|
6558
|
+
if (next === "stop" || next === "rerun_capped") this.activeSplit?.skip.add(failure.conversationId);
|
|
6559
|
+
this.emit({ type: "processing_failure", ...base, result, next });
|
|
6560
|
+
}
|
|
6561
|
+
/**
|
|
6562
|
+
* The server answered `none` or `paused`: the conversation is held at the
|
|
6563
|
+
* anchor the report read through, so no later cycle (a message elsewhere,
|
|
6564
|
+
* a wake frame, a reconnect, an `unanswered` re-wake, a turn's end) runs it
|
|
6565
|
+
* again on the same input. A newer message moves its anchor and lifts the
|
|
6566
|
+
* hold (`paceCards`); a card read meanwhile leaves the tray, and the hold
|
|
6567
|
+
* with it (`forgetCleared`). What waits for it does not start: a run or
|
|
6568
|
+
* reply turn queued at that anchor (after `paused`, the server's mute, any),
|
|
6569
|
+
* and a reply turn to start again after the gate.
|
|
6570
|
+
*/
|
|
6571
|
+
holdFailed(failure, reason) {
|
|
6572
|
+
const id = failure.conversationId;
|
|
6573
|
+
const anchor = failure.anchor;
|
|
6574
|
+
this.pacer.holdFailed(`${CONVERSATION_KEY}${id}`, anchor, reason);
|
|
6575
|
+
this.dropWaiting(id, reason === "paused" ? null : anchor);
|
|
6576
|
+
this.turnRerunsAfterGate = this.turnRerunsAfterGate.filter(
|
|
6577
|
+
({ reply }) => reply.conversationId !== id || reason !== "paused" && anchorFrom(reply.card) !== anchor
|
|
6578
|
+
);
|
|
6579
|
+
}
|
|
6580
|
+
/**
|
|
6581
|
+
* Run the conversation again after a `retry` or `hidden` answer, after the
|
|
6582
|
+
* others already queued, and say what was done. Its run or reply turn takes
|
|
6583
|
+
* a place at the back of the queue (it reads the conversation itself, so it
|
|
6584
|
+
* sees a hidden span as the note); a reply turn asked for while a gate is
|
|
6585
|
+
* deciding starts after that cycle's own turns. A gate call sees only the
|
|
6586
|
+
* card: after `retry` it joins the end of the running gate's single-card
|
|
6587
|
+
* calls (once, with the card as this cycle read it); after `hidden`, or
|
|
6588
|
+
* when no gate is running for it, the next cycle reads the tray again and
|
|
6589
|
+
* gates it alone. `running`: nothing started, because its next attempt is
|
|
6590
|
+
* under way already (running, queued, or being decided by the gate now).
|
|
6591
|
+
*/
|
|
6592
|
+
rerun(failure, hidden) {
|
|
6593
|
+
if (this.stopped) return "stop";
|
|
6594
|
+
const id = failure.conversationId;
|
|
6595
|
+
const reruns = failure.reruns + 1;
|
|
6596
|
+
this.pacer.release(`conversation:${id}`);
|
|
6597
|
+
if (failure.holding !== void 0) this.tracker.end(failure.holding);
|
|
6598
|
+
const split = this.activeSplit;
|
|
6599
|
+
const current = split?.members.get(id);
|
|
6600
|
+
if (failure.source === "gate") {
|
|
6601
|
+
this.rememberGateAlone(id);
|
|
6602
|
+
if (split !== null && current !== void 0 && !hidden) {
|
|
6603
|
+
split.skip.delete(id);
|
|
6604
|
+
if (split.queue.some((item) => item.card.conversation_id === id)) return "running";
|
|
6605
|
+
split.queue.push({ card: current, reruns });
|
|
6606
|
+
} else {
|
|
6607
|
+
split?.skip.add(id);
|
|
6608
|
+
this.wakeFrom("more");
|
|
6609
|
+
}
|
|
6610
|
+
this.failureCounts.reruns += 1;
|
|
6611
|
+
return "rerun";
|
|
6612
|
+
}
|
|
6613
|
+
if (this.tracker.conversationRunning(id) || current !== void 0) return "running";
|
|
6614
|
+
const texts = this.options.texts.current();
|
|
6615
|
+
this.failureCounts.reruns += 1;
|
|
6616
|
+
if (texts !== null && failure.source === "turn" && failure.reply !== void 0) {
|
|
6617
|
+
if (split !== null) this.turnRerunsAfterGate.push({ reply: failure.reply, reruns });
|
|
6618
|
+
else void this.startReplyTurn(failure.reply, texts, reruns);
|
|
6619
|
+
} else if (texts !== null && failure.source === "run" && conversationRunTemplate(texts) !== null) {
|
|
6620
|
+
void this.startConversationRun(failure.card, texts, reruns);
|
|
6621
|
+
} else {
|
|
6622
|
+
this.wakeFrom("more");
|
|
6623
|
+
}
|
|
6624
|
+
return "rerun";
|
|
6625
|
+
}
|
|
6626
|
+
/** The reply turns asked for while a gate was deciding: started now, after the cycle's own turns. */
|
|
6627
|
+
flushTurnReruns() {
|
|
6628
|
+
const due = this.turnRerunsAfterGate.splice(0);
|
|
6629
|
+
if (this.stopped || due.length === 0) return;
|
|
6630
|
+
const texts = this.options.texts.current();
|
|
6631
|
+
for (const { reply, reruns } of due) {
|
|
6632
|
+
if (this.tracker.conversationRunning(reply.conversationId)) continue;
|
|
6633
|
+
if (texts === null) this.wakeFrom("more");
|
|
6634
|
+
else void this.startReplyTurn(reply, texts, reruns);
|
|
6635
|
+
}
|
|
6636
|
+
}
|
|
6637
|
+
/**
|
|
6638
|
+
* A held conversation: a run or turn still waiting for a slot for it does
|
|
6639
|
+
* not start; with an `anchor`, only one for that anchor (one queued for a
|
|
6640
|
+
* newer message is new input, and goes on).
|
|
6641
|
+
*/
|
|
6642
|
+
dropWaiting(conversationId, anchor) {
|
|
6643
|
+
for (const entry of [...this.waiting]) {
|
|
6644
|
+
const turn = entry.handle.turn;
|
|
6645
|
+
if (turn.kind !== "reply" || turn.conversationId !== conversationId) continue;
|
|
6646
|
+
if (anchor !== null && turn.anchor !== anchor) continue;
|
|
6647
|
+
this.waiting.splice(this.waiting.indexOf(entry), 1);
|
|
6648
|
+
this.tracker.end(entry.handle);
|
|
6649
|
+
entry.drop?.();
|
|
6650
|
+
this.failureCounts.startsHeld += 1;
|
|
6651
|
+
}
|
|
6652
|
+
}
|
|
6653
|
+
rememberGateAlone(conversationId) {
|
|
6654
|
+
this.gateAlone.delete(conversationId);
|
|
6655
|
+
this.gateAlone.add(conversationId);
|
|
6656
|
+
while (this.gateAlone.size > SILENT_MEMORY) {
|
|
6657
|
+
const oldest = this.gateAlone.values().next().value;
|
|
6658
|
+
if (oldest === void 0) break;
|
|
6659
|
+
this.gateAlone.delete(oldest);
|
|
6660
|
+
}
|
|
6661
|
+
}
|
|
5689
6662
|
// ─── small things ───────────────────────────────────────────────────────
|
|
5690
6663
|
/** Recording under way after a turn's confirmed end: `stop()` waits for it. */
|
|
5691
6664
|
track(work) {
|
|
@@ -5723,8 +6696,12 @@ invite_id: ${invite.invite_id}`,
|
|
|
5723
6696
|
turnsQueued: 0,
|
|
5724
6697
|
heldByTurns: 0,
|
|
5725
6698
|
paced: 0,
|
|
6699
|
+
heldAfterFailure: 0,
|
|
5726
6700
|
chunksOk: 0,
|
|
5727
6701
|
chunksFailed: 0,
|
|
6702
|
+
splitCallsOk: 0,
|
|
6703
|
+
splitCallsFailed: 0,
|
|
6704
|
+
splitStopped: false,
|
|
5728
6705
|
truncated: false,
|
|
5729
6706
|
failure: null,
|
|
5730
6707
|
...parts
|
|
@@ -5755,6 +6732,23 @@ conversation_id: ${conversationId}`;
|
|
|
5755
6732
|
|
|
5756
6733
|
conversation_id: ${conversationId}`;
|
|
5757
6734
|
}
|
|
6735
|
+
function conversationRunTemplate(texts) {
|
|
6736
|
+
const raw = texts.conversation_run_template;
|
|
6737
|
+
const template = typeof raw === "string" ? raw.trim() : "";
|
|
6738
|
+
return template.length > 0 ? template : null;
|
|
6739
|
+
}
|
|
6740
|
+
function conversationRunPrompt(texts, conversationId) {
|
|
6741
|
+
const template = conversationRunTemplate(texts);
|
|
6742
|
+
if (template === null) return replyPrompt(texts, conversationId);
|
|
6743
|
+
if (template.includes(REPLY_TURN_PLACEHOLDER)) return template.split(REPLY_TURN_PLACEHOLDER).join(conversationId);
|
|
6744
|
+
return `${template}
|
|
6745
|
+
|
|
6746
|
+
conversation_id: ${conversationId}`;
|
|
6747
|
+
}
|
|
6748
|
+
function untouched(card, fresh) {
|
|
6749
|
+
if (fresh === null) return false;
|
|
6750
|
+
return turnOutcome({ turn: { status: "failed" }, gateCard: card, freshTray: fresh }).reason === "turn_failed";
|
|
6751
|
+
}
|
|
5758
6752
|
function chunkFailure(chunks) {
|
|
5759
6753
|
const failed = chunks.filter((chunk) => chunk.outcome === "fail_closed");
|
|
5760
6754
|
const first = failed[0];
|
|
@@ -5763,6 +6757,34 @@ function chunkFailure(chunks) {
|
|
|
5763
6757
|
if (chunks.length === 1) return reason;
|
|
5764
6758
|
return `${failed.length} of ${chunks.length} gate chunks failed; chunk ${first.index}: ${reason}`;
|
|
5765
6759
|
}
|
|
6760
|
+
function gateFailureText(chunks, split) {
|
|
6761
|
+
const parts = [];
|
|
6762
|
+
const chunked = chunkFailure(chunks);
|
|
6763
|
+
if (chunked !== null) parts.push(chunked);
|
|
6764
|
+
const failed = split.filter((call2) => call2.outcome === "fail_closed");
|
|
6765
|
+
const first = failed[0];
|
|
6766
|
+
if (first !== void 0) {
|
|
6767
|
+
const reason = first.failure ?? "no usable verdicts in the answer";
|
|
6768
|
+
parts.push(`${failed.length} of ${split.length} single-card calls failed; ${first.conversationId ?? "a card"}: ${reason}`);
|
|
6769
|
+
}
|
|
6770
|
+
return parts.length === 0 ? null : parts.join("; ");
|
|
6771
|
+
}
|
|
6772
|
+
function gateFailure(card, startedAt, reruns, timedOut, failedAt) {
|
|
6773
|
+
return {
|
|
6774
|
+
conversationId: card.conversation_id,
|
|
6775
|
+
source: "gate",
|
|
6776
|
+
card,
|
|
6777
|
+
anchor: anchorFrom(card),
|
|
6778
|
+
startedAt,
|
|
6779
|
+
reruns,
|
|
6780
|
+
outcome: timedOut ? "timed_out" : "failed",
|
|
6781
|
+
failedAt
|
|
6782
|
+
};
|
|
6783
|
+
}
|
|
6784
|
+
function decidedBy(result) {
|
|
6785
|
+
const undecided = new Set(result.undecided);
|
|
6786
|
+
return result.verdicts.map((verdict) => verdict.conversation_id).filter((id) => !undecided.has(id));
|
|
6787
|
+
}
|
|
5766
6788
|
function timeoutOption(value, fallback, name) {
|
|
5767
6789
|
if (value === void 0) return fallback;
|
|
5768
6790
|
if (value === 0 || value === Number.POSITIVE_INFINITY) return Number.POSITIVE_INFINITY;
|
|
@@ -5770,6 +6792,7 @@ function timeoutOption(value, fallback, name) {
|
|
|
5770
6792
|
return value;
|
|
5771
6793
|
}
|
|
5772
6794
|
function launchKey(launch) {
|
|
6795
|
+
if (launch === "held") return "heldAfterFailure";
|
|
5773
6796
|
return launch === "started" ? "turnsStarted" : launch === "queued" ? "turnsQueued" : "turnsRefused";
|
|
5774
6797
|
}
|
|
5775
6798
|
function trayInvites(tray) {
|