agentchatme 1.4.0 → 1.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +124 -0
- package/README.md +27 -6
- package/dist/index.cjs +1123 -100
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +723 -43
- package/dist/index.d.ts +723 -43
- package/dist/index.js +1123 -100
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -45,6 +45,11 @@ var ErrorCode = {
|
|
|
45
45
|
RESTRICTED: "RESTRICTED",
|
|
46
46
|
CONVERSATION_NOT_FOUND: "CONVERSATION_NOT_FOUND",
|
|
47
47
|
MESSAGE_NOT_FOUND: "MESSAGE_NOT_FOUND",
|
|
48
|
+
/**
|
|
49
|
+
* 410 when you download the attachment of a message hidden from you
|
|
50
|
+
* because your machine could not process it (`Message.could_not_process`).
|
|
51
|
+
*/
|
|
52
|
+
COULD_NOT_PROCESS: "COULD_NOT_PROCESS",
|
|
48
53
|
GROUP_DELETED: "GROUP_DELETED",
|
|
49
54
|
RATE_LIMITED: "RATE_LIMITED",
|
|
50
55
|
RECIPIENT_BACKLOGGED: "RECIPIENT_BACKLOGGED",
|
|
@@ -306,7 +311,7 @@ function createAgentChatError(body, status, headers) {
|
|
|
306
311
|
}
|
|
307
312
|
|
|
308
313
|
// src/version.ts
|
|
309
|
-
var VERSION = "1.
|
|
314
|
+
var VERSION = "1.5.1" ;
|
|
310
315
|
|
|
311
316
|
// src/runtime.ts
|
|
312
317
|
function detectRuntime() {
|
|
@@ -672,6 +677,7 @@ function parseBacklogWarning(header2) {
|
|
|
672
677
|
if (!Number.isFinite(undeliveredCount) || !Number.isInteger(undeliveredCount)) return null;
|
|
673
678
|
return { recipientHandle, undeliveredCount };
|
|
674
679
|
}
|
|
680
|
+
var PROCESSING_FAILURE_ACTIONS = /* @__PURE__ */ new Set(["retry", "hidden", "paused", "none"]);
|
|
675
681
|
function intHeader(headers, name) {
|
|
676
682
|
const raw = headers?.get?.(name);
|
|
677
683
|
if (typeof raw !== "string" || !/^\d+$/.test(raw.trim())) return null;
|
|
@@ -763,7 +769,8 @@ var AgentChatClient = class _AgentChatClient {
|
|
|
763
769
|
record: (conversationId, opts) => this.recordDecision(conversationId, opts)
|
|
764
770
|
};
|
|
765
771
|
this.conversations = {
|
|
766
|
-
markRead: (conversationId, throughMessageId, opts) => this.markConversationRead(conversationId, throughMessageId, opts)
|
|
772
|
+
markRead: (conversationId, throughMessageId, opts) => this.markConversationRead(conversationId, throughMessageId, opts),
|
|
773
|
+
reportProcessingFailure: (conversationId, opts) => this.reportProcessingFailure(conversationId, opts)
|
|
767
774
|
};
|
|
768
775
|
this.texts = {
|
|
769
776
|
get: (opts) => this.getTexts(opts)
|
|
@@ -835,6 +842,36 @@ var AgentChatClient = class _AgentChatClient {
|
|
|
835
842
|
options
|
|
836
843
|
);
|
|
837
844
|
}
|
|
845
|
+
async reportProcessingFailure(conversationId, options) {
|
|
846
|
+
const body = {
|
|
847
|
+
through_message_id: options.throughMessageId,
|
|
848
|
+
outcome: options.outcome,
|
|
849
|
+
evidence: options.evidence
|
|
850
|
+
};
|
|
851
|
+
let res;
|
|
852
|
+
try {
|
|
853
|
+
res = await this.post(
|
|
854
|
+
`/v1/conversations/${encodeURIComponent(conversationId)}/processing-failures`,
|
|
855
|
+
body,
|
|
856
|
+
// One report, one count: a retried request carries the same key.
|
|
857
|
+
{ ...options, idempotencyKey: options.idempotencyKey ?? generateClientMsgId() }
|
|
858
|
+
);
|
|
859
|
+
} catch (err) {
|
|
860
|
+
if (err instanceof AgentChatError && err.status === 404 && err.code !== "CONVERSATION_NOT_FOUND" && err.code !== "MESSAGE_NOT_FOUND") {
|
|
861
|
+
return { action: "none", attempts: 0, hidden_message_ids: [], paused_until: null };
|
|
862
|
+
}
|
|
863
|
+
throw err;
|
|
864
|
+
}
|
|
865
|
+
const r = res !== null && typeof res === "object" ? res : {};
|
|
866
|
+
const action = PROCESSING_FAILURE_ACTIONS.has(r.action) ? r.action : "none";
|
|
867
|
+
return {
|
|
868
|
+
action,
|
|
869
|
+
attempts: typeof r.attempts === "number" && Number.isFinite(r.attempts) ? r.attempts : 0,
|
|
870
|
+
hidden_message_ids: Array.isArray(r.hidden_message_ids) ? r.hidden_message_ids.filter((id) => typeof id === "string") : [],
|
|
871
|
+
...typeof r.hidden_count === "number" && Number.isFinite(r.hidden_count) ? { hidden_count: r.hidden_count } : {},
|
|
872
|
+
paused_until: typeof r.paused_until === "string" ? r.paused_until : null
|
|
873
|
+
};
|
|
874
|
+
}
|
|
838
875
|
getTexts(options) {
|
|
839
876
|
const params = new URLSearchParams({ runtime: options.runtime });
|
|
840
877
|
return this.get(`/v1/texts?${params.toString()}`, options);
|
|
@@ -1099,24 +1136,48 @@ var AgentChatClient = class _AgentChatClient {
|
|
|
1099
1136
|
* most one:
|
|
1100
1137
|
* - `beforeSeq` — backwards scrollback (rows with seq < N, newest first)
|
|
1101
1138
|
* - `afterSeq` — forwards gap-fill (rows with seq > N, oldest first)
|
|
1139
|
+
* - `afterMessageId` — forwards from a message you hold (instead of `afterSeq`)
|
|
1102
1140
|
* - `aroundMessageId` — backwards window ending at that exact message
|
|
1103
1141
|
*
|
|
1104
1142
|
* `afterSeq` is the path `RealtimeClient` uses for in-order recovery
|
|
1105
1143
|
* when a per-conversation seq gap is detected. Application code usually
|
|
1106
1144
|
* only needs `beforeSeq` for normal pagination.
|
|
1145
|
+
*
|
|
1146
|
+
* With `waitMs` > 0 (and `afterSeq` or `afterMessageId`) this is the
|
|
1147
|
+
* scoped wait: the server holds the request until a message from someone
|
|
1148
|
+
* else arrives after the cursor, or answers `[]` after `waitMs`. One
|
|
1149
|
+
* conversation only, no sync ack. `getMessagesPage` also returns how long
|
|
1150
|
+
* the server held it.
|
|
1107
1151
|
*/
|
|
1108
|
-
getMessages(conversationId, options) {
|
|
1152
|
+
async getMessages(conversationId, options) {
|
|
1153
|
+
return (await this.getMessagesPage(conversationId, options)).messages;
|
|
1154
|
+
}
|
|
1155
|
+
/** `getMessages()` with the response facts a waiter needs: how long the server held a scoped wait (`waitedMs`). */
|
|
1156
|
+
async getMessagesPage(conversationId, options) {
|
|
1109
1157
|
const params = new URLSearchParams();
|
|
1110
1158
|
params.set("limit", String(options?.limit ?? 50));
|
|
1111
1159
|
if (options?.beforeSeq !== void 0) params.set("before_seq", String(options.beforeSeq));
|
|
1112
1160
|
if (options?.afterSeq !== void 0) params.set("after_seq", String(options.afterSeq));
|
|
1161
|
+
if (options?.afterMessageId !== void 0) params.set("after_message_id", options.afterMessageId);
|
|
1113
1162
|
if (options?.aroundMessageId !== void 0) {
|
|
1114
1163
|
params.set("around_message_id", options.aroundMessageId);
|
|
1115
1164
|
}
|
|
1116
|
-
|
|
1165
|
+
const waitMs = Math.floor(options?.waitMs ?? 0);
|
|
1166
|
+
if (waitMs > 0) params.set("wait_ms", String(waitMs));
|
|
1167
|
+
const timeoutMs = options?.timeoutMs !== void 0 ? options.timeoutMs : waitMs > 0 ? waitMs + INBOX_WAIT_TIMEOUT_GRACE_MS : void 0;
|
|
1168
|
+
const res = await this.http.request(
|
|
1169
|
+
"GET",
|
|
1117
1170
|
`/v1/messages/${encodeURIComponent(conversationId)}?${params.toString()}`,
|
|
1118
|
-
options
|
|
1171
|
+
this.toRequestOpts({ ...options, timeoutMs })
|
|
1119
1172
|
);
|
|
1173
|
+
const waitedMs = intHeader(res.headers, "x-agentchat-waited-ms");
|
|
1174
|
+
return {
|
|
1175
|
+
messages: res.data,
|
|
1176
|
+
waitedMs,
|
|
1177
|
+
scopedWaitSupported: waitMs > 0 ? waitedMs !== null : null,
|
|
1178
|
+
requestId: res.requestId,
|
|
1179
|
+
headers: res.headers
|
|
1180
|
+
};
|
|
1120
1181
|
}
|
|
1121
1182
|
/**
|
|
1122
1183
|
* Hide a message from your own view (hide-for-me). Either side of the
|
|
@@ -1408,11 +1469,13 @@ var AgentChatClient = class _AgentChatClient {
|
|
|
1408
1469
|
}
|
|
1409
1470
|
// ─── Mutes ────────────────────────────────────────────────────────────────
|
|
1410
1471
|
//
|
|
1411
|
-
// Mute
|
|
1412
|
-
//
|
|
1413
|
-
//
|
|
1414
|
-
//
|
|
1415
|
-
//
|
|
1472
|
+
// Mute stops the wake for a specific agent or conversation without
|
|
1473
|
+
// blocking or leaving: no real-time WebSocket push and no wake webhook.
|
|
1474
|
+
// Nothing is hidden — the conversation stays on the inbox tray
|
|
1475
|
+
// (`muted: true`), envelopes still land in `/v1/messages/sync`, and the
|
|
1476
|
+
// unread counter still bumps. A muted message reaches the machine at its
|
|
1477
|
+
// next sync (every reconnect) or when the agent opens the conversation;
|
|
1478
|
+
// until then the sender sees it as `stored`, not `delivered`.
|
|
1416
1479
|
//
|
|
1417
1480
|
// All mute APIs are idempotent:
|
|
1418
1481
|
// - Re-muting with a different `mutedUntil` refreshes the expiry.
|
|
@@ -3560,6 +3623,7 @@ __export(integration_exports, {
|
|
|
3560
3623
|
anchor: () => anchor_exports,
|
|
3561
3624
|
applyInviteOutcome: () => applyInviteOutcome,
|
|
3562
3625
|
applyOutcome: () => applyOutcome,
|
|
3626
|
+
conversationRunPrompt: () => conversationRunPrompt,
|
|
3563
3627
|
hostedMcpUrl: () => hostedMcpUrl,
|
|
3564
3628
|
inviteOutcome: () => inviteOutcome,
|
|
3565
3629
|
isTexts: () => isTexts,
|
|
@@ -4795,6 +4859,7 @@ function readResult(text2) {
|
|
|
4795
4859
|
// src/integration/receive-loop.ts
|
|
4796
4860
|
var TRAY_LIMIT = 50;
|
|
4797
4861
|
var SILENT_MEMORY = 1e3;
|
|
4862
|
+
var CONVERSATION_KEY = "conversation:";
|
|
4798
4863
|
var PACE_MS = 6e4;
|
|
4799
4864
|
var TURN_OVERDUE_MS = 30 * 6e4;
|
|
4800
4865
|
var NO_PROGRESS_BACKOFF_MS = 1e3;
|
|
@@ -4804,6 +4869,11 @@ var CYCLE_TIMEOUT_MS = 10 * 6e4;
|
|
|
4804
4869
|
var GATE_CHUNK_SIZE = 10;
|
|
4805
4870
|
var REPLY_TURN_PLACEHOLDER = "<conversation_id>";
|
|
4806
4871
|
var RECENT_CYCLES = 20;
|
|
4872
|
+
var MAX_RERUNS_PER_CYCLE = 3;
|
|
4873
|
+
var SPLIT_STOP_AFTER_FAILURES = 2;
|
|
4874
|
+
var RUNTIME_CHECK_TIMEOUT_MS = 6e4;
|
|
4875
|
+
var AWAITING_EVIDENCE_MS = 10 * 6e4;
|
|
4876
|
+
var EVIDENCE_MEMORY = 100;
|
|
4807
4877
|
var Pacer = class {
|
|
4808
4878
|
constructor(paceMs, now2) {
|
|
4809
4879
|
this.paceMs = paceMs;
|
|
@@ -4812,30 +4882,121 @@ var Pacer = class {
|
|
|
4812
4882
|
paceMs;
|
|
4813
4883
|
now;
|
|
4814
4884
|
until = /* @__PURE__ */ new Map();
|
|
4885
|
+
/** Orders holds and tray reads: a read lifts only holds older than its start. */
|
|
4886
|
+
seq = 0;
|
|
4887
|
+
/** A D5 pace. It never replaces a could-not-process hold at the same anchor (that input stays held). */
|
|
4815
4888
|
hold(key, anchor) {
|
|
4816
|
-
this.until.
|
|
4817
|
-
|
|
4818
|
-
|
|
4819
|
-
|
|
4820
|
-
|
|
4821
|
-
|
|
4822
|
-
|
|
4823
|
-
|
|
4824
|
-
|
|
4889
|
+
const prior = this.until.get(key);
|
|
4890
|
+
if (prior?.failure !== void 0 && prior.anchor === anchor) return;
|
|
4891
|
+
this.set(key, { anchor, until: this.now() + this.paceMs });
|
|
4892
|
+
}
|
|
4893
|
+
/** Could not process: hold the key at this anchor until a newer message moves it (held again at it: since the first time). */
|
|
4894
|
+
holdFailed(key, anchor, reason) {
|
|
4895
|
+
const prior = this.until.get(key);
|
|
4896
|
+
const since = prior?.failure !== void 0 && prior.anchor === anchor ? prior.failure.since : this.now();
|
|
4897
|
+
this.set(key, { anchor, until: Number.POSITIVE_INFINITY, failure: { reason, since, seq: ++this.seq } });
|
|
4898
|
+
}
|
|
4899
|
+
/** What holds the key at this anchor now: a pace, a could-not-process hold, or nothing (a stale entry goes). */
|
|
4900
|
+
check(key, anchor) {
|
|
4825
4901
|
const entry = this.until.get(key);
|
|
4826
|
-
if (!entry) return
|
|
4902
|
+
if (!entry) return null;
|
|
4827
4903
|
if (entry.anchor !== anchor || this.now() >= entry.until) {
|
|
4828
4904
|
this.until.delete(key);
|
|
4829
|
-
return
|
|
4905
|
+
return null;
|
|
4830
4906
|
}
|
|
4831
|
-
return
|
|
4907
|
+
return entry.failure ? "failed" : "paced";
|
|
4908
|
+
}
|
|
4909
|
+
held(key, anchor) {
|
|
4910
|
+
return this.check(key, anchor) !== null;
|
|
4832
4911
|
}
|
|
4912
|
+
/** A could-not-process hold on the key at exactly this anchor; changes nothing (a queued run's anchor may be older than the card's). */
|
|
4913
|
+
heldFailedAt(key, anchor) {
|
|
4914
|
+
const entry = this.until.get(key);
|
|
4915
|
+
return entry?.failure !== void 0 && entry.anchor === anchor;
|
|
4916
|
+
}
|
|
4917
|
+
/** Lift whatever holds the key (a pace or a could-not-process hold). */
|
|
4833
4918
|
release(key) {
|
|
4834
4919
|
this.until.delete(key);
|
|
4835
4920
|
}
|
|
4921
|
+
/** Lift a pace; a could-not-process hold stays. */
|
|
4922
|
+
releasePace(key) {
|
|
4923
|
+
if (this.until.get(key)?.failure === void 0) this.until.delete(key);
|
|
4924
|
+
}
|
|
4836
4925
|
releaseInvites() {
|
|
4837
4926
|
for (const key of [...this.until.keys()]) if (key.startsWith("invite:")) this.until.delete(key);
|
|
4838
4927
|
}
|
|
4928
|
+
/** Taken before a tray read: `keepFailedOn` lifts only holds made before it. */
|
|
4929
|
+
mark() {
|
|
4930
|
+
return this.seq;
|
|
4931
|
+
}
|
|
4932
|
+
/**
|
|
4933
|
+
* After a tray read from `mark` (`anchors`: each card's key and anchor):
|
|
4934
|
+
* a hold whose card shows another anchor is over (a newer message came),
|
|
4935
|
+
* and, on a complete tray (`complete`: not truncated), so is one whose
|
|
4936
|
+
* card is gone (read meanwhile: a verdict, a mark-read). Only holds made
|
|
4937
|
+
* before the read started: a read that began earlier says nothing of a
|
|
4938
|
+
* later hold.
|
|
4939
|
+
*/
|
|
4940
|
+
keepFailedOn(anchors, mark, complete) {
|
|
4941
|
+
for (const [key, entry] of this.until) {
|
|
4942
|
+
if (entry.failure === void 0 || entry.failure.seq > mark) continue;
|
|
4943
|
+
const shown = anchors.get(key);
|
|
4944
|
+
if (shown === void 0 ? complete : shown !== entry.anchor) this.until.delete(key);
|
|
4945
|
+
}
|
|
4946
|
+
}
|
|
4947
|
+
/** The could-not-process holds, oldest first (by `since`). */
|
|
4948
|
+
failed() {
|
|
4949
|
+
const out = [];
|
|
4950
|
+
for (const [key, entry] of this.until) {
|
|
4951
|
+
if (entry.failure !== void 0 && entry.anchor !== null) {
|
|
4952
|
+
out.push({ key, anchor: entry.anchor, reason: entry.failure.reason, since: entry.failure.since });
|
|
4953
|
+
}
|
|
4954
|
+
}
|
|
4955
|
+
return out.sort((a, b) => a.since - b.since);
|
|
4956
|
+
}
|
|
4957
|
+
set(key, entry) {
|
|
4958
|
+
this.until.delete(key);
|
|
4959
|
+
this.until.set(key, entry);
|
|
4960
|
+
let paces = 0;
|
|
4961
|
+
let holds = 0;
|
|
4962
|
+
for (const held of this.until.values()) {
|
|
4963
|
+
if (held.failure === void 0) paces += 1;
|
|
4964
|
+
else holds += 1;
|
|
4965
|
+
}
|
|
4966
|
+
for (const [key2, held] of this.until) {
|
|
4967
|
+
if (paces <= SILENT_MEMORY && holds <= SILENT_MEMORY) break;
|
|
4968
|
+
if (held.failure === void 0 && paces > SILENT_MEMORY) {
|
|
4969
|
+
this.until.delete(key2);
|
|
4970
|
+
paces -= 1;
|
|
4971
|
+
} else if (held.failure !== void 0 && holds > SILENT_MEMORY) {
|
|
4972
|
+
this.until.delete(key2);
|
|
4973
|
+
holds -= 1;
|
|
4974
|
+
}
|
|
4975
|
+
}
|
|
4976
|
+
}
|
|
4977
|
+
};
|
|
4978
|
+
function clears(completion, failure) {
|
|
4979
|
+
return completion.kind === failure.source && completion.conversations.has(failure.conversationId);
|
|
4980
|
+
}
|
|
4981
|
+
function evidences(completion, failure) {
|
|
4982
|
+
return completion.kind === failure.source && completion.at > failure.startedAt && completion.conversations.size > 0 && !completion.conversations.has(failure.conversationId);
|
|
4983
|
+
}
|
|
4984
|
+
var EvidenceClock = class {
|
|
4985
|
+
last = 0;
|
|
4986
|
+
completions = [];
|
|
4987
|
+
start() {
|
|
4988
|
+
return ++this.last;
|
|
4989
|
+
}
|
|
4990
|
+
complete(kind, conversations) {
|
|
4991
|
+
const entry = { at: ++this.last, kind, conversations: new Set(conversations) };
|
|
4992
|
+
this.completions.push(entry);
|
|
4993
|
+
if (this.completions.length > EVIDENCE_MEMORY) this.completions.shift();
|
|
4994
|
+
return entry;
|
|
4995
|
+
}
|
|
4996
|
+
/** A completion since the failed attempt started that is evidence for it (`evidences`). */
|
|
4997
|
+
since(failure) {
|
|
4998
|
+
return this.completions.some((c) => evidences(c, failure));
|
|
4999
|
+
}
|
|
4839
5000
|
};
|
|
4840
5001
|
var ReceiveLoop = class {
|
|
4841
5002
|
options;
|
|
@@ -4854,15 +5015,68 @@ var ReceiveLoop = class {
|
|
|
4854
5015
|
/** When a single-run invite's silent run was counted (D5 pacing in single-run mode). */
|
|
4855
5016
|
silentAt = /* @__PURE__ */ new Map();
|
|
4856
5017
|
paceMs;
|
|
5018
|
+
/** Turns and runs waiting for a slot; `drop` ends what else the entry holds (an invite run's other invites). */
|
|
4857
5019
|
waiting = [];
|
|
5020
|
+
/** Conversation and invite runs started and not yet concluded: `idle()` and `stop()` wait for them. */
|
|
5021
|
+
runWork = /* @__PURE__ */ new Set();
|
|
5022
|
+
/** Could not process: the order of attempts and completions. */
|
|
5023
|
+
evidence = new EvidenceClock();
|
|
5024
|
+
/** Could not process: failures waiting for a completion (the evidence a report needs), by conversation. */
|
|
5025
|
+
awaitingEvidence = /* @__PURE__ */ new Map();
|
|
5026
|
+
/** Waiting failures the runtime check has already been asked about (once each). */
|
|
5027
|
+
checkedFailures = /* @__PURE__ */ new WeakSet();
|
|
5028
|
+
/**
|
|
5029
|
+
* Per kind of work, the conversations whose attempt failed since that kind
|
|
5030
|
+
* last completed. Two or more: several failing together is the runtime, so
|
|
5031
|
+
* a passing `checkRuntime()` is no evidence for any of them.
|
|
5032
|
+
*/
|
|
5033
|
+
failingSince = { gate: /* @__PURE__ */ new Set(), turn: /* @__PURE__ */ new Set(), run: /* @__PURE__ */ new Set() };
|
|
5034
|
+
/** gateCall: conversations whose last gate call failed; gated alone, after the others. */
|
|
5035
|
+
gateAlone = /* @__PURE__ */ new Set();
|
|
5036
|
+
/**
|
|
5037
|
+
* gateCall: the running cycle's gate: its cards (`members`) and its
|
|
5038
|
+
* single-card queue. A `retry` answer for one of its cards joins the queue;
|
|
5039
|
+
* a `hidden` one skips the card here (its card, read before the hide, is
|
|
5040
|
+
* stale) and gates it in the next cycle.
|
|
5041
|
+
*/
|
|
5042
|
+
activeSplit = null;
|
|
5043
|
+
/**
|
|
5044
|
+
* gateCall: reply turns to start again on a `retry` / `hidden` answer that
|
|
5045
|
+
* came while a gate was deciding: they start after that cycle's own turns
|
|
5046
|
+
* (no turn starts while cards of a cycle still wait for their gate call).
|
|
5047
|
+
*/
|
|
5048
|
+
turnRerunsAfterGate = [];
|
|
5049
|
+
lastRuntimeCheck = null;
|
|
5050
|
+
failureCounts = {
|
|
5051
|
+
reported: 0,
|
|
5052
|
+
retry: 0,
|
|
5053
|
+
hidden: 0,
|
|
5054
|
+
paused: 0,
|
|
5055
|
+
none: 0,
|
|
5056
|
+
reportErrors: 0,
|
|
5057
|
+
reruns: 0,
|
|
5058
|
+
rerunsCapped: 0,
|
|
5059
|
+
runtimeChecksOk: 0,
|
|
5060
|
+
runtimeChecksFailed: 0,
|
|
5061
|
+
splitsStopped: 0,
|
|
5062
|
+
startsHeld: 0
|
|
5063
|
+
};
|
|
4858
5064
|
/** Conversations whose verdict a gate call is recording right now (an abandoned cycle's included). */
|
|
4859
5065
|
recordsInFlight = /* @__PURE__ */ new Set();
|
|
5066
|
+
/**
|
|
5067
|
+
* Could not process: conversations whose report is out on another's
|
|
5068
|
+
* completion (`takeDue`): held out of the gate and of new runs, and a run
|
|
5069
|
+
* or turn of theirs waiting for a slot waits, until the server answers.
|
|
5070
|
+
*/
|
|
5071
|
+
reporting = /* @__PURE__ */ new Map();
|
|
4860
5072
|
/** Turns holding a slot (admitted, not yet ended or released). */
|
|
4861
5073
|
slotted = /* @__PURE__ */ new Set();
|
|
4862
5074
|
handle;
|
|
4863
5075
|
started = false;
|
|
4864
5076
|
stopped = false;
|
|
4865
5077
|
heldWhileRunning = false;
|
|
5078
|
+
/** A cycle left a card out because its could-not-process report was out: `letGo` reads the tray again. */
|
|
5079
|
+
heldWhileReporting = false;
|
|
4866
5080
|
activeTurns = 0;
|
|
4867
5081
|
noProgressStreak = 0;
|
|
4868
5082
|
moreTimer = null;
|
|
@@ -4889,6 +5103,8 @@ var ReceiveLoop = class {
|
|
|
4889
5103
|
runsNotStarted: 0,
|
|
4890
5104
|
chunksOk: 0,
|
|
4891
5105
|
chunksFailed: 0,
|
|
5106
|
+
splitCallsOk: 0,
|
|
5107
|
+
splitCallsFailed: 0,
|
|
4892
5108
|
outcomeRecordFailures: 0
|
|
4893
5109
|
};
|
|
4894
5110
|
outcomes = {};
|
|
@@ -4986,7 +5202,10 @@ var ReceiveLoop = class {
|
|
|
4986
5202
|
this.clearMoreTimer();
|
|
4987
5203
|
this.abort.abort();
|
|
4988
5204
|
this.cycleAbort?.abort();
|
|
4989
|
-
for (const { handle } of this.waiting.splice(0))
|
|
5205
|
+
for (const { handle, drop } of this.waiting.splice(0)) {
|
|
5206
|
+
this.tracker.end(handle);
|
|
5207
|
+
drop?.();
|
|
5208
|
+
}
|
|
4990
5209
|
for (const off of this.unsubscribe.splice(0)) {
|
|
4991
5210
|
try {
|
|
4992
5211
|
off();
|
|
@@ -4998,11 +5217,21 @@ var ReceiveLoop = class {
|
|
|
4998
5217
|
} catch {
|
|
4999
5218
|
}
|
|
5000
5219
|
await this.coalescer.idle();
|
|
5001
|
-
await Promise.allSettled([...this.pendingWork]);
|
|
5220
|
+
await Promise.allSettled([...this.pendingWork, ...this.runWork]);
|
|
5002
5221
|
}
|
|
5003
|
-
/**
|
|
5004
|
-
|
|
5005
|
-
|
|
5222
|
+
/**
|
|
5223
|
+
* Resolves when the running cycle has finished (tests, graceful stops) and,
|
|
5224
|
+
* in single-run mode, no run is running or waiting for a slot (conversation
|
|
5225
|
+
* runs go on after the cycle that started them). Gate-call turns are not
|
|
5226
|
+
* waited for.
|
|
5227
|
+
*/
|
|
5228
|
+
async idle() {
|
|
5229
|
+
for (; ; ) {
|
|
5230
|
+
await this.coalescer.idle();
|
|
5231
|
+
const runs = [...this.runWork];
|
|
5232
|
+
if (runs.length === 0) return;
|
|
5233
|
+
await Promise.allSettled(runs);
|
|
5234
|
+
}
|
|
5006
5235
|
}
|
|
5007
5236
|
/** Let go of a turn whose end the runtime will never confirm (an operator's call). Nothing is recorded for it. */
|
|
5008
5237
|
releaseTurn(key) {
|
|
@@ -5042,6 +5271,17 @@ var ReceiveLoop = class {
|
|
|
5042
5271
|
turnsWaiting: this.waiting.length,
|
|
5043
5272
|
outcomes: { ...this.outcomes }
|
|
5044
5273
|
},
|
|
5274
|
+
processingFailures: {
|
|
5275
|
+
...this.failureCounts,
|
|
5276
|
+
awaitingEvidence: this.awaitingEvidence.size,
|
|
5277
|
+
lastRuntimeCheck: this.lastRuntimeCheck === null ? null : { ...this.lastRuntimeCheck },
|
|
5278
|
+
held: this.pacer.failed().map(({ key, anchor, reason, since }) => ({
|
|
5279
|
+
conversationId: key.slice(CONVERSATION_KEY.length),
|
|
5280
|
+
anchor,
|
|
5281
|
+
since: new Date(since).toISOString(),
|
|
5282
|
+
reason
|
|
5283
|
+
}))
|
|
5284
|
+
},
|
|
5045
5285
|
lastError: this.lastError
|
|
5046
5286
|
};
|
|
5047
5287
|
}
|
|
@@ -5070,7 +5310,7 @@ var ReceiveLoop = class {
|
|
|
5070
5310
|
const id = typeof payload.conversation_id === "string" ? payload.conversation_id : void 0;
|
|
5071
5311
|
const named = Array.isArray(payload.conversation_ids) ? payload.conversation_ids.filter((x) => typeof x === "string") : [];
|
|
5072
5312
|
if (reason === "unanswered") {
|
|
5073
|
-
for (const conversationId of named) this.pacer.
|
|
5313
|
+
for (const conversationId of named) this.pacer.releasePace(`${CONVERSATION_KEY}${conversationId}`);
|
|
5074
5314
|
this.pacer.releaseInvites();
|
|
5075
5315
|
}
|
|
5076
5316
|
this.wakeFrom("wake_frame", id, {
|
|
@@ -5113,6 +5353,8 @@ var ReceiveLoop = class {
|
|
|
5113
5353
|
const work = (async () => {
|
|
5114
5354
|
const summary = this.options.mode === "gateCall" ? await this.gateCallCycle(run3) : await this.singleRunCycle(run3);
|
|
5115
5355
|
if (!run3.abandoned()) this.finishCycle(summary);
|
|
5356
|
+
this.flushTurnReruns();
|
|
5357
|
+
void this.track(this.checkParked(this.options.mode === "gateCall" ? "turn" : "run"));
|
|
5116
5358
|
})();
|
|
5117
5359
|
let timer = null;
|
|
5118
5360
|
const overdue = new Promise((resolve) => {
|
|
@@ -5180,12 +5422,13 @@ var ReceiveLoop = class {
|
|
|
5180
5422
|
* cycle that is making progress is not abandoned for it. With the deadline
|
|
5181
5423
|
* off the watchdog is the calls' only bound, so they count.
|
|
5182
5424
|
*/
|
|
5183
|
-
boundedModelCall(run3) {
|
|
5184
|
-
const call2 = this.deadlineModelCall(run3);
|
|
5425
|
+
boundedModelCall(run3, timedOut) {
|
|
5426
|
+
const call2 = this.deadlineModelCall(run3, timedOut);
|
|
5185
5427
|
if (!Number.isFinite(this.gateCallTimeoutMs)) return call2;
|
|
5186
5428
|
return (prompt, options) => run3.exclude(() => call2(prompt, options));
|
|
5187
5429
|
}
|
|
5188
|
-
|
|
5430
|
+
/** `timedOut` is set when a call runs past its deadline (a could-not-process report then says `timed_out`). */
|
|
5431
|
+
deadlineModelCall(run3, timedOut) {
|
|
5189
5432
|
return (prompt) => new Promise((resolve, reject) => {
|
|
5190
5433
|
const controller = new AbortController();
|
|
5191
5434
|
const abort = () => {
|
|
@@ -5199,6 +5442,7 @@ var ReceiveLoop = class {
|
|
|
5199
5442
|
run3.signal.addEventListener("abort", abort, { once: true });
|
|
5200
5443
|
const timer = Number.isFinite(this.gateCallTimeoutMs) ? this._setTimeout(() => {
|
|
5201
5444
|
run3.signal.removeEventListener("abort", abort);
|
|
5445
|
+
if (timedOut) timedOut.value = true;
|
|
5202
5446
|
controller.abort();
|
|
5203
5447
|
reject(new Error(`no answer within ${this.gateCallTimeoutMs} ms`));
|
|
5204
5448
|
}, this.gateCallTimeoutMs) : null;
|
|
@@ -5246,20 +5490,22 @@ var ReceiveLoop = class {
|
|
|
5246
5490
|
}
|
|
5247
5491
|
}
|
|
5248
5492
|
async gateCallCycle(run3) {
|
|
5493
|
+
const mark = this.pacer.mark();
|
|
5249
5494
|
const tray = await this.readTray();
|
|
5250
5495
|
if (run3.stale()) return this.summary("abandoned");
|
|
5251
5496
|
if (tray === null) return this.summary("tray_failed", { failure: this.lastError });
|
|
5252
5497
|
const cards = cardsOf(tray);
|
|
5498
|
+
this.forgetCleared(tray, mark);
|
|
5253
5499
|
const invites = trayInvites(tray);
|
|
5254
5500
|
const unheld = this.unheld(cards);
|
|
5255
5501
|
const heldByTurns = cards.length - unheld.length;
|
|
5256
5502
|
if (heldByTurns > 0) this.heldWhileRunning = true;
|
|
5257
|
-
const gateCards
|
|
5503
|
+
const { open: gateCards, paced: pacedCards, heldAfterFailure } = this.paceCards(unheld);
|
|
5258
5504
|
const openInvites = invites.filter((invite) => !this.tracker.inviteRunning(invite.invite_id));
|
|
5259
5505
|
const owedInvites = openInvites.filter((invite) => !this.pacer.held(`invite:${invite.invite_id}`, null));
|
|
5260
|
-
const paced =
|
|
5506
|
+
const paced = pacedCards + (openInvites.length - owedInvites.length);
|
|
5261
5507
|
const owedCards = gateCards.filter((card) => card.muted !== true);
|
|
5262
|
-
const base = { cards: cards.length, invites: invites.length, heldByTurns, paced, truncated: tray.truncated === true };
|
|
5508
|
+
const base = { cards: cards.length, invites: invites.length, heldByTurns, paced, heldAfterFailure, truncated: tray.truncated === true };
|
|
5263
5509
|
if (cards.length === 0 && invites.length === 0) return this.summary("empty", base);
|
|
5264
5510
|
if (owedCards.length === 0 && owedInvites.length === 0) return this.summary("nothing_owed", base);
|
|
5265
5511
|
const texts = await this.options.texts.ensure();
|
|
@@ -5280,14 +5526,19 @@ var ReceiveLoop = class {
|
|
|
5280
5526
|
turnsRefused: 0,
|
|
5281
5527
|
turnsQueued: 0,
|
|
5282
5528
|
chunksOk: 0,
|
|
5283
|
-
chunksFailed: 0
|
|
5529
|
+
chunksFailed: 0,
|
|
5530
|
+
splitCallsOk: 0,
|
|
5531
|
+
splitCallsFailed: 0,
|
|
5532
|
+
heldAfterFailure
|
|
5284
5533
|
};
|
|
5285
5534
|
let heldDuringGate = 0;
|
|
5535
|
+
let splitStopped = false;
|
|
5286
5536
|
let outcome = "invites_only";
|
|
5287
5537
|
let failure = null;
|
|
5288
5538
|
if (owedCards.length > 0) {
|
|
5289
5539
|
const gate = await this.gateInChunks(run3, tray, gateCards, texts);
|
|
5290
5540
|
heldDuringGate = gate.held;
|
|
5541
|
+
tally.heldAfterFailure += gate.heldAfterFailure;
|
|
5291
5542
|
tally.gated = gate.gated;
|
|
5292
5543
|
tally.reply = gate.verdicts.reply;
|
|
5293
5544
|
tally.dismiss = gate.verdicts.dismiss;
|
|
@@ -5297,24 +5548,27 @@ var ReceiveLoop = class {
|
|
|
5297
5548
|
tally.recordFailures = gate.recordFailures;
|
|
5298
5549
|
tally.chunksOk = gate.chunks.filter((c) => c.outcome === "decided").length;
|
|
5299
5550
|
tally.chunksFailed = gate.chunks.length - tally.chunksOk;
|
|
5300
|
-
|
|
5301
|
-
|
|
5551
|
+
tally.splitCallsOk = gate.split.filter((c) => c.outcome === "decided").length;
|
|
5552
|
+
tally.splitCallsFailed = gate.split.length - tally.splitCallsOk;
|
|
5553
|
+
splitStopped = gate.splitStopped;
|
|
5554
|
+
failure = gateFailureText(gate.chunks, gate.split);
|
|
5555
|
+
if (tally.chunksOk + tally.splitCallsOk > 0) {
|
|
5302
5556
|
outcome = "decided";
|
|
5303
5557
|
for (const reply of gate.reply) {
|
|
5304
5558
|
if (run3.stale()) break;
|
|
5305
5559
|
tally[launchKey(await run3.exclude(() => this.startReplyTurn(reply, texts)))] += 1;
|
|
5306
5560
|
}
|
|
5307
|
-
} else if (tally.chunksFailed > 0) {
|
|
5561
|
+
} else if (tally.chunksFailed + tally.splitCallsFailed > 0) {
|
|
5308
5562
|
outcome = "fail_closed";
|
|
5309
5563
|
} else if (owedInvites.length === 0 && !run3.stale()) {
|
|
5310
|
-
return this.summary("nothing_owed", { ...base, heldByTurns: heldByTurns + heldDuringGate });
|
|
5564
|
+
return this.summary("nothing_owed", { ...base, heldByTurns: heldByTurns + heldDuringGate, heldAfterFailure: tally.heldAfterFailure });
|
|
5311
5565
|
}
|
|
5312
5566
|
}
|
|
5313
5567
|
for (const invite of owedInvites) {
|
|
5314
5568
|
if (run3.stale()) break;
|
|
5315
5569
|
tally[launchKey(await run3.exclude(() => this.startInviteTurn(invite, texts)))] += 1;
|
|
5316
5570
|
}
|
|
5317
|
-
const parts = { ...base, heldByTurns: heldByTurns + heldDuringGate, ...tally };
|
|
5571
|
+
const parts = { ...base, heldByTurns: heldByTurns + heldDuringGate, ...tally, splitStopped };
|
|
5318
5572
|
if (run3.stale()) return this.summary("abandoned", parts);
|
|
5319
5573
|
if (tray.truncated === true && outcome === "decided") {
|
|
5320
5574
|
this.readTheRest(tally.recorded > 0 || tally.turnsStarted > 0);
|
|
@@ -5324,10 +5578,54 @@ var ReceiveLoop = class {
|
|
|
5324
5578
|
/**
|
|
5325
5579
|
* Held out of the gate: a running or waiting turn, or a verdict still being
|
|
5326
5580
|
* recorded (an abandoned cycle's last call): gating it again could record
|
|
5327
|
-
* the same verdict twice (a second defer climbs the ladder).
|
|
5581
|
+
* the same verdict twice (a second defer climbs the ladder). Also a
|
|
5582
|
+
* conversation whose could-not-process report is out (`reporting`): its
|
|
5583
|
+
* answer may hold it.
|
|
5328
5584
|
*/
|
|
5329
5585
|
unheld(cards) {
|
|
5330
|
-
return this.tracker.withoutRunning(cards).filter((card) =>
|
|
5586
|
+
return this.tracker.withoutRunning(cards).filter((card) => {
|
|
5587
|
+
if (this.recordsInFlight.has(card.conversation_id)) return false;
|
|
5588
|
+
if (!this.reporting.has(card.conversation_id)) return true;
|
|
5589
|
+
this.heldWhileReporting = true;
|
|
5590
|
+
return false;
|
|
5591
|
+
});
|
|
5592
|
+
}
|
|
5593
|
+
/**
|
|
5594
|
+
* The cards neither paced (D5) nor held after a could-not-process answer
|
|
5595
|
+
* (`none` / `paused` through the anchor they still show), and how many of
|
|
5596
|
+
* each were left out. A card whose anchor moved is no longer held.
|
|
5597
|
+
*/
|
|
5598
|
+
paceCards(cards) {
|
|
5599
|
+
const open = [];
|
|
5600
|
+
let paced = 0;
|
|
5601
|
+
let heldAfterFailure = 0;
|
|
5602
|
+
for (const card of cards) {
|
|
5603
|
+
const state = this.pacer.check(`${CONVERSATION_KEY}${card.conversation_id}`, anchorFrom(card));
|
|
5604
|
+
if (state === "paced") paced += 1;
|
|
5605
|
+
else if (state === "failed") heldAfterFailure += 1;
|
|
5606
|
+
else open.push(card);
|
|
5607
|
+
}
|
|
5608
|
+
return { open, paced, heldAfterFailure };
|
|
5609
|
+
}
|
|
5610
|
+
/** Held after a could-not-process answer, at the anchor this card still shows. */
|
|
5611
|
+
heldAfterFailure(card) {
|
|
5612
|
+
return this.pacer.check(`${CONVERSATION_KEY}${card.conversation_id}`, anchorFrom(card)) === "failed";
|
|
5613
|
+
}
|
|
5614
|
+
/**
|
|
5615
|
+
* A tray read after `mark`, every card on it (a turn's included): a held
|
|
5616
|
+
* conversation whose card shows a newer anchor is no longer held, and on
|
|
5617
|
+
* a complete tray (not truncated) neither is one whose card is gone (read
|
|
5618
|
+
* meanwhile: a verdict, a mark-read). A truncated tray proves nothing
|
|
5619
|
+
* about the cards it does not show.
|
|
5620
|
+
*/
|
|
5621
|
+
forgetCleared(tray, mark) {
|
|
5622
|
+
const anchors = new Map(cardsOf(tray).map((card) => [`${CONVERSATION_KEY}${card.conversation_id}`, anchorFrom(card)]));
|
|
5623
|
+
this.pacer.keepFailedOn(anchors, mark, tray.truncated !== true);
|
|
5624
|
+
}
|
|
5625
|
+
/** The start of a run or reply turn: held after a could-not-process answer at the anchor it was started for. */
|
|
5626
|
+
failureHeld(handle) {
|
|
5627
|
+
const turn = handle.turn;
|
|
5628
|
+
return turn.kind === "reply" && this.pacer.heldFailedAt(`${CONVERSATION_KEY}${turn.conversationId}`, turn.anchor);
|
|
5331
5629
|
}
|
|
5332
5630
|
/**
|
|
5333
5631
|
* The gate in chunks (contract v1 §3.6): the cards, in tray order, split
|
|
@@ -5342,62 +5640,192 @@ var ReceiveLoop = class {
|
|
|
5342
5640
|
* cards only (or none) is not called. Stops once the cycle is stale; a call
|
|
5343
5641
|
* a stop cut short is not counted as a failed chunk, and a cycle the
|
|
5344
5642
|
* watchdog abandoned reports and counts no chunk after its summary.
|
|
5643
|
+
*
|
|
5644
|
+
* The split (could not process): every owed card of a chunk of more than
|
|
5645
|
+
* one card that failed closed is then gated again alone, one call per card,
|
|
5646
|
+
* in tray order, before any turn starts, so one card the model cannot
|
|
5647
|
+
* handle fails only its own call. A card whose last gate call failed is
|
|
5648
|
+
* kept out of the chunks and gated alone after them. A single-card call that
|
|
5649
|
+
* fails is the card's failure (reported with evidence; a `retry` answer
|
|
5650
|
+
* gates it once more at the end of the queue). When the queue is empty, a
|
|
5651
|
+
* failure still waiting gets the runtime check. After
|
|
5652
|
+
* `SPLIT_STOP_AFTER_FAILURES` single-card calls in a row failed with nothing
|
|
5653
|
+
* decided in the cycle, the split goes on only if `checkRuntime()` passes;
|
|
5654
|
+
* otherwise it stops (`split_stopped`).
|
|
5345
5655
|
*/
|
|
5346
5656
|
async gateInChunks(run3, tray, cards, texts) {
|
|
5347
5657
|
const size = this.gateChunkSize;
|
|
5348
|
-
const of = Math.ceil(cards.length / size);
|
|
5349
|
-
const callModel = this.boundedModelCall(run3);
|
|
5350
5658
|
const client = this.recordingClient(run3);
|
|
5351
5659
|
const out = {
|
|
5352
5660
|
gated: 0,
|
|
5353
5661
|
held: 0,
|
|
5662
|
+
heldAfterFailure: 0,
|
|
5354
5663
|
reply: [],
|
|
5355
5664
|
verdicts: { reply: 0, dismiss: 0, defer: 0 },
|
|
5356
5665
|
undecided: 0,
|
|
5357
5666
|
recorded: 0,
|
|
5358
5667
|
recordFailures: 0,
|
|
5359
|
-
chunks: []
|
|
5668
|
+
chunks: [],
|
|
5669
|
+
split: [],
|
|
5670
|
+
splitStopped: false
|
|
5360
5671
|
};
|
|
5361
|
-
|
|
5362
|
-
|
|
5363
|
-
|
|
5364
|
-
const
|
|
5365
|
-
out.held += planned.length - chunk.length;
|
|
5366
|
-
if (chunk.length < planned.length) this.heldWhileRunning = true;
|
|
5367
|
-
if (!chunk.some((card) => card.muted !== true)) continue;
|
|
5368
|
-
const result = await run({
|
|
5369
|
-
tray: { ...tray, conversations: chunk },
|
|
5370
|
-
callModel,
|
|
5371
|
-
client,
|
|
5372
|
-
texts,
|
|
5373
|
-
signal: run3.signal,
|
|
5374
|
-
...this.options.toolNamer ? { toolNamer: this.options.toolNamer } : {}
|
|
5375
|
-
});
|
|
5376
|
-
if (run3.abandoned()) break;
|
|
5377
|
-
if (run3.stale() && result.outcome !== "decided") break;
|
|
5378
|
-
out.gated += chunk.length;
|
|
5672
|
+
const gated = /* @__PURE__ */ new Set();
|
|
5673
|
+
const decided = /* @__PURE__ */ new Set();
|
|
5674
|
+
const take = (result, sent) => {
|
|
5675
|
+
for (const card of sent) gated.add(card.conversation_id);
|
|
5379
5676
|
const undecided = new Set(result.undecided);
|
|
5380
5677
|
for (const verdict of result.verdicts) {
|
|
5381
|
-
if (
|
|
5678
|
+
if (undecided.has(verdict.conversation_id) || decided.has(verdict.conversation_id)) continue;
|
|
5679
|
+
decided.add(verdict.conversation_id);
|
|
5680
|
+
out.verdicts[verdict.decision] += 1;
|
|
5382
5681
|
}
|
|
5383
5682
|
out.reply.push(...result.reply);
|
|
5384
|
-
out.undecided += result.undecided.length;
|
|
5385
5683
|
out.recorded += result.recorded;
|
|
5386
5684
|
out.recordFailures += result.recordFailures.length;
|
|
5387
|
-
|
|
5388
|
-
|
|
5389
|
-
|
|
5390
|
-
|
|
5391
|
-
|
|
5392
|
-
|
|
5393
|
-
|
|
5394
|
-
|
|
5395
|
-
|
|
5396
|
-
|
|
5397
|
-
|
|
5398
|
-
|
|
5399
|
-
|
|
5685
|
+
};
|
|
5686
|
+
const gateOne = (card, timedOut) => run({
|
|
5687
|
+
tray: { ...tray, conversations: [card] },
|
|
5688
|
+
callModel: this.boundedModelCall(run3, timedOut),
|
|
5689
|
+
client,
|
|
5690
|
+
texts,
|
|
5691
|
+
signal: run3.signal,
|
|
5692
|
+
...this.options.toolNamer ? { toolNamer: this.options.toolNamer } : {}
|
|
5693
|
+
});
|
|
5694
|
+
const queue = [];
|
|
5695
|
+
const alone = cards.filter((card) => card.muted !== true && this.gateAlone.has(card.conversation_id));
|
|
5696
|
+
const chunked = alone.length === 0 ? cards : cards.filter((card) => !alone.includes(card));
|
|
5697
|
+
const split = {
|
|
5698
|
+
token: run3.token,
|
|
5699
|
+
members: new Map(cards.map((card) => [card.conversation_id, card])),
|
|
5700
|
+
queue,
|
|
5701
|
+
skip: /* @__PURE__ */ new Set()
|
|
5702
|
+
};
|
|
5703
|
+
this.activeSplit = split;
|
|
5704
|
+
try {
|
|
5705
|
+
const of = Math.ceil(chunked.length / size);
|
|
5706
|
+
for (let index2 = 1; index2 <= of; index2++) {
|
|
5707
|
+
if (run3.stale()) break;
|
|
5708
|
+
const slice = chunked.slice((index2 - 1) * size, index2 * size);
|
|
5709
|
+
const planned = slice.filter((card) => !this.heldAfterFailure(card));
|
|
5710
|
+
out.heldAfterFailure += slice.length - planned.length;
|
|
5711
|
+
const chunk = this.unheld(planned);
|
|
5712
|
+
out.held += planned.length - chunk.length;
|
|
5713
|
+
if (chunk.length < planned.length) this.heldWhileRunning = true;
|
|
5714
|
+
if (!chunk.some((card) => card.muted !== true)) continue;
|
|
5715
|
+
const startedAt = this.evidence.start();
|
|
5716
|
+
const timedOut = { value: false };
|
|
5717
|
+
const result = await run({
|
|
5718
|
+
tray: { ...tray, conversations: chunk },
|
|
5719
|
+
callModel: this.boundedModelCall(run3, timedOut),
|
|
5720
|
+
client,
|
|
5721
|
+
texts,
|
|
5722
|
+
signal: run3.signal,
|
|
5723
|
+
...this.options.toolNamer ? { toolNamer: this.options.toolNamer } : {}
|
|
5724
|
+
});
|
|
5725
|
+
if (run3.abandoned()) break;
|
|
5726
|
+
if (run3.stale() && result.outcome !== "decided") break;
|
|
5727
|
+
take(result, chunk);
|
|
5728
|
+
const report = {
|
|
5729
|
+
cycle: run3.token,
|
|
5730
|
+
index: index2,
|
|
5731
|
+
of,
|
|
5732
|
+
cards: chunk.length,
|
|
5733
|
+
outcome: result.outcome === "decided" ? "decided" : "fail_closed",
|
|
5734
|
+
undecided: result.undecided.length,
|
|
5735
|
+
failure: result.failure
|
|
5736
|
+
};
|
|
5737
|
+
out.chunks.push(report);
|
|
5738
|
+
if (report.outcome === "decided") this.counts.chunksOk += 1;
|
|
5739
|
+
else this.counts.chunksFailed += 1;
|
|
5740
|
+
this.emit({ type: "gate_chunk", ...report });
|
|
5741
|
+
if (report.outcome === "decided") {
|
|
5742
|
+
const through = decidedBy(result);
|
|
5743
|
+
for (const id of through) this.gateAlone.delete(id);
|
|
5744
|
+
await run3.exclude(() => this.reportDue(this.takeDue(this.completed("gate", through))));
|
|
5745
|
+
continue;
|
|
5746
|
+
}
|
|
5747
|
+
this.failedAttempt("gate", chunk.filter((card) => card.muted !== true).map((card) => card.conversation_id));
|
|
5748
|
+
if (chunk.length === 1) {
|
|
5749
|
+
this.rememberGateAlone(chunk[0].conversation_id);
|
|
5750
|
+
await run3.exclude(() => this.failed(gateFailure(chunk[0], startedAt, 0, timedOut.value, this.now())));
|
|
5751
|
+
} else {
|
|
5752
|
+
for (const card of chunk) if (card.muted !== true) queue.push({ card, reruns: 0 });
|
|
5753
|
+
}
|
|
5754
|
+
}
|
|
5755
|
+
for (const card of alone) {
|
|
5756
|
+
if (!queue.some((item) => item.card.conversation_id === card.conversation_id)) queue.push({ card, reruns: 0 });
|
|
5757
|
+
}
|
|
5758
|
+
let index = 0;
|
|
5759
|
+
let failuresInARow = 0;
|
|
5760
|
+
splitting: for (; ; ) {
|
|
5761
|
+
while (queue.length > 0) {
|
|
5762
|
+
if (run3.stale()) break splitting;
|
|
5763
|
+
const { card, reruns } = queue.shift();
|
|
5764
|
+
const id = card.conversation_id;
|
|
5765
|
+
if (decided.has(id) || split.skip.has(id) || this.heldAfterFailure(card)) continue;
|
|
5766
|
+
if (this.unheld([card]).length === 0) {
|
|
5767
|
+
if (!gated.has(id)) out.held += 1;
|
|
5768
|
+
this.heldWhileRunning = true;
|
|
5769
|
+
continue;
|
|
5770
|
+
}
|
|
5771
|
+
index += 1;
|
|
5772
|
+
const startedAt = this.evidence.start();
|
|
5773
|
+
const timedOut = { value: false };
|
|
5774
|
+
const result = await gateOne(card, timedOut);
|
|
5775
|
+
if (run3.abandoned()) break splitting;
|
|
5776
|
+
if (run3.stale() && result.outcome !== "decided") break splitting;
|
|
5777
|
+
take(result, [card]);
|
|
5778
|
+
const report = {
|
|
5779
|
+
cycle: run3.token,
|
|
5780
|
+
index,
|
|
5781
|
+
of: index + queue.length,
|
|
5782
|
+
cards: 1,
|
|
5783
|
+
outcome: result.outcome === "decided" ? "decided" : "fail_closed",
|
|
5784
|
+
undecided: result.undecided.length,
|
|
5785
|
+
failure: result.failure,
|
|
5786
|
+
split: true,
|
|
5787
|
+
conversationId: id
|
|
5788
|
+
};
|
|
5789
|
+
out.split.push(report);
|
|
5790
|
+
if (report.outcome === "decided") this.counts.splitCallsOk += 1;
|
|
5791
|
+
else this.counts.splitCallsFailed += 1;
|
|
5792
|
+
this.emit({ type: "gate_chunk", ...report });
|
|
5793
|
+
if (report.outcome === "decided") {
|
|
5794
|
+
failuresInARow = 0;
|
|
5795
|
+
this.gateAlone.delete(id);
|
|
5796
|
+
await run3.exclude(() => this.reportDue(this.takeDue(this.completed("gate", decidedBy(result)))));
|
|
5797
|
+
continue;
|
|
5798
|
+
}
|
|
5799
|
+
this.rememberGateAlone(id);
|
|
5800
|
+
this.failedAttempt("gate", [id]);
|
|
5801
|
+
await run3.exclude(() => this.failed(gateFailure(card, startedAt, reruns, timedOut.value, this.now())));
|
|
5802
|
+
failuresInARow += 1;
|
|
5803
|
+
if (failuresInARow >= SPLIT_STOP_AFTER_FAILURES && decided.size === 0) {
|
|
5804
|
+
const ok = this.options.checkRuntime ? await run3.exclude(() => this.runtimeCheck()) : false;
|
|
5805
|
+
if (run3.stale()) break splitting;
|
|
5806
|
+
if (!ok) {
|
|
5807
|
+
out.splitStopped = true;
|
|
5808
|
+
this.failureCounts.splitsStopped += 1;
|
|
5809
|
+
this.emit({
|
|
5810
|
+
type: "split_stopped",
|
|
5811
|
+
cycle: run3.token,
|
|
5812
|
+
remaining: queue.filter((item) => !decided.has(item.card.conversation_id)).length,
|
|
5813
|
+
reason: this.options.checkRuntime ? "runtime_check_failed" : "no_runtime_check"
|
|
5814
|
+
});
|
|
5815
|
+
break splitting;
|
|
5816
|
+
}
|
|
5817
|
+
failuresInARow = 0;
|
|
5818
|
+
}
|
|
5819
|
+
}
|
|
5820
|
+
if (run3.stale()) break;
|
|
5821
|
+
await run3.exclude(() => this.checkParked("gate"));
|
|
5822
|
+
if (queue.length === 0) break;
|
|
5823
|
+
}
|
|
5824
|
+
} finally {
|
|
5825
|
+
if (this.activeSplit === split) this.activeSplit = null;
|
|
5400
5826
|
}
|
|
5827
|
+
out.gated = gated.size;
|
|
5828
|
+
out.undecided = gated.size - decided.size;
|
|
5401
5829
|
return out;
|
|
5402
5830
|
}
|
|
5403
5831
|
/**
|
|
@@ -5430,12 +5858,14 @@ var ReceiveLoop = class {
|
|
|
5430
5858
|
}
|
|
5431
5859
|
}
|
|
5432
5860
|
async singleRunCycle(run3) {
|
|
5861
|
+
const mark = this.pacer.mark();
|
|
5433
5862
|
const tray = await this.readTray();
|
|
5434
5863
|
if (run3.stale()) return this.summary("abandoned");
|
|
5435
5864
|
let cards = null;
|
|
5436
5865
|
let invites = null;
|
|
5437
5866
|
let base = { cards: 0, invites: 0, heldByTurns: 0, truncated: false };
|
|
5438
5867
|
if (tray !== null) {
|
|
5868
|
+
this.forgetCleared(tray, mark);
|
|
5439
5869
|
cards = cardsOf(tray).filter((card) => card.muted !== true).length;
|
|
5440
5870
|
invites = trayInvites(tray).length;
|
|
5441
5871
|
base = { cards: cardsOf(tray).length, invites, heldByTurns: 0, truncated: tray.truncated === true };
|
|
@@ -5445,27 +5875,202 @@ var ReceiveLoop = class {
|
|
|
5445
5875
|
const texts = await this.options.texts.ensure();
|
|
5446
5876
|
if (run3.stale()) return this.summary("abandoned", base);
|
|
5447
5877
|
if (texts === null) return this.summary("texts_missing", { ...base, failure: "the served texts are not available" });
|
|
5878
|
+
if (this.options.conversationRuns !== false && conversationRunTemplate(texts) !== null) {
|
|
5879
|
+
if (tray !== null) return this.conversationRunCycle(run3, tray, texts);
|
|
5880
|
+
if (this.runWork.size > 0 || this.waiting.length > 0) this.heldWhileRunning = true;
|
|
5881
|
+
return this.summary("tray_failed", { ...base, failure: this.lastError });
|
|
5882
|
+
}
|
|
5448
5883
|
const unready = await this.checkReady();
|
|
5449
5884
|
if (run3.stale()) return this.summary("abandoned", base);
|
|
5450
5885
|
if (unready !== null) return this.summary("tools_unavailable", { ...base, failure: unready });
|
|
5451
5886
|
const prompt = cards === 0 && (invites ?? 0) > 0 ? texts.invite_prompt.trim() : (invites ?? 0) > 0 ? `${texts.wake_instruction.trim()}
|
|
5452
5887
|
|
|
5453
5888
|
${texts.invite_prompt.trim()}` : texts.wake_instruction.trim();
|
|
5889
|
+
const kind = cards === 0 && (invites ?? 0) > 0 ? "invite" : "tray";
|
|
5454
5890
|
this.counts.runsStarted += 1;
|
|
5455
|
-
|
|
5891
|
+
const end = await run3.exclude(() => this.callRun({ prompt, texts, cards, invites, signal: this.abort.signal, kind }));
|
|
5892
|
+
this.countRunEnd(end);
|
|
5893
|
+
this.emit({ type: "run", end, kind });
|
|
5894
|
+
const startedWith = tray === null || this.options.dismissIgnoredInvites === false ? [] : trayInvites(tray);
|
|
5895
|
+
if (startedWith.length > 0) await this.track(this.afterRunInvites(startedWith, end));
|
|
5896
|
+
const outcome = end.status === "completed" ? "run_completed" : end.status === "failed" ? "run_failed" : "run_not_started";
|
|
5897
|
+
return this.summary(outcome, { ...base, failure: end.status === "completed" ? null : end.detail ?? null });
|
|
5898
|
+
}
|
|
5899
|
+
/**
|
|
5900
|
+
* Conversation runs (single-run mode, texts with `conversation_run_template`):
|
|
5901
|
+
* one run per owed card (unmuted, with a preview: a notices-only card is not
|
|
5902
|
+
* owed), prompted for that one conversation, and one invite run for the
|
|
5903
|
+
* pending invites. A conversation whose run is running or
|
|
5904
|
+
* waiting for a slot is held out (the tracker), and so is one paced after a
|
|
5905
|
+
* completed run that left its card at the same anchor (D5). Runs take a slot
|
|
5906
|
+
* each (`maxConcurrentTurns`) in the order queued: the cards whose last run
|
|
5907
|
+
* failed go last, after the invite run, so they never hold up the others.
|
|
5908
|
+
* The loop records no verdict for a card: the run decides.
|
|
5909
|
+
*/
|
|
5910
|
+
async conversationRunCycle(run3, tray, texts) {
|
|
5911
|
+
const cards = cardsOf(tray);
|
|
5912
|
+
const invites = trayInvites(tray);
|
|
5913
|
+
const unheld = this.unheld(cards);
|
|
5914
|
+
const heldByTurns = cards.length - unheld.length;
|
|
5915
|
+
if (heldByTurns > 0) this.heldWhileRunning = true;
|
|
5916
|
+
const { open: runCards, paced: pacedCards, heldAfterFailure } = this.paceCards(unheld);
|
|
5917
|
+
const openInvites = invites.filter((invite) => !this.tracker.inviteRunning(invite.invite_id));
|
|
5918
|
+
const owedInvites = openInvites.filter((invite) => !this.pacer.held(`invite:${invite.invite_id}`, null));
|
|
5919
|
+
const paced = pacedCards + (openInvites.length - owedInvites.length);
|
|
5920
|
+
const owedCards = runCards.filter((card) => card.muted !== true && card.preview != null);
|
|
5921
|
+
const base = { cards: cards.length, invites: invites.length, heldByTurns, paced, heldAfterFailure, truncated: tray.truncated === true };
|
|
5922
|
+
if (owedCards.length === 0 && owedInvites.length === 0) return this.summary("nothing_owed", base);
|
|
5923
|
+
const unready = await this.checkReady();
|
|
5924
|
+
if (run3.stale()) return this.summary("abandoned", base);
|
|
5925
|
+
if (unready !== null) return this.summary("tools_unavailable", { ...base, failure: unready });
|
|
5926
|
+
const tally = { turnsStarted: 0, turnsRefused: 0, turnsQueued: 0, heldAfterFailure };
|
|
5927
|
+
const failedLast = owedCards.filter((card) => this.awaitingEvidence.has(card.conversation_id));
|
|
5928
|
+
for (const card of owedCards.filter((card2) => !failedLast.includes(card2))) {
|
|
5929
|
+
if (run3.stale()) break;
|
|
5930
|
+
tally[launchKey(await this.startConversationRun(card, texts))] += 1;
|
|
5931
|
+
}
|
|
5932
|
+
if (owedInvites.length > 0 && !run3.stale()) tally[launchKey(await this.startInviteRun(owedInvites, texts))] += 1;
|
|
5933
|
+
for (const card of failedLast) {
|
|
5934
|
+
if (run3.stale()) break;
|
|
5935
|
+
tally[launchKey(await this.startConversationRun(card, texts))] += 1;
|
|
5936
|
+
}
|
|
5937
|
+
const parts = { ...base, ...tally };
|
|
5938
|
+
if (run3.stale()) return this.summary("abandoned", parts);
|
|
5939
|
+
const launched = tally.turnsStarted + tally.turnsQueued;
|
|
5940
|
+
if (tray.truncated === true) this.readTheRest(launched > 0);
|
|
5941
|
+
return this.summary(launched > 0 ? "runs_started" : "nothing_owed", parts);
|
|
5942
|
+
}
|
|
5943
|
+
/** One conversation run: a slot (or a place in the queue), the run, then what its end says. */
|
|
5944
|
+
async startConversationRun(card, texts, reruns = 0) {
|
|
5945
|
+
const conversationId = card.conversation_id;
|
|
5946
|
+
const anchor = anchorFrom(card);
|
|
5947
|
+
const handle = this.tracker.begin({ kind: "reply", conversationId, anchor });
|
|
5948
|
+
if (handle === null) return "refused";
|
|
5949
|
+
const request = {
|
|
5950
|
+
kind: "conversation",
|
|
5951
|
+
conversationId,
|
|
5952
|
+
prompt: conversationRunPrompt(texts, conversationId),
|
|
5953
|
+
texts,
|
|
5954
|
+
cards: 1,
|
|
5955
|
+
invites: 0,
|
|
5956
|
+
signal: this.abort.signal
|
|
5957
|
+
};
|
|
5958
|
+
return this.launch(handle, async () => {
|
|
5959
|
+
const attempt = { conversationId, source: "run", card, anchor, startedAt: this.evidence.start(), reruns };
|
|
5960
|
+
this.counts.runsStarted += 1;
|
|
5961
|
+
this.trackRun(this.afterConversationRun(handle, attempt, this.callRun(request)));
|
|
5962
|
+
return true;
|
|
5963
|
+
});
|
|
5964
|
+
}
|
|
5965
|
+
async afterConversationRun(handle, attempt, done) {
|
|
5966
|
+
const end = await done;
|
|
5967
|
+
const held = this.tracker.holds(handle);
|
|
5968
|
+
let completion = null;
|
|
5969
|
+
let failure = null;
|
|
5970
|
+
let due = [];
|
|
5971
|
+
let ended = false;
|
|
5456
5972
|
try {
|
|
5457
|
-
|
|
5458
|
-
|
|
5459
|
-
|
|
5973
|
+
this.countRunEnd(end);
|
|
5974
|
+
this.emit({ type: "run", end, kind: "conversation", conversationId: attempt.conversationId });
|
|
5975
|
+
if (!held) return;
|
|
5976
|
+
if (end.status === "completed") {
|
|
5977
|
+
completion = this.completed("run", [attempt.conversationId]);
|
|
5978
|
+
this.pacer.hold(`conversation:${attempt.conversationId}`, attempt.anchor);
|
|
5979
|
+
} else if (end.status === "failed" && !this.stopped) {
|
|
5980
|
+
this.failedAttempt("run", [attempt.conversationId]);
|
|
5981
|
+
if (untouched(attempt.card, await this.readTray())) {
|
|
5982
|
+
failure = { ...attempt, outcome: end.timedOut === true ? "timed_out" : "failed", failedAt: this.now(), holding: handle };
|
|
5983
|
+
}
|
|
5984
|
+
}
|
|
5985
|
+
ended = true;
|
|
5986
|
+
} finally {
|
|
5987
|
+
if (failure === null) this.tracker.end(handle);
|
|
5988
|
+
if (ended && !this.stopped && completion !== null) due = this.takeDue(completion);
|
|
5989
|
+
this.releaseSlot(handle);
|
|
5990
|
+
if (failure === null) this.afterTurnEnded();
|
|
5991
|
+
}
|
|
5992
|
+
if (failure !== null) {
|
|
5993
|
+
try {
|
|
5994
|
+
if (!this.stopped) await this.failed(failure);
|
|
5995
|
+
} finally {
|
|
5996
|
+
this.tracker.end(handle);
|
|
5997
|
+
this.afterTurnEnded();
|
|
5998
|
+
}
|
|
5460
5999
|
}
|
|
6000
|
+
await this.reportDue(due);
|
|
6001
|
+
await this.checkParked("run");
|
|
6002
|
+
}
|
|
6003
|
+
/** The invite run: one run for the pending invites, holding each of them until it ends. */
|
|
6004
|
+
async startInviteRun(invites, texts) {
|
|
6005
|
+
const handles = [];
|
|
6006
|
+
const held = [];
|
|
6007
|
+
for (const invite of invites) {
|
|
6008
|
+
const handle = this.tracker.begin({ kind: "invite", inviteId: invite.invite_id });
|
|
6009
|
+
if (handle === null) continue;
|
|
6010
|
+
handles.push(handle);
|
|
6011
|
+
held.push(invite);
|
|
6012
|
+
}
|
|
6013
|
+
const slot = handles[0];
|
|
6014
|
+
if (slot === void 0) return "refused";
|
|
6015
|
+
const endAll = () => {
|
|
6016
|
+
for (const handle of handles) this.tracker.end(handle);
|
|
6017
|
+
};
|
|
6018
|
+
const request = {
|
|
6019
|
+
kind: "invite",
|
|
6020
|
+
prompt: texts.invite_prompt.trim(),
|
|
6021
|
+
texts,
|
|
6022
|
+
cards: 0,
|
|
6023
|
+
invites: held.length,
|
|
6024
|
+
signal: this.abort.signal
|
|
6025
|
+
};
|
|
6026
|
+
return this.launch(
|
|
6027
|
+
slot,
|
|
6028
|
+
async () => {
|
|
6029
|
+
this.counts.runsStarted += 1;
|
|
6030
|
+
this.trackRun(this.afterInviteRun(slot, held, endAll, this.callRun(request)));
|
|
6031
|
+
return true;
|
|
6032
|
+
},
|
|
6033
|
+
endAll
|
|
6034
|
+
);
|
|
6035
|
+
}
|
|
6036
|
+
async afterInviteRun(slot, invites, endAll, done) {
|
|
6037
|
+
const end = await done;
|
|
6038
|
+
const held = this.tracker.holds(slot);
|
|
6039
|
+
let completion = null;
|
|
6040
|
+
let due = [];
|
|
6041
|
+
let ended = false;
|
|
6042
|
+
try {
|
|
6043
|
+
this.countRunEnd(end);
|
|
6044
|
+
this.emit({ type: "run", end, kind: "invite" });
|
|
6045
|
+
if (!held) return;
|
|
6046
|
+
if (end.status === "completed") {
|
|
6047
|
+
completion = this.completed("invite", []);
|
|
6048
|
+
for (const invite of invites) this.pacer.hold(`invite:${invite.invite_id}`, null);
|
|
6049
|
+
}
|
|
6050
|
+
if (this.options.dismissIgnoredInvites !== false) await this.afterRunInvites(invites, end);
|
|
6051
|
+
ended = true;
|
|
6052
|
+
} finally {
|
|
6053
|
+
endAll();
|
|
6054
|
+
if (ended && !this.stopped && completion !== null) due = this.takeDue(completion);
|
|
6055
|
+
this.releaseSlot(slot);
|
|
6056
|
+
this.afterTurnEnded();
|
|
6057
|
+
}
|
|
6058
|
+
await this.reportDue(due);
|
|
6059
|
+
await this.checkParked("run");
|
|
6060
|
+
}
|
|
6061
|
+
/** The runtime's `startRun`, never rejecting: a throw is a run that did not start. */
|
|
6062
|
+
callRun(request) {
|
|
6063
|
+
return Promise.resolve().then(() => this.options.startRun(request)).catch((err) => ({ status: "not_started", detail: err instanceof Error ? err.message : String(err) }));
|
|
6064
|
+
}
|
|
6065
|
+
countRunEnd(end) {
|
|
5461
6066
|
if (end.status === "completed") this.counts.runsCompleted += 1;
|
|
5462
6067
|
else if (end.status === "failed") this.counts.runsFailed += 1;
|
|
5463
6068
|
else this.counts.runsNotStarted += 1;
|
|
5464
|
-
|
|
5465
|
-
|
|
5466
|
-
|
|
5467
|
-
const
|
|
5468
|
-
|
|
6069
|
+
}
|
|
6070
|
+
/** A run's whole course after its start: `idle()` and `stop()` wait for it. */
|
|
6071
|
+
trackRun(work) {
|
|
6072
|
+
const tracked = work.catch((err) => this.fail(err)).finally(() => this.runWork.delete(tracked));
|
|
6073
|
+
this.runWork.add(tracked);
|
|
5469
6074
|
}
|
|
5470
6075
|
async afterRunInvites(invites, end) {
|
|
5471
6076
|
const completed = end.status === "completed";
|
|
@@ -5476,7 +6081,7 @@ ${texts.invite_prompt.trim()}` : texts.wake_instruction.trim();
|
|
|
5476
6081
|
}
|
|
5477
6082
|
}
|
|
5478
6083
|
// ─── turns (gate-call mode) ─────────────────────────────────────────────
|
|
5479
|
-
async startReplyTurn(reply, texts) {
|
|
6084
|
+
async startReplyTurn(reply, texts, reruns = 0) {
|
|
5480
6085
|
const anchor = anchorFrom(reply.card);
|
|
5481
6086
|
const handle = this.tracker.begin({ kind: "reply", conversationId: reply.conversationId, anchor });
|
|
5482
6087
|
if (handle === null) return "refused";
|
|
@@ -5491,6 +6096,7 @@ ${texts.invite_prompt.trim()}` : texts.wake_instruction.trim();
|
|
|
5491
6096
|
signal: this.abort.signal
|
|
5492
6097
|
};
|
|
5493
6098
|
return this.launch(handle, async () => {
|
|
6099
|
+
const startedAt = this.evidence.start();
|
|
5494
6100
|
const admission = await this.admit(this.options.startTurn, request);
|
|
5495
6101
|
if (!admission.started) {
|
|
5496
6102
|
this.tracker.end(handle);
|
|
@@ -5500,7 +6106,8 @@ ${texts.invite_prompt.trim()}` : texts.wake_instruction.trim();
|
|
|
5500
6106
|
}
|
|
5501
6107
|
this.counts.turnsStarted += 1;
|
|
5502
6108
|
this.emit({ type: "turn_started", request: withoutRuntime(request) });
|
|
5503
|
-
|
|
6109
|
+
const attempt = { conversationId: reply.conversationId, source: "turn", card: reply.card, anchor, startedAt, reruns, reply };
|
|
6110
|
+
void this.afterReplyTurn(handle, attempt, admission.done);
|
|
5504
6111
|
return true;
|
|
5505
6112
|
});
|
|
5506
6113
|
}
|
|
@@ -5540,9 +6147,15 @@ invite_id: ${invite.invite_id}`,
|
|
|
5540
6147
|
* queues behind them instead of taking the slot. A refused admission here
|
|
5541
6148
|
* hands its slot on the same way.
|
|
5542
6149
|
*/
|
|
5543
|
-
async launch(handle, admit) {
|
|
5544
|
-
if (this.
|
|
5545
|
-
this.
|
|
6150
|
+
async launch(handle, admit, drop) {
|
|
6151
|
+
if (this.failureHeld(handle)) {
|
|
6152
|
+
this.tracker.end(handle);
|
|
6153
|
+
drop?.();
|
|
6154
|
+
this.failureCounts.startsHeld += 1;
|
|
6155
|
+
return "held";
|
|
6156
|
+
}
|
|
6157
|
+
if (this.waiting.length > 0 || this.activeTurns >= this.maxTurns || this.awaitsAnswer(handle)) {
|
|
6158
|
+
this.waiting.push({ handle, admit, ...drop ? { drop } : {} });
|
|
5546
6159
|
this.counts.turnsQueued += 1;
|
|
5547
6160
|
if (this.activeTurns < this.maxTurns) void this.pump();
|
|
5548
6161
|
return "queued";
|
|
@@ -5562,30 +6175,63 @@ invite_id: ${invite.invite_id}`,
|
|
|
5562
6175
|
releaseSlot(handle, options = {}) {
|
|
5563
6176
|
if (this.slotted.delete(handle)) this.activeTurns = Math.max(0, this.activeTurns - 1);
|
|
5564
6177
|
const queued = this.waiting.findIndex((w) => w.handle === handle);
|
|
5565
|
-
if (queued >= 0) this.waiting.splice(queued, 1);
|
|
6178
|
+
if (queued >= 0) this.waiting.splice(queued, 1)[0].drop?.();
|
|
5566
6179
|
if (options.pump !== false) void this.pump();
|
|
5567
6180
|
}
|
|
5568
6181
|
async pump() {
|
|
5569
|
-
while (this.
|
|
5570
|
-
const
|
|
6182
|
+
while (this.activeTurns < this.maxTurns && !this.stopped) {
|
|
6183
|
+
const index = this.waiting.findIndex((w) => !this.awaitsAnswer(w.handle));
|
|
6184
|
+
if (index < 0) break;
|
|
6185
|
+
const next = this.waiting.splice(index, 1)[0];
|
|
5571
6186
|
if (!this.tracker.holds(next.handle)) continue;
|
|
6187
|
+
if (this.failureHeld(next.handle)) {
|
|
6188
|
+
this.tracker.end(next.handle);
|
|
6189
|
+
next.drop?.();
|
|
6190
|
+
this.failureCounts.startsHeld += 1;
|
|
6191
|
+
continue;
|
|
6192
|
+
}
|
|
5572
6193
|
await this.admitInSlot(next.handle, next.admit);
|
|
5573
6194
|
}
|
|
5574
6195
|
}
|
|
5575
|
-
|
|
6196
|
+
awaitsAnswer(handle) {
|
|
6197
|
+
return handle.turn.kind === "reply" && this.reporting.has(handle.turn.conversationId);
|
|
6198
|
+
}
|
|
6199
|
+
async afterReplyTurn(handle, attempt, done) {
|
|
6200
|
+
const reply = attempt.reply;
|
|
6201
|
+
let completion = null;
|
|
6202
|
+
let failure = null;
|
|
6203
|
+
let due = [];
|
|
6204
|
+
let ended = false;
|
|
5576
6205
|
try {
|
|
5577
6206
|
const end = await this.tracker.confirmedEnd(handle, done);
|
|
5578
6207
|
if (end === null || !this.tracker.holds(handle)) return;
|
|
5579
|
-
|
|
6208
|
+
if (end.status === "completed") completion = this.completed("turn", [reply.conversationId]);
|
|
6209
|
+
const recorded = await this.track(this.recordReplyOutcome(handle, reply, end));
|
|
6210
|
+
if (end.status === "failed" && !this.stopped) this.failedAttempt("turn", [reply.conversationId]);
|
|
6211
|
+
if (end.status === "failed" && recorded?.freshRead === true && recorded.outcome.reason === "turn_failed") {
|
|
6212
|
+
failure = { ...attempt, outcome: end.timedOut === true ? "timed_out" : "failed", failedAt: this.now(), holding: handle };
|
|
6213
|
+
}
|
|
6214
|
+
ended = true;
|
|
5580
6215
|
} finally {
|
|
5581
|
-
this.tracker.end(handle);
|
|
6216
|
+
if (failure === null) this.tracker.end(handle);
|
|
6217
|
+
if (ended && !this.stopped && completion !== null) due = this.takeDue(completion);
|
|
5582
6218
|
this.releaseSlot(handle);
|
|
5583
|
-
this.afterTurnEnded();
|
|
6219
|
+
if (failure === null) this.afterTurnEnded();
|
|
6220
|
+
}
|
|
6221
|
+
if (failure !== null) {
|
|
6222
|
+
try {
|
|
6223
|
+
if (!this.stopped) await this.track(this.failed(failure));
|
|
6224
|
+
} finally {
|
|
6225
|
+
this.tracker.end(handle);
|
|
6226
|
+
this.afterTurnEnded();
|
|
6227
|
+
}
|
|
5584
6228
|
}
|
|
6229
|
+
await this.track(this.reportDue(due));
|
|
6230
|
+
await this.track(this.checkParked("turn"));
|
|
5585
6231
|
}
|
|
5586
6232
|
async recordReplyOutcome(handle, reply, end) {
|
|
5587
6233
|
const fresh = await this.readTray();
|
|
5588
|
-
if (!this.tracker.holds(handle)) return;
|
|
6234
|
+
if (!this.tracker.holds(handle)) return null;
|
|
5589
6235
|
const anchor = anchorFrom(reply.card);
|
|
5590
6236
|
const key = `${reply.conversationId}|${anchor ?? ""}`;
|
|
5591
6237
|
const silentBefore = this.silent.get(key) ?? 0;
|
|
@@ -5615,6 +6261,7 @@ invite_id: ${invite.invite_id}`,
|
|
|
5615
6261
|
}
|
|
5616
6262
|
}
|
|
5617
6263
|
this.emit({ type: "turn_outcome", outcome, recorded });
|
|
6264
|
+
return { outcome, freshRead: fresh !== null };
|
|
5618
6265
|
}
|
|
5619
6266
|
async afterInviteTurn(handle, invite, done) {
|
|
5620
6267
|
try {
|
|
@@ -5684,6 +6331,332 @@ invite_id: ${invite.invite_id}`,
|
|
|
5684
6331
|
this.wakeFrom("more");
|
|
5685
6332
|
}
|
|
5686
6333
|
}
|
|
6334
|
+
// ─── could not process: failures, evidence, the server's answer ─────────
|
|
6335
|
+
/**
|
|
6336
|
+
* A run, turn or gate call that completed, on the evidence clock. A failure
|
|
6337
|
+
* it clears (the same kind of attempt for the same conversation went
|
|
6338
|
+
* through since) waits no more. Call it the moment the work ended, before
|
|
6339
|
+
* its slot frees, so the next attempt starts after it.
|
|
6340
|
+
*/
|
|
6341
|
+
completed(kind, conversations) {
|
|
6342
|
+
const completion = this.evidence.complete(kind, conversations);
|
|
6343
|
+
if (kind !== "invite") this.failingSince[kind].clear();
|
|
6344
|
+
for (const id of conversations) {
|
|
6345
|
+
const failure = this.awaitingEvidence.get(id);
|
|
6346
|
+
if (failure !== void 0 && clears(completion, failure)) this.awaitingEvidence.delete(id);
|
|
6347
|
+
}
|
|
6348
|
+
return completion;
|
|
6349
|
+
}
|
|
6350
|
+
/**
|
|
6351
|
+
* The failures this completion is the evidence for (each once), taken off
|
|
6352
|
+
* the waiting list. Their conversations are held from now until the
|
|
6353
|
+
* server has answered (`reporting`): no gate, and a run or turn of theirs
|
|
6354
|
+
* waiting for a slot waits for the answer, so it never starts on the input
|
|
6355
|
+
* the answer is about. Take them before the completed work's slot frees.
|
|
6356
|
+
* One that waited past `AWAITING_EVIDENCE_MS` is dropped instead: a
|
|
6357
|
+
* completion that late says little about the runtime when it failed.
|
|
6358
|
+
*/
|
|
6359
|
+
takeDue(completion) {
|
|
6360
|
+
const now2 = this.now();
|
|
6361
|
+
const due = [];
|
|
6362
|
+
for (const failure of [...this.awaitingEvidence.values()]) {
|
|
6363
|
+
if (now2 - failure.failedAt > AWAITING_EVIDENCE_MS) {
|
|
6364
|
+
this.awaitingEvidence.delete(failure.conversationId);
|
|
6365
|
+
continue;
|
|
6366
|
+
}
|
|
6367
|
+
if (evidences(completion, failure)) {
|
|
6368
|
+
this.awaitingEvidence.delete(failure.conversationId);
|
|
6369
|
+
this.holdForReport(failure);
|
|
6370
|
+
due.push(failure);
|
|
6371
|
+
}
|
|
6372
|
+
}
|
|
6373
|
+
return due;
|
|
6374
|
+
}
|
|
6375
|
+
/** Report what `takeDue` took, letting go of each conversation once its answer is in: what waits for it goes on (or was dropped). */
|
|
6376
|
+
async reportDue(due) {
|
|
6377
|
+
const held = [...due];
|
|
6378
|
+
try {
|
|
6379
|
+
while (held.length > 0) {
|
|
6380
|
+
const failure = held[0];
|
|
6381
|
+
await this.reportFailure(failure, "other_run_completed");
|
|
6382
|
+
held.shift();
|
|
6383
|
+
this.letGo([failure]);
|
|
6384
|
+
}
|
|
6385
|
+
} finally {
|
|
6386
|
+
this.letGo(held);
|
|
6387
|
+
}
|
|
6388
|
+
}
|
|
6389
|
+
/** A report for this failure is due: its conversation is held until `letGo` (counted: two reports out hold it until both are answered). */
|
|
6390
|
+
holdForReport(failure) {
|
|
6391
|
+
this.reporting.set(failure.conversationId, (this.reporting.get(failure.conversationId) ?? 0) + 1);
|
|
6392
|
+
}
|
|
6393
|
+
/**
|
|
6394
|
+
* The answers are in (or the report is off): those conversations are no
|
|
6395
|
+
* longer held for it, what waits for them goes on, and a card a cycle left
|
|
6396
|
+
* out meanwhile for a report is read again (only then: a wake for
|
|
6397
|
+
* anything else could spin while a turn runs).
|
|
6398
|
+
*/
|
|
6399
|
+
letGo(failures) {
|
|
6400
|
+
if (failures.length === 0) return;
|
|
6401
|
+
for (const { conversationId } of failures) {
|
|
6402
|
+
const left = (this.reporting.get(conversationId) ?? 0) - 1;
|
|
6403
|
+
if (left > 0) this.reporting.set(conversationId, left);
|
|
6404
|
+
else this.reporting.delete(conversationId);
|
|
6405
|
+
}
|
|
6406
|
+
if (this.stopped) return;
|
|
6407
|
+
if (this.waiting.length > 0) void this.pump();
|
|
6408
|
+
if (this.heldWhileReporting) {
|
|
6409
|
+
this.heldWhileReporting = false;
|
|
6410
|
+
this.wakeFrom("more");
|
|
6411
|
+
}
|
|
6412
|
+
}
|
|
6413
|
+
/** An attempt of this kind failed for these conversations (whatever comes of it): see `failingSince`. */
|
|
6414
|
+
failedAttempt(kind, conversations) {
|
|
6415
|
+
for (const id of conversations) this.failingSince[kind].add(id);
|
|
6416
|
+
}
|
|
6417
|
+
/**
|
|
6418
|
+
* A conversation's run, reply turn or single-card gate call failed. It is
|
|
6419
|
+
* reported only with evidence that the runtime otherwise works:
|
|
6420
|
+
* (a) a later completion of the same kind of work for another conversation
|
|
6421
|
+
* (`evidences`), or (b) `checkRuntime()` passing while no other
|
|
6422
|
+
* conversation of that kind has failed since that kind last completed.
|
|
6423
|
+
* Without (a) it waits: the work already queued runs first (cheap
|
|
6424
|
+
* evidence), and the runtime check is asked only once nothing of that kind
|
|
6425
|
+
* is queued (`checkParked`). It waits at most `AWAITING_EVIDENCE_MS`.
|
|
6426
|
+
* Nothing is reported after `stop()`: a run the stop cut short did not
|
|
6427
|
+
* fail on its conversation.
|
|
6428
|
+
*/
|
|
6429
|
+
async failed(failure) {
|
|
6430
|
+
if (this.stopped) return;
|
|
6431
|
+
this.awaitingEvidence.delete(failure.conversationId);
|
|
6432
|
+
if (this.evidence.since(failure)) {
|
|
6433
|
+
await this.reportFailure(failure, "other_run_completed");
|
|
6434
|
+
return;
|
|
6435
|
+
}
|
|
6436
|
+
if (!this.canReport(failure)) return;
|
|
6437
|
+
this.awaitingEvidence.set(failure.conversationId, failure);
|
|
6438
|
+
while (this.awaitingEvidence.size > SILENT_MEMORY) {
|
|
6439
|
+
const oldest = this.awaitingEvidence.keys().next().value;
|
|
6440
|
+
if (oldest === void 0) break;
|
|
6441
|
+
this.awaitingEvidence.delete(oldest);
|
|
6442
|
+
}
|
|
6443
|
+
if (failure.source !== "gate") await this.checkParked(failure.source);
|
|
6444
|
+
}
|
|
6445
|
+
/**
|
|
6446
|
+
* Evidence (b), once nothing of this kind is queued: ask `checkRuntime()`
|
|
6447
|
+
* about the failure still waiting for evidence, if it is the only
|
|
6448
|
+
* conversation of that kind failing since that kind last completed. Several
|
|
6449
|
+
* failing together is the runtime: none of them is reported on (b), and
|
|
6450
|
+
* they keep waiting for (a). A `timed_out` failure is never reported on
|
|
6451
|
+
* (b): a timeout is ambiguous (a brownout, a hung hook) and the check does
|
|
6452
|
+
* not exercise those paths; it waits for (a). Each failure is asked about
|
|
6453
|
+
* once. A check that fails drops it (a broken runtime reports nothing).
|
|
6454
|
+
*/
|
|
6455
|
+
async checkParked(kind) {
|
|
6456
|
+
if (!this.options.checkRuntime || this.stopped || !this.idleFor(kind)) return;
|
|
6457
|
+
const failing = this.failingSince[kind];
|
|
6458
|
+
const due = [...this.awaitingEvidence.values()].filter(
|
|
6459
|
+
(failure) => failure.source === kind && // A timeout is ambiguous (a brownout, a hung hook) and the runtime
|
|
6460
|
+
// check does not exercise those paths: only (a) can report it.
|
|
6461
|
+
failure.outcome !== "timed_out" && !this.checkedFailures.has(failure) && this.now() - failure.failedAt <= AWAITING_EVIDENCE_MS && [...failing].every((id) => id === failure.conversationId)
|
|
6462
|
+
);
|
|
6463
|
+
if (due.length === 0) return;
|
|
6464
|
+
for (const failure of due) {
|
|
6465
|
+
this.checkedFailures.add(failure);
|
|
6466
|
+
this.holdForReport(failure);
|
|
6467
|
+
}
|
|
6468
|
+
try {
|
|
6469
|
+
const ok = await this.runtimeCheck();
|
|
6470
|
+
for (const failure of due) {
|
|
6471
|
+
if (this.stopped || this.awaitingEvidence.get(failure.conversationId) !== failure) continue;
|
|
6472
|
+
if (!ok) {
|
|
6473
|
+
this.awaitingEvidence.delete(failure.conversationId);
|
|
6474
|
+
continue;
|
|
6475
|
+
}
|
|
6476
|
+
if (![...failing].every((id) => id === failure.conversationId)) continue;
|
|
6477
|
+
this.awaitingEvidence.delete(failure.conversationId);
|
|
6478
|
+
await this.reportFailure(failure, "runtime_check");
|
|
6479
|
+
}
|
|
6480
|
+
} finally {
|
|
6481
|
+
this.letGo(due);
|
|
6482
|
+
}
|
|
6483
|
+
}
|
|
6484
|
+
/** Nothing of this kind is running or queued: the runtime check may be asked now. */
|
|
6485
|
+
idleFor(kind) {
|
|
6486
|
+
if (kind === "gate") return this.activeSplit === null || this.activeSplit.queue.length === 0;
|
|
6487
|
+
if (kind === "turn" && this.activeSplit !== null) return false;
|
|
6488
|
+
return this.activeTurns === 0 && this.waiting.length === 0;
|
|
6489
|
+
}
|
|
6490
|
+
canReport(failure) {
|
|
6491
|
+
return failure.anchor !== null && typeof this.client.conversations?.reportProcessingFailure === "function";
|
|
6492
|
+
}
|
|
6493
|
+
/** `checkRuntime()`, bounded by `RUNTIME_CHECK_TIMEOUT_MS`; a throw or a timeout is not ok. */
|
|
6494
|
+
async runtimeCheck() {
|
|
6495
|
+
const check = this.options.checkRuntime;
|
|
6496
|
+
let timer = null;
|
|
6497
|
+
let answer;
|
|
6498
|
+
try {
|
|
6499
|
+
answer = await Promise.race([
|
|
6500
|
+
Promise.resolve().then(() => check()),
|
|
6501
|
+
new Promise((resolve) => {
|
|
6502
|
+
timer = this._setTimeout(
|
|
6503
|
+
() => resolve({ ok: false, detail: `no answer within ${RUNTIME_CHECK_TIMEOUT_MS} ms` }),
|
|
6504
|
+
RUNTIME_CHECK_TIMEOUT_MS
|
|
6505
|
+
);
|
|
6506
|
+
timer.unref?.();
|
|
6507
|
+
})
|
|
6508
|
+
]);
|
|
6509
|
+
} catch (err) {
|
|
6510
|
+
answer = { ok: false, detail: `the check threw: ${err instanceof Error ? err.message : String(err)}` };
|
|
6511
|
+
} finally {
|
|
6512
|
+
if (timer !== null) this._clearTimeout(timer);
|
|
6513
|
+
}
|
|
6514
|
+
const ok = answer?.ok === true;
|
|
6515
|
+
if (ok) this.failureCounts.runtimeChecksOk += 1;
|
|
6516
|
+
else this.failureCounts.runtimeChecksFailed += 1;
|
|
6517
|
+
this.lastRuntimeCheck = { at: new Date(this.now()).toISOString(), ok, detail: typeof answer?.detail === "string" ? answer.detail : null };
|
|
6518
|
+
return ok;
|
|
6519
|
+
}
|
|
6520
|
+
/** Tell the server, and do what it answers: run the conversation again (`retry`, `hidden`) or stop (`paused`, `none`). */
|
|
6521
|
+
async reportFailure(failure, evidence) {
|
|
6522
|
+
const conversations = this.client.conversations;
|
|
6523
|
+
if (!this.canReport(failure) || conversations === void 0 || this.stopped) return;
|
|
6524
|
+
const base = {
|
|
6525
|
+
conversationId: failure.conversationId,
|
|
6526
|
+
source: failure.source,
|
|
6527
|
+
throughMessageId: failure.anchor,
|
|
6528
|
+
outcome: failure.outcome,
|
|
6529
|
+
evidence
|
|
6530
|
+
};
|
|
6531
|
+
let result;
|
|
6532
|
+
try {
|
|
6533
|
+
result = await conversations.reportProcessingFailure(failure.conversationId, {
|
|
6534
|
+
throughMessageId: failure.anchor,
|
|
6535
|
+
outcome: failure.outcome,
|
|
6536
|
+
evidence
|
|
6537
|
+
});
|
|
6538
|
+
} catch (err) {
|
|
6539
|
+
this.failureCounts.reportErrors += 1;
|
|
6540
|
+
this.fail(err);
|
|
6541
|
+
this.emit({ type: "processing_failure", ...base, result: null, error: err instanceof Error ? err.message : String(err), next: "stop" });
|
|
6542
|
+
return;
|
|
6543
|
+
}
|
|
6544
|
+
this.failureCounts.reported += 1;
|
|
6545
|
+
this.failureCounts[result.action] += 1;
|
|
6546
|
+
let next = "stop";
|
|
6547
|
+
if (result.action === "retry" || result.action === "hidden") {
|
|
6548
|
+
if (failure.reruns >= MAX_RERUNS_PER_CYCLE) {
|
|
6549
|
+
next = "rerun_capped";
|
|
6550
|
+
this.failureCounts.rerunsCapped += 1;
|
|
6551
|
+
} else {
|
|
6552
|
+
next = this.rerun(failure, result.action === "hidden");
|
|
6553
|
+
}
|
|
6554
|
+
}
|
|
6555
|
+
if (result.action === "none" || result.action === "paused") this.holdFailed(failure, result.action);
|
|
6556
|
+
if (next === "stop" || next === "rerun_capped") this.activeSplit?.skip.add(failure.conversationId);
|
|
6557
|
+
this.emit({ type: "processing_failure", ...base, result, next });
|
|
6558
|
+
}
|
|
6559
|
+
/**
|
|
6560
|
+
* The server answered `none` or `paused`: the conversation is held at the
|
|
6561
|
+
* anchor the report read through, so no later cycle (a message elsewhere,
|
|
6562
|
+
* a wake frame, a reconnect, an `unanswered` re-wake, a turn's end) runs it
|
|
6563
|
+
* again on the same input. A newer message moves its anchor and lifts the
|
|
6564
|
+
* hold (`paceCards`); a card read meanwhile leaves the tray, and the hold
|
|
6565
|
+
* with it (`forgetCleared`). What waits for it does not start: a run or
|
|
6566
|
+
* reply turn queued at that anchor (after `paused`, the server's mute, any),
|
|
6567
|
+
* and a reply turn to start again after the gate.
|
|
6568
|
+
*/
|
|
6569
|
+
holdFailed(failure, reason) {
|
|
6570
|
+
const id = failure.conversationId;
|
|
6571
|
+
const anchor = failure.anchor;
|
|
6572
|
+
this.pacer.holdFailed(`${CONVERSATION_KEY}${id}`, anchor, reason);
|
|
6573
|
+
this.dropWaiting(id, reason === "paused" ? null : anchor);
|
|
6574
|
+
this.turnRerunsAfterGate = this.turnRerunsAfterGate.filter(
|
|
6575
|
+
({ reply }) => reply.conversationId !== id || reason !== "paused" && anchorFrom(reply.card) !== anchor
|
|
6576
|
+
);
|
|
6577
|
+
}
|
|
6578
|
+
/**
|
|
6579
|
+
* Run the conversation again after a `retry` or `hidden` answer, after the
|
|
6580
|
+
* others already queued, and say what was done. Its run or reply turn takes
|
|
6581
|
+
* a place at the back of the queue (it reads the conversation itself, so it
|
|
6582
|
+
* sees a hidden span as the note); a reply turn asked for while a gate is
|
|
6583
|
+
* deciding starts after that cycle's own turns. A gate call sees only the
|
|
6584
|
+
* card: after `retry` it joins the end of the running gate's single-card
|
|
6585
|
+
* calls (once, with the card as this cycle read it); after `hidden`, or
|
|
6586
|
+
* when no gate is running for it, the next cycle reads the tray again and
|
|
6587
|
+
* gates it alone. `running`: nothing started, because its next attempt is
|
|
6588
|
+
* under way already (running, queued, or being decided by the gate now).
|
|
6589
|
+
*/
|
|
6590
|
+
rerun(failure, hidden) {
|
|
6591
|
+
if (this.stopped) return "stop";
|
|
6592
|
+
const id = failure.conversationId;
|
|
6593
|
+
const reruns = failure.reruns + 1;
|
|
6594
|
+
this.pacer.release(`conversation:${id}`);
|
|
6595
|
+
if (failure.holding !== void 0) this.tracker.end(failure.holding);
|
|
6596
|
+
const split = this.activeSplit;
|
|
6597
|
+
const current = split?.members.get(id);
|
|
6598
|
+
if (failure.source === "gate") {
|
|
6599
|
+
this.rememberGateAlone(id);
|
|
6600
|
+
if (split !== null && current !== void 0 && !hidden) {
|
|
6601
|
+
split.skip.delete(id);
|
|
6602
|
+
if (split.queue.some((item) => item.card.conversation_id === id)) return "running";
|
|
6603
|
+
split.queue.push({ card: current, reruns });
|
|
6604
|
+
} else {
|
|
6605
|
+
split?.skip.add(id);
|
|
6606
|
+
this.wakeFrom("more");
|
|
6607
|
+
}
|
|
6608
|
+
this.failureCounts.reruns += 1;
|
|
6609
|
+
return "rerun";
|
|
6610
|
+
}
|
|
6611
|
+
if (this.tracker.conversationRunning(id) || current !== void 0) return "running";
|
|
6612
|
+
const texts = this.options.texts.current();
|
|
6613
|
+
this.failureCounts.reruns += 1;
|
|
6614
|
+
if (texts !== null && failure.source === "turn" && failure.reply !== void 0) {
|
|
6615
|
+
if (split !== null) this.turnRerunsAfterGate.push({ reply: failure.reply, reruns });
|
|
6616
|
+
else void this.startReplyTurn(failure.reply, texts, reruns);
|
|
6617
|
+
} else if (texts !== null && failure.source === "run" && conversationRunTemplate(texts) !== null) {
|
|
6618
|
+
void this.startConversationRun(failure.card, texts, reruns);
|
|
6619
|
+
} else {
|
|
6620
|
+
this.wakeFrom("more");
|
|
6621
|
+
}
|
|
6622
|
+
return "rerun";
|
|
6623
|
+
}
|
|
6624
|
+
/** The reply turns asked for while a gate was deciding: started now, after the cycle's own turns. */
|
|
6625
|
+
flushTurnReruns() {
|
|
6626
|
+
const due = this.turnRerunsAfterGate.splice(0);
|
|
6627
|
+
if (this.stopped || due.length === 0) return;
|
|
6628
|
+
const texts = this.options.texts.current();
|
|
6629
|
+
for (const { reply, reruns } of due) {
|
|
6630
|
+
if (this.tracker.conversationRunning(reply.conversationId)) continue;
|
|
6631
|
+
if (texts === null) this.wakeFrom("more");
|
|
6632
|
+
else void this.startReplyTurn(reply, texts, reruns);
|
|
6633
|
+
}
|
|
6634
|
+
}
|
|
6635
|
+
/**
|
|
6636
|
+
* A held conversation: a run or turn still waiting for a slot for it does
|
|
6637
|
+
* not start; with an `anchor`, only one for that anchor (one queued for a
|
|
6638
|
+
* newer message is new input, and goes on).
|
|
6639
|
+
*/
|
|
6640
|
+
dropWaiting(conversationId, anchor) {
|
|
6641
|
+
for (const entry of [...this.waiting]) {
|
|
6642
|
+
const turn = entry.handle.turn;
|
|
6643
|
+
if (turn.kind !== "reply" || turn.conversationId !== conversationId) continue;
|
|
6644
|
+
if (anchor !== null && turn.anchor !== anchor) continue;
|
|
6645
|
+
this.waiting.splice(this.waiting.indexOf(entry), 1);
|
|
6646
|
+
this.tracker.end(entry.handle);
|
|
6647
|
+
entry.drop?.();
|
|
6648
|
+
this.failureCounts.startsHeld += 1;
|
|
6649
|
+
}
|
|
6650
|
+
}
|
|
6651
|
+
rememberGateAlone(conversationId) {
|
|
6652
|
+
this.gateAlone.delete(conversationId);
|
|
6653
|
+
this.gateAlone.add(conversationId);
|
|
6654
|
+
while (this.gateAlone.size > SILENT_MEMORY) {
|
|
6655
|
+
const oldest = this.gateAlone.values().next().value;
|
|
6656
|
+
if (oldest === void 0) break;
|
|
6657
|
+
this.gateAlone.delete(oldest);
|
|
6658
|
+
}
|
|
6659
|
+
}
|
|
5687
6660
|
// ─── small things ───────────────────────────────────────────────────────
|
|
5688
6661
|
/** Recording under way after a turn's confirmed end: `stop()` waits for it. */
|
|
5689
6662
|
track(work) {
|
|
@@ -5721,8 +6694,12 @@ invite_id: ${invite.invite_id}`,
|
|
|
5721
6694
|
turnsQueued: 0,
|
|
5722
6695
|
heldByTurns: 0,
|
|
5723
6696
|
paced: 0,
|
|
6697
|
+
heldAfterFailure: 0,
|
|
5724
6698
|
chunksOk: 0,
|
|
5725
6699
|
chunksFailed: 0,
|
|
6700
|
+
splitCallsOk: 0,
|
|
6701
|
+
splitCallsFailed: 0,
|
|
6702
|
+
splitStopped: false,
|
|
5726
6703
|
truncated: false,
|
|
5727
6704
|
failure: null,
|
|
5728
6705
|
...parts
|
|
@@ -5753,6 +6730,23 @@ conversation_id: ${conversationId}`;
|
|
|
5753
6730
|
|
|
5754
6731
|
conversation_id: ${conversationId}`;
|
|
5755
6732
|
}
|
|
6733
|
+
function conversationRunTemplate(texts) {
|
|
6734
|
+
const raw = texts.conversation_run_template;
|
|
6735
|
+
const template = typeof raw === "string" ? raw.trim() : "";
|
|
6736
|
+
return template.length > 0 ? template : null;
|
|
6737
|
+
}
|
|
6738
|
+
function conversationRunPrompt(texts, conversationId) {
|
|
6739
|
+
const template = conversationRunTemplate(texts);
|
|
6740
|
+
if (template === null) return replyPrompt(texts, conversationId);
|
|
6741
|
+
if (template.includes(REPLY_TURN_PLACEHOLDER)) return template.split(REPLY_TURN_PLACEHOLDER).join(conversationId);
|
|
6742
|
+
return `${template}
|
|
6743
|
+
|
|
6744
|
+
conversation_id: ${conversationId}`;
|
|
6745
|
+
}
|
|
6746
|
+
function untouched(card, fresh) {
|
|
6747
|
+
if (fresh === null) return false;
|
|
6748
|
+
return turnOutcome({ turn: { status: "failed" }, gateCard: card, freshTray: fresh }).reason === "turn_failed";
|
|
6749
|
+
}
|
|
5756
6750
|
function chunkFailure(chunks) {
|
|
5757
6751
|
const failed = chunks.filter((chunk) => chunk.outcome === "fail_closed");
|
|
5758
6752
|
const first = failed[0];
|
|
@@ -5761,6 +6755,34 @@ function chunkFailure(chunks) {
|
|
|
5761
6755
|
if (chunks.length === 1) return reason;
|
|
5762
6756
|
return `${failed.length} of ${chunks.length} gate chunks failed; chunk ${first.index}: ${reason}`;
|
|
5763
6757
|
}
|
|
6758
|
+
function gateFailureText(chunks, split) {
|
|
6759
|
+
const parts = [];
|
|
6760
|
+
const chunked = chunkFailure(chunks);
|
|
6761
|
+
if (chunked !== null) parts.push(chunked);
|
|
6762
|
+
const failed = split.filter((call2) => call2.outcome === "fail_closed");
|
|
6763
|
+
const first = failed[0];
|
|
6764
|
+
if (first !== void 0) {
|
|
6765
|
+
const reason = first.failure ?? "no usable verdicts in the answer";
|
|
6766
|
+
parts.push(`${failed.length} of ${split.length} single-card calls failed; ${first.conversationId ?? "a card"}: ${reason}`);
|
|
6767
|
+
}
|
|
6768
|
+
return parts.length === 0 ? null : parts.join("; ");
|
|
6769
|
+
}
|
|
6770
|
+
function gateFailure(card, startedAt, reruns, timedOut, failedAt) {
|
|
6771
|
+
return {
|
|
6772
|
+
conversationId: card.conversation_id,
|
|
6773
|
+
source: "gate",
|
|
6774
|
+
card,
|
|
6775
|
+
anchor: anchorFrom(card),
|
|
6776
|
+
startedAt,
|
|
6777
|
+
reruns,
|
|
6778
|
+
outcome: timedOut ? "timed_out" : "failed",
|
|
6779
|
+
failedAt
|
|
6780
|
+
};
|
|
6781
|
+
}
|
|
6782
|
+
function decidedBy(result) {
|
|
6783
|
+
const undecided = new Set(result.undecided);
|
|
6784
|
+
return result.verdicts.map((verdict) => verdict.conversation_id).filter((id) => !undecided.has(id));
|
|
6785
|
+
}
|
|
5764
6786
|
function timeoutOption(value, fallback, name) {
|
|
5765
6787
|
if (value === void 0) return fallback;
|
|
5766
6788
|
if (value === 0 || value === Number.POSITIVE_INFINITY) return Number.POSITIVE_INFINITY;
|
|
@@ -5768,6 +6790,7 @@ function timeoutOption(value, fallback, name) {
|
|
|
5768
6790
|
return value;
|
|
5769
6791
|
}
|
|
5770
6792
|
function launchKey(launch) {
|
|
6793
|
+
if (launch === "held") return "heldAfterFailure";
|
|
5771
6794
|
return launch === "started" ? "turnsStarted" : launch === "queued" ? "turnsQueued" : "turnsRefused";
|
|
5772
6795
|
}
|
|
5773
6796
|
function trayInvites(tray) {
|