agentchatme 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +90 -0
- package/README.md +27 -6
- package/dist/index.cjs +858 -78
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +602 -39
- package/dist/index.d.ts +602 -39
- package/dist/index.js +858 -78
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -45,6 +45,11 @@ var ErrorCode = {
|
|
|
45
45
|
RESTRICTED: "RESTRICTED",
|
|
46
46
|
CONVERSATION_NOT_FOUND: "CONVERSATION_NOT_FOUND",
|
|
47
47
|
MESSAGE_NOT_FOUND: "MESSAGE_NOT_FOUND",
|
|
48
|
+
/**
|
|
49
|
+
* 410 when you download the attachment of a message hidden from you
|
|
50
|
+
* because your machine could not process it (`Message.could_not_process`).
|
|
51
|
+
*/
|
|
52
|
+
COULD_NOT_PROCESS: "COULD_NOT_PROCESS",
|
|
48
53
|
GROUP_DELETED: "GROUP_DELETED",
|
|
49
54
|
RATE_LIMITED: "RATE_LIMITED",
|
|
50
55
|
RECIPIENT_BACKLOGGED: "RECIPIENT_BACKLOGGED",
|
|
@@ -306,7 +311,7 @@ function createAgentChatError(body, status, headers) {
|
|
|
306
311
|
}
|
|
307
312
|
|
|
308
313
|
// src/version.ts
|
|
309
|
-
var VERSION = "1.
|
|
314
|
+
var VERSION = "1.5.0" ;
|
|
310
315
|
|
|
311
316
|
// src/runtime.ts
|
|
312
317
|
function detectRuntime() {
|
|
@@ -672,6 +677,7 @@ function parseBacklogWarning(header2) {
|
|
|
672
677
|
if (!Number.isFinite(undeliveredCount) || !Number.isInteger(undeliveredCount)) return null;
|
|
673
678
|
return { recipientHandle, undeliveredCount };
|
|
674
679
|
}
|
|
680
|
+
var PROCESSING_FAILURE_ACTIONS = /* @__PURE__ */ new Set(["retry", "hidden", "paused", "none"]);
|
|
675
681
|
function intHeader(headers, name) {
|
|
676
682
|
const raw = headers?.get?.(name);
|
|
677
683
|
if (typeof raw !== "string" || !/^\d+$/.test(raw.trim())) return null;
|
|
@@ -763,7 +769,8 @@ var AgentChatClient = class _AgentChatClient {
|
|
|
763
769
|
record: (conversationId, opts) => this.recordDecision(conversationId, opts)
|
|
764
770
|
};
|
|
765
771
|
this.conversations = {
|
|
766
|
-
markRead: (conversationId, throughMessageId, opts) => this.markConversationRead(conversationId, throughMessageId, opts)
|
|
772
|
+
markRead: (conversationId, throughMessageId, opts) => this.markConversationRead(conversationId, throughMessageId, opts),
|
|
773
|
+
reportProcessingFailure: (conversationId, opts) => this.reportProcessingFailure(conversationId, opts)
|
|
767
774
|
};
|
|
768
775
|
this.texts = {
|
|
769
776
|
get: (opts) => this.getTexts(opts)
|
|
@@ -835,6 +842,36 @@ var AgentChatClient = class _AgentChatClient {
|
|
|
835
842
|
options
|
|
836
843
|
);
|
|
837
844
|
}
|
|
845
|
+
async reportProcessingFailure(conversationId, options) {
|
|
846
|
+
const body = {
|
|
847
|
+
through_message_id: options.throughMessageId,
|
|
848
|
+
outcome: options.outcome,
|
|
849
|
+
evidence: options.evidence
|
|
850
|
+
};
|
|
851
|
+
let res;
|
|
852
|
+
try {
|
|
853
|
+
res = await this.post(
|
|
854
|
+
`/v1/conversations/${encodeURIComponent(conversationId)}/processing-failures`,
|
|
855
|
+
body,
|
|
856
|
+
// One report, one count: a retried request carries the same key.
|
|
857
|
+
{ ...options, idempotencyKey: options.idempotencyKey ?? generateClientMsgId() }
|
|
858
|
+
);
|
|
859
|
+
} catch (err) {
|
|
860
|
+
if (err instanceof AgentChatError && err.status === 404 && err.code !== "CONVERSATION_NOT_FOUND" && err.code !== "MESSAGE_NOT_FOUND") {
|
|
861
|
+
return { action: "none", attempts: 0, hidden_message_ids: [], paused_until: null };
|
|
862
|
+
}
|
|
863
|
+
throw err;
|
|
864
|
+
}
|
|
865
|
+
const r = res !== null && typeof res === "object" ? res : {};
|
|
866
|
+
const action = PROCESSING_FAILURE_ACTIONS.has(r.action) ? r.action : "none";
|
|
867
|
+
return {
|
|
868
|
+
action,
|
|
869
|
+
attempts: typeof r.attempts === "number" && Number.isFinite(r.attempts) ? r.attempts : 0,
|
|
870
|
+
hidden_message_ids: Array.isArray(r.hidden_message_ids) ? r.hidden_message_ids.filter((id) => typeof id === "string") : [],
|
|
871
|
+
...typeof r.hidden_count === "number" && Number.isFinite(r.hidden_count) ? { hidden_count: r.hidden_count } : {},
|
|
872
|
+
paused_until: typeof r.paused_until === "string" ? r.paused_until : null
|
|
873
|
+
};
|
|
874
|
+
}
|
|
838
875
|
getTexts(options) {
|
|
839
876
|
const params = new URLSearchParams({ runtime: options.runtime });
|
|
840
877
|
return this.get(`/v1/texts?${params.toString()}`, options);
|
|
@@ -1099,24 +1136,48 @@ var AgentChatClient = class _AgentChatClient {
|
|
|
1099
1136
|
* most one:
|
|
1100
1137
|
* - `beforeSeq` — backwards scrollback (rows with seq < N, newest first)
|
|
1101
1138
|
* - `afterSeq` — forwards gap-fill (rows with seq > N, oldest first)
|
|
1139
|
+
* - `afterMessageId` — forwards from a message you hold (instead of `afterSeq`)
|
|
1102
1140
|
* - `aroundMessageId` — backwards window ending at that exact message
|
|
1103
1141
|
*
|
|
1104
1142
|
* `afterSeq` is the path `RealtimeClient` uses for in-order recovery
|
|
1105
1143
|
* when a per-conversation seq gap is detected. Application code usually
|
|
1106
1144
|
* only needs `beforeSeq` for normal pagination.
|
|
1145
|
+
*
|
|
1146
|
+
* With `waitMs` > 0 (and `afterSeq` or `afterMessageId`) this is the
|
|
1147
|
+
* scoped wait: the server holds the request until a message from someone
|
|
1148
|
+
* else arrives after the cursor, or answers `[]` after `waitMs`. One
|
|
1149
|
+
* conversation only, no sync ack. `getMessagesPage` also returns how long
|
|
1150
|
+
* the server held it.
|
|
1107
1151
|
*/
|
|
1108
|
-
getMessages(conversationId, options) {
|
|
1152
|
+
async getMessages(conversationId, options) {
|
|
1153
|
+
return (await this.getMessagesPage(conversationId, options)).messages;
|
|
1154
|
+
}
|
|
1155
|
+
/** `getMessages()` with the response facts a waiter needs: how long the server held a scoped wait (`waitedMs`). */
|
|
1156
|
+
async getMessagesPage(conversationId, options) {
|
|
1109
1157
|
const params = new URLSearchParams();
|
|
1110
1158
|
params.set("limit", String(options?.limit ?? 50));
|
|
1111
1159
|
if (options?.beforeSeq !== void 0) params.set("before_seq", String(options.beforeSeq));
|
|
1112
1160
|
if (options?.afterSeq !== void 0) params.set("after_seq", String(options.afterSeq));
|
|
1161
|
+
if (options?.afterMessageId !== void 0) params.set("after_message_id", options.afterMessageId);
|
|
1113
1162
|
if (options?.aroundMessageId !== void 0) {
|
|
1114
1163
|
params.set("around_message_id", options.aroundMessageId);
|
|
1115
1164
|
}
|
|
1116
|
-
|
|
1165
|
+
const waitMs = Math.floor(options?.waitMs ?? 0);
|
|
1166
|
+
if (waitMs > 0) params.set("wait_ms", String(waitMs));
|
|
1167
|
+
const timeoutMs = options?.timeoutMs !== void 0 ? options.timeoutMs : waitMs > 0 ? waitMs + INBOX_WAIT_TIMEOUT_GRACE_MS : void 0;
|
|
1168
|
+
const res = await this.http.request(
|
|
1169
|
+
"GET",
|
|
1117
1170
|
`/v1/messages/${encodeURIComponent(conversationId)}?${params.toString()}`,
|
|
1118
|
-
options
|
|
1171
|
+
this.toRequestOpts({ ...options, timeoutMs })
|
|
1119
1172
|
);
|
|
1173
|
+
const waitedMs = intHeader(res.headers, "x-agentchat-waited-ms");
|
|
1174
|
+
return {
|
|
1175
|
+
messages: res.data,
|
|
1176
|
+
waitedMs,
|
|
1177
|
+
scopedWaitSupported: waitMs > 0 ? waitedMs !== null : null,
|
|
1178
|
+
requestId: res.requestId,
|
|
1179
|
+
headers: res.headers
|
|
1180
|
+
};
|
|
1120
1181
|
}
|
|
1121
1182
|
/**
|
|
1122
1183
|
* Hide a message from your own view (hide-for-me). Either side of the
|
|
@@ -1408,11 +1469,13 @@ var AgentChatClient = class _AgentChatClient {
|
|
|
1408
1469
|
}
|
|
1409
1470
|
// ─── Mutes ────────────────────────────────────────────────────────────────
|
|
1410
1471
|
//
|
|
1411
|
-
// Mute
|
|
1412
|
-
//
|
|
1413
|
-
//
|
|
1414
|
-
//
|
|
1415
|
-
//
|
|
1472
|
+
// Mute stops the wake for a specific agent or conversation without
|
|
1473
|
+
// blocking or leaving: no real-time WebSocket push and no wake webhook.
|
|
1474
|
+
// Nothing is hidden — the conversation stays on the inbox tray
|
|
1475
|
+
// (`muted: true`), envelopes still land in `/v1/messages/sync`, and the
|
|
1476
|
+
// unread counter still bumps. A muted message reaches the machine at its
|
|
1477
|
+
// next sync (every reconnect) or when the agent opens the conversation;
|
|
1478
|
+
// until then the sender sees it as `stored`, not `delivered`.
|
|
1416
1479
|
//
|
|
1417
1480
|
// All mute APIs are idempotent:
|
|
1418
1481
|
// - Re-muting with a different `mutedUntil` refreshes the expiry.
|
|
@@ -3560,6 +3623,7 @@ __export(integration_exports, {
|
|
|
3560
3623
|
anchor: () => anchor_exports,
|
|
3561
3624
|
applyInviteOutcome: () => applyInviteOutcome,
|
|
3562
3625
|
applyOutcome: () => applyOutcome,
|
|
3626
|
+
conversationRunPrompt: () => conversationRunPrompt,
|
|
3563
3627
|
hostedMcpUrl: () => hostedMcpUrl,
|
|
3564
3628
|
inviteOutcome: () => inviteOutcome,
|
|
3565
3629
|
isTexts: () => isTexts,
|
|
@@ -4804,6 +4868,11 @@ var CYCLE_TIMEOUT_MS = 10 * 6e4;
|
|
|
4804
4868
|
var GATE_CHUNK_SIZE = 10;
|
|
4805
4869
|
var REPLY_TURN_PLACEHOLDER = "<conversation_id>";
|
|
4806
4870
|
var RECENT_CYCLES = 20;
|
|
4871
|
+
var MAX_RERUNS_PER_CYCLE = 3;
|
|
4872
|
+
var SPLIT_STOP_AFTER_FAILURES = 2;
|
|
4873
|
+
var RUNTIME_CHECK_TIMEOUT_MS = 6e4;
|
|
4874
|
+
var AWAITING_EVIDENCE_MS = 10 * 6e4;
|
|
4875
|
+
var EVIDENCE_MEMORY = 100;
|
|
4807
4876
|
var Pacer = class {
|
|
4808
4877
|
constructor(paceMs, now2) {
|
|
4809
4878
|
this.paceMs = paceMs;
|
|
@@ -4837,6 +4906,29 @@ var Pacer = class {
|
|
|
4837
4906
|
for (const key of [...this.until.keys()]) if (key.startsWith("invite:")) this.until.delete(key);
|
|
4838
4907
|
}
|
|
4839
4908
|
};
|
|
4909
|
+
function clears(completion, failure) {
|
|
4910
|
+
return completion.kind === failure.source && completion.conversations.has(failure.conversationId);
|
|
4911
|
+
}
|
|
4912
|
+
function evidences(completion, failure) {
|
|
4913
|
+
return completion.kind === failure.source && completion.at > failure.startedAt && completion.conversations.size > 0 && !completion.conversations.has(failure.conversationId);
|
|
4914
|
+
}
|
|
4915
|
+
var EvidenceClock = class {
|
|
4916
|
+
last = 0;
|
|
4917
|
+
completions = [];
|
|
4918
|
+
start() {
|
|
4919
|
+
return ++this.last;
|
|
4920
|
+
}
|
|
4921
|
+
complete(kind, conversations) {
|
|
4922
|
+
const entry = { at: ++this.last, kind, conversations: new Set(conversations) };
|
|
4923
|
+
this.completions.push(entry);
|
|
4924
|
+
if (this.completions.length > EVIDENCE_MEMORY) this.completions.shift();
|
|
4925
|
+
return entry;
|
|
4926
|
+
}
|
|
4927
|
+
/** A completion since the failed attempt started that is evidence for it (`evidences`). */
|
|
4928
|
+
since(failure) {
|
|
4929
|
+
return this.completions.some((c) => evidences(c, failure));
|
|
4930
|
+
}
|
|
4931
|
+
};
|
|
4840
4932
|
var ReceiveLoop = class {
|
|
4841
4933
|
options;
|
|
4842
4934
|
client;
|
|
@@ -4854,7 +4946,51 @@ var ReceiveLoop = class {
|
|
|
4854
4946
|
/** When a single-run invite's silent run was counted (D5 pacing in single-run mode). */
|
|
4855
4947
|
silentAt = /* @__PURE__ */ new Map();
|
|
4856
4948
|
paceMs;
|
|
4949
|
+
/** Turns and runs waiting for a slot; `drop` ends what else the entry holds (an invite run's other invites). */
|
|
4857
4950
|
waiting = [];
|
|
4951
|
+
/** Conversation and invite runs started and not yet concluded: `idle()` and `stop()` wait for them. */
|
|
4952
|
+
runWork = /* @__PURE__ */ new Set();
|
|
4953
|
+
/** Could not process: the order of attempts and completions. */
|
|
4954
|
+
evidence = new EvidenceClock();
|
|
4955
|
+
/** Could not process: failures waiting for a completion (the evidence a report needs), by conversation. */
|
|
4956
|
+
awaitingEvidence = /* @__PURE__ */ new Map();
|
|
4957
|
+
/** Waiting failures the runtime check has already been asked about (once each). */
|
|
4958
|
+
checkedFailures = /* @__PURE__ */ new WeakSet();
|
|
4959
|
+
/**
|
|
4960
|
+
* Per kind of work, the conversations whose attempt failed since that kind
|
|
4961
|
+
* last completed. Two or more: several failing together is the runtime, so
|
|
4962
|
+
* a passing `checkRuntime()` is no evidence for any of them.
|
|
4963
|
+
*/
|
|
4964
|
+
failingSince = { gate: /* @__PURE__ */ new Set(), turn: /* @__PURE__ */ new Set(), run: /* @__PURE__ */ new Set() };
|
|
4965
|
+
/** gateCall: conversations whose last gate call failed; gated alone, after the others. */
|
|
4966
|
+
gateAlone = /* @__PURE__ */ new Set();
|
|
4967
|
+
/**
|
|
4968
|
+
* gateCall: the running cycle's gate: its cards (`members`) and its
|
|
4969
|
+
* single-card queue. A `retry` answer for one of its cards joins the queue;
|
|
4970
|
+
* a `hidden` one skips the card here (its card, read before the hide, is
|
|
4971
|
+
* stale) and gates it in the next cycle.
|
|
4972
|
+
*/
|
|
4973
|
+
activeSplit = null;
|
|
4974
|
+
/**
|
|
4975
|
+
* gateCall: reply turns to start again on a `retry` / `hidden` answer that
|
|
4976
|
+
* came while a gate was deciding: they start after that cycle's own turns
|
|
4977
|
+
* (no turn starts while cards of a cycle still wait for their gate call).
|
|
4978
|
+
*/
|
|
4979
|
+
turnRerunsAfterGate = [];
|
|
4980
|
+
lastRuntimeCheck = null;
|
|
4981
|
+
failureCounts = {
|
|
4982
|
+
reported: 0,
|
|
4983
|
+
retry: 0,
|
|
4984
|
+
hidden: 0,
|
|
4985
|
+
paused: 0,
|
|
4986
|
+
none: 0,
|
|
4987
|
+
reportErrors: 0,
|
|
4988
|
+
reruns: 0,
|
|
4989
|
+
rerunsCapped: 0,
|
|
4990
|
+
runtimeChecksOk: 0,
|
|
4991
|
+
runtimeChecksFailed: 0,
|
|
4992
|
+
splitsStopped: 0
|
|
4993
|
+
};
|
|
4858
4994
|
/** Conversations whose verdict a gate call is recording right now (an abandoned cycle's included). */
|
|
4859
4995
|
recordsInFlight = /* @__PURE__ */ new Set();
|
|
4860
4996
|
/** Turns holding a slot (admitted, not yet ended or released). */
|
|
@@ -4889,6 +5025,8 @@ var ReceiveLoop = class {
|
|
|
4889
5025
|
runsNotStarted: 0,
|
|
4890
5026
|
chunksOk: 0,
|
|
4891
5027
|
chunksFailed: 0,
|
|
5028
|
+
splitCallsOk: 0,
|
|
5029
|
+
splitCallsFailed: 0,
|
|
4892
5030
|
outcomeRecordFailures: 0
|
|
4893
5031
|
};
|
|
4894
5032
|
outcomes = {};
|
|
@@ -4986,7 +5124,10 @@ var ReceiveLoop = class {
|
|
|
4986
5124
|
this.clearMoreTimer();
|
|
4987
5125
|
this.abort.abort();
|
|
4988
5126
|
this.cycleAbort?.abort();
|
|
4989
|
-
for (const { handle } of this.waiting.splice(0))
|
|
5127
|
+
for (const { handle, drop } of this.waiting.splice(0)) {
|
|
5128
|
+
this.tracker.end(handle);
|
|
5129
|
+
drop?.();
|
|
5130
|
+
}
|
|
4990
5131
|
for (const off of this.unsubscribe.splice(0)) {
|
|
4991
5132
|
try {
|
|
4992
5133
|
off();
|
|
@@ -4998,11 +5139,21 @@ var ReceiveLoop = class {
|
|
|
4998
5139
|
} catch {
|
|
4999
5140
|
}
|
|
5000
5141
|
await this.coalescer.idle();
|
|
5001
|
-
await Promise.allSettled([...this.pendingWork]);
|
|
5142
|
+
await Promise.allSettled([...this.pendingWork, ...this.runWork]);
|
|
5002
5143
|
}
|
|
5003
|
-
/**
|
|
5004
|
-
|
|
5005
|
-
|
|
5144
|
+
/**
|
|
5145
|
+
* Resolves when the running cycle has finished (tests, graceful stops) and,
|
|
5146
|
+
* in single-run mode, no run is running or waiting for a slot (conversation
|
|
5147
|
+
* runs go on after the cycle that started them). Gate-call turns are not
|
|
5148
|
+
* waited for.
|
|
5149
|
+
*/
|
|
5150
|
+
async idle() {
|
|
5151
|
+
for (; ; ) {
|
|
5152
|
+
await this.coalescer.idle();
|
|
5153
|
+
const runs = [...this.runWork];
|
|
5154
|
+
if (runs.length === 0) return;
|
|
5155
|
+
await Promise.allSettled(runs);
|
|
5156
|
+
}
|
|
5006
5157
|
}
|
|
5007
5158
|
/** Let go of a turn whose end the runtime will never confirm (an operator's call). Nothing is recorded for it. */
|
|
5008
5159
|
releaseTurn(key) {
|
|
@@ -5042,6 +5193,11 @@ var ReceiveLoop = class {
|
|
|
5042
5193
|
turnsWaiting: this.waiting.length,
|
|
5043
5194
|
outcomes: { ...this.outcomes }
|
|
5044
5195
|
},
|
|
5196
|
+
processingFailures: {
|
|
5197
|
+
...this.failureCounts,
|
|
5198
|
+
awaitingEvidence: this.awaitingEvidence.size,
|
|
5199
|
+
lastRuntimeCheck: this.lastRuntimeCheck === null ? null : { ...this.lastRuntimeCheck }
|
|
5200
|
+
},
|
|
5045
5201
|
lastError: this.lastError
|
|
5046
5202
|
};
|
|
5047
5203
|
}
|
|
@@ -5113,6 +5269,8 @@ var ReceiveLoop = class {
|
|
|
5113
5269
|
const work = (async () => {
|
|
5114
5270
|
const summary = this.options.mode === "gateCall" ? await this.gateCallCycle(run3) : await this.singleRunCycle(run3);
|
|
5115
5271
|
if (!run3.abandoned()) this.finishCycle(summary);
|
|
5272
|
+
this.flushTurnReruns();
|
|
5273
|
+
void this.track(this.checkParked(this.options.mode === "gateCall" ? "turn" : "run"));
|
|
5116
5274
|
})();
|
|
5117
5275
|
let timer = null;
|
|
5118
5276
|
const overdue = new Promise((resolve) => {
|
|
@@ -5180,12 +5338,13 @@ var ReceiveLoop = class {
|
|
|
5180
5338
|
* cycle that is making progress is not abandoned for it. With the deadline
|
|
5181
5339
|
* off the watchdog is the calls' only bound, so they count.
|
|
5182
5340
|
*/
|
|
5183
|
-
boundedModelCall(run3) {
|
|
5184
|
-
const call2 = this.deadlineModelCall(run3);
|
|
5341
|
+
boundedModelCall(run3, timedOut) {
|
|
5342
|
+
const call2 = this.deadlineModelCall(run3, timedOut);
|
|
5185
5343
|
if (!Number.isFinite(this.gateCallTimeoutMs)) return call2;
|
|
5186
5344
|
return (prompt, options) => run3.exclude(() => call2(prompt, options));
|
|
5187
5345
|
}
|
|
5188
|
-
|
|
5346
|
+
/** `timedOut` is set when a call runs past its deadline (a could-not-process report then says `timed_out`). */
|
|
5347
|
+
deadlineModelCall(run3, timedOut) {
|
|
5189
5348
|
return (prompt) => new Promise((resolve, reject) => {
|
|
5190
5349
|
const controller = new AbortController();
|
|
5191
5350
|
const abort = () => {
|
|
@@ -5199,6 +5358,7 @@ var ReceiveLoop = class {
|
|
|
5199
5358
|
run3.signal.addEventListener("abort", abort, { once: true });
|
|
5200
5359
|
const timer = Number.isFinite(this.gateCallTimeoutMs) ? this._setTimeout(() => {
|
|
5201
5360
|
run3.signal.removeEventListener("abort", abort);
|
|
5361
|
+
if (timedOut) timedOut.value = true;
|
|
5202
5362
|
controller.abort();
|
|
5203
5363
|
reject(new Error(`no answer within ${this.gateCallTimeoutMs} ms`));
|
|
5204
5364
|
}, this.gateCallTimeoutMs) : null;
|
|
@@ -5280,9 +5440,12 @@ var ReceiveLoop = class {
|
|
|
5280
5440
|
turnsRefused: 0,
|
|
5281
5441
|
turnsQueued: 0,
|
|
5282
5442
|
chunksOk: 0,
|
|
5283
|
-
chunksFailed: 0
|
|
5443
|
+
chunksFailed: 0,
|
|
5444
|
+
splitCallsOk: 0,
|
|
5445
|
+
splitCallsFailed: 0
|
|
5284
5446
|
};
|
|
5285
5447
|
let heldDuringGate = 0;
|
|
5448
|
+
let splitStopped = false;
|
|
5286
5449
|
let outcome = "invites_only";
|
|
5287
5450
|
let failure = null;
|
|
5288
5451
|
if (owedCards.length > 0) {
|
|
@@ -5297,14 +5460,17 @@ var ReceiveLoop = class {
|
|
|
5297
5460
|
tally.recordFailures = gate.recordFailures;
|
|
5298
5461
|
tally.chunksOk = gate.chunks.filter((c) => c.outcome === "decided").length;
|
|
5299
5462
|
tally.chunksFailed = gate.chunks.length - tally.chunksOk;
|
|
5300
|
-
|
|
5301
|
-
|
|
5463
|
+
tally.splitCallsOk = gate.split.filter((c) => c.outcome === "decided").length;
|
|
5464
|
+
tally.splitCallsFailed = gate.split.length - tally.splitCallsOk;
|
|
5465
|
+
splitStopped = gate.splitStopped;
|
|
5466
|
+
failure = gateFailureText(gate.chunks, gate.split);
|
|
5467
|
+
if (tally.chunksOk + tally.splitCallsOk > 0) {
|
|
5302
5468
|
outcome = "decided";
|
|
5303
5469
|
for (const reply of gate.reply) {
|
|
5304
5470
|
if (run3.stale()) break;
|
|
5305
5471
|
tally[launchKey(await run3.exclude(() => this.startReplyTurn(reply, texts)))] += 1;
|
|
5306
5472
|
}
|
|
5307
|
-
} else if (tally.chunksFailed > 0) {
|
|
5473
|
+
} else if (tally.chunksFailed + tally.splitCallsFailed > 0) {
|
|
5308
5474
|
outcome = "fail_closed";
|
|
5309
5475
|
} else if (owedInvites.length === 0 && !run3.stale()) {
|
|
5310
5476
|
return this.summary("nothing_owed", { ...base, heldByTurns: heldByTurns + heldDuringGate });
|
|
@@ -5314,7 +5480,7 @@ var ReceiveLoop = class {
|
|
|
5314
5480
|
if (run3.stale()) break;
|
|
5315
5481
|
tally[launchKey(await run3.exclude(() => this.startInviteTurn(invite, texts)))] += 1;
|
|
5316
5482
|
}
|
|
5317
|
-
const parts = { ...base, heldByTurns: heldByTurns + heldDuringGate, ...tally };
|
|
5483
|
+
const parts = { ...base, heldByTurns: heldByTurns + heldDuringGate, ...tally, splitStopped };
|
|
5318
5484
|
if (run3.stale()) return this.summary("abandoned", parts);
|
|
5319
5485
|
if (tray.truncated === true && outcome === "decided") {
|
|
5320
5486
|
this.readTheRest(tally.recorded > 0 || tally.turnsStarted > 0);
|
|
@@ -5342,11 +5508,21 @@ var ReceiveLoop = class {
|
|
|
5342
5508
|
* cards only (or none) is not called. Stops once the cycle is stale; a call
|
|
5343
5509
|
* a stop cut short is not counted as a failed chunk, and a cycle the
|
|
5344
5510
|
* watchdog abandoned reports and counts no chunk after its summary.
|
|
5511
|
+
*
|
|
5512
|
+
* The split (could not process): every owed card of a chunk of more than
|
|
5513
|
+
* one card that failed closed is then gated again alone, one call per card,
|
|
5514
|
+
* in tray order, before any turn starts, so one card the model cannot
|
|
5515
|
+
* handle fails only its own call. A card whose last gate call failed is
|
|
5516
|
+
* kept out of the chunks and gated alone after them. A single-card call that
|
|
5517
|
+
* fails is the card's failure (reported with evidence; a `retry` answer
|
|
5518
|
+
* gates it once more at the end of the queue). When the queue is empty, a
|
|
5519
|
+
* failure still waiting gets the runtime check. After
|
|
5520
|
+
* `SPLIT_STOP_AFTER_FAILURES` single-card calls in a row failed with nothing
|
|
5521
|
+
* decided in the cycle, the split goes on only if `checkRuntime()` passes;
|
|
5522
|
+
* otherwise it stops (`split_stopped`).
|
|
5345
5523
|
*/
|
|
5346
5524
|
async gateInChunks(run3, tray, cards, texts) {
|
|
5347
5525
|
const size = this.gateChunkSize;
|
|
5348
|
-
const of = Math.ceil(cards.length / size);
|
|
5349
|
-
const callModel = this.boundedModelCall(run3);
|
|
5350
5526
|
const client = this.recordingClient(run3);
|
|
5351
5527
|
const out = {
|
|
5352
5528
|
gated: 0,
|
|
@@ -5356,48 +5532,165 @@ var ReceiveLoop = class {
|
|
|
5356
5532
|
undecided: 0,
|
|
5357
5533
|
recorded: 0,
|
|
5358
5534
|
recordFailures: 0,
|
|
5359
|
-
chunks: []
|
|
5535
|
+
chunks: [],
|
|
5536
|
+
split: [],
|
|
5537
|
+
splitStopped: false
|
|
5360
5538
|
};
|
|
5361
|
-
|
|
5362
|
-
|
|
5363
|
-
|
|
5364
|
-
const
|
|
5365
|
-
out.held += planned.length - chunk.length;
|
|
5366
|
-
if (chunk.length < planned.length) this.heldWhileRunning = true;
|
|
5367
|
-
if (!chunk.some((card) => card.muted !== true)) continue;
|
|
5368
|
-
const result = await run({
|
|
5369
|
-
tray: { ...tray, conversations: chunk },
|
|
5370
|
-
callModel,
|
|
5371
|
-
client,
|
|
5372
|
-
texts,
|
|
5373
|
-
signal: run3.signal,
|
|
5374
|
-
...this.options.toolNamer ? { toolNamer: this.options.toolNamer } : {}
|
|
5375
|
-
});
|
|
5376
|
-
if (run3.abandoned()) break;
|
|
5377
|
-
if (run3.stale() && result.outcome !== "decided") break;
|
|
5378
|
-
out.gated += chunk.length;
|
|
5539
|
+
const gated = /* @__PURE__ */ new Set();
|
|
5540
|
+
const decided = /* @__PURE__ */ new Set();
|
|
5541
|
+
const take = (result, sent) => {
|
|
5542
|
+
for (const card of sent) gated.add(card.conversation_id);
|
|
5379
5543
|
const undecided = new Set(result.undecided);
|
|
5380
5544
|
for (const verdict of result.verdicts) {
|
|
5381
|
-
if (
|
|
5545
|
+
if (undecided.has(verdict.conversation_id) || decided.has(verdict.conversation_id)) continue;
|
|
5546
|
+
decided.add(verdict.conversation_id);
|
|
5547
|
+
out.verdicts[verdict.decision] += 1;
|
|
5382
5548
|
}
|
|
5383
5549
|
out.reply.push(...result.reply);
|
|
5384
|
-
out.undecided += result.undecided.length;
|
|
5385
5550
|
out.recorded += result.recorded;
|
|
5386
5551
|
out.recordFailures += result.recordFailures.length;
|
|
5387
|
-
|
|
5388
|
-
|
|
5389
|
-
|
|
5390
|
-
|
|
5391
|
-
|
|
5392
|
-
|
|
5393
|
-
|
|
5394
|
-
|
|
5395
|
-
|
|
5396
|
-
|
|
5397
|
-
|
|
5398
|
-
|
|
5399
|
-
|
|
5552
|
+
};
|
|
5553
|
+
const gateOne = (card, timedOut) => run({
|
|
5554
|
+
tray: { ...tray, conversations: [card] },
|
|
5555
|
+
callModel: this.boundedModelCall(run3, timedOut),
|
|
5556
|
+
client,
|
|
5557
|
+
texts,
|
|
5558
|
+
signal: run3.signal,
|
|
5559
|
+
...this.options.toolNamer ? { toolNamer: this.options.toolNamer } : {}
|
|
5560
|
+
});
|
|
5561
|
+
const queue = [];
|
|
5562
|
+
const alone = cards.filter((card) => card.muted !== true && this.gateAlone.has(card.conversation_id));
|
|
5563
|
+
const chunked = alone.length === 0 ? cards : cards.filter((card) => !alone.includes(card));
|
|
5564
|
+
const split = {
|
|
5565
|
+
token: run3.token,
|
|
5566
|
+
members: new Map(cards.map((card) => [card.conversation_id, card])),
|
|
5567
|
+
queue,
|
|
5568
|
+
skip: /* @__PURE__ */ new Set()
|
|
5569
|
+
};
|
|
5570
|
+
this.activeSplit = split;
|
|
5571
|
+
try {
|
|
5572
|
+
const of = Math.ceil(chunked.length / size);
|
|
5573
|
+
for (let index2 = 1; index2 <= of; index2++) {
|
|
5574
|
+
if (run3.stale()) break;
|
|
5575
|
+
const planned = chunked.slice((index2 - 1) * size, index2 * size);
|
|
5576
|
+
const chunk = this.unheld(planned);
|
|
5577
|
+
out.held += planned.length - chunk.length;
|
|
5578
|
+
if (chunk.length < planned.length) this.heldWhileRunning = true;
|
|
5579
|
+
if (!chunk.some((card) => card.muted !== true)) continue;
|
|
5580
|
+
const startedAt = this.evidence.start();
|
|
5581
|
+
const timedOut = { value: false };
|
|
5582
|
+
const result = await run({
|
|
5583
|
+
tray: { ...tray, conversations: chunk },
|
|
5584
|
+
callModel: this.boundedModelCall(run3, timedOut),
|
|
5585
|
+
client,
|
|
5586
|
+
texts,
|
|
5587
|
+
signal: run3.signal,
|
|
5588
|
+
...this.options.toolNamer ? { toolNamer: this.options.toolNamer } : {}
|
|
5589
|
+
});
|
|
5590
|
+
if (run3.abandoned()) break;
|
|
5591
|
+
if (run3.stale() && result.outcome !== "decided") break;
|
|
5592
|
+
take(result, chunk);
|
|
5593
|
+
const report = {
|
|
5594
|
+
cycle: run3.token,
|
|
5595
|
+
index: index2,
|
|
5596
|
+
of,
|
|
5597
|
+
cards: chunk.length,
|
|
5598
|
+
outcome: result.outcome === "decided" ? "decided" : "fail_closed",
|
|
5599
|
+
undecided: result.undecided.length,
|
|
5600
|
+
failure: result.failure
|
|
5601
|
+
};
|
|
5602
|
+
out.chunks.push(report);
|
|
5603
|
+
if (report.outcome === "decided") this.counts.chunksOk += 1;
|
|
5604
|
+
else this.counts.chunksFailed += 1;
|
|
5605
|
+
this.emit({ type: "gate_chunk", ...report });
|
|
5606
|
+
if (report.outcome === "decided") {
|
|
5607
|
+
const through = decidedBy(result);
|
|
5608
|
+
for (const id of through) this.gateAlone.delete(id);
|
|
5609
|
+
await run3.exclude(() => this.reportDue(this.completed("gate", through)));
|
|
5610
|
+
continue;
|
|
5611
|
+
}
|
|
5612
|
+
this.failedAttempt("gate", chunk.filter((card) => card.muted !== true).map((card) => card.conversation_id));
|
|
5613
|
+
if (chunk.length === 1) {
|
|
5614
|
+
this.rememberGateAlone(chunk[0].conversation_id);
|
|
5615
|
+
await run3.exclude(() => this.failed(gateFailure(chunk[0], startedAt, 0, timedOut.value, this.now())));
|
|
5616
|
+
} else {
|
|
5617
|
+
for (const card of chunk) if (card.muted !== true) queue.push({ card, reruns: 0 });
|
|
5618
|
+
}
|
|
5619
|
+
}
|
|
5620
|
+
for (const card of alone) {
|
|
5621
|
+
if (!queue.some((item) => item.card.conversation_id === card.conversation_id)) queue.push({ card, reruns: 0 });
|
|
5622
|
+
}
|
|
5623
|
+
let index = 0;
|
|
5624
|
+
let failuresInARow = 0;
|
|
5625
|
+
splitting: for (; ; ) {
|
|
5626
|
+
while (queue.length > 0) {
|
|
5627
|
+
if (run3.stale()) break splitting;
|
|
5628
|
+
const { card, reruns } = queue.shift();
|
|
5629
|
+
const id = card.conversation_id;
|
|
5630
|
+
if (decided.has(id) || split.skip.has(id)) continue;
|
|
5631
|
+
if (this.unheld([card]).length === 0) {
|
|
5632
|
+
if (!gated.has(id)) out.held += 1;
|
|
5633
|
+
this.heldWhileRunning = true;
|
|
5634
|
+
continue;
|
|
5635
|
+
}
|
|
5636
|
+
index += 1;
|
|
5637
|
+
const startedAt = this.evidence.start();
|
|
5638
|
+
const timedOut = { value: false };
|
|
5639
|
+
const result = await gateOne(card, timedOut);
|
|
5640
|
+
if (run3.abandoned()) break splitting;
|
|
5641
|
+
if (run3.stale() && result.outcome !== "decided") break splitting;
|
|
5642
|
+
take(result, [card]);
|
|
5643
|
+
const report = {
|
|
5644
|
+
cycle: run3.token,
|
|
5645
|
+
index,
|
|
5646
|
+
of: index + queue.length,
|
|
5647
|
+
cards: 1,
|
|
5648
|
+
outcome: result.outcome === "decided" ? "decided" : "fail_closed",
|
|
5649
|
+
undecided: result.undecided.length,
|
|
5650
|
+
failure: result.failure,
|
|
5651
|
+
split: true,
|
|
5652
|
+
conversationId: id
|
|
5653
|
+
};
|
|
5654
|
+
out.split.push(report);
|
|
5655
|
+
if (report.outcome === "decided") this.counts.splitCallsOk += 1;
|
|
5656
|
+
else this.counts.splitCallsFailed += 1;
|
|
5657
|
+
this.emit({ type: "gate_chunk", ...report });
|
|
5658
|
+
if (report.outcome === "decided") {
|
|
5659
|
+
failuresInARow = 0;
|
|
5660
|
+
this.gateAlone.delete(id);
|
|
5661
|
+
await run3.exclude(() => this.reportDue(this.completed("gate", decidedBy(result))));
|
|
5662
|
+
continue;
|
|
5663
|
+
}
|
|
5664
|
+
this.rememberGateAlone(id);
|
|
5665
|
+
this.failedAttempt("gate", [id]);
|
|
5666
|
+
await run3.exclude(() => this.failed(gateFailure(card, startedAt, reruns, timedOut.value, this.now())));
|
|
5667
|
+
failuresInARow += 1;
|
|
5668
|
+
if (failuresInARow >= SPLIT_STOP_AFTER_FAILURES && decided.size === 0) {
|
|
5669
|
+
const ok = this.options.checkRuntime ? await run3.exclude(() => this.runtimeCheck()) : false;
|
|
5670
|
+
if (run3.stale()) break splitting;
|
|
5671
|
+
if (!ok) {
|
|
5672
|
+
out.splitStopped = true;
|
|
5673
|
+
this.failureCounts.splitsStopped += 1;
|
|
5674
|
+
this.emit({
|
|
5675
|
+
type: "split_stopped",
|
|
5676
|
+
cycle: run3.token,
|
|
5677
|
+
remaining: queue.filter((item) => !decided.has(item.card.conversation_id)).length,
|
|
5678
|
+
reason: this.options.checkRuntime ? "runtime_check_failed" : "no_runtime_check"
|
|
5679
|
+
});
|
|
5680
|
+
break splitting;
|
|
5681
|
+
}
|
|
5682
|
+
failuresInARow = 0;
|
|
5683
|
+
}
|
|
5684
|
+
}
|
|
5685
|
+
if (run3.stale()) break;
|
|
5686
|
+
await run3.exclude(() => this.checkParked("gate"));
|
|
5687
|
+
if (queue.length === 0) break;
|
|
5688
|
+
}
|
|
5689
|
+
} finally {
|
|
5690
|
+
if (this.activeSplit === split) this.activeSplit = null;
|
|
5400
5691
|
}
|
|
5692
|
+
out.gated = gated.size;
|
|
5693
|
+
out.undecided = gated.size - decided.size;
|
|
5401
5694
|
return out;
|
|
5402
5695
|
}
|
|
5403
5696
|
/**
|
|
@@ -5445,27 +5738,194 @@ var ReceiveLoop = class {
|
|
|
5445
5738
|
const texts = await this.options.texts.ensure();
|
|
5446
5739
|
if (run3.stale()) return this.summary("abandoned", base);
|
|
5447
5740
|
if (texts === null) return this.summary("texts_missing", { ...base, failure: "the served texts are not available" });
|
|
5741
|
+
if (this.options.conversationRuns !== false && conversationRunTemplate(texts) !== null) {
|
|
5742
|
+
if (tray !== null) return this.conversationRunCycle(run3, tray, texts);
|
|
5743
|
+
if (this.runWork.size > 0 || this.waiting.length > 0) this.heldWhileRunning = true;
|
|
5744
|
+
return this.summary("tray_failed", { ...base, failure: this.lastError });
|
|
5745
|
+
}
|
|
5448
5746
|
const unready = await this.checkReady();
|
|
5449
5747
|
if (run3.stale()) return this.summary("abandoned", base);
|
|
5450
5748
|
if (unready !== null) return this.summary("tools_unavailable", { ...base, failure: unready });
|
|
5451
5749
|
const prompt = cards === 0 && (invites ?? 0) > 0 ? texts.invite_prompt.trim() : (invites ?? 0) > 0 ? `${texts.wake_instruction.trim()}
|
|
5452
5750
|
|
|
5453
5751
|
${texts.invite_prompt.trim()}` : texts.wake_instruction.trim();
|
|
5752
|
+
const kind = cards === 0 && (invites ?? 0) > 0 ? "invite" : "tray";
|
|
5454
5753
|
this.counts.runsStarted += 1;
|
|
5455
|
-
|
|
5754
|
+
const end = await run3.exclude(() => this.callRun({ prompt, texts, cards, invites, signal: this.abort.signal, kind }));
|
|
5755
|
+
this.countRunEnd(end);
|
|
5756
|
+
this.emit({ type: "run", end, kind });
|
|
5757
|
+
const startedWith = tray === null || this.options.dismissIgnoredInvites === false ? [] : trayInvites(tray);
|
|
5758
|
+
if (startedWith.length > 0) await this.track(this.afterRunInvites(startedWith, end));
|
|
5759
|
+
const outcome = end.status === "completed" ? "run_completed" : end.status === "failed" ? "run_failed" : "run_not_started";
|
|
5760
|
+
return this.summary(outcome, { ...base, failure: end.status === "completed" ? null : end.detail ?? null });
|
|
5761
|
+
}
|
|
5762
|
+
/**
|
|
5763
|
+
* Conversation runs (single-run mode, texts with `conversation_run_template`):
|
|
5764
|
+
* one run per owed card (unmuted, with a preview: a notices-only card is not
|
|
5765
|
+
* owed), prompted for that one conversation, and one invite run for the
|
|
5766
|
+
* pending invites. A conversation whose run is running or
|
|
5767
|
+
* waiting for a slot is held out (the tracker), and so is one paced after a
|
|
5768
|
+
* completed run that left its card at the same anchor (D5). Runs take a slot
|
|
5769
|
+
* each (`maxConcurrentTurns`) in the order queued: the cards whose last run
|
|
5770
|
+
* failed go last, after the invite run, so they never hold up the others.
|
|
5771
|
+
* The loop records no verdict for a card: the run decides.
|
|
5772
|
+
*/
|
|
5773
|
+
async conversationRunCycle(run3, tray, texts) {
|
|
5774
|
+
const cards = cardsOf(tray);
|
|
5775
|
+
const invites = trayInvites(tray);
|
|
5776
|
+
const unheld = this.unheld(cards);
|
|
5777
|
+
const heldByTurns = cards.length - unheld.length;
|
|
5778
|
+
if (heldByTurns > 0) this.heldWhileRunning = true;
|
|
5779
|
+
const runCards = unheld.filter((card) => !this.pacer.held(`conversation:${card.conversation_id}`, anchorFrom(card)));
|
|
5780
|
+
const openInvites = invites.filter((invite) => !this.tracker.inviteRunning(invite.invite_id));
|
|
5781
|
+
const owedInvites = openInvites.filter((invite) => !this.pacer.held(`invite:${invite.invite_id}`, null));
|
|
5782
|
+
const paced = unheld.length - runCards.length + (openInvites.length - owedInvites.length);
|
|
5783
|
+
const owedCards = runCards.filter((card) => card.muted !== true && card.preview != null);
|
|
5784
|
+
const base = { cards: cards.length, invites: invites.length, heldByTurns, paced, truncated: tray.truncated === true };
|
|
5785
|
+
if (owedCards.length === 0 && owedInvites.length === 0) return this.summary("nothing_owed", base);
|
|
5786
|
+
const unready = await this.checkReady();
|
|
5787
|
+
if (run3.stale()) return this.summary("abandoned", base);
|
|
5788
|
+
if (unready !== null) return this.summary("tools_unavailable", { ...base, failure: unready });
|
|
5789
|
+
const tally = { turnsStarted: 0, turnsRefused: 0, turnsQueued: 0 };
|
|
5790
|
+
const failedLast = owedCards.filter((card) => this.awaitingEvidence.has(card.conversation_id));
|
|
5791
|
+
for (const card of owedCards.filter((card2) => !failedLast.includes(card2))) {
|
|
5792
|
+
if (run3.stale()) break;
|
|
5793
|
+
tally[launchKey(await this.startConversationRun(card, texts))] += 1;
|
|
5794
|
+
}
|
|
5795
|
+
if (owedInvites.length > 0 && !run3.stale()) tally[launchKey(await this.startInviteRun(owedInvites, texts))] += 1;
|
|
5796
|
+
for (const card of failedLast) {
|
|
5797
|
+
if (run3.stale()) break;
|
|
5798
|
+
tally[launchKey(await this.startConversationRun(card, texts))] += 1;
|
|
5799
|
+
}
|
|
5800
|
+
const parts = { ...base, ...tally };
|
|
5801
|
+
if (run3.stale()) return this.summary("abandoned", parts);
|
|
5802
|
+
const launched = tally.turnsStarted + tally.turnsQueued;
|
|
5803
|
+
if (tray.truncated === true) this.readTheRest(launched > 0);
|
|
5804
|
+
return this.summary(launched > 0 ? "runs_started" : "nothing_owed", parts);
|
|
5805
|
+
}
|
|
5806
|
+
/** One conversation run: a slot (or a place in the queue), the run, then what its end says. */
|
|
5807
|
+
async startConversationRun(card, texts, reruns = 0) {
|
|
5808
|
+
const conversationId = card.conversation_id;
|
|
5809
|
+
const anchor = anchorFrom(card);
|
|
5810
|
+
const handle = this.tracker.begin({ kind: "reply", conversationId, anchor });
|
|
5811
|
+
if (handle === null) return "refused";
|
|
5812
|
+
const request = {
|
|
5813
|
+
kind: "conversation",
|
|
5814
|
+
conversationId,
|
|
5815
|
+
prompt: conversationRunPrompt(texts, conversationId),
|
|
5816
|
+
texts,
|
|
5817
|
+
cards: 1,
|
|
5818
|
+
invites: 0,
|
|
5819
|
+
signal: this.abort.signal
|
|
5820
|
+
};
|
|
5821
|
+
return this.launch(handle, async () => {
|
|
5822
|
+
const attempt = { conversationId, source: "run", card, anchor, startedAt: this.evidence.start(), reruns };
|
|
5823
|
+
this.counts.runsStarted += 1;
|
|
5824
|
+
this.trackRun(this.afterConversationRun(handle, attempt, this.callRun(request)));
|
|
5825
|
+
return true;
|
|
5826
|
+
});
|
|
5827
|
+
}
|
|
5828
|
+
async afterConversationRun(handle, attempt, done) {
|
|
5829
|
+
const end = await done;
|
|
5830
|
+
const held = this.tracker.holds(handle);
|
|
5831
|
+
let completion = null;
|
|
5832
|
+
let failure = null;
|
|
5456
5833
|
try {
|
|
5457
|
-
|
|
5458
|
-
|
|
5459
|
-
|
|
5834
|
+
this.countRunEnd(end);
|
|
5835
|
+
this.emit({ type: "run", end, kind: "conversation", conversationId: attempt.conversationId });
|
|
5836
|
+
if (!held) return;
|
|
5837
|
+
if (end.status === "completed") {
|
|
5838
|
+
completion = this.completed("run", [attempt.conversationId]);
|
|
5839
|
+
this.pacer.hold(`conversation:${attempt.conversationId}`, attempt.anchor);
|
|
5840
|
+
} else if (end.status === "failed" && !this.stopped) {
|
|
5841
|
+
this.failedAttempt("run", [attempt.conversationId]);
|
|
5842
|
+
if (untouched(attempt.card, await this.readTray())) {
|
|
5843
|
+
failure = { ...attempt, outcome: end.timedOut === true ? "timed_out" : "failed", failedAt: this.now(), holding: handle };
|
|
5844
|
+
}
|
|
5845
|
+
}
|
|
5846
|
+
} finally {
|
|
5847
|
+
if (failure === null) this.tracker.end(handle);
|
|
5848
|
+
this.releaseSlot(handle);
|
|
5849
|
+
if (failure === null) this.afterTurnEnded();
|
|
5850
|
+
}
|
|
5851
|
+
if (failure !== null) {
|
|
5852
|
+
try {
|
|
5853
|
+
if (!this.stopped) await this.failed(failure);
|
|
5854
|
+
} finally {
|
|
5855
|
+
this.tracker.end(handle);
|
|
5856
|
+
this.afterTurnEnded();
|
|
5857
|
+
}
|
|
5460
5858
|
}
|
|
5859
|
+
if (!this.stopped && completion !== null) await this.reportDue(completion);
|
|
5860
|
+
await this.checkParked("run");
|
|
5861
|
+
}
|
|
5862
|
+
/** The invite run: one run for the pending invites, holding each of them until it ends. */
|
|
5863
|
+
async startInviteRun(invites, texts) {
|
|
5864
|
+
const handles = [];
|
|
5865
|
+
const held = [];
|
|
5866
|
+
for (const invite of invites) {
|
|
5867
|
+
const handle = this.tracker.begin({ kind: "invite", inviteId: invite.invite_id });
|
|
5868
|
+
if (handle === null) continue;
|
|
5869
|
+
handles.push(handle);
|
|
5870
|
+
held.push(invite);
|
|
5871
|
+
}
|
|
5872
|
+
const slot = handles[0];
|
|
5873
|
+
if (slot === void 0) return "refused";
|
|
5874
|
+
const endAll = () => {
|
|
5875
|
+
for (const handle of handles) this.tracker.end(handle);
|
|
5876
|
+
};
|
|
5877
|
+
const request = {
|
|
5878
|
+
kind: "invite",
|
|
5879
|
+
prompt: texts.invite_prompt.trim(),
|
|
5880
|
+
texts,
|
|
5881
|
+
cards: 0,
|
|
5882
|
+
invites: held.length,
|
|
5883
|
+
signal: this.abort.signal
|
|
5884
|
+
};
|
|
5885
|
+
return this.launch(
|
|
5886
|
+
slot,
|
|
5887
|
+
async () => {
|
|
5888
|
+
this.counts.runsStarted += 1;
|
|
5889
|
+
this.trackRun(this.afterInviteRun(slot, held, endAll, this.callRun(request)));
|
|
5890
|
+
return true;
|
|
5891
|
+
},
|
|
5892
|
+
endAll
|
|
5893
|
+
);
|
|
5894
|
+
}
|
|
5895
|
+
async afterInviteRun(slot, invites, endAll, done) {
|
|
5896
|
+
const end = await done;
|
|
5897
|
+
const held = this.tracker.holds(slot);
|
|
5898
|
+
let completion = null;
|
|
5899
|
+
try {
|
|
5900
|
+
this.countRunEnd(end);
|
|
5901
|
+
this.emit({ type: "run", end, kind: "invite" });
|
|
5902
|
+
if (!held) return;
|
|
5903
|
+
if (end.status === "completed") {
|
|
5904
|
+
completion = this.completed("invite", []);
|
|
5905
|
+
for (const invite of invites) this.pacer.hold(`invite:${invite.invite_id}`, null);
|
|
5906
|
+
}
|
|
5907
|
+
if (this.options.dismissIgnoredInvites !== false) await this.afterRunInvites(invites, end);
|
|
5908
|
+
} finally {
|
|
5909
|
+
endAll();
|
|
5910
|
+
this.releaseSlot(slot);
|
|
5911
|
+
this.afterTurnEnded();
|
|
5912
|
+
}
|
|
5913
|
+
if (!this.stopped && completion !== null) await this.reportDue(completion);
|
|
5914
|
+
await this.checkParked("run");
|
|
5915
|
+
}
|
|
5916
|
+
/** The runtime's `startRun`, never rejecting: a throw is a run that did not start. */
|
|
5917
|
+
callRun(request) {
|
|
5918
|
+
return Promise.resolve().then(() => this.options.startRun(request)).catch((err) => ({ status: "not_started", detail: err instanceof Error ? err.message : String(err) }));
|
|
5919
|
+
}
|
|
5920
|
+
countRunEnd(end) {
|
|
5461
5921
|
if (end.status === "completed") this.counts.runsCompleted += 1;
|
|
5462
5922
|
else if (end.status === "failed") this.counts.runsFailed += 1;
|
|
5463
5923
|
else this.counts.runsNotStarted += 1;
|
|
5464
|
-
|
|
5465
|
-
|
|
5466
|
-
|
|
5467
|
-
const
|
|
5468
|
-
|
|
5924
|
+
}
|
|
5925
|
+
/** A run's whole course after its start: `idle()` and `stop()` wait for it. */
|
|
5926
|
+
trackRun(work) {
|
|
5927
|
+
const tracked = work.catch((err) => this.fail(err)).finally(() => this.runWork.delete(tracked));
|
|
5928
|
+
this.runWork.add(tracked);
|
|
5469
5929
|
}
|
|
5470
5930
|
async afterRunInvites(invites, end) {
|
|
5471
5931
|
const completed = end.status === "completed";
|
|
@@ -5476,7 +5936,7 @@ ${texts.invite_prompt.trim()}` : texts.wake_instruction.trim();
|
|
|
5476
5936
|
}
|
|
5477
5937
|
}
|
|
5478
5938
|
// ─── turns (gate-call mode) ─────────────────────────────────────────────
|
|
5479
|
-
async startReplyTurn(reply, texts) {
|
|
5939
|
+
async startReplyTurn(reply, texts, reruns = 0) {
|
|
5480
5940
|
const anchor = anchorFrom(reply.card);
|
|
5481
5941
|
const handle = this.tracker.begin({ kind: "reply", conversationId: reply.conversationId, anchor });
|
|
5482
5942
|
if (handle === null) return "refused";
|
|
@@ -5491,6 +5951,7 @@ ${texts.invite_prompt.trim()}` : texts.wake_instruction.trim();
|
|
|
5491
5951
|
signal: this.abort.signal
|
|
5492
5952
|
};
|
|
5493
5953
|
return this.launch(handle, async () => {
|
|
5954
|
+
const startedAt = this.evidence.start();
|
|
5494
5955
|
const admission = await this.admit(this.options.startTurn, request);
|
|
5495
5956
|
if (!admission.started) {
|
|
5496
5957
|
this.tracker.end(handle);
|
|
@@ -5500,7 +5961,8 @@ ${texts.invite_prompt.trim()}` : texts.wake_instruction.trim();
|
|
|
5500
5961
|
}
|
|
5501
5962
|
this.counts.turnsStarted += 1;
|
|
5502
5963
|
this.emit({ type: "turn_started", request: withoutRuntime(request) });
|
|
5503
|
-
|
|
5964
|
+
const attempt = { conversationId: reply.conversationId, source: "turn", card: reply.card, anchor, startedAt, reruns, reply };
|
|
5965
|
+
void this.afterReplyTurn(handle, attempt, admission.done);
|
|
5504
5966
|
return true;
|
|
5505
5967
|
});
|
|
5506
5968
|
}
|
|
@@ -5540,9 +6002,9 @@ invite_id: ${invite.invite_id}`,
|
|
|
5540
6002
|
* queues behind them instead of taking the slot. A refused admission here
|
|
5541
6003
|
* hands its slot on the same way.
|
|
5542
6004
|
*/
|
|
5543
|
-
async launch(handle, admit) {
|
|
6005
|
+
async launch(handle, admit, drop) {
|
|
5544
6006
|
if (this.waiting.length > 0 || this.activeTurns >= this.maxTurns) {
|
|
5545
|
-
this.waiting.push({ handle, admit });
|
|
6007
|
+
this.waiting.push({ handle, admit, ...drop ? { drop } : {} });
|
|
5546
6008
|
this.counts.turnsQueued += 1;
|
|
5547
6009
|
if (this.activeTurns < this.maxTurns) void this.pump();
|
|
5548
6010
|
return "queued";
|
|
@@ -5562,7 +6024,7 @@ invite_id: ${invite.invite_id}`,
|
|
|
5562
6024
|
releaseSlot(handle, options = {}) {
|
|
5563
6025
|
if (this.slotted.delete(handle)) this.activeTurns = Math.max(0, this.activeTurns - 1);
|
|
5564
6026
|
const queued = this.waiting.findIndex((w) => w.handle === handle);
|
|
5565
|
-
if (queued >= 0) this.waiting.splice(queued, 1);
|
|
6027
|
+
if (queued >= 0) this.waiting.splice(queued, 1)[0].drop?.();
|
|
5566
6028
|
if (options.pump !== false) void this.pump();
|
|
5567
6029
|
}
|
|
5568
6030
|
async pump() {
|
|
@@ -5572,20 +6034,38 @@ invite_id: ${invite.invite_id}`,
|
|
|
5572
6034
|
await this.admitInSlot(next.handle, next.admit);
|
|
5573
6035
|
}
|
|
5574
6036
|
}
|
|
5575
|
-
async afterReplyTurn(handle,
|
|
6037
|
+
async afterReplyTurn(handle, attempt, done) {
|
|
6038
|
+
const reply = attempt.reply;
|
|
6039
|
+
let completion = null;
|
|
6040
|
+
let failure = null;
|
|
5576
6041
|
try {
|
|
5577
6042
|
const end = await this.tracker.confirmedEnd(handle, done);
|
|
5578
6043
|
if (end === null || !this.tracker.holds(handle)) return;
|
|
5579
|
-
|
|
6044
|
+
if (end.status === "completed") completion = this.completed("turn", [reply.conversationId]);
|
|
6045
|
+
const recorded = await this.track(this.recordReplyOutcome(handle, reply, end));
|
|
6046
|
+
if (end.status === "failed" && !this.stopped) this.failedAttempt("turn", [reply.conversationId]);
|
|
6047
|
+
if (end.status === "failed" && recorded?.freshRead === true && recorded.outcome.reason === "turn_failed") {
|
|
6048
|
+
failure = { ...attempt, outcome: end.timedOut === true ? "timed_out" : "failed", failedAt: this.now(), holding: handle };
|
|
6049
|
+
}
|
|
5580
6050
|
} finally {
|
|
5581
|
-
this.tracker.end(handle);
|
|
6051
|
+
if (failure === null) this.tracker.end(handle);
|
|
5582
6052
|
this.releaseSlot(handle);
|
|
5583
|
-
this.afterTurnEnded();
|
|
6053
|
+
if (failure === null) this.afterTurnEnded();
|
|
5584
6054
|
}
|
|
6055
|
+
if (failure !== null) {
|
|
6056
|
+
try {
|
|
6057
|
+
if (!this.stopped) await this.track(this.failed(failure));
|
|
6058
|
+
} finally {
|
|
6059
|
+
this.tracker.end(handle);
|
|
6060
|
+
this.afterTurnEnded();
|
|
6061
|
+
}
|
|
6062
|
+
}
|
|
6063
|
+
if (!this.stopped && completion !== null) await this.track(this.reportDue(completion));
|
|
6064
|
+
await this.track(this.checkParked("turn"));
|
|
5585
6065
|
}
|
|
5586
6066
|
async recordReplyOutcome(handle, reply, end) {
|
|
5587
6067
|
const fresh = await this.readTray();
|
|
5588
|
-
if (!this.tracker.holds(handle)) return;
|
|
6068
|
+
if (!this.tracker.holds(handle)) return null;
|
|
5589
6069
|
const anchor = anchorFrom(reply.card);
|
|
5590
6070
|
const key = `${reply.conversationId}|${anchor ?? ""}`;
|
|
5591
6071
|
const silentBefore = this.silent.get(key) ?? 0;
|
|
@@ -5615,6 +6095,7 @@ invite_id: ${invite.invite_id}`,
|
|
|
5615
6095
|
}
|
|
5616
6096
|
}
|
|
5617
6097
|
this.emit({ type: "turn_outcome", outcome, recorded });
|
|
6098
|
+
return { outcome, freshRead: fresh !== null };
|
|
5618
6099
|
}
|
|
5619
6100
|
async afterInviteTurn(handle, invite, done) {
|
|
5620
6101
|
try {
|
|
@@ -5684,6 +6165,257 @@ invite_id: ${invite.invite_id}`,
|
|
|
5684
6165
|
this.wakeFrom("more");
|
|
5685
6166
|
}
|
|
5686
6167
|
}
|
|
6168
|
+
// ─── could not process: failures, evidence, the server's answer ─────────
|
|
6169
|
+
/**
|
|
6170
|
+
* A run, turn or gate call that completed, on the evidence clock. A failure
|
|
6171
|
+
* it clears (the same kind of attempt for the same conversation went
|
|
6172
|
+
* through since) waits no more. Call it the moment the work ended, before
|
|
6173
|
+
* its slot frees, so the next attempt starts after it.
|
|
6174
|
+
*/
|
|
6175
|
+
completed(kind, conversations) {
|
|
6176
|
+
const completion = this.evidence.complete(kind, conversations);
|
|
6177
|
+
if (kind !== "invite") this.failingSince[kind].clear();
|
|
6178
|
+
for (const id of conversations) {
|
|
6179
|
+
const failure = this.awaitingEvidence.get(id);
|
|
6180
|
+
if (failure !== void 0 && clears(completion, failure)) this.awaitingEvidence.delete(id);
|
|
6181
|
+
}
|
|
6182
|
+
return completion;
|
|
6183
|
+
}
|
|
6184
|
+
/**
|
|
6185
|
+
* Report the failures this completion is the evidence for (each once). One
|
|
6186
|
+
* that waited past `AWAITING_EVIDENCE_MS` is dropped instead: a completion
|
|
6187
|
+
* that late says little about the runtime when it failed.
|
|
6188
|
+
*/
|
|
6189
|
+
async reportDue(completion) {
|
|
6190
|
+
const now2 = this.now();
|
|
6191
|
+
const due = [];
|
|
6192
|
+
for (const failure of [...this.awaitingEvidence.values()]) {
|
|
6193
|
+
if (now2 - failure.failedAt > AWAITING_EVIDENCE_MS) {
|
|
6194
|
+
this.awaitingEvidence.delete(failure.conversationId);
|
|
6195
|
+
continue;
|
|
6196
|
+
}
|
|
6197
|
+
if (evidences(completion, failure)) {
|
|
6198
|
+
this.awaitingEvidence.delete(failure.conversationId);
|
|
6199
|
+
due.push(failure);
|
|
6200
|
+
}
|
|
6201
|
+
}
|
|
6202
|
+
for (const failure of due) await this.reportFailure(failure, "other_run_completed");
|
|
6203
|
+
}
|
|
6204
|
+
/** An attempt of this kind failed for these conversations (whatever comes of it): see `failingSince`. */
|
|
6205
|
+
failedAttempt(kind, conversations) {
|
|
6206
|
+
for (const id of conversations) this.failingSince[kind].add(id);
|
|
6207
|
+
}
|
|
6208
|
+
/**
|
|
6209
|
+
* A conversation's run, reply turn or single-card gate call failed. It is
|
|
6210
|
+
* reported only with evidence that the runtime otherwise works:
|
|
6211
|
+
* (a) a later completion of the same kind of work for another conversation
|
|
6212
|
+
* (`evidences`), or (b) `checkRuntime()` passing while no other
|
|
6213
|
+
* conversation of that kind has failed since that kind last completed.
|
|
6214
|
+
* Without (a) it waits: the work already queued runs first (cheap
|
|
6215
|
+
* evidence), and the runtime check is asked only once nothing of that kind
|
|
6216
|
+
* is queued (`checkParked`). It waits at most `AWAITING_EVIDENCE_MS`.
|
|
6217
|
+
* Nothing is reported after `stop()`: a run the stop cut short did not
|
|
6218
|
+
* fail on its conversation.
|
|
6219
|
+
*/
|
|
6220
|
+
async failed(failure) {
|
|
6221
|
+
if (this.stopped) return;
|
|
6222
|
+
this.awaitingEvidence.delete(failure.conversationId);
|
|
6223
|
+
if (this.evidence.since(failure)) {
|
|
6224
|
+
await this.reportFailure(failure, "other_run_completed");
|
|
6225
|
+
return;
|
|
6226
|
+
}
|
|
6227
|
+
if (!this.canReport(failure)) return;
|
|
6228
|
+
this.awaitingEvidence.set(failure.conversationId, failure);
|
|
6229
|
+
while (this.awaitingEvidence.size > SILENT_MEMORY) {
|
|
6230
|
+
const oldest = this.awaitingEvidence.keys().next().value;
|
|
6231
|
+
if (oldest === void 0) break;
|
|
6232
|
+
this.awaitingEvidence.delete(oldest);
|
|
6233
|
+
}
|
|
6234
|
+
if (failure.source !== "gate") await this.checkParked(failure.source);
|
|
6235
|
+
}
|
|
6236
|
+
/**
|
|
6237
|
+
* Evidence (b), once nothing of this kind is queued: ask `checkRuntime()`
|
|
6238
|
+
* about the failure still waiting for evidence, if it is the only
|
|
6239
|
+
* conversation of that kind failing since that kind last completed. Several
|
|
6240
|
+
* failing together is the runtime: none of them is reported on (b), and
|
|
6241
|
+
* they keep waiting for (a). A `timed_out` failure is never reported on
|
|
6242
|
+
* (b): a timeout is ambiguous (a brownout, a hung hook) and the check does
|
|
6243
|
+
* not exercise those paths; it waits for (a). Each failure is asked about
|
|
6244
|
+
* once. A check that fails drops it (a broken runtime reports nothing).
|
|
6245
|
+
*/
|
|
6246
|
+
async checkParked(kind) {
|
|
6247
|
+
if (!this.options.checkRuntime || this.stopped || !this.idleFor(kind)) return;
|
|
6248
|
+
const failing = this.failingSince[kind];
|
|
6249
|
+
const due = [...this.awaitingEvidence.values()].filter(
|
|
6250
|
+
(failure) => failure.source === kind && // A timeout is ambiguous (a brownout, a hung hook) and the runtime
|
|
6251
|
+
// check does not exercise those paths: only (a) can report it.
|
|
6252
|
+
failure.outcome !== "timed_out" && !this.checkedFailures.has(failure) && this.now() - failure.failedAt <= AWAITING_EVIDENCE_MS && [...failing].every((id) => id === failure.conversationId)
|
|
6253
|
+
);
|
|
6254
|
+
if (due.length === 0) return;
|
|
6255
|
+
for (const failure of due) this.checkedFailures.add(failure);
|
|
6256
|
+
const ok = await this.runtimeCheck();
|
|
6257
|
+
for (const failure of due) {
|
|
6258
|
+
if (this.stopped || this.awaitingEvidence.get(failure.conversationId) !== failure) continue;
|
|
6259
|
+
if (!ok) {
|
|
6260
|
+
this.awaitingEvidence.delete(failure.conversationId);
|
|
6261
|
+
continue;
|
|
6262
|
+
}
|
|
6263
|
+
if (![...failing].every((id) => id === failure.conversationId)) continue;
|
|
6264
|
+
this.awaitingEvidence.delete(failure.conversationId);
|
|
6265
|
+
await this.reportFailure(failure, "runtime_check");
|
|
6266
|
+
}
|
|
6267
|
+
}
|
|
6268
|
+
/** Nothing of this kind is running or queued: the runtime check may be asked now. */
|
|
6269
|
+
idleFor(kind) {
|
|
6270
|
+
if (kind === "gate") return this.activeSplit === null || this.activeSplit.queue.length === 0;
|
|
6271
|
+
if (kind === "turn" && this.activeSplit !== null) return false;
|
|
6272
|
+
return this.activeTurns === 0 && this.waiting.length === 0;
|
|
6273
|
+
}
|
|
6274
|
+
canReport(failure) {
|
|
6275
|
+
return failure.anchor !== null && typeof this.client.conversations?.reportProcessingFailure === "function";
|
|
6276
|
+
}
|
|
6277
|
+
/** `checkRuntime()`, bounded by `RUNTIME_CHECK_TIMEOUT_MS`; a throw or a timeout is not ok. */
|
|
6278
|
+
async runtimeCheck() {
|
|
6279
|
+
const check = this.options.checkRuntime;
|
|
6280
|
+
let timer = null;
|
|
6281
|
+
let answer;
|
|
6282
|
+
try {
|
|
6283
|
+
answer = await Promise.race([
|
|
6284
|
+
Promise.resolve().then(() => check()),
|
|
6285
|
+
new Promise((resolve) => {
|
|
6286
|
+
timer = this._setTimeout(
|
|
6287
|
+
() => resolve({ ok: false, detail: `no answer within ${RUNTIME_CHECK_TIMEOUT_MS} ms` }),
|
|
6288
|
+
RUNTIME_CHECK_TIMEOUT_MS
|
|
6289
|
+
);
|
|
6290
|
+
timer.unref?.();
|
|
6291
|
+
})
|
|
6292
|
+
]);
|
|
6293
|
+
} catch (err) {
|
|
6294
|
+
answer = { ok: false, detail: `the check threw: ${err instanceof Error ? err.message : String(err)}` };
|
|
6295
|
+
} finally {
|
|
6296
|
+
if (timer !== null) this._clearTimeout(timer);
|
|
6297
|
+
}
|
|
6298
|
+
const ok = answer?.ok === true;
|
|
6299
|
+
if (ok) this.failureCounts.runtimeChecksOk += 1;
|
|
6300
|
+
else this.failureCounts.runtimeChecksFailed += 1;
|
|
6301
|
+
this.lastRuntimeCheck = { at: new Date(this.now()).toISOString(), ok, detail: typeof answer?.detail === "string" ? answer.detail : null };
|
|
6302
|
+
return ok;
|
|
6303
|
+
}
|
|
6304
|
+
/** Tell the server, and do what it answers: run the conversation again (`retry`, `hidden`) or stop (`paused`, `none`). */
|
|
6305
|
+
async reportFailure(failure, evidence) {
|
|
6306
|
+
const conversations = this.client.conversations;
|
|
6307
|
+
if (!this.canReport(failure) || conversations === void 0 || this.stopped) return;
|
|
6308
|
+
const base = {
|
|
6309
|
+
conversationId: failure.conversationId,
|
|
6310
|
+
source: failure.source,
|
|
6311
|
+
throughMessageId: failure.anchor,
|
|
6312
|
+
outcome: failure.outcome,
|
|
6313
|
+
evidence
|
|
6314
|
+
};
|
|
6315
|
+
let result;
|
|
6316
|
+
try {
|
|
6317
|
+
result = await conversations.reportProcessingFailure(failure.conversationId, {
|
|
6318
|
+
throughMessageId: failure.anchor,
|
|
6319
|
+
outcome: failure.outcome,
|
|
6320
|
+
evidence
|
|
6321
|
+
});
|
|
6322
|
+
} catch (err) {
|
|
6323
|
+
this.failureCounts.reportErrors += 1;
|
|
6324
|
+
this.fail(err);
|
|
6325
|
+
this.emit({ type: "processing_failure", ...base, result: null, error: err instanceof Error ? err.message : String(err), next: "stop" });
|
|
6326
|
+
return;
|
|
6327
|
+
}
|
|
6328
|
+
this.failureCounts.reported += 1;
|
|
6329
|
+
this.failureCounts[result.action] += 1;
|
|
6330
|
+
let next = "stop";
|
|
6331
|
+
if (result.action === "retry" || result.action === "hidden") {
|
|
6332
|
+
if (failure.reruns >= MAX_RERUNS_PER_CYCLE) {
|
|
6333
|
+
next = "rerun_capped";
|
|
6334
|
+
this.failureCounts.rerunsCapped += 1;
|
|
6335
|
+
} else {
|
|
6336
|
+
next = this.rerun(failure, result.action === "hidden");
|
|
6337
|
+
}
|
|
6338
|
+
}
|
|
6339
|
+
if (result.action === "paused") this.dropWaiting(failure.conversationId);
|
|
6340
|
+
if (next === "stop" || next === "rerun_capped") this.activeSplit?.skip.add(failure.conversationId);
|
|
6341
|
+
this.emit({ type: "processing_failure", ...base, result, next });
|
|
6342
|
+
}
|
|
6343
|
+
/**
|
|
6344
|
+
* Run the conversation again after a `retry` or `hidden` answer, after the
|
|
6345
|
+
* others already queued, and say what was done. Its run or reply turn takes
|
|
6346
|
+
* a place at the back of the queue (it reads the conversation itself, so it
|
|
6347
|
+
* sees a hidden span as the note); a reply turn asked for while a gate is
|
|
6348
|
+
* deciding starts after that cycle's own turns. A gate call sees only the
|
|
6349
|
+
* card: after `retry` it joins the end of the running gate's single-card
|
|
6350
|
+
* calls (once, with the card as this cycle read it); after `hidden`, or
|
|
6351
|
+
* when no gate is running for it, the next cycle reads the tray again and
|
|
6352
|
+
* gates it alone. `running`: nothing started, because its next attempt is
|
|
6353
|
+
* under way already (running, queued, or being decided by the gate now).
|
|
6354
|
+
*/
|
|
6355
|
+
rerun(failure, hidden) {
|
|
6356
|
+
if (this.stopped) return "stop";
|
|
6357
|
+
const id = failure.conversationId;
|
|
6358
|
+
const reruns = failure.reruns + 1;
|
|
6359
|
+
this.pacer.release(`conversation:${id}`);
|
|
6360
|
+
if (failure.holding !== void 0) this.tracker.end(failure.holding);
|
|
6361
|
+
const split = this.activeSplit;
|
|
6362
|
+
const current = split?.members.get(id);
|
|
6363
|
+
if (failure.source === "gate") {
|
|
6364
|
+
this.rememberGateAlone(id);
|
|
6365
|
+
if (split !== null && current !== void 0 && !hidden) {
|
|
6366
|
+
split.skip.delete(id);
|
|
6367
|
+
if (split.queue.some((item) => item.card.conversation_id === id)) return "running";
|
|
6368
|
+
split.queue.push({ card: current, reruns });
|
|
6369
|
+
} else {
|
|
6370
|
+
split?.skip.add(id);
|
|
6371
|
+
this.wakeFrom("more");
|
|
6372
|
+
}
|
|
6373
|
+
this.failureCounts.reruns += 1;
|
|
6374
|
+
return "rerun";
|
|
6375
|
+
}
|
|
6376
|
+
if (this.tracker.conversationRunning(id) || current !== void 0) return "running";
|
|
6377
|
+
const texts = this.options.texts.current();
|
|
6378
|
+
this.failureCounts.reruns += 1;
|
|
6379
|
+
if (texts !== null && failure.source === "turn" && failure.reply !== void 0) {
|
|
6380
|
+
if (split !== null) this.turnRerunsAfterGate.push({ reply: failure.reply, reruns });
|
|
6381
|
+
else void this.startReplyTurn(failure.reply, texts, reruns);
|
|
6382
|
+
} else if (texts !== null && failure.source === "run" && conversationRunTemplate(texts) !== null) {
|
|
6383
|
+
void this.startConversationRun(failure.card, texts, reruns);
|
|
6384
|
+
} else {
|
|
6385
|
+
this.wakeFrom("more");
|
|
6386
|
+
}
|
|
6387
|
+
return "rerun";
|
|
6388
|
+
}
|
|
6389
|
+
/** The reply turns asked for while a gate was deciding: started now, after the cycle's own turns. */
|
|
6390
|
+
flushTurnReruns() {
|
|
6391
|
+
const due = this.turnRerunsAfterGate.splice(0);
|
|
6392
|
+
if (this.stopped || due.length === 0) return;
|
|
6393
|
+
const texts = this.options.texts.current();
|
|
6394
|
+
for (const { reply, reruns } of due) {
|
|
6395
|
+
if (this.tracker.conversationRunning(reply.conversationId)) continue;
|
|
6396
|
+
if (texts === null) this.wakeFrom("more");
|
|
6397
|
+
else void this.startReplyTurn(reply, texts, reruns);
|
|
6398
|
+
}
|
|
6399
|
+
}
|
|
6400
|
+
/** A `paused` conversation: a run or turn still waiting for a slot for it does not start. */
|
|
6401
|
+
dropWaiting(conversationId) {
|
|
6402
|
+
for (const entry of [...this.waiting]) {
|
|
6403
|
+
const turn = entry.handle.turn;
|
|
6404
|
+
if (turn.kind !== "reply" || turn.conversationId !== conversationId) continue;
|
|
6405
|
+
this.waiting.splice(this.waiting.indexOf(entry), 1);
|
|
6406
|
+
this.tracker.end(entry.handle);
|
|
6407
|
+
entry.drop?.();
|
|
6408
|
+
}
|
|
6409
|
+
}
|
|
6410
|
+
rememberGateAlone(conversationId) {
|
|
6411
|
+
this.gateAlone.delete(conversationId);
|
|
6412
|
+
this.gateAlone.add(conversationId);
|
|
6413
|
+
while (this.gateAlone.size > SILENT_MEMORY) {
|
|
6414
|
+
const oldest = this.gateAlone.values().next().value;
|
|
6415
|
+
if (oldest === void 0) break;
|
|
6416
|
+
this.gateAlone.delete(oldest);
|
|
6417
|
+
}
|
|
6418
|
+
}
|
|
5687
6419
|
// ─── small things ───────────────────────────────────────────────────────
|
|
5688
6420
|
/** Recording under way after a turn's confirmed end: `stop()` waits for it. */
|
|
5689
6421
|
track(work) {
|
|
@@ -5723,6 +6455,9 @@ invite_id: ${invite.invite_id}`,
|
|
|
5723
6455
|
paced: 0,
|
|
5724
6456
|
chunksOk: 0,
|
|
5725
6457
|
chunksFailed: 0,
|
|
6458
|
+
splitCallsOk: 0,
|
|
6459
|
+
splitCallsFailed: 0,
|
|
6460
|
+
splitStopped: false,
|
|
5726
6461
|
truncated: false,
|
|
5727
6462
|
failure: null,
|
|
5728
6463
|
...parts
|
|
@@ -5753,6 +6488,23 @@ conversation_id: ${conversationId}`;
|
|
|
5753
6488
|
|
|
5754
6489
|
conversation_id: ${conversationId}`;
|
|
5755
6490
|
}
|
|
6491
|
+
function conversationRunTemplate(texts) {
|
|
6492
|
+
const raw = texts.conversation_run_template;
|
|
6493
|
+
const template = typeof raw === "string" ? raw.trim() : "";
|
|
6494
|
+
return template.length > 0 ? template : null;
|
|
6495
|
+
}
|
|
6496
|
+
function conversationRunPrompt(texts, conversationId) {
|
|
6497
|
+
const template = conversationRunTemplate(texts);
|
|
6498
|
+
if (template === null) return replyPrompt(texts, conversationId);
|
|
6499
|
+
if (template.includes(REPLY_TURN_PLACEHOLDER)) return template.split(REPLY_TURN_PLACEHOLDER).join(conversationId);
|
|
6500
|
+
return `${template}
|
|
6501
|
+
|
|
6502
|
+
conversation_id: ${conversationId}`;
|
|
6503
|
+
}
|
|
6504
|
+
function untouched(card, fresh) {
|
|
6505
|
+
if (fresh === null) return false;
|
|
6506
|
+
return turnOutcome({ turn: { status: "failed" }, gateCard: card, freshTray: fresh }).reason === "turn_failed";
|
|
6507
|
+
}
|
|
5756
6508
|
function chunkFailure(chunks) {
|
|
5757
6509
|
const failed = chunks.filter((chunk) => chunk.outcome === "fail_closed");
|
|
5758
6510
|
const first = failed[0];
|
|
@@ -5761,6 +6513,34 @@ function chunkFailure(chunks) {
|
|
|
5761
6513
|
if (chunks.length === 1) return reason;
|
|
5762
6514
|
return `${failed.length} of ${chunks.length} gate chunks failed; chunk ${first.index}: ${reason}`;
|
|
5763
6515
|
}
|
|
6516
|
+
function gateFailureText(chunks, split) {
|
|
6517
|
+
const parts = [];
|
|
6518
|
+
const chunked = chunkFailure(chunks);
|
|
6519
|
+
if (chunked !== null) parts.push(chunked);
|
|
6520
|
+
const failed = split.filter((call2) => call2.outcome === "fail_closed");
|
|
6521
|
+
const first = failed[0];
|
|
6522
|
+
if (first !== void 0) {
|
|
6523
|
+
const reason = first.failure ?? "no usable verdicts in the answer";
|
|
6524
|
+
parts.push(`${failed.length} of ${split.length} single-card calls failed; ${first.conversationId ?? "a card"}: ${reason}`);
|
|
6525
|
+
}
|
|
6526
|
+
return parts.length === 0 ? null : parts.join("; ");
|
|
6527
|
+
}
|
|
6528
|
+
function gateFailure(card, startedAt, reruns, timedOut, failedAt) {
|
|
6529
|
+
return {
|
|
6530
|
+
conversationId: card.conversation_id,
|
|
6531
|
+
source: "gate",
|
|
6532
|
+
card,
|
|
6533
|
+
anchor: anchorFrom(card),
|
|
6534
|
+
startedAt,
|
|
6535
|
+
reruns,
|
|
6536
|
+
outcome: timedOut ? "timed_out" : "failed",
|
|
6537
|
+
failedAt
|
|
6538
|
+
};
|
|
6539
|
+
}
|
|
6540
|
+
function decidedBy(result) {
|
|
6541
|
+
const undecided = new Set(result.undecided);
|
|
6542
|
+
return result.verdicts.map((verdict) => verdict.conversation_id).filter((id) => !undecided.has(id));
|
|
6543
|
+
}
|
|
5764
6544
|
function timeoutOption(value, fallback, name) {
|
|
5765
6545
|
if (value === void 0) return fallback;
|
|
5766
6546
|
if (value === 0 || value === Number.POSITIVE_INFINITY) return Number.POSITIVE_INFINITY;
|