agentchatme 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +90 -0
- package/README.md +27 -6
- package/dist/index.cjs +858 -78
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +602 -39
- package/dist/index.d.ts +602 -39
- package/dist/index.js +858 -78
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -47,6 +47,11 @@ var ErrorCode = {
|
|
|
47
47
|
RESTRICTED: "RESTRICTED",
|
|
48
48
|
CONVERSATION_NOT_FOUND: "CONVERSATION_NOT_FOUND",
|
|
49
49
|
MESSAGE_NOT_FOUND: "MESSAGE_NOT_FOUND",
|
|
50
|
+
/**
|
|
51
|
+
* 410 when you download the attachment of a message hidden from you
|
|
52
|
+
* because your machine could not process it (`Message.could_not_process`).
|
|
53
|
+
*/
|
|
54
|
+
COULD_NOT_PROCESS: "COULD_NOT_PROCESS",
|
|
50
55
|
GROUP_DELETED: "GROUP_DELETED",
|
|
51
56
|
RATE_LIMITED: "RATE_LIMITED",
|
|
52
57
|
RECIPIENT_BACKLOGGED: "RECIPIENT_BACKLOGGED",
|
|
@@ -308,7 +313,7 @@ function createAgentChatError(body, status, headers) {
|
|
|
308
313
|
}
|
|
309
314
|
|
|
310
315
|
// src/version.ts
|
|
311
|
-
var VERSION = "1.
|
|
316
|
+
var VERSION = "1.5.0" ;
|
|
312
317
|
|
|
313
318
|
// src/runtime.ts
|
|
314
319
|
function detectRuntime() {
|
|
@@ -674,6 +679,7 @@ function parseBacklogWarning(header2) {
|
|
|
674
679
|
if (!Number.isFinite(undeliveredCount) || !Number.isInteger(undeliveredCount)) return null;
|
|
675
680
|
return { recipientHandle, undeliveredCount };
|
|
676
681
|
}
|
|
682
|
+
var PROCESSING_FAILURE_ACTIONS = /* @__PURE__ */ new Set(["retry", "hidden", "paused", "none"]);
|
|
677
683
|
function intHeader(headers, name) {
|
|
678
684
|
const raw = headers?.get?.(name);
|
|
679
685
|
if (typeof raw !== "string" || !/^\d+$/.test(raw.trim())) return null;
|
|
@@ -765,7 +771,8 @@ var AgentChatClient = class _AgentChatClient {
|
|
|
765
771
|
record: (conversationId, opts) => this.recordDecision(conversationId, opts)
|
|
766
772
|
};
|
|
767
773
|
this.conversations = {
|
|
768
|
-
markRead: (conversationId, throughMessageId, opts) => this.markConversationRead(conversationId, throughMessageId, opts)
|
|
774
|
+
markRead: (conversationId, throughMessageId, opts) => this.markConversationRead(conversationId, throughMessageId, opts),
|
|
775
|
+
reportProcessingFailure: (conversationId, opts) => this.reportProcessingFailure(conversationId, opts)
|
|
769
776
|
};
|
|
770
777
|
this.texts = {
|
|
771
778
|
get: (opts) => this.getTexts(opts)
|
|
@@ -837,6 +844,36 @@ var AgentChatClient = class _AgentChatClient {
|
|
|
837
844
|
options
|
|
838
845
|
);
|
|
839
846
|
}
|
|
847
|
+
async reportProcessingFailure(conversationId, options) {
|
|
848
|
+
const body = {
|
|
849
|
+
through_message_id: options.throughMessageId,
|
|
850
|
+
outcome: options.outcome,
|
|
851
|
+
evidence: options.evidence
|
|
852
|
+
};
|
|
853
|
+
let res;
|
|
854
|
+
try {
|
|
855
|
+
res = await this.post(
|
|
856
|
+
`/v1/conversations/${encodeURIComponent(conversationId)}/processing-failures`,
|
|
857
|
+
body,
|
|
858
|
+
// One report, one count: a retried request carries the same key.
|
|
859
|
+
{ ...options, idempotencyKey: options.idempotencyKey ?? generateClientMsgId() }
|
|
860
|
+
);
|
|
861
|
+
} catch (err) {
|
|
862
|
+
if (err instanceof AgentChatError && err.status === 404 && err.code !== "CONVERSATION_NOT_FOUND" && err.code !== "MESSAGE_NOT_FOUND") {
|
|
863
|
+
return { action: "none", attempts: 0, hidden_message_ids: [], paused_until: null };
|
|
864
|
+
}
|
|
865
|
+
throw err;
|
|
866
|
+
}
|
|
867
|
+
const r = res !== null && typeof res === "object" ? res : {};
|
|
868
|
+
const action = PROCESSING_FAILURE_ACTIONS.has(r.action) ? r.action : "none";
|
|
869
|
+
return {
|
|
870
|
+
action,
|
|
871
|
+
attempts: typeof r.attempts === "number" && Number.isFinite(r.attempts) ? r.attempts : 0,
|
|
872
|
+
hidden_message_ids: Array.isArray(r.hidden_message_ids) ? r.hidden_message_ids.filter((id) => typeof id === "string") : [],
|
|
873
|
+
...typeof r.hidden_count === "number" && Number.isFinite(r.hidden_count) ? { hidden_count: r.hidden_count } : {},
|
|
874
|
+
paused_until: typeof r.paused_until === "string" ? r.paused_until : null
|
|
875
|
+
};
|
|
876
|
+
}
|
|
840
877
|
getTexts(options) {
|
|
841
878
|
const params = new URLSearchParams({ runtime: options.runtime });
|
|
842
879
|
return this.get(`/v1/texts?${params.toString()}`, options);
|
|
@@ -1101,24 +1138,48 @@ var AgentChatClient = class _AgentChatClient {
|
|
|
1101
1138
|
* most one:
|
|
1102
1139
|
* - `beforeSeq` — backwards scrollback (rows with seq < N, newest first)
|
|
1103
1140
|
* - `afterSeq` — forwards gap-fill (rows with seq > N, oldest first)
|
|
1141
|
+
* - `afterMessageId` — forwards from a message you hold (instead of `afterSeq`)
|
|
1104
1142
|
* - `aroundMessageId` — backwards window ending at that exact message
|
|
1105
1143
|
*
|
|
1106
1144
|
* `afterSeq` is the path `RealtimeClient` uses for in-order recovery
|
|
1107
1145
|
* when a per-conversation seq gap is detected. Application code usually
|
|
1108
1146
|
* only needs `beforeSeq` for normal pagination.
|
|
1147
|
+
*
|
|
1148
|
+
* With `waitMs` > 0 (and `afterSeq` or `afterMessageId`) this is the
|
|
1149
|
+
* scoped wait: the server holds the request until a message from someone
|
|
1150
|
+
* else arrives after the cursor, or answers `[]` after `waitMs`. One
|
|
1151
|
+
* conversation only, no sync ack. `getMessagesPage` also returns how long
|
|
1152
|
+
* the server held it.
|
|
1109
1153
|
*/
|
|
1110
|
-
getMessages(conversationId, options) {
|
|
1154
|
+
async getMessages(conversationId, options) {
|
|
1155
|
+
return (await this.getMessagesPage(conversationId, options)).messages;
|
|
1156
|
+
}
|
|
1157
|
+
/** `getMessages()` with the response facts a waiter needs: how long the server held a scoped wait (`waitedMs`). */
|
|
1158
|
+
async getMessagesPage(conversationId, options) {
|
|
1111
1159
|
const params = new URLSearchParams();
|
|
1112
1160
|
params.set("limit", String(options?.limit ?? 50));
|
|
1113
1161
|
if (options?.beforeSeq !== void 0) params.set("before_seq", String(options.beforeSeq));
|
|
1114
1162
|
if (options?.afterSeq !== void 0) params.set("after_seq", String(options.afterSeq));
|
|
1163
|
+
if (options?.afterMessageId !== void 0) params.set("after_message_id", options.afterMessageId);
|
|
1115
1164
|
if (options?.aroundMessageId !== void 0) {
|
|
1116
1165
|
params.set("around_message_id", options.aroundMessageId);
|
|
1117
1166
|
}
|
|
1118
|
-
|
|
1167
|
+
const waitMs = Math.floor(options?.waitMs ?? 0);
|
|
1168
|
+
if (waitMs > 0) params.set("wait_ms", String(waitMs));
|
|
1169
|
+
const timeoutMs = options?.timeoutMs !== void 0 ? options.timeoutMs : waitMs > 0 ? waitMs + INBOX_WAIT_TIMEOUT_GRACE_MS : void 0;
|
|
1170
|
+
const res = await this.http.request(
|
|
1171
|
+
"GET",
|
|
1119
1172
|
`/v1/messages/${encodeURIComponent(conversationId)}?${params.toString()}`,
|
|
1120
|
-
options
|
|
1173
|
+
this.toRequestOpts({ ...options, timeoutMs })
|
|
1121
1174
|
);
|
|
1175
|
+
const waitedMs = intHeader(res.headers, "x-agentchat-waited-ms");
|
|
1176
|
+
return {
|
|
1177
|
+
messages: res.data,
|
|
1178
|
+
waitedMs,
|
|
1179
|
+
scopedWaitSupported: waitMs > 0 ? waitedMs !== null : null,
|
|
1180
|
+
requestId: res.requestId,
|
|
1181
|
+
headers: res.headers
|
|
1182
|
+
};
|
|
1122
1183
|
}
|
|
1123
1184
|
/**
|
|
1124
1185
|
* Hide a message from your own view (hide-for-me). Either side of the
|
|
@@ -1410,11 +1471,13 @@ var AgentChatClient = class _AgentChatClient {
|
|
|
1410
1471
|
}
|
|
1411
1472
|
// ─── Mutes ────────────────────────────────────────────────────────────────
|
|
1412
1473
|
//
|
|
1413
|
-
// Mute
|
|
1414
|
-
//
|
|
1415
|
-
//
|
|
1416
|
-
//
|
|
1417
|
-
//
|
|
1474
|
+
// Mute stops the wake for a specific agent or conversation without
|
|
1475
|
+
// blocking or leaving: no real-time WebSocket push and no wake webhook.
|
|
1476
|
+
// Nothing is hidden — the conversation stays on the inbox tray
|
|
1477
|
+
// (`muted: true`), envelopes still land in `/v1/messages/sync`, and the
|
|
1478
|
+
// unread counter still bumps. A muted message reaches the machine at its
|
|
1479
|
+
// next sync (every reconnect) or when the agent opens the conversation;
|
|
1480
|
+
// until then the sender sees it as `stored`, not `delivered`.
|
|
1418
1481
|
//
|
|
1419
1482
|
// All mute APIs are idempotent:
|
|
1420
1483
|
// - Re-muting with a different `mutedUntil` refreshes the expiry.
|
|
@@ -3562,6 +3625,7 @@ __export(integration_exports, {
|
|
|
3562
3625
|
anchor: () => anchor_exports,
|
|
3563
3626
|
applyInviteOutcome: () => applyInviteOutcome,
|
|
3564
3627
|
applyOutcome: () => applyOutcome,
|
|
3628
|
+
conversationRunPrompt: () => conversationRunPrompt,
|
|
3565
3629
|
hostedMcpUrl: () => hostedMcpUrl,
|
|
3566
3630
|
inviteOutcome: () => inviteOutcome,
|
|
3567
3631
|
isTexts: () => isTexts,
|
|
@@ -4806,6 +4870,11 @@ var CYCLE_TIMEOUT_MS = 10 * 6e4;
|
|
|
4806
4870
|
var GATE_CHUNK_SIZE = 10;
|
|
4807
4871
|
var REPLY_TURN_PLACEHOLDER = "<conversation_id>";
|
|
4808
4872
|
var RECENT_CYCLES = 20;
|
|
4873
|
+
var MAX_RERUNS_PER_CYCLE = 3;
|
|
4874
|
+
var SPLIT_STOP_AFTER_FAILURES = 2;
|
|
4875
|
+
var RUNTIME_CHECK_TIMEOUT_MS = 6e4;
|
|
4876
|
+
var AWAITING_EVIDENCE_MS = 10 * 6e4;
|
|
4877
|
+
var EVIDENCE_MEMORY = 100;
|
|
4809
4878
|
var Pacer = class {
|
|
4810
4879
|
constructor(paceMs, now2) {
|
|
4811
4880
|
this.paceMs = paceMs;
|
|
@@ -4839,6 +4908,29 @@ var Pacer = class {
|
|
|
4839
4908
|
for (const key of [...this.until.keys()]) if (key.startsWith("invite:")) this.until.delete(key);
|
|
4840
4909
|
}
|
|
4841
4910
|
};
|
|
4911
|
+
function clears(completion, failure) {
|
|
4912
|
+
return completion.kind === failure.source && completion.conversations.has(failure.conversationId);
|
|
4913
|
+
}
|
|
4914
|
+
function evidences(completion, failure) {
|
|
4915
|
+
return completion.kind === failure.source && completion.at > failure.startedAt && completion.conversations.size > 0 && !completion.conversations.has(failure.conversationId);
|
|
4916
|
+
}
|
|
4917
|
+
var EvidenceClock = class {
|
|
4918
|
+
last = 0;
|
|
4919
|
+
completions = [];
|
|
4920
|
+
start() {
|
|
4921
|
+
return ++this.last;
|
|
4922
|
+
}
|
|
4923
|
+
complete(kind, conversations) {
|
|
4924
|
+
const entry = { at: ++this.last, kind, conversations: new Set(conversations) };
|
|
4925
|
+
this.completions.push(entry);
|
|
4926
|
+
if (this.completions.length > EVIDENCE_MEMORY) this.completions.shift();
|
|
4927
|
+
return entry;
|
|
4928
|
+
}
|
|
4929
|
+
/** A completion since the failed attempt started that is evidence for it (`evidences`). */
|
|
4930
|
+
since(failure) {
|
|
4931
|
+
return this.completions.some((c) => evidences(c, failure));
|
|
4932
|
+
}
|
|
4933
|
+
};
|
|
4842
4934
|
var ReceiveLoop = class {
|
|
4843
4935
|
options;
|
|
4844
4936
|
client;
|
|
@@ -4856,7 +4948,51 @@ var ReceiveLoop = class {
|
|
|
4856
4948
|
/** When a single-run invite's silent run was counted (D5 pacing in single-run mode). */
|
|
4857
4949
|
silentAt = /* @__PURE__ */ new Map();
|
|
4858
4950
|
paceMs;
|
|
4951
|
+
/** Turns and runs waiting for a slot; `drop` ends what else the entry holds (an invite run's other invites). */
|
|
4859
4952
|
waiting = [];
|
|
4953
|
+
/** Conversation and invite runs started and not yet concluded: `idle()` and `stop()` wait for them. */
|
|
4954
|
+
runWork = /* @__PURE__ */ new Set();
|
|
4955
|
+
/** Could not process: the order of attempts and completions. */
|
|
4956
|
+
evidence = new EvidenceClock();
|
|
4957
|
+
/** Could not process: failures waiting for a completion (the evidence a report needs), by conversation. */
|
|
4958
|
+
awaitingEvidence = /* @__PURE__ */ new Map();
|
|
4959
|
+
/** Waiting failures the runtime check has already been asked about (once each). */
|
|
4960
|
+
checkedFailures = /* @__PURE__ */ new WeakSet();
|
|
4961
|
+
/**
|
|
4962
|
+
* Per kind of work, the conversations whose attempt failed since that kind
|
|
4963
|
+
* last completed. Two or more: several failing together is the runtime, so
|
|
4964
|
+
* a passing `checkRuntime()` is no evidence for any of them.
|
|
4965
|
+
*/
|
|
4966
|
+
failingSince = { gate: /* @__PURE__ */ new Set(), turn: /* @__PURE__ */ new Set(), run: /* @__PURE__ */ new Set() };
|
|
4967
|
+
/** gateCall: conversations whose last gate call failed; gated alone, after the others. */
|
|
4968
|
+
gateAlone = /* @__PURE__ */ new Set();
|
|
4969
|
+
/**
|
|
4970
|
+
* gateCall: the running cycle's gate: its cards (`members`) and its
|
|
4971
|
+
* single-card queue. A `retry` answer for one of its cards joins the queue;
|
|
4972
|
+
* a `hidden` one skips the card here (its card, read before the hide, is
|
|
4973
|
+
* stale) and gates it in the next cycle.
|
|
4974
|
+
*/
|
|
4975
|
+
activeSplit = null;
|
|
4976
|
+
/**
|
|
4977
|
+
* gateCall: reply turns to start again on a `retry` / `hidden` answer that
|
|
4978
|
+
* came while a gate was deciding: they start after that cycle's own turns
|
|
4979
|
+
* (no turn starts while cards of a cycle still wait for their gate call).
|
|
4980
|
+
*/
|
|
4981
|
+
turnRerunsAfterGate = [];
|
|
4982
|
+
lastRuntimeCheck = null;
|
|
4983
|
+
failureCounts = {
|
|
4984
|
+
reported: 0,
|
|
4985
|
+
retry: 0,
|
|
4986
|
+
hidden: 0,
|
|
4987
|
+
paused: 0,
|
|
4988
|
+
none: 0,
|
|
4989
|
+
reportErrors: 0,
|
|
4990
|
+
reruns: 0,
|
|
4991
|
+
rerunsCapped: 0,
|
|
4992
|
+
runtimeChecksOk: 0,
|
|
4993
|
+
runtimeChecksFailed: 0,
|
|
4994
|
+
splitsStopped: 0
|
|
4995
|
+
};
|
|
4860
4996
|
/** Conversations whose verdict a gate call is recording right now (an abandoned cycle's included). */
|
|
4861
4997
|
recordsInFlight = /* @__PURE__ */ new Set();
|
|
4862
4998
|
/** Turns holding a slot (admitted, not yet ended or released). */
|
|
@@ -4891,6 +5027,8 @@ var ReceiveLoop = class {
|
|
|
4891
5027
|
runsNotStarted: 0,
|
|
4892
5028
|
chunksOk: 0,
|
|
4893
5029
|
chunksFailed: 0,
|
|
5030
|
+
splitCallsOk: 0,
|
|
5031
|
+
splitCallsFailed: 0,
|
|
4894
5032
|
outcomeRecordFailures: 0
|
|
4895
5033
|
};
|
|
4896
5034
|
outcomes = {};
|
|
@@ -4988,7 +5126,10 @@ var ReceiveLoop = class {
|
|
|
4988
5126
|
this.clearMoreTimer();
|
|
4989
5127
|
this.abort.abort();
|
|
4990
5128
|
this.cycleAbort?.abort();
|
|
4991
|
-
for (const { handle } of this.waiting.splice(0))
|
|
5129
|
+
for (const { handle, drop } of this.waiting.splice(0)) {
|
|
5130
|
+
this.tracker.end(handle);
|
|
5131
|
+
drop?.();
|
|
5132
|
+
}
|
|
4992
5133
|
for (const off of this.unsubscribe.splice(0)) {
|
|
4993
5134
|
try {
|
|
4994
5135
|
off();
|
|
@@ -5000,11 +5141,21 @@ var ReceiveLoop = class {
|
|
|
5000
5141
|
} catch {
|
|
5001
5142
|
}
|
|
5002
5143
|
await this.coalescer.idle();
|
|
5003
|
-
await Promise.allSettled([...this.pendingWork]);
|
|
5144
|
+
await Promise.allSettled([...this.pendingWork, ...this.runWork]);
|
|
5004
5145
|
}
|
|
5005
|
-
/**
|
|
5006
|
-
|
|
5007
|
-
|
|
5146
|
+
/**
|
|
5147
|
+
* Resolves when the running cycle has finished (tests, graceful stops) and,
|
|
5148
|
+
* in single-run mode, no run is running or waiting for a slot (conversation
|
|
5149
|
+
* runs go on after the cycle that started them). Gate-call turns are not
|
|
5150
|
+
* waited for.
|
|
5151
|
+
*/
|
|
5152
|
+
async idle() {
|
|
5153
|
+
for (; ; ) {
|
|
5154
|
+
await this.coalescer.idle();
|
|
5155
|
+
const runs = [...this.runWork];
|
|
5156
|
+
if (runs.length === 0) return;
|
|
5157
|
+
await Promise.allSettled(runs);
|
|
5158
|
+
}
|
|
5008
5159
|
}
|
|
5009
5160
|
/** Let go of a turn whose end the runtime will never confirm (an operator's call). Nothing is recorded for it. */
|
|
5010
5161
|
releaseTurn(key) {
|
|
@@ -5044,6 +5195,11 @@ var ReceiveLoop = class {
|
|
|
5044
5195
|
turnsWaiting: this.waiting.length,
|
|
5045
5196
|
outcomes: { ...this.outcomes }
|
|
5046
5197
|
},
|
|
5198
|
+
processingFailures: {
|
|
5199
|
+
...this.failureCounts,
|
|
5200
|
+
awaitingEvidence: this.awaitingEvidence.size,
|
|
5201
|
+
lastRuntimeCheck: this.lastRuntimeCheck === null ? null : { ...this.lastRuntimeCheck }
|
|
5202
|
+
},
|
|
5047
5203
|
lastError: this.lastError
|
|
5048
5204
|
};
|
|
5049
5205
|
}
|
|
@@ -5115,6 +5271,8 @@ var ReceiveLoop = class {
|
|
|
5115
5271
|
const work = (async () => {
|
|
5116
5272
|
const summary = this.options.mode === "gateCall" ? await this.gateCallCycle(run3) : await this.singleRunCycle(run3);
|
|
5117
5273
|
if (!run3.abandoned()) this.finishCycle(summary);
|
|
5274
|
+
this.flushTurnReruns();
|
|
5275
|
+
void this.track(this.checkParked(this.options.mode === "gateCall" ? "turn" : "run"));
|
|
5118
5276
|
})();
|
|
5119
5277
|
let timer = null;
|
|
5120
5278
|
const overdue = new Promise((resolve) => {
|
|
@@ -5182,12 +5340,13 @@ var ReceiveLoop = class {
|
|
|
5182
5340
|
* cycle that is making progress is not abandoned for it. With the deadline
|
|
5183
5341
|
* off the watchdog is the calls' only bound, so they count.
|
|
5184
5342
|
*/
|
|
5185
|
-
boundedModelCall(run3) {
|
|
5186
|
-
const call2 = this.deadlineModelCall(run3);
|
|
5343
|
+
boundedModelCall(run3, timedOut) {
|
|
5344
|
+
const call2 = this.deadlineModelCall(run3, timedOut);
|
|
5187
5345
|
if (!Number.isFinite(this.gateCallTimeoutMs)) return call2;
|
|
5188
5346
|
return (prompt, options) => run3.exclude(() => call2(prompt, options));
|
|
5189
5347
|
}
|
|
5190
|
-
|
|
5348
|
+
/** `timedOut` is set when a call runs past its deadline (a could-not-process report then says `timed_out`). */
|
|
5349
|
+
deadlineModelCall(run3, timedOut) {
|
|
5191
5350
|
return (prompt) => new Promise((resolve, reject) => {
|
|
5192
5351
|
const controller = new AbortController();
|
|
5193
5352
|
const abort = () => {
|
|
@@ -5201,6 +5360,7 @@ var ReceiveLoop = class {
|
|
|
5201
5360
|
run3.signal.addEventListener("abort", abort, { once: true });
|
|
5202
5361
|
const timer = Number.isFinite(this.gateCallTimeoutMs) ? this._setTimeout(() => {
|
|
5203
5362
|
run3.signal.removeEventListener("abort", abort);
|
|
5363
|
+
if (timedOut) timedOut.value = true;
|
|
5204
5364
|
controller.abort();
|
|
5205
5365
|
reject(new Error(`no answer within ${this.gateCallTimeoutMs} ms`));
|
|
5206
5366
|
}, this.gateCallTimeoutMs) : null;
|
|
@@ -5282,9 +5442,12 @@ var ReceiveLoop = class {
|
|
|
5282
5442
|
turnsRefused: 0,
|
|
5283
5443
|
turnsQueued: 0,
|
|
5284
5444
|
chunksOk: 0,
|
|
5285
|
-
chunksFailed: 0
|
|
5445
|
+
chunksFailed: 0,
|
|
5446
|
+
splitCallsOk: 0,
|
|
5447
|
+
splitCallsFailed: 0
|
|
5286
5448
|
};
|
|
5287
5449
|
let heldDuringGate = 0;
|
|
5450
|
+
let splitStopped = false;
|
|
5288
5451
|
let outcome = "invites_only";
|
|
5289
5452
|
let failure = null;
|
|
5290
5453
|
if (owedCards.length > 0) {
|
|
@@ -5299,14 +5462,17 @@ var ReceiveLoop = class {
|
|
|
5299
5462
|
tally.recordFailures = gate.recordFailures;
|
|
5300
5463
|
tally.chunksOk = gate.chunks.filter((c) => c.outcome === "decided").length;
|
|
5301
5464
|
tally.chunksFailed = gate.chunks.length - tally.chunksOk;
|
|
5302
|
-
|
|
5303
|
-
|
|
5465
|
+
tally.splitCallsOk = gate.split.filter((c) => c.outcome === "decided").length;
|
|
5466
|
+
tally.splitCallsFailed = gate.split.length - tally.splitCallsOk;
|
|
5467
|
+
splitStopped = gate.splitStopped;
|
|
5468
|
+
failure = gateFailureText(gate.chunks, gate.split);
|
|
5469
|
+
if (tally.chunksOk + tally.splitCallsOk > 0) {
|
|
5304
5470
|
outcome = "decided";
|
|
5305
5471
|
for (const reply of gate.reply) {
|
|
5306
5472
|
if (run3.stale()) break;
|
|
5307
5473
|
tally[launchKey(await run3.exclude(() => this.startReplyTurn(reply, texts)))] += 1;
|
|
5308
5474
|
}
|
|
5309
|
-
} else if (tally.chunksFailed > 0) {
|
|
5475
|
+
} else if (tally.chunksFailed + tally.splitCallsFailed > 0) {
|
|
5310
5476
|
outcome = "fail_closed";
|
|
5311
5477
|
} else if (owedInvites.length === 0 && !run3.stale()) {
|
|
5312
5478
|
return this.summary("nothing_owed", { ...base, heldByTurns: heldByTurns + heldDuringGate });
|
|
@@ -5316,7 +5482,7 @@ var ReceiveLoop = class {
|
|
|
5316
5482
|
if (run3.stale()) break;
|
|
5317
5483
|
tally[launchKey(await run3.exclude(() => this.startInviteTurn(invite, texts)))] += 1;
|
|
5318
5484
|
}
|
|
5319
|
-
const parts = { ...base, heldByTurns: heldByTurns + heldDuringGate, ...tally };
|
|
5485
|
+
const parts = { ...base, heldByTurns: heldByTurns + heldDuringGate, ...tally, splitStopped };
|
|
5320
5486
|
if (run3.stale()) return this.summary("abandoned", parts);
|
|
5321
5487
|
if (tray.truncated === true && outcome === "decided") {
|
|
5322
5488
|
this.readTheRest(tally.recorded > 0 || tally.turnsStarted > 0);
|
|
@@ -5344,11 +5510,21 @@ var ReceiveLoop = class {
|
|
|
5344
5510
|
* cards only (or none) is not called. Stops once the cycle is stale; a call
|
|
5345
5511
|
* a stop cut short is not counted as a failed chunk, and a cycle the
|
|
5346
5512
|
* watchdog abandoned reports and counts no chunk after its summary.
|
|
5513
|
+
*
|
|
5514
|
+
* The split (could not process): every owed card of a chunk of more than
|
|
5515
|
+
* one card that failed closed is then gated again alone, one call per card,
|
|
5516
|
+
* in tray order, before any turn starts, so one card the model cannot
|
|
5517
|
+
* handle fails only its own call. A card whose last gate call failed is
|
|
5518
|
+
* kept out of the chunks and gated alone after them. A single-card call that
|
|
5519
|
+
* fails is the card's failure (reported with evidence; a `retry` answer
|
|
5520
|
+
* gates it once more at the end of the queue). When the queue is empty, a
|
|
5521
|
+
* failure still waiting gets the runtime check. After
|
|
5522
|
+
* `SPLIT_STOP_AFTER_FAILURES` single-card calls in a row failed with nothing
|
|
5523
|
+
* decided in the cycle, the split goes on only if `checkRuntime()` passes;
|
|
5524
|
+
* otherwise it stops (`split_stopped`).
|
|
5347
5525
|
*/
|
|
5348
5526
|
async gateInChunks(run3, tray, cards, texts) {
|
|
5349
5527
|
const size = this.gateChunkSize;
|
|
5350
|
-
const of = Math.ceil(cards.length / size);
|
|
5351
|
-
const callModel = this.boundedModelCall(run3);
|
|
5352
5528
|
const client = this.recordingClient(run3);
|
|
5353
5529
|
const out = {
|
|
5354
5530
|
gated: 0,
|
|
@@ -5358,48 +5534,165 @@ var ReceiveLoop = class {
|
|
|
5358
5534
|
undecided: 0,
|
|
5359
5535
|
recorded: 0,
|
|
5360
5536
|
recordFailures: 0,
|
|
5361
|
-
chunks: []
|
|
5537
|
+
chunks: [],
|
|
5538
|
+
split: [],
|
|
5539
|
+
splitStopped: false
|
|
5362
5540
|
};
|
|
5363
|
-
|
|
5364
|
-
|
|
5365
|
-
|
|
5366
|
-
const
|
|
5367
|
-
out.held += planned.length - chunk.length;
|
|
5368
|
-
if (chunk.length < planned.length) this.heldWhileRunning = true;
|
|
5369
|
-
if (!chunk.some((card) => card.muted !== true)) continue;
|
|
5370
|
-
const result = await run({
|
|
5371
|
-
tray: { ...tray, conversations: chunk },
|
|
5372
|
-
callModel,
|
|
5373
|
-
client,
|
|
5374
|
-
texts,
|
|
5375
|
-
signal: run3.signal,
|
|
5376
|
-
...this.options.toolNamer ? { toolNamer: this.options.toolNamer } : {}
|
|
5377
|
-
});
|
|
5378
|
-
if (run3.abandoned()) break;
|
|
5379
|
-
if (run3.stale() && result.outcome !== "decided") break;
|
|
5380
|
-
out.gated += chunk.length;
|
|
5541
|
+
const gated = /* @__PURE__ */ new Set();
|
|
5542
|
+
const decided = /* @__PURE__ */ new Set();
|
|
5543
|
+
const take = (result, sent) => {
|
|
5544
|
+
for (const card of sent) gated.add(card.conversation_id);
|
|
5381
5545
|
const undecided = new Set(result.undecided);
|
|
5382
5546
|
for (const verdict of result.verdicts) {
|
|
5383
|
-
if (
|
|
5547
|
+
if (undecided.has(verdict.conversation_id) || decided.has(verdict.conversation_id)) continue;
|
|
5548
|
+
decided.add(verdict.conversation_id);
|
|
5549
|
+
out.verdicts[verdict.decision] += 1;
|
|
5384
5550
|
}
|
|
5385
5551
|
out.reply.push(...result.reply);
|
|
5386
|
-
out.undecided += result.undecided.length;
|
|
5387
5552
|
out.recorded += result.recorded;
|
|
5388
5553
|
out.recordFailures += result.recordFailures.length;
|
|
5389
|
-
|
|
5390
|
-
|
|
5391
|
-
|
|
5392
|
-
|
|
5393
|
-
|
|
5394
|
-
|
|
5395
|
-
|
|
5396
|
-
|
|
5397
|
-
|
|
5398
|
-
|
|
5399
|
-
|
|
5400
|
-
|
|
5401
|
-
|
|
5554
|
+
};
|
|
5555
|
+
const gateOne = (card, timedOut) => run({
|
|
5556
|
+
tray: { ...tray, conversations: [card] },
|
|
5557
|
+
callModel: this.boundedModelCall(run3, timedOut),
|
|
5558
|
+
client,
|
|
5559
|
+
texts,
|
|
5560
|
+
signal: run3.signal,
|
|
5561
|
+
...this.options.toolNamer ? { toolNamer: this.options.toolNamer } : {}
|
|
5562
|
+
});
|
|
5563
|
+
const queue = [];
|
|
5564
|
+
const alone = cards.filter((card) => card.muted !== true && this.gateAlone.has(card.conversation_id));
|
|
5565
|
+
const chunked = alone.length === 0 ? cards : cards.filter((card) => !alone.includes(card));
|
|
5566
|
+
const split = {
|
|
5567
|
+
token: run3.token,
|
|
5568
|
+
members: new Map(cards.map((card) => [card.conversation_id, card])),
|
|
5569
|
+
queue,
|
|
5570
|
+
skip: /* @__PURE__ */ new Set()
|
|
5571
|
+
};
|
|
5572
|
+
this.activeSplit = split;
|
|
5573
|
+
try {
|
|
5574
|
+
const of = Math.ceil(chunked.length / size);
|
|
5575
|
+
for (let index2 = 1; index2 <= of; index2++) {
|
|
5576
|
+
if (run3.stale()) break;
|
|
5577
|
+
const planned = chunked.slice((index2 - 1) * size, index2 * size);
|
|
5578
|
+
const chunk = this.unheld(planned);
|
|
5579
|
+
out.held += planned.length - chunk.length;
|
|
5580
|
+
if (chunk.length < planned.length) this.heldWhileRunning = true;
|
|
5581
|
+
if (!chunk.some((card) => card.muted !== true)) continue;
|
|
5582
|
+
const startedAt = this.evidence.start();
|
|
5583
|
+
const timedOut = { value: false };
|
|
5584
|
+
const result = await run({
|
|
5585
|
+
tray: { ...tray, conversations: chunk },
|
|
5586
|
+
callModel: this.boundedModelCall(run3, timedOut),
|
|
5587
|
+
client,
|
|
5588
|
+
texts,
|
|
5589
|
+
signal: run3.signal,
|
|
5590
|
+
...this.options.toolNamer ? { toolNamer: this.options.toolNamer } : {}
|
|
5591
|
+
});
|
|
5592
|
+
if (run3.abandoned()) break;
|
|
5593
|
+
if (run3.stale() && result.outcome !== "decided") break;
|
|
5594
|
+
take(result, chunk);
|
|
5595
|
+
const report = {
|
|
5596
|
+
cycle: run3.token,
|
|
5597
|
+
index: index2,
|
|
5598
|
+
of,
|
|
5599
|
+
cards: chunk.length,
|
|
5600
|
+
outcome: result.outcome === "decided" ? "decided" : "fail_closed",
|
|
5601
|
+
undecided: result.undecided.length,
|
|
5602
|
+
failure: result.failure
|
|
5603
|
+
};
|
|
5604
|
+
out.chunks.push(report);
|
|
5605
|
+
if (report.outcome === "decided") this.counts.chunksOk += 1;
|
|
5606
|
+
else this.counts.chunksFailed += 1;
|
|
5607
|
+
this.emit({ type: "gate_chunk", ...report });
|
|
5608
|
+
if (report.outcome === "decided") {
|
|
5609
|
+
const through = decidedBy(result);
|
|
5610
|
+
for (const id of through) this.gateAlone.delete(id);
|
|
5611
|
+
await run3.exclude(() => this.reportDue(this.completed("gate", through)));
|
|
5612
|
+
continue;
|
|
5613
|
+
}
|
|
5614
|
+
this.failedAttempt("gate", chunk.filter((card) => card.muted !== true).map((card) => card.conversation_id));
|
|
5615
|
+
if (chunk.length === 1) {
|
|
5616
|
+
this.rememberGateAlone(chunk[0].conversation_id);
|
|
5617
|
+
await run3.exclude(() => this.failed(gateFailure(chunk[0], startedAt, 0, timedOut.value, this.now())));
|
|
5618
|
+
} else {
|
|
5619
|
+
for (const card of chunk) if (card.muted !== true) queue.push({ card, reruns: 0 });
|
|
5620
|
+
}
|
|
5621
|
+
}
|
|
5622
|
+
for (const card of alone) {
|
|
5623
|
+
if (!queue.some((item) => item.card.conversation_id === card.conversation_id)) queue.push({ card, reruns: 0 });
|
|
5624
|
+
}
|
|
5625
|
+
let index = 0;
|
|
5626
|
+
let failuresInARow = 0;
|
|
5627
|
+
splitting: for (; ; ) {
|
|
5628
|
+
while (queue.length > 0) {
|
|
5629
|
+
if (run3.stale()) break splitting;
|
|
5630
|
+
const { card, reruns } = queue.shift();
|
|
5631
|
+
const id = card.conversation_id;
|
|
5632
|
+
if (decided.has(id) || split.skip.has(id)) continue;
|
|
5633
|
+
if (this.unheld([card]).length === 0) {
|
|
5634
|
+
if (!gated.has(id)) out.held += 1;
|
|
5635
|
+
this.heldWhileRunning = true;
|
|
5636
|
+
continue;
|
|
5637
|
+
}
|
|
5638
|
+
index += 1;
|
|
5639
|
+
const startedAt = this.evidence.start();
|
|
5640
|
+
const timedOut = { value: false };
|
|
5641
|
+
const result = await gateOne(card, timedOut);
|
|
5642
|
+
if (run3.abandoned()) break splitting;
|
|
5643
|
+
if (run3.stale() && result.outcome !== "decided") break splitting;
|
|
5644
|
+
take(result, [card]);
|
|
5645
|
+
const report = {
|
|
5646
|
+
cycle: run3.token,
|
|
5647
|
+
index,
|
|
5648
|
+
of: index + queue.length,
|
|
5649
|
+
cards: 1,
|
|
5650
|
+
outcome: result.outcome === "decided" ? "decided" : "fail_closed",
|
|
5651
|
+
undecided: result.undecided.length,
|
|
5652
|
+
failure: result.failure,
|
|
5653
|
+
split: true,
|
|
5654
|
+
conversationId: id
|
|
5655
|
+
};
|
|
5656
|
+
out.split.push(report);
|
|
5657
|
+
if (report.outcome === "decided") this.counts.splitCallsOk += 1;
|
|
5658
|
+
else this.counts.splitCallsFailed += 1;
|
|
5659
|
+
this.emit({ type: "gate_chunk", ...report });
|
|
5660
|
+
if (report.outcome === "decided") {
|
|
5661
|
+
failuresInARow = 0;
|
|
5662
|
+
this.gateAlone.delete(id);
|
|
5663
|
+
await run3.exclude(() => this.reportDue(this.completed("gate", decidedBy(result))));
|
|
5664
|
+
continue;
|
|
5665
|
+
}
|
|
5666
|
+
this.rememberGateAlone(id);
|
|
5667
|
+
this.failedAttempt("gate", [id]);
|
|
5668
|
+
await run3.exclude(() => this.failed(gateFailure(card, startedAt, reruns, timedOut.value, this.now())));
|
|
5669
|
+
failuresInARow += 1;
|
|
5670
|
+
if (failuresInARow >= SPLIT_STOP_AFTER_FAILURES && decided.size === 0) {
|
|
5671
|
+
const ok = this.options.checkRuntime ? await run3.exclude(() => this.runtimeCheck()) : false;
|
|
5672
|
+
if (run3.stale()) break splitting;
|
|
5673
|
+
if (!ok) {
|
|
5674
|
+
out.splitStopped = true;
|
|
5675
|
+
this.failureCounts.splitsStopped += 1;
|
|
5676
|
+
this.emit({
|
|
5677
|
+
type: "split_stopped",
|
|
5678
|
+
cycle: run3.token,
|
|
5679
|
+
remaining: queue.filter((item) => !decided.has(item.card.conversation_id)).length,
|
|
5680
|
+
reason: this.options.checkRuntime ? "runtime_check_failed" : "no_runtime_check"
|
|
5681
|
+
});
|
|
5682
|
+
break splitting;
|
|
5683
|
+
}
|
|
5684
|
+
failuresInARow = 0;
|
|
5685
|
+
}
|
|
5686
|
+
}
|
|
5687
|
+
if (run3.stale()) break;
|
|
5688
|
+
await run3.exclude(() => this.checkParked("gate"));
|
|
5689
|
+
if (queue.length === 0) break;
|
|
5690
|
+
}
|
|
5691
|
+
} finally {
|
|
5692
|
+
if (this.activeSplit === split) this.activeSplit = null;
|
|
5402
5693
|
}
|
|
5694
|
+
out.gated = gated.size;
|
|
5695
|
+
out.undecided = gated.size - decided.size;
|
|
5403
5696
|
return out;
|
|
5404
5697
|
}
|
|
5405
5698
|
/**
|
|
@@ -5447,27 +5740,194 @@ var ReceiveLoop = class {
|
|
|
5447
5740
|
const texts = await this.options.texts.ensure();
|
|
5448
5741
|
if (run3.stale()) return this.summary("abandoned", base);
|
|
5449
5742
|
if (texts === null) return this.summary("texts_missing", { ...base, failure: "the served texts are not available" });
|
|
5743
|
+
if (this.options.conversationRuns !== false && conversationRunTemplate(texts) !== null) {
|
|
5744
|
+
if (tray !== null) return this.conversationRunCycle(run3, tray, texts);
|
|
5745
|
+
if (this.runWork.size > 0 || this.waiting.length > 0) this.heldWhileRunning = true;
|
|
5746
|
+
return this.summary("tray_failed", { ...base, failure: this.lastError });
|
|
5747
|
+
}
|
|
5450
5748
|
const unready = await this.checkReady();
|
|
5451
5749
|
if (run3.stale()) return this.summary("abandoned", base);
|
|
5452
5750
|
if (unready !== null) return this.summary("tools_unavailable", { ...base, failure: unready });
|
|
5453
5751
|
const prompt = cards === 0 && (invites ?? 0) > 0 ? texts.invite_prompt.trim() : (invites ?? 0) > 0 ? `${texts.wake_instruction.trim()}
|
|
5454
5752
|
|
|
5455
5753
|
${texts.invite_prompt.trim()}` : texts.wake_instruction.trim();
|
|
5754
|
+
const kind = cards === 0 && (invites ?? 0) > 0 ? "invite" : "tray";
|
|
5456
5755
|
this.counts.runsStarted += 1;
|
|
5457
|
-
|
|
5756
|
+
const end = await run3.exclude(() => this.callRun({ prompt, texts, cards, invites, signal: this.abort.signal, kind }));
|
|
5757
|
+
this.countRunEnd(end);
|
|
5758
|
+
this.emit({ type: "run", end, kind });
|
|
5759
|
+
const startedWith = tray === null || this.options.dismissIgnoredInvites === false ? [] : trayInvites(tray);
|
|
5760
|
+
if (startedWith.length > 0) await this.track(this.afterRunInvites(startedWith, end));
|
|
5761
|
+
const outcome = end.status === "completed" ? "run_completed" : end.status === "failed" ? "run_failed" : "run_not_started";
|
|
5762
|
+
return this.summary(outcome, { ...base, failure: end.status === "completed" ? null : end.detail ?? null });
|
|
5763
|
+
}
|
|
5764
|
+
/**
|
|
5765
|
+
* Conversation runs (single-run mode, texts with `conversation_run_template`):
|
|
5766
|
+
* one run per owed card (unmuted, with a preview: a notices-only card is not
|
|
5767
|
+
* owed), prompted for that one conversation, and one invite run for the
|
|
5768
|
+
* pending invites. A conversation whose run is running or
|
|
5769
|
+
* waiting for a slot is held out (the tracker), and so is one paced after a
|
|
5770
|
+
* completed run that left its card at the same anchor (D5). Runs take a slot
|
|
5771
|
+
* each (`maxConcurrentTurns`) in the order queued: the cards whose last run
|
|
5772
|
+
* failed go last, after the invite run, so they never hold up the others.
|
|
5773
|
+
* The loop records no verdict for a card: the run decides.
|
|
5774
|
+
*/
|
|
5775
|
+
async conversationRunCycle(run3, tray, texts) {
|
|
5776
|
+
const cards = cardsOf(tray);
|
|
5777
|
+
const invites = trayInvites(tray);
|
|
5778
|
+
const unheld = this.unheld(cards);
|
|
5779
|
+
const heldByTurns = cards.length - unheld.length;
|
|
5780
|
+
if (heldByTurns > 0) this.heldWhileRunning = true;
|
|
5781
|
+
const runCards = unheld.filter((card) => !this.pacer.held(`conversation:${card.conversation_id}`, anchorFrom(card)));
|
|
5782
|
+
const openInvites = invites.filter((invite) => !this.tracker.inviteRunning(invite.invite_id));
|
|
5783
|
+
const owedInvites = openInvites.filter((invite) => !this.pacer.held(`invite:${invite.invite_id}`, null));
|
|
5784
|
+
const paced = unheld.length - runCards.length + (openInvites.length - owedInvites.length);
|
|
5785
|
+
const owedCards = runCards.filter((card) => card.muted !== true && card.preview != null);
|
|
5786
|
+
const base = { cards: cards.length, invites: invites.length, heldByTurns, paced, truncated: tray.truncated === true };
|
|
5787
|
+
if (owedCards.length === 0 && owedInvites.length === 0) return this.summary("nothing_owed", base);
|
|
5788
|
+
const unready = await this.checkReady();
|
|
5789
|
+
if (run3.stale()) return this.summary("abandoned", base);
|
|
5790
|
+
if (unready !== null) return this.summary("tools_unavailable", { ...base, failure: unready });
|
|
5791
|
+
const tally = { turnsStarted: 0, turnsRefused: 0, turnsQueued: 0 };
|
|
5792
|
+
const failedLast = owedCards.filter((card) => this.awaitingEvidence.has(card.conversation_id));
|
|
5793
|
+
for (const card of owedCards.filter((card2) => !failedLast.includes(card2))) {
|
|
5794
|
+
if (run3.stale()) break;
|
|
5795
|
+
tally[launchKey(await this.startConversationRun(card, texts))] += 1;
|
|
5796
|
+
}
|
|
5797
|
+
if (owedInvites.length > 0 && !run3.stale()) tally[launchKey(await this.startInviteRun(owedInvites, texts))] += 1;
|
|
5798
|
+
for (const card of failedLast) {
|
|
5799
|
+
if (run3.stale()) break;
|
|
5800
|
+
tally[launchKey(await this.startConversationRun(card, texts))] += 1;
|
|
5801
|
+
}
|
|
5802
|
+
const parts = { ...base, ...tally };
|
|
5803
|
+
if (run3.stale()) return this.summary("abandoned", parts);
|
|
5804
|
+
const launched = tally.turnsStarted + tally.turnsQueued;
|
|
5805
|
+
if (tray.truncated === true) this.readTheRest(launched > 0);
|
|
5806
|
+
return this.summary(launched > 0 ? "runs_started" : "nothing_owed", parts);
|
|
5807
|
+
}
|
|
5808
|
+
/** One conversation run: a slot (or a place in the queue), the run, then what its end says. */
|
|
5809
|
+
async startConversationRun(card, texts, reruns = 0) {
|
|
5810
|
+
const conversationId = card.conversation_id;
|
|
5811
|
+
const anchor = anchorFrom(card);
|
|
5812
|
+
const handle = this.tracker.begin({ kind: "reply", conversationId, anchor });
|
|
5813
|
+
if (handle === null) return "refused";
|
|
5814
|
+
const request = {
|
|
5815
|
+
kind: "conversation",
|
|
5816
|
+
conversationId,
|
|
5817
|
+
prompt: conversationRunPrompt(texts, conversationId),
|
|
5818
|
+
texts,
|
|
5819
|
+
cards: 1,
|
|
5820
|
+
invites: 0,
|
|
5821
|
+
signal: this.abort.signal
|
|
5822
|
+
};
|
|
5823
|
+
return this.launch(handle, async () => {
|
|
5824
|
+
const attempt = { conversationId, source: "run", card, anchor, startedAt: this.evidence.start(), reruns };
|
|
5825
|
+
this.counts.runsStarted += 1;
|
|
5826
|
+
this.trackRun(this.afterConversationRun(handle, attempt, this.callRun(request)));
|
|
5827
|
+
return true;
|
|
5828
|
+
});
|
|
5829
|
+
}
|
|
5830
|
+
async afterConversationRun(handle, attempt, done) {
|
|
5831
|
+
const end = await done;
|
|
5832
|
+
const held = this.tracker.holds(handle);
|
|
5833
|
+
let completion = null;
|
|
5834
|
+
let failure = null;
|
|
5458
5835
|
try {
|
|
5459
|
-
|
|
5460
|
-
|
|
5461
|
-
|
|
5836
|
+
this.countRunEnd(end);
|
|
5837
|
+
this.emit({ type: "run", end, kind: "conversation", conversationId: attempt.conversationId });
|
|
5838
|
+
if (!held) return;
|
|
5839
|
+
if (end.status === "completed") {
|
|
5840
|
+
completion = this.completed("run", [attempt.conversationId]);
|
|
5841
|
+
this.pacer.hold(`conversation:${attempt.conversationId}`, attempt.anchor);
|
|
5842
|
+
} else if (end.status === "failed" && !this.stopped) {
|
|
5843
|
+
this.failedAttempt("run", [attempt.conversationId]);
|
|
5844
|
+
if (untouched(attempt.card, await this.readTray())) {
|
|
5845
|
+
failure = { ...attempt, outcome: end.timedOut === true ? "timed_out" : "failed", failedAt: this.now(), holding: handle };
|
|
5846
|
+
}
|
|
5847
|
+
}
|
|
5848
|
+
} finally {
|
|
5849
|
+
if (failure === null) this.tracker.end(handle);
|
|
5850
|
+
this.releaseSlot(handle);
|
|
5851
|
+
if (failure === null) this.afterTurnEnded();
|
|
5852
|
+
}
|
|
5853
|
+
if (failure !== null) {
|
|
5854
|
+
try {
|
|
5855
|
+
if (!this.stopped) await this.failed(failure);
|
|
5856
|
+
} finally {
|
|
5857
|
+
this.tracker.end(handle);
|
|
5858
|
+
this.afterTurnEnded();
|
|
5859
|
+
}
|
|
5462
5860
|
}
|
|
5861
|
+
if (!this.stopped && completion !== null) await this.reportDue(completion);
|
|
5862
|
+
await this.checkParked("run");
|
|
5863
|
+
}
|
|
5864
|
+
/** The invite run: one run for the pending invites, holding each of them until it ends. */
|
|
5865
|
+
async startInviteRun(invites, texts) {
|
|
5866
|
+
const handles = [];
|
|
5867
|
+
const held = [];
|
|
5868
|
+
for (const invite of invites) {
|
|
5869
|
+
const handle = this.tracker.begin({ kind: "invite", inviteId: invite.invite_id });
|
|
5870
|
+
if (handle === null) continue;
|
|
5871
|
+
handles.push(handle);
|
|
5872
|
+
held.push(invite);
|
|
5873
|
+
}
|
|
5874
|
+
const slot = handles[0];
|
|
5875
|
+
if (slot === void 0) return "refused";
|
|
5876
|
+
const endAll = () => {
|
|
5877
|
+
for (const handle of handles) this.tracker.end(handle);
|
|
5878
|
+
};
|
|
5879
|
+
const request = {
|
|
5880
|
+
kind: "invite",
|
|
5881
|
+
prompt: texts.invite_prompt.trim(),
|
|
5882
|
+
texts,
|
|
5883
|
+
cards: 0,
|
|
5884
|
+
invites: held.length,
|
|
5885
|
+
signal: this.abort.signal
|
|
5886
|
+
};
|
|
5887
|
+
return this.launch(
|
|
5888
|
+
slot,
|
|
5889
|
+
async () => {
|
|
5890
|
+
this.counts.runsStarted += 1;
|
|
5891
|
+
this.trackRun(this.afterInviteRun(slot, held, endAll, this.callRun(request)));
|
|
5892
|
+
return true;
|
|
5893
|
+
},
|
|
5894
|
+
endAll
|
|
5895
|
+
);
|
|
5896
|
+
}
|
|
5897
|
+
async afterInviteRun(slot, invites, endAll, done) {
|
|
5898
|
+
const end = await done;
|
|
5899
|
+
const held = this.tracker.holds(slot);
|
|
5900
|
+
let completion = null;
|
|
5901
|
+
try {
|
|
5902
|
+
this.countRunEnd(end);
|
|
5903
|
+
this.emit({ type: "run", end, kind: "invite" });
|
|
5904
|
+
if (!held) return;
|
|
5905
|
+
if (end.status === "completed") {
|
|
5906
|
+
completion = this.completed("invite", []);
|
|
5907
|
+
for (const invite of invites) this.pacer.hold(`invite:${invite.invite_id}`, null);
|
|
5908
|
+
}
|
|
5909
|
+
if (this.options.dismissIgnoredInvites !== false) await this.afterRunInvites(invites, end);
|
|
5910
|
+
} finally {
|
|
5911
|
+
endAll();
|
|
5912
|
+
this.releaseSlot(slot);
|
|
5913
|
+
this.afterTurnEnded();
|
|
5914
|
+
}
|
|
5915
|
+
if (!this.stopped && completion !== null) await this.reportDue(completion);
|
|
5916
|
+
await this.checkParked("run");
|
|
5917
|
+
}
|
|
5918
|
+
/** The runtime's `startRun`, never rejecting: a throw is a run that did not start. */
|
|
5919
|
+
callRun(request) {
|
|
5920
|
+
return Promise.resolve().then(() => this.options.startRun(request)).catch((err) => ({ status: "not_started", detail: err instanceof Error ? err.message : String(err) }));
|
|
5921
|
+
}
|
|
5922
|
+
countRunEnd(end) {
|
|
5463
5923
|
if (end.status === "completed") this.counts.runsCompleted += 1;
|
|
5464
5924
|
else if (end.status === "failed") this.counts.runsFailed += 1;
|
|
5465
5925
|
else this.counts.runsNotStarted += 1;
|
|
5466
|
-
|
|
5467
|
-
|
|
5468
|
-
|
|
5469
|
-
const
|
|
5470
|
-
|
|
5926
|
+
}
|
|
5927
|
+
/** A run's whole course after its start: `idle()` and `stop()` wait for it. */
|
|
5928
|
+
trackRun(work) {
|
|
5929
|
+
const tracked = work.catch((err) => this.fail(err)).finally(() => this.runWork.delete(tracked));
|
|
5930
|
+
this.runWork.add(tracked);
|
|
5471
5931
|
}
|
|
5472
5932
|
async afterRunInvites(invites, end) {
|
|
5473
5933
|
const completed = end.status === "completed";
|
|
@@ -5478,7 +5938,7 @@ ${texts.invite_prompt.trim()}` : texts.wake_instruction.trim();
|
|
|
5478
5938
|
}
|
|
5479
5939
|
}
|
|
5480
5940
|
// ─── turns (gate-call mode) ─────────────────────────────────────────────
|
|
5481
|
-
async startReplyTurn(reply, texts) {
|
|
5941
|
+
async startReplyTurn(reply, texts, reruns = 0) {
|
|
5482
5942
|
const anchor = anchorFrom(reply.card);
|
|
5483
5943
|
const handle = this.tracker.begin({ kind: "reply", conversationId: reply.conversationId, anchor });
|
|
5484
5944
|
if (handle === null) return "refused";
|
|
@@ -5493,6 +5953,7 @@ ${texts.invite_prompt.trim()}` : texts.wake_instruction.trim();
|
|
|
5493
5953
|
signal: this.abort.signal
|
|
5494
5954
|
};
|
|
5495
5955
|
return this.launch(handle, async () => {
|
|
5956
|
+
const startedAt = this.evidence.start();
|
|
5496
5957
|
const admission = await this.admit(this.options.startTurn, request);
|
|
5497
5958
|
if (!admission.started) {
|
|
5498
5959
|
this.tracker.end(handle);
|
|
@@ -5502,7 +5963,8 @@ ${texts.invite_prompt.trim()}` : texts.wake_instruction.trim();
|
|
|
5502
5963
|
}
|
|
5503
5964
|
this.counts.turnsStarted += 1;
|
|
5504
5965
|
this.emit({ type: "turn_started", request: withoutRuntime(request) });
|
|
5505
|
-
|
|
5966
|
+
const attempt = { conversationId: reply.conversationId, source: "turn", card: reply.card, anchor, startedAt, reruns, reply };
|
|
5967
|
+
void this.afterReplyTurn(handle, attempt, admission.done);
|
|
5506
5968
|
return true;
|
|
5507
5969
|
});
|
|
5508
5970
|
}
|
|
@@ -5542,9 +6004,9 @@ invite_id: ${invite.invite_id}`,
|
|
|
5542
6004
|
* queues behind them instead of taking the slot. A refused admission here
|
|
5543
6005
|
* hands its slot on the same way.
|
|
5544
6006
|
*/
|
|
5545
|
-
async launch(handle, admit) {
|
|
6007
|
+
async launch(handle, admit, drop) {
|
|
5546
6008
|
if (this.waiting.length > 0 || this.activeTurns >= this.maxTurns) {
|
|
5547
|
-
this.waiting.push({ handle, admit });
|
|
6009
|
+
this.waiting.push({ handle, admit, ...drop ? { drop } : {} });
|
|
5548
6010
|
this.counts.turnsQueued += 1;
|
|
5549
6011
|
if (this.activeTurns < this.maxTurns) void this.pump();
|
|
5550
6012
|
return "queued";
|
|
@@ -5564,7 +6026,7 @@ invite_id: ${invite.invite_id}`,
|
|
|
5564
6026
|
releaseSlot(handle, options = {}) {
|
|
5565
6027
|
if (this.slotted.delete(handle)) this.activeTurns = Math.max(0, this.activeTurns - 1);
|
|
5566
6028
|
const queued = this.waiting.findIndex((w) => w.handle === handle);
|
|
5567
|
-
if (queued >= 0) this.waiting.splice(queued, 1);
|
|
6029
|
+
if (queued >= 0) this.waiting.splice(queued, 1)[0].drop?.();
|
|
5568
6030
|
if (options.pump !== false) void this.pump();
|
|
5569
6031
|
}
|
|
5570
6032
|
async pump() {
|
|
@@ -5574,20 +6036,38 @@ invite_id: ${invite.invite_id}`,
|
|
|
5574
6036
|
await this.admitInSlot(next.handle, next.admit);
|
|
5575
6037
|
}
|
|
5576
6038
|
}
|
|
5577
|
-
async afterReplyTurn(handle,
|
|
6039
|
+
async afterReplyTurn(handle, attempt, done) {
|
|
6040
|
+
const reply = attempt.reply;
|
|
6041
|
+
let completion = null;
|
|
6042
|
+
let failure = null;
|
|
5578
6043
|
try {
|
|
5579
6044
|
const end = await this.tracker.confirmedEnd(handle, done);
|
|
5580
6045
|
if (end === null || !this.tracker.holds(handle)) return;
|
|
5581
|
-
|
|
6046
|
+
if (end.status === "completed") completion = this.completed("turn", [reply.conversationId]);
|
|
6047
|
+
const recorded = await this.track(this.recordReplyOutcome(handle, reply, end));
|
|
6048
|
+
if (end.status === "failed" && !this.stopped) this.failedAttempt("turn", [reply.conversationId]);
|
|
6049
|
+
if (end.status === "failed" && recorded?.freshRead === true && recorded.outcome.reason === "turn_failed") {
|
|
6050
|
+
failure = { ...attempt, outcome: end.timedOut === true ? "timed_out" : "failed", failedAt: this.now(), holding: handle };
|
|
6051
|
+
}
|
|
5582
6052
|
} finally {
|
|
5583
|
-
this.tracker.end(handle);
|
|
6053
|
+
if (failure === null) this.tracker.end(handle);
|
|
5584
6054
|
this.releaseSlot(handle);
|
|
5585
|
-
this.afterTurnEnded();
|
|
6055
|
+
if (failure === null) this.afterTurnEnded();
|
|
5586
6056
|
}
|
|
6057
|
+
if (failure !== null) {
|
|
6058
|
+
try {
|
|
6059
|
+
if (!this.stopped) await this.track(this.failed(failure));
|
|
6060
|
+
} finally {
|
|
6061
|
+
this.tracker.end(handle);
|
|
6062
|
+
this.afterTurnEnded();
|
|
6063
|
+
}
|
|
6064
|
+
}
|
|
6065
|
+
if (!this.stopped && completion !== null) await this.track(this.reportDue(completion));
|
|
6066
|
+
await this.track(this.checkParked("turn"));
|
|
5587
6067
|
}
|
|
5588
6068
|
async recordReplyOutcome(handle, reply, end) {
|
|
5589
6069
|
const fresh = await this.readTray();
|
|
5590
|
-
if (!this.tracker.holds(handle)) return;
|
|
6070
|
+
if (!this.tracker.holds(handle)) return null;
|
|
5591
6071
|
const anchor = anchorFrom(reply.card);
|
|
5592
6072
|
const key = `${reply.conversationId}|${anchor ?? ""}`;
|
|
5593
6073
|
const silentBefore = this.silent.get(key) ?? 0;
|
|
@@ -5617,6 +6097,7 @@ invite_id: ${invite.invite_id}`,
|
|
|
5617
6097
|
}
|
|
5618
6098
|
}
|
|
5619
6099
|
this.emit({ type: "turn_outcome", outcome, recorded });
|
|
6100
|
+
return { outcome, freshRead: fresh !== null };
|
|
5620
6101
|
}
|
|
5621
6102
|
async afterInviteTurn(handle, invite, done) {
|
|
5622
6103
|
try {
|
|
@@ -5686,6 +6167,257 @@ invite_id: ${invite.invite_id}`,
|
|
|
5686
6167
|
this.wakeFrom("more");
|
|
5687
6168
|
}
|
|
5688
6169
|
}
|
|
6170
|
+
// ─── could not process: failures, evidence, the server's answer ─────────
|
|
6171
|
+
/**
|
|
6172
|
+
* A run, turn or gate call that completed, on the evidence clock. A failure
|
|
6173
|
+
* it clears (the same kind of attempt for the same conversation went
|
|
6174
|
+
* through since) waits no more. Call it the moment the work ended, before
|
|
6175
|
+
* its slot frees, so the next attempt starts after it.
|
|
6176
|
+
*/
|
|
6177
|
+
completed(kind, conversations) {
|
|
6178
|
+
const completion = this.evidence.complete(kind, conversations);
|
|
6179
|
+
if (kind !== "invite") this.failingSince[kind].clear();
|
|
6180
|
+
for (const id of conversations) {
|
|
6181
|
+
const failure = this.awaitingEvidence.get(id);
|
|
6182
|
+
if (failure !== void 0 && clears(completion, failure)) this.awaitingEvidence.delete(id);
|
|
6183
|
+
}
|
|
6184
|
+
return completion;
|
|
6185
|
+
}
|
|
6186
|
+
/**
|
|
6187
|
+
* Report the failures this completion is the evidence for (each once). One
|
|
6188
|
+
* that waited past `AWAITING_EVIDENCE_MS` is dropped instead: a completion
|
|
6189
|
+
* that late says little about the runtime when it failed.
|
|
6190
|
+
*/
|
|
6191
|
+
async reportDue(completion) {
|
|
6192
|
+
const now2 = this.now();
|
|
6193
|
+
const due = [];
|
|
6194
|
+
for (const failure of [...this.awaitingEvidence.values()]) {
|
|
6195
|
+
if (now2 - failure.failedAt > AWAITING_EVIDENCE_MS) {
|
|
6196
|
+
this.awaitingEvidence.delete(failure.conversationId);
|
|
6197
|
+
continue;
|
|
6198
|
+
}
|
|
6199
|
+
if (evidences(completion, failure)) {
|
|
6200
|
+
this.awaitingEvidence.delete(failure.conversationId);
|
|
6201
|
+
due.push(failure);
|
|
6202
|
+
}
|
|
6203
|
+
}
|
|
6204
|
+
for (const failure of due) await this.reportFailure(failure, "other_run_completed");
|
|
6205
|
+
}
|
|
6206
|
+
/** An attempt of this kind failed for these conversations (whatever comes of it): see `failingSince`. */
|
|
6207
|
+
failedAttempt(kind, conversations) {
|
|
6208
|
+
for (const id of conversations) this.failingSince[kind].add(id);
|
|
6209
|
+
}
|
|
6210
|
+
/**
|
|
6211
|
+
* A conversation's run, reply turn or single-card gate call failed. It is
|
|
6212
|
+
* reported only with evidence that the runtime otherwise works:
|
|
6213
|
+
* (a) a later completion of the same kind of work for another conversation
|
|
6214
|
+
* (`evidences`), or (b) `checkRuntime()` passing while no other
|
|
6215
|
+
* conversation of that kind has failed since that kind last completed.
|
|
6216
|
+
* Without (a) it waits: the work already queued runs first (cheap
|
|
6217
|
+
* evidence), and the runtime check is asked only once nothing of that kind
|
|
6218
|
+
* is queued (`checkParked`). It waits at most `AWAITING_EVIDENCE_MS`.
|
|
6219
|
+
* Nothing is reported after `stop()`: a run the stop cut short did not
|
|
6220
|
+
* fail on its conversation.
|
|
6221
|
+
*/
|
|
6222
|
+
async failed(failure) {
|
|
6223
|
+
if (this.stopped) return;
|
|
6224
|
+
this.awaitingEvidence.delete(failure.conversationId);
|
|
6225
|
+
if (this.evidence.since(failure)) {
|
|
6226
|
+
await this.reportFailure(failure, "other_run_completed");
|
|
6227
|
+
return;
|
|
6228
|
+
}
|
|
6229
|
+
if (!this.canReport(failure)) return;
|
|
6230
|
+
this.awaitingEvidence.set(failure.conversationId, failure);
|
|
6231
|
+
while (this.awaitingEvidence.size > SILENT_MEMORY) {
|
|
6232
|
+
const oldest = this.awaitingEvidence.keys().next().value;
|
|
6233
|
+
if (oldest === void 0) break;
|
|
6234
|
+
this.awaitingEvidence.delete(oldest);
|
|
6235
|
+
}
|
|
6236
|
+
if (failure.source !== "gate") await this.checkParked(failure.source);
|
|
6237
|
+
}
|
|
6238
|
+
/**
|
|
6239
|
+
* Evidence (b), once nothing of this kind is queued: ask `checkRuntime()`
|
|
6240
|
+
* about the failure still waiting for evidence, if it is the only
|
|
6241
|
+
* conversation of that kind failing since that kind last completed. Several
|
|
6242
|
+
* failing together is the runtime: none of them is reported on (b), and
|
|
6243
|
+
* they keep waiting for (a). A `timed_out` failure is never reported on
|
|
6244
|
+
* (b): a timeout is ambiguous (a brownout, a hung hook) and the check does
|
|
6245
|
+
* not exercise those paths; it waits for (a). Each failure is asked about
|
|
6246
|
+
* once. A check that fails drops it (a broken runtime reports nothing).
|
|
6247
|
+
*/
|
|
6248
|
+
async checkParked(kind) {
|
|
6249
|
+
if (!this.options.checkRuntime || this.stopped || !this.idleFor(kind)) return;
|
|
6250
|
+
const failing = this.failingSince[kind];
|
|
6251
|
+
const due = [...this.awaitingEvidence.values()].filter(
|
|
6252
|
+
(failure) => failure.source === kind && // A timeout is ambiguous (a brownout, a hung hook) and the runtime
|
|
6253
|
+
// check does not exercise those paths: only (a) can report it.
|
|
6254
|
+
failure.outcome !== "timed_out" && !this.checkedFailures.has(failure) && this.now() - failure.failedAt <= AWAITING_EVIDENCE_MS && [...failing].every((id) => id === failure.conversationId)
|
|
6255
|
+
);
|
|
6256
|
+
if (due.length === 0) return;
|
|
6257
|
+
for (const failure of due) this.checkedFailures.add(failure);
|
|
6258
|
+
const ok = await this.runtimeCheck();
|
|
6259
|
+
for (const failure of due) {
|
|
6260
|
+
if (this.stopped || this.awaitingEvidence.get(failure.conversationId) !== failure) continue;
|
|
6261
|
+
if (!ok) {
|
|
6262
|
+
this.awaitingEvidence.delete(failure.conversationId);
|
|
6263
|
+
continue;
|
|
6264
|
+
}
|
|
6265
|
+
if (![...failing].every((id) => id === failure.conversationId)) continue;
|
|
6266
|
+
this.awaitingEvidence.delete(failure.conversationId);
|
|
6267
|
+
await this.reportFailure(failure, "runtime_check");
|
|
6268
|
+
}
|
|
6269
|
+
}
|
|
6270
|
+
/** Nothing of this kind is running or queued: the runtime check may be asked now. */
|
|
6271
|
+
idleFor(kind) {
|
|
6272
|
+
if (kind === "gate") return this.activeSplit === null || this.activeSplit.queue.length === 0;
|
|
6273
|
+
if (kind === "turn" && this.activeSplit !== null) return false;
|
|
6274
|
+
return this.activeTurns === 0 && this.waiting.length === 0;
|
|
6275
|
+
}
|
|
6276
|
+
canReport(failure) {
|
|
6277
|
+
return failure.anchor !== null && typeof this.client.conversations?.reportProcessingFailure === "function";
|
|
6278
|
+
}
|
|
6279
|
+
/** `checkRuntime()`, bounded by `RUNTIME_CHECK_TIMEOUT_MS`; a throw or a timeout is not ok. */
|
|
6280
|
+
async runtimeCheck() {
|
|
6281
|
+
const check = this.options.checkRuntime;
|
|
6282
|
+
let timer = null;
|
|
6283
|
+
let answer;
|
|
6284
|
+
try {
|
|
6285
|
+
answer = await Promise.race([
|
|
6286
|
+
Promise.resolve().then(() => check()),
|
|
6287
|
+
new Promise((resolve) => {
|
|
6288
|
+
timer = this._setTimeout(
|
|
6289
|
+
() => resolve({ ok: false, detail: `no answer within ${RUNTIME_CHECK_TIMEOUT_MS} ms` }),
|
|
6290
|
+
RUNTIME_CHECK_TIMEOUT_MS
|
|
6291
|
+
);
|
|
6292
|
+
timer.unref?.();
|
|
6293
|
+
})
|
|
6294
|
+
]);
|
|
6295
|
+
} catch (err) {
|
|
6296
|
+
answer = { ok: false, detail: `the check threw: ${err instanceof Error ? err.message : String(err)}` };
|
|
6297
|
+
} finally {
|
|
6298
|
+
if (timer !== null) this._clearTimeout(timer);
|
|
6299
|
+
}
|
|
6300
|
+
const ok = answer?.ok === true;
|
|
6301
|
+
if (ok) this.failureCounts.runtimeChecksOk += 1;
|
|
6302
|
+
else this.failureCounts.runtimeChecksFailed += 1;
|
|
6303
|
+
this.lastRuntimeCheck = { at: new Date(this.now()).toISOString(), ok, detail: typeof answer?.detail === "string" ? answer.detail : null };
|
|
6304
|
+
return ok;
|
|
6305
|
+
}
|
|
6306
|
+
/** Tell the server, and do what it answers: run the conversation again (`retry`, `hidden`) or stop (`paused`, `none`). */
|
|
6307
|
+
async reportFailure(failure, evidence) {
|
|
6308
|
+
const conversations = this.client.conversations;
|
|
6309
|
+
if (!this.canReport(failure) || conversations === void 0 || this.stopped) return;
|
|
6310
|
+
const base = {
|
|
6311
|
+
conversationId: failure.conversationId,
|
|
6312
|
+
source: failure.source,
|
|
6313
|
+
throughMessageId: failure.anchor,
|
|
6314
|
+
outcome: failure.outcome,
|
|
6315
|
+
evidence
|
|
6316
|
+
};
|
|
6317
|
+
let result;
|
|
6318
|
+
try {
|
|
6319
|
+
result = await conversations.reportProcessingFailure(failure.conversationId, {
|
|
6320
|
+
throughMessageId: failure.anchor,
|
|
6321
|
+
outcome: failure.outcome,
|
|
6322
|
+
evidence
|
|
6323
|
+
});
|
|
6324
|
+
} catch (err) {
|
|
6325
|
+
this.failureCounts.reportErrors += 1;
|
|
6326
|
+
this.fail(err);
|
|
6327
|
+
this.emit({ type: "processing_failure", ...base, result: null, error: err instanceof Error ? err.message : String(err), next: "stop" });
|
|
6328
|
+
return;
|
|
6329
|
+
}
|
|
6330
|
+
this.failureCounts.reported += 1;
|
|
6331
|
+
this.failureCounts[result.action] += 1;
|
|
6332
|
+
let next = "stop";
|
|
6333
|
+
if (result.action === "retry" || result.action === "hidden") {
|
|
6334
|
+
if (failure.reruns >= MAX_RERUNS_PER_CYCLE) {
|
|
6335
|
+
next = "rerun_capped";
|
|
6336
|
+
this.failureCounts.rerunsCapped += 1;
|
|
6337
|
+
} else {
|
|
6338
|
+
next = this.rerun(failure, result.action === "hidden");
|
|
6339
|
+
}
|
|
6340
|
+
}
|
|
6341
|
+
if (result.action === "paused") this.dropWaiting(failure.conversationId);
|
|
6342
|
+
if (next === "stop" || next === "rerun_capped") this.activeSplit?.skip.add(failure.conversationId);
|
|
6343
|
+
this.emit({ type: "processing_failure", ...base, result, next });
|
|
6344
|
+
}
|
|
6345
|
+
/**
|
|
6346
|
+
* Run the conversation again after a `retry` or `hidden` answer, after the
|
|
6347
|
+
* others already queued, and say what was done. Its run or reply turn takes
|
|
6348
|
+
* a place at the back of the queue (it reads the conversation itself, so it
|
|
6349
|
+
* sees a hidden span as the note); a reply turn asked for while a gate is
|
|
6350
|
+
* deciding starts after that cycle's own turns. A gate call sees only the
|
|
6351
|
+
* card: after `retry` it joins the end of the running gate's single-card
|
|
6352
|
+
* calls (once, with the card as this cycle read it); after `hidden`, or
|
|
6353
|
+
* when no gate is running for it, the next cycle reads the tray again and
|
|
6354
|
+
* gates it alone. `running`: nothing started, because its next attempt is
|
|
6355
|
+
* under way already (running, queued, or being decided by the gate now).
|
|
6356
|
+
*/
|
|
6357
|
+
rerun(failure, hidden) {
|
|
6358
|
+
if (this.stopped) return "stop";
|
|
6359
|
+
const id = failure.conversationId;
|
|
6360
|
+
const reruns = failure.reruns + 1;
|
|
6361
|
+
this.pacer.release(`conversation:${id}`);
|
|
6362
|
+
if (failure.holding !== void 0) this.tracker.end(failure.holding);
|
|
6363
|
+
const split = this.activeSplit;
|
|
6364
|
+
const current = split?.members.get(id);
|
|
6365
|
+
if (failure.source === "gate") {
|
|
6366
|
+
this.rememberGateAlone(id);
|
|
6367
|
+
if (split !== null && current !== void 0 && !hidden) {
|
|
6368
|
+
split.skip.delete(id);
|
|
6369
|
+
if (split.queue.some((item) => item.card.conversation_id === id)) return "running";
|
|
6370
|
+
split.queue.push({ card: current, reruns });
|
|
6371
|
+
} else {
|
|
6372
|
+
split?.skip.add(id);
|
|
6373
|
+
this.wakeFrom("more");
|
|
6374
|
+
}
|
|
6375
|
+
this.failureCounts.reruns += 1;
|
|
6376
|
+
return "rerun";
|
|
6377
|
+
}
|
|
6378
|
+
if (this.tracker.conversationRunning(id) || current !== void 0) return "running";
|
|
6379
|
+
const texts = this.options.texts.current();
|
|
6380
|
+
this.failureCounts.reruns += 1;
|
|
6381
|
+
if (texts !== null && failure.source === "turn" && failure.reply !== void 0) {
|
|
6382
|
+
if (split !== null) this.turnRerunsAfterGate.push({ reply: failure.reply, reruns });
|
|
6383
|
+
else void this.startReplyTurn(failure.reply, texts, reruns);
|
|
6384
|
+
} else if (texts !== null && failure.source === "run" && conversationRunTemplate(texts) !== null) {
|
|
6385
|
+
void this.startConversationRun(failure.card, texts, reruns);
|
|
6386
|
+
} else {
|
|
6387
|
+
this.wakeFrom("more");
|
|
6388
|
+
}
|
|
6389
|
+
return "rerun";
|
|
6390
|
+
}
|
|
6391
|
+
/** The reply turns asked for while a gate was deciding: started now, after the cycle's own turns. */
|
|
6392
|
+
flushTurnReruns() {
|
|
6393
|
+
const due = this.turnRerunsAfterGate.splice(0);
|
|
6394
|
+
if (this.stopped || due.length === 0) return;
|
|
6395
|
+
const texts = this.options.texts.current();
|
|
6396
|
+
for (const { reply, reruns } of due) {
|
|
6397
|
+
if (this.tracker.conversationRunning(reply.conversationId)) continue;
|
|
6398
|
+
if (texts === null) this.wakeFrom("more");
|
|
6399
|
+
else void this.startReplyTurn(reply, texts, reruns);
|
|
6400
|
+
}
|
|
6401
|
+
}
|
|
6402
|
+
/** A `paused` conversation: a run or turn still waiting for a slot for it does not start. */
|
|
6403
|
+
dropWaiting(conversationId) {
|
|
6404
|
+
for (const entry of [...this.waiting]) {
|
|
6405
|
+
const turn = entry.handle.turn;
|
|
6406
|
+
if (turn.kind !== "reply" || turn.conversationId !== conversationId) continue;
|
|
6407
|
+
this.waiting.splice(this.waiting.indexOf(entry), 1);
|
|
6408
|
+
this.tracker.end(entry.handle);
|
|
6409
|
+
entry.drop?.();
|
|
6410
|
+
}
|
|
6411
|
+
}
|
|
6412
|
+
rememberGateAlone(conversationId) {
|
|
6413
|
+
this.gateAlone.delete(conversationId);
|
|
6414
|
+
this.gateAlone.add(conversationId);
|
|
6415
|
+
while (this.gateAlone.size > SILENT_MEMORY) {
|
|
6416
|
+
const oldest = this.gateAlone.values().next().value;
|
|
6417
|
+
if (oldest === void 0) break;
|
|
6418
|
+
this.gateAlone.delete(oldest);
|
|
6419
|
+
}
|
|
6420
|
+
}
|
|
5689
6421
|
// ─── small things ───────────────────────────────────────────────────────
|
|
5690
6422
|
/** Recording under way after a turn's confirmed end: `stop()` waits for it. */
|
|
5691
6423
|
track(work) {
|
|
@@ -5725,6 +6457,9 @@ invite_id: ${invite.invite_id}`,
|
|
|
5725
6457
|
paced: 0,
|
|
5726
6458
|
chunksOk: 0,
|
|
5727
6459
|
chunksFailed: 0,
|
|
6460
|
+
splitCallsOk: 0,
|
|
6461
|
+
splitCallsFailed: 0,
|
|
6462
|
+
splitStopped: false,
|
|
5728
6463
|
truncated: false,
|
|
5729
6464
|
failure: null,
|
|
5730
6465
|
...parts
|
|
@@ -5755,6 +6490,23 @@ conversation_id: ${conversationId}`;
|
|
|
5755
6490
|
|
|
5756
6491
|
conversation_id: ${conversationId}`;
|
|
5757
6492
|
}
|
|
6493
|
+
function conversationRunTemplate(texts) {
|
|
6494
|
+
const raw = texts.conversation_run_template;
|
|
6495
|
+
const template = typeof raw === "string" ? raw.trim() : "";
|
|
6496
|
+
return template.length > 0 ? template : null;
|
|
6497
|
+
}
|
|
6498
|
+
function conversationRunPrompt(texts, conversationId) {
|
|
6499
|
+
const template = conversationRunTemplate(texts);
|
|
6500
|
+
if (template === null) return replyPrompt(texts, conversationId);
|
|
6501
|
+
if (template.includes(REPLY_TURN_PLACEHOLDER)) return template.split(REPLY_TURN_PLACEHOLDER).join(conversationId);
|
|
6502
|
+
return `${template}
|
|
6503
|
+
|
|
6504
|
+
conversation_id: ${conversationId}`;
|
|
6505
|
+
}
|
|
6506
|
+
function untouched(card, fresh) {
|
|
6507
|
+
if (fresh === null) return false;
|
|
6508
|
+
return turnOutcome({ turn: { status: "failed" }, gateCard: card, freshTray: fresh }).reason === "turn_failed";
|
|
6509
|
+
}
|
|
5758
6510
|
function chunkFailure(chunks) {
|
|
5759
6511
|
const failed = chunks.filter((chunk) => chunk.outcome === "fail_closed");
|
|
5760
6512
|
const first = failed[0];
|
|
@@ -5763,6 +6515,34 @@ function chunkFailure(chunks) {
|
|
|
5763
6515
|
if (chunks.length === 1) return reason;
|
|
5764
6516
|
return `${failed.length} of ${chunks.length} gate chunks failed; chunk ${first.index}: ${reason}`;
|
|
5765
6517
|
}
|
|
6518
|
+
function gateFailureText(chunks, split) {
|
|
6519
|
+
const parts = [];
|
|
6520
|
+
const chunked = chunkFailure(chunks);
|
|
6521
|
+
if (chunked !== null) parts.push(chunked);
|
|
6522
|
+
const failed = split.filter((call2) => call2.outcome === "fail_closed");
|
|
6523
|
+
const first = failed[0];
|
|
6524
|
+
if (first !== void 0) {
|
|
6525
|
+
const reason = first.failure ?? "no usable verdicts in the answer";
|
|
6526
|
+
parts.push(`${failed.length} of ${split.length} single-card calls failed; ${first.conversationId ?? "a card"}: ${reason}`);
|
|
6527
|
+
}
|
|
6528
|
+
return parts.length === 0 ? null : parts.join("; ");
|
|
6529
|
+
}
|
|
6530
|
+
function gateFailure(card, startedAt, reruns, timedOut, failedAt) {
|
|
6531
|
+
return {
|
|
6532
|
+
conversationId: card.conversation_id,
|
|
6533
|
+
source: "gate",
|
|
6534
|
+
card,
|
|
6535
|
+
anchor: anchorFrom(card),
|
|
6536
|
+
startedAt,
|
|
6537
|
+
reruns,
|
|
6538
|
+
outcome: timedOut ? "timed_out" : "failed",
|
|
6539
|
+
failedAt
|
|
6540
|
+
};
|
|
6541
|
+
}
|
|
6542
|
+
function decidedBy(result) {
|
|
6543
|
+
const undecided = new Set(result.undecided);
|
|
6544
|
+
return result.verdicts.map((verdict) => verdict.conversation_id).filter((id) => !undecided.has(id));
|
|
6545
|
+
}
|
|
5766
6546
|
function timeoutOption(value, fallback, name) {
|
|
5767
6547
|
if (value === void 0) return fallback;
|
|
5768
6548
|
if (value === 0 || value === Number.POSITIVE_INFINITY) return Number.POSITIVE_INFINITY;
|