@sentry/junior 0.136.1 → 0.138.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{agent-hooks-2KDSPPO4.js → agent-hooks-SCR74Q3E.js} +11 -9
- package/dist/api/conversations/projection.d.ts +1 -0
- package/dist/api/conversations/stats.d.ts +1 -0
- package/dist/api/schema/conversation.d.ts +2 -0
- package/dist/api/schema.js +1 -1
- package/dist/api.js +187 -46
- package/dist/app.d.ts +8 -1
- package/dist/app.js +519 -860
- package/dist/{catalog-runtime-WSZQGC4D.js → catalog-runtime-IU3QF4GG.js} +5 -5
- package/dist/chat/agent/request.d.ts +32 -2
- package/dist/chat/agent-dispatch/heartbeat.d.ts +5 -0
- package/dist/chat/agent-dispatch/types.d.ts +1 -1
- package/dist/chat/agent-invocations/errors.d.ts +8 -0
- package/dist/chat/agent-invocations/spawn.d.ts +12 -0
- package/dist/chat/agent-invocations/store.d.ts +44 -0
- package/dist/chat/agent-invocations/types.d.ts +449 -0
- package/dist/chat/agent-invocations/work.d.ts +31 -0
- package/dist/chat/config.d.ts +1 -0
- package/dist/chat/configuration/defaults.d.ts +1 -1
- package/dist/chat/conversations/sql/store.d.ts +8 -0
- package/dist/chat/conversations/store.d.ts +9 -1
- package/dist/chat/experimental.d.ts +16 -0
- package/dist/chat/pi/client.d.ts +1 -8
- package/dist/chat/pi/gateway-auth.d.ts +16 -0
- package/dist/chat/runtime/agent-runner.d.ts +2 -1
- package/dist/chat/sandbox/docker.d.ts +17 -0
- package/dist/chat/services/agent-continue.d.ts +2 -0
- package/dist/chat/services/auth-pause.d.ts +0 -1
- package/dist/chat/services/mcp-auth-orchestration.d.ts +3 -2
- package/dist/chat/services/plugin-auth-orchestration.d.ts +3 -2
- package/dist/chat/services/subscribed-reply-router-policy.d.ts +12 -0
- package/dist/chat/services/turn-session-routing.d.ts +17 -0
- package/dist/chat/slack/message.d.ts +6 -1
- package/dist/chat/slack/source-link.d.ts +8 -2
- package/dist/chat/slack/team-domain.d.ts +4 -0
- package/dist/chat/source.d.ts +5 -5
- package/dist/chat/state/turn-session.d.ts +0 -2
- package/dist/chat/task-execution/state.d.ts +23 -11
- package/dist/chat/task-execution/store.d.ts +20 -11
- package/dist/chat/task-execution/worker.d.ts +2 -2
- package/dist/chat/tools/runtime/spawn-agent.d.ts +31 -0
- package/dist/chat/tools/types.d.ts +2 -0
- package/dist/chunk-3Y47VMHV.js +6 -0
- package/dist/chunk-5IDFPCON.js +46 -0
- package/dist/chunk-5LTJ4ITU.js +1523 -0
- package/dist/{chunk-KLBYCSLJ.js → chunk-5MXWHTCK.js} +17 -434
- package/dist/{chunk-CZZDCH3Q.js → chunk-A7WCHXZV.js} +206 -78
- package/dist/{chunk-4YQ6I6VF.js → chunk-BOLJOV5T.js} +2 -56
- package/dist/{chunk-XCZ6COFO.js → chunk-BYW4B4PR.js} +90 -54
- package/dist/{chunk-JQRTXJHD.js → chunk-C7ZVPO4Z.js} +2 -1
- package/dist/{chunk-MW2QRPGW.js → chunk-DCLLQGS6.js} +119 -50
- package/dist/{chunk-CDVCCOMF.js → chunk-E5UYAIKJ.js} +75 -1475
- package/dist/{chunk-3SUK3UXX.js → chunk-ELKW5FIB.js} +5 -5
- package/dist/chunk-EM225ITJ.js +56 -0
- package/dist/{chunk-Y6JNHTRD.js → chunk-FEUNPTGV.js} +34 -70
- package/dist/{chunk-TYNMSQKA.js → chunk-FNLBTB4K.js} +1 -1
- package/dist/{chunk-TRZSM67C.js → chunk-HCIVH3JS.js} +1 -0
- package/dist/chunk-HIWJWWMT.js +668 -0
- package/dist/chunk-IUFM5JWV.js +24 -0
- package/dist/{chunk-XWPA5KZF.js → chunk-JNNJZURV.js} +1 -1
- package/dist/{chunk-6ZK6FZIV.js → chunk-KCBZSJRU.js} +6 -27
- package/dist/{chunk-BG3KWWDJ.js → chunk-KDCHVT25.js} +1 -1
- package/dist/{chunk-VH6MI4L7.js → chunk-KFRHRSCJ.js} +1 -1
- package/dist/chunk-LPVUNO2W.js +320 -0
- package/dist/{chunk-G66RYKRS.js → chunk-N7WARMC6.js} +16 -776
- package/dist/chunk-PB3CXNVL.js +41 -0
- package/dist/chunk-Q6QZTTLS.js +55 -0
- package/dist/{chunk-3QNJOHKG.js → chunk-QS527AV5.js} +1 -1
- package/dist/chunk-R776HKPK.js +81 -0
- package/dist/{chunk-OTJMQYVP.js → chunk-RZIBSCYC.js} +517 -608
- package/dist/chunk-SD3OFRWY.js +714 -0
- package/dist/{chunk-HIRIWTWD.js → chunk-SIPKWSXK.js} +58 -10
- package/dist/chunk-TCS76X5D.js +441 -0
- package/dist/{chunk-UX2TNH5D.js → chunk-TDJCLNDH.js} +3 -3
- package/dist/{chunk-NVVBEJUB.js → chunk-TUUDXEXU.js} +336 -300
- package/dist/{chunk-22Z6MGMK.js → chunk-VPZFVJND.js} +1 -1
- package/dist/{chunk-RFDD3Z2I.js → chunk-YU5QUF67.js} +1 -1
- package/dist/chunk-ZBSMCXKV.js +710 -0
- package/dist/{chunk-FSYXYBWU.js → chunk-ZXNTJJIE.js} +2 -2
- package/dist/cli/chat.js +65 -26
- package/dist/cli/check.js +7 -7
- package/dist/cli/plugins.js +14 -12
- package/dist/cli/snapshot-create.js +10 -9
- package/dist/cli/upgrade.js +7 -7
- package/dist/conversation-work-KATFHLG3.js +56 -0
- package/dist/db/schema/agent-invocations.d.ts +544 -0
- package/dist/db/schema.d.ts +543 -1
- package/dist/{db-Z2NNL2JC.js → db-UO2IWXGA.js} +7 -7
- package/dist/experimental-HH6OM6UH.js +13 -0
- package/dist/instrumentation.js +4 -4
- package/dist/nitro.js +4 -3
- package/dist/{oauth-callback-server-PO3M5JMG.js → oauth-callback-server-MAQI43LX.js} +26 -18
- package/dist/{runner-QHQYZYJA.js → runner-PIUQEXKE.js} +26 -14
- package/dist/{skills-KXMT3DRA.js → skills-4ARJADHQ.js} +6 -6
- package/dist/spawn-G2AWPMCT.js +38 -0
- package/dist/{validation-Y42ULTT6.js → validation-S3EE3HWL.js} +6 -6
- package/dist/version.js +1 -1
- package/dist/work-UARRUAW3.js +46 -0
- package/dist/worker-DOCIRVD3.js +24 -0
- package/migrations/0021_first_warhawk.sql +35 -0
- package/migrations/meta/0021_snapshot.json +2176 -0
- package/migrations/meta/_journal.json +8 -1
- package/package.json +6 -5
|
@@ -0,0 +1,668 @@
|
|
|
1
|
+
import {
|
|
2
|
+
CONVERSATION_WORK_CHECK_IN_INTERVAL_MS,
|
|
3
|
+
CONVERSATION_WORK_MAX_RETRIES,
|
|
4
|
+
ackMessages,
|
|
5
|
+
beginConversationResume,
|
|
6
|
+
checkInConversationWork,
|
|
7
|
+
clearConsumedConversationWake,
|
|
8
|
+
completeConversationWork,
|
|
9
|
+
countPendingConversationMessages,
|
|
10
|
+
deadLetterAttempt,
|
|
11
|
+
drainConversationMailbox,
|
|
12
|
+
ensureConversationWake,
|
|
13
|
+
getConversationWorkState,
|
|
14
|
+
isFinalAttempt,
|
|
15
|
+
isInvalidConversationRecordError,
|
|
16
|
+
recordAttemptFailure,
|
|
17
|
+
recordConversationRetry,
|
|
18
|
+
releaseConversationWork,
|
|
19
|
+
requestConversationContinuation,
|
|
20
|
+
startConversationWork
|
|
21
|
+
} from "./chunk-5LTJ4ITU.js";
|
|
22
|
+
import {
|
|
23
|
+
ConversationQueueMessageRejectedError
|
|
24
|
+
} from "./chunk-IUFM5JWV.js";
|
|
25
|
+
import {
|
|
26
|
+
getChatConfig,
|
|
27
|
+
isProviderRetryError
|
|
28
|
+
} from "./chunk-BYW4B4PR.js";
|
|
29
|
+
import {
|
|
30
|
+
logException,
|
|
31
|
+
logInfo,
|
|
32
|
+
logWarn,
|
|
33
|
+
withLogContext
|
|
34
|
+
} from "./chunk-JNNJZURV.js";
|
|
35
|
+
|
|
36
|
+
// src/chat/task-execution/worker.ts
|
|
37
|
+
var CONVERSATION_WORK_DEFER_DELAY_MS = 15e3;
|
|
38
|
+
function now(options) {
|
|
39
|
+
return options.nowMs?.() ?? Date.now();
|
|
40
|
+
}
|
|
41
|
+
function selectContiguousActorBatch(messages) {
|
|
42
|
+
const first = messages[0];
|
|
43
|
+
if (!first) {
|
|
44
|
+
return [];
|
|
45
|
+
}
|
|
46
|
+
const nextActorIndex = messages.findIndex(
|
|
47
|
+
(message) => message.input.authorId !== first.input.authorId
|
|
48
|
+
);
|
|
49
|
+
return messages.slice(
|
|
50
|
+
0,
|
|
51
|
+
nextActorIndex === -1 ? messages.length : nextActorIndex
|
|
52
|
+
);
|
|
53
|
+
}
|
|
54
|
+
function selectAttemptMessages(work) {
|
|
55
|
+
const messages = work.messages;
|
|
56
|
+
const interrupts = messages.filter(
|
|
57
|
+
(message) => message.delivery === "interrupt"
|
|
58
|
+
);
|
|
59
|
+
if (interrupts.length > 0) {
|
|
60
|
+
return selectContiguousActorBatch(interrupts);
|
|
61
|
+
}
|
|
62
|
+
return work.execution.status === "awaiting_resume" ? [] : selectContiguousActorBatch(messages);
|
|
63
|
+
}
|
|
64
|
+
function nudgeIdempotencyKey(reason, conversationId, nowMs) {
|
|
65
|
+
return `${reason}:${conversationId}:${nowMs}`;
|
|
66
|
+
}
|
|
67
|
+
async function requestLostLeaseRecovery(args) {
|
|
68
|
+
const before = await getConversationWorkState({
|
|
69
|
+
conversationId: args.conversationId,
|
|
70
|
+
state: args.options.state
|
|
71
|
+
});
|
|
72
|
+
const retry = await recordConversationRetry({
|
|
73
|
+
conversationId: args.conversationId,
|
|
74
|
+
leaseToken: args.leaseToken,
|
|
75
|
+
conversationStore: args.options.conversationStore,
|
|
76
|
+
nowMs: args.nowMs,
|
|
77
|
+
state: args.options.state
|
|
78
|
+
});
|
|
79
|
+
if (retry === "lost_lease") {
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
if (retry === "stopped") {
|
|
83
|
+
logException(
|
|
84
|
+
new Error("Conversation work stopped after repeated failed attempts"),
|
|
85
|
+
"conversation.work.retry.exhausted",
|
|
86
|
+
{
|
|
87
|
+
"app.run.id": before?.execution.runId ?? "unknown",
|
|
88
|
+
"app.worker.last_progress_at_ms": before?.execution.lastProgressAtMs ?? before?.createdAtMs ?? 0,
|
|
89
|
+
"app.worker.retry_count": (before?.execution.retryCount ?? 0) + 1
|
|
90
|
+
}
|
|
91
|
+
);
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
const resumeRequested = await requestConversationContinuation({
|
|
95
|
+
conversationId: args.conversationId,
|
|
96
|
+
destination: args.destination,
|
|
97
|
+
leaseToken: args.leaseToken,
|
|
98
|
+
conversationStore: args.options.conversationStore,
|
|
99
|
+
nowMs: args.nowMs,
|
|
100
|
+
state: args.options.state
|
|
101
|
+
});
|
|
102
|
+
if (!resumeRequested) {
|
|
103
|
+
return;
|
|
104
|
+
}
|
|
105
|
+
const released = await releaseConversationWork({
|
|
106
|
+
conversationId: args.conversationId,
|
|
107
|
+
leaseToken: args.leaseToken,
|
|
108
|
+
conversationStore: args.options.conversationStore,
|
|
109
|
+
nowMs: args.nowMs,
|
|
110
|
+
state: args.options.state
|
|
111
|
+
});
|
|
112
|
+
if (!released) {
|
|
113
|
+
return;
|
|
114
|
+
}
|
|
115
|
+
await ensureConversationWake({
|
|
116
|
+
conversationId: args.conversationId,
|
|
117
|
+
conversationStore: args.options.conversationStore,
|
|
118
|
+
idempotencyKey: nudgeIdempotencyKey(
|
|
119
|
+
"lost_lease",
|
|
120
|
+
args.conversationId,
|
|
121
|
+
args.nowMs
|
|
122
|
+
),
|
|
123
|
+
nowMs: args.nowMs,
|
|
124
|
+
queue: args.options.queue,
|
|
125
|
+
replaceExistingWake: true,
|
|
126
|
+
state: args.options.state
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
async function recordFailedDeliveryAttempt(args) {
|
|
130
|
+
const failure = await recordAttemptFailure({
|
|
131
|
+
conversationId: args.conversationId,
|
|
132
|
+
inboundMessageIds: args.messageIds,
|
|
133
|
+
leaseToken: args.leaseToken,
|
|
134
|
+
conversationStore: args.options.conversationStore,
|
|
135
|
+
nowMs: args.nowMs,
|
|
136
|
+
state: args.options.state
|
|
137
|
+
});
|
|
138
|
+
for (const message of failure.deadLetteredMessages) {
|
|
139
|
+
logWarn("conversation.work.dead_lettered", {
|
|
140
|
+
"app.conversation.source": message.source,
|
|
141
|
+
"app.inbound.attempt_count": message.attemptCount ?? 0,
|
|
142
|
+
"app.inbound.message_id": message.inboundMessageId,
|
|
143
|
+
"app.inbound.pending_count": failure.pendingCount
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
return failure;
|
|
147
|
+
}
|
|
148
|
+
function isTerminalFailure(failure) {
|
|
149
|
+
return failure.status === "recorded" && failure.deadLetteredMessages.length > 0 && failure.pendingCount === 0;
|
|
150
|
+
}
|
|
151
|
+
function logRetryExhausted(args) {
|
|
152
|
+
logException(args.error, "conversation.work.retry.exhausted", {
|
|
153
|
+
"app.run.id": args.work.execution.runId ?? "unknown",
|
|
154
|
+
"app.worker.last_progress_at_ms": args.work.execution.lastProgressAtMs ?? args.work.createdAtMs,
|
|
155
|
+
"app.worker.retry_count": args.retryCount
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
function startLeaseCheckIn(args) {
|
|
159
|
+
const timer = setInterval(() => {
|
|
160
|
+
const nowMs = now(args.options);
|
|
161
|
+
void checkInConversationWork({
|
|
162
|
+
conversationId: args.conversationId,
|
|
163
|
+
leaseToken: args.leaseToken,
|
|
164
|
+
conversationStore: args.options.conversationStore,
|
|
165
|
+
nowMs,
|
|
166
|
+
state: args.options.state
|
|
167
|
+
}).then(
|
|
168
|
+
(checkedIn) => {
|
|
169
|
+
if (!checkedIn) {
|
|
170
|
+
args.onLostLease();
|
|
171
|
+
logWarn("conversation.work.check_in.failed");
|
|
172
|
+
}
|
|
173
|
+
},
|
|
174
|
+
(error) => {
|
|
175
|
+
logException(error, "conversation.work.check_in.failed");
|
|
176
|
+
}
|
|
177
|
+
);
|
|
178
|
+
}, args.options.checkInIntervalMs ?? CONVERSATION_WORK_CHECK_IN_INTERVAL_MS);
|
|
179
|
+
timer.unref?.();
|
|
180
|
+
return timer;
|
|
181
|
+
}
|
|
182
|
+
async function processConversationWork(message, options) {
|
|
183
|
+
return withLogContext(
|
|
184
|
+
{ conversationId: message.conversationId },
|
|
185
|
+
() => processConversationWorkInContext(message, options)
|
|
186
|
+
);
|
|
187
|
+
}
|
|
188
|
+
async function processConversationWorkInContext(message, options) {
|
|
189
|
+
const conversationId = message.conversationId;
|
|
190
|
+
let initial;
|
|
191
|
+
try {
|
|
192
|
+
initial = await getConversationWorkState({
|
|
193
|
+
conversationId,
|
|
194
|
+
state: options.state
|
|
195
|
+
});
|
|
196
|
+
} catch (error) {
|
|
197
|
+
if (isInvalidConversationRecordError(error)) {
|
|
198
|
+
throw new ConversationQueueMessageRejectedError(
|
|
199
|
+
"invalid_record",
|
|
200
|
+
`Conversation record failed validation for ${conversationId}`,
|
|
201
|
+
{ conversationId }
|
|
202
|
+
);
|
|
203
|
+
}
|
|
204
|
+
throw error;
|
|
205
|
+
}
|
|
206
|
+
if (!initial || countPendingConversationMessages(initial) === 0 && initial.execution.status === "idle" && !initial.execution.lease) {
|
|
207
|
+
if (initial) {
|
|
208
|
+
await clearConsumedConversationWake({
|
|
209
|
+
conversationId,
|
|
210
|
+
conversationStore: options.conversationStore,
|
|
211
|
+
nowMs: now(options),
|
|
212
|
+
state: options.state
|
|
213
|
+
});
|
|
214
|
+
}
|
|
215
|
+
return { status: "no_work" };
|
|
216
|
+
}
|
|
217
|
+
const destination = initial.destination;
|
|
218
|
+
const lease = await startConversationWork({
|
|
219
|
+
conversationId,
|
|
220
|
+
conversationStore: options.conversationStore,
|
|
221
|
+
nowMs: now(options),
|
|
222
|
+
state: options.state
|
|
223
|
+
});
|
|
224
|
+
if (lease.status === "no_work") {
|
|
225
|
+
await clearConsumedConversationWake({
|
|
226
|
+
conversationId,
|
|
227
|
+
conversationStore: options.conversationStore,
|
|
228
|
+
nowMs: now(options),
|
|
229
|
+
state: options.state
|
|
230
|
+
});
|
|
231
|
+
return { status: "no_work" };
|
|
232
|
+
}
|
|
233
|
+
if (lease.status === "active") {
|
|
234
|
+
const nudgeNowMs = now(options);
|
|
235
|
+
await ensureConversationWake({
|
|
236
|
+
conversationId,
|
|
237
|
+
conversationStore: options.conversationStore,
|
|
238
|
+
delayMs: CONVERSATION_WORK_DEFER_DELAY_MS,
|
|
239
|
+
idempotencyKey: nudgeIdempotencyKey("active", conversationId, nudgeNowMs),
|
|
240
|
+
nowMs: nudgeNowMs,
|
|
241
|
+
queue: options.queue,
|
|
242
|
+
replaceExistingWake: true,
|
|
243
|
+
state: options.state
|
|
244
|
+
});
|
|
245
|
+
logInfo("conversation.work.nudge.deferred", {
|
|
246
|
+
"app.lease.expires_at_ms": lease.leaseExpiresAtMs
|
|
247
|
+
});
|
|
248
|
+
return { status: "active" };
|
|
249
|
+
}
|
|
250
|
+
const startedAtMs = now(options);
|
|
251
|
+
const softYieldDeadlineMs = startedAtMs + (options.softYieldAfterMs ?? getChatConfig().conversationWorkSoftYieldAfterMs);
|
|
252
|
+
let attemptMessageIds = [];
|
|
253
|
+
let attemptSelectedMessageIds = /* @__PURE__ */ new Set();
|
|
254
|
+
let attemptStartMessageIds = /* @__PURE__ */ new Set();
|
|
255
|
+
let failedWithoutProgress = false;
|
|
256
|
+
let leaseLost = false;
|
|
257
|
+
const markLeaseLost = () => {
|
|
258
|
+
leaseLost = true;
|
|
259
|
+
};
|
|
260
|
+
const timer = startLeaseCheckIn({
|
|
261
|
+
conversationId,
|
|
262
|
+
leaseToken: lease.leaseToken,
|
|
263
|
+
onLostLease: markLeaseLost,
|
|
264
|
+
options
|
|
265
|
+
});
|
|
266
|
+
logInfo("conversation.work.lease.acquired", {
|
|
267
|
+
"app.lease.expires_at_ms": lease.leaseExpiresAtMs,
|
|
268
|
+
"app.worker.soft_yield_deadline_ms": softYieldDeadlineMs
|
|
269
|
+
});
|
|
270
|
+
const drain = (handle) => drainConversationMailbox({
|
|
271
|
+
conversationId,
|
|
272
|
+
leaseToken: lease.leaseToken,
|
|
273
|
+
conversationStore: options.conversationStore,
|
|
274
|
+
handle: async (messages) => {
|
|
275
|
+
const candidates = messages.filter(
|
|
276
|
+
(message2) => message2.delivery === "interrupt" && (attemptSelectedMessageIds.has(message2.inboundMessageId) || !attemptStartMessageIds.has(message2.inboundMessageId))
|
|
277
|
+
);
|
|
278
|
+
if (candidates.length === 0) {
|
|
279
|
+
return [];
|
|
280
|
+
}
|
|
281
|
+
return await handle(candidates) ?? candidates.map((message2) => message2.inboundMessageId);
|
|
282
|
+
},
|
|
283
|
+
nowMs: now(options),
|
|
284
|
+
state: options.state
|
|
285
|
+
});
|
|
286
|
+
const shouldYield = () => leaseLost || now(options) >= softYieldDeadlineMs;
|
|
287
|
+
const checkIn = async () => {
|
|
288
|
+
const checkedIn = await checkInConversationWork({
|
|
289
|
+
conversationId,
|
|
290
|
+
leaseToken: lease.leaseToken,
|
|
291
|
+
conversationStore: options.conversationStore,
|
|
292
|
+
nowMs: now(options),
|
|
293
|
+
state: options.state
|
|
294
|
+
});
|
|
295
|
+
if (!checkedIn) {
|
|
296
|
+
markLeaseLost();
|
|
297
|
+
}
|
|
298
|
+
return checkedIn;
|
|
299
|
+
};
|
|
300
|
+
const yieldWork = async () => {
|
|
301
|
+
const yieldNowMs = now(options);
|
|
302
|
+
await ensureConversationWake({
|
|
303
|
+
conversationId,
|
|
304
|
+
conversationStore: options.conversationStore,
|
|
305
|
+
idempotencyKey: nudgeIdempotencyKey("yield", conversationId, yieldNowMs),
|
|
306
|
+
nowMs: yieldNowMs,
|
|
307
|
+
queue: options.queue,
|
|
308
|
+
replaceExistingWake: true,
|
|
309
|
+
state: options.state
|
|
310
|
+
});
|
|
311
|
+
const released = await releaseConversationWork({
|
|
312
|
+
conversationId,
|
|
313
|
+
leaseToken: lease.leaseToken,
|
|
314
|
+
conversationStore: options.conversationStore,
|
|
315
|
+
nowMs: yieldNowMs,
|
|
316
|
+
state: options.state
|
|
317
|
+
});
|
|
318
|
+
if (!released) {
|
|
319
|
+
return { status: "lost_lease" };
|
|
320
|
+
}
|
|
321
|
+
logInfo("conversation.work.yielded", {
|
|
322
|
+
"app.worker.elapsed_ms": now(options) - startedAtMs,
|
|
323
|
+
"app.worker.soft_yield_deadline_ms": softYieldDeadlineMs
|
|
324
|
+
});
|
|
325
|
+
return { status: "yielded" };
|
|
326
|
+
};
|
|
327
|
+
try {
|
|
328
|
+
let hasRun = false;
|
|
329
|
+
while (true) {
|
|
330
|
+
attemptMessageIds = [];
|
|
331
|
+
const leasedWork = await getConversationWorkState({
|
|
332
|
+
conversationId,
|
|
333
|
+
state: options.state
|
|
334
|
+
});
|
|
335
|
+
if (!leasedWork || leasedWork.lease?.leaseToken !== lease.leaseToken || leaseLost) {
|
|
336
|
+
markLeaseLost();
|
|
337
|
+
await requestLostLeaseRecovery({
|
|
338
|
+
conversationId,
|
|
339
|
+
destination,
|
|
340
|
+
leaseToken: lease.leaseToken,
|
|
341
|
+
nowMs: now(options),
|
|
342
|
+
options
|
|
343
|
+
});
|
|
344
|
+
return { status: "lost_lease" };
|
|
345
|
+
}
|
|
346
|
+
const resumePending = leasedWork.execution.status === "awaiting_resume";
|
|
347
|
+
const attemptMessages = selectAttemptMessages(leasedWork);
|
|
348
|
+
attemptStartMessageIds = new Set(
|
|
349
|
+
leasedWork.messages.map((message2) => message2.inboundMessageId)
|
|
350
|
+
);
|
|
351
|
+
if (hasRun && shouldYield()) {
|
|
352
|
+
if (resumePending) {
|
|
353
|
+
return await yieldWork();
|
|
354
|
+
}
|
|
355
|
+
break;
|
|
356
|
+
}
|
|
357
|
+
if (resumePending && attemptMessages.length === 0) {
|
|
358
|
+
const resumeStarted = await beginConversationResume({
|
|
359
|
+
conversationId,
|
|
360
|
+
leaseToken: lease.leaseToken,
|
|
361
|
+
conversationStore: options.conversationStore,
|
|
362
|
+
nowMs: now(options),
|
|
363
|
+
state: options.state
|
|
364
|
+
});
|
|
365
|
+
if (!resumeStarted) {
|
|
366
|
+
markLeaseLost();
|
|
367
|
+
await requestLostLeaseRecovery({
|
|
368
|
+
conversationId,
|
|
369
|
+
destination,
|
|
370
|
+
leaseToken: lease.leaseToken,
|
|
371
|
+
nowMs: now(options),
|
|
372
|
+
options
|
|
373
|
+
});
|
|
374
|
+
return { status: "lost_lease" };
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
attemptMessageIds = attemptMessages.map(
|
|
378
|
+
(message2) => message2.inboundMessageId
|
|
379
|
+
);
|
|
380
|
+
attemptSelectedMessageIds = new Set(attemptMessageIds);
|
|
381
|
+
const ack = async () => {
|
|
382
|
+
const acknowledged = await ackMessages({
|
|
383
|
+
conversationId,
|
|
384
|
+
inboundMessageIds: attemptMessageIds,
|
|
385
|
+
leaseToken: lease.leaseToken,
|
|
386
|
+
conversationStore: options.conversationStore,
|
|
387
|
+
nowMs: now(options),
|
|
388
|
+
state: options.state
|
|
389
|
+
});
|
|
390
|
+
if (!acknowledged) {
|
|
391
|
+
markLeaseLost();
|
|
392
|
+
throw new Error(
|
|
393
|
+
`Conversation work lease lost before inbox ack for ${conversationId}`
|
|
394
|
+
);
|
|
395
|
+
}
|
|
396
|
+
};
|
|
397
|
+
const workerContext = {
|
|
398
|
+
attempt: {
|
|
399
|
+
ack,
|
|
400
|
+
conversationId,
|
|
401
|
+
destination,
|
|
402
|
+
drain,
|
|
403
|
+
isFinalAttempt: attemptMessages.some(
|
|
404
|
+
(message2) => isFinalAttempt(message2)
|
|
405
|
+
),
|
|
406
|
+
messages: attemptMessages
|
|
407
|
+
},
|
|
408
|
+
conversationId,
|
|
409
|
+
destination,
|
|
410
|
+
shouldYield,
|
|
411
|
+
checkIn
|
|
412
|
+
};
|
|
413
|
+
const result = await options.run(workerContext);
|
|
414
|
+
hasRun = true;
|
|
415
|
+
if (result.status === "lost_lease") {
|
|
416
|
+
await requestLostLeaseRecovery({
|
|
417
|
+
conversationId,
|
|
418
|
+
destination,
|
|
419
|
+
leaseToken: lease.leaseToken,
|
|
420
|
+
nowMs: now(options),
|
|
421
|
+
options
|
|
422
|
+
});
|
|
423
|
+
return { status: "lost_lease" };
|
|
424
|
+
}
|
|
425
|
+
if (leaseLost) {
|
|
426
|
+
await requestLostLeaseRecovery({
|
|
427
|
+
conversationId,
|
|
428
|
+
destination,
|
|
429
|
+
leaseToken: lease.leaseToken,
|
|
430
|
+
nowMs: now(options),
|
|
431
|
+
options
|
|
432
|
+
});
|
|
433
|
+
return { status: "lost_lease" };
|
|
434
|
+
}
|
|
435
|
+
if (result.status === "yielded") {
|
|
436
|
+
const resumeRequested = await requestConversationContinuation({
|
|
437
|
+
conversationId,
|
|
438
|
+
destination,
|
|
439
|
+
leaseToken: lease.leaseToken,
|
|
440
|
+
conversationStore: options.conversationStore,
|
|
441
|
+
nowMs: now(options),
|
|
442
|
+
state: options.state
|
|
443
|
+
});
|
|
444
|
+
if (!resumeRequested) {
|
|
445
|
+
return { status: "lost_lease" };
|
|
446
|
+
}
|
|
447
|
+
return await yieldWork();
|
|
448
|
+
}
|
|
449
|
+
if (result.status === "deferred") {
|
|
450
|
+
const deferredNowMs = now(options);
|
|
451
|
+
const released = await releaseConversationWork({
|
|
452
|
+
conversationId,
|
|
453
|
+
leaseToken: lease.leaseToken,
|
|
454
|
+
conversationStore: options.conversationStore,
|
|
455
|
+
nowMs: deferredNowMs,
|
|
456
|
+
state: options.state
|
|
457
|
+
});
|
|
458
|
+
if (!released) {
|
|
459
|
+
return { status: "lost_lease" };
|
|
460
|
+
}
|
|
461
|
+
const wake = await ensureConversationWake({
|
|
462
|
+
conversationId,
|
|
463
|
+
conversationStore: options.conversationStore,
|
|
464
|
+
idempotencyKey: nudgeIdempotencyKey(
|
|
465
|
+
"deferred",
|
|
466
|
+
conversationId,
|
|
467
|
+
deferredNowMs
|
|
468
|
+
),
|
|
469
|
+
nowMs: deferredNowMs,
|
|
470
|
+
queue: options.queue,
|
|
471
|
+
state: options.state
|
|
472
|
+
});
|
|
473
|
+
return wake.status === "enqueued" ? { status: "pending_requeued" } : { status: "completed" };
|
|
474
|
+
}
|
|
475
|
+
if (attemptMessageIds.length > 0) {
|
|
476
|
+
const failure = await recordFailedDeliveryAttempt({
|
|
477
|
+
conversationId,
|
|
478
|
+
leaseToken: lease.leaseToken,
|
|
479
|
+
nowMs: now(options),
|
|
480
|
+
messageIds: attemptMessageIds,
|
|
481
|
+
options
|
|
482
|
+
});
|
|
483
|
+
if (isTerminalFailure(failure)) {
|
|
484
|
+
if (failure.retryCount >= CONVERSATION_WORK_MAX_RETRIES) {
|
|
485
|
+
logRetryExhausted({
|
|
486
|
+
error: new Error(
|
|
487
|
+
"Conversation work stopped after repeated failed attempts"
|
|
488
|
+
),
|
|
489
|
+
retryCount: failure.retryCount,
|
|
490
|
+
work: leasedWork
|
|
491
|
+
});
|
|
492
|
+
}
|
|
493
|
+
await deadLetterAttempt({
|
|
494
|
+
conversationId,
|
|
495
|
+
leaseToken: lease.leaseToken,
|
|
496
|
+
conversationStore: options.conversationStore,
|
|
497
|
+
nowMs: now(options),
|
|
498
|
+
state: options.state
|
|
499
|
+
});
|
|
500
|
+
return { status: "failed" };
|
|
501
|
+
}
|
|
502
|
+
if (failure.status === "lost_lease") {
|
|
503
|
+
return { status: "lost_lease" };
|
|
504
|
+
}
|
|
505
|
+
if (failure.status === "recorded") {
|
|
506
|
+
failedWithoutProgress = true;
|
|
507
|
+
break;
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
const next = await getConversationWorkState({
|
|
511
|
+
conversationId,
|
|
512
|
+
state: options.state
|
|
513
|
+
});
|
|
514
|
+
if (!next || next.lease?.leaseToken !== lease.leaseToken) {
|
|
515
|
+
return { status: "lost_lease" };
|
|
516
|
+
}
|
|
517
|
+
if (next.execution.status !== "awaiting_resume" && countPendingConversationMessages(next) === 0) {
|
|
518
|
+
break;
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
const completion = await completeConversationWork({
|
|
522
|
+
conversationId,
|
|
523
|
+
leaseToken: lease.leaseToken,
|
|
524
|
+
madeProgress: !failedWithoutProgress,
|
|
525
|
+
conversationStore: options.conversationStore,
|
|
526
|
+
nowMs: now(options),
|
|
527
|
+
state: options.state
|
|
528
|
+
});
|
|
529
|
+
if (completion === "lost_lease") {
|
|
530
|
+
return { status: "lost_lease" };
|
|
531
|
+
}
|
|
532
|
+
if (completion === "pending") {
|
|
533
|
+
const nudgeNowMs = now(options);
|
|
534
|
+
const wake = await ensureConversationWake({
|
|
535
|
+
conversationId,
|
|
536
|
+
conversationStore: options.conversationStore,
|
|
537
|
+
idempotencyKey: nudgeIdempotencyKey(
|
|
538
|
+
"pending",
|
|
539
|
+
conversationId,
|
|
540
|
+
nudgeNowMs
|
|
541
|
+
),
|
|
542
|
+
nowMs: nudgeNowMs,
|
|
543
|
+
queue: options.queue,
|
|
544
|
+
state: options.state
|
|
545
|
+
});
|
|
546
|
+
return wake.status === "enqueued" ? { status: "pending_requeued" } : { status: "completed" };
|
|
547
|
+
}
|
|
548
|
+
logInfo("conversation.work.completed", {
|
|
549
|
+
"app.worker.elapsed_ms": now(options) - startedAtMs
|
|
550
|
+
});
|
|
551
|
+
return { status: "completed" };
|
|
552
|
+
} catch (error) {
|
|
553
|
+
const errorNowMs = now(options);
|
|
554
|
+
let recoveryRecorded = false;
|
|
555
|
+
try {
|
|
556
|
+
const failure = attemptMessageIds.length > 0 ? await recordFailedDeliveryAttempt({
|
|
557
|
+
conversationId,
|
|
558
|
+
leaseToken: lease.leaseToken,
|
|
559
|
+
nowMs: errorNowMs,
|
|
560
|
+
messageIds: attemptMessageIds,
|
|
561
|
+
options
|
|
562
|
+
}) : void 0;
|
|
563
|
+
if (failure && isTerminalFailure(failure)) {
|
|
564
|
+
if (failure.retryCount >= CONVERSATION_WORK_MAX_RETRIES) {
|
|
565
|
+
logRetryExhausted({
|
|
566
|
+
error,
|
|
567
|
+
retryCount: failure.retryCount,
|
|
568
|
+
work: initial
|
|
569
|
+
});
|
|
570
|
+
}
|
|
571
|
+
await deadLetterAttempt({
|
|
572
|
+
conversationId,
|
|
573
|
+
leaseToken: lease.leaseToken,
|
|
574
|
+
conversationStore: options.conversationStore,
|
|
575
|
+
nowMs: errorNowMs,
|
|
576
|
+
state: options.state
|
|
577
|
+
});
|
|
578
|
+
} else if (failure?.status === "recorded") {
|
|
579
|
+
await releaseConversationWork({
|
|
580
|
+
conversationId,
|
|
581
|
+
leaseToken: lease.leaseToken,
|
|
582
|
+
conversationStore: options.conversationStore,
|
|
583
|
+
nowMs: errorNowMs,
|
|
584
|
+
state: options.state
|
|
585
|
+
});
|
|
586
|
+
await ensureConversationWake({
|
|
587
|
+
conversationId,
|
|
588
|
+
conversationStore: options.conversationStore,
|
|
589
|
+
idempotencyKey: nudgeIdempotencyKey(
|
|
590
|
+
"error",
|
|
591
|
+
conversationId,
|
|
592
|
+
errorNowMs
|
|
593
|
+
),
|
|
594
|
+
nowMs: errorNowMs,
|
|
595
|
+
queue: options.queue,
|
|
596
|
+
replaceExistingWake: true,
|
|
597
|
+
state: options.state
|
|
598
|
+
});
|
|
599
|
+
} else {
|
|
600
|
+
const retry = failure ? failure.status : await recordConversationRetry({
|
|
601
|
+
conversationId,
|
|
602
|
+
leaseToken: lease.leaseToken,
|
|
603
|
+
conversationStore: options.conversationStore,
|
|
604
|
+
nowMs: errorNowMs,
|
|
605
|
+
state: options.state
|
|
606
|
+
});
|
|
607
|
+
if (retry === "stopped") {
|
|
608
|
+
logException(error, "conversation.work.retry.exhausted", {
|
|
609
|
+
"app.run.id": initial.execution.runId ?? "unknown",
|
|
610
|
+
"app.worker.last_progress_at_ms": initial.execution.lastProgressAtMs ?? initial.createdAtMs,
|
|
611
|
+
"app.worker.retry_count": failure?.retryCount ?? (initial.execution.retryCount ?? 0) + 1
|
|
612
|
+
});
|
|
613
|
+
recoveryRecorded = true;
|
|
614
|
+
return { status: "failed" };
|
|
615
|
+
}
|
|
616
|
+
const resumeRequested = await requestConversationContinuation({
|
|
617
|
+
conversationId,
|
|
618
|
+
destination,
|
|
619
|
+
leaseToken: lease.leaseToken,
|
|
620
|
+
conversationStore: options.conversationStore,
|
|
621
|
+
nowMs: errorNowMs,
|
|
622
|
+
state: options.state
|
|
623
|
+
});
|
|
624
|
+
if (resumeRequested) {
|
|
625
|
+
await ensureConversationWake({
|
|
626
|
+
conversationId,
|
|
627
|
+
conversationStore: options.conversationStore,
|
|
628
|
+
idempotencyKey: nudgeIdempotencyKey(
|
|
629
|
+
"error",
|
|
630
|
+
conversationId,
|
|
631
|
+
errorNowMs
|
|
632
|
+
),
|
|
633
|
+
nowMs: errorNowMs,
|
|
634
|
+
queue: options.queue,
|
|
635
|
+
replaceExistingWake: true,
|
|
636
|
+
state: options.state
|
|
637
|
+
});
|
|
638
|
+
}
|
|
639
|
+
await releaseConversationWork({
|
|
640
|
+
conversationId,
|
|
641
|
+
leaseToken: lease.leaseToken,
|
|
642
|
+
conversationStore: options.conversationStore,
|
|
643
|
+
nowMs: errorNowMs,
|
|
644
|
+
state: options.state
|
|
645
|
+
});
|
|
646
|
+
}
|
|
647
|
+
recoveryRecorded = true;
|
|
648
|
+
} catch (recoveryError) {
|
|
649
|
+
logException(recoveryError, "conversation.work.requeue.failed");
|
|
650
|
+
}
|
|
651
|
+
if (!isProviderRetryError(error)) {
|
|
652
|
+
logException(error, "conversation.work.failed", {
|
|
653
|
+
"app.worker.elapsed_ms": now(options) - startedAtMs
|
|
654
|
+
});
|
|
655
|
+
}
|
|
656
|
+
if (!recoveryRecorded) {
|
|
657
|
+
throw error;
|
|
658
|
+
}
|
|
659
|
+
return { status: "failed" };
|
|
660
|
+
} finally {
|
|
661
|
+
clearInterval(timer);
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
export {
|
|
666
|
+
CONVERSATION_WORK_DEFER_DELAY_MS,
|
|
667
|
+
processConversationWork
|
|
668
|
+
};
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
// src/chat/task-execution/queue.ts
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
var conversationQueueMessageSchema = z.object({
|
|
4
|
+
conversationId: z.string().trim().min(1)
|
|
5
|
+
}).strict();
|
|
6
|
+
var ConversationQueueMessageRejectedError = class extends Error {
|
|
7
|
+
conversationId;
|
|
8
|
+
reason;
|
|
9
|
+
constructor(reason, message, options = {}) {
|
|
10
|
+
super(message);
|
|
11
|
+
this.name = "ConversationQueueMessageRejectedError";
|
|
12
|
+
this.reason = reason;
|
|
13
|
+
this.conversationId = options.conversationId;
|
|
14
|
+
}
|
|
15
|
+
};
|
|
16
|
+
function isConversationQueueMessageRejectedError(error) {
|
|
17
|
+
return error instanceof ConversationQueueMessageRejectedError;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export {
|
|
21
|
+
conversationQueueMessageSchema,
|
|
22
|
+
ConversationQueueMessageRejectedError,
|
|
23
|
+
isConversationQueueMessageRejectedError
|
|
24
|
+
};
|