openclaw-lark-multi-agent 1.4.12 → 1.4.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/feishu-bot.d.ts +8 -0
- package/dist/feishu-bot.js +198 -7
- package/dist/message-store.d.ts +30 -0
- package/dist/message-store.js +94 -0
- package/dist/openclaw-client.d.ts +8 -4
- package/dist/openclaw-client.js +94 -20
- package/dist/terminal-errors.d.ts +49 -0
- package/dist/terminal-errors.js +116 -0
- package/package.json +1 -1
package/dist/feishu-bot.d.ts
CHANGED
|
@@ -57,6 +57,8 @@ export declare class FeishuBot {
|
|
|
57
57
|
* message falls back to a normal run, cancel these before submission so a
|
|
58
58
|
* later identical steer cannot consume the stale observer/correlation. */
|
|
59
59
|
private pendingSteerCleanups;
|
|
60
|
+
private errorNoticeTimers;
|
|
61
|
+
private errorNoticeChecks;
|
|
60
62
|
private adminOpenId;
|
|
61
63
|
private locale;
|
|
62
64
|
private configPath?;
|
|
@@ -201,6 +203,11 @@ export declare class FeishuBot {
|
|
|
201
203
|
private deliverySourceId;
|
|
202
204
|
private sendSystemDelivery;
|
|
203
205
|
private enqueueAndDispatchDelivery;
|
|
206
|
+
private inspectErrorNotice;
|
|
207
|
+
private deferErrorNotice;
|
|
208
|
+
private scheduleErrorNoticeCheck;
|
|
209
|
+
/** Bounded read-only diagnosis; known live tasks retain low-frequency observation. */
|
|
210
|
+
private reconcileErrorNotices;
|
|
204
211
|
private scheduleDeliveryRetry;
|
|
205
212
|
private dispatchPendingDeliveries;
|
|
206
213
|
private hasFreeModeBot;
|
|
@@ -277,6 +284,7 @@ export declare class FeishuBot {
|
|
|
277
284
|
/**
|
|
278
285
|
* Send a proactive message to a chat (not a reply).
|
|
279
286
|
*/
|
|
287
|
+
private sendErrorNotice;
|
|
280
288
|
private sendFinalMessage;
|
|
281
289
|
private sendMessage;
|
|
282
290
|
/**
|
package/dist/feishu-bot.js
CHANGED
|
@@ -6,6 +6,7 @@ import { InactiveRunObservation } from "./openclaw-client.js";
|
|
|
6
6
|
import { SessionWaitPaused, isWaitTimeout, normalizeSessionRuntimeStatus, formatSessionFooter, FOREGROUND_WAIT_MS } from "./session-status.js";
|
|
7
7
|
import { LiveStatusController } from "./live-status.js";
|
|
8
8
|
import { CompactProgressController } from "./compact-progress.js";
|
|
9
|
+
import { errorIdentity } from "./terminal-errors.js";
|
|
9
10
|
import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "fs";
|
|
10
11
|
import { basename, extname, join, resolve } from "path";
|
|
11
12
|
import { getBridgeAttachmentsDir, getDataDir } from "./paths.js";
|
|
@@ -117,6 +118,8 @@ export class FeishuBot {
|
|
|
117
118
|
* message falls back to a normal run, cancel these before submission so a
|
|
118
119
|
* later identical steer cannot consume the stale observer/correlation. */
|
|
119
120
|
pendingSteerCleanups = new Map();
|
|
121
|
+
errorNoticeTimers = new Map();
|
|
122
|
+
errorNoticeChecks = new Map();
|
|
120
123
|
adminOpenId;
|
|
121
124
|
locale;
|
|
122
125
|
configPath;
|
|
@@ -249,6 +252,19 @@ export class FeishuBot {
|
|
|
249
252
|
this.openclawClient.setVerboseTranscriptDelivery(sessionKey, this.store.getBotVerbose(this.config.name, chatId));
|
|
250
253
|
await this.openclawClient.subscribeSession(sessionKey, async (text, meta) => {
|
|
251
254
|
try {
|
|
255
|
+
let provenance = meta?.error;
|
|
256
|
+
if (!provenance && meta?.genericErrorCandidate && meta.runId && meta.sessionKey) {
|
|
257
|
+
provenance = this.store.getErrorNotice(this.config.name, chatId, errorIdentity(meta.sessionKey, meta.runId))?.provenance;
|
|
258
|
+
}
|
|
259
|
+
if (meta?.sourceType === "run_error" || provenance) {
|
|
260
|
+
if (provenance) {
|
|
261
|
+
this.store.observeRunError(this.config.name, chatId, provenance);
|
|
262
|
+
await this.reconcileErrorNotices(chatId);
|
|
263
|
+
}
|
|
264
|
+
else
|
|
265
|
+
console.warn(`[${this.config.name}] Uncorrelated run_error retained in logs; no finality evidence`);
|
|
266
|
+
return; // errors never cancel pending results or complete live cards
|
|
267
|
+
}
|
|
252
268
|
const sourceType = meta?.sourceType === "verbose_transcript" ? "verbose_transcript" : "assistant_visible";
|
|
253
269
|
console.log(`[${this.config.name}] ${sourceType === "verbose_transcript" ? "Verbose transcript" : "Proactive message"} for ${chatId.slice(-8)}`);
|
|
254
270
|
const parsed = this.extractBridgeAttachments(text);
|
|
@@ -260,7 +276,7 @@ export class FeishuBot {
|
|
|
260
276
|
? candidateTarget
|
|
261
277
|
: undefined);
|
|
262
278
|
try {
|
|
263
|
-
await this.enqueueAndDispatchDelivery(chatId, sourceType, this.deliverySourceId(sourceType === "verbose_transcript" ? "verbose" : "proactive", `${Date.now()}:${Math.random()}:${parsed.text.trim()}|${JSON.stringify(parsed.attachments)}`), parsed.text.trim(), parsed.attachments, activeTarget?.messageId, activeTarget ? `trigger:${activeTarget.triggerId}` : undefined, sourceType === "verbose_transcript" ? undefined : this.config.model);
|
|
279
|
+
await this.enqueueAndDispatchDelivery(chatId, sourceType, this.deliverySourceId(sourceType === "verbose_transcript" ? "verbose" : "proactive", `${Date.now()}:${Math.random()}:${parsed.text.trim()}|${JSON.stringify(parsed.attachments)}`), parsed.text.trim(), parsed.attachments, activeTarget?.messageId, activeTarget ? `trigger:${activeTarget.triggerId}` : undefined, sourceType === "verbose_transcript" ? undefined : this.config.model, meta);
|
|
264
280
|
if (sourceType !== "verbose_transcript" && (parsed.text.trim() || parsed.attachments.length > 0)) {
|
|
265
281
|
await activeTarget?.liveStatus?.complete().catch(() => { });
|
|
266
282
|
if (meta?.runId && runTarget)
|
|
@@ -363,6 +379,7 @@ export class FeishuBot {
|
|
|
363
379
|
}
|
|
364
380
|
// Re-subscribe to existing sessions
|
|
365
381
|
await this.ensureSession(chat.chatId);
|
|
382
|
+
drainTasks.push(this.reconcileErrorNotices(chat.chatId));
|
|
366
383
|
// Drain only messages that were explicitly marked as reply triggers.
|
|
367
384
|
// Context-only messages should not start an OpenClaw run after restart.
|
|
368
385
|
const pendingTriggerIds = this.store.getPendingTriggerIds(this.config.name, chat.chatId);
|
|
@@ -2263,23 +2280,33 @@ export class FeishuBot {
|
|
|
2263
2280
|
const sourceId = `discussion-system:${Date.now()}:${Math.random().toString(36).slice(2)}`;
|
|
2264
2281
|
await this.enqueueAndDispatchDelivery(chatId, "discussion_system", sourceId, text);
|
|
2265
2282
|
}
|
|
2266
|
-
async enqueueAndDispatchDelivery(chatId, sourceType, sourceId, text, attachments = [], replyToMessageId, deliveryKey, finalReplyModel) {
|
|
2283
|
+
async enqueueAndDispatchDelivery(chatId, sourceType, sourceId, text, attachments = [], replyToMessageId, deliveryKey, finalReplyModel, provenance, errorNoticeId) {
|
|
2267
2284
|
if (!text.trim() && attachments.length === 0)
|
|
2268
2285
|
return;
|
|
2269
2286
|
const attachmentsJson = JSON.stringify(attachments);
|
|
2270
2287
|
const normalizedPayload = `${text.trim()}|${attachmentsJson}`;
|
|
2271
2288
|
const contentHash = this.stableHash(normalizedPayload);
|
|
2272
2289
|
const finalDeliveryKey = deliveryKey || sourceId;
|
|
2273
|
-
const deliveryMetaJson =
|
|
2274
|
-
?
|
|
2275
|
-
:
|
|
2290
|
+
const deliveryMetaJson = JSON.stringify({
|
|
2291
|
+
...(finalReplyModel?.trim() ? { model: finalReplyModel.trim() } : {}),
|
|
2292
|
+
...(provenance ? { provenance } : {}), ...(errorNoticeId ? { errorNoticeId } : {}),
|
|
2293
|
+
});
|
|
2294
|
+
const correlatedFinal = sourceType === "assistant_visible" && provenance?.final && Boolean(provenance.runId && provenance.sessionKey);
|
|
2295
|
+
if (correlatedFinal) {
|
|
2296
|
+
const identity = { sessionKey: provenance.sessionKey, runId: provenance.runId };
|
|
2297
|
+
this.store.promoteExactRunFinal(this.config.name, chatId, identity.sessionKey, identity.runId, contentHash, text, attachmentsJson);
|
|
2298
|
+
if (!deliveryKey && this.store.hasExactRunFinalPayload(this.config.name, chatId, identity, text, attachmentsJson)) {
|
|
2299
|
+
await this.dispatchPendingDeliveries(chatId);
|
|
2300
|
+
return;
|
|
2301
|
+
}
|
|
2302
|
+
}
|
|
2276
2303
|
if (sourceType === "verbose_transcript") {
|
|
2277
2304
|
if (this.store.hasRecentSimilarDelivery(this.config.name, chatId, contentHash, 60_000, ["verbose_transcript"]))
|
|
2278
2305
|
return;
|
|
2279
2306
|
if (this.store.hasRecentOverlappingDelivery(this.config.name, chatId, text, attachmentsJson, 60_000, 8, ["verbose_transcript"]))
|
|
2280
2307
|
return;
|
|
2281
2308
|
}
|
|
2282
|
-
else if (!deliveryKey) {
|
|
2309
|
+
else if (!deliveryKey && !correlatedFinal) {
|
|
2283
2310
|
if (this.store.hasRecentSimilarDelivery(this.config.name, chatId, contentHash, 60_000, [sourceType]))
|
|
2284
2311
|
return;
|
|
2285
2312
|
if (this.store.hasRecentOverlappingDelivery(this.config.name, chatId, text, attachmentsJson, 60_000, 8, [sourceType]))
|
|
@@ -2306,6 +2333,15 @@ export class FeishuBot {
|
|
|
2306
2333
|
// answer again on a second card. The original outbox row remains the sole
|
|
2307
2334
|
// owner of text delivery; a different newly-frozen card is only collapsed.
|
|
2308
2335
|
const existing = this.store.getDeliveryByKey(this.config.name, chatId, finalDeliveryKey);
|
|
2336
|
+
if (sourceType === "run_error") {
|
|
2337
|
+
// A withdrawn unsent notice may become terminal later. Keep its one
|
|
2338
|
+
// task key; never use the result recovery/correction/attachment paths.
|
|
2339
|
+
if (existing?.status === "failed" && !existing.textDelivered && existing.attempts < DELIVERY_MAX_ATTEMPTS) {
|
|
2340
|
+
this.store.renewUnsentErrorDelivery(existing.id, text, contentHash, deliveryMetaJson);
|
|
2341
|
+
}
|
|
2342
|
+
await this.dispatchPendingDeliveries(chatId);
|
|
2343
|
+
return;
|
|
2344
|
+
}
|
|
2309
2345
|
// A failed owner must not permanently poison a logical trigger. Re-enqueue
|
|
2310
2346
|
// the payload under a deterministic recovery key so a later authoritative
|
|
2311
2347
|
// delivery gets a fresh retry budget.
|
|
@@ -2408,6 +2444,124 @@ export class FeishuBot {
|
|
|
2408
2444
|
this.lastRealDeliveryAt.set(chatId, Date.now());
|
|
2409
2445
|
}
|
|
2410
2446
|
}
|
|
2447
|
+
async inspectErrorNotice(record) {
|
|
2448
|
+
if (this.store.hasRunFinalDelivery(this.config.name, record.chatId, record.provenance, true)) {
|
|
2449
|
+
return { state: "suppressed", reason: "exact_run_final_platform_delivery_confirmed" };
|
|
2450
|
+
}
|
|
2451
|
+
if (this.store.hasRunFinalDelivery(this.config.name, record.chatId, record.provenance, false)) {
|
|
2452
|
+
return { state: "pending", reason: "exact_run_final_delivery_pending" };
|
|
2453
|
+
}
|
|
2454
|
+
let verdict;
|
|
2455
|
+
try {
|
|
2456
|
+
verdict = await this.openclawClient.inspectRunError(record.provenance);
|
|
2457
|
+
}
|
|
2458
|
+
catch {
|
|
2459
|
+
verdict = { state: "pending", reason: "inspection_unavailable" };
|
|
2460
|
+
}
|
|
2461
|
+
// A stop or final may have arrived during the read-only lookup.
|
|
2462
|
+
const latest = this.store.getErrorNotice(this.config.name, record.chatId, record.noticeId);
|
|
2463
|
+
if (latest?.provenance.userStop) {
|
|
2464
|
+
record.provenance.userStop = true;
|
|
2465
|
+
return { state: "stopped", reason: "explicit_bridge_stop" };
|
|
2466
|
+
}
|
|
2467
|
+
if (latest && (latest.provenance.source !== record.provenance.source || latest.provenance.state !== record.provenance.state || latest.provenance.stopReason !== record.provenance.stopReason)) {
|
|
2468
|
+
record.provenance = latest.provenance;
|
|
2469
|
+
record.checks = latest.checks;
|
|
2470
|
+
return { state: "pending", reason: "stronger_terminal_evidence_observed" };
|
|
2471
|
+
}
|
|
2472
|
+
if (this.store.hasRunFinalDelivery(this.config.name, record.chatId, record.provenance, true)) {
|
|
2473
|
+
return { state: "suppressed", reason: "exact_run_final_platform_delivery_confirmed" };
|
|
2474
|
+
}
|
|
2475
|
+
if (this.store.hasRunFinalDelivery(this.config.name, record.chatId, record.provenance, false)) {
|
|
2476
|
+
return { state: "pending", reason: "exact_run_final_delivery_pending" };
|
|
2477
|
+
}
|
|
2478
|
+
return verdict;
|
|
2479
|
+
}
|
|
2480
|
+
deferErrorNotice(record, verdict) {
|
|
2481
|
+
const delays = [15_000, 60_000, 120_000, 300_000, FOREGROUND_WAIT_MS];
|
|
2482
|
+
record.verdict = verdict;
|
|
2483
|
+
// Known-live task/delivery state is not an unknown diagnosis. Continue
|
|
2484
|
+
// low-frequency checks across the Gateway's 30-minute retry window, bounded
|
|
2485
|
+
// by the existing 24-hour observation horizon. Unknown evidence stays capped.
|
|
2486
|
+
if (verdict.reason === "matching_task_or_delivery_continuing")
|
|
2487
|
+
record.hasObservedContinuingTask = true;
|
|
2488
|
+
const continuing = record.hasObservedContinuingTask === true;
|
|
2489
|
+
const budget = continuing ? 300 : 6;
|
|
2490
|
+
record.status = record.checks >= budget || Date.now() - record.createdAt >= 24 * 60 * 60_000 ? "parked" : "pending";
|
|
2491
|
+
record.nextCheckAt = Date.now() + (continuing && record.checks >= 4 ? 300_000 : (delays[Math.max(0, record.checks - 1)] || FOREGROUND_WAIT_MS));
|
|
2492
|
+
this.store.saveErrorNotice(record);
|
|
2493
|
+
this.scheduleErrorNoticeCheck(record.chatId);
|
|
2494
|
+
}
|
|
2495
|
+
scheduleErrorNoticeCheck(chatId) {
|
|
2496
|
+
const previous = this.errorNoticeTimers.get(chatId);
|
|
2497
|
+
if (previous)
|
|
2498
|
+
clearTimeout(previous);
|
|
2499
|
+
this.errorNoticeTimers.delete(chatId);
|
|
2500
|
+
const next = this.store.listPendingErrorNotices(this.config.name, chatId)[0];
|
|
2501
|
+
if (!next)
|
|
2502
|
+
return;
|
|
2503
|
+
const timer = setTimeout(() => {
|
|
2504
|
+
this.errorNoticeTimers.delete(chatId);
|
|
2505
|
+
void this.reconcileErrorNotices(chatId).catch(err => console.warn(`[${this.config.name}] Error notice check unavailable:`, this.errorSummary(err)));
|
|
2506
|
+
}, Math.max(1000, next.nextCheckAt - Date.now()));
|
|
2507
|
+
timer.unref?.();
|
|
2508
|
+
this.errorNoticeTimers.set(chatId, timer);
|
|
2509
|
+
}
|
|
2510
|
+
/** Bounded read-only diagnosis; known live tasks retain low-frequency observation. */
|
|
2511
|
+
async reconcileErrorNotices(chatId) {
|
|
2512
|
+
const running = this.errorNoticeChecks.get(chatId);
|
|
2513
|
+
if (running)
|
|
2514
|
+
return running;
|
|
2515
|
+
const work = async () => {
|
|
2516
|
+
const timer = this.errorNoticeTimers.get(chatId);
|
|
2517
|
+
if (timer)
|
|
2518
|
+
clearTimeout(timer);
|
|
2519
|
+
this.errorNoticeTimers.delete(chatId);
|
|
2520
|
+
for (const snapshot of this.store.listPendingErrorNotices(this.config.name, chatId)) {
|
|
2521
|
+
// Earlier awaits may have observed a stop/stronger envelope for this row.
|
|
2522
|
+
const record = this.store.getErrorNotice(this.config.name, chatId, snapshot.noticeId);
|
|
2523
|
+
if (!record || record.status !== "pending")
|
|
2524
|
+
continue;
|
|
2525
|
+
if (record.nextCheckAt > Date.now())
|
|
2526
|
+
continue;
|
|
2527
|
+
if (record.checks >= (record.hasObservedContinuingTask || record.verdict?.reason === "matching_task_or_delivery_continuing" ? 300 : 6)) {
|
|
2528
|
+
// Normal exhausted checks are already parked. A still-pending row at
|
|
2529
|
+
// the cap can be a crash after budget reservation / before verdict or
|
|
2530
|
+
// outbox creation. Reserve one durable recovery probe, never per boot.
|
|
2531
|
+
if (record.recoveryProbeUsed) {
|
|
2532
|
+
this.deferErrorNotice(record, record.verdict || { state: "pending", reason: "budget_exhausted" });
|
|
2533
|
+
continue;
|
|
2534
|
+
}
|
|
2535
|
+
record.recoveryProbeUsed = true;
|
|
2536
|
+
}
|
|
2537
|
+
record.checks++;
|
|
2538
|
+
// Reserve the budget before awaiting RPC, including crash/disconnect.
|
|
2539
|
+
this.store.saveErrorNotice(record);
|
|
2540
|
+
const verdict = await this.inspectErrorNotice(record);
|
|
2541
|
+
console.log(`[${this.config.name}] Error notice ${record.noticeId}: ${verdict.state}/${verdict.reason}`);
|
|
2542
|
+
if (verdict.state === "pending") {
|
|
2543
|
+
this.deferErrorNotice(record, verdict);
|
|
2544
|
+
continue;
|
|
2545
|
+
}
|
|
2546
|
+
record.verdict = verdict;
|
|
2547
|
+
this.store.saveErrorNotice(record);
|
|
2548
|
+
if (verdict.state === "terminal" && verdict.terminalKey && verdict.text) {
|
|
2549
|
+
// Independent identity: never take the result's trigger/final key.
|
|
2550
|
+
await this.enqueueAndDispatchDelivery(chatId, "run_error", verdict.terminalKey, verdict.text, [], undefined, verdict.terminalKey, undefined, { sourceType: "run_error", runId: record.provenance.runId, error: record.provenance, terminalEvidence: verdict }, record.noticeId);
|
|
2551
|
+
// The send-time recheck may have parked or suppressed the observation.
|
|
2552
|
+
const refreshed = this.store.getErrorNotice(this.config.name, chatId, record.noticeId);
|
|
2553
|
+
if (refreshed?.verdict && ["pending", "suppressed", "stopped"].includes(refreshed.verdict.state))
|
|
2554
|
+
continue;
|
|
2555
|
+
}
|
|
2556
|
+
record.status = verdict.state;
|
|
2557
|
+
this.store.saveErrorNotice(record);
|
|
2558
|
+
}
|
|
2559
|
+
this.scheduleErrorNoticeCheck(chatId);
|
|
2560
|
+
};
|
|
2561
|
+
const promise = work().finally(() => { this.errorNoticeChecks.delete(chatId); });
|
|
2562
|
+
this.errorNoticeChecks.set(chatId, promise);
|
|
2563
|
+
return promise;
|
|
2564
|
+
}
|
|
2411
2565
|
scheduleDeliveryRetry(chatId, attempt) {
|
|
2412
2566
|
if (this.deliveryRetryTimers.has(chatId))
|
|
2413
2567
|
return;
|
|
@@ -2454,7 +2608,32 @@ export class FeishuBot {
|
|
|
2454
2608
|
const finalStatus = needsStatus ? formatSessionFooter(await this.readSessionStatus(chatId)) : undefined;
|
|
2455
2609
|
if (needsStatus && !finalModel)
|
|
2456
2610
|
finalModel = this.config.model;
|
|
2457
|
-
if (
|
|
2611
|
+
if (item.sourceType === "run_error") {
|
|
2612
|
+
const meta = JSON.parse(item.deliveryMetaJson || "{}");
|
|
2613
|
+
const record = this.store.getErrorNotice(this.config.name, chatId, meta.errorNoticeId || "");
|
|
2614
|
+
const verdict = record ? await this.inspectErrorNotice(record) : { state: "pending", reason: "missing_persisted_provenance" };
|
|
2615
|
+
if (verdict.state !== "terminal" || verdict.terminalKey !== item.deliveryKey || verdict.text !== item.content) {
|
|
2616
|
+
this.store.withdrawErrorDelivery(item.id);
|
|
2617
|
+
if (record) {
|
|
2618
|
+
if (verdict.state === "pending" || verdict.state === "terminal")
|
|
2619
|
+
this.deferErrorNotice(record, { ...verdict, state: "pending" });
|
|
2620
|
+
else {
|
|
2621
|
+
record.status = verdict.state;
|
|
2622
|
+
record.verdict = verdict;
|
|
2623
|
+
this.store.saveErrorNotice(record);
|
|
2624
|
+
}
|
|
2625
|
+
}
|
|
2626
|
+
return;
|
|
2627
|
+
}
|
|
2628
|
+
record.verdict = verdict;
|
|
2629
|
+
this.store.saveErrorNotice(record);
|
|
2630
|
+
}
|
|
2631
|
+
if (item.sourceType === "run_error") {
|
|
2632
|
+
// Exactly one platform operation per guarded attempt. A card/text
|
|
2633
|
+
// fallback must not bypass a fresh finality check.
|
|
2634
|
+
await this.sendErrorNotice(chatId, item.content);
|
|
2635
|
+
}
|
|
2636
|
+
else if (shouldReplyToSource) {
|
|
2458
2637
|
try {
|
|
2459
2638
|
await this.replyFinalMessage(replyTarget, item.content, finalModel, finalStatus);
|
|
2460
2639
|
}
|
|
@@ -2515,6 +2694,8 @@ export class FeishuBot {
|
|
|
2515
2694
|
return;
|
|
2516
2695
|
}
|
|
2517
2696
|
this.store.markDeliveryFailed(item.id);
|
|
2697
|
+
if (item.sourceType === "run_error")
|
|
2698
|
+
return;
|
|
2518
2699
|
const label = failureStage === "attachment" ? "附件发送失败" : "最终回复发送失败";
|
|
2519
2700
|
const errorText = `⚠️ ${label}(已重试 ${DELIVERY_MAX_ATTEMPTS} 次):${this.errorSummary(err)}`;
|
|
2520
2701
|
const replyTarget = item.replyToMessageId || replyToMessageId;
|
|
@@ -3280,6 +3461,16 @@ export class FeishuBot {
|
|
|
3280
3461
|
/**
|
|
3281
3462
|
* Send a proactive message to a chat (not a reply).
|
|
3282
3463
|
*/
|
|
3464
|
+
async sendErrorNotice(chatId, text) {
|
|
3465
|
+
const res = await this.client.im.message.create({
|
|
3466
|
+
params: { receive_id_type: "chat_id" },
|
|
3467
|
+
data: { receive_id: chatId, msg_type: "text", content: JSON.stringify({ text }) },
|
|
3468
|
+
});
|
|
3469
|
+
const id = res?.data?.message_id || res?.message_id;
|
|
3470
|
+
if (!id)
|
|
3471
|
+
throw new Error("Error notice send returned no confirmed message id");
|
|
3472
|
+
return id;
|
|
3473
|
+
}
|
|
3283
3474
|
async sendFinalMessage(chatId, text, model, status) {
|
|
3284
3475
|
if (!model?.trim())
|
|
3285
3476
|
return this.sendMessage(chatId, text);
|
package/dist/message-store.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type ErrorProvenance, type ErrorVerdict } from "./terminal-errors.js";
|
|
1
2
|
export type BotChatMode = "normal" | "free" | "mute";
|
|
2
3
|
export interface ChatInfo {
|
|
3
4
|
chatId: string;
|
|
@@ -60,9 +61,38 @@ export interface DeliveryOutboxItem {
|
|
|
60
61
|
createdAt: number;
|
|
61
62
|
updatedAt: number;
|
|
62
63
|
}
|
|
64
|
+
export interface PendingErrorNotice {
|
|
65
|
+
botName: string;
|
|
66
|
+
chatId: string;
|
|
67
|
+
noticeId: string;
|
|
68
|
+
provenance: ErrorProvenance;
|
|
69
|
+
checks: number;
|
|
70
|
+
nextCheckAt: number;
|
|
71
|
+
createdAt: number;
|
|
72
|
+
status: "pending" | "parked" | "terminal" | "suppressed" | "stopped";
|
|
73
|
+
verdict?: ErrorVerdict;
|
|
74
|
+
hasObservedContinuingTask?: boolean;
|
|
75
|
+
recoveryProbeUsed?: boolean;
|
|
76
|
+
}
|
|
63
77
|
export declare class MessageStore {
|
|
64
78
|
private db;
|
|
65
79
|
constructor(dbPath?: string);
|
|
80
|
+
observeRunError(botName: string, chatId: string, provenance: ErrorProvenance): PendingErrorNotice;
|
|
81
|
+
getErrorNotice(botName: string, chatId: string, noticeId: string): PendingErrorNotice | undefined;
|
|
82
|
+
saveErrorNotice(record: PendingErrorNotice): void;
|
|
83
|
+
listPendingErrorNotices(botName: string, chatId: string): PendingErrorNotice[];
|
|
84
|
+
/** A later exact final may be content-deduped against its own transcript.
|
|
85
|
+
* Upgrade only identical payloads with the same canonical session/run; never
|
|
86
|
+
* infer identity from text similarity or another run's successful answer. */
|
|
87
|
+
promoteExactRunFinal(botName: string, chatId: string, sessionKey: string, runId: string, contentHash: string, content: string, attachmentsJson: string): void;
|
|
88
|
+
hasExactRunFinalPayload(botName: string, chatId: string, p: {
|
|
89
|
+
sessionKey: string;
|
|
90
|
+
runId: string;
|
|
91
|
+
}, content: string, attachmentsJson: string): boolean;
|
|
92
|
+
hasRunFinalDelivery(botName: string, chatId: string, p: ErrorProvenance, confirmed: boolean): boolean;
|
|
93
|
+
renewUnsentErrorDelivery(id: number, content: string, contentHash: string, metadata: string): void;
|
|
94
|
+
/** Withdraw only an unsent error notice. Never claim it was delivered or touch a result row. */
|
|
95
|
+
withdrawErrorDelivery(id: number): void;
|
|
66
96
|
private init;
|
|
67
97
|
/**
|
|
68
98
|
* Insert a message. Returns the auto-increment id, or -1 if duplicate.
|
package/dist/message-store.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { errorIdentity } from "./terminal-errors.js";
|
|
1
2
|
import Database from "better-sqlite3";
|
|
2
3
|
import { resolve } from "path";
|
|
3
4
|
import { mkdirSync } from "fs";
|
|
@@ -11,8 +12,101 @@ export class MessageStore {
|
|
|
11
12
|
this.db.pragma("journal_mode = WAL");
|
|
12
13
|
this.init();
|
|
13
14
|
}
|
|
15
|
+
observeRunError(botName, chatId, provenance) {
|
|
16
|
+
const noticeId = errorIdentity(provenance.sessionKey, provenance.runId);
|
|
17
|
+
const previous = this.getErrorNotice(botName, chatId, noticeId);
|
|
18
|
+
if (previous) {
|
|
19
|
+
if (provenance.userStop) {
|
|
20
|
+
previous.provenance.userStop = true;
|
|
21
|
+
previous.status = "stopped";
|
|
22
|
+
previous.verdict = { state: "stopped", reason: "explicit_bridge_stop" };
|
|
23
|
+
this.saveErrorNotice(previous);
|
|
24
|
+
}
|
|
25
|
+
// A committed transcript abort is weaker than a gateway terminal error.
|
|
26
|
+
// One stronger envelope can wake a parked observation; repeats cannot.
|
|
27
|
+
const terminalChat = (p) => p.source === "chat" && (p.state === "error" || (p.state === "final" && p.stopReason === "error"));
|
|
28
|
+
if (!provenance.userStop && !previous.provenance.userStop && terminalChat(provenance) && !terminalChat(previous.provenance)
|
|
29
|
+
&& ["pending", "parked"].includes(previous.status)) {
|
|
30
|
+
previous.provenance = provenance;
|
|
31
|
+
previous.status = "pending";
|
|
32
|
+
previous.checks = Math.min(previous.checks, 5);
|
|
33
|
+
previous.nextCheckAt = Date.now();
|
|
34
|
+
this.saveErrorNotice(previous);
|
|
35
|
+
}
|
|
36
|
+
return previous; // duplicate/late events never reset the budget
|
|
37
|
+
}
|
|
38
|
+
const record = { botName, chatId, noticeId, provenance, checks: 0,
|
|
39
|
+
nextCheckAt: Date.now(), createdAt: Date.now(), status: "pending" };
|
|
40
|
+
this.saveErrorNotice(record);
|
|
41
|
+
return record;
|
|
42
|
+
}
|
|
43
|
+
getErrorNotice(botName, chatId, noticeId) {
|
|
44
|
+
const row = this.db.prepare("SELECT record_json FROM error_notices WHERE bot_name=? AND chat_id=? AND notice_id=?")
|
|
45
|
+
.get(botName, chatId, noticeId);
|
|
46
|
+
return row ? JSON.parse(row.record_json) : undefined;
|
|
47
|
+
}
|
|
48
|
+
saveErrorNotice(record) {
|
|
49
|
+
this.db.prepare(`INSERT INTO error_notices (bot_name,chat_id,notice_id,record_json,next_check_at,status)
|
|
50
|
+
VALUES (?,?,?,?,?,?) ON CONFLICT(bot_name,chat_id,notice_id) DO UPDATE SET
|
|
51
|
+
record_json=excluded.record_json,next_check_at=excluded.next_check_at,status=excluded.status`)
|
|
52
|
+
.run(record.botName, record.chatId, record.noticeId, JSON.stringify(record), record.nextCheckAt, record.status);
|
|
53
|
+
}
|
|
54
|
+
listPendingErrorNotices(botName, chatId) {
|
|
55
|
+
return this.db.prepare("SELECT record_json FROM error_notices WHERE bot_name=? AND chat_id=? AND status='pending' ORDER BY next_check_at LIMIT 100")
|
|
56
|
+
.all(botName, chatId).map(row => JSON.parse(row.record_json));
|
|
57
|
+
}
|
|
58
|
+
/** A later exact final may be content-deduped against its own transcript.
|
|
59
|
+
* Upgrade only identical payloads with the same canonical session/run; never
|
|
60
|
+
* infer identity from text similarity or another run's successful answer. */
|
|
61
|
+
promoteExactRunFinal(botName, chatId, sessionKey, runId, contentHash, content, attachmentsJson) {
|
|
62
|
+
this.db.prepare(`UPDATE delivery_outbox
|
|
63
|
+
SET delivery_meta_json=json_set(delivery_meta_json,'$.provenance.final',json('true'))
|
|
64
|
+
WHERE bot_name=? AND chat_id=? AND source_type='assistant_visible' AND content_hash=?
|
|
65
|
+
AND content=? AND attachments_json=?
|
|
66
|
+
AND json_valid(delivery_meta_json)
|
|
67
|
+
AND json_extract(delivery_meta_json,'$.provenance.sessionKey')=?
|
|
68
|
+
AND json_extract(delivery_meta_json,'$.provenance.runId')=?`).run(botName, chatId, contentHash, content, attachmentsJson, sessionKey, runId);
|
|
69
|
+
}
|
|
70
|
+
hasExactRunFinalPayload(botName, chatId, p, content, attachmentsJson) {
|
|
71
|
+
return Boolean(this.db.prepare(`SELECT 1 FROM delivery_outbox
|
|
72
|
+
WHERE bot_name=? AND chat_id=? AND source_type='assistant_visible'
|
|
73
|
+
AND content=? AND attachments_json=? AND status IN ('pending','delivering','delivered')
|
|
74
|
+
AND json_valid(delivery_meta_json) AND json_extract(delivery_meta_json,'$.provenance.final')=1
|
|
75
|
+
AND json_extract(delivery_meta_json,'$.provenance.sessionKey')=?
|
|
76
|
+
AND json_extract(delivery_meta_json,'$.provenance.runId')=? LIMIT 1`)
|
|
77
|
+
.get(botName, chatId, content, attachmentsJson, p.sessionKey, p.runId));
|
|
78
|
+
}
|
|
79
|
+
hasRunFinalDelivery(botName, chatId, p, confirmed) {
|
|
80
|
+
return Boolean(p.runId && this.db.prepare(`SELECT 1 FROM delivery_outbox
|
|
81
|
+
WHERE bot_name=? AND chat_id=? AND source_type='assistant_visible'
|
|
82
|
+
AND json_valid(delivery_meta_json)
|
|
83
|
+
AND json_extract(delivery_meta_json,'$.provenance.final')=1
|
|
84
|
+
AND json_extract(delivery_meta_json,'$.provenance.sessionKey')=?
|
|
85
|
+
AND json_extract(delivery_meta_json,'$.provenance.runId')=?
|
|
86
|
+
AND ${confirmed ? "status='delivered'" : "status IN ('pending','delivering')"} LIMIT 1`)
|
|
87
|
+
.get(botName, chatId, p.sessionKey, p.runId));
|
|
88
|
+
}
|
|
89
|
+
renewUnsentErrorDelivery(id, content, contentHash, metadata) {
|
|
90
|
+
this.db.prepare(`UPDATE delivery_outbox SET status='pending',content=?,content_hash=?,delivery_meta_json=?,updated_at=?
|
|
91
|
+
WHERE id=? AND source_type='run_error' AND status='failed' AND text_delivered=0`)
|
|
92
|
+
.run(content, contentHash, metadata, Date.now(), id);
|
|
93
|
+
}
|
|
94
|
+
/** Withdraw only an unsent error notice. Never claim it was delivered or touch a result row. */
|
|
95
|
+
withdrawErrorDelivery(id) {
|
|
96
|
+
// claimDelivery reserves a send attempt before the asynchronous evidence
|
|
97
|
+
// guard. Withdrawing here means no platform send was attempted: refund only
|
|
98
|
+
// that current claim, never an earlier failed/ambiguous platform attempt.
|
|
99
|
+
this.db.prepare("UPDATE delivery_outbox SET status='failed', attempts=MAX(0,attempts-1), updated_at=? WHERE id=? AND source_type='run_error' AND status='delivering' AND text_delivered=0")
|
|
100
|
+
.run(Date.now(), id);
|
|
101
|
+
}
|
|
14
102
|
init() {
|
|
15
103
|
this.db.exec(`
|
|
104
|
+
CREATE TABLE IF NOT EXISTS error_notices (
|
|
105
|
+
bot_name TEXT NOT NULL, chat_id TEXT NOT NULL, notice_id TEXT NOT NULL,
|
|
106
|
+
record_json TEXT NOT NULL, next_check_at INTEGER NOT NULL,
|
|
107
|
+
status TEXT NOT NULL, PRIMARY KEY(bot_name, chat_id, notice_id)
|
|
108
|
+
);
|
|
109
|
+
CREATE INDEX IF NOT EXISTS idx_error_notices_due ON error_notices(bot_name, chat_id, status, next_check_at);
|
|
16
110
|
CREATE TABLE IF NOT EXISTS messages (
|
|
17
111
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
18
112
|
chat_id TEXT NOT NULL,
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type ErrorProvenance, type ProactiveMessageMeta } from "./terminal-errors.js";
|
|
1
2
|
import { SessionWaitPaused, type SessionRuntimeStatus } from "./session-status.js";
|
|
2
3
|
import { OpenClawConfig } from "./config.js";
|
|
3
4
|
import { ChatMessage } from "./message-store.js";
|
|
@@ -76,6 +77,11 @@ export declare class OpenClawClient {
|
|
|
76
77
|
private mutedProactiveSessionCounts;
|
|
77
78
|
/** Sessions force-aborted by /stop; collectReply should finish immediately instead of waiting for idle timeout. */
|
|
78
79
|
private forceAbortedSessions;
|
|
80
|
+
/** Explicit-stop evidence comes from exact authorized abort response run IDs,
|
|
81
|
+
* never the persistent session flag used by the original collector. */
|
|
82
|
+
private explicitlyStoppedRuns;
|
|
83
|
+
private pendingExplicitStops;
|
|
84
|
+
private isExplicitlyStoppedRun;
|
|
79
85
|
/** Global limiter for chat.send RPC calls; large multi-bot fan-out can
|
|
80
86
|
* saturate the Gateway before collectReply even starts. The slot is released
|
|
81
87
|
* as soon as the chat.send RPC returns a runId; collectReply does not hold it.
|
|
@@ -208,6 +214,7 @@ export declare class OpenClawClient {
|
|
|
208
214
|
abortChat(sessionKey: string, runId?: string): Promise<any>;
|
|
209
215
|
private sessionKeyVariants;
|
|
210
216
|
private trackChatEventSession;
|
|
217
|
+
inspectRunError(provenance: ErrorProvenance): Promise<import("./terminal-errors.js").ErrorVerdict>;
|
|
211
218
|
private extractTextFromChatMessage;
|
|
212
219
|
private emitProactiveForSession;
|
|
213
220
|
private suppressSessionKeys;
|
|
@@ -279,10 +286,7 @@ export declare class OpenClawClient {
|
|
|
279
286
|
* Subscribe to message events for a session.
|
|
280
287
|
* When the agent proactively produces a message, the callback fires.
|
|
281
288
|
*/
|
|
282
|
-
subscribeSession(sessionKey: string, onMessage: (text: string, meta?:
|
|
283
|
-
sourceType?: string;
|
|
284
|
-
runId?: string;
|
|
285
|
-
}) => void): Promise<void>;
|
|
289
|
+
subscribeSession(sessionKey: string, onMessage: (text: string, meta?: ProactiveMessageMeta) => void): Promise<void>;
|
|
286
290
|
setVerboseTranscriptDelivery(sessionKey: string, enabled: boolean): void;
|
|
287
291
|
unsubscribeSession(sessionKey: string): Promise<void>;
|
|
288
292
|
/**
|
package/dist/openclaw-client.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import WebSocket from "ws";
|
|
2
|
+
import { inspectError, genericFailure } from "./terminal-errors.js";
|
|
2
3
|
import { SessionWaitPaused, normalizeSessionRuntimeStatus, isWaitTimeout, FOREGROUND_WAIT_MS } from "./session-status.js";
|
|
3
4
|
import { createHash, randomUUID } from "crypto";
|
|
4
5
|
import { mkdirSync, readFileSync, statSync, writeFileSync } from "fs";
|
|
@@ -80,6 +81,23 @@ export class OpenClawClient {
|
|
|
80
81
|
mutedProactiveSessionCounts = new Map();
|
|
81
82
|
/** Sessions force-aborted by /stop; collectReply should finish immediately instead of waiting for idle timeout. */
|
|
82
83
|
forceAbortedSessions = new Set();
|
|
84
|
+
/** Explicit-stop evidence comes from exact authorized abort response run IDs,
|
|
85
|
+
* never the persistent session flag used by the original collector. */
|
|
86
|
+
explicitlyStoppedRuns = new Map();
|
|
87
|
+
pendingExplicitStops = new Map();
|
|
88
|
+
isExplicitlyStoppedRun(sessionKey, runId) {
|
|
89
|
+
if (!runId)
|
|
90
|
+
return false;
|
|
91
|
+
const key = JSON.stringify([this.canonicalSessionKey(sessionKey), runId]);
|
|
92
|
+
const at = this.explicitlyStoppedRuns.get(key);
|
|
93
|
+
if (at === undefined)
|
|
94
|
+
return false;
|
|
95
|
+
if (Date.now() - at > 24 * 60 * 60_000) {
|
|
96
|
+
this.explicitlyStoppedRuns.delete(key);
|
|
97
|
+
return false;
|
|
98
|
+
}
|
|
99
|
+
return true;
|
|
100
|
+
}
|
|
83
101
|
/** Global limiter for chat.send RPC calls; large multi-bot fan-out can
|
|
84
102
|
* saturate the Gateway before collectReply even starts. The slot is released
|
|
85
103
|
* as soon as the chat.send RPC returns a runId; collectReply does not hold it.
|
|
@@ -639,6 +657,18 @@ export class OpenClawClient {
|
|
|
639
657
|
// structured content arrays rather than a plain string. Extract only visible
|
|
640
658
|
// text parts and ignore thinking/tool blocks so the bridge can deliver final
|
|
641
659
|
// cron results via the bot.
|
|
660
|
+
if (msg?.role === "assistant" && ["error", "aborted"].includes(msg?.stopReason)) {
|
|
661
|
+
if (this.mutedProactiveSessions.has(rawKey) || this.mutedProactiveSessions.has(shortKey))
|
|
662
|
+
return false;
|
|
663
|
+
if (runId && this.ownedDeliveryRuns.has(runId))
|
|
664
|
+
return false;
|
|
665
|
+
return this.emitProactiveForSession(rawKey, String(msg.errorMessage || "Agent run failed"), {
|
|
666
|
+
sourceType: "run_error", runId,
|
|
667
|
+
error: { sessionKey: this.canonicalSessionKey(rawKey), runId: runId || "", source: "session.message",
|
|
668
|
+
state: "final", stopReason: msg.stopReason, detail: String(msg.errorMessage || ""),
|
|
669
|
+
...(this.isExplicitlyStoppedRun(rawKey, runId) ? { userStop: true } : {}) },
|
|
670
|
+
});
|
|
671
|
+
}
|
|
642
672
|
const allowVerboseTranscript = this.isVerboseTranscriptEnabled(rawKey);
|
|
643
673
|
const proactiveText = this.extractVisibleAssistantText(msg, { allowMixedToolText: allowVerboseTranscript });
|
|
644
674
|
if (!proactiveText)
|
|
@@ -658,7 +688,8 @@ export class OpenClawClient {
|
|
|
658
688
|
const cb = this.sessionMessageCallbacks.get(rawKey) || this.sessionMessageCallbacks.get(shortKey);
|
|
659
689
|
if (cb) {
|
|
660
690
|
if (runId)
|
|
661
|
-
cb(proactiveText, { runId
|
|
691
|
+
cb(proactiveText, { runId, sessionKey: this.canonicalSessionKey(rawKey), final: msg?.stopReason === "stop",
|
|
692
|
+
...(!msg?.stopReason && genericFailure(proactiveText) ? { genericErrorCandidate: true } : {}) });
|
|
662
693
|
else
|
|
663
694
|
cb(proactiveText);
|
|
664
695
|
}
|
|
@@ -1789,13 +1820,39 @@ export class OpenClawClient {
|
|
|
1789
1820
|
// chat.abort supports { sessionKey } with no runId to abort ALL active runs
|
|
1790
1821
|
// for that session. Used by /stop to force-clear a stuck run.
|
|
1791
1822
|
if (!runId) {
|
|
1823
|
+
for (const [identity, at] of this.explicitlyStoppedRuns)
|
|
1824
|
+
if (Date.now() - at > 24 * 60 * 60_000)
|
|
1825
|
+
this.explicitlyStoppedRuns.delete(identity);
|
|
1792
1826
|
// Mark the session so any in-flight collectReply finishes immediately
|
|
1793
1827
|
// instead of treating the cancelled lifecycle as a transient state.
|
|
1794
|
-
this.
|
|
1795
|
-
|
|
1828
|
+
if (this.runStateChecks.has(key)) {
|
|
1829
|
+
this.forceAbortedSessions.add(key);
|
|
1830
|
+
this.forceAbortedSessions.add(this.shortKey(key));
|
|
1831
|
+
}
|
|
1796
1832
|
}
|
|
1797
1833
|
const params = runId ? { sessionKey: key, runId } : { sessionKey: key };
|
|
1798
|
-
|
|
1834
|
+
if (runId)
|
|
1835
|
+
return this.rpc("chat.abort", params, 5000).catch(() => { });
|
|
1836
|
+
let resolveStop;
|
|
1837
|
+
const barrier = new Promise(resolve => { resolveStop = resolve; });
|
|
1838
|
+
this.pendingExplicitStops.set(key, barrier);
|
|
1839
|
+
try {
|
|
1840
|
+
const result = await this.rpc("chat.abort", params, 5000).catch(() => undefined);
|
|
1841
|
+
// The authorized abort response names the exact runs actually stopped;
|
|
1842
|
+
// it also covers queued/unseen runs with no local status/delta event.
|
|
1843
|
+
if (result?.aborted === true && Array.isArray(result.runIds)) {
|
|
1844
|
+
for (const stopped of result.runIds)
|
|
1845
|
+
if (typeof stopped === "string" && stopped) {
|
|
1846
|
+
this.explicitlyStoppedRuns.set(JSON.stringify([key, stopped]), Date.now());
|
|
1847
|
+
}
|
|
1848
|
+
}
|
|
1849
|
+
return result;
|
|
1850
|
+
}
|
|
1851
|
+
finally {
|
|
1852
|
+
resolveStop();
|
|
1853
|
+
if (this.pendingExplicitStops.get(key) === barrier)
|
|
1854
|
+
this.pendingExplicitStops.delete(key);
|
|
1855
|
+
}
|
|
1799
1856
|
}
|
|
1800
1857
|
sessionKeyVariants(key) {
|
|
1801
1858
|
const shortKey = this.shortKey(key);
|
|
@@ -1812,39 +1869,55 @@ export class OpenClawClient {
|
|
|
1812
1869
|
// still reach the proactive callback.
|
|
1813
1870
|
return;
|
|
1814
1871
|
}
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1872
|
+
const stopReason = [payload?.stopReason, payload?.message?.stopReason].find(reason => reason === "error" || reason === "aborted")
|
|
1873
|
+
|| payload?.stopReason || payload?.message?.stopReason;
|
|
1874
|
+
const isError = state === "error" || state === "aborted" || stopReason === "error" || stopReason === "aborted";
|
|
1875
|
+
if (isError) {
|
|
1876
|
+
const detail = String(payload?.errorMessage || payload?.message?.errorMessage || payload?.error || payload?.reason || "").trim();
|
|
1877
|
+
const aborted = state === "aborted" || stopReason === "aborted";
|
|
1878
|
+
const text = `⚠️ Agent run ${aborted ? "was aborted" : "failed"}${detail ? `: ${detail}` : ""}`;
|
|
1879
|
+
this.emitProactiveForSession(sessionKey, text, {
|
|
1880
|
+
sourceType: "run_error", runId,
|
|
1881
|
+
error: { sessionKey: this.canonicalSessionKey(sessionKey), runId, source: "chat", state: state || "final",
|
|
1882
|
+
...(typeof stopReason === "string" ? { stopReason } : {}), detail,
|
|
1883
|
+
...(this.isExplicitlyStoppedRun(sessionKey, runId) ? { userStop: true } : {}) },
|
|
1884
|
+
});
|
|
1885
|
+
this.releaseSuppressedSessionKeysAfter(keys, 0);
|
|
1886
|
+
}
|
|
1887
|
+
else if (state === "delta") {
|
|
1818
1888
|
this.suppressSessionKeys(keys);
|
|
1819
1889
|
}
|
|
1820
1890
|
else if (state === "final") {
|
|
1821
1891
|
const text = this.extractTextFromChatMessage(payload?.message);
|
|
1822
1892
|
if (text)
|
|
1823
|
-
this.emitProactiveForSession(sessionKey, text,
|
|
1893
|
+
this.emitProactiveForSession(sessionKey, text, {
|
|
1894
|
+
runId, sessionKey: this.canonicalSessionKey(sessionKey), final: payload?.yielded !== true,
|
|
1895
|
+
...(!stopReason && genericFailure(text) ? { genericErrorCandidate: true } : {}),
|
|
1896
|
+
});
|
|
1824
1897
|
this.releaseSuppressedSessionKeysAfter(keys, 30000);
|
|
1825
1898
|
}
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
this.emitProactiveForSession(sessionKey, text, runId, "run_error");
|
|
1832
|
-
this.releaseSuppressedSessionKeysAfter(keys, 0);
|
|
1899
|
+
}
|
|
1900
|
+
async inspectRunError(provenance) {
|
|
1901
|
+
await this.pendingExplicitStops.get(this.canonicalSessionKey(provenance.sessionKey));
|
|
1902
|
+
if (this.isExplicitlyStoppedRun(provenance.sessionKey, provenance.runId)) {
|
|
1903
|
+
return { state: "stopped", reason: "exact_run_abort_receipt" };
|
|
1833
1904
|
}
|
|
1905
|
+
const verdict = await inspectError(provenance, (method, params, timeout) => this.rpc(method, params, timeout));
|
|
1906
|
+
// A stop request can begin while task/wait evidence is in flight.
|
|
1907
|
+
await this.pendingExplicitStops.get(this.canonicalSessionKey(provenance.sessionKey));
|
|
1908
|
+
return this.isExplicitlyStoppedRun(provenance.sessionKey, provenance.runId)
|
|
1909
|
+
? { state: "stopped", reason: "exact_run_abort_receipt" } : verdict;
|
|
1834
1910
|
}
|
|
1835
1911
|
extractTextFromChatMessage(message) {
|
|
1836
1912
|
const parts = Array.isArray(message?.content) ? message.content : [];
|
|
1837
1913
|
return parts.filter((part) => part?.type === "text" && typeof part.text === "string").map((part) => part.text).join("\n").trim();
|
|
1838
1914
|
}
|
|
1839
|
-
emitProactiveForSession(sessionKey, text,
|
|
1915
|
+
emitProactiveForSession(sessionKey, text, meta) {
|
|
1840
1916
|
const [shortKey, fullKey] = this.sessionKeyVariants(sessionKey);
|
|
1841
1917
|
const cb = this.sessionMessageCallbacks.get(fullKey) || this.sessionMessageCallbacks.get(shortKey);
|
|
1842
1918
|
if (!cb)
|
|
1843
1919
|
return false;
|
|
1844
|
-
|
|
1845
|
-
cb(text, { ...(runId ? { runId } : {}), ...(sourceType ? { sourceType } : {}) });
|
|
1846
|
-
else
|
|
1847
|
-
cb(text);
|
|
1920
|
+
cb(text, meta);
|
|
1848
1921
|
return true;
|
|
1849
1922
|
}
|
|
1850
1923
|
suppressSessionKeys(keys) {
|
|
@@ -2221,6 +2294,7 @@ export class OpenClawClient {
|
|
|
2221
2294
|
this.pendingSteerTexts.clear();
|
|
2222
2295
|
this.steerConsumedCallbacks.clear();
|
|
2223
2296
|
this.ownedDeliveryRuns.clear();
|
|
2297
|
+
this.explicitlyStoppedRuns.clear();
|
|
2224
2298
|
if (this.ws) {
|
|
2225
2299
|
this.ws.close();
|
|
2226
2300
|
this.ws = null;
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/** Only gateway envelopes / committed assistant metadata populate this record. */
|
|
2
|
+
export interface ErrorProvenance {
|
|
3
|
+
sessionKey: string;
|
|
4
|
+
runId: string;
|
|
5
|
+
source: "chat" | "session.message";
|
|
6
|
+
state: string;
|
|
7
|
+
stopReason?: string;
|
|
8
|
+
detail?: string;
|
|
9
|
+
userStop?: boolean;
|
|
10
|
+
}
|
|
11
|
+
export interface ProactiveMessageMeta {
|
|
12
|
+
sourceType?: string;
|
|
13
|
+
runId?: string;
|
|
14
|
+
sessionKey?: string;
|
|
15
|
+
final?: boolean;
|
|
16
|
+
error?: ErrorProvenance;
|
|
17
|
+
terminalEvidence?: ErrorVerdict;
|
|
18
|
+
/** Missing stopReason: only a persisted error for this exact run may qualify. */
|
|
19
|
+
genericErrorCandidate?: boolean;
|
|
20
|
+
}
|
|
21
|
+
export interface TaskEvidence {
|
|
22
|
+
id: string;
|
|
23
|
+
kind?: string;
|
|
24
|
+
runId?: string;
|
|
25
|
+
sourceId?: string;
|
|
26
|
+
sessionKey?: string;
|
|
27
|
+
ownerKey?: string;
|
|
28
|
+
childSessionKey?: string;
|
|
29
|
+
parentTaskId?: string;
|
|
30
|
+
status: string;
|
|
31
|
+
deliveryStatus?: string;
|
|
32
|
+
terminalOutcome?: string;
|
|
33
|
+
}
|
|
34
|
+
export interface ErrorVerdict {
|
|
35
|
+
state: "pending" | "suppressed" | "terminal" | "stopped";
|
|
36
|
+
reason: string;
|
|
37
|
+
task?: TaskEvidence;
|
|
38
|
+
terminalKey?: string;
|
|
39
|
+
text?: string;
|
|
40
|
+
}
|
|
41
|
+
export type ReadRPC = (method: string, params: Record<string, unknown>, timeoutMs: number) => Promise<any>;
|
|
42
|
+
export declare const errorIdentity: (sessionKey: string, runId: string) => string;
|
|
43
|
+
export declare const genericFailure: (text: string) => boolean;
|
|
44
|
+
export declare function announceCandidate(p: ErrorProvenance): {
|
|
45
|
+
runId: string;
|
|
46
|
+
childSessionKey?: string;
|
|
47
|
+
} | undefined;
|
|
48
|
+
/** Read-only and bounded: no model invocation, replay, cancellation or session-status heuristic. */
|
|
49
|
+
export declare function inspectError(p: ErrorProvenance, rpc: ReadRPC): Promise<ErrorVerdict>;
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
export const errorIdentity = (sessionKey, runId) => createHash("sha256").update(JSON.stringify([sessionKey, runId])).digest("hex");
|
|
3
|
+
export const genericFailure = (text) => text.trim() === "The agent run failed before producing a reply.";
|
|
4
|
+
// These are candidate locators, NOT business identities. Verify every component
|
|
5
|
+
// against tasks.list + a fresh tasks.get. Unsupported/batch forms stay unknown.
|
|
6
|
+
export function announceCandidate(p) {
|
|
7
|
+
const uuid = "[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}";
|
|
8
|
+
const direct = p.runId.match(new RegExp(`^announce:v1:(agent:[^:]+:.+):(${uuid})$`));
|
|
9
|
+
if (direct)
|
|
10
|
+
return { childSessionKey: direct[1], runId: direct[2] };
|
|
11
|
+
const agent = p.sessionKey.match(/^agent:([^:]+):/)?.[1];
|
|
12
|
+
const prefix = `announce:requester-settle:${agent}:${p.sessionKey}:`;
|
|
13
|
+
if (!agent || !p.runId.startsWith(prefix))
|
|
14
|
+
return;
|
|
15
|
+
const wake = p.runId.slice(prefix.length).match(new RegExp(`^(${uuid})(?::yield-\\d+)?(?::retry-\\d+)?$`));
|
|
16
|
+
if (wake)
|
|
17
|
+
return { runId: wake[1] };
|
|
18
|
+
}
|
|
19
|
+
function projectTask(t) {
|
|
20
|
+
const out = {};
|
|
21
|
+
for (const k of ["id", "kind", "runId", "sourceId", "sessionKey", "ownerKey", "childSessionKey", "parentTaskId", "status", "deliveryStatus", "terminalOutcome"]) {
|
|
22
|
+
if (typeof t?.[k] === "string")
|
|
23
|
+
out[k] = t[k];
|
|
24
|
+
}
|
|
25
|
+
return out;
|
|
26
|
+
}
|
|
27
|
+
function matches(t, p, c) {
|
|
28
|
+
return Boolean(t.id && t.kind === "subagent" && t.runId === c.runId
|
|
29
|
+
&& t.sessionKey === p.sessionKey && t.ownerKey === p.sessionKey
|
|
30
|
+
&& t.childSessionKey && (!c.childSessionKey || t.childSessionKey === c.childSessionKey));
|
|
31
|
+
}
|
|
32
|
+
/** Read-only and bounded: no model invocation, replay, cancellation or session-status heuristic. */
|
|
33
|
+
export async function inspectError(p, rpc) {
|
|
34
|
+
if (p.userStop)
|
|
35
|
+
return { state: "stopped", reason: "explicit_bridge_stop" };
|
|
36
|
+
const pending = (reason, task) => ({ state: "pending", reason, task });
|
|
37
|
+
const announce = p.runId.startsWith("announce:");
|
|
38
|
+
if (!p.runId)
|
|
39
|
+
return pending("missing_run_identity");
|
|
40
|
+
if (!announce) {
|
|
41
|
+
// Ordinary chat.error is a gateway terminal envelope (after gateway retry
|
|
42
|
+
// grace). Do not rewrite normal collector/auth/parameter/tool-error handling.
|
|
43
|
+
if (p.source === "chat" && (p.state === "error" || (p.state === "final" && p.stopReason === "error")))
|
|
44
|
+
return {
|
|
45
|
+
state: "terminal", reason: "ordinary_gateway_error", terminalKey: `run-error:${errorIdentity(p.sessionKey, p.runId)}`,
|
|
46
|
+
text: `⚠️ Agent run failed${p.detail ? `: ${p.detail}` : ""}`,
|
|
47
|
+
};
|
|
48
|
+
try {
|
|
49
|
+
const wait = await rpc("agent.wait", { runId: p.runId, timeoutMs: 1 }, 2000);
|
|
50
|
+
const ended = wait?.endedAt;
|
|
51
|
+
const started = wait?.startedAt;
|
|
52
|
+
const terminalTime = typeof ended === "number" && Number.isFinite(ended) && ended > 0
|
|
53
|
+
&& (started === undefined || (typeof started === "number" && Number.isFinite(started) && started >= 0 && ended >= started));
|
|
54
|
+
if (wait?.runId === p.runId && wait?.status === "error" && wait?.pendingError !== true
|
|
55
|
+
&& wait?.yielded !== true && terminalTime)
|
|
56
|
+
return {
|
|
57
|
+
state: "terminal", reason: "exact_run_terminal_error", terminalKey: `run-error:${errorIdentity(p.sessionKey, p.runId)}`,
|
|
58
|
+
text: "⚠️ 该次运行已终止,未能完成回复。",
|
|
59
|
+
};
|
|
60
|
+
return pending("wait_not_terminal_failure");
|
|
61
|
+
}
|
|
62
|
+
catch {
|
|
63
|
+
return pending("wait_unavailable");
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
const candidate = announceCandidate(p);
|
|
67
|
+
if (!candidate)
|
|
68
|
+
return pending("unsupported_announce_lineage");
|
|
69
|
+
try {
|
|
70
|
+
// At most two pages; absence or truncation is never evidence of failure.
|
|
71
|
+
let cursor;
|
|
72
|
+
const matchesFound = [];
|
|
73
|
+
for (let page = 0; page < 2; page++) {
|
|
74
|
+
const result = await rpc("tasks.list", { sessionKey: p.sessionKey, limit: 100, ...(cursor ? { cursor } : {}) }, 2000);
|
|
75
|
+
if (!Array.isArray(result?.tasks))
|
|
76
|
+
return pending("unsupported_task_response");
|
|
77
|
+
for (const raw of result.tasks) {
|
|
78
|
+
const task = projectTask(raw);
|
|
79
|
+
if (matches(task, p, candidate))
|
|
80
|
+
matchesFound.push(task);
|
|
81
|
+
}
|
|
82
|
+
cursor = typeof result.nextCursor === "string" ? result.nextCursor : undefined;
|
|
83
|
+
if (!cursor)
|
|
84
|
+
break;
|
|
85
|
+
}
|
|
86
|
+
const ids = [...new Set(matchesFound.map(t => t.id))];
|
|
87
|
+
if (ids.length !== 1 || cursor)
|
|
88
|
+
return pending("task_lineage_missing_ambiguous_or_truncated");
|
|
89
|
+
const task = projectTask((await rpc("tasks.get", { taskId: ids[0] }, 2000))?.task);
|
|
90
|
+
if (task.id !== ids[0] || !matches(task, p, candidate))
|
|
91
|
+
return pending("task_lineage_changed");
|
|
92
|
+
if (["queued", "running"].includes(task.status) || ["pending", "session_queued"].includes(task.deliveryStatus || "")) {
|
|
93
|
+
return pending("matching_task_or_delivery_continuing", task);
|
|
94
|
+
}
|
|
95
|
+
if (task.status === "completed" && task.deliveryStatus === "delivered" && task.terminalOutcome !== "blocked") {
|
|
96
|
+
return { state: "suppressed", reason: "matching_task_completed_delivery_confirmed", task };
|
|
97
|
+
}
|
|
98
|
+
const terminalKey = `task-error:${errorIdentity(p.sessionKey, task.id)}`;
|
|
99
|
+
// A failed child is not the whole user request. Do not borrow another child's
|
|
100
|
+
// successful result, or substitute raw child output for the parent answer.
|
|
101
|
+
if (["failed", "timed_out"].includes(task.status) && ["delivered", "failed", "not_applicable"].includes(task.deliveryStatus || "")) {
|
|
102
|
+
return { state: "terminal", reason: "matching_child_terminal_failure", task, terminalKey,
|
|
103
|
+
text: `⚠️ 子任务 ${task.id} ${task.status === "timed_out" ? "已超时终止" : "已失败"};这不代表同会话其他任务失败。` };
|
|
104
|
+
}
|
|
105
|
+
// Public schema has no requester-settle retry counter. Only the explicit
|
|
106
|
+
// blocked + failed delivery projection proves that continuation is blocked.
|
|
107
|
+
if (task.status === "completed" && task.deliveryStatus === "failed" && task.terminalOutcome === "blocked") {
|
|
108
|
+
return { state: "terminal", reason: "matching_completion_delivery_blocked", task, terminalKey,
|
|
109
|
+
text: `⚠️ 子任务 ${task.id} 已完成,但结果回传已终止,需要人工处理;并非任务执行失败。` };
|
|
110
|
+
}
|
|
111
|
+
return pending("task_does_not_prove_attempt_chain_terminal", task);
|
|
112
|
+
}
|
|
113
|
+
catch {
|
|
114
|
+
return pending("task_lookup_unavailable");
|
|
115
|
+
}
|
|
116
|
+
}
|