bereach-openclaw 1.5.5 → 1.5.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -3
- package/node_modules/@bereach/tools/src/definitions.ts +6 -0
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
- package/src/commands/connector.ts +119 -39
- package/src/connector/manager.ts +168 -15
- package/src/hooks/lifecycle.ts +22 -2
- package/src/index.ts +10 -4
package/README.md
CHANGED
|
@@ -21,8 +21,11 @@ openclaw plugins install bereach-openclaw
|
|
|
21
21
|
**Before upgrading:** note your `BEREACH_API_KEY` - uninstall may remove `plugins.entries.bereach-openclaw`.
|
|
22
22
|
|
|
23
23
|
```bash
|
|
24
|
-
# 1. Uninstall
|
|
24
|
+
# 1. Uninstall + fully delete (rm prevents stale .old copies causing 401 loops)
|
|
25
25
|
openclaw plugins uninstall bereach-openclaw
|
|
26
|
+
rm -rf /data/.openclaw/extensions/bereach-openclaw*
|
|
27
|
+
rm -rf /data/.openclaw/node_modules/bereach-openclaw
|
|
28
|
+
rm -rf /data/.openclaw/node_modules/bereach
|
|
26
29
|
|
|
27
30
|
# 2. Reinstall latest
|
|
28
31
|
openclaw plugins install bereach-openclaw
|
|
@@ -108,9 +111,9 @@ OpenClaw loads plugins from **two locations**:
|
|
|
108
111
|
|
|
109
112
|
If `extensions/bereach-openclaw/` is corrupt or incomplete, you get trim/undefined errors. Fix:
|
|
110
113
|
|
|
111
|
-
**1.
|
|
114
|
+
**1. Fully remove the active extension** (do NOT rename to `.bak` - OpenClaw loads `.bak`/`.old` copies as duplicate plugins):
|
|
112
115
|
```bash
|
|
113
|
-
|
|
116
|
+
rm -rf /data/.openclaw/extensions/bereach-openclaw*
|
|
114
117
|
```
|
|
115
118
|
|
|
116
119
|
**2. Reinstall from npm (inside the container):**
|
|
@@ -1289,6 +1289,11 @@ export const definitions: ToolDefinition[] = [
|
|
|
1289
1289
|
dailyActionLimit: { type: "integer", minimum: 1, description: "Max actions per day." },
|
|
1290
1290
|
dailyTarget: { type: "integer", minimum: 1, description: "Daily target goal (e.g. 50 leads/day). Set null to remove." },
|
|
1291
1291
|
totalTarget: { type: "integer", minimum: 1, description: "Total campaign goal (e.g. 500 leads total). Auto-completes when reached. Set null to remove." },
|
|
1292
|
+
handoverMode: {
|
|
1293
|
+
type: "string",
|
|
1294
|
+
enum: ["on_reply", "on_goal", "manual"],
|
|
1295
|
+
description: "When agent hands over conversation to human. on_reply (default): stop autonomous outreach when contact replies. on_goal: keep replying until meeting booked or converted. manual: never auto-stop.",
|
|
1296
|
+
},
|
|
1292
1297
|
taskOverrides: {
|
|
1293
1298
|
type: "object",
|
|
1294
1299
|
description: "Per-task-type config overrides. Keys are task types (e.g. 'lead-gen-visit', 'lead-gen-qualify'). Values are config objects with: maxRunsPerDay, minIntervalMinutes, defaultBatchSize, maxCreditsPerRun.",
|
|
@@ -1428,6 +1433,7 @@ export const definitions: ToolDefinition[] = [
|
|
|
1428
1433
|
lifecycleStage: { type: "string", enum: ["contact", "lead", "qualified", "approved", "rejected"], description: "New stage (forward-only, or rejected)." },
|
|
1429
1434
|
hotScore: { type: "integer", minimum: 0, maximum: 100 },
|
|
1430
1435
|
qualificationNotes: { type: "string", description: "Agent reasoning for qualification/rejection." },
|
|
1436
|
+
leadBrief: { type: "string", description: "2-3 sentence human-readable lead summary for sales prep. NOT ICP analysis — a cheat sheet: who this person is, what they care about, notable background, conversation hooks." },
|
|
1431
1437
|
notes: { type: "string" },
|
|
1432
1438
|
name: { type: "string", description: "Update contact name." },
|
|
1433
1439
|
profileData: { type: "object", description: "Full LinkedIn profile snapshot (JSON)." },
|
package/openclaw.plugin.json
CHANGED
package/package.json
CHANGED
|
@@ -58,10 +58,19 @@ type TaskResult = {
|
|
|
58
58
|
commentsPosted?: number;
|
|
59
59
|
invitationsAccepted?: number;
|
|
60
60
|
error?: string;
|
|
61
|
+
reason?: string;
|
|
61
62
|
};
|
|
62
63
|
|
|
63
64
|
const MIN_POLL_MS = 5000;
|
|
64
65
|
|
|
66
|
+
/** Thrown when API returns 401/403 - credentials are invalid, retrying won't help */
|
|
67
|
+
export class AuthError extends Error {
|
|
68
|
+
constructor(public readonly statusCode: number, body: string) {
|
|
69
|
+
super(`Auth failed (HTTP ${statusCode}): ${body.slice(0, 200)}`);
|
|
70
|
+
this.name = "AuthError";
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
65
74
|
// ---------------------------------------------------------------------------
|
|
66
75
|
// TaskResult extraction from OpenClaw agent output
|
|
67
76
|
// ---------------------------------------------------------------------------
|
|
@@ -166,6 +175,9 @@ async function httpPost(url: string, body: unknown, headers: Record<string, stri
|
|
|
166
175
|
});
|
|
167
176
|
if (!res.ok) {
|
|
168
177
|
const text = await res.text().catch(() => "");
|
|
178
|
+
if (res.status === 401 || res.status === 403) {
|
|
179
|
+
throw new AuthError(res.status, text);
|
|
180
|
+
}
|
|
169
181
|
throw new Error(`HTTP ${res.status}: ${text.slice(0, 200)}`);
|
|
170
182
|
}
|
|
171
183
|
return res.json();
|
|
@@ -230,13 +242,12 @@ async function isWebhookAvailable(config: ConnectorConfig): Promise<boolean> {
|
|
|
230
242
|
if (res.status === 401 || res.status === 403) return false;
|
|
231
243
|
if (res.ok || res.status === 405 || res.status === 204) return true;
|
|
232
244
|
|
|
233
|
-
//
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
});
|
|
238
|
-
|
|
239
|
-
return rootRes.ok || rootRes.status === 404;
|
|
245
|
+
// OPTIONS returned non-2xx/405 (e.g. 404): /hooks/agent doesn't exist.
|
|
246
|
+
// Don't assume hooks are available just because the gateway root responds.
|
|
247
|
+
// This prevents a false-positive where the connector keeps dispatching
|
|
248
|
+
// to a webhook URL that doesn't exist, failing every task with 404.
|
|
249
|
+
console.log(`[connector] Webhook probe: OPTIONS /hooks/agent returned ${res.status}, hooks not available`);
|
|
250
|
+
return false;
|
|
240
251
|
} catch {
|
|
241
252
|
return false;
|
|
242
253
|
}
|
|
@@ -480,9 +491,18 @@ async function executeOnOpenClaw(
|
|
|
480
491
|
config: ConnectorConfig,
|
|
481
492
|
task: NonNullable<PullResponse["task"]>,
|
|
482
493
|
webhookAvailable: boolean,
|
|
483
|
-
): Promise<{ result: TaskResult | null; error: string | null }> {
|
|
494
|
+
): Promise<{ result: TaskResult | null; error: string | null; webhookDead?: boolean }> {
|
|
484
495
|
if (webhookAvailable) {
|
|
485
|
-
|
|
496
|
+
const res = await executeViaWebhook(config, task);
|
|
497
|
+
// Detect webhook 404 — hooks endpoint doesn't exist (plugin version mismatch,
|
|
498
|
+
// gateway restart needed). Fall back to execFile for this task AND signal the
|
|
499
|
+
// main loop to flip webhookAvailable so subsequent tasks don't keep failing.
|
|
500
|
+
if (res.error?.startsWith("Webhook HTTP 404")) {
|
|
501
|
+
console.warn(`[connector] Webhook returned 404, falling back to execFile for task ${task.id}`);
|
|
502
|
+
const fallback = await executeViaExecFile(task);
|
|
503
|
+
return { ...fallback, webhookDead: true };
|
|
504
|
+
}
|
|
505
|
+
return res;
|
|
486
506
|
}
|
|
487
507
|
console.log(`[connector] Webhook unavailable, falling back to execFile`);
|
|
488
508
|
return executeViaExecFile(task);
|
|
@@ -504,7 +524,7 @@ async function updateTaskStatus(
|
|
|
504
524
|
}
|
|
505
525
|
|
|
506
526
|
export type ConnectorStatus = {
|
|
507
|
-
state: "stopped" | "starting" | "running" | "stopping" | "error" | "disabled";
|
|
527
|
+
state: "stopped" | "starting" | "running" | "stopping" | "error" | "disabled" | "dormant";
|
|
508
528
|
uptime: number;
|
|
509
529
|
lastHeartbeat: number;
|
|
510
530
|
currentTaskId: string | undefined;
|
|
@@ -515,16 +535,26 @@ export type ConnectorStatus = {
|
|
|
515
535
|
|
|
516
536
|
export async function runConnectorLoop(
|
|
517
537
|
config: ConnectorConfig,
|
|
518
|
-
options?: { signal?: AbortSignal },
|
|
538
|
+
options?: { signal?: AbortSignal; onHeartbeat?: () => void },
|
|
519
539
|
): Promise<void> {
|
|
520
540
|
let pollInterval = config.pollIntervalMs;
|
|
521
541
|
let currentTaskId: string | undefined;
|
|
522
542
|
let consecutiveErrors = 0;
|
|
543
|
+
let consecutiveAuthErrors = 0;
|
|
523
544
|
let totalTasksExecuted = 0;
|
|
524
545
|
let webhookAvailable = false;
|
|
525
546
|
|
|
526
547
|
const signal = options?.signal;
|
|
527
548
|
|
|
549
|
+
// Internal AbortController: links heartbeat + poll loops so either can abort both.
|
|
550
|
+
// When heartbeat detects auth failure → aborts internal → poll loop exits too.
|
|
551
|
+
// External signal (from ConnectorManager) is forwarded to internal.
|
|
552
|
+
const internalAbort = new AbortController();
|
|
553
|
+
const internalSignal = internalAbort.signal;
|
|
554
|
+
if (signal) {
|
|
555
|
+
signal.addEventListener("abort", () => internalAbort.abort(signal.reason), { once: true });
|
|
556
|
+
}
|
|
557
|
+
|
|
528
558
|
console.log(`[connector] Starting connector loop`);
|
|
529
559
|
console.log(`[connector] API: ${config.apiUrl}`);
|
|
530
560
|
console.log(`[connector] Auth: ${config.apiKey ? "API key (Bearer)" : "connector token (legacy)"}`);
|
|
@@ -541,32 +571,47 @@ export async function runConnectorLoop(
|
|
|
541
571
|
|
|
542
572
|
// Heartbeat runs independently - does NOT touch pollInterval or consecutiveErrors.
|
|
543
573
|
// Otherwise it would neutralize exponential backoff during error recovery.
|
|
574
|
+
// Uses internalAbort so auth failures kill both heartbeat AND poll loops.
|
|
544
575
|
let consecutiveHeartbeatFailures = 0;
|
|
576
|
+
let consecutiveHeartbeatAuthErrors = 0;
|
|
545
577
|
const heartbeatLoop = async () => {
|
|
546
|
-
|
|
578
|
+
let heartbeatIntervalMs = 30_000;
|
|
579
|
+
while (!internalSignal.aborted) {
|
|
547
580
|
try {
|
|
548
581
|
await heartbeat(config, currentTaskId);
|
|
549
582
|
consecutiveHeartbeatFailures = 0;
|
|
583
|
+
consecutiveHeartbeatAuthErrors = 0;
|
|
584
|
+
heartbeatIntervalMs = 30_000;
|
|
585
|
+
options?.onHeartbeat?.();
|
|
550
586
|
} catch (err) {
|
|
551
587
|
consecutiveHeartbeatFailures++;
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
console.
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
588
|
+
if (err instanceof AuthError) {
|
|
589
|
+
consecutiveHeartbeatAuthErrors++;
|
|
590
|
+
console.error(`[connector] Heartbeat auth failed (${consecutiveHeartbeatAuthErrors}/3): ${err.message}`);
|
|
591
|
+
if (consecutiveHeartbeatAuthErrors >= 3) {
|
|
592
|
+
console.error("[connector] FATAL: 3 consecutive auth failures in heartbeat, API key is invalid");
|
|
593
|
+
internalAbort.abort(err);
|
|
594
|
+
return;
|
|
595
|
+
}
|
|
596
|
+
} else {
|
|
597
|
+
consecutiveHeartbeatAuthErrors = 0;
|
|
598
|
+
console.error(`[connector] Heartbeat failed (${consecutiveHeartbeatFailures}):`, err instanceof Error ? err.message : String(err));
|
|
599
|
+
if (consecutiveHeartbeatFailures === 5) {
|
|
600
|
+
console.warn("[connector] WARNING: 5 consecutive heartbeat failures, API may be unreachable");
|
|
601
|
+
}
|
|
602
|
+
// Exponential backoff: 30s → 60s → 120s → max 5min (never die for transient errors)
|
|
603
|
+
heartbeatIntervalMs = Math.min(heartbeatIntervalMs * 2, 5 * 60 * 1000);
|
|
560
604
|
}
|
|
561
605
|
}
|
|
562
606
|
await new Promise((r) => {
|
|
563
|
-
const timer = setTimeout(r,
|
|
564
|
-
|
|
607
|
+
const timer = setTimeout(r, heartbeatIntervalMs);
|
|
608
|
+
internalSignal.addEventListener("abort", () => { clearTimeout(timer); r(undefined); }, { once: true });
|
|
565
609
|
});
|
|
566
610
|
}
|
|
567
611
|
};
|
|
568
612
|
heartbeatLoop().catch((err) => {
|
|
569
613
|
console.error("[connector] Heartbeat loop crashed:", err instanceof Error ? err.message : String(err));
|
|
614
|
+
if (!internalSignal.aborted) internalAbort.abort(err);
|
|
570
615
|
});
|
|
571
616
|
|
|
572
617
|
// Task execution lock - prevents concurrent task execution.
|
|
@@ -574,11 +619,11 @@ export async function runConnectorLoop(
|
|
|
574
619
|
let isExecuting = false;
|
|
575
620
|
let idlePollCount = 0;
|
|
576
621
|
|
|
577
|
-
while (!
|
|
622
|
+
while (!internalSignal.aborted) {
|
|
578
623
|
try {
|
|
579
624
|
// Skip polling while a task is running - prevents claiming concurrent tasks
|
|
580
625
|
if (isExecuting) {
|
|
581
|
-
await sleep(pollInterval,
|
|
626
|
+
await sleep(pollInterval, internalSignal);
|
|
582
627
|
continue;
|
|
583
628
|
}
|
|
584
629
|
|
|
@@ -612,6 +657,7 @@ export async function runConnectorLoop(
|
|
|
612
657
|
);
|
|
613
658
|
|
|
614
659
|
try {
|
|
660
|
+
(task as any)._startedAt = Date.now();
|
|
615
661
|
await updateTaskStatus(config, task.id, "accepted");
|
|
616
662
|
await updateTaskStatus(config, task.id, "running");
|
|
617
663
|
|
|
@@ -625,9 +671,9 @@ export async function runConnectorLoop(
|
|
|
625
671
|
}, watchdogMs);
|
|
626
672
|
watchdogTimer.unref(); // Don't prevent graceful shutdown
|
|
627
673
|
|
|
628
|
-
const
|
|
674
|
+
const execResult = await Promise.race([
|
|
629
675
|
executeOnOpenClaw(config, task, webhookAvailable),
|
|
630
|
-
new Promise<{ result: null; error: string }>((resolve) => {
|
|
676
|
+
new Promise<{ result: null; error: string; webhookDead?: boolean }>((resolve) => {
|
|
631
677
|
const check = setTimeout(() => {
|
|
632
678
|
if (watchdogFired) {
|
|
633
679
|
resolve({ result: null, error: `Watchdog: task execution exceeded ${Math.round(watchdogMs / 1000)}s timeout` });
|
|
@@ -636,21 +682,36 @@ export async function runConnectorLoop(
|
|
|
636
682
|
check.unref();
|
|
637
683
|
}),
|
|
638
684
|
]);
|
|
685
|
+
const { result, error } = execResult;
|
|
686
|
+
// If webhook returned 404, disable it for future tasks
|
|
687
|
+
if (execResult.webhookDead) {
|
|
688
|
+
console.warn(`[connector] Disabling webhook mode — /hooks/agent not available`);
|
|
689
|
+
webhookAvailable = false;
|
|
690
|
+
}
|
|
639
691
|
clearTimeout(watchdogTimer);
|
|
640
692
|
|
|
641
693
|
const taskStatus = error ? "failed" : (result?.success !== false ? "succeeded" : "failed");
|
|
694
|
+
const execDuration = Date.now() - (task as any)._startedAt;
|
|
642
695
|
console.log(
|
|
643
|
-
`[connector] Task ${task.id} ${taskStatus}${error ? `: ${error.slice(0, 100)}` : ""}${task.workflowRunId ? ` workflow=${task.workflowRunId}` : ""}`,
|
|
696
|
+
`[connector] Task ${task.id} ${taskStatus} (${Math.round(execDuration / 1000)}s)${error ? `: ${error.slice(0, 100)}` : ""}${result?.reason ? ` reason=${result.reason}` : ""}${task.workflowRunId ? ` workflow=${task.workflowRunId}` : ""}`,
|
|
644
697
|
);
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
// when lifecycle hook may not have fired (agent crash).
|
|
649
|
-
// Don't submit if result looks successful — lifecycle hook already posted.
|
|
650
|
-
if (!webhookAvailable || error) {
|
|
651
|
-
await submitResult(config, task.id, result, error);
|
|
698
|
+
// Flag suspicious fast failures with no tools for investigation
|
|
699
|
+
if (!error && result?.success === false && (result as any)?.toolCallCount === 0 && execDuration < 10_000) {
|
|
700
|
+
console.warn(`[connector] DIAGNOSTIC: Task ${task.id} failed in <10s with 0 tool calls — likely context injection failure or model error`);
|
|
652
701
|
}
|
|
653
702
|
|
|
703
|
+
// Always submit from connector as safety net. In webhook mode the lifecycle
|
|
704
|
+
// hook usually reports first, but if it fails silently (network, crash) the
|
|
705
|
+
// result is lost and the task stays "running" forever. The result endpoint
|
|
706
|
+
// uses an optimistic lock so double-submissions are harmless (second is a no-op).
|
|
707
|
+
// Derive error when result indicates failure but no explicit error exists.
|
|
708
|
+
const derivedError = error ?? (
|
|
709
|
+
result?.success === false
|
|
710
|
+
? (result.error ?? result.reason ?? "Task failed (no details from agent)")
|
|
711
|
+
: undefined
|
|
712
|
+
);
|
|
713
|
+
await submitResult(config, task.id, result, derivedError ?? null);
|
|
714
|
+
|
|
654
715
|
totalTasksExecuted++;
|
|
655
716
|
pollInterval = 5_000;
|
|
656
717
|
} finally {
|
|
@@ -666,10 +727,22 @@ export async function runConnectorLoop(
|
|
|
666
727
|
isExecuting = false;
|
|
667
728
|
currentTaskId = undefined;
|
|
668
729
|
(globalThis as any).__bereachCurrentTaskId = undefined;
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
730
|
+
|
|
731
|
+
if (err instanceof AuthError) {
|
|
732
|
+
consecutiveAuthErrors++;
|
|
733
|
+
console.error(`[connector] Poll auth failed (${consecutiveAuthErrors}/3): ${err.message}`);
|
|
734
|
+
if (consecutiveAuthErrors >= 3) {
|
|
735
|
+
console.error("[connector] FATAL: 3 consecutive auth failures in poll loop, API key is invalid");
|
|
736
|
+
internalAbort.abort(err);
|
|
737
|
+
break;
|
|
738
|
+
}
|
|
739
|
+
} else {
|
|
740
|
+
consecutiveAuthErrors = 0;
|
|
741
|
+
console.error(
|
|
742
|
+
`[connector] Poll error (${consecutiveErrors}):`,
|
|
743
|
+
err instanceof Error ? err.message : String(err),
|
|
744
|
+
);
|
|
745
|
+
}
|
|
673
746
|
|
|
674
747
|
pollInterval = Math.min(
|
|
675
748
|
config.pollIntervalMs * Math.pow(2, consecutiveErrors),
|
|
@@ -686,10 +759,17 @@ export async function runConnectorLoop(
|
|
|
686
759
|
}
|
|
687
760
|
}
|
|
688
761
|
|
|
689
|
-
await sleep(pollInterval,
|
|
762
|
+
await sleep(pollInterval, internalSignal);
|
|
690
763
|
}
|
|
691
764
|
|
|
692
765
|
console.log("[connector] Loop stopped (abort signal received)");
|
|
766
|
+
|
|
767
|
+
// If internal abort fired (auth failure) but external didn't (not a graceful shutdown),
|
|
768
|
+
// throw so ConnectorManager.handleCrash() triggers with the auth error.
|
|
769
|
+
if (internalSignal.aborted && !signal?.aborted) {
|
|
770
|
+
const reason = internalSignal.reason;
|
|
771
|
+
throw reason instanceof Error ? reason : new Error("Connector loop aborted internally");
|
|
772
|
+
}
|
|
693
773
|
}
|
|
694
774
|
|
|
695
775
|
/** Re-probe webhook availability. Called by ConnectorManager on retry. */
|
package/src/connector/manager.ts
CHANGED
|
@@ -4,39 +4,75 @@
|
|
|
4
4
|
* Auto-starts the polling loop from register() when an API key is present.
|
|
5
5
|
* Handles crash recovery with exponential backoff, graceful shutdown on
|
|
6
6
|
* SIGTERM/SIGINT, and prevents duplicate instances.
|
|
7
|
+
*
|
|
8
|
+
* Key resilience features:
|
|
9
|
+
* - AuthError detection: distinguishes auth failures from transient errors
|
|
10
|
+
* - Dormant mode: after persistent auth failures, enters low-power probe mode
|
|
11
|
+
* instead of burning restarts. Probes every 15 min and auto-recovers.
|
|
12
|
+
* - Config resolver: re-resolves API key on each restart/probe
|
|
13
|
+
* - Health check: detects zombie state (running but heartbeat dead)
|
|
7
14
|
*/
|
|
8
15
|
|
|
9
|
-
import { runConnectorLoop, isWebhookAvailable, type ConnectorConfig, type ConnectorStatus } from "../commands/connector.js";
|
|
16
|
+
import { runConnectorLoop, isWebhookAvailable, AuthError, type ConnectorConfig, type ConnectorStatus } from "../commands/connector.js";
|
|
10
17
|
|
|
11
18
|
const MAX_RESTARTS = 10;
|
|
12
19
|
const INITIAL_RESTART_DELAY_MS = 1_000;
|
|
13
20
|
const MAX_RESTART_DELAY_MS = 30_000;
|
|
21
|
+
const DORMANT_PROBE_INTERVAL_MS = 15 * 60 * 1000; // 15 min
|
|
14
22
|
|
|
15
|
-
export type ConnectorManagerState = "stopped" | "starting" | "running" | "stopping" | "error" | "disabled";
|
|
23
|
+
export type ConnectorManagerState = "stopped" | "starting" | "running" | "stopping" | "error" | "disabled" | "dormant";
|
|
16
24
|
|
|
17
25
|
class ConnectorManager {
|
|
18
26
|
private state: ConnectorManagerState = "stopped";
|
|
19
27
|
private abortController: AbortController | null = null;
|
|
20
28
|
private config: ConnectorConfig | null = null;
|
|
29
|
+
private resolveConfig: (() => ConnectorConfig) | null = null;
|
|
21
30
|
private restartCount = 0;
|
|
22
31
|
private startedAt = 0;
|
|
23
32
|
private lastHeartbeatAt = 0;
|
|
24
33
|
private totalTasksExecuted = 0;
|
|
25
34
|
private consecutiveErrors = 0;
|
|
35
|
+
private consecutiveAuthCrashes = 0;
|
|
26
36
|
private executionMode: "webhook" | "execFile" = "execFile";
|
|
27
37
|
private loopPromise: Promise<void> | null = null;
|
|
28
38
|
private signalHandlersInstalled = false;
|
|
39
|
+
private stabilityTimer: ReturnType<typeof setTimeout> | null = null;
|
|
40
|
+
private dormantTimer: ReturnType<typeof setTimeout> | null = null;
|
|
29
41
|
|
|
30
42
|
/**
|
|
31
|
-
* Start the connector.
|
|
43
|
+
* Start the connector. Accepts either a static config or a resolver function
|
|
44
|
+
* that re-resolves config (including API key) on each restart.
|
|
45
|
+
*
|
|
46
|
+
* Idempotent with smart recovery:
|
|
47
|
+
* - If dormant: wakes up and attempts restart (user may have changed key)
|
|
48
|
+
* - If running but unhealthy: forces restart
|
|
49
|
+
* - If running and healthy: skips
|
|
32
50
|
*/
|
|
33
|
-
async start(
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
51
|
+
async start(configOrResolver: ConnectorConfig | (() => ConnectorConfig)): Promise<void> {
|
|
52
|
+
// Store resolver for future re-resolution
|
|
53
|
+
if (typeof configOrResolver === "function") {
|
|
54
|
+
this.resolveConfig = configOrResolver;
|
|
55
|
+
this.config = configOrResolver();
|
|
56
|
+
} else {
|
|
57
|
+
this.resolveConfig = null;
|
|
58
|
+
this.config = configOrResolver;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
if (this.state === "dormant") {
|
|
62
|
+
// User may have changed their key - wake up and try
|
|
63
|
+
console.log("[bereach:connector] waking from dormant mode (register() called)");
|
|
64
|
+
if (this.dormantTimer) { clearTimeout(this.dormantTimer); this.dormantTimer = null; }
|
|
65
|
+
this.consecutiveAuthCrashes = 0;
|
|
66
|
+
// Fall through to normal start
|
|
67
|
+
} else if (this.state === "running" || this.state === "starting") {
|
|
68
|
+
if (this.isHealthy()) {
|
|
69
|
+
console.log("[bereach:connector] skip duplicate start (already running and healthy)");
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
console.warn("[bereach:connector] connector unhealthy, forcing restart");
|
|
73
|
+
await this.stop();
|
|
37
74
|
}
|
|
38
75
|
|
|
39
|
-
this.config = config;
|
|
40
76
|
this.state = "starting";
|
|
41
77
|
this.restartCount = 0;
|
|
42
78
|
|
|
@@ -73,11 +109,11 @@ class ConnectorManager {
|
|
|
73
109
|
process.on("SIGINT", () => shutdown("SIGINT"));
|
|
74
110
|
}
|
|
75
111
|
|
|
76
|
-
// Probe webhook availability with retry
|
|
112
|
+
// Probe webhook availability with retry - the gateway HTTP server may not be
|
|
77
113
|
// listening yet when register() fires during plugin load.
|
|
78
114
|
let webhookOk = false;
|
|
79
115
|
for (let attempt = 1; attempt <= 5; attempt++) {
|
|
80
|
-
webhookOk = await isWebhookAvailable(config);
|
|
116
|
+
webhookOk = await isWebhookAvailable(this.config);
|
|
81
117
|
if (webhookOk) break;
|
|
82
118
|
if (attempt < 5) {
|
|
83
119
|
console.log(`[bereach:connector] webhook probe attempt ${attempt}/5 failed, retrying in 3s...`);
|
|
@@ -95,6 +131,9 @@ class ConnectorManager {
|
|
|
95
131
|
async stop(): Promise<void> {
|
|
96
132
|
if (this.state === "stopped" || this.state === "disabled") return;
|
|
97
133
|
|
|
134
|
+
// Clean up dormant timer if active
|
|
135
|
+
if (this.dormantTimer) { clearTimeout(this.dormantTimer); this.dormantTimer = null; }
|
|
136
|
+
|
|
98
137
|
this.state = "stopping";
|
|
99
138
|
console.log("[bereach:connector] stopping...");
|
|
100
139
|
|
|
@@ -128,10 +167,43 @@ class ConnectorManager {
|
|
|
128
167
|
};
|
|
129
168
|
}
|
|
130
169
|
|
|
131
|
-
|
|
170
|
+
/**
|
|
171
|
+
* Check if the connector is actually healthy (not a zombie).
|
|
172
|
+
* Unhealthy = running state but no heartbeat in 5+ minutes,
|
|
173
|
+
* or running for 2+ minutes without ever getting a heartbeat.
|
|
174
|
+
*/
|
|
175
|
+
private isHealthy(): boolean {
|
|
176
|
+
if (this.state !== "running") return false;
|
|
177
|
+
const now = Date.now();
|
|
178
|
+
const uptime = now - this.startedAt;
|
|
179
|
+
// Just started (< 2 min), give it time
|
|
180
|
+
if (uptime < 2 * 60 * 1000) return true;
|
|
181
|
+
// No heartbeat ever received after 2 min = zombie
|
|
182
|
+
if (!this.lastHeartbeatAt) return false;
|
|
183
|
+
// Last heartbeat > 5 min ago = zombie
|
|
184
|
+
return (now - this.lastHeartbeatAt) < 5 * 60 * 1000;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* Re-resolve config from the resolver (picks up API key changes).
|
|
189
|
+
* Falls back to stored config if no resolver.
|
|
190
|
+
*/
|
|
191
|
+
private freshConfig(): ConnectorConfig | null {
|
|
192
|
+
if (this.resolveConfig) {
|
|
193
|
+
try {
|
|
194
|
+
this.config = this.resolveConfig();
|
|
195
|
+
const masked = this.config.apiKey ? `...${this.config.apiKey.slice(-6)}` : "NOT SET";
|
|
196
|
+
console.log(`[bereach:connector] re-resolved config (API key: ${masked})`);
|
|
197
|
+
} catch (err) {
|
|
198
|
+
console.error("[bereach:connector] config resolver failed:", err instanceof Error ? err.message : String(err));
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
return this.config;
|
|
202
|
+
}
|
|
132
203
|
|
|
133
204
|
private startLoop(): void {
|
|
134
|
-
|
|
205
|
+
const config = this.freshConfig();
|
|
206
|
+
if (!config) return;
|
|
135
207
|
|
|
136
208
|
this.abortController = new AbortController();
|
|
137
209
|
this.startedAt = Date.now();
|
|
@@ -151,7 +223,10 @@ class ConnectorManager {
|
|
|
151
223
|
}, 5 * 60 * 1000);
|
|
152
224
|
this.stabilityTimer.unref();
|
|
153
225
|
|
|
154
|
-
this.loopPromise = runConnectorLoop(
|
|
226
|
+
this.loopPromise = runConnectorLoop(config, {
|
|
227
|
+
signal: this.abortController.signal,
|
|
228
|
+
onHeartbeat: () => { this.lastHeartbeatAt = Date.now(); },
|
|
229
|
+
})
|
|
155
230
|
.then(() => {
|
|
156
231
|
// Normal exit (abort signal)
|
|
157
232
|
if (this.state !== "stopping") {
|
|
@@ -160,14 +235,29 @@ class ConnectorManager {
|
|
|
160
235
|
})
|
|
161
236
|
.catch((err) => {
|
|
162
237
|
console.error(`[bereach:connector] loop crashed: ${err instanceof Error ? err.message : String(err)}`);
|
|
163
|
-
this.handleCrash();
|
|
238
|
+
this.handleCrash(err);
|
|
164
239
|
});
|
|
165
240
|
}
|
|
166
241
|
|
|
167
|
-
private handleCrash(): void {
|
|
242
|
+
private handleCrash(err?: unknown): void {
|
|
168
243
|
this.restartCount++;
|
|
169
244
|
this.consecutiveErrors++;
|
|
170
245
|
|
|
246
|
+
// Track consecutive auth-related crashes for dormant mode.
|
|
247
|
+
// 2 auth crashes = 2 full start→3-failures→crash cycles = ~6 total 401s.
|
|
248
|
+
const isAuthCrash = err instanceof AuthError ||
|
|
249
|
+
(err instanceof Error && err.message.includes("Auth failed"));
|
|
250
|
+
if (isAuthCrash) {
|
|
251
|
+
this.consecutiveAuthCrashes++;
|
|
252
|
+
console.error(`[bereach:connector] auth crash ${this.consecutiveAuthCrashes}/2`);
|
|
253
|
+
if (this.consecutiveAuthCrashes >= 2) {
|
|
254
|
+
this.enterDormant();
|
|
255
|
+
return;
|
|
256
|
+
}
|
|
257
|
+
} else {
|
|
258
|
+
this.consecutiveAuthCrashes = 0;
|
|
259
|
+
}
|
|
260
|
+
|
|
171
261
|
if (this.restartCount > MAX_RESTARTS) {
|
|
172
262
|
// Don't give up permanently - enter cooldown then restart.
|
|
173
263
|
// The user's VPS has no supervisor to restart us, so we must self-recover.
|
|
@@ -199,6 +289,69 @@ class ConnectorManager {
|
|
|
199
289
|
this.startLoop();
|
|
200
290
|
}, delay);
|
|
201
291
|
}
|
|
292
|
+
|
|
293
|
+
/**
|
|
294
|
+
* Enter dormant mode - lightweight 15-min probe instead of full restart loop.
|
|
295
|
+
* Used when API key is confirmed invalid (persistent auth failures).
|
|
296
|
+
* Auto-recovers when the key becomes valid again.
|
|
297
|
+
*/
|
|
298
|
+
private enterDormant(): void {
|
|
299
|
+
this.state = "dormant";
|
|
300
|
+
this.abortController = null;
|
|
301
|
+
this.loopPromise = null;
|
|
302
|
+
console.error(
|
|
303
|
+
"[bereach:connector] API key invalid - entering dormant mode. " +
|
|
304
|
+
"Will probe every 15 min and auto-recover when key is fixed.",
|
|
305
|
+
);
|
|
306
|
+
this.scheduleDormantProbe();
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
private scheduleDormantProbe(): void {
|
|
310
|
+
if (this.dormantTimer) clearTimeout(this.dormantTimer);
|
|
311
|
+
this.dormantTimer = setTimeout(() => this.runDormantProbe(), DORMANT_PROBE_INTERVAL_MS);
|
|
312
|
+
this.dormantTimer.unref();
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
private async runDormantProbe(): Promise<void> {
|
|
316
|
+
if (this.state !== "dormant") return;
|
|
317
|
+
|
|
318
|
+
// Re-resolve config to pick up any key changes
|
|
319
|
+
const config = this.freshConfig();
|
|
320
|
+
if (!config?.apiKey) {
|
|
321
|
+
console.log("[bereach:connector] dormant probe: no API key configured, will retry in 15 min");
|
|
322
|
+
this.scheduleDormantProbe();
|
|
323
|
+
return;
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
// Single lightweight heartbeat probe
|
|
327
|
+
try {
|
|
328
|
+
const res = await fetch(`${config.apiUrl}/connectors/heartbeat`, {
|
|
329
|
+
method: "POST",
|
|
330
|
+
headers: {
|
|
331
|
+
"Content-Type": "application/json",
|
|
332
|
+
"Authorization": `Bearer ${config.apiKey}`,
|
|
333
|
+
},
|
|
334
|
+
body: JSON.stringify({}),
|
|
335
|
+
signal: AbortSignal.timeout(10_000),
|
|
336
|
+
});
|
|
337
|
+
if (res.ok) {
|
|
338
|
+
console.log("[bereach:connector] dormant probe succeeded! Restarting connector.");
|
|
339
|
+
this.consecutiveAuthCrashes = 0;
|
|
340
|
+
this.restartCount = 0;
|
|
341
|
+
this.consecutiveErrors = 0;
|
|
342
|
+
this.startLoop();
|
|
343
|
+
return;
|
|
344
|
+
}
|
|
345
|
+
if (res.status === 401 || res.status === 403) {
|
|
346
|
+
console.log("[bereach:connector] dormant probe: still 401, will retry in 15 min");
|
|
347
|
+
} else {
|
|
348
|
+
console.log(`[bereach:connector] dormant probe: HTTP ${res.status}, will retry in 15 min`);
|
|
349
|
+
}
|
|
350
|
+
} catch (err) {
|
|
351
|
+
console.log(`[bereach:connector] dormant probe failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
352
|
+
}
|
|
353
|
+
this.scheduleDormantProbe();
|
|
354
|
+
}
|
|
202
355
|
}
|
|
203
356
|
|
|
204
357
|
// Module-level singleton
|
package/src/hooks/lifecycle.ts
CHANGED
|
@@ -125,9 +125,25 @@ export function registerLifecycleHook(
|
|
|
125
125
|
creditsUsed: state.creditsUsedThisSession,
|
|
126
126
|
toolCallCount: state.toolCallCount,
|
|
127
127
|
visitCount: state.visitCount,
|
|
128
|
-
...(state.toolCallCount === 0 ? { reason: "no_tools_called" } : {}),
|
|
128
|
+
...(state.toolCallCount === 0 ? { reason: "no_tools_called", error: "Task completed with no tool calls" } : {}),
|
|
129
129
|
};
|
|
130
130
|
|
|
131
|
+
// Diagnostic logging for no_tools_called — capture WHY the agent did nothing.
|
|
132
|
+
// This helps diagnose context injection failures, model errors, etc.
|
|
133
|
+
if (state.toolCallCount === 0) {
|
|
134
|
+
const msgCount = allMessages.length;
|
|
135
|
+
const userMsgCount = userMsgs.length;
|
|
136
|
+
const assistantMsgCount = assistantMsgs.length;
|
|
137
|
+
const lastAssistantText = outputText?.slice(0, 300) || "(empty)";
|
|
138
|
+
const hasTaskMeta = userMsgs.some((m: any) => extractTextFromContent(m.content).includes("TASK_META"));
|
|
139
|
+
const endSuccess = (endCtx as any)?.success;
|
|
140
|
+
const endError = (endCtx as any)?.error;
|
|
141
|
+
log(`NO_TOOLS_CALLED diagnostic for ${taskMode.taskId}: ` +
|
|
142
|
+
`msgs=${msgCount} (user=${userMsgCount} assistant=${assistantMsgCount}) ` +
|
|
143
|
+
`hasTaskMeta=${hasTaskMeta} endSuccess=${endSuccess} endError=${endError ? String(endError).slice(0, 200) : "none"} ` +
|
|
144
|
+
`lastAssistant="${lastAssistantText}"`);
|
|
145
|
+
}
|
|
146
|
+
|
|
131
147
|
// B17 fix: In execFile mode, the connector also submits results after
|
|
132
148
|
// parsing the OpenClaw --json wrapper (extractTaskResultFromWrapper).
|
|
133
149
|
// The lifecycle hook fires first (inside subprocess), so it wins the race.
|
|
@@ -148,10 +164,14 @@ export function registerLifecycleHook(
|
|
|
148
164
|
attempts++;
|
|
149
165
|
try {
|
|
150
166
|
if (attempt > 0) await new Promise((r) => setTimeout(r, 2000 * attempt));
|
|
167
|
+
// Derive error from result when task failed but no explicit error exists
|
|
168
|
+
const taskError = taskResult.success === false
|
|
169
|
+
? (taskResult.error ?? taskResult.reason ?? "Task failed (no details captured)")
|
|
170
|
+
: undefined;
|
|
151
171
|
const res = await fetch(`${API_BASE}/tasks/${taskMode.taskId}/result`, {
|
|
152
172
|
method: "POST",
|
|
153
173
|
headers: { Authorization: `Bearer ${key}`, "Content-Type": "application/json" },
|
|
154
|
-
body: JSON.stringify({ result: taskResult }),
|
|
174
|
+
body: JSON.stringify({ result: taskResult, ...(taskError ? { error: taskError } : {}) }),
|
|
155
175
|
signal: AbortSignal.timeout(10000),
|
|
156
176
|
});
|
|
157
177
|
if (res.ok) {
|
package/src/index.ts
CHANGED
|
@@ -18,6 +18,9 @@ const log = createLogger("init");
|
|
|
18
18
|
const registeredApis = new WeakSet<object>();
|
|
19
19
|
let registeredCount = 0;
|
|
20
20
|
let autoDetectDone = false;
|
|
21
|
+
let lastRegisterAt = 0;
|
|
22
|
+
/** After initial burst, suppress noisy re-registration logs */
|
|
23
|
+
const REGISTER_LOG_THRESHOLD = 5;
|
|
21
24
|
|
|
22
25
|
/**
|
|
23
26
|
* Resolve the BeReach API key from all supported sources (in priority order):
|
|
@@ -120,8 +123,11 @@ export default function register(api: any) {
|
|
|
120
123
|
}
|
|
121
124
|
|
|
122
125
|
registeredCount++;
|
|
123
|
-
|
|
126
|
+
lastRegisterAt = Date.now();
|
|
127
|
+
if (registeredCount > 1 && registeredCount <= REGISTER_LOG_THRESHOLD) {
|
|
124
128
|
log(`register() called again (call #${registeredCount}) — re-registering on new api instance`);
|
|
129
|
+
} else if (registeredCount === REGISTER_LOG_THRESHOLD + 1) {
|
|
130
|
+
log(`register() call #${registeredCount} — suppressing further re-registration logs`);
|
|
125
131
|
}
|
|
126
132
|
if (api && typeof api === "object") {
|
|
127
133
|
registeredApis.add(api);
|
|
@@ -226,14 +232,14 @@ export default function register(api: any) {
|
|
|
226
232
|
const gatewayUrl = config.gatewayUrl || readEnv("OPENCLAW_GATEWAY_URL") || "http://localhost:18789";
|
|
227
233
|
const pollMs = config.connectorPollIntervalMs ?? 15000;
|
|
228
234
|
|
|
229
|
-
connectorManager.start({
|
|
230
|
-
apiKey,
|
|
235
|
+
connectorManager.start(() => ({
|
|
236
|
+
apiKey: resolveApiKey(api),
|
|
231
237
|
apiUrl: API_BASE,
|
|
232
238
|
openclawUrl: readEnv("OPENCLAW_URL") || "http://localhost:3579",
|
|
233
239
|
pollIntervalMs: Math.max(5000, pollMs),
|
|
234
240
|
gatewayUrl,
|
|
235
241
|
hooksToken,
|
|
236
|
-
}).catch((err) => {
|
|
242
|
+
})).catch((err) => {
|
|
237
243
|
log(`connector auto-start failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
238
244
|
});
|
|
239
245
|
} else if (!connectorEnabled) {
|