@calltelemetry/openclaw-linear 0.6.1 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +115 -17
- package/index.ts +57 -22
- package/openclaw.plugin.json +37 -4
- package/package.json +2 -1
- package/prompts.yaml +47 -0
- package/src/api/linear-api.test.ts +494 -0
- package/src/api/linear-api.ts +193 -19
- package/src/gateway/dispatch-methods.ts +243 -0
- package/src/infra/cli.ts +284 -29
- package/src/infra/codex-worktree.ts +83 -0
- package/src/infra/commands.ts +156 -0
- package/src/infra/doctor.test.ts +4 -4
- package/src/infra/doctor.ts +7 -29
- package/src/infra/file-lock.test.ts +61 -0
- package/src/infra/file-lock.ts +49 -0
- package/src/infra/multi-repo.ts +85 -0
- package/src/infra/notify.test.ts +357 -108
- package/src/infra/notify.ts +222 -43
- package/src/infra/observability.ts +48 -0
- package/src/infra/resilience.test.ts +94 -0
- package/src/infra/resilience.ts +101 -0
- package/src/pipeline/artifacts.ts +38 -2
- package/src/pipeline/dag-dispatch.test.ts +553 -0
- package/src/pipeline/dag-dispatch.ts +390 -0
- package/src/pipeline/dispatch-service.ts +48 -1
- package/src/pipeline/dispatch-state.ts +2 -42
- package/src/pipeline/pipeline.ts +91 -17
- package/src/pipeline/planner.test.ts +334 -0
- package/src/pipeline/planner.ts +287 -0
- package/src/pipeline/planning-state.test.ts +236 -0
- package/src/pipeline/planning-state.ts +178 -0
- package/src/pipeline/tier-assess.test.ts +175 -0
- package/src/pipeline/webhook.ts +90 -17
- package/src/tools/dispatch-history-tool.ts +201 -0
- package/src/tools/orchestration-tools.test.ts +158 -0
- package/src/tools/planner-tools.test.ts +535 -0
- package/src/tools/planner-tools.ts +450 -0
package/src/infra/notify.ts
CHANGED
|
@@ -1,24 +1,30 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* notify.ts —
|
|
2
|
+
* notify.ts — Unified notification provider for dispatch lifecycle events.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
4
|
+
* Uses OpenClaw's native runtime channel API for all providers (Discord, Slack,
|
|
5
|
+
* Telegram, Signal, etc). One formatter, one send function, config-driven
|
|
6
|
+
* fan-out with per-event-type toggles.
|
|
7
|
+
*
|
|
8
|
+
* Modeled on DevClaw's notify.ts pattern — the runtime handles token resolution,
|
|
9
|
+
* formatting differences (markdown vs mrkdwn), and delivery per channel.
|
|
7
10
|
*/
|
|
11
|
+
import type { PluginRuntime } from "openclaw/plugin-sdk";
|
|
8
12
|
|
|
9
13
|
// ---------------------------------------------------------------------------
|
|
10
14
|
// Types
|
|
11
15
|
// ---------------------------------------------------------------------------
|
|
12
16
|
|
|
13
17
|
export type NotifyKind =
|
|
14
|
-
| "dispatch"
|
|
15
|
-
| "working"
|
|
16
|
-
| "auditing"
|
|
17
|
-
| "audit_pass"
|
|
18
|
-
| "audit_fail"
|
|
19
|
-
| "escalation"
|
|
20
|
-
| "stuck"
|
|
21
|
-
| "watchdog_kill"
|
|
18
|
+
| "dispatch" // issue dispatched to worker
|
|
19
|
+
| "working" // worker started
|
|
20
|
+
| "auditing" // audit triggered
|
|
21
|
+
| "audit_pass" // audit passed → done
|
|
22
|
+
| "audit_fail" // audit failed → rework
|
|
23
|
+
| "escalation" // 2x fail or stale → stuck
|
|
24
|
+
| "stuck" // stale detection
|
|
25
|
+
| "watchdog_kill" // agent killed by inactivity watchdog
|
|
26
|
+
| "project_progress" // DAG dispatch progress update
|
|
27
|
+
| "project_complete"; // all project issues dispatched
|
|
22
28
|
|
|
23
29
|
export interface NotifyPayload {
|
|
24
30
|
identifier: string;
|
|
@@ -32,58 +38,231 @@ export interface NotifyPayload {
|
|
|
32
38
|
export type NotifyFn = (kind: NotifyKind, payload: NotifyPayload) => Promise<void>;
|
|
33
39
|
|
|
34
40
|
// ---------------------------------------------------------------------------
|
|
35
|
-
//
|
|
41
|
+
// Provider config
|
|
42
|
+
// ---------------------------------------------------------------------------
|
|
43
|
+
|
|
44
|
+
export interface NotifyTarget {
|
|
45
|
+
/** OpenClaw channel name: "discord", "slack", "telegram", "signal", etc. */
|
|
46
|
+
channel: string;
|
|
47
|
+
/** Channel/group/user ID to send to */
|
|
48
|
+
target: string;
|
|
49
|
+
/** Optional account ID for multi-account channel setups */
|
|
50
|
+
accountId?: string;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export interface NotificationsConfig {
|
|
54
|
+
targets?: NotifyTarget[];
|
|
55
|
+
events?: Partial<Record<NotifyKind, boolean>>;
|
|
56
|
+
/** Opt-in: send rich embeds (Discord) and HTML (Telegram) instead of plain text. */
|
|
57
|
+
richFormat?: boolean;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// ---------------------------------------------------------------------------
|
|
61
|
+
// Rich message types (Discord embeds + Telegram HTML)
|
|
36
62
|
// ---------------------------------------------------------------------------
|
|
37
63
|
|
|
38
|
-
|
|
64
|
+
export interface DiscordEmbed {
|
|
65
|
+
title?: string;
|
|
66
|
+
description?: string;
|
|
67
|
+
color?: number;
|
|
68
|
+
fields?: Array<{ name: string; value: string; inline?: boolean }>;
|
|
69
|
+
footer?: { text: string };
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export interface RichMessage {
|
|
73
|
+
text: string;
|
|
74
|
+
discord?: { embeds: DiscordEmbed[] };
|
|
75
|
+
telegram?: { html: string };
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// ---------------------------------------------------------------------------
|
|
79
|
+
// Unified message formatter
|
|
80
|
+
// ---------------------------------------------------------------------------
|
|
39
81
|
|
|
40
|
-
function
|
|
41
|
-
const
|
|
82
|
+
export function formatMessage(kind: NotifyKind, payload: NotifyPayload): string {
|
|
83
|
+
const id = payload.identifier;
|
|
42
84
|
switch (kind) {
|
|
43
85
|
case "dispatch":
|
|
44
|
-
return `${
|
|
86
|
+
return `${id} dispatched — ${payload.title}`;
|
|
45
87
|
case "working":
|
|
46
|
-
return `${
|
|
88
|
+
return `${id} worker started (attempt ${payload.attempt ?? 0})`;
|
|
47
89
|
case "auditing":
|
|
48
|
-
return `${
|
|
90
|
+
return `${id} audit in progress`;
|
|
49
91
|
case "audit_pass":
|
|
50
|
-
return `${
|
|
92
|
+
return `${id} passed audit. PR ready.`;
|
|
51
93
|
case "audit_fail": {
|
|
52
94
|
const gaps = payload.verdict?.gaps?.join(", ") ?? "unspecified";
|
|
53
|
-
return `${
|
|
95
|
+
return `${id} failed audit (attempt ${payload.attempt ?? 0}). Gaps: ${gaps}`;
|
|
54
96
|
}
|
|
55
97
|
case "escalation":
|
|
56
|
-
return `🚨 ${
|
|
98
|
+
return `🚨 ${id} needs human review — ${payload.reason ?? "audit failed 2x"}`;
|
|
57
99
|
case "stuck":
|
|
58
|
-
return `⏰ ${
|
|
100
|
+
return `⏰ ${id} stuck — ${payload.reason ?? "stale 2h"}`;
|
|
59
101
|
case "watchdog_kill":
|
|
60
|
-
return `⚡ ${
|
|
102
|
+
return `⚡ ${id} killed by watchdog (${payload.reason ?? "no I/O for 120s"}). ${
|
|
61
103
|
payload.attempt != null ? `Retrying (attempt ${payload.attempt}).` : "Will retry."
|
|
62
104
|
}`;
|
|
105
|
+
case "project_progress":
|
|
106
|
+
return `📊 ${payload.title} (${id}): ${payload.status}`;
|
|
107
|
+
case "project_complete":
|
|
108
|
+
return `✅ ${payload.title} (${id}): ${payload.status}`;
|
|
63
109
|
default:
|
|
64
|
-
return `${
|
|
110
|
+
return `${id} — ${kind}: ${payload.status}`;
|
|
65
111
|
}
|
|
66
112
|
}
|
|
67
113
|
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
114
|
+
// ---------------------------------------------------------------------------
|
|
115
|
+
// Rich message formatter (Discord embeds + Telegram HTML)
|
|
116
|
+
// ---------------------------------------------------------------------------
|
|
117
|
+
|
|
118
|
+
const EVENT_COLORS: Record<string, number> = {
|
|
119
|
+
dispatch: 0x3498db, // blue
|
|
120
|
+
working: 0x3498db, // blue
|
|
121
|
+
auditing: 0xf39c12, // yellow
|
|
122
|
+
audit_pass: 0x2ecc71, // green
|
|
123
|
+
audit_fail: 0xe74c3c, // red
|
|
124
|
+
escalation: 0xe74c3c, // red
|
|
125
|
+
stuck: 0xe67e22, // orange
|
|
126
|
+
watchdog_kill: 0x9b59b6, // purple
|
|
127
|
+
project_progress: 0x3498db,
|
|
128
|
+
project_complete: 0x2ecc71,
|
|
129
|
+
};
|
|
130
|
+
|
|
131
|
+
export function formatRichMessage(kind: NotifyKind, payload: NotifyPayload): RichMessage {
|
|
132
|
+
const text = formatMessage(kind, payload);
|
|
133
|
+
const color = EVENT_COLORS[kind] ?? 0x95a5a6;
|
|
134
|
+
|
|
135
|
+
// Discord embed
|
|
136
|
+
const fields: DiscordEmbed["fields"] = [];
|
|
137
|
+
if (payload.attempt != null) fields.push({ name: "Attempt", value: String(payload.attempt), inline: true });
|
|
138
|
+
if (payload.status) fields.push({ name: "Status", value: payload.status, inline: true });
|
|
139
|
+
if (payload.verdict?.gaps?.length) {
|
|
140
|
+
fields.push({ name: "Gaps", value: payload.verdict.gaps.join("\n").slice(0, 1024) });
|
|
141
|
+
}
|
|
142
|
+
if (payload.reason) fields.push({ name: "Reason", value: payload.reason });
|
|
143
|
+
|
|
144
|
+
const embed: DiscordEmbed = {
|
|
145
|
+
title: `${payload.identifier} — ${kind.replace(/_/g, " ")}`,
|
|
146
|
+
description: payload.title,
|
|
147
|
+
color,
|
|
148
|
+
fields: fields.length > 0 ? fields : undefined,
|
|
149
|
+
footer: { text: `Linear Agent • ${kind}` },
|
|
150
|
+
};
|
|
151
|
+
|
|
152
|
+
// Telegram HTML
|
|
153
|
+
const htmlParts: string[] = [
|
|
154
|
+
`<b>${escapeHtml(payload.identifier)}</b> — ${escapeHtml(kind.replace(/_/g, " "))}`,
|
|
155
|
+
`<i>${escapeHtml(payload.title)}</i>`,
|
|
156
|
+
];
|
|
157
|
+
if (payload.attempt != null) htmlParts.push(`Attempt: <code>${payload.attempt}</code>`);
|
|
158
|
+
if (payload.status) htmlParts.push(`Status: <code>${escapeHtml(payload.status)}</code>`);
|
|
159
|
+
if (payload.verdict?.gaps?.length) {
|
|
160
|
+
htmlParts.push(`Gaps:\n${payload.verdict.gaps.map(g => `• ${escapeHtml(g)}`).join("\n")}`);
|
|
161
|
+
}
|
|
162
|
+
if (payload.reason) htmlParts.push(`Reason: ${escapeHtml(payload.reason)}`);
|
|
163
|
+
|
|
164
|
+
return {
|
|
165
|
+
text,
|
|
166
|
+
discord: { embeds: [embed] },
|
|
167
|
+
telegram: { html: htmlParts.join("\n") },
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function escapeHtml(s: string): string {
|
|
172
|
+
return s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
// ---------------------------------------------------------------------------
|
|
176
|
+
// Unified send — routes to OpenClaw runtime channel API
|
|
177
|
+
// ---------------------------------------------------------------------------
|
|
178
|
+
|
|
179
|
+
export async function sendToTarget(
|
|
180
|
+
target: NotifyTarget,
|
|
181
|
+
message: string | RichMessage,
|
|
182
|
+
runtime: PluginRuntime,
|
|
183
|
+
): Promise<void> {
|
|
184
|
+
const ch = target.channel;
|
|
185
|
+
const to = target.target;
|
|
186
|
+
const isRich = typeof message !== "string";
|
|
187
|
+
const plainText = isRich ? message.text : message;
|
|
188
|
+
|
|
189
|
+
if (ch === "discord") {
|
|
190
|
+
if (isRich && message.discord) {
|
|
191
|
+
await runtime.channel.discord.sendMessageDiscord(to, plainText, { embeds: message.discord.embeds });
|
|
192
|
+
} else {
|
|
193
|
+
await runtime.channel.discord.sendMessageDiscord(to, plainText);
|
|
86
194
|
}
|
|
195
|
+
} else if (ch === "slack") {
|
|
196
|
+
await runtime.channel.slack.sendMessageSlack(to, plainText, {
|
|
197
|
+
accountId: target.accountId,
|
|
198
|
+
});
|
|
199
|
+
} else if (ch === "telegram") {
|
|
200
|
+
if (isRich && message.telegram) {
|
|
201
|
+
await runtime.channel.telegram.sendMessageTelegram(to, message.telegram.html, { silent: true, textMode: "html" });
|
|
202
|
+
} else {
|
|
203
|
+
await runtime.channel.telegram.sendMessageTelegram(to, plainText, { silent: true });
|
|
204
|
+
}
|
|
205
|
+
} else if (ch === "signal") {
|
|
206
|
+
await runtime.channel.signal.sendMessageSignal(to, plainText);
|
|
207
|
+
} else {
|
|
208
|
+
// Fallback: use CLI for any channel the runtime doesn't expose directly
|
|
209
|
+
const { execFileSync } = await import("node:child_process");
|
|
210
|
+
execFileSync("openclaw", ["message", "send", "--channel", ch, "--target", to, "--message", plainText, "--json"], {
|
|
211
|
+
timeout: 30_000,
|
|
212
|
+
stdio: "ignore",
|
|
213
|
+
});
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
// ---------------------------------------------------------------------------
|
|
218
|
+
// Config-driven factory
|
|
219
|
+
// ---------------------------------------------------------------------------
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* Parse notification config from plugin config.
|
|
223
|
+
*/
|
|
224
|
+
export function parseNotificationsConfig(
|
|
225
|
+
pluginConfig: Record<string, unknown> | undefined,
|
|
226
|
+
): NotificationsConfig {
|
|
227
|
+
const raw = pluginConfig?.notifications as NotificationsConfig | undefined;
|
|
228
|
+
return {
|
|
229
|
+
targets: raw?.targets ?? [],
|
|
230
|
+
events: raw?.events ?? {},
|
|
231
|
+
richFormat: raw?.richFormat ?? false,
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/**
|
|
236
|
+
* Create a notifier from plugin config. Returns a NotifyFn that:
|
|
237
|
+
* 1. Checks event toggles (skip suppressed events)
|
|
238
|
+
* 2. Formats the message
|
|
239
|
+
* 3. Fans out to all configured targets (failures isolated via Promise.allSettled)
|
|
240
|
+
*/
|
|
241
|
+
export function createNotifierFromConfig(
|
|
242
|
+
pluginConfig: Record<string, unknown> | undefined,
|
|
243
|
+
runtime: PluginRuntime,
|
|
244
|
+
): NotifyFn {
|
|
245
|
+
const config = parseNotificationsConfig(pluginConfig);
|
|
246
|
+
|
|
247
|
+
if (!config.targets?.length) return createNoopNotifier();
|
|
248
|
+
|
|
249
|
+
const useRich = config.richFormat === true;
|
|
250
|
+
|
|
251
|
+
return async (kind, payload) => {
|
|
252
|
+
// Check event toggle — default is enabled (true)
|
|
253
|
+
if (config.events?.[kind] === false) return;
|
|
254
|
+
|
|
255
|
+
const message = useRich ? formatRichMessage(kind, payload) : formatMessage(kind, payload);
|
|
256
|
+
|
|
257
|
+
await Promise.allSettled(
|
|
258
|
+
config.targets!.map(async (target) => {
|
|
259
|
+
try {
|
|
260
|
+
await sendToTarget(target, message, runtime);
|
|
261
|
+
} catch (err) {
|
|
262
|
+
console.error(`Notify error (${target.channel}:${target.target}):`, err);
|
|
263
|
+
}
|
|
264
|
+
}),
|
|
265
|
+
);
|
|
87
266
|
};
|
|
88
267
|
}
|
|
89
268
|
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* observability.ts — Structured diagnostic event logging.
|
|
3
|
+
*
|
|
4
|
+
* Emits structured JSON log lines via api.logger for lifecycle telemetry.
|
|
5
|
+
* Consumers (log aggregators, monitoring) can parse these for dashboards.
|
|
6
|
+
*
|
|
7
|
+
* Pattern: `[linear:diagnostic] {...json...}`
|
|
8
|
+
*/
|
|
9
|
+
import type { OpenClawPluginApi } from "openclaw/plugin-sdk";
|
|
10
|
+
|
|
11
|
+
export type DiagnosticEvent =
|
|
12
|
+
| "webhook_received"
|
|
13
|
+
| "dispatch_started"
|
|
14
|
+
| "phase_transition"
|
|
15
|
+
| "audit_triggered"
|
|
16
|
+
| "verdict_processed"
|
|
17
|
+
| "watchdog_kill"
|
|
18
|
+
| "notify_sent"
|
|
19
|
+
| "notify_failed"
|
|
20
|
+
| "health_check";
|
|
21
|
+
|
|
22
|
+
export interface DiagnosticPayload {
|
|
23
|
+
event: DiagnosticEvent;
|
|
24
|
+
identifier?: string;
|
|
25
|
+
issueId?: string;
|
|
26
|
+
phase?: string;
|
|
27
|
+
from?: string;
|
|
28
|
+
to?: string;
|
|
29
|
+
attempt?: number;
|
|
30
|
+
tier?: string;
|
|
31
|
+
webhookType?: string;
|
|
32
|
+
webhookAction?: string;
|
|
33
|
+
channel?: string;
|
|
34
|
+
target?: string;
|
|
35
|
+
error?: string;
|
|
36
|
+
durationMs?: number;
|
|
37
|
+
[key: string]: unknown;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const PREFIX = "[linear:diagnostic]";
|
|
41
|
+
|
|
42
|
+
export function emitDiagnostic(api: OpenClawPluginApi, payload: DiagnosticPayload): void {
|
|
43
|
+
try {
|
|
44
|
+
api.logger.info(`${PREFIX} ${JSON.stringify(payload)}`);
|
|
45
|
+
} catch {
|
|
46
|
+
// Never throw from telemetry
|
|
47
|
+
}
|
|
48
|
+
}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach } from "vitest";
|
|
2
|
+
import {
|
|
3
|
+
createRetryPolicy,
|
|
4
|
+
createCircuitBreaker,
|
|
5
|
+
withResilience,
|
|
6
|
+
resetDefaultPolicy,
|
|
7
|
+
} from "./resilience.js";
|
|
8
|
+
import { BrokenCircuitError } from "cockatiel";
|
|
9
|
+
|
|
10
|
+
beforeEach(() => {
|
|
11
|
+
resetDefaultPolicy();
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
describe("createRetryPolicy", () => {
|
|
15
|
+
it("retries on transient failure then succeeds", async () => {
|
|
16
|
+
let calls = 0;
|
|
17
|
+
const policy = createRetryPolicy({ attempts: 3, initialDelay: 10, maxDelay: 20 });
|
|
18
|
+
const result = await policy.execute(async () => {
|
|
19
|
+
calls++;
|
|
20
|
+
if (calls < 3) throw new Error("transient");
|
|
21
|
+
return "ok";
|
|
22
|
+
});
|
|
23
|
+
expect(result).toBe("ok");
|
|
24
|
+
expect(calls).toBe(3);
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
it("throws after exhausting retries", async () => {
|
|
28
|
+
const policy = createRetryPolicy({ attempts: 2, initialDelay: 10, maxDelay: 20 });
|
|
29
|
+
await expect(
|
|
30
|
+
policy.execute(async () => {
|
|
31
|
+
throw new Error("permanent");
|
|
32
|
+
}),
|
|
33
|
+
).rejects.toThrow("permanent");
|
|
34
|
+
});
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
describe("createCircuitBreaker", () => {
|
|
38
|
+
it("opens after consecutive failures", async () => {
|
|
39
|
+
const breaker = createCircuitBreaker({ threshold: 3, halfOpenAfter: 60_000 });
|
|
40
|
+
|
|
41
|
+
// Fail 3 times to trip the breaker
|
|
42
|
+
for (let i = 0; i < 3; i++) {
|
|
43
|
+
try {
|
|
44
|
+
await breaker.execute(async () => {
|
|
45
|
+
throw new Error("fail");
|
|
46
|
+
});
|
|
47
|
+
} catch {}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// Next call should fail fast with BrokenCircuitError
|
|
51
|
+
await expect(
|
|
52
|
+
breaker.execute(async () => "should not run"),
|
|
53
|
+
).rejects.toThrow(BrokenCircuitError);
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
it("allows calls when under threshold", async () => {
|
|
57
|
+
const breaker = createCircuitBreaker({ threshold: 5, halfOpenAfter: 60_000 });
|
|
58
|
+
|
|
59
|
+
// Fail twice then succeed — should not trip
|
|
60
|
+
let calls = 0;
|
|
61
|
+
for (let i = 0; i < 2; i++) {
|
|
62
|
+
try {
|
|
63
|
+
await breaker.execute(async () => {
|
|
64
|
+
throw new Error("fail");
|
|
65
|
+
});
|
|
66
|
+
} catch {}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const result = await breaker.execute(async () => {
|
|
70
|
+
calls++;
|
|
71
|
+
return "ok";
|
|
72
|
+
});
|
|
73
|
+
expect(result).toBe("ok");
|
|
74
|
+
expect(calls).toBe(1);
|
|
75
|
+
});
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
describe("withResilience", () => {
|
|
79
|
+
it("returns result on success", async () => {
|
|
80
|
+
const result = await withResilience(async () => 42);
|
|
81
|
+
expect(result).toBe(42);
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
it("retries transient failures", async () => {
|
|
85
|
+
let calls = 0;
|
|
86
|
+
const result = await withResilience(async () => {
|
|
87
|
+
calls++;
|
|
88
|
+
if (calls < 2) throw new Error("transient");
|
|
89
|
+
return "recovered";
|
|
90
|
+
});
|
|
91
|
+
expect(result).toBe("recovered");
|
|
92
|
+
expect(calls).toBe(2);
|
|
93
|
+
});
|
|
94
|
+
});
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* resilience.ts — Retry + circuit breaker for external API calls.
|
|
3
|
+
*
|
|
4
|
+
* Wraps functions with exponential backoff retry and a circuit breaker
|
|
5
|
+
* that opens after consecutive failures to prevent cascading overload.
|
|
6
|
+
*/
|
|
7
|
+
import {
|
|
8
|
+
retry,
|
|
9
|
+
handleAll,
|
|
10
|
+
ExponentialBackoff,
|
|
11
|
+
CircuitBreakerPolicy,
|
|
12
|
+
circuitBreaker,
|
|
13
|
+
ConsecutiveBreaker,
|
|
14
|
+
wrap,
|
|
15
|
+
type IPolicy,
|
|
16
|
+
} from "cockatiel";
|
|
17
|
+
|
|
18
|
+
// ---------------------------------------------------------------------------
|
|
19
|
+
// Retry policy
|
|
20
|
+
// ---------------------------------------------------------------------------
|
|
21
|
+
|
|
22
|
+
const DEFAULT_RETRY_ATTEMPTS = 3;
|
|
23
|
+
const DEFAULT_BACKOFF = { initialDelay: 500, maxDelay: 5_000 };
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Create a retry policy with exponential backoff.
|
|
27
|
+
*/
|
|
28
|
+
export function createRetryPolicy(opts?: {
|
|
29
|
+
attempts?: number;
|
|
30
|
+
initialDelay?: number;
|
|
31
|
+
maxDelay?: number;
|
|
32
|
+
}): IPolicy {
|
|
33
|
+
const attempts = opts?.attempts ?? DEFAULT_RETRY_ATTEMPTS;
|
|
34
|
+
const initialDelay = opts?.initialDelay ?? DEFAULT_BACKOFF.initialDelay;
|
|
35
|
+
const maxDelay = opts?.maxDelay ?? DEFAULT_BACKOFF.maxDelay;
|
|
36
|
+
|
|
37
|
+
return retry(handleAll, {
|
|
38
|
+
maxAttempts: attempts,
|
|
39
|
+
backoff: new ExponentialBackoff({
|
|
40
|
+
initialDelay,
|
|
41
|
+
maxDelay,
|
|
42
|
+
}),
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// ---------------------------------------------------------------------------
|
|
47
|
+
// Circuit breaker
|
|
48
|
+
// ---------------------------------------------------------------------------
|
|
49
|
+
|
|
50
|
+
const DEFAULT_BREAKER_THRESHOLD = 5;
|
|
51
|
+
const DEFAULT_HALF_OPEN_AFTER = 30_000;
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Create a circuit breaker that opens after consecutive failures.
|
|
55
|
+
*/
|
|
56
|
+
export function createCircuitBreaker(opts?: {
|
|
57
|
+
threshold?: number;
|
|
58
|
+
halfOpenAfter?: number;
|
|
59
|
+
}): CircuitBreakerPolicy {
|
|
60
|
+
const threshold = opts?.threshold ?? DEFAULT_BREAKER_THRESHOLD;
|
|
61
|
+
const halfOpenAfter = opts?.halfOpenAfter ?? DEFAULT_HALF_OPEN_AFTER;
|
|
62
|
+
|
|
63
|
+
return circuitBreaker(handleAll, {
|
|
64
|
+
breaker: new ConsecutiveBreaker(threshold),
|
|
65
|
+
halfOpenAfter,
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// ---------------------------------------------------------------------------
|
|
70
|
+
// Combined policy
|
|
71
|
+
// ---------------------------------------------------------------------------
|
|
72
|
+
|
|
73
|
+
let _defaultPolicy: IPolicy | null = null;
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Get the default combined retry + circuit breaker policy (singleton).
|
|
77
|
+
* 3 retries with exponential backoff (500ms → 5s) + circuit breaker
|
|
78
|
+
* (opens after 5 consecutive failures, half-opens after 30s).
|
|
79
|
+
*/
|
|
80
|
+
export function getDefaultPolicy(): IPolicy {
|
|
81
|
+
if (!_defaultPolicy) {
|
|
82
|
+
const retryPolicy = createRetryPolicy();
|
|
83
|
+
const breaker = createCircuitBreaker();
|
|
84
|
+
_defaultPolicy = wrap(retryPolicy, breaker);
|
|
85
|
+
}
|
|
86
|
+
return _defaultPolicy;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Execute a function with the default retry + circuit breaker policy.
|
|
91
|
+
*/
|
|
92
|
+
export async function withResilience<T>(fn: () => Promise<T>): Promise<T> {
|
|
93
|
+
return getDefaultPolicy().execute(fn);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Reset the default policy singleton (for testing).
|
|
98
|
+
*/
|
|
99
|
+
export function resetDefaultPolicy(): void {
|
|
100
|
+
_defaultPolicy = null;
|
|
101
|
+
}
|
|
@@ -229,22 +229,58 @@ export function buildSummaryFromArtifacts(worktreePath: string): string | null {
|
|
|
229
229
|
// Memory integration
|
|
230
230
|
// ---------------------------------------------------------------------------
|
|
231
231
|
|
|
232
|
+
export interface DispatchMemoryMetadata {
|
|
233
|
+
type: "dispatch";
|
|
234
|
+
issue: string;
|
|
235
|
+
title: string;
|
|
236
|
+
tier: string;
|
|
237
|
+
status: string;
|
|
238
|
+
project?: string;
|
|
239
|
+
attempts: number;
|
|
240
|
+
model: string;
|
|
241
|
+
date: string;
|
|
242
|
+
}
|
|
243
|
+
|
|
232
244
|
/**
|
|
233
|
-
* Write dispatch summary to the orchestrator's memory directory
|
|
245
|
+
* Write dispatch summary to the orchestrator's memory directory
|
|
246
|
+
* with YAML frontmatter for searchable metadata.
|
|
234
247
|
* Auto-indexed by OpenClaw's sqlite+embeddings memory system.
|
|
235
248
|
*/
|
|
236
249
|
export function writeDispatchMemory(
|
|
237
250
|
issueIdentifier: string,
|
|
238
251
|
summary: string,
|
|
239
252
|
workspaceDir: string,
|
|
253
|
+
metadata?: Partial<DispatchMemoryMetadata>,
|
|
240
254
|
): void {
|
|
241
255
|
const memDir = join(workspaceDir, "memory");
|
|
242
256
|
if (!existsSync(memDir)) {
|
|
243
257
|
mkdirSync(memDir, { recursive: true });
|
|
244
258
|
}
|
|
259
|
+
|
|
260
|
+
const fm: DispatchMemoryMetadata = {
|
|
261
|
+
type: "dispatch",
|
|
262
|
+
issue: issueIdentifier,
|
|
263
|
+
title: metadata?.title ?? issueIdentifier,
|
|
264
|
+
tier: metadata?.tier ?? "unknown",
|
|
265
|
+
status: metadata?.status ?? "unknown",
|
|
266
|
+
project: metadata?.project,
|
|
267
|
+
attempts: metadata?.attempts ?? 0,
|
|
268
|
+
model: metadata?.model ?? "unknown",
|
|
269
|
+
date: metadata?.date ?? new Date().toISOString().slice(0, 10),
|
|
270
|
+
};
|
|
271
|
+
|
|
272
|
+
const frontmatter = [
|
|
273
|
+
"---",
|
|
274
|
+
...Object.entries(fm)
|
|
275
|
+
.filter(([, v]) => v !== undefined)
|
|
276
|
+
.map(([k, v]) => `${k}: ${typeof v === "string" ? `"${v}"` : v}`),
|
|
277
|
+
"---",
|
|
278
|
+
"",
|
|
279
|
+
].join("\n");
|
|
280
|
+
|
|
245
281
|
writeFileSync(
|
|
246
282
|
join(memDir, `dispatch-${issueIdentifier}.md`),
|
|
247
|
-
summary,
|
|
283
|
+
frontmatter + summary,
|
|
248
284
|
"utf-8",
|
|
249
285
|
);
|
|
250
286
|
}
|