@calltelemetry/openclaw-linear 0.7.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/index.ts +39 -0
- package/openclaw.plugin.json +3 -3
- package/package.json +2 -1
- package/src/api/linear-api.test.ts +494 -0
- package/src/api/linear-api.ts +14 -11
- package/src/gateway/dispatch-methods.ts +243 -0
- package/src/infra/cli.ts +97 -29
- package/src/infra/codex-worktree.ts +83 -0
- package/src/infra/commands.ts +156 -0
- package/src/infra/file-lock.test.ts +61 -0
- package/src/infra/file-lock.ts +49 -0
- package/src/infra/multi-repo.ts +85 -0
- package/src/infra/notify.ts +115 -15
- package/src/infra/observability.ts +48 -0
- package/src/infra/resilience.test.ts +94 -0
- package/src/infra/resilience.ts +101 -0
- package/src/pipeline/artifacts.ts +38 -2
- package/src/pipeline/dag-dispatch.test.ts +553 -0
- package/src/pipeline/dag-dispatch.ts +390 -0
- package/src/pipeline/dispatch-service.ts +48 -1
- package/src/pipeline/dispatch-state.ts +2 -42
- package/src/pipeline/pipeline.ts +91 -17
- package/src/pipeline/planner.ts +6 -1
- package/src/pipeline/planning-state.ts +2 -40
- package/src/pipeline/tier-assess.test.ts +175 -0
- package/src/pipeline/webhook.ts +21 -0
- package/src/tools/dispatch-history-tool.ts +201 -0
- package/src/tools/orchestration-tools.test.ts +158 -0
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { describe, it, expect, afterEach } from "vitest";
|
|
2
|
+
import { acquireLock, releaseLock } from "./file-lock.js";
|
|
3
|
+
import fs from "node:fs/promises";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import os from "node:os";
|
|
6
|
+
|
|
7
|
+
const tmpDir = os.tmpdir();
|
|
8
|
+
const testState = path.join(tmpDir, `file-lock-test-${process.pid}.json`);
|
|
9
|
+
const lockFile = testState + ".lock";
|
|
10
|
+
|
|
11
|
+
afterEach(async () => {
|
|
12
|
+
try { await fs.unlink(lockFile); } catch {}
|
|
13
|
+
try { await fs.unlink(testState); } catch {}
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
describe("acquireLock / releaseLock", () => {
|
|
17
|
+
it("creates and removes a lock file", async () => {
|
|
18
|
+
await acquireLock(testState);
|
|
19
|
+
const stat = await fs.stat(lockFile);
|
|
20
|
+
expect(stat.isFile()).toBe(true);
|
|
21
|
+
|
|
22
|
+
await releaseLock(testState);
|
|
23
|
+
await expect(fs.stat(lockFile)).rejects.toThrow();
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
it("blocks concurrent acquires until released", async () => {
|
|
27
|
+
await acquireLock(testState);
|
|
28
|
+
|
|
29
|
+
let secondAcquired = false;
|
|
30
|
+
const secondLock = acquireLock(testState).then(() => {
|
|
31
|
+
secondAcquired = true;
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
// Give the second acquire a moment to spin
|
|
35
|
+
await new Promise((r) => setTimeout(r, 120));
|
|
36
|
+
expect(secondAcquired).toBe(false);
|
|
37
|
+
|
|
38
|
+
await releaseLock(testState);
|
|
39
|
+
await secondLock;
|
|
40
|
+
expect(secondAcquired).toBe(true);
|
|
41
|
+
|
|
42
|
+
await releaseLock(testState);
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
it("releaseLock is safe to call when no lock exists", async () => {
|
|
46
|
+
await expect(releaseLock(testState)).resolves.toBeUndefined();
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
it("recovers from stale lock", async () => {
|
|
50
|
+
// Write a lock file with an old timestamp (> 30s ago)
|
|
51
|
+
await fs.writeFile(lockFile, String(Date.now() - 60_000), { flag: "w" });
|
|
52
|
+
|
|
53
|
+
// Should succeed by detecting stale lock
|
|
54
|
+
await acquireLock(testState);
|
|
55
|
+
const content = await fs.readFile(lockFile, "utf-8");
|
|
56
|
+
const lockTime = Number(content);
|
|
57
|
+
expect(Date.now() - lockTime).toBeLessThan(5000);
|
|
58
|
+
|
|
59
|
+
await releaseLock(testState);
|
|
60
|
+
});
|
|
61
|
+
});
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* file-lock.ts — Shared file-level locking for state files.
|
|
3
|
+
*
|
|
4
|
+
* Used by dispatch-state.ts and planning-state.ts to prevent
|
|
5
|
+
* concurrent read-modify-write races on JSON state files.
|
|
6
|
+
*/
|
|
7
|
+
import fs from "node:fs/promises";
|
|
8
|
+
|
|
9
|
+
const LOCK_STALE_MS = 30_000;
|
|
10
|
+
const LOCK_RETRY_MS = 50;
|
|
11
|
+
const LOCK_TIMEOUT_MS = 10_000;
|
|
12
|
+
|
|
13
|
+
function lockPath(statePath: string): string {
|
|
14
|
+
return statePath + ".lock";
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export async function acquireLock(statePath: string): Promise<void> {
|
|
18
|
+
const lock = lockPath(statePath);
|
|
19
|
+
const deadline = Date.now() + LOCK_TIMEOUT_MS;
|
|
20
|
+
|
|
21
|
+
while (Date.now() < deadline) {
|
|
22
|
+
try {
|
|
23
|
+
await fs.writeFile(lock, String(Date.now()), { flag: "wx" });
|
|
24
|
+
return;
|
|
25
|
+
} catch (err: any) {
|
|
26
|
+
if (err.code !== "EEXIST") throw err;
|
|
27
|
+
|
|
28
|
+
// Check for stale lock
|
|
29
|
+
try {
|
|
30
|
+
const content = await fs.readFile(lock, "utf-8");
|
|
31
|
+
const lockTime = Number(content);
|
|
32
|
+
if (Date.now() - lockTime > LOCK_STALE_MS) {
|
|
33
|
+
try { await fs.unlink(lock); } catch { /* race */ }
|
|
34
|
+
continue;
|
|
35
|
+
}
|
|
36
|
+
} catch { /* lock disappeared — retry */ }
|
|
37
|
+
|
|
38
|
+
await new Promise((r) => setTimeout(r, LOCK_RETRY_MS));
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
// Last resort: force remove potentially stale lock
|
|
43
|
+
try { await fs.unlink(lockPath(statePath)); } catch { /* ignore */ }
|
|
44
|
+
await fs.writeFile(lock, String(Date.now()), { flag: "wx" });
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export async function releaseLock(statePath: string): Promise<void> {
|
|
48
|
+
try { await fs.unlink(lockPath(statePath)); } catch { /* already removed */ }
|
|
49
|
+
}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* multi-repo.ts — Multi-repo resolution for dispatches spanning multiple git repos.
|
|
3
|
+
*
|
|
4
|
+
* Three-tier resolution:
|
|
5
|
+
* 1. Issue body markers: <!-- repos: api, frontend --> or [repos: api, frontend]
|
|
6
|
+
* 2. Linear labels: repo:api, repo:frontend
|
|
7
|
+
* 3. Config default: Falls back to single codexBaseRepo
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import path from "node:path";
|
|
11
|
+
|
|
12
|
+
export interface RepoConfig {
|
|
13
|
+
name: string;
|
|
14
|
+
path: string;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export interface RepoResolution {
|
|
18
|
+
repos: RepoConfig[];
|
|
19
|
+
source: "issue_body" | "labels" | "config_default";
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Resolve which repos a dispatch should work with.
|
|
24
|
+
*/
|
|
25
|
+
export function resolveRepos(
|
|
26
|
+
description: string | null | undefined,
|
|
27
|
+
labels: string[],
|
|
28
|
+
pluginConfig?: Record<string, unknown>,
|
|
29
|
+
): RepoResolution {
|
|
30
|
+
// 1. Check issue body for repo markers
|
|
31
|
+
// Match: <!-- repos: name1, name2 --> or [repos: name1, name2]
|
|
32
|
+
const htmlComment = description?.match(/<!--\s*repos:\s*([^>]+?)\s*-->/i);
|
|
33
|
+
const bracketMatch = description?.match(/\[repos:\s*([^\]]+)\]/i);
|
|
34
|
+
const bodyMatch = htmlComment?.[1] ?? bracketMatch?.[1];
|
|
35
|
+
|
|
36
|
+
if (bodyMatch) {
|
|
37
|
+
const names = bodyMatch.split(",").map(s => s.trim()).filter(Boolean);
|
|
38
|
+
if (names.length > 0) {
|
|
39
|
+
const repoMap = getRepoMap(pluginConfig);
|
|
40
|
+
const repos = names.map(name => ({
|
|
41
|
+
name,
|
|
42
|
+
path: repoMap[name] ?? resolveRepoPath(name, pluginConfig),
|
|
43
|
+
}));
|
|
44
|
+
return { repos, source: "issue_body" };
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// 2. Check labels for repo: prefix
|
|
49
|
+
const repoLabels = labels
|
|
50
|
+
.filter(l => l.startsWith("repo:"))
|
|
51
|
+
.map(l => l.slice(5).trim())
|
|
52
|
+
.filter(Boolean);
|
|
53
|
+
|
|
54
|
+
if (repoLabels.length > 0) {
|
|
55
|
+
const repoMap = getRepoMap(pluginConfig);
|
|
56
|
+
const repos = repoLabels.map(name => ({
|
|
57
|
+
name,
|
|
58
|
+
path: repoMap[name] ?? resolveRepoPath(name, pluginConfig),
|
|
59
|
+
}));
|
|
60
|
+
return { repos, source: "labels" };
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// 3. Config default: single repo
|
|
64
|
+
const baseRepo = (pluginConfig?.codexBaseRepo as string) ?? "/home/claw/ai-workspace";
|
|
65
|
+
return {
|
|
66
|
+
repos: [{ name: "default", path: baseRepo }],
|
|
67
|
+
source: "config_default",
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function getRepoMap(pluginConfig?: Record<string, unknown>): Record<string, string> {
|
|
72
|
+
const repos = pluginConfig?.repos as Record<string, string> | undefined;
|
|
73
|
+
return repos ?? {};
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function resolveRepoPath(name: string, pluginConfig?: Record<string, unknown>): string {
|
|
77
|
+
// Convention: {parentDir}/{name}
|
|
78
|
+
const baseRepo = (pluginConfig?.codexBaseRepo as string) ?? "/home/claw/ai-workspace";
|
|
79
|
+
const parentDir = path.dirname(baseRepo);
|
|
80
|
+
return path.join(parentDir, name);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export function isMultiRepo(resolution: RepoResolution): boolean {
|
|
84
|
+
return resolution.repos.length > 1;
|
|
85
|
+
}
|
package/src/infra/notify.ts
CHANGED
|
@@ -15,14 +15,16 @@ import type { PluginRuntime } from "openclaw/plugin-sdk";
|
|
|
15
15
|
// ---------------------------------------------------------------------------
|
|
16
16
|
|
|
17
17
|
export type NotifyKind =
|
|
18
|
-
| "dispatch"
|
|
19
|
-
| "working"
|
|
20
|
-
| "auditing"
|
|
21
|
-
| "audit_pass"
|
|
22
|
-
| "audit_fail"
|
|
23
|
-
| "escalation"
|
|
24
|
-
| "stuck"
|
|
25
|
-
| "watchdog_kill"
|
|
18
|
+
| "dispatch" // issue dispatched to worker
|
|
19
|
+
| "working" // worker started
|
|
20
|
+
| "auditing" // audit triggered
|
|
21
|
+
| "audit_pass" // audit passed → done
|
|
22
|
+
| "audit_fail" // audit failed → rework
|
|
23
|
+
| "escalation" // 2x fail or stale → stuck
|
|
24
|
+
| "stuck" // stale detection
|
|
25
|
+
| "watchdog_kill" // agent killed by inactivity watchdog
|
|
26
|
+
| "project_progress" // DAG dispatch progress update
|
|
27
|
+
| "project_complete"; // all project issues dispatched
|
|
26
28
|
|
|
27
29
|
export interface NotifyPayload {
|
|
28
30
|
identifier: string;
|
|
@@ -51,6 +53,26 @@ export interface NotifyTarget {
|
|
|
51
53
|
export interface NotificationsConfig {
|
|
52
54
|
targets?: NotifyTarget[];
|
|
53
55
|
events?: Partial<Record<NotifyKind, boolean>>;
|
|
56
|
+
/** Opt-in: send rich embeds (Discord) and HTML (Telegram) instead of plain text. */
|
|
57
|
+
richFormat?: boolean;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// ---------------------------------------------------------------------------
|
|
61
|
+
// Rich message types (Discord embeds + Telegram HTML)
|
|
62
|
+
// ---------------------------------------------------------------------------
|
|
63
|
+
|
|
64
|
+
export interface DiscordEmbed {
|
|
65
|
+
title?: string;
|
|
66
|
+
description?: string;
|
|
67
|
+
color?: number;
|
|
68
|
+
fields?: Array<{ name: string; value: string; inline?: boolean }>;
|
|
69
|
+
footer?: { text: string };
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export interface RichMessage {
|
|
73
|
+
text: string;
|
|
74
|
+
discord?: { embeds: DiscordEmbed[] };
|
|
75
|
+
telegram?: { html: string };
|
|
54
76
|
}
|
|
55
77
|
|
|
56
78
|
// ---------------------------------------------------------------------------
|
|
@@ -80,37 +102,112 @@ export function formatMessage(kind: NotifyKind, payload: NotifyPayload): string
|
|
|
80
102
|
return `⚡ ${id} killed by watchdog (${payload.reason ?? "no I/O for 120s"}). ${
|
|
81
103
|
payload.attempt != null ? `Retrying (attempt ${payload.attempt}).` : "Will retry."
|
|
82
104
|
}`;
|
|
105
|
+
case "project_progress":
|
|
106
|
+
return `📊 ${payload.title} (${id}): ${payload.status}`;
|
|
107
|
+
case "project_complete":
|
|
108
|
+
return `✅ ${payload.title} (${id}): ${payload.status}`;
|
|
83
109
|
default:
|
|
84
110
|
return `${id} — ${kind}: ${payload.status}`;
|
|
85
111
|
}
|
|
86
112
|
}
|
|
87
113
|
|
|
114
|
+
// ---------------------------------------------------------------------------
|
|
115
|
+
// Rich message formatter (Discord embeds + Telegram HTML)
|
|
116
|
+
// ---------------------------------------------------------------------------
|
|
117
|
+
|
|
118
|
+
const EVENT_COLORS: Record<string, number> = {
|
|
119
|
+
dispatch: 0x3498db, // blue
|
|
120
|
+
working: 0x3498db, // blue
|
|
121
|
+
auditing: 0xf39c12, // yellow
|
|
122
|
+
audit_pass: 0x2ecc71, // green
|
|
123
|
+
audit_fail: 0xe74c3c, // red
|
|
124
|
+
escalation: 0xe74c3c, // red
|
|
125
|
+
stuck: 0xe67e22, // orange
|
|
126
|
+
watchdog_kill: 0x9b59b6, // purple
|
|
127
|
+
project_progress: 0x3498db,
|
|
128
|
+
project_complete: 0x2ecc71,
|
|
129
|
+
};
|
|
130
|
+
|
|
131
|
+
export function formatRichMessage(kind: NotifyKind, payload: NotifyPayload): RichMessage {
|
|
132
|
+
const text = formatMessage(kind, payload);
|
|
133
|
+
const color = EVENT_COLORS[kind] ?? 0x95a5a6;
|
|
134
|
+
|
|
135
|
+
// Discord embed
|
|
136
|
+
const fields: DiscordEmbed["fields"] = [];
|
|
137
|
+
if (payload.attempt != null) fields.push({ name: "Attempt", value: String(payload.attempt), inline: true });
|
|
138
|
+
if (payload.status) fields.push({ name: "Status", value: payload.status, inline: true });
|
|
139
|
+
if (payload.verdict?.gaps?.length) {
|
|
140
|
+
fields.push({ name: "Gaps", value: payload.verdict.gaps.join("\n").slice(0, 1024) });
|
|
141
|
+
}
|
|
142
|
+
if (payload.reason) fields.push({ name: "Reason", value: payload.reason });
|
|
143
|
+
|
|
144
|
+
const embed: DiscordEmbed = {
|
|
145
|
+
title: `${payload.identifier} — ${kind.replace(/_/g, " ")}`,
|
|
146
|
+
description: payload.title,
|
|
147
|
+
color,
|
|
148
|
+
fields: fields.length > 0 ? fields : undefined,
|
|
149
|
+
footer: { text: `Linear Agent • ${kind}` },
|
|
150
|
+
};
|
|
151
|
+
|
|
152
|
+
// Telegram HTML
|
|
153
|
+
const htmlParts: string[] = [
|
|
154
|
+
`<b>${escapeHtml(payload.identifier)}</b> — ${escapeHtml(kind.replace(/_/g, " "))}`,
|
|
155
|
+
`<i>${escapeHtml(payload.title)}</i>`,
|
|
156
|
+
];
|
|
157
|
+
if (payload.attempt != null) htmlParts.push(`Attempt: <code>${payload.attempt}</code>`);
|
|
158
|
+
if (payload.status) htmlParts.push(`Status: <code>${escapeHtml(payload.status)}</code>`);
|
|
159
|
+
if (payload.verdict?.gaps?.length) {
|
|
160
|
+
htmlParts.push(`Gaps:\n${payload.verdict.gaps.map(g => `• ${escapeHtml(g)}`).join("\n")}`);
|
|
161
|
+
}
|
|
162
|
+
if (payload.reason) htmlParts.push(`Reason: ${escapeHtml(payload.reason)}`);
|
|
163
|
+
|
|
164
|
+
return {
|
|
165
|
+
text,
|
|
166
|
+
discord: { embeds: [embed] },
|
|
167
|
+
telegram: { html: htmlParts.join("\n") },
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function escapeHtml(s: string): string {
|
|
172
|
+
return s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");
|
|
173
|
+
}
|
|
174
|
+
|
|
88
175
|
// ---------------------------------------------------------------------------
|
|
89
176
|
// Unified send — routes to OpenClaw runtime channel API
|
|
90
177
|
// ---------------------------------------------------------------------------
|
|
91
178
|
|
|
92
179
|
export async function sendToTarget(
|
|
93
180
|
target: NotifyTarget,
|
|
94
|
-
message: string,
|
|
181
|
+
message: string | RichMessage,
|
|
95
182
|
runtime: PluginRuntime,
|
|
96
183
|
): Promise<void> {
|
|
97
184
|
const ch = target.channel;
|
|
98
185
|
const to = target.target;
|
|
186
|
+
const isRich = typeof message !== "string";
|
|
187
|
+
const plainText = isRich ? message.text : message;
|
|
99
188
|
|
|
100
189
|
if (ch === "discord") {
|
|
101
|
-
|
|
190
|
+
if (isRich && message.discord) {
|
|
191
|
+
await runtime.channel.discord.sendMessageDiscord(to, plainText, { embeds: message.discord.embeds });
|
|
192
|
+
} else {
|
|
193
|
+
await runtime.channel.discord.sendMessageDiscord(to, plainText);
|
|
194
|
+
}
|
|
102
195
|
} else if (ch === "slack") {
|
|
103
|
-
await runtime.channel.slack.sendMessageSlack(to,
|
|
196
|
+
await runtime.channel.slack.sendMessageSlack(to, plainText, {
|
|
104
197
|
accountId: target.accountId,
|
|
105
198
|
});
|
|
106
199
|
} else if (ch === "telegram") {
|
|
107
|
-
|
|
200
|
+
if (isRich && message.telegram) {
|
|
201
|
+
await runtime.channel.telegram.sendMessageTelegram(to, message.telegram.html, { silent: true, textMode: "html" });
|
|
202
|
+
} else {
|
|
203
|
+
await runtime.channel.telegram.sendMessageTelegram(to, plainText, { silent: true });
|
|
204
|
+
}
|
|
108
205
|
} else if (ch === "signal") {
|
|
109
|
-
await runtime.channel.signal.sendMessageSignal(to,
|
|
206
|
+
await runtime.channel.signal.sendMessageSignal(to, plainText);
|
|
110
207
|
} else {
|
|
111
208
|
// Fallback: use CLI for any channel the runtime doesn't expose directly
|
|
112
209
|
const { execFileSync } = await import("node:child_process");
|
|
113
|
-
execFileSync("openclaw", ["message", "send", "--channel", ch, "--target", to, "--message",
|
|
210
|
+
execFileSync("openclaw", ["message", "send", "--channel", ch, "--target", to, "--message", plainText, "--json"], {
|
|
114
211
|
timeout: 30_000,
|
|
115
212
|
stdio: "ignore",
|
|
116
213
|
});
|
|
@@ -131,6 +228,7 @@ export function parseNotificationsConfig(
|
|
|
131
228
|
return {
|
|
132
229
|
targets: raw?.targets ?? [],
|
|
133
230
|
events: raw?.events ?? {},
|
|
231
|
+
richFormat: raw?.richFormat ?? false,
|
|
134
232
|
};
|
|
135
233
|
}
|
|
136
234
|
|
|
@@ -148,11 +246,13 @@ export function createNotifierFromConfig(
|
|
|
148
246
|
|
|
149
247
|
if (!config.targets?.length) return createNoopNotifier();
|
|
150
248
|
|
|
249
|
+
const useRich = config.richFormat === true;
|
|
250
|
+
|
|
151
251
|
return async (kind, payload) => {
|
|
152
252
|
// Check event toggle — default is enabled (true)
|
|
153
253
|
if (config.events?.[kind] === false) return;
|
|
154
254
|
|
|
155
|
-
const message = formatMessage(kind, payload);
|
|
255
|
+
const message = useRich ? formatRichMessage(kind, payload) : formatMessage(kind, payload);
|
|
156
256
|
|
|
157
257
|
await Promise.allSettled(
|
|
158
258
|
config.targets!.map(async (target) => {
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* observability.ts — Structured diagnostic event logging.
|
|
3
|
+
*
|
|
4
|
+
* Emits structured JSON log lines via api.logger for lifecycle telemetry.
|
|
5
|
+
* Consumers (log aggregators, monitoring) can parse these for dashboards.
|
|
6
|
+
*
|
|
7
|
+
* Pattern: `[linear:diagnostic] {...json...}`
|
|
8
|
+
*/
|
|
9
|
+
import type { OpenClawPluginApi } from "openclaw/plugin-sdk";
|
|
10
|
+
|
|
11
|
+
export type DiagnosticEvent =
|
|
12
|
+
| "webhook_received"
|
|
13
|
+
| "dispatch_started"
|
|
14
|
+
| "phase_transition"
|
|
15
|
+
| "audit_triggered"
|
|
16
|
+
| "verdict_processed"
|
|
17
|
+
| "watchdog_kill"
|
|
18
|
+
| "notify_sent"
|
|
19
|
+
| "notify_failed"
|
|
20
|
+
| "health_check";
|
|
21
|
+
|
|
22
|
+
export interface DiagnosticPayload {
|
|
23
|
+
event: DiagnosticEvent;
|
|
24
|
+
identifier?: string;
|
|
25
|
+
issueId?: string;
|
|
26
|
+
phase?: string;
|
|
27
|
+
from?: string;
|
|
28
|
+
to?: string;
|
|
29
|
+
attempt?: number;
|
|
30
|
+
tier?: string;
|
|
31
|
+
webhookType?: string;
|
|
32
|
+
webhookAction?: string;
|
|
33
|
+
channel?: string;
|
|
34
|
+
target?: string;
|
|
35
|
+
error?: string;
|
|
36
|
+
durationMs?: number;
|
|
37
|
+
[key: string]: unknown;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const PREFIX = "[linear:diagnostic]";
|
|
41
|
+
|
|
42
|
+
export function emitDiagnostic(api: OpenClawPluginApi, payload: DiagnosticPayload): void {
|
|
43
|
+
try {
|
|
44
|
+
api.logger.info(`${PREFIX} ${JSON.stringify(payload)}`);
|
|
45
|
+
} catch {
|
|
46
|
+
// Never throw from telemetry
|
|
47
|
+
}
|
|
48
|
+
}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach } from "vitest";
|
|
2
|
+
import {
|
|
3
|
+
createRetryPolicy,
|
|
4
|
+
createCircuitBreaker,
|
|
5
|
+
withResilience,
|
|
6
|
+
resetDefaultPolicy,
|
|
7
|
+
} from "./resilience.js";
|
|
8
|
+
import { BrokenCircuitError } from "cockatiel";
|
|
9
|
+
|
|
10
|
+
beforeEach(() => {
|
|
11
|
+
resetDefaultPolicy();
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
describe("createRetryPolicy", () => {
|
|
15
|
+
it("retries on transient failure then succeeds", async () => {
|
|
16
|
+
let calls = 0;
|
|
17
|
+
const policy = createRetryPolicy({ attempts: 3, initialDelay: 10, maxDelay: 20 });
|
|
18
|
+
const result = await policy.execute(async () => {
|
|
19
|
+
calls++;
|
|
20
|
+
if (calls < 3) throw new Error("transient");
|
|
21
|
+
return "ok";
|
|
22
|
+
});
|
|
23
|
+
expect(result).toBe("ok");
|
|
24
|
+
expect(calls).toBe(3);
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
it("throws after exhausting retries", async () => {
|
|
28
|
+
const policy = createRetryPolicy({ attempts: 2, initialDelay: 10, maxDelay: 20 });
|
|
29
|
+
await expect(
|
|
30
|
+
policy.execute(async () => {
|
|
31
|
+
throw new Error("permanent");
|
|
32
|
+
}),
|
|
33
|
+
).rejects.toThrow("permanent");
|
|
34
|
+
});
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
describe("createCircuitBreaker", () => {
|
|
38
|
+
it("opens after consecutive failures", async () => {
|
|
39
|
+
const breaker = createCircuitBreaker({ threshold: 3, halfOpenAfter: 60_000 });
|
|
40
|
+
|
|
41
|
+
// Fail 3 times to trip the breaker
|
|
42
|
+
for (let i = 0; i < 3; i++) {
|
|
43
|
+
try {
|
|
44
|
+
await breaker.execute(async () => {
|
|
45
|
+
throw new Error("fail");
|
|
46
|
+
});
|
|
47
|
+
} catch {}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// Next call should fail fast with BrokenCircuitError
|
|
51
|
+
await expect(
|
|
52
|
+
breaker.execute(async () => "should not run"),
|
|
53
|
+
).rejects.toThrow(BrokenCircuitError);
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
it("allows calls when under threshold", async () => {
|
|
57
|
+
const breaker = createCircuitBreaker({ threshold: 5, halfOpenAfter: 60_000 });
|
|
58
|
+
|
|
59
|
+
// Fail twice then succeed — should not trip
|
|
60
|
+
let calls = 0;
|
|
61
|
+
for (let i = 0; i < 2; i++) {
|
|
62
|
+
try {
|
|
63
|
+
await breaker.execute(async () => {
|
|
64
|
+
throw new Error("fail");
|
|
65
|
+
});
|
|
66
|
+
} catch {}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const result = await breaker.execute(async () => {
|
|
70
|
+
calls++;
|
|
71
|
+
return "ok";
|
|
72
|
+
});
|
|
73
|
+
expect(result).toBe("ok");
|
|
74
|
+
expect(calls).toBe(1);
|
|
75
|
+
});
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
describe("withResilience", () => {
|
|
79
|
+
it("returns result on success", async () => {
|
|
80
|
+
const result = await withResilience(async () => 42);
|
|
81
|
+
expect(result).toBe(42);
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
it("retries transient failures", async () => {
|
|
85
|
+
let calls = 0;
|
|
86
|
+
const result = await withResilience(async () => {
|
|
87
|
+
calls++;
|
|
88
|
+
if (calls < 2) throw new Error("transient");
|
|
89
|
+
return "recovered";
|
|
90
|
+
});
|
|
91
|
+
expect(result).toBe("recovered");
|
|
92
|
+
expect(calls).toBe(2);
|
|
93
|
+
});
|
|
94
|
+
});
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* resilience.ts — Retry + circuit breaker for external API calls.
|
|
3
|
+
*
|
|
4
|
+
* Wraps functions with exponential backoff retry and a circuit breaker
|
|
5
|
+
* that opens after consecutive failures to prevent cascading overload.
|
|
6
|
+
*/
|
|
7
|
+
import {
|
|
8
|
+
retry,
|
|
9
|
+
handleAll,
|
|
10
|
+
ExponentialBackoff,
|
|
11
|
+
CircuitBreakerPolicy,
|
|
12
|
+
circuitBreaker,
|
|
13
|
+
ConsecutiveBreaker,
|
|
14
|
+
wrap,
|
|
15
|
+
type IPolicy,
|
|
16
|
+
} from "cockatiel";
|
|
17
|
+
|
|
18
|
+
// ---------------------------------------------------------------------------
|
|
19
|
+
// Retry policy
|
|
20
|
+
// ---------------------------------------------------------------------------
|
|
21
|
+
|
|
22
|
+
const DEFAULT_RETRY_ATTEMPTS = 3;
|
|
23
|
+
const DEFAULT_BACKOFF = { initialDelay: 500, maxDelay: 5_000 };
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Create a retry policy with exponential backoff.
|
|
27
|
+
*/
|
|
28
|
+
export function createRetryPolicy(opts?: {
|
|
29
|
+
attempts?: number;
|
|
30
|
+
initialDelay?: number;
|
|
31
|
+
maxDelay?: number;
|
|
32
|
+
}): IPolicy {
|
|
33
|
+
const attempts = opts?.attempts ?? DEFAULT_RETRY_ATTEMPTS;
|
|
34
|
+
const initialDelay = opts?.initialDelay ?? DEFAULT_BACKOFF.initialDelay;
|
|
35
|
+
const maxDelay = opts?.maxDelay ?? DEFAULT_BACKOFF.maxDelay;
|
|
36
|
+
|
|
37
|
+
return retry(handleAll, {
|
|
38
|
+
maxAttempts: attempts,
|
|
39
|
+
backoff: new ExponentialBackoff({
|
|
40
|
+
initialDelay,
|
|
41
|
+
maxDelay,
|
|
42
|
+
}),
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// ---------------------------------------------------------------------------
|
|
47
|
+
// Circuit breaker
|
|
48
|
+
// ---------------------------------------------------------------------------
|
|
49
|
+
|
|
50
|
+
const DEFAULT_BREAKER_THRESHOLD = 5;
|
|
51
|
+
const DEFAULT_HALF_OPEN_AFTER = 30_000;
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Create a circuit breaker that opens after consecutive failures.
|
|
55
|
+
*/
|
|
56
|
+
export function createCircuitBreaker(opts?: {
|
|
57
|
+
threshold?: number;
|
|
58
|
+
halfOpenAfter?: number;
|
|
59
|
+
}): CircuitBreakerPolicy {
|
|
60
|
+
const threshold = opts?.threshold ?? DEFAULT_BREAKER_THRESHOLD;
|
|
61
|
+
const halfOpenAfter = opts?.halfOpenAfter ?? DEFAULT_HALF_OPEN_AFTER;
|
|
62
|
+
|
|
63
|
+
return circuitBreaker(handleAll, {
|
|
64
|
+
breaker: new ConsecutiveBreaker(threshold),
|
|
65
|
+
halfOpenAfter,
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// ---------------------------------------------------------------------------
|
|
70
|
+
// Combined policy
|
|
71
|
+
// ---------------------------------------------------------------------------
|
|
72
|
+
|
|
73
|
+
let _defaultPolicy: IPolicy | null = null;
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Get the default combined retry + circuit breaker policy (singleton).
|
|
77
|
+
* 3 retries with exponential backoff (500ms → 5s) + circuit breaker
|
|
78
|
+
* (opens after 5 consecutive failures, half-opens after 30s).
|
|
79
|
+
*/
|
|
80
|
+
export function getDefaultPolicy(): IPolicy {
|
|
81
|
+
if (!_defaultPolicy) {
|
|
82
|
+
const retryPolicy = createRetryPolicy();
|
|
83
|
+
const breaker = createCircuitBreaker();
|
|
84
|
+
_defaultPolicy = wrap(retryPolicy, breaker);
|
|
85
|
+
}
|
|
86
|
+
return _defaultPolicy;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Execute a function with the default retry + circuit breaker policy.
|
|
91
|
+
*/
|
|
92
|
+
export async function withResilience<T>(fn: () => Promise<T>): Promise<T> {
|
|
93
|
+
return getDefaultPolicy().execute(fn);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Reset the default policy singleton (for testing).
|
|
98
|
+
*/
|
|
99
|
+
export function resetDefaultPolicy(): void {
|
|
100
|
+
_defaultPolicy = null;
|
|
101
|
+
}
|
|
@@ -229,22 +229,58 @@ export function buildSummaryFromArtifacts(worktreePath: string): string | null {
|
|
|
229
229
|
// Memory integration
|
|
230
230
|
// ---------------------------------------------------------------------------
|
|
231
231
|
|
|
232
|
+
export interface DispatchMemoryMetadata {
|
|
233
|
+
type: "dispatch";
|
|
234
|
+
issue: string;
|
|
235
|
+
title: string;
|
|
236
|
+
tier: string;
|
|
237
|
+
status: string;
|
|
238
|
+
project?: string;
|
|
239
|
+
attempts: number;
|
|
240
|
+
model: string;
|
|
241
|
+
date: string;
|
|
242
|
+
}
|
|
243
|
+
|
|
232
244
|
/**
|
|
233
|
-
* Write dispatch summary to the orchestrator's memory directory
|
|
245
|
+
* Write dispatch summary to the orchestrator's memory directory
|
|
246
|
+
* with YAML frontmatter for searchable metadata.
|
|
234
247
|
* Auto-indexed by OpenClaw's sqlite+embeddings memory system.
|
|
235
248
|
*/
|
|
236
249
|
export function writeDispatchMemory(
|
|
237
250
|
issueIdentifier: string,
|
|
238
251
|
summary: string,
|
|
239
252
|
workspaceDir: string,
|
|
253
|
+
metadata?: Partial<DispatchMemoryMetadata>,
|
|
240
254
|
): void {
|
|
241
255
|
const memDir = join(workspaceDir, "memory");
|
|
242
256
|
if (!existsSync(memDir)) {
|
|
243
257
|
mkdirSync(memDir, { recursive: true });
|
|
244
258
|
}
|
|
259
|
+
|
|
260
|
+
const fm: DispatchMemoryMetadata = {
|
|
261
|
+
type: "dispatch",
|
|
262
|
+
issue: issueIdentifier,
|
|
263
|
+
title: metadata?.title ?? issueIdentifier,
|
|
264
|
+
tier: metadata?.tier ?? "unknown",
|
|
265
|
+
status: metadata?.status ?? "unknown",
|
|
266
|
+
project: metadata?.project,
|
|
267
|
+
attempts: metadata?.attempts ?? 0,
|
|
268
|
+
model: metadata?.model ?? "unknown",
|
|
269
|
+
date: metadata?.date ?? new Date().toISOString().slice(0, 10),
|
|
270
|
+
};
|
|
271
|
+
|
|
272
|
+
const frontmatter = [
|
|
273
|
+
"---",
|
|
274
|
+
...Object.entries(fm)
|
|
275
|
+
.filter(([, v]) => v !== undefined)
|
|
276
|
+
.map(([k, v]) => `${k}: ${typeof v === "string" ? `"${v}"` : v}`),
|
|
277
|
+
"---",
|
|
278
|
+
"",
|
|
279
|
+
].join("\n");
|
|
280
|
+
|
|
245
281
|
writeFileSync(
|
|
246
282
|
join(memDir, `dispatch-${issueIdentifier}.md`),
|
|
247
|
-
summary,
|
|
283
|
+
frontmatter + summary,
|
|
248
284
|
"utf-8",
|
|
249
285
|
);
|
|
250
286
|
}
|