@unblocklabs/skill-usage-audit 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +138 -0
- package/evaluate-skill-health.mjs +919 -0
- package/index.ts +1523 -0
- package/openclaw.plugin.json +66 -0
- package/package.json +38 -0
package/index.ts
ADDED
|
@@ -0,0 +1,1523 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Skill usage audit plugin
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import { mkdir, readdir, readFile, stat } from "node:fs/promises";
|
|
6
|
+
import { dirname, basename, resolve, join, relative } from "node:path";
|
|
7
|
+
|
|
8
|
+
import { createHash } from "node:crypto";
|
|
9
|
+
|
|
10
|
+
import type { OpenClawPluginApi } from "openclaw/plugin-sdk";
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
const DEFAULT_DB_PATH = "~/.openclaw/audits/skill-usage.db";
|
|
14
|
+
const DEFAULT_INCLUDE_TOOL_PARAMS = false;
|
|
15
|
+
const DEFAULT_CAPTURE_MESSAGE_CONTENT = false;
|
|
16
|
+
const DEFAULT_CONTEXT_WINDOW_SIZE = 5;
|
|
17
|
+
const DEFAULT_CONTEXT_TIMEOUT_MS = 60000;
|
|
18
|
+
const MAX_MESSAGE_LEN = 200;
|
|
19
|
+
const MAX_HISTORY_PER_SCOPE = 80;
|
|
20
|
+
|
|
21
|
+
const DEFAULT_REDACT_KEYS = [
|
|
22
|
+
"token",
|
|
23
|
+
"apikey",
|
|
24
|
+
"api_key",
|
|
25
|
+
"apiKey",
|
|
26
|
+
"password",
|
|
27
|
+
"passwd",
|
|
28
|
+
"auth",
|
|
29
|
+
"authorization",
|
|
30
|
+
"secret",
|
|
31
|
+
"secretToken",
|
|
32
|
+
"refreshToken",
|
|
33
|
+
"client_secret",
|
|
34
|
+
];
|
|
35
|
+
|
|
36
|
+
const SECRET_PATTERNS: RegExp[] = [
|
|
37
|
+
/Authorization:\s*Bearer\s+[A-Za-z0-9._~+/=-]{10,}/gi,
|
|
38
|
+
/\bsk-[A-Za-z0-9]{10,}/gi,
|
|
39
|
+
/\bxox(?:b|p|o|s|r|u)-[A-Za-z0-9-]{10,}/gi,
|
|
40
|
+
/\bgh[oprstuv]_[A-Za-z0-9]{20,}/gi,
|
|
41
|
+
/(?:api[_-]?key|secret|token)[\s=:]\"?[A-Za-z0-9._~+/=-]{16,}\"?/gi,
|
|
42
|
+
];
|
|
43
|
+
|
|
44
|
+
const EMAIL_PATTERN = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g;
|
|
45
|
+
const PHONE_PATTERN = /(?:(?:\+?\d{1,3}[-.\s]?)?(?:\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4})(?:\b|$))/g;
|
|
46
|
+
const URL_WITH_QUERY_PATTERN = /https?:\/\/[^\s<>"']+\?[^\s<>"']*/gi;
|
|
47
|
+
|
|
48
|
+
const NEGATIVE_MESSAGE_PATTERNS = [
|
|
49
|
+
/wrong/i,
|
|
50
|
+
/try again/i,
|
|
51
|
+
/that didn['’]?t work/i,
|
|
52
|
+
/no that['’]?s not/i,
|
|
53
|
+
/\bredo\b/i,
|
|
54
|
+
/\bfix\b/i,
|
|
55
|
+
/\bbroken\b/i,
|
|
56
|
+
/\bbad\b/i,
|
|
57
|
+
/\bincorrect\b/i,
|
|
58
|
+
/sorry/i,
|
|
59
|
+
/apolog/i,
|
|
60
|
+
/did not/i,
|
|
61
|
+
];
|
|
62
|
+
|
|
63
|
+
interface PluginConfig {
|
|
64
|
+
dbPath?: string;
|
|
65
|
+
captureMessageContent?: unknown;
|
|
66
|
+
includeToolParams?: boolean;
|
|
67
|
+
redactKeys?: unknown;
|
|
68
|
+
skillBlockDetection?: boolean;
|
|
69
|
+
contextWindowSize?: unknown;
|
|
70
|
+
contextTimeoutMs?: unknown;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
interface RawContext {
|
|
74
|
+
sessionId?: string;
|
|
75
|
+
runId?: string;
|
|
76
|
+
sessionKey?: string;
|
|
77
|
+
agentId?: string;
|
|
78
|
+
messageProvider?: string;
|
|
79
|
+
channelId?: string;
|
|
80
|
+
trigger?: string;
|
|
81
|
+
conversationId?: string;
|
|
82
|
+
accountId?: string;
|
|
83
|
+
[key: string]: unknown;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
interface RawEvent {
|
|
87
|
+
[key: string]: unknown;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
interface MessageCapture {
|
|
91
|
+
ts: string;
|
|
92
|
+
role: "user" | "assistant";
|
|
93
|
+
length: number;
|
|
94
|
+
text?: string;
|
|
95
|
+
metadata?: unknown;
|
|
96
|
+
signalLabels?: string[];
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
type ImpliedOutcome = "positive" | "negative" | "unclear";
|
|
100
|
+
|
|
101
|
+
interface SkillExecutionState {
|
|
102
|
+
id: number;
|
|
103
|
+
finalized: boolean;
|
|
104
|
+
ts: string;
|
|
105
|
+
startAt: number;
|
|
106
|
+
|
|
107
|
+
sessionId?: string;
|
|
108
|
+
sessionKey?: string;
|
|
109
|
+
runId?: string;
|
|
110
|
+
|
|
111
|
+
scopeKeys: string[];
|
|
112
|
+
skillName: string;
|
|
113
|
+
skillPath: string;
|
|
114
|
+
versionHash?: string | null;
|
|
115
|
+
versionHashPromise?: Promise<string | null>;
|
|
116
|
+
|
|
117
|
+
intentContext: MessageCapture[];
|
|
118
|
+
followupMessages: MessageCapture[];
|
|
119
|
+
|
|
120
|
+
toolReadCount: number;
|
|
121
|
+
encounteredSkillPaths: Set<string>;
|
|
122
|
+
sameSkillRetried: boolean;
|
|
123
|
+
fallbackSkillRetried: boolean;
|
|
124
|
+
|
|
125
|
+
inFlightToolCalls: Set<string>;
|
|
126
|
+
hadToolCall: boolean;
|
|
127
|
+
mechanicalSuccess: boolean;
|
|
128
|
+
error?: string;
|
|
129
|
+
|
|
130
|
+
inFollowup: boolean;
|
|
131
|
+
followupTimer?: ReturnType<typeof setTimeout>;
|
|
132
|
+
followupStartedAt?: number;
|
|
133
|
+
impliedOutcome?: ImpliedOutcome;
|
|
134
|
+
|
|
135
|
+
contextWindowSize: number;
|
|
136
|
+
contextTimeoutMs: number;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
interface SqliteBackend {
|
|
140
|
+
kind: string;
|
|
141
|
+
close: () => void;
|
|
142
|
+
exec: (sql: string) => void;
|
|
143
|
+
prepare: (sql: string) => {
|
|
144
|
+
run: (params?: Record<string, unknown>) => unknown;
|
|
145
|
+
get: (params?: Record<string, unknown>) => Record<string, unknown> | undefined;
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
interface DbPrepared {
|
|
150
|
+
insertEvent: (params: Record<string, unknown>) => void;
|
|
151
|
+
insertVersion: (params: Record<string, unknown>) => void;
|
|
152
|
+
upsertSkill: (params: Record<string, unknown>) => void;
|
|
153
|
+
getLatestSkillVersion: (params: Record<string, unknown>) => { version_hash?: string } | undefined;
|
|
154
|
+
insertExecution: (params: Record<string, unknown>) => void;
|
|
155
|
+
insertFeedback: (params: Record<string, unknown>) => void;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
interface DbState {
|
|
159
|
+
backend: SqliteBackend | null;
|
|
160
|
+
statements: DbPrepared | null;
|
|
161
|
+
error?: string;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function resolveHomePath(pathLike: string): string {
|
|
165
|
+
if (!pathLike.startsWith("~")) return resolve(pathLike);
|
|
166
|
+
const home = process.env.HOME || process.env.USERPROFILE;
|
|
167
|
+
if (!home) return resolve(pathLike);
|
|
168
|
+
return resolve(home, pathLike.slice(2));
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
function resolveDbPath(pathLike: string | undefined): string {
|
|
173
|
+
return resolveHomePath(pathLike && pathLike.trim().length > 0 ? pathLike : DEFAULT_DB_PATH);
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
function toStringLike(value: unknown): string | undefined {
|
|
177
|
+
if (typeof value !== "string") return undefined;
|
|
178
|
+
const t = value.trim();
|
|
179
|
+
return t.length ? t : undefined;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
function parseIntConfig(value: unknown, fallback: number, min?: number): number {
|
|
183
|
+
const parsed =
|
|
184
|
+
typeof value === "number"
|
|
185
|
+
? Math.floor(value)
|
|
186
|
+
: Number.parseInt(typeof value === "string" ? value.trim() : "", 10);
|
|
187
|
+
|
|
188
|
+
if (!Number.isFinite(parsed)) return fallback;
|
|
189
|
+
if (min !== undefined && parsed < min) return fallback;
|
|
190
|
+
return parsed;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
function normalizeText(value: unknown, redactKeys: Set<string>): string {
|
|
194
|
+
const text = typeof value !== "string" ? String(value ?? "") : value;
|
|
195
|
+
return sanitizeValue(redactParams(text, redactKeys), MAX_MESSAGE_LEN);
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
function sanitizeValue(value: unknown, maxLen = 400): string {
|
|
199
|
+
if (value === null || value === undefined) return "";
|
|
200
|
+
|
|
201
|
+
if (typeof value === "string") {
|
|
202
|
+
const scrubbed = scrubSecrets(value);
|
|
203
|
+
return scrubbed.length <= maxLen
|
|
204
|
+
? scrubbed
|
|
205
|
+
: `${scrubbed.slice(0, maxLen)}…[truncated ${scrubbed.length - maxLen} chars]`;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
if (Array.isArray(value)) {
|
|
209
|
+
return JSON.stringify(value.map((entry) => sanitizeValue(entry, maxLen))).slice(0, maxLen);
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
if (typeof value === "object") {
|
|
213
|
+
return JSON.stringify(redactParams(value, new Set())).slice(0, maxLen);
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
return String(value).slice(0, maxLen);
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
function redactUrlsWithQuery(value: string): string {
|
|
220
|
+
return value.replace(URL_WITH_QUERY_PATTERN, (match) => {
|
|
221
|
+
const idx = match.indexOf("?");
|
|
222
|
+
if (idx < 0) return match;
|
|
223
|
+
return `${match.slice(0, idx)}[query-removed]`;
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
function scrubSecrets(value: string): string {
|
|
228
|
+
const cleaned = sanitizePII(EMAIL_PATTERN, value);
|
|
229
|
+
const scrubbedQuery = redactUrlsWithQuery(cleaned);
|
|
230
|
+
const scrubbedPhone = sanitizePII(PHONE_PATTERN, scrubbedQuery);
|
|
231
|
+
return SECRET_PATTERNS.reduce((next, pattern) => next.replace(pattern, "[REDACTED]"), scrubbedPhone);
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
function sanitizePII(pattern: RegExp, value: string): string {
|
|
235
|
+
return value.replace(pattern, "[REDACTED]");
|
|
236
|
+
}
|
|
237
|
+
function shouldRedactKey(key: string, redactKeys: Set<string>): boolean {
|
|
238
|
+
const candidate = key.toLowerCase();
|
|
239
|
+
if (redactKeys.has(candidate)) return true;
|
|
240
|
+
return candidate.includes("token") || candidate.includes("secret") || candidate.includes("auth") || candidate.includes("password");
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
function redactParams(value: unknown, redactKeys: Set<string>): unknown {
|
|
244
|
+
if (value === null || value === undefined) return value;
|
|
245
|
+
|
|
246
|
+
if (typeof value === "string") return scrubSecrets(value);
|
|
247
|
+
if (Array.isArray(value)) return value.map((entry) => redactParams(entry, redactKeys));
|
|
248
|
+
if (typeof value !== "object") return value;
|
|
249
|
+
|
|
250
|
+
const entries = value as Record<string, unknown>;
|
|
251
|
+
const out: Record<string, unknown> = {};
|
|
252
|
+
for (const [k, v] of Object.entries(entries)) {
|
|
253
|
+
out[k] = shouldRedactKey(k, redactKeys) ? "[REDACTED]" : redactParams(v, redactKeys);
|
|
254
|
+
}
|
|
255
|
+
return out;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
function buildToolParams(toolName: string, params: Record<string, unknown>, redactKeys: Set<string>) {
|
|
259
|
+
if (toolName === "write" || toolName === "message") {
|
|
260
|
+
const scrubbed = redactParams(params, redactKeys);
|
|
261
|
+
if (typeof scrubbed === "object" && scrubbed && !Array.isArray(scrubbed)) {
|
|
262
|
+
const copy = { ...(scrubbed as Record<string, unknown>) };
|
|
263
|
+
if (Object.prototype.hasOwnProperty.call(copy, "text")) copy.text = "[REDACTED]";
|
|
264
|
+
if (Object.prototype.hasOwnProperty.call(copy, "content")) copy.content = "[REDACTED]";
|
|
265
|
+
if (Object.prototype.hasOwnProperty.call(copy, "body")) copy.body = "[REDACTED]";
|
|
266
|
+
if (Object.prototype.hasOwnProperty.call(copy, "data")) copy.data = "[REDACTED]";
|
|
267
|
+
if (Object.prototype.hasOwnProperty.call(copy, "message")) copy.message = "[REDACTED]";
|
|
268
|
+
return copy;
|
|
269
|
+
}
|
|
270
|
+
return scrubbed;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
if (toolName === "exec" && typeof params === "object" && params) {
|
|
274
|
+
const copy = { ...(params as Record<string, unknown>), command: "[REDACTED]" };
|
|
275
|
+
return copy;
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
return redactParams(params, redactKeys);
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
function detectMessageSignals(execution: SkillExecutionState | undefined, text: string): string[] {
|
|
282
|
+
const lower = text.toLowerCase();
|
|
283
|
+
const labels = new Set<string>();
|
|
284
|
+
|
|
285
|
+
if (NEGATIVE_MESSAGE_PATTERNS.some((pattern) => pattern.test(lower))) {
|
|
286
|
+
labels.add("negative_phrase_detected");
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
if (/(retry|redo|again|rerun|re-run)/i.test(lower)) {
|
|
290
|
+
const token = execution?.skillName?.toLowerCase();
|
|
291
|
+
if (!token || lower.includes(token)) {
|
|
292
|
+
labels.add("skill_retry_detected");
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
const mention = /(?:\/skill:|using\s+skill\s+|skill\s+name\s+['"]?)([a-z0-9._-]+)/i.exec(lower);
|
|
297
|
+
if (mention?.[1] && execution && mention[1] !== execution.skillName.toLowerCase() && /fallback|instead|retry/.test(lower)) {
|
|
298
|
+
labels.add("fallback_or_skill_switch_detected");
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
if (/(\/skill:|skill\s+name\s+['"]?[a-z0-9._-]+)/i.test(lower)) {
|
|
302
|
+
labels.add("skill_reference_detected");
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
return [...labels];
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
function isNegativeSignal(signals: string[]): boolean {
|
|
309
|
+
return signals.includes("negative_phrase_detected") || signals.includes("skill_retry_detected") || signals.includes("fallback_or_skill_switch_detected");
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
function makeMessageCapture(
|
|
313
|
+
text: unknown,
|
|
314
|
+
role: MessageCapture["role"],
|
|
315
|
+
metadata: unknown,
|
|
316
|
+
redactKeys: Set<string>,
|
|
317
|
+
captureContent: boolean,
|
|
318
|
+
signalLabels: string[] = [],
|
|
319
|
+
): MessageCapture | null {
|
|
320
|
+
if (typeof text !== "string") return null;
|
|
321
|
+
|
|
322
|
+
const safeText = normalizeText(text, redactKeys);
|
|
323
|
+
const snap: MessageCapture = {
|
|
324
|
+
ts: new Date().toISOString(),
|
|
325
|
+
role,
|
|
326
|
+
length: text.length,
|
|
327
|
+
signalLabels: signalLabels.length ? [...new Set(signalLabels)] : undefined,
|
|
328
|
+
};
|
|
329
|
+
|
|
330
|
+
if (captureContent) {
|
|
331
|
+
snap.text = safeText;
|
|
332
|
+
if (metadata !== undefined) {
|
|
333
|
+
snap.metadata = normalizeText(metadata, redactKeys);
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
return snap;
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
function inferSkillName(skillPath: string): string {
|
|
341
|
+
const resolved = resolve(skillPath);
|
|
342
|
+
if (basename(resolved).toLowerCase() === "skill.md") {
|
|
343
|
+
return basename(dirname(resolved));
|
|
344
|
+
}
|
|
345
|
+
return basename(resolved);
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
function inferSkillSource(skillPath: string): string {
|
|
349
|
+
const abs = resolve(skillPath);
|
|
350
|
+
const home = resolve(process.env.HOME || process.env.USERPROFILE || "");
|
|
351
|
+
if (abs.includes(`${home}/.openclaw/extensions`) || abs.includes(`${home}/.openclaw/extensions/`)) return "extension";
|
|
352
|
+
if (abs.includes(`${home}/.openclaw/skills`) || abs.includes(`${home}/.openclaw/skills/`)) return "bundled";
|
|
353
|
+
if (abs.includes("/skills/") || abs.includes("\\skills\\")) return "workspace";
|
|
354
|
+
return "unknown";
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
function extractSkillPathFromParams(params: Record<string, unknown>): string | undefined {
|
|
358
|
+
const candidates = [
|
|
359
|
+
params.path,
|
|
360
|
+
params.file_path,
|
|
361
|
+
params.filePath,
|
|
362
|
+
params.target,
|
|
363
|
+
params.targetPath,
|
|
364
|
+
];
|
|
365
|
+
|
|
366
|
+
for (const candidate of candidates) {
|
|
367
|
+
const p = toStringLike(candidate);
|
|
368
|
+
if (!p) continue;
|
|
369
|
+
if (basename(p).toLowerCase() === "skill.md") return p;
|
|
370
|
+
}
|
|
371
|
+
return undefined;
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
function buildBase(event: { sessionId?: string; runId?: string } | undefined, ctx: RawContext | undefined) {
|
|
375
|
+
return {
|
|
376
|
+
sessionId: toStringLike(event?.sessionId) || toStringLike(ctx?.sessionId) || undefined,
|
|
377
|
+
runId: toStringLike(event?.runId) || toStringLike(ctx?.runId) || undefined,
|
|
378
|
+
sessionKey: toStringLike(ctx?.sessionKey) || undefined,
|
|
379
|
+
agentId: toStringLike(ctx?.agentId) || undefined,
|
|
380
|
+
channelId: toStringLike(ctx?.channelId) || undefined,
|
|
381
|
+
messageProvider: toStringLike(ctx?.messageProvider) || undefined,
|
|
382
|
+
trigger: toStringLike(ctx?.trigger) || undefined,
|
|
383
|
+
};
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
function buildScopeKeys(ctx: RawContext | undefined, event: RawEvent | undefined): string[] {
|
|
387
|
+
const keys: string[] = [];
|
|
388
|
+
const add = (key: string) => {
|
|
389
|
+
if (key && !keys.includes(key)) keys.push(key);
|
|
390
|
+
};
|
|
391
|
+
|
|
392
|
+
const metadata = (event?.metadata as RawEvent) || {};
|
|
393
|
+
const channel = toStringLike(ctx?.channelId) || toStringLike(metadata.channelId) || "unknown";
|
|
394
|
+
const conv =
|
|
395
|
+
toStringLike(ctx?.conversationId) ||
|
|
396
|
+
toStringLike(metadata.conversationId) ||
|
|
397
|
+
toStringLike((metadata as RawEvent).threadTs) ||
|
|
398
|
+
toStringLike((metadata as RawEvent).threadId);
|
|
399
|
+
const account = toStringLike(ctx?.accountId) || toStringLike(metadata.accountId);
|
|
400
|
+
const sessionKey = toStringLike(ctx?.sessionKey) || toStringLike(metadata.sessionKey);
|
|
401
|
+
const sessionId = toStringLike(ctx?.sessionId);
|
|
402
|
+
const runId = toStringLike(ctx?.runId) || toStringLike(metadata.runId) || toStringLike(event?.runId);
|
|
403
|
+
|
|
404
|
+
if (sessionKey) add(`sk:${sessionKey}`);
|
|
405
|
+
if (runId) add(`run:${runId}`);
|
|
406
|
+
if (sessionId) add(`sid:${sessionId}`);
|
|
407
|
+
if (conv) {
|
|
408
|
+
add(`conv:${channel}:${conv}`);
|
|
409
|
+
if (account) add(`conv:${channel}:${account}:${conv}`);
|
|
410
|
+
}
|
|
411
|
+
if (account) add(`acct:${channel}:${account}`);
|
|
412
|
+
if (channel) add(`ch:${channel}`);
|
|
413
|
+
|
|
414
|
+
if (!keys.length) add("global");
|
|
415
|
+
|
|
416
|
+
return keys;
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
function buildMessageScope(ctx: RawContext | undefined, event: RawEvent | undefined): string {
|
|
420
|
+
const scopeCandidates = buildScopeKeys(ctx, event);
|
|
421
|
+
return scopeCandidates.find((key) => key.startsWith("conv:")) || scopeCandidates.find((key) => key.startsWith("acct:")) || scopeCandidates[0];
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
function createPreparedStatements(db: SqliteBackend): DbPrepared {
|
|
425
|
+
const insertEvent = db.prepare(`
|
|
426
|
+
INSERT INTO skill_events (
|
|
427
|
+
ts, type, session_id, session_key, run_id, agent_id, channel_id, message_provider,
|
|
428
|
+
tool_name, tool_call_id, params, duration_ms, success, error,
|
|
429
|
+
skill_name, skill_path, skill_source,
|
|
430
|
+
skill_block_count, skill_block_names, skill_block_locations
|
|
431
|
+
) VALUES (
|
|
432
|
+
@ts, @type, @session_id, @session_key, @run_id, @agent_id, @channel_id, @message_provider,
|
|
433
|
+
@tool_name, @tool_call_id, @params, @duration_ms, @success, @error,
|
|
434
|
+
@skill_name, @skill_path, @skill_source,
|
|
435
|
+
@skill_block_count, @skill_block_names, @skill_block_locations
|
|
436
|
+
)
|
|
437
|
+
`);
|
|
438
|
+
|
|
439
|
+
const insertVersion = db.prepare(`
|
|
440
|
+
INSERT INTO skill_versions (skill_name, skill_path, version_hash, first_seen_at, notes)
|
|
441
|
+
VALUES (@skill_name, @skill_path, @version_hash, @first_seen_at, @notes)
|
|
442
|
+
`);
|
|
443
|
+
|
|
444
|
+
const upsertSkill = db.prepare(`
|
|
445
|
+
INSERT INTO skills (
|
|
446
|
+
skill_name,
|
|
447
|
+
skill_path,
|
|
448
|
+
current_version_hash,
|
|
449
|
+
status,
|
|
450
|
+
last_modified_at,
|
|
451
|
+
last_used_at,
|
|
452
|
+
total_executions
|
|
453
|
+
) VALUES (
|
|
454
|
+
@skill_name,
|
|
455
|
+
@skill_path,
|
|
456
|
+
@current_version_hash,
|
|
457
|
+
COALESCE(@status, 'stable'),
|
|
458
|
+
@last_modified_at,
|
|
459
|
+
@last_used_at,
|
|
460
|
+
1
|
|
461
|
+
)
|
|
462
|
+
ON CONFLICT(skill_name) DO UPDATE SET
|
|
463
|
+
skill_path = excluded.skill_path,
|
|
464
|
+
current_version_hash = excluded.current_version_hash,
|
|
465
|
+
status = COALESCE(skills.status, excluded.status),
|
|
466
|
+
last_modified_at = excluded.last_modified_at,
|
|
467
|
+
last_used_at = excluded.last_used_at,
|
|
468
|
+
total_executions = COALESCE(skills.total_executions, 0) + 1
|
|
469
|
+
`);
|
|
470
|
+
|
|
471
|
+
const getLatestSkillVersion = db.prepare(`
|
|
472
|
+
SELECT version_hash
|
|
473
|
+
FROM skill_versions
|
|
474
|
+
WHERE skill_name = @skill_name AND skill_path = @skill_path
|
|
475
|
+
ORDER BY id DESC
|
|
476
|
+
LIMIT 1
|
|
477
|
+
`);
|
|
478
|
+
|
|
479
|
+
const getExactSkillVersion = db.prepare(`
|
|
480
|
+
SELECT version_hash
|
|
481
|
+
FROM skill_versions
|
|
482
|
+
WHERE skill_name = @skill_name AND skill_path = @skill_path AND version_hash = @version_hash
|
|
483
|
+
LIMIT 1
|
|
484
|
+
`);
|
|
485
|
+
|
|
486
|
+
const insertExecution = db.prepare(`
|
|
487
|
+
INSERT INTO skill_executions (
|
|
488
|
+
ts,
|
|
489
|
+
session_key,
|
|
490
|
+
run_id,
|
|
491
|
+
skill_name,
|
|
492
|
+
skill_path,
|
|
493
|
+
version_hash,
|
|
494
|
+
intent_context,
|
|
495
|
+
mechanical_success,
|
|
496
|
+
semantic_outcome,
|
|
497
|
+
followup_messages,
|
|
498
|
+
implied_outcome,
|
|
499
|
+
error,
|
|
500
|
+
duration_ms
|
|
501
|
+
) VALUES (
|
|
502
|
+
@ts,
|
|
503
|
+
@session_key,
|
|
504
|
+
@run_id,
|
|
505
|
+
@skill_name,
|
|
506
|
+
@skill_path,
|
|
507
|
+
@version_hash,
|
|
508
|
+
@intent_context,
|
|
509
|
+
@mechanical_success,
|
|
510
|
+
@semantic_outcome,
|
|
511
|
+
@followup_messages,
|
|
512
|
+
@implied_outcome,
|
|
513
|
+
@error,
|
|
514
|
+
@duration_ms
|
|
515
|
+
)
|
|
516
|
+
`);
|
|
517
|
+
|
|
518
|
+
const insertFeedback = db.prepare(`
|
|
519
|
+
INSERT INTO skill_feedback (execution_id, source, label, notes)
|
|
520
|
+
VALUES (@execution_id, @source, @label, @notes)
|
|
521
|
+
`);
|
|
522
|
+
|
|
523
|
+
return {
|
|
524
|
+
insertEvent: (params) => {
|
|
525
|
+
try {
|
|
526
|
+
insertEvent.run(params);
|
|
527
|
+
} catch {
|
|
528
|
+
// ignore duplicate event shape issues; keep write path non-blocking
|
|
529
|
+
}
|
|
530
|
+
},
|
|
531
|
+
insertVersion: (params) => {
|
|
532
|
+
try {
|
|
533
|
+
const existing = getExactSkillVersion.get(params) as { version_hash?: string } | undefined;
|
|
534
|
+
if (!existing?.version_hash) {
|
|
535
|
+
insertVersion.run(params);
|
|
536
|
+
}
|
|
537
|
+
} catch {
|
|
538
|
+
// best effort
|
|
539
|
+
}
|
|
540
|
+
},
|
|
541
|
+
upsertSkill: (params) => {
|
|
542
|
+
try {
|
|
543
|
+
upsertSkill.run(params);
|
|
544
|
+
} catch {
|
|
545
|
+
// best effort
|
|
546
|
+
}
|
|
547
|
+
},
|
|
548
|
+
getLatestSkillVersion: (params) => getLatestSkillVersion.get(params) as { version_hash?: string } | undefined,
|
|
549
|
+
insertExecution: (params) => insertExecution.run(params),
|
|
550
|
+
insertFeedback: (params) => insertFeedback.run(params),
|
|
551
|
+
};
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
async function initSqlite(path: string, log: { info: (msg: string) => void; error: (msg: string) => void; }) {
|
|
555
|
+
await mkdir(dirname(path), { recursive: true });
|
|
556
|
+
|
|
557
|
+
let backend: SqliteBackend | null = null;
|
|
558
|
+
|
|
559
|
+
try {
|
|
560
|
+
const sqlite3 = require("better-sqlite3");
|
|
561
|
+
const BetterSqlite3 = sqlite3?.default || sqlite3;
|
|
562
|
+
if (typeof BetterSqlite3 === "function") {
|
|
563
|
+
const db = new BetterSqlite3(path);
|
|
564
|
+
db.pragma("journal_mode = WAL");
|
|
565
|
+
db.pragma("foreign_keys = ON");
|
|
566
|
+
backend = {
|
|
567
|
+
kind: "better-sqlite3",
|
|
568
|
+
close: () => db.close(),
|
|
569
|
+
exec: (sql) => db.exec(sql),
|
|
570
|
+
prepare: (sql) => {
|
|
571
|
+
const stmt = db.prepare(sql);
|
|
572
|
+
return {
|
|
573
|
+
run: (params) => stmt.run(params),
|
|
574
|
+
get: (params) => stmt.get(params) as Record<string, unknown> | undefined,
|
|
575
|
+
};
|
|
576
|
+
},
|
|
577
|
+
};
|
|
578
|
+
}
|
|
579
|
+
} catch {
|
|
580
|
+
// fallback below
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
if (!backend) {
|
|
584
|
+
try {
|
|
585
|
+
const sqlite = require("node:sqlite");
|
|
586
|
+
const DatabaseSync = sqlite.DatabaseSync;
|
|
587
|
+
if (typeof DatabaseSync === "function") {
|
|
588
|
+
const db = new DatabaseSync(path);
|
|
589
|
+
db.exec("PRAGMA journal_mode = WAL;");
|
|
590
|
+
db.exec("PRAGMA foreign_keys = ON;");
|
|
591
|
+
backend = {
|
|
592
|
+
kind: "node:sqlite",
|
|
593
|
+
close: () => db.close(),
|
|
594
|
+
exec: (sql) => db.exec(sql),
|
|
595
|
+
prepare: (sql) => {
|
|
596
|
+
const stmt = db.prepare(sql);
|
|
597
|
+
return {
|
|
598
|
+
run: (params) => stmt.run(params),
|
|
599
|
+
get: (params) => stmt.get(params) as Record<string, unknown> | undefined,
|
|
600
|
+
};
|
|
601
|
+
},
|
|
602
|
+
} as SqliteBackend;
|
|
603
|
+
}
|
|
604
|
+
} catch {
|
|
605
|
+
// none
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
if (!backend) {
|
|
610
|
+
return {
|
|
611
|
+
backend: null,
|
|
612
|
+
statements: null,
|
|
613
|
+
error: "No sqlite backend available; plugin will continue without sqlite writes",
|
|
614
|
+
} as DbState;
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
backend.exec(`
|
|
618
|
+
CREATE TABLE IF NOT EXISTS skill_events (
|
|
619
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
620
|
+
ts TEXT NOT NULL,
|
|
621
|
+
type TEXT NOT NULL,
|
|
622
|
+
session_id TEXT,
|
|
623
|
+
session_key TEXT,
|
|
624
|
+
run_id TEXT,
|
|
625
|
+
agent_id TEXT,
|
|
626
|
+
channel_id TEXT,
|
|
627
|
+
message_provider TEXT,
|
|
628
|
+
tool_name TEXT,
|
|
629
|
+
tool_call_id TEXT,
|
|
630
|
+
params TEXT,
|
|
631
|
+
duration_ms INTEGER,
|
|
632
|
+
success INTEGER,
|
|
633
|
+
error TEXT,
|
|
634
|
+
skill_name TEXT,
|
|
635
|
+
skill_path TEXT,
|
|
636
|
+
skill_source TEXT,
|
|
637
|
+
skill_block_count INTEGER,
|
|
638
|
+
skill_block_names TEXT,
|
|
639
|
+
skill_block_locations TEXT
|
|
640
|
+
);
|
|
641
|
+
|
|
642
|
+
CREATE TABLE IF NOT EXISTS skill_versions (
|
|
643
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
644
|
+
skill_name TEXT NOT NULL,
|
|
645
|
+
skill_path TEXT NOT NULL,
|
|
646
|
+
version_hash TEXT NOT NULL,
|
|
647
|
+
first_seen_at TEXT NOT NULL,
|
|
648
|
+
notes TEXT,
|
|
649
|
+
UNIQUE(skill_name, skill_path, version_hash)
|
|
650
|
+
);
|
|
651
|
+
|
|
652
|
+
CREATE TABLE IF NOT EXISTS skills (
|
|
653
|
+
skill_name TEXT PRIMARY KEY,
|
|
654
|
+
skill_path TEXT NOT NULL,
|
|
655
|
+
current_version_hash TEXT,
|
|
656
|
+
status TEXT DEFAULT 'stable',
|
|
657
|
+
last_modified_at TEXT,
|
|
658
|
+
last_used_at TEXT,
|
|
659
|
+
total_executions INTEGER DEFAULT 0
|
|
660
|
+
);
|
|
661
|
+
|
|
662
|
+
CREATE TABLE IF NOT EXISTS skill_executions (
|
|
663
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
664
|
+
ts TEXT NOT NULL,
|
|
665
|
+
session_key TEXT,
|
|
666
|
+
run_id TEXT,
|
|
667
|
+
skill_name TEXT NOT NULL,
|
|
668
|
+
skill_path TEXT NOT NULL,
|
|
669
|
+
version_hash TEXT,
|
|
670
|
+
intent_context TEXT,
|
|
671
|
+
mechanical_success INTEGER,
|
|
672
|
+
semantic_outcome TEXT,
|
|
673
|
+
followup_messages TEXT,
|
|
674
|
+
implied_outcome TEXT,
|
|
675
|
+
error TEXT,
|
|
676
|
+
duration_ms INTEGER,
|
|
677
|
+
created_at TEXT DEFAULT CURRENT_TIMESTAMP
|
|
678
|
+
);
|
|
679
|
+
|
|
680
|
+
CREATE TABLE IF NOT EXISTS skill_feedback (
|
|
681
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
682
|
+
execution_id INTEGER REFERENCES skill_executions(id),
|
|
683
|
+
source TEXT,
|
|
684
|
+
label TEXT,
|
|
685
|
+
notes TEXT,
|
|
686
|
+
created_at TEXT DEFAULT CURRENT_TIMESTAMP
|
|
687
|
+
);
|
|
688
|
+
|
|
689
|
+
CREATE TABLE IF NOT EXISTS skill_health_snapshots (
|
|
690
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
691
|
+
ts TEXT NOT NULL,
|
|
692
|
+
skill_name TEXT NOT NULL,
|
|
693
|
+
version_hash TEXT,
|
|
694
|
+
usage_count INTEGER DEFAULT 0,
|
|
695
|
+
mechanical_failure_rate REAL DEFAULT 0,
|
|
696
|
+
implied_negative_rate REAL DEFAULT 0,
|
|
697
|
+
status_recommendation TEXT,
|
|
698
|
+
notes TEXT,
|
|
699
|
+
created_at TEXT DEFAULT CURRENT_TIMESTAMP
|
|
700
|
+
);
|
|
701
|
+
`);
|
|
702
|
+
|
|
703
|
+
log.info(`skill-usage-audit: sqlite initialized at ${path} (${backend.kind})`);
|
|
704
|
+
return {
|
|
705
|
+
backend,
|
|
706
|
+
statements: createPreparedStatements(backend),
|
|
707
|
+
} as DbState;
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
async function computeSkillVersionHash(skillPath: string): Promise<string | null> {
|
|
711
|
+
const dir = resolve(dirname(skillPath));
|
|
712
|
+
const scriptDir = join(dir, "scripts");
|
|
713
|
+
const files: string[] = [skillPath];
|
|
714
|
+
try {
|
|
715
|
+
const statDir = await stat(scriptDir);
|
|
716
|
+
if (statDir.isDirectory()) {
|
|
717
|
+
const scriptEntries = await readdir(scriptDir, { withFileTypes: true });
|
|
718
|
+
for (const entry of scriptEntries.sort((a, b) => a.name.localeCompare(b.name))) {
|
|
719
|
+
if (entry.isFile()) files.push(join(scriptDir, entry.name));
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
} catch {
|
|
723
|
+
// no scripts
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
if (!files.length) return null;
|
|
727
|
+
|
|
728
|
+
const hash = createHash("sha256");
|
|
729
|
+
let hasBytes = false;
|
|
730
|
+
for (const p of files) {
|
|
731
|
+
try {
|
|
732
|
+
const data = await readFile(p);
|
|
733
|
+
hash.update(relative(dir, p));
|
|
734
|
+
hash.update("\0");
|
|
735
|
+
hash.update(data);
|
|
736
|
+
hash.update("\0");
|
|
737
|
+
hasBytes = true;
|
|
738
|
+
} catch {
|
|
739
|
+
// skip unreadable
|
|
740
|
+
}
|
|
741
|
+
}
|
|
742
|
+
|
|
743
|
+
if (!hasBytes) return null;
|
|
744
|
+
return hash.digest("hex");
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
export default function register(api: OpenClawPluginApi) {
|
|
748
|
+
const log = api.logger;
|
|
749
|
+
const cfg = (api.pluginConfig as PluginConfig) || {};
|
|
750
|
+
|
|
751
|
+
const includeToolParams = cfg.includeToolParams ?? DEFAULT_INCLUDE_TOOL_PARAMS;
|
|
752
|
+
const captureMessageContent = cfg.captureMessageContent === undefined ? DEFAULT_CAPTURE_MESSAGE_CONTENT : cfg.captureMessageContent === true;
|
|
753
|
+
const redactKeys = new Set((Array.isArray(cfg.redactKeys) ? cfg.redactKeys : DEFAULT_REDACT_KEYS).map((k) => String(k).toLowerCase()));
|
|
754
|
+
const contextWindowSize = parseIntConfig(cfg.contextWindowSize, DEFAULT_CONTEXT_WINDOW_SIZE, 1);
|
|
755
|
+
const contextTimeoutMs = parseIntConfig(cfg.contextTimeoutMs, DEFAULT_CONTEXT_TIMEOUT_MS, 0);
|
|
756
|
+
const detectSkillBlocks = cfg.skillBlockDetection !== false;
|
|
757
|
+
const dbPath = resolveDbPath(cfg.dbPath);
|
|
758
|
+
|
|
759
|
+
const dbState: DbState = { backend: null, statements: null };
|
|
760
|
+
const dbInitPromise = initSqlite(dbPath, log);
|
|
761
|
+
let dbChain = Promise.resolve<void>(undefined);
|
|
762
|
+
let hasLoggedDbIssue = false;
|
|
763
|
+
|
|
764
|
+
let shutdownInProgress = false;
|
|
765
|
+
let shutdownPromise: Promise<void> | null = null;
|
|
766
|
+
|
|
767
|
+
const messageHistory = new Map<string, MessageCapture[]>();
|
|
768
|
+
const executionsById = new Map<number, SkillExecutionState>();
|
|
769
|
+
const execByScope = new Map<string, SkillExecutionState[]>();
|
|
770
|
+
const execByTool = new Map<string, number>();
|
|
771
|
+
let executionSeq = 0;
|
|
772
|
+
|
|
773
|
+
async function ensureDbReady(): Promise<DbState> {
|
|
774
|
+
if (dbState.backend || dbState.statements || dbState.error) {
|
|
775
|
+
return dbState;
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
const state = await dbInitPromise;
|
|
779
|
+
if (!dbState.statements) {
|
|
780
|
+
dbState.backend = state.backend;
|
|
781
|
+
dbState.statements = state.statements;
|
|
782
|
+
dbState.error = state.error;
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
if (state.error && !hasLoggedDbIssue) {
|
|
786
|
+
hasLoggedDbIssue = true;
|
|
787
|
+
log.info(`skill-usage-audit: sqlite unavailable: ${state.error}`);
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
return dbState;
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
function queueDbInsert(rowType: RawEvent): void {
|
|
794
|
+
if (rowType.type !== "session_start" && rowType.type !== "session_end" && rowType.type !== "tool_call_start" && rowType.type !== "tool_call_end" && rowType.type !== "skill_file_read" && rowType.type !== "skill_block_detected") {
|
|
795
|
+
return;
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
dbChain = dbChain
|
|
799
|
+
.then(async () => {
|
|
800
|
+
const state = await ensureDbReady();
|
|
801
|
+
if (!state.statements) return;
|
|
802
|
+
|
|
803
|
+
const row = {
|
|
804
|
+
ts: toStringLike(rowType.ts) || new Date().toISOString(),
|
|
805
|
+
type: rowType.type,
|
|
806
|
+
session_id: toStringLike(rowType.sessionId) || null,
|
|
807
|
+
session_key: toStringLike(rowType.sessionKey) || null,
|
|
808
|
+
run_id: toStringLike(rowType.runId) || null,
|
|
809
|
+
agent_id: toStringLike(rowType.agentId) || null,
|
|
810
|
+
channel_id: toStringLike(rowType.channelId) || null,
|
|
811
|
+
message_provider: toStringLike(rowType.messageProvider) || null,
|
|
812
|
+
tool_name: toStringLike(rowType.toolName) || null,
|
|
813
|
+
tool_call_id: toStringLike(rowType.toolCallId) || null,
|
|
814
|
+
params: rowType.params ? JSON.stringify(rowType.params) : null,
|
|
815
|
+
duration_ms: typeof rowType.durationMs === "number" ? Math.max(0, Math.floor(rowType.durationMs)) : null,
|
|
816
|
+
success: typeof rowType.success === "boolean" ? (rowType.success ? 1 : 0) : null,
|
|
817
|
+
error: toStringLike(rowType.error) || null,
|
|
818
|
+
skill_name: toStringLike(rowType.skillName) || null,
|
|
819
|
+
skill_path: toStringLike(rowType.skillPath) || null,
|
|
820
|
+
skill_source: toStringLike(rowType.skillSource) || null,
|
|
821
|
+
skill_block_count: typeof rowType.skillBlockCount === "number" && Number.isFinite(rowType.skillBlockCount)
|
|
822
|
+
? Math.max(0, Math.floor(rowType.skillBlockCount))
|
|
823
|
+
: null,
|
|
824
|
+
skill_block_names: Array.isArray(rowType.skillBlockNames) ? JSON.stringify(rowType.skillBlockNames) : null,
|
|
825
|
+
skill_block_locations: Array.isArray(rowType.skillBlockLocations) ? JSON.stringify(rowType.skillBlockLocations) : null,
|
|
826
|
+
};
|
|
827
|
+
|
|
828
|
+
state.statements?.insertEvent(row);
|
|
829
|
+
})
|
|
830
|
+
.catch((err) => {
|
|
831
|
+
log.error(`skill-usage-audit: db write failed: ${String(err)}`);
|
|
832
|
+
});
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
function queueSkillVersionWrite(skillName: string, skillPath: string, ts: string, versionHash: string | null | undefined): void {
|
|
836
|
+
dbChain = dbChain
|
|
837
|
+
.then(async () => {
|
|
838
|
+
const state = await ensureDbReady();
|
|
839
|
+
if (!state.statements) return;
|
|
840
|
+
|
|
841
|
+
const hash = versionHash ?? (await computeSkillVersionHash(skillPath));
|
|
842
|
+
if (!hash) return;
|
|
843
|
+
|
|
844
|
+
state.statements.insertVersion({
|
|
845
|
+
skill_name: skillName,
|
|
846
|
+
skill_path: resolve(skillPath),
|
|
847
|
+
version_hash: hash,
|
|
848
|
+
first_seen_at: ts,
|
|
849
|
+
notes: null,
|
|
850
|
+
});
|
|
851
|
+
|
|
852
|
+
state.statements.upsertSkill({
|
|
853
|
+
skill_name: skillName,
|
|
854
|
+
skill_path: resolve(skillPath),
|
|
855
|
+
current_version_hash: hash,
|
|
856
|
+
status: "stable",
|
|
857
|
+
last_modified_at: ts,
|
|
858
|
+
last_used_at: ts,
|
|
859
|
+
});
|
|
860
|
+
})
|
|
861
|
+
.catch((err) => {
|
|
862
|
+
log.error(`skill-usage-audit: failed writing skill version: ${String(err)}`);
|
|
863
|
+
});
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
function enqueueEvent(event: RawEvent): void {
|
|
867
|
+
queueDbInsert(event);
|
|
868
|
+
}
|
|
869
|
+
|
|
870
|
+
function addMessage(scope: string, message: MessageCapture): void {
|
|
871
|
+
const list = messageHistory.get(scope) || [];
|
|
872
|
+
list.push(message);
|
|
873
|
+
if (list.length > MAX_HISTORY_PER_SCOPE) {
|
|
874
|
+
list.splice(0, list.length - MAX_HISTORY_PER_SCOPE);
|
|
875
|
+
}
|
|
876
|
+
messageHistory.set(scope, list);
|
|
877
|
+
}
|
|
878
|
+
|
|
879
|
+
function getRecentMessages(scopeKeys: string[], windowSize: number): MessageCapture[] {
|
|
880
|
+
const all: MessageCapture[] = [];
|
|
881
|
+
for (const key of scopeKeys) {
|
|
882
|
+
const list = messageHistory.get(key) || [];
|
|
883
|
+
all.push(...list);
|
|
884
|
+
}
|
|
885
|
+
return all.sort((a, b) => (a.ts < b.ts ? -1 : a.ts > b.ts ? 1 : 0)).slice(-windowSize);
|
|
886
|
+
}
|
|
887
|
+
|
|
888
|
+
function trackExecution(execution: SkillExecutionState): void {
|
|
889
|
+
for (const key of execution.scopeKeys) {
|
|
890
|
+
const list = execByScope.get(key) || [];
|
|
891
|
+
if (!list.includes(execution)) list.push(execution);
|
|
892
|
+
execByScope.set(key, list);
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
|
|
896
|
+
function untrackExecution(execution: SkillExecutionState): void {
|
|
897
|
+
for (const key of execution.scopeKeys) {
|
|
898
|
+
const list = execByScope.get(key);
|
|
899
|
+
if (!list) continue;
|
|
900
|
+
const next = list.filter((entry) => entry.id !== execution.id);
|
|
901
|
+
if (next.length) execByScope.set(key, next);
|
|
902
|
+
else execByScope.delete(key);
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
for (const [toolId, id] of execByTool.entries()) {
|
|
906
|
+
if (id === execution.id) {
|
|
907
|
+
execByTool.delete(toolId);
|
|
908
|
+
}
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
executionsById.delete(execution.id);
|
|
912
|
+
}
|
|
913
|
+
|
|
914
|
+
function candidatesForScope(keys: string[]): SkillExecutionState[] {
|
|
915
|
+
const set = new Map<number, SkillExecutionState>();
|
|
916
|
+
for (const key of keys) {
|
|
917
|
+
const list = execByScope.get(key) || [];
|
|
918
|
+
for (const execution of list) {
|
|
919
|
+
if (!execution.finalized) set.set(execution.id, execution);
|
|
920
|
+
}
|
|
921
|
+
}
|
|
922
|
+
return [...set.values()].sort((a, b) => b.startAt - a.startAt);
|
|
923
|
+
}
|
|
924
|
+
|
|
925
|
+
function pickExecution(keys: string[], requireFollowup = false): SkillExecutionState | undefined {
|
|
926
|
+
const list = candidatesForScope(keys);
|
|
927
|
+
if (!list.length) return undefined;
|
|
928
|
+
if (requireFollowup) {
|
|
929
|
+
return list.find((execution) => execution.inFollowup) || list[0];
|
|
930
|
+
}
|
|
931
|
+
return list[0];
|
|
932
|
+
}
|
|
933
|
+
|
|
934
|
+
function markFollowup(execution: SkillExecutionState): void {
|
|
935
|
+
if (execution.inFollowup) return;
|
|
936
|
+
|
|
937
|
+
execution.inFollowup = true;
|
|
938
|
+
execution.followupStartedAt = Date.now();
|
|
939
|
+
|
|
940
|
+
if (contextTimeoutMs <= 0) {
|
|
941
|
+
finalizeExecution(execution.id, "followup-timeout");
|
|
942
|
+
return;
|
|
943
|
+
}
|
|
944
|
+
|
|
945
|
+
execution.followupTimer = setTimeout(() => {
|
|
946
|
+
finalizeExecution(execution.id, "followup-timeout");
|
|
947
|
+
}, contextTimeoutMs);
|
|
948
|
+
}
|
|
949
|
+
|
|
950
|
+
function stopFollowup(execution: SkillExecutionState): void {
|
|
951
|
+
if (!execution.inFollowup) return;
|
|
952
|
+
execution.inFollowup = false;
|
|
953
|
+
execution.followupStartedAt = undefined;
|
|
954
|
+
if (execution.followupTimer) clearTimeout(execution.followupTimer);
|
|
955
|
+
execution.followupTimer = undefined;
|
|
956
|
+
execution.followupMessages = [];
|
|
957
|
+
}
|
|
958
|
+
|
|
959
|
+
function isNegativeMessage(execution: SkillExecutionState, text: string | undefined): boolean {
|
|
960
|
+
return isNegativeSignal(detectMessageSignals(execution, text || ""));
|
|
961
|
+
}
|
|
962
|
+
|
|
963
|
+
function determineOutcome(execution: SkillExecutionState): ImpliedOutcome {
|
|
964
|
+
if (execution.fallbackSkillRetried || execution.sameSkillRetried) return "negative";
|
|
965
|
+
if (execution.followupMessages.some((m) => isNegativeMessage(execution, m.text))) return "negative";
|
|
966
|
+
if (execution.followupMessages.some((m) => isNegativeSignal(m.signalLabels || []))) return "negative";
|
|
967
|
+
if (execution.hadToolCall && execution.mechanicalSuccess) return "positive";
|
|
968
|
+
return "unclear";
|
|
969
|
+
}
|
|
970
|
+
|
|
971
|
+
function finalizeExecution(executionId: number, reason: string): void {
|
|
972
|
+
const execution = executionsById.get(executionId);
|
|
973
|
+
if (!execution || execution.finalized) return;
|
|
974
|
+
|
|
975
|
+
execution.finalized = true;
|
|
976
|
+
if (execution.followupTimer) clearTimeout(execution.followupTimer);
|
|
977
|
+
execution.followupTimer = undefined;
|
|
978
|
+
|
|
979
|
+
untrackExecution(execution);
|
|
980
|
+
|
|
981
|
+
const durationMs = Math.max(0, Date.now() - execution.startAt);
|
|
982
|
+
const outcome = determineOutcome(execution);
|
|
983
|
+
|
|
984
|
+
dbChain = dbChain
|
|
985
|
+
.then(async () => {
|
|
986
|
+
const state = await ensureDbReady();
|
|
987
|
+
if (!state.statements) return;
|
|
988
|
+
|
|
989
|
+
let versionHash = execution.versionHash ?? null;
|
|
990
|
+
if (!versionHash && execution.versionHashPromise) {
|
|
991
|
+
try {
|
|
992
|
+
versionHash = await execution.versionHashPromise;
|
|
993
|
+
} catch {
|
|
994
|
+
versionHash = null;
|
|
995
|
+
}
|
|
996
|
+
}
|
|
997
|
+
|
|
998
|
+
if (!versionHash) {
|
|
999
|
+
const row = state.statements.getLatestSkillVersion({
|
|
1000
|
+
skill_name: execution.skillName,
|
|
1001
|
+
skill_path: execution.skillPath,
|
|
1002
|
+
});
|
|
1003
|
+
if (row?.version_hash) versionHash = String(row.version_hash);
|
|
1004
|
+
}
|
|
1005
|
+
|
|
1006
|
+
const mechanicalSuccess = execution.hadToolCall ? (execution.mechanicalSuccess ? 1 : 0) : null;
|
|
1007
|
+
state.statements.insertExecution({
|
|
1008
|
+
ts: execution.ts,
|
|
1009
|
+
session_key: execution.sessionKey || null,
|
|
1010
|
+
run_id: execution.runId || null,
|
|
1011
|
+
skill_name: execution.skillName,
|
|
1012
|
+
skill_path: execution.skillPath,
|
|
1013
|
+
version_hash: versionHash || null,
|
|
1014
|
+
intent_context: JSON.stringify(execution.intentContext),
|
|
1015
|
+
mechanical_success: mechanicalSuccess,
|
|
1016
|
+
semantic_outcome: "unclear",
|
|
1017
|
+
followup_messages: JSON.stringify(execution.followupMessages),
|
|
1018
|
+
implied_outcome: outcome,
|
|
1019
|
+
error: execution.error || null,
|
|
1020
|
+
duration_ms: durationMs,
|
|
1021
|
+
});
|
|
1022
|
+
})
|
|
1023
|
+
.catch((err) => {
|
|
1024
|
+
log.error(`skill-usage-audit: failed inserting execution ${executionId}: ${String(err)} (${reason})`);
|
|
1025
|
+
});
|
|
1026
|
+
}
|
|
1027
|
+
|
|
1028
|
+
function normalizeSkillExecutionPath(rawPath: string | undefined): string | undefined {
|
|
1029
|
+
if (!rawPath) return undefined;
|
|
1030
|
+
const trimmed = rawPath.trim();
|
|
1031
|
+
if (!trimmed) return undefined;
|
|
1032
|
+
return trimmed.startsWith("~") ? resolveHomePath(trimmed) : resolve(trimmed);
|
|
1033
|
+
}
|
|
1034
|
+
|
|
1035
|
+
function resolveSkillPathFromBlock(name: string, location: string | undefined): string | undefined {
|
|
1036
|
+
if (location) {
|
|
1037
|
+
const normalized = normalizeSkillExecutionPath(location);
|
|
1038
|
+
if (!normalized) return undefined;
|
|
1039
|
+
const lower = normalized.toLowerCase();
|
|
1040
|
+
if (lower.endsWith("skill.md")) return normalized;
|
|
1041
|
+
return `${normalized}/SKILL.md`;
|
|
1042
|
+
}
|
|
1043
|
+
|
|
1044
|
+
if (!name) return undefined;
|
|
1045
|
+
const trimmed = name.trim();
|
|
1046
|
+
if (!trimmed) return undefined;
|
|
1047
|
+
if (trimmed.startsWith("~") || trimmed.includes("/") || trimmed.endsWith(".md")) {
|
|
1048
|
+
const normalized = normalizeSkillExecutionPath(trimmed);
|
|
1049
|
+
if (normalized && normalized.toLowerCase().endsWith(".md")) return normalized;
|
|
1050
|
+
}
|
|
1051
|
+
return undefined;
|
|
1052
|
+
}
|
|
1053
|
+
|
|
1054
|
+
function findMatchingExecution(
|
|
1055
|
+
scopeKeys: string[],
|
|
1056
|
+
skillName: string,
|
|
1057
|
+
skillPath: string | undefined,
|
|
1058
|
+
runId: string | undefined,
|
|
1059
|
+
): SkillExecutionState | undefined {
|
|
1060
|
+
const candidates = candidatesForScope(scopeKeys).filter((execution) => {
|
|
1061
|
+
if (execution.skillName.toLowerCase() === skillName.toLowerCase()) return true;
|
|
1062
|
+
return !!(skillPath && execution.encounteredSkillPaths.has(skillPath));
|
|
1063
|
+
});
|
|
1064
|
+
|
|
1065
|
+
if (!candidates.length) return undefined;
|
|
1066
|
+
|
|
1067
|
+
if (runId) {
|
|
1068
|
+
const sameRun = candidates.filter((e) => e.runId === runId);
|
|
1069
|
+
if (sameRun.length) {
|
|
1070
|
+
const exact = skillPath ? sameRun.filter((e) => e.encounteredSkillPaths.has(skillPath)) : [];
|
|
1071
|
+
if (exact.length) return exact[0];
|
|
1072
|
+
return sameRun[0];
|
|
1073
|
+
}
|
|
1074
|
+
}
|
|
1075
|
+
|
|
1076
|
+
const exactPath = skillPath ? candidates.filter((e) => e.encounteredSkillPaths.has(skillPath)) : [];
|
|
1077
|
+
if (exactPath.length) return exactPath[0];
|
|
1078
|
+
|
|
1079
|
+
return candidates[0];
|
|
1080
|
+
}
|
|
1081
|
+
|
|
1082
|
+
function syncExecutionPath(execution: SkillExecutionState, skillName: string, skillPath: string | undefined): void {
|
|
1083
|
+
const normalized = normalizeSkillExecutionPath(skillPath);
|
|
1084
|
+
if (normalized) {
|
|
1085
|
+
execution.encounteredSkillPaths.add(normalized);
|
|
1086
|
+
if (execution.skillPath === execution.skillName || execution.skillPath === "") {
|
|
1087
|
+
execution.skillPath = normalized;
|
|
1088
|
+
execution.versionHashPromise = computeSkillVersionHash(normalized);
|
|
1089
|
+
execution.versionHash = null;
|
|
1090
|
+
}
|
|
1091
|
+
}
|
|
1092
|
+
|
|
1093
|
+
if (skillName && execution.skillName !== skillName) {
|
|
1094
|
+
execution.skillName = skillName;
|
|
1095
|
+
}
|
|
1096
|
+
}
|
|
1097
|
+
|
|
1098
|
+
function attachToolCall(keys: string[], event: RawEvent, execution?: SkillExecutionState): SkillExecutionState | undefined {
|
|
1099
|
+
const target = execution || pickExecution(keys);
|
|
1100
|
+
if (!target || target.finalized) return;
|
|
1101
|
+
|
|
1102
|
+
if (target.inFollowup) stopFollowup(target);
|
|
1103
|
+
|
|
1104
|
+
target.hadToolCall = true;
|
|
1105
|
+
target.toolReadCount += 1;
|
|
1106
|
+
|
|
1107
|
+
const callId = toStringLike(event.toolCallId)
|
|
1108
|
+
? `tool:${toStringLike(event.toolCallId)}`
|
|
1109
|
+
: `anon:${target.id}:${target.toolReadCount}`;
|
|
1110
|
+
target.inFlightToolCalls.add(callId);
|
|
1111
|
+
|
|
1112
|
+
execByTool.set(callId, target.id);
|
|
1113
|
+
|
|
1114
|
+
const toolName = toStringLike(event.toolName) || "";
|
|
1115
|
+
if (toolName === "read" && typeof event.params === "object") {
|
|
1116
|
+
const p = event.params as Record<string, unknown>;
|
|
1117
|
+
const rawPath = extractSkillPathFromParams(p);
|
|
1118
|
+
if (rawPath) {
|
|
1119
|
+
const inferredSkillName = inferSkillName(rawPath);
|
|
1120
|
+
syncExecutionPath(target, inferredSkillName, rawPath);
|
|
1121
|
+
|
|
1122
|
+
const normalized = normalizeSkillExecutionPath(rawPath);
|
|
1123
|
+
if (normalized) {
|
|
1124
|
+
if (normalized === target.skillPath) {
|
|
1125
|
+
if (target.toolReadCount > 1) target.sameSkillRetried = true;
|
|
1126
|
+
} else {
|
|
1127
|
+
target.fallbackSkillRetried = true;
|
|
1128
|
+
}
|
|
1129
|
+
target.encounteredSkillPaths.add(normalized);
|
|
1130
|
+
}
|
|
1131
|
+
}
|
|
1132
|
+
}
|
|
1133
|
+
|
|
1134
|
+
return target;
|
|
1135
|
+
}
|
|
1136
|
+
|
|
1137
|
+
function detachToolCall(event: RawEvent, scopeKeys: string[]): SkillExecutionState | undefined {
|
|
1138
|
+
const toolId = toStringLike(event.toolCallId);
|
|
1139
|
+
const direct = toolId ? execByTool.get(`tool:${toolId}`) : undefined;
|
|
1140
|
+
const target = direct ? executionsById.get(direct) : pickExecution(scopeKeys);
|
|
1141
|
+
if (!target || target.finalized) return;
|
|
1142
|
+
|
|
1143
|
+
const key = toolId ? `tool:${toolId}` : [...target.inFlightToolCalls].find((id) => id.startsWith(`anon:${target.id}:`));
|
|
1144
|
+
if (key) {
|
|
1145
|
+
target.inFlightToolCalls.delete(key);
|
|
1146
|
+
execByTool.delete(key);
|
|
1147
|
+
}
|
|
1148
|
+
|
|
1149
|
+
if (target.inFlightToolCalls.size === 0) {
|
|
1150
|
+
markFollowup(target);
|
|
1151
|
+
}
|
|
1152
|
+
|
|
1153
|
+
const toolName = toStringLike(event.toolName) || "";
|
|
1154
|
+
if (toolName === "read" && typeof event.params === "object" && event.error) {
|
|
1155
|
+
target.mechanicalSuccess = false;
|
|
1156
|
+
target.error = target.error || toStringLike(event.error) || "tool failure";
|
|
1157
|
+
}
|
|
1158
|
+
|
|
1159
|
+
if (event.error) {
|
|
1160
|
+
target.mechanicalSuccess = false;
|
|
1161
|
+
target.error = target.error || toStringLike(event.error) || "tool failure";
|
|
1162
|
+
}
|
|
1163
|
+
|
|
1164
|
+
return target;
|
|
1165
|
+
}
|
|
1166
|
+
|
|
1167
|
+
function onFollowupMessage(execution: SkillExecutionState | undefined, message: MessageCapture): void {
|
|
1168
|
+
if (!execution) return;
|
|
1169
|
+
|
|
1170
|
+
if (!execution.inFollowup) {
|
|
1171
|
+
execution.inFollowup = true;
|
|
1172
|
+
execution.followupStartedAt = Date.now();
|
|
1173
|
+
}
|
|
1174
|
+
execution.followupMessages.push(message);
|
|
1175
|
+
|
|
1176
|
+
if (execution.followupMessages.length >= execution.contextWindowSize) {
|
|
1177
|
+
finalizeExecution(execution.id, "followup-limit");
|
|
1178
|
+
return;
|
|
1179
|
+
}
|
|
1180
|
+
|
|
1181
|
+
if (isNegativeMessage(execution, message.text) || isNegativeSignal(message.signalLabels || [])) {
|
|
1182
|
+
finalizeExecution(execution.id, "negative-signal");
|
|
1183
|
+
return;
|
|
1184
|
+
}
|
|
1185
|
+
|
|
1186
|
+
if (contextTimeoutMs <= 0 && execution.followupMessages.length > 0) {
|
|
1187
|
+
finalizeExecution(execution.id, "no-timeout");
|
|
1188
|
+
}
|
|
1189
|
+
}
|
|
1190
|
+
|
|
1191
|
+
function startExecutionFromSkillRead(
|
|
1192
|
+
ctx: RawContext | undefined,
|
|
1193
|
+
event: RawEvent,
|
|
1194
|
+
skillPath: string,
|
|
1195
|
+
now: string,
|
|
1196
|
+
isFromSkillBlock = false,
|
|
1197
|
+
): SkillExecutionState {
|
|
1198
|
+
const initialPath = normalizeSkillExecutionPath(skillPath) || skillPath;
|
|
1199
|
+
const skillName = inferSkillName(initialPath);
|
|
1200
|
+
const scopeKeys = buildScopeKeys(ctx, event);
|
|
1201
|
+
const runId = toStringLike(event.runId) || toStringLike(ctx?.runId) || toStringLike(ctx?.sessionId);
|
|
1202
|
+
|
|
1203
|
+
const existing = findMatchingExecution(scopeKeys, skillName, initialPath, runId);
|
|
1204
|
+
if (existing) {
|
|
1205
|
+
syncExecutionPath(existing, skillName, initialPath);
|
|
1206
|
+
return existing;
|
|
1207
|
+
}
|
|
1208
|
+
|
|
1209
|
+
const intentContext = getRecentMessages(scopeKeys, contextWindowSize);
|
|
1210
|
+
const execution: SkillExecutionState = {
|
|
1211
|
+
id: ++executionSeq,
|
|
1212
|
+
finalized: false,
|
|
1213
|
+
ts: now,
|
|
1214
|
+
startAt: Date.now(),
|
|
1215
|
+
sessionId: toStringLike(ctx?.sessionId),
|
|
1216
|
+
sessionKey: toStringLike(ctx?.sessionKey),
|
|
1217
|
+
runId,
|
|
1218
|
+
scopeKeys,
|
|
1219
|
+
skillName,
|
|
1220
|
+
skillPath: initialPath,
|
|
1221
|
+
versionHash: null,
|
|
1222
|
+
versionHashPromise: computeSkillVersionHash(initialPath),
|
|
1223
|
+
intentContext,
|
|
1224
|
+
followupMessages: [],
|
|
1225
|
+
toolReadCount: 0,
|
|
1226
|
+
encounteredSkillPaths: new Set(initialPath ? [initialPath] : []),
|
|
1227
|
+
sameSkillRetried: false,
|
|
1228
|
+
fallbackSkillRetried: false,
|
|
1229
|
+
inFlightToolCalls: new Set(),
|
|
1230
|
+
hadToolCall: false,
|
|
1231
|
+
mechanicalSuccess: true,
|
|
1232
|
+
inFollowup: false,
|
|
1233
|
+
contextWindowSize,
|
|
1234
|
+
contextTimeoutMs,
|
|
1235
|
+
};
|
|
1236
|
+
|
|
1237
|
+
execution.versionHashPromise
|
|
1238
|
+
?.then((h) => {
|
|
1239
|
+
execution.versionHash = h;
|
|
1240
|
+
})
|
|
1241
|
+
.catch(() => {
|
|
1242
|
+
// keep null; lookup later on finalize
|
|
1243
|
+
});
|
|
1244
|
+
|
|
1245
|
+
trackExecution(execution);
|
|
1246
|
+
executionsById.set(execution.id, execution);
|
|
1247
|
+
|
|
1248
|
+
queueSkillVersionWrite(skillName, initialPath, now, execution.versionHash);
|
|
1249
|
+
|
|
1250
|
+
if (!isFromSkillBlock) {
|
|
1251
|
+
enqueueEvent({
|
|
1252
|
+
v: 1,
|
|
1253
|
+
ts: now,
|
|
1254
|
+
type: "skill_file_read",
|
|
1255
|
+
...buildBase(event as { sessionId?: string; runId?: string }, ctx),
|
|
1256
|
+
skillName,
|
|
1257
|
+
skillPath: initialPath,
|
|
1258
|
+
skillSource: inferSkillSource(initialPath),
|
|
1259
|
+
toolName: toStringLike(event.toolName),
|
|
1260
|
+
toolCallId: toStringLike(event.toolCallId),
|
|
1261
|
+
});
|
|
1262
|
+
}
|
|
1263
|
+
|
|
1264
|
+
return execution;
|
|
1265
|
+
}
|
|
1266
|
+
|
|
1267
|
+
interface ParsedSkillBlock {
|
|
1268
|
+
name: string;
|
|
1269
|
+
location?: string;
|
|
1270
|
+
}
|
|
1271
|
+
|
|
1272
|
+
function parseSkillBlock(prompt: string): { blocks: ParsedSkillBlock[]; names: string[]; locations: string[]; count: number } {
|
|
1273
|
+
const rgx = /<\s*skill\b([^>]*)>/gi;
|
|
1274
|
+
const blocks: ParsedSkillBlock[] = [];
|
|
1275
|
+
const names: string[] = [];
|
|
1276
|
+
const locations: string[] = [];
|
|
1277
|
+
let match = rgx.exec(prompt);
|
|
1278
|
+
|
|
1279
|
+
while (match) {
|
|
1280
|
+
const attrs = match[1] || "";
|
|
1281
|
+
const nameMatch = /\bname="([^"]+)"/i.exec(attrs);
|
|
1282
|
+
const locMatch = /\blocation="([^"]+)"/i.exec(attrs);
|
|
1283
|
+
const name = nameMatch?.[1];
|
|
1284
|
+
if (!name) {
|
|
1285
|
+
match = rgx.exec(prompt);
|
|
1286
|
+
continue;
|
|
1287
|
+
}
|
|
1288
|
+
|
|
1289
|
+
const location = locMatch?.[1];
|
|
1290
|
+
blocks.push({ name, location });
|
|
1291
|
+
names.push(name);
|
|
1292
|
+
if (location) locations.push(location);
|
|
1293
|
+
match = rgx.exec(prompt);
|
|
1294
|
+
}
|
|
1295
|
+
|
|
1296
|
+
return { blocks, names, locations, count: names.length };
|
|
1297
|
+
}
|
|
1298
|
+
|
|
1299
|
+
|
|
1300
|
+
api.on("session_start", async (event, ctx: RawContext) => {
|
|
1301
|
+
enqueueEvent({
|
|
1302
|
+
v: 1,
|
|
1303
|
+
ts: new Date().toISOString(),
|
|
1304
|
+
type: "session_start",
|
|
1305
|
+
...buildBase(event as { sessionId?: string }, ctx),
|
|
1306
|
+
});
|
|
1307
|
+
});
|
|
1308
|
+
|
|
1309
|
+
api.on("session_end", async (event, ctx: RawContext) => {
|
|
1310
|
+
const keys = buildScopeKeys(ctx, event);
|
|
1311
|
+
const remaining = candidatesForScope(keys);
|
|
1312
|
+
for (const execution of remaining) {
|
|
1313
|
+
finalizeExecution(execution.id, "session-end");
|
|
1314
|
+
}
|
|
1315
|
+
|
|
1316
|
+
enqueueEvent({
|
|
1317
|
+
v: 1,
|
|
1318
|
+
ts: new Date().toISOString(),
|
|
1319
|
+
type: "session_end",
|
|
1320
|
+
...buildBase(event as { sessionId?: string }, ctx),
|
|
1321
|
+
durationMs: (event as RawEvent).durationMs,
|
|
1322
|
+
messageCount: (event as RawEvent).messageCount,
|
|
1323
|
+
});
|
|
1324
|
+
});
|
|
1325
|
+
|
|
1326
|
+
api.on("before_tool_call", async (event, ctx: RawContext) => {
|
|
1327
|
+
const now = new Date().toISOString();
|
|
1328
|
+
const toolName = toStringLike((event as RawEvent).toolName) || "";
|
|
1329
|
+
const params = ((event as RawEvent).params as Record<string, unknown>) || {};
|
|
1330
|
+
|
|
1331
|
+
enqueueEvent({
|
|
1332
|
+
v: 1,
|
|
1333
|
+
ts: now,
|
|
1334
|
+
type: "tool_call_start",
|
|
1335
|
+
...buildBase(event as { sessionId?: string; runId?: string }, ctx),
|
|
1336
|
+
toolName,
|
|
1337
|
+
toolCallId: toStringLike((event as RawEvent).toolCallId),
|
|
1338
|
+
params: includeToolParams ? buildToolParams(toolName, params, redactKeys) : undefined,
|
|
1339
|
+
});
|
|
1340
|
+
|
|
1341
|
+
const scopeKeys = buildScopeKeys(ctx, event);
|
|
1342
|
+
const skillPath = extractSkillPathFromParams(params);
|
|
1343
|
+
|
|
1344
|
+
if (toolName === "read" && skillPath) {
|
|
1345
|
+
const normalized = resolve(skillPath);
|
|
1346
|
+
const execution = startExecutionFromSkillRead(ctx, event as RawEvent, normalized, now);
|
|
1347
|
+
attachToolCall(scopeKeys, event as RawEvent, execution);
|
|
1348
|
+
return;
|
|
1349
|
+
}
|
|
1350
|
+
|
|
1351
|
+
attachToolCall(scopeKeys, event as RawEvent);
|
|
1352
|
+
});
|
|
1353
|
+
|
|
1354
|
+
api.on("after_tool_call", async (event, ctx: RawContext) => {
|
|
1355
|
+
const now = new Date().toISOString();
|
|
1356
|
+
const toolName = toStringLike((event as RawEvent).toolName) || "";
|
|
1357
|
+
const params = ((event as RawEvent).params as Record<string, unknown>) || {};
|
|
1358
|
+
const scopeKeys = buildScopeKeys(ctx, event);
|
|
1359
|
+
const linked = detachToolCall(event as RawEvent, scopeKeys);
|
|
1360
|
+
|
|
1361
|
+
enqueueEvent({
|
|
1362
|
+
v: 1,
|
|
1363
|
+
ts: now,
|
|
1364
|
+
type: "tool_call_end",
|
|
1365
|
+
...buildBase(event as { sessionId?: string; runId?: string }, ctx),
|
|
1366
|
+
toolName,
|
|
1367
|
+
toolCallId: toStringLike((event as RawEvent).toolCallId),
|
|
1368
|
+
params: includeToolParams ? buildToolParams(toolName, params, redactKeys) : undefined,
|
|
1369
|
+
durationMs: typeof (event as RawEvent).durationMs === "number" ? Number((event as RawEvent).durationMs) : undefined,
|
|
1370
|
+
success: !(typeof (event as RawEvent).error === "string"),
|
|
1371
|
+
error: toStringLike((event as RawEvent).error),
|
|
1372
|
+
skillName: linked?.skillName,
|
|
1373
|
+
skillPath: linked?.skillPath,
|
|
1374
|
+
});
|
|
1375
|
+
});
|
|
1376
|
+
|
|
1377
|
+
api.on("message_received", async (event, ctx: RawContext) => {
|
|
1378
|
+
const text = toStringLike((event as RawEvent).content);
|
|
1379
|
+
if (text === undefined) return;
|
|
1380
|
+
|
|
1381
|
+
const scopeKeys = buildScopeKeys(ctx, event);
|
|
1382
|
+
const execution = pickExecution(scopeKeys, true);
|
|
1383
|
+
const msg = makeMessageCapture(text, "user", (event as RawEvent).metadata, redactKeys, captureMessageContent, detectMessageSignals(execution, text));
|
|
1384
|
+
if (!msg) return;
|
|
1385
|
+
|
|
1386
|
+
const scope = buildMessageScope(ctx, event);
|
|
1387
|
+
addMessage(scope, msg);
|
|
1388
|
+
|
|
1389
|
+
if (execution) {
|
|
1390
|
+
onFollowupMessage(execution, msg);
|
|
1391
|
+
}
|
|
1392
|
+
});
|
|
1393
|
+
|
|
1394
|
+
api.on("message_sent", async (event, ctx: RawContext) => {
|
|
1395
|
+
const text = toStringLike((event as RawEvent).content);
|
|
1396
|
+
if (text === undefined) return;
|
|
1397
|
+
|
|
1398
|
+
const scopeKeys = buildScopeKeys(ctx, event);
|
|
1399
|
+
const execution = pickExecution(scopeKeys, true);
|
|
1400
|
+
const msg = makeMessageCapture(text, "assistant", (event as RawEvent).metadata, redactKeys, captureMessageContent, detectMessageSignals(execution, text));
|
|
1401
|
+
if (!msg) return;
|
|
1402
|
+
|
|
1403
|
+
const scope = buildMessageScope(ctx, event);
|
|
1404
|
+
addMessage(scope, msg);
|
|
1405
|
+
|
|
1406
|
+
if (execution) {
|
|
1407
|
+
onFollowupMessage(execution, msg);
|
|
1408
|
+
}
|
|
1409
|
+
});
|
|
1410
|
+
|
|
1411
|
+
api.on("before_prompt_build", async (event, ctx: RawContext) => {
|
|
1412
|
+
if (!detectSkillBlocks) return;
|
|
1413
|
+
const prompt = toStringLike((event as RawEvent).prompt);
|
|
1414
|
+
if (!prompt) return;
|
|
1415
|
+
|
|
1416
|
+
const info = parseSkillBlock(prompt);
|
|
1417
|
+
if (!info.count) return;
|
|
1418
|
+
|
|
1419
|
+
const timestamp = new Date().toISOString();
|
|
1420
|
+
|
|
1421
|
+
enqueueEvent({
|
|
1422
|
+
v: 1,
|
|
1423
|
+
ts: timestamp,
|
|
1424
|
+
type: "skill_block_detected",
|
|
1425
|
+
...buildBase(undefined, ctx),
|
|
1426
|
+
skillBlockCount: info.count,
|
|
1427
|
+
skillBlockNames: info.names,
|
|
1428
|
+
skillBlockLocations: info.locations,
|
|
1429
|
+
});
|
|
1430
|
+
|
|
1431
|
+
for (const block of info.blocks) {
|
|
1432
|
+
const resolvedPath = resolveSkillPathFromBlock(block.name, block.location) || block.location || block.name;
|
|
1433
|
+
if (!resolvedPath) continue;
|
|
1434
|
+
startExecutionFromSkillRead(ctx, event as RawEvent, resolvedPath, timestamp, true);
|
|
1435
|
+
}
|
|
1436
|
+
});
|
|
1437
|
+
|
|
1438
|
+
function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T> {
|
|
1439
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
1440
|
+
return Promise.race([
|
|
1441
|
+
promise,
|
|
1442
|
+
new Promise<T>((_resolve, reject) => {
|
|
1443
|
+
timer = setTimeout(() => reject(new Error("flush-timeout")), timeoutMs);
|
|
1444
|
+
}),
|
|
1445
|
+
]).finally(() => {
|
|
1446
|
+
if (timer) clearTimeout(timer);
|
|
1447
|
+
});
|
|
1448
|
+
}
|
|
1449
|
+
|
|
1450
|
+
function finalizeAllExecutions(reason: string): void {
|
|
1451
|
+
for (const execution of [...executionsById.values()]) {
|
|
1452
|
+
finalizeExecution(execution.id, reason);
|
|
1453
|
+
}
|
|
1454
|
+
}
|
|
1455
|
+
|
|
1456
|
+
async function flushPendingWrites(reason: string): Promise<void> {
|
|
1457
|
+
finalizeAllExecutions(reason);
|
|
1458
|
+
|
|
1459
|
+
try {
|
|
1460
|
+
await withTimeout(dbChain, 2500);
|
|
1461
|
+
} catch (err) {
|
|
1462
|
+
log.error(`skill-usage-audit: shutdown flush failed: ${String(err)}`);
|
|
1463
|
+
}
|
|
1464
|
+
|
|
1465
|
+
if (dbState.backend) {
|
|
1466
|
+
try {
|
|
1467
|
+
dbState.backend.close();
|
|
1468
|
+
} catch {
|
|
1469
|
+
// ignore
|
|
1470
|
+
}
|
|
1471
|
+
dbState.backend = null;
|
|
1472
|
+
dbState.statements = null;
|
|
1473
|
+
}
|
|
1474
|
+
}
|
|
1475
|
+
|
|
1476
|
+
function requestFlush(reason: string): Promise<void> {
|
|
1477
|
+
if (!shutdownPromise) {
|
|
1478
|
+
shutdownInProgress = true;
|
|
1479
|
+
shutdownPromise = flushPendingWrites(reason);
|
|
1480
|
+
}
|
|
1481
|
+
return shutdownPromise;
|
|
1482
|
+
}
|
|
1483
|
+
|
|
1484
|
+
function handleSignal(signal: string): void {
|
|
1485
|
+
const timeout = setTimeout(() => {
|
|
1486
|
+
log.info(`skill-usage-audit: forced shutdown after ${signal}`);
|
|
1487
|
+
if (dbState.backend) {
|
|
1488
|
+
try {
|
|
1489
|
+
dbState.backend.close();
|
|
1490
|
+
} catch {
|
|
1491
|
+
// ignore
|
|
1492
|
+
}
|
|
1493
|
+
}
|
|
1494
|
+
process.exit(0);
|
|
1495
|
+
}, 3500);
|
|
1496
|
+
|
|
1497
|
+
void requestFlush(`signal:${signal}`).finally(() => {
|
|
1498
|
+
clearTimeout(timeout);
|
|
1499
|
+
process.exit(0);
|
|
1500
|
+
});
|
|
1501
|
+
}
|
|
1502
|
+
|
|
1503
|
+
process.on("SIGTERM", () => handleSignal("SIGTERM"));
|
|
1504
|
+
process.on("SIGINT", () => handleSignal("SIGINT"));
|
|
1505
|
+
process.on("beforeExit", () => {
|
|
1506
|
+
if (!shutdownInProgress) {
|
|
1507
|
+
void requestFlush("beforeExit");
|
|
1508
|
+
}
|
|
1509
|
+
});
|
|
1510
|
+
process.on("exit", () => {
|
|
1511
|
+
if (dbState.backend) {
|
|
1512
|
+
try {
|
|
1513
|
+
dbState.backend.close();
|
|
1514
|
+
} catch {
|
|
1515
|
+
// ignore
|
|
1516
|
+
}
|
|
1517
|
+
}
|
|
1518
|
+
});
|
|
1519
|
+
|
|
1520
|
+
// eager init for logs
|
|
1521
|
+
void ensureDbReady();
|
|
1522
|
+
log.info("skill-usage-audit plugin registered");
|
|
1523
|
+
}
|