@klhapp/skillmux 1.7.1 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +35 -0
- package/README.md +15 -5
- package/docs/assets/architecture-dark.svg +160 -0
- package/docs/assets/{architecture.svg → architecture-light.svg} +40 -34
- package/docs/assets/logo-dark.png +0 -0
- package/docs/assets/logo-light.png +0 -0
- package/docs/cli.md +63 -5
- package/docs/concepts.md +10 -0
- package/docs/configuration.md +20 -20
- package/docs/deployment.md +20 -7
- package/docs/getting-started.md +11 -0
- package/docs/mcp-routing.md +41 -7
- package/docs/schema.json +31 -4
- package/docs/skill-management.md +32 -0
- package/package.json +1 -1
- package/src/audit.ts +1 -0
- package/src/cli.ts +62 -8
- package/src/commands/audit.ts +82 -0
- package/src/commands/eval.ts +81 -0
- package/src/commands/outdated.ts +112 -0
- package/src/commands/update.ts +253 -0
- package/src/config.ts +6 -0
- package/src/db.ts +152 -45
- package/src/eval.ts +69 -0
- package/src/install.ts +82 -3
- package/src/provenance.ts +99 -0
- package/src/router-core.ts +109 -26
- package/src/scan.ts +7 -1
- package/src/server.ts +21 -6
- package/src/stats.ts +119 -13
- package/src/sync.ts +38 -8
- package/src/types.ts +22 -0
- package/src/vault.ts +44 -4
- package/docs/assets/logo.png +0 -0
package/src/router-core.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Database } from "bun:sqlite";
|
|
2
|
-
import { existsSync, watch } from "node:fs";
|
|
2
|
+
import { existsSync, lstatSync, watch } from "node:fs";
|
|
3
3
|
import { join } from "node:path";
|
|
4
4
|
import { buildAuditRow } from "./audit";
|
|
5
5
|
import { embeddingDimension, embeddingFingerprint, expandHome, loadConfig } from "./config";
|
|
@@ -8,11 +8,15 @@ import {
|
|
|
8
8
|
deleteSkill,
|
|
9
9
|
findExactMatch,
|
|
10
10
|
ftsSearch,
|
|
11
|
+
getAuditRowByRequestId,
|
|
11
12
|
getIndexMeta,
|
|
12
13
|
getSkillRow,
|
|
13
14
|
ingestVault,
|
|
14
15
|
insertAudit,
|
|
16
|
+
insertFetch,
|
|
17
|
+
openAudit,
|
|
15
18
|
openIndex,
|
|
19
|
+
pruneAudit,
|
|
16
20
|
replaceSkills,
|
|
17
21
|
setIndexMeta,
|
|
18
22
|
skillCount,
|
|
@@ -22,7 +26,7 @@ import {
|
|
|
22
26
|
upsertVector,
|
|
23
27
|
vectorTopK,
|
|
24
28
|
} from "./db";
|
|
25
|
-
import type { SkillRow } from "./db";
|
|
29
|
+
import type { PruneResult, SkillRow } from "./db";
|
|
26
30
|
import type {
|
|
27
31
|
RankedCandidate,
|
|
28
32
|
RetrievalCapability,
|
|
@@ -73,26 +77,45 @@ const defaultClients: Clients = {
|
|
|
73
77
|
};
|
|
74
78
|
|
|
75
79
|
let overrides: Overrides = {};
|
|
76
|
-
|
|
80
|
+
type Env = { config: Config; db: Database; auditDb: Database };
|
|
81
|
+
|
|
82
|
+
let envPromise: Promise<Env> | null = null;
|
|
83
|
+
let resolvedEnv: Env | null = null;
|
|
84
|
+
let lastAuditPruneAt: number | null = null;
|
|
85
|
+
|
|
86
|
+
const AUDIT_PRUNE_INTERVAL_MS = 24 * 60 * 60 * 1000;
|
|
77
87
|
|
|
78
88
|
/** Replace config/client overrides wholesale (tests, ops). Resets the cached index handle. */
|
|
79
89
|
export function configure(opts: Overrides): void {
|
|
80
90
|
overrides = opts;
|
|
81
|
-
|
|
91
|
+
envPromise = null;
|
|
92
|
+
resolvedEnv = null;
|
|
93
|
+
lastAuditPruneAt = null;
|
|
82
94
|
}
|
|
83
95
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
+
/**
|
|
97
|
+
* Memoizes the in-flight promise, not just the resolved value: startup fires
|
|
98
|
+
* initializeRuntime()'s getRuntime() and pruneAuditIfDue() back-to-back before
|
|
99
|
+
* either has awaited anything, so caching only the resolved env would let both
|
|
100
|
+
* open their own index/audit handles and race an ingestVault (AC14 regression).
|
|
101
|
+
*/
|
|
102
|
+
async function getEnv(): Promise<Env> {
|
|
103
|
+
if (envPromise) return envPromise;
|
|
104
|
+
envPromise = (async () => {
|
|
105
|
+
const config = overrides.config ?? (await loadConfig());
|
|
106
|
+
const stateDir = expandHome(config.state_dir);
|
|
107
|
+
const db = openIndex(stateDir);
|
|
108
|
+
const auditDb = openAudit(stateDir);
|
|
109
|
+
if (skillCount(db) === 0) {
|
|
110
|
+
const vaultPath = expandHome(config.vault_path);
|
|
111
|
+
const localVaultPaths = config.local_vault_paths.map(expandHome);
|
|
112
|
+
ingestVault(db, await scanVaults(vaultPath, localVaultPaths));
|
|
113
|
+
setIndexMeta(db, "last_indexed_mtime", String(maxVaultMtime(vaultPath, localVaultPaths)));
|
|
114
|
+
}
|
|
115
|
+
resolvedEnv = { config, db, auditDb };
|
|
116
|
+
return resolvedEnv;
|
|
117
|
+
})();
|
|
118
|
+
return envPromise;
|
|
96
119
|
}
|
|
97
120
|
|
|
98
121
|
function getClients(): Clients {
|
|
@@ -100,14 +123,21 @@ function getClients(): Clients {
|
|
|
100
123
|
}
|
|
101
124
|
|
|
102
125
|
/** Runtime accessor for the eval harness and CLI — not part of the MCP surface. */
|
|
103
|
-
export async function getRuntime(): Promise<{
|
|
104
|
-
|
|
105
|
-
|
|
126
|
+
export async function getRuntime(): Promise<{
|
|
127
|
+
config: Config;
|
|
128
|
+
db: Database;
|
|
129
|
+
auditDb: Database;
|
|
130
|
+
clients: Clients;
|
|
131
|
+
}> {
|
|
132
|
+
const { config, db, auditDb } = await getEnv();
|
|
133
|
+
return { config, db, auditDb, clients: getClients() };
|
|
106
134
|
}
|
|
107
135
|
|
|
108
136
|
export function closeRuntime(): void {
|
|
109
|
-
|
|
110
|
-
|
|
137
|
+
resolvedEnv?.db.close();
|
|
138
|
+
resolvedEnv?.auditDb.close();
|
|
139
|
+
envPromise = null;
|
|
140
|
+
resolvedEnv = null;
|
|
111
141
|
}
|
|
112
142
|
|
|
113
143
|
/**
|
|
@@ -131,8 +161,20 @@ async function deliverSkill(db: Database, config: Config, skillId: string): Prom
|
|
|
131
161
|
let raw = "";
|
|
132
162
|
for (let i = 0; i < candidates.length; i++) {
|
|
133
163
|
const candidate = candidates[i]!;
|
|
134
|
-
const
|
|
164
|
+
const skillDir = join(candidate, skillId);
|
|
165
|
+
const path = join(skillDir, "SKILL.md");
|
|
166
|
+
const file = Bun.file(path);
|
|
135
167
|
if (!(await file.exists())) continue;
|
|
168
|
+
// A symlinked SKILL.md must never be read here: this is the direct "zero-loss
|
|
169
|
+
// delivery" read path that bypasses the index and readSkill's own symlink guard
|
|
170
|
+
// (vault.ts), and its `body` is served straight into the agent's context.
|
|
171
|
+
//
|
|
172
|
+
// The skill directory itself must be checked too, separately from SKILL.md's
|
|
173
|
+
// leaf check above: `lstat` only refuses to follow the *final* path component,
|
|
174
|
+
// so a symlinked skill directory (e.g. a tampered local_vault_paths override)
|
|
175
|
+
// containing a real, non-symlink SKILL.md at its target silently passes the
|
|
176
|
+
// leaf check while still resolving straight through to arbitrary host content.
|
|
177
|
+
if (lstatSync(skillDir).isSymbolicLink() || lstatSync(path).isSymbolicLink()) continue;
|
|
136
178
|
const candidateBytes = await file.bytes();
|
|
137
179
|
const candidateRaw = decodeUtf8Strict(candidateBytes);
|
|
138
180
|
if (i < candidates.length - 1) {
|
|
@@ -365,7 +407,7 @@ export async function startVaultWatcher(): Promise<() => void> {
|
|
|
365
407
|
|
|
366
408
|
export async function resolveSkill(input: ResolveSkillInput): Promise<ResolveResult> {
|
|
367
409
|
const t0 = performance.now();
|
|
368
|
-
const { config, db } = await getEnv();
|
|
410
|
+
const { config, db, auditDb } = await getEnv();
|
|
369
411
|
await syncVaultIfNeeded();
|
|
370
412
|
|
|
371
413
|
if (input.top_k !== undefined) {
|
|
@@ -392,7 +434,10 @@ export async function resolveSkill(input: ResolveSkillInput): Promise<ResolveRes
|
|
|
392
434
|
score: c.score,
|
|
393
435
|
}));
|
|
394
436
|
|
|
437
|
+
const requestId = crypto.randomUUID();
|
|
438
|
+
|
|
395
439
|
const result: ResolveResult = {
|
|
440
|
+
request_id: requestId,
|
|
396
441
|
retrieval,
|
|
397
442
|
...(retrievalResult.degraded_from
|
|
398
443
|
? {
|
|
@@ -404,10 +449,11 @@ export async function resolveSkill(input: ResolveSkillInput): Promise<ResolveRes
|
|
|
404
449
|
};
|
|
405
450
|
|
|
406
451
|
insertAudit(
|
|
407
|
-
|
|
452
|
+
auditDb,
|
|
408
453
|
buildAuditRow({
|
|
409
454
|
id: 0, // assigned by SQLite
|
|
410
455
|
ts: new Date().toISOString(),
|
|
456
|
+
request_id: requestId,
|
|
411
457
|
query: input.query,
|
|
412
458
|
retrieval,
|
|
413
459
|
degraded_from: retrievalResult.degraded_from ?? null,
|
|
@@ -592,11 +638,48 @@ export async function retrieveAndRerank(
|
|
|
592
638
|
};
|
|
593
639
|
}
|
|
594
640
|
|
|
641
|
+
/**
|
|
642
|
+
* AC14: runs at most once per 24 hours per process. Callers must not await
|
|
643
|
+
* this on the startup or resolve path -- it is meant to be fired and left to
|
|
644
|
+
* resolve in the background so it never blocks readiness or a resolve.
|
|
645
|
+
*/
|
|
646
|
+
export async function pruneAuditIfDue(now: Date = new Date()): Promise<PruneResult | null> {
|
|
647
|
+
const { config, auditDb } = await getEnv();
|
|
648
|
+
const retentionDays = config.audit?.retention_days ?? 90;
|
|
649
|
+
if (retentionDays <= 0) return null;
|
|
650
|
+
if (lastAuditPruneAt !== null && now.getTime() - lastAuditPruneAt < AUDIT_PRUNE_INTERVAL_MS) {
|
|
651
|
+
return null;
|
|
652
|
+
}
|
|
653
|
+
lastAuditPruneAt = now.getTime();
|
|
654
|
+
return pruneAudit(auditDb, retentionDays, now);
|
|
655
|
+
}
|
|
656
|
+
|
|
595
657
|
export async function fetchSkill(input: FetchSkillInput): Promise<FetchSkillResult> {
|
|
596
|
-
const { config, db } = await getEnv();
|
|
658
|
+
const { config, db, auditDb } = await getEnv();
|
|
597
659
|
await syncVaultIfNeeded();
|
|
598
660
|
if (getSkillRow(db, input.skill_id) === null) {
|
|
599
661
|
throw new Error(`SKILL_NOT_FOUND: no skill '${input.skill_id}' in the index`);
|
|
600
662
|
}
|
|
601
|
-
|
|
663
|
+
const result = await deliverSkill(db, config, input.skill_id);
|
|
664
|
+
|
|
665
|
+
let resolveAuditId: number | null = null;
|
|
666
|
+
let rankAtResolve: number | null = null;
|
|
667
|
+
if (input.request_id) {
|
|
668
|
+
const resolveRow = getAuditRowByRequestId(auditDb, input.request_id);
|
|
669
|
+
if (resolveRow) {
|
|
670
|
+
resolveAuditId = resolveRow.id;
|
|
671
|
+
const index = resolveRow.candidates.findIndex((c) => c.skill_id === input.skill_id);
|
|
672
|
+
rankAtResolve = index === -1 ? null : index + 1;
|
|
673
|
+
}
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
insertFetch(auditDb, {
|
|
677
|
+
ts: new Date().toISOString(),
|
|
678
|
+
skill_id: input.skill_id,
|
|
679
|
+
request_id: input.request_id ?? null,
|
|
680
|
+
resolve_audit_id: resolveAuditId,
|
|
681
|
+
rank_at_resolve: rankAtResolve,
|
|
682
|
+
});
|
|
683
|
+
|
|
684
|
+
return result;
|
|
602
685
|
}
|
package/src/scan.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { existsSync } from "node:fs";
|
|
1
|
+
import { existsSync, lstatSync } from "node:fs";
|
|
2
2
|
import { basename, dirname, join } from "node:path";
|
|
3
3
|
import { decodeUtf8Strict, listSupportingFiles, scanVault } from "./vault";
|
|
4
4
|
|
|
@@ -162,8 +162,14 @@ interface ScanContentTarget {
|
|
|
162
162
|
content: string;
|
|
163
163
|
}
|
|
164
164
|
|
|
165
|
+
/** Refuses (returns null for) a symlinked path instead of following it — every
|
|
166
|
+
* caller feeds vault content into scan findings, so a symlink swapped in after
|
|
167
|
+
* an earlier symlink-filtering pass (e.g. listSupportingFiles' walk, which runs
|
|
168
|
+
* before this read, not at it) must still be caught right here, not trusted to
|
|
169
|
+
* have stayed excluded. Same defense-in-depth pattern as readSkill/hashSkillContent. */
|
|
165
170
|
export async function readTextFileOrNull(path: string): Promise<string | null> {
|
|
166
171
|
try {
|
|
172
|
+
if (lstatSync(path).isSymbolicLink()) return null;
|
|
167
173
|
const bytes = await Bun.file(path).bytes();
|
|
168
174
|
return decodeUtf8Strict(bytes);
|
|
169
175
|
} catch {
|
package/src/server.ts
CHANGED
|
@@ -12,6 +12,7 @@ import {
|
|
|
12
12
|
backfillEmbeddings,
|
|
13
13
|
configure,
|
|
14
14
|
fetchSkill,
|
|
15
|
+
pruneAuditIfDue,
|
|
15
16
|
resolveSkill,
|
|
16
17
|
} from "./router-core";
|
|
17
18
|
import { closeRuntime, getRuntime, startVaultWatcher } from "./router-core";
|
|
@@ -113,12 +114,16 @@ export function createMcpServer(): McpServer {
|
|
|
113
114
|
{
|
|
114
115
|
description:
|
|
115
116
|
"Fetch a skill's SKILL.md verbatim by skill_id, with sha256 and supporting-file paths. " +
|
|
116
|
-
"Independent of any prior resolve_skill outcome."
|
|
117
|
-
|
|
117
|
+
"Independent of any prior resolve_skill outcome. Pass the request_id from a prior " +
|
|
118
|
+
"resolve_skill call to link this fetch to it for quality measurement.",
|
|
119
|
+
inputSchema: {
|
|
120
|
+
skill_id: z.string().regex(SKILL_ID_PATTERN),
|
|
121
|
+
request_id: z.string().min(1).max(128).optional(),
|
|
122
|
+
},
|
|
118
123
|
},
|
|
119
|
-
async ({ skill_id }) => {
|
|
124
|
+
async ({ skill_id, request_id }) => {
|
|
120
125
|
try {
|
|
121
|
-
const result = await fetchSkill({ skill_id });
|
|
126
|
+
const result = await fetchSkill({ skill_id, request_id });
|
|
122
127
|
const { body, ...meta } = result;
|
|
123
128
|
return {
|
|
124
129
|
content: [{ type: "text" as const, text: body }],
|
|
@@ -175,6 +180,14 @@ export async function startServer(opts?: {
|
|
|
175
180
|
.then(() => metricsRegistry.setReadiness(readinessState.get()))
|
|
176
181
|
.catch((err) => console.error("skillmux runtime init error:", err));
|
|
177
182
|
|
|
183
|
+
// AC14: fire-and-forget so this never delays readiness or blocks a resolve;
|
|
184
|
+
// not chained onto initPromise, which is awaited below for HTTP transport.
|
|
185
|
+
const runAuditPrune = () =>
|
|
186
|
+
pruneAuditIfDue().catch((err) => console.error("skillmux audit prune error:", err));
|
|
187
|
+
runAuditPrune();
|
|
188
|
+
const auditPruneInterval = setInterval(runAuditPrune, 24 * 60 * 60 * 1000);
|
|
189
|
+
auditPruneInterval.unref();
|
|
190
|
+
|
|
178
191
|
const server = createMcpServer();
|
|
179
192
|
|
|
180
193
|
const transportType = opts?.transport ?? "stdio";
|
|
@@ -345,13 +358,13 @@ export async function startServer(opts?: {
|
|
|
345
358
|
{ status: 400, headers: { "Content-Type": "application/json" } },
|
|
346
359
|
);
|
|
347
360
|
}
|
|
348
|
-
const {
|
|
361
|
+
const { auditDb } = await getRuntime();
|
|
349
362
|
const headers = new Headers({ "Content-Type": "application/json" });
|
|
350
363
|
if (allowOriginHeader)
|
|
351
364
|
headers.set("Access-Control-Allow-Origin", allowOriginHeader);
|
|
352
365
|
for (const [key, value] of Object.entries(rateLimitResult.headers))
|
|
353
366
|
headers.set(key, value);
|
|
354
|
-
return new Response(JSON.stringify(getStats(
|
|
367
|
+
return new Response(JSON.stringify(getStats(auditDb, since)), {
|
|
355
368
|
status: 200,
|
|
356
369
|
headers,
|
|
357
370
|
});
|
|
@@ -521,6 +534,7 @@ export async function startServer(opts?: {
|
|
|
521
534
|
async stop() {
|
|
522
535
|
if (stopped) return;
|
|
523
536
|
stopped = true;
|
|
537
|
+
clearInterval(auditPruneInterval);
|
|
524
538
|
readinessState.set({ ...readinessState.get(), status: "stopping" });
|
|
525
539
|
metricsRegistry.setReadiness(readinessState.get());
|
|
526
540
|
bunServer.stop(true);
|
|
@@ -540,6 +554,7 @@ export async function startServer(opts?: {
|
|
|
540
554
|
async stop() {
|
|
541
555
|
if (stopped) return;
|
|
542
556
|
stopped = true;
|
|
557
|
+
clearInterval(auditPruneInterval);
|
|
543
558
|
readinessState.set({ ...readinessState.get(), status: "stopping" });
|
|
544
559
|
metricsRegistry.setReadiness(readinessState.get());
|
|
545
560
|
configWatcher?.stop();
|
package/src/stats.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Database } from "bun:sqlite";
|
|
2
|
-
import type { AuditCandidate, AuditRow } from "./types";
|
|
2
|
+
import type { AuditCandidate, AuditRow, FetchAuditRow } from "./types";
|
|
3
3
|
|
|
4
4
|
export const SINCE_PATTERN = /^(\d+[hdwmy]|\d{4}-\d{2}-\d{2}([T ].+)?)$/;
|
|
5
5
|
|
|
@@ -20,6 +20,18 @@ export interface RetrievalTotals {
|
|
|
20
20
|
lexical: number;
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
+
export type AcceptanceSignal =
|
|
24
|
+
| { available: false; uncorrelated_fetch_count: number }
|
|
25
|
+
| {
|
|
26
|
+
available: true;
|
|
27
|
+
resolves_with_candidates: number;
|
|
28
|
+
accepted_count: number;
|
|
29
|
+
acceptance_rate: number;
|
|
30
|
+
observed_mrr: number;
|
|
31
|
+
top1_acceptance_rate: number;
|
|
32
|
+
uncorrelated_fetch_count: number;
|
|
33
|
+
};
|
|
34
|
+
|
|
23
35
|
export interface StatsResponse {
|
|
24
36
|
since: string;
|
|
25
37
|
until: string;
|
|
@@ -31,6 +43,8 @@ export interface StatsResponse {
|
|
|
31
43
|
average_latency_ms: number;
|
|
32
44
|
skills: SkillStat[];
|
|
33
45
|
top_empty_shortlist_queries: EmptyShortlistQuery[];
|
|
46
|
+
acceptance: AcceptanceSignal;
|
|
47
|
+
top_unused_shortlist_queries: EmptyShortlistQuery[];
|
|
34
48
|
}
|
|
35
49
|
|
|
36
50
|
const RELATIVE_WINDOW = /^(\d+)([hdwmy])$/;
|
|
@@ -63,10 +77,16 @@ function compareCodeUnits(a: string, b: string): number {
|
|
|
63
77
|
return 0;
|
|
64
78
|
}
|
|
65
79
|
|
|
66
|
-
export function computeStats(
|
|
80
|
+
export function computeStats(
|
|
81
|
+
rows: AuditRow[],
|
|
82
|
+
since: Date,
|
|
83
|
+
until: Date,
|
|
84
|
+
fetchRows: FetchAuditRow[] = [],
|
|
85
|
+
): StatsResponse {
|
|
67
86
|
const retrieval_totals: RetrievalTotals = { exact: 0, reranked: 0, hybrid: 0, lexical: 0 };
|
|
68
87
|
const skillCounts = new Map<string, number>();
|
|
69
88
|
const emptyShortlistCounts = new Map<string, number>();
|
|
89
|
+
const resolvesWithCandidates = new Map<number, AuditRow>();
|
|
70
90
|
let empty_shortlist_count = 0;
|
|
71
91
|
let degraded_count = 0;
|
|
72
92
|
let total_latency_ms = 0;
|
|
@@ -86,6 +106,7 @@ export function computeStats(rows: AuditRow[], since: Date, until: Date): StatsR
|
|
|
86
106
|
empty_shortlist_count++;
|
|
87
107
|
emptyShortlistCounts.set(row.query, (emptyShortlistCounts.get(row.query) ?? 0) + 1);
|
|
88
108
|
} else {
|
|
109
|
+
resolvesWithCandidates.set(row.id, row);
|
|
89
110
|
const seenInRow = new Set<string>();
|
|
90
111
|
for (const candidate of row.candidates) {
|
|
91
112
|
if (!candidate.skill_id) continue;
|
|
@@ -96,6 +117,52 @@ export function computeStats(rows: AuditRow[], since: Date, until: Date): StatsR
|
|
|
96
117
|
}
|
|
97
118
|
}
|
|
98
119
|
|
|
120
|
+
let uncorrelated_fetch_count = 0;
|
|
121
|
+
const firstFetchByResolve = new Map<number, FetchAuditRow>();
|
|
122
|
+
for (const fetch of fetchRows) {
|
|
123
|
+
if (fetch.resolve_audit_id === null) {
|
|
124
|
+
uncorrelated_fetch_count++;
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
if (!resolvesWithCandidates.has(fetch.resolve_audit_id)) continue;
|
|
128
|
+
const existing = firstFetchByResolve.get(fetch.resolve_audit_id);
|
|
129
|
+
if (!existing || fetch.ts < existing.ts) {
|
|
130
|
+
firstFetchByResolve.set(fetch.resolve_audit_id, fetch);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
const acceptedResolveIds = new Set(firstFetchByResolve.keys());
|
|
135
|
+
const accepted_count = acceptedResolveIds.size;
|
|
136
|
+
const acceptance: AcceptanceSignal =
|
|
137
|
+
accepted_count > 0
|
|
138
|
+
? (() => {
|
|
139
|
+
let reciprocalRankSum = 0;
|
|
140
|
+
let top1Count = 0;
|
|
141
|
+
for (const fetch of firstFetchByResolve.values()) {
|
|
142
|
+
const rank = fetch.rank_at_resolve;
|
|
143
|
+
if (rank !== null) {
|
|
144
|
+
reciprocalRankSum += 1 / rank;
|
|
145
|
+
if (rank === 1) top1Count++;
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
return {
|
|
149
|
+
available: true as const,
|
|
150
|
+
resolves_with_candidates: resolvesWithCandidates.size,
|
|
151
|
+
accepted_count,
|
|
152
|
+
acceptance_rate: accepted_count / resolvesWithCandidates.size,
|
|
153
|
+
observed_mrr: reciprocalRankSum / accepted_count,
|
|
154
|
+
top1_acceptance_rate: top1Count / accepted_count,
|
|
155
|
+
uncorrelated_fetch_count,
|
|
156
|
+
};
|
|
157
|
+
})()
|
|
158
|
+
: { available: false as const, uncorrelated_fetch_count };
|
|
159
|
+
|
|
160
|
+
const unusedShortlistCounts = new Map<string, number>();
|
|
161
|
+
for (const [id, row] of resolvesWithCandidates) {
|
|
162
|
+
if (acceptedResolveIds.has(id)) continue;
|
|
163
|
+
unusedShortlistCounts.set(row.query, (unusedShortlistCounts.get(row.query) ?? 0) + 1);
|
|
164
|
+
}
|
|
165
|
+
|
|
99
166
|
const total_requests = rows.length;
|
|
100
167
|
const empty_shortlist_rate = total_requests > 0 ? empty_shortlist_count / total_requests : 0;
|
|
101
168
|
const average_latency_ms = total_requests > 0 ? total_latency_ms / total_requests : 0;
|
|
@@ -109,15 +176,8 @@ export function computeStats(rows: AuditRow[], since: Date, until: Date): StatsR
|
|
|
109
176
|
return compareCodeUnits(a.skill_id, b.skill_id);
|
|
110
177
|
});
|
|
111
178
|
|
|
112
|
-
const top_empty_shortlist_queries
|
|
113
|
-
|
|
114
|
-
.sort((a, b) => {
|
|
115
|
-
if (b.count !== a.count) {
|
|
116
|
-
return b.count - a.count;
|
|
117
|
-
}
|
|
118
|
-
return compareCodeUnits(a.query, b.query);
|
|
119
|
-
})
|
|
120
|
-
.slice(0, 20);
|
|
179
|
+
const top_empty_shortlist_queries = topQueryCounts(emptyShortlistCounts);
|
|
180
|
+
const top_unused_shortlist_queries = topQueryCounts(unusedShortlistCounts);
|
|
121
181
|
|
|
122
182
|
return {
|
|
123
183
|
since: since.toISOString(),
|
|
@@ -130,12 +190,27 @@ export function computeStats(rows: AuditRow[], since: Date, until: Date): StatsR
|
|
|
130
190
|
average_latency_ms,
|
|
131
191
|
skills,
|
|
132
192
|
top_empty_shortlist_queries,
|
|
193
|
+
acceptance,
|
|
194
|
+
top_unused_shortlist_queries,
|
|
133
195
|
};
|
|
134
196
|
}
|
|
135
197
|
|
|
198
|
+
function topQueryCounts(counts: Map<string, number>): EmptyShortlistQuery[] {
|
|
199
|
+
return [...counts.entries()]
|
|
200
|
+
.map(([query, count]) => ({ query, count }))
|
|
201
|
+
.sort((a, b) => {
|
|
202
|
+
if (b.count !== a.count) {
|
|
203
|
+
return b.count - a.count;
|
|
204
|
+
}
|
|
205
|
+
return compareCodeUnits(a.query, b.query);
|
|
206
|
+
})
|
|
207
|
+
.slice(0, 20);
|
|
208
|
+
}
|
|
209
|
+
|
|
136
210
|
interface AuditTableRow {
|
|
137
211
|
id: number;
|
|
138
212
|
ts: string;
|
|
213
|
+
request_id: string | null;
|
|
139
214
|
query: string;
|
|
140
215
|
retrieval: AuditRow["retrieval"];
|
|
141
216
|
degraded_from: string | null;
|
|
@@ -147,7 +222,7 @@ interface AuditTableRow {
|
|
|
147
222
|
export function queryAuditRows(db: Database, sinceIso: string): AuditRow[] {
|
|
148
223
|
const rows = db
|
|
149
224
|
.query(
|
|
150
|
-
"SELECT id, ts, query, retrieval, degraded_from, degradation_reason, candidates, latency_ms FROM audit WHERE ts >= ? ORDER BY ts ASC",
|
|
225
|
+
"SELECT id, ts, request_id, query, retrieval, degraded_from, degradation_reason, candidates, latency_ms FROM audit WHERE ts >= ? ORDER BY ts ASC",
|
|
151
226
|
)
|
|
152
227
|
.all(sinceIso) as AuditTableRow[];
|
|
153
228
|
|
|
@@ -178,6 +253,7 @@ export function queryAuditRows(db: Database, sinceIso: string): AuditRow[] {
|
|
|
178
253
|
const result: AuditRow = {
|
|
179
254
|
id: row.id,
|
|
180
255
|
ts: row.ts,
|
|
256
|
+
request_id: row.request_id,
|
|
181
257
|
query: row.query,
|
|
182
258
|
retrieval: row.retrieval,
|
|
183
259
|
candidates,
|
|
@@ -193,10 +269,19 @@ export function queryAuditRows(db: Database, sinceIso: string): AuditRow[] {
|
|
|
193
269
|
});
|
|
194
270
|
}
|
|
195
271
|
|
|
272
|
+
export function queryFetchRows(db: Database, sinceIso: string): FetchAuditRow[] {
|
|
273
|
+
return db
|
|
274
|
+
.query(
|
|
275
|
+
"SELECT id, ts, skill_id, request_id, resolve_audit_id, rank_at_resolve FROM fetch WHERE ts >= ? ORDER BY ts ASC",
|
|
276
|
+
)
|
|
277
|
+
.all(sinceIso) as FetchAuditRow[];
|
|
278
|
+
}
|
|
279
|
+
|
|
196
280
|
export function getStats(db: Database, since: string, now: Date = new Date()): StatsResponse {
|
|
197
281
|
const sinceDate = parseSince(since, now);
|
|
198
282
|
const rows = queryAuditRows(db, sinceDate.toISOString());
|
|
199
|
-
|
|
283
|
+
const fetchRows = queryFetchRows(db, sinceDate.toISOString());
|
|
284
|
+
return computeStats(rows, sinceDate, now, fetchRows);
|
|
200
285
|
}
|
|
201
286
|
|
|
202
287
|
export function renderStatsText(stats: StatsResponse): string {
|
|
@@ -230,5 +315,26 @@ export function renderStatsText(stats: StatsResponse): string {
|
|
|
230
315
|
}
|
|
231
316
|
}
|
|
232
317
|
|
|
318
|
+
if (stats.acceptance.available) {
|
|
319
|
+
lines.push(
|
|
320
|
+
`acceptance: acceptance_rate=${stats.acceptance.acceptance_rate.toFixed(3)} ` +
|
|
321
|
+
`observed_mrr=${stats.acceptance.observed_mrr.toFixed(3)} ` +
|
|
322
|
+
`top1_acceptance_rate=${stats.acceptance.top1_acceptance_rate.toFixed(3)} ` +
|
|
323
|
+
`(accepted=${stats.acceptance.accepted_count}/${stats.acceptance.resolves_with_candidates}, ` +
|
|
324
|
+
`uncorrelated_fetch_count=${stats.acceptance.uncorrelated_fetch_count})`,
|
|
325
|
+
);
|
|
326
|
+
} else {
|
|
327
|
+
lines.push(`acceptance: unavailable (uncorrelated_fetch_count=${stats.acceptance.uncorrelated_fetch_count})`);
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
lines.push("top unused shortlist queries:");
|
|
331
|
+
if (stats.top_unused_shortlist_queries.length === 0) {
|
|
332
|
+
lines.push(" (none)");
|
|
333
|
+
} else {
|
|
334
|
+
for (const entry of stats.top_unused_shortlist_queries) {
|
|
335
|
+
lines.push(` "${entry.query}" (${entry.count})`);
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
|
|
233
339
|
return lines.join("\n");
|
|
234
340
|
}
|
package/src/sync.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { chmodSync, existsSync, mkdirSync, readFileSync, readdirSync, rmSync, symlinkSync, unlinkSync, writeFileSync } from "node:fs";
|
|
2
2
|
import { homedir } from "node:os";
|
|
3
3
|
import { join, relative } from "node:path";
|
|
4
|
+
import { findSymlinks } from "./install";
|
|
4
5
|
import { resolveSkillRoot } from "./vault";
|
|
5
6
|
|
|
6
7
|
export const SKILLMUX_MARKER_FILENAME = ".skillmux";
|
|
@@ -101,25 +102,47 @@ export interface SyncTargetParams {
|
|
|
101
102
|
export interface SyncTargetResult {
|
|
102
103
|
added: string[];
|
|
103
104
|
removed: string[];
|
|
105
|
+
/** Core skills whose directory contains an internal symlink and were refused —
|
|
106
|
+
* syncTarget symlinks the whole skill directory into the target, so an internal
|
|
107
|
+
* symlink would otherwise be exposed to the agent regardless of how it got into
|
|
108
|
+
* the vault (install/update already reject them, but a shared git-backed vault
|
|
109
|
+
* can still be tampered with directly). */
|
|
110
|
+
skipped: string[];
|
|
104
111
|
}
|
|
105
112
|
|
|
106
113
|
export interface SyncTargetOptions {
|
|
107
114
|
dryRun?: boolean;
|
|
108
115
|
}
|
|
109
116
|
|
|
117
|
+
/** Splits `skillIds` into those safe to symlink into a target and those refused
|
|
118
|
+
* for containing an internal symlink (see SyncTargetResult.skipped). */
|
|
119
|
+
function partitionSyncable(
|
|
120
|
+
skillIds: string[],
|
|
121
|
+
skillSource: (skillId: string) => string,
|
|
122
|
+
): { syncable: string[]; skipped: string[] } {
|
|
123
|
+
const syncable: string[] = [];
|
|
124
|
+
const skipped: string[] = [];
|
|
125
|
+
for (const skillId of skillIds) {
|
|
126
|
+
if (findSymlinks(join(skillSource(skillId), skillId)).length > 0) skipped.push(skillId);
|
|
127
|
+
else syncable.push(skillId);
|
|
128
|
+
}
|
|
129
|
+
return { syncable, skipped };
|
|
130
|
+
}
|
|
131
|
+
|
|
110
132
|
export function syncTarget(params: SyncTargetParams, options: SyncTargetOptions = {}): SyncTargetResult {
|
|
111
133
|
const { vaultPath, targetDir, targetName, coreSkillIds, localVaultPaths = [] } = params;
|
|
112
134
|
const { dryRun = false } = options;
|
|
113
135
|
const skillSource = (skillId: string) => resolveSkillRoot(skillId, vaultPath, localVaultPaths) ?? vaultPath;
|
|
114
136
|
|
|
115
137
|
if (!existsSync(targetDir)) {
|
|
116
|
-
|
|
138
|
+
const { syncable, skipped } = partitionSyncable(coreSkillIds, skillSource);
|
|
139
|
+
if (dryRun) return { added: syncable, removed: [], skipped };
|
|
117
140
|
mkdirSync(targetDir, { recursive: true });
|
|
118
|
-
for (const skillId of
|
|
141
|
+
for (const skillId of syncable) {
|
|
119
142
|
symlinkSync(join(skillSource(skillId), skillId), join(targetDir, skillId));
|
|
120
143
|
}
|
|
121
|
-
writeTargetMarker(targetDir, targetName, vaultPath,
|
|
122
|
-
return { added:
|
|
144
|
+
writeTargetMarker(targetDir, targetName, vaultPath, syncable);
|
|
145
|
+
return { added: syncable, removed: [], skipped };
|
|
123
146
|
}
|
|
124
147
|
|
|
125
148
|
let marker = readSkillmuxMarker(targetDir);
|
|
@@ -167,14 +190,21 @@ export function syncTarget(params: SyncTargetParams, options: SyncTargetOptions
|
|
|
167
190
|
}
|
|
168
191
|
|
|
169
192
|
const removed = [...managedEntries].filter((name) => existing.includes(name) && !desired.has(name));
|
|
170
|
-
const
|
|
171
|
-
|
|
193
|
+
const addedCandidates = coreSkillIds.filter((skillId) => !existing.includes(skillId));
|
|
194
|
+
const { syncable: added, skipped } = partitionSyncable(addedCandidates, skillSource);
|
|
195
|
+
if (dryRun) return { added, removed, skipped };
|
|
172
196
|
|
|
173
197
|
for (const name of removed) unlinkSync(join(targetDir, name));
|
|
174
198
|
for (const skillId of added) symlinkSync(join(skillSource(skillId), skillId), join(targetDir, skillId));
|
|
175
|
-
writeTargetMarker(
|
|
199
|
+
writeTargetMarker(
|
|
200
|
+
targetDir,
|
|
201
|
+
targetName,
|
|
202
|
+
vaultPath,
|
|
203
|
+
coreSkillIds.filter((skillId) => !skipped.includes(skillId)),
|
|
204
|
+
marker.created_at,
|
|
205
|
+
);
|
|
176
206
|
|
|
177
|
-
return { added, removed };
|
|
207
|
+
return { added, removed, skipped };
|
|
178
208
|
}
|
|
179
209
|
|
|
180
210
|
export interface AdoptTargetResult {
|
package/src/types.ts
CHANGED
|
@@ -107,6 +107,11 @@ export interface ConfigPolicy {
|
|
|
107
107
|
environment_overrides?: boolean;
|
|
108
108
|
}
|
|
109
109
|
|
|
110
|
+
export interface AuditConfig {
|
|
111
|
+
/** Age in days beyond which audit rows are pruned. 0 disables pruning. */
|
|
112
|
+
retention_days: number;
|
|
113
|
+
}
|
|
114
|
+
|
|
110
115
|
export interface Config {
|
|
111
116
|
config?: ConfigPolicy;
|
|
112
117
|
vault_path: string;
|
|
@@ -116,6 +121,7 @@ export interface Config {
|
|
|
116
121
|
output: OutputConfig;
|
|
117
122
|
inference: InferenceConfig;
|
|
118
123
|
server?: ServerConfig;
|
|
124
|
+
audit?: AuditConfig;
|
|
119
125
|
}
|
|
120
126
|
|
|
121
127
|
export interface RankedCandidate {
|
|
@@ -136,6 +142,7 @@ export type DegradationReason =
|
|
|
136
142
|
| "reranker_protocol_error";
|
|
137
143
|
|
|
138
144
|
export interface ResolveResult {
|
|
145
|
+
request_id: string;
|
|
139
146
|
retrieval: RetrievalCapability;
|
|
140
147
|
degraded_from?: "reranked" | "hybrid";
|
|
141
148
|
degradation_reason?: DegradationReason;
|
|
@@ -151,6 +158,7 @@ export interface ResolveSkillInput {
|
|
|
151
158
|
|
|
152
159
|
export interface FetchSkillInput {
|
|
153
160
|
skill_id: string;
|
|
161
|
+
request_id?: string;
|
|
154
162
|
}
|
|
155
163
|
|
|
156
164
|
export interface FetchSkillResult {
|
|
@@ -169,6 +177,8 @@ export interface AuditCandidate {
|
|
|
169
177
|
export interface AuditRow {
|
|
170
178
|
id: number;
|
|
171
179
|
ts: string;
|
|
180
|
+
/** Null for rows written before request_id existed (AC4). */
|
|
181
|
+
request_id: string | null;
|
|
172
182
|
query: string;
|
|
173
183
|
retrieval: RetrievalCapability;
|
|
174
184
|
degraded_from?: "reranked" | "hybrid" | null;
|
|
@@ -177,6 +187,18 @@ export interface AuditRow {
|
|
|
177
187
|
latency_ms: number;
|
|
178
188
|
}
|
|
179
189
|
|
|
190
|
+
export interface FetchAuditRow {
|
|
191
|
+
id: number;
|
|
192
|
+
ts: string;
|
|
193
|
+
skill_id: string;
|
|
194
|
+
/** Exactly as supplied by the caller, including an unknown value. Null when the caller sent none. */
|
|
195
|
+
request_id: string | null;
|
|
196
|
+
/** Null when the fetch is uncorrelated: no request_id, an unknown/malformed one, or a pruned resolve row. */
|
|
197
|
+
resolve_audit_id: number | null;
|
|
198
|
+
/** Rank of skill_id in the correlated resolve's shortlist. Null when uncorrelated or absent from the shortlist. */
|
|
199
|
+
rank_at_resolve: number | null;
|
|
200
|
+
}
|
|
201
|
+
|
|
180
202
|
export interface Clients {
|
|
181
203
|
embed(texts: string[]): Promise<Float32Array[]>;
|
|
182
204
|
rerank?: (query: string, docs: { skill_id: string; text: string }[]) => Promise<number[]>;
|