@klhapp/skillmux 1.11.2 → 1.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/README.md +20 -10
- package/bin/skillmux.js +89 -0
- package/docs/README.md +4 -4
- package/docs/cli.md +15 -5
- package/docs/concepts.md +4 -3
- package/docs/configuration.md +22 -5
- package/docs/deployment.md +3 -3
- package/docs/getting-started.md +32 -7
- package/docs/releasing.md +17 -2
- package/docs/sdd/cli-remote-target-parity/schema-bucket-b.yaml +297 -0
- package/docs/sdd/cli-remote-target-parity/spec-bucket-b.md +95 -0
- package/docs/sdd/cli-remote-target-parity/spec.md +111 -0
- package/docs/sdd/cli-remote-target-parity/think.md +183 -0
- package/docs/sdd/cli-surface-consistency/think.md +149 -0
- package/docs/sdd/config-authority-rerank-resilience/plan.md +338 -0
- package/docs/sdd/config-authority-rerank-resilience/review.md +124 -0
- package/docs/sdd/config-authority-rerank-resilience/spec.md +78 -0
- package/docs/sdd/feat-packaging-deployment/review.md +95 -0
- package/docs/sdd/fetch-outcome-flywheel/review.md +103 -0
- package/docs/sdd/fetch-outcome-flywheel/schema.json +347 -0
- package/docs/sdd/fetch-outcome-flywheel/spec.md +185 -0
- package/docs/sdd/npm-native-binaries/review.md +103 -0
- package/docs/sdd/npm-native-binaries/spec.md +148 -0
- package/docs/sdd/ranked-shortlist-2/pr3-calibration-removal.md +56 -0
- package/docs/sdd/refactor-builtin-target-paths/spec.md +41 -0
- package/docs/sdd/runtime-resource-hardening/review.md +86 -0
- package/docs/sdd/runtime-resource-hardening/spec.md +43 -0
- package/docs/sdd/security-hardening-phase/think.md +129 -0
- package/docs/sdd/security-observability/review.md +96 -0
- package/docs/sdd/security-observability/spec.md +55 -0
- package/docs/sdd/skill-provenance-update/review.md +152 -0
- package/docs/sdd/skill-provenance-update/schema.json +193 -0
- package/docs/sdd/skill-provenance-update/spec.md +134 -0
- package/docs/sdd/supply-chain-hardening/review.md +50 -0
- package/docs/sdd/supply-chain-hardening/spec.md +42 -0
- package/docs/sdd/target-marker-rehome/spec.md +33 -0
- package/docs/skill-management.md +1 -1
- package/docs/troubleshooting.md +18 -2
- package/package.json +15 -8
- package/src/adapters.ts +0 -438
- package/src/audit.ts +0 -21
- package/src/cli.ts +0 -928
- package/src/clients.ts +0 -368
- package/src/commands/audit.ts +0 -85
- package/src/commands/config.ts +0 -224
- package/src/commands/context.ts +0 -104
- package/src/commands/core.ts +0 -56
- package/src/commands/doctor.ts +0 -97
- package/src/commands/eval.ts +0 -88
- package/src/commands/init.ts +0 -672
- package/src/commands/install.ts +0 -155
- package/src/commands/local-vault.ts +0 -60
- package/src/commands/models.ts +0 -10
- package/src/commands/outdated.ts +0 -115
- package/src/commands/project.ts +0 -580
- package/src/commands/report.ts +0 -66
- package/src/commands/scan.ts +0 -71
- package/src/commands/shared.ts +0 -38
- package/src/commands/skill.ts +0 -33
- package/src/commands/sync.ts +0 -233
- package/src/commands/target.ts +0 -225
- package/src/commands/update.ts +0 -277
- package/src/completions.ts +0 -196
- package/src/concurrency-limiter.ts +0 -61
- package/src/config-service.ts +0 -398
- package/src/config-watcher.ts +0 -253
- package/src/config.ts +0 -583
- package/src/context.ts +0 -188
- package/src/db-audit.ts +0 -286
- package/src/db-index.ts +0 -238
- package/src/db.ts +0 -3
- package/src/deployment.ts +0 -39
- package/src/doctor.ts +0 -205
- package/src/eval.ts +0 -302
- package/src/global-flags.ts +0 -46
- package/src/init-agents.ts +0 -329
- package/src/init-instructions.ts +0 -192
- package/src/init.ts +0 -389
- package/src/install.ts +0 -232
- package/src/lifecycle.ts +0 -51
- package/src/logger.ts +0 -26
- package/src/manifest.ts +0 -352
- package/src/mcp-registration.ts +0 -89
- package/src/metrics.ts +0 -121
- package/src/models.ts +0 -20
- package/src/output.ts +0 -214
- package/src/project-setup.ts +0 -36
- package/src/prompts.ts +0 -124
- package/src/provenance.ts +0 -99
- package/src/rate-limiter.ts +0 -137
- package/src/readiness.ts +0 -30
- package/src/redact.ts +0 -52
- package/src/router-core.ts +0 -666
- package/src/rrf.ts +0 -31
- package/src/scan.ts +0 -300
- package/src/server.ts +0 -930
- package/src/setup.ts +0 -145
- package/src/snapshot.ts +0 -135
- package/src/stats.ts +0 -340
- package/src/sync.ts +0 -482
- package/src/toml-writer.ts +0 -51
- package/src/types.ts +0 -212
- package/src/vault.ts +0 -207
package/src/router-core.ts
DELETED
|
@@ -1,666 +0,0 @@
|
|
|
1
|
-
import type { Database } from "bun:sqlite";
|
|
2
|
-
import { existsSync, lstatSync, watch } from "node:fs";
|
|
3
|
-
import { join } from "node:path";
|
|
4
|
-
import { buildAuditRow } from "./audit";
|
|
5
|
-
import { embeddingDimension, embeddingFingerprint, expandHome, loadConfig } from "./config";
|
|
6
|
-
import { RemoteInferenceError } from "./clients";
|
|
7
|
-
import { log } from "./logger";
|
|
8
|
-
import { warn } from "./output";
|
|
9
|
-
import {
|
|
10
|
-
deleteSkill,
|
|
11
|
-
findExactMatch,
|
|
12
|
-
ftsSearch,
|
|
13
|
-
getAuditRowByRequestId,
|
|
14
|
-
getIndexMeta,
|
|
15
|
-
getSkillRow,
|
|
16
|
-
ingestVault,
|
|
17
|
-
insertAudit,
|
|
18
|
-
insertFetch,
|
|
19
|
-
openAudit,
|
|
20
|
-
openIndex,
|
|
21
|
-
pruneAudit,
|
|
22
|
-
replaceSkills,
|
|
23
|
-
setIndexMeta,
|
|
24
|
-
skillCount,
|
|
25
|
-
skillsNeedingVectors,
|
|
26
|
-
toSkillRow,
|
|
27
|
-
upsertSkill,
|
|
28
|
-
upsertVector,
|
|
29
|
-
vectorTopK,
|
|
30
|
-
} from "./db";
|
|
31
|
-
import type { PruneResult, SkillRow } from "./db";
|
|
32
|
-
import type {
|
|
33
|
-
RankedCandidate,
|
|
34
|
-
RetrievalCapability,
|
|
35
|
-
Clients,
|
|
36
|
-
Config,
|
|
37
|
-
DegradationReason,
|
|
38
|
-
FetchSkillInput,
|
|
39
|
-
FetchSkillResult,
|
|
40
|
-
ResolveResult,
|
|
41
|
-
ResolveSkillInput,
|
|
42
|
-
} from "./types";
|
|
43
|
-
import { reciprocalRankFusion } from "./rrf";
|
|
44
|
-
import {
|
|
45
|
-
decodeUtf8Strict,
|
|
46
|
-
getVaultMaxMtime,
|
|
47
|
-
listSupportingFiles,
|
|
48
|
-
parseSkillMd,
|
|
49
|
-
readSkill,
|
|
50
|
-
scanVaults,
|
|
51
|
-
vaultResolutionOrder,
|
|
52
|
-
sha256Hex,
|
|
53
|
-
SKILL_ID_PATTERN,
|
|
54
|
-
} from "./vault";
|
|
55
|
-
|
|
56
|
-
function maxVaultMtime(vaultPath: string, localVaultPaths: string[]): number {
|
|
57
|
-
return Math.max(getVaultMaxMtime(vaultPath), ...localVaultPaths.map(getVaultMaxMtime));
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
export { buildAuditRow } from "./audit";
|
|
61
|
-
export { loadConfig } from "./config";
|
|
62
|
-
export type * from "./types";
|
|
63
|
-
|
|
64
|
-
const NO_MATCH_MESSAGE =
|
|
65
|
-
"No skill in the vault passed the relevance threshold for this task. " +
|
|
66
|
-
"Proceed under your normal workflow; do not load an unrelated skill.";
|
|
67
|
-
|
|
68
|
-
interface Overrides {
|
|
69
|
-
config?: Config;
|
|
70
|
-
clients?: Partial<Clients>;
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
// Remote clients default to failing fast, which routes resolveSkill into the
|
|
74
|
-
// Production clients are installed during server startup; tests can inject fakes.
|
|
75
|
-
const defaultClients: Clients = {
|
|
76
|
-
embed: async () => {
|
|
77
|
-
throw new Error("embedding client not configured");
|
|
78
|
-
},
|
|
79
|
-
};
|
|
80
|
-
|
|
81
|
-
let overrides: Overrides = {};
|
|
82
|
-
type Env = { config: Config; db: Database; auditDb: Database };
|
|
83
|
-
|
|
84
|
-
let envPromise: Promise<Env> | null = null;
|
|
85
|
-
let resolvedEnv: Env | null = null;
|
|
86
|
-
let lastAuditPruneAt: number | null = null;
|
|
87
|
-
|
|
88
|
-
const AUDIT_PRUNE_INTERVAL_MS = 24 * 60 * 60 * 1000;
|
|
89
|
-
|
|
90
|
-
/** Replace config/client overrides wholesale (tests, ops). Resets the cached index handle. */
|
|
91
|
-
export function configure(opts: Overrides): void {
|
|
92
|
-
overrides = opts;
|
|
93
|
-
envPromise = null;
|
|
94
|
-
resolvedEnv = null;
|
|
95
|
-
lastAuditPruneAt = null;
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
/**
|
|
99
|
-
* Memoizes the in-flight promise, not just the resolved value: startup fires
|
|
100
|
-
* initializeRuntime()'s getRuntime() and pruneAuditIfDue() back-to-back before
|
|
101
|
-
* either has awaited anything, so caching only the resolved env would let both
|
|
102
|
-
* open their own index/audit handles and race an ingestVault (AC14 regression).
|
|
103
|
-
*/
|
|
104
|
-
async function getEnv(): Promise<Env> {
|
|
105
|
-
if (envPromise) return envPromise;
|
|
106
|
-
envPromise = (async () => {
|
|
107
|
-
const config = overrides.config ?? (await loadConfig());
|
|
108
|
-
const stateDir = expandHome(config.state_dir);
|
|
109
|
-
const db = openIndex(stateDir);
|
|
110
|
-
const auditDb = openAudit(stateDir);
|
|
111
|
-
if (skillCount(db) === 0) {
|
|
112
|
-
const vaultPath = expandHome(config.vault_path);
|
|
113
|
-
const localVaultPaths = config.local_vault_paths.map(expandHome);
|
|
114
|
-
ingestVault(db, await scanVaults(vaultPath, localVaultPaths));
|
|
115
|
-
setIndexMeta(db, "last_indexed_mtime", String(maxVaultMtime(vaultPath, localVaultPaths)));
|
|
116
|
-
}
|
|
117
|
-
resolvedEnv = { config, db, auditDb };
|
|
118
|
-
return resolvedEnv;
|
|
119
|
-
})();
|
|
120
|
-
return envPromise;
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
function getClients(): Clients {
|
|
124
|
-
return { ...defaultClients, ...overrides.clients };
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
/** Runtime accessor for the eval harness and CLI — not part of the MCP surface. */
|
|
128
|
-
export async function getRuntime(): Promise<{
|
|
129
|
-
config: Config;
|
|
130
|
-
db: Database;
|
|
131
|
-
auditDb: Database;
|
|
132
|
-
clients: Clients;
|
|
133
|
-
}> {
|
|
134
|
-
const { config, db, auditDb } = await getEnv();
|
|
135
|
-
return { config, db, auditDb, clients: getClients() };
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
export function closeRuntime(): void {
|
|
139
|
-
resolvedEnv?.db.close();
|
|
140
|
-
resolvedEnv?.auditDb.close();
|
|
141
|
-
envPromise = null;
|
|
142
|
-
resolvedEnv = null;
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
/**
|
|
146
|
-
* Zero-loss delivery: read SKILL.md from disk now, hash it, and if the
|
|
147
|
-
* index is stale re-index that skill — never serve stale bytes.
|
|
148
|
-
*/
|
|
149
|
-
async function deliverSkill(db: Database, config: Config, skillId: string): Promise<FetchSkillResult> {
|
|
150
|
-
const vaultPath = expandHome(config.vault_path);
|
|
151
|
-
const localVaultPaths = config.local_vault_paths.map(expandHome);
|
|
152
|
-
const candidates = vaultResolutionOrder(vaultPath, localVaultPaths);
|
|
153
|
-
|
|
154
|
-
// Existence alone (resolveSkillRoot's check) isn't enough here: an unparseable
|
|
155
|
-
// local_vault_paths override would otherwise shadow a perfectly valid vault_path
|
|
156
|
-
// copy of the same skill_id, since it's checked first. Skip a broken override and
|
|
157
|
-
// fall through to the next root — mirroring scanVault/rebuildIndex's tolerance for
|
|
158
|
-
// unparseable content. The last candidate (always vault_path) is never skipped on
|
|
159
|
-
// parse failure, matching the pre-existing single-root behavior where a broken
|
|
160
|
-
// vault_path copy propagates its parse error rather than being silently swallowed.
|
|
161
|
-
let root: string | null = null;
|
|
162
|
-
let bytes: Uint8Array | null = null;
|
|
163
|
-
let raw = "";
|
|
164
|
-
for (let i = 0; i < candidates.length; i++) {
|
|
165
|
-
const candidate = candidates[i]!;
|
|
166
|
-
const skillDir = join(candidate, skillId);
|
|
167
|
-
const path = join(skillDir, "SKILL.md");
|
|
168
|
-
const file = Bun.file(path);
|
|
169
|
-
if (!(await file.exists())) continue;
|
|
170
|
-
// A symlinked SKILL.md must never be read here: this is the direct "zero-loss
|
|
171
|
-
// delivery" read path that bypasses the index and readSkill's own symlink guard
|
|
172
|
-
// (vault.ts), and its `body` is served straight into the agent's context.
|
|
173
|
-
//
|
|
174
|
-
// The skill directory itself must be checked too, separately from SKILL.md's
|
|
175
|
-
// leaf check above: `lstat` only refuses to follow the *final* path component,
|
|
176
|
-
// so a symlinked skill directory (e.g. a tampered local_vault_paths override)
|
|
177
|
-
// containing a real, non-symlink SKILL.md at its target silently passes the
|
|
178
|
-
// leaf check while still resolving straight through to arbitrary host content.
|
|
179
|
-
if (lstatSync(skillDir).isSymbolicLink() || lstatSync(path).isSymbolicLink()) continue;
|
|
180
|
-
const candidateBytes = await file.bytes();
|
|
181
|
-
const candidateRaw = decodeUtf8Strict(candidateBytes);
|
|
182
|
-
if (i < candidates.length - 1) {
|
|
183
|
-
try {
|
|
184
|
-
parseSkillMd(skillId, candidateRaw);
|
|
185
|
-
} catch {
|
|
186
|
-
continue;
|
|
187
|
-
}
|
|
188
|
-
}
|
|
189
|
-
root = candidate;
|
|
190
|
-
bytes = candidateBytes;
|
|
191
|
-
raw = candidateRaw;
|
|
192
|
-
break;
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
if (root === null || bytes === null) {
|
|
196
|
-
// Deleted on disk but the watcher hasn't caught up: drop the stale row and
|
|
197
|
-
// surface the schema's error code rather than a raw ENOENT.
|
|
198
|
-
deleteSkill(db, skillId);
|
|
199
|
-
throw new Error(`SKILL_NOT_FOUND: skill '${skillId}' no longer exists in the vault`);
|
|
200
|
-
}
|
|
201
|
-
const contentSha256 = sha256Hex(bytes);
|
|
202
|
-
let row = getSkillRow(db, skillId);
|
|
203
|
-
if (row === null || row.content_sha256 !== contentSha256) {
|
|
204
|
-
const fresh = parseSkillMd(skillId, raw);
|
|
205
|
-
upsertSkill(db, fresh);
|
|
206
|
-
row = getSkillRow(db, skillId)!;
|
|
207
|
-
}
|
|
208
|
-
return {
|
|
209
|
-
skill_id: skillId,
|
|
210
|
-
title: row.title,
|
|
211
|
-
content_sha256: contentSha256,
|
|
212
|
-
body: raw,
|
|
213
|
-
files: listSupportingFiles(root, skillId),
|
|
214
|
-
};
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
const rerankText = (r: SkillRow) => `${r.title}\n${r.description}\n${r.aliases}`;
|
|
218
|
-
|
|
219
|
-
export interface RebuildReport {
|
|
220
|
-
indexed: number;
|
|
221
|
-
retained: string[];
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
/**
|
|
225
|
-
* Full from-scratch rebuild of the lexical index. Skills whose SKILL.md
|
|
226
|
-
* fails to parse keep their previously indexed row (`retained`) so a bad write
|
|
227
|
-
* never evicts a working skill. Vectors persist by content hash; changed
|
|
228
|
-
* content is re-embedded by the next backfill.
|
|
229
|
-
*/
|
|
230
|
-
export async function rebuildIndex(
|
|
231
|
-
onInvalid?: (skillId: string, error: unknown) => void,
|
|
232
|
-
): Promise<RebuildReport> {
|
|
233
|
-
const { config, db } = await getEnv();
|
|
234
|
-
const vaultPath = expandHome(config.vault_path);
|
|
235
|
-
const localVaultPaths = config.local_vault_paths.map(expandHome);
|
|
236
|
-
const currentMtime = maxVaultMtime(vaultPath, localVaultPaths);
|
|
237
|
-
const invalidIds: string[] = [];
|
|
238
|
-
const skills = await scanVaults(vaultPath, localVaultPaths, (skillId, error) => {
|
|
239
|
-
invalidIds.push(skillId);
|
|
240
|
-
onInvalid?.(skillId, error);
|
|
241
|
-
});
|
|
242
|
-
const rows = skills.map(toSkillRow);
|
|
243
|
-
// A skill_id invalid in one root (e.g. a local_vault_paths entry being edited) can
|
|
244
|
-
// still be valid in another (e.g. vault_path) — scanVaults already resolved that
|
|
245
|
-
// in `skills`. Only retain the previous row for ids that came back invalid
|
|
246
|
-
// everywhere; otherwise this duplicates the skill_id and violates the
|
|
247
|
-
// skills.skill_id PRIMARY KEY on replaceSkills's plain INSERT.
|
|
248
|
-
const validIds = new Set(skills.map((s) => s.skill_id));
|
|
249
|
-
const retained: string[] = [];
|
|
250
|
-
for (const skillId of invalidIds) {
|
|
251
|
-
if (validIds.has(skillId)) continue;
|
|
252
|
-
const previous = getSkillRow(db, skillId);
|
|
253
|
-
if (previous) {
|
|
254
|
-
rows.push(previous);
|
|
255
|
-
retained.push(skillId);
|
|
256
|
-
}
|
|
257
|
-
}
|
|
258
|
-
replaceSkills(db, rows);
|
|
259
|
-
setIndexMeta(db, "last_indexed_mtime", String(currentMtime));
|
|
260
|
-
return { indexed: rows.length, retained };
|
|
261
|
-
}
|
|
262
|
-
|
|
263
|
-
/**
|
|
264
|
-
* On-Demand Lazy Indexing (First Principles #2):
|
|
265
|
-
* Checks the max mtime of the vault directory and re-indexes only if files have changed.
|
|
266
|
-
* This runs synchronously to block queries until the lexical index is correct.
|
|
267
|
-
*/
|
|
268
|
-
export async function syncVaultIfNeeded(): Promise<void> {
|
|
269
|
-
const { config, db } = await getEnv();
|
|
270
|
-
const vaultPath = expandHome(config.vault_path);
|
|
271
|
-
const localVaultPaths = config.local_vault_paths.map(expandHome);
|
|
272
|
-
const currentMtime = maxVaultMtime(vaultPath, localVaultPaths);
|
|
273
|
-
const lastIndexed = getIndexMeta(db, "last_indexed_mtime");
|
|
274
|
-
|
|
275
|
-
if (lastIndexed === null || currentMtime > Number(lastIndexed)) {
|
|
276
|
-
const invalidIds: string[] = [];
|
|
277
|
-
const skills = await scanVaults(vaultPath, localVaultPaths, (skillId, error) => {
|
|
278
|
-
invalidIds.push(skillId);
|
|
279
|
-
warn(`keeping previous index entry for ${skillId}: ${error}`);
|
|
280
|
-
});
|
|
281
|
-
const rows = skills.map(toSkillRow);
|
|
282
|
-
// See rebuildIndex: a skill_id invalid in one root can still be valid in
|
|
283
|
-
// another that scanVaults already resolved — don't re-add its previous row.
|
|
284
|
-
const validIds = new Set(skills.map((s) => s.skill_id));
|
|
285
|
-
for (const skillId of invalidIds) {
|
|
286
|
-
if (validIds.has(skillId)) continue;
|
|
287
|
-
const previous = getSkillRow(db, skillId);
|
|
288
|
-
if (previous) {
|
|
289
|
-
rows.push(previous);
|
|
290
|
-
}
|
|
291
|
-
}
|
|
292
|
-
replaceSkills(db, rows);
|
|
293
|
-
setIndexMeta(db, "last_indexed_mtime", String(currentMtime));
|
|
294
|
-
backfillEmbeddings().catch(() => {});
|
|
295
|
-
}
|
|
296
|
-
}
|
|
297
|
-
|
|
298
|
-
/**
|
|
299
|
-
* Embed every skill missing a current vector (new or content changed).
|
|
300
|
-
* Called by `skillmux index` and at server startup; failure is tolerated —
|
|
301
|
-
* resolve falls back to lexical-only recall until vectors exist.
|
|
302
|
-
*/
|
|
303
|
-
export async function backfillEmbeddings(): Promise<number> {
|
|
304
|
-
const { config, db } = await getEnv();
|
|
305
|
-
const clients = getClients();
|
|
306
|
-
const fingerprint = embeddingFingerprint(config);
|
|
307
|
-
const pending = skillsNeedingVectors(db, embeddingDimension(config), fingerprint);
|
|
308
|
-
if (pending.length === 0) return 0;
|
|
309
|
-
|
|
310
|
-
const BATCH_SIZE = 10;
|
|
311
|
-
let count = 0;
|
|
312
|
-
for (let i = 0; i < pending.length; i += BATCH_SIZE) {
|
|
313
|
-
const chunk = pending.slice(i, i + BATCH_SIZE);
|
|
314
|
-
try {
|
|
315
|
-
const vectors = await clients.embed(chunk.map(rerankText));
|
|
316
|
-
db.transaction(() => {
|
|
317
|
-
chunk.forEach((row, j) => {
|
|
318
|
-
const vector = vectors[j];
|
|
319
|
-
if (!vector) throw new Error("Embedding client returned an incomplete batch.");
|
|
320
|
-
upsertVector(db, row.skill_id, row.content_sha256, fingerprint, vector);
|
|
321
|
-
});
|
|
322
|
-
})();
|
|
323
|
-
count += chunk.length;
|
|
324
|
-
} catch (err) {
|
|
325
|
-
if (
|
|
326
|
-
i === 0 ||
|
|
327
|
-
(err instanceof RemoteInferenceError &&
|
|
328
|
-
(err.kind === "configuration" || err.kind === "protocol"))
|
|
329
|
-
) {
|
|
330
|
-
throw err;
|
|
331
|
-
}
|
|
332
|
-
break;
|
|
333
|
-
}
|
|
334
|
-
}
|
|
335
|
-
return count;
|
|
336
|
-
}
|
|
337
|
-
|
|
338
|
-
const WATCH_DEBOUNCE_MS = 300;
|
|
339
|
-
const STABLE_STAT_INTERVAL_MS = 100;
|
|
340
|
-
const STABLE_STAT_MAX_TRIES = 10;
|
|
341
|
-
|
|
342
|
-
/** Wait until SKILL.md stops changing (two identical stats in a row) or give up. */
|
|
343
|
-
async function waitForStableFile(path: string): Promise<void> {
|
|
344
|
-
let previous = "";
|
|
345
|
-
for (let i = 0; i < STABLE_STAT_MAX_TRIES; i++) {
|
|
346
|
-
const file = Bun.file(path);
|
|
347
|
-
if (!(await file.exists())) return;
|
|
348
|
-
const current = `${file.size}:${file.lastModified}`;
|
|
349
|
-
if (current === previous) return;
|
|
350
|
-
previous = current;
|
|
351
|
-
await Bun.sleep(STABLE_STAT_INTERVAL_MS);
|
|
352
|
-
}
|
|
353
|
-
}
|
|
354
|
-
|
|
355
|
-
async function reindexOneSkill(db: Database, vaultPath: string, skillId: string): Promise<void> {
|
|
356
|
-
const skillMd = join(vaultPath, skillId, "SKILL.md");
|
|
357
|
-
await waitForStableFile(skillMd);
|
|
358
|
-
if (!(await Bun.file(skillMd).exists())) {
|
|
359
|
-
deleteSkill(db, skillId);
|
|
360
|
-
return;
|
|
361
|
-
}
|
|
362
|
-
try {
|
|
363
|
-
upsertSkill(db, await readSkill(vaultPath, skillId));
|
|
364
|
-
backfillEmbeddings().catch(() => {});
|
|
365
|
-
} catch (error) {
|
|
366
|
-
warn(`keeping previous index entry for ${skillId}: ${error}`);
|
|
367
|
-
}
|
|
368
|
-
}
|
|
369
|
-
|
|
370
|
-
/**
|
|
371
|
-
* Watch the vault and fold file changes into the index within seconds.
|
|
372
|
-
* Events are debounced per skill; a write that fails to parse keeps the
|
|
373
|
-
* previous index entry. Returns a stop function.
|
|
374
|
-
*/
|
|
375
|
-
export async function startVaultWatcher(): Promise<() => void> {
|
|
376
|
-
const { config, db } = await getEnv();
|
|
377
|
-
const vaultPath = expandHome(config.vault_path);
|
|
378
|
-
const timers = new Map<string, ReturnType<typeof setTimeout>>();
|
|
379
|
-
|
|
380
|
-
// A fresh installation may not have a vault checkout yet. Serving an empty
|
|
381
|
-
// vault is valid; leave live indexing inactive until the next server start
|
|
382
|
-
// finds a checkout rather than failing startup with ENOENT.
|
|
383
|
-
if (!existsSync(vaultPath)) return () => {};
|
|
384
|
-
|
|
385
|
-
const watcher = watch(vaultPath, { recursive: true }, (_event, filename) => {
|
|
386
|
-
const skillId = filename?.split(/[\\/]/)[0];
|
|
387
|
-
if (!skillId || !SKILL_ID_PATTERN.test(skillId)) return;
|
|
388
|
-
clearTimeout(timers.get(skillId));
|
|
389
|
-
timers.set(
|
|
390
|
-
skillId,
|
|
391
|
-
setTimeout(() => {
|
|
392
|
-
timers.delete(skillId);
|
|
393
|
-
void reindexOneSkill(db, vaultPath, skillId);
|
|
394
|
-
}, WATCH_DEBOUNCE_MS),
|
|
395
|
-
);
|
|
396
|
-
});
|
|
397
|
-
// A watcher error (e.g. the vault root disappearing) must degrade the index,
|
|
398
|
-
// not crash the server — an unhandled 'error' event would throw.
|
|
399
|
-
watcher.on("error", (error) => {
|
|
400
|
-
warn(`vault watcher error, live updates paused: ${error}`);
|
|
401
|
-
});
|
|
402
|
-
|
|
403
|
-
return () => {
|
|
404
|
-
watcher.close();
|
|
405
|
-
for (const timer of timers.values()) clearTimeout(timer);
|
|
406
|
-
timers.clear();
|
|
407
|
-
};
|
|
408
|
-
}
|
|
409
|
-
|
|
410
|
-
export async function resolveSkill(input: ResolveSkillInput): Promise<ResolveResult> {
|
|
411
|
-
const t0 = performance.now();
|
|
412
|
-
const { config, db, auditDb } = await getEnv();
|
|
413
|
-
await syncVaultIfNeeded();
|
|
414
|
-
|
|
415
|
-
if (input.top_k !== undefined) {
|
|
416
|
-
if (!Number.isInteger(input.top_k) || input.top_k < 1) {
|
|
417
|
-
throw new Error(`Invalid top_k: ${input.top_k} must be a positive integer`);
|
|
418
|
-
}
|
|
419
|
-
if (input.top_k > config.output.max_top_k) {
|
|
420
|
-
throw new Error(
|
|
421
|
-
`Invalid top_k: ${input.top_k} exceeds max_top_k of ${config.output.max_top_k}`,
|
|
422
|
-
);
|
|
423
|
-
}
|
|
424
|
-
}
|
|
425
|
-
|
|
426
|
-
const effectiveTopK = input.top_k ?? config.output.top_k;
|
|
427
|
-
const retrievalResult = await retrieveAndRerank(input);
|
|
428
|
-
const { retrieval, candidates: rankedCandidates } = retrievalResult;
|
|
429
|
-
|
|
430
|
-
const candidates: RankedCandidate[] = rankedCandidates
|
|
431
|
-
.slice(0, effectiveTopK)
|
|
432
|
-
.map((c, index) => ({
|
|
433
|
-
rank: index + 1,
|
|
434
|
-
skill_id: c.skill_id,
|
|
435
|
-
description: c.description,
|
|
436
|
-
score: c.score,
|
|
437
|
-
}));
|
|
438
|
-
|
|
439
|
-
const requestId = crypto.randomUUID();
|
|
440
|
-
|
|
441
|
-
const result: ResolveResult = {
|
|
442
|
-
request_id: requestId,
|
|
443
|
-
retrieval,
|
|
444
|
-
...(retrievalResult.degraded_from
|
|
445
|
-
? {
|
|
446
|
-
degraded_from: retrievalResult.degraded_from,
|
|
447
|
-
degradation_reason: retrievalResult.degradation_reason,
|
|
448
|
-
}
|
|
449
|
-
: {}),
|
|
450
|
-
candidates,
|
|
451
|
-
};
|
|
452
|
-
|
|
453
|
-
insertAudit(
|
|
454
|
-
auditDb,
|
|
455
|
-
buildAuditRow({
|
|
456
|
-
id: 0, // assigned by SQLite
|
|
457
|
-
ts: new Date().toISOString(),
|
|
458
|
-
request_id: requestId,
|
|
459
|
-
query: input.query,
|
|
460
|
-
retrieval,
|
|
461
|
-
degraded_from: retrievalResult.degraded_from ?? null,
|
|
462
|
-
degradation_reason: retrievalResult.degradation_reason ?? null,
|
|
463
|
-
candidates: candidates.map((c) => ({ skill_id: c.skill_id, score: c.score })),
|
|
464
|
-
latency_ms: Math.round(performance.now() - t0),
|
|
465
|
-
}),
|
|
466
|
-
);
|
|
467
|
-
|
|
468
|
-
return result;
|
|
469
|
-
}
|
|
470
|
-
|
|
471
|
-
export interface RawCandidate {
|
|
472
|
-
skill_id: string;
|
|
473
|
-
title: string;
|
|
474
|
-
description: string;
|
|
475
|
-
score: number | null;
|
|
476
|
-
}
|
|
477
|
-
|
|
478
|
-
export interface RetrievalResult {
|
|
479
|
-
retrieval: Exclude<RetrievalCapability, "exact">;
|
|
480
|
-
degraded_from?: "reranked" | "hybrid";
|
|
481
|
-
degradation_reason?: DegradationReason;
|
|
482
|
-
candidates: RawCandidate[];
|
|
483
|
-
trace: Array<{
|
|
484
|
-
skill_id: string;
|
|
485
|
-
lexical_rank: number | null;
|
|
486
|
-
fused_rank: number | null;
|
|
487
|
-
reranked_rank: number | null;
|
|
488
|
-
}>;
|
|
489
|
-
}
|
|
490
|
-
|
|
491
|
-
export function classifyInferenceError(
|
|
492
|
-
stage: "embedding" | "reranker",
|
|
493
|
-
error: unknown,
|
|
494
|
-
): DegradationReason {
|
|
495
|
-
const isTimeout =
|
|
496
|
-
(error as { name?: string })?.name === "TimeoutError" ||
|
|
497
|
-
(error as { name?: string })?.name === "AbortError" ||
|
|
498
|
-
String(error).toLowerCase().includes("timeout") ||
|
|
499
|
-
String(error).toLowerCase().includes("aborted");
|
|
500
|
-
|
|
501
|
-
if (isTimeout) {
|
|
502
|
-
return stage === "embedding" ? "embedding_timeout" : "reranker_timeout";
|
|
503
|
-
}
|
|
504
|
-
|
|
505
|
-
if (error instanceof RemoteInferenceError) {
|
|
506
|
-
if (error.kind === "protocol") {
|
|
507
|
-
return stage === "embedding" ? "embedding_protocol_error" : "reranker_protocol_error";
|
|
508
|
-
}
|
|
509
|
-
return stage === "embedding" ? "embedding_unavailable" : "reranker_unavailable";
|
|
510
|
-
}
|
|
511
|
-
|
|
512
|
-
const msg = error instanceof Error ? error.message.toLowerCase() : String(error).toLowerCase();
|
|
513
|
-
if (
|
|
514
|
-
msg.includes("protocol") ||
|
|
515
|
-
msg.includes("malformed") ||
|
|
516
|
-
msg.includes("invalid") ||
|
|
517
|
-
msg.includes("json")
|
|
518
|
-
) {
|
|
519
|
-
return stage === "embedding" ? "embedding_protocol_error" : "reranker_protocol_error";
|
|
520
|
-
}
|
|
521
|
-
|
|
522
|
-
return stage === "embedding" ? "embedding_unavailable" : "reranker_unavailable";
|
|
523
|
-
}
|
|
524
|
-
|
|
525
|
-
/**
|
|
526
|
-
* Retrieve the full fused candidate set, optionally rerank it, and synchronize
|
|
527
|
-
* the vault before reading the index.
|
|
528
|
-
*/
|
|
529
|
-
export async function retrieveAndRerank(
|
|
530
|
-
input: ResolveSkillInput,
|
|
531
|
-
): Promise<RetrievalResult> {
|
|
532
|
-
await syncVaultIfNeeded();
|
|
533
|
-
const { config, db } = await getEnv();
|
|
534
|
-
const clients = getClients();
|
|
535
|
-
|
|
536
|
-
const embedPromise =
|
|
537
|
-
!input.forceLexical && clients.embed
|
|
538
|
-
? clients.embed([input.query]).then(
|
|
539
|
-
(res) => res,
|
|
540
|
-
(err) => ({ error: err }),
|
|
541
|
-
)
|
|
542
|
-
: null;
|
|
543
|
-
|
|
544
|
-
const lexical = ftsSearch(db, input.query, config.recall.k_lexical);
|
|
545
|
-
const lexicalRanks = new Map(lexical.map((row, index) => [row.skill_id, index + 1]));
|
|
546
|
-
|
|
547
|
-
let retrieval: RetrievalResult["retrieval"] = "lexical";
|
|
548
|
-
let degraded_from: RetrievalResult["degraded_from"] = undefined;
|
|
549
|
-
let degradation_reason: RetrievalResult["degradation_reason"] = undefined;
|
|
550
|
-
let rows = lexical;
|
|
551
|
-
let fusedRows: SkillRow[] | null = null;
|
|
552
|
-
|
|
553
|
-
if (embedPromise) {
|
|
554
|
-
const embedRes = await embedPromise;
|
|
555
|
-
if (embedRes && typeof embedRes === "object" && "error" in embedRes) {
|
|
556
|
-
retrieval = "lexical";
|
|
557
|
-
degraded_from = clients.rerank ? "reranked" : "hybrid";
|
|
558
|
-
degradation_reason = classifyInferenceError("embedding", embedRes.error);
|
|
559
|
-
log.warn("embedding", { degraded_from, reason: degradation_reason });
|
|
560
|
-
} else {
|
|
561
|
-
try {
|
|
562
|
-
const queryVec = (embedRes as Float32Array[])[0];
|
|
563
|
-
if (!queryVec) throw new Error("Embedding client returned no query vector.");
|
|
564
|
-
const nearest = vectorTopK(db, queryVec, config.recall.k_vector);
|
|
565
|
-
rows = reciprocalRankFusion(lexical, nearest);
|
|
566
|
-
fusedRows = rows;
|
|
567
|
-
retrieval = "hybrid";
|
|
568
|
-
} catch (embedError) {
|
|
569
|
-
retrieval = "lexical";
|
|
570
|
-
degraded_from = clients.rerank ? "reranked" : "hybrid";
|
|
571
|
-
degradation_reason = classifyInferenceError("embedding", embedError);
|
|
572
|
-
log.warn("embedding", { degraded_from, reason: degradation_reason });
|
|
573
|
-
}
|
|
574
|
-
}
|
|
575
|
-
}
|
|
576
|
-
|
|
577
|
-
let scores: number[] | null = null;
|
|
578
|
-
if (clients.rerank && retrieval === "hybrid" && rows.length > 0) {
|
|
579
|
-
const kRerank = config.recall.k_rerank ?? 10;
|
|
580
|
-
const rerankCandidates = rows.slice(0, kRerank);
|
|
581
|
-
try {
|
|
582
|
-
scores = await clients.rerank(
|
|
583
|
-
input.query,
|
|
584
|
-
rerankCandidates.map((r) => ({ skill_id: r.skill_id, text: rerankText(r) })),
|
|
585
|
-
);
|
|
586
|
-
retrieval = "reranked";
|
|
587
|
-
rows = rerankCandidates;
|
|
588
|
-
} catch (rerankError) {
|
|
589
|
-
scores = null;
|
|
590
|
-
degraded_from = "reranked";
|
|
591
|
-
degradation_reason = classifyInferenceError("reranker", rerankError);
|
|
592
|
-
log.warn("reranker", { degraded_from, reason: degradation_reason });
|
|
593
|
-
}
|
|
594
|
-
}
|
|
595
|
-
|
|
596
|
-
const candidates = rows
|
|
597
|
-
.map((r, i) => ({
|
|
598
|
-
skill_id: r.skill_id,
|
|
599
|
-
title: r.title,
|
|
600
|
-
description: r.description,
|
|
601
|
-
score: scores?.[i] ?? null,
|
|
602
|
-
}))
|
|
603
|
-
.sort((a, b) => scores === null ? 0 : (b.score ?? -Infinity) - (a.score ?? -Infinity));
|
|
604
|
-
const rerankedRanks = retrieval === "reranked"
|
|
605
|
-
? new Map(candidates.map((candidate, index) => [candidate.skill_id, index + 1]))
|
|
606
|
-
: new Map<string, number>();
|
|
607
|
-
const traceRows = fusedRows ?? rows;
|
|
608
|
-
|
|
609
|
-
return {
|
|
610
|
-
retrieval,
|
|
611
|
-
...(degraded_from ? { degraded_from, degradation_reason } : {}),
|
|
612
|
-
candidates,
|
|
613
|
-
trace: traceRows.map((row, index) => ({
|
|
614
|
-
skill_id: row.skill_id,
|
|
615
|
-
lexical_rank: lexicalRanks.get(row.skill_id) ?? null,
|
|
616
|
-
fused_rank: fusedRows ? index + 1 : null,
|
|
617
|
-
reranked_rank: rerankedRanks.get(row.skill_id) ?? null,
|
|
618
|
-
})),
|
|
619
|
-
};
|
|
620
|
-
}
|
|
621
|
-
|
|
622
|
-
/**
|
|
623
|
-
* AC14: runs at most once per 24 hours per process. Callers must not await
|
|
624
|
-
* this on the startup or resolve path -- it is meant to be fired and left to
|
|
625
|
-
* resolve in the background so it never blocks readiness or a resolve.
|
|
626
|
-
*/
|
|
627
|
-
export async function pruneAuditIfDue(now: Date = new Date()): Promise<PruneResult | null> {
|
|
628
|
-
const { config, auditDb } = await getEnv();
|
|
629
|
-
const retentionDays = config.audit?.retention_days ?? 90;
|
|
630
|
-
if (retentionDays <= 0) return null;
|
|
631
|
-
if (lastAuditPruneAt !== null && now.getTime() - lastAuditPruneAt < AUDIT_PRUNE_INTERVAL_MS) {
|
|
632
|
-
return null;
|
|
633
|
-
}
|
|
634
|
-
lastAuditPruneAt = now.getTime();
|
|
635
|
-
return pruneAudit(auditDb, retentionDays, now);
|
|
636
|
-
}
|
|
637
|
-
|
|
638
|
-
export async function fetchSkill(input: FetchSkillInput): Promise<FetchSkillResult> {
|
|
639
|
-
const { config, db, auditDb } = await getEnv();
|
|
640
|
-
await syncVaultIfNeeded();
|
|
641
|
-
if (getSkillRow(db, input.skill_id) === null) {
|
|
642
|
-
throw new Error(`SKILL_NOT_FOUND: no skill '${input.skill_id}' in the index`);
|
|
643
|
-
}
|
|
644
|
-
const result = await deliverSkill(db, config, input.skill_id);
|
|
645
|
-
|
|
646
|
-
let resolveAuditId: number | null = null;
|
|
647
|
-
let rankAtResolve: number | null = null;
|
|
648
|
-
if (input.request_id) {
|
|
649
|
-
const resolveRow = getAuditRowByRequestId(auditDb, input.request_id);
|
|
650
|
-
if (resolveRow) {
|
|
651
|
-
resolveAuditId = resolveRow.id;
|
|
652
|
-
const index = resolveRow.candidates.findIndex((c) => c.skill_id === input.skill_id);
|
|
653
|
-
rankAtResolve = index === -1 ? null : index + 1;
|
|
654
|
-
}
|
|
655
|
-
}
|
|
656
|
-
|
|
657
|
-
insertFetch(auditDb, {
|
|
658
|
-
ts: new Date().toISOString(),
|
|
659
|
-
skill_id: input.skill_id,
|
|
660
|
-
request_id: input.request_id ?? null,
|
|
661
|
-
resolve_audit_id: resolveAuditId,
|
|
662
|
-
rank_at_resolve: rankAtResolve,
|
|
663
|
-
});
|
|
664
|
-
|
|
665
|
-
return result;
|
|
666
|
-
}
|
package/src/rrf.ts
DELETED
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
export interface RankedItem {
|
|
2
|
-
skill_id: string;
|
|
3
|
-
}
|
|
4
|
-
|
|
5
|
-
export function reciprocalRankFusion<T extends RankedItem>(lexical: T[], semantic: T[], rankConstant = 60): T[] {
|
|
6
|
-
const byId = new Map<string, { item: T; score: number; lexical: number; semantic: number }>();
|
|
7
|
-
const add = (items: T[], lane: "lexical" | "semantic") => {
|
|
8
|
-
items.forEach((item, index) => {
|
|
9
|
-
const current = byId.get(item.skill_id) ?? {
|
|
10
|
-
item,
|
|
11
|
-
score: 0,
|
|
12
|
-
lexical: Number.POSITIVE_INFINITY,
|
|
13
|
-
semantic: Number.POSITIVE_INFINITY,
|
|
14
|
-
};
|
|
15
|
-
current.score += 1 / (rankConstant + index + 1);
|
|
16
|
-
current[lane] = index;
|
|
17
|
-
byId.set(item.skill_id, current);
|
|
18
|
-
});
|
|
19
|
-
};
|
|
20
|
-
add(lexical, "lexical");
|
|
21
|
-
add(semantic, "semantic");
|
|
22
|
-
return [...byId.values()]
|
|
23
|
-
.sort((a, b) =>
|
|
24
|
-
b.score - a.score
|
|
25
|
-
|| Number(a.semantic === Infinity) - Number(b.semantic === Infinity)
|
|
26
|
-
|| a.semantic - b.semantic
|
|
27
|
-
|| a.lexical - b.lexical
|
|
28
|
-
|| a.item.skill_id.localeCompare(b.item.skill_id),
|
|
29
|
-
)
|
|
30
|
-
.map((entry) => entry.item);
|
|
31
|
-
}
|