capman 0.5.5 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1 -1
- package/bin/lib/cmd-generate.js +156 -12
- package/bin/lib/cmd-help.js +3 -0
- package/dist/cjs/cache.d.ts +9 -0
- package/dist/cjs/cache.d.ts.map +1 -1
- package/dist/cjs/cache.js +37 -7
- package/dist/cjs/cache.js.map +1 -1
- package/dist/cjs/engine.d.ts +68 -1
- package/dist/cjs/engine.d.ts.map +1 -1
- package/dist/cjs/engine.js +313 -13
- package/dist/cjs/engine.js.map +1 -1
- package/dist/cjs/generator.d.ts.map +1 -1
- package/dist/cjs/generator.js +28 -6
- package/dist/cjs/generator.js.map +1 -1
- package/dist/cjs/index.d.ts +3 -1
- package/dist/cjs/index.d.ts.map +1 -1
- package/dist/cjs/index.js +5 -1
- package/dist/cjs/index.js.map +1 -1
- package/dist/cjs/learning.d.ts +7 -0
- package/dist/cjs/learning.d.ts.map +1 -1
- package/dist/cjs/learning.js +44 -23
- package/dist/cjs/learning.js.map +1 -1
- package/dist/cjs/matcher.d.ts +92 -0
- package/dist/cjs/matcher.d.ts.map +1 -1
- package/dist/cjs/matcher.js +354 -35
- package/dist/cjs/matcher.js.map +1 -1
- package/dist/cjs/parser.js +27 -9
- package/dist/cjs/parser.js.map +1 -1
- package/dist/cjs/resolver.d.ts +2 -2
- package/dist/cjs/resolver.d.ts.map +1 -1
- package/dist/cjs/resolver.js +66 -26
- package/dist/cjs/resolver.js.map +1 -1
- package/dist/cjs/schema.d.ts +865 -94
- package/dist/cjs/schema.d.ts.map +1 -1
- package/dist/cjs/schema.js +62 -12
- package/dist/cjs/schema.js.map +1 -1
- package/dist/cjs/types.d.ts +153 -9
- package/dist/cjs/types.d.ts.map +1 -1
- package/dist/cjs/version.d.ts +1 -1
- package/dist/cjs/version.js +1 -1
- package/dist/esm/cache.d.ts +9 -0
- package/dist/esm/cache.js +37 -7
- package/dist/esm/engine.d.ts +68 -1
- package/dist/esm/engine.js +314 -14
- package/dist/esm/generator.js +28 -6
- package/dist/esm/index.d.ts +3 -1
- package/dist/esm/index.js +2 -0
- package/dist/esm/learning.d.ts +7 -0
- package/dist/esm/learning.js +45 -24
- package/dist/esm/matcher.d.ts +92 -0
- package/dist/esm/matcher.js +346 -35
- package/dist/esm/parser.js +27 -9
- package/dist/esm/resolver.d.ts +2 -2
- package/dist/esm/resolver.js +66 -26
- package/dist/esm/schema.d.ts +865 -94
- package/dist/esm/schema.js +62 -12
- package/dist/esm/types.d.ts +153 -9
- package/dist/esm/version.d.ts +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +1 -1
package/dist/esm/engine.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { match as _match, matchWithLLM as _matchWithLLM, resolverToIntent, extractParams,
|
|
1
|
+
import { match as _match, matchWithLLM as _matchWithLLM, resolverToIntent, extractParams, LLMParseError, tokenize, buildBM25Index, sanitizeForPrompt, calibrateCeiling as _calibrateCeiling } from './matcher';
|
|
2
2
|
import { resolve as _resolve, checkPrivacy } from './resolver';
|
|
3
3
|
import { MemoryLearningStore } from './learning';
|
|
4
4
|
import { logger } from './logger';
|
|
@@ -17,6 +17,7 @@ export class CapmanEngine {
|
|
|
17
17
|
this.mode = options.mode ?? 'balanced';
|
|
18
18
|
this.llm = options.llm;
|
|
19
19
|
this.baseUrl = options.baseUrl;
|
|
20
|
+
this.environment = options.environment;
|
|
20
21
|
this.auth = options.auth;
|
|
21
22
|
this.headers = options.headers;
|
|
22
23
|
this.threshold = options.threshold ?? 50;
|
|
@@ -27,6 +28,12 @@ export class CapmanEngine {
|
|
|
27
28
|
this.llmCircuitBreakerResetMs = options.llmCircuitBreakerResetMs ?? 60_000;
|
|
28
29
|
this.fuzzyMatch = options.fuzzyMatch ?? false;
|
|
29
30
|
this.fuzzyThreshold = options.fuzzyThreshold ?? 0.4;
|
|
31
|
+
this.bm25K1 = options.bm25K1 ?? 1.5;
|
|
32
|
+
this.bm25B = options.bm25B ?? 0.75;
|
|
33
|
+
this.bm25Index = buildBM25Index(options.manifest.capabilities);
|
|
34
|
+
this.bm25Ceiling = this.calibrateBM25Ceiling();
|
|
35
|
+
this.marginAwareLLM = options.marginAwareLLM ?? false;
|
|
36
|
+
this.adaptiveMargin = options.adaptiveMarginOverride ?? this.calibrateAdaptiveMargin();
|
|
30
37
|
// Cache — default MemoryCache (no filesystem writes), or disabled with false
|
|
31
38
|
// Use FileCache or ComboCache explicitly for persistence across restarts
|
|
32
39
|
this.cache = options.cache === false
|
|
@@ -90,12 +97,16 @@ export class CapmanEngine {
|
|
|
90
97
|
resolvedVia: 'cache',
|
|
91
98
|
totalMs: Date.now() - start,
|
|
92
99
|
};
|
|
100
|
+
const { verdict: cacheVerdict, margin: cacheMargin } = this.computeVerdict(matchWithFreshParams);
|
|
93
101
|
const result = {
|
|
94
102
|
match: matchWithFreshParams,
|
|
95
103
|
resolution,
|
|
96
104
|
resolvedVia: 'cache',
|
|
97
105
|
durationMs: Date.now() - start,
|
|
98
106
|
trace,
|
|
107
|
+
verdict: cacheVerdict,
|
|
108
|
+
margin: cacheMargin,
|
|
109
|
+
missingParams: undefined
|
|
99
110
|
};
|
|
100
111
|
await this.recordLearning(query, matchWithFreshParams, 'cache');
|
|
101
112
|
return result;
|
|
@@ -114,6 +125,7 @@ export class CapmanEngine {
|
|
|
114
125
|
// ── Step 2.5: Apply learning boost ───────────────────────────────────────
|
|
115
126
|
matchResult = await this.applyBoostToMatchResult(query, matchResult, resolvedVia);
|
|
116
127
|
// ── Step 3: Privacy check ────────────────────────────────────────────────
|
|
128
|
+
let privacyFailed = false;
|
|
117
129
|
if (matchResult.capability) {
|
|
118
130
|
const privacyError = checkPrivacy(matchResult.capability, this.auth);
|
|
119
131
|
steps.push({
|
|
@@ -122,8 +134,30 @@ export class CapmanEngine {
|
|
|
122
134
|
durationMs: 0,
|
|
123
135
|
detail: privacyError ?? `level: ${matchResult.capability.privacy.level}`,
|
|
124
136
|
});
|
|
137
|
+
// Warn on deprecated or sunset capabilities — never silently fail
|
|
138
|
+
this.checkCapabilityLifecycle(matchResult.capability);
|
|
139
|
+
// Log when engine mode differs from capability's preferred mode
|
|
140
|
+
this.checkMatchHint(matchResult.capability);
|
|
141
|
+
// Short-circuit: if privacy fails, skip disambiguation to avoid burning an LLM
|
|
142
|
+
// call on a request that _resolve() will block anyway. privacyFailed propagates
|
|
143
|
+
// to Step 4a so the mode guard check is clean and explicit.
|
|
144
|
+
if (privacyError)
|
|
145
|
+
privacyFailed = true;
|
|
125
146
|
}
|
|
126
|
-
// ── Step
|
|
147
|
+
// ── Step 4a: Compute verdict + optional margin-aware LLM disambiguation ──
|
|
148
|
+
let { verdict, margin } = this.computeVerdict(matchResult);
|
|
149
|
+
if (verdict === 'marginal' &&
|
|
150
|
+
this.marginAwareLLM &&
|
|
151
|
+
this.llm &&
|
|
152
|
+
!privacyFailed &&
|
|
153
|
+
(this.mode === 'balanced' || this.mode === 'accurate')) {
|
|
154
|
+
matchResult = await this.disambiguateLLM(query, matchResult, steps);
|
|
155
|
+
// Recompute verdict after disambiguation
|
|
156
|
+
const recomputed = this.computeVerdict(matchResult);
|
|
157
|
+
verdict = recomputed.verdict;
|
|
158
|
+
margin = recomputed.margin;
|
|
159
|
+
}
|
|
160
|
+
// ── Step 4b: Resolve ──────────────────────────────────────────────────────
|
|
127
161
|
const resolveStart = Date.now();
|
|
128
162
|
const resolution = await _resolve(matchResult, matchResult.extractedParams, this.resolveOptions(overrides));
|
|
129
163
|
steps.push({
|
|
@@ -145,6 +179,68 @@ export class CapmanEngine {
|
|
|
145
179
|
await this.cache.set(capKey, matchResult);
|
|
146
180
|
// capKey always starts with 'cap:' — structurally distinct from queryKey
|
|
147
181
|
}
|
|
182
|
+
// ── Step 5b: Compute missingParams ───────────────────────────────────────
|
|
183
|
+
// Spec: LLM attempts extraction first when available. missingParams is last resort.
|
|
184
|
+
let missingParams;
|
|
185
|
+
if (matchResult.capability && resolvedVia !== 'llm') {
|
|
186
|
+
const cap = matchResult.capability;
|
|
187
|
+
const unresolved = cap.params.filter(p => p.source === 'user_query' && p.required
|
|
188
|
+
&& matchResult.extractedParams[p.name] === null);
|
|
189
|
+
if (unresolved.length > 0 && this.llm && this.mode !== 'cheap') {
|
|
190
|
+
// LLM available — attempt targeted param extraction before declaring incomplete
|
|
191
|
+
const skipReason = this.checkLLMAllowed();
|
|
192
|
+
if (!skipReason) {
|
|
193
|
+
try {
|
|
194
|
+
const paramExtractionStart = Date.now();
|
|
195
|
+
const paramDescriptions = unresolved
|
|
196
|
+
.map(p => `- ${p.name}: ${p.description}`)
|
|
197
|
+
.join('\n');
|
|
198
|
+
const paramPrompt = `Extract the following parameters from this user query.\n` +
|
|
199
|
+
`Query: ${JSON.stringify({ user_query: query })}\n\n` +
|
|
200
|
+
`Parameters to extract:\n${paramDescriptions}\n\n` +
|
|
201
|
+
`Respond ONLY with valid JSON: { "params": { "<name>": "<value or null>" } }`;
|
|
202
|
+
const raw = await this.llm(paramPrompt);
|
|
203
|
+
const clean = raw.replace(/```json|```/g, '').trim();
|
|
204
|
+
const parsed = JSON.parse(clean);
|
|
205
|
+
this.recordLLMSuccess();
|
|
206
|
+
steps.push({
|
|
207
|
+
type: 'llm_match',
|
|
208
|
+
status: 'pass',
|
|
209
|
+
durationMs: Date.now() - paramExtractionStart,
|
|
210
|
+
detail: `param extraction: ${unresolved.map(p => p.name).join(', ')}`,
|
|
211
|
+
});
|
|
212
|
+
// Merge LLM-extracted values — validate type before accepting
|
|
213
|
+
for (const p of unresolved) {
|
|
214
|
+
const val = parsed?.params?.[p.name];
|
|
215
|
+
if (val && typeof val === 'string' && val.trim().length > 0) {
|
|
216
|
+
matchResult.extractedParams[p.name] = val.trim();
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
catch (err) {
|
|
221
|
+
const isParseError = err instanceof SyntaxError;
|
|
222
|
+
if (isParseError) {
|
|
223
|
+
// JSON parse failure: refund the rate-limit slot but don't open circuit breaker
|
|
224
|
+
// The llm is reachable - the response format was just bad
|
|
225
|
+
this.llmCallsThisMinute = Math.max(0, this.llmCallsThisMinute - 1);
|
|
226
|
+
}
|
|
227
|
+
else {
|
|
228
|
+
// Hard failure (timeout, network): refund slot and increment fail counter
|
|
229
|
+
this.recordLLMFailure();
|
|
230
|
+
}
|
|
231
|
+
logger.warn(`LLM param extraction failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
232
|
+
// fall through to missingParams below
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
// After LLM attempt (or if skipped/unavailable), report what's still missing
|
|
237
|
+
const stillMissing = cap.params
|
|
238
|
+
.filter(p => p.source === 'user_query' && p.required
|
|
239
|
+
&& matchResult.extractedParams[p.name] === null)
|
|
240
|
+
.map(p => p.name);
|
|
241
|
+
if (stillMissing.length > 0)
|
|
242
|
+
missingParams = stillMissing;
|
|
243
|
+
}
|
|
148
244
|
// ── Step 6: Build reasoning array ────────────────────────────────────────
|
|
149
245
|
const reasoning = [];
|
|
150
246
|
if (matchResult.candidates.length) {
|
|
@@ -189,6 +285,9 @@ export class CapmanEngine {
|
|
|
189
285
|
resolvedVia,
|
|
190
286
|
durationMs: Date.now() - start,
|
|
191
287
|
trace,
|
|
288
|
+
verdict,
|
|
289
|
+
margin,
|
|
290
|
+
missingParams,
|
|
192
291
|
};
|
|
193
292
|
}
|
|
194
293
|
/**
|
|
@@ -216,6 +315,20 @@ export class CapmanEngine {
|
|
|
216
315
|
await this.cache.clear();
|
|
217
316
|
}
|
|
218
317
|
checkManifestVersion(manifest) {
|
|
318
|
+
// ── Schema version check ─────────────────────────────────────────────────
|
|
319
|
+
// schemaVersion tracks manifest format — "1" for v0.6+.
|
|
320
|
+
// Manifests without schemaVersion are pre-v0.6 — warn but allow.
|
|
321
|
+
const CURRENT_SCHEMA_VERSION = '1';
|
|
322
|
+
if (!manifest.schemaVersion) {
|
|
323
|
+
console.warn(`[capman] Manifest is missing schemaVersion — it was generated with capman < 0.6. ` +
|
|
324
|
+
`Regenerate with: npx capman generate`);
|
|
325
|
+
}
|
|
326
|
+
else if (manifest.schemaVersion !== CURRENT_SCHEMA_VERSION) {
|
|
327
|
+
console.warn(`[capman] Manifest schemaVersion "${manifest.schemaVersion}" differs from ` +
|
|
328
|
+
`engine's expected "${CURRENT_SCHEMA_VERSION}". ` +
|
|
329
|
+
`Regenerate with: npx capman generate`);
|
|
330
|
+
}
|
|
331
|
+
// ── Package version check ────────────────────────────────────────────────
|
|
219
332
|
if (!manifest.version)
|
|
220
333
|
return;
|
|
221
334
|
const SEMVER_RE = /^\d+\.\d+\.\d+$/;
|
|
@@ -223,8 +336,8 @@ export class CapmanEngine {
|
|
|
223
336
|
const [mMaj, mMin] = manifest.version.split('.').map(Number);
|
|
224
337
|
const [eMaj, eMin] = VERSION.split('.').map(Number);
|
|
225
338
|
if (mMaj !== eMaj || mMin !== eMin) {
|
|
226
|
-
console.warn(`[capman] Manifest
|
|
227
|
-
`
|
|
339
|
+
console.warn(`[capman] Manifest was generated with capman "${manifest.version}" ` +
|
|
340
|
+
`but engine is "${VERSION}". This is usually fine across patch versions. ` +
|
|
228
341
|
`If you experience unexpected matching issues, regenerate with: npx capman generate`);
|
|
229
342
|
}
|
|
230
343
|
}
|
|
@@ -233,6 +346,42 @@ export class CapmanEngine {
|
|
|
233
346
|
`to engine version "${VERSION}" — version strings are not valid semver.`);
|
|
234
347
|
}
|
|
235
348
|
}
|
|
349
|
+
checkCapabilityLifecycle(capability) {
|
|
350
|
+
const lc = capability.lifecycle;
|
|
351
|
+
if (!lc || lc.status === 'stable' || lc.status === 'beta' || lc.status === 'experimental') {
|
|
352
|
+
if (lc?.status === 'beta') {
|
|
353
|
+
logger.warn(`Capability "${capability.id}" is in beta — behavior may change`);
|
|
354
|
+
}
|
|
355
|
+
if (lc?.status === 'experimental') {
|
|
356
|
+
logger.warn(`Capability "${capability.id}" is experimental — use with caution`);
|
|
357
|
+
}
|
|
358
|
+
return;
|
|
359
|
+
}
|
|
360
|
+
if (lc.status === 'deprecated') {
|
|
361
|
+
const sunsetPassed = lc.sunsetAt && new Date(lc.sunsetAt) < new Date();
|
|
362
|
+
if (sunsetPassed) {
|
|
363
|
+
// Sunset date has passed — strongest warning
|
|
364
|
+
console.warn(`[capman] ⚠️ Capability "${capability.id}" passed its sunset date (${lc.sunsetAt}). ` +
|
|
365
|
+
`It may be removed in a future version.` +
|
|
366
|
+
(lc.successor ? ` Use "${lc.successor}" instead.` : '') +
|
|
367
|
+
(lc.note ? ` Note: ${lc.note}` : ''));
|
|
368
|
+
}
|
|
369
|
+
else {
|
|
370
|
+
logger.warn(`Capability "${capability.id}" is deprecated.` +
|
|
371
|
+
(lc.sunsetAt ? ` Sunset: ${lc.sunsetAt}.` : '') +
|
|
372
|
+
(lc.successor ? ` Use "${lc.successor}" instead.` : '') +
|
|
373
|
+
(lc.note ? ` Note: ${lc.note}` : ''));
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
checkMatchHint(capability) {
|
|
378
|
+
const hint = capability.matchHint?.preferredMode;
|
|
379
|
+
if (!hint || hint === this.mode)
|
|
380
|
+
return;
|
|
381
|
+
// Advisory only — log but never enforce
|
|
382
|
+
logger.warn(`Capability "${capability.id}" prefers mode "${hint}" but engine is in "${this.mode}" mode. ` +
|
|
383
|
+
`Set mode: '${hint}' in EngineOptions to honor this hint.`);
|
|
384
|
+
}
|
|
236
385
|
/**
|
|
237
386
|
* Replaces the active manifest without creating a new engine instance.
|
|
238
387
|
* Useful for hot-reloading manifests in long-running servers without
|
|
@@ -248,11 +397,12 @@ export class CapmanEngine {
|
|
|
248
397
|
async loadManifest(manifest) {
|
|
249
398
|
this.checkManifestVersion(manifest);
|
|
250
399
|
this.manifest = manifest;
|
|
400
|
+
this.bm25Index = buildBM25Index(manifest.capabilities);
|
|
401
|
+
this.bm25Ceiling = this.calibrateBM25Ceiling();
|
|
402
|
+
this.adaptiveMargin = this.calibrateAdaptiveMargin();
|
|
403
|
+
// resolveBaseUrl() reads from this.manifest.servers on each call —
|
|
404
|
+
// server selection updates automatically after loadManifest()
|
|
251
405
|
await this.clearCache();
|
|
252
|
-
// Note: LLM rate limiter state (llmCallsThisMinute, llmConsecutiveFails,
|
|
253
|
-
// llmCircuitOpenAt) is intentionally preserved across manifest reloads.
|
|
254
|
-
// The LLM provider has not changed, so circuit breaker state remains valid.
|
|
255
|
-
// If you need a clean rate limiter state, create a new CapmanEngine instance.
|
|
256
406
|
}
|
|
257
407
|
/**
|
|
258
408
|
* Explain what would happen for a query — without executing it.
|
|
@@ -291,7 +441,8 @@ export class CapmanEngine {
|
|
|
291
441
|
// ── Apply learning boost (same as ask()) ─────────────────────────────────
|
|
292
442
|
matchResult = await this.applyBoostToMatchResult(query, matchResult, resolvedVia);
|
|
293
443
|
// ── Build candidate explanations ─────────────────────────────────────────
|
|
294
|
-
const
|
|
444
|
+
const qTokens = tokenize(query);
|
|
445
|
+
const qWordSet = new Set(qTokens);
|
|
295
446
|
const candidates = matchResult.candidates
|
|
296
447
|
.sort((a, b) => b.score - a.score)
|
|
297
448
|
.map(c => {
|
|
@@ -305,8 +456,8 @@ export class CapmanEngine {
|
|
|
305
456
|
}
|
|
306
457
|
else if (c.score >= 50) {
|
|
307
458
|
const matchedWords = (cap?.examples ?? [])
|
|
308
|
-
.flatMap(e => e
|
|
309
|
-
.filter(w => qWordSet.has(w)
|
|
459
|
+
.flatMap(e => tokenize(e))
|
|
460
|
+
.filter(w => qWordSet.has(w));
|
|
310
461
|
const unique = [...new Set(matchedWords)].slice(0, 3);
|
|
311
462
|
explanation = unique.length
|
|
312
463
|
? `Matched keywords: ${unique.join(', ')} (${c.score}%)`
|
|
@@ -496,6 +647,10 @@ export class CapmanEngine {
|
|
|
496
647
|
const fuzzyOpts = {
|
|
497
648
|
fuzzyMatch: this.fuzzyMatch,
|
|
498
649
|
fuzzyThreshold: this.fuzzyThreshold,
|
|
650
|
+
bm25Index: this.bm25Index,
|
|
651
|
+
bm25Ceiling: this.bm25Ceiling,
|
|
652
|
+
bm25K1: this.bm25K1,
|
|
653
|
+
bm25B: this.bm25B,
|
|
499
654
|
};
|
|
500
655
|
switch (this.mode) {
|
|
501
656
|
case 'cheap': {
|
|
@@ -604,7 +759,11 @@ export class CapmanEngine {
|
|
|
604
759
|
break;
|
|
605
760
|
}
|
|
606
761
|
}
|
|
607
|
-
|
|
762
|
+
if (matchResult === undefined) {
|
|
763
|
+
const exhaustive = this.mode;
|
|
764
|
+
throw new Error(`_runMatch: unhandled MatchMode "${exhaustive}"`);
|
|
765
|
+
}
|
|
766
|
+
return { matchResult, resolvedVia };
|
|
608
767
|
}
|
|
609
768
|
/**
|
|
610
769
|
* Applies learning boost to a MatchResult and returns the updated result.
|
|
@@ -663,7 +822,7 @@ export class CapmanEngine {
|
|
|
663
822
|
const stats = await this.learning.getStats();
|
|
664
823
|
if (!stats || Object.keys(stats.index).length === 0)
|
|
665
824
|
return candidates;
|
|
666
|
-
const qWords = query
|
|
825
|
+
const qWords = tokenize(query);
|
|
667
826
|
if (qWords.length === 0)
|
|
668
827
|
return candidates;
|
|
669
828
|
return candidates.map(candidate => {
|
|
@@ -689,10 +848,26 @@ export class CapmanEngine {
|
|
|
689
848
|
};
|
|
690
849
|
});
|
|
691
850
|
}
|
|
851
|
+
/**
|
|
852
|
+
* Resolves the effective baseUrl from manifest.servers[] or EngineOptions.baseUrl.
|
|
853
|
+
* Priority: environment-matched server > first server > explicit baseUrl > undefined
|
|
854
|
+
*/
|
|
855
|
+
resolveBaseUrl() {
|
|
856
|
+
const servers = this.manifest.servers;
|
|
857
|
+
if (!servers?.length)
|
|
858
|
+
return this.baseUrl;
|
|
859
|
+
if (this.environment) {
|
|
860
|
+
const match = servers.find(s => s.environment === this.environment);
|
|
861
|
+
if (match)
|
|
862
|
+
return match.url.replace(/\/$/, '');
|
|
863
|
+
}
|
|
864
|
+
// Fallback to first server
|
|
865
|
+
return servers[0].url.replace(/\/$/, '');
|
|
866
|
+
}
|
|
692
867
|
// ── Private helpers ────────────────────────────────────────────────────────
|
|
693
868
|
resolveOptions(overrides = {}) {
|
|
694
869
|
return {
|
|
695
|
-
baseUrl: this.
|
|
870
|
+
baseUrl: this.resolveBaseUrl(),
|
|
696
871
|
auth: this.auth,
|
|
697
872
|
headers: this.headers,
|
|
698
873
|
...overrides,
|
|
@@ -711,6 +886,131 @@ export class CapmanEngine {
|
|
|
711
886
|
timestamp: new Date().toISOString(),
|
|
712
887
|
});
|
|
713
888
|
}
|
|
889
|
+
calibrateBM25Ceiling() {
|
|
890
|
+
return _calibrateCeiling(this.manifest.capabilities, this.bm25Index, this.bm25K1, this.bm25B);
|
|
891
|
+
}
|
|
892
|
+
/**
|
|
893
|
+
* Calibrates the adaptive margin threshold from the manifest's own score
|
|
894
|
+
* distribution. Runs each capability's first example against all other
|
|
895
|
+
* capabilities to find the typical inter-capability score spread.
|
|
896
|
+
* Dense overlapping vocabulary → lower margin (harder to separate).
|
|
897
|
+
* Sparse vocabulary → higher margin (easier to separate).
|
|
898
|
+
*
|
|
899
|
+
* Complexity: O(capabilities²) — runs at constructor time and on loadManifest().
|
|
900
|
+
* For manifests with ≤100 capabilities this is negligible (<10ms).
|
|
901
|
+
* For very large manifests (500+ capabilities), consider passing
|
|
902
|
+
* `adaptiveMarginOverride` to skip calibration.
|
|
903
|
+
*/
|
|
904
|
+
calibrateAdaptiveMargin() {
|
|
905
|
+
if (this.manifest.capabilities.length < 2)
|
|
906
|
+
return 20;
|
|
907
|
+
const margins = [];
|
|
908
|
+
const fuzzyOpts = {
|
|
909
|
+
fuzzyMatch: false, // calibration uses keyword only — deterministic
|
|
910
|
+
bm25Index: this.bm25Index,
|
|
911
|
+
bm25Ceiling: this.bm25Ceiling,
|
|
912
|
+
bm25K1: this.bm25K1,
|
|
913
|
+
bm25B: this.bm25B,
|
|
914
|
+
};
|
|
915
|
+
for (const cap of this.manifest.capabilities) {
|
|
916
|
+
if (!cap.examples?.length)
|
|
917
|
+
continue;
|
|
918
|
+
// Use all examples and take the maximum margin — same rationale as
|
|
919
|
+
// calibrateBM25Ceiling(): a weak first example skews the calibration.
|
|
920
|
+
for (const example of cap.examples) {
|
|
921
|
+
const result = _match(example, this.manifest, fuzzyOpts);
|
|
922
|
+
const sorted = [...result.candidates].sort((a, b) => b.score - a.score);
|
|
923
|
+
if (sorted.length >= 2) {
|
|
924
|
+
margins.push(sorted[0].score - sorted[1].score);
|
|
925
|
+
}
|
|
926
|
+
}
|
|
927
|
+
}
|
|
928
|
+
if (margins.length === 0)
|
|
929
|
+
return 20;
|
|
930
|
+
// Use 25th percentile of margins as the threshold — manifests where
|
|
931
|
+
// capabilities are naturally close together get a tighter threshold
|
|
932
|
+
margins.sort((a, b) => a - b);
|
|
933
|
+
const p25 = margins[Math.floor(margins.length * 0.25)];
|
|
934
|
+
return Math.max(10, Math.min(30, Math.round(p25 * 0.6)));
|
|
935
|
+
}
|
|
936
|
+
computeVerdict(matchResult) {
|
|
937
|
+
if (!matchResult.capability)
|
|
938
|
+
return { verdict: 'uncertain', margin: 0 };
|
|
939
|
+
const sorted = [...matchResult.candidates].sort((a, b) => b.score - a.score);
|
|
940
|
+
const best = sorted[0]?.score ?? 0;
|
|
941
|
+
const second = sorted[1]?.score ?? 0;
|
|
942
|
+
const margin = best - second;
|
|
943
|
+
if (best < 60)
|
|
944
|
+
return { verdict: 'uncertain', margin };
|
|
945
|
+
if (margin < this.adaptiveMargin)
|
|
946
|
+
return { verdict: 'marginal', margin };
|
|
947
|
+
return { verdict: 'clear', margin };
|
|
948
|
+
}
|
|
949
|
+
/**
|
|
950
|
+
* Targeted disambiguation between top-2 candidates.
|
|
951
|
+
* Sends ~200 tokens instead of full manifest (~4000 tokens) — 93% cost reduction.
|
|
952
|
+
* Returns updated matchResult with LLM-preferred winner, or original on failure.
|
|
953
|
+
*/
|
|
954
|
+
async disambiguateLLM(query, matchResult, steps) {
|
|
955
|
+
if (!this.llm)
|
|
956
|
+
return matchResult;
|
|
957
|
+
const sorted = [...matchResult.candidates]
|
|
958
|
+
.sort((a, b) => b.score - a.score)
|
|
959
|
+
.slice(0, 2);
|
|
960
|
+
if (sorted.length < 2)
|
|
961
|
+
return matchResult;
|
|
962
|
+
const capA = this.manifest.capabilities.find(c => c.id === sorted[0].capabilityId);
|
|
963
|
+
const capB = this.manifest.capabilities.find(c => c.id === sorted[1].capabilityId);
|
|
964
|
+
if (!capA || !capB)
|
|
965
|
+
return matchResult;
|
|
966
|
+
const skipReason = this.checkLLMAllowed();
|
|
967
|
+
if (skipReason) {
|
|
968
|
+
logger.warn(`Disambiguation LLM skipped — ${skipReason}`);
|
|
969
|
+
steps.push({ type: 'llm_match', status: 'skip', durationMs: 0, detail: `disambiguation skipped: ${skipReason}` });
|
|
970
|
+
return matchResult;
|
|
971
|
+
}
|
|
972
|
+
const prompt = `Two capabilities are close matches for this query. Pick the best one.
|
|
973
|
+
|
|
974
|
+
Query: ${JSON.stringify({ user_query: query })}
|
|
975
|
+
|
|
976
|
+
Option A: ${capA.id} — ${sanitizeForPrompt(capA.description, 150)}
|
|
977
|
+
Option B: ${capB.id} — ${sanitizeForPrompt(capB.description, 150)}
|
|
978
|
+
|
|
979
|
+
Respond ONLY with valid JSON:
|
|
980
|
+
{ "winner": "<capability_id>", "confidence": <0-100>, "reasoning": "<one sentence>" }`;
|
|
981
|
+
const t = Date.now();
|
|
982
|
+
try {
|
|
983
|
+
const raw = await this.llm(prompt);
|
|
984
|
+
const clean = raw.replace(/```json|```/g, '').trim();
|
|
985
|
+
const parsed = JSON.parse(clean);
|
|
986
|
+
this.recordLLMSuccess();
|
|
987
|
+
const winner = this.manifest.capabilities.find(c => c.id === parsed.winner);
|
|
988
|
+
if (!winner) {
|
|
989
|
+
steps.push({ type: 'llm_match', status: 'fail', durationMs: Date.now() - t, detail: 'disambiguation returned unknown id' });
|
|
990
|
+
return matchResult;
|
|
991
|
+
}
|
|
992
|
+
steps.push({ type: 'llm_match', status: 'pass', durationMs: Date.now() - t, detail: `disambiguation: ${winner.id} (${parsed.confidence}%)` });
|
|
993
|
+
const confidence = typeof parsed.confidence === 'number' && !isNaN(parsed.confidence)
|
|
994
|
+
? Math.min(100, Math.max(0, Math.round(parsed.confidence)))
|
|
995
|
+
: matchResult.confidence; // fallback to original if LLM returned bad value
|
|
996
|
+
return {
|
|
997
|
+
...matchResult,
|
|
998
|
+
capability: winner,
|
|
999
|
+
confidence,
|
|
1000
|
+
intent: resolverToIntent(winner),
|
|
1001
|
+
extractedParams: extractParams(query, winner),
|
|
1002
|
+
candidates: matchResult.candidates.map(c => ({ ...c, matched: c.capabilityId === winner.id })),
|
|
1003
|
+
reasoning: parsed.reasoning ?? `Disambiguated to "${winner.id}"`,
|
|
1004
|
+
};
|
|
1005
|
+
}
|
|
1006
|
+
catch (err) {
|
|
1007
|
+
const isParseError = err instanceof LLMParseError;
|
|
1008
|
+
if (!isParseError)
|
|
1009
|
+
this.recordLLMFailure();
|
|
1010
|
+
steps.push({ type: 'llm_match', status: 'fail', durationMs: Date.now() - t, detail: String(err) });
|
|
1011
|
+
return matchResult;
|
|
1012
|
+
}
|
|
1013
|
+
}
|
|
714
1014
|
}
|
|
715
1015
|
/** Maximum allowed query length in characters. Queries exceeding this throw RangeError. */
|
|
716
1016
|
CapmanEngine.MAX_QUERY_LENGTH = 1000;
|
package/dist/esm/generator.js
CHANGED
|
@@ -5,10 +5,14 @@ import { validateConfig, validateManifest } from './schema';
|
|
|
5
5
|
import { logger } from './logger';
|
|
6
6
|
export function generate(config) {
|
|
7
7
|
return {
|
|
8
|
+
schemaVersion: '1',
|
|
8
9
|
version: VERSION,
|
|
9
10
|
app: config.app,
|
|
10
11
|
generatedAt: new Date().toISOString(),
|
|
11
|
-
capabilities: config.capabilities.map(cap => ({ ...cap
|
|
12
|
+
capabilities: config.capabilities.map(cap => ({ ...cap })),
|
|
13
|
+
...(config.info ? { info: config.info } : {}),
|
|
14
|
+
...(config.tagRegistry ? { tagRegistry: config.tagRegistry } : {}),
|
|
15
|
+
...(config.servers ? { servers: config.servers } : {}),
|
|
12
16
|
};
|
|
13
17
|
}
|
|
14
18
|
export function loadConfig(configPath) {
|
|
@@ -33,6 +37,10 @@ export function loadConfig(configPath) {
|
|
|
33
37
|
// Use a CJS config file or convert with: module.exports = { ... }
|
|
34
38
|
// Full ESM config support is planned for v0.5.
|
|
35
39
|
try {
|
|
40
|
+
// Bust the module cache before loading — require() caches by resolved path,
|
|
41
|
+
// so a second call without this returns the stale version from the first call.
|
|
42
|
+
// This matters in watch mode and test suites that change config between calls.
|
|
43
|
+
delete require.cache[require.resolve(resolved)];
|
|
36
44
|
const mod = require(resolved);
|
|
37
45
|
raw = mod.default ?? mod;
|
|
38
46
|
}
|
|
@@ -80,7 +88,12 @@ export function writeManifest(manifest, outputPath = 'manifest.json') {
|
|
|
80
88
|
throw new Error(`writeManifest: output path "${outputPath}" resolves outside the working directory.\n` +
|
|
81
89
|
`Resolved: ${resolved}\nAllowed: ${cwd}`);
|
|
82
90
|
}
|
|
83
|
-
|
|
91
|
+
// Write atomically via tmp → rename — same pattern used by FileCache and
|
|
92
|
+
// FileLearningStore. A crash or SIGKILL mid-write leaves the .tmp file, not
|
|
93
|
+
// a truncated manifest.json, so the next readManifest() can still parse it.
|
|
94
|
+
const tmp = `${resolved}.tmp`;
|
|
95
|
+
fs.writeFileSync(tmp, JSON.stringify(manifest, null, 2));
|
|
96
|
+
fs.renameSync(tmp, resolved);
|
|
84
97
|
return resolved;
|
|
85
98
|
}
|
|
86
99
|
export function readManifest(manifestPath = 'manifest.json') {
|
|
@@ -122,14 +135,23 @@ export function validate(manifest) {
|
|
|
122
135
|
return { valid: errors.length === 0, errors, warnings };
|
|
123
136
|
}
|
|
124
137
|
export function generateStarterConfig() {
|
|
125
|
-
return `// capman.config.js
|
|
126
|
-
//
|
|
127
|
-
// Replace the examples below with your own app's capabilities.
|
|
138
|
+
return `// capman.config.js
|
|
139
|
+
// Auto-generated starter config — edit before use
|
|
128
140
|
|
|
129
141
|
module.exports = {
|
|
130
|
-
app: '
|
|
142
|
+
app: 'my-app',
|
|
131
143
|
baseUrl: 'https://api.your-app.com',
|
|
132
144
|
|
|
145
|
+
// Optional metadata block — used for documentation and provenance
|
|
146
|
+
info: {
|
|
147
|
+
title: 'My App',
|
|
148
|
+
description: 'Brief description of what this app does',
|
|
149
|
+
version: '1.0.0',
|
|
150
|
+
homepage: 'https://your-app.com',
|
|
151
|
+
contact: { name: 'Your Name', email: 'you@your-app.com' },
|
|
152
|
+
license: { name: 'MIT' },
|
|
153
|
+
},
|
|
154
|
+
|
|
133
155
|
capabilities: [
|
|
134
156
|
{
|
|
135
157
|
id: 'get_resource',
|
package/dist/esm/index.d.ts
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
export { setLogLevel } from './logger';
|
|
2
2
|
export type { LogLevel } from './logger';
|
|
3
|
-
export type { Capability, CapabilityParam, CapmanConfig, Manifest, MatchResult, ExecutionTrace, TraceStep, MatchCandidate, ResolveResult, ApiCallResult, ValidationResult, Resolver, ApiResolver, NavResolver, HybridResolver, PrivacyScope, ResolverType, HttpMethod, ExplainResult, ExplainCandidate, } from './types';
|
|
3
|
+
export type { Capability, CapabilityParam, CapmanConfig, Manifest, MatchResult, ExecutionTrace, TraceStep, MatchCandidate, ResolveResult, ApiCallResult, ValidationResult, Resolver, ApiResolver, NavResolver, HybridResolver, PrivacyScope, ResolverType, HttpMethod, ExplainResult, ExplainCandidate, ManifestInfo, Server, LifecycleInfo, LifecycleStatus, CapabilityError, Endpoint, ParamType, MatchHint, } from './types';
|
|
4
4
|
export { generate, loadConfig, writeManifest, readManifest, validate, generateStarterConfig, } from './generator';
|
|
5
5
|
export { match, matchWithLLM, extractParams, } from './matcher';
|
|
6
6
|
export { LLMParseError } from './matcher';
|
|
7
7
|
export type { LLMMatcherOptions } from './matcher';
|
|
8
|
+
export { TYPE_PATTERNS } from './matcher';
|
|
9
|
+
export { filterByTags } from './matcher';
|
|
8
10
|
export { resolve } from './resolver';
|
|
9
11
|
export type { ResolveOptions, AuthContext } from './resolver';
|
|
10
12
|
export { CapmanEngine } from './engine';
|
package/dist/esm/index.js
CHANGED
|
@@ -2,6 +2,8 @@ export { setLogLevel } from './logger';
|
|
|
2
2
|
export { generate, loadConfig, writeManifest, readManifest, validate, generateStarterConfig, } from './generator';
|
|
3
3
|
export { match, matchWithLLM, extractParams, } from './matcher';
|
|
4
4
|
export { LLMParseError } from './matcher';
|
|
5
|
+
export { TYPE_PATTERNS } from './matcher';
|
|
6
|
+
export { filterByTags } from './matcher';
|
|
5
7
|
export { resolve } from './resolver';
|
|
6
8
|
// ─── Engine (recommended API) ─────────────────────────────────────────────────
|
|
7
9
|
export { CapmanEngine } from './engine';
|
package/dist/esm/learning.d.ts
CHANGED
|
@@ -6,6 +6,13 @@ export interface LearningEntry {
|
|
|
6
6
|
extractedParams: Record<string, string | null>;
|
|
7
7
|
resolvedVia: 'keyword' | 'llm' | 'cache';
|
|
8
8
|
timestamp: string;
|
|
9
|
+
/**
|
|
10
|
+
* Confidence-derived weight stored at record time (confidence / 100, floor 0.1).
|
|
11
|
+
* Used by subtract() to reverse the exact contribution made by update(),
|
|
12
|
+
* preventing index drift when high-confidence entries are pruned.
|
|
13
|
+
* Optional for backwards-compatibility with persisted entries written before v0.5.5.
|
|
14
|
+
*/
|
|
15
|
+
weight?: number;
|
|
9
16
|
}
|
|
10
17
|
export interface KeywordStats {
|
|
11
18
|
/** keyword → Map of capabilityId → hit count */
|