capman 0.4.2 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +153 -0
- package/CODEBASE.md +393 -0
- package/README.md +1 -1
- package/bin/capman.js +11 -724
- package/bin/lib/cmd-demo.js +180 -0
- package/bin/lib/cmd-explain.js +72 -0
- package/bin/lib/cmd-generate.js +280 -0
- package/bin/lib/cmd-help.js +26 -0
- package/bin/lib/cmd-init.js +19 -0
- package/bin/lib/cmd-inspect.js +33 -0
- package/bin/lib/cmd-run.js +71 -0
- package/bin/lib/cmd-validate.js +32 -0
- package/bin/lib/shared.js +77 -0
- package/dist/cjs/cache.d.ts.map +1 -1
- package/dist/cjs/cache.js +8 -2
- package/dist/cjs/cache.js.map +1 -1
- package/dist/cjs/engine.d.ts +58 -1
- package/dist/cjs/engine.d.ts.map +1 -1
- package/dist/cjs/engine.js +312 -12
- package/dist/cjs/engine.js.map +1 -1
- package/dist/cjs/generator.d.ts.map +1 -1
- package/dist/cjs/generator.js +4 -0
- package/dist/cjs/generator.js.map +1 -1
- package/dist/cjs/index.d.ts +1 -1
- package/dist/cjs/index.d.ts.map +1 -1
- package/dist/cjs/index.js.map +1 -1
- package/dist/cjs/learning.d.ts.map +1 -1
- package/dist/cjs/learning.js +7 -2
- package/dist/cjs/learning.js.map +1 -1
- package/dist/cjs/matcher.d.ts.map +1 -1
- package/dist/cjs/matcher.js +23 -27
- package/dist/cjs/matcher.js.map +1 -1
- package/dist/cjs/parser.js +2 -1
- package/dist/cjs/parser.js.map +1 -1
- package/dist/cjs/resolver.js +6 -2
- package/dist/cjs/resolver.js.map +1 -1
- package/dist/cjs/types.d.ts +27 -0
- package/dist/cjs/types.d.ts.map +1 -1
- package/dist/cjs/version.d.ts +1 -1
- package/dist/cjs/version.js +1 -1
- package/dist/esm/cache.d.ts +49 -0
- package/dist/esm/cache.js +8 -2
- package/dist/esm/engine.d.ts +138 -0
- package/dist/esm/engine.js +312 -12
- package/dist/esm/generator.d.ts +7 -0
- package/dist/esm/generator.js +4 -0
- package/dist/esm/index.d.ts +47 -0
- package/dist/esm/learning.d.ts +55 -0
- package/dist/esm/learning.js +7 -2
- package/dist/esm/logger.d.ts +21 -0
- package/dist/esm/matcher.d.ts +6 -0
- package/dist/esm/matcher.js +23 -27
- package/dist/esm/parser.d.ts +10 -0
- package/dist/esm/parser.js +2 -1
- package/dist/esm/resolver.d.ts +21 -0
- package/dist/esm/resolver.js +6 -2
- package/dist/esm/schema.d.ts +740 -0
- package/dist/esm/types.d.ts +136 -0
- package/dist/esm/version.d.ts +1 -0
- package/dist/esm/version.js +1 -1
- package/package.json +5 -3
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import type { Manifest, MatchResult, ResolveResult, ExecutionTrace, ExplainResult } from './types';
|
|
2
|
+
import type { LLMMatcherOptions } from './matcher';
|
|
3
|
+
import type { ResolveOptions, AuthContext } from './resolver';
|
|
4
|
+
import type { CacheStore } from './cache';
|
|
5
|
+
import type { LearningStore } from './learning';
|
|
6
|
+
import type { MatchMode } from './index';
|
|
7
|
+
export interface EngineOptions {
|
|
8
|
+
/** The capability manifest to use */
|
|
9
|
+
manifest: Manifest;
|
|
10
|
+
/**
|
|
11
|
+
* Matching mode
|
|
12
|
+
* - 'cheap' — keyword only, no LLM
|
|
13
|
+
* - 'balanced' — keyword first, LLM fallback (default)
|
|
14
|
+
* - 'accurate' — LLM first, keyword fallback
|
|
15
|
+
*/
|
|
16
|
+
mode?: MatchMode;
|
|
17
|
+
/** LLM function for accurate/balanced matching */
|
|
18
|
+
llm?: LLMMatcherOptions['llm'];
|
|
19
|
+
/** Cache store — defaults to MemoryCache. Use FileCache or ComboCache for persistence. */
|
|
20
|
+
cache?: CacheStore | false;
|
|
21
|
+
/** Learning store — defaults to MemoryLearningStore. Use FileLearningStore for persistence. */
|
|
22
|
+
learning?: LearningStore | false;
|
|
23
|
+
/** Base URL for API resolvers */
|
|
24
|
+
baseUrl?: string;
|
|
25
|
+
/** Auth context for privacy-scoped capabilities */
|
|
26
|
+
auth?: AuthContext;
|
|
27
|
+
/** Custom headers for API calls */
|
|
28
|
+
headers?: Record<string, string>;
|
|
29
|
+
/** Confidence threshold for keyword matcher (default: 50) */
|
|
30
|
+
threshold?: number;
|
|
31
|
+
/**
|
|
32
|
+
* Maximum LLM calls per minute in balanced/accurate mode.
|
|
33
|
+
* After limit is hit, falls back to keyword result.
|
|
34
|
+
* @default 60
|
|
35
|
+
*/
|
|
36
|
+
maxLLMCallsPerMinute?: number;
|
|
37
|
+
/**
|
|
38
|
+
* Minimum milliseconds between consecutive LLM calls.
|
|
39
|
+
* Useful for free-tier models with burst limits.
|
|
40
|
+
* @default 0
|
|
41
|
+
*/
|
|
42
|
+
llmCooldownMs?: number;
|
|
43
|
+
/**
|
|
44
|
+
* Maximum consecutive LLM failures before circuit breaker opens.
|
|
45
|
+
* When open, LLM calls are skipped for llmCircuitBreakerResetMs.
|
|
46
|
+
* @default 3
|
|
47
|
+
*/
|
|
48
|
+
llmCircuitBreakerThreshold?: number;
|
|
49
|
+
/**
|
|
50
|
+
* Milliseconds to wait before retrying LLM after circuit breaker opens.
|
|
51
|
+
* @default 60000
|
|
52
|
+
*/
|
|
53
|
+
llmCircuitBreakerResetMs?: number;
|
|
54
|
+
}
|
|
55
|
+
export interface EngineResult {
|
|
56
|
+
match: MatchResult;
|
|
57
|
+
resolution: ResolveResult;
|
|
58
|
+
resolvedVia: 'cache' | 'keyword' | 'llm';
|
|
59
|
+
durationMs: number;
|
|
60
|
+
/** Full execution trace — always present */
|
|
61
|
+
trace: ExecutionTrace;
|
|
62
|
+
}
|
|
63
|
+
export declare class CapmanEngine {
|
|
64
|
+
private manifest;
|
|
65
|
+
private mode;
|
|
66
|
+
private llm?;
|
|
67
|
+
private cache;
|
|
68
|
+
private learning;
|
|
69
|
+
private baseUrl?;
|
|
70
|
+
private auth?;
|
|
71
|
+
private headers?;
|
|
72
|
+
private threshold;
|
|
73
|
+
private maxLLMCallsPerMinute;
|
|
74
|
+
private llmCooldownMs;
|
|
75
|
+
private llmCircuitBreakerThreshold;
|
|
76
|
+
private llmCircuitBreakerResetMs;
|
|
77
|
+
private llmCallsThisMinute;
|
|
78
|
+
private llmWindowStart;
|
|
79
|
+
private llmLastCallAt;
|
|
80
|
+
private llmConsecutiveFails;
|
|
81
|
+
private llmCircuitOpenAt;
|
|
82
|
+
constructor(options: EngineOptions);
|
|
83
|
+
/**
|
|
84
|
+
* Ask the engine a natural language query.
|
|
85
|
+
* Automatically handles caching, matching, resolution, and learning.
|
|
86
|
+
*
|
|
87
|
+
* @example
|
|
88
|
+
* const engine = new CapmanEngine({ manifest, llm: myLLM })
|
|
89
|
+
* const result = await engine.ask("Check availability for blue jacket")
|
|
90
|
+
* console.log(result.match.capability?.id) // check_product_availability
|
|
91
|
+
* console.log(result.resolution.apiCalls) // [{ url: '...', method: 'GET' }]
|
|
92
|
+
* console.log(result.resolvedVia) // 'keyword' | 'llm' | 'cache'
|
|
93
|
+
*/
|
|
94
|
+
ask(query: string, overrides?: Partial<ResolveOptions>): Promise<EngineResult>;
|
|
95
|
+
/**
|
|
96
|
+
* Get stats from the learning store.
|
|
97
|
+
* Shows which capabilities are most used, LLM vs keyword ratio, cache hit rate.
|
|
98
|
+
*/
|
|
99
|
+
getStats(): Promise<import("./learning").KeywordStats | null>;
|
|
100
|
+
/**
|
|
101
|
+
* Get the most frequently matched capabilities.
|
|
102
|
+
*/
|
|
103
|
+
getTopCapabilities(limit?: number): Promise<{
|
|
104
|
+
id: string;
|
|
105
|
+
hits: number;
|
|
106
|
+
}[]>;
|
|
107
|
+
/**
|
|
108
|
+
* Clear the cache.
|
|
109
|
+
*/
|
|
110
|
+
clearCache(): Promise<void>;
|
|
111
|
+
/**
|
|
112
|
+
* Explain what would happen for a query — without executing it.
|
|
113
|
+
* Shows matched capability, all candidate scores with reasoning,
|
|
114
|
+
* and what action would be taken.
|
|
115
|
+
*
|
|
116
|
+
* @example
|
|
117
|
+
* const explanation = await engine.explain('track order 1234')
|
|
118
|
+
* console.log(explanation.matched.reasoning)
|
|
119
|
+
* console.log(explanation.wouldExecute.action)
|
|
120
|
+
* console.log(explanation.candidates)
|
|
121
|
+
*/
|
|
122
|
+
explain(query: string): Promise<ExplainResult>;
|
|
123
|
+
/**
|
|
124
|
+
* Checks all rate limiting and circuit breaker conditions.
|
|
125
|
+
* Returns null if LLM call is allowed, or a skip reason string if it should be skipped.
|
|
126
|
+
*/
|
|
127
|
+
private checkLLMAllowed;
|
|
128
|
+
/**
|
|
129
|
+
* Records a successful LLM call — updates rate limit counters.
|
|
130
|
+
*/
|
|
131
|
+
private recordLLMSuccess;
|
|
132
|
+
/**
|
|
133
|
+
* Records a failed LLM call — may open the circuit breaker.
|
|
134
|
+
*/
|
|
135
|
+
private recordLLMFailure;
|
|
136
|
+
private resolveOptions;
|
|
137
|
+
private recordLearning;
|
|
138
|
+
}
|
package/dist/esm/engine.js
CHANGED
|
@@ -6,6 +6,12 @@ import { MemoryCache, normalizeQuery } from './cache';
|
|
|
6
6
|
// ─── CapmanEngine ─────────────────────────────────────────────────────────────
|
|
7
7
|
export class CapmanEngine {
|
|
8
8
|
constructor(options) {
|
|
9
|
+
// ── LLM rate limiting state ────────────────────────────────────────────────
|
|
10
|
+
this.llmCallsThisMinute = 0;
|
|
11
|
+
this.llmWindowStart = Date.now();
|
|
12
|
+
this.llmLastCallAt = 0;
|
|
13
|
+
this.llmConsecutiveFails = 0;
|
|
14
|
+
this.llmCircuitOpenAt = 0;
|
|
9
15
|
this.manifest = options.manifest;
|
|
10
16
|
this.mode = options.mode ?? 'balanced';
|
|
11
17
|
this.llm = options.llm;
|
|
@@ -13,6 +19,10 @@ export class CapmanEngine {
|
|
|
13
19
|
this.auth = options.auth;
|
|
14
20
|
this.headers = options.headers;
|
|
15
21
|
this.threshold = options.threshold ?? 50;
|
|
22
|
+
this.maxLLMCallsPerMinute = options.maxLLMCallsPerMinute ?? 60;
|
|
23
|
+
this.llmCooldownMs = options.llmCooldownMs ?? 0;
|
|
24
|
+
this.llmCircuitBreakerThreshold = options.llmCircuitBreakerThreshold ?? 3;
|
|
25
|
+
this.llmCircuitBreakerResetMs = options.llmCircuitBreakerResetMs ?? 60000;
|
|
16
26
|
// Cache — default MemoryCache (no filesystem writes), or disabled with false
|
|
17
27
|
// Use FileCache or ComboCache explicitly for persistence across restarts
|
|
18
28
|
this.cache = options.cache === false
|
|
@@ -51,7 +61,7 @@ export class CapmanEngine {
|
|
|
51
61
|
const resolution = await _resolve(cached.result, cached.result.extractedParams, this.resolveOptions(overrides));
|
|
52
62
|
const trace = {
|
|
53
63
|
query,
|
|
54
|
-
candidates: cached.result.candidates
|
|
64
|
+
candidates: cached.result.candidates,
|
|
55
65
|
reasoning: [`Served from cache (original: ${cached.result.reasoning})`],
|
|
56
66
|
steps,
|
|
57
67
|
resolvedVia: 'cache',
|
|
@@ -83,10 +93,30 @@ export class CapmanEngine {
|
|
|
83
93
|
}
|
|
84
94
|
case 'accurate': {
|
|
85
95
|
if (this.llm) {
|
|
86
|
-
const
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
96
|
+
const skipReason = this.checkLLMAllowed();
|
|
97
|
+
if (skipReason) {
|
|
98
|
+
logger.warn(`LLM skipped — ${skipReason} — falling back to keyword`);
|
|
99
|
+
const t = Date.now();
|
|
100
|
+
matchResult = _match(query, this.manifest);
|
|
101
|
+
steps.push({ type: 'keyword_match', status: 'pass', durationMs: Date.now() - t, detail: `llm skipped: ${skipReason}` });
|
|
102
|
+
}
|
|
103
|
+
else {
|
|
104
|
+
const t = Date.now();
|
|
105
|
+
try {
|
|
106
|
+
matchResult = await _matchWithLLM(query, this.manifest, { llm: this.llm });
|
|
107
|
+
this.recordLLMSuccess();
|
|
108
|
+
resolvedVia = 'llm';
|
|
109
|
+
steps.push({ type: 'llm_match', status: 'pass', durationMs: Date.now() - t, detail: `confidence: ${matchResult.confidence}%` });
|
|
110
|
+
}
|
|
111
|
+
catch (err) {
|
|
112
|
+
this.recordLLMFailure();
|
|
113
|
+
logger.warn(`LLM call failed — falling back to keyword: ${err}`);
|
|
114
|
+
const t2 = Date.now();
|
|
115
|
+
matchResult = _match(query, this.manifest);
|
|
116
|
+
steps.push({ type: 'llm_match', status: 'fail', durationMs: Date.now() - t, detail: String(err) });
|
|
117
|
+
steps.push({ type: 'keyword_match', status: 'pass', durationMs: Date.now() - t2, detail: 'fallback after llm failure' });
|
|
118
|
+
}
|
|
119
|
+
}
|
|
90
120
|
}
|
|
91
121
|
else {
|
|
92
122
|
logger.warn('accurate mode requires llm — falling back to keyword');
|
|
@@ -105,11 +135,28 @@ export class CapmanEngine {
|
|
|
105
135
|
matchResult = keywordResult;
|
|
106
136
|
}
|
|
107
137
|
else {
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
138
|
+
const skipReason = this.checkLLMAllowed();
|
|
139
|
+
if (skipReason) {
|
|
140
|
+
logger.warn(`LLM skipped — ${skipReason}`);
|
|
141
|
+
steps.push({ type: 'llm_match', status: 'skip', durationMs: 0, detail: skipReason });
|
|
142
|
+
matchResult = keywordResult;
|
|
143
|
+
}
|
|
144
|
+
else {
|
|
145
|
+
logger.info(`Low confidence (${keywordResult.confidence}%) — escalating to LLM`);
|
|
146
|
+
const t2 = Date.now();
|
|
147
|
+
try {
|
|
148
|
+
matchResult = await _matchWithLLM(query, this.manifest, { llm: this.llm });
|
|
149
|
+
this.recordLLMSuccess();
|
|
150
|
+
resolvedVia = 'llm';
|
|
151
|
+
steps.push({ type: 'llm_match', status: 'pass', durationMs: Date.now() - t2, detail: `confidence: ${matchResult.confidence}%` });
|
|
152
|
+
}
|
|
153
|
+
catch (err) {
|
|
154
|
+
this.recordLLMFailure();
|
|
155
|
+
logger.warn(`LLM call failed — falling back to keyword: ${err}`);
|
|
156
|
+
steps.push({ type: 'llm_match', status: 'fail', durationMs: Date.now() - t2, detail: String(err) });
|
|
157
|
+
matchResult = keywordResult;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
113
160
|
}
|
|
114
161
|
break;
|
|
115
162
|
}
|
|
@@ -140,7 +187,7 @@ export class CapmanEngine {
|
|
|
140
187
|
});
|
|
141
188
|
// ── Step 6: Build reasoning array ────────────────────────────────────────
|
|
142
189
|
const reasoning = [];
|
|
143
|
-
if (matchResult.candidates
|
|
190
|
+
if (matchResult.candidates.length) {
|
|
144
191
|
const winner = matchResult.candidates.find(c => c.matched);
|
|
145
192
|
const rejected = matchResult.candidates
|
|
146
193
|
.filter(c => !c.matched && c.score > 0)
|
|
@@ -167,7 +214,7 @@ export class CapmanEngine {
|
|
|
167
214
|
await this.recordLearning(query, matchResult, resolvedVia);
|
|
168
215
|
const trace = {
|
|
169
216
|
query,
|
|
170
|
-
candidates: matchResult.candidates
|
|
217
|
+
candidates: matchResult.candidates,
|
|
171
218
|
reasoning,
|
|
172
219
|
steps,
|
|
173
220
|
resolvedVia,
|
|
@@ -205,6 +252,259 @@ export class CapmanEngine {
|
|
|
205
252
|
if (this.cache)
|
|
206
253
|
await this.cache.clear();
|
|
207
254
|
}
|
|
255
|
+
/**
|
|
256
|
+
* Explain what would happen for a query — without executing it.
|
|
257
|
+
* Shows matched capability, all candidate scores with reasoning,
|
|
258
|
+
* and what action would be taken.
|
|
259
|
+
*
|
|
260
|
+
* @example
|
|
261
|
+
* const explanation = await engine.explain('track order 1234')
|
|
262
|
+
* console.log(explanation.matched.reasoning)
|
|
263
|
+
* console.log(explanation.wouldExecute.action)
|
|
264
|
+
* console.log(explanation.candidates)
|
|
265
|
+
*/
|
|
266
|
+
async explain(query) {
|
|
267
|
+
const start = Date.now();
|
|
268
|
+
// ── Match — mirrors ask() logic including rate limiting ───────────────────
|
|
269
|
+
let matchResult;
|
|
270
|
+
let resolvedVia = 'keyword';
|
|
271
|
+
if (this.mode === 'accurate') {
|
|
272
|
+
if (this.llm) {
|
|
273
|
+
const skipReason = this.checkLLMAllowed();
|
|
274
|
+
if (skipReason) {
|
|
275
|
+
logger.warn(`explain(): LLM skipped — ${skipReason} — falling back to keyword`);
|
|
276
|
+
matchResult = _match(query, this.manifest);
|
|
277
|
+
}
|
|
278
|
+
else {
|
|
279
|
+
try {
|
|
280
|
+
matchResult = await _matchWithLLM(query, this.manifest, { llm: this.llm });
|
|
281
|
+
this.recordLLMSuccess();
|
|
282
|
+
resolvedVia = 'llm';
|
|
283
|
+
}
|
|
284
|
+
catch (err) {
|
|
285
|
+
this.recordLLMFailure();
|
|
286
|
+
logger.warn(`explain(): LLM call failed — falling back to keyword: ${err}`);
|
|
287
|
+
matchResult = _match(query, this.manifest);
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
else {
|
|
292
|
+
matchResult = _match(query, this.manifest);
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
else if (this.mode === 'balanced' && this.llm) {
|
|
296
|
+
// Keyword first — escalate to LLM if low confidence (same as ask())
|
|
297
|
+
const keywordResult = _match(query, this.manifest);
|
|
298
|
+
if (keywordResult.confidence >= this.threshold) {
|
|
299
|
+
matchResult = keywordResult;
|
|
300
|
+
}
|
|
301
|
+
else {
|
|
302
|
+
const skipReason = this.checkLLMAllowed();
|
|
303
|
+
if (skipReason) {
|
|
304
|
+
logger.warn(`explain(): LLM skipped — ${skipReason}`);
|
|
305
|
+
matchResult = keywordResult;
|
|
306
|
+
}
|
|
307
|
+
else {
|
|
308
|
+
try {
|
|
309
|
+
matchResult = await _matchWithLLM(query, this.manifest, { llm: this.llm });
|
|
310
|
+
this.recordLLMSuccess();
|
|
311
|
+
resolvedVia = 'llm';
|
|
312
|
+
}
|
|
313
|
+
catch (err) {
|
|
314
|
+
this.recordLLMFailure();
|
|
315
|
+
logger.warn(`explain(): LLM call failed — falling back to keyword: ${err}`);
|
|
316
|
+
matchResult = keywordResult;
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
else {
|
|
322
|
+
// cheap mode or no llm — keyword only
|
|
323
|
+
matchResult = _match(query, this.manifest);
|
|
324
|
+
}
|
|
325
|
+
// ── Build candidate explanations ─────────────────────────────────────────
|
|
326
|
+
const candidates = matchResult.candidates
|
|
327
|
+
.sort((a, b) => b.score - a.score)
|
|
328
|
+
.map(c => {
|
|
329
|
+
const cap = this.manifest.capabilities.find(mc => mc.id === c.capabilityId);
|
|
330
|
+
let explanation = '';
|
|
331
|
+
if (c.score === 0) {
|
|
332
|
+
explanation = 'No keyword overlap with examples or description';
|
|
333
|
+
}
|
|
334
|
+
else if (c.score >= 90) {
|
|
335
|
+
explanation = `Strong match (${c.score}%) — query closely matches examples`;
|
|
336
|
+
}
|
|
337
|
+
else if (c.score >= 50) {
|
|
338
|
+
const qWords = query.toLowerCase().split(/\W+/).filter(Boolean);
|
|
339
|
+
const matchedWords = (cap?.examples ?? [])
|
|
340
|
+
.flatMap(e => e.toLowerCase().split(/\s+/))
|
|
341
|
+
.filter(w => qWords.includes(w) && w.length > 2);
|
|
342
|
+
const unique = [...new Set(matchedWords)].slice(0, 3);
|
|
343
|
+
explanation = unique.length
|
|
344
|
+
? `Matched keywords: ${unique.join(', ')} (${c.score}%)`
|
|
345
|
+
: `Partial match (${c.score}%) — some keyword overlap`;
|
|
346
|
+
}
|
|
347
|
+
else {
|
|
348
|
+
explanation = `Weak match (${c.score}%) — below 50% confidence threshold, rejected`;
|
|
349
|
+
}
|
|
350
|
+
return { capabilityId: c.capabilityId, score: c.score, matched: c.matched, explanation };
|
|
351
|
+
});
|
|
352
|
+
// ── Build reasoning array ────────────────────────────────────────────────
|
|
353
|
+
const reasoning = [];
|
|
354
|
+
const winner = candidates.find(c => c.matched);
|
|
355
|
+
const rejected = candidates.filter(c => !c.matched && c.score > 0).slice(0, 3);
|
|
356
|
+
if (winner) {
|
|
357
|
+
reasoning.push(`Matched "${winner.capabilityId}" with ${winner.score}% confidence`);
|
|
358
|
+
}
|
|
359
|
+
else {
|
|
360
|
+
reasoning.push('No capability matched above the 50% confidence threshold');
|
|
361
|
+
}
|
|
362
|
+
if (rejected.length) {
|
|
363
|
+
reasoning.push(`Rejected: ${rejected.map(r => `${r.capabilityId} (${r.score}%)`).join(', ')}`);
|
|
364
|
+
}
|
|
365
|
+
reasoning.push(`Resolved via: ${resolvedVia}`);
|
|
366
|
+
if (matchResult.extractedParams && Object.keys(matchResult.extractedParams).length) {
|
|
367
|
+
const params = Object.entries(matchResult.extractedParams)
|
|
368
|
+
.filter(([, v]) => v !== null)
|
|
369
|
+
.map(([k, v]) => `${k}=${v}`)
|
|
370
|
+
.join(', ');
|
|
371
|
+
if (params)
|
|
372
|
+
reasoning.push(`Would extract params: ${params}`);
|
|
373
|
+
}
|
|
374
|
+
// ── Build wouldExecute ───────────────────────────────────────────────────
|
|
375
|
+
const cap = matchResult.capability;
|
|
376
|
+
let action = null;
|
|
377
|
+
let blocked = null;
|
|
378
|
+
let privacy = null;
|
|
379
|
+
let resolverType = null;
|
|
380
|
+
if (cap) {
|
|
381
|
+
privacy = cap.privacy.level;
|
|
382
|
+
resolverType = cap.resolver.type;
|
|
383
|
+
// Check if privacy would block — mirrors checkPrivacy() in resolver.ts
|
|
384
|
+
if (cap.privacy.level === 'user_owned') {
|
|
385
|
+
if (!this.auth?.isAuthenticated) {
|
|
386
|
+
blocked = `Capability "${cap.id}" requires authentication (privacy: user_owned)`;
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
else if (cap.privacy.level === 'admin') {
|
|
390
|
+
if (!this.auth?.isAuthenticated) {
|
|
391
|
+
blocked = `Capability "${cap.id}" requires authentication (privacy: admin)`;
|
|
392
|
+
}
|
|
393
|
+
else if (this.auth.role !== 'admin') {
|
|
394
|
+
blocked = `Capability "${cap.id}" requires admin role (current role: ${this.auth.role ?? 'none'})`;
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
if (!blocked) {
|
|
398
|
+
// Build action string
|
|
399
|
+
const params = matchResult.extractedParams;
|
|
400
|
+
if (cap.resolver.type === 'api') {
|
|
401
|
+
const endpoint = cap.resolver.endpoints[0];
|
|
402
|
+
let path = endpoint.path;
|
|
403
|
+
for (const [k, v] of Object.entries(params)) {
|
|
404
|
+
if (v)
|
|
405
|
+
path = path.replace(`{${k}}`, v);
|
|
406
|
+
}
|
|
407
|
+
const base = this.baseUrl ?? '';
|
|
408
|
+
action = `${endpoint.method} ${base}${path}`;
|
|
409
|
+
}
|
|
410
|
+
else if (cap.resolver.type === 'nav') {
|
|
411
|
+
let dest = cap.resolver.destination;
|
|
412
|
+
for (const [k, v] of Object.entries(params)) {
|
|
413
|
+
if (v)
|
|
414
|
+
dest = dest.replace(`{${k}}`, v);
|
|
415
|
+
}
|
|
416
|
+
action = `navigate → ${dest}`;
|
|
417
|
+
}
|
|
418
|
+
else if (cap.resolver.type === 'hybrid') {
|
|
419
|
+
const hybrid = cap.resolver;
|
|
420
|
+
const endpoint = hybrid.api.endpoints[0];
|
|
421
|
+
let path = endpoint.path;
|
|
422
|
+
for (const [k, v] of Object.entries(params)) {
|
|
423
|
+
if (v)
|
|
424
|
+
path = path.replace(`{${k}}`, v);
|
|
425
|
+
}
|
|
426
|
+
let dest = hybrid.nav.destination;
|
|
427
|
+
for (const [k, v] of Object.entries(params)) {
|
|
428
|
+
if (v)
|
|
429
|
+
dest = dest.replace(`{${k}}`, v);
|
|
430
|
+
}
|
|
431
|
+
const base = this.baseUrl ?? '';
|
|
432
|
+
action = `${endpoint.method} ${base}${path} + navigate → ${dest}`;
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
return {
|
|
437
|
+
query,
|
|
438
|
+
matched: {
|
|
439
|
+
capability: matchResult.capability,
|
|
440
|
+
confidence: matchResult.confidence,
|
|
441
|
+
intent: matchResult.intent,
|
|
442
|
+
reasoning,
|
|
443
|
+
},
|
|
444
|
+
candidates,
|
|
445
|
+
wouldExecute: { resolverType, action, privacy, blocked },
|
|
446
|
+
resolvedVia,
|
|
447
|
+
durationMs: Date.now() - start,
|
|
448
|
+
};
|
|
449
|
+
}
|
|
450
|
+
/**
|
|
451
|
+
* Checks all rate limiting and circuit breaker conditions.
|
|
452
|
+
* Returns null if LLM call is allowed, or a skip reason string if it should be skipped.
|
|
453
|
+
*/
|
|
454
|
+
checkLLMAllowed() {
|
|
455
|
+
const now = Date.now();
|
|
456
|
+
// ── Circuit breaker ──────────────────────────────────────────────────────
|
|
457
|
+
if (this.llmCircuitOpenAt > 0) {
|
|
458
|
+
const elapsed = now - this.llmCircuitOpenAt;
|
|
459
|
+
if (elapsed < this.llmCircuitBreakerResetMs) {
|
|
460
|
+
const remainingSec = Math.ceil((this.llmCircuitBreakerResetMs - elapsed) / 1000);
|
|
461
|
+
return `circuit breaker open — ${remainingSec}s remaining`;
|
|
462
|
+
}
|
|
463
|
+
// Reset circuit breaker — try again
|
|
464
|
+
logger.info('LLM circuit breaker reset — trying again');
|
|
465
|
+
this.llmCircuitOpenAt = 0;
|
|
466
|
+
this.llmConsecutiveFails = 0;
|
|
467
|
+
}
|
|
468
|
+
// ── Cooldown between calls ───────────────────────────────────────────────
|
|
469
|
+
if (this.llmCooldownMs > 0 && this.llmLastCallAt > 0) {
|
|
470
|
+
const elapsed = now - this.llmLastCallAt;
|
|
471
|
+
if (elapsed < this.llmCooldownMs) {
|
|
472
|
+
const remainingMs = this.llmCooldownMs - elapsed;
|
|
473
|
+
return `cooldown active — ${remainingMs}ms remaining`;
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
// ── Per-minute rate limit ────────────────────────────────────────────────
|
|
477
|
+
const windowElapsed = now - this.llmWindowStart;
|
|
478
|
+
if (windowElapsed >= 60000) {
|
|
479
|
+
this.llmCallsThisMinute = 0;
|
|
480
|
+
this.llmWindowStart = now;
|
|
481
|
+
}
|
|
482
|
+
if (this.llmCallsThisMinute >= this.maxLLMCallsPerMinute) {
|
|
483
|
+
// Recalculate elapsed after possible window reset above
|
|
484
|
+
const resetIn = Math.ceil((60000 - (now - this.llmWindowStart)) / 1000);
|
|
485
|
+
return `rate limit reached (${this.maxLLMCallsPerMinute}/min) — resets in ${Math.max(0, resetIn)}s`;
|
|
486
|
+
}
|
|
487
|
+
// Reserve the slot atomically before the call happens
|
|
488
|
+
this.llmCallsThisMinute++;
|
|
489
|
+
this.llmLastCallAt = Date.now();
|
|
490
|
+
return null;
|
|
491
|
+
}
|
|
492
|
+
/**
|
|
493
|
+
* Records a successful LLM call — updates rate limit counters.
|
|
494
|
+
*/
|
|
495
|
+
recordLLMSuccess() {
|
|
496
|
+
this.llmConsecutiveFails = 0;
|
|
497
|
+
}
|
|
498
|
+
/**
|
|
499
|
+
* Records a failed LLM call — may open the circuit breaker.
|
|
500
|
+
*/
|
|
501
|
+
recordLLMFailure() {
|
|
502
|
+
this.llmConsecutiveFails++;
|
|
503
|
+
if (this.llmConsecutiveFails >= this.llmCircuitBreakerThreshold) {
|
|
504
|
+
this.llmCircuitOpenAt = Date.now();
|
|
505
|
+
logger.warn(`LLM circuit breaker opened after ${this.llmConsecutiveFails} consecutive failures — pausing for ${this.llmCircuitBreakerResetMs / 1000}s`);
|
|
506
|
+
}
|
|
507
|
+
}
|
|
208
508
|
// ── Private helpers ────────────────────────────────────────────────────────
|
|
209
509
|
resolveOptions(overrides = {}) {
|
|
210
510
|
return {
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { CapmanConfig, Manifest, ValidationResult } from './types';
|
|
2
|
+
export declare function generate(config: CapmanConfig): Manifest;
|
|
3
|
+
export declare function loadConfig(configPath?: string): CapmanConfig;
|
|
4
|
+
export declare function writeManifest(manifest: Manifest, outputPath?: string): string;
|
|
5
|
+
export declare function readManifest(manifestPath?: string): Manifest;
|
|
6
|
+
export declare function validate(manifest: Manifest): ValidationResult;
|
|
7
|
+
export declare function generateStarterConfig(): string;
|
package/dist/esm/generator.js
CHANGED
|
@@ -28,6 +28,10 @@ export function loadConfig(configPath) {
|
|
|
28
28
|
if (fs.existsSync(resolved)) {
|
|
29
29
|
let raw;
|
|
30
30
|
// Catch syntax errors in config file
|
|
31
|
+
// Note: require() only works with CJS config files (.js, .json)
|
|
32
|
+
// ESM config files (.mjs or "type": "module") are not supported.
|
|
33
|
+
// Use a CJS config file or convert with: module.exports = { ... }
|
|
34
|
+
// Full ESM config support is planned for v0.5.
|
|
31
35
|
try {
|
|
32
36
|
const mod = require(resolved);
|
|
33
37
|
raw = mod.default ?? mod;
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
export { setLogLevel } from './logger';
|
|
2
|
+
export type { LogLevel } from './logger';
|
|
3
|
+
export type { Capability, CapabilityParam, CapmanConfig, Manifest, MatchResult, ExecutionTrace, TraceStep, MatchCandidate, ResolveResult, ApiCallResult, ValidationResult, Resolver, ApiResolver, NavResolver, HybridResolver, PrivacyScope, ResolverType, HttpMethod, ExplainResult, ExplainCandidate, } from './types';
|
|
4
|
+
export { generate, loadConfig, writeManifest, readManifest, validate, generateStarterConfig, } from './generator';
|
|
5
|
+
export { match, matchWithLLM, } from './matcher';
|
|
6
|
+
export type { LLMMatcherOptions } from './matcher';
|
|
7
|
+
export { resolve } from './resolver';
|
|
8
|
+
export type { ResolveOptions, AuthContext } from './resolver';
|
|
9
|
+
export { CapmanEngine } from './engine';
|
|
10
|
+
export type { EngineOptions, EngineResult } from './engine';
|
|
11
|
+
export { MemoryCache, FileCache, ComboCache, buildCacheKey, normalizeQuery } from './cache';
|
|
12
|
+
export type { CacheStore, CacheEntry } from './cache';
|
|
13
|
+
export { FileLearningStore, MemoryLearningStore } from './learning';
|
|
14
|
+
export type { LearningStore, LearningEntry, KeywordStats } from './learning';
|
|
15
|
+
export { parseOpenAPI } from './parser';
|
|
16
|
+
export type { ParseResult } from './parser';
|
|
17
|
+
import type { Manifest, MatchResult, ResolveResult } from './types';
|
|
18
|
+
import type { LLMMatcherOptions } from './matcher';
|
|
19
|
+
import type { ResolveOptions } from './resolver';
|
|
20
|
+
export type MatchMode = 'cheap' | 'balanced' | 'accurate';
|
|
21
|
+
export interface AskOptions extends ResolveOptions {
|
|
22
|
+
llm?: LLMMatcherOptions['llm'];
|
|
23
|
+
/**
|
|
24
|
+
* Controls how intent matching is performed.
|
|
25
|
+
* - 'cheap' — keyword only, no LLM calls
|
|
26
|
+
* - 'balanced' — keyword first, LLM fallback if confidence < 50% (default)
|
|
27
|
+
* - 'accurate' — LLM first, keyword fallback
|
|
28
|
+
* @default 'balanced'
|
|
29
|
+
*/
|
|
30
|
+
mode?: MatchMode;
|
|
31
|
+
}
|
|
32
|
+
export interface AskResult {
|
|
33
|
+
match: MatchResult;
|
|
34
|
+
resolution: ResolveResult;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* One-shot convenience: match + resolve in a single call.
|
|
38
|
+
* Delegates to CapmanEngine internally.
|
|
39
|
+
*
|
|
40
|
+
* @deprecated For full features including trace and caching, use CapmanEngine directly.
|
|
41
|
+
*
|
|
42
|
+
* @example
|
|
43
|
+
* const result = await ask("show me the dashboard", manifest, {
|
|
44
|
+
* baseUrl: 'https://api.your-app.com',
|
|
45
|
+
* })
|
|
46
|
+
*/
|
|
47
|
+
export declare function ask(query: string, manifest: Manifest, options?: AskOptions): Promise<AskResult>;
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
export interface LearningEntry {
|
|
2
|
+
query: string;
|
|
3
|
+
capabilityId: string | null;
|
|
4
|
+
confidence: number;
|
|
5
|
+
intent: string;
|
|
6
|
+
extractedParams: Record<string, string | null>;
|
|
7
|
+
resolvedVia: 'keyword' | 'llm' | 'cache';
|
|
8
|
+
timestamp: string;
|
|
9
|
+
}
|
|
10
|
+
export interface KeywordStats {
|
|
11
|
+
/** keyword → Map of capabilityId → hit count */
|
|
12
|
+
index: Record<string, Record<string, number>>;
|
|
13
|
+
/** Total queries processed */
|
|
14
|
+
totalQueries: number;
|
|
15
|
+
/** Queries that went to LLM */
|
|
16
|
+
llmQueries: number;
|
|
17
|
+
/** Queries served from cache */
|
|
18
|
+
cacheHits: number;
|
|
19
|
+
/** Out of scope queries */
|
|
20
|
+
outOfScope: number;
|
|
21
|
+
}
|
|
22
|
+
export interface LearningStore {
|
|
23
|
+
record(entry: LearningEntry): Promise<void>;
|
|
24
|
+
getStats(): Promise<KeywordStats>;
|
|
25
|
+
getTopCapabilities(limit?: number): Promise<Array<{
|
|
26
|
+
id: string;
|
|
27
|
+
hits: number;
|
|
28
|
+
}>>;
|
|
29
|
+
clear(): Promise<void>;
|
|
30
|
+
}
|
|
31
|
+
export declare class FileLearningStore implements LearningStore {
|
|
32
|
+
private filePath;
|
|
33
|
+
private entries;
|
|
34
|
+
private loaded;
|
|
35
|
+
constructor(filePath?: string);
|
|
36
|
+
private load;
|
|
37
|
+
private save;
|
|
38
|
+
record(entry: LearningEntry): Promise<void>;
|
|
39
|
+
getStats(): Promise<KeywordStats>;
|
|
40
|
+
getTopCapabilities(limit?: number): Promise<Array<{
|
|
41
|
+
id: string;
|
|
42
|
+
hits: number;
|
|
43
|
+
}>>;
|
|
44
|
+
clear(): Promise<void>;
|
|
45
|
+
}
|
|
46
|
+
export declare class MemoryLearningStore implements LearningStore {
|
|
47
|
+
private entries;
|
|
48
|
+
record(entry: LearningEntry): Promise<void>;
|
|
49
|
+
getStats(): Promise<KeywordStats>;
|
|
50
|
+
getTopCapabilities(limit?: number): Promise<Array<{
|
|
51
|
+
id: string;
|
|
52
|
+
hits: number;
|
|
53
|
+
}>>;
|
|
54
|
+
clear(): Promise<void>;
|
|
55
|
+
}
|
package/dist/esm/learning.js
CHANGED
|
@@ -57,8 +57,13 @@ export class FileLearningStore {
|
|
|
57
57
|
try {
|
|
58
58
|
const raw = await fs.promises.readFile(this.filePath, 'utf-8');
|
|
59
59
|
const parsed = JSON.parse(raw);
|
|
60
|
-
|
|
61
|
-
|
|
60
|
+
if (parsed && typeof parsed === 'object' && !Array.isArray(parsed) && Array.isArray(parsed.entries)) {
|
|
61
|
+
this.entries = parsed.entries;
|
|
62
|
+
logger.debug(`Learning store loaded: ${this.entries.length} entries`);
|
|
63
|
+
}
|
|
64
|
+
else {
|
|
65
|
+
logger.warn(`Learning store at ${this.filePath} contained unexpected format — starting fresh`);
|
|
66
|
+
}
|
|
62
67
|
}
|
|
63
68
|
catch {
|
|
64
69
|
// File doesn't exist yet — start fresh
|