capman 0.5.5 → 0.6.0

package/dist/esm/engine.js

@@ -1,4 +1,4 @@
-import { match as _match, matchWithLLM as _matchWithLLM, resolverToIntent, extractParams, STOPWORDS, LLMParseError } from './matcher';
+import { match as _match, matchWithLLM as _matchWithLLM, resolverToIntent, extractParams, LLMParseError, tokenize, buildBM25Index, scoreCapability as _scoreCapability, sanitizeForPrompt } from './matcher';
 import { resolve as _resolve, checkPrivacy } from './resolver';
 import { MemoryLearningStore } from './learning';
 import { logger } from './logger';
@@ -27,6 +27,12 @@ export class CapmanEngine {
         this.llmCircuitBreakerResetMs = options.llmCircuitBreakerResetMs ?? 60_000;
         this.fuzzyMatch = options.fuzzyMatch ?? false;
         this.fuzzyThreshold = options.fuzzyThreshold ?? 0.4;
+        this.bm25K1 = options.bm25K1 ?? 1.5;
+        this.bm25B = options.bm25B ?? 0.75;
+        this.bm25Index = buildBM25Index(options.manifest.capabilities);
+        this.bm25Ceiling = this.calibrateBM25Ceiling();
+        this.marginAwareLLM = options.marginAwareLLM ?? false;
+        this.adaptiveMargin = options.adaptiveMarginOverride ?? this.calibrateAdaptiveMargin();
         // Cache — default MemoryCache (no filesystem writes), or disabled with false
         // Use FileCache or ComboCache explicitly for persistence across restarts
         this.cache = options.cache === false
@@ -90,12 +96,16 @@ export class CapmanEngine {
                     resolvedVia: 'cache',
                     totalMs: Date.now() - start,
                 };
+                const { verdict: cacheVerdict, margin: cacheMargin } = this.computeVerdict(matchWithFreshParams);
                 const result = {
                     match: matchWithFreshParams,
                     resolution,
                     resolvedVia: 'cache',
                     durationMs: Date.now() - start,
                     trace,
+                    verdict: cacheVerdict,
+                    margin: cacheMargin,
+                    missingParams: undefined
                 };
                 await this.recordLearning(query, matchWithFreshParams, 'cache');
                 return result;
@@ -123,7 +133,19 @@ export class CapmanEngine {
                 detail: privacyError ?? `level: ${matchResult.capability.privacy.level}`,
             });
         }
-        // ── Step 4: Resolve ──────────────────────────────────────────────────────
+        // ── Step 4a: Compute verdict + optional margin-aware LLM disambiguation ──
+        let { verdict, margin } = this.computeVerdict(matchResult);
+        if (verdict === 'marginal' &&
+            this.marginAwareLLM &&
+            this.llm &&
+            this.mode === 'balanced') {
+            matchResult = await this.disambiguateLLM(query, matchResult, steps);
+            // Recompute verdict after disambiguation
+            const recomputed = this.computeVerdict(matchResult);
+            verdict = recomputed.verdict;
+            margin = recomputed.margin;
+        }
+        // ── Step 4b: Resolve ──────────────────────────────────────────────────────
         const resolveStart = Date.now();
         const resolution = await _resolve(matchResult, matchResult.extractedParams, this.resolveOptions(overrides));
         steps.push({
@@ -145,6 +167,57 @@ export class CapmanEngine {
             await this.cache.set(capKey, matchResult);
             // capKey always starts with 'cap:' — structurally distinct from queryKey
         }
+        // ── Step 5b: Compute missingParams ───────────────────────────────────────
+        // Spec: LLM attempts extraction first when available. missingParams is last resort.
+        let missingParams;
+        if (matchResult.capability && resolvedVia !== 'llm') {
+            const cap = matchResult.capability;
+            const unresolved = cap.params.filter(p => p.source === 'user_query' && p.required
+                && matchResult.extractedParams[p.name] === null);
+            if (unresolved.length > 0 && this.llm && this.mode !== 'cheap') {
+                // LLM available — attempt targeted param extraction before declaring incomplete
+                const skipReason = this.checkLLMAllowed();
+                if (!skipReason) {
+                    try {
+                        const paramExtractionStart = Date.now();
+                        const paramDescriptions = unresolved
+                            .map(p => `- ${p.name}: ${p.description}`)
+                            .join('\n');
+                        const paramPrompt = `Extract the following parameters from this user query.\n` +
+                            `Query: ${JSON.stringify({ user_query: query })}\n\n` +
+                            `Parameters to extract:\n${paramDescriptions}\n\n` +
+                            `Respond ONLY with valid JSON: { "params": { "<name>": "<value or null>" } }`;
+                        const raw = await this.llm(paramPrompt);
+                        const clean = raw.replace(/```json|```/g, '').trim();
+                        const parsed = JSON.parse(clean);
+                        this.recordLLMSuccess();
+                        steps.push({
+                            type: 'llm_match',
+                            status: 'pass',
+                            durationMs: Date.now() - paramExtractionStart,
+                            detail: `param extraction: ${unresolved.map(p => p.name).join(', ')}`,
+                        });
+                        // Merge LLM-extracted values — validate type before accepting
+                        for (const p of unresolved) {
+                            const val = parsed?.params?.[p.name];
+                            if (val && typeof val === 'string' && val.trim().length > 0) {
+                                matchResult.extractedParams[p.name] = val.trim();
+                            }
+                        }
+                    }
+                    catch {
+                        // LLM param extraction failed — fall through to missingParams below
+                    }
+                }
+            }
+            // After LLM attempt (or if skipped/unavailable), report what's still missing
+            const stillMissing = cap.params
+                .filter(p => p.source === 'user_query' && p.required
+                && matchResult.extractedParams[p.name] === null)
+                .map(p => p.name);
+            if (stillMissing.length > 0)
+                missingParams = stillMissing;
+        }
         // ── Step 6: Build reasoning array ────────────────────────────────────────
         const reasoning = [];
         if (matchResult.candidates.length) {
@@ -189,6 +262,9 @@ export class CapmanEngine {
             resolvedVia,
             durationMs: Date.now() - start,
             trace,
+            verdict,
+            margin,
+            missingParams,
         };
     }
     /**
@@ -248,11 +324,10 @@ export class CapmanEngine {
     async loadManifest(manifest) {
         this.checkManifestVersion(manifest);
         this.manifest = manifest;
+        this.bm25Index = buildBM25Index(manifest.capabilities);
+        this.bm25Ceiling = this.calibrateBM25Ceiling();
+        this.adaptiveMargin = this.calibrateAdaptiveMargin();
         await this.clearCache();
-        // Note: LLM rate limiter state (llmCallsThisMinute, llmConsecutiveFails,
-        // llmCircuitOpenAt) is intentionally preserved across manifest reloads.
-        // The LLM provider has not changed, so circuit breaker state remains valid.
-        // If you need a clean rate limiter state, create a new CapmanEngine instance.
     }
     /**
      * Explain what would happen for a query — without executing it.
@@ -291,7 +366,8 @@ export class CapmanEngine {
         // ── Apply learning boost (same as ask()) ─────────────────────────────────
         matchResult = await this.applyBoostToMatchResult(query, matchResult, resolvedVia);
         // ── Build candidate explanations ─────────────────────────────────────────
-        const qWordSet = new Set(query.toLowerCase().split(/\W+/).filter(Boolean));
+        const qTokens = tokenize(query);
+        const qWordSet = new Set(qTokens);
         const candidates = matchResult.candidates
             .sort((a, b) => b.score - a.score)
             .map(c => {
@@ -305,8 +381,8 @@ export class CapmanEngine {
             }
             else if (c.score >= 50) {
                 const matchedWords = (cap?.examples ?? [])
-                    .flatMap(e => e.toLowerCase().split(/\s+/))
-                    .filter(w => qWordSet.has(w) && w.length > 2);
+                    .flatMap(e => tokenize(e))
+                    .filter(w => qWordSet.has(w));
                 const unique = [...new Set(matchedWords)].slice(0, 3);
                 explanation = unique.length
                     ? `Matched keywords: ${unique.join(', ')} (${c.score}%)`
@@ -496,6 +572,10 @@ export class CapmanEngine {
         const fuzzyOpts = {
             fuzzyMatch: this.fuzzyMatch,
             fuzzyThreshold: this.fuzzyThreshold,
+            bm25Index: this.bm25Index,
+            bm25Ceiling: this.bm25Ceiling,
+            bm25K1: this.bm25K1,
+            bm25B: this.bm25B,
         };
         switch (this.mode) {
             case 'cheap': {
@@ -663,7 +743,7 @@ export class CapmanEngine {
         const stats = await this.learning.getStats();
         if (!stats || Object.keys(stats.index).length === 0)
             return candidates;
-        const qWords = query.toLowerCase().split(/\W+/).filter(w => w.length > 2 && !STOPWORDS.has(w));
+        const qWords = tokenize(query);
         if (qWords.length === 0)
             return candidates;
         return candidates.map(candidate => {
@@ -711,6 +791,136 @@ export class CapmanEngine {
             timestamp: new Date().toISOString(),
         });
     }
+    calibrateBM25Ceiling() {
+        let max = 0;
+        for (const cap of this.manifest.capabilities) {
+            if (!cap.examples?.length)
+                continue;
+            const selfWords = new Set(tokenize(cap.examples[0]));
+            const raw = _scoreCapability(selfWords, cap, this.bm25Index, this.bm25K1, this.bm25B);
+            if (raw > max)
+                max = raw;
+        }
+        return max > 0 ? max : 100;
+    }
+    /**
+     * Calibrates the adaptive margin threshold from the manifest's own score
+     * distribution. Runs each capability's first example against all other
+     * capabilities to find the typical inter-capability score spread.
+     * Dense overlapping vocabulary → lower margin (harder to separate).
+     * Sparse vocabulary → higher margin (easier to separate).
+     *
+     * Complexity: O(capabilities²) — runs at constructor time and on loadManifest().
+     * For manifests with ≤100 capabilities this is negligible (<10ms).
+     * For very large manifests (500+ capabilities), consider passing
+     * `adaptiveMarginOverride` to skip calibration.
+     */
+    calibrateAdaptiveMargin() {
+        if (this.manifest.capabilities.length < 2)
+            return 20;
+        const margins = [];
+        const fuzzyOpts = {
+            fuzzyMatch: false, // calibration uses keyword only — deterministic
+            bm25Index: this.bm25Index,
+            bm25Ceiling: this.bm25Ceiling,
+            bm25K1: this.bm25K1,
+            bm25B: this.bm25B,
+        };
+        for (const cap of this.manifest.capabilities) {
+            if (!cap.examples?.length)
+                continue;
+            const result = _match(cap.examples[0], this.manifest, fuzzyOpts);
+            const sorted = [...result.candidates].sort((a, b) => b.score - a.score);
+            if (sorted.length >= 2) {
+                margins.push(sorted[0].score - sorted[1].score);
+            }
+        }
+        if (margins.length === 0)
+            return 20;
+        // Use 25th percentile of margins as the threshold — manifests where
+        // capabilities are naturally close together get a tighter threshold
+        margins.sort((a, b) => a - b);
+        const p25 = margins[Math.floor(margins.length * 0.25)];
+        return Math.max(10, Math.min(30, Math.round(p25 * 0.6)));
+    }
+    computeVerdict(matchResult) {
+        if (!matchResult.capability)
+            return { verdict: 'uncertain', margin: 0 };
+        const sorted = [...matchResult.candidates].sort((a, b) => b.score - a.score);
+        const best = sorted[0]?.score ?? 0;
+        const second = sorted[1]?.score ?? 0;
+        const margin = best - second;
+        if (best < 60)
+            return { verdict: 'uncertain', margin };
+        if (margin < this.adaptiveMargin)
+            return { verdict: 'marginal', margin };
+        return { verdict: 'clear', margin };
+    }
+    /**
+       * Targeted disambiguation between top-2 candidates.
+       * Sends ~200 tokens instead of full manifest (~4000 tokens) — 93% cost reduction.
+       * Returns updated matchResult with LLM-preferred winner, or original on failure.
+       */
+    async disambiguateLLM(query, matchResult, steps) {
+        if (!this.llm)
+            return matchResult;
+        const sorted = [...matchResult.candidates]
+            .sort((a, b) => b.score - a.score)
+            .slice(0, 2);
+        if (sorted.length < 2)
+            return matchResult;
+        const capA = this.manifest.capabilities.find(c => c.id === sorted[0].capabilityId);
+        const capB = this.manifest.capabilities.find(c => c.id === sorted[1].capabilityId);
+        if (!capA || !capB)
+            return matchResult;
+        const skipReason = this.checkLLMAllowed();
+        if (skipReason) {
+            logger.warn(`Disambiguation LLM skipped — ${skipReason}`);
+            steps.push({ type: 'llm_match', status: 'skip', durationMs: 0, detail: `disambiguation skipped: ${skipReason}` });
+            return matchResult;
+        }
+        const prompt = `Two capabilities are close matches for this query. Pick the best one.
+
+  Query: ${JSON.stringify({ user_query: query })}
+
+  Option A: ${capA.id} — ${sanitizeForPrompt(capA.description, 150)}
+  Option B: ${capB.id} — ${sanitizeForPrompt(capB.description, 150)}
+
+  Respond ONLY with valid JSON:
+  { "winner": "<capability_id>", "confidence": <0-100>, "reasoning": "<one sentence>" }`;
+        const t = Date.now();
+        try {
+            const raw = await this.llm(prompt);
+            const clean = raw.replace(/```json|```/g, '').trim();
+            const parsed = JSON.parse(clean);
+            this.recordLLMSuccess();
+            const winner = this.manifest.capabilities.find(c => c.id === parsed.winner);
+            if (!winner) {
+                steps.push({ type: 'llm_match', status: 'fail', durationMs: Date.now() - t, detail: 'disambiguation returned unknown id' });
+                return matchResult;
+            }
+            steps.push({ type: 'llm_match', status: 'pass', durationMs: Date.now() - t, detail: `disambiguation: ${winner.id} (${parsed.confidence}%)` });
+            const confidence = typeof parsed.confidence === 'number' && !isNaN(parsed.confidence)
+                ? Math.min(100, Math.max(0, Math.round(parsed.confidence)))
+                : matchResult.confidence; // fallback to original if LLM returned bad value
+            return {
+                ...matchResult,
+                capability: winner,
+                confidence,
+                intent: resolverToIntent(winner),
+                extractedParams: extractParams(query, winner),
+                candidates: matchResult.candidates.map(c => ({ ...c, matched: c.capabilityId === winner.id })),
+                reasoning: parsed.reasoning ?? `Disambiguated to "${winner.id}"`,
+            };
+        }
+        catch (err) {
+            const isParseError = err instanceof LLMParseError;
+            if (!isParseError)
+                this.recordLLMFailure();
+            steps.push({ type: 'llm_match', status: 'fail', durationMs: Date.now() - t, detail: String(err) });
+            return matchResult;
+        }
+    }
 }
 /** Maximum allowed query length in characters. Queries exceeding this throw RangeError. */
 CapmanEngine.MAX_QUERY_LENGTH = 1000;

package/dist/esm/index.d.ts

@@ -5,6 +5,7 @@ export { generate, loadConfig, writeManifest, readManifest, validate, generateSt
 export { match, matchWithLLM, extractParams, } from './matcher';
 export { LLMParseError } from './matcher';
 export type { LLMMatcherOptions } from './matcher';
+export { TYPE_PATTERNS } from './matcher';
 export { resolve } from './resolver';
 export type { ResolveOptions, AuthContext } from './resolver';
 export { CapmanEngine } from './engine';

package/dist/esm/index.js

@@ -2,6 +2,7 @@ export { setLogLevel } from './logger';
 export { generate, loadConfig, writeManifest, readManifest, validate, generateStarterConfig, } from './generator';
 export { match, matchWithLLM, extractParams, } from './matcher';
 export { LLMParseError } from './matcher';
+export { TYPE_PATTERNS } from './matcher';
 export { resolve } from './resolver';
 // ─── Engine (recommended API) ─────────────────────────────────────────────────
 export { CapmanEngine } from './engine';

package/dist/esm/learning.js

@@ -2,7 +2,7 @@ import * as fs from 'fs';
 import * as path from 'path';
 import { logger } from './logger';
 const MAX_LEARNING_ENTRIES = 10_000;
-import { STOPWORDS } from './matcher';
+import { tokenize } from './matcher';
 // Module-level registry — tracks all active FileLearningStore instances
 // for process exit flushing. Handlers registered once to avoid accumulation.
 const activeStores = new Set();
@@ -71,13 +71,15 @@ class LearningIndex {
         if (!entry.capabilityId)
             this.statsCounter.outOfScope++;
         if (entry.capabilityId) {
-            const words = entry.query.toLowerCase()
-                .split(/\W+/)
-                .filter(w => w.length > 2 && !STOPWORDS.has(w));
+            // Confidence-weighted contribution — a 95% match contributes 9.5×
+            // more signal than a 51% borderline match. Floor of 0.1 ensures
+            // borderline matches still contribute, just proportionally less.
+            const weight = Math.max(0.1, entry.confidence / 100);
+            const words = tokenize(entry.query);
             for (const word of words) {
                 this.index[word] ??= {};
                 this.index[word][entry.capabilityId] =
-                    (this.index[word][entry.capabilityId] ?? 0) + 1;
+                    (this.index[word][entry.capabilityId] ?? 0) + weight;
             }
         }
     }
@@ -93,14 +95,14 @@ class LearningIndex {
             return;
         }
         // Keyword index cleanup
-        const words = entry.query.toLowerCase()
-            .split(/\W+/)
-            .filter(w => w.length > 2 && !STOPWORDS.has(w));
+        const words = tokenize(entry.query);
         for (const word of words) {
             if (!this.index[word])
                 continue;
+            // Subtract estimated weight (0.5 average) — exact weight not stored.
+            // Minor drift on prune is acceptable; index is rebuilt when drift matters.
             this.index[word][entry.capabilityId] =
-                (this.index[word][entry.capabilityId] ?? 1) - 1;
+                (this.index[word][entry.capabilityId] ?? 0.5) - 0.5;
             if (this.index[word][entry.capabilityId] <= 0) {
                 delete this.index[word][entry.capabilityId];
             }
@@ -255,11 +257,7 @@ export class FileLearningStore {
         // not be persisted to disk under GDPR/CCPA data retention requirements.
         const sanitized = {
             ...entry,
-            query: entry.query
-                .toLowerCase()
-                .split(/\W+/)
-                .filter(w => w.length > 2 && !STOPWORDS.has(w))
-                .join(' '),
+            query: tokenize(entry.query).join(' '),
         };
         this.entries.push(sanitized);
         this.learningIndex.update(sanitized);
@@ -308,11 +306,7 @@ export class MemoryLearningStore {
     async record(entry) {
         const sanitized = {
             ...entry,
-            query: entry.query
-                .toLowerCase()
-                .split(/\W+/)
-                .filter(w => w.length > 2 && !STOPWORDS.has(w))
-                .join(' '),
+            query: tokenize(entry.query).join(' '),
         };
         this.entries.push(sanitized);
         this.learningIndex.update(sanitized);

package/dist/esm/matcher.d.ts

@@ -3,7 +3,58 @@ export declare class LLMParseError extends Error {
     constructor(message: string);
 }
 export declare const STOPWORDS: Set<string>;
+/**
+ * Regex patterns for common param types.
+ * Used when a CapabilityParam has `pattern` set to a named type.
+ */
+export declare const TYPE_PATTERNS: Record<string, RegExp>;
+/**
+ * Simplified suffix-stripping stemmer — 10 most common English morphological
+ * patterns covering ~80% of benefit at ~25% the complexity of Porter stemmer.
+ * Applied symmetrically to both query words and capability index words.
+ */
+export declare function stem(word: string): string;
+/**
+ * Shared tokenizer — used by scorer, learning index, and boost system.
+ * Applies stopword filtering AND stemming symmetrically.
+ * Any site that tokenizes text for matching MUST use this function
+ * to avoid silent mismatches between query and index tokens.
+ */
+export declare function tokenize(text: string): string[];
+export interface BM25Index {
+    /** Document frequency — how many capabilities contain each term */
+    df: Record<string, number>;
+    /** Average field length per field type */
+    avgdl: {
+        examples: number;
+        description: number;
+        name: number;
+    };
+    /** Total number of capabilities */
+    N: number;
+    /** Bigram sets per capability — post-stopword, post-stem, examples only */
+    bigrams: Record<string, Set<string>>;
+}
+/** Build a BM25 index over all capabilities. Call once at manifest load. */
+export declare function buildBM25Index(capabilities: Capability[]): BM25Index;
+/**
+ * BM25 scoring with field weights.
+ * k1 = 1.5 (TF saturation), b = 0.75 (length normalization)
+ * Field weights: examples 0.6, description 0.3, name 0.1
+ */
+export declare function scoreCapability(qWordSet: Set<string>, cap: Capability, index: BM25Index, k1?: number, b?: number): number;
+/**
+ * Extracts bigrams from a token array as "token1__token2" strings.
+ * Input must already be post-stopword and post-stem (use tokenize() first).
+ */
+export declare function extractBigrams(tokens: string[]): Set<string>;
 export declare function resolverToIntent(cap: Capability): MatchResult['intent'];
+/**
+ * Strips characters that could break LLM prompt structure from
+ * capability field values before injection into the system prompt.
+ * Removes control characters, newlines, and delimiter-like sequences.
+ */
+export declare function sanitizeForPrompt(value: string, maxLen: number): string;
 /**
  * Extracts parameter values from a user query using keyword heuristics.
  *
@@ -22,6 +73,10 @@ export declare function extractParams(query: string, cap: Capability): Record<st
 export interface MatchOptions {
     fuzzyMatch?: boolean;
     fuzzyThreshold?: number;
+    bm25Index?: BM25Index;
+    bm25K1?: number;
+    bm25B?: number;
+    bm25Ceiling?: number;
 }
 export declare function match(query: string, manifest: Manifest, options?: MatchOptions): MatchResult;
 export interface LLMMatcherOptions {