capman 0.5.5 → 0.6.1

package/dist/esm/learning.js

@@ -2,7 +2,7 @@ import * as fs from 'fs';
 import * as path from 'path';
 import { logger } from './logger';
 const MAX_LEARNING_ENTRIES = 10_000;
-import { STOPWORDS } from './matcher';
+import { tokenize } from './matcher';
 // Module-level registry — tracks all active FileLearningStore instances
 // for process exit flushing. Handlers registered once to avoid accumulation.
 const activeStores = new Set();
@@ -71,13 +71,19 @@ class LearningIndex {
         if (!entry.capabilityId)
             this.statsCounter.outOfScope++;
         if (entry.capabilityId) {
-            const words = entry.query.toLowerCase()
-                .split(/\W+/)
-                .filter(w => w.length > 2 && !STOPWORDS.has(w));
+            // Confidence-weighted contribution — a 95% match contributes 9.5×
+            // more signal than a 51% borderline match. Floor of 0.1 ensures
+            // borderline matches still contribute, just proportionally less.
+            const weight = Math.max(0.1, entry.confidence / 100);
+            // Store weight on the entry so subtract() can reverse the exact amount.
+            // Without this, subtract() would have to use a hardcoded estimate (0.5)
+            // that causes index drift after pruning high-confidence entries.
+            entry.weight = weight;
+            const words = tokenize(entry.query);
             for (const word of words) {
                 this.index[word] ??= {};
                 this.index[word][entry.capabilityId] =
-                    (this.index[word][entry.capabilityId] ?? 0) + 1;
+                    (this.index[word][entry.capabilityId] ?? 0) + weight;
             }
         }
     }
@@ -93,14 +99,16 @@ class LearningIndex {
             return;
         }
         // Keyword index cleanup
-        const words = entry.query.toLowerCase()
-            .split(/\W+/)
-            .filter(w => w.length > 2 && !STOPWORDS.has(w));
+        const words = tokenize(entry.query);
         for (const word of words) {
             if (!this.index[word])
                 continue;
+            // Use the weight stored at record time for exact symmetric subtraction.
+            // Fallback recalculates from confidence for entries persisted before the
+            // weight field was added (backwards-compatible with older learning.json files).
+            const weight = entry.weight ?? Math.max(0.1, entry.confidence / 100);
             this.index[word][entry.capabilityId] =
-                (this.index[word][entry.capabilityId] ?? 1) - 1;
+                (this.index[word][entry.capabilityId] ?? weight) - weight;
             if (this.index[word][entry.capabilityId] <= 0) {
                 delete this.index[word][entry.capabilityId];
             }
@@ -166,8 +174,10 @@ export class FileLearningStore {
             fs.writeFileSync(tmp, payload);
             fs.renameSync(tmp, this.filePath);
         }
-        catch {
-            // Best-effort in exit handler
+        catch (err) {
+            // Use process.stderr.write — never console.error in an exit handler,
+            // as stdout may already be flushed or closed at this point.
+            process.stderr.write(`[capman] Failed to flush learning store to ${this.filePath}: ${err}\n`);
         }
     }
     /**
@@ -200,7 +210,26 @@ export class FileLearningStore {
             const raw = await fs.promises.readFile(this.filePath, 'utf-8');
             const parsed = JSON.parse(raw);
             if (parsed && typeof parsed === 'object' && !Array.isArray(parsed) && Array.isArray(parsed.entries)) {
-                this.entries = parsed.entries;
+                // Validate each entry — corrupted entries (null capability, wrong types) must
+                // not propagate into the engine where they cause runtime errors deep in matching.
+                const validEntries = [];
+                let skipped = 0;
+                for (const entry of parsed.entries) {
+                    if (entry !== null && typeof entry === 'object' &&
+                        typeof entry.query === 'string' &&
+                        (entry.capabilityId === null || typeof entry.capabilityId === 'string') &&
+                        typeof entry.confidence === 'number' &&
+                        typeof entry.resolvedVia === 'string') {
+                        validEntries.push(entry);
+                    }
+                    else {
+                        skipped++;
+                    }
+                }
+                if (skipped > 0) {
+                    logger.warn(`Learning store: skipped ${skipped} invalid entries during load`);
+                }
+                this.entries = validEntries;
                 this.learningIndex.rebuild(this.entries);
                 logger.debug(`Learning store loaded: ${this.entries.length} entries`);
             }
@@ -255,11 +284,7 @@ export class FileLearningStore {
         // not be persisted to disk under GDPR/CCPA data retention requirements.
         const sanitized = {
             ...entry,
-            query: entry.query
-                .toLowerCase()
-                .split(/\W+/)
-                .filter(w => w.length > 2 && !STOPWORDS.has(w))
-                .join(' '),
+            query: tokenize(entry.query).join(' '),
         };
         this.entries.push(sanitized);
         this.learningIndex.update(sanitized);
@@ -308,19 +333,15 @@ export class MemoryLearningStore {
     async record(entry) {
         const sanitized = {
             ...entry,
-            query: entry.query
-                .toLowerCase()
-                .split(/\W+/)
-                .filter(w => w.length > 2 && !STOPWORDS.has(w))
-                .join(' '),
+            query: tokenize(entry.query).join(' '),
         };
         this.entries.push(sanitized);
         this.learningIndex.update(sanitized);
         if (this.entries.length > MAX_LEARNING_ENTRIES) {
             const excess = this.entries.length - MAX_LEARNING_ENTRIES;
             const pruned = this.entries.splice(0, excess);
-            for (const entry of pruned) {
-                this.learningIndex.subtract(entry);
+            for (const staleEntry of pruned) {
+                this.learningIndex.subtract(staleEntry);
             }
         }
     }

package/dist/esm/matcher.d.ts

@@ -3,7 +3,89 @@ export declare class LLMParseError extends Error {
     constructor(message: string);
 }
 export declare const STOPWORDS: Set<string>;
+/**
+ * Regex patterns for common param types.
+ * Used when a CapabilityParam has `pattern` set to a named type.
+ */
+export declare const TYPE_PATTERNS: Record<string, RegExp>;
+/**
+ * Simplified suffix-stripping stemmer — 10 most common English morphological
+ * patterns covering ~80% of benefit at ~25% the complexity of Porter stemmer.
+ * Applied symmetrically to both query words and capability index words.
+ */
+export declare function stem(word: string): string;
+/**
+ * Shared tokenizer — used by scorer, learning index, and boost system.
+ * Applies stopword filtering AND stemming symmetrically.
+ * Any site that tokenizes text for matching MUST use this function
+ * to avoid silent mismatches between query and index tokens.
+ */
+export declare function tokenize(text: string): string[];
+export interface BM25Index {
+    /** Document frequency — how many capabilities contain each term */
+    df: Record<string, number>;
+    /** Average field length per field type */
+    avgdl: {
+        examples: number;
+        description: number;
+        name: number;
+    };
+    /** Total number of capabilities */
+    N: number;
+    /** Bigram sets per capability — post-stopword, post-stem, examples only */
+    bigrams: Record<string, Set<string>>;
+    /**
+     * Pre-computed token arrays per capability, per field.
+     * Avoids re-tokenizing capability text on every scoreCapability() call.
+     * At 50 capabilities × 100 req/s, that is 5,000 redundant tokenization
+     * calls per second — each involving stem() and split/filter chains.
+     */
+    capTokens: Record<string, {
+        examples: string[];
+        description: string[];
+        name: string[];
+    }>;
+}
+/** Build a BM25 index over all capabilities. Call once at manifest load. */
+export declare function buildBM25Index(capabilities: Capability[]): BM25Index;
+/**
+ * BM25 scoring with field weights.
+ * k1 = 1.5 (TF saturation), b = 0.75 (length normalization)
+ * Field weights: examples 0.6, description 0.3, name 0.1
+ */
+export declare function scoreCapability(qWordSet: Set<string>, cap: Capability, index: BM25Index, k1?: number, b?: number): number;
+/**
+ * Extracts bigrams from a token array as "token1__token2" strings.
+ * Input must already be post-stopword and post-stem (use tokenize() first).
+ */
+export declare function extractBigrams(tokens: string[]): Set<string>;
+/**
+ * Returns a sub-manifest containing only capabilities that match ALL provided tags.
+ * Capabilities without tags are excluded when tags filter is active.
+ * Enables token-efficient LLM prompts for large manifests:
+ *
+ * @example
+ * // Only send order-related capabilities to LLM
+ * const orderManifest = filterByTags(manifest, ['orders'])
+ * const result = await matchWithLLM(query, orderManifest, { llm })
+ *
+ * @example
+ * // Match by any of multiple tags (union) — call filterByTags per tag and merge
+ * const ordersOrPayments = [
+ *   ...filterByTags(manifest, ['orders']).capabilities,
+ *   ...filterByTags(manifest, ['payments']).capabilities,
+ * ]
+ */
+export declare function filterByTags(manifest: Manifest, tags: string[]): Manifest;
 export declare function resolverToIntent(cap: Capability): MatchResult['intent'];
+/**
+ * Strips characters that could break LLM prompt structure from
+ * capability field values before injection into the system prompt.
+ * Removes control characters, newlines, delimiter sequences, and braces
+ * anywhere in the string (not just at line starts) to resist prompt injection
+ * from third-party OpenAPI spec content ingested via parseOpenAPI().
+ */
+export declare function sanitizeForPrompt(value: string, maxLen: number): string;
 /**
  * Extracts parameter values from a user query using keyword heuristics.
  *
@@ -22,7 +104,17 @@ export declare function extractParams(query: string, cap: Capability): Record<st
 export interface MatchOptions {
     fuzzyMatch?: boolean;
     fuzzyThreshold?: number;
+    bm25Index?: BM25Index;
+    bm25K1?: number;
+    bm25B?: number;
+    bm25Ceiling?: number;
 }
+/**
+ * Calibrates a BM25 normalization ceiling from the manifest.
+ * Scores each capability against all of its own examples and returns the maximum.
+ * Call once at manifest load time — O(capabilities × examples).
+ */
+export declare function calibrateCeiling(capabilities: Capability[], bm25Index: BM25Index, k1: number, b: number): number;
 export declare function match(query: string, manifest: Manifest, options?: MatchOptions): MatchResult;
 export interface LLMMatcherOptions {
     llm: (prompt: string) => Promise<string>;

package/dist/esm/matcher.js

@@ -18,40 +18,265 @@ export const STOPWORDS = new Set([
     'it', 'its', 'how', 'when', 'where', 'who', 'which', 'all',
     'just', 'some', 'any', 'there', 'their', 'them', 'they',
 ]);
-function filterStopwords(words) {
-    return words.filter(w => !STOPWORDS.has(w.toLowerCase()) && w.length > 1);
+// ─── Type Patterns ────────────────────────────────────────────────────────────
+/**
+ * Regex patterns for common param types.
+ * Used when a CapabilityParam has `pattern` set to a named type.
+ */
+export const TYPE_PATTERNS = {
+    email: /\b[\w.+-]+@[\w-]+\.[a-zA-Z]{2,}\b/,
+    date: /\b\d{4}-\d{2}-\d{2}\b|\b(?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\w*\s+\d{1,2}\b/i,
+    orderId: /\b[A-Z]{2,}-?\d{4,}\b|\b\d{6,}\b/,
+    url: /https?:\/\/[^\s]+/,
+};
+/**
+ * Extracts a value from a query using an example template pattern.
+ * e.g. template "order {orderId}", query "track order 12345" → "12345"
+ * e.g. template "booking {ref}", query "cancel booking ABC-001" → "ABC-001"
+ */
+function extractFromTemplate(query, template, paramName) {
+    // Split template on {paramName} to get prefix and suffix
+    const placeholder = `{${paramName}}`;
+    const idx = template.indexOf(placeholder);
+    if (idx === -1)
+        return null;
+    const prefix = template.slice(0, idx).trim().toLowerCase();
+    const suffix = template.slice(idx + placeholder.length).trim().toLowerCase();
+    const q = query.toLowerCase();
+    if (prefix) {
+        const prefixIdx = q.indexOf(prefix);
+        if (prefixIdx === -1)
+            return null;
+        const after = query.slice(prefixIdx + prefix.length).trim();
+        const tokens = after.split(/\s+/).filter(t => t.length > 0);
+        if (!tokens.length)
+            return null;
+        // If there's a suffix, find it and take what's between
+        if (suffix) {
+            const suffixIdx = after.toLowerCase().indexOf(suffix);
+            if (suffixIdx > 0) {
+                return after.slice(0, suffixIdx).trim().split(/\s+/)[0] ?? null;
+            }
+        }
+        return tokens[0].replace(/[^a-zA-Z0-9\-_.@]/g, '') || null;
+    }
+    // Prefix is empty — placeholder is at start of template e.g. "{email} unsubscribe"
+    if (!prefix) {
+        if (suffix) {
+            // Find suffix in query — take what comes before it
+            const suffixIdx = query.toLowerCase().indexOf(suffix);
+            if (suffixIdx > 0) {
+                return query.slice(0, suffixIdx).trim().split(/\s+/).pop()
+                    ?.replace(/[^a-zA-Z0-9\-_.@]/g, '') || null;
+            }
+        }
+        // No prefix, no suffix — template is just "{paramName}"; take last meaningful word
+        const words = query.trim().split(/\s+/);
+        return words[words.length - 1]?.replace(/[^a-zA-Z0-9\-_.@]/g, '') || null;
+    }
+    return null;
 }
-function scoreCapability(qWordSet, cap) {
+// ─── Stem cache ───────────────────────────────────────────────────────────────
+// Each word stemmed exactly once per process — O(1) on repeat lookups.
+// Module-level — persists for the process lifetime. Vocabulary in production
+// is finite (capability names + user query vocabulary) so growth is bounded
+// in practice. In test environments with synthetic random strings, this may
+// grow larger but remains functionally harmless.
+const stemCache = new Map();
+/**
+ * Simplified suffix-stripping stemmer — 10 most common English morphological
+ * patterns covering ~80% of benefit at ~25% the complexity of Porter stemmer.
+ * Applied symmetrically to both query words and capability index words.
+ */
+export function stem(word) {
+    const cached = stemCache.get(word);
+    if (cached !== undefined)
+        return cached;
+    let s = word;
+    if (s.length > 7 && s.endsWith('ation'))
+        s = s.slice(0, -5); // cancellation → cancell
+    else if (s.length > 6 && s.endsWith('tion'))
+        s = s.slice(0, -4); // completion → comple
+    else if (s.length > 6 && s.endsWith('ing'))
+        s = s.slice(0, -3); // tracking → track
+    else if (s.length > 6 && s.endsWith('ity'))
+        s = s.slice(0, -3); // availability → availabil
+    else if (s.length > 5 && s.endsWith('ion'))
+        s = s.slice(0, -3); // version → vers
+    else if (s.length > 6 && s.endsWith('est'))
+        s = s.slice(0, -3); // fastest → fast
+    else if (s.length > 4 && s.endsWith('er'))
+        s = s.slice(0, -2); // tracker → track
+    else if (s.length > 4 && s.endsWith('ed'))
+        s = s.slice(0, -2); // ordered → order
+    else if (s.length > 4 && s.endsWith('ly'))
+        s = s.slice(0, -2); // quickly → quick
+    else if (s.length > 4 && s.endsWith('es'))
+        s = s.slice(0, -2); // fetches → fetch
+    else if (s.length > 3 && s.endsWith('s') &&
+        !s.endsWith('ss'))
+        s = s.slice(0, -1); // orders → order
+    stemCache.set(word, s);
+    return s;
+}
+/**
+ * Shared tokenizer — used by scorer, learning index, and boost system.
+ * Applies stopword filtering AND stemming symmetrically.
+ * Any site that tokenizes text for matching MUST use this function
+ * to avoid silent mismatches between query and index tokens.
+ */
+export function tokenize(text) {
+    return text
+        .toLowerCase()
+        .split(/\W+/)
+        .filter(w => w.length > 2 && !STOPWORDS.has(w))
+        .map(stem);
+}
+/** Build a BM25 index over all capabilities. Call once at manifest load. */
+export function buildBM25Index(capabilities) {
+    const N = capabilities.length;
+    if (N === 0)
+        return { df: {}, avgdl: { examples: 0, description: 0, name: 0 }, N: 0, bigrams: {}, capTokens: {}, };
+    const df = {};
+    let totalExLen = 0;
+    let totalDescLen = 0;
+    let totalNameLen = 0;
+    // Pre-compute token arrays for every capability in a single pass.
+    // scoreCapability() reads from capTokens instead of re-tokenizing on every call.
+    const capTokens = {};
+    for (const cap of capabilities) {
+        const exTokens = tokenize((cap.examples ?? []).join(' '));
+        const descTokens = tokenize(cap.description);
+        const nameTokens = tokenize(cap.name);
+        capTokens[cap.id] = { examples: exTokens, description: descTokens, name: nameTokens };
+        totalExLen += exTokens.length;
+        totalDescLen += descTokens.length;
+        totalNameLen += nameTokens.length;
+        // Count document frequency — each term counted once per capability
+        const seen = new Set();
+        for (const t of [...exTokens, ...descTokens, ...nameTokens]) {
+            if (!seen.has(t)) {
+                df[t] = (df[t] ?? 0) + 1;
+                seen.add(t);
+            }
+        }
+    }
+    // Build bigram sets per capability — examples field only
+    // Clean bigrams only: post-stopword, post-stem tokens
+    const bigrams = {};
+    for (const cap of capabilities) {
+        const set = new Set();
+        for (const example of cap.examples ?? []) {
+            for (const bg of extractBigrams(tokenize(example)))
+                set.add(bg);
+        }
+        bigrams[cap.id] = set;
+    }
+    return {
+        df,
+        avgdl: {
+            examples: totalExLen / N,
+            description: totalDescLen / N,
+            name: totalNameLen / N,
+        },
+        N,
+        bigrams,
+        capTokens,
+    };
+}
+/**
+ * BM25 scoring with field weights.
+ * k1 = 1.5 (TF saturation), b = 0.75 (length normalization)
+ * Field weights: examples 0.6, description 0.3, name 0.1
+ */
+export function scoreCapability(qWordSet, cap, index, k1 = 1.5, b = 0.75) {
+    if (index.N === 0)
+        return 0;
+    // Use pre-computed token arrays from the index — avoids re-tokenizing
+    // capability text on every call. Falls back to live tokenization only when
+    // scoreCapability() is called outside CapmanEngine (e.g. unit tests that
+    // build a BM25Index manually without capTokens populated).
+    const tokens = index.capTokens[cap.id];
+    const exTokens = tokens?.examples ?? tokenize((cap.examples ?? []).join(' '));
+    const descTokens = tokens?.description ?? tokenize(cap.description);
+    const nameTokens = tokens?.name ?? tokenize(cap.name);
+    const score = bm25Field(qWordSet, exTokens, index, 'examples', k1, b) * 0.6
+        + bm25Field(qWordSet, descTokens, index, 'description', k1, b) * 0.3
+        + bm25Field(qWordSet, nameTokens, index, 'name', k1, b) * 0.1;
+    return score;
+}
+function bm25Field(queryTerms, fieldTokens, index, field, k1, b) {
+    if (fieldTokens.length === 0)
+        return 0;
+    const avgdl = index.avgdl[field] || 1;
+    const dl = fieldTokens.length;
+    const tf = new Map();
+    for (const t of fieldTokens) {
+        tf.set(t, (tf.get(t) ?? 0) + 1);
+    }
     let score = 0;
-    // Check examples — take the best single example match, not the sum.
-    // Accumulating across examples rewards bloated example lists over precise ones:
-    // 10 examples at 50% overlap = 300 points (clamped to 60) beats 1 perfect example at 60.
-    // Taking Math.max means quality of examples matters, not quantity.
-    let bestExampleScore = 0;
-    for (const example of cap.examples ?? []) {
-        const exWords = filterStopwords(example.toLowerCase().split(/\s+/));
-        if (exWords.length === 0)
+    for (const term of queryTerms) {
+        const termTf = tf.get(term) ?? 0;
+        if (termTf === 0)
             continue;
-        const overlap = exWords.filter(w => qWordSet.has(w)).length;
-        const contribution = (overlap / exWords.length) * 60;
-        bestExampleScore = Math.max(bestExampleScore, contribution);
+        const df = index.df[term] ?? 0;
+        const idf = Math.log((index.N - df + 0.5) / (df + 0.5) + 1);
+        const tfNorm = (termTf * (k1 + 1)) / (termTf + k1 * (1 - b + b * (dl / avgdl)));
+        score += idf * tfNorm;
     }
-    score += bestExampleScore;
-    // Check description words — normalize against min(length, 10) to avoid
-    // penalizing rich documentation (many words = lower ratio) while also
-    // preventing single-word descriptions from maxing out on any match.
-    const descWords = filterStopwords(cap.description.toLowerCase().split(/\W+/).filter(Boolean));
-    if (descWords.length > 0) {
-        const descOverlap = descWords.filter(w => qWordSet.has(w)).length;
-        score += Math.min((descOverlap / Math.min(descWords.length, 10)) * 30, 30);
+    return score;
+}
+/**
+ * Extracts bigrams from a token array as "token1__token2" strings.
+ * Input must already be post-stopword and post-stem (use tokenize() first).
+ */
+export function extractBigrams(tokens) {
+    const bigrams = new Set();
+    for (let i = 0; i < tokens.length - 1; i++) {
+        bigrams.add(`${tokens[i]}__${tokens[i + 1]}`);
     }
-    // Check name words
-    const nameWords = filterStopwords(cap.name.toLowerCase().split(/\W+/).filter(Boolean));
-    if (nameWords.length > 0) {
-        const nameOverlap = nameWords.filter(w => qWordSet.has(w)).length;
-        score += (nameOverlap / nameWords.length) * 10;
+    return bigrams;
+}
+/**
+ * Returns a sub-manifest containing only capabilities that match ALL provided tags.
+ * Capabilities without tags are excluded when tags filter is active.
+ * Enables token-efficient LLM prompts for large manifests:
+ *
+ * @example
+ * // Only send order-related capabilities to LLM
+ * const orderManifest = filterByTags(manifest, ['orders'])
+ * const result = await matchWithLLM(query, orderManifest, { llm })
+ *
+ * @example
+ * // Match by any of multiple tags (union) — call filterByTags per tag and merge
+ * const ordersOrPayments = [
+ *   ...filterByTags(manifest, ['orders']).capabilities,
+ *   ...filterByTags(manifest, ['payments']).capabilities,
+ * ]
+ */
+export function filterByTags(manifest, tags) {
+    if (tags.length === 0)
+        return manifest;
+    const tagSet = new Set(tags);
+    return {
+        ...manifest,
+        capabilities: manifest.capabilities.filter(cap => cap.tags?.length && tags.every(t => cap.tags.includes(t))),
+    };
+}
+/**
+ * Returns a fixed bonus in normalized points (0–15), applied after BM25 normalization.
+ * 5 points per matching bigram, saturates at 3 bigrams (15 points).
+ * Fixed point value regardless of manifest size — ceiling-independent.
+ */
+function bigramBonus(queryBigrams, capBigrams) {
+    if (queryBigrams.size === 0 || capBigrams.size === 0)
+        return 0;
+    let overlap = 0;
+    for (const bigram of queryBigrams) {
+        if (capBigrams.has(bigram))
+            overlap++;
     }
-    return Math.min(Math.round(score), 100);
+    return Math.min(overlap * 5, 15); // normalized points — 3 bigrams saturate at 15
 }
 export function resolverToIntent(cap) {
     const t = cap.resolver.type;
@@ -66,13 +291,18 @@ export function resolverToIntent(cap) {
 /**
  * Strips characters that could break LLM prompt structure from
  * capability field values before injection into the system prompt.
- * Removes control characters, newlines, and delimiter-like sequences.
+ * Removes control characters, newlines, delimiter sequences, and braces
+ * anywhere in the string (not just at line starts) to resist prompt injection
+ * from third-party OpenAPI spec content ingested via parseOpenAPI().
  */
-function sanitizeForPrompt(value, maxLen) {
+export function sanitizeForPrompt(value, maxLen) {
     return value
-        .replace(/[\r\n\t]/g, ' ') // newlines → space
+        .replace(/[\r\n\t]/g, ' ') // newlines/tabs → space
         .replace(/---+/g, '—') // horizontal rules → em dash
-        .replace(/^\s*[{}\[\]]/gm, ' ') // leading braces/brackets → space
+        .replace(/[{}\[\]]/g, ' ') // all braces/brackets anywhere → space (was: leading only)
+        .split(' ') // per-word cap — limits injection payload per token
+        .map(w => w.slice(0, 200)) // no single token longer than 200 chars
+        .join(' ')
         .replace(/\s+/g, ' ') // collapse whitespace
         .trim()
         .slice(0, maxLen);
@@ -104,6 +334,42 @@ export function extractParams(query, cap) {
             result[param.name] = null;
             continue;
         }
+        // ── Type-implied pattern extraction ───────────────────────────────────
+        // param.type implies a TYPE_PATTERNS match — no need to set pattern explicitly
+        if (param.type && !param.pattern) {
+            // Map param types that have direct regex equivalents
+            const typeToPattern = {
+                email: TYPE_PATTERNS.email,
+                date: TYPE_PATTERNS.date,
+                url: TYPE_PATTERNS.url,
+            };
+            const impliedPattern = typeToPattern[param.type];
+            if (impliedPattern) {
+                const match = query.match(impliedPattern);
+                if (match) {
+                    result[param.name] = match[0];
+                    continue;
+                }
+            }
+        }
+        // ── Explicit pattern extraction (highest priority when set) ───────────
+        if (param.pattern) {
+            const namedPattern = TYPE_PATTERNS[param.pattern];
+            if (namedPattern) {
+                const match = query.match(namedPattern);
+                if (match) {
+                    result[param.name] = match[0];
+                    continue;
+                }
+            }
+            else if (param.pattern.includes(`{${param.name}}`)) {
+                const extracted = extractFromTemplate(query, param.pattern, param.name);
+                if (extracted) {
+                    result[param.name] = extracted;
+                    continue;
+                }
+            }
+        }
         // Try to extract value after known keywords
         // e.g. "profile for johndoe" → johndoe
         //      "articles by jane"   → jane
@@ -157,10 +423,36 @@ export function extractParams(query, cap) {
                 extracted = candidate;
             }
         }
+        // ── Enum validation ───────────────────────────────────────────────────
+        if (extracted !== null && param.type === 'enum' && param.enum?.length) {
+            if (!param.enum.includes(extracted)) {
+                // Extracted value not in allowed list — treat as not found
+                extracted = null;
+            }
+        }
         result[param.name] = extracted;
     }
     return result;
 }
+/**
+ * Calibrates a BM25 normalization ceiling from the manifest.
+ * Scores each capability against all of its own examples and returns the maximum.
+ * Call once at manifest load time — O(capabilities × examples).
+ */
+export function calibrateCeiling(capabilities, bm25Index, k1, b) {
+    let max = 0;
+    for (const cap of capabilities) {
+        if (!cap.examples?.length)
+            continue;
+        for (const example of cap.examples) {
+            const selfWords = new Set(tokenize(example));
+            const raw = scoreCapability(selfWords, cap, bm25Index, k1, b);
+            if (raw > max)
+                max = raw;
+        }
+    }
+    return max > 0 ? max : 100;
+}
 export function match(query, manifest, options = {}) {
     if (!query?.trim()) {
         logger.warn('Empty query received');
@@ -225,10 +517,23 @@ export function match(query, manifest, options = {}) {
     }
     // ── Score all capabilities ────────────────────────────────────────────────
     // Build qWordSet once — O(1) lookups instead of O(n) Array.includes per word
-    const qWordSet = new Set(filterStopwords(query.toLowerCase().split(/\W+/).filter(Boolean)));
+    const qTokens = tokenize(query);
+    const qWordSet = new Set(qTokens);
+    // Build query bigrams for phrase bonus
+    const qBigrams = extractBigrams(qTokens);
+    // Build BM25 index for this manifest — O(capabilities × tokens)
+    // In CapmanEngine this is pre-built; for direct match() calls it's built per-call
+    const bm25Index = options.bm25Index ?? buildBM25Index(manifest.capabilities);
+    const k1 = options.bm25K1 ?? 1.5;
+    const b = options.bm25B ?? 0.75;
+    // Calibrate ceiling — max self-score for normalization
+    const ceiling = options.bm25Ceiling ?? calibrateCeiling(manifest.capabilities, bm25Index, k1, b);
     const allScores = [];
     for (const cap of manifest.capabilities) {
-        const keywordScore = scoreCapability(qWordSet, cap);
+        const rawBM25 = scoreCapability(qWordSet, cap, bm25Index, k1, b);
+        const bm25Score = Math.min(100, Math.round((rawBM25 / ceiling) * 100));
+        const bonusPoints = bigramBonus(qBigrams, bm25Index.bigrams[cap.id] ?? new Set());
+        const keywordScore = Math.min(100, bm25Score + bonusPoints);
         const fuzzyScore = fuzzyScoreMap.get(cap.id) ?? 0;
         const via = fuzzyScore > keywordScore ? 'fuzzy' : 'keyword';
         const score = Math.min(100, Math.round(Math.max(keywordScore, fuzzyScore)));
@@ -345,7 +650,13 @@ ${JSON.stringify({ user_query: query })}
     // Build full candidate list — all capabilities scored, LLM winner marked as matched.
     // This aligns the shape with keyword match results and allows the learning boost
     // to surface alternatives if the LLM made a wrong call.
-    const llmConfidence = effectivelyOOS ? 0 : parsed.confidence;
+    // Clamp and round confidence — LLM may return values outside 0–100 with
+    // misconfigured models or prompt drift. Unclamped values corrupt learning
+    // weights (weight = confidence/100 can exceed 1.0) and verdict margins.
+    // disambiguateLLM() already does this; apply the same treatment here.
+    const llmConfidence = effectivelyOOS
+        ? 0
+        : Math.min(100, Math.max(0, Math.round(parsed.confidence)));
     const allCandidates = manifest.capabilities.map(c => ({
         capabilityId: c.id,
         score: c.id === capability?.id ? llmConfidence : 0,