capman 0.5.5 → 0.6.0

package/dist/esm/matcher.js

@@ -18,40 +18,226 @@ export const STOPWORDS = new Set([
     'it', 'its', 'how', 'when', 'where', 'who', 'which', 'all',
     'just', 'some', 'any', 'there', 'their', 'them', 'they',
 ]);
-function filterStopwords(words) {
-    return words.filter(w => !STOPWORDS.has(w.toLowerCase()) && w.length > 1);
+// ─── Type Patterns ────────────────────────────────────────────────────────────
+/**
+ * Regex patterns for common param types.
+ * Used when a CapabilityParam has `pattern` set to a named type.
+ */
+export const TYPE_PATTERNS = {
+    email: /\b[\w.+-]+@[\w-]+\.[a-zA-Z]{2,}\b/,
+    date: /\b\d{4}-\d{2}-\d{2}\b|\b(?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\w*\s+\d{1,2}\b/i,
+    orderId: /\b[A-Z]{2,}-?\d{4,}\b|\b\d{6,}\b/,
+    url: /https?:\/\/[^\s]+/,
+};
+/**
+ * Extracts a value from a query using an example template pattern.
+ * e.g. template "order {orderId}", query "track order 12345" → "12345"
+ * e.g. template "booking {ref}", query "cancel booking ABC-001" → "ABC-001"
+ */
+function extractFromTemplate(query, template, paramName) {
+    // Split template on {paramName} to get prefix and suffix
+    const placeholder = `{${paramName}}`;
+    const idx = template.indexOf(placeholder);
+    if (idx === -1)
+        return null;
+    const prefix = template.slice(0, idx).trim().toLowerCase();
+    const suffix = template.slice(idx + placeholder.length).trim().toLowerCase();
+    const q = query.toLowerCase();
+    if (prefix) {
+        const prefixIdx = q.indexOf(prefix);
+        if (prefixIdx === -1)
+            return null;
+        const after = query.slice(prefixIdx + prefix.length).trim();
+        const tokens = after.split(/\s+/).filter(t => t.length > 0);
+        if (!tokens.length)
+            return null;
+        // If there's a suffix, find it and take what's between
+        if (suffix) {
+            const suffixIdx = after.toLowerCase().indexOf(suffix);
+            if (suffixIdx > 0) {
+                return after.slice(0, suffixIdx).trim().split(/\s+/)[0] ?? null;
+            }
+        }
+        return tokens[0].replace(/[^a-zA-Z0-9\-_.@]/g, '') || null;
+    }
+    // Prefix is empty — placeholder is at start of template e.g. "{email} unsubscribe"
+    if (!prefix) {
+        if (suffix) {
+            // Find suffix in query — take what comes before it
+            const suffixIdx = query.toLowerCase().indexOf(suffix);
+            if (suffixIdx > 0) {
+                return query.slice(0, suffixIdx).trim().split(/\s+/).pop()
+                    ?.replace(/[^a-zA-Z0-9\-_.@]/g, '') || null;
+            }
+        }
+        // No prefix, no suffix — template is just "{paramName}"; take last meaningful word
+        const words = query.trim().split(/\s+/);
+        return words[words.length - 1]?.replace(/[^a-zA-Z0-9\-_.@]/g, '') || null;
+    }
+    return null;
+}
+// ─── Stem cache ───────────────────────────────────────────────────────────────
+// Each word stemmed exactly once per process — O(1) on repeat lookups.
+// Module-level — persists for the process lifetime. Vocabulary in production
+// is finite (capability names + user query vocabulary) so growth is bounded
+// in practice. In test environments with synthetic random strings, this may
+// grow larger but remains functionally harmless.
+const stemCache = new Map();
+/**
+ * Simplified suffix-stripping stemmer — 10 most common English morphological
+ * patterns covering ~80% of benefit at ~25% the complexity of Porter stemmer.
+ * Applied symmetrically to both query words and capability index words.
+ */
+export function stem(word) {
+    const cached = stemCache.get(word);
+    if (cached !== undefined)
+        return cached;
+    let s = word;
+    if (s.length > 7 && s.endsWith('ation'))
+        s = s.slice(0, -5); // cancellation → cancell
+    else if (s.length > 6 && s.endsWith('tion'))
+        s = s.slice(0, -4); // completion → comple
+    else if (s.length > 6 && s.endsWith('ing'))
+        s = s.slice(0, -3); // tracking → track
+    else if (s.length > 6 && s.endsWith('ity'))
+        s = s.slice(0, -3); // availability → availabil
+    else if (s.length > 5 && s.endsWith('ion'))
+        s = s.slice(0, -3); // version → vers
+    else if (s.length > 6 && s.endsWith('est'))
+        s = s.slice(0, -3); // fastest → fast
+    else if (s.length > 4 && s.endsWith('er'))
+        s = s.slice(0, -2); // tracker → track
+    else if (s.length > 4 && s.endsWith('ed'))
+        s = s.slice(0, -2); // ordered → order
+    else if (s.length > 4 && s.endsWith('ly'))
+        s = s.slice(0, -2); // quickly → quick
+    else if (s.length > 4 && s.endsWith('es'))
+        s = s.slice(0, -2); // fetches → fetch
+    else if (s.length > 3 && s.endsWith('s') &&
+        !s.endsWith('ss'))
+        s = s.slice(0, -1); // orders → order
+    stemCache.set(word, s);
+    return s;
+}
+/**
+ * Shared tokenizer — used by scorer, learning index, and boost system.
+ * Applies stopword filtering AND stemming symmetrically.
+ * Any site that tokenizes text for matching MUST use this function
+ * to avoid silent mismatches between query and index tokens.
+ */
+export function tokenize(text) {
+    return text
+        .toLowerCase()
+        .split(/\W+/)
+        .filter(w => w.length > 2 && !STOPWORDS.has(w))
+        .map(stem);
+}
+/** Build a BM25 index over all capabilities. Call once at manifest load. */
+export function buildBM25Index(capabilities) {
+    const N = capabilities.length;
+    if (N === 0)
+        return { df: {}, avgdl: { examples: 0, description: 0, name: 0 }, N: 0, bigrams: {}, };
+    const df = {};
+    let totalExLen = 0;
+    let totalDescLen = 0;
+    let totalNameLen = 0;
+    for (const cap of capabilities) {
+        const exTokens = tokenize((cap.examples ?? []).join(' '));
+        const descTokens = tokenize(cap.description);
+        const nameTokens = tokenize(cap.name);
+        totalExLen += exTokens.length;
+        totalDescLen += descTokens.length;
+        totalNameLen += nameTokens.length;
+        // Count document frequency — each term counted once per capability
+        const seen = new Set();
+        for (const t of [...exTokens, ...descTokens, ...nameTokens]) {
+            if (!seen.has(t)) {
+                df[t] = (df[t] ?? 0) + 1;
+                seen.add(t);
+            }
+        }
+    }
+    // Build bigram sets per capability — examples field only
+    // Clean bigrams only: post-stopword, post-stem tokens
+    const bigrams = {};
+    for (const cap of capabilities) {
+        const set = new Set();
+        for (const example of cap.examples ?? []) {
+            for (const bg of extractBigrams(tokenize(example)))
+                set.add(bg);
+        }
+        bigrams[cap.id] = set;
+    }
+    return {
+        df,
+        avgdl: {
+            examples: totalExLen / N,
+            description: totalDescLen / N,
+            name: totalNameLen / N,
+        },
+        N,
+        bigrams,
+    };
 }
-function scoreCapability(qWordSet, cap) {
+/**
+ * BM25 scoring with field weights.
+ * k1 = 1.5 (TF saturation), b = 0.75 (length normalization)
+ * Field weights: examples 0.6, description 0.3, name 0.1
+ */
+export function scoreCapability(qWordSet, cap, index, k1 = 1.5, b = 0.75) {
+    if (index.N === 0)
+        return 0;
+    const score = bm25Field(qWordSet, tokenize((cap.examples ?? []).join(' ')), index, 'examples', k1, b) * 0.6
+        + bm25Field(qWordSet, tokenize(cap.description), index, 'description', k1, b) * 0.3
+        + bm25Field(qWordSet, tokenize(cap.name), index, 'name', k1, b) * 0.1;
+    return score;
+}
+function bm25Field(queryTerms, fieldTokens, index, field, k1, b) {
+    if (fieldTokens.length === 0)
+        return 0;
+    const avgdl = index.avgdl[field] || 1;
+    const dl = fieldTokens.length;
+    const tf = new Map();
+    for (const t of fieldTokens) {
+        tf.set(t, (tf.get(t) ?? 0) + 1);
+    }
     let score = 0;
-    // Check examples — take the best single example match, not the sum.
-    // Accumulating across examples rewards bloated example lists over precise ones:
-    // 10 examples at 50% overlap = 300 points (clamped to 60) beats 1 perfect example at 60.
-    // Taking Math.max means quality of examples matters, not quantity.
-    let bestExampleScore = 0;
-    for (const example of cap.examples ?? []) {
-        const exWords = filterStopwords(example.toLowerCase().split(/\s+/));
-        if (exWords.length === 0)
+    for (const term of queryTerms) {
+        const termTf = tf.get(term) ?? 0;
+        if (termTf === 0)
             continue;
-        const overlap = exWords.filter(w => qWordSet.has(w)).length;
-        const contribution = (overlap / exWords.length) * 60;
-        bestExampleScore = Math.max(bestExampleScore, contribution);
+        const df = index.df[term] ?? 0;
+        const idf = Math.log((index.N - df + 0.5) / (df + 0.5) + 1);
+        const tfNorm = (termTf * (k1 + 1)) / (termTf + k1 * (1 - b + b * (dl / avgdl)));
+        score += idf * tfNorm;
     }
-    score += bestExampleScore;
-    // Check description words — normalize against min(length, 10) to avoid
-    // penalizing rich documentation (many words = lower ratio) while also
-    // preventing single-word descriptions from maxing out on any match.
-    const descWords = filterStopwords(cap.description.toLowerCase().split(/\W+/).filter(Boolean));
-    if (descWords.length > 0) {
-        const descOverlap = descWords.filter(w => qWordSet.has(w)).length;
-        score += Math.min((descOverlap / Math.min(descWords.length, 10)) * 30, 30);
+    return score;
+}
+/**
+ * Extracts bigrams from a token array as "token1__token2" strings.
+ * Input must already be post-stopword and post-stem (use tokenize() first).
+ */
+export function extractBigrams(tokens) {
+    const bigrams = new Set();
+    for (let i = 0; i < tokens.length - 1; i++) {
+        bigrams.add(`${tokens[i]}__${tokens[i + 1]}`);
     }
-    // Check name words
-    const nameWords = filterStopwords(cap.name.toLowerCase().split(/\W+/).filter(Boolean));
-    if (nameWords.length > 0) {
-        const nameOverlap = nameWords.filter(w => qWordSet.has(w)).length;
-        score += (nameOverlap / nameWords.length) * 10;
+    return bigrams;
+}
+/**
+ * Returns a fixed bonus in normalized points (0–15), applied after BM25 normalization.
+ * 5 points per matching bigram, saturates at 3 bigrams (15 points).
+ * Fixed point value regardless of manifest size — ceiling-independent.
+ */
+function bigramBonus(queryBigrams, capBigrams) {
+    if (queryBigrams.size === 0 || capBigrams.size === 0)
+        return 0;
+    let overlap = 0;
+    for (const bigram of queryBigrams) {
+        if (capBigrams.has(bigram))
+            overlap++;
     }
-    return Math.min(Math.round(score), 100);
+    return Math.min(overlap * 5, 15); // normalized points — 3 bigrams saturate at 15
 }
 export function resolverToIntent(cap) {
     const t = cap.resolver.type;
@@ -68,7 +254,7 @@ export function resolverToIntent(cap) {
  * capability field values before injection into the system prompt.
  * Removes control characters, newlines, and delimiter-like sequences.
  */
-function sanitizeForPrompt(value, maxLen) {
+export function sanitizeForPrompt(value, maxLen) {
     return value
         .replace(/[\r\n\t]/g, ' ') // newlines → space
         .replace(/---+/g, '—') // horizontal rules → em dash
@@ -104,6 +290,26 @@ export function extractParams(query, cap) {
             result[param.name] = null;
             continue;
         }
+        // ── Pattern extraction (highest priority) ─────────────────────────────
+        if (param.pattern) {
+            const namedPattern = TYPE_PATTERNS[param.pattern];
+            if (namedPattern) {
+                // Named type pattern — match regex directly against full query
+                const match = query.match(namedPattern);
+                if (match) {
+                    result[param.name] = match[0];
+                    continue;
+                }
+            }
+            else if (param.pattern.includes(`{${param.name}}`)) {
+                // Example template — positional extraction
+                const extracted = extractFromTemplate(query, param.pattern, param.name);
+                if (extracted) {
+                    result[param.name] = extracted;
+                    continue;
+                }
+            }
+        }
         // Try to extract value after known keywords
         // e.g. "profile for johndoe" → johndoe
         //      "articles by jane"   → jane
@@ -225,10 +431,34 @@ export function match(query, manifest, options = {}) {
     }
     // ── Score all capabilities ────────────────────────────────────────────────
     // Build qWordSet once — O(1) lookups instead of O(n) Array.includes per word
-    const qWordSet = new Set(filterStopwords(query.toLowerCase().split(/\W+/).filter(Boolean)));
+    const qTokens = tokenize(query);
+    const qWordSet = new Set(qTokens);
+    // Build query bigrams for phrase bonus
+    const qBigrams = extractBigrams(qTokens);
+    // Build BM25 index for this manifest — O(capabilities × tokens)
+    // In CapmanEngine this is pre-built; for direct match() calls it's built per-call
+    const bm25Index = options.bm25Index ?? buildBM25Index(manifest.capabilities);
+    const k1 = options.bm25K1 ?? 1.5;
+    const b = options.bm25B ?? 0.75;
+    // Calibrate ceiling — max self-score for normalization
+    const ceiling = options.bm25Ceiling ?? (() => {
+        let max = 0;
+        for (const cap of manifest.capabilities) {
+            if (!cap.examples?.length)
+                continue;
+            const selfWords = new Set(tokenize(cap.examples[0]));
+            const raw = scoreCapability(selfWords, cap, bm25Index, k1, b);
+            if (raw > max)
+                max = raw;
+        }
+        return max > 0 ? max : 100;
+    })();
     const allScores = [];
     for (const cap of manifest.capabilities) {
-        const keywordScore = scoreCapability(qWordSet, cap);
+        const rawBM25 = scoreCapability(qWordSet, cap, bm25Index, k1, b);
+        const bm25Score = Math.min(100, Math.round((rawBM25 / ceiling) * 100));
+        const bonusPoints = bigramBonus(qBigrams, bm25Index.bigrams[cap.id] ?? new Set());
+        const keywordScore = Math.min(100, bm25Score + bonusPoints);
         const fuzzyScore = fuzzyScoreMap.get(cap.id) ?? 0;
         const via = fuzzyScore > keywordScore ? 'fuzzy' : 'keyword';
         const score = Math.min(100, Math.round(Math.max(keywordScore, fuzzyScore)));

package/dist/esm/schema.d.ts

@@ -13,18 +13,21 @@ export declare const CapmanConfigSchema: z.ZodEffects<z.ZodObject<{
             required: z.ZodBoolean;
             source: z.ZodEnum<["user_query", "session"]>;
             default: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
+            pattern: z.ZodOptional<z.ZodString>;
         }, "strip", z.ZodTypeAny, {
             name: string;
-            required: boolean;
             description: string;
+            required: boolean;
             source: "user_query" | "session";
             default?: string | number | boolean | undefined;
+            pattern?: string | undefined;
         }, {
             name: string;
-            required: boolean;
             description: string;
+            required: boolean;
             source: "user_query" | "session";
             default?: string | number | boolean | undefined;
+            pattern?: string | undefined;
         }>, "many">;
         returns: z.ZodArray<z.ZodString, "many">;
         resolver: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
@@ -147,14 +150,15 @@ export declare const CapmanConfigSchema: z.ZodEffects<z.ZodObject<{
     }, "strip", z.ZodTypeAny, {
         name: string;
         id: string;
+        description: string;
         params: {
             name: string;
-            required: boolean;
             description: string;
+            required: boolean;
             source: "user_query" | "session";
             default?: string | number | boolean | undefined;
+            pattern?: string | undefined;
         }[];
-        description: string;
         returns: string[];
         resolver: {
             type: "api";
@@ -189,14 +193,15 @@ export declare const CapmanConfigSchema: z.ZodEffects<z.ZodObject<{
     }, {
         name: string;
         id: string;
+        description: string;
         params: {
             name: string;
-            required: boolean;
             description: string;
+            required: boolean;
             source: "user_query" | "session";
             default?: string | number | boolean | undefined;
+            pattern?: string | undefined;
         }[];
-        description: string;
         returns: string[];
         resolver: {
             type: "api";
@@ -231,14 +236,15 @@ export declare const CapmanConfigSchema: z.ZodEffects<z.ZodObject<{
     }>, "many">, {
         name: string;
         id: string;
+        description: string;
         params: {
             name: string;
-            required: boolean;
             description: string;
+            required: boolean;
             source: "user_query" | "session";
             default?: string | number | boolean | undefined;
+            pattern?: string | undefined;
         }[];
-        description: string;
         returns: string[];
         resolver: {
             type: "api";
@@ -273,14 +279,15 @@ export declare const CapmanConfigSchema: z.ZodEffects<z.ZodObject<{
     }[], {
         name: string;
         id: string;
+        description: string;
         params: {
             name: string;
-            required: boolean;
             description: string;
+            required: boolean;
             source: "user_query" | "session";
             default?: string | number | boolean | undefined;
+            pattern?: string | undefined;
         }[];
-        description: string;
         returns: string[];
         resolver: {
             type: "api";
@@ -318,14 +325,15 @@ export declare const CapmanConfigSchema: z.ZodEffects<z.ZodObject<{
     capabilities: {
         name: string;
         id: string;
+        description: string;
         params: {
             name: string;
-            required: boolean;
             description: string;
+            required: boolean;
             source: "user_query" | "session";
             default?: string | number | boolean | undefined;
+            pattern?: string | undefined;
         }[];
-        description: string;
         returns: string[];
         resolver: {
             type: "api";
@@ -364,14 +372,15 @@ export declare const CapmanConfigSchema: z.ZodEffects<z.ZodObject<{
     capabilities: {
         name: string;
         id: string;
+        description: string;
         params: {
             name: string;
-            required: boolean;
             description: string;
+            required: boolean;
             source: "user_query" | "session";
             default?: string | number | boolean | undefined;
+            pattern?: string | undefined;
         }[];
-        description: string;
         returns: string[];
         resolver: {
             type: "api";
@@ -410,14 +419,15 @@ export declare const CapmanConfigSchema: z.ZodEffects<z.ZodObject<{
     capabilities: {
         name: string;
         id: string;
+        description: string;
         params: {
             name: string;
-            required: boolean;
             description: string;
+            required: boolean;
             source: "user_query" | "session";
             default?: string | number | boolean | undefined;
+            pattern?: string | undefined;
         }[];
-        description: string;
         returns: string[];
         resolver: {
             type: "api";
@@ -456,14 +466,15 @@ export declare const CapmanConfigSchema: z.ZodEffects<z.ZodObject<{
     capabilities: {
         name: string;
         id: string;
+        description: string;
         params: {
             name: string;
-            required: boolean;
             description: string;
+            required: boolean;
             source: "user_query" | "session";
             default?: string | number | boolean | undefined;
+            pattern?: string | undefined;
         }[];
-        description: string;
         returns: string[];
         resolver: {
             type: "api";
@@ -513,18 +524,21 @@ export declare const ManifestSchema: z.ZodObject<{
             required: z.ZodBoolean;
             source: z.ZodEnum<["user_query", "session"]>;
             default: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
+            pattern: z.ZodOptional<z.ZodString>;
         }, "strip", z.ZodTypeAny, {
             name: string;
-            required: boolean;
             description: string;
+            required: boolean;
             source: "user_query" | "session";
             default?: string | number | boolean | undefined;
+            pattern?: string | undefined;
         }, {
             name: string;
-            required: boolean;
             description: string;
+            required: boolean;
             source: "user_query" | "session";
             default?: string | number | boolean | undefined;
+            pattern?: string | undefined;
         }>, "many">;
         returns: z.ZodArray<z.ZodString, "many">;
         resolver: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
@@ -647,14 +661,15 @@ export declare const ManifestSchema: z.ZodObject<{
     }, "strip", z.ZodTypeAny, {
         name: string;
         id: string;
+        description: string;
         params: {
             name: string;
-            required: boolean;
             description: string;
+            required: boolean;
             source: "user_query" | "session";
             default?: string | number | boolean | undefined;
+            pattern?: string | undefined;
         }[];
-        description: string;
         returns: string[];
         resolver: {
             type: "api";
@@ -689,14 +704,15 @@ export declare const ManifestSchema: z.ZodObject<{
     }, {
         name: string;
         id: string;
+        description: string;
         params: {
             name: string;
-            required: boolean;
             description: string;
+            required: boolean;
             source: "user_query" | "session";
             default?: string | number | boolean | undefined;
+            pattern?: string | undefined;
         }[];
-        description: string;
         returns: string[];
         resolver: {
             type: "api";
@@ -735,14 +751,15 @@ export declare const ManifestSchema: z.ZodObject<{
     capabilities: {
         name: string;
         id: string;
+        description: string;
         params: {
             name: string;
-            required: boolean;
             description: string;
+            required: boolean;
             source: "user_query" | "session";
             default?: string | number | boolean | undefined;
+            pattern?: string | undefined;
         }[];
-        description: string;
         returns: string[];
         resolver: {
             type: "api";
@@ -782,14 +799,15 @@ export declare const ManifestSchema: z.ZodObject<{
     capabilities: {
         name: string;
         id: string;
+        description: string;
         params: {
             name: string;
-            required: boolean;
             description: string;
+            required: boolean;
             source: "user_query" | "session";
             default?: string | number | boolean | undefined;
+            pattern?: string | undefined;
         }[];
-        description: string;
         returns: string[];
         resolver: {
             type: "api";

package/dist/esm/schema.js

@@ -6,6 +6,7 @@ const CapabilityParamSchema = z.object({
     required: z.boolean(),
     source: z.enum(['user_query', 'session']),
     default: z.union([z.string(), z.number(), z.boolean()]).optional(),
+    pattern: z.string().optional(),
 });
 // ─── Resolver Schemas ─────────────────────────────────────────────────────────
 const ApiResolverSchema = z.object({

package/dist/esm/types.d.ts

@@ -5,7 +5,13 @@ export interface CapabilityParam {
     description: string;
     required: boolean;
     source: 'user_query' | 'session';
-    default?: string | number | boolean;
+    /**
+     * Optional extraction hint. Either a named type or an example template.
+     * Named types: 'email' | 'date' | 'orderId' | 'url'
+     * Example template: "order {paramName}" — extracts token after "order"
+     * When provided, pattern matching runs before keyword heuristics.
+     */
+    pattern?: string;
 }
 export interface ApiResolver {
     type: 'api';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "capman",
-  "version": "0.5.5",
+  "version": "0.6.0",
   "description": "Capability Manifest Engine — let AI agents interact with your app without navigating the UI",
   "main": "./dist/cjs/index.js",
   "module": "./dist/esm/index.js",