capman 0.5.1 → 0.5.3

package/dist/esm/learning.js

@@ -1,37 +1,18 @@
 import * as fs from 'fs';
 import * as path from 'path';
 import { logger } from './logger';
-const MAX_LEARNING_ENTRIES = 10000;
+const MAX_LEARNING_ENTRIES = 10_000;
 import { STOPWORDS } from './matcher';
-// ─── Shared computation helpers ───────────────────────────────────────────────
-function computeStats(entries) {
-    const index = {};
-    let totalQueries = 0;
-    let llmQueries = 0;
-    let cacheHits = 0;
-    let outOfScope = 0;
-    for (const entry of entries) {
-        totalQueries++;
-        if (entry.resolvedVia === 'llm')
-            llmQueries++;
-        if (entry.resolvedVia === 'cache')
-            cacheHits++;
-        if (!entry.capabilityId)
-            outOfScope++;
-        if (entry.capabilityId) {
-            const words = entry.query.toLowerCase()
-                .split(/\W+/)
-                .filter(w => w.length > 2 && !STOPWORDS.has(w));
-            for (const word of words) {
-                if (!index[word])
-                    index[word] = {};
-                index[word][entry.capabilityId] =
-                    (index[word][entry.capabilityId] ?? 0) + 1;
-            }
-        }
+// Module-level registry — tracks all active FileLearningStore instances
+// for process exit flushing. Handlers registered once to avoid accumulation.
+const activeStores = new Set();
+let exitHandlersRegistered = false;
+function flushAllStores() {
+    for (const store of activeStores) {
+        store.flushSync();
     }
-    return { index, totalQueries, llmQueries, cacheHits, outOfScope };
 }
+// ─── Shared computation helpers ───────────────────────────────────────────────
 function computeTopCapabilities(entries, limit) {
     const counts = {};
     for (const entry of entries) {
@@ -44,22 +25,18 @@ function computeTopCapabilities(entries, limit) {
         .slice(0, limit)
         .map(([id, hits]) => ({ id, hits }));
 }
-// ─── File Learning Store ──────────────────────────────────────────────────────
-export class FileLearningStore {
-    constructor(filePath = '.capman/learning.json') {
-        this.entries = [];
-        this.loaded = false;
-        this.saveQueue = Promise.resolve();
-        // ── Incremental index — updated in record(), not rebuilt in getStats() ────
+// ─── Shared Learning Index ────────────────────────────────────────────────────
+// Encapsulates keyword index and stats counters.
+// Both FileLearningStore and MemoryLearningStore compose this instead of
+// duplicating the same ~80 lines of index management logic.
+class LearningIndex {
+    constructor() {
         this.index = {};
         this.statsCounter = {
             totalQueries: 0, llmQueries: 0, cacheHits: 0, outOfScope: 0,
         };
-        this.filePath = path.resolve(process.cwd(), filePath);
-        logger.info(`FileLearningStore initialized — writing to: ${this.filePath}`);
     }
-    updateIndex(entry) {
-        var _a;
+    update(entry) {
         this.statsCounter.totalQueries++;
         if (entry.resolvedVia === 'llm')
             this.statsCounter.llmQueries++;
@@ -72,27 +49,24 @@ export class FileLearningStore {
                 .split(/\W+/)
                 .filter(w => w.length > 2 && !STOPWORDS.has(w));
             for (const word of words) {
-                (_a = this.index)[word] ?? (_a[word] = {});
+                this.index[word] ??= {};
                 this.index[word][entry.capabilityId] =
                     (this.index[word][entry.capabilityId] ?? 0) + 1;
             }
         }
     }
-    subtractFromIndex(entry) {
-        if (!entry.capabilityId) {
-            this.statsCounter.outOfScope = Math.max(0, this.statsCounter.outOfScope - 1);
-            this.statsCounter.totalQueries = Math.max(0, this.statsCounter.totalQueries - 1);
-            if (entry.resolvedVia === 'llm')
-                this.statsCounter.llmQueries = Math.max(0, this.statsCounter.llmQueries - 1);
-            if (entry.resolvedVia === 'cache')
-                this.statsCounter.cacheHits = Math.max(0, this.statsCounter.cacheHits - 1);
-            return;
-        }
+    subtract(entry) {
+        // Shared counter decrements regardless of capabilityId
         this.statsCounter.totalQueries = Math.max(0, this.statsCounter.totalQueries - 1);
         if (entry.resolvedVia === 'llm')
             this.statsCounter.llmQueries = Math.max(0, this.statsCounter.llmQueries - 1);
         if (entry.resolvedVia === 'cache')
             this.statsCounter.cacheHits = Math.max(0, this.statsCounter.cacheHits - 1);
+        if (!entry.capabilityId) {
+            this.statsCounter.outOfScope = Math.max(0, this.statsCounter.outOfScope - 1);
+            return;
+        }
+        // Keyword index cleanup
         const words = entry.query.toLowerCase()
             .split(/\W+/)
             .filter(w => w.length > 2 && !STOPWORDS.has(w));
@@ -109,22 +83,102 @@ export class FileLearningStore {
             }
         }
     }
-    rebuildIndex() {
+    rebuild(entries) {
+        this.index = {};
+        this.statsCounter = { totalQueries: 0, llmQueries: 0, cacheHits: 0, outOfScope: 0 };
+        for (const entry of entries) {
+            this.update(entry);
+        }
+    }
+    reset() {
         this.index = {};
         this.statsCounter = { totalQueries: 0, llmQueries: 0, cacheHits: 0, outOfScope: 0 };
-        for (const entry of this.entries) {
-            this.updateIndex(entry);
+    }
+    getStats() {
+        return { ...this.statsCounter, index: structuredClone(this.index) };
+    }
+    getIndex() {
+        return structuredClone(this.index);
+    }
+}
+// ─── File Learning Store ──────────────────────────────────────────────────────
+export class FileLearningStore {
+    constructor(filePath = '.capman/learning.json') {
+        this.entries = [];
+        this.loadPromise = null;
+        this.saveQueue = Promise.resolve();
+        this.learningIndex = new LearningIndex();
+        this.dirty = false;
+        this.saveTimer = null;
+        const cwd = process.cwd();
+        const resolved = path.resolve(cwd, filePath);
+        const allowedPrefix = cwd === '/' ? '/' : cwd + path.sep;
+        if (!resolved.startsWith(allowedPrefix)) {
+            throw new Error(`FileLearningStore path "${filePath}" resolves outside the working directory.\n` +
+                `Resolved: ${resolved}\nAllowed:  ${cwd}`);
+        }
+        this.filePath = resolved;
+        logger.info(`FileLearningStore initialized — writing to: ${this.filePath}`);
+        activeStores.add(this);
+        if (!exitHandlersRegistered) {
+            exitHandlersRegistered = true;
+            process.on('exit', flushAllStores);
+            process.on('SIGTERM', () => { flushAllStores(); process.exit(0); });
+            process.on('SIGINT', () => { flushAllStores(); process.exit(0); });
         }
     }
-    async load() {
-        if (this.loaded)
+    flushSync() {
+        // Cancel pending timer — prevents scheduleSave firing after sync write
+        if (this.saveTimer) {
+            clearTimeout(this.saveTimer);
+            this.saveTimer = null;
+        }
+        if (!this.dirty)
             return;
+        this.dirty = false;
+        try {
+            const dir = path.dirname(this.filePath);
+            fs.mkdirSync(dir, { recursive: true });
+            const tmp = `${this.filePath}.tmp`;
+            const payload = JSON.stringify({ entries: this.entries, updatedAt: new Date().toISOString() }, null, 2);
+            // Write to .tmp then rename — matches _doSave() pattern so they can't interleave
+            fs.writeFileSync(tmp, payload);
+            fs.renameSync(tmp, this.filePath);
+        }
+        catch {
+            // Best-effort in exit handler
+        }
+    }
+    /**
+     * Removes this store from the exit flush registry and cancels any pending save timer.
+     * Call when the store is no longer needed to prevent memory leaks in long-running servers.
+     */
+    async destroy() {
+        if (this.saveTimer) {
+            clearTimeout(this.saveTimer);
+            this.saveTimer = null;
+        }
+        if (this.dirty) {
+            this.dirty = false;
+            // Await final flush before removing from registry —
+            // ensures data is written before the store becomes unreachable
+            await this.save();
+        }
+        activeStores.delete(this);
+    }
+    load() {
+        if (!this.loadPromise) {
+            this.loadPromise = this._doLoad();
+        }
+        return this.loadPromise;
+    }
+    async _doLoad() {
         try {
             const raw = await fs.promises.readFile(this.filePath, 'utf-8');
             const parsed = JSON.parse(raw);
             if (parsed && typeof parsed === 'object' && !Array.isArray(parsed) && Array.isArray(parsed.entries)) {
                 this.entries = parsed.entries;
-                this.rebuildIndex();
+                this.learningIndex.rebuild(this.entries);
                 logger.debug(`Learning store loaded: ${this.entries.length} entries`);
             }
             else {
@@ -134,7 +188,19 @@ export class FileLearningStore {
         catch {
             // File doesn't exist yet — start fresh
         }
-        this.loaded = true;
+    }
+    scheduleSave(urgencyMs = 5_000) {
+        this.dirty = true;
+        if (!this.saveTimer) {
+            this.saveTimer = setTimeout(() => {
+                this.saveTimer = null;
+                if (this.dirty) {
+                    this.dirty = false;
+                    // Route through saveQueue — serializes with all other saves
+                    this.save();
+                }
+            }, urgencyMs);
+        }
     }
     save() {
         this.saveQueue = this.saveQueue.then(() => this._doSave());
@@ -144,10 +210,12 @@ export class FileLearningStore {
         try {
             const dir = path.dirname(this.filePath);
             await fs.promises.mkdir(dir, { recursive: true });
-            await fs.promises.writeFile(this.filePath, JSON.stringify({
+            const tmp = `${this.filePath}.tmp`;
+            await fs.promises.writeFile(tmp, JSON.stringify({
                 entries: this.entries,
                 updatedAt: new Date().toISOString(),
             }, null, 2));
+            await fs.promises.rename(tmp, this.filePath);
         }
         catch (err) {
             logger.warn(`Failed to save learning store to ${this.filePath}: ${err instanceof Error ? err.message : String(err)}`);
@@ -155,35 +223,52 @@ export class FileLearningStore {
     }
     async record(entry) {
         await this.load();
-        this.entries.push(entry);
-        this.updateIndex(entry);
+        // Store only tokenized keywords — never raw query text.
+        // Raw queries may contain PII (emails, names, order IDs) that should
+        // not be persisted to disk under GDPR/CCPA data retention requirements.
+        const sanitized = {
+            ...entry,
+            query: entry.query
+                .toLowerCase()
+                .split(/\W+/)
+                .filter(w => w.length > 2 && !STOPWORDS.has(w))
+                .join(' '),
+        };
+        this.entries.push(sanitized);
+        this.learningIndex.update(sanitized);
         if (this.entries.length > MAX_LEARNING_ENTRIES) {
             const excess = this.entries.length - MAX_LEARNING_ENTRIES;
             const pruned = this.entries.splice(0, excess);
             // Subtract pruned entries from index — O(pruned × w) instead of O(n × w) full rebuild
             for (const entry of pruned) {
-                this.subtractFromIndex(entry);
+                this.learningIndex.subtract(entry);
             }
             logger.debug(`Learning store pruned ${excess} oldest entries (cap: ${MAX_LEARNING_ENTRIES})`);
         }
-        await this.save();
+        this.scheduleSave();
     }
     async getStats() {
         await this.load();
-        return { ...this.statsCounter, index: structuredClone(this.index) };
+        return this.learningIndex.getStats();
     }
     async getIndex() {
         await this.load();
-        return structuredClone(this.index);
+        return this.learningIndex.getIndex();
     }
     async getTopCapabilities(limit = 5) {
         await this.load();
         return computeTopCapabilities(this.entries, limit);
     }
     async clear() {
+        // Cancel any pending debounced save — prevents stale data being written
+        // after clear() resets state
+        if (this.saveTimer) {
+            clearTimeout(this.saveTimer);
+            this.saveTimer = null;
+        }
+        this.dirty = false;
         this.entries = [];
-        this.index = {};
-        this.statsCounter = { totalQueries: 0, llmQueries: 0, cacheHits: 0, outOfScope: 0 };
+        this.learningIndex.reset();
         await this.save();
     }
 }
@@ -191,92 +276,41 @@ export class FileLearningStore {
 export class MemoryLearningStore {
     constructor() {
         this.entries = [];
-        this.index = {};
-        this.statsCounter = {
-            totalQueries: 0, llmQueries: 0, cacheHits: 0, outOfScope: 0,
-        };
+        this.learningIndex = new LearningIndex();
     }
     async record(entry) {
-        this.entries.push(entry);
-        this.updateIndex(entry);
+        const sanitized = {
+            ...entry,
+            query: entry.query
+                .toLowerCase()
+                .split(/\W+/)
+                .filter(w => w.length > 2 && !STOPWORDS.has(w))
+                .join(' '),
+        };
+        this.entries.push(sanitized);
+        this.learningIndex.update(sanitized);
         if (this.entries.length > MAX_LEARNING_ENTRIES) {
             const excess = this.entries.length - MAX_LEARNING_ENTRIES;
             const pruned = this.entries.splice(0, excess);
             for (const entry of pruned) {
-                this.subtractFromIndex(entry);
+                this.learningIndex.subtract(entry);
             }
         }
     }
     async getStats() {
-        return { ...this.statsCounter, index: structuredClone(this.index) };
+        return this.learningIndex.getStats();
     }
     async getIndex() {
-        return structuredClone(this.index);
-    }
-    updateIndex(entry) {
-        var _a;
-        this.statsCounter.totalQueries++;
-        if (entry.resolvedVia === 'llm')
-            this.statsCounter.llmQueries++;
-        if (entry.resolvedVia === 'cache')
-            this.statsCounter.cacheHits++;
-        if (!entry.capabilityId)
-            this.statsCounter.outOfScope++;
-        if (entry.capabilityId) {
-            const words = entry.query.toLowerCase()
-                .split(/\W+/)
-                .filter(w => w.length > 2 && !STOPWORDS.has(w));
-            for (const word of words) {
-                (_a = this.index)[word] ?? (_a[word] = {});
-                this.index[word][entry.capabilityId] =
-                    (this.index[word][entry.capabilityId] ?? 0) + 1;
-            }
-        }
-    }
-    subtractFromIndex(entry) {
-        if (!entry.capabilityId) {
-            this.statsCounter.outOfScope = Math.max(0, this.statsCounter.outOfScope - 1);
-            this.statsCounter.totalQueries = Math.max(0, this.statsCounter.totalQueries - 1);
-            if (entry.resolvedVia === 'llm')
-                this.statsCounter.llmQueries = Math.max(0, this.statsCounter.llmQueries - 1);
-            if (entry.resolvedVia === 'cache')
-                this.statsCounter.cacheHits = Math.max(0, this.statsCounter.cacheHits - 1);
-            return;
-        }
-        this.statsCounter.totalQueries = Math.max(0, this.statsCounter.totalQueries - 1);
-        if (entry.resolvedVia === 'llm')
-            this.statsCounter.llmQueries = Math.max(0, this.statsCounter.llmQueries - 1);
-        if (entry.resolvedVia === 'cache')
-            this.statsCounter.cacheHits = Math.max(0, this.statsCounter.cacheHits - 1);
-        const words = entry.query.toLowerCase()
-            .split(/\W+/)
-            .filter(w => w.length > 2 && !STOPWORDS.has(w));
-        for (const word of words) {
-            if (!this.index[word])
-                continue;
-            this.index[word][entry.capabilityId] =
-                (this.index[word][entry.capabilityId] ?? 1) - 1;
-            if (this.index[word][entry.capabilityId] <= 0) {
-                delete this.index[word][entry.capabilityId];
-            }
-            if (Object.keys(this.index[word]).length === 0) {
-                delete this.index[word];
-            }
-        }
-    }
-    rebuildIndex() {
-        this.index = {};
-        this.statsCounter = { totalQueries: 0, llmQueries: 0, cacheHits: 0, outOfScope: 0 };
-        for (const entry of this.entries) {
-            this.updateIndex(entry);
-        }
+        return this.learningIndex.getIndex();
     }
     async getTopCapabilities(limit = 5) {
         return computeTopCapabilities(this.entries, limit);
     }
     async clear() {
         this.entries = [];
-        this.index = {};
-        this.statsCounter = { totalQueries: 0, llmQueries: 0, cacheHits: 0, outOfScope: 0 };
+        this.learningIndex.reset();
+    }
+    async destroy() {
+        // No-op for memory store — nothing to flush or deregister
     }
 }

package/dist/esm/matcher.d.ts

@@ -1,4 +1,7 @@
-import type { Capability, Manifest, MatchResult } from './types';
+import type { Manifest, Capability, MatchResult } from './types';
+export declare class LLMParseError extends Error {
+    constructor(message: string);
+}
 export declare const STOPWORDS: Set<string>;
 export declare function resolverToIntent(cap: Capability): MatchResult['intent'];
 /**

package/dist/esm/matcher.js

@@ -1,4 +1,11 @@
 import { logger } from './logger';
+// ─── Typed error for LLM parse failures ──────────────────────────────────────
+export class LLMParseError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'LLMParseError';
+    }
+}
 export const STOPWORDS = new Set([
     'show', 'me', 'the', 'get', 'find', 'fetch', 'give', 'please',
     'can', 'you', 'i', 'want', 'to', 'a', 'an', 'my', 'our', 'your',
@@ -65,7 +72,7 @@ export function extractParams(query, cap) {
     for (const param of cap.params) {
         // Session params come from auth context, not query
         if (param.source === 'session') {
-            result[param.name] = '[from_session]';
+            result[param.name] = null; // injected by resolver from auth context — not extracted from query
             continue;
         }
         if (param.source !== 'user_query') {
@@ -141,7 +148,8 @@ export function match(query, manifest) {
             candidates: [],
         };
     }
-    logger.info(`Matching query: "${query}"`);
+    logger.info(`Matching query (${query.length} chars)`);
+    logger.debug(`Full query: "${query}"`);
     logger.debug(`Manifest has ${manifest.capabilities.length} capabilities`);
     let best = null;
     let bestScore = 0;
@@ -197,7 +205,13 @@ export function match(query, manifest) {
  * fields can influence LLM routing decisions.
  */
 export async function matchWithLLM(query, manifest, options) {
-    const manifestSummary = manifest.capabilities.map(c => `- ${c.id} (${c.resolver.type}): ${c.description}${c.examples?.length ? `\n  examples: ${c.examples.slice(0, 2).join(', ')}` : ''}`).join('\n');
+    // Truncate description and examples — prevents context window overflow and
+    // reduces prompt injection surface from third-party OpenAPI spec content.
+    const MAX_DESC_LEN = 200;
+    const MAX_EXAMPLE_LEN = 100;
+    const manifestSummary = manifest.capabilities.map(c => `- ${c.id} (${c.resolver.type}): ${c.description.slice(0, MAX_DESC_LEN)}${c.description.length > MAX_DESC_LEN ? '…' : ''}${c.examples?.length
+        ? `\n  examples: ${c.examples.slice(0, 2).map(e => e.slice(0, MAX_EXAMPLE_LEN)).join(', ')}`
+        : ''}`).join('\n');
     const prompt = `You are an intent matcher for an AI agent system.
 
 App: ${manifest.app}
@@ -228,13 +242,13 @@ ${JSON.stringify({ user_query: query })}
         parsed = JSON.parse(clean);
     }
     catch {
-        throw new Error(`LLM_PARSE_ERROR: LLM returned invalid JSON. First 300 chars: ${clean.slice(0, 300)}`);
+        throw new LLMParseError(`LLM returned invalid JSON. First 300 chars: ${clean.slice(0, 300)}`);
     }
     if (typeof parsed.matched_capability !== 'string') {
-        throw new Error(`LLM_PARSE_ERROR: missing "matched_capability" field in response`);
+        throw new LLMParseError(`missing "matched_capability" field in response`);
     }
     if (typeof parsed.confidence !== 'number') {
-        throw new Error(`LLM_PARSE_ERROR: missing numeric "confidence" field in response`);
+        throw new LLMParseError(`missing numeric "confidence" field in response`);
     }
     const isOOS = parsed.matched_capability === 'OUT_OF_SCOPE';
     const capability = isOOS
@@ -245,16 +259,21 @@ ${JSON.stringify({ user_query: query })}
     if (!isOOS && capability === null) {
         logger.warn(`LLM returned unknown capability ID: "${parsed.matched_capability}" — treating as out_of_scope`);
     }
+    // Build full candidate list — all capabilities scored, LLM winner marked as matched.
+    // This aligns the shape with keyword match results and allows the learning boost
+    // to surface alternatives if the LLM made a wrong call.
+    const llmConfidence = effectivelyOOS ? 0 : parsed.confidence;
+    const allCandidates = manifest.capabilities.map(c => ({
+        capabilityId: c.id,
+        score: c.id === capability?.id ? llmConfidence : 0,
+        matched: c.id === capability?.id,
+    }));
     return {
         capability,
-        confidence: effectivelyOOS ? 0 : parsed.confidence,
+        confidence: llmConfidence,
         intent: effectivelyOOS ? 'out_of_scope' : parsed.intent,
         extractedParams: (parsed.extracted_params ?? {}),
         reasoning: parsed.reasoning ?? 'No reasoning provided',
-        candidates: capability ? [{
-                capabilityId: capability.id,
-                score: parsed.confidence,
-                matched: true,
-            }] : [],
+        candidates: allCandidates,
     };
 }

package/dist/esm/parser.js

@@ -11,7 +11,7 @@ async function loadSpec(source) {
     if (source.startsWith('http://') || source.startsWith('https://')) {
         logger.info(`Fetching OpenAPI spec from: ${source}`);
         const controller = new AbortController();
-        const timer = setTimeout(() => controller.abort(), 10000);
+        const timer = setTimeout(() => controller.abort(), 10_000);
         // eslint-disable-next-line prefer-const
         let res;
         try {
@@ -36,7 +36,7 @@ async function loadSpec(source) {
         throw new Error(`Spec file not found: ${resolved}`);
     }
     logger.info(`Reading OpenAPI spec from: ${resolved}`);
-    const text = fs.readFileSync(resolved, 'utf-8');
+    const text = await fs.promises.readFile(resolved, 'utf-8');
     return parseSpecText(text, source);
 }
 function parseSpecText(text, source) {
@@ -50,8 +50,14 @@ function parseSpecText(text, source) {
         const yaml = require('js-yaml');
         return yaml.load(text);
     }
-    catch {
-        // js-yaml not installed — try basic YAML detection
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        // Distinguish "module not found" from actual YAML parse errors
+        const code = err.code;
+        if (code !== 'MODULE_NOT_FOUND') {
+            throw new Error(`YAML parse error in "${source}": ${msg}`);
+        }
+        // js-yaml not installed — fall through to extension check
         if (source.endsWith('.yaml') || source.endsWith('.yml')) {
             throw new Error('YAML spec detected but js-yaml is not installed.\n' +
                 'Install it: npm install js-yaml\n' +

package/dist/esm/resolver.d.ts

@@ -17,5 +17,12 @@ export interface ResolveOptions {
     retries?: number;
     /** Timeout in milliseconds (default: 5000) */
     timeoutMs?: number;
+    /**
+     * When true, retries all HTTP methods including POST/PUT/PATCH/DELETE.
+     * Use only for idempotent write operations — retrying non-idempotent
+     * methods can cause duplicate side effects (duplicate orders, double charges).
+     * @default false
+     */
+    retryAllMethods?: boolean;
 }
 export declare function resolve(matchResult: MatchResult, params?: Record<string, unknown>, options?: ResolveOptions): Promise<ResolveResult>;