@getmikk/ai-context 1.7.1 → 1.9.0

package/src/providers.ts

@@ -28,6 +28,16 @@ export class ClaudeProvider implements ContextProvider {
         lines.push(`  <seeds_found>${context.meta?.seedCount ?? 0}</seeds_found>`)
         lines.push(`  <functions_selected>${context.meta?.selectedFunctions ?? 0} of ${context.meta?.totalFunctionsConsidered ?? 0}</functions_selected>`)
         lines.push(`  <estimated_tokens>${context.meta?.estimatedTokens ?? 0}</estimated_tokens>`)
+        if (context.meta?.reasons && context.meta.reasons.length > 0) {
+            for (const reason of context.meta.reasons) {
+                lines.push(`  <reason>${esc(reason)}</reason>`)
+            }
+        }
+        if (context.meta?.suggestions && context.meta.suggestions.length > 0) {
+            for (const s of context.meta.suggestions) {
+                lines.push(`  <suggestion>${esc(s)}</suggestion>`)
+            }
+        }
         lines.push('</context_meta>')
         lines.push('')
 
@@ -79,6 +89,23 @@ export class ClaudeProvider implements ContextProvider {
             lines.push('')
         }
 
+        if (context.modules.length === 0 && context.meta?.reasons?.length) {
+            lines.push('<no_match_reason>')
+            for (const reason of context.meta.reasons) {
+                lines.push(`  <item>${esc(reason)}</item>`)
+            }
+            lines.push('</no_match_reason>')
+            lines.push('')
+            if (context.meta.suggestions && context.meta.suggestions.length > 0) {
+                lines.push('<did_you_mean>')
+                for (const suggestion of context.meta.suggestions) {
+                    lines.push(`  <item>${esc(suggestion)}</item>`)
+                }
+                lines.push('</did_you_mean>')
+                lines.push('')
+            }
+        }
+
         // ── Context files (schemas, data models, config) ───────────────────
         if (context.contextFiles && context.contextFiles.length > 0) {
             lines.push('<context_files>')
@@ -160,6 +187,20 @@ export class CompactProvider implements ContextProvider {
             `Task keywords: ${context.meta?.keywords?.join(', ') ?? ''}`,
             '',
         ]
+        if (context.modules.length === 0 && context.meta?.reasons?.length) {
+            lines.push('No exact context selected:')
+            for (const reason of context.meta.reasons) {
+                lines.push(`- ${reason}`)
+            }
+            if (context.meta.suggestions && context.meta.suggestions.length > 0) {
+                lines.push('')
+                lines.push('Did you mean:')
+                for (const suggestion of context.meta.suggestions) {
+                    lines.push(`- ${suggestion}`)
+                }
+            }
+            lines.push('')
+        }
         for (const mod of context.modules) {
             lines.push(`## ${mod.name}`)
             for (const fn of mod.functions) {
@@ -218,4 +259,4 @@ function esc(s: string): string {
         .replace(/</g, '&lt;')
         .replace(/>/g, '&gt;')
         .replace(/"/g, '&quot;')
-}
+}

package/src/token-counter.ts

@@ -0,0 +1,224 @@
+/**
+ * Improved Token Counter
+ * 
+ * Provides more accurate token counting than the simple length/4 approximation.
+ * Uses a GPT-4 compatible tokenizer approximation for better budget management.
+ */
+
+// Character-based token approximation (more accurate than simple division)
+const CHARS_PER_TOKEN = 3.8 // Average for GPT-4 tokenizer
+const MIN_CHARS_PER_TOKEN = 2.0   // For dense code
+const MAX_CHARS_PER_TOKEN = 6.0   // For sparse text
+
+// Special token patterns that affect tokenization
+const TOKEN_PATTERNS = {
+    // Common programming patterns that typically tokenize as single tokens
+    SINGLE_TOKEN_PATTERNS: [
+        /\b(if|else|for|while|function|return|const|let|var|class|import|export)\b/g,
+        /\b(true|false|null|undefined)\b/g,
+        /\b(async|await|try|catch|throw|new|this)\b/g,
+        // Operators and punctuation
+        /[+\-*\/=<>!&|]+/g,
+        /[{}()\[\];,\.]/g,
+        // Common function names
+        /\b(console\.log|console\.error|console\.warn)\b/g,
+        /\b(Math\.(floor|ceil|round|max|min))\b/g,
+    ],
+    
+    // Patterns that typically increase token count
+    HIGH_TOKEN_PATTERNS: [
+        // String literals (each character ~0.25 tokens)
+        /'[^']*'/g,
+        /"[^"]*"/g,
+        /`[^`]*`/g,
+        // Numbers (digits ~0.5 tokens each)
+        /\b\d+\.?\d*\b/g,
+        // Long identifiers (split into multiple tokens)
+        /\b[a-z][a-zA-Z0-9]{8,}\b/g,
+    ]
+}
+
+/**
+ * Count tokens with improved accuracy using position-based pattern matching
+ */
+export function countTokens(text: string): number {
+    if (!text || text.length === 0) return 0
+    
+    let tokenCount = 0
+    const processedPositions = new Set<number>() // Track positions to avoid double-counting
+    
+    // Count single-token patterns with position tracking
+    for (const pattern of TOKEN_PATTERNS.SINGLE_TOKEN_PATTERNS) {
+        for (const match of text.matchAll(pattern)) {
+            const start = match.index!
+            const end = start + match[0].length
+            
+            // Check if this range overlaps with already processed ranges
+            let overlaps = false
+            for (let i = start; i < end; i++) {
+                if (processedPositions.has(i)) {
+                    overlaps = true
+                    break
+                }
+            }
+            
+            if (!overlaps) {
+                tokenCount += 1
+                // Mark positions as processed
+                for (let i = start; i < end; i++) {
+                    processedPositions.add(i)
+                }
+            }
+        }
+    }
+    
+    // Count high-token patterns (strings, numbers, long identifiers)
+    for (const pattern of TOKEN_PATTERNS.HIGH_TOKEN_PATTERNS) {
+        for (const match of text.matchAll(pattern)) {
+            const start = match.index!
+            const end = start + match[0].length
+            
+            // Check for overlaps
+            let overlaps = false
+            for (let i = start; i < end; i++) {
+                if (processedPositions.has(i)) {
+                    overlaps = true
+                    break
+                }
+            }
+            
+            if (!overlaps) {
+                let tokensToAdd = 0
+                if (match[0].startsWith('\'') || match[0].startsWith('"') || match[0].startsWith('`')) {
+                    // String literal: roughly 1 token per 4 characters
+                    tokensToAdd = Math.ceil(match[0].length / 4)
+                } else if (/^\d/.test(match[0])) {
+                    // Number: roughly 1 token per 2 digits
+                    tokensToAdd = Math.ceil(match[0].length / 2)
+                } else {
+                    // Long identifier: roughly 1 token per 6 characters
+                    tokensToAdd = Math.ceil(match[0].length / 6)
+                }
+                
+                tokenCount += tokensToAdd
+                // Mark positions as processed
+                for (let i = start; i < end; i++) {
+                    processedPositions.add(i)
+                }
+            }
+        }
+    }
+    
+    // Count remaining characters (general text)
+    const remainingText = Array.from(text.split(''))
+        .map((char, index) => processedPositions.has(index) ? '' : char)
+        .join('')
+    
+    if (remainingText.length > 0) {
+        // Use variable rate based on character density
+        const avgWordLength = remainingText.split(/\s+/).reduce((sum, word) => sum + word.length, 0) / Math.max(remainingText.split(/\s+/).length, 1)
+        
+        let charsPerToken = CHARS_PER_TOKEN
+        if (avgWordLength < 4) {
+            charsPerToken = MIN_CHARS_PER_TOKEN // Dense code
+        } else if (avgWordLength > 8) {
+            charsPerToken = MAX_CHARS_PER_TOKEN // Sparse text
+        }
+        
+        tokenCount += Math.ceil(remainingText.length / charsPerToken)
+    }
+    
+    // Apply bounds checking for sanity
+    const minEstimate = Math.ceil(text.length / MAX_CHARS_PER_TOKEN)
+    const maxEstimate = Math.ceil(text.length / MIN_CHARS_PER_TOKEN)
+    
+    return Math.max(minEstimate, Math.min(maxEstimate, tokenCount))
+}
+
+/**
+ * Fast token count for quick estimates (still more accurate than length/4)
+ */
+export function countTokensFast(text: string): number {
+    if (!text || text.length === 0) return 0
+    
+    // Quick heuristic based on character patterns
+    const codeDensity = (text.match(/[a-zA-Z0-9]/g) || []).length / text.length
+    const stringRatio = (text.match(/['"`]/g) || []).length / text.length
+    
+    // Adjust chars per token based on content type
+    let charsPerToken = CHARS_PER_TOKEN
+    if (codeDensity > 0.7) {
+        charsPerToken = 3.2 // Dense code
+    } else if (stringRatio > 0.2) {
+        charsPerToken = 4.5 // String-heavy
+    } else if (codeDensity < 0.3) {
+        charsPerToken = 5.0 // Sparse text/comments
+    }
+    
+    return Math.ceil(text.length / charsPerToken)
+}
+
+/**
+ * Estimate tokens for a file with content type awareness
+ */
+export function estimateFileTokens(content: string, filePath: string): number {
+    const extension = filePath.split('.').pop()?.toLowerCase()
+    
+    // Adjust counting based on file type
+    switch (extension) {
+        case 'json':
+            // JSON is token-heavy due to strings and structure
+            return countTokens(content) * 1.1
+        case 'md':
+            // Markdown has more natural language
+            return countTokens(content) * 0.9
+        case 'ts':
+        case 'tsx':
+        case 'js':
+        case 'jsx':
+            // Code files benefit from pattern recognition
+            return countTokens(content)
+        default:
+            // Use standard counting for unknown types
+            return countTokens(content)
+    }
+}
+
+/**
+ * Token budget manager with overflow protection
+ */
+export class TokenBudget {
+    constructor(private maxTokens: number, private overflowAllowance: number = 0.1) {}
+    
+    /**
+     * Check if content fits within budget
+     */
+    fits(content: string): boolean {
+        const tokens = countTokens(content)
+        return tokens <= this.maxTokens * (1 + this.overflowAllowance)
+    }
+    
+    /**
+     * Get remaining token count
+     */
+    remaining(usedTokens: number): number {
+        return Math.max(0, this.maxTokens - usedTokens)
+    }
+    
+    /**
+     * Truncate content to fit within budget
+     */
+    truncate(content: string, usedTokens: number = 0): string {
+        const available = this.remaining(usedTokens)
+        if (available <= 0) return ''
+        
+        const estimatedTokens = countTokens(content)
+        if (estimatedTokens <= available) return content
+        
+        // Rough truncation based on character ratio
+        const ratio = available / estimatedTokens
+        const truncateAt = Math.floor(content.length * ratio * 0.9) // 10% buffer
+        
+        return content.substring(0, truncateAt) + '\n... [truncated due to token budget]'
+    }
+}

package/src/types.ts

@@ -24,6 +24,8 @@ export interface AIContext {
         selectedFunctions: number
         estimatedTokens: number
         keywords: string[]
+        reasons?: string[]
+        suggestions?: string[]
     }
 }
 
@@ -72,6 +74,18 @@ export interface ContextQuery {
     includeCallGraph?: boolean
     /** Include function bodies for top-scored functions (default true) */
     includeBodies?: boolean
+    /** Relevance mode: balanced (default) or strict (high-precision filtering) */
+    relevanceMode?: 'balanced' | 'strict'
+    /** Additional required terms (comma-separated in CLI) that must be respected */
+    requiredKeywords?: string[]
+    /** In strict mode, require all extracted/required keywords to match */
+    requireAllKeywords?: boolean
+    /** Minimum number of matched keywords required in strict mode (default 1) */
+    minKeywordMatches?: number
+    /** Hard gate in strict mode: final output keeps only strict keyword matches */
+    exactOnly?: boolean
+    /** In strict mode, return empty context if no exact matches are found */
+    failFast?: boolean
     /** Absolute filesystem path to the project root (needed for body reading) */
     projectRoot?: string
 }
@@ -81,4 +95,4 @@ export interface ContextProvider {
     name: string
     formatContext(context: AIContext): string
     maxTokens: number
-}
+}

package/tests/claude-md.test.ts

@@ -64,8 +64,8 @@ describe('ClaudeMdGenerator', () => {
     test('generates valid markdown', () => {
         const gen = new ClaudeMdGenerator(mockContract, mockLock)
         const md = gen.generate()
-        expect(md).toContain('# TestProject')
-        expect(md).toContain('Architecture Overview')
+        expect(md).toContain('<name>TestProject</name>')
+        expect(md).toContain('<repository_context>')
     })
 
     test('includes project description', () => {
@@ -77,8 +77,8 @@ describe('ClaudeMdGenerator', () => {
     test('includes module sections', () => {
         const gen = new ClaudeMdGenerator(mockContract, mockLock)
         const md = gen.generate()
-        expect(md).toContain('Authentication module')
-        expect(md).toContain('API module')
+        expect(md).toContain('<module id="auth">')
+        expect(md).toContain('<module id="api">')
     })
 
     test('includes function names', () => {
@@ -105,7 +105,7 @@ describe('ClaudeMdGenerator', () => {
         const gen = new ClaudeMdGenerator(mockContract, mockLock)
         const md = gen.generate()
         // API depends on Auth (handleLogin calls verifyToken)
-        expect(md).toContain('Depends on')
+        expect(md).toContain('<depends_on>Authentication</depends_on>')
     })
 
     test('respects token budget', () => {
@@ -119,8 +119,8 @@ describe('ClaudeMdGenerator', () => {
     test('includes stats', () => {
         const gen = new ClaudeMdGenerator(mockContract, mockLock)
         const md = gen.generate()
-        expect(md).toContain('3 functions')
-        expect(md).toContain('2 modules')
+        expect(md).toContain('<functions>3</functions>')
+        expect(md).toContain('<modules>2</modules>')
     })
 
     test('shows purpose when available', () => {
@@ -132,6 +132,86 @@ describe('ClaudeMdGenerator', () => {
     test('shows calledBy count for key functions', () => {
         const gen = new ClaudeMdGenerator(mockContract, mockLock)
         const md = gen.generate()
-        expect(md).toContain('called by 1')
+        expect(md).toContain('callers="1"')
+    })
+
+    describe('Edge Cases and Fault Tolerance', () => {
+        test('handles completely empty lock and contract without throwing', () => {
+            const emptyContract: MikkContract = {
+                version: '1',
+                project: { name: 'Empty', language: 'TS', description: '', entryPoints: [] },
+                declared: { modules: [], constraints: [], decisions: [] },
+                overwrite: { mode: 'never', requireConfirmation: false }
+            }
+            const emptyLock: MikkLock = {
+                version: '1', generatedAt: new Date().toISOString(), generatorVersion: '1', projectRoot: '', syncState: { status: 'clean', lastSyncAt: '', lockHash: '', contractHash: '' },
+                files: {}, functions: {}, classes: {}, modules: {}, graph: { nodes: 0, edges: 0, rootHash: '' }
+            }
+            const gen = new ClaudeMdGenerator(emptyContract, emptyLock)
+            const md = gen.generate()
+            expect(md).toContain('<name>Empty</name>')
+            expect(md).toContain('<modules>0</modules>')
+            expect(md).toContain('<functions>0</functions>')
+        })
+
+        test('handles missing optional fields gracefully', () => {
+            const partialContract: MikkContract = {
+                version: '1',
+                project: { name: 'Partial', language: 'TS', description: '', entryPoints: [] },
+                declared: { modules: [{ id: 'core', name: 'Core', description: '', paths: [] }], constraints: [], decisions: [] },
+                overwrite: { mode: 'never', requireConfirmation: false }
+            }
+            const partialLock: MikkLock = {
+                version: '1', generatedAt: new Date().toISOString(), generatorVersion: '1', projectRoot: '', syncState: { status: 'clean', lastSyncAt: '', lockHash: '', contractHash: '' },
+                files: {},
+                functions: {
+                    'f1': { id: 'f1', name: 'func', file: 'a.ts', startLine: 1, endLine: 2, hash: 'h', calls: [], calledBy: [], moduleId: 'core' } 
+                },
+                classes: {},
+                modules: {
+                    'core': { id: 'core', files: [], hash: 'h', fragmentPath: 'p' }
+                },
+                graph: { nodes: 0, edges: 0, rootHash: '' }
+            }
+            const gen = new ClaudeMdGenerator(partialContract, partialLock)
+            const md = gen.generate()
+            expect(md).toContain('<name>Core</name>')
+            expect(md).toContain('func')
+        })
+
+        test('handles circular dependencies between functions gracefully', () => {
+            const circLock: MikkLock = {
+                ...mockLock,
+                functions: {
+                    'fn:a': { id: 'fn:a', name: 'A', file: 'a.ts', startLine: 1, endLine: 2, hash: 'h', calls: ['fn:b'], calledBy: ['fn:b'], moduleId: 'auth' },
+                    'fn:b': { id: 'fn:b', name: 'B', file: 'b.ts', startLine: 1, endLine: 2, hash: 'h', calls: ['fn:a'], calledBy: ['fn:a'], moduleId: 'auth' },
+                }
+            }
+            const gen = new ClaudeMdGenerator(mockContract, circLock)
+            const md = gen.generate()
+            expect(md).toContain('A')
+            expect(md).toContain('B')
+            expect(md.length).toBeLessThan(10000)
+        })
+
+        test('truncates extremely strict token budgets without losing core layout', () => {
+            const gen = new ClaudeMdGenerator(mockContract, mockLock, 50) 
+            const md = gen.generate()
+            expect(md).toContain('<repository_context>')
+            expect(md).not.toContain('Use JWT') 
+        })
+        
+        test('handles foreign or orphaned modules robustly', () => {
+            const orphanedLock: MikkLock = {
+                ...mockLock,
+                functions: {
+                    'fn:orphan': { id: 'fn:orphan', name: 'OrphanFn', file: 'o.ts', startLine: 1, endLine: 2, hash: 'h', calls: [], calledBy: [], moduleId: 'unknown-module' }
+                }
+            }
+            const gen = new ClaudeMdGenerator(mockContract, orphanedLock)
+            const md = gen.generate()
+            // Unknown module functions are typically skipped entirely. The generator should not crash.
+            expect(md).not.toContain('OrphanFn')
+        })
     })
 })

package/tests/context-builder.test.ts

@@ -0,0 +1,159 @@
+import { describe, expect, test } from 'bun:test'
+import { ContextBuilder } from '../src/context-builder.js'
+import type { ContextQuery } from '../src/types.js'
+
+function makeFixture() {
+    const contract = {
+        project: {
+            name: 'mikk',
+            language: 'typescript',
+            description: 'fixture',
+        },
+        declared: {
+            modules: [
+                { id: 'core-parser', name: 'Core Parser', description: '', paths: [], entryFunctions: [] },
+                { id: 'ui', name: 'UI', description: '', paths: [], entryFunctions: [] },
+            ],
+            constraints: [],
+            decisions: [],
+        },
+    } as any
+
+    const fnResolver = {
+        id: 'fn:parser:resolver',
+        name: 'resolveImports',
+        file: 'packages/core/src/parser/ts-resolver.ts',
+        moduleId: 'core-parser',
+        startLine: 1,
+        endLine: 10,
+        params: [],
+        returnType: 'void',
+        isAsync: false,
+        isExported: true,
+        purpose: 'resolve ts imports',
+        calls: ['fn:parser:helper'],
+        calledBy: [],
+        edgeCasesHandled: [],
+        errorHandling: [],
+    }
+
+    const fnHelper = {
+        id: 'fn:parser:helper',
+        name: 'normalizeTsPath',
+        file: 'packages/core/src/parser/path.ts',
+        moduleId: 'core-parser',
+        startLine: 1,
+        endLine: 8,
+        params: [],
+        returnType: 'string',
+        isAsync: false,
+        isExported: false,
+        purpose: 'normalize ts path',
+        calls: [],
+        calledBy: ['fn:parser:resolver'],
+        edgeCasesHandled: [],
+        errorHandling: [],
+    }
+
+    const fnUnrelated = {
+        id: 'fn:ui:render',
+        name: 'renderHeader',
+        file: 'apps/web/components/header.tsx',
+        moduleId: 'ui',
+        startLine: 1,
+        endLine: 8,
+        params: [],
+        returnType: 'void',
+        isAsync: false,
+        isExported: true,
+        purpose: 'render ui header',
+        calls: [],
+        calledBy: [],
+        edgeCasesHandled: [],
+        errorHandling: [],
+    }
+
+    const lock = {
+        functions: {
+            [fnResolver.id]: fnResolver,
+            [fnHelper.id]: fnHelper,
+            [fnUnrelated.id]: fnUnrelated,
+        },
+        files: {
+            [fnResolver.file]: { path: fnResolver.file, moduleId: fnResolver.moduleId, imports: [] },
+            [fnHelper.file]: { path: fnHelper.file, moduleId: fnHelper.moduleId, imports: [] },
+            [fnUnrelated.file]: { path: fnUnrelated.file, moduleId: fnUnrelated.moduleId, imports: [] },
+        },
+        routes: [],
+        contextFiles: [],
+    } as any
+
+    return { contract, lock }
+}
+
+function namesFrom(query: ContextQuery): string[] {
+    const { contract, lock } = makeFixture()
+    const builder = new ContextBuilder(contract, lock)
+    const ctx = builder.build(query)
+    return ctx.modules.flatMap(m => m.functions.map(f => f.name))
+}
+
+describe('ContextBuilder strict relevance mode', () => {
+    test('strict mode filters unrelated entry-point noise', () => {
+        const balanced = namesFrom({
+            task: 'fix ts resolver imports',
+            tokenBudget: 1200,
+            includeBodies: false,
+            includeCallGraph: false,
+            relevanceMode: 'balanced',
+        })
+        const strict = namesFrom({
+            task: 'fix ts resolver imports',
+            tokenBudget: 1200,
+            includeBodies: false,
+            includeCallGraph: false,
+            relevanceMode: 'strict',
+            minKeywordMatches: 1,
+        })
+
+        expect(balanced).toContain('renderHeader')
+        expect(strict).not.toContain('renderHeader')
+        expect(strict).toContain('resolveImports')
+    })
+
+    test('requiredKeywords enforces exact focus in strict mode', () => {
+        const strict = namesFrom({
+            task: 'resolver imports',
+            tokenBudget: 1200,
+            includeBodies: false,
+            includeCallGraph: false,
+            relevanceMode: 'strict',
+            requiredKeywords: ['ts'],
+            minKeywordMatches: 1,
+        })
+
+        expect(strict).toContain('resolveImports')
+        expect(strict).toContain('normalizeTsPath')
+        expect(strict).not.toContain('renderHeader')
+    })
+
+    test('failFast returns empty context when exact match is impossible', () => {
+        const { contract, lock } = makeFixture()
+        const builder = new ContextBuilder(contract, lock)
+        const ctx = builder.build({
+            task: 'resolver imports',
+            tokenBudget: 1200,
+            includeBodies: false,
+            includeCallGraph: false,
+            relevanceMode: 'strict',
+            requiredKeywords: ['nonexistent'],
+            exactOnly: true,
+            failFast: true,
+        })
+
+        expect(ctx.modules.length).toBe(0)
+        expect(ctx.meta.selectedFunctions).toBe(0)
+        expect((ctx.meta.reasons?.length ?? 0) > 0).toBe(true)
+        expect((ctx.meta.suggestions?.length ?? 0) > 0).toBe(true)
+    })
+})