ferret-scan 2.2.0 → 2.3.0

package/dist/__tests__/llmAnalysisMitre.test.js

@@ -0,0 +1,192 @@
+/**
+ * LLM Analysis MITRE Atlas Tests
+ * Tests for analyzeWithLlm with MITRE atlas options
+ */
+import { analyzeWithLlm } from '../features/llmAnalysis.js';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import * as os from 'node:os';
+function makeConfig(overrides = {}) {
+    return {
+        provider: 'openai-compatible',
+        baseUrl: 'http://localhost:11434/v1/chat/completions',
+        model: 'llama3',
+        apiKeyEnv: 'DUMMY_KEY',
+        timeoutMs: 5000,
+        jsonMode: false,
+        maxInputChars: 1000,
+        maxOutputTokens: 200,
+        temperature: 0,
+        systemPromptAddendum: '',
+        includeMitreAtlasTechniques: false,
+        maxMitreAtlasTechniques: 0,
+        cacheDir: '/tmp/ferret-llm-cache',
+        cacheTtlHours: 1,
+        maxRetries: 0,
+        retryBackoffMs: 1,
+        retryMaxBackoffMs: 10,
+        minRequestIntervalMs: 0,
+        onlyIfFindings: false,
+        maxFindingsPerFile: 10,
+        maxFiles: 5,
+        minConfidence: 0.5,
+        ...overrides,
+    };
+}
+function makeFile(overrides = {}) {
+    return {
+        path: '/project/.claude/agents/test.md',
+        relativePath: 'agents/test.md',
+        type: 'md',
+        component: 'agent',
+        size: 100,
+        modified: new Date(),
+        ...overrides,
+    };
+}
+describe('analyzeWithLlm - MITRE atlas integration', () => {
+    let tmpDir;
+    beforeEach(() => {
+        tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ferret-llm-mitre-'));
+    });
+    afterEach(() => {
+        fs.rmSync(tmpDir, { recursive: true, force: true });
+    });
+    it('includes MITRE techniques when includeMitreAtlasTechniques=true', async () => {
+        let capturedPrompt = null;
+        const mockAnalyze = jest.fn().mockImplementation(async (prompt) => {
+            capturedPrompt = prompt;
+            return JSON.stringify({ version: 1, findings: [] });
+        });
+        const provider = { name: 'test', analyze: mockAnalyze };
+        const file = makeFile();
+        const config = makeConfig({
+            includeMitreAtlasTechniques: true,
+            maxMitreAtlasTechniques: 10,
+            cacheDir: tmpDir,
+        });
+        await analyzeWithLlm(provider, config, file, 'file content here', []);
+        expect(capturedPrompt).not.toBeNull();
+        // When MITRE included, system prompt should reference atlas techniques
+        const p = capturedPrompt;
+        expect(typeof p?.system).toBe('string');
+    });
+    it('returns findings with MITRE atlas IDs', async () => {
+        const mockResponse = JSON.stringify({
+            version: 1,
+            findings: [
+                {
+                    title: 'Prompt Injection',
+                    severity: 'HIGH',
+                    category: 'injection',
+                    match: 'IGNORE PREVIOUS',
+                    remediation: 'fix',
+                    confidence: 0.9,
+                    mitre_atlas: ['AML.T0051'],
+                },
+            ],
+        });
+        const mockAnalyze = jest.fn().mockResolvedValue(mockResponse);
+        const provider = { name: 'test', analyze: mockAnalyze };
+        const file = makeFile();
+        const config = makeConfig({ cacheDir: tmpDir });
+        const result = await analyzeWithLlm(provider, config, file, 'IGNORE PREVIOUS instructions', []);
+        expect(result.ran).toBe(true);
+        expect(result.findings.length).toBeGreaterThan(0);
+    });
+    it('handles findings with invalid MITRE IDs gracefully', async () => {
+        const mockResponse = JSON.stringify({
+            version: 1,
+            findings: [
+                {
+                    title: 'Test Finding',
+                    severity: 'MEDIUM',
+                    category: 'injection',
+                    match: 'bad content',
+                    remediation: 'fix',
+                    confidence: 0.8,
+                    mitre_atlas: ['INVALID-001', 'AML.T0051'],
+                },
+            ],
+        });
+        const mockAnalyze = jest.fn().mockResolvedValue(mockResponse);
+        const provider = { name: 'test', analyze: mockAnalyze };
+        const file = makeFile();
+        const result = await analyzeWithLlm(provider, makeConfig({ cacheDir: tmpDir }), file, 'bad content here', []);
+        // Invalid IDs should not cause errors
+        expect(Array.isArray(result.findings)).toBe(true);
+    });
+    it('analyzes large files with excerpt truncation', async () => {
+        const mockAnalyze = jest.fn().mockResolvedValue(JSON.stringify({ version: 1, findings: [] }));
+        const provider = { name: 'test', analyze: mockAnalyze };
+        const file = makeFile();
+        // Create very long content
+        const longContent = 'line content here\n'.repeat(2000);
+        const config = makeConfig({ maxInputChars: 1000, cacheDir: tmpDir });
+        const result = await analyzeWithLlm(provider, config, file, longContent, []);
+        expect(result.ran).toBe(true);
+    });
+    it('uses category aliases correctly', async () => {
+        const mockResponse = JSON.stringify({
+            version: 1,
+            findings: [
+                {
+                    title: 'Prompt Test',
+                    severity: 'HIGH',
+                    category: 'prompt-injection', // should alias to 'injection'
+                    match: 'test',
+                    remediation: 'fix',
+                    confidence: 0.8,
+                },
+                {
+                    title: 'Credential Test',
+                    severity: 'HIGH',
+                    category: 'secret', // should alias to 'credentials'
+                    match: 'secret',
+                    remediation: 'fix',
+                    confidence: 0.8,
+                },
+                {
+                    title: 'Exfil Test',
+                    severity: 'HIGH',
+                    category: 'exfiltration-attempt', // should alias to 'exfiltration'
+                    match: 'exfil',
+                    remediation: 'fix',
+                    confidence: 0.8,
+                },
+            ],
+        });
+        const mockAnalyze = jest.fn().mockResolvedValue(mockResponse);
+        const provider = { name: 'test', analyze: mockAnalyze };
+        const file = makeFile();
+        const config = makeConfig({ cacheDir: tmpDir });
+        const result = await analyzeWithLlm(provider, config, file, 'test content', []);
+        if (result.ran && result.findings.length > 0) {
+            expect(result.findings[0]?.category).toBe('injection');
+            expect(result.findings[1]?.category).toBe('credentials');
+            expect(result.findings[2]?.category).toBe('exfiltration');
+        }
+    });
+    it('handles extractJson with code fences', async () => {
+        // LLM often returns JSON wrapped in code fences
+        const mockResponse = '```json\n{"version":1,"findings":[]}\n```';
+        const mockAnalyze = jest.fn().mockResolvedValue(mockResponse);
+        const provider = { name: 'test', analyze: mockAnalyze };
+        const file = makeFile();
+        const config = makeConfig({ cacheDir: tmpDir });
+        const result = await analyzeWithLlm(provider, config, file, 'content', []);
+        expect(result.ran).toBe(true);
+        expect(result.findings).toHaveLength(0);
+    });
+    it('handles extractJson with best-effort extraction', async () => {
+        // LLM returns JSON embedded in text
+        const mockResponse = 'Here are the findings: {"version":1,"findings":[]} Done.';
+        const mockAnalyze = jest.fn().mockResolvedValue(mockResponse);
+        const provider = { name: 'test', analyze: mockAnalyze };
+        const file = makeFile();
+        const config = makeConfig({ cacheDir: tmpDir });
+        const result = await analyzeWithLlm(provider, config, file, 'content', []);
+        expect(result.ran).toBe(true);
+    });
+});
+//# sourceMappingURL=llmAnalysisMitre.test.js.map

package/dist/__tests__/llmGroqTPM.test.d.ts

@@ -0,0 +1,6 @@
+/**
+ * LLM Groq TPM Throttle Tests
+ * Tests for the Groq-specific TPM throttling in createOpenAICompatibleProvider
+ */
+export {};
+//# sourceMappingURL=llmGroqTPM.test.d.ts.map

package/dist/__tests__/llmGroqTPM.test.js

@@ -0,0 +1,89 @@
+/**
+ * LLM Groq TPM Throttle Tests
+ * Tests for the Groq-specific TPM throttling in createOpenAICompatibleProvider
+ */
+import { createOpenAICompatibleProvider } from '../features/llmAnalysis.js';
+function makeGroqConfig(overrides = {}) {
+    return {
+        provider: 'openai-compatible',
+        baseUrl: 'https://api.groq.com/openai/v1/chat/completions',
+        model: 'llama-3.3-70b',
+        apiKeyEnv: 'GROQ_TEST_KEY',
+        timeoutMs: 5000,
+        jsonMode: false,
+        maxInputChars: 500,
+        maxOutputTokens: 100,
+        temperature: 0,
+        systemPromptAddendum: '',
+        includeMitreAtlasTechniques: false,
+        maxMitreAtlasTechniques: 0,
+        cacheDir: '/tmp/ferret-llm-cache',
+        cacheTtlHours: 1,
+        maxRetries: 0,
+        retryBackoffMs: 1,
+        retryMaxBackoffMs: 10,
+        minRequestIntervalMs: 1,
+        onlyIfFindings: false,
+        maxFindingsPerFile: 10,
+        maxFiles: 5,
+        minConfidence: 0.5,
+        ...overrides,
+    };
+}
+describe('createOpenAICompatibleProvider - Groq adaptations', () => {
+    beforeEach(() => {
+        process.env['GROQ_TEST_KEY'] = 'gsk_test_key_for_groq_tests_abc123';
+    });
+    afterEach(() => {
+        delete process.env['GROQ_TEST_KEY'];
+    });
+    it('creates a Groq provider with reduced output tokens', async () => {
+        const provider = createOpenAICompatibleProvider(makeGroqConfig({
+            maxOutputTokens: 1000, // Should be reduced to 400 for Groq
+        }));
+        expect(provider).not.toBeNull();
+        expect(provider?.name).toBe('openai-compatible');
+    });
+    it('provider analyze calls fetch for Groq endpoint', async () => {
+        globalThis.fetch = jest.fn().mockResolvedValue({
+            ok: true,
+            status: 200,
+            json: () => Promise.resolve({
+                choices: [{ message: { content: '{"version":1,"findings":[]}' } }],
+            }),
+        });
+        const provider = createOpenAICompatibleProvider(makeGroqConfig());
+        expect(provider).not.toBeNull();
+        const result = await provider.analyze({ system: 'test', user: 'content' });
+        expect(typeof result).toBe('string');
+        expect(globalThis.fetch).toHaveBeenCalledWith(expect.stringContaining('groq.com'), expect.any(Object));
+    });
+    it('uses larger minRequestIntervalMs for Groq (at least 1000ms)', async () => {
+        // Create with small interval but Groq endpoint should enforce minimum
+        const provider = createOpenAICompatibleProvider(makeGroqConfig({
+            minRequestIntervalMs: 50, // Should be bumped to 1000ms for Groq
+        }));
+        expect(provider).not.toBeNull();
+        // Just verify it was created
+        expect(provider?.name).toBe('openai-compatible');
+    });
+    it('handles very large prompt estimates exceeding TPM limit', async () => {
+        globalThis.fetch = jest.fn().mockResolvedValue({
+            ok: true,
+            status: 200,
+            json: () => Promise.resolve({
+                choices: [{ message: { content: '{}' } }],
+            }),
+        });
+        const provider = createOpenAICompatibleProvider(makeGroqConfig({
+            maxInputChars: 100000, // Very large input that would exceed Groq TPM
+            maxOutputTokens: 400,
+        }));
+        expect(provider).not.toBeNull();
+        // Large prompt - should warn about exceeding limit but still attempt
+        const largePrompt = { system: 'x'.repeat(10000), user: 'y'.repeat(10000) };
+        const result = await provider.analyze(largePrompt);
+        expect(typeof result).toBe('string');
+    });
+});
+//# sourceMappingURL=llmGroqTPM.test.js.map

package/dist/__tests__/llmProviderRetry.test.d.ts

@@ -0,0 +1,6 @@
+/**
+ * LLM Provider Retry Tests
+ * Tests for retry behavior in createOpenAICompatibleProvider
+ */
+export {};
+//# sourceMappingURL=llmProviderRetry.test.d.ts.map

package/dist/__tests__/llmProviderRetry.test.js

@@ -0,0 +1,172 @@
+/**
+ * LLM Provider Retry Tests
+ * Tests for retry behavior in createOpenAICompatibleProvider
+ */
+import { createOpenAICompatibleProvider } from '../features/llmAnalysis.js';
+function makeConfig(overrides = {}) {
+    return {
+        provider: 'openai-compatible',
+        baseUrl: 'http://localhost:11434/v1/chat/completions',
+        model: 'llama3',
+        apiKeyEnv: 'DUMMY_KEY',
+        timeoutMs: 5000,
+        jsonMode: false,
+        maxInputChars: 1000,
+        maxOutputTokens: 100,
+        temperature: 0,
+        systemPromptAddendum: '',
+        includeMitreAtlasTechniques: false,
+        maxMitreAtlasTechniques: 0,
+        cacheDir: '/tmp/ferret-llm-cache',
+        cacheTtlHours: 1,
+        maxRetries: 2,
+        retryBackoffMs: 1,
+        retryMaxBackoffMs: 10,
+        minRequestIntervalMs: 0,
+        onlyIfFindings: false,
+        maxFindingsPerFile: 10,
+        maxFiles: 5,
+        minConfidence: 0.5,
+        ...overrides,
+    };
+}
+describe('createOpenAICompatibleProvider - retry behavior', () => {
+    it('retries on 429 rate limit and succeeds on second attempt', async () => {
+        let callCount = 0;
+        globalThis.fetch = jest.fn().mockImplementation(async () => {
+            callCount++;
+            if (callCount === 1) {
+                return {
+                    ok: false,
+                    status: 429,
+                    headers: { get: () => null },
+                    text: () => Promise.resolve('Rate limit exceeded'),
+                };
+            }
+            return {
+                ok: true,
+                status: 200,
+                json: () => Promise.resolve({
+                    choices: [{ message: { content: '{"version":1,"findings":[]}' } }],
+                }),
+            };
+        });
+        const provider = createOpenAICompatibleProvider(makeConfig({ maxRetries: 2 }));
+        expect(provider).not.toBeNull();
+        const result = await provider.analyze({ system: 'sys', user: 'usr' });
+        expect(callCount).toBe(2);
+        expect(result).toContain('findings');
+    });
+    it('retries on 500 server error', async () => {
+        let callCount = 0;
+        globalThis.fetch = jest.fn().mockImplementation(async () => {
+            callCount++;
+            if (callCount < 2) {
+                return {
+                    ok: false,
+                    status: 500,
+                    headers: { get: () => null },
+                    text: () => Promise.resolve('Server error'),
+                };
+            }
+            return {
+                ok: true,
+                status: 200,
+                json: () => Promise.resolve({
+                    choices: [{ message: { content: '{}' } }],
+                }),
+            };
+        });
+        const provider = createOpenAICompatibleProvider(makeConfig({ maxRetries: 2 }));
+        const result = await provider.analyze({ system: 'sys', user: 'usr' });
+        expect(callCount).toBe(2);
+        expect(typeof result).toBe('string');
+    });
+    it('throws after exhausting retries', async () => {
+        globalThis.fetch = jest.fn().mockResolvedValue({
+            ok: false,
+            status: 429,
+            headers: { get: () => null },
+            text: () => Promise.resolve('Rate limit'),
+        });
+        const provider = createOpenAICompatibleProvider(makeConfig({ maxRetries: 1 }));
+        await expect(provider.analyze({ system: 'sys', user: 'usr' })).rejects.toThrow('LLM HTTP 429');
+    });
+    it('throws immediately for non-retryable 400 errors', async () => {
+        globalThis.fetch = jest.fn().mockResolvedValue({
+            ok: false,
+            status: 400,
+            headers: { get: () => null },
+            text: () => Promise.resolve('Bad request'),
+        });
+        const provider = createOpenAICompatibleProvider(makeConfig({ maxRetries: 3 }));
+        await expect(provider.analyze({ system: 'sys', user: 'usr' })).rejects.toThrow('LLM HTTP 400');
+        // Should only have called fetch once (no retry for 400)
+        expect(globalThis.fetch.mock.calls).toHaveLength(1);
+    });
+    it('respects Retry-After header', async () => {
+        let callCount = 0;
+        globalThis.fetch = jest.fn().mockImplementation(async () => {
+            callCount++;
+            if (callCount === 1) {
+                return {
+                    ok: false,
+                    status: 429,
+                    headers: { get: (name) => name === 'retry-after' ? '0' : null },
+                    text: () => Promise.resolve('Rate limited'),
+                };
+            }
+            return {
+                ok: true,
+                status: 200,
+                json: () => Promise.resolve({
+                    choices: [{ message: { content: '{}' } }],
+                }),
+            };
+        });
+        const provider = createOpenAICompatibleProvider(makeConfig({ maxRetries: 2 }));
+        const result = await provider.analyze({ system: 'sys', user: 'usr' });
+        expect(typeof result).toBe('string');
+    });
+    it('falls back from jsonMode when unsupported (HTTP 400 with response_format error)', async () => {
+        let callCount = 0;
+        globalThis.fetch = jest.fn().mockImplementation(async (_url, opts) => {
+            callCount++;
+            const body = JSON.parse(opts.body);
+            if (callCount === 1 && body.response_format) {
+                // Simulate provider returning a 400 with response_format rejection
+                return {
+                    ok: false,
+                    status: 400,
+                    headers: { get: () => null },
+                    text: () => Promise.resolve('unknown field: response_format - json_validate_failed'),
+                };
+            }
+            return {
+                ok: true,
+                status: 200,
+                json: () => Promise.resolve({
+                    choices: [{ message: { content: '{"findings":[]}' } }],
+                }),
+            };
+        });
+        // Enable jsonMode - it should fallback when unsupported (400 with response_format error message)
+        const provider = createOpenAICompatibleProvider(makeConfig({ jsonMode: true, maxRetries: 0 }));
+        const result = await provider.analyze({ system: 'sys', user: 'usr' });
+        expect(typeof result).toBe('string');
+        expect(callCount).toBe(2); // First with jsonMode, second without
+    });
+    it('handles minRequestIntervalMs throttling', async () => {
+        globalThis.fetch = jest.fn().mockResolvedValue({
+            ok: true,
+            status: 200,
+            json: () => Promise.resolve({
+                choices: [{ message: { content: '{}' } }],
+            }),
+        });
+        const provider = createOpenAICompatibleProvider(makeConfig({ minRequestIntervalMs: 0 }));
+        const result = await provider.analyze({ system: 'sys', user: 'usr' });
+        expect(typeof result).toBe('string');
+    });
+});
+//# sourceMappingURL=llmProviderRetry.test.js.map

package/dist/__tests__/mcpValidator.extra.test.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Additional MCP Validator Tests
+ */
+export {};
+//# sourceMappingURL=mcpValidator.extra.test.d.ts.map