hallucination-validator 1.0.0

package/README.md

@@ -0,0 +1,64 @@
+# hallucination-validator
+
+**AI Output Validator for Security and Fact-Checking**
+
+`hallucination-validator` is a Node.js library that validates the output of Large Language Models (LLMs) to catch "hallucinations" that are actually security risks. It checks for broken links, dangerous code patterns, and verifies that quotes actually exist in your source context.
+
+## Why use this?
+
+LLMs often:
+1.  Invent URLs that look real but are dead (or worse, hijacked).
+2.  Suggest "standard" but insecure code (like `eval()`).
+3.  Fabricate quotes from documents they are summarizing.
+
+This package provides a programmatic way to flag these issues before showing the response to a user.
+
+## Features
+
+*   **Link Validation**: Extracts URLs and pings them (HEAD request) to ensure they are 200 OK.
+*   **Code Safety Scanning**: Detects `eval()`, `child_process`, and other dangerous Node.js/JS patterns.
+*   **Quote Verification**: Fuzzy-matching to verify that a quoted string actually exists in the provided context source.
+
+## Installation
+
+```bash
+npm install hallucination-validator
+```
+
+## Usage
+
+```javascript
+const HallucinationValidator = require('hallucination-validator');
+
+const validator = new HallucinationValidator();
+
+// 1. Validate Links
+const text = "Check out valid.com and broken.link/404";
+validator.validateLinks(text).then(broken => {
+    console.log(broken); 
+    // [{ url: 'http://broken.link/404', status: 404, ... }]
+});
+
+// 2. Scan Code
+const code = "function run() { eval(input); }";
+const risks = validator.scanCodeSafety(code);
+console.log(risks); 
+// ['eval()']
+
+// 3. Verify Quotes
+const source = "The quick brown fox jumps over the lazy dog.";
+const aiQuote = "The quick brown fox jumped over a lazy cat.";
+
+const isValid = validator.verifyQuote(aiQuote, source);
+console.log(isValid); 
+// false (too different)
+```
+
+## Dependencies
+
+*   `fast-levenshtein`: For robust string comparison.
+*   **Node 18+**: uses native `fetch`.
+
+## License
+
+MIT

package/index.js

@@ -0,0 +1,125 @@
+const levenshtein = require('fast-levenshtein');
+
+class HallucinationValidator {
+
+    /**
+     * Scans text for URLs and verifies they are reachable (200 OK).
+     * @param {string} text 
+     * @returns {Promise<Array<{url: string, status: number, error: string}>>} List of broken links
+     */
+    async validateLinks(text) {
+        const urlRegex = /https?:\/\/[^\s)]+/g;
+        const matches = text.match(urlRegex) || [];
+        const brokenLinks = [];
+
+        // De-duplicate URLs
+        const uniqueUrls = [...new Set(matches)];
+
+        for (const url of uniqueUrls) {
+            // Remove trailing punctuation often captured by regex
+            const cleanUrl = url.replace(/[.,;]$/, '');
+
+            try {
+                const response = await fetch(cleanUrl, { method: 'HEAD' });
+                if (!response.ok) {
+                    brokenLinks.push({ url: cleanUrl, status: response.status, error: 'Non-200 status' });
+                }
+            } catch (err) {
+                brokenLinks.push({ url: cleanUrl, status: 0, error: err.message });
+            }
+        }
+
+        return brokenLinks;
+    }
+
+    /**
+     * Scans code snippets for potentially dangerous patterns often hallucinated.
+     * @param {string} code 
+     * @returns {Array<string>} List of found dangerous patterns
+     */
+    scanCodeSafety(code) {
+        const dangerousPatterns = [
+            { pattern: /eval\s*\(/g, name: 'eval()' },
+            { pattern: /child_process/g, name: 'child_process' },
+            { pattern: /exec\s*\(/g, name: 'exec()' },
+            { pattern: /spawn\s*\(/g, name: 'spawn()' },
+            { pattern: /innerHTML/g, name: 'innerHTML usage' },
+            { pattern: /document\.write/g, name: 'document.write' }
+        ];
+
+        const findings = [];
+        for (const { pattern, name } of dangerousPatterns) {
+            if (pattern.test(code)) {
+                findings.push(name);
+            }
+        }
+        return findings;
+    }
+
+    /**
+     * Verifies if a quote exists within a source context, allowing for typo tolerance.
+     * @param {string} quote 
+     * @param {string} context 
+     * @param {number} threshold - Allowed matching distance ratio (0.0 to 1.0)
+     * @returns {boolean}
+     */
+    verifyQuote(quote, context, threshold = 0.2) {
+        if (!quote || !context) return false;
+
+        // 1. Direct inclusion check (fastest)
+        if (context.includes(quote)) return true;
+
+        // 2. Normalized inclusion check (ignore case/punctuation)
+        const normalize = (s) => s.toLowerCase().replace(/[^\w\s]/g, '').replace(/\s+/g, ' ');
+        const nQuote = normalize(quote);
+        const nContext = normalize(context);
+
+        if (nContext.includes(nQuote)) return true;
+
+        // 3. Fuzzy match 
+        // We scan the context for a window of text similar in length to the quote
+        // and check Levenshtein distance. This is O(N*M) worst case but valid for verification.
+        const qLen = nQuote.length;
+        const cLen = nContext.length;
+
+        // Optimization: Don't scan if size diff is huge or context smaller logic
+        if (cLen < qLen * (1 - threshold)) return false;
+
+        // Sliding window scan
+        // To be performant, we only check windows starting at word boundaries
+        // or just brute force every N chars if high precision needed.
+        // For "Senior Dev" approach: Let's use a simplified heuristic. 
+        // If the context is massive, real fuzzy search is complex.
+        // We will do a character-by-character validation for small/medium texts.
+
+        // WindowStep: 1 char is safest.
+        // Limit: This is sync and blocking.
+        // If usage assumes short texts (like checking a paragraph), this is fine.
+
+        let minDistance = Infinity;
+
+        // Safety cap: if context > 10kb, maybe warn or skip fuzzy?
+        // Let's implement looking for the best window.
+
+        for (let i = 0; i <= cLen - qLen; i++) {
+            // Heuristic: check first char matches to skip redundant calc
+            if (nContext[i] !== nQuote[0]) continue;
+
+            const window = nContext.substring(i, i + qLen);
+            const dist = levenshtein.get(nQuote, window);
+            if (dist < minDistance) {
+                minDistance = dist;
+            }
+            if (minDistance / qLen <= threshold) return true;
+        }
+
+        // Check allows for slightly larger or smaller windows?
+        // nQuote might be missing a word, so window size in context might vary.
+        // But `verifyQuote` usually implies checking if the STATED quote is in source.
+        // So checking strict length window is a fair approximation for "is this string present".
+
+        return (minDistance / qLen) <= threshold;
+    }
+}
+
+module.exports = HallucinationValidator;

package/package.json

@@ -0,0 +1,25 @@
+{
+    "name": "hallucination-validator",
+    "version": "1.0.0",
+    "description": "Validates AI outputs for linkrot, dangerous code, and hallucinations.",
+    "main": "index.js",
+    "scripts": {
+        "test": "node --test"
+    },
+    "keywords": [
+        "ai",
+        "security",
+        "hallucination",
+        "validation",
+        "llm",
+        "fact-check"
+    ],
+    "author": "Godfrey Lebo <emorylebo@gmail.com>",
+    "license": "MIT",
+    "dependencies": {
+        "fast-levenshtein": "^3.0.0"
+    },
+    "engines": {
+        "node": ">=18.0.0"
+    }
+}

package/test/index.test.js

@@ -0,0 +1,57 @@
+const { test, mock } = require('node:test');
+const assert = require('node:assert');
+const HallucinationValidator = require('../index.js');
+
+test('validateLinks detects broken links', async (t) => {
+    // Mock global fetch
+    const originalFetch = global.fetch;
+
+    global.fetch = async (url) => {
+        if (url.includes('google.com')) {
+            return { ok: true, status: 200 };
+        }
+        if (url.includes('broken.link')) {
+            return { ok: false, status: 404 };
+        }
+        throw new Error('Network error');
+    };
+
+    const validator = new HallucinationValidator();
+    const text = "Check this out: https://google.com and https://broken.link/resource.";
+
+    const results = await validator.validateLinks(text);
+
+    assert.strictEqual(results.length, 1);
+    assert.strictEqual(results[0].url, 'https://broken.link/resource');
+    assert.strictEqual(results[0].status, 404);
+
+    // Restore fetch
+    global.fetch = originalFetch;
+});
+
+test('scanCodeSafety finds dangerous coding patterns', (t) => {
+    const validator = new HallucinationValidator();
+    const unsafeCode = `
+        function bad() {
+            eval("alert('hacked')");
+            const cp = require('child_process');
+        }
+    `;
+
+    const findings = validator.scanCodeSafety(unsafeCode);
+    assert.ok(findings.includes('eval()'));
+    assert.ok(findings.includes('child_process'));
+    assert.strictEqual(findings.length, 2);
+});
+
+test('verifyQuote confirms existence of fuzzy quote', (t) => {
+    const validator = new HallucinationValidator();
+    const context = "The quick brown fox jumps over the lazy dog.";
+    const exactQuote = "quick brown fox";
+    const fuzzyQuote = "quick brwn fox"; // typo
+    const wrongQuote = "lazy cat";
+
+    assert.strictEqual(validator.verifyQuote(exactQuote, context), true);
+    assert.strictEqual(validator.verifyQuote(fuzzyQuote, context, 0.3), true);
+    assert.strictEqual(validator.verifyQuote(wrongQuote, context), false);
+});