micro-models-agent 0.39.0 → 0.40.0

package/dist/modules/hallucination/detector.js

@@ -1,17 +1,17 @@
-import { FactualCheck } from './factual';
-import { ConsistencyCheck } from './consistency';
-import { ConfidenceCheck } from './confidence';
+import { ConsistencyCheck } from "./consistency";
+import { ConfidenceCheck } from "./confidence";
+import { FactualCheck } from "./factual";
+import { LLMJudge } from "./llm-judge";
 export class HallucinationDetector {
-    factual;
     consistency;
     confidence;
-    constructor() {
-        this.factual = new FactualCheck();
+    factual;
+    judge;
+    constructor(baseDir, llmProvider) {
         this.consistency = new ConsistencyCheck();
         this.confidence = new ConfidenceCheck();
-    }
-    getFactualCheck() {
-        return this.factual;
+        this.factual = baseDir ? new FactualCheck(baseDir) : null;
+        this.judge = llmProvider ? new LLMJudge(llmProvider) : null;
     }
     getConsistencyCheck() {
         return this.consistency;
@@ -19,23 +19,28 @@ export class HallucinationDetector {
     getConfidenceCheck() {
         return this.confidence;
     }
-    validate(response) {
+    async validate(response) {
         const confidenceResult = this.confidence.validate(response);
-        if (confidenceResult.status === 'retry' || confidenceResult.status === 'block') {
+        if (confidenceResult.status === "retry" ||
+            confidenceResult.status === "block") {
             return confidenceResult;
         }
-        const factualResult = this.factual.validate(response);
-        const consistencyResult = this.consistency.validate(response);
+        const factualResult = this.factual?.validate(response);
+        if (factualResult && factualResult.status !== "pass") {
+            return factualResult;
+        }
+        // LLM-as-judge consistency: fast path (no decisions) skips any LLM call.
+        const judgeResult = this.judge
+            ? await this.judge.validate(response, this.consistency)
+            : null;
         const warnings = [];
-        if (factualResult.status === 'warn')
-            warnings.push(factualResult.reason || '');
-        if (consistencyResult.status === 'warn')
-            warnings.push(consistencyResult.reason || '');
-        if (confidenceResult.status === 'warn')
-            warnings.push(confidenceResult.reason || '');
+        if (judgeResult && judgeResult.status === "warn")
+            warnings.push(judgeResult.reason || "");
+        if (confidenceResult.status === "warn")
+            warnings.push(confidenceResult.reason || "");
         if (warnings.length > 0) {
-            return { status: 'warn', reason: warnings.join('; ') };
+            return { status: "warn", reason: warnings.join("; ") };
         }
-        return { status: 'pass' };
+        return { status: "pass" };
     }
 }

package/dist/modules/hallucination/factual.js

@@ -1,170 +1,190 @@
+import { existsSync, readdirSync } from "fs";
+import { resolve, isAbsolute, join } from "path";
 import { t } from "../../i18n/index";
-import { existsSync } from "fs";
-import { join } from "path";
-const FILE_EXTENSIONS = new Set([
-    "ts",
-    "tsx",
-    "js",
-    "jsx",
-    "mjs",
-    "cjs",
-    "mts",
-    "cts",
-    "json",
-    "md",
-    "yaml",
-    "yml",
-    "py",
-    "rs",
-    "go",
-    "java",
-    "c",
-    "cpp",
-    "h",
-    "hpp",
-    "html",
-    "css",
-    "scss",
-    "less",
-    "vue",
-    "svelte",
-    "sh",
-    "bash",
-    "zsh",
-    "ps1",
-    "bat",
-    "cmd",
-    "txt",
-    "env",
-    "env.local",
-    "env.production",
-    "gitignore",
-    "dockerignore",
-    "dockerfile",
-    "makefile",
-    "cmake",
-    "toml",
-    "lock",
-    "config",
-    "log",
-    "xml",
-    "sql",
-    "graphql",
-    "proto",
-    "wasm",
+import { isJsMemberAccess } from "./js-identifiers";
+/** Generated/cached dirs to skip when scanning for bare file names. */
+const IGNORED_DIRS = new Set([
+    "node_modules",
+    ".git",
+    "dist",
+    "build",
+    "coverage",
+    ".mma",
+    "_testing",
 ]);
-const VERSION_PATTERN = /^\d+(\.\d+)*$/;
-const COMMON_WORDS = new Set([
-    "node.js", "Node.js",
-    "console.log", "console.error", "console.warn", "console.info",
-    "Math.floor", "Math.ceil", "Math.round", "Math.max", "Math.min",
-    "JSON.parse", "JSON.stringify",
-    "Object.keys", "Object.values", "Object.entries",
-    "Array.from", "Array.isArray",
-    "Date.now", "Date.parse",
-    "RegExp", "Promise",
+const NON_FILE_EXTENSIONS = new Set([
+    "com",
+    "org",
+    "net",
+    "gov",
+    "edu",
+    "io",
+    "dev",
+    "app",
+    "co",
+    "ai",
+    "ru",
+    "de",
+    "fr",
+    "uk",
+    "us",
+    "es",
+    "it",
+    "jp",
+    "cn",
+    "br",
 ]);
+/** Well-known runtime/library names that end in .js/.ts but are NOT files. */
+const TECH_NAMES = new Set([
+    "node.js",
+    "react.js",
+    "vue.js",
+    "next.js",
+    "angular.js",
+    "express.js",
+    "deno.js",
+    "qwik.js",
+    "svelte.js",
+    "jquery.js",
+]);
+/**
+ * JS/TS global identifiers whose member access (`process.env`, `console.log`)
+ * looks like a file path to the regex but is code, not a file.
+ */
+function looksLikeFilePath(s) {
+    const lower = s.toLowerCase();
+    // "Node.js" etc. are technologies, not files — never flag them.
+    if (TECH_NAMES.has(lower))
+        return false;
+    if (!s.includes("/") && !s.includes("\\") && !s.match(/^\w+\.[a-z]{2,4}$/i))
+        return false;
+    // process.env / console.log / Math.max — member access on JS globals,
+    // not file paths. Only applies to bare `word.word` (no slashes).
+    if (isJsMemberAccess(s))
+        return false;
+    const ext = s.split(".").pop()?.toLowerCase() || "";
+    return !NON_FILE_EXTENSIONS.has(ext);
+}
 export class FactualCheck {
-    knownPaths = new Set();
-    createdPaths = new Set();
-    readFiles = new Set();
-    baseDir = process.cwd();
-    setBaseDir(dir) {
-        this.baseDir = dir;
+    baseDir;
+    nameCache = null;
+    nameCacheTime = 0;
+    lastRescan = 0;
+    static NAME_CACHE_TTL_MS = 5_000;
+    static RESCAN_COOLDOWN_MS = 2_000;
+    static MAX_INDEXED_FILES = 20_000;
+    constructor(baseDir) {
+        this.baseDir = baseDir;
     }
-    trackReadPath(path) {
-        this.knownPaths.add(path);
-        const base = path.split(/[/\\]/).pop();
-        if (base && base !== path)
-            this.knownPaths.add(base);
-        // Track that this file was actually read by the agent
-        this.readFiles.add(base || path);
+    validate(response) {
+        const filePaths = this.extractFilePaths(response);
+        if (filePaths.length === 0)
+            return { status: "pass" };
+        const nonExistent = [];
+        for (const fp of filePaths) {
+            if (!this.pathExists(fp)) {
+                nonExistent.push(fp);
+            }
+        }
+        if (nonExistent.length > 0) {
+            return {
+                status: "warn",
+                reason: t("hall.unknown_paths", {
+                    paths: nonExistent.slice(0, 5).join(", "),
+                }),
+            };
+        }
+        return { status: "pass" };
+    }
+    /**
+     * Bare filenames ("bikes.json") and relative paths ("src/data/bikes.json")
+     * both land on the project tree. Absolute paths are checked as-is.
+     */
+    pathExists(fp) {
+        if (isAbsolute(fp))
+            return existsSync(fp);
+        // "foo.json" has no directory part — treat it as a bare name that may
+        // live anywhere under baseDir (e.g. src/data/bikes.json). Resolving it
+        // against the workspace root alone produced false "unknown file" warnings.
+        if (!fp.includes("/") && !fp.includes("\\")) {
+            return this.bareNameExists(fp);
+        }
+        return existsSync(resolve(this.baseDir, fp));
     }
-    trackCreatedPath(path) {
-        this.knownPaths.add(path);
-        const base = path.split(/[/\\]/).pop();
-        if (base && base !== path)
-            this.knownPaths.add(base);
-        this.createdPaths.add(path);
+    bareNameExists(name) {
+        if (existsSync(resolve(this.baseDir, name)))
+            return true;
+        if (this.indexHas(name))
+            return true;
+        // A write_file from this session may have created the file after the
+        // index was built. Do a one-off rescan (rate-limited) before warning.
+        const now = Date.now();
+        if (now - this.lastRescan < FactualCheck.RESCAN_COOLDOWN_MS)
+            return false;
+        this.lastRescan = now;
+        this.nameCache = null;
+        this.nameCacheTime = 0;
+        return this.indexHas(name);
     }
-    trackDeletedPath(path) {
-        this.knownPaths.add(path);
-        const base = path.split(/[/\\]/).pop();
-        if (base && base !== path)
-            this.knownPaths.add(base);
-        this.createdPaths.delete(path);
+    indexHas(name) {
+        return (this.getNameIndex().get(name)?.length ?? 0) > 0;
     }
     /**
-     * Register file paths found in a document (e.g. structure.md, README).
-     * Files mentioned in project documentation are not hallucinations.
+     * Basename → paths index, rebuilt every NAME_CACHE_TTL_MS so newly created
+     * files (write_file) are picked up without a full scan on every response.
      */
-    trackDocumentContent(content) {
-        // Match common path patterns in documentation
-        const pathPatterns = /(?:^|\s)([\w\-./]+\.\w{1,10})(?:\s|$|[,;)])/gm;
-        let match;
-        while ((match = pathPatterns.exec(content)) !== null) {
-            const file = match[1];
-            if (file.includes('/') || file.includes('\\')) {
-                this.knownPaths.add(file);
-            }
-            const base = file.split(/[/\\]/).pop();
-            if (base)
-                this.knownPaths.add(base);
+    getNameIndex() {
+        const now = Date.now();
+        if (this.nameCache &&
+            now - this.nameCacheTime < FactualCheck.NAME_CACHE_TTL_MS) {
+            return this.nameCache;
         }
+        const index = new Map();
+        const scanned = this.scanDir(this.baseDir, index, 0);
+        if (scanned > 0 || index.size > 0) {
+            this.nameCache = index;
+            this.nameCacheTime = now;
+        }
+        return index;
     }
-    pathExistsOnDisk(path) {
+    scanDir(dir, index, count) {
+        if (count >= FactualCheck.MAX_INDEXED_FILES)
+            return count;
+        let entries;
         try {
-            // Skip absolute paths — they're either system paths or outside the project.
-            // On Windows, existsSync('/...') can hang on certain paths.
-            if (path.startsWith("/") || path.startsWith("~") || /^[A-Za-z]:/.test(path)) {
-                return false;
-            }
-            return existsSync(join(this.baseDir, path));
+            entries = readdirSync(dir, { withFileTypes: true });
         }
         catch {
-            return false;
+            return count;
         }
-    }
-    validate(response) {
-        const pathRegex = /[\w\-./]+\.\w+/g;
-        const mentionedPaths = response.match(pathRegex) || [];
-        const unknownPaths = mentionedPaths.filter((p) => {
-            if (this.knownPaths.has(p))
-                return false;
-            if (VERSION_PATTERN.test(p))
-                return false;
-            if (COMMON_WORDS.has(p))
-                return false;
-            // Skip absolute paths — agent responses use relative paths; absolute
-            // paths are either system paths or URLs, and existsSync can hang on
-            // Windows for root-relative paths like "/page.html".
-            if (p.startsWith("/") || p.startsWith("~") || /^[A-Za-z]:/.test(p))
-                return false;
-            if (this.pathExistsOnDisk(p))
-                return false;
-            const ext = p.split(".").pop()?.toLowerCase() || "";
-            if (!FILE_EXTENSIONS.has(ext))
-                return false;
-            const basename = p
-                .split("/")
-                .pop()
-                ?.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-            if (basename) {
-                const urlPattern = new RegExp(`(https?|file)://[^\\s]*${basename}`);
-                if (urlPattern.test(response))
-                    return false;
+        for (const entry of entries) {
+            if (count >= FactualCheck.MAX_INDEXED_FILES)
+                break;
+            const full = join(dir, entry.name);
+            if (entry.isDirectory()) {
+                if (!IGNORED_DIRS.has(entry.name)) {
+                    count = this.scanDir(full, index, count);
+                }
+            }
+            else if (entry.isFile()) {
+                const list = index.get(entry.name);
+                if (list)
+                    list.push(full);
+                else
+                    index.set(entry.name, [full]);
+                count++;
             }
-            return true;
-        });
-        if (unknownPaths.length > 0) {
-            const uniquePaths = [...new Set(unknownPaths)];
-            return {
-                status: "warn",
-                reason: t("hall.unknown_paths", {
-                    paths: uniquePaths.slice(0, 3).join(", "),
-                }),
-            };
         }
-        return { status: "pass" };
+        return count;
+    }
+    extractFilePaths(text) {
+        // Capture absolute Windows paths (with drive letter) and relative paths.
+        // The old pattern started at a word boundary, so "E:\project\src\a.ts"
+        // only matched "project\src\a.ts" — resolve() then glued that onto
+        // baseDir producing a bogus path and a false "unknown file" warning
+        // even for files that actually exist on disk.
+        const pattern = /\b(?:[a-zA-Z]:[\\/])?[\w./\\-]+\.[a-z]{2,6}\b/gi;
+        const matches = text.match(pattern) || [];
+        return [...new Set(matches)].filter(looksLikeFilePath);
     }
 }

package/dist/modules/hallucination/index.js

@@ -1,4 +1,5 @@
-export { HallucinationDetector } from './detector';
-export { FactualCheck } from './factual';
-export { ConsistencyCheck } from './consistency';
-export { ConfidenceCheck } from './confidence';
+export { HallucinationDetector, } from "./detector";
+export { ConsistencyCheck } from "./consistency";
+export { ConfidenceCheck } from "./confidence";
+export { FactualCheck } from "./factual";
+export { LLMJudge } from "./llm-judge";

package/dist/modules/hallucination/js-identifiers.js

@@ -0,0 +1,72 @@
+/**
+ * JS/TS global identifiers whose member access (`process.env`, `console.log`,
+ * `Math.max`) looks like a file path to `\b[\w./\\-]+\.[a-z]+\b`-style regexes
+ * but is code, not a file. Shared by the hallucination factual check and the
+ * execution auditor so both stop flagging member access as "missing files".
+ */
+const JS_GLOBALS = new Set([
+    "process",
+    "console",
+    "math",
+    "json",
+    "global",
+    "globalthis",
+    "window",
+    "document",
+    "buffer",
+    "module",
+    "require",
+    "exports",
+    "url",
+    "promise",
+    "array",
+    "object",
+    "string",
+    "number",
+    "boolean",
+    "symbol",
+    "date",
+    "regexp",
+    "error",
+    "map",
+    "set",
+    "weakmap",
+    "weakset",
+    "proxy",
+    "reflect",
+    "intl",
+    "crypto",
+    "performance",
+    "fetch",
+    "navigator",
+    "location",
+    "settimeout",
+    "setinterval",
+    "cleartimeout",
+    "clearinterval",
+    "queuemicrotask",
+    "structuredclone",
+    "textencoder",
+    "textdecoder",
+    "atomics",
+    "sharedarraybuffer",
+    "dataview",
+    "arraybuffer",
+    "bigint",
+    "infinity",
+    "nan",
+    "undefined",
+    "bun",
+    "deno",
+    "node",
+]);
+/**
+ * True when `candidate` starts with a known JS/TS global followed by a dot
+ * (`process.env`, `console.log`, `process.env.NODE` partial matches from
+ * `\b[\w./\\-]+\.[a-z]+`-style regexes) — member access, not a file path.
+ */
+export function isJsMemberAccess(candidate) {
+    const lower = candidate.toLowerCase();
+    const m = lower.match(/^([\w-]+)\./);
+    return m !== null && JS_GLOBALS.has(m[1]);
+}

package/dist/modules/hallucination/llm-judge.js

@@ -0,0 +1,103 @@
+import { t } from "../../i18n/index";
+const JUDGE_TIMEOUT_MS = 15000;
+const MAX_DECISIONS_SHOWN = 6;
+const MAX_RESPONSE_CHARS = 2000;
+/**
+ * LLM-as-judge consistency check. Asks a model whether the agent's latest
+ * response contradicts any of its earlier decisions — replacing the fragile
+ * word-matching heuristic that only caught exact substring reversals.
+ *
+ * Fast path: when no decisions are tracked, returns pass without any LLM call.
+ * Any judge failure (timeout, provider error, unparseable verdict) degrades to
+ * pass so the agent loop is never blocked by the judge.
+ */
+export class LLMJudge {
+    provider;
+    constructor(provider) {
+        this.provider = provider;
+    }
+    async validate(response, consistency) {
+        const decisions = consistency.getDecisions();
+        if (decisions.length === 0) {
+            return { status: "pass" };
+        }
+        const shown = decisions.slice(-MAX_DECISIONS_SHOWN);
+        const decisionLines = shown
+            .map((d, i) => `${i + 1}. [${d.location || "agent"}] "${d.decision}"`)
+            .join("\n");
+        const system = `You are a consistency checker for an AI coding agent. The agent previously made these decisions:
+
+${decisionLines}
+
+Decide whether the agent's NEW response contradicts any of these decisions. A contradiction means reversing or abandoning a previously stated approach (e.g. switching to a different framework, changing a chosen strategy, or undoing a decision that was explicitly made).
+
+Reply with ONLY valid JSON, no other text:
+- {"contradicts": false}
+- {"contradicts": true, "reason": "short explanation"}`;
+        const user = `Agent's new response:\n\n${response.slice(0, MAX_RESPONSE_CHARS)}`;
+        try {
+            const verdictText = await this.askJudge([
+                { role: "system", content: system },
+                { role: "user", content: user },
+            ]);
+            const verdict = this.parseVerdict(verdictText);
+            if (verdict.contradicts) {
+                const reason = verdict.reason?.trim()
+                    ? ` — ${verdict.reason.trim()}`
+                    : "";
+                return {
+                    status: "warn",
+                    reason: t("hall.contradiction_llm", { reason }),
+                };
+            }
+            return { status: "pass" };
+        }
+        catch {
+            return { status: "pass" };
+        }
+    }
+    async askJudge(messages) {
+        let text = "";
+        let timer;
+        const consume = (async () => {
+            for await (const chunk of this.provider.chat(messages, [])) {
+                if (chunk.type === "text" && chunk.content) {
+                    text += chunk.content;
+                }
+            }
+            return text;
+        })();
+        const timeout = new Promise((_, reject) => {
+            timer = setTimeout(() => reject(new Error("LLM judge timed out")), JUDGE_TIMEOUT_MS);
+        });
+        try {
+            return await Promise.race([consume, timeout]);
+        }
+        finally {
+            if (timer)
+                clearTimeout(timer);
+        }
+    }
+    parseVerdict(text) {
+        const block = text.match(/\{[\s\S]*\}/);
+        if (block) {
+            try {
+                const obj = JSON.parse(block[0]);
+                if (typeof obj.contradicts === "boolean") {
+                    return {
+                        contradicts: obj.contradicts,
+                        reason: typeof obj.reason === "string" ? obj.reason : undefined,
+                    };
+                }
+            }
+            catch {
+                // fall through to lenient parsing
+            }
+        }
+        const lower = text.toLowerCase();
+        if (lower.includes('"contradicts": true') || /^yes\b/.test(lower.trim())) {
+            return { contradicts: true, reason: text.slice(0, 200) };
+        }
+        return { contradicts: false };
+    }
+}

package/dist/modules/index.js

@@ -1,5 +1,5 @@
-export { ModuleRegistry } from './registry';
-export { SkillsLoader, SkillsMatcher, SkillsModule } from './skills';
-export { PluginManager } from './plugins';
-export { PluginLoader } from './plugins/loader';
-export { MCPClient, MCPRegistry } from './mcp';
+export { ModuleRegistry } from "./registry";
+export { SkillsLoader, SkillsModule } from "./skills";
+export { PluginManager } from "./plugins";
+export { PluginLoader } from "./plugins/loader";
+export { MCPClient, MCPRegistry } from "./mcp";