micro-models-agent 0.28.9 → 0.28.17

package/dist/modules/certification/cli.js

@@ -0,0 +1,176 @@
+import { rmSync } from "fs";
+import { homedir } from "os";
+import { join, dirname } from "path";
+import { fileURLToPath } from "url";
+import { existsSync, readFileSync } from "fs";
+import { t } from "../../i18n/index";
+import { pc } from "../../ui/colors";
+import { loadScenarios, filterByTags } from "./loader";
+import { runScenario, findMmaRoot } from "./runner";
+import { readManifest, upsertCertification, removeCertification, } from "./manifest";
+const HERE = dirname(fileURLToPath(import.meta.url));
+const MMA_ROOT = findMmaRoot(HERE);
+const USER_SCENARIO_DIR = join(homedir(), ".mma", "certification", "scenarios");
+function readVersion() {
+    const candidates = [join(MMA_ROOT, "package.json")];
+    for (const p of candidates) {
+        if (existsSync(p)) {
+            try {
+                const raw = JSON.parse(readFileSync(p, "utf-8"));
+                if (raw.version)
+                    return raw.version;
+            }
+            catch {
+                // Broken package.json — fall back to the default version
+            }
+        }
+    }
+    return "0.0.0";
+}
+export function parseTags(s) {
+    return s
+        .split(",")
+        .map((x) => x.trim())
+        .filter(Boolean);
+}
+export async function certify(opts) {
+    const providerUrl = opts.providerUrl || opts.config.provider.baseUrl;
+    if (opts.tags.includes("security") && !opts.config.security?.enabled) {
+        console.error(pc.red(t("cli.cert_security_required")));
+        process.exitCode = 1;
+        return;
+    }
+    const { scenarios, errors } = loadScenarios(USER_SCENARIO_DIR);
+    for (const e of errors)
+        console.error(pc.yellow(`  ${e}`));
+    const selected = filterByTags(scenarios, opts.tags);
+    if (selected.length === 0) {
+        console.error(pc.red(t("cli.cert_no_scenarios", { tags: opts.tags.join(",") })));
+        process.exitCode = 1;
+        return;
+    }
+    const manifest = readManifest();
+    const existing = manifest.certifications.find((e) => e.model === opts.name && e.providerUrl === providerUrl);
+    if (existing && !opts.force) {
+        console.error(pc.yellow(t("cli.cert_exists", { model: opts.name })));
+        console.error(pc.yellow(t("cli.cert_exists_hint")));
+        process.exitCode = 1;
+        return;
+    }
+    console.log(t("cli.cert_started", { model: opts.name, provider: providerUrl }));
+    const sandboxBase = join(process.cwd(), ".mma", "certification");
+    const results = [];
+    const total = selected.length;
+    let idx = 0;
+    for (const scenario of selected) {
+        idx++;
+        if (scenario.mode === "skip") {
+            console.log(pc.dim(`[${idx}/${total}] ${scenario.id} ... skipped`));
+            results.push({
+                id: scenario.id,
+                title: scenario.title,
+                status: "skipped",
+                passed: 0,
+                of: 0,
+            });
+            continue;
+        }
+        const res = await runScenario(scenario, {
+            model: opts.name,
+            providerUrl,
+            providerKey: opts.providerKey,
+            contextWindow: opts.contextWindow,
+            mmaRoot: MMA_ROOT,
+            sandboxBase,
+            defaultReps: opts.reps,
+            defaultThreshold: 2,
+            onRep: (id, rep, reps, passed, failures) => {
+                const word = passed
+                    ? pc.green(t("cli.cert_rep_pass"))
+                    : pc.red(t("cli.cert_rep_fail"));
+                console.log(`[${idx}/${total}] ${id} (${rep}/${reps})... ${word}`);
+                if (!passed)
+                    console.log(`    ${pc.dim(failures.join("; "))}`);
+            },
+        });
+        results.push(res);
+    }
+    if (opts.clean) {
+        try {
+            rmSync(sandboxBase, { recursive: true, force: true });
+        }
+        catch {
+            /* ignore */
+        }
+    }
+    const suite = summarize(results);
+    const entry = {
+        model: opts.name,
+        providerUrl,
+        mmaVersion: readVersion(),
+        certifiedAt: new Date().toISOString(),
+        suite,
+        results,
+    };
+    upsertCertification(entry);
+    console.log(t("cli.cert_done", {
+        passed: String(suite.passed),
+        failed: String(suite.failed),
+        skipped: String(suite.skipped),
+        total: String(suite.total),
+    }));
+    printResults(results);
+}
+export async function certStatus(name, config) {
+    const m = readManifest();
+    const providerUrl = config.provider.baseUrl;
+    const entry = m.certifications.find((e) => e.model === name && e.providerUrl === providerUrl);
+    if (!entry) {
+        console.log(t("cli.cert_not_found", { model: name }));
+        return;
+    }
+    console.log(`${t("cli.cert_provider_col")}: ${entry.providerUrl}`);
+    console.log(`${t("cli.cert_version_col")}: ${entry.mmaVersion}  ${t("cli.cert_date_col")}: ${entry.certifiedAt.slice(0, 10)}`);
+    console.log(`${t("cli.cert_suite_col")}: ${entry.suite.passed} pass / ${entry.suite.failed} fail / ${entry.suite.skipped} skipped`);
+    printResults(entry.results);
+}
+export async function certList() {
+    const m = readManifest();
+    if (m.certifications.length === 0) {
+        console.log(t("cli.cert_empty"));
+        return;
+    }
+    for (const e of m.certifications) {
+        console.log(`  ${pc.green("✔")} ${e.model}  ${pc.dim(e.providerUrl)}  ${e.mmaVersion}  ${e.certifiedAt.slice(0, 10)}  ${e.suite.passed}/${e.suite.total} pass`);
+    }
+}
+export async function uncertify(name, config) {
+    const removed = removeCertification(name, config.provider.baseUrl);
+    if (removed)
+        console.log(t("cli.cert_uncertified", { model: name }));
+    else
+        console.log(t("cli.cert_not_found", { model: name }));
+}
+function summarize(results) {
+    return {
+        passed: results.filter((r) => r.status === "pass").length,
+        failed: results.filter((r) => r.status === "fail").length,
+        skipped: results.filter((r) => r.status === "skipped").length,
+        total: results.length,
+    };
+}
+function printResults(results) {
+    for (const r of results) {
+        const icon = r.status === "pass"
+            ? pc.green("✔")
+            : r.status === "fail"
+                ? pc.red("✘")
+                : r.status === "skipped"
+                    ? pc.dim("–")
+                    : pc.yellow("!");
+        const detail = r.status === "skipped" ? pc.dim(r.title) : `${r.passed}/${r.of}`;
+        console.log(`  ${icon} ${r.id}  ${detail}`);
+        if (r.error)
+            console.log(`    ${pc.dim(r.error)}`);
+    }
+}

package/dist/modules/certification/fact-checker.js

@@ -0,0 +1,84 @@
+import { existsSync, readFileSync, statSync } from "fs";
+import { join } from "path";
+export function checkSandbox(sandboxDir, checks, exitCode, output) {
+    const failures = [];
+    for (const check of checks) {
+        if (!runCheck(sandboxDir, check, exitCode, output)) {
+            failures.push(describe(check));
+        }
+    }
+    return { pass: failures.length === 0, failures };
+}
+function runCheck(sandboxDir, check, exitCode, output) {
+    switch (check.type) {
+        case "exitCode":
+            return exitCode === (check.code ?? 0);
+        case "outputContains":
+            return output.includes(check.text);
+        case "fileExists":
+            return isFile(join(sandboxDir, check.path));
+        case "fileNotExists":
+            return !existsSync(join(sandboxDir, check.path));
+        case "dirExists":
+            return isDir(join(sandboxDir, check.path));
+        case "fileContent": {
+            const abs = join(sandboxDir, check.path);
+            if (!isFile(abs))
+                return false;
+            const content = readFileSync(abs, "utf-8");
+            if (check.contains !== undefined)
+                return content.includes(check.contains);
+            if (check.equals !== undefined)
+                return content === check.equals;
+            return false;
+        }
+        case "fileRegex": {
+            const abs = join(sandboxDir, check.path);
+            if (!isFile(abs))
+                return false;
+            return new RegExp(check.pattern).test(readFileSync(abs, "utf-8"));
+        }
+        default:
+            return false;
+    }
+}
+function isFile(p) {
+    try {
+        return existsSync(p) && statSync(p).isFile();
+    }
+    catch {
+        return false;
+    }
+}
+function isDir(p) {
+    try {
+        return existsSync(p) && statSync(p).isDirectory();
+    }
+    catch {
+        return false;
+    }
+}
+function describe(check) {
+    switch (check.type) {
+        case "exitCode":
+            return `expected exit code ${check.code ?? 0}`;
+        case "outputContains":
+            return `output missing "${check.text}"`;
+        case "fileExists":
+            return `file missing: ${check.path}`;
+        case "fileNotExists":
+            return `file unexpectedly exists: ${check.path}`;
+        case "dirExists":
+            return `directory missing: ${check.path}`;
+        case "fileContent": {
+            const what = check.contains !== undefined
+                ? `contains "${check.contains}"`
+                : `equals "${check.equals}"`;
+            return `${check.path} does not ${what}`;
+        }
+        case "fileRegex":
+            return `${check.path} does not match ${check.pattern}`;
+        default:
+            return "unknown check";
+    }
+}

package/dist/modules/certification/loader.js

@@ -0,0 +1,111 @@
+import { existsSync, readdirSync, readFileSync } from "fs";
+import { join } from "path";
+import { parse as parseYaml } from "yaml";
+import { BUILTIN_SCENARIOS } from "./scenarios";
+const TAGS = ["core", "security", "image", "network", "browser"];
+const CHECK_TYPES = [
+    "fileExists",
+    "fileNotExists",
+    "dirExists",
+    "fileContent",
+    "fileRegex",
+    "exitCode",
+    "outputContains",
+];
+export function validateScenario(s) {
+    const errors = [];
+    const isSkip = s.mode === "skip";
+    if (!s.id || !s.title || (!isSkip && !s.prompt)) {
+        errors.push(`[${s.id ?? "?"}] missing id, title or prompt`);
+    }
+    if (!Array.isArray(s.tags) || s.tags.length === 0) {
+        errors.push(`[${s.id}] tags required`);
+    }
+    for (const tag of s.tags) {
+        if (!TAGS.includes(tag))
+            errors.push(`[${s.id}] unknown tag: ${tag}`);
+    }
+    if (s.mode !== "run" && s.mode !== "skip") {
+        errors.push(`[${s.id}] mode must be run or skip`);
+    }
+    if (!Array.isArray(s.checks) || (!isSkip && s.checks.length === 0)) {
+        errors.push(`[${s.id}] at least one check required`);
+    }
+    for (const c of s.checks) {
+        if (!CHECK_TYPES.includes(c.type)) {
+            errors.push(`[${s.id}] unknown check type: ${c.type}`);
+        }
+        if (c.type === "fileContent" &&
+            c.contains === undefined &&
+            c.equals === undefined) {
+            errors.push(`[${s.id}] fileContent needs contains or equals`);
+        }
+    }
+    if (s.reps !== undefined && (!Number.isInteger(s.reps) || s.reps < 1)) {
+        errors.push(`[${s.id}] reps must be a positive integer`);
+    }
+    if (s.passThreshold !== undefined &&
+        (!Number.isInteger(s.passThreshold) || s.passThreshold < 1)) {
+        errors.push(`[${s.id}] passThreshold must be a positive integer`);
+    }
+    if (s.passThreshold !== undefined &&
+        s.reps !== undefined &&
+        s.passThreshold > s.reps) {
+        errors.push(`[${s.id}] passThreshold > reps (must be <= reps)`);
+    }
+    return errors;
+}
+export function loadScenarios(userDir) {
+    const errors = [];
+    const scenarios = [];
+    for (const s of BUILTIN_SCENARIOS) {
+        const errs = validateScenario(s);
+        if (errs.length > 0)
+            errors.push(...errs);
+        else
+            scenarios.push(s);
+    }
+    if (userDir && existsSync(userDir)) {
+        for (const file of readdirSync(userDir)) {
+            if (!file.endsWith(".yaml") && !file.endsWith(".yml"))
+                continue;
+            try {
+                const raw = readFileSync(join(userDir, file), "utf-8");
+                const data = parseYaml(raw);
+                const parsed = normalizeScenario(data, file);
+                const errs = validateScenario(parsed);
+                if (errs.length > 0)
+                    errors.push(...errs);
+                else
+                    scenarios.push(parsed);
+            }
+            catch (e) {
+                errors.push(`[${file}] failed to parse: ${e.message}`);
+            }
+        }
+    }
+    return { scenarios, errors };
+}
+function normalizeScenario(data, file) {
+    const d = (data ?? {});
+    return {
+        id: String(d.id ?? file),
+        title: String(d.title ?? d.id ?? file),
+        tags: Array.isArray(d.tags) ? d.tags : [],
+        mode: d.mode === "skip" ? "skip" : "run",
+        prompt: String(d.prompt ?? ""),
+        reps: typeof d.reps === "number" ? d.reps : undefined,
+        passThreshold: typeof d.passThreshold === "number" ? d.passThreshold : undefined,
+        fixtures: Array.isArray(d.fixtures)
+            ? d.fixtures
+            : undefined,
+        checks: Array.isArray(d.checks) ? d.checks : [],
+        skipReason: typeof d.skipReason === "string" ? d.skipReason : undefined,
+    };
+}
+export function filterByTags(scenarios, tags) {
+    if (!tags || tags.length === 0) {
+        return scenarios.filter((s) => s.tags.includes("core"));
+    }
+    return scenarios.filter((s) => s.tags.some((t) => tags.includes(t)));
+}

package/dist/modules/certification/manifest.js

@@ -0,0 +1,50 @@
+import { existsSync, readFileSync, mkdirSync, writeFileSync } from "fs";
+import { homedir } from "os";
+import { join } from "path";
+export const MANIFEST_PATH = join(homedir(), ".mma", "certifications.json");
+export function readManifest(path = MANIFEST_PATH) {
+    try {
+        if (existsSync(path)) {
+            const raw = JSON.parse(readFileSync(path, "utf-8"));
+            return { version: 1, certifications: raw.certifications ?? [] };
+        }
+    }
+    catch {
+        /* corrupted manifest → start fresh */
+    }
+    return { version: 1, certifications: [] };
+}
+export function saveManifest(m, path = MANIFEST_PATH) {
+    mkdirSync(join(homedir(), ".mma"), { recursive: true });
+    writeFileSync(path, JSON.stringify(m, null, 2), "utf-8");
+}
+export function upsertCertification(entry, path = MANIFEST_PATH) {
+    const m = readManifest(path);
+    const idx = m.certifications.findIndex((e) => e.model === entry.model && e.providerUrl === entry.providerUrl);
+    if (idx >= 0)
+        m.certifications[idx] = entry;
+    else
+        m.certifications.push(entry);
+    saveManifest(m, path);
+    return m;
+}
+export function removeCertification(model, providerUrl, path = MANIFEST_PATH) {
+    const m = readManifest(path);
+    const before = m.certifications.length;
+    m.certifications = m.certifications.filter((e) => !(e.model === model && e.providerUrl === providerUrl));
+    if (m.certifications.length !== before) {
+        saveManifest(m, path);
+        return true;
+    }
+    return false;
+}
+export function isStale(entry, currentVersion) {
+    return entry.mmaVersion !== currentVersion;
+}
+export function getCertMark(model, providerUrl, currentVersion, path = MANIFEST_PATH) {
+    const m = readManifest(path);
+    const entry = m.certifications.find((e) => e.model === model && e.providerUrl === providerUrl);
+    if (!entry)
+        return "none";
+    return isStale(entry, currentVersion) ? "stale" : "certified";
+}

package/dist/modules/certification/runner.js

@@ -0,0 +1,162 @@
+import { spawn } from "child_process";
+import { existsSync, mkdirSync, rmSync, cpSync } from "fs";
+import { platform } from "os";
+import { join, resolve, dirname } from "path";
+import { checkSandbox } from "./fact-checker";
+export async function runScenario(scenario, opts) {
+    if (scenario.mode === "skip") {
+        return {
+            id: scenario.id,
+            title: scenario.title,
+            status: "skipped",
+            passed: 0,
+            of: 0,
+        };
+    }
+    const reps = scenario.reps ?? opts.defaultReps;
+    const threshold = Math.min(scenario.passThreshold ?? opts.defaultThreshold, reps);
+    const runner = opts.runner ?? defaultRunner;
+    const timeoutMs = opts.timeoutMs ?? 120_000;
+    const entryPoint = resolveMmaEntry(opts.mmaRoot);
+    let passed = 0;
+    let firstError;
+    for (let i = 1; i <= reps; i++) {
+        const sandbox = join(opts.sandboxBase, `run-${scenario.id}-${i}`);
+        let failures = [];
+        let exitCode = -1;
+        let output = "";
+        try {
+            prepareSandbox(sandbox, scenario, opts.mmaRoot);
+            const args = [
+                entryPoint,
+                "--no-agents-md",
+                "--exit-on-complete",
+                "-d",
+                sandbox,
+                scenario.prompt,
+            ];
+            const env = {
+                ...process.env,
+                MMA_MODEL: opts.model,
+                MMA_PROVIDER_BASEURL: opts.providerUrl,
+                MMA_CONTEXT_WINDOW: String(opts.contextWindow ?? 32000),
+            };
+            if (opts.providerKey)
+                env.MMA_PROVIDER_APIKEY = opts.providerKey;
+            const res = await runner(env, opts.mmaRoot, args, timeoutMs);
+            output = `${res.stdout}\n${res.stderr}`;
+            exitCode = res.code ?? -1;
+            const outcome = checkSandbox(sandbox, scenario.checks, exitCode, output);
+            failures = outcome.failures;
+            if (res.timedOut)
+                failures.push(`rep ${i} timed out`);
+        }
+        catch (e) {
+            return {
+                id: scenario.id,
+                title: scenario.title,
+                status: "error",
+                passed,
+                of: reps,
+                error: e.message,
+            };
+        }
+        const pass = failures.length === 0;
+        if (pass)
+            passed++;
+        else if (!firstError)
+            firstError = failures.join("; ");
+        opts.onRep?.(scenario.id, i, reps, pass, failures);
+    }
+    const status = passed >= threshold ? "pass" : "fail";
+    return {
+        id: scenario.id,
+        title: scenario.title,
+        status,
+        passed,
+        of: reps,
+        error: status === "pass" ? undefined : firstError,
+    };
+}
+function prepareSandbox(sandbox, scenario, mmaRoot) {
+    rmSync(sandbox, { recursive: true, force: true });
+    mkdirSync(sandbox, { recursive: true });
+    for (const f of scenario.fixtures ?? []) {
+        const src = join(mmaRoot, f.source);
+        if (!existsSync(src)) {
+            throw new Error(`fixture missing: ${f.source}`);
+        }
+        const dest = join(sandbox, f.dest);
+        mkdirSync(dirname(dest), { recursive: true });
+        cpSync(src, dest);
+    }
+}
+export function resolveMmaEntry(mmaRoot) {
+    const dev = join(mmaRoot, "src", "cli", "main.ts");
+    if (existsSync(dev))
+        return dev;
+    return join(mmaRoot, "dist", "main.js");
+}
+export function findMmaRoot(fromDir) {
+    const candidates = [
+        resolve(fromDir, "..", "..", ".."),
+        resolve(fromDir, ".."),
+    ];
+    for (const c of candidates) {
+        if (existsSync(join(c, "package.json")))
+            return c;
+    }
+    return process.cwd();
+}
+export const defaultRunner = (env, cwd, args, timeoutMs) => new Promise((resolvePromise) => {
+    const child = spawn(process.execPath, args, {
+        cwd,
+        env,
+        windowsHide: true,
+        stdio: ["ignore", "pipe", "pipe"],
+    });
+    let stdout = "";
+    let stderr = "";
+    let timedOut = false;
+    child.stdout?.on("data", (c) => {
+        stdout += c.toString("utf-8");
+    });
+    child.stderr?.on("data", (c) => {
+        stderr += c.toString("utf-8");
+    });
+    const timer = setTimeout(() => {
+        timedOut = true;
+        killTree(child);
+    }, timeoutMs);
+    child.on("close", (code) => {
+        clearTimeout(timer);
+        resolvePromise({ code, stdout, stderr, timedOut });
+    });
+    child.on("error", () => {
+        clearTimeout(timer);
+        resolvePromise({ code: null, stdout, stderr, timedOut });
+    });
+});
+function killTree(child) {
+    const pid = child.pid;
+    if (!pid)
+        return;
+    if (platform() === "win32") {
+        spawn("taskkill", ["/pid", String(pid), "/T", "/F"], {
+            windowsHide: true,
+            stdio: "ignore",
+        });
+        return;
+    }
+    try {
+        process.kill(-pid, "SIGTERM");
+    }
+    catch {
+        try {
+            child.kill("SIGKILL");
+        }
+        catch {
+            /* already dead */
+        }
+    }
+}