micro-models-agent 0.52.1 → 0.54.0

package/dist/modules/certification/cli.js

@@ -0,0 +1,213 @@
+import { rmSync, writeFileSync } from "fs";
+import { join, dirname } from "path";
+import { fileURLToPath } from "url";
+import { existsSync, readFileSync } from "fs";
+import { t } from "../../i18n/index";
+import { pc } from "../../ui/colors";
+import { loadScenarios, filterByTags } from "./loader";
+import { runScenario, findMmaRoot } from "./runner";
+import { readManifest, upsertCertification, removeCertification, isFullyPassed } from "./manifest";
+const HERE = dirname(fileURLToPath(import.meta.url));
+const MMA_ROOT = findMmaRoot(HERE);
+function readVersion() {
+    const candidates = [join(MMA_ROOT, "package.json")];
+    for (const p of candidates) {
+        if (existsSync(p)) {
+            try {
+                const raw = JSON.parse(readFileSync(p, "utf-8"));
+                if (raw.version)
+                    return raw.version;
+            }
+            catch {
+                // Broken package.json — fall back to the default version
+            }
+        }
+    }
+    return "0.0.0";
+}
+export function parseTags(s) {
+    return s
+        .split(",")
+        .map((x) => x.trim())
+        .filter(Boolean);
+}
+export async function certify(opts) {
+    const providerUrl = opts.providerUrl || opts.config.provider.baseUrl;
+    const { scenarios, errors } = loadScenarios(join(opts.projectDir, ".mma", "certification", "scenarios"));
+    for (const e of errors)
+        console.error(pc.yellow(`  ${e}`));
+    const selected = filterByTags(scenarios, opts.tags);
+    if (selected.length === 0) {
+        console.error(pc.red(t("cli.cert_no_scenarios", { tags: opts.tags.join(",") })));
+        process.exitCode = 1;
+        return;
+    }
+    const manifest = readManifest(opts.projectDir);
+    const existing = manifest.certifications.find((e) => e.model === opts.name && e.providerUrl === providerUrl);
+    if (existing && !opts.force) {
+        console.error(pc.yellow(t("cli.cert_exists", { model: opts.name })));
+        console.error(pc.yellow(t("cli.cert_exists_hint")));
+        process.exitCode = 1;
+        return;
+    }
+    console.log(t("cli.cert_started", { model: opts.name, provider: providerUrl }));
+    const sandboxBase = join(process.cwd(), ".mma", "certification");
+    const results = [];
+    const total = selected.length;
+    let idx = 0;
+    for (const scenario of selected) {
+        idx++;
+        if (scenario.mode === "skip") {
+            console.log(pc.dim(`[${idx}/${total}] ${scenario.id} ... skipped`));
+            results.push({
+                id: scenario.id,
+                title: scenario.title,
+                status: "skipped",
+                passed: 0,
+                of: 0,
+            });
+            continue;
+        }
+        const res = await runScenario(scenario, {
+            model: opts.name,
+            providerUrl,
+            providerKey: opts.providerKey,
+            contextWindow: opts.contextWindow,
+            mmaRoot: MMA_ROOT,
+            sandboxBase,
+            defaultReps: opts.reps,
+            defaultThreshold: 2,
+            baseConfig: opts.config,
+            onRep: (id, rep, reps, passed, failures) => {
+                const word = passed ? pc.green(t("cli.cert_rep_pass")) : pc.red(t("cli.cert_rep_fail"));
+                console.log(`[${idx}/${total}] ${id} (${rep}/${reps})... ${word}`);
+                if (!passed)
+                    console.log(`    ${pc.dim(failures.join("; "))}`);
+            },
+        });
+        results.push(res);
+    }
+    if (opts.clean) {
+        try {
+            rmSync(sandboxBase, { recursive: true, force: true });
+        }
+        catch {
+            /* ignore */
+        }
+    }
+    const suite = summarize(results);
+    const entry = {
+        model: opts.name,
+        providerUrl,
+        mmaVersion: readVersion(),
+        certifiedAt: new Date().toISOString(),
+        suite,
+        results,
+    };
+    upsertCertification(entry, opts.projectDir);
+    console.log(t("cli.cert_done", {
+        passed: String(suite.passed),
+        failed: String(suite.failed),
+        skipped: String(suite.skipped),
+        total: String(suite.total),
+    }));
+    printResults(results);
+    if (isFullyPassed(entry)) {
+        console.log(pc.green(`\n✔ ${opts.name} is certified`));
+    }
+    else {
+        console.log(pc.yellow(`\n⚠ ${opts.name} is NOT certified — all scenarios must pass`));
+    }
+    // Always write report file
+    writeReport(entry, opts.projectDir);
+}
+export async function certStatus(name, config, projectDir) {
+    const m = readManifest(projectDir);
+    const providerUrl = config.provider.baseUrl;
+    const entry = m.certifications.find((e) => e.model === name && e.providerUrl === providerUrl);
+    if (!entry) {
+        console.log(t("cli.cert_not_found", { model: name }));
+        return;
+    }
+    console.log(`${t("cli.cert_provider_col")}: ${entry.providerUrl}`);
+    console.log(`${t("cli.cert_version_col")}: ${entry.mmaVersion}  ${t("cli.cert_date_col")}: ${entry.certifiedAt.slice(0, 10)}`);
+    console.log(`${t("cli.cert_suite_col")}: ${entry.suite.passed} pass / ${entry.suite.failed} fail / ${entry.suite.skipped} skipped`);
+    const status = isFullyPassed(entry)
+        ? pc.green("✔ certified")
+        : pc.yellow("⚠ NOT certified");
+    console.log(`Status: ${status}`);
+    printResults(entry.results);
+}
+export async function certList(projectDir) {
+    const m = readManifest(projectDir);
+    if (m.certifications.length === 0) {
+        console.log(t("cli.cert_empty"));
+        return;
+    }
+    for (const e of m.certifications) {
+        const mark = isFullyPassed(e) ? pc.green("✔") : pc.red("✘");
+        console.log(`  ${mark} ${e.model}  ${pc.dim(e.providerUrl)}  ${e.mmaVersion}  ${e.certifiedAt.slice(0, 10)}  ${e.suite.passed}/${e.suite.total} pass`);
+    }
+}
+export async function uncertify(name, config, projectDir) {
+    const removed = removeCertification(name, config.provider.baseUrl, projectDir);
+    if (removed)
+        console.log(t("cli.cert_uncertified", { model: name }));
+    else
+        console.log(t("cli.cert_not_found", { model: name }));
+}
+function summarize(results) {
+    return {
+        passed: results.filter((r) => r.status === "pass").length,
+        failed: results.filter((r) => r.status === "fail").length,
+        skipped: results.filter((r) => r.status === "skipped").length,
+        total: results.length,
+    };
+}
+function printResults(results) {
+    for (const r of results) {
+        const icon = r.status === "pass"
+            ? pc.green("✔")
+            : r.status === "fail"
+                ? pc.red("✘")
+                : r.status === "skipped"
+                    ? pc.dim("–")
+                    : pc.yellow("!");
+        const detail = r.status === "skipped" ? pc.dim(r.title) : `${r.passed}/${r.of}`;
+        console.log(`  ${icon} ${r.id}  ${detail}`);
+        if (r.error)
+            console.log(`    ${pc.dim(r.error)}`);
+        if (r.diagnostics)
+            console.log(r.diagnostics);
+    }
+}
+function writeReport(entry, projectDir) {
+    const reportDir = join(projectDir, "certification");
+    const ts = entry.certifiedAt.replace(/[:.]/g, "-").slice(0, 19);
+    const filename = `report-${entry.model.replace(/[/\\:]/g, "_")}-${ts}.json`;
+    const reportPath = join(reportDir, filename);
+    const report = {
+        model: entry.model,
+        providerUrl: entry.providerUrl,
+        mmaVersion: entry.mmaVersion,
+        certifiedAt: entry.certifiedAt,
+        certified: isFullyPassed(entry),
+        suite: entry.suite,
+        results: entry.results.map((r) => ({
+            id: r.id,
+            title: r.title,
+            status: r.status,
+            passed: r.passed,
+            of: r.of,
+            error: r.error,
+            diagnostics: r.diagnostics,
+        })),
+    };
+    try {
+        writeFileSync(reportPath, JSON.stringify(report, null, 2), "utf-8");
+        console.log(pc.dim(`\nReport: ${reportPath}`));
+    }
+    catch (e) {
+        console.error(pc.yellow(`  Failed to write report: ${e.message}`));
+    }
+}

package/dist/modules/certification/fact-checker.js

@@ -0,0 +1,82 @@
+import { existsSync, readFileSync, statSync } from "fs";
+import { join } from "path";
+export function checkSandbox(sandboxDir, checks, exitCode, output) {
+    const failures = [];
+    for (const check of checks) {
+        if (!runCheck(sandboxDir, check, exitCode, output)) {
+            failures.push(describe(check));
+        }
+    }
+    return { pass: failures.length === 0, failures };
+}
+function runCheck(sandboxDir, check, exitCode, output) {
+    switch (check.type) {
+        case "exitCode":
+            return exitCode === (check.code ?? 0);
+        case "outputContains":
+            return output.includes(check.text);
+        case "fileExists":
+            return isFile(join(sandboxDir, check.path));
+        case "fileNotExists":
+            return !existsSync(join(sandboxDir, check.path));
+        case "dirExists":
+            return isDir(join(sandboxDir, check.path));
+        case "fileContent": {
+            const abs = join(sandboxDir, check.path);
+            if (!isFile(abs))
+                return false;
+            const content = readFileSync(abs, "utf-8");
+            if (check.contains !== undefined)
+                return content.includes(check.contains);
+            if (check.equals !== undefined)
+                return content === check.equals;
+            return false;
+        }
+        case "fileRegex": {
+            const abs = join(sandboxDir, check.path);
+            if (!isFile(abs))
+                return false;
+            return new RegExp(check.pattern).test(readFileSync(abs, "utf-8"));
+        }
+        default:
+            return false;
+    }
+}
+function isFile(p) {
+    try {
+        return existsSync(p) && statSync(p).isFile();
+    }
+    catch {
+        return false;
+    }
+}
+function isDir(p) {
+    try {
+        return existsSync(p) && statSync(p).isDirectory();
+    }
+    catch {
+        return false;
+    }
+}
+function describe(check) {
+    switch (check.type) {
+        case "exitCode":
+            return `expected exit code ${check.code ?? 0}`;
+        case "outputContains":
+            return `output missing "${check.text}"`;
+        case "fileExists":
+            return `file missing: ${check.path}`;
+        case "fileNotExists":
+            return `file unexpectedly exists: ${check.path}`;
+        case "dirExists":
+            return `directory missing: ${check.path}`;
+        case "fileContent": {
+            const what = check.contains !== undefined ? `contains "${check.contains}"` : `equals "${check.equals}"`;
+            return `${check.path} does not ${what}`;
+        }
+        case "fileRegex":
+            return `${check.path} does not match ${check.pattern}`;
+        default:
+            return "unknown check";
+    }
+}

package/dist/modules/certification/loader.js

@@ -0,0 +1,106 @@
+import { existsSync, readdirSync, readFileSync } from "fs";
+import { join } from "path";
+import { parse as parseYaml } from "yaml";
+import { BUILTIN_SCENARIOS } from "./scenarios";
+const TAGS = ["core", "security", "image", "network", "browser", "moe"];
+const CHECK_TYPES = [
+    "fileExists",
+    "fileNotExists",
+    "dirExists",
+    "fileContent",
+    "fileRegex",
+    "exitCode",
+    "outputContains",
+];
+export function validateScenario(s) {
+    const errors = [];
+    const isSkip = s.mode === "skip";
+    if (!s.id || !s.title || (!isSkip && !s.prompt)) {
+        errors.push(`[${s.id ?? "?"}] missing id, title or prompt`);
+    }
+    if (!Array.isArray(s.tags) || s.tags.length === 0) {
+        errors.push(`[${s.id}] tags required`);
+    }
+    for (const tag of s.tags) {
+        if (!TAGS.includes(tag))
+            errors.push(`[${s.id}] unknown tag: ${tag}`);
+    }
+    if (s.mode !== "run" && s.mode !== "skip") {
+        errors.push(`[${s.id}] mode must be run or skip`);
+    }
+    if (!Array.isArray(s.checks) || (!isSkip && s.checks.length === 0)) {
+        errors.push(`[${s.id}] at least one check required`);
+    }
+    for (const c of s.checks) {
+        if (!CHECK_TYPES.includes(c.type)) {
+            errors.push(`[${s.id}] unknown check type: ${c.type}`);
+        }
+        if (c.type === "fileContent" && c.contains === undefined && c.equals === undefined) {
+            errors.push(`[${s.id}] fileContent needs contains or equals`);
+        }
+    }
+    if (s.reps !== undefined && (!Number.isInteger(s.reps) || s.reps < 1)) {
+        errors.push(`[${s.id}] reps must be a positive integer`);
+    }
+    if (s.passThreshold !== undefined &&
+        (!Number.isInteger(s.passThreshold) || s.passThreshold < 1)) {
+        errors.push(`[${s.id}] passThreshold must be a positive integer`);
+    }
+    if (s.passThreshold !== undefined && s.reps !== undefined && s.passThreshold > s.reps) {
+        errors.push(`[${s.id}] passThreshold > reps (must be <= reps)`);
+    }
+    return errors;
+}
+export function loadScenarios(userDir) {
+    const errors = [];
+    const scenarios = [];
+    for (const s of BUILTIN_SCENARIOS) {
+        const errs = validateScenario(s);
+        if (errs.length > 0)
+            errors.push(...errs);
+        else
+            scenarios.push(s);
+    }
+    if (userDir && existsSync(userDir)) {
+        for (const file of readdirSync(userDir)) {
+            if (!file.endsWith(".yaml") && !file.endsWith(".yml"))
+                continue;
+            try {
+                const raw = readFileSync(join(userDir, file), "utf-8");
+                const data = parseYaml(raw);
+                const parsed = normalizeScenario(data, file);
+                const errs = validateScenario(parsed);
+                if (errs.length > 0)
+                    errors.push(...errs);
+                else
+                    scenarios.push(parsed);
+            }
+            catch (e) {
+                errors.push(`[${file}] failed to parse: ${e.message}`);
+            }
+        }
+    }
+    return { scenarios, errors };
+}
+function normalizeScenario(data, file) {
+    const d = (data ?? {});
+    return {
+        id: String(d.id ?? file),
+        title: String(d.title ?? d.id ?? file),
+        tags: Array.isArray(d.tags) ? d.tags : [],
+        mode: d.mode === "skip" ? "skip" : "run",
+        prompt: String(d.prompt ?? ""),
+        reps: typeof d.reps === "number" ? d.reps : undefined,
+        passThreshold: typeof d.passThreshold === "number" ? d.passThreshold : undefined,
+        fixtures: Array.isArray(d.fixtures) ? d.fixtures : undefined,
+        checks: Array.isArray(d.checks) ? d.checks : [],
+        skipReason: typeof d.skipReason === "string" ? d.skipReason : undefined,
+        config: typeof d.config === "object" && d.config !== null ? d.config : undefined,
+    };
+}
+export function filterByTags(scenarios, tags) {
+    if (!tags || tags.length === 0) {
+        return scenarios.filter((s) => s.tags.includes("core"));
+    }
+    return scenarios.filter((s) => s.tags.some((t) => tags.includes(t)));
+}

package/dist/modules/certification/manifest.js

@@ -0,0 +1,58 @@
+import { existsSync, readFileSync, mkdirSync, writeFileSync } from "fs";
+import { join } from "path";
+export function manifestPath(projectDir) {
+    return join(projectDir, "certification", "certifications.json");
+}
+export function readManifest(projectDir) {
+    const path = manifestPath(projectDir);
+    try {
+        if (existsSync(path)) {
+            const raw = JSON.parse(readFileSync(path, "utf-8"));
+            return { version: 1, certifications: raw.certifications ?? [] };
+        }
+    }
+    catch {
+        /* corrupted manifest → start fresh */
+    }
+    return { version: 1, certifications: [] };
+}
+export function saveManifest(m, projectDir) {
+    const path = manifestPath(projectDir);
+    mkdirSync(join(projectDir, "certification"), { recursive: true });
+    writeFileSync(path, JSON.stringify(m, null, 2), "utf-8");
+}
+export function upsertCertification(entry, projectDir) {
+    const m = readManifest(projectDir);
+    const idx = m.certifications.findIndex((e) => e.model === entry.model && e.providerUrl === entry.providerUrl);
+    if (idx >= 0)
+        m.certifications[idx] = entry;
+    else
+        m.certifications.push(entry);
+    saveManifest(m, projectDir);
+    return m;
+}
+export function removeCertification(model, providerUrl, projectDir) {
+    const m = readManifest(projectDir);
+    const before = m.certifications.length;
+    m.certifications = m.certifications.filter((e) => !(e.model === model && e.providerUrl === providerUrl));
+    if (m.certifications.length !== before) {
+        saveManifest(m, projectDir);
+        return true;
+    }
+    return false;
+}
+export function isStale(entry, currentVersion) {
+    return entry.mmaVersion !== currentVersion;
+}
+export function isFullyPassed(entry) {
+    return entry.suite.failed === 0 && entry.suite.passed > 0;
+}
+export function getCertMark(model, providerUrl, currentVersion, projectDir) {
+    const m = readManifest(projectDir);
+    const entry = m.certifications.find((e) => e.model === model && e.providerUrl === providerUrl);
+    if (!entry)
+        return "none";
+    if (isStale(entry, currentVersion))
+        return "stale";
+    return isFullyPassed(entry) ? "certified" : "none";
+}