micro-models-agent 0.55.0 → 0.56.0

package/dist/modules/certification/cli.js

@@ -1,213 +0,0 @@
-import { rmSync, writeFileSync } from "fs";
-import { join, dirname } from "path";
-import { fileURLToPath } from "url";
-import { existsSync, readFileSync } from "fs";
-import { t } from "../../i18n/index";
-import { pc } from "../../ui/colors";
-import { loadScenarios, filterByTags } from "./loader";
-import { runScenario, findMmaRoot } from "./runner";
-import { readManifest, upsertCertification, removeCertification, isFullyPassed } from "./manifest";
-const HERE = dirname(fileURLToPath(import.meta.url));
-const MMA_ROOT = findMmaRoot(HERE);
-function readVersion() {
-    const candidates = [join(MMA_ROOT, "package.json")];
-    for (const p of candidates) {
-        if (existsSync(p)) {
-            try {
-                const raw = JSON.parse(readFileSync(p, "utf-8"));
-                if (raw.version)
-                    return raw.version;
-            }
-            catch {
-                // Broken package.json — fall back to the default version
-            }
-        }
-    }
-    return "0.0.0";
-}
-export function parseTags(s) {
-    return s
-        .split(",")
-        .map((x) => x.trim())
-        .filter(Boolean);
-}
-export async function certify(opts) {
-    const providerUrl = opts.providerUrl || opts.config.provider.baseUrl;
-    const { scenarios, errors } = loadScenarios(join(opts.projectDir, ".mma", "certification", "scenarios"));
-    for (const e of errors)
-        console.error(pc.yellow(`  ${e}`));
-    const selected = filterByTags(scenarios, opts.tags);
-    if (selected.length === 0) {
-        console.error(pc.red(t("cli.cert_no_scenarios", { tags: opts.tags.join(",") })));
-        process.exitCode = 1;
-        return;
-    }
-    const manifest = readManifest(opts.projectDir);
-    const existing = manifest.certifications.find((e) => e.model === opts.name && e.providerUrl === providerUrl);
-    if (existing && !opts.force) {
-        console.error(pc.yellow(t("cli.cert_exists", { model: opts.name })));
-        console.error(pc.yellow(t("cli.cert_exists_hint")));
-        process.exitCode = 1;
-        return;
-    }
-    console.log(t("cli.cert_started", { model: opts.name, provider: providerUrl }));
-    const sandboxBase = join(process.cwd(), ".mma", "certification");
-    const results = [];
-    const total = selected.length;
-    let idx = 0;
-    for (const scenario of selected) {
-        idx++;
-        if (scenario.mode === "skip") {
-            console.log(pc.dim(`[${idx}/${total}] ${scenario.id} ... skipped`));
-            results.push({
-                id: scenario.id,
-                title: scenario.title,
-                status: "skipped",
-                passed: 0,
-                of: 0,
-            });
-            continue;
-        }
-        const res = await runScenario(scenario, {
-            model: opts.name,
-            providerUrl,
-            providerKey: opts.providerKey,
-            contextWindow: opts.contextWindow,
-            mmaRoot: MMA_ROOT,
-            sandboxBase,
-            defaultReps: opts.reps,
-            defaultThreshold: 2,
-            baseConfig: opts.config,
-            onRep: (id, rep, reps, passed, failures) => {
-                const word = passed ? pc.green(t("cli.cert_rep_pass")) : pc.red(t("cli.cert_rep_fail"));
-                console.log(`[${idx}/${total}] ${id} (${rep}/${reps})... ${word}`);
-                if (!passed)
-                    console.log(`    ${pc.dim(failures.join("; "))}`);
-            },
-        });
-        results.push(res);
-    }
-    if (opts.clean) {
-        try {
-            rmSync(sandboxBase, { recursive: true, force: true });
-        }
-        catch {
-            /* ignore */
-        }
-    }
-    const suite = summarize(results);
-    const entry = {
-        model: opts.name,
-        providerUrl,
-        mmaVersion: readVersion(),
-        certifiedAt: new Date().toISOString(),
-        suite,
-        results,
-    };
-    upsertCertification(entry, opts.projectDir);
-    console.log(t("cli.cert_done", {
-        passed: String(suite.passed),
-        failed: String(suite.failed),
-        skipped: String(suite.skipped),
-        total: String(suite.total),
-    }));
-    printResults(results);
-    if (isFullyPassed(entry)) {
-        console.log(pc.green(`\n✔ ${opts.name} is certified`));
-    }
-    else {
-        console.log(pc.yellow(`\n⚠ ${opts.name} is NOT certified — all scenarios must pass`));
-    }
-    // Always write report file
-    writeReport(entry, opts.projectDir);
-}
-export async function certStatus(name, config, projectDir) {
-    const m = readManifest(projectDir);
-    const providerUrl = config.provider.baseUrl;
-    const entry = m.certifications.find((e) => e.model === name && e.providerUrl === providerUrl);
-    if (!entry) {
-        console.log(t("cli.cert_not_found", { model: name }));
-        return;
-    }
-    console.log(`${t("cli.cert_provider_col")}: ${entry.providerUrl}`);
-    console.log(`${t("cli.cert_version_col")}: ${entry.mmaVersion}  ${t("cli.cert_date_col")}: ${entry.certifiedAt.slice(0, 10)}`);
-    console.log(`${t("cli.cert_suite_col")}: ${entry.suite.passed} pass / ${entry.suite.failed} fail / ${entry.suite.skipped} skipped`);
-    const status = isFullyPassed(entry)
-        ? pc.green("✔ certified")
-        : pc.yellow("⚠ NOT certified");
-    console.log(`Status: ${status}`);
-    printResults(entry.results);
-}
-export async function certList(projectDir) {
-    const m = readManifest(projectDir);
-    if (m.certifications.length === 0) {
-        console.log(t("cli.cert_empty"));
-        return;
-    }
-    for (const e of m.certifications) {
-        const mark = isFullyPassed(e) ? pc.green("✔") : pc.red("✘");
-        console.log(`  ${mark} ${e.model}  ${pc.dim(e.providerUrl)}  ${e.mmaVersion}  ${e.certifiedAt.slice(0, 10)}  ${e.suite.passed}/${e.suite.total} pass`);
-    }
-}
-export async function uncertify(name, config, projectDir) {
-    const removed = removeCertification(name, config.provider.baseUrl, projectDir);
-    if (removed)
-        console.log(t("cli.cert_uncertified", { model: name }));
-    else
-        console.log(t("cli.cert_not_found", { model: name }));
-}
-function summarize(results) {
-    return {
-        passed: results.filter((r) => r.status === "pass").length,
-        failed: results.filter((r) => r.status === "fail").length,
-        skipped: results.filter((r) => r.status === "skipped").length,
-        total: results.length,
-    };
-}
-function printResults(results) {
-    for (const r of results) {
-        const icon = r.status === "pass"
-            ? pc.green("✔")
-            : r.status === "fail"
-                ? pc.red("✘")
-                : r.status === "skipped"
-                    ? pc.dim("–")
-                    : pc.yellow("!");
-        const detail = r.status === "skipped" ? pc.dim(r.title) : `${r.passed}/${r.of}`;
-        console.log(`  ${icon} ${r.id}  ${detail}`);
-        if (r.error)
-            console.log(`    ${pc.dim(r.error)}`);
-        if (r.diagnostics)
-            console.log(r.diagnostics);
-    }
-}
-function writeReport(entry, projectDir) {
-    const reportDir = join(projectDir, "certification");
-    const ts = entry.certifiedAt.replace(/[:.]/g, "-").slice(0, 19);
-    const filename = `report-${entry.model.replace(/[/\\:]/g, "_")}-${ts}.json`;
-    const reportPath = join(reportDir, filename);
-    const report = {
-        model: entry.model,
-        providerUrl: entry.providerUrl,
-        mmaVersion: entry.mmaVersion,
-        certifiedAt: entry.certifiedAt,
-        certified: isFullyPassed(entry),
-        suite: entry.suite,
-        results: entry.results.map((r) => ({
-            id: r.id,
-            title: r.title,
-            status: r.status,
-            passed: r.passed,
-            of: r.of,
-            error: r.error,
-            diagnostics: r.diagnostics,
-        })),
-    };
-    try {
-        writeFileSync(reportPath, JSON.stringify(report, null, 2), "utf-8");
-        console.log(pc.dim(`\nReport: ${reportPath}`));
-    }
-    catch (e) {
-        console.error(pc.yellow(`  Failed to write report: ${e.message}`));
-    }
-}

package/dist/modules/certification/fact-checker.js

@@ -1,82 +0,0 @@
-import { existsSync, readFileSync, statSync } from "fs";
-import { join } from "path";
-export function checkSandbox(sandboxDir, checks, exitCode, output) {
-    const failures = [];
-    for (const check of checks) {
-        if (!runCheck(sandboxDir, check, exitCode, output)) {
-            failures.push(describe(check));
-        }
-    }
-    return { pass: failures.length === 0, failures };
-}
-function runCheck(sandboxDir, check, exitCode, output) {
-    switch (check.type) {
-        case "exitCode":
-            return exitCode === (check.code ?? 0);
-        case "outputContains":
-            return output.includes(check.text);
-        case "fileExists":
-            return isFile(join(sandboxDir, check.path));
-        case "fileNotExists":
-            return !existsSync(join(sandboxDir, check.path));
-        case "dirExists":
-            return isDir(join(sandboxDir, check.path));
-        case "fileContent": {
-            const abs = join(sandboxDir, check.path);
-            if (!isFile(abs))
-                return false;
-            const content = readFileSync(abs, "utf-8");
-            if (check.contains !== undefined)
-                return content.includes(check.contains);
-            if (check.equals !== undefined)
-                return content === check.equals;
-            return false;
-        }
-        case "fileRegex": {
-            const abs = join(sandboxDir, check.path);
-            if (!isFile(abs))
-                return false;
-            return new RegExp(check.pattern).test(readFileSync(abs, "utf-8"));
-        }
-        default:
-            return false;
-    }
-}
-function isFile(p) {
-    try {
-        return existsSync(p) && statSync(p).isFile();
-    }
-    catch {
-        return false;
-    }
-}
-function isDir(p) {
-    try {
-        return existsSync(p) && statSync(p).isDirectory();
-    }
-    catch {
-        return false;
-    }
-}
-function describe(check) {
-    switch (check.type) {
-        case "exitCode":
-            return `expected exit code ${check.code ?? 0}`;
-        case "outputContains":
-            return `output missing "${check.text}"`;
-        case "fileExists":
-            return `file missing: ${check.path}`;
-        case "fileNotExists":
-            return `file unexpectedly exists: ${check.path}`;
-        case "dirExists":
-            return `directory missing: ${check.path}`;
-        case "fileContent": {
-            const what = check.contains !== undefined ? `contains "${check.contains}"` : `equals "${check.equals}"`;
-            return `${check.path} does not ${what}`;
-        }
-        case "fileRegex":
-            return `${check.path} does not match ${check.pattern}`;
-        default:
-            return "unknown check";
-    }
-}

package/dist/modules/certification/loader.js

@@ -1,106 +0,0 @@
-import { existsSync, readdirSync, readFileSync } from "fs";
-import { join } from "path";
-import { parse as parseYaml } from "yaml";
-import { BUILTIN_SCENARIOS } from "./scenarios";
-const TAGS = ["core", "security", "image", "network", "browser", "moe"];
-const CHECK_TYPES = [
-    "fileExists",
-    "fileNotExists",
-    "dirExists",
-    "fileContent",
-    "fileRegex",
-    "exitCode",
-    "outputContains",
-];
-export function validateScenario(s) {
-    const errors = [];
-    const isSkip = s.mode === "skip";
-    if (!s.id || !s.title || (!isSkip && !s.prompt)) {
-        errors.push(`[${s.id ?? "?"}] missing id, title or prompt`);
-    }
-    if (!Array.isArray(s.tags) || s.tags.length === 0) {
-        errors.push(`[${s.id}] tags required`);
-    }
-    for (const tag of s.tags) {
-        if (!TAGS.includes(tag))
-            errors.push(`[${s.id}] unknown tag: ${tag}`);
-    }
-    if (s.mode !== "run" && s.mode !== "skip") {
-        errors.push(`[${s.id}] mode must be run or skip`);
-    }
-    if (!Array.isArray(s.checks) || (!isSkip && s.checks.length === 0)) {
-        errors.push(`[${s.id}] at least one check required`);
-    }
-    for (const c of s.checks) {
-        if (!CHECK_TYPES.includes(c.type)) {
-            errors.push(`[${s.id}] unknown check type: ${c.type}`);
-        }
-        if (c.type === "fileContent" && c.contains === undefined && c.equals === undefined) {
-            errors.push(`[${s.id}] fileContent needs contains or equals`);
-        }
-    }
-    if (s.reps !== undefined && (!Number.isInteger(s.reps) || s.reps < 1)) {
-        errors.push(`[${s.id}] reps must be a positive integer`);
-    }
-    if (s.passThreshold !== undefined &&
-        (!Number.isInteger(s.passThreshold) || s.passThreshold < 1)) {
-        errors.push(`[${s.id}] passThreshold must be a positive integer`);
-    }
-    if (s.passThreshold !== undefined && s.reps !== undefined && s.passThreshold > s.reps) {
-        errors.push(`[${s.id}] passThreshold > reps (must be <= reps)`);
-    }
-    return errors;
-}
-export function loadScenarios(userDir) {
-    const errors = [];
-    const scenarios = [];
-    for (const s of BUILTIN_SCENARIOS) {
-        const errs = validateScenario(s);
-        if (errs.length > 0)
-            errors.push(...errs);
-        else
-            scenarios.push(s);
-    }
-    if (userDir && existsSync(userDir)) {
-        for (const file of readdirSync(userDir)) {
-            if (!file.endsWith(".yaml") && !file.endsWith(".yml"))
-                continue;
-            try {
-                const raw = readFileSync(join(userDir, file), "utf-8");
-                const data = parseYaml(raw);
-                const parsed = normalizeScenario(data, file);
-                const errs = validateScenario(parsed);
-                if (errs.length > 0)
-                    errors.push(...errs);
-                else
-                    scenarios.push(parsed);
-            }
-            catch (e) {
-                errors.push(`[${file}] failed to parse: ${e.message}`);
-            }
-        }
-    }
-    return { scenarios, errors };
-}
-function normalizeScenario(data, file) {
-    const d = (data ?? {});
-    return {
-        id: String(d.id ?? file),
-        title: String(d.title ?? d.id ?? file),
-        tags: Array.isArray(d.tags) ? d.tags : [],
-        mode: d.mode === "skip" ? "skip" : "run",
-        prompt: String(d.prompt ?? ""),
-        reps: typeof d.reps === "number" ? d.reps : undefined,
-        passThreshold: typeof d.passThreshold === "number" ? d.passThreshold : undefined,
-        fixtures: Array.isArray(d.fixtures) ? d.fixtures : undefined,
-        checks: Array.isArray(d.checks) ? d.checks : [],
-        skipReason: typeof d.skipReason === "string" ? d.skipReason : undefined,
-        config: typeof d.config === "object" && d.config !== null ? d.config : undefined,
-    };
-}
-export function filterByTags(scenarios, tags) {
-    if (!tags || tags.length === 0) {
-        return scenarios.filter((s) => s.tags.includes("core"));
-    }
-    return scenarios.filter((s) => s.tags.some((t) => tags.includes(t)));
-}

package/dist/modules/certification/manifest.js

@@ -1,58 +0,0 @@
-import { existsSync, readFileSync, mkdirSync, writeFileSync } from "fs";
-import { join } from "path";
-export function manifestPath(projectDir) {
-    return join(projectDir, "certification", "certifications.json");
-}
-export function readManifest(projectDir) {
-    const path = manifestPath(projectDir);
-    try {
-        if (existsSync(path)) {
-            const raw = JSON.parse(readFileSync(path, "utf-8"));
-            return { version: 1, certifications: raw.certifications ?? [] };
-        }
-    }
-    catch {
-        /* corrupted manifest → start fresh */
-    }
-    return { version: 1, certifications: [] };
-}
-export function saveManifest(m, projectDir) {
-    const path = manifestPath(projectDir);
-    mkdirSync(join(projectDir, "certification"), { recursive: true });
-    writeFileSync(path, JSON.stringify(m, null, 2), "utf-8");
-}
-export function upsertCertification(entry, projectDir) {
-    const m = readManifest(projectDir);
-    const idx = m.certifications.findIndex((e) => e.model === entry.model && e.providerUrl === entry.providerUrl);
-    if (idx >= 0)
-        m.certifications[idx] = entry;
-    else
-        m.certifications.push(entry);
-    saveManifest(m, projectDir);
-    return m;
-}
-export function removeCertification(model, providerUrl, projectDir) {
-    const m = readManifest(projectDir);
-    const before = m.certifications.length;
-    m.certifications = m.certifications.filter((e) => !(e.model === model && e.providerUrl === providerUrl));
-    if (m.certifications.length !== before) {
-        saveManifest(m, projectDir);
-        return true;
-    }
-    return false;
-}
-export function isStale(entry, currentVersion) {
-    return entry.mmaVersion !== currentVersion;
-}
-export function isFullyPassed(entry) {
-    return entry.suite.failed === 0 && entry.suite.passed > 0;
-}
-export function getCertMark(model, providerUrl, currentVersion, projectDir) {
-    const m = readManifest(projectDir);
-    const entry = m.certifications.find((e) => e.model === model && e.providerUrl === providerUrl);
-    if (!entry)
-        return "none";
-    if (isStale(entry, currentVersion))
-        return "stale";
-    return isFullyPassed(entry) ? "certified" : "none";
-}