micro-models-agent 0.44.0 → 0.45.0

package/dist/modules/certification/cli.js

@@ -1,176 +0,0 @@
-import { rmSync } from "fs";
-import { homedir } from "os";
-import { join, dirname } from "path";
-import { fileURLToPath } from "url";
-import { existsSync, readFileSync } from "fs";
-import { t } from "../../i18n/index";
-import { pc } from "../../ui/colors";
-import { loadScenarios, filterByTags } from "./loader";
-import { runScenario, findMmaRoot } from "./runner";
-import { readManifest, upsertCertification, removeCertification, } from "./manifest";
-const HERE = dirname(fileURLToPath(import.meta.url));
-const MMA_ROOT = findMmaRoot(HERE);
-const USER_SCENARIO_DIR = join(homedir(), ".mma", "certification", "scenarios");
-function readVersion() {
-    const candidates = [join(MMA_ROOT, "package.json")];
-    for (const p of candidates) {
-        if (existsSync(p)) {
-            try {
-                const raw = JSON.parse(readFileSync(p, "utf-8"));
-                if (raw.version)
-                    return raw.version;
-            }
-            catch {
-                // Broken package.json — fall back to the default version
-            }
-        }
-    }
-    return "0.0.0";
-}
-export function parseTags(s) {
-    return s
-        .split(",")
-        .map((x) => x.trim())
-        .filter(Boolean);
-}
-export async function certify(opts) {
-    const providerUrl = opts.providerUrl || opts.config.provider.baseUrl;
-    if (opts.tags.includes("security") && !opts.config.security?.enabled) {
-        console.error(pc.red(t("cli.cert_security_required")));
-        process.exitCode = 1;
-        return;
-    }
-    const { scenarios, errors } = loadScenarios(USER_SCENARIO_DIR);
-    for (const e of errors)
-        console.error(pc.yellow(`  ${e}`));
-    const selected = filterByTags(scenarios, opts.tags);
-    if (selected.length === 0) {
-        console.error(pc.red(t("cli.cert_no_scenarios", { tags: opts.tags.join(",") })));
-        process.exitCode = 1;
-        return;
-    }
-    const manifest = readManifest();
-    const existing = manifest.certifications.find((e) => e.model === opts.name && e.providerUrl === providerUrl);
-    if (existing && !opts.force) {
-        console.error(pc.yellow(t("cli.cert_exists", { model: opts.name })));
-        console.error(pc.yellow(t("cli.cert_exists_hint")));
-        process.exitCode = 1;
-        return;
-    }
-    console.log(t("cli.cert_started", { model: opts.name, provider: providerUrl }));
-    const sandboxBase = join(process.cwd(), ".mma", "certification");
-    const results = [];
-    const total = selected.length;
-    let idx = 0;
-    for (const scenario of selected) {
-        idx++;
-        if (scenario.mode === "skip") {
-            console.log(pc.dim(`[${idx}/${total}] ${scenario.id} ... skipped`));
-            results.push({
-                id: scenario.id,
-                title: scenario.title,
-                status: "skipped",
-                passed: 0,
-                of: 0,
-            });
-            continue;
-        }
-        const res = await runScenario(scenario, {
-            model: opts.name,
-            providerUrl,
-            providerKey: opts.providerKey,
-            contextWindow: opts.contextWindow,
-            mmaRoot: MMA_ROOT,
-            sandboxBase,
-            defaultReps: opts.reps,
-            defaultThreshold: 2,
-            onRep: (id, rep, reps, passed, failures) => {
-                const word = passed
-                    ? pc.green(t("cli.cert_rep_pass"))
-                    : pc.red(t("cli.cert_rep_fail"));
-                console.log(`[${idx}/${total}] ${id} (${rep}/${reps})... ${word}`);
-                if (!passed)
-                    console.log(`    ${pc.dim(failures.join("; "))}`);
-            },
-        });
-        results.push(res);
-    }
-    if (opts.clean) {
-        try {
-            rmSync(sandboxBase, { recursive: true, force: true });
-        }
-        catch {
-            /* ignore */
-        }
-    }
-    const suite = summarize(results);
-    const entry = {
-        model: opts.name,
-        providerUrl,
-        mmaVersion: readVersion(),
-        certifiedAt: new Date().toISOString(),
-        suite,
-        results,
-    };
-    upsertCertification(entry);
-    console.log(t("cli.cert_done", {
-        passed: String(suite.passed),
-        failed: String(suite.failed),
-        skipped: String(suite.skipped),
-        total: String(suite.total),
-    }));
-    printResults(results);
-}
-export async function certStatus(name, config) {
-    const m = readManifest();
-    const providerUrl = config.provider.baseUrl;
-    const entry = m.certifications.find((e) => e.model === name && e.providerUrl === providerUrl);
-    if (!entry) {
-        console.log(t("cli.cert_not_found", { model: name }));
-        return;
-    }
-    console.log(`${t("cli.cert_provider_col")}: ${entry.providerUrl}`);
-    console.log(`${t("cli.cert_version_col")}: ${entry.mmaVersion}  ${t("cli.cert_date_col")}: ${entry.certifiedAt.slice(0, 10)}`);
-    console.log(`${t("cli.cert_suite_col")}: ${entry.suite.passed} pass / ${entry.suite.failed} fail / ${entry.suite.skipped} skipped`);
-    printResults(entry.results);
-}
-export async function certList() {
-    const m = readManifest();
-    if (m.certifications.length === 0) {
-        console.log(t("cli.cert_empty"));
-        return;
-    }
-    for (const e of m.certifications) {
-        console.log(`  ${pc.green("✔")} ${e.model}  ${pc.dim(e.providerUrl)}  ${e.mmaVersion}  ${e.certifiedAt.slice(0, 10)}  ${e.suite.passed}/${e.suite.total} pass`);
-    }
-}
-export async function uncertify(name, config) {
-    const removed = removeCertification(name, config.provider.baseUrl);
-    if (removed)
-        console.log(t("cli.cert_uncertified", { model: name }));
-    else
-        console.log(t("cli.cert_not_found", { model: name }));
-}
-function summarize(results) {
-    return {
-        passed: results.filter((r) => r.status === "pass").length,
-        failed: results.filter((r) => r.status === "fail").length,
-        skipped: results.filter((r) => r.status === "skipped").length,
-        total: results.length,
-    };
-}
-function printResults(results) {
-    for (const r of results) {
-        const icon = r.status === "pass"
-            ? pc.green("✔")
-            : r.status === "fail"
-                ? pc.red("✘")
-                : r.status === "skipped"
-                    ? pc.dim("–")
-                    : pc.yellow("!");
-        const detail = r.status === "skipped" ? pc.dim(r.title) : `${r.passed}/${r.of}`;
-        console.log(`  ${icon} ${r.id}  ${detail}`);
-        if (r.error)
-            console.log(`    ${pc.dim(r.error)}`);
-    }
-}

package/dist/modules/certification/fact-checker.js

@@ -1,84 +0,0 @@
-import { existsSync, readFileSync, statSync } from "fs";
-import { join } from "path";
-export function checkSandbox(sandboxDir, checks, exitCode, output) {
-    const failures = [];
-    for (const check of checks) {
-        if (!runCheck(sandboxDir, check, exitCode, output)) {
-            failures.push(describe(check));
-        }
-    }
-    return { pass: failures.length === 0, failures };
-}
-function runCheck(sandboxDir, check, exitCode, output) {
-    switch (check.type) {
-        case "exitCode":
-            return exitCode === (check.code ?? 0);
-        case "outputContains":
-            return output.includes(check.text);
-        case "fileExists":
-            return isFile(join(sandboxDir, check.path));
-        case "fileNotExists":
-            return !existsSync(join(sandboxDir, check.path));
-        case "dirExists":
-            return isDir(join(sandboxDir, check.path));
-        case "fileContent": {
-            const abs = join(sandboxDir, check.path);
-            if (!isFile(abs))
-                return false;
-            const content = readFileSync(abs, "utf-8");
-            if (check.contains !== undefined)
-                return content.includes(check.contains);
-            if (check.equals !== undefined)
-                return content === check.equals;
-            return false;
-        }
-        case "fileRegex": {
-            const abs = join(sandboxDir, check.path);
-            if (!isFile(abs))
-                return false;
-            return new RegExp(check.pattern).test(readFileSync(abs, "utf-8"));
-        }
-        default:
-            return false;
-    }
-}
-function isFile(p) {
-    try {
-        return existsSync(p) && statSync(p).isFile();
-    }
-    catch {
-        return false;
-    }
-}
-function isDir(p) {
-    try {
-        return existsSync(p) && statSync(p).isDirectory();
-    }
-    catch {
-        return false;
-    }
-}
-function describe(check) {
-    switch (check.type) {
-        case "exitCode":
-            return `expected exit code ${check.code ?? 0}`;
-        case "outputContains":
-            return `output missing "${check.text}"`;
-        case "fileExists":
-            return `file missing: ${check.path}`;
-        case "fileNotExists":
-            return `file unexpectedly exists: ${check.path}`;
-        case "dirExists":
-            return `directory missing: ${check.path}`;
-        case "fileContent": {
-            const what = check.contains !== undefined
-                ? `contains "${check.contains}"`
-                : `equals "${check.equals}"`;
-            return `${check.path} does not ${what}`;
-        }
-        case "fileRegex":
-            return `${check.path} does not match ${check.pattern}`;
-        default:
-            return "unknown check";
-    }
-}

package/dist/modules/certification/loader.js

@@ -1,111 +0,0 @@
-import { existsSync, readdirSync, readFileSync } from "fs";
-import { join } from "path";
-import { parse as parseYaml } from "yaml";
-import { BUILTIN_SCENARIOS } from "./scenarios";
-const TAGS = ["core", "security", "image", "network", "browser"];
-const CHECK_TYPES = [
-    "fileExists",
-    "fileNotExists",
-    "dirExists",
-    "fileContent",
-    "fileRegex",
-    "exitCode",
-    "outputContains",
-];
-export function validateScenario(s) {
-    const errors = [];
-    const isSkip = s.mode === "skip";
-    if (!s.id || !s.title || (!isSkip && !s.prompt)) {
-        errors.push(`[${s.id ?? "?"}] missing id, title or prompt`);
-    }
-    if (!Array.isArray(s.tags) || s.tags.length === 0) {
-        errors.push(`[${s.id}] tags required`);
-    }
-    for (const tag of s.tags) {
-        if (!TAGS.includes(tag))
-            errors.push(`[${s.id}] unknown tag: ${tag}`);
-    }
-    if (s.mode !== "run" && s.mode !== "skip") {
-        errors.push(`[${s.id}] mode must be run or skip`);
-    }
-    if (!Array.isArray(s.checks) || (!isSkip && s.checks.length === 0)) {
-        errors.push(`[${s.id}] at least one check required`);
-    }
-    for (const c of s.checks) {
-        if (!CHECK_TYPES.includes(c.type)) {
-            errors.push(`[${s.id}] unknown check type: ${c.type}`);
-        }
-        if (c.type === "fileContent" &&
-            c.contains === undefined &&
-            c.equals === undefined) {
-            errors.push(`[${s.id}] fileContent needs contains or equals`);
-        }
-    }
-    if (s.reps !== undefined && (!Number.isInteger(s.reps) || s.reps < 1)) {
-        errors.push(`[${s.id}] reps must be a positive integer`);
-    }
-    if (s.passThreshold !== undefined &&
-        (!Number.isInteger(s.passThreshold) || s.passThreshold < 1)) {
-        errors.push(`[${s.id}] passThreshold must be a positive integer`);
-    }
-    if (s.passThreshold !== undefined &&
-        s.reps !== undefined &&
-        s.passThreshold > s.reps) {
-        errors.push(`[${s.id}] passThreshold > reps (must be <= reps)`);
-    }
-    return errors;
-}
-export function loadScenarios(userDir) {
-    const errors = [];
-    const scenarios = [];
-    for (const s of BUILTIN_SCENARIOS) {
-        const errs = validateScenario(s);
-        if (errs.length > 0)
-            errors.push(...errs);
-        else
-            scenarios.push(s);
-    }
-    if (userDir && existsSync(userDir)) {
-        for (const file of readdirSync(userDir)) {
-            if (!file.endsWith(".yaml") && !file.endsWith(".yml"))
-                continue;
-            try {
-                const raw = readFileSync(join(userDir, file), "utf-8");
-                const data = parseYaml(raw);
-                const parsed = normalizeScenario(data, file);
-                const errs = validateScenario(parsed);
-                if (errs.length > 0)
-                    errors.push(...errs);
-                else
-                    scenarios.push(parsed);
-            }
-            catch (e) {
-                errors.push(`[${file}] failed to parse: ${e.message}`);
-            }
-        }
-    }
-    return { scenarios, errors };
-}
-function normalizeScenario(data, file) {
-    const d = (data ?? {});
-    return {
-        id: String(d.id ?? file),
-        title: String(d.title ?? d.id ?? file),
-        tags: Array.isArray(d.tags) ? d.tags : [],
-        mode: d.mode === "skip" ? "skip" : "run",
-        prompt: String(d.prompt ?? ""),
-        reps: typeof d.reps === "number" ? d.reps : undefined,
-        passThreshold: typeof d.passThreshold === "number" ? d.passThreshold : undefined,
-        fixtures: Array.isArray(d.fixtures)
-            ? d.fixtures
-            : undefined,
-        checks: Array.isArray(d.checks) ? d.checks : [],
-        skipReason: typeof d.skipReason === "string" ? d.skipReason : undefined,
-    };
-}
-export function filterByTags(scenarios, tags) {
-    if (!tags || tags.length === 0) {
-        return scenarios.filter((s) => s.tags.includes("core"));
-    }
-    return scenarios.filter((s) => s.tags.some((t) => tags.includes(t)));
-}

package/dist/modules/certification/manifest.js

@@ -1,50 +0,0 @@
-import { existsSync, readFileSync, mkdirSync, writeFileSync } from "fs";
-import { homedir } from "os";
-import { join } from "path";
-export const MANIFEST_PATH = join(homedir(), ".mma", "certifications.json");
-export function readManifest(path = MANIFEST_PATH) {
-    try {
-        if (existsSync(path)) {
-            const raw = JSON.parse(readFileSync(path, "utf-8"));
-            return { version: 1, certifications: raw.certifications ?? [] };
-        }
-    }
-    catch {
-        /* corrupted manifest → start fresh */
-    }
-    return { version: 1, certifications: [] };
-}
-export function saveManifest(m, path = MANIFEST_PATH) {
-    mkdirSync(join(homedir(), ".mma"), { recursive: true });
-    writeFileSync(path, JSON.stringify(m, null, 2), "utf-8");
-}
-export function upsertCertification(entry, path = MANIFEST_PATH) {
-    const m = readManifest(path);
-    const idx = m.certifications.findIndex((e) => e.model === entry.model && e.providerUrl === entry.providerUrl);
-    if (idx >= 0)
-        m.certifications[idx] = entry;
-    else
-        m.certifications.push(entry);
-    saveManifest(m, path);
-    return m;
-}
-export function removeCertification(model, providerUrl, path = MANIFEST_PATH) {
-    const m = readManifest(path);
-    const before = m.certifications.length;
-    m.certifications = m.certifications.filter((e) => !(e.model === model && e.providerUrl === providerUrl));
-    if (m.certifications.length !== before) {
-        saveManifest(m, path);
-        return true;
-    }
-    return false;
-}
-export function isStale(entry, currentVersion) {
-    return entry.mmaVersion !== currentVersion;
-}
-export function getCertMark(model, providerUrl, currentVersion, path = MANIFEST_PATH) {
-    const m = readManifest(path);
-    const entry = m.certifications.find((e) => e.model === model && e.providerUrl === providerUrl);
-    if (!entry)
-        return "none";
-    return isStale(entry, currentVersion) ? "stale" : "certified";
-}

package/dist/modules/certification/runner.js

@@ -1,162 +0,0 @@
-import { spawn } from "child_process";
-import { existsSync, mkdirSync, rmSync, cpSync } from "fs";
-import { platform } from "os";
-import { join, resolve, dirname } from "path";
-import { checkSandbox } from "./fact-checker";
-export async function runScenario(scenario, opts) {
-    if (scenario.mode === "skip") {
-        return {
-            id: scenario.id,
-            title: scenario.title,
-            status: "skipped",
-            passed: 0,
-            of: 0,
-        };
-    }
-    const reps = scenario.reps ?? opts.defaultReps;
-    const threshold = Math.min(scenario.passThreshold ?? opts.defaultThreshold, reps);
-    const runner = opts.runner ?? defaultRunner;
-    const timeoutMs = opts.timeoutMs ?? 120_000;
-    const entryPoint = resolveMmaEntry(opts.mmaRoot);
-    let passed = 0;
-    let firstError;
-    for (let i = 1; i <= reps; i++) {
-        const sandbox = join(opts.sandboxBase, `run-${scenario.id}-${i}`);
-        let failures = [];
-        let exitCode = -1;
-        let output = "";
-        try {
-            prepareSandbox(sandbox, scenario, opts.mmaRoot);
-            const args = [
-                entryPoint,
-                "--no-agents-md",
-                "--exit-on-complete",
-                "-d",
-                sandbox,
-                scenario.prompt,
-            ];
-            const env = {
-                ...process.env,
-                MMA_MODEL: opts.model,
-                MMA_PROVIDER_BASEURL: opts.providerUrl,
-                MMA_CONTEXT_WINDOW: String(opts.contextWindow ?? 32000),
-            };
-            if (opts.providerKey)
-                env.MMA_PROVIDER_APIKEY = opts.providerKey;
-            const res = await runner(env, opts.mmaRoot, args, timeoutMs);
-            output = `${res.stdout}\n${res.stderr}`;
-            exitCode = res.code ?? -1;
-            const outcome = checkSandbox(sandbox, scenario.checks, exitCode, output);
-            failures = outcome.failures;
-            if (res.timedOut)
-                failures.push(`rep ${i} timed out`);
-        }
-        catch (e) {
-            return {
-                id: scenario.id,
-                title: scenario.title,
-                status: "error",
-                passed,
-                of: reps,
-                error: e.message,
-            };
-        }
-        const pass = failures.length === 0;
-        if (pass)
-            passed++;
-        else if (!firstError)
-            firstError = failures.join("; ");
-        opts.onRep?.(scenario.id, i, reps, pass, failures);
-    }
-    const status = passed >= threshold ? "pass" : "fail";
-    return {
-        id: scenario.id,
-        title: scenario.title,
-        status,
-        passed,
-        of: reps,
-        error: status === "pass" ? undefined : firstError,
-    };
-}
-function prepareSandbox(sandbox, scenario, mmaRoot) {
-    rmSync(sandbox, { recursive: true, force: true });
-    mkdirSync(sandbox, { recursive: true });
-    for (const f of scenario.fixtures ?? []) {
-        const src = join(mmaRoot, f.source);
-        if (!existsSync(src)) {
-            throw new Error(`fixture missing: ${f.source}`);
-        }
-        const dest = join(sandbox, f.dest);
-        mkdirSync(dirname(dest), { recursive: true });
-        cpSync(src, dest);
-    }
-}
-export function resolveMmaEntry(mmaRoot) {
-    const dev = join(mmaRoot, "src", "cli", "main.ts");
-    if (existsSync(dev))
-        return dev;
-    return join(mmaRoot, "dist", "main.js");
-}
-export function findMmaRoot(fromDir) {
-    const candidates = [
-        resolve(fromDir, "..", "..", ".."),
-        resolve(fromDir, ".."),
-    ];
-    for (const c of candidates) {
-        if (existsSync(join(c, "package.json")))
-            return c;
-    }
-    return process.cwd();
-}
-export const defaultRunner = (env, cwd, args, timeoutMs) => new Promise((resolvePromise) => {
-    const child = spawn(process.execPath, args, {
-        cwd,
-        env,
-        windowsHide: true,
-        stdio: ["ignore", "pipe", "pipe"],
-    });
-    let stdout = "";
-    let stderr = "";
-    let timedOut = false;
-    child.stdout?.on("data", (c) => {
-        stdout += c.toString("utf-8");
-    });
-    child.stderr?.on("data", (c) => {
-        stderr += c.toString("utf-8");
-    });
-    const timer = setTimeout(() => {
-        timedOut = true;
-        killTree(child);
-    }, timeoutMs);
-    child.on("close", (code) => {
-        clearTimeout(timer);
-        resolvePromise({ code, stdout, stderr, timedOut });
-    });
-    child.on("error", () => {
-        clearTimeout(timer);
-        resolvePromise({ code: null, stdout, stderr, timedOut });
-    });
-});
-function killTree(child) {
-    const pid = child.pid;
-    if (!pid)
-        return;
-    if (platform() === "win32") {
-        spawn("taskkill", ["/pid", String(pid), "/T", "/F"], {
-            windowsHide: true,
-            stdio: "ignore",
-        });
-        return;
-    }
-    try {
-        process.kill(-pid, "SIGTERM");
-    }
-    catch {
-        try {
-            child.kill("SIGKILL");
-        }
-        catch {
-            /* already dead */
-        }
-    }
-}