@mrsf/cli 0.6.0 → 0.7.0

package/dist/evaluation/reanchor-gates.d.ts

@@ -0,0 +1,29 @@
+export type ReanchorGateMode = "correctness" | "performance";
+export interface ReanchorProfileReport {
+    profile: string;
+    metrics: {
+        passRate: number;
+        incorrectConfidentRelocations: number;
+    };
+    timingMs: {
+        perComment: number;
+        p95: number;
+        max: number;
+        reconciliationMax: number;
+    };
+}
+export interface ReanchorGatePolicy {
+    version: 1;
+    correctness: Record<string, {
+        minimum_pass_rate: number;
+        maximum_incorrect_confident_relocations: number;
+    }>;
+    performance: Record<string, {
+        maximum_ms_per_comment: number;
+        maximum_p95_comment_ms: number;
+        maximum_single_comment_ms: number;
+        maximum_reconciliation_ms: number;
+    }>;
+}
+export declare function evaluateReanchorGate(report: ReanchorProfileReport, policy: ReanchorGatePolicy, mode: ReanchorGateMode): string[];
+//# sourceMappingURL=reanchor-gates.d.ts.map

package/dist/evaluation/reanchor-gates.js

@@ -0,0 +1,47 @@
+export function evaluateReanchorGate(report, policy, mode) {
+    const failures = [];
+    const correctness = policy.correctness[report.profile];
+    if (!correctness) {
+        failures.push(`No correctness gate configured for profile "${report.profile}".`);
+    }
+    else {
+        if (report.metrics.passRate < correctness.minimum_pass_rate) {
+            failures.push(`Pass rate ${report.metrics.passRate.toFixed(4)} is below `
+                + `${correctness.minimum_pass_rate.toFixed(4)}.`);
+        }
+        if (report.metrics.incorrectConfidentRelocations
+            > correctness.maximum_incorrect_confident_relocations) {
+            failures.push(`${report.metrics.incorrectConfidentRelocations} incorrect confident `
+                + `relocations exceed the allowed `
+                + `${correctness.maximum_incorrect_confident_relocations}.`);
+        }
+    }
+    if (mode === "performance") {
+        const performance = policy.performance[report.profile];
+        if (!performance) {
+            failures.push(`No performance gate configured for profile "${report.profile}".`);
+        }
+        else {
+            if (report.timingMs.perComment > performance.maximum_ms_per_comment) {
+                failures.push(`${report.timingMs.perComment.toFixed(3)} ms/comment exceeds `
+                    + `${performance.maximum_ms_per_comment.toFixed(3)} ms/comment.`);
+            }
+            if (report.timingMs.p95 > performance.maximum_p95_comment_ms) {
+                failures.push(`p95 ${report.timingMs.p95.toFixed(3)} ms exceeds `
+                    + `${performance.maximum_p95_comment_ms.toFixed(3)} ms.`);
+            }
+            if (report.timingMs.max > performance.maximum_single_comment_ms) {
+                failures.push(`maximum ${report.timingMs.max.toFixed(3)} ms exceeds `
+                    + `${performance.maximum_single_comment_ms.toFixed(3)} ms.`);
+            }
+            if (report.timingMs.reconciliationMax
+                > performance.maximum_reconciliation_ms) {
+                failures.push(`reconciliation maximum `
+                    + `${report.timingMs.reconciliationMax.toFixed(3)} ms exceeds `
+                    + `${performance.maximum_reconciliation_ms.toFixed(3)} ms.`);
+            }
+        }
+    }
+    return failures;
+}
+//# sourceMappingURL=reanchor-gates.js.map

package/dist/evaluation/run-reanchor-eval.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=run-reanchor-eval.d.ts.map

package/dist/evaluation/run-reanchor-eval.js

@@ -0,0 +1,103 @@
+#!/usr/bin/env node
+import path from "node:path";
+import process from "node:process";
+import { readFile, writeFile } from "node:fs/promises";
+import { createEvaluationBaseline, evaluateCases, findBaselineDifferences, loadEvaluationCases, } from "./reanchor-eval.js";
+async function main() {
+    const options = parseArguments(process.argv.slice(2));
+    const casesPath = path.resolve(options.casesPath);
+    const schemaPath = path.resolve(options.schemaPath);
+    const cases = await loadEvaluationCases(casesPath, schemaPath);
+    const summary = await evaluateCases(cases);
+    const baseline = createEvaluationBaseline(summary);
+    if (options.json) {
+        process.stdout.write(`${JSON.stringify(summary, null, 2)}\n`);
+    }
+    else {
+        printSummary(summary);
+    }
+    if (options.writeBaselinePath) {
+        await writeFile(path.resolve(options.writeBaselinePath), `${JSON.stringify(baseline, null, 2)}\n`);
+        return;
+    }
+    if (options.baselinePath) {
+        const expected = JSON.parse(await readFile(path.resolve(options.baselinePath), "utf8"));
+        const differences = findBaselineDifferences(expected, baseline);
+        if (differences.length > 0) {
+            for (const difference of differences) {
+                console.error(`BASELINE ${difference}`);
+            }
+            process.exitCode = 1;
+        }
+        else if (!options.json) {
+            console.log("Baseline matches.");
+        }
+    }
+    else if (summary.failed > 0) {
+        process.exitCode = 1;
+    }
+}
+function parseArguments(args) {
+    let casesPath = "../evaluation/reanchor/cases";
+    let schemaPath = "../evaluation/reanchor/schema.json";
+    let json = false;
+    let baselinePath;
+    let writeBaselinePath;
+    for (let index = 0; index < args.length; index += 1) {
+        const argument = args[index];
+        if (argument === "--json") {
+            json = true;
+        }
+        else if (argument === "--cases" && args[index + 1]) {
+            casesPath = args[index += 1];
+        }
+        else if (argument === "--schema" && args[index + 1]) {
+            schemaPath = args[index += 1];
+        }
+        else if (argument === "--baseline" && args[index + 1]) {
+            baselinePath = args[index += 1];
+        }
+        else if (argument === "--write-baseline" && args[index + 1]) {
+            writeBaselinePath = args[index += 1];
+        }
+        else {
+            throw new Error(`Unknown or incomplete argument: ${argument}`);
+        }
+    }
+    if (baselinePath && writeBaselinePath) {
+        throw new Error("--baseline and --write-baseline cannot be used together.");
+    }
+    return {
+        casesPath,
+        schemaPath,
+        json,
+        baselinePath,
+        writeBaselinePath,
+    };
+}
+function printSummary(summary) {
+    const passRate = summary.comments === 0
+        ? 0
+        : (summary.passed / summary.comments) * 100;
+    console.log(`Reanchoring evaluation (${summary.algorithm})`);
+    console.log(`Cases: ${summary.cases}`);
+    console.log(`Comments: ${summary.comments}`);
+    console.log(`Passed: ${summary.passed}/${summary.comments} (${passRate.toFixed(1)}%)`);
+    console.log(`Status matches: ${summary.statusMatches}/${summary.comments}`);
+    console.log(`Exact range matches: ${summary.exactRangeMatches}/${summary.comments}`);
+    console.log(`Incorrect confident relocations: ${summary.incorrectConfidentRelocations}`);
+    console.log(`Duration: ${summary.durationMs.toFixed(2)} ms`);
+    console.log(`Per-comment timing: median ${summary.timingMs.median.toFixed(3)} ms, `
+        + `p95 ${summary.timingMs.p95.toFixed(3)} ms, `
+        + `max ${summary.timingMs.max.toFixed(3)} ms`);
+    for (const result of summary.results.filter((item) => !item.passed)) {
+        console.log(`FAIL ${result.caseId}/${result.commentId}: expected ${result.expected.status}, `
+            + `received ${result.actual.status} (${result.actual.reason})`);
+    }
+}
+main().catch((error) => {
+    const message = error instanceof Error ? error.message : String(error);
+    console.error(message);
+    process.exitCode = 1;
+});
+//# sourceMappingURL=run-reanchor-eval.js.map

package/dist/evaluation/run-reanchor-generator.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=run-reanchor-generator.d.ts.map

package/dist/evaluation/run-reanchor-generator.js

@@ -0,0 +1,63 @@
+#!/usr/bin/env node
+import { mkdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+import process from "node:process";
+import { generateEvaluationCases } from "./generate-reanchor-cases.js";
+async function main() {
+    const options = parseArguments(process.argv.slice(2));
+    const outputPath = path.resolve(options.output);
+    const cases = generateEvaluationCases(options);
+    await mkdir(path.dirname(outputPath), { recursive: true });
+    await writeFile(outputPath, `${JSON.stringify(cases, null, 2)}\n`);
+    console.log(`Generated ${cases.length} reanchoring cases at ${outputPath} `
+        + `(seeds ${options.seed}-${options.seed + cases.length - 1}).`);
+}
+function parseArguments(args) {
+    const values = new Map();
+    for (let index = 0; index < args.length; index += 2) {
+        const name = args[index];
+        const value = args[index + 1];
+        if (!name?.startsWith("--") || value == null) {
+            throw new Error(`Unknown or incomplete argument: ${name ?? ""}`);
+        }
+        values.set(name, value);
+    }
+    const supported = new Set([
+        "--seed",
+        "--cases",
+        "--blocks",
+        "--comments",
+        "--mutations",
+        "--output",
+    ]);
+    for (const name of values.keys()) {
+        if (!supported.has(name))
+            throw new Error(`Unknown argument: ${name}`);
+    }
+    const seed = integerArgument(values, "--seed", 1);
+    return {
+        seed,
+        caseCount: integerArgument(values, "--cases", 10),
+        blocksPerCase: integerArgument(values, "--blocks", 12),
+        commentsPerCase: integerArgument(values, "--comments", 4),
+        mutationsPerCase: integerArgument(values, "--mutations", 5),
+        output: values.get("--output")
+            ?? `../evaluation/reanchor/generated/seed-${seed}.json`,
+    };
+}
+function integerArgument(values, name, fallback) {
+    const raw = values.get(name);
+    if (raw == null)
+        return fallback;
+    const value = Number(raw);
+    if (!Number.isSafeInteger(value)) {
+        throw new Error(`${name} must be an integer.`);
+    }
+    return value;
+}
+main().catch((error) => {
+    const message = error instanceof Error ? error.message : String(error);
+    console.error(message);
+    process.exitCode = 1;
+});
+//# sourceMappingURL=run-reanchor-generator.js.map

package/dist/evaluation/run-reanchor-profile.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=run-reanchor-profile.d.ts.map

package/dist/evaluation/run-reanchor-profile.js

@@ -0,0 +1,129 @@
+#!/usr/bin/env node
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import process from "node:process";
+import { generateEvaluationCases } from "./generate-reanchor-cases.js";
+import { evaluateCases } from "./reanchor-eval.js";
+import { evaluateReanchorGate, } from "./reanchor-gates.js";
+async function main() {
+    const options = parseArguments(process.argv.slice(2));
+    const workloads = JSON.parse(await readFile(path.resolve(options.workloadsPath), "utf8"));
+    assertWorkloadFile(workloads);
+    const config = workloads.profiles[options.profile];
+    if (!config) {
+        throw new Error(`Unknown workload profile "${options.profile}". Available profiles: `
+            + `${Object.keys(workloads.profiles).sort().join(", ")}.`);
+    }
+    const generated = generateEvaluationCases({
+        seed: config.seed,
+        caseCount: config.case_count,
+        blocksPerCase: config.blocks_per_case,
+        commentsPerCase: config.comments_per_case,
+        mutationsPerCase: config.mutations_per_case,
+    });
+    const cases = generated.map((value) => ({
+        casePath: `<generated:${value.id}>`,
+        value,
+    }));
+    const summary = await evaluateCases(cases);
+    const report = {
+        profile: options.profile,
+        config,
+        metrics: {
+            cases: summary.cases,
+            comments: summary.comments,
+            passed: summary.passed,
+            failed: summary.failed,
+            passRate: summary.comments > 0
+                ? summary.passed / summary.comments
+                : 0,
+            incorrectConfidentRelocations: summary.incorrectConfidentRelocations,
+        },
+        timingMs: {
+            total: summary.durationMs,
+            ...summary.timingMs,
+        },
+    };
+    let gateFailures = [];
+    if (options.gatePath) {
+        const policy = JSON.parse(await readFile(path.resolve(options.gatePath), "utf8"));
+        gateFailures = evaluateReanchorGate(report, policy, options.gateMode);
+    }
+    if (options.json) {
+        console.log(JSON.stringify(report, null, 2));
+    }
+    else {
+        console.log(`Reanchoring workload: ${options.profile}`);
+        console.log(`${report.metrics.cases} cases, ${report.metrics.comments} comments, `
+            + `${report.metrics.passed} passed, ${report.metrics.failed} failed`);
+        console.log(`${report.metrics.incorrectConfidentRelocations} incorrect confident relocations`);
+        console.log(`Timing: ${report.timingMs.total.toFixed(2)} ms total, `
+            + `${report.timingMs.perComment.toFixed(3)} ms/comment, `
+            + `${report.timingMs.p95.toFixed(3)} ms p95`);
+    }
+    if (gateFailures.length > 0) {
+        throw new Error(`Reanchoring ${options.gateMode} gate failed for ${options.profile}:\n`
+            + gateFailures.map((failure) => `- ${failure}`).join("\n"));
+    }
+}
+function parseArguments(args) {
+    let profile = "small";
+    let workloadsPath = "../evaluation/reanchor/workloads.json";
+    let json = false;
+    let gatePath;
+    let gateMode = "correctness";
+    for (let index = 0; index < args.length; index += 1) {
+        const argument = args[index];
+        if (argument === "--json") {
+            json = true;
+        }
+        else if (argument === "--profile" && args[index + 1]) {
+            profile = args[index += 1];
+        }
+        else if (argument === "--workloads" && args[index + 1]) {
+            workloadsPath = args[index += 1];
+        }
+        else if (argument === "--gate" && args[index + 1]) {
+            gatePath = args[index += 1];
+        }
+        else if (argument === "--gate-mode"
+            && (args[index + 1] === "correctness"
+                || args[index + 1] === "performance")) {
+            gateMode = args[index += 1];
+        }
+        else {
+            throw new Error(`Unknown or incomplete argument: ${argument}`);
+        }
+    }
+    return { profile, workloadsPath, json, gatePath, gateMode };
+}
+function assertWorkloadFile(value) {
+    if (!isRecord(value) || value.version !== 1 || !isRecord(value.profiles)) {
+        throw new Error("Workload file must contain version 1 and a profiles object.");
+    }
+    for (const [name, profile] of Object.entries(value.profiles)) {
+        if (!isRecord(profile)
+            || !isNonNegativeInteger(profile.seed)
+            || !isPositiveInteger(profile.case_count)
+            || !isPositiveInteger(profile.blocks_per_case)
+            || !isPositiveInteger(profile.comments_per_case)
+            || !isPositiveInteger(profile.mutations_per_case)) {
+            throw new Error(`Workload profile "${name}" is invalid.`);
+        }
+    }
+}
+function isRecord(value) {
+    return typeof value === "object" && value != null && !Array.isArray(value);
+}
+function isPositiveInteger(value) {
+    return Number.isSafeInteger(value) && Number(value) > 0;
+}
+function isNonNegativeInteger(value) {
+    return Number.isSafeInteger(value) && Number(value) >= 0;
+}
+main().catch((error) => {
+    const message = error instanceof Error ? error.message : String(error);
+    console.error(message);
+    process.exitCode = 1;
+});
+//# sourceMappingURL=run-reanchor-profile.js.map