minovative-mind-cli 2.11.5 → 2.13.0

package/dist/services/swebench/gitDiffExtractor.js

@@ -0,0 +1,209 @@
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+const execFileAsync = promisify(execFile);
+/**
+ * Safely executes a git command in the target workspace directory.
+ */
+async function runGit(args, cwd, maxBuffer = 20 * 1024 * 1024) {
+    try {
+        const { stdout, stderr } = await execFileAsync('git', args, {
+            cwd,
+            maxBuffer,
+            encoding: 'utf-8',
+            env: {
+                ...process.env,
+                GIT_PAGER: 'cat',
+                GIT_TERMINAL_PROMPT: '0',
+            },
+        });
+        return { stdout, stderr };
+    }
+    catch (err) {
+        const execErr = err;
+        throw new Error(`Git command failed: git ${args.join(' ')} (in ${cwd})\n${execErr.stderr || execErr.message}`);
+    }
+}
+/**
+ * Checks whether the target directory contains a valid git repository.
+ */
+export async function isGitRepository(workspaceDir) {
+    try {
+        const gitDir = path.join(workspaceDir, '.git');
+        if (!fs.existsSync(gitDir)) {
+            return false;
+        }
+        const { stdout } = await runGit(['rev-parse', '--is-inside-work-tree'], workspaceDir);
+        return stdout.trim() === 'true';
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Gets the current HEAD commit hash of the git repository.
+ */
+export async function getHeadCommit(workspaceDir) {
+    const { stdout } = await runGit(['rev-parse', 'HEAD'], workspaceDir);
+    return stdout.trim();
+}
+/**
+ * Resets the workspace git repository to a clean state matching baseCommit or HEAD.
+ */
+export async function resetWorkspaceRepo(workspaceDir, baseCommit) {
+    const isRepo = await isGitRepository(workspaceDir);
+    if (!isRepo) {
+        throw new Error(`Directory is not a git repository: ${workspaceDir}`);
+    }
+    if (baseCommit) {
+        await runGit(['checkout', baseCommit, '--force'], workspaceDir);
+        await runGit(['reset', '--hard', baseCommit], workspaceDir);
+    }
+    else {
+        await runGit(['reset', '--hard', 'HEAD'], workspaceDir);
+    }
+    // Clean untracked files and directories
+    await runGit(['clean', '-fdx'], workspaceDir);
+}
+/**
+ * Extracts a unified git patch representing all modifications made by the model.
+ * Produces standard SWE-bench format (patch starting with `diff --git` or empty string).
+ */
+export async function extractGitPatch(workspaceDir, options = {}) {
+    const opts = typeof options === 'string' ? { baseCommit: options } : options;
+    const isRepo = await isGitRepository(workspaceDir);
+    if (!isRepo) {
+        throw new Error(`Cannot extract patch: Directory is not a git repository (${workspaceDir})`);
+    }
+    const maxBuffer = opts.maxBuffer ?? 20 * 1024 * 1024;
+    // Include untracked files in the git index as intent-to-add so git diff includes newly created files
+    if (opts.includeUntracked !== false) {
+        try {
+            await runGit(['add', '-N', '.'], workspaceDir, maxBuffer);
+        }
+        catch {
+            // Intent-to-add might fail on locked files or non-standard repos; continue anyway
+        }
+    }
+    let patch = '';
+    if (opts.baseCommit) {
+        try {
+            const { stdout } = await runGit(['diff', '--binary', opts.baseCommit, '--'], workspaceDir, maxBuffer);
+            patch = stdout;
+        }
+        catch {
+            // Fallback without --binary
+            const { stdout } = await runGit(['diff', opts.baseCommit, '--'], workspaceDir, maxBuffer);
+            patch = stdout;
+        }
+    }
+    else {
+        // If no base commit is specified, diff against HEAD
+        const { stdout } = await runGit(['diff', '--binary', 'HEAD', '--'], workspaceDir, maxBuffer);
+        patch = stdout;
+    }
+    return normalizePatch(patch);
+}
+/**
+ * Normalizes git unified diff patch text:
+ * - Trims superfluous leading/trailing whitespace
+ * - Ensures empty diff returns exact empty string ""
+ * - Preserves trailing newline if content exists
+ */
+export function normalizePatch(patch) {
+    if (!patch || patch.trim().length === 0) {
+        return '';
+    }
+    const trimmed = patch.trimEnd();
+    if (trimmed.length === 0) {
+        return '';
+    }
+    return trimmed + '\n';
+}
+/**
+ * Validates whether a patch conforms to standard unified diff structure.
+ */
+export function validatePatchFormat(patch) {
+    if (!patch || patch.trim().length === 0) {
+        return { valid: true };
+    }
+    const issues = [];
+    const hasDiffHeader = /^diff --git a\/.+ b\/.+$/m.test(patch);
+    const hasFileHeader = /^--- (a\/.+|\/dev\/null)$/m.test(patch) && /^\+\+\+ (b\/.+|\/dev\/null)$/m.test(patch);
+    const hasHunkHeader = /^@@ -\d+(,\d+)? \+\d+(,\d+)? @@/m.test(patch);
+    if (!hasDiffHeader) {
+        issues.push('Missing "diff --git a/... b/..." header');
+    }
+    if (!hasFileHeader) {
+        issues.push('Missing "--- a/..." and "+++ b/..." unified diff headers');
+    }
+    if (!hasHunkHeader) {
+        issues.push('Missing "@@ ... @@" hunk header');
+    }
+    return {
+        valid: issues.length === 0,
+        issues: issues.length > 0 ? issues : undefined,
+    };
+}
+/**
+ * Checks out a specific base commit in the repository.
+ */
+export async function checkoutBaseCommit(workspaceDir, baseCommit, options) {
+    const isRepo = await isGitRepository(workspaceDir);
+    if (!isRepo) {
+        return { success: false, error: `Directory ${workspaceDir} is not a valid git repository.` };
+    }
+    try {
+        await runGit(['checkout', '-f', baseCommit], workspaceDir);
+        return { success: true };
+    }
+    catch (err) {
+        if (options?.fetchIfMissing) {
+            try {
+                await runGit(['fetch', 'origin', baseCommit], workspaceDir);
+                await runGit(['checkout', '-f', baseCommit], workspaceDir);
+                return { success: true };
+            }
+            catch (fetchErr) {
+                return { success: false, error: `Failed to checkout base commit ${baseCommit}: ${fetchErr?.message || err?.message}` };
+            }
+        }
+        return { success: false, error: `Failed to checkout base commit ${baseCommit}: ${err?.message || String(err)}` };
+    }
+}
+/**
+ * Clones a repository if it doesn't already exist or ensures it is checked out.
+ */
+export async function cloneOrEnsureRepo(repo, targetDir, options) {
+    const isRepo = await isGitRepository(targetDir);
+    if (isRepo) {
+        if (options?.baseCommit) {
+            const checkoutRes = await checkoutBaseCommit(targetDir, options.baseCommit, { fetchIfMissing: true });
+            if (!checkoutRes.success) {
+                return { success: false, repoDir: targetDir, error: checkoutRes.error };
+            }
+        }
+        return { success: true, repoDir: targetDir };
+    }
+    try {
+        await fs.promises.mkdir(path.dirname(targetDir), { recursive: true });
+        const url = options?.cloneUrl || `https://github.com/${repo}.git`;
+        const cloneArgs = ['clone'];
+        if (options?.shallow) {
+            cloneArgs.push('--depth', '1');
+        }
+        cloneArgs.push(url, targetDir);
+        await runGit(cloneArgs, path.dirname(targetDir));
+        if (options?.baseCommit) {
+            const checkoutRes = await checkoutBaseCommit(targetDir, options.baseCommit, { fetchIfMissing: true });
+            if (!checkoutRes.success) {
+                return { success: false, repoDir: targetDir, error: checkoutRes.error };
+            }
+        }
+        return { success: true, repoDir: targetDir };
+    }
+    catch (err) {
+        return { success: false, repoDir: targetDir, error: `Failed to clone repository ${repo}: ${err?.message || String(err)}` };
+    }
+}

package/dist/services/swebench/index.d.ts

@@ -0,0 +1,4 @@
+export * from './types.js';
+export * from './instanceLoader.js';
+export * from './gitDiffExtractor.js';
+export * from './sweBenchRunnerService.js';

package/dist/services/swebench/index.js

@@ -0,0 +1,4 @@
+export * from './types.js';
+export * from './instanceLoader.js';
+export * from './gitDiffExtractor.js';
+export * from './sweBenchRunnerService.js';

package/dist/services/swebench/instanceLoader.d.ts

@@ -0,0 +1,21 @@
+import { SWEBenchInstance, SWEBenchPrediction, InstanceFilterOptions } from './types.js';
+/**
+ * Loads SWE-bench instances from a JSON or JSONL file.
+ */
+export declare function loadInstancesFromFile(filePath: string, filterOptions?: InstanceFilterOptions): Promise<SWEBenchInstance[]>;
+/**
+ * Applies filtering, pagination, and deterministic shuffling to an array of instances.
+ */
+export declare function filterInstances(instances: SWEBenchInstance[], options?: InstanceFilterOptions): SWEBenchInstance[];
+/**
+ * Parses a SWE-bench predictions JSONL file into an array of predictions.
+ */
+export declare function parsePredictions(predictionsPath: string): Promise<SWEBenchPrediction[]>;
+/**
+ * Saves a list of SWE-bench predictions to a JSONL file.
+ */
+export declare function savePredictions(predictions: SWEBenchPrediction[], outputPath: string): Promise<void>;
+/**
+ * Appends a single prediction object to a JSONL file immediately.
+ */
+export declare function appendPrediction(prediction: SWEBenchPrediction, outputPath: string): Promise<void>;

package/dist/services/swebench/instanceLoader.js

@@ -0,0 +1,171 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import * as readline from 'node:readline';
+/**
+ * Loads SWE-bench instances from a JSON or JSONL file.
+ */
+export async function loadInstancesFromFile(filePath, filterOptions = {}) {
+    const resolvedPath = path.resolve(filePath);
+    if (!fs.existsSync(resolvedPath)) {
+        throw new Error(`Failed to load instances: file not found at ${resolvedPath}`);
+    }
+    const ext = path.extname(resolvedPath).toLowerCase();
+    let instances = [];
+    try {
+        if (ext === '.jsonl') {
+            instances = await loadJsonlInstances(resolvedPath);
+        }
+        else {
+            instances = await loadJsonInstances(resolvedPath);
+        }
+    }
+    catch (err) {
+        throw new Error(`Failed to load instances from ${resolvedPath}: ${err?.message || String(err)}`);
+    }
+    return filterInstances(instances, filterOptions);
+}
+/**
+ * Loads instances from a JSON file.
+ */
+async function loadJsonInstances(filePath) {
+    const content = await fs.promises.readFile(filePath, 'utf-8');
+    const parsed = JSON.parse(content);
+    if (Array.isArray(parsed)) {
+        return parsed;
+    }
+    if (parsed && typeof parsed === 'object') {
+        if (Array.isArray(parsed.instances)) {
+            return parsed.instances;
+        }
+        if (Array.isArray(parsed.data)) {
+            return parsed.data;
+        }
+    }
+    throw new Error(`Unrecognized JSON format in ${filePath}. Expected an array or { instances: [...] }`);
+}
+/**
+ * Loads instances from a JSONL file line-by-line.
+ */
+async function loadJsonlInstances(filePath) {
+    const instances = [];
+    const fileStream = fs.createReadStream(filePath, { encoding: 'utf-8' });
+    const rl = readline.createInterface({
+        input: fileStream,
+        crlfDelay: Infinity,
+    });
+    let lineNum = 0;
+    for await (const line of rl) {
+        lineNum++;
+        const trimmed = line.trim();
+        if (!trimmed || trimmed.startsWith('#') || trimmed.startsWith('//')) {
+            continue;
+        }
+        try {
+            const parsed = JSON.parse(trimmed);
+            if (parsed && typeof parsed === 'object' && parsed.instance_id) {
+                instances.push(parsed);
+            }
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            throw new Error(`Failed to parse line ${lineNum} in JSONL ${filePath}: ${msg}`);
+        }
+    }
+    return instances;
+}
+/**
+ * Applies filtering, pagination, and deterministic shuffling to an array of instances.
+ */
+export function filterInstances(instances, options = {}) {
+    let result = [...instances];
+    if (options.repo) {
+        const targetRepo = options.repo.toLowerCase();
+        result = result.filter((inst) => inst.repo && inst.repo.toLowerCase().includes(targetRepo));
+    }
+    if (options.instanceIds && options.instanceIds.length > 0) {
+        const idSet = new Set(options.instanceIds);
+        result = result.filter((inst) => idSet.has(inst.instance_id));
+    }
+    if (options.shuffle) {
+        result = pseudoShuffle(result, options.seed ?? 42);
+    }
+    if (typeof options.offset === 'number' && options.offset > 0) {
+        result = result.slice(options.offset);
+    }
+    if (typeof options.limit === 'number' && options.limit > 0) {
+        result = result.slice(0, options.limit);
+    }
+    return result;
+}
+/**
+ * Deterministic pseudo-random shuffle (Fisher-Yates) using a simple LCG algorithm.
+ */
+function pseudoShuffle(items, seed) {
+    const array = [...items];
+    let currentSeed = seed;
+    const random = () => {
+        currentSeed = (currentSeed * 9301 + 49297) % 233280;
+        return currentSeed / 233280;
+    };
+    for (let i = array.length - 1; i > 0; i--) {
+        const j = Math.floor(random() * (i + 1));
+        const temp = array[i];
+        array[i] = array[j];
+        array[j] = temp;
+    }
+    return array;
+}
+/**
+ * Parses a SWE-bench predictions JSONL file into an array of predictions.
+ */
+export async function parsePredictions(predictionsPath) {
+    const resolvedPath = path.resolve(predictionsPath);
+    if (!fs.existsSync(resolvedPath)) {
+        return [];
+    }
+    const predictions = [];
+    const fileStream = fs.createReadStream(resolvedPath, { encoding: 'utf-8' });
+    const rl = readline.createInterface({
+        input: fileStream,
+        crlfDelay: Infinity,
+    });
+    for await (const line of rl) {
+        const trimmed = line.trim();
+        if (!trimmed)
+            continue;
+        try {
+            const parsed = JSON.parse(trimmed);
+            if (parsed && parsed.instance_id) {
+                predictions.push(parsed);
+            }
+        }
+        catch {
+            // Skip malformed individual prediction lines
+        }
+    }
+    return predictions;
+}
+/**
+ * Saves a list of SWE-bench predictions to a JSONL file.
+ */
+export async function savePredictions(predictions, outputPath) {
+    const resolvedPath = path.resolve(outputPath);
+    const dir = path.dirname(resolvedPath);
+    if (!fs.existsSync(dir)) {
+        await fs.promises.mkdir(dir, { recursive: true });
+    }
+    const lines = predictions.map((pred) => JSON.stringify(pred)).join('\n') + (predictions.length > 0 ? '\n' : '');
+    await fs.promises.writeFile(resolvedPath, lines, 'utf-8');
+}
+/**
+ * Appends a single prediction object to a JSONL file immediately.
+ */
+export async function appendPrediction(prediction, outputPath) {
+    const resolvedPath = path.resolve(outputPath);
+    const dir = path.dirname(resolvedPath);
+    if (!fs.existsSync(dir)) {
+        await fs.promises.mkdir(dir, { recursive: true });
+    }
+    const line = JSON.stringify(prediction) + '\n';
+    await fs.promises.appendFile(resolvedPath, line, 'utf-8');
+}

package/dist/services/swebench/sweBenchRunnerService.d.ts

@@ -0,0 +1,38 @@
+import { SWEBenchInstance, SWEBenchInstanceResult, SWEBenchEvaluationOptions, SWEBenchEvaluationSummary, SWEBenchEvaluationMetrics, RunInstanceOptions } from './types.js';
+export declare const DEFAULT_MODEL_NAME = "minovative-mind-agent";
+export declare const DEFAULT_TIMEOUT_MS: number;
+/**
+ * Formats a SWE-bench instance into a comprehensive problem prompt for the agent.
+ */
+export declare function buildInstancePrompt(instance: SWEBenchInstance): string;
+/**
+ * Resolves the workspace directory for an instance based on configuration.
+ */
+export declare function resolveInstanceWorkspaceDir(instance: SWEBenchInstance, options?: {
+    workspacesBaseDir?: string;
+    workspaceDirMap?: Record<string, string>;
+    defaultDir?: string;
+    autoCloneRepos?: boolean;
+}): string;
+/**
+ * Executes the mmcli agent loop on a single SWE-bench instance and extracts the resulting patch.
+ */
+export declare function runSWEBenchInstance(instance: SWEBenchInstance, options?: RunInstanceOptions): Promise<SWEBenchInstanceResult>;
+/**
+ * Runs evaluation across a suite of SWE-bench instances.
+ */
+export declare function runSWEBenchEvaluation(options: SWEBenchEvaluationOptions): Promise<SWEBenchEvaluationSummary>;
+/**
+ * Computes aggregate metrics across SWE-bench evaluation results including token usage,
+ * repository pass rates, diff sizes, and timing statistics.
+ */
+export declare function calculateEvaluationMetrics(results: SWEBenchInstanceResult[], totalDurationMs: number, modelName?: string): SWEBenchEvaluationMetrics;
+/**
+ * Saves a structured, public-ready evaluation report JSON file containing summary statistics,
+ * token economics, repository breakdown, and individual instance outcomes.
+ */
+export declare function saveEvaluationReport(summary: SWEBenchEvaluationSummary, reportPath: string): Promise<void>;
+/**
+ * Formats a clean, high-impact ASCII terminal summary dashboard for SWE-bench evaluation runs.
+ */
+export declare function formatEvaluationReportSummary(summary: SWEBenchEvaluationSummary): string;