@mirnoorata/codexa 0.2.0

package/dist/eval/baseline.js

@@ -0,0 +1,146 @@
+import { execFileSync } from "node:child_process";
+import path from "node:path";
+function runBaseline(command, cwd) {
+    assertAllowedBaseline(command);
+    if (command[0] === "rg") {
+        return runRipgrepBaseline(command.slice(1), cwd);
+    }
+    return runBaselineCommand(command, cwd);
+}
+export function runScenarioBaselines(scenario) {
+    const commands = scenario.baselineCommands?.length ? scenario.baselineCommands : scenario.baselineCommand ? [scenario.baselineCommand] : [];
+    if (commands.length === 0) {
+        return null;
+    }
+    const cwd = scenario.baselineCwd ?? scenario.repoRoot;
+    return commands.map((command) => ({ command, output: runBaseline(command, cwd) }));
+}
+function assertAllowedBaseline(command) {
+    const executable = command[0];
+    for (const arg of command) {
+        if (isUnsafeBaselineArgument(arg)) {
+            throw new Error(`baseline command contains unsafe argument: ${arg}`);
+        }
+    }
+    if (executable === "rg" && isAllowedRipgrepBaseline(command.slice(1))) {
+        return;
+    }
+    if (isAllowedGitStatusBaseline(command) || isAllowedGitGrepBaseline(command)) {
+        return;
+    }
+    throw new Error(`unsupported baseline executable: ${executable}`);
+}
+function isAllowedGitStatusBaseline(command) {
+    return command[0] === "git" && command[1] === "status" && command.length === 3 && (command[2] === "--short" || command[2] === "--porcelain");
+}
+function isAllowedGitGrepBaseline(command) {
+    if (command[0] !== "git" || command[1] !== "grep") {
+        return false;
+    }
+    const allowedFlags = new Set(["-n", "--line-number", "-E", "-F", "-e", "-m", "--"]);
+    for (let i = 2; i < command.length; i += 1) {
+        const arg = command[i];
+        if (command[i - 1] === "-e" || command[i - 1] === "-m") {
+            continue;
+        }
+        if (arg.startsWith("-") && !allowedFlags.has(arg)) {
+            return false;
+        }
+    }
+    return true;
+}
+function isAllowedRipgrepBaseline(args) {
+    return parseRipgrepBaselineArgs(args) !== undefined;
+}
+function isUnsafeBaselineArgument(value) {
+    if (value === ".") {
+        return false;
+    }
+    return path.isAbsolute(value) || value === ".codex" || value.includes(".codex/") || value.includes(".codex\\") || value.includes("../") || value.includes("..\\");
+}
+function runRipgrepBaseline(args, cwd) {
+    try {
+        return execFileSync("rg", args, {
+            cwd,
+            encoding: "utf8",
+            timeout: 10_000,
+            maxBuffer: 2 * 1024 * 1024,
+            stdio: ["ignore", "pipe", "pipe"]
+        });
+    }
+    catch (error) {
+        if (isMissingExecutable(error)) {
+            return runGitGrepBaseline(args, cwd);
+        }
+        return handleBaselineError(error);
+    }
+}
+function runGitGrepBaseline(rgArgs, cwd) {
+    const parsed = parseRipgrepBaselineArgs(rgArgs);
+    if (!parsed) {
+        return "";
+    }
+    return runBaselineCommand(["git", "grep", "-n", "-E", "-m", "25", "-e", parsed.pattern, "--", ...parsed.paths, ":(exclude).codex/**"], cwd);
+}
+function parseRipgrepBaselineArgs(args) {
+    let pattern;
+    const paths = [];
+    for (let i = 0; i < args.length; i += 1) {
+        const arg = args[i];
+        if (arg === "-n" || arg === "--line-number") {
+            continue;
+        }
+        if (arg === "--") {
+            continue;
+        }
+        if (arg === "-e") {
+            if (!args[i + 1]) {
+                return undefined;
+            }
+            pattern = args[i + 1];
+            i += 1;
+            continue;
+        }
+        if (arg.startsWith("-")) {
+            return undefined;
+        }
+        if (!pattern) {
+            pattern = arg;
+            continue;
+        }
+        paths.push(arg);
+    }
+    return pattern ? { pattern, paths: paths.length > 0 ? paths : ["."] } : undefined;
+}
+function runBaselineCommand(command, cwd) {
+    try {
+        return execFileSync(command[0], command.slice(1), {
+            cwd,
+            encoding: "utf8",
+            timeout: 10_000,
+            maxBuffer: 2 * 1024 * 1024,
+            stdio: ["ignore", "pipe", "pipe"]
+        });
+    }
+    catch (error) {
+        return handleBaselineError(error);
+    }
+}
+function handleBaselineError(error) {
+    const status = error && typeof error === "object" && "status" in error ? Number(error.status) : undefined;
+    if (status !== undefined && status > 1) {
+        throw error;
+    }
+    if (error && typeof error === "object" && "stdout" in error) {
+        return String(error.stdout ?? "");
+    }
+    return "";
+}
+function isMissingExecutable(error) {
+    return Boolean(error && typeof error === "object" && "code" in error && error.code === "ENOENT");
+}
+export function formatBaselineCommands(scenario) {
+    const commands = scenario.baselineCommands?.length ? scenario.baselineCommands : scenario.baselineCommand ? [scenario.baselineCommand] : [];
+    return commands.length > 0 ? commands.map((command) => command.join(" ")).join(" && ") : "unknown";
+}
+//# sourceMappingURL=baseline.js.map

package/dist/eval/baseline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"baseline.js","sourceRoot":"","sources":["../../src/eval/baseline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,IAAI,MAAM,WAAW,CAAC;AAQ7B,SAAS,WAAW,CAAC,OAAiB,EAAE,GAAW;IACjD,qBAAqB,CAAC,OAAO,CAAC,CAAC;IAC/B,IAAI,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACxB,OAAO,kBAAkB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACnD,CAAC;IACD,OAAO,kBAAkB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;AAC1C,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,QAAsB;IACzD,MAAM,QAAQ,GAAG,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5I,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,GAAG,GAAG,QAAQ,CAAC,WAAW,IAAI,QAAQ,CAAC,QAAQ,CAAC;IACtD,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;AACrF,CAAC;AAED,SAAS,qBAAqB,CAAC,OAAiB;IAC9C,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IAC9B,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,IAAI,wBAAwB,CAAC,GAAG,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,8CAA8C,GAAG,EAAE,CAAC,CAAC;QACvE,CAAC;IACH,CAAC;IACD,IAAI,UAAU,KAAK,IAAI,IAAI,wBAAwB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACtE,OAAO;IACT,CAAC;IACD,IAAI,0BAA0B,CAAC,OAAO,CAAC,IAAI,wBAAwB,CAAC,OAAO,CAAC,EAAE,CAAC;QAC7E,OAAO;IACT,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,oCAAoC,UAAU,EAAE,CAAC,CAAC;AACpE,CAAC;AAED,SAAS,0BAA0B,CAAC,OAAiB;IACnD,OAAO,OAAO,CAAC,CAAC,CAAC,KAAK,KAAK,IAAI,OAAO,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,OAAO,CAAC,CAAC,CAAC,KAAK,aAAa,CAAC,CAAC;AAC/I,CAAC;AAED,SAAS,wBAAwB,CAAC,OAAiB;IACjD,IAAI,OAAO,CAAC,CAAC,CAAC,KAAK,KAAK,IAAI,OAAO,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE,CAAC;QAClD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;IACpF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3C,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACvB,IAAI,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,IAAI,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACvD,SAAS;QACX,CAAC;QACD,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAClD,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,wBAAwB,CAAC,IAAc;IAC9C,OAAO,wBAAwB,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC;AACtD,CAAC;AAED,SAAS,wBAAwB,CAAC,KAAa;IAC7C,IAAI,KAAK,KAAK,GAAG,EAAE,CAAC;QAClB,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;AACpK,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAc,EAAE,GAAW;IACrD,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE;YAC9B,GAAG;YACH,QAAQ,EAAE,MAAM;YAChB,OAAO,EAAE,MAAM;YACf,SAAS,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI;YAC1B,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;SAClC,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,mBAAmB,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,kBAAkB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACvC,CAAC;QACD,OAAO,mBAAmB,CAAC,KAAK,CAAC,CAAC;IACpC,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,MAAgB,EAAE,GAAW;IACvD,MAAM,MAAM,GAAG,wBAAwB,CAAC,MAAM,CAAC,CAAC;IAChD,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,OAAO,kBAAkB,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,OAAO,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC,KAAK,EAAE,qBAAqB,CAAC,EAAE,GAAG,CAAC,CAAC;AAC9I,CAAC;AAED,SAAS,wBAAwB,CAAC,IAAc;IAC9C,IAAI,OAA2B,CAAC;IAChC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACxC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,eAAe,EAAE,CAAC;YAC5C,SAAS;QACX,CAAC;QACD,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YACjB,SAAS;QACX,CAAC;QACD,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YACjB,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBACjB,OAAO,SAAS,CAAC;YACnB,CAAC;YACD,OAAO,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACtB,CAAC,IAAI,CAAC,CAAC;YACP,SAAS;QACX,CAAC;QACD,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACxB,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,GAAG,GAAG,CAAC;YACd,SAAS;QACX,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAClB,CAAC;IACD,OAAO,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;AACpF,CAAC;AAED,SAAS,kBAAkB,CAAC,OAAiB,EAAE,GAAW;IACxD,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;YAChD,GAAG;YACH,QAAQ,EAAE,MAAM;YAChB,OAAO,EAAE,MAAM;YACf,SAAS,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI;YAC1B,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;SAClC,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,mBAAmB,CAAC,KAAK,CAAC,CAAC;IACpC,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAc;IACzC,MAAM,MAAM,GAAG,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,QAAQ,IAAI,KAAK,CAAC,CAAC,CAAC,MAAM,CAAE,KAA8B,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACpI,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QACvC,MAAM,KAAK,CAAC;IACd,CAAC;IACD,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC5D,OAAO,MAAM,CAAE,KAA8B,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;IAC9D,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAc;IACzC,OAAO,OAAO,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,IAAI,KAAK,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;AAC9H,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,QAAsB;IAC3D,MAAM,QAAQ,GAAG,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5I,OAAO,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AACrG,CAAC"}

package/dist/eval/historical.d.ts

@@ -0,0 +1,4 @@
+import type { QueryOptions } from "../types.js";
+import type { EvalScenario } from "./types.js";
+export declare function historicalFixtureScenarios(seed: string, options: QueryOptions): Promise<EvalScenario[]>;
+export declare function externalHistoricalTaskPackScenarios(repoRoot: string, options: QueryOptions, taskPackPath: string): Promise<EvalScenario[]>;