even-pf 0.2.4 → 0.2.6

package/bin/even-pf.js

@@ -0,0 +1,47 @@
+#!/usr/bin/env bun
+"use strict";
+
+const { spawnSync } = require("child_process");
+const path = require("path");
+
+// Map process.platform + process.arch to the sub-package name and binary filename
+const PLATFORM_MAP = {
+  "linux-x64":   { pkg: "even-pf-linux-x64",   bin: "even-pf" },
+  "linux-arm64": { pkg: "even-pf-linux-arm64",  bin: "even-pf" },
+  "win32-x64":   { pkg: "even-pf-windows-x64",  bin: "even-pf.exe" },
+  "darwin-x64":  { pkg: "even-pf-darwin-x64",   bin: "even-pf" },
+  "darwin-arm64":{ pkg: "even-pf-darwin-arm64", bin: "even-pf" },
+};
+
+const key = `${process.platform}-${process.arch}`;
+const entry = PLATFORM_MAP[key];
+
+if (!entry) {
+  console.error(
+    `even-pf: Unsupported platform/architecture: ${key}\n` +
+    `Supported: ${Object.keys(PLATFORM_MAP).join(", ")}`
+  );
+  process.exit(1);
+}
+
+let binaryPath;
+try {
+  // resolve the binary inside the optionally-installed platform sub-package
+  binaryPath = require.resolve(`${entry.pkg}/bin/${entry.bin}`);
+} catch {
+  console.error(
+    `even-pf: Could not find the platform binary package "${entry.pkg}".\n` +
+    `Try reinstalling even-pf, or install the package manually:\n` +
+    `  npm install ${entry.pkg}`
+  );
+  process.exit(1);
+}
+
+const result = spawnSync(binaryPath, process.argv.slice(2), { stdio: "inherit" });
+
+if (result.error) {
+  console.error(`even-pf: Failed to launch binary: ${result.error.message}`);
+  process.exit(1);
+}
+
+process.exit(result.status ?? 0);

package/package.json

@@ -1,16 +1,21 @@
 {
   "name": "even-pf",
-  "version": "0.2.4",
-  "module": "src/index.ts",
+  "version": "0.2.6",
+  "description": "AI-assisted responsible grading tool for programming assignments",
+  "module": "src/cli.ts",
   "type": "module",
   "license": "UNLICENSED",
   "scripts": {
-    "start": "bun run src/index.ts",
-    "build": "bun build src/index.ts --compile --outfile build/epf",
+    "start": "bun run src/cli.ts",
+    "build-dev": "bun build src/cli.ts --compile --outfile build/epf",
+    "build:all": "bun scripts/build-all.ts",
+    "publish:all": "bun scripts/publish-all.ts",
+    "publish:dry": "bun scripts/publish-all.ts --dry-run",
+    "bump": "bun scripts/bump-version.ts",
     "config-gen": "bun run --console-depth 6 src/generate-config.ts"
   },
   "bin": {
-    "even-pf": "src/index.ts"
+    "even-pf": "bin/even-pf.js"
   },
   "devDependencies": {
     "@types/bun": "latest"
@@ -24,14 +29,16 @@
     "smol-toml": "^1.6.0",
     "zod-defaults": "^0.2.3"
   },
+  "optionalDependencies": {
+    "even-pf-linux-x64": "0.2.6",
+    "even-pf-linux-arm64": "0.2.6",
+    "even-pf-windows-x64": "0.2.6",
+    "even-pf-darwin-x64": "0.2.6",
+    "even-pf-darwin-arm64": "0.2.6"
+  },
   "files": [
-    ".gitignore",
-    "src",
-    "bun.lock",
-    "bunfig.toml",
-    "epf.example.toml",
+    "bin/even-pf.js",
     "package.json",
-    "README.md",
-    "tsconfig.json"
+    "README.md"
   ]
 }

package/.gitignore

@@ -1,4 +0,0 @@
-/node_modules
-/build
-
-/outputs

package/bun.lock

@@ -1,42 +0,0 @@
-{
-  "lockfileVersion": 1,
-  "configVersion": 1,
-  "workspaces": {
-    "": {
-      "name": "tools",
-      "dependencies": {
-        "@openrouter/sdk": "^0.5.1",
-        "chalk": "^5.6.2",
-        "smol-toml": "^1.6.0",
-        "zod-defaults": "^0.2.3",
-      },
-      "devDependencies": {
-        "@types/bun": "latest",
-      },
-      "peerDependencies": {
-        "typescript": "^5.9.3",
-      },
-    },
-  },
-  "packages": {
-    "@openrouter/sdk": ["@openrouter/sdk@0.5.1", "", { "dependencies": { "zod": "^3.25.0 || ^4.0.0" } }, "sha512-Kl0N1jIj7A3lnkM5dO3SGP8JP3jAozzs6JWcHVuZUBt5DsGKxFGNH1Y15bCfsJiLNA2ylAQpCN3aNcgEYkkL5Q=="],
-
-    "@types/bun": ["@types/bun@1.3.9", "", { "dependencies": { "bun-types": "1.3.9" } }, "sha512-KQ571yULOdWJiMH+RIWIOZ7B2RXQGpL1YQrBtLIV3FqDcCu6FsbFUBwhdKUlCKUpS3PJDsHlJ1QKlpxoVR+xtw=="],
-
-    "@types/node": ["@types/node@24.10.1", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ=="],
-
-    "bun-types": ["bun-types@1.3.9", "", { "dependencies": { "@types/node": "*" } }, "sha512-+UBWWOakIP4Tswh0Bt0QD0alpTY8cb5hvgiYeWCMet9YukHbzuruIEeXC2D7nMJPB12kbh8C7XJykSexEqGKJg=="],
-
-    "chalk": ["chalk@5.6.2", "", {}, "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA=="],
-
-    "smol-toml": ["smol-toml@1.6.0", "", {}, "sha512-4zemZi0HvTnYwLfrpk/CF9LOd9Lt87kAt50GnqhMpyF9U3poDAP2+iukq2bZsO/ufegbYehBkqINbsWxj4l4cw=="],
-
-    "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
-
-    "undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="],
-
-    "zod": ["zod@4.1.12", "", {}, "sha512-JInaHOamG8pt5+Ey8kGmdcAcg3OL9reK8ltczgHTAwNhMys/6ThXHityHxVV2p3fkw/c+MAvBHFVYHFZDmjMCQ=="],
-
-    "zod-defaults": ["zod-defaults@0.2.3", "", { "peerDependencies": { "zod": "^4.1.12" } }, "sha512-7pYkOH1/c+Ril5AZUYtsbhMkehkI8CMqzFZ7YZXfC9SMLRvZuLyonQE7BAIVSNxeTpqTZmW5BLxGSzWMnKNdIw=="],
-  }
-}

package/bunfig.toml

@@ -1,2 +0,0 @@
-[console]
-depth = 6

package/epf.example.toml

@@ -1,58 +0,0 @@
-[llm.models.general_analysis]
-sdk = "openrouter"
-model_name = ""
-max_completion_tokens = 20000
-temperature = 0.9
-top_p = 1
-frequency_penalty = 0
-presence_penalty = 0
-reasoning_effort = "high"
-
-[llm.models.output_comparison]
-sdk = "openrouter"
-model_name = ""
-max_completion_tokens = 20000
-temperature = 0
-top_p = 1
-frequency_penalty = 0
-presence_penalty = 0
-reasoning_effort = "high"
-
-[llm.prompt_replacement]
-role = "role_placeholder"
-
-[vendors.openrouter]
-api_key = ""
-
-[[analysis_workflows]]
-slug = ""
-model = "general_analysis"
-runs = 1
-output_filename = ""
-prompt = ""
-
-[[analysis_workflows.input_files_searches]]
-file_glob = ""
-search_directory = "."
-excluded_files = []
-
-[[testing_workflows]]
-slug = ""
-model = "general_analysis"
-runs = 1
-output_filename = ""
-setup_commands = []
-cleanup_commands = []
-
-[[testing_workflows.test_cases]]
-name = ""
-work_directory = "."
-single_run_command = ""
-interactive_steps = []
-
-[testing_workflows.test_cases.single_run_expected_output]
-prefix_strip_string = ""
-postfix_strip_string = ""
-substring = ""
-llm_judge_input_mode = "NONE"
-llm_judge_prompt = ""

package/src/generate-config.ts

@@ -1,40 +0,0 @@
-import {stringify} from "smol-toml";
-import {getDefaultsForSchema} from "zod-defaults";
-
-import {
-    ConfigSchema,
-    AnalysisWorkflowEntrySchema,
-    FileSearchEntrySchema,
-    TestingWorkflowEntrySchema, TestCaseSchema, LLMConfigSchema, ModelConfigSchema
-} from "./util/config-schema.ts";
-
-
-console.log("generate-config.ts");
-
-let defaultConfig = getDefaultsForSchema(ConfigSchema);
-
-let defaultLLMConfig = getDefaultsForSchema(LLMConfigSchema);
-defaultLLMConfig.prompt_replacement["role"] = "role_placeholder";
-let defaultModelConfig = getDefaultsForSchema(ModelConfigSchema);
-defaultLLMConfig.models["general_analysis"] = defaultModelConfig;
-defaultLLMConfig.models["output_comparison"] = structuredClone(defaultModelConfig);
-defaultLLMConfig.models["output_comparison"].temperature = 0;
-defaultConfig.llm = defaultLLMConfig;
-
-let defaultAnalysisWorkflowEntry = getDefaultsForSchema(AnalysisWorkflowEntrySchema);
-let defaultFileSearchEntry = getDefaultsForSchema(FileSearchEntrySchema);
-defaultAnalysisWorkflowEntry.input_files_searches = [defaultFileSearchEntry];
-defaultConfig.analysis_workflows = [defaultAnalysisWorkflowEntry];
-
-let defaultTestingWorkflowEntry = getDefaultsForSchema(TestingWorkflowEntrySchema);
-defaultTestingWorkflowEntry.test_cases = [getDefaultsForSchema(TestCaseSchema)];
-defaultConfig.testing_workflows = [defaultTestingWorkflowEntry];
-
-console.log(defaultConfig);
-
-const tomlString = stringify(defaultConfig);
-const outputFilename = "epf.example.toml";
-await Bun.file(outputFilename).write(tomlString);
-console.log(`Default config written to ${outputFilename}`);
-
-console.log("generate-config.ts done");

package/src/index.ts

@@ -1,59 +0,0 @@
-#!/usr/bin/env bun
-
-import {OpenRouter} from "@openrouter/sdk";
-
-import {CONFIG} from "./util/config.ts";
-import {executeTestingWorkflow} from "./workflow/testing-workflow.ts";
-import {executeAnalysisWorkflow} from "./workflow/analysis-workflow.ts";
-import type {WorkflowDependencies} from "./workflow";
-import {OutputViewer} from "./util/output-viewer.ts";
-
-
-console.log("EPF index.ts");
-
-const workflowDependencies: WorkflowDependencies = {
-    seed: Math.floor(Date.now() / 1000),
-    openRouter: new OpenRouter({
-        apiKey: CONFIG.vendors.openrouter.api_key,
-    }),
-    outputViewer: new OutputViewer(),
-}
-
-// Parallelize workflows with Promise.allSettled
-const analysisWorkflows = CONFIG.analysis_workflows;
-const testingWorkflows = CONFIG.testing_workflows;
-console.log(`Starting execution of ${analysisWorkflows.length} workflows...`);
-console.log(analysisWorkflows.map((w) => w.slug));
-let workflowRuns: Promise<void>[] = [];
-analysisWorkflows.forEach((workflow) => {
-    for (let i = 0; i < workflow.runs; i++) {
-        workflowRuns.push(executeAnalysisWorkflow(workflow, i+1, workflowDependencies));
-    }
-});
-testingWorkflows.forEach((workflow) => {
-    for (let i = 0; i < workflow.runs; i++) {
-        workflowRuns.push(executeTestingWorkflow(workflow, i+1, workflowDependencies));
-    }
-});
-
-const workflowsResults = await Promise.allSettled(workflowRuns);
-// Summarize with indices to include slugs in failure logs
-const failedIndices: number[] = [];
-const succeededIndices: number[] = [];
-workflowsResults.forEach((r, i) => {
-    if (r.status === "rejected") failedIndices.push(i);
-    else succeededIndices.push(i);
-});
-
-console.log(`Workflows completed. Succeeded: ${succeededIndices.length}; Failed: ${failedIndices.length}`);
-if (failedIndices.length > 0) {
-    failedIndices.forEach((i) => {
-        const r = workflowsResults[i] as PromiseRejectedResult;
-        const slug = analysisWorkflows[i]?.slug ?? `#${i + 1}`;
-        console.warn(`Workflow '${slug}' failed:`, r.reason);
-    });
-}
-
-workflowDependencies.outputViewer.display();
-
-console.log("index.ts done");

package/src/util/args.ts

@@ -1,37 +0,0 @@
-import {parseArgs} from "util";
-
-// console.log(Bun.argv);
-export const ARGS = parseArgs({
-    args: Bun.argv,
-    options: {
-        // mode: {
-        //     type: "string",
-        //     short: "M",
-        //     default: "run",
-        // },
-        config: {
-            type: "string",
-            short: "C",
-        },
-        dir: {
-            type: "string",
-            short: "D",
-            default: ".",
-        },
-        skip_workflow: {
-            type: "string",
-            short: "S",
-            multiple: true,
-        },
-        completion_inputs_destination: {
-            type: "string",
-        },
-    },
-    strict: true,
-    allowPositionals: true,
-});
-
-// export enum RunMode {
-//     Run = "run",
-//     Eval = "eval",
-// }

package/src/util/config-schema.ts

@@ -1,80 +0,0 @@
-import {z} from "zod";
-
-export const ModelConfigSchema = z.object({
-    sdk: z.enum(["openrouter"]).default("openrouter"),
-    model_name: z.string().default(""),
-    max_completion_tokens: z.number().min(1).default(20000),
-    temperature: z.number().min(0).max(1).default(0.9),
-    top_p: z.number().min(0).max(1).default(1),
-    frequency_penalty: z.number().min(-2).max(2).default(0),
-    presence_penalty: z.number().min(-2).max(2).default(0),
-    reasoning_effort: z.enum(["low", "medium", "high"]).default("high"),
-});
-
-export const LLMConfigSchema = z.object({
-    models: z.record(z.string(), ModelConfigSchema),
-    prompt_replacement: z.record(z.string(), z.string()),
-});
-
-export const FileSearchEntrySchema = z.object({
-    file_glob: z.string().min(1),
-    search_directory: z.string().default("."),
-    excluded_files: z.array(z.string()).default([]),
-});
-
-export const BaseWorkflowEntrySchema = z.object({
-    slug: z.string(),
-    model: z.string().default("general_analysis"),
-    runs: z.number().min(1).default(1),
-    input_files_searches: z.array(FileSearchEntrySchema).default([]),
-    output_filename: z.string().min(1),
-});
-
-export const AnalysisWorkflowEntrySchema = BaseWorkflowEntrySchema.extend({
-    prompt: z.string(),
-})
-
-export enum LLMJudgeInputModeEnum{
-    None = "NONE",
-    Diff = "DIFF",
-    Full = "FULL",
-}
-const LLMJudgeInputModeSchema = z.enum(LLMJudgeInputModeEnum);
-
-const ExpectedOutputSchema = z.object({
-    prefix_strip_string: z.string().min(0),
-    postfix_strip_string: z.string().min(0),
-    substring: z.string().min(0),
-    llm_judge_input_mode: LLMJudgeInputModeSchema.default(LLMJudgeInputModeEnum.None),
-    llm_judge_prompt: z.string().min(0),
-});
-
-export const TestCaseSchema = z.object({
-    name: z.string(),
-    work_directory: z.string().default("."),
-    single_run_command: z.string(),
-    single_run_expected_output: ExpectedOutputSchema,
-    interactive_steps: z.array(z.object({
-        input: z.string(),
-        expected_output: ExpectedOutputSchema,
-    })),
-});
-
-export const TestingWorkflowEntrySchema = BaseWorkflowEntrySchema.extend({
-    setup_commands: z.array(z.string()).default([]),
-    test_cases: z.array(TestCaseSchema).default([]),
-    cleanup_commands: z.array(z.string()).default([]),
-}).omit({
-    input_files_searches: true,
-});
-
-export const ConfigSchema = z.object({
-    llm: LLMConfigSchema,
-    vendors: z.object({
-        openrouter: z.object({
-            api_key: z.string(),
-        }),
-    }),
-    analysis_workflows: z.array(AnalysisWorkflowEntrySchema),
-    testing_workflows: z.array(TestingWorkflowEntrySchema),
-});

package/src/util/config.ts

@@ -1,69 +0,0 @@
-import os from "node:os";
-import { readFileSync, existsSync } from "node:fs";
-
-import {z} from "zod";
-
-import {ARGS} from "./args.ts";
-import {ConfigSchema} from "./config-schema.ts";
-
-
-const homeDir: string = os.homedir();
-const defaultConfigFileName = "epf.toml";
-const configURLEnvVar = "EPF_CONFIG_URL";
-
-type Config = z.infer<typeof ConfigSchema>;
-
-async function readConfig() {
-    console.log(`Loading config`);
-    
-    let configFilePath: string;
-    if (ARGS.values.config && ARGS.values.config.trim().length > 0) {
-        configFilePath = ARGS.values.config.trim();
-        console.log(`Found config from command line argument: ${configFilePath}`);
-    }
-    else if (process.env[configURLEnvVar]) {
-        configFilePath = process.env[configURLEnvVar]!;
-        console.log(`Found config from environment variable ${configURLEnvVar}`);
-    }
-    else {
-        if (existsSync(defaultConfigFileName)) {
-            configFilePath = defaultConfigFileName;
-            console.log(`Found config from current directory`);
-        }
-        else if (existsSync(`${homeDir}/${defaultConfigFileName}`)) {
-            configFilePath = `${homeDir}/${defaultConfigFileName}`;
-            console.log(`Found config from home directory`);
-        }
-        else {
-            throw new Error(`Config file ${defaultConfigFileName} not found`);
-        }
-    }
-    
-    let configFileContents: string;
-    if (/^https?:\/\//.test(configFilePath)) {
-        console.log(`Fetching config from URL: ${configFilePath}`);
-        const configResponse = await fetch(configFilePath);
-        if (!configResponse.ok) {
-            throw new Error(`Failed to fetch config from URL: ${configFilePath}, status: ${configResponse.status}`);
-        }
-        configFileContents = await configResponse.text();
-    } else {
-        console.log(`Loading config from local file: ${configFilePath}`);
-        configFileContents = readFileSync(configFilePath).toString();
-    }
-    
-    console.assert(configFileContents.trim().length > 0, "Config file is empty");
-    
-    let obj =  Bun.TOML.parse(configFileContents);
-    const parsedConfig = ConfigSchema.safeParse(obj);
-    if (!parsedConfig.success) {
-        console.error("Config file is invalid:", parsedConfig.error.format());
-        console.log(`Config file contents:\n${configFileContents}`);
-        console.log(parsedConfig);
-        throw new Error("Config file is invalid");
-    }
-    console.log(`Config loaded from ${configFilePath}`);
-    return parsedConfig.data as Config;
-}
-
-export const CONFIG = await readConfig();

package/src/util/eval-harness.ts

@@ -1,18 +0,0 @@
-import { appendFile } from "node:fs/promises";
-
-import {ARGS} from "./args.ts";
-
-export async function recordCompletionInput(inputs: {role: string, content: any}[]) {
-    let completionInputsDestination = ARGS.values.completion_inputs_destination;
-    if (!completionInputsDestination) {
-        return;
-    }
-    
-    if (!(await Bun.file(completionInputsDestination).exists())){
-        console.warn(`Completion inputs destination file ${completionInputsDestination} does not exist`);
-        return;
-    }
-    
-    await appendFile(completionInputsDestination, JSON.stringify(completionInputsDestination)+"\n");
-    console.log(`Recorded completion inputs to ${completionInputsDestination}`);
-}

package/src/util/file-payload.ts

@@ -1,87 +0,0 @@
-/**
- * Helper class for generating file content payloads with language-specific formatting
- */
-export class FilePayloadGenerator {
-    private static readonly LANGUAGE_MAP: Record<string, string> = {
-        '.cs': 'csharp',
-        '.cpp': 'cpp',
-        '.cc': 'cpp',
-        '.cxx': 'cpp',
-        '.c': 'c',
-        '.h': 'cpp',
-        '.hpp': 'cpp',
-        '.hxx': 'cpp',
-        '.java': 'java',
-        '.js': 'javascript',
-        '.ts': 'typescript',
-        '.py': 'python',
-        '.rb': 'ruby',
-        '.go': 'go',
-        '.rs': 'rust',
-        '.swift': 'swift',
-        '.kt': 'kotlin',
-        '.php': 'php',
-        '.scala': 'scala',
-        '.sh': 'bash',
-        '.bash': 'bash',
-        '.zsh': 'zsh',
-        '.ps1': 'powershell',
-        '.md': 'markdown',
-        '.json': 'json',
-        '.xml': 'xml',
-        '.yaml': 'yaml',
-        '.yml': 'yaml',
-        '.toml': 'toml',
-        '.html': 'html',
-        '.css': 'css',
-        '.scss': 'scss',
-        '.sql': 'sql',
-    };
-
-    /**
-     * Get the language identifier for a file based on its extension
-     * @param filePath The path to the file
-     * @returns The language identifier (e.g., 'csharp', 'cpp')
-     */
-    private static getLanguageFromPath(filePath: string): string {
-        const ext = filePath.substring(filePath.lastIndexOf('.')).toLowerCase();
-        return this.LANGUAGE_MAP[ext] || 'text';
-    }
-
-    /**
-     * Generate a formatted payload for a single file
-     * @param filePath The path to the file
-     * @param content The content of the file
-     * @returns A formatted string with file path and content in a code block
-     */
-    static formatFileContent(filePath: string, content: string): string {
-        const language = this.getLanguageFromPath(filePath);
-        return `${filePath}\n\`\`\`${language}\n${content}\n\`\`\``;
-    }
-
-    /**
-     * Generate payloads for multiple files
-     * @param files Array of file paths
-     * @returns Array of formatted file content strings
-     */
-    static async generatePayloads(files: string[]): Promise<string[]> {
-        const payloads: string[] = [];
-        
-        for (const file of files) {
-            const content = await Bun.file(file).text();
-            payloads.push(this.formatFileContent(file, content));
-        }
-        
-        return payloads;
-    }
-
-    /**
-     * Add a custom language mapping
-     * @param extension The file extension (including the dot, e.g., '.custom')
-     * @param language The language identifier to use
-     */
-    static addLanguageMapping(extension: string, language: string): void {
-        this.LANGUAGE_MAP[extension.toLowerCase()] = language;
-    }
-}
-

package/src/util/llm.ts

@@ -1,80 +0,0 @@
-import type {SystemMessage, UserMessage} from "@openrouter/sdk/models";
-
-import {CONFIG} from "./config.ts";
-import type {WorkflowDependencies} from "../workflow";
-import {recordCompletionInput} from "./eval-harness.ts";
-
-
-export async function generateCompletion(deps: WorkflowDependencies,
-                                         log: (..._: any[])=>void,
-                                         warn: (..._: any[])=>void,
-                                         model: string,
-                                         systemPrompt: string,
-                                         content: UserMessage["content"]) {
-    let modelSettings = CONFIG.llm.models[model];
-    if (!modelSettings) {
-        throw new Error(`No model settings found for model "${model}"`);
-    }
-    
-    let replacedCount = 0;
-    for (const [replacementKey, replacementValue] of Object.entries(CONFIG.llm.prompt_replacement)) {
-        if (systemPrompt.includes(replacementKey)) {replacedCount++}
-        systemPrompt = systemPrompt.replaceAll(`{{${replacementKey}}}`, replacementValue);
-        if (typeof content === "string") {
-            if (content.includes(replacementKey)) {replacedCount++}
-            content = content.replaceAll(`{{${replacementKey}}}`, replacementValue);
-        }
-        else {
-            for (let i = 0; i < content.length; i++) {
-                const element = content[i];
-                if (element && "type" in element && element.type === "text" && typeof element.text === "string") {
-                    if (element.text.includes(replacementKey)) {replacedCount++}
-                    content[i] = {
-                        ...element,
-                        text: element.text.replaceAll(`{{${replacementKey}}}`, replacementValue),
-                    }
-                }
-            }
-            
-        }
-    }
-    log(`Replaced ${replacedCount} instances of prompt variables in system prompt and content`);
-    
-    let messages: (SystemMessage | UserMessage)[] = [
-        {
-            role: "system",
-            content: systemPrompt,
-        },
-        {
-            role: "user",
-            content: content,
-        }
-    ];
-    setTimeout(async ()=> await recordCompletionInput(messages), 5);
-    
-    log("Sending chat completion request...");
-    let startTime = Date.now();
-    let completion = await deps.openRouter.chat.send({
-        model: modelSettings.model_name,
-        maxCompletionTokens: modelSettings.max_completion_tokens,
-        messages: messages,
-        stream: false,
-        seed: deps.seed,
-        frequencyPenalty: modelSettings.frequency_penalty,
-        presencePenalty: modelSettings.presence_penalty,
-        temperature: modelSettings.temperature,
-        reasoning: {
-            effort: modelSettings.reasoning_effort,
-        },
-    });
-    log(`Completion response generated in ${(Date.now() - startTime) / 1000} seconds`);
-    if (completion.choices.length < 1){
-        warn("No choices returned from completion");
-        console.log(completion);
-    }
-    
-    return {
-        text: completion.choices[0]?.message.content?.toString() ?? "",
-        model: completion.model,
-    };
-}

package/src/util/output-viewer.ts

@@ -1,54 +0,0 @@
-import chalk from "chalk";
-
-type FileRecord = {
-    type: "markdown" | "text";
-    content: string;
-}
-
-export class OutputViewer {
-    filesRecords: Record<string, FileRecord> = {};
-    
-    
-    addFile(filename: string, _: FileRecord): void {
-        this.filesRecords[filename] = _;
-    }
-    
-    private serve(): void {
-        let files = Object.entries(this.filesRecords).sort((a, b) => a[0].localeCompare(b[0]));
-        
-        let server = Bun.serve({
-            port: 0,
-            routes: {
-                "/:slug": (req) => {
-                    let slug = req.params.slug;
-                    console.log(`Request for slug: "${slug}"`);
-                    return new Response(this.filesRecords[slug]?.content ?? "Not Found");
-                }
-            },
-            fetch(req) {
-                return new Response("Not Found (fallback)", { status: 404 });
-            },
-        });
-        console.log(server.url);
-    }
-    
-    display() {
-        if (Object.keys(this.filesRecords).length === 0) {
-            console.warn("No files to display");
-            return;
-        }
-        
-        console.log("Click the following links to view the outputs in your browser:");
-        
-        const FRONTEND_URL = "https://ta-tools-dashboard.vercel.app/tools/md-viewer"; //TODO: not hardcode this
-        let files = Object.entries(this.filesRecords).sort((a, b) => a[0].localeCompare(b[0]));
-        for (const [filename, fileRecord] of files) {
-            let params = new URLSearchParams();
-            params.set("name", filename);
-            params.set("comp", "gzip");
-            params.set("data", Bun.gzipSync(fileRecord.content).toBase64());
-            let url = `${FRONTEND_URL}#${params.toString()}`;
-            console.log(`${chalk.cyan(filename)}: ${url}`);
-        }
-    }
-}

package/src/workflow/analysis-workflow.ts

@@ -1,60 +0,0 @@
-import {Glob} from "bun";
-
-import chalk from "chalk";
-
-import {CONFIG} from "../util/config.ts";
-import {FilePayloadGenerator} from "../util/file-payload.ts";
-import type {WorkflowDependencies} from "./index.ts";
-import {generateCompletion} from "../util/llm.ts";
-
-
-export async function executeAnalysisWorkflow(workflow: typeof CONFIG.analysis_workflows[number], runNum: number, deps: WorkflowDependencies) {
-    console.log(`Executing analysis workflow: ${workflow.slug}`);
-    const log = (...args: Parameters<typeof console.log>) => {
-        console.log(chalk.cyan(`[${workflow.slug}]`), ...args);
-    }
-    const warn = (...args: Parameters<typeof console.warn>) => {
-        console.warn(chalk.red(`[${workflow.slug}]`), ...args);
-    }
-    
-    let allFiles = (
-        await Promise.all(
-            workflow.input_files_searches.map(async (fileSearch) => {
-                const fileExclusionsSet = new Set(fileSearch.excluded_files);
-                const glob = new Glob(fileSearch.file_glob);
-                const matches: string[] = [];
-                for await (const file of glob.scan(fileSearch.search_directory)) {
-                    if (fileExclusionsSet.has(file)) {
-                        log(`Excluding file: ${file}`);
-                        continue;
-                    }
-                    matches.push(file);
-                }
-                log(`Found ${matches.length} files for search: ${fileSearch.file_glob} in ${fileSearch.search_directory}`, matches);
-                return matches;
-            })
-        )
-    ).flat();
-    
-    if (allFiles.length === 0) {
-        warn(`No files found for workflow, skipping...`);
-        return;
-    }
-    log(`Found ${allFiles.length} files for workflow`);
-    const fileContentsPayload = await FilePayloadGenerator.generatePayloads(allFiles);
-    const completion = await generateCompletion(deps, log, warn, workflow.model, workflow.prompt, fileContentsPayload.map((file) => {
-        return {
-            type: "text",
-            text: file,
-        }
-    }));
-
-    const outputFileName = workflow.output_filename
-        .replaceAll("[seed]", deps.seed.toString())
-        .replaceAll("[slug]", workflow.slug)
-        .replaceAll("[model]", `(${completion.model.replaceAll("/", "--")})`)
-        .replaceAll("[run]", runNum.toString());
-    await Bun.write(outputFileName, completion.text);
-    log(`Completion written to ${outputFileName}`);
-    deps.outputViewer.addFile(outputFileName, {type: "markdown", content: completion.text});
-}

package/src/workflow/index.ts

@@ -1,9 +0,0 @@
-import type {OpenRouter} from "@openrouter/sdk";
-
-import type {OutputViewer} from "../util/output-viewer.ts";
-
-export type WorkflowDependencies = {
-    seed: number,
-    openRouter: OpenRouter,
-    outputViewer: OutputViewer,
-}

package/src/workflow/testing-workflow.ts

@@ -1,127 +0,0 @@
-import {$} from "bun";
-
-import {CONFIG} from "../util/config.ts";
-import chalk from "chalk";
-import {LLMJudgeInputModeEnum} from "../util/config-schema.ts";
-import type {WorkflowDependencies} from "./index.ts";
-import {generateCompletion} from "../util/llm.ts";
-
-export async function executeTestingWorkflow(workflow: typeof CONFIG.testing_workflows[number], runNum: number, deps: WorkflowDependencies) {
-    console.log(`Executing testing workflow: ${workflow.slug}`);
-    const log = (...args: Parameters<typeof console.log>) => {
-        console.log(chalk.cyan(`[${workflow.slug}]`), ...args);
-    }
-    const warn = (...args: Parameters<typeof console.warn>) => {
-        console.warn(chalk.red(`[${workflow.slug}]`), ...args);
-    }
-    const debug = (...args: Parameters<typeof console.debug>) => {
-        console.debug(chalk.gray(`[${workflow.slug}]`), ...args.map(e => typeof e === "string" ? chalk.gray(e) : e));
-    }
-
-    log(`$PATH: ${process.env.PATH}`);
-    
-    for (const command of workflow.setup_commands){
-        log(`Running setup command: ${command}`);
-        await $`${{raw: command}}`.nothrow();
-    }
-
-    let testCasesResults: boolean[] = new Array(workflow.test_cases.length);
-    let testCasesResultsExplanation: string[] = new Array(workflow.test_cases.length);
-    for (let i = 0; i < workflow.test_cases.length; i++) {
-        try {
-            const testCase = workflow.test_cases[i]!;
-            log(`Running test case ${i + 1}/${workflow.test_cases.length}: ${testCase.name}`);
-            
-            if (testCase.interactive_steps.length > 0) {
-                warn("Interactive steps are not supported in this version. Skipping interactive steps.");
-                continue;
-            }
-            
-            let {stdout, stderr, exitCode} = await $`${{raw: testCase.single_run_command}}`.cwd(testCase.work_directory).nothrow().quiet();
-            console.log();  // Blank line for readability
-            debug(`Test case stdout (${stdout.length}):\n${stdout}`);
-            debug(`Test case stderr (${stderr.length}):\n${stderr}`);
-            debug(`Exit code: ${exitCode}`);
-            
-            let commandOutput = stdout.toString();
-            if (testCase.single_run_expected_output.prefix_strip_string.length > 0) {
-                let prefixIndex = commandOutput.indexOf(testCase.single_run_expected_output.prefix_strip_string);
-                if (prefixIndex !== -1) {
-                    commandOutput = commandOutput.substring(prefixIndex + testCase.single_run_expected_output.prefix_strip_string.length);
-                }
-            }
-            if (testCase.single_run_expected_output.postfix_strip_string.length > 0) {
-                let postfixIndex = commandOutput.lastIndexOf(testCase.single_run_expected_output.postfix_strip_string);
-                if (postfixIndex !== -1) {
-                    commandOutput = commandOutput.substring(0, postfixIndex);
-                }
-            }
-            debug("Sanitized command output for evaluation:\n", commandOutput);
-            
-            if (testCase.single_run_expected_output.llm_judge_input_mode == LLMJudgeInputModeEnum.None){
-                if (stdout.includes(testCase.single_run_expected_output.substring)) {
-                    log(`Test case '${testCase.name}' passed: expected substring found in output.`);
-                    testCasesResults[i] = true;
-                }
-                else {
-                    warn(`Test case '${testCase.name}' failed: expected substring NOT found in output.`);
-                    testCasesResults[i] = false;
-                }
-            }
-            else {
-                switch (testCase.single_run_expected_output.llm_judge_input_mode) {
-                    case LLMJudgeInputModeEnum.Full:
-                        log("Evaluating full output with LLM judge...");
-                        const completion = await generateCompletion(deps, log, warn, workflow.model, testCase.single_run_expected_output.llm_judge_prompt, JSON.stringify({
-                            "expected_output_substring": testCase.single_run_expected_output.substring,
-                            "actual_output": commandOutput,
-                        }));
-                        const completionText = completion.text;
-                        log(`LLM judge completion:\n${completionText}`);
-                        const llmJudgeResult = completionText.toLowerCase().includes("pass");  // TODO: More robust parsing
-                        if (llmJudgeResult) {
-                            log(chalk.green(`Test case '${testCase.name}' passed according to LLM judge.`));
-                            testCasesResults[i] = true;
-                        }
-                        else {
-                            warn(chalk.yellowBright(`Test case '${testCase.name}' failed according to LLM judge.`));
-                            testCasesResults[i] = false;
-                        }
-                        
-                        try {
-                            const judgeResultObject = JSON.parse(completionText);
-                            
-                            if ("summary" in judgeResultObject) {
-                                testCasesResultsExplanation[i] = judgeResultObject.summary;
-                            }
-                        } catch (e) {
-                            warn("Failed to parse LLM judge output as JSON. Make sure the LLM prompt requests JSON output.");
-                        }
-                        break;
-                    default:
-                        warn(`LLM judge input mode '${testCase.single_run_expected_output.llm_judge_input_mode}' is not supported in this version. Skipping LLM judging.`);
-                        break;
-                }
-            }
-        } catch (e) {
-            warn(`Error occurred while executing test case ${i + 1}:`, e);
-            testCasesResults[i] = false;
-        }
-    }
-    for (const command of workflow.cleanup_commands){
-        log(`Running cleanup command: ${command}`);
-        await $`${{raw: command}}`.nothrow();
-    }
-    
-    const passedCount = testCasesResults.filter((r) => r).length;
-    log(`Testing workflow completed. Passed ${passedCount}/${workflow.test_cases.length} test cases.`);
-    console.table(testCasesResults.map((entry, idx) => {
-        return [
-            workflow.test_cases[idx]?.name,
-            entry ? chalk.green("PASS") : chalk.red("FAIL"),
-            testCasesResultsExplanation[idx] || "",
-        ];
-    }));
-
-    log(`Finished testing workflow: ${workflow.slug}`);
-}

package/tsconfig.json

@@ -1,29 +0,0 @@
-{
-  "compilerOptions": {
-    // Environment setup & latest features
-    "lib": ["ESNext"],
-    "target": "ESNext",
-    "module": "Preserve",
-    "moduleDetection": "force",
-    "jsx": "react-jsx",
-    "allowJs": true,
-
-    // Bundler mode
-    "moduleResolution": "bundler",
-    "allowImportingTsExtensions": true,
-    "verbatimModuleSyntax": true,
-    "noEmit": true,
-
-    // Best practices
-    "strict": true,
-    "skipLibCheck": true,
-    "noFallthroughCasesInSwitch": true,
-    "noUncheckedIndexedAccess": true,
-    "noImplicitOverride": true,
-
-    // Some stricter flags (disabled by default)
-    "noUnusedLocals": false,
-    "noUnusedParameters": false,
-    "noPropertyAccessFromIndexSignature": false
-  }
-}