even-pf 0.1.0 → 0.2.0

package/README.md

@@ -1,6 +1,14 @@
 # Project Even-Playfield (E-PF)
 AI-assisted responsible grading tool for programming assignments. A GPT-wrapper in CLI for CS graders.
 
+## Usage
+Make sure you have [Bun](https://bun.com) installed.
+To run:
+```bash
+bunx even-pf [config]
+```
+
+## Development
 To install dependencies:
 ```bash
 bun install
@@ -11,10 +19,6 @@ To install as a tool globally:
 bun link
 ```
 
-To run:
-```bash
-bunx e-pf
-```
 Make sure you have a config file in your home or current directory. Alternatively, you can set environment variable `EPF_CONFIG_URL`.
 
 This project was created using `bun init` in bun v1.3.2. [Bun](https://bun.com) is a fast all-in-one JavaScript runtime.

package/bun.lock

@@ -7,33 +7,29 @@
       "dependencies": {
         "@openrouter/sdk": "^0.5.1",
         "chalk": "^5.6.2",
-        "smol-toml": "^1.5.2",
+        "smol-toml": "^1.6.0",
         "zod-defaults": "^0.2.3",
       },
       "devDependencies": {
         "@types/bun": "latest",
       },
       "peerDependencies": {
-        "typescript": "^5",
+        "typescript": "^5.9.3",
       },
     },
   },
   "packages": {
     "@openrouter/sdk": ["@openrouter/sdk@0.5.1", "", { "dependencies": { "zod": "^3.25.0 || ^4.0.0" } }, "sha512-Kl0N1jIj7A3lnkM5dO3SGP8JP3jAozzs6JWcHVuZUBt5DsGKxFGNH1Y15bCfsJiLNA2ylAQpCN3aNcgEYkkL5Q=="],
 
-    "@types/bun": ["@types/bun@1.3.2", "", { "dependencies": { "bun-types": "1.3.2" } }, "sha512-t15P7k5UIgHKkxwnMNkJbWlh/617rkDGEdSsDbu+qNHTaz9SKf7aC8fiIlUdD5RPpH6GEkP0cK7WlvmrEBRtWg=="],
+    "@types/bun": ["@types/bun@1.3.9", "", { "dependencies": { "bun-types": "1.3.9" } }, "sha512-KQ571yULOdWJiMH+RIWIOZ7B2RXQGpL1YQrBtLIV3FqDcCu6FsbFUBwhdKUlCKUpS3PJDsHlJ1QKlpxoVR+xtw=="],
 
     "@types/node": ["@types/node@24.10.1", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ=="],
 
-    "@types/react": ["@types/react@19.2.6", "", { "dependencies": { "csstype": "^3.2.2" } }, "sha512-p/jUvulfgU7oKtj6Xpk8cA2Y1xKTtICGpJYeJXz2YVO2UcvjQgeRMLDGfDeqeRW2Ta+0QNFwcc8X3GH8SxZz6w=="],
-
-    "bun-types": ["bun-types@1.3.2", "", { "dependencies": { "@types/node": "*" }, "peerDependencies": { "@types/react": "^19" } }, "sha512-i/Gln4tbzKNuxP70OWhJRZz1MRfvqExowP7U6JKoI8cntFrtxg7RJK3jvz7wQW54UuvNC8tbKHHri5fy74FVqg=="],
+    "bun-types": ["bun-types@1.3.9", "", { "dependencies": { "@types/node": "*" } }, "sha512-+UBWWOakIP4Tswh0Bt0QD0alpTY8cb5hvgiYeWCMet9YukHbzuruIEeXC2D7nMJPB12kbh8C7XJykSexEqGKJg=="],
 
     "chalk": ["chalk@5.6.2", "", {}, "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA=="],
 
-    "csstype": ["csstype@3.2.3", "", {}, "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ=="],
-
-    "smol-toml": ["smol-toml@1.5.2", "", {}, "sha512-QlaZEqcAH3/RtNyet1IPIYPsEWAaYyXXv1Krsi+1L/QHppjX4Ifm8MQsBISz9vE8cHicIq3clogsheili5vhaQ=="],
+    "smol-toml": ["smol-toml@1.6.0", "", {}, "sha512-4zemZi0HvTnYwLfrpk/CF9LOd9Lt87kAt50GnqhMpyF9U3poDAP2+iukq2bZsO/ufegbYehBkqINbsWxj4l4cw=="],
 
     "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
 

package/epf.example.toml

@@ -1,8 +1,6 @@
-[openrouter]
-api_key = ""
-model = ""
-
-[hyperparameters]
+[llm.models.general_analysis]
+sdk = "openrouter"
+model_name = ""
 max_completion_tokens = 20000
 temperature = 0.9
 top_p = 1
@@ -10,8 +8,25 @@ frequency_penalty = 0
 presence_penalty = 0
 reasoning_effort = "high"
 
+[llm.models.output_comparison]
+sdk = "openrouter"
+model_name = ""
+max_completion_tokens = 20000
+temperature = 0
+top_p = 1
+frequency_penalty = 0
+presence_penalty = 0
+reasoning_effort = "high"
+
+[llm.prompt_replacement]
+role = "role_placeholder"
+
+[vendors.openrouter]
+api_key = ""
+
 [[analysis_workflows]]
 slug = ""
+model = "general_analysis"
 runs = 1
 output_filename = ""
 prompt = ""
@@ -23,6 +38,7 @@ excluded_files = []
 
 [[testing_workflows]]
 slug = ""
+model = "general_analysis"
 runs = 1
 output_filename = ""
 setup_commands = []

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "even-pf",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "module": "src/index.ts",
   "type": "module",
   "license": "UNLICENSED",
@@ -10,18 +10,18 @@
     "config-gen": "bun run --console-depth 6 src/generate-config.ts"
   },
   "bin": {
-    "e-pf": "src/index.ts"
+    "even-pf": "src/index.ts"
   },
   "devDependencies": {
     "@types/bun": "latest"
   },
   "peerDependencies": {
-    "typescript": "^5"
+    "typescript": "^5.9.3"
   },
   "dependencies": {
     "@openrouter/sdk": "^0.5.1",
     "chalk": "^5.6.2",
-    "smol-toml": "^1.5.2",
+    "smol-toml": "^1.6.0",
     "zod-defaults": "^0.2.3"
   },
   "files": [

package/src/generate-config.ts

@@ -5,17 +5,27 @@ import {
     ConfigSchema,
     AnalysisWorkflowEntrySchema,
     FileSearchEntrySchema,
-    TestingWorkflowEntrySchema, TestCaseSchema
+    TestingWorkflowEntrySchema, TestCaseSchema, LLMConfigSchema, ModelConfigSchema
 } from "./util/config-schema.ts";
 
 
 console.log("generate-config.ts");
 
 let defaultConfig = getDefaultsForSchema(ConfigSchema);
+
+let defaultLLMConfig = getDefaultsForSchema(LLMConfigSchema);
+defaultLLMConfig.prompt_replacement["role"] = "role_placeholder";
+let defaultModelConfig = getDefaultsForSchema(ModelConfigSchema);
+defaultLLMConfig.models["general_analysis"] = defaultModelConfig;
+defaultLLMConfig.models["output_comparison"] = structuredClone(defaultModelConfig);
+defaultLLMConfig.models["output_comparison"].temperature = 0;
+defaultConfig.llm = defaultLLMConfig;
+
 let defaultAnalysisWorkflowEntry = getDefaultsForSchema(AnalysisWorkflowEntrySchema);
 let defaultFileSearchEntry = getDefaultsForSchema(FileSearchEntrySchema);
 defaultAnalysisWorkflowEntry.input_files_searches = [defaultFileSearchEntry];
 defaultConfig.analysis_workflows = [defaultAnalysisWorkflowEntry];
+
 let defaultTestingWorkflowEntry = getDefaultsForSchema(TestingWorkflowEntrySchema);
 defaultTestingWorkflowEntry.test_cases = [getDefaultsForSchema(TestCaseSchema)];
 defaultConfig.testing_workflows = [defaultTestingWorkflowEntry];

package/src/index.ts

@@ -11,8 +11,9 @@ import type {WorkflowDependencies} from "./workflow";
 console.log("EPF index.ts");
 
 const workflowDependencies: WorkflowDependencies = {
+    seed: Math.floor(Date.now() / 1000),
     openRouter: new OpenRouter({
-        apiKey: CONFIG.openrouter.api_key,
+        apiKey: CONFIG.vendors.openrouter.api_key,
     })
 }
 

package/src/util/args.ts

@@ -0,0 +1,37 @@
+import {parseArgs} from "util";
+
+// console.log(Bun.argv);
+export const ARGS = parseArgs({
+    args: Bun.argv,
+    options: {
+        // mode: {
+        //     type: "string",
+        //     short: "M",
+        //     default: "run",
+        // },
+        config: {
+            type: "string",
+            short: "C",
+        },
+        dir: {
+            type: "string",
+            short: "D",
+            default: ".",
+        },
+        skip_workflow: {
+            type: "string",
+            short: "S",
+            multiple: true,
+        },
+        completion_inputs_destination: {
+            type: "string",
+        },
+    },
+    strict: true,
+    allowPositionals: true,
+});
+
+// export enum RunMode {
+//     Run = "run",
+//     Eval = "eval",
+// }

package/src/util/config-schema.ts

@@ -1,5 +1,21 @@
 import {z} from "zod";
 
+export const ModelConfigSchema = z.object({
+    sdk: z.enum(["openrouter"]).default("openrouter"),
+    model_name: z.string().default(""),
+    max_completion_tokens: z.number().min(1).default(20000),
+    temperature: z.number().min(0).max(1).default(0.9),
+    top_p: z.number().min(0).max(1).default(1),
+    frequency_penalty: z.number().min(-2).max(2).default(0),
+    presence_penalty: z.number().min(-2).max(2).default(0),
+    reasoning_effort: z.enum(["low", "medium", "high"]).default("high"),
+});
+
+export const LLMConfigSchema = z.object({
+    models: z.record(z.string(), ModelConfigSchema),
+    prompt_replacement: z.record(z.string(), z.string()),
+});
+
 export const FileSearchEntrySchema = z.object({
     file_glob: z.string().min(1),
     search_directory: z.string().default("."),
@@ -8,6 +24,7 @@ export const FileSearchEntrySchema = z.object({
 
 export const BaseWorkflowEntrySchema = z.object({
     slug: z.string(),
+    model: z.string().default("general_analysis"),
     runs: z.number().min(1).default(1),
     input_files_searches: z.array(FileSearchEntrySchema).default([]),
     output_filename: z.string().min(1),
@@ -52,17 +69,11 @@ export const TestingWorkflowEntrySchema = BaseWorkflowEntrySchema.extend({
 });
 
 export const ConfigSchema = z.object({
-    openrouter: z.object({
-        api_key: z.string(),
-        model: z.string(),
-    }),
-    hyperparameters: z.object({
-        max_completion_tokens: z.number().min(1).default(20000),
-        temperature: z.number().min(0).max(1).default(0.9),
-        top_p: z.number().min(0).max(1).default(1),
-        frequency_penalty: z.number().min(-2).max(2).default(0),
-        presence_penalty: z.number().min(-2).max(2).default(0),
-        reasoning_effort: z.enum(["low", "medium", "high"]).default("high"),
+    llm: LLMConfigSchema,
+    vendors: z.object({
+        openrouter: z.object({
+            api_key: z.string(),
+        }),
     }),
     analysis_workflows: z.array(AnalysisWorkflowEntrySchema),
     testing_workflows: z.array(TestingWorkflowEntrySchema),

package/src/util/config.ts

@@ -3,6 +3,7 @@ import { readFileSync, existsSync } from "node:fs";
 
 import {z} from "zod";
 
+import {ARGS} from "./args.ts";
 import {ConfigSchema} from "./config-schema.ts";
 
 
@@ -16,8 +17,8 @@ async function readConfig() {
     console.log(`Loading config`);
     
     let configFilePath: string;
-    if (process.argv.length >= 3) {
-        configFilePath = process.argv[2]!;
+    if (ARGS.values.config && ARGS.values.config.trim().length > 0) {
+        configFilePath = ARGS.values.config.trim();
         console.log(`Found config from command line argument: ${configFilePath}`);
     }
     else if (process.env[configURLEnvVar]) {
@@ -47,14 +48,18 @@ async function readConfig() {
         }
         configFileContents = await configResponse.text();
     } else {
-        console.log(`Loading config from file: ${configFilePath}`);
+        console.log(`Loading config from local file: ${configFilePath}`);
         configFileContents = readFileSync(configFilePath).toString();
     }
-
+    
+    console.assert(configFileContents.trim().length > 0, "Config file is empty");
+    
     let obj =  Bun.TOML.parse(configFileContents);
     const parsedConfig = ConfigSchema.safeParse(obj);
     if (!parsedConfig.success) {
         console.error("Config file is invalid:", parsedConfig.error.format());
+        console.log(`Config file contents:\n${configFileContents}`);
+        console.log(parsedConfig);
         throw new Error("Config file is invalid");
     }
     console.log(`Config loaded from ${configFilePath}`);

package/src/util/eval-harness.ts

@@ -0,0 +1,18 @@
+import { appendFile } from "node:fs/promises";
+
+import {ARGS} from "./args.ts";
+
+export async function recordCompletionInput(inputs: {role: string, content: any}[]) {
+    let completionInputsDestination = ARGS.values.completion_inputs_destination;
+    if (!completionInputsDestination) {
+        return;
+    }
+    
+    if (!(await Bun.file(completionInputsDestination).exists())){
+        console.warn(`Completion inputs destination file ${completionInputsDestination} does not exist`);
+        return;
+    }
+    
+    await appendFile(completionInputsDestination, JSON.stringify(completionInputsDestination)+"\n");
+    console.log(`Recorded completion inputs to ${completionInputsDestination}`);
+}

package/src/util/llm.ts

@@ -0,0 +1,80 @@
+import type {SystemMessage, UserMessage} from "@openrouter/sdk/models";
+
+import {CONFIG} from "./config.ts";
+import type {WorkflowDependencies} from "../workflow";
+import {recordCompletionInput} from "./eval-harness.ts";
+
+
+export async function generateCompletion(deps: WorkflowDependencies,
+                                         log: (..._: any[])=>void,
+                                         warn: (..._: any[])=>void,
+                                         model: string,
+                                         systemPrompt: string,
+                                         content: UserMessage["content"]) {
+    let modelSettings = CONFIG.llm.models[model];
+    if (!modelSettings) {
+        throw new Error(`No model settings found for model "${model}"`);
+    }
+    
+    let replacedCount = 0;
+    for (const [replacementKey, replacementValue] of Object.entries(CONFIG.llm.prompt_replacement)) {
+        if (systemPrompt.includes(replacementKey)) {replacedCount++}
+        systemPrompt = systemPrompt.replaceAll(`{{${replacementKey}}}`, replacementValue);
+        if (typeof content === "string") {
+            if (content.includes(replacementKey)) {replacedCount++}
+            content = content.replaceAll(`{{${replacementKey}}}`, replacementValue);
+        }
+        else {
+            for (let i = 0; i < content.length; i++) {
+                const element = content[i];
+                if (element && "type" in element && element.type === "text" && typeof element.text === "string") {
+                    if (element.text.includes(replacementKey)) {replacedCount++}
+                    content[i] = {
+                        ...element,
+                        text: element.text.replaceAll(`{{${replacementKey}}}`, replacementValue),
+                    }
+                }
+            }
+            
+        }
+    }
+    log(`Replaced ${replacedCount} instances of prompt variables in system prompt and content`);
+    
+    let messages: (SystemMessage | UserMessage)[] = [
+        {
+            role: "system",
+            content: systemPrompt,
+        },
+        {
+            role: "user",
+            content: content,
+        }
+    ];
+    setTimeout(async ()=> await recordCompletionInput(messages), 5);
+    
+    log("Sending chat completion request...");
+    let startTime = Date.now();
+    let completion = await deps.openRouter.chat.send({
+        model: modelSettings.model_name,
+        maxCompletionTokens: modelSettings.max_completion_tokens,
+        messages: messages,
+        stream: false,
+        seed: deps.seed,
+        frequencyPenalty: modelSettings.frequency_penalty,
+        presencePenalty: modelSettings.presence_penalty,
+        temperature: modelSettings.temperature,
+        reasoning: {
+            effort: modelSettings.reasoning_effort,
+        },
+    });
+    log(`Completion response generated in ${(Date.now() - startTime) / 1000} seconds`);
+    if (completion.choices.length < 1){
+        warn("No choices returned from completion");
+        console.log(completion);
+    }
+    
+    return {
+        text: completion.choices[0]?.message.content?.toString() ?? "",
+        model: completion.model,
+    };
+}

package/src/util/output-viewer.ts

@@ -0,0 +1,32 @@
+type FileRecord = {
+    type: "markdown" | "text";
+    content: string;
+}
+
+export class OutputViewer {
+    filesRecords: Record<string, FileRecord> = {};
+    
+    
+    addFile(filename: string, _: FileRecord): void {
+        this.filesRecords[filename] = _;
+    }
+    
+    display(): void {
+        let files = Object.entries(this.filesRecords).sort((a, b) => a[0].localeCompare(b[0]));
+        
+        let server = Bun.serve({
+            port: 0,
+            routes: {
+                "/:slug": (req) => {
+                    let slug = req.params.slug;
+                    console.log(`Request for slug: "${slug}"`);
+                    return new Response(this.filesRecords[slug]?.content ?? "Not Found");
+                }
+            },
+            fetch(req) {
+                return new Response("Not Found (fallback)", { status: 404 });
+            },
+        });
+        console.log(server.url);
+    }
+}

package/src/workflow/analysis-workflow.ts

@@ -1,9 +1,11 @@
 import {Glob} from "bun";
 
+import chalk from "chalk";
+
 import {CONFIG} from "../util/config.ts";
 import {FilePayloadGenerator} from "../util/file-payload.ts";
 import type {WorkflowDependencies} from "./index.ts";
-import chalk from "chalk";
+import {generateCompletion} from "../util/llm.ts";
 
 
 export async function executeAnalysisWorkflow(workflow: typeof CONFIG.analysis_workflows[number], runNum: number, deps: WorkflowDependencies) {
@@ -40,49 +42,18 @@ export async function executeAnalysisWorkflow(workflow: typeof CONFIG.analysis_w
     }
     log(`Found ${allFiles.length} files for workflow`);
     const fileContentsPayload = await FilePayloadGenerator.generatePayloads(allFiles);
-    
-    log("Sending chat completion request...");
-    let startTime = Date.now();
-    const seed = Math.floor(Date.now() / 1000);
-    let completion = await deps.openRouter.chat.send({
-        model: CONFIG.openrouter.model,
-        maxCompletionTokens: CONFIG.hyperparameters.max_completion_tokens,
-        messages: [
-            {
-                role: "system",
-                content: workflow.prompt,
-            },
-            {
-                role: "user",
-                content: fileContentsPayload.map((file) => {
-                    return {
-                        type: "text",
-                        text: file,
-                    }
-                }),
-            }
-        ],
-        stream: false,
-        seed: seed,
-        frequencyPenalty: CONFIG.hyperparameters.frequency_penalty,
-        presencePenalty: CONFIG.hyperparameters.presence_penalty,
-        temperature: CONFIG.hyperparameters.temperature,
-        reasoning: {
-            effort: CONFIG.hyperparameters.reasoning_effort,
-        },
-    });
-    log(`Completion response generated in ${(Date.now() - startTime) / 1000} seconds`);
-    if (completion.choices.length < 1){
-        warn("No choices returned from completion");
-        console.log(completion);
-    }
-    const completionText = completion.choices[0]?.message.content?.toString() ?? "";
-    // TODO: Add more template variables
+    const completion = await generateCompletion(deps, log, warn, workflow.model, workflow.prompt, fileContentsPayload.map((file) => {
+        return {
+            type: "text",
+            text: file,
+        }
+    }));
+
     const outputFileName = workflow.output_filename
-        .replaceAll("[seed]", seed.toString())
+        .replaceAll("[seed]", deps.seed.toString())
         .replaceAll("[slug]", workflow.slug)
         .replaceAll("[model]", `(${completion.model.replaceAll("/", "--")})`)
         .replaceAll("[run]", runNum.toString());
-    await Bun.write(outputFileName, completionText);
+    await Bun.write(outputFileName, completion.text);
     log(`Completion written to ${outputFileName}`);
 }

package/src/workflow/index.ts

@@ -1,5 +1,6 @@
 import type {OpenRouter} from "@openrouter/sdk";
 
 export type WorkflowDependencies = {
+    seed: number,
     openRouter: OpenRouter,
 }

package/src/workflow/testing-workflow.ts

@@ -4,6 +4,7 @@ import {CONFIG} from "../util/config.ts";
 import chalk from "chalk";
 import {LLMJudgeInputModeEnum} from "../util/config-schema.ts";
 import type {WorkflowDependencies} from "./index.ts";
+import {generateCompletion} from "../util/llm.ts";
 
 export async function executeTestingWorkflow(workflow: typeof CONFIG.testing_workflows[number], runNum: number, deps: WorkflowDependencies) {
     console.log(`Executing testing workflow: ${workflow.slug}`);
@@ -71,37 +72,11 @@ export async function executeTestingWorkflow(workflow: typeof CONFIG.testing_wor
                 switch (testCase.single_run_expected_output.llm_judge_input_mode) {
                     case LLMJudgeInputModeEnum.Full:
                         log("Evaluating full output with LLM judge...");
-                        const seed = Math.floor(Date.now() / 1000);
-                        let completion = await deps.openRouter.chat.send({
-                            model: CONFIG.openrouter.model,
-                            maxCompletionTokens: CONFIG.hyperparameters.max_completion_tokens,
-                            messages: [
-                                {
-                                    role: "system",
-                                    content: testCase.single_run_expected_output.llm_judge_prompt,
-                                },
-                                {
-                                    role: "user",
-                                    content: JSON.stringify({
-                                        "expected_output_substring": testCase.single_run_expected_output.substring,
-                                        "actual_output": commandOutput,
-                                    }),
-                                }
-                            ],
-                            stream: false,
-                            seed: seed,
-                            frequencyPenalty: CONFIG.hyperparameters.frequency_penalty,
-                            presencePenalty: CONFIG.hyperparameters.presence_penalty,
-                            temperature: 0,
-                            reasoning: {
-                                effort: CONFIG.hyperparameters.reasoning_effort,
-                            },
-                        });
-                        if (completion.choices.length < 1){
-                            warn("No choices returned from completion");
-                            console.log(completion);
-                        }
-                        const completionText = completion.choices[0]?.message.content?.toString() ?? "";
+                        const completion = await generateCompletion(deps, log, warn, workflow.model, testCase.single_run_expected_output.llm_judge_prompt, JSON.stringify({
+                            "expected_output_substring": testCase.single_run_expected_output.substring,
+                            "actual_output": commandOutput,
+                        }));
+                        const completionText = completion.text;
                         log(`LLM judge completion:\n${completionText}`);
                         const llmJudgeResult = completionText.toLowerCase().includes("pass");  // TODO: More robust parsing
                         if (llmJudgeResult) {