even-pf 0.3.4 → 0.4.0

package/README.md

@@ -8,6 +8,11 @@ To run:
 bunx --bun even-pf [config]
 ```
 
+The `config` can be an absolute or relative path. Can also be a URL. 
+If not specified, it will first try to get `EPF_CONFIG_URL` environment variable, then look for `epf.toml` in the current and the home directory.
+
+If you're using the tool in a resource-constrained environment, you can use platform-specific executables like [even-pf-linux-x64](https://www.npmjs.com/package/even-pf-linux-x64).
+
 ## Development
 To install dependencies:
 ```bash
@@ -16,13 +21,18 @@ bun install
 
 To install as a tool globally:
 ```bash
-bun link
+bun link -g e-pf
 ```
 
 Make sure you have a config file in your home or current directory. Alternatively, you can set environment variable `EPF_CONFIG_URL`.
 
 This project was created using `bun init` in bun v1.3.2. [Bun](https://bun.com) is a fast all-in-one JavaScript runtime.
 
+After making changes, you might want to bump the version.
+```bash
+bun run bump <semver>
+```
+
 ## Specs
 ### File-viewer Frontend
 In consideration of the tool might be running at a remote server, for easily viewing the Markdown files, we will use a simple file-viewer frontend.

package/bun.lock

@@ -14,11 +14,11 @@
         "@types/bun": "latest",
       },
       "optionalDependencies": {
-        "even-pf-darwin-arm64": "0.3.3",
-        "even-pf-darwin-x64": "0.3.3",
-        "even-pf-linux-arm64": "0.3.3",
-        "even-pf-linux-x64": "0.3.3",
-        "even-pf-windows-x64": "0.3.3",
+        "even-pf-darwin-arm64": "0.3.4",
+        "even-pf-darwin-x64": "0.3.4",
+        "even-pf-linux-arm64": "0.3.4",
+        "even-pf-linux-x64": "0.3.4",
+        "even-pf-windows-x64": "0.3.4",
       },
       "peerDependencies": {
         "typescript": "^5.9.3",
@@ -36,15 +36,15 @@
 
     "chalk": ["chalk@5.6.2", "", {}, "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA=="],
 
-    "even-pf-darwin-arm64": ["even-pf-darwin-arm64@0.3.3", "", { "os": "darwin", "cpu": "arm64", "bin": { "even-pf": "bin/even-pf" } }, "sha512-pwgjCNFXqECN16/WeGxYN3xU9Zd/VXQD/ntmFqXZExIo8u+Wauo2AxgzRFEXtQXkO5Uf8GcCkRlUbJT+l8ZIuQ=="],
+    "even-pf-darwin-arm64": ["even-pf-darwin-arm64@0.3.4", "", { "os": "darwin", "cpu": "arm64", "bin": { "even-pf": "bin/even-pf" } }, "sha512-x2vTM0ogvlFhUiHqb13kXJTKPRPU/VdoZa1G51c3IHsZz7wdDpkD/DxcEvxAmO28MbJtfjxig8nRFMvld5J6jg=="],
 
-    "even-pf-darwin-x64": ["even-pf-darwin-x64@0.3.3", "", { "os": "darwin", "cpu": "x64", "bin": { "even-pf": "bin/even-pf" } }, "sha512-WUZsqjDSjIJajoeOP8Lt/i4+of9hjxKLgGX4jg6tBpf/FEsnohLhg0tWzyCGGlOFV6t44Ug3Ke/H1eA4MCZAAQ=="],
+    "even-pf-darwin-x64": ["even-pf-darwin-x64@0.3.4", "", { "os": "darwin", "cpu": "x64", "bin": { "even-pf": "bin/even-pf" } }, "sha512-L2hzBvSLFcWMB/MJQeZTQHI8mqpGMQ7T0tSPXjv4S1tFglF8ZtdxggDAhmItEyyqVfsAT6LY+HyOpJnUAga9tg=="],
 
-    "even-pf-linux-arm64": ["even-pf-linux-arm64@0.3.3", "", { "os": "linux", "cpu": "arm64", "bin": { "even-pf": "bin/even-pf" } }, "sha512-DPPMH2xpRqsv7MpjrzV1t0c3QpL4nqd2tuWQp+G3dZFgi9Cd7xk3O4i4C1e0iyrPXVgZs1oE03TJM/9l3Br07Q=="],
+    "even-pf-linux-arm64": ["even-pf-linux-arm64@0.3.4", "", { "os": "linux", "cpu": "arm64", "bin": { "even-pf": "bin/even-pf" } }, "sha512-/5nLtKs+8xvTHEkrVPQQ5XQBTKROmF42z6+fo4AOkOj/TbDGwCher6RYYMHQ6pD7M0jjF5AdSlj5HLEGf/N9Qg=="],
 
-    "even-pf-linux-x64": ["even-pf-linux-x64@0.3.3", "", { "os": "linux", "cpu": "x64", "bin": { "even-pf": "bin/even-pf" } }, "sha512-1hXDMHqvX7iG6wPEL6CxVoS/F9Hz6hkiHccBPu0zTXV9Zircic95tOMRK5Z9OwG0g6nztJXEljJTi9qIl4uBJw=="],
+    "even-pf-linux-x64": ["even-pf-linux-x64@0.3.4", "", { "os": "linux", "cpu": "x64", "bin": { "even-pf": "bin/even-pf" } }, "sha512-UN0wz2svjcjckugzFyc4tHxllrTM7IScSmnLDq5z9AB5cplHZrvAg8cYcvz20YEcHsr7aUkxrhA7iDv5KKYhkA=="],
 
-    "even-pf-windows-x64": ["even-pf-windows-x64@0.3.3", "", { "os": "win32", "cpu": "x64", "bin": { "even-pf": "bin/even-pf.exe" } }, "sha512-4hPLGNr4ZZ9h71PypVbT0pjpwLWFhveZ09budw/5A7P/12UE/BhJ+aemHEN1Ge9IJ2gPXNkLa2KSA6q99Zr/Rw=="],
+    "even-pf-windows-x64": ["even-pf-windows-x64@0.3.4", "", { "os": "win32", "cpu": "x64", "bin": { "even-pf": "bin/even-pf.exe" } }, "sha512-ni84uLUdo95TlACDUyz7Ia7+4wigSByvUuR+IrXbLzkN90mZTsJoZVbAoJMR8CnOlPPEClcPHqkTcYl1lbLOwA=="],
 
     "smol-toml": ["smol-toml@1.6.0", "", {}, "sha512-4zemZi0HvTnYwLfrpk/CF9LOd9Lt87kAt50GnqhMpyF9U3poDAP2+iukq2bZsO/ufegbYehBkqINbsWxj4l4cw=="],
 

package/epf.example.toml

@@ -1,3 +1,8 @@
+[output_viewing]
+mode = "webui"
+api_port = 0
+webui_base_url = "https://ta-tools-dashboard.vercel.app"
+
 [llm.models.general_analysis]
 sdk = "openrouter"
 model_name = ""

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "even-pf",
-  "version": "0.3.4",
+  "version": "0.4.0",
   "description": "AI-assisted responsible grading tool for programming assignments",
   "module": "src/cli.ts",
   "type": "module",
@@ -15,7 +15,8 @@
     "config-gen": "bun run --console-depth 6 src/generate-config.ts"
   },
   "bin": {
-    "even-pf": "bin/even-pf.js"
+    "even-pf": "bin/even-pf.js",
+    "e-pf": "src/cli.ts"
   },
   "devDependencies": {
     "@types/bun": "latest"
@@ -30,11 +31,11 @@
     "zod-defaults": "^0.2.3"
   },
   "optionalDependencies": {
-    "even-pf-linux-x64": "0.3.4",
-    "even-pf-linux-arm64": "0.3.4",
-    "even-pf-windows-x64": "0.3.4",
-    "even-pf-darwin-x64": "0.3.4",
-    "even-pf-darwin-arm64": "0.3.4"
+    "even-pf-linux-x64": "0.4.0",
+    "even-pf-linux-arm64": "0.4.0",
+    "even-pf-windows-x64": "0.4.0",
+    "even-pf-darwin-x64": "0.4.0",
+    "even-pf-darwin-arm64": "0.4.0"
   },
   "files": [
     "bin/even-pf.js",

package/src/cli.ts

@@ -35,7 +35,7 @@ testingWorkflows.forEach((workflow) => {
         workflowRuns.push(executeTestingWorkflow(workflow, i+1, workflowDependencies));
     }
 });
-
+workflowDependencies.outputViewer.display();  // For start the server early.
 const workflowsResults = await Promise.allSettled(workflowRuns);
 // Summarize with indices to include slugs in failure logs
 const failedIndices: number[] = [];

package/src/util/config-schema.ts

@@ -1,80 +1,92 @@
-import {z} from "zod";
-
-export const ModelConfigSchema = z.object({
-    sdk: z.enum(["openrouter"]).default("openrouter"),
-    model_name: z.string().default(""),
-    max_completion_tokens: z.number().min(1).default(20000),
-    temperature: z.number().min(0).max(1).default(0.9),
-    top_p: z.number().min(0).max(1).default(1),
-    frequency_penalty: z.number().min(-2).max(2).default(0),
-    presence_penalty: z.number().min(-2).max(2).default(0),
-    reasoning_effort: z.enum(["low", "medium", "high"]).default("high"),
-});
-
-export const LLMConfigSchema = z.object({
-    models: z.record(z.string(), ModelConfigSchema),
-    prompt_replacement: z.record(z.string(), z.string()),
-});
-
-export const FileSearchEntrySchema = z.object({
-    file_glob: z.string().min(1),
-    search_directory: z.string().default("."),
-    excluded_files: z.array(z.string()).default([]),
-});
-
-export const BaseWorkflowEntrySchema = z.object({
-    slug: z.string(),
-    model: z.string().default("general_analysis"),
-    runs: z.number().min(1).default(1),
-    input_files_searches: z.array(FileSearchEntrySchema).default([]),
-    output_filename: z.string().min(1),
-});
-
-export const AnalysisWorkflowEntrySchema = BaseWorkflowEntrySchema.extend({
-    prompt: z.string(),
-})
-
-export enum LLMJudgeInputModeEnum{
-    None = "NONE",
-    Diff = "DIFF",
-    Full = "FULL",
-}
-const LLMJudgeInputModeSchema = z.enum(LLMJudgeInputModeEnum);
-
-const ExpectedOutputSchema = z.object({
-    prefix_strip_string: z.string().min(0),
-    postfix_strip_string: z.string().min(0),
-    substring: z.string().min(0),
-    llm_judge_input_mode: LLMJudgeInputModeSchema.default(LLMJudgeInputModeEnum.None),
-    llm_judge_prompt: z.string().min(0),
-});
-
-export const TestCaseSchema = z.object({
-    name: z.string(),
-    work_directory: z.string().default("."),
-    single_run_command: z.string(),
-    single_run_expected_output: ExpectedOutputSchema,
-    interactive_steps: z.array(z.object({
-        input: z.string(),
-        expected_output: ExpectedOutputSchema,
-    })),
-});
-
-export const TestingWorkflowEntrySchema = BaseWorkflowEntrySchema.extend({
-    setup_commands: z.array(z.string()).default([]),
-    test_cases: z.array(TestCaseSchema).default([]),
-    cleanup_commands: z.array(z.string()).default([]),
-}).omit({
-    input_files_searches: true,
-});
-
-export const ConfigSchema = z.object({
-    llm: LLMConfigSchema,
-    vendors: z.object({
-        openrouter: z.object({
-            api_key: z.string(),
-        }),
-    }),
-    analysis_workflows: z.array(AnalysisWorkflowEntrySchema),
-    testing_workflows: z.array(TestingWorkflowEntrySchema),
-});
+import {z} from "zod";
+
+export enum OutputViewingModeEnum {
+    Local = "local",
+    WebUI = "webui",
+}
+
+export const OutputViewingConfigSchema = z.object({
+    mode: z.enum(OutputViewingModeEnum).default(OutputViewingModeEnum.WebUI),
+    api_port: z.number().min(0).max(65535).default(0), // 0 means random available port
+    webui_base_url: z.string().default("https://ta-tools-dashboard.vercel.app"),
+});
+
+export const ModelConfigSchema = z.object({
+    sdk: z.enum(["openrouter"]).default("openrouter"),
+    model_name: z.string().default(""),
+    max_completion_tokens: z.number().min(1).default(20000),
+    temperature: z.number().min(0).max(1).default(0.9),
+    top_p: z.number().min(0).max(1).default(1),
+    frequency_penalty: z.number().min(-2).max(2).default(0),
+    presence_penalty: z.number().min(-2).max(2).default(0),
+    reasoning_effort: z.enum(["low", "medium", "high"]).default("high"),
+});
+
+export const LLMConfigSchema = z.object({
+    models: z.record(z.string(), ModelConfigSchema),
+    prompt_replacement: z.record(z.string(), z.string()),
+});
+
+export const FileSearchEntrySchema = z.object({
+    file_glob: z.string().min(1),
+    search_directory: z.string().default("."),
+    excluded_files: z.array(z.string()).default([]),
+});
+
+export const BaseWorkflowEntrySchema = z.object({
+    slug: z.string(),
+    model: z.string().default("general_analysis"),
+    runs: z.number().min(1).default(1),
+    input_files_searches: z.array(FileSearchEntrySchema).default([]),
+    output_filename: z.string().min(1),
+});
+
+export const AnalysisWorkflowEntrySchema = BaseWorkflowEntrySchema.extend({
+    prompt: z.string(),
+})
+
+export enum LLMJudgeInputModeEnum{
+    None = "NONE",
+    Diff = "DIFF",
+    Full = "FULL",
+}
+const LLMJudgeInputModeSchema = z.enum(LLMJudgeInputModeEnum);
+
+const ExpectedOutputSchema = z.object({
+    prefix_strip_string: z.string().min(0),
+    postfix_strip_string: z.string().min(0),
+    substring: z.string().min(0),
+    llm_judge_input_mode: LLMJudgeInputModeSchema.default(LLMJudgeInputModeEnum.None),
+    llm_judge_prompt: z.string().min(0),
+});
+
+export const TestCaseSchema = z.object({
+    name: z.string(),
+    work_directory: z.string().default("."),
+    single_run_command: z.string(),
+    single_run_expected_output: ExpectedOutputSchema,
+    interactive_steps: z.array(z.object({
+        input: z.string(),
+        expected_output: ExpectedOutputSchema,
+    })),
+});
+
+export const TestingWorkflowEntrySchema = BaseWorkflowEntrySchema.extend({
+    setup_commands: z.array(z.string()).default([]),
+    test_cases: z.array(TestCaseSchema).default([]),
+    cleanup_commands: z.array(z.string()).default([]),
+}).omit({
+    input_files_searches: true,
+});
+
+export const ConfigSchema = z.object({
+    output_viewing: OutputViewingConfigSchema,
+    llm: LLMConfigSchema,
+    vendors: z.object({
+        openrouter: z.object({
+            api_key: z.string(),
+        }),
+    }),
+    analysis_workflows: z.array(AnalysisWorkflowEntrySchema).default([]),
+    testing_workflows: z.array(TestingWorkflowEntrySchema).default([]),
+});

package/src/util/output-viewer.ts

@@ -1,54 +1,119 @@
-import chalk from "chalk";
-
-type FileRecord = {
-    type: "markdown" | "text";
-    content: string;
-}
-
-export class OutputViewer {
-    filesRecords: Record<string, FileRecord> = {};
-    
-    
-    addFile(filename: string, _: FileRecord): void {
-        this.filesRecords[filename] = _;
-    }
-    
-    private serve(): void {
-        let files = Object.entries(this.filesRecords).sort((a, b) => a[0].localeCompare(b[0]));
-        
-        let server = Bun.serve({
-            port: 0,
-            routes: {
-                "/:slug": (req) => {
-                    let slug = req.params.slug;
-                    console.log(`Request for slug: "${slug}"`);
-                    return new Response(this.filesRecords[slug]?.content ?? "Not Found");
-                }
-            },
-            fetch(req) {
-                return new Response("Not Found (fallback)", { status: 404 });
-            },
-        });
-        console.log(server.url);
-    }
-    
-    display() {
-        if (Object.keys(this.filesRecords).length === 0) {
-            console.warn("No files to display");
-            return;
-        }
-        
-        console.log("Click the following links to view the outputs in your browser:");
-        
-        const FRONTEND_URL = "https://ta-tools-dashboard.vercel.app/tools/md-viewer"; //TODO: not hardcode this
-        let files = Object.entries(this.filesRecords).sort((a, b) => a[0].localeCompare(b[0]));
-        for (const [filename, fileRecord] of files) {
-            let params = new URLSearchParams();
-            params.set("name", filename);
-            params.set("comp", "gzip");
-            params.set("data", Bun.gzipSync(fileRecord.content).toBase64());
-            let url = `${FRONTEND_URL}#${params.toString()}`;
-            console.log(`${chalk.cyan(filename)}: ${url}` + "\n");
-        }
-    }
-}
+import chalk from "chalk";
+
+import {CONFIG} from "./config.ts";
+import {OutputViewingModeEnum} from "./config-schema.ts";
+
+type FileRecord = {
+    type: "markdown" | "text";
+    content: string;
+}
+
+const CORS_HEADERS = {
+    "Access-Control-Allow-Origin": "*",
+    "Access-Control-Allow-Methods": "GET, OPTIONS",
+    "Access-Control-Allow-Headers": "Content-Type",
+};
+
+function jsonResponse(data: unknown, status = 200): Response {
+    return new Response(JSON.stringify(data), {
+        status,
+        headers: {
+            "Content-Type": "application/json",
+            ...CORS_HEADERS,
+        },
+    });
+}
+
+export class OutputViewer {
+    filesRecords: Record<string, FileRecord> = {};
+    displayed: boolean = false;
+    
+    addFile(filename: string, _: FileRecord): void {
+        this.filesRecords[filename] = _;
+    }
+    
+    serve(): string {
+        let files = Object.entries(this.filesRecords).sort((a, b) => a[0].localeCompare(b[0]));
+        
+        let server = Bun.serve({
+            port: CONFIG.output_viewing.api_port,
+            routes: {
+                "/": (req) => {
+                    if (req.method === "OPTIONS") {
+                        return new Response(null, { status: 204, headers: CORS_HEADERS });
+                    }
+                    return jsonResponse({
+                        files: files.map(([filename, fileRecord]) => ({
+                            name: filename,
+                            type: fileRecord.type,
+                        })),
+                    });
+                },
+                "/:slug": (req) => {
+                    if (req.method === "OPTIONS") {
+                        return new Response(null, { status: 204, headers: CORS_HEADERS });
+                    }
+                    let slug = req.params.slug;
+                    let record = this.filesRecords[slug];
+                    if (!record) {
+                        return jsonResponse({ error: "Not Found" }, 404);
+                    }
+                    return jsonResponse({
+                        name: slug,
+                        type: record.type,
+                        content: record.content,
+                    });
+                }
+            },
+            fetch(req) {
+                if (req.method === "OPTIONS") {
+                    return new Response(null, { status: 204, headers: CORS_HEADERS });
+                }
+                return jsonResponse({ error: "Not Found" }, 404);
+            },
+        });
+        console.log(server.url);
+        return server.url.toString();
+    }
+    
+    display() {
+        let frontendURL = "";
+        switch (CONFIG.output_viewing.mode) {
+            case OutputViewingModeEnum.Local:
+                if (Object.keys(this.filesRecords).length === 0) {
+                    console.warn("No files to display (you can probably ignore this warning if your workflows haven't completed yet)");
+                    return;
+                }
+                
+                console.log("Click the following links to view the outputs in your browser:");
+                
+                let files = Object.entries(this.filesRecords).sort((a, b) => a[0].localeCompare(b[0]));
+                for (const [filename, fileRecord] of files) {
+                    let params = new URLSearchParams();
+                    params.set("name", filename);
+                    params.set("comp", "gzip");
+                    params.set("data", Bun.gzipSync(fileRecord.content).toBase64());
+                    frontendURL = `${CONFIG.output_viewing.webui_base_url}/tools/results-viewer#${params.toString()}`;
+                    console.log(`${chalk.cyan(filename)}: ${frontendURL}` + "\n");
+                }
+                break
+            case OutputViewingModeEnum.WebUI:
+                if (this.displayed){
+                    console.log("Output viewer API is already running");
+                    console.log(frontendURL + "\n");
+                    console.log("Press Ctrl+C to stop")
+                    return;
+                }
+                this.displayed = true;
+                let apiURL = this.serve();
+                let params = new URLSearchParams();
+                params.set("api", apiURL);
+                frontendURL = `${CONFIG.output_viewing.webui_base_url}/tools/results-viewer#${params.toString()}`;
+                
+                console.log(chalk.cyan("Open the following URL to view all outputs:"));
+                console.log(frontendURL);
+                console.log("Press Ctrl+C to stop the server")
+        }
+        
+    }
+}

package/src/version.ts

@@ -1,5 +1,6 @@
 // Injected at compile time by Bun.build({ define }) in scripts/build-all.ts
 declare const EPF_VERSION: string;
 
-console.log(`even-pf v${EPF_VERSION}`);
+const version = typeof EPF_VERSION !== "undefined" ? `v${EPF_VERSION}` : "dev";
+console.log(`even-pf ${version}`);