editcodewithai 0.0.1 → 0.1.0

package/README.md

@@ -1,5 +1,6 @@
 # editcodewithai
-Edit Code With AI 
+
+Edit Code With AI
 
 ## A Simple Open Source AI Code Editor
 
@@ -21,30 +22,77 @@ npm install
 ## Usage
 
 ```typescript
-import { performAiEdit } from 'editcodewithai';
+import { performAiEdit } from "editcodewithai";
+import { VizFiles } from "@vizhub/viz-types";
+
+// Define your LLM function that will process the prompt
+const myLlmFunction = async (prompt: string) => {
+  // Call your preferred LLM API here
+  // This example assumes using OpenRouter
+  const response = await fetch(
+    "https://openrouter.ai/api/v1/chat/completions",
+    {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${apiKey}`,
+      },
+      body: JSON.stringify({
+        model: "openai/gpt-4",
+        messages: [{ role: "user", content: prompt }],
+      }),
+    }
+  );
+
+  const data = await response.json();
+  return {
+    content: data.choices[0].message.content,
+    generationId: data.id,
+  };
+};
+
+// Your files
+const files: VizFiles = {
+  file1: {
+    name: "index.js",
+    text: "console.log('Hello world');",
+  },
+};
 
+// Perform the AI edit
 const result = await performAiEdit({
   prompt: "Update the code to use async/await",
-  modelName: "openai/gpt-4",
-  files: {
-    // Your files object
-  },
+  files: files,
+  llmFunction: myLlmFunction,
   apiKey: "your-openrouter-api-key",
-  baseURL: "https://openrouter.ai/api/v1"
 });
 ```
 
-The `files` object should be a map of file IDs to file objects containing:
+### Parameters
+
+The `performAiEdit` function accepts the following parameters:
+
+- `prompt`: A string containing instructions for the AI
+- `files`: A `VizFiles` object (map of file IDs to file objects)
+- `llmFunction`: A function that takes a prompt string and returns a Promise with the LLM response
+- `apiKey`: Your OpenRouter API key for retrieving cost metadata
+
+Each file in the `files` object should contain:
+
 - `name`: The filename (e.g. "index.js")
 - `text`: The file contents as a string
 
-The function returns:
+### Return Value
+
+The function returns an object with:
+
 - `changedFiles`: Updated files with modifications
 - `openRouterGenerationId`: ID of the generation
 - `upstreamCostCents`: Cost in cents
 - `provider`: The AI provider used
 - `inputTokens`: Number of input tokens
 - `outputTokens`: Number of output tokens
+- `promptTemplateVersion`: Version of the prompt template used
 
 ## Similar open source projects
 

package/dist/benchmark.d.ts

@@ -0,0 +1,37 @@
+import { LlmFunction } from "./types";
+/**
+ * Creates an OpenRouter LLM function with the provided API key and model using LangChain
+ * @param apiKey OpenRouter API key
+ * @param model Model to use (defaults to "anthropic/claude-3.5-sonnet")
+ * @returns LLM function that can be used with performAiEdit
+ */
+export declare function createOpenRouterLlmFunction(apiKey: string, model?: string): LlmFunction;
+/**
+ * Runs a benchmark test using OpenRouter to implement an "add" function
+ * @param apiKey OpenRouter API key
+ * @param model Optional model to use (defaults to "openai/gpt-4")
+ * @returns Benchmark results including the implementation and performance metrics
+ */
+export declare function runAddFunctionBenchmark(apiKey: string, model?: string): Promise<{
+    success: boolean;
+    implementation: any;
+    isCorrect: boolean;
+    testOutput: any;
+    elapsedTime: number;
+    costCents: any;
+    provider: any;
+    inputTokens: any;
+    outputTokens: any;
+    error?: undefined;
+} | {
+    success: boolean;
+    error: string;
+    implementation?: undefined;
+    isCorrect?: undefined;
+    testOutput?: undefined;
+    elapsedTime?: undefined;
+    costCents?: undefined;
+    provider?: undefined;
+    inputTokens?: undefined;
+    outputTokens?: undefined;
+}>;

package/dist/benchmark.js

@@ -0,0 +1,123 @@
+import { performAiEdit } from "./index";
+import { ChatOpenAI } from "@langchain/openai";
+import { StringOutputParser } from "@langchain/core/output_parsers";
+/**
+ * Creates an OpenRouter LLM function with the provided API key and model using LangChain
+ * @param apiKey OpenRouter API key
+ * @param model Model to use (defaults to "anthropic/claude-3.5-sonnet")
+ * @returns LLM function that can be used with performAiEdit
+ */
+export function createOpenRouterLlmFunction(apiKey, model = "anthropic/claude-3.5-sonnet") {
+    return async (prompt) => {
+        try {
+            const options = {
+                modelName: model,
+                configuration: {
+                    apiKey,
+                    baseURL: "https://openrouter.ai/api/v1",
+                },
+                streaming: false,
+            };
+            const chatModel = new ChatOpenAI(options);
+            const result = await chatModel.invoke(prompt);
+            const parser = new StringOutputParser();
+            const resultString = await parser.invoke(result);
+            return {
+                content: resultString,
+                generationId: Date.now().toString(), // OpenRouter doesn't return an ID through LangChain
+            };
+        }
+        catch (error) {
+            if (error instanceof Error) {
+                throw new Error(`OpenRouter API error: ${error.message}`);
+            }
+            else {
+                throw new Error(`OpenRouter API error: ${String(error)}`);
+            }
+        }
+    };
+}
+/**
+ * Runs a benchmark test using OpenRouter to implement an "add" function
+ * @param apiKey OpenRouter API key
+ * @param model Optional model to use (defaults to "openai/gpt-4")
+ * @returns Benchmark results including the implementation and performance metrics
+ */
+export async function runAddFunctionBenchmark(apiKey, model = "openai/gpt-4") {
+    console.log(`Running benchmark with model: ${model}`);
+    // Create test file with empty add function
+    const files = {
+        file1: {
+            name: "index.js",
+            text: "function add(a, b) {\n  // TODO: Implement this function\n}\n\nmodule.exports = { add };",
+        },
+    };
+    const prompt = "Implement the 'add' function to add two numbers together and return the result.";
+    // Create LLM function
+    const llmFunction = createOpenRouterLlmFunction(apiKey, model);
+    console.log("Sending request to OpenRouter...");
+    const startTime = Date.now();
+    try {
+        // Perform the AI edit
+        const result = await performAiEdit({
+            prompt,
+            files,
+            llmFunction,
+            apiKey,
+        });
+        const endTime = Date.now();
+        const elapsedTime = (endTime - startTime) / 1000;
+        // Extract the implementation
+        const implementation = result.changedFiles.file1?.text || "";
+        // Validate the implementation
+        let isCorrect = false;
+        let testOutput = null;
+        try {
+            // Create a function from the implementation to test it
+            const funcStr = implementation + "\nreturn add(3, 4);";
+            const testFunc = new Function(funcStr);
+            testOutput = testFunc();
+            isCorrect = testOutput === 7;
+        }
+        catch (error) {
+            console.error("Error testing implementation:", error);
+        }
+        return {
+            success: true,
+            implementation,
+            isCorrect,
+            testOutput,
+            elapsedTime,
+            costCents: result.upstreamCostCents,
+            provider: result.provider,
+            inputTokens: result.inputTokens,
+            outputTokens: result.outputTokens,
+        };
+    }
+    catch (error) {
+        console.error("Benchmark failed:", error);
+        return {
+            success: false,
+            error: error instanceof Error ? error.message : String(error),
+        };
+    }
+}
+/**
+ * Example usage of the benchmark function
+ */
+// Check if this file is being executed directly (CommonJS approach)
+if (require.main === module) {
+    // This code runs when the file is executed directly
+    const apiKey = process.env.OPENROUTER_API_KEY;
+    if (!apiKey) {
+        console.error("Please set the OPENROUTER_API_KEY environment variable");
+        process.exit(1);
+    }
+    runAddFunctionBenchmark(apiKey)
+        .then((result) => {
+        console.log("Benchmark result:", JSON.stringify(result, null, 2));
+    })
+        .catch((error) => {
+        console.error("Error running benchmark:", error);
+    });
+}

package/dist/benchmarks/benchmarkRunner.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Main function: runs the challenges on different models, calls the AI to fill them in,
+ * executes each test, and writes pass/fail results to CSV.
+ */
+export declare function runAllChallenges(): Promise<void>;

package/dist/benchmarks/benchmarkRunner.js

@@ -0,0 +1,85 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runAllChallenges = runAllChallenges;
+const file_system_1 = require("langchain/cache/file_system");
+const index_1 = require("../index");
+const llm_1 = require("./llm");
+const fileUtils_1 = require("./fileUtils");
+const runner_1 = require("./runner");
+const challenges_1 = require("./challenges");
+const models_1 = require("./models");
+/**
+ * Main function: runs the challenges on different models, calls the AI to fill them in,
+ * executes each test, and writes pass/fail results to CSV.
+ */
+async function runAllChallenges() {
+    const apiKey = process.env.OPENROUTER_API_KEY || "";
+    if (!apiKey) {
+        throw new Error("Please set OPENROUTER_API_KEY in your .env");
+    }
+    // Setup cache
+    const cacheDirBase = "benchmarks/llmResponseCache";
+    (0, fileUtils_1.ensureCacheDir)(cacheDirBase);
+    const cache = await file_system_1.LocalFileCache.create(cacheDirBase);
+    // Store results
+    const results = [];
+    // Run each model against each challenge
+    for (const model of models_1.models) {
+        try {
+            await runModelChallenges(model, apiKey, cache, results);
+        }
+        catch (error) {
+            console.error(`Error running model ${model}:`, error);
+            // Add failure results for remaining challenges
+            for (const challenge of challenges_1.challenges) {
+                results.push({
+                    challenge: challenge.name,
+                    model,
+                    passFail: "error",
+                });
+            }
+        }
+    }
+    // Write results to CSV
+    const csv = (0, fileUtils_1.writeResultsToCsv)(results, "benchmarks/results.csv");
+    console.log("\nAll challenges completed. See 'benchmarks/results.csv' for summary.\n");
+    console.log(csv);
+}
+/**
+ * Runs all challenges for a specific model
+ */
+async function runModelChallenges(model, apiKey, cache, results) {
+    // Create the LLM function for this model
+    const llmFunction = (0, llm_1.createOpenRouterLlmFunction)(model, apiKey, cache);
+    // Run each challenge for this model
+    for (const challenge of challenges_1.challenges) {
+        console.log(`\n=== Challenge: ${challenge.name} | Model: ${model} ===`);
+        try {
+            // Ask AI to fill out the placeholder TODOs
+            const aiResult = await (0, index_1.performAiEdit)({
+                prompt: challenge.prompt,
+                files: challenge.files,
+                llmFunction,
+            });
+            // Write returned files to disk, including LLM response
+            const challengeDir = (0, fileUtils_1.writeChallengeFiles)(challenge.name, model, aiResult.changedFiles, aiResult.rawResponse);
+            // Run index.js in a child process
+            const exitCode = (0, runner_1.runNodeTest)(challengeDir);
+            const passFail = exitCode === 0 ? "pass" : "fail";
+            // Record the result
+            results.push({
+                challenge: challenge.name,
+                model,
+                passFail,
+            });
+        }
+        catch (error) {
+            console.error(`Error running challenge ${challenge.name}:`, error);
+            results.push({
+                challenge: challenge.name,
+                model,
+                passFail: "error",
+            });
+        }
+    }
+}

package/dist/benchmarks/challenges.d.ts

@@ -0,0 +1,2 @@
+import { Challenge } from "./types";
+export declare const challenges: Challenge[];

package/dist/benchmarks/challenges.js

@@ -0,0 +1,158 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.challenges = void 0;
+// Challenges definition. Each has multiple files (some with TODO).
+// "index.mjs" includes the test code that calls process.exit(1) if it fails.
+exports.challenges = [
+    {
+        name: "add",
+        prompt: "Implement the 'add' function to correctly add two numbers (a+b) and pass the test in index.mjs.",
+        files: {
+            file1: {
+                name: "index.mjs",
+                text: `
+import { add } from "./functions.mjs";
+
+// A simple test:
+const result = add(3, 4);
+if (result !== 7) {
+  console.error("Test failed: expected 7, got", result);
+  process.exit(1);
+}
+console.log("Add test passed");
+process.exit(0);
+        `,
+            },
+            file2: {
+                name: "functions.mjs",
+                text: `
+// TODO: Implement the add function
+export function add(a, b) {
+  // TODO
+}
+        `,
+            },
+        },
+    },
+    {
+        name: "multiply",
+        prompt: "Implement the 'multiply' function to correctly multiply two numbers and pass the unit test in index.mjs.",
+        files: {
+            file1: {
+                name: "index.mjs",
+                text: `
+import { multiply } from "./functions.mjs";
+
+const result = multiply(6, 7);
+if (result !== 42) {
+  console.error("Test failed: expected 42, got", result);
+  process.exit(1);
+}
+console.log("Multiply test passed");
+process.exit(0);
+      `,
+            },
+            file2: {
+                name: "functions.mjs",
+                text: `
+// TODO: Implement the multiply function
+export function multiply(a, b) {
+  // TODO
+}
+      `,
+            },
+        },
+    },
+    {
+        name: "square",
+        prompt: "Implement the 'square' function that returns x*x.",
+        files: {
+            file1: {
+                name: "index.mjs",
+                text: `
+import { square } from "./functions.mjs";
+
+const input = 5;
+const result = square(input);
+if (result !== 25) {
+  console.error("Test failed: expected 25, got", result);
+  process.exit(1);
+}
+console.log("Square test passed");
+process.exit(0);
+      `,
+            },
+            file2: {
+                name: "functions.mjs",
+                text: `
+// TODO: Implement the square function
+export function square(x) {
+  // TODO
+}
+      `,
+            },
+        },
+    },
+    {
+        name: "toUpperCase",
+        prompt: "Implement the toUpperCase function that returns the given string in uppercase.",
+        files: {
+            file1: {
+                name: "index.mjs",
+                text: `
+import { toUpperCase } from "./functions.mjs";
+
+const input = "hello";
+const result = toUpperCase(input);
+if (result !== "HELLO") {
+  console.error("Test failed: expected 'HELLO', got", result);
+  process.exit(1);
+}
+console.log("toUpperCase test passed");
+process.exit(0);
+      `,
+            },
+            file2: {
+                name: "functions.mjs",
+                text: `
+// TODO: Implement the toUpperCase function
+export function toUpperCase(str) {
+  // TODO
+}
+      `,
+            },
+        },
+    },
+    {
+        name: "reverseString",
+        prompt: "Implement the reverseString function that reverses the given string.",
+        files: {
+            file1: {
+                name: "index.mjs",
+                text: `
+import { reverseString } from "./functions.mjs";
+
+const input = "OpenAI";
+const expected = "IAnepO";
+
+const result = reverseString(input);
+if (result !== expected) {
+  console.error("Test failed: expected", expected, "but got", result);
+  process.exit(1);
+}
+console.log("reverseString test passed");
+process.exit(0);
+      `,
+            },
+            file2: {
+                name: "functions.mjs",
+                text: `
+// TODO: Implement the reverseString function
+export function reverseString(str) {
+  // TODO
+}
+      `,
+            },
+        },
+    },
+];

package/dist/benchmarks/fileUtils.d.ts

@@ -0,0 +1,15 @@
+import { VizFiles } from "@vizhub/viz-types";
+import { ChallengeResult } from "./types";
+/**
+ * Writes the files for a given challenge into a folder named after challenge.name + model
+ * e.g. "benchmarks/challenges/add/gpt4" to keep them separate per model.
+ */
+export declare function writeChallengeFiles(challengeName: string, model: string, changedFiles: VizFiles, llmResponse?: string): string;
+/**
+ * Writes results to a CSV file
+ */
+export declare function writeResultsToCsv(results: ChallengeResult[], filePath?: string): string;
+/**
+ * Ensures a cache directory exists
+ */
+export declare function ensureCacheDir(cacheDirBase: string): void;

package/dist/benchmarks/fileUtils.js

@@ -0,0 +1,55 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.writeChallengeFiles = writeChallengeFiles;
+exports.writeResultsToCsv = writeResultsToCsv;
+exports.ensureCacheDir = ensureCacheDir;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+/**
+ * Writes the files for a given challenge into a folder named after challenge.name + model
+ * e.g. "benchmarks/challenges/add/gpt4" to keep them separate per model.
+ */
+function writeChallengeFiles(challengeName, model, changedFiles, llmResponse) {
+    const challengeDir = path_1.default.join("benchmarks", "challenges", challengeName, model);
+    if (!fs_1.default.existsSync(challengeDir)) {
+        fs_1.default.mkdirSync(challengeDir, { recursive: true });
+    }
+    Object.keys(changedFiles).forEach((fileKey) => {
+        const { name, text } = changedFiles[fileKey];
+        const filePath = path_1.default.join(challengeDir, name);
+        fs_1.default.writeFileSync(filePath, text, "utf-8");
+    });
+    // Write the LLM response to a file if provided
+    if (llmResponse) {
+        const responsePath = path_1.default.join(challengeDir, "llmResponse.md");
+        fs_1.default.writeFileSync(responsePath, llmResponse, "utf-8");
+    }
+    return challengeDir;
+}
+/**
+ * Writes results to a CSV file
+ */
+function writeResultsToCsv(results, filePath = "benchmarks/results.csv") {
+    let csv = "challenge,model,passFail\n";
+    for (const r of results) {
+        csv += `${r.challenge},${r.model},${r.passFail}\n`;
+    }
+    // Ensure benchmarks directory exists
+    const dir = path_1.default.dirname(filePath);
+    if (!fs_1.default.existsSync(dir)) {
+        fs_1.default.mkdirSync(dir, { recursive: true });
+    }
+    fs_1.default.writeFileSync(filePath, csv, "utf-8");
+    return csv;
+}
+/**
+ * Ensures a cache directory exists
+ */
+function ensureCacheDir(cacheDirBase) {
+    if (!fs_1.default.existsSync(cacheDirBase)) {
+        fs_1.default.mkdirSync(cacheDirBase, { recursive: true });
+    }
+}

package/dist/benchmarks/index.d.ts

@@ -0,0 +1,5 @@
+export { runAllChallenges } from "./benchmarkRunner";
+export { challenges } from "./challenges";
+export { createOpenRouterLlmFunction } from "./llm";
+export { runNodeTest } from "./runner";
+export { writeChallengeFiles, writeResultsToCsv } from "./fileUtils";

package/dist/benchmarks/index.js

@@ -0,0 +1,44 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.writeResultsToCsv = exports.writeChallengeFiles = exports.runNodeTest = exports.createOpenRouterLlmFunction = exports.challenges = exports.runAllChallenges = void 0;
+/**********************************************************************
+ * benchmark.ts
+ * --------------------------------------------------------------------
+ * Requires environment variables to be set in a .env file
+ * --------------------------------------------------------------------
+ * Defines 5 challenges, each with index.js (ES module) that performs
+ * a simple unit test. The code calls process.exit(0) on success or
+ * process.exit(1) on failure.
+ *
+ * Runs all challenges on 5 different models:
+ *    ["gpt3", "gpt4", "gpt5", "deepseek", "qwen32"]
+ *
+ * Writes final results to "results.csv" with columns:
+ *    challenge, model, passFail, exitCode
+ **********************************************************************/
+const dotenv_1 = __importDefault(require("dotenv"));
+const benchmarkRunner_1 = require("./benchmarkRunner");
+// Load environment variables from .env file
+dotenv_1.default.config();
+// Run if invoked directly
+if (require.main === module) {
+    (0, benchmarkRunner_1.runAllChallenges)().catch((error) => {
+        console.error("Error running challenges:", error);
+        process.exit(1);
+    });
+}
+// Re-export for programmatic usage
+var benchmarkRunner_2 = require("./benchmarkRunner");
+Object.defineProperty(exports, "runAllChallenges", { enumerable: true, get: function () { return benchmarkRunner_2.runAllChallenges; } });
+var challenges_1 = require("./challenges");
+Object.defineProperty(exports, "challenges", { enumerable: true, get: function () { return challenges_1.challenges; } });
+var llm_1 = require("./llm");
+Object.defineProperty(exports, "createOpenRouterLlmFunction", { enumerable: true, get: function () { return llm_1.createOpenRouterLlmFunction; } });
+var runner_1 = require("./runner");
+Object.defineProperty(exports, "runNodeTest", { enumerable: true, get: function () { return runner_1.runNodeTest; } });
+var fileUtils_1 = require("./fileUtils");
+Object.defineProperty(exports, "writeChallengeFiles", { enumerable: true, get: function () { return fileUtils_1.writeChallengeFiles; } });
+Object.defineProperty(exports, "writeResultsToCsv", { enumerable: true, get: function () { return fileUtils_1.writeResultsToCsv; } });

package/dist/benchmarks/llm.d.ts

@@ -0,0 +1,10 @@
+import { LlmFunction } from "../types";
+import { LocalFileCache } from "langchain/cache/file_system";
+/**
+ * Creates an LLM function that connects to OpenRouter
+ * @param model The model identifier to use
+ * @param apiKey OpenRouter API key
+ * @param cache Optional local file cache to avoid duplicate requests
+ * @returns A function that can be used with performAiEdit
+ */
+export declare function createOpenRouterLlmFunction(model: string, apiKey: string, cache?: LocalFileCache): LlmFunction;

package/dist/benchmarks/llm.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createOpenRouterLlmFunction = createOpenRouterLlmFunction;
+const openai_1 = require("@langchain/openai");
+const output_parsers_1 = require("@langchain/core/output_parsers");
+/**
+ * Creates an LLM function that connects to OpenRouter
+ * @param model The model identifier to use
+ * @param apiKey OpenRouter API key
+ * @param cache Optional local file cache to avoid duplicate requests
+ * @returns A function that can be used with performAiEdit
+ */
+function createOpenRouterLlmFunction(model, apiKey, cache) {
+    return async (prompt) => {
+        // Create OpenAI chat model with OpenRouter configuration
+        const chatModel = new openai_1.ChatOpenAI({
+            modelName: model,
+            configuration: { apiKey, baseURL: "https://openrouter.ai/api/v1" },
+            streaming: false,
+            cache,
+        });
+        // Invoke the model
+        const result = await chatModel.invoke(prompt);
+        // Parse to string
+        const parser = new output_parsers_1.StringOutputParser();
+        const resultString = await parser.invoke(result);
+        return { content: resultString, generationId: result.lc_kwargs.id };
+    };
+}

package/dist/benchmarks/models.d.ts

@@ -0,0 +1 @@
+export declare const models: string[];