aimyapi 1.0.8 → 1.0.10

package/prompts/generate-code-prompt.md

@@ -0,0 +1,37 @@
+
+Most recent user query: 
+{{USER_QUERY}}
+
+---
+
+Based on the prior conversation, write a typescript script that performs the operation requested by the user and call the `submitCode` tool with the script. 
+
+## API documentation
+
+You can make use of this api:
+
+`api.ts`:
+```ts
+{{API}}
+```
+
+## Instructions
+
+Use the global object defined above to make calls to the API. The API runs in a development environment using mocked data.
+
+The provided tools will execute code in a sandbox and you will only have access to the API provided above and the standard javascript api's. 
+
+You may use top level `await` since your code will be wrapped in an async function before being run.
+You should write safe code and catch exceptions. For errors, use `console.error` with a helpful message.
+Avoid logging PII or sensitive information.
+
+You may use `runCode` to experiment or test. Console logs will be returned with the tool results, including 'traces' which show calls to the API.
+
+When finished, you must call `submitCode` to check and run the code. Examine the log traces that get returned to confirm that the expected calls were made and the code is running as expected. If not, you may call submit code again. Once finished, respond with "Successfully submitted code".
+
+---
+
+{{DOCUMENTATION}}
+
+{{CONTEXT}}
+

package/src/index.ts

@@ -2,16 +2,17 @@ require('dotenv').config()
 import fs from 'fs';
 import { OpenAI } from "openai";
 
+import { LogSource, SandboxLog } from "./logger";
 import createSandbox, { CodeRunResult } from "./sandbox";
 import { rejectOpenPromises } from './sandbox-wrappers';
 
 import { ChatCompletionMessageParam } from 'openai/resources/index.mjs';
-import { QuickJSWASMModule, getQuickJS } from "quickjs-emscripten";
+import { getQuickJS, QuickJSWASMModule } from "quickjs-emscripten";
 
 import { AutoParseableTool } from 'openai/lib/parser.js';
-import { ZodSchema, z } from 'zod';
-import { zodToJsonSchema } from 'zod-to-json-schema';
-import { check } from 'zod/v4';
+import { z } from 'zod';
+
+import path from 'path';
 
 const openai = new OpenAI( { baseURL: process.env.OPENAI_API_BASE_URL || undefined, apiKey: process.env.OPENAI_API_KEY || undefined } );
 
@@ -46,14 +47,13 @@ export interface GenerateCodeResult {
     comments?: string;
 }
 
+function createLog( message: string, source: LogSource) {
+    return { type: "log", args: [message], source } as SandboxLog
+}
 
 const wrapCode = (apiGlobalName:string, apiFileDefPath:string, code:string) => `
 (async () => {
-    try {
-        ${code}
-    } catch(err) { 
-        console.error(err); 
-    }
+${code}
 })();        
 `;
 
@@ -63,7 +63,7 @@ ${code}
 \`\`\`
 `;
 
-export function zodParseJSON<T>(schema: ZodSchema<T>) {
+export function zodParseJSON<T>(schema: { parse: any }) {
   return (input: string): T => schema.parse(JSON.parse(input));
 }
 
@@ -72,7 +72,7 @@ export function createBasePrompt(apiFilePath:string, apiGlobalName: string, docu
     const documentationText: string = documentationPath ? fs.readFileSync(documentationPath, 'utf8') : '';
 
     //console.time("Loading templates");
-    const createTaskPrompt: string = fs.readFileSync( './prompts/create-task-prompt.md', 'utf8').replace("{{DOCUMENTATION}}", documentationText).replace("{{API}}", apiText).replace("{{API_GLOBAL_NAME}}", apiGlobalName || "");   
+    const createTaskPrompt: string = fs.readFileSync( path.resolve(__dirname, '../prompts/generate-code-prompt.md'), 'utf8').replace("{{DOCUMENTATION}}", documentationText).replace("{{API}}", apiText).replace("{{API_GLOBAL_NAME}}", apiGlobalName || "");   
     //console.timeEnd("Loading templates")
     
     return createTaskPrompt;
@@ -99,7 +99,7 @@ export const generateCode = async function(instance:AIMyAPIInstance, queryText:s
         console.log("Generating code for query:", queryText);
     }
 
-    const newHistoryItems = [];
+    const newHistoryItems: any[] = [];
     // checkcode
     const codingTools: AutoParseableTool<any, true>[] = [
         {
@@ -111,7 +111,15 @@ export const generateCode = async function(instance:AIMyAPIInstance, queryText:s
                     if(debug) {
                         console.log("Running code...", code);                 
                     }
-                    
+                    const checkRes = await instance.checkCode(code);
+                    if(!checkRes.success) {
+                        return {
+                            success: false,
+                            reason: "Code check failed",
+                            logs: checkRes.logs,
+                        }
+                    }
+
                     const res = await instance.runCode(code);
 
                     if(hideLogsFromAgent) {
@@ -119,30 +127,30 @@ export const generateCode = async function(instance:AIMyAPIInstance, queryText:s
                     }
                     return res;
                 },
-                description: "Run the provided code in a sandboxed environment. The code will have access to the API object. Returns the result of the code execution.",
+                description: "Check and run the provided code in a sandboxed environment. The code will have access to the API object. Returns the result of the code execution. Use this for testing and experimentation.",
                 parse: zodParseJSON(runCodeSchema),
-                parameters: zodToJsonSchema(runCodeSchema),            
+                parameters: runCodeSchema.toJSONSchema(),
             }
         },
-        {
-            type: "function",
-            function: {
-                name: "checkCode",
-                // @ts-ignore
-                function: async ({code}: {code: string}) => {   
-                    if(debug) {
-                        console.log("Checking code...", code);                 
-                    }
-                    const res = await instance.checkCode(code);
+        // {
+        //     type: "function",
+        //     function: {
+        //         name: "checkCode",
+        //         // @ts-ignore
+        //         function: async ({code}: {code: string}) => {   
+        //             if(debug) {
+        //                 console.log("Checking code...", code);                 
+        //             }
+        //             const res = await instance.checkCode(code);
                     
-                    return res;
-
-                },
-                description: "Check the provided code for errors or issues without running it. Returns the result of the code check.",
-                parse: zodParseJSON(checkCodeSchema),
-                parameters: zodToJsonSchema(checkCodeSchema),
-            }
-        },        
+        //             return res;
+
+        //         },
+        //         description: "Check the provided code for errors or issues without running it. Returns the result of the code check.",
+        //         parse: zodParseJSON(checkCodeSchema),
+        //         parameters: checkCodeSchema.toJSONSchema(),
+        //     }
+        // },        
         {
             type: "function",
             function: {
@@ -158,14 +166,21 @@ export const generateCode = async function(instance:AIMyAPIInstance, queryText:s
                         }
                     }
 
-                    finalSubmittedCode = code;
-                    return {
-                        success: true,
+                    const res = await instance.runCode(code);
+
+                    if(hideLogsFromAgent) {
+                        res.logs = [];
+                    }
+
+                    if(res.success) {
+                        finalSubmittedCode = code;
                     }
+                    return res;                    
+
                 },
-                description: "Submit the provided code as the final code to run. This will end the code generation process and the code will be run in the sandbox. Use this when you are confident in the code you have generated and want to run it.",
+                description: "Submit the provided code as a candidate for the final code. The code will be checked and run and output logs and traces will be provided. Check the logs, if satisifed, respond to the user otherwise submit again.",
                 parse: zodParseJSON(submitCodeSchema),
-                parameters: zodToJsonSchema(submitCodeSchema),
+                parameters: submitCodeSchema.toJSONSchema(),
             }
         }
     ];
@@ -193,22 +208,23 @@ export const generateCode = async function(instance:AIMyAPIInstance, queryText:s
         })
         .on('message', async (message: ChatCompletionMessageParam) => {
             if(debug) {
-                console.log("Message:", JSON.stringify(message, null, 2));
+                if("reasoning_content" in message) {
+                    console.log(message.reasoning_content);
+                }
+                //console.log("Message:", JSON.stringify(message, null, 2));
             }
 
             newHistoryItems.push(message);
         })
         .on('functionToolCall', (functionCall: OpenAI.Chat.Completions.ChatCompletionMessageFunctionToolCall.Function) => {
-            if(debug) {
-                console.log("Function Call:", JSON.stringify(functionCall, null, 2));
-            }
-            
-
+            // if(debug) {
+            //     console.log("Function Call:", JSON.stringify(functionCall, null, 2));
+            // }
         })
         .on('functionToolCallResult', (functionCall) => {
-            if(debug) {
-                console.log("Function Call Result:", JSON.stringify(functionCall, null, 2));
-            }
+            // if(debug) {
+            //     console.log("Function Call Result:", JSON.stringify(functionCall, null, 2));
+            // }
         })
     
       const finalContent = await runner.finalContent();
@@ -333,7 +349,8 @@ async function createWithAPI(options:AIMyAPIOptions): Promise<AIMyAPIInstance> {
                     console.error("Unknown failure checking code");
                     return {
                         success: false,
-                        logs: [],
+                        logs: [createLog("Unknown failure checking code", "compiler")],
+
                     };
                 } 
         
@@ -386,7 +403,7 @@ async function createWithAPI(options:AIMyAPIOptions): Promise<AIMyAPIInstance> {
         
             // Create sandbox           
             // NOTE: not sure why I have to replace the backslash here
-            const {vm, runCode: runTask, isAsyncProcessRunning} = await createSandbox(QuickJS, apiExports ? {
+            const {vm, runCode: sandbox_runCode, isAsyncProcessRunning} = await createSandbox(QuickJS, apiExports ? {
                 [apiDefFilePath]: apiExports,
             } : undefined, { 
                 ...apiGlobals,
@@ -401,7 +418,7 @@ async function createWithAPI(options:AIMyAPIOptions): Promise<AIMyAPIInstance> {
                     console.log("Running Code:\n", code);
                 }
 
-                const res = await runTask(wrapCode(apiGlobalName, apiDefFilePath, code));
+                const res = await sandbox_runCode(wrapCode(apiGlobalName, apiDefFilePath, code));
         
                 if(!res) {
                     console.error("Unknown failure running task");
@@ -441,29 +458,36 @@ async function createWithAPI(options:AIMyAPIOptions): Promise<AIMyAPIInstance> {
             // Create prompt
             
             if(debug) {
-                console.log("Query:", userQuery);
+                console.log("[debug] Query:", userQuery);
             }
         
             // Generate Code
             console.time("Generate Task")
+            const prompt = createTaskPrompt.replace("{{CONTEXT}}", currentContext ? JSON.stringify(currentContext, null, 2) : "" );
             const generatedCode:GenerateCodeResult | null = await generateCode(instance, 
                 userQuery, 
                 [], 
-                createTaskPrompt.replace("{{CONTEXT}}", currentContext ? JSON.stringify(currentContext, null, 2) : "" ), 
+                prompt, 
                 debug, 
                 hideLogsFromAgent, 
                 model, 
                 additionalModelOptions
             );
             console.timeEnd("Generate Task")
+
+            console.log(`generated code result\nSuccess?: ${generatedCode?.success}\nComments: ${generatedCode?.comments}\nRunning Code...`);
         
-            console.time("Run Code")
+
+            console.time("Run Code took")
             if(generatedCode?.code) {
-                await instance.runCode(generatedCode.code);
+                const codeRunResult = await instance.runCode(generatedCode.code);
+                if(!codeRunResult.success) {
+                    console.error("Error running generated code");
+                }
             } else {
                 console.error("No code generated", generatedCode);
             }
-            console.timeEnd("Run Code")
+            console.timeEnd("Run Code took")
 
             return generatedCode;
         }

package/src/log-tester-inject.ts

@@ -0,0 +1,233 @@
+type LogSource = "sandbox" | "compiler" | "runner";
+type LogEntryType = "log" | "error" | "trace";
+
+type SandboxLog = { 
+  type: "log" | "error"; 
+  args: any[]; 
+  source: LogSource 
+} | {
+  type: "trace";
+  fnName: string,
+  fnArgs: any[],
+  result: any
+};
+
+const has = Object.prototype.hasOwnProperty;
+
+function dequal(foo: any, bar: any) {
+    let ctor, len;
+    if (foo === bar) return true;
+
+    if (foo && bar && (ctor=foo.constructor) === bar.constructor) {
+        if (ctor === Date) return foo.getTime() === bar.getTime();
+        if (ctor === RegExp) return foo.toString() === bar.toString();
+
+        if (ctor === Array) {
+            if ((len=foo.length) === bar.length) {
+                while (len-- && dequal(foo[len], bar[len]));
+            }
+            return len === -1;
+        }
+
+        if (!ctor || typeof foo === 'object') {
+            len = 0;
+            for (ctor in foo) {
+                if (has.call(foo, ctor) && ++len && !has.call(bar, ctor)) return false;
+                if (!(ctor in bar) || !dequal(foo[ctor], bar[ctor])) return false;
+            }
+            return Object.keys(bar).length === len;
+        }
+    }
+
+    return foo !== foo && bar !== bar;
+}
+
+/** Return true to pass. Returning a falsy value or throwing fails the check. */
+type ArgumentsEvaluator = (args: any[]) => any;
+/** Return true to pass. Returning a falsy value or throwing fails the check. */
+type ResultEvaluator = (result: any) => any;
+
+/**
+ * One step in a test sequence. Gets the logs and the current position, checks
+ * the entries there, and returns the position of the next unchecked entry.
+ * Throws TestFailed if the check fails.
+ */
+type TestStep = (logs: SandboxLog[], index: number) => number;
+
+class TestFailed extends Error {
+    constructor(message: string) {
+        super(message);
+        this.name = "TestFailed";
+    }
+}
+
+function describeEntry(entry: SandboxLog | undefined): string {
+    if (!entry) return "end of logs";
+    const short = (v: any) => JSON.stringify(v);
+    if (entry.type === "trace") {
+        return `trace ${entry.fnName}(${entry.fnArgs.map(short).join(", ")}) -> ${short(entry.result)}`;
+    }
+    return `${entry.type} [${entry.source}] ${entry.args.map(short).join(" ")}`;
+}
+
+function nameMatches(pattern: string, name: string): boolean {
+    return pattern === "*" || pattern === name;
+}
+
+/** Runs an evaluator and throws TestFailed if it returns falsy or throws. */
+function evaluate(evaluator: ((value: any) => any) | undefined, value: any, what: string, index: number, entry: SandboxLog) {
+    if (!evaluator) return;
+    let passed: any;
+    try {
+        passed = evaluator(value);
+    } catch (e: any) {
+        throw new TestFailed(`Log #${index}: ${what} check threw: ${e?.message ?? e}. Entry: ${describeEntry(entry)}`);
+    }
+    if (!passed) {
+        throw new TestFailed(`Log #${index}: ${what} check failed. Entry: ${describeEntry(entry)}`);
+    }
+}
+
+function checkTrace(logs: SandboxLog[], index: number, functionName: string, argumentsEvaluator?: ArgumentsEvaluator, resultEvaluator?: ResultEvaluator) {
+    const entry = logs[index];
+    if (!entry || entry.type !== "trace" || !nameMatches(functionName, entry.fnName)) {
+        throw new TestFailed(`Log #${index}: expected trace for "${functionName}", got ${describeEntry(entry)}`);
+    }
+    evaluate(argumentsEvaluator, entry.fnArgs, `arguments of "${entry.fnName}"`, index, entry);
+    evaluate(resultEvaluator, entry.result, `result of "${entry.fnName}"`, index, entry);
+}
+
+function expectEntryType(type: "log" | "error", argumentsEvaluator?: ArgumentsEvaluator): TestStep {
+    return (logs, index) => {
+        const entry = logs[index];
+        if (!entry || entry.type !== type) {
+            throw new TestFailed(`Log #${index}: expected ${type}, got ${describeEntry(entry)}`);
+        }
+        evaluate(argumentsEvaluator, entry.args, `${type} arguments`, index, entry);
+        return index + 1;
+    };
+}
+
+/**
+ * Checks the logs from a sandbox run against an expected sequence of entries.
+ *
+ * @example
+ * test.sequence(
+ *     test.expectTrace("getSalesData"),
+ *     test.expectTrace("display", (args) => args[0].includes("The month with the highest sales was")),
+ * );
+ */
+class LogTester {
+    private source: SandboxLog[];
+
+    constructor(source: SandboxLog[]) {
+        this.source = source;
+    }
+
+    private get logs(): SandboxLog[] {
+        return this.source;
+    }
+
+    /**
+     * Runs the steps in order, starting at the first log entry. Throws TestFailed
+     * on the first failing step. Entries after the last step are ignored.
+     * Accepts steps as separate arguments or as one array.
+     */
+    sequence(...steps: (TestStep | TestStep[])[]): true {
+        const logs = this.logs;
+        let index = 0;
+        for (const step of steps.flat()) {
+            if (typeof step !== "function") {
+                throw new TestFailed(`sequence() expects test steps (e.g. test.expectTrace(...)), got ${typeof step}`);
+            }
+            index = step(logs, index);
+        }
+        return true;
+    }
+
+    /** Next entry must be a trace for `functionName` ("*" for any function). */
+    expectTrace(functionName: string, argumentsEvaluator?: ArgumentsEvaluator, resultEvaluator?: ResultEvaluator): TestStep {
+        return (logs, index) => {
+            checkTrace(logs, index, functionName, argumentsEvaluator, resultEvaluator);
+            return index + 1;
+        };
+    }
+
+    /** Next entry can be anything, but there must be one. */
+    expectAny(): TestStep {
+        return (logs, index) => {
+            if (index >= logs.length) {
+                throw new TestFailed(`Log #${index}: expected any entry, got end of logs`);
+            }
+            return index + 1;
+        };
+    }
+
+    /** Skips ahead to just before the next trace for `functionName` ("*" for any). Fails if there is none. */
+    skipToTrace(functionName: string): TestStep {
+        return (logs, index) => {
+            for (let i = index; i < logs.length; i++) {
+                const entry = logs[i];
+                if (entry.type === "trace" && nameMatches(functionName, entry.fnName)) {
+                    return i;
+                }
+            }
+            throw new TestFailed(`Log #${index}: no trace for "${functionName}" found from here to the end of the logs`);
+        };
+    }
+
+    /**
+     * Consecutive traces for `functionName` ("*" for any).
+     * With `num` > 0, exactly that many must be next (more are left for later steps).
+     * With `num` 0 or undefined, takes as many as are next, including none.
+     * The evaluators are checked against every matched trace.
+     */
+    expectMultipleTraces(functionName: string, num?: number, argumentsEvaluator?: ArgumentsEvaluator, resultEvaluator?: ResultEvaluator): TestStep {
+        return (logs, index) => {
+            if (num && num > 0) {
+                for (let n = 0; n < num; n++) {
+                    const entry = logs[index + n];
+                    if (!entry || entry.type !== "trace" || !nameMatches(functionName, entry.fnName)) {
+                        throw new TestFailed(`Log #${index + n}: expected ${num} traces for "${functionName}" but found ${n}. Got ${describeEntry(entry)}`);
+                    }
+                    checkTrace(logs, index + n, functionName, argumentsEvaluator, resultEvaluator);
+                }
+                return index + num;
+            }
+
+            let i = index;
+            while (i < logs.length) {
+                const entry = logs[i];
+                if (entry.type !== "trace" || !nameMatches(functionName, entry.fnName)) break;
+                checkTrace(logs, i, functionName, argumentsEvaluator, resultEvaluator);
+                i++;
+            }
+            return i;
+        };
+    }
+
+    /** Next entry must be a log (console.log etc). The evaluator gets the logged arguments. */
+    expectLog(argumentsEvaluator?: ArgumentsEvaluator): TestStep {
+        return expectEntryType("log", argumentsEvaluator);
+    }
+
+    /** Next entry must be an error. The evaluator gets the error arguments. */
+    expectError(argumentsEvaluator?: ArgumentsEvaluator): TestStep {
+        return expectEntryType("error", argumentsEvaluator);
+    }
+
+    /** Deep equality. Returns true if equal, otherwise throws TestFailed, so it works inside evaluators. */
+    equals(actual: any, expected: any): true {
+        if (!dequal(actual, expected)) {
+            throw new TestFailed(`Expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`);
+        }
+        return true;
+    }
+}
+
+
+// const test = new LogTester(logs);
+// test.sequence(
+//     test.expectTrace("display", (args) => args[0][0].sales == 100)
+// )
+// console.log("success");