agency-lang 0.0.52 → 0.0.54

package/dist/lib/backends/agencyGenerator.d.ts

@@ -15,6 +15,7 @@ import { WhileLoop } from "../types/whileLoop.js";
 import { BaseGenerator } from "./baseGenerator.js";
 import { AgencyConfig } from "../config.js";
 import { MessageThread } from "../types/messageThread.js";
+import { Skill } from "../types/skill.js";
 export declare class AgencyGenerator extends BaseGenerator {
     private indentLevel;
     private indentSize;
@@ -64,5 +65,6 @@ export declare class AgencyGenerator extends BaseGenerator {
     protected processAwaitStatement(node: AwaitStatement): string;
     protected processNewLine(_node: NewLine): string;
     protected processMessageThread(node: MessageThread): string;
+    protected processSkill(node: Skill): string;
 }
 export declare function generateAgency(program: AgencyProgram): string;

package/dist/lib/backends/agencyGenerator.js

@@ -141,7 +141,10 @@ export class AgencyGenerator extends BaseGenerator {
             }
         }
         result += '"""';
-        return result;
+        return result
+            .split("\n")
+            .map((line) => this.indentStr(line))
+            .join("\n");
     }
     processPromptLiteral(variableName, variableType, node) {
         // For agency code, prompts are just part of assignments
@@ -380,8 +383,6 @@ export class AgencyGenerator extends BaseGenerator {
         return "";
     }
     processUsesTool(node) {
-        // Track tool usage but don't generate code
-        this.toolsUsed.push(...node.toolNames);
         return this.indentStr(`uses ${node.toolNames.join(", ")}`);
     }
     processSpecialVar(node) {
@@ -408,6 +409,9 @@ export class AgencyGenerator extends BaseGenerator {
         const threadType = node.subthread ? "subthread" : "thread";
         return this.indentStr(`${threadType} {\n${bodyCodeStr}${this.indentStr("}")}`);
     }
+    processSkill(node) {
+        return this.indentStr(`skill "${node.filepath}"`);
+    }
 }
 export function generateAgency(program) {
     const generator = new AgencyGenerator();

package/dist/lib/backends/baseGenerator.d.ts

@@ -14,6 +14,7 @@ import { UsesTool } from "../types/tools.js";
 import { WhileLoop } from "../types/whileLoop.js";
 import { AgencyConfig } from "../config.js";
 import { MessageThread } from "../types/messageThread.js";
+import { Skill } from "../types/skill.js";
 type Scope = GlobalScope | FunctionScope | NodeScope;
 type GlobalScope = {
     type: "global";
@@ -34,7 +35,6 @@ export declare class BaseGenerator {
     protected functionScopedVariables: string[];
     protected globalScopedVariables: string[];
     protected functionParameters: string[];
-    protected toolsUsed: string[];
     protected typeAliases: Record<string, VariableType>;
     protected functionsUsed: Set<string>;
     protected importStatements: string[];
@@ -59,6 +59,7 @@ export declare class BaseGenerator {
     protected collectFunctionSignature(node: FunctionDefinition): void;
     protected processGraphNodeName(node: GraphNodeDefinition): void;
     protected processNode(node: AgencyNode): string;
+    protected processSkill(node: Skill): string;
     protected processMessageThread(node: MessageThread): string;
     protected processNewLine(_node: NewLine): string;
     protected processAwaitStatement(node: AwaitStatement): string;

package/dist/lib/backends/baseGenerator.js

@@ -7,8 +7,6 @@ export class BaseGenerator {
     functionScopedVariables = [];
     globalScopedVariables = [];
     functionParameters = [];
-    // collect tools for a prompt
-    toolsUsed = [];
     typeAliases = {};
     // collect functions used to see what builtin helpers to include
     functionsUsed = new Set();
@@ -175,10 +173,15 @@ export class BaseGenerator {
                 return node.value;
             case "messageThread":
                 return this.processMessageThread(node);
+            case "skill":
+                return this.processSkill(node);
             default:
                 throw new Error(`Unhandled Agency node type: ${node.type}`);
         }
     }
+    processSkill(node) {
+        return "processSkill not implemented";
+    }
     processMessageThread(node) {
         return "processMessageThread not implemented";
     }

package/dist/lib/backends/graphGenerator.js

@@ -46,7 +46,7 @@ export class GraphGenerator extends TypeScriptGenerator {
                     return `return ${returnCode}\n`;
                 }
             }
-            return `return { ...state, data: ${returnCode}}\n`;
+            return `return { messages: __self.messages_0, data: ${returnCode}}\n`;
         }
     }
     processGraphNodeName(node) {

package/dist/lib/backends/typescriptGenerator.d.ts

@@ -14,6 +14,7 @@ import { WhileLoop } from "../types/whileLoop.js";
 import { BaseGenerator } from "./baseGenerator.js";
 import { AgencyConfig } from "../config.js";
 import { MessageThread } from "../types/messageThread.js";
+import { Skill } from "../types/skill.js";
 export declare class TypeScriptGenerator extends BaseGenerator {
     protected currentMessageThreadNodeId: string[];
     constructor(args?: {
@@ -51,7 +52,11 @@ export declare class TypeScriptGenerator extends BaseGenerator {
     protected generateLiteral(literal: Literal): string;
     protected getScopeReturnType(): VariableType | undefined;
     protected generateImports(): string;
-    buildPromptString(segments: PromptSegment[], typeHints: TypeHintMap): string;
+    buildPromptString({ segments, typeHints, skills, }: {
+        segments: PromptSegment[];
+        typeHints: TypeHintMap;
+        skills: Skill[];
+    }): string;
     generateStringLiteral(segments: PromptSegment[]): string;
     /**
      * Generates an async for prompt-based assignments
@@ -71,6 +76,7 @@ export declare class TypeScriptGenerator extends BaseGenerator {
     protected processTimeBlock(node: TimeBlock, timingVarName: string): string;
     protected processAwaitStatement(node: AwaitStatement): string;
     protected processMessageThread(node: MessageThread, varName?: string): string;
+    protected processSkill(node: Skill): string;
     protected processBodyAsParts(body: AgencyNode[]): string[];
 }
 export declare function generateTypeScript(program: AgencyProgram, config?: AgencyConfig): string;

package/dist/lib/backends/typescriptGenerator.js

@@ -1,4 +1,4 @@
-import { TYPES_THAT_DONT_TRIGGER_NEW_PART } from "../config.js";
+import { BUILTIN_TOOLS, TYPES_THAT_DONT_TRIGGER_NEW_PART } from "../config.js";
 import * as renderSpecialVar from "../templates/backends/graphGenerator/specialVar.js";
 import * as renderTime from "../templates/backends/typescriptGenerator/builtinFunctions/time.js";
 import * as builtinTools from "../templates/backends/typescriptGenerator/builtinTools.js";
@@ -10,12 +10,14 @@ import * as renderMessageThread from "../templates/backends/typescriptGenerator/
 import * as promptFunction from "../templates/backends/typescriptGenerator/promptFunction.js";
 import * as renderTool from "../templates/backends/typescriptGenerator/tool.js";
 import * as renderToolCall from "../templates/backends/typescriptGenerator/toolCall.js";
+import * as renderSkillPrompt from "../templates/prompts/skill.js";
 import { escape, uniq } from "../utils.js";
 import { BaseGenerator } from "./baseGenerator.js";
 import { generateBuiltinHelpers, mapFunctionName, } from "./typescriptGenerator/builtins.js";
 import { variableTypeToString } from "./typescriptGenerator/typeToString.js";
 import { DEFAULT_SCHEMA, mapTypeToZodSchema, } from "./typescriptGenerator/typeToZodSchema.js";
 import { TypescriptPreprocessor } from "../preprocessors/typescriptPreprocessor.js";
+import path from "path";
 const DEFAULT_PROMPT_NAME = "__promptVar";
 export class TypeScriptGenerator extends BaseGenerator {
     currentMessageThreadNodeId = ["0"];
@@ -220,15 +222,18 @@ export class TypeScriptGenerator extends BaseGenerator {
             name: functionName,
             description: node.docString?.value || "No description provided.",
             schema: Object.keys(properties).length > 0 ? `{${schema}}` : "{}",
+            parameters: JSON.stringify(parameters.map((p) => p.name)),
         });
     }
     processUsesTool(node) {
         node.toolNames.forEach((toolName) => {
-            if (!this.functionDefinitions[toolName]) {
+            if (BUILTIN_TOOLS.includes(toolName))
+                return;
+            if (!this.functionDefinitions[toolName] &&
+                !this.isImportedTool(toolName)) {
                 throw new Error(`Tool '${toolName}' is being used but no function definition found for it. Make sure to define a function for this tool.`);
             }
         });
-        this.toolsUsed.push(...node.toolNames);
         return "";
     }
     /**
@@ -348,7 +353,7 @@ export class TypeScriptGenerator extends BaseGenerator {
         arr.push(builtinTools.default({}));
         return arr.join("\n");
     }
-    buildPromptString(segments, typeHints) {
+    buildPromptString({ segments, typeHints, skills, }) {
         const promptParts = [];
         for (const segment of segments) {
             if (segment.type === "text") {
@@ -368,6 +373,21 @@ export class TypeScriptGenerator extends BaseGenerator {
                 }
             }
         }
+        if (skills.length > 0) {
+            const skillsArr = skills.map((skill) => {
+                // strip the directory and extension from the filepath to get the skill name
+                const skillName = path.basename(skill.filepath, path.extname(skill.filepath));
+                if (skill.description) {
+                    return `- ${skillName} (filepath: ${skill.filepath}): ${skill.description}`;
+                }
+                else {
+                    return `- ${skillName} (filepath: ${skill.filepath})`;
+                }
+            });
+            promptParts.push(renderSkillPrompt.default({
+                skills: skillsArr.join("\n"),
+            }));
+        }
         return "`" + promptParts.join("") + "`";
     }
     generateStringLiteral(segments) {
@@ -397,28 +417,50 @@ export class TypeScriptGenerator extends BaseGenerator {
         const zodSchema = mapTypeToZodSchema(_variableType, this.typeAliases);
         const typeString = variableTypeToString(_variableType, this.typeAliases);
         // Build prompt construction code
-        const promptCode = this.buildPromptString(prompt.segments, this.typeHints);
+        const promptCode = this.buildPromptString({
+            segments: prompt.segments,
+            typeHints: this.typeHints,
+            skills: prompt.skills || [],
+        });
         const parts = functionArgs.map((arg) => `${arg.replace(".", "_")}: ${variableTypeToString(this.typeHints[arg] || { type: "primitiveType", value: "string" }, this.typeAliases)}`);
         parts.push("__metadata?: Record<string, any>");
         const argsStr = parts.join(", ");
-        const _tools = this.toolsUsed
-            .map((toolName) => `__${toolName}Tool`)
-            .join(", ");
+        let _tools = "";
+        if (prompt.tools) {
+            _tools = prompt.tools.toolNames.map((name) => `__${name}Tool`).join(", ");
+        }
         const tools = _tools.length > 0 ? `[${_tools}]` : "undefined";
-        const functionCalls = this.toolsUsed
+        /* What's going on here? This is a nine. We change all agency functions to take an array of arguments.
+    So, for example, `function add(a, b)` would get turned into `function add(arr)`.
+    Earlier, the arguments were getting converted into an object like `function add({a, b})` because LLMs
+    pass back an object of parameters for the function calls, so that made it easy to pass those arguments
+    straight to a function.
+    
+    But that meant if a function was defined in another file, we would need to parse the contents of the file
+    to understand the names of the function parameters, hence the idea of an array. Unfortunately, the array
+    still doesn't solve the problem completely, because if what we get back from a tool call is an object
+    like `{a:1, b:2}`, we need to know how to convert it into an array. We need to know the order of the parameters.
+    
+    So now, every function call generates a related tool definition, but it also generats an array of parameter names.
+    So if the user defines `add`, we'll generate `export const __addTool`, which is the tool definition,
+    and `export const __addToolParams`, which is an array of param names.
+    
+    It's a mess and needs rethinking 🤮
+    
+    This still doesn't work perfectly, because I can't introspect imported functions to see if they throw an interrupt,
+    so I have to always assume they do, which reduces some opportunity for parallelism. Maybe I just need to bite the bullet
+    and commit to having a preprocessed step where all the files get read.
+    I'll probably need to do that for supporting type checking anyway.
+    */
+        const functionCalls = (prompt.tools || { type: "usesTool", toolNames: [] }).toolNames
+            .filter((t) => BUILTIN_TOOLS.includes(t))
             .map((toolName) => {
-            const func = this.functionDefinitions[toolName];
-            if (!func) {
+            if (!this.functionDefinitions[toolName] &&
+                !this.isImportedTool(toolName)) {
                 throw new Error(`Tool '${toolName}' is being used but no function definition found for it. Make sure to define a function for this tool.`);
             }
-            const paramsStr = func.parameters
-                .map((param, index) => {
-                return `args["${param.name}"]`;
-            })
-                .join(", ");
             return renderToolCall.default({
                 name: toolName,
-                paramsStr,
             });
         })
             .join("\n");
@@ -426,7 +468,6 @@ export class TypeScriptGenerator extends BaseGenerator {
         const metadataObj = `{
       messages: __self.messages_${this.currentMessageThreadNodeId.at(-1)}.getMessages(),
     }`;
-        this.toolsUsed = []; // reset after use
         const scopedFunctionArgs = functionArgs.map((arg) => this.generateScopedVariableName(arg));
         return promptFunction.default({
             variableName,
@@ -442,6 +483,7 @@ export class TypeScriptGenerator extends BaseGenerator {
             nodeContext: this.getCurrentScope().type === "node",
             isStreaming: prompt.isStreaming || false,
             isAsync: prompt.async || false,
+            messagesVar: `__self.messages_${this.currentMessageThreadNodeId.at(-1)}`,
         });
     }
     processImportStatement(node) {
@@ -452,7 +494,11 @@ export class TypeScriptGenerator extends BaseGenerator {
     }
     processImportToolStatement(node) {
         const importNames = node.importedTools
-            .map((toolName) => [toolName, `__${toolName}Tool`])
+            .map((toolName) => [
+            toolName,
+            `__${toolName}Tool`,
+            `__${toolName}ToolParams`,
+        ])
             .flat();
         return `import { ${importNames.join(", ")} } from "${node.agencyFile.replace(/\.agency$/, ".ts")}";`;
     }
@@ -530,6 +576,9 @@ export class TypeScriptGenerator extends BaseGenerator {
             parentNodeId: node.parentNodeId || "0",
         });
     }
+    processSkill(node) {
+        return "";
+    }
     /* This generates the body of a node or function separated into multiple parts.
     You can think of a part as roughly corresponding to a single statement
     (although some statements don't need their own parts, such as a newlines or type definitions).

package/dist/lib/cli/commands.d.ts

@@ -0,0 +1,12 @@
+import { AgencyConfig } from "../config.js";
+import { AgencyProgram } from "../index.js";
+export declare function loadConfig(configPath?: string): AgencyConfig;
+export declare function readStdin(): Promise<string>;
+export declare function parse(contents: string, config: AgencyConfig): AgencyProgram;
+export declare function readFile(inputFile: string): string;
+export declare function renderGraph(contents: string, config: AgencyConfig): void;
+export declare function getImports(program: AgencyProgram): string[];
+export declare function compile(config: AgencyConfig, inputFile: string, _outputFile?: string): string | null;
+export declare function run(config: AgencyConfig, inputFile: string, outputFile?: string): void;
+export declare function format(contents: string, config: AgencyConfig): Promise<string>;
+export declare function formatFile(inputPath: string, inPlace: boolean, config: AgencyConfig): void;

package/dist/lib/cli/commands.js

@@ -0,0 +1,217 @@
+import { generateAgency } from "../backends/agencyGenerator.js";
+import { generateGraph } from "../index.js";
+import { TypescriptPreprocessor } from "../preprocessors/typescriptPreprocessor.js";
+import { renderMermaidAscii } from "beautiful-mermaid";
+import { spawn } from "child_process";
+import * as fs from "fs";
+import * as path from "path";
+import { parseAgency } from "../parser.js";
+// Load configuration from agency.json
+export function loadConfig(configPath) {
+    let config = {};
+    // Determine config file path
+    const defaultConfigPath = path.join(process.cwd(), "agency.json");
+    const finalConfigPath = configPath || defaultConfigPath;
+    // Check if config file exists
+    if (fs.existsSync(finalConfigPath)) {
+        try {
+            const configContent = fs.readFileSync(finalConfigPath, "utf-8");
+            config = JSON.parse(configContent);
+            if (config.verbose) {
+                console.log(`Loaded config from ${finalConfigPath}`);
+            }
+        }
+        catch (error) {
+            console.error(`Error loading config from ${finalConfigPath}:`, error);
+            process.exit(1);
+        }
+    }
+    return config;
+}
+export function readStdin() {
+    return new Promise((resolve, reject) => {
+        let data = "";
+        process.stdin.setEncoding("utf-8");
+        process.stdin.on("data", (chunk) => {
+            data += chunk;
+        });
+        process.stdin.on("end", () => {
+            resolve(data);
+        });
+        process.stdin.on("error", (err) => {
+            reject(err);
+        });
+    });
+}
+export function parse(contents, config) {
+    const verbose = config.verbose ?? false;
+    const parseResult = parseAgency(contents, verbose);
+    // Check if parsing was successful
+    if (!parseResult.success) {
+        console.error("Parse error:");
+        console.error(parseResult);
+        process.exit(1);
+    }
+    return parseResult.result;
+}
+export function readFile(inputFile) {
+    // Validate input file
+    if (!fs.existsSync(inputFile)) {
+        console.error(`Error: Input file '${inputFile}' not found`);
+        process.exit(1);
+    }
+    // Read and parse the Agency file
+    const contents = fs.readFileSync(inputFile, "utf-8");
+    return contents;
+}
+export function renderGraph(contents, config) {
+    const parsedProgram = parse(contents, config);
+    const preprocessor = new TypescriptPreprocessor(parsedProgram, config);
+    preprocessor.preprocess();
+    const mermaid = preprocessor.renderMermaid();
+    console.log("Program Mermaid Diagram:\n");
+    mermaid.forEach((subgraph) => {
+        const ascii = renderMermaidAscii(subgraph);
+        console.log(ascii);
+    });
+    console.log("==========");
+    mermaid.forEach((subgraph) => {
+        console.log(subgraph);
+    });
+}
+export function getImports(program) {
+    const toolAndNodeImports = program.nodes
+        .filter((node) => node.type === "importNodeStatement" ||
+        node.type === "importToolStatement")
+        .map((node) => node.agencyFile.trim());
+    // this makes compile() try to parse non-agency files
+    const importStatements = program.nodes
+        .filter((node) => node.type === "importStatement" && node.modulePath.endsWith(".agency"))
+        .map((node) => node.modulePath.trim());
+    return [...toolAndNodeImports, ...importStatements];
+}
+const compiledFiles = new Set();
+const dirSearched = new Set();
+export function compile(config, inputFile, _outputFile) {
+    // Check if the input is a directory
+    const stats = fs.statSync(inputFile);
+    const verbose = config.verbose ?? false;
+    if (stats.isDirectory()) {
+        dirSearched.add(path.resolve(inputFile));
+        // Find all .agency files in the directory
+        const files = fs.readdirSync(inputFile);
+        const agencyFiles = files.filter((file) => file.endsWith(".agency"));
+        for (const file of agencyFiles) {
+            const fullPath = path.join(inputFile, file);
+            compile(config, fullPath, undefined);
+        }
+        // Find all subdirectories and compile their .agency files
+        const subdirs = files.filter((file) => {
+            const fullPath = path.join(inputFile, file);
+            return fs.statSync(fullPath).isDirectory();
+        });
+        for (const subdir of subdirs) {
+            const fullSubdirPath = path.join(inputFile, subdir);
+            const resolvedSubdirPath = path.resolve(fullSubdirPath);
+            if (!dirSearched.has(resolvedSubdirPath)) {
+                compile(config, fullSubdirPath, undefined);
+            }
+        }
+        return null;
+    }
+    // Resolve the absolute path of the input file to avoid duplicates
+    const absoluteInputFile = path.resolve(inputFile);
+    let outputFile = _outputFile || inputFile.replace(".agency", ".ts");
+    if (config.outDir && !_outputFile) {
+        const outputDir = path.resolve(config.outDir);
+        if (!fs.existsSync(outputDir)) {
+            fs.mkdirSync(outputDir, { recursive: true });
+        }
+        outputFile = path.join(outputDir, outputFile);
+    }
+    // Skip if already compiled
+    if (compiledFiles.has(absoluteInputFile)) {
+        return outputFile;
+    }
+    compiledFiles.add(absoluteInputFile);
+    const contents = readFile(inputFile);
+    const parsedProgram = parse(contents, config);
+    const imports = getImports(parsedProgram);
+    const inputDir = path.dirname(absoluteInputFile);
+    for (const importPath of imports) {
+        const absPath = path.resolve(inputDir, importPath);
+        compile(config, absPath, undefined);
+    }
+    // Update the import path in the AST to reference the new .ts file
+    parsedProgram.nodes.forEach((node) => {
+        if (node.type === "importStatement") {
+            node.modulePath = node.modulePath.replace(".agency", ".ts");
+        }
+    });
+    const generatedCode = generateGraph(parsedProgram, config);
+    fs.writeFileSync(outputFile, generatedCode, "utf-8");
+    console.log(`${inputFile} → ${outputFile}`);
+    return outputFile;
+}
+export function run(config, inputFile, outputFile) {
+    // Compile the file
+    const output = compile(config, inputFile, outputFile);
+    if (output === null) {
+        console.error("Error: No output file generated.");
+        process.exit(1);
+    }
+    // Run the generated TypeScript file with Node.js
+    console.log(`Running ${output}...`);
+    console.log("---");
+    const nodeProcess = spawn("node", [output], {
+        stdio: "inherit",
+        shell: false,
+    });
+    nodeProcess.on("error", (error) => {
+        console.error(`Failed to run ${output}:`, error);
+        process.exit(1);
+    });
+    nodeProcess.on("exit", (code) => {
+        if (code !== 0) {
+            process.exit(code || 1);
+        }
+    });
+}
+export async function format(contents, config) {
+    const parsedProgram = parse(contents, config);
+    const generatedCode = generateAgency(parsedProgram);
+    return generatedCode;
+}
+export function formatFile(inputPath, inPlace, config) {
+    const stats = fs.statSync(inputPath);
+    if (stats.isDirectory()) {
+        // Format all .agency files in directory
+        const files = fs.readdirSync(inputPath);
+        const agencyFiles = files.filter((file) => file.endsWith(".agency"));
+        for (const file of agencyFiles) {
+            const fullPath = path.join(inputPath, file);
+            formatFile(fullPath, inPlace, config);
+        }
+        // Recursively format subdirectories
+        const subdirs = files.filter((file) => {
+            const fullPath = path.join(inputPath, file);
+            return fs.statSync(fullPath).isDirectory();
+        });
+        for (const subdir of subdirs) {
+            const fullSubdirPath = path.join(inputPath, subdir);
+            formatFile(fullSubdirPath, inPlace, config);
+        }
+        return;
+    }
+    // Format single file
+    const contents = readFile(inputPath);
+    const parsedProgram = parse(contents, config);
+    const generatedCode = generateAgency(parsedProgram);
+    if (inPlace) {
+        fs.writeFileSync(inputPath, generatedCode, "utf-8");
+        console.log(`Formatted ${inputPath}`);
+    }
+    else {
+        console.log(generatedCode);
+    }
+}

package/dist/lib/cli/evaluate.d.ts

@@ -0,0 +1 @@
+export declare function evaluate(target?: string, argsFilePath?: string, resultsFilePath?: string): Promise<void>;