agency-lang 0.6.1 → 0.6.2

package/dist/lib/agents/agency-agent/agent.js

@@ -1444,7 +1444,6 @@ async function __loadAgentsMd_impl(dir) {
       ]);
       await runner.handle(2, async (__data) => approve(), async (runner2) => {
         await runner2.step(0, async (runner3) => {
-          __self.__retryable = false;
           __stack.locals.result = await __call(read, {
             type: "positional",
             args: [`AGENTS.md`, __stack.args.dir]
@@ -1456,10 +1455,7 @@ async function __loadAgentsMd_impl(dir) {
           }
         });
       });
-      await runner.step(3, async (runner2) => {
-        __self.__retryable = false;
-      });
-      await runner.ifElse(4, [
+      await runner.ifElse(3, [
         {
           condition: async () => await isFailure(__stack.locals.result),
           body: async (runner2) => {
@@ -1471,7 +1467,7 @@ async function __loadAgentsMd_impl(dir) {
           }
         }
       ]);
-      await runner.step(5, async (runner2) => {
+      await runner.step(4, async (runner2) => {
         __functionCompleted = true;
         runner2.halt(`
 
@@ -3147,7 +3143,6 @@ async function __printHeader_impl() {
         }
       });
       await runner.step(2, async (runner2) => {
-        __self.__retryable = false;
         __stack.locals.data = await __call(box, {
           type: "named",
           positionalArgs: [],
@@ -4899,7 +4894,7 @@ Agent crashed: ${__error.message}`);
   }
 }
 var stdin_default = graph;
-const __sourceMap = { "dist/lib/agents/agency-agent/agent.agency:__cb_top_0": { "1": { "line": 97, "col": 2 }, "1.0": { "line": 98, "col": 4 } }, "dist/lib/agents/agency-agent/agent.agency:__cb_top_1": { "1": { "line": 103, "col": 2 }, "1.0.0": { "line": 104, "col": 4 }, "1.0.1": { "line": 105, "col": 6 }, "1.0.2": { "line": 106, "col": 11 }, "1.0.3": { "line": 107, "col": 6 }, "1.0.4": { "line": 109, "col": 6 }, "1.0": { "line": 104, "col": 4 } }, "dist/lib/agents/agency-agent/agent.agency:__cb_top_2": { "1": { "line": 115, "col": 2 }, "1.0": { "line": 116, "col": 4 } }, "dist/lib/agents/agency-agent/agent.agency:__cb_top_3": { "1": { "line": 121, "col": 2 }, "1.0": { "line": 122, "col": 4 } }, "dist/lib/agents/agency-agent/agent.agency:_showTraces": { "1": { "line": 93, "col": 2 } }, "dist/lib/agents/agency-agent/agent.agency:renderLLMCallResponse": { "1": { "line": 142, "col": 2 }, "2": { "line": 143, "col": 2 }, "3": { "line": 146, "col": 2 }, "5": { "line": 151, "col": 2 }, "2.0": { "line": 144, "col": 4 }, "3.0.0": { "line": 148, "col": 6 }, "3.0": { "line": 147, "col": 4 } }, "dist/lib/agents/agency-agent/agent.agency:loadAgentsMd": { "1": { "line": 162, "col": 2 }, "2": { "line": 165, "col": 2 }, "4": { "line": 166, "col": 2 }, "5": { "line": 169, "col": 2 }, "1.0": { "line": 163, "col": 4 }, "2.0": { "line": 165, "col": 2 }, "4.0": { "line": 167, "col": 4 } }, "dist/lib/agents/agency-agent/agent.agency:builtinPalette": { "1": { "line": 183, "col": 2 } }, "dist/lib/agents/agency-agent/agent.agency:mergedPalette": { "1": { "line": 195, "col": 2 }, "2": { "line": 196, "col": 2 }, "3": { "line": 203, "col": 2 }, "4": { "line": 204, "col": 2 }, "5": { "line": 207, "col": 2 }, "2.0": { "line": 197, "col": 4 }, "2.1.0": { "line": 199, "col": 6 }, "2.1": { "line": 198, "col": 4 }, "2.2": { "line": 201, "col": 4 }, "4.0": { "line": 205, "col": 4 } }, "dist/lib/agents/agency-agent/agent.agency:_runTurn": { "2": { "line": 218, "col": 2 }, "3": { "line": 219, "col": 2 }, "4": { "line": 222, "col": 2 }, "5": { "line": 225, "col": 2 }, "7": { "line": 229, "col": 2 }, "9": { "line": 233, "col": 2 }, "11": { "line": 261, "col": 2 }, "12": { "line": 262, "col": 2 }, "14": { "line": 267, "col": 2 }, "3.0": { "line": 220, "col": 4 }, "4.0": { "line": 223, "col": 4 }, "5.0": { "line": 226, "col": 4 }, "5.1": { "line": 227, "col": 4 }, "7.0": { "line": 230, "col": 4 }, "7.1": { "line": 231, "col": 4 }, "9.1": { "line": 241, "col": 4 }, "9.2": { "line": 242, "col": 4 }, "9.3": { "line": 243, "col": 4 }, "9.4.0": { "line": 245, "col": 6 }, "9.4.1": { "line": 246, "col": 6 }, "9.4": { "line": 244, "col": 4 }, "9.5": { "line": 248, "col": 4 }, "9.6": { "line": 249, "col": 4 }, "9.7.0": { "line": 251, "col": 6 }, "9.7.1": { "line": 252, "col": 6 }, "9.7.2.0": { "line": 254, "col": 8 }, "9.7.2": { "line": 253, "col": 6 }, "9.7": { "line": 250, "col": 4 }, "9.9": { "line": 257, "col": 4 }, "12.0": { "line": 263, "col": 4 }, "12.1": { "line": 265, "col": 4 } }, "dist/lib/agents/agency-agent/agent.agency:mainAgent": { "1": { "line": 450, "col": 2 }, "3": { "line": 464, "col": 2 }, "1.0": { "line": 451, "col": 4 }, "1.1.1": { "line": 456, "col": 6 }, "1.1.2": { "line": 457, "col": 6 }, "1.1": { "line": 452, "col": 4 }, "1.3": { "line": 459, "col": 4 } }, "dist/lib/agents/agency-agent/agent.agency:agentReplyVia": { "1": { "line": 473, "col": 2 }, "2": { "line": 474, "col": 2 }, "4": { "line": 477, "col": 2 }, "6": { "line": 480, "col": 2 }, "8": { "line": 483, "col": 2 }, "10": { "line": 486, "col": 2 }, "12": { "line": 489, "col": 2 }, "2.0": { "line": 475, "col": 4 }, "4.0": { "line": 478, "col": 4 }, "6.0": { "line": 481, "col": 4 }, "8.0": { "line": 484, "col": 4 }, "10.0": { "line": 487, "col": 4 } }, "dist/lib/agents/agency-agent/agent.agency:agentReply": { "1": { "line": 499, "col": 2 } }, "dist/lib/agents/agency-agent/agent.agency:roundedCost": { "1": { "line": 503, "col": 2 } }, "dist/lib/agents/agency-agent/agent.agency:_buildStatus": { "1": { "line": 507, "col": 2 } }, "dist/lib/agents/agency-agent/agent.agency:sample": { "1": { "line": 515, "col": 2 } }, "dist/lib/agents/agency-agent/agent.agency:printHeader": { "1": { "line": 519, "col": 2 }, "2": { "line": 520, "col": 2 }, "3": { "line": 542, "col": 2 } }, "dist/lib/agents/agency-agent/agent.agency:__block_0": { "2.0": { "line": 527, "col": 4 } }, "dist/lib/agents/agency-agent/agent.agency:__block_1": { "2.0.0": { "line": 528, "col": 6 }, "2.0.1": { "line": 536, "col": 6 }, "2.0.2": { "line": 537, "col": 6 } }, "dist/lib/agents/agency-agent/agent.agency:__block_2": { "2.0.0.0": { "line": 529, "col": 8 }, "2.0.0.1": { "line": 530, "col": 8 }, "2.0.0.2": { "line": 531, "col": 8 }, "2.0.0.3": { "line": 532, "col": 8 }, "2.0.0.4": { "line": 533, "col": 8 }, "2.0.0.5": { "line": 534, "col": 8 } }, "dist/lib/agents/agency-agent/agent.agency:__block_3": { "2.0.2.0": { "line": 538, "col": 8 } }, "dist/lib/agents/agency-agent/agent.agency:givePolicyChoice": { "1": { "line": 546, "col": 2 }, "2": { "line": 547, "col": 2 }, "3": { "line": 548, "col": 2 }, "4": { "line": 559, "col": 2 }, "5": { "line": 560, "col": 2 } }, "dist/lib/agents/agency-agent/agent.agency:setupSession": { "2": { "line": 578, "col": 2 }, "3": { "line": 583, "col": 2 }, "4": { "line": 584, "col": 2 }, "5": { "line": 586, "col": 2 }, "6": { "line": 587, "col": 2 }, "8": { "line": 604, "col": 2 }, "3.0": { "line": 583, "col": 2 }, "6.0": { "line": 587, "col": 2 }, "6.1.0.0": { "line": 590, "col": 8 }, "6.1.0.1": { "line": 591, "col": 8 }, "6.1.0.2": { "line": 593, "col": 8 }, "6.1.0": { "line": 589, "col": 6 }, "6.1.2": { "line": 596, "col": 6 }, "6.1.3": { "line": 597, "col": 6 }, "6.1": { "line": 588, "col": 4 }, "6.3": { "line": 600, "col": 4 } }, "dist/lib/agents/agency-agent/agent.agency:oneShotAgent": { "1": { "line": 614, "col": 2 }, "2": { "line": 615, "col": 2 }, "3": { "line": 616, "col": 2 }, "4": { "line": 617, "col": 2 }, "5": { "line": 622, "col": 2 }, "4.0": { "line": 618, "col": 4 } }, "dist/lib/agents/agency-agent/agent.agency:_runSeedTurn": { "1": { "line": 629, "col": 2 }, "2": { "line": 630, "col": 2 }, "3": { "line": 631, "col": 2 }, "3.0": { "line": 632, "col": 4 }, "3.1": { "line": 634, "col": 4 } }, "dist/lib/agents/agency-agent/agent.agency:startInteractive": { "1": { "line": 645, "col": 2 }, "3": { "line": 660, "col": 2 }, "1.0.0": { "line": 647, "col": 6 }, "1.0": { "line": 646, "col": 4 }, "1.1": { "line": 649, "col": 4 } }, "dist/lib/agents/agency-agent/agent.agency:main": { "2": { "line": 666, "col": 2 }, "3": { "line": 714, "col": 2 }, "4": { "line": 717, "col": 2 }, "6": { "line": 728, "col": 2 }, "7": { "line": 738, "col": 2 }, "8": { "line": 739, "col": 2 }, "9": { "line": 740, "col": 2 }, "10": { "line": 741, "col": 2 }, "11": { "line": 744, "col": 2 }, "13": { "line": 759, "col": 2 }, "15": { "line": 778, "col": 2 }, "16": { "line": 779, "col": 2 }, "18": { "line": 797, "col": 2 }, "19": { "line": 798, "col": 2 }, "20": { "line": 799, "col": 2 }, "21": { "line": 800, "col": 2 }, "22": { "line": 801, "col": 2 }, "3.0": { "line": 715, "col": 4 }, "4.0": { "line": 718, "col": 4 }, "11.0": { "line": 745, "col": 4 }, "11.1": { "line": 750, "col": 4 }, "13.0": { "line": 760, "col": 4 }, "13.1": { "line": 761, "col": 4 }, "13.2": { "line": 762, "col": 4 }, "13.3": { "line": 763, "col": 4 }, "13.4": { "line": 764, "col": 4 }, "13.5": { "line": 765, "col": 4 }, "16.0": { "line": 780, "col": 4 }, "16.1.0": { "line": 782, "col": 6 }, "16.1.1.0": { "line": 784, "col": 8 }, "16.1.1": { "line": 783, "col": 6 }, "16.1.2": { "line": 786, "col": 6 }, "16.1": { "line": 781, "col": 4 }, "16.2": { "line": 788, "col": 4 }, "16.3": { "line": 789, "col": 4 } } };
+const __sourceMap = { "dist/lib/agents/agency-agent/agent.agency:__cb_top_0": { "1": { "line": 97, "col": 2 }, "1.0": { "line": 98, "col": 4 } }, "dist/lib/agents/agency-agent/agent.agency:__cb_top_1": { "1": { "line": 103, "col": 2 }, "1.0.0": { "line": 104, "col": 4 }, "1.0.1": { "line": 105, "col": 6 }, "1.0.2": { "line": 106, "col": 11 }, "1.0.3": { "line": 107, "col": 6 }, "1.0.4": { "line": 109, "col": 6 }, "1.0": { "line": 104, "col": 4 } }, "dist/lib/agents/agency-agent/agent.agency:__cb_top_2": { "1": { "line": 115, "col": 2 }, "1.0": { "line": 116, "col": 4 } }, "dist/lib/agents/agency-agent/agent.agency:__cb_top_3": { "1": { "line": 121, "col": 2 }, "1.0": { "line": 122, "col": 4 } }, "dist/lib/agents/agency-agent/agent.agency:_showTraces": { "1": { "line": 93, "col": 2 } }, "dist/lib/agents/agency-agent/agent.agency:renderLLMCallResponse": { "1": { "line": 142, "col": 2 }, "2": { "line": 143, "col": 2 }, "3": { "line": 146, "col": 2 }, "5": { "line": 151, "col": 2 }, "2.0": { "line": 144, "col": 4 }, "3.0.0": { "line": 148, "col": 6 }, "3.0": { "line": 147, "col": 4 } }, "dist/lib/agents/agency-agent/agent.agency:loadAgentsMd": { "1": { "line": 162, "col": 2 }, "2": { "line": 165, "col": 2 }, "3": { "line": 166, "col": 2 }, "4": { "line": 169, "col": 2 }, "1.0": { "line": 163, "col": 4 }, "2.0": { "line": 165, "col": 2 }, "3.0": { "line": 167, "col": 4 } }, "dist/lib/agents/agency-agent/agent.agency:builtinPalette": { "1": { "line": 183, "col": 2 } }, "dist/lib/agents/agency-agent/agent.agency:mergedPalette": { "1": { "line": 195, "col": 2 }, "2": { "line": 196, "col": 2 }, "3": { "line": 203, "col": 2 }, "4": { "line": 204, "col": 2 }, "5": { "line": 207, "col": 2 }, "2.0": { "line": 197, "col": 4 }, "2.1.0": { "line": 199, "col": 6 }, "2.1": { "line": 198, "col": 4 }, "2.2": { "line": 201, "col": 4 }, "4.0": { "line": 205, "col": 4 } }, "dist/lib/agents/agency-agent/agent.agency:_runTurn": { "2": { "line": 218, "col": 2 }, "3": { "line": 219, "col": 2 }, "4": { "line": 222, "col": 2 }, "5": { "line": 225, "col": 2 }, "7": { "line": 229, "col": 2 }, "9": { "line": 233, "col": 2 }, "11": { "line": 261, "col": 2 }, "12": { "line": 262, "col": 2 }, "14": { "line": 267, "col": 2 }, "3.0": { "line": 220, "col": 4 }, "4.0": { "line": 223, "col": 4 }, "5.0": { "line": 226, "col": 4 }, "5.1": { "line": 227, "col": 4 }, "7.0": { "line": 230, "col": 4 }, "7.1": { "line": 231, "col": 4 }, "9.1": { "line": 241, "col": 4 }, "9.2": { "line": 242, "col": 4 }, "9.3": { "line": 243, "col": 4 }, "9.4.0": { "line": 245, "col": 6 }, "9.4.1": { "line": 246, "col": 6 }, "9.4": { "line": 244, "col": 4 }, "9.5": { "line": 248, "col": 4 }, "9.6": { "line": 249, "col": 4 }, "9.7.0": { "line": 251, "col": 6 }, "9.7.1": { "line": 252, "col": 6 }, "9.7.2.0": { "line": 254, "col": 8 }, "9.7.2": { "line": 253, "col": 6 }, "9.7": { "line": 250, "col": 4 }, "9.9": { "line": 257, "col": 4 }, "12.0": { "line": 263, "col": 4 }, "12.1": { "line": 265, "col": 4 } }, "dist/lib/agents/agency-agent/agent.agency:mainAgent": { "1": { "line": 450, "col": 2 }, "3": { "line": 464, "col": 2 }, "1.0": { "line": 451, "col": 4 }, "1.1.1": { "line": 456, "col": 6 }, "1.1.2": { "line": 457, "col": 6 }, "1.1": { "line": 452, "col": 4 }, "1.3": { "line": 459, "col": 4 } }, "dist/lib/agents/agency-agent/agent.agency:agentReplyVia": { "1": { "line": 473, "col": 2 }, "2": { "line": 474, "col": 2 }, "4": { "line": 477, "col": 2 }, "6": { "line": 480, "col": 2 }, "8": { "line": 483, "col": 2 }, "10": { "line": 486, "col": 2 }, "12": { "line": 489, "col": 2 }, "2.0": { "line": 475, "col": 4 }, "4.0": { "line": 478, "col": 4 }, "6.0": { "line": 481, "col": 4 }, "8.0": { "line": 484, "col": 4 }, "10.0": { "line": 487, "col": 4 } }, "dist/lib/agents/agency-agent/agent.agency:agentReply": { "1": { "line": 499, "col": 2 } }, "dist/lib/agents/agency-agent/agent.agency:roundedCost": { "1": { "line": 503, "col": 2 } }, "dist/lib/agents/agency-agent/agent.agency:_buildStatus": { "1": { "line": 507, "col": 2 } }, "dist/lib/agents/agency-agent/agent.agency:sample": { "1": { "line": 515, "col": 2 } }, "dist/lib/agents/agency-agent/agent.agency:printHeader": { "1": { "line": 519, "col": 2 }, "2": { "line": 520, "col": 2 }, "3": { "line": 542, "col": 2 } }, "dist/lib/agents/agency-agent/agent.agency:__block_0": { "2.0": { "line": 527, "col": 4 } }, "dist/lib/agents/agency-agent/agent.agency:__block_1": { "2.0.0": { "line": 528, "col": 6 }, "2.0.1": { "line": 536, "col": 6 }, "2.0.2": { "line": 537, "col": 6 } }, "dist/lib/agents/agency-agent/agent.agency:__block_2": { "2.0.0.0": { "line": 529, "col": 8 }, "2.0.0.1": { "line": 530, "col": 8 }, "2.0.0.2": { "line": 531, "col": 8 }, "2.0.0.3": { "line": 532, "col": 8 }, "2.0.0.4": { "line": 533, "col": 8 }, "2.0.0.5": { "line": 534, "col": 8 } }, "dist/lib/agents/agency-agent/agent.agency:__block_3": { "2.0.2.0": { "line": 538, "col": 8 } }, "dist/lib/agents/agency-agent/agent.agency:givePolicyChoice": { "1": { "line": 546, "col": 2 }, "2": { "line": 547, "col": 2 }, "3": { "line": 548, "col": 2 }, "4": { "line": 559, "col": 2 }, "5": { "line": 560, "col": 2 } }, "dist/lib/agents/agency-agent/agent.agency:setupSession": { "2": { "line": 578, "col": 2 }, "3": { "line": 583, "col": 2 }, "4": { "line": 584, "col": 2 }, "5": { "line": 586, "col": 2 }, "6": { "line": 587, "col": 2 }, "8": { "line": 604, "col": 2 }, "3.0": { "line": 583, "col": 2 }, "6.0": { "line": 587, "col": 2 }, "6.1.0.0": { "line": 590, "col": 8 }, "6.1.0.1": { "line": 591, "col": 8 }, "6.1.0.2": { "line": 593, "col": 8 }, "6.1.0": { "line": 589, "col": 6 }, "6.1.2": { "line": 596, "col": 6 }, "6.1.3": { "line": 597, "col": 6 }, "6.1": { "line": 588, "col": 4 }, "6.3": { "line": 600, "col": 4 } }, "dist/lib/agents/agency-agent/agent.agency:oneShotAgent": { "1": { "line": 614, "col": 2 }, "2": { "line": 615, "col": 2 }, "3": { "line": 616, "col": 2 }, "4": { "line": 617, "col": 2 }, "5": { "line": 622, "col": 2 }, "4.0": { "line": 618, "col": 4 } }, "dist/lib/agents/agency-agent/agent.agency:_runSeedTurn": { "1": { "line": 629, "col": 2 }, "2": { "line": 630, "col": 2 }, "3": { "line": 631, "col": 2 }, "3.0": { "line": 632, "col": 4 }, "3.1": { "line": 634, "col": 4 } }, "dist/lib/agents/agency-agent/agent.agency:startInteractive": { "1": { "line": 645, "col": 2 }, "3": { "line": 660, "col": 2 }, "1.0.0": { "line": 647, "col": 6 }, "1.0": { "line": 646, "col": 4 }, "1.1": { "line": 649, "col": 4 } }, "dist/lib/agents/agency-agent/agent.agency:main": { "2": { "line": 666, "col": 2 }, "3": { "line": 714, "col": 2 }, "4": { "line": 717, "col": 2 }, "6": { "line": 728, "col": 2 }, "7": { "line": 738, "col": 2 }, "8": { "line": 739, "col": 2 }, "9": { "line": 740, "col": 2 }, "10": { "line": 741, "col": 2 }, "11": { "line": 744, "col": 2 }, "13": { "line": 759, "col": 2 }, "15": { "line": 778, "col": 2 }, "16": { "line": 779, "col": 2 }, "18": { "line": 797, "col": 2 }, "19": { "line": 798, "col": 2 }, "20": { "line": 799, "col": 2 }, "21": { "line": 800, "col": 2 }, "22": { "line": 801, "col": 2 }, "3.0": { "line": 715, "col": 4 }, "4.0": { "line": 718, "col": 4 }, "11.0": { "line": 745, "col": 4 }, "11.1": { "line": 750, "col": 4 }, "13.0": { "line": 760, "col": 4 }, "13.1": { "line": 761, "col": 4 }, "13.2": { "line": 762, "col": 4 }, "13.3": { "line": 763, "col": 4 }, "13.4": { "line": 764, "col": 4 }, "13.5": { "line": 765, "col": 4 }, "16.0": { "line": 780, "col": 4 }, "16.1.0": { "line": 782, "col": 6 }, "16.1.1.0": { "line": 784, "col": 8 }, "16.1.1": { "line": 783, "col": 6 }, "16.1.2": { "line": 786, "col": 6 }, "16.1": { "line": 781, "col": 4 }, "16.2": { "line": 788, "col": 4 }, "16.3": { "line": 789, "col": 4 } } };
 export {
   __getCheckpoints,
   __mainNodeParams,

package/dist/lib/agents/agency-agent/subagents/review.js

@@ -1175,7 +1175,6 @@ async function ___typecheck_impl(agencyCode) {
         });
       });
       await runner.step(1, async (runner2) => {
-        __self.__retryable = false;
         __stack.locals.result = await __call(typecheck, {
           type: "positional",
           args: [__stack.args.agencyCode]
@@ -1378,7 +1377,6 @@ async function ___parse_impl(agencyCode) {
         });
       });
       await runner.step(1, async (runner2) => {
-        __self.__retryable = false;
         __stack.locals.result = await __call(parseAST, {
           type: "positional",
           args: [__stack.args.agencyCode]

package/dist/lib/agents/docs/cli/cli/optimize.md

@@ -5,117 +5,237 @@ description: Documents `agency eval optimize` — the eval-driven loop that rewr
 
 # Optimizing agents
 
-`agency eval optimize` (also `agency optimize`) improves an agent by rewriting the declarations you mark with the `optimize` modifier. It evaluates the baseline, asks a mutator model to propose new values for those declarations, runs and grades each candidate against your inputs, and keeps the best one.
+`agency optimize` improves an agent by rewriting your prompts for you.
 
-```bash
-agency optimize agent.agency --goal "Return the capital of the given country."
-agency optimize agent.agency --inputs inputs.json --graders grading.ts --iterations 5
-agency optimize agent.agency:main --inputs inputs.json --validation-split 0.3 --no-writeback
+For example, let's say you are writing an agent to return the capital of India. Here's your code:
+
+```ts
+node main() {
+  const prompt = "What is the capital of France?"
+  const response = llm(prompt)
+  return response
+}
+```
+
+Notice that the prompt is incorrectly asking for the capital of France. We're going to have the optimizer change this prompt to India. It's really easy to get started with the optimizer for a toy example like this. First, we need to mark the targets we want the optimizer to optimize:
+
+```ts
+node main() {
+  // added `optimize` to next line
+  optimize const prompt = "What is the capital of France?"
+  const response = llm(prompt)
+  return response
+}
+```
+
+The only change needed is the `optimize` modifier on the `prompt` variable declaration. Now call the `optimize` command, giving it your agency file and a goal:
+
+```
+agency optimize foo.agency --goal 'Return the capital of India'
+```
+
+If you run this command, you'll see output similar to this:
+
+```
+  grading:
+    - goal
+    first input: input-1 — goal: Return the capital of India
+
+== optimize greedy (run demo-run): 1 target(s), 1 input(s), up to 5 iteration(s) ==
+  - bar.agency:main:prompt = "What is the capital of France?"
+  baseline   objective 0.000
+  iter 1/5  accepted objective 1.000 (6.3s)
+  ~ bar.agency:main:prompt:
+      - What is the capital of France?
+      + What is the capital of India?
+      The change focuses on directly addressing the goal of retrieving the capital of India by modifying the prompt to reflect…
+  reached the maximum objective (1.000) — stopping early
+
+== Optimized variables ==
+  ~ bar.agency:main:prompt:
+      - What is the capital of France?
+      + What is the capital of India?
+
+Complete: champion iteration 1, accepted 1, rejected 0, invalid 0 (10.0s)
+Optimize demo-run completed: 1 accepted, 0 rejected
 ```
 
-## Marking what to optimize
+You can put `optimize` on any string `const` `let` to tell the the optimizer to rewrite it.
+
+## Inputs, graders, optimizers
+
+The `--goal` flag makes it really easy to get started with the optimizer, but gives you limited control. Now let's look at a more real-world example. But first I need to explain how the optimizer works.
 
-Put `optimize` on any string `const`/`let` the optimizer may rewrite. Discovery starts at the agent file and follows local relative `.agency` imports.
+The optimizer has three core things: inputs, graders, and the optimizer itself.
 
-```agency
-optimize const systemPrompt = "Answer accurately."
+### Inputs
+Inputs are examples you give to the optimizer. They are example input-output pairs.
 
-node main(question: string): string {
-  optimize const prompt = "Answer accurately: ${question}"
-  const answer: string = llm(prompt)
-  return answer
+For example, let's say we're optimizing this code:
+
+```ts
+node main(country) {
+  // note prompt incorrectly says "area" instead of "capital"
+  optimize const prompt = `What is the area of ${country}?`
+  const response = llm(prompt)
+  return response
 }
 ```
 
-A rewritten value must preserve every interpolation placeholder the original used (`${question}` here). Legacy `@optimize(...)` tags are not supported.
+It is very similar to the code we just saw, but now there's a `country` parameter for the node. We might give these inputs to the optimizer:
+
+```
+{
+  "inputs": [
+    { "args": { "country": "India" },  "expected": "New Delhi" },
+    { "args": { "country": "Japan" },  "expected": "Tokyo" },
+    { "args": { "country": "Brazil" }, "expected": "Brasília" }
+  ]
+}
+```
+  
+Save this as inputs.json and run the optimizer again:
+
+```
+agency optimize foo.agency --goal 'Return the capital of India' --inputs inputs.json
+
+```
 
-## Inputs and the goal
+This will run the optimizer the same as earlier, except now it also has three example inputs to look at. The optimizer will run foo.agency once for each input. That means it will run your agent, setting country to `"India"` for the first iteration, `"Japan"` for the second iteration etc, and look at the return value of the node.
 
-You describe what to optimize against with inputs and/or a goal. An input is one invocation of the agent: `args` for the node, plus optional `goal`, `expected`, `node`, `working_dir`, and freeform `metadata`.
+You can optionally also provide other values:
 
-```json
-{ "inputs": [
-  { "id": "india",  "args": { "country": "India" },  "expected": "New Delhi" },
-  { "id": "japan",  "args": { "country": "Japan" },  "expected": "Tokyo" }
-] }
+```ts
+export type Input = {
+  /** Unique id. Generated for you if not given.*/
+  id?: string;
+  /** What the agent should accomplish — read by the goal judge and the
+   *  pairwise judge suite. This is a per-input goal.*/
+  goal?: string;
+  /** Entry node to run. Defaults to `main`. */
+  node?: string;
+  /** Freeform, grader-agnostic metadata (tags, expectedOutput, …). */
+  metadata?: Record<string, any>;
+};
 ```
 
-- `--inputs <file|dir>` — the input suite.
-- `--goal <text>` — an overall goal. **Combinable with `--inputs`**: it fills in as the goal for any input that doesn't set its own. Used alone, it creates one inline no-argument input (and fails upfront if the node requires arguments).
-- At least one of `--inputs` / `--goal` is required.
+Notice that you can pass in a per-input goal, or an overall goal, as we have been doing with the `--goal` flag. You can pass in either one or both, but at least one goal is required. The `--goal` flag only fills in goals for inputs that don't have their own; they don't get combined. So if an input already has a goal, the `--goal` flag's value won't be used.
+
+### Graders
+So, we pass in an input, an expected output, and a goal to the optimizer. How does the optimizer measure the expected output? In our example with capitals, the expected output for India was `"New Delhi"`. What if the agent instead returned `"the capital of India is New Delhi"`? It's the job of the *grader* to decide how well the agent did. Let's look at some examples of graders.
+
+#### ExactMatchGrader
+Returns a binary pass-fail. Not the most useful grader, because it would give both of these the same score, which makes it hard for the optimizer to see if its changes to the agent are making any progress:
 
-`expected` is the gold output for an input (any JSON). It's read by the built-in match graders and surfaced to the optimizer's reflection — see below.
+```
+// these responses would get the same score:
+response1 = "asdadasdasd"
+response2 = "the capital of India is New Delhi"
+```
 
-## Options
+#### ContainsGrader
+Also returns a binary pass/fail like exact match, but this one checks to see if the expected output is anywhere in the response. Slightly better.
 
-| Flag | Meaning |
-| --- | --- |
-| `<file>[:<node>]` | Required agent target. A directory resolves to `main.agency`; the node defaults to `main`. |
-| `--inputs <file\|dir>` | Input suite file or directory. |
-| `--goal <text>` | Overall goal (combinable with `--inputs`; or a single inline input on its own). |
-| `--graders <file>` | A TypeScript grading module that replaces the default goal judge. See [Custom graders](#custom-graders). |
-| `--validation-inputs <file\|dir>` | Held-out validation suite. See [Validation sets](#validation-sets). |
-| `--validation-split <ratio>` | Hold out this fraction of `--inputs` (seeded by `--seed`) when `--validation-inputs` is absent. |
-| `--optimizer <name>` | `greedy` (default), `gepa`, or `example`. |
-| `--iterations <n>` | Max candidate iterations after the baseline. Default `5`. |
-| `--minibatch <n>` | GEPA minibatch size (gepa only). Default `8`. |
-| `--seed <n>` | RNG seed for reproducible search / validation split. |
-| `--mutator-model <model>` | Model override for proposing mutations. |
-| `--no-writeback` | Don't write the champion back to the source files. |
-| `--silent` | Print nothing; artifacts are still written. |
-| `--run-id <id>` | Output run id (must not already exist). |
-| `--runs-dir <path>` | Output root. Defaults to `eval.optimizeRunsDir`, then `eval.runsDir/optimize`, then `runs/optimize`. |
+#### SimilarityGrader
+Calculates the levenshtein distance and returns a score between 0 and 1 (0 = no match, 1 = perfect match).
 
-The baseline runs the unmutated program first; if a baseline input fails (or fails a `mustPass` gate), the run aborts and reports the failing inputs — a failure before any mutation means the program or suite is broken, not the optimization.
+#### LLM Judge
+Asks an LLM to return a score between 0 and 1 (0 = no match, 1 = perfect match) for how well the response matches the expected output.
 
-## Custom graders
+This is the default grader.
 
-By default a run is graded by one built-in LLM judge that scores each output against the input's `goal` (or the overall `--goal`). To grade differently — match a known answer, run a deterministic check, combine several graders — pass `--graders ./grading.ts` (or set `eval.optimize.graders` in `agency.json`). The module **replaces** the default judge.
+### Custom graders
 
-A grading module **default-exports one grader or an array of graders**. A "grader" is any of:
+So far, we have just been using the LLM Judge, which is the default grader. But we can also specify a custom grader using the `--graders` flag.
+
+First write a grader file:
 
 ```ts
-import { grader, scalar, ExactMatch, Contains, LlmJudge, type Grader } from "agency-lang/optimize";
+// graders.ts
+import { type Grader } from "agency-lang/optimize";
+
+// `input` is the typed Input; the gold answer is at `input.expected`
+// `output` is the actual response from your agent.
+const exact: Grader = ({ output, input }) => {
+   // return a number (0..1), a boolean, or a Grade
+  return output === input.expected ? 1 : 0;
+}
+
+export default exact;
+```
+
+Use the grader:
+
+```
+agency optimize foo.agency --goal 'Return the capital of India' --graders graders.ts
+```
+
+That's a really simple example where we're writing a custom function to use as the grader. It's an exact match function which, as we know, isn't very good. We can easily change this though. Let's see some options.
 
-// (a) a metric function: ctx = { output, input, judge }
-//     `input` is the typed Input; the gold answer is `input.expected`
-//     (extra per-input data can also live under `input.metadata`).
-const exact: Grader = ({ output, input }) =>
-  output === input.expected ? 1 : 0;   // return a number (0..1), a boolean, or a Grade
+We could call an LLM judge, passing it a custom judge prompt:
 
-// returning feedback too? use the scalar()/binary() constructors instead of a raw Grade literal:
+```ts
+import { scalar, type Grader } from "agency-lang/optimize";
 const judged: Grader = async ({ output, input, judge }) => {
-  const v = await judge({ goal: `Return ${input.expected}.`, output });
-  return scalar(v.score, v.reasoning);   // ← vs { score: { kind: "scalar", value: v.score }, feedback: v.reasoning }
+  const v = await judge({ goal:
+    `Hi this is my custom LLM judge prompt. The output should match this expected value: ${input.expected}.`,
+    output
+  });
+
+  // Agency func to return a scalar score + reasoning for the score.
+  // Generates something like:
+  // 
+  // ```
+  // { score: { kind: "scalar", value: v.score }, feedback: v.reasoning }
+  // ```
+  return scalar(v.score, v.reasoning);   
 };
+```
 
-// (b) a wrapped function carrying policy (mustPass gate, weight, threshold, samples, inputScope)
-const gate = grader(exact, { mustPass: true, name: "capital-exact" });
-
-// (c) a configured built-in — matchOn defaults to ["expected"]
-const has = new Contains({});                                    // output contains input.expected
-const judge = new LlmJudge({ goal: "Return the capital.", samples: 3 });
+We could use a built-in grader:
 
-export default [gate, judged];   // or `export default exact` for the simple case
+```ts
+import { Contains } from "agency-lang/optimize";
+export default (new Contains({}));
 ```
 
-A metric function returns a **number** (0..1 scalar), a **boolean** (1.0/0.0), or a full **Grade**. For a Grade with feedback, the `scalar(value, feedback?)` and `binary(pass, feedback?)` constructors are the ergonomic way to build one.
+Instead of a single grader, we can also return an array of graders:
 
-**How grades become the objective.** Every grade counts: a number contributes its value (0..1), and a boolean / `ExactMatch` / `Contains` result contributes `1.0` (pass) or `0.0` (fail) — so a binary-only grader gives you plain accuracy. The objective for an input is the weighted mean of its grades, and the run objective is the mean across inputs. `mustPass` is an orthogonal **gate**: a failed `mustPass` grader zeroes that input regardless of its other grades.
+```ts
+import { Contains, Grader, scalar } from "agency-lang/optimize";
 
-> **Pick a grader that has a gradient.** Exact `===` against free-form LLM output almost never matches (`"The capital is New Delhi."` ≠ `"New Delhi"`), so it scores 0 for every candidate and the search can't climb. Use `Contains`, `Similarity`, or an `LlmJudge` (or constrain the prompt to emit only the value) so a better candidate actually scores higher.
+const judged: Grader = async ({ output, input, judge }) => {
+    const v = await judge({
+        goal:
+            `Hi this is my custom LLM judge prompt. The output should match this expected value: ${input.expected}.`,
+        output
+    });
 
-`ctx.judge({ goal, output })` runs the bundled LLM goal judge from inside a metric function, so you can mix deterministic and LLM grading. When a grading module is configured, a per-input `goal` is optional.
+    return scalar(v.score, v.reasoning);
+};
 
-### Steering the search without a goal
+export default [new Contains({}), judged];
+```
 
-The optimizer's reflection is fed each input's `expected` answer **and** each grader's `feedback`, so a self-explaining grader (one that returns `{ score, feedback }`) or labeled `expected` outputs can drive the rewrites *without* a `--goal` — `--goal` is then an optional extra steer. A grader that returns only a bare score and inputs with no `expected` leave the mutator nothing to learn from, so it can only guess from the current prompt; provide one or the other.
+Finally, you can use the `grader` function to wrap a custom function and supply some metadata:
 
-The mutator is instructed **not** to hard-code the expected answers into the prompt. A [validation set](#validation-sets) is the backstop that fails any prompt which memorizes them anyway.
+```ts
+// use the `exact` function as the grader.
+// mustPass = if this grader fails, consider this entire iteration failed.
+// name = shown in debug output.
+const gate = grader(exact, { mustPass: true, name: "capital-exact" });
+```
+
+To recap:
+- A grading module **default-exports one grader or an array of graders**. 
+- A metric function returns a **number** (0..1 scalar), a **boolean** (1.0/0.0), or a full **Grade**. For a Grade with feedback, the `scalar(value, feedback?)` and `binary(pass, feedback?)` constructors are the ergonomic way to build one.
+
+#### How grades become the objective
+Every grade counts: a number contributes its value (0..1), and a boolean / `ExactMatch` / `Contains` result contributes `1.0` (pass) or `0.0` (fail) — so a binary-only grader gives you plain accuracy. The objective for an input is the weighted mean of its grades, and the run objective is the mean across inputs. `mustPass` is an orthogonal **gate**: a failed `mustPass` grader zeroes that input regardless of its other grades.
 
 ## Validation sets
 
-Pass `--validation-inputs <file|dir>` to grade the champion against held-out inputs, or `--validation-split <ratio>` to hold out a seeded fraction of `--inputs`. Search and candidate acceptance run on the **training** inputs; with the default `greedy` optimizer the champion written back is the one with the best **validation** objective, and `report.md` shows train-vs-validation side by side so an overfit prompt (high train, flat validation) is visible. `gepa` and `example` report a validation objective but select on training; the report says so.
+Pass `--validation-inputs <file|dir>` to grade the champion against held-out inputs, or `--validation-split <ratio>` to hold out a seeded fraction of `--inputs`. Search and candidate acceptance run on the **training** inputs; with the default `greedy` optimizer the champion written back is the one with the best **validation** objective, and `report.md` shows train-vs-validation side by side so an overfit prompt (high train, flat validation) is visible.
 
 ## Configuration
 
@@ -152,6 +272,9 @@ runs/optimize/<run-id>/
 
 By default the optimizer also prints progress to the console (the resolved grading setup, per-iteration decisions, and the start→end value of every optimized variable). `--silent` suppresses console output; artifacts are still written.
 
+## Optimizers
+Agency comes with two built-in optimizers, `greedy` and `gepa`. `greedy` is the default. You can specify the optimizer using the `--optimizer` flag. You can also write your own optimizers.
+
 ## Writing your own optimizer
 
 `greedy`, `gepa`, and `example` are built on a shared `BaseOptimizer`, which you can extend. Write a module that default-exports a **factory** `(config) => Optimizer`, then point `--optimizer` (or `eval.optimize.optimizer`) at its path — exactly like `--graders`:
@@ -178,4 +301,4 @@ agency optimize foo.agency --inputs inputs.json --optimizer ./myOptimizer.ts
 
 ## Notes
 
-The CLI installs an approval handler for the internal `std::agency.run(...)` calls used by eval execution. The stdlib `agency.eval.optimize(...)` function does **not** install a handler; Agency callers should wrap it in their own handler when they want auto-approval.
+The CLI installs an approval handler for the internal `std::agency.run(...)` calls used by eval execution. The stdlib `agency.eval.optimize(...)` function does **not** install a handler; Agency callers should wrap it in their own handler when they want auto-approval.