agency-lang 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/agentTarget.d.ts +52 -0
- package/dist/lib/agentTarget.js +83 -0
- package/dist/lib/agentTarget.test.js +76 -0
- package/dist/lib/agents/agency-agent/agent.js +8 -4
- package/dist/lib/agents/agency-agent/lib/coordinator.js +4 -4
- package/dist/lib/agents/agency-agent/lib/policy.js +4 -4
- package/dist/lib/agents/agency-agent/prompts/codeSample.js +4 -0
- package/dist/lib/agents/eval/goalJudge.js +5 -1
- package/dist/lib/agents/policy/agent.js +4 -0
- package/dist/lib/agents/review/agent.js +4 -0
- package/dist/lib/analysis/closure.d.ts +26 -0
- package/dist/lib/analysis/closure.js +99 -0
- package/dist/lib/analysis/closure.test.js +43 -0
- package/dist/lib/analysis/imports.d.ts +10 -0
- package/dist/lib/analysis/imports.js +25 -0
- package/dist/lib/backends/agencyGenerator.d.ts +14 -1
- package/dist/lib/backends/agencyGenerator.js +54 -15
- package/dist/lib/backends/agencyGenerator.matchBlock.test.js +59 -0
- package/dist/lib/backends/agencyGenerator.roundtrip.test.js +17 -0
- package/dist/lib/backends/agencyGenerator.test.js +44 -1
- package/dist/lib/backends/docstringInterpolationRuntime.test.js +1 -1
- package/dist/lib/backends/topLevelRefusal.test.js +40 -0
- package/dist/lib/backends/typescriptBuilder/directRunArgsCheck.d.ts +11 -0
- package/dist/lib/backends/typescriptBuilder/directRunArgsCheck.js +19 -0
- package/dist/lib/backends/typescriptBuilder/nameClassifier.d.ts +7 -0
- package/dist/lib/backends/typescriptBuilder/nameClassifier.js +1 -1
- package/dist/lib/backends/typescriptBuilder.js +70 -36
- package/dist/lib/cli/commands.d.ts +5 -24
- package/dist/lib/cli/commands.js +8 -49
- package/dist/lib/cli/coverage.js +1 -1
- package/dist/lib/cli/debug.js +1 -1
- package/dist/lib/cli/doc.js +1 -1
- package/dist/lib/cli/eval/grade.d.ts +15 -0
- package/dist/lib/cli/eval/grade.js +33 -0
- package/dist/lib/cli/eval/grade.test.js +89 -0
- package/dist/lib/cli/eval/graders.d.ts +12 -0
- package/dist/lib/cli/eval/graders.js +33 -0
- package/dist/lib/cli/eval/logs.d.ts +7 -0
- package/dist/lib/cli/eval/logs.js +42 -0
- package/dist/lib/cli/eval/logs.test.js +42 -0
- package/dist/lib/cli/eval/optimize.js +20 -42
- package/dist/lib/cli/eval/optimize.test.js +13 -18
- package/dist/lib/cli/eval/run.d.ts +23 -57
- package/dist/lib/cli/eval/run.js +99 -214
- package/dist/lib/cli/eval/run.test.js +230 -21
- package/dist/lib/cli/evalExtract.js +11 -0
- package/dist/lib/cli/evalJudge.d.ts +1 -1
- package/dist/lib/cli/evalJudge.js +8 -34
- package/dist/lib/cli/evalJudge.test.js +10 -29
- package/dist/lib/cli/literate.js +1 -1
- package/dist/lib/cli/pack.js +1 -1
- package/dist/lib/cli/runBundledAgent.js +9 -4
- package/dist/lib/cli/runBundledAgent.test.js +21 -0
- package/dist/lib/cli/serve.js +2 -1
- package/dist/lib/cli/test.js +5 -2
- package/dist/lib/cli/util.d.ts +1 -14
- package/dist/lib/cli/util.js +4 -59
- package/dist/lib/cli/validatorInterrupts.spawn.test.js +167 -0
- package/dist/lib/cli/watch.js +1 -1
- package/dist/lib/cli/watch.test.js +2 -2
- package/dist/lib/compiler/buildSession.js +2 -1
- package/dist/lib/compiler/buildSession.test.js +1 -1
- package/dist/lib/compiler/compile.js +1 -1
- package/dist/lib/compiler/defaultSession.d.ts +24 -0
- package/dist/lib/compiler/defaultSession.js +45 -0
- package/dist/lib/config.d.ts +36 -0
- package/dist/lib/config.js +27 -0
- package/dist/lib/debugger/driver.test.js +1 -1
- package/dist/lib/debugger/exportFrames.test.js +1 -1
- package/dist/lib/debugger/testSession.test.js +1 -1
- package/dist/lib/debugger/thread.test.js +1 -1
- package/dist/lib/debugger/trace.test.js +1 -1
- package/dist/lib/eval/extract.d.ts +0 -6
- package/dist/lib/eval/extract.js +23 -43
- package/dist/lib/eval/extract.test.js +97 -29
- package/dist/lib/{optimize → eval}/grading/baseGrader.test.js +3 -2
- package/dist/lib/{optimize → eval}/grading/functionGrader.d.ts +5 -0
- package/dist/lib/{optimize → eval}/grading/functionGrader.js +8 -2
- package/dist/lib/{optimize → eval}/grading/functionGrader.test.js +7 -6
- package/dist/lib/{optimize → eval/grading}/goalJudgeFile.js +1 -1
- package/dist/lib/eval/grading/gradeBreakdown.d.ts +54 -0
- package/dist/lib/eval/grading/gradeBreakdown.js +83 -0
- package/dist/lib/{optimize → eval/grading}/gradeBreakdown.test.js +5 -4
- package/dist/lib/eval/grading/gradeRun.d.ts +44 -0
- package/dist/lib/eval/grading/gradeRun.js +180 -0
- package/dist/lib/eval/grading/gradeRun.test.js +175 -0
- package/dist/lib/eval/grading/gradeSuite.d.ts +10 -0
- package/dist/lib/eval/grading/gradeSuite.js +27 -0
- package/dist/lib/{optimize → eval}/grading/graders/builtinGraders.test.js +7 -6
- package/dist/lib/{optimize → eval}/grading/graders/humanGrader.js +1 -1
- package/dist/lib/{optimize → eval}/grading/graders/humanGrader.test.js +3 -2
- package/dist/lib/{optimize → eval}/grading/graders/llmJudge.js +1 -1
- package/dist/lib/{optimize → eval}/grading/graders/llmJudge.test.js +6 -5
- package/dist/lib/{optimize → eval/grading}/gradingModule.d.ts +2 -2
- package/dist/lib/{optimize → eval/grading}/gradingModule.js +24 -3
- package/dist/lib/{optimize → eval/grading}/gradingModule.test.js +1 -1
- package/dist/lib/eval/grading/recordGrading.d.ts +12 -0
- package/dist/lib/eval/grading/recordGrading.js +27 -0
- package/dist/lib/{optimize → eval}/grading/scorecard.d.ts +5 -2
- package/dist/lib/{optimize → eval}/grading/scorecard.test.js +14 -13
- package/dist/lib/eval/grading/testUtils.d.ts +7 -0
- package/dist/lib/eval/grading/testUtils.js +31 -0
- package/dist/lib/{optimize → eval}/grading/types.d.ts +9 -3
- package/dist/lib/eval/judge/suite.d.ts +4 -5
- package/dist/lib/eval/judge/suite.js +41 -23
- package/dist/lib/eval/judge/suite.test.js +42 -20
- package/dist/lib/eval/judge/types.d.ts +4 -0
- package/dist/lib/eval/loadInputs.d.ts +9 -0
- package/dist/lib/eval/loadInputs.js +138 -21
- package/dist/lib/eval/loadInputs.test.js +192 -16
- package/dist/lib/eval/normalize.d.ts +4 -0
- package/dist/lib/eval/normalize.js +2 -1
- package/dist/lib/eval/public.d.ts +12 -0
- package/dist/lib/eval/public.js +10 -0
- package/dist/lib/eval/public.test.d.ts +1 -0
- package/dist/lib/eval/public.test.js +49 -0
- package/dist/lib/eval/readRun.js +43 -4
- package/dist/lib/eval/readRun.test.js +37 -3
- package/dist/lib/eval/run/commandLine.d.ts +9 -0
- package/dist/lib/eval/run/commandLine.js +54 -0
- package/dist/lib/eval/run/commandLine.test.d.ts +1 -0
- package/dist/lib/eval/run/commandLine.test.js +34 -0
- package/dist/lib/eval/run/costTail.d.ts +8 -0
- package/dist/lib/eval/run/costTail.js +26 -0
- package/dist/lib/eval/run/extract.d.ts +24 -0
- package/dist/lib/eval/run/extract.js +36 -0
- package/dist/lib/eval/run/runAgent.d.ts +68 -0
- package/dist/lib/eval/run/runAgent.js +205 -0
- package/dist/lib/eval/run/runAgent.test.d.ts +1 -0
- package/dist/lib/eval/run/runAgent.test.js +193 -0
- package/dist/lib/eval/run/runSuite.d.ts +52 -0
- package/dist/lib/eval/run/runSuite.js +247 -0
- package/dist/lib/eval/run/runSuite.test.d.ts +1 -0
- package/dist/lib/eval/run/runSuite.test.js +321 -0
- package/dist/lib/eval/run/seed.d.ts +40 -0
- package/dist/lib/eval/run/seed.js +101 -0
- package/dist/lib/eval/run/seed.test.d.ts +1 -0
- package/dist/lib/eval/run/seed.test.js +118 -0
- package/dist/lib/eval/run/spawnRunner.d.ts +27 -0
- package/dist/lib/eval/run/spawnRunner.js +208 -0
- package/dist/lib/eval/run/spawnRunner.test.d.ts +1 -0
- package/dist/lib/eval/run/spawnRunner.test.js +149 -0
- package/dist/lib/eval/run/statusBoard.d.ts +22 -0
- package/dist/lib/eval/run/statusBoard.js +78 -0
- package/dist/lib/eval/run/subprocess.d.ts +54 -0
- package/dist/lib/eval/run/subprocess.js +170 -0
- package/dist/lib/eval/run/subprocess.test.d.ts +1 -0
- package/dist/lib/eval/run/subprocess.test.js +64 -0
- package/dist/lib/eval/runArtifacts.d.ts +55 -14
- package/dist/lib/eval/runArtifacts.js +76 -52
- package/dist/lib/eval/runArtifacts.test.js +11 -30
- package/dist/lib/eval/runTypes.d.ts +59 -21
- package/dist/lib/eval/sources.d.ts +26 -0
- package/dist/lib/eval/sources.js +148 -0
- package/dist/lib/eval/sources.test.d.ts +1 -0
- package/dist/lib/eval/sources.test.js +96 -0
- package/dist/lib/eval/testUtils.d.ts +7 -0
- package/dist/lib/eval/testUtils.js +22 -0
- package/dist/lib/eval/types.d.ts +32 -24
- package/dist/lib/importPaths.d.ts +3 -0
- package/dist/lib/importPaths.js +23 -18
- package/dist/lib/logsViewer/follow.js +14 -59
- package/dist/lib/logsViewer/parse.d.ts +1 -11
- package/dist/lib/logsViewer/parse.js +5 -61
- package/dist/lib/lowering/patternLowering.js +47 -1
- package/dist/lib/lowering/patternLowering.test.js +11 -5
- package/dist/lib/optimize/baseOptimizer.d.ts +15 -25
- package/dist/lib/optimize/baseOptimizer.js +50 -60
- package/dist/lib/optimize/baseOptimizer.test.js +12 -14
- package/dist/lib/optimize/baseOptimizer.workdir.test.js +18 -18
- package/dist/lib/optimize/evalCache.d.ts +2 -3
- package/dist/lib/optimize/evalCache.js +1 -1
- package/dist/lib/optimize/evalCache.test.js +6 -4
- package/dist/lib/optimize/gepaReflect.d.ts +1 -1
- package/dist/lib/optimize/gepaReflect.test.js +1 -1
- package/dist/lib/optimize/mutator.test.js +2 -2
- package/dist/lib/optimize/optimizer.d.ts +2 -2
- package/dist/lib/optimize/optimizers/example.d.ts +2 -2
- package/dist/lib/optimize/optimizers/example.js +1 -1
- package/dist/lib/optimize/optimizers/example.test.js +7 -5
- package/dist/lib/optimize/optimizers/gepa.d.ts +2 -2
- package/dist/lib/optimize/optimizers/gepa.js +1 -1
- package/dist/lib/optimize/optimizers/gepa.test.js +7 -5
- package/dist/lib/optimize/optimizers/greedyReflective.d.ts +1 -1
- package/dist/lib/optimize/optimizers/greedyReflective.test.js +22 -19
- package/dist/lib/optimize/public.d.ts +2 -12
- package/dist/lib/optimize/public.js +4 -10
- package/dist/lib/optimize/reflectionFeedback.d.ts +1 -1
- package/dist/lib/optimize/reflectionFeedback.js +6 -15
- package/dist/lib/optimize/reflectionFeedback.test.js +18 -20
- package/dist/lib/optimize/report.js +1 -1
- package/dist/lib/optimize/report.test.js +3 -3
- package/dist/lib/optimize/reporter.d.ts +2 -62
- package/dist/lib/optimize/reporter.js +0 -100
- package/dist/lib/optimize/reporter.test.js +2 -172
- package/dist/lib/optimize/sourceMutator.js +2 -1
- package/dist/lib/optimize/targets.d.ts +0 -6
- package/dist/lib/optimize/targets.js +4 -57
- package/dist/lib/optimize/testUtils.d.ts +11 -0
- package/dist/lib/optimize/testUtils.js +64 -0
- package/dist/lib/optimize/types.d.ts +1 -35
- package/dist/lib/optimize/validationSplit.test.js +1 -1
- package/dist/lib/optimize/workspace.d.ts +4 -4
- package/dist/lib/optimize/workspace.js +2 -2
- package/dist/lib/optimize/workspace.test.js +1 -1
- package/dist/lib/parsers/body.test.js +101 -0
- package/dist/lib/parsers/codeLiteral.test.js +90 -0
- package/dist/lib/parsers/parsers.d.ts +1 -1
- package/dist/lib/parsers/parsers.js +107 -9
- package/dist/lib/parsers/splice.test.js +26 -2
- package/dist/lib/preprocessors/expandSplices.js +47 -2
- package/dist/lib/preprocessors/expandSplices.test.js +50 -0
- package/dist/lib/runtime/configOverrides.d.ts +17 -0
- package/dist/lib/runtime/configOverrides.js +26 -0
- package/dist/lib/runtime/configOverrides.test.js +26 -1
- package/dist/lib/runtime/initPlanWorkedExamples.test.js +1 -1
- package/dist/lib/runtime/interrupts.js +23 -5
- package/dist/lib/runtime/interrupts.test.js +28 -0
- package/dist/lib/runtime/ipc.d.ts +30 -1
- package/dist/lib/runtime/ipc.js +30 -1
- package/dist/lib/runtime/ipc.test.js +28 -1
- package/dist/lib/runtime/prompt.js +2 -0
- package/dist/lib/runtime/state/context.d.ts +9 -0
- package/dist/lib/runtime/state/context.js +16 -2
- package/dist/lib/runtime/state/context.test.js +24 -0
- package/dist/lib/runtime/staticInit.crossModule.test.js +1 -1
- package/dist/lib/runtime/subprocess-bootstrap.js +9 -4
- package/dist/lib/runtime/template/aliasTable.d.ts +10 -0
- package/dist/lib/runtime/template/aliasTable.js +35 -0
- package/dist/lib/runtime/template/explainMismatch.d.ts +14 -0
- package/dist/lib/runtime/template/explainMismatch.js +49 -0
- package/dist/lib/runtime/template/fill.js +112 -55
- package/dist/lib/runtime/template/fill.test.js +465 -0
- package/dist/lib/runtime/template/lift.d.ts +4 -0
- package/dist/lib/runtime/template/lift.js +4 -0
- package/dist/lib/runtime/template/synthesizeType.d.ts +31 -0
- package/dist/lib/runtime/template/synthesizeType.js +118 -0
- package/dist/lib/runtime/template/synthesizeType.test.d.ts +1 -0
- package/dist/lib/runtime/template/synthesizeType.test.js +111 -0
- package/dist/lib/runtime/toolSchemaSize.d.ts +70 -0
- package/dist/lib/runtime/toolSchemaSize.js +87 -0
- package/dist/lib/runtime/toolSchemaSize.test.d.ts +1 -0
- package/dist/lib/runtime/toolSchemaSize.test.js +88 -0
- package/dist/lib/runtime/topsortCycleErrors.test.js +1 -1
- package/dist/lib/runtime/validateChain.d.ts +4 -3
- package/dist/lib/runtime/validateChain.js +32 -4
- package/dist/lib/runtime/validateChain.test.js +39 -9
- package/dist/lib/serve/http/functionFrame.integration.test.js +1 -1
- package/dist/lib/serve/public.d.ts +1 -0
- package/dist/lib/serve/public.js +4 -0
- package/dist/lib/statelog/appendReader.d.ts +20 -0
- package/dist/lib/statelog/appendReader.js +53 -0
- package/dist/lib/statelog/parse.d.ts +11 -0
- package/dist/lib/statelog/parse.js +61 -0
- package/dist/lib/statelog/tail.d.ts +12 -0
- package/dist/lib/statelog/tail.js +21 -0
- package/dist/lib/statelogClient.d.ts +10 -4
- package/dist/lib/statelogClient.js +2 -2
- package/dist/lib/stdlib/agencyEval.d.ts +1 -62
- package/dist/lib/stdlib/agencyEval.js +1 -125
- package/dist/lib/stdlib/llm.d.ts +19 -0
- package/dist/lib/stdlib/llm.js +44 -2
- package/dist/lib/stdlib/shell.js +21 -0
- package/dist/lib/templates/backends/typescriptGenerator/directRunArgsCheck.d.ts +7 -0
- package/dist/lib/templates/backends/typescriptGenerator/directRunArgsCheck.js +13 -0
- package/dist/lib/typeChecker/diagnosticExplanations.js +9 -0
- package/dist/lib/typeChecker/diagnostics.d.ts +15 -0
- package/dist/lib/typeChecker/diagnostics.js +15 -0
- package/dist/lib/typeChecker/flow.d.ts +10 -0
- package/dist/lib/typeChecker/flowBuilder.js +44 -4
- package/dist/lib/typeChecker/flowNarrowing.test.js +259 -0
- package/dist/lib/typeChecker/handlerParamTyping.test.js +7 -7
- package/dist/lib/typeChecker/index.js +4 -0
- package/dist/lib/typeChecker/matchArmNarrowing.test.js +200 -0
- package/dist/lib/typeChecker/narrowing.js +9 -3
- package/dist/lib/typeChecker/spliceDiagnostics.test.js +2 -0
- package/dist/lib/typeChecker/topLevelStatements.d.ts +8 -0
- package/dist/lib/typeChecker/topLevelStatements.js +20 -0
- package/dist/lib/typeChecker/topLevelStatements.test.d.ts +1 -0
- package/dist/lib/typeChecker/topLevelStatements.test.js +83 -0
- package/dist/lib/typeChecker.test.js +37 -19
- package/dist/lib/types/matchBlock.d.ts +5 -0
- package/dist/lib/types/splice.d.ts +1 -1
- package/dist/lib/utils/findRecursively.d.ts +5 -0
- package/dist/lib/utils/findRecursively.js +31 -0
- package/dist/lib/utils/hash.d.ts +1 -0
- package/dist/lib/utils/hash.js +4 -0
- package/dist/lib/utils/holes.d.ts +4 -1
- package/dist/lib/utils/holes.js +11 -2
- package/dist/lib/utils/topLevel.d.ts +10 -0
- package/dist/lib/utils/topLevel.js +138 -0
- package/dist/lib/utils/topLevel.test.d.ts +1 -0
- package/dist/lib/utils/topLevel.test.js +151 -0
- package/dist/scripts/agency.js +86 -16
- package/package.json +7 -2
- package/stdlib/agency/eval.agency +16 -214
- package/stdlib/agency/eval.js +6 -701
- package/stdlib/agency.agency +22 -2
- package/stdlib/agency.js +121 -7
- package/stdlib/agents/coding.agency +2 -1
- package/stdlib/agents/coding.js +3 -2
- package/stdlib/agents/composable/promptRewriter.agency +27 -0
- package/stdlib/agents/composable/promptRewriter.js +393 -0
- package/stdlib/agents/composable/researcher.agency +52 -0
- package/stdlib/agents/composable/researcher.js +591 -0
- package/stdlib/agents/composable/utils.agency +7 -0
- package/stdlib/agents/composable/utils.js +353 -0
- package/stdlib/agents/lib/expertGuidance.js +4 -4
- package/stdlib/agents/lib/feedback.js +4 -4
- package/stdlib/agents/lib/search.agency +20 -7
- package/stdlib/agents/lib/search.js +164 -47
- package/stdlib/agents/planner.js +12 -12
- package/stdlib/data/finance/dbnomics.js +4 -4
- package/stdlib/data/finance/edgar.js +4 -4
- package/stdlib/data/finance/fred.js +8 -8
- package/stdlib/data/finance/gdelt.js +4 -4
- package/stdlib/data/people/littlesis.js +12 -12
- package/stdlib/data/tech/hackernews.js +16 -16
- package/stdlib/data/tech/yc.js +8 -8
- package/stdlib/data/usaspending.js +8 -8
- package/stdlib/data/wikidata.js +12 -12
- package/stdlib/docs/cli/eval-judge.md +3 -4
- package/stdlib/docs/cli/eval.md +270 -18
- package/stdlib/docs/cli/optimize.md +10 -9
- package/stdlib/docs/cli/run.md +26 -1
- package/stdlib/docs/diagnostics/effects.md +20 -0
- package/stdlib/docs/diagnostics/index.md +3 -0
- package/stdlib/docs/diagnostics/templates.md +10 -0
- package/stdlib/docs/guide/agency-config-file.md +1 -0
- package/stdlib/docs/guide/type-validation.md +2 -16
- package/stdlib/docs/stdlib/agency/eval.md +27 -252
- package/stdlib/docs/stdlib/agency.md +30 -28
- package/stdlib/docs/stdlib/agents/coding.md +2 -2
- package/stdlib/docs/stdlib/agents/composable/promptRewriter.md +26 -0
- package/stdlib/docs/stdlib/agents/composable/researcher.md +39 -0
- package/stdlib/docs/stdlib/agents/composable/utils.md +25 -0
- package/stdlib/docs/stdlib/agents/lib/search.md +5 -4
- package/stdlib/docs/stdlib/safeBash.md +5 -5
- package/stdlib/docs/stdlib/syntax.md +36 -19
- package/stdlib/safeBash.agency +12 -28
- package/stdlib/safeBash.js +244 -291
- package/stdlib/syntax.agency +19 -12
- package/stdlib/syntax.js +7 -5
- package/dist/lib/cli/eval/run.workdir.test.js +0 -111
- package/dist/lib/eval/runEvalInput.d.ts +0 -63
- package/dist/lib/eval/runEvalInput.js +0 -79
- package/dist/lib/eval/runWorkdir.d.ts +0 -25
- package/dist/lib/eval/runWorkdir.js +0 -46
- package/dist/lib/eval/runWorkdir.test.js +0 -42
- package/dist/lib/optimize/artifacts.d.ts +0 -46
- package/dist/lib/optimize/artifacts.js +0 -151
- package/dist/lib/optimize/artifacts.test.js +0 -181
- package/dist/lib/optimize/gradeBreakdown.d.ts +0 -22
- package/dist/lib/optimize/gradeBreakdown.js +0 -20
- package/dist/lib/optimize/history.d.ts +0 -16
- package/dist/lib/optimize/history.js +0 -29
- package/dist/lib/optimize/history.test.js +0 -37
- package/dist/lib/optimize/inputs.d.ts +0 -2
- package/dist/lib/optimize/inputs.js +0 -9
- package/dist/lib/optimize/inputs.test.js +0 -17
- package/dist/lib/optimize/loop.d.ts +0 -24
- package/dist/lib/optimize/loop.js +0 -330
- package/dist/lib/optimize/loop.test.js +0 -377
- package/dist/lib/utils/projectTree.d.ts +0 -13
- package/dist/lib/utils/projectTree.js +0 -42
- package/dist/lib/utils/projectTree.test.js +0 -49
- /package/dist/lib/{cli/eval/run.workdir.test.d.ts → agentTarget.test.d.ts} +0 -0
- /package/dist/lib/{eval/runWorkdir.test.d.ts → analysis/closure.test.d.ts} +0 -0
- /package/dist/lib/{optimize/artifacts.test.d.ts → backends/topLevelRefusal.test.d.ts} +0 -0
- /package/dist/lib/{optimize/grading → cli/eval}/grade.test.d.ts +0 -0
- /package/dist/lib/{optimize/goalJudgeFile.test.d.ts → cli/eval/logs.test.d.ts} +0 -0
- /package/dist/lib/{optimize/gradeBreakdown.test.d.ts → cli/validatorInterrupts.spawn.test.d.ts} +0 -0
- /package/dist/lib/{optimize → eval}/grading/agencyRunner.d.ts +0 -0
- /package/dist/lib/{optimize → eval}/grading/agencyRunner.js +0 -0
- /package/dist/lib/{optimize → eval}/grading/agencyRunner.test.d.ts +0 -0
- /package/dist/lib/{optimize → eval}/grading/agencyRunner.test.js +0 -0
- /package/dist/lib/{optimize → eval}/grading/aggregate.d.ts +0 -0
- /package/dist/lib/{optimize → eval}/grading/aggregate.js +0 -0
- /package/dist/lib/{optimize → eval}/grading/aggregate.test.d.ts +0 -0
- /package/dist/lib/{optimize → eval}/grading/aggregate.test.js +0 -0
- /package/dist/lib/{optimize → eval}/grading/baseGrader.d.ts +0 -0
- /package/dist/lib/{optimize → eval}/grading/baseGrader.js +0 -0
- /package/dist/lib/{optimize → eval}/grading/baseGrader.test.d.ts +0 -0
- /package/dist/lib/{optimize → eval}/grading/functionGrader.test.d.ts +0 -0
- /package/dist/lib/{optimize → eval}/grading/getPath.d.ts +0 -0
- /package/dist/lib/{optimize → eval}/grading/getPath.js +0 -0
- /package/dist/lib/{optimize → eval}/grading/getPath.test.d.ts +0 -0
- /package/dist/lib/{optimize → eval}/grading/getPath.test.js +0 -0
- /package/dist/lib/{optimize → eval/grading}/goalJudgeFile.d.ts +0 -0
- /package/dist/lib/{optimize/grading/graders/builtinGraders.test.d.ts → eval/grading/goalJudgeFile.test.d.ts} +0 -0
- /package/dist/lib/{optimize → eval/grading}/goalJudgeFile.test.js +0 -0
- /package/dist/lib/{optimize → eval}/grading/grade.d.ts +0 -0
- /package/dist/lib/{optimize → eval}/grading/grade.js +0 -0
- /package/dist/lib/{optimize/grading/graders/humanGrader.test.d.ts → eval/grading/grade.test.d.ts} +0 -0
- /package/dist/lib/{optimize → eval}/grading/grade.test.js +0 -0
- /package/dist/lib/{optimize/grading/graders/llmJudge.test.d.ts → eval/grading/gradeBreakdown.test.d.ts} +0 -0
- /package/dist/lib/{optimize/grading/scorecard.test.d.ts → eval/grading/gradeRun.test.d.ts} +0 -0
- /package/dist/lib/{optimize → eval}/grading/graders/builtinGraders.d.ts +0 -0
- /package/dist/lib/{optimize → eval}/grading/graders/builtinGraders.js +0 -0
- /package/dist/lib/{optimize/grading/types.js → eval/grading/graders/builtinGraders.test.d.ts} +0 -0
- /package/dist/lib/{optimize → eval}/grading/graders/humanGrader.d.ts +0 -0
- /package/dist/lib/{optimize/gradingModule.test.d.ts → eval/grading/graders/humanGrader.test.d.ts} +0 -0
- /package/dist/lib/{optimize → eval}/grading/graders/llmJudge.d.ts +0 -0
- /package/dist/lib/{optimize/history.test.d.ts → eval/grading/graders/llmJudge.test.d.ts} +0 -0
- /package/dist/lib/{optimize/inputs.test.d.ts → eval/grading/gradingModule.test.d.ts} +0 -0
- /package/dist/lib/{optimize → eval}/grading/scorecard.js +0 -0
- /package/dist/lib/{optimize/loop.test.d.ts → eval/grading/scorecard.test.d.ts} +0 -0
- /package/dist/lib/{utils/projectTree.test.d.ts → eval/grading/types.js} +0 -0
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* How an agent target string is parsed and resolved. A target names an agent
|
|
3
|
+
* entry point: `path`, `path:node`, or a directory (meaning its main.agency).
|
|
4
|
+
* Consumed by the eval commands, the optimizer, and `agency test`.
|
|
5
|
+
*/
|
|
6
|
+
export declare function parseTarget(target: string): {
|
|
7
|
+
filename: string;
|
|
8
|
+
nodeName: string;
|
|
9
|
+
};
|
|
10
|
+
/** Resolve a target into the agent file, node (default "main"), and the
|
|
11
|
+
* display label run results carry. */
|
|
12
|
+
export declare function resolveEvalRunTarget(target: string): {
|
|
13
|
+
agentFile: string;
|
|
14
|
+
node: string;
|
|
15
|
+
label: string;
|
|
16
|
+
};
|
|
17
|
+
/** What runs as the agent, and what evidence it produces. `file` is the
|
|
18
|
+
* classic seeded-and-compiled .agency target; `command` is a CLI run in the
|
|
19
|
+
* workdir (Agency CLIs only — the statelog is the evidence contract). A
|
|
20
|
+
* future non-Agency variant would be a third kind with a weaker record. */
|
|
21
|
+
export type EvalTarget = {
|
|
22
|
+
kind: "file";
|
|
23
|
+
agentFile: string;
|
|
24
|
+
node: string;
|
|
25
|
+
label: string;
|
|
26
|
+
} | {
|
|
27
|
+
kind: "command";
|
|
28
|
+
tokens: string[];
|
|
29
|
+
label: string;
|
|
30
|
+
};
|
|
31
|
+
/** Resolve the runner-side agent choice. Exactly one of --agent /
|
|
32
|
+
* --agent-cmd; the command's {task} placeholder is validated here, before
|
|
33
|
+
* any run. Commands come ONLY from these flags, never from suite content —
|
|
34
|
+
* suites can be remote git sources, and a suite that named its own command
|
|
35
|
+
* would be remote code execution. */
|
|
36
|
+
export declare function resolveEvalTarget(opts: {
|
|
37
|
+
agent?: string;
|
|
38
|
+
agentCmd?: string;
|
|
39
|
+
}): EvalTarget;
|
|
40
|
+
/**
|
|
41
|
+
* Fail fast when the entry node cannot receive a task: eval delivers the
|
|
42
|
+
* input's task as the node's single positional parameter, so the node must
|
|
43
|
+
* take exactly one. The subprocess bootstrap enforces the same rule at run
|
|
44
|
+
* time (resolveNodeCallArgs), but by then a workdir has been seeded and the
|
|
45
|
+
* agent compiled — and the optimizer would pay that once per candidate.
|
|
46
|
+
* A misconfigured agent should cost one error, not a suite of run failures.
|
|
47
|
+
*
|
|
48
|
+
* Best-effort: an unreadable or unparseable file, or a node defined
|
|
49
|
+
* elsewhere than the entry file, is left for compile/run to report with
|
|
50
|
+
* better errors than a pre-parse could.
|
|
51
|
+
*/
|
|
52
|
+
export declare function assertEvalEntryNodeTakesOneParameter(agentFile: string, node: string): void;
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import * as fs from "fs";
|
|
2
|
+
import * as path from "path";
|
|
3
|
+
import { MISSING_TASK_PLACEHOLDER_ERROR, TASK_PLACEHOLDER, tokenizeCommand } from "./eval/run/commandLine.js";
|
|
4
|
+
import { parseAgency } from "./parser.js";
|
|
5
|
+
/**
|
|
6
|
+
* How an agent target string is parsed and resolved. A target names an agent
|
|
7
|
+
* entry point: `path`, `path:node`, or a directory (meaning its main.agency).
|
|
8
|
+
* Consumed by the eval commands, the optimizer, and `agency test`.
|
|
9
|
+
*/
|
|
10
|
+
export function parseTarget(target) {
|
|
11
|
+
const colonIndex = target.lastIndexOf(":");
|
|
12
|
+
if (colonIndex === -1) {
|
|
13
|
+
return { filename: target, nodeName: "" };
|
|
14
|
+
}
|
|
15
|
+
const filename = target.slice(0, colonIndex);
|
|
16
|
+
const nodeName = target.slice(colonIndex + 1);
|
|
17
|
+
return { filename, nodeName };
|
|
18
|
+
}
|
|
19
|
+
/** Resolve a target into the agent file, node (default "main"), and the
|
|
20
|
+
* display label run results carry. */
|
|
21
|
+
export function resolveEvalRunTarget(target) {
|
|
22
|
+
const parsed = parseTarget(target);
|
|
23
|
+
const resolved = path.resolve(parsed.filename);
|
|
24
|
+
const agentFile = fs.existsSync(resolved) && fs.statSync(resolved).isDirectory()
|
|
25
|
+
? path.join(resolved, "main.agency")
|
|
26
|
+
: resolved;
|
|
27
|
+
const node = parsed.nodeName || "main";
|
|
28
|
+
return { agentFile, node, label: `${agentFile}:${node}` };
|
|
29
|
+
}
|
|
30
|
+
/** Resolve the runner-side agent choice. Exactly one of --agent /
|
|
31
|
+
* --agent-cmd; the command's {task} placeholder is validated here, before
|
|
32
|
+
* any run. Commands come ONLY from these flags, never from suite content —
|
|
33
|
+
* suites can be remote git sources, and a suite that named its own command
|
|
34
|
+
* would be remote code execution. */
|
|
35
|
+
export function resolveEvalTarget(opts) {
|
|
36
|
+
if ((opts.agent ? 1 : 0) + (opts.agentCmd ? 1 : 0) !== 1) {
|
|
37
|
+
throw new Error("Provide exactly one of --agent or --agent-cmd");
|
|
38
|
+
}
|
|
39
|
+
if (opts.agentCmd) {
|
|
40
|
+
const tokens = tokenizeCommand(opts.agentCmd);
|
|
41
|
+
if (!tokens.some((t) => t.includes(TASK_PLACEHOLDER))) {
|
|
42
|
+
throw new Error(MISSING_TASK_PLACEHOLDER_ERROR);
|
|
43
|
+
}
|
|
44
|
+
return { kind: "command", tokens, label: opts.agentCmd };
|
|
45
|
+
}
|
|
46
|
+
return { kind: "file", ...resolveEvalRunTarget(opts.agent) };
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Fail fast when the entry node cannot receive a task: eval delivers the
|
|
50
|
+
* input's task as the node's single positional parameter, so the node must
|
|
51
|
+
* take exactly one. The subprocess bootstrap enforces the same rule at run
|
|
52
|
+
* time (resolveNodeCallArgs), but by then a workdir has been seeded and the
|
|
53
|
+
* agent compiled — and the optimizer would pay that once per candidate.
|
|
54
|
+
* A misconfigured agent should cost one error, not a suite of run failures.
|
|
55
|
+
*
|
|
56
|
+
* Best-effort: an unreadable or unparseable file, or a node defined
|
|
57
|
+
* elsewhere than the entry file, is left for compile/run to report with
|
|
58
|
+
* better errors than a pre-parse could.
|
|
59
|
+
*/
|
|
60
|
+
export function assertEvalEntryNodeTakesOneParameter(agentFile, node) {
|
|
61
|
+
let source;
|
|
62
|
+
try {
|
|
63
|
+
source = fs.readFileSync(agentFile, "utf-8");
|
|
64
|
+
}
|
|
65
|
+
catch {
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
const parsed = parseAgency(source, {}, false);
|
|
69
|
+
if (!parsed.success)
|
|
70
|
+
return;
|
|
71
|
+
for (const candidate of parsed.result.nodes) {
|
|
72
|
+
if (candidate.type !== "graphNode" || candidate.nodeName !== node)
|
|
73
|
+
continue;
|
|
74
|
+
const count = candidate.parameters.length;
|
|
75
|
+
if (count !== 1) {
|
|
76
|
+
const detail = count === 0
|
|
77
|
+
? `takes none — add one (it may go unused: \`node ${node}(task: string) { ... }\`)`
|
|
78
|
+
: `takes ${count} (${candidate.parameters.map((p) => p.name).join(", ")}) — add a one-parameter adapter node`;
|
|
79
|
+
throw new Error(`eval delivers the input's task as the entry node's single parameter, ` +
|
|
80
|
+
`but node "${node}" in ${agentFile} ${detail}.`);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import * as fs from "fs";
|
|
2
|
+
import * as os from "os";
|
|
3
|
+
import * as path from "path";
|
|
4
|
+
import { afterEach, describe, expect, it } from "vitest";
|
|
5
|
+
import { assertEvalEntryNodeTakesOneParameter, parseTarget, resolveEvalRunTarget, resolveEvalTarget } from "./agentTarget.js";
|
|
6
|
+
const dirs = [];
|
|
7
|
+
afterEach(() => {
|
|
8
|
+
// Raw rmSync, not safeDelete: mkdtemp paths sit outside any project root,
|
|
9
|
+
// which safeDelete refuses by design. Same reasoning as runArtifacts.ts.
|
|
10
|
+
for (const tempDir of dirs.splice(0)) {
|
|
11
|
+
fs.rmSync(tempDir, { recursive: true, force: true });
|
|
12
|
+
}
|
|
13
|
+
});
|
|
14
|
+
describe("agent targets", () => {
|
|
15
|
+
it("splits path from node on the last colon", () => {
|
|
16
|
+
expect(parseTarget("a.agency:main")).toEqual({ filename: "a.agency", nodeName: "main" });
|
|
17
|
+
expect(parseTarget("a.agency")).toEqual({ filename: "a.agency", nodeName: "" });
|
|
18
|
+
});
|
|
19
|
+
it("resolves file and directory agent targets", () => {
|
|
20
|
+
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "agent-target-"));
|
|
21
|
+
dirs.push(tmpDir);
|
|
22
|
+
const file = path.join(tmpDir, "agent.agency");
|
|
23
|
+
fs.writeFileSync(file, "node main() {}\n");
|
|
24
|
+
const dir = path.join(tmpDir, "agent-dir");
|
|
25
|
+
fs.mkdirSync(dir);
|
|
26
|
+
fs.writeFileSync(path.join(dir, "main.agency"), "node main() {}\n");
|
|
27
|
+
expect(resolveEvalRunTarget(`${file}:evalMain`)).toEqual({ agentFile: file, node: "evalMain", label: `${file}:evalMain` });
|
|
28
|
+
expect(resolveEvalRunTarget(file)).toEqual({ agentFile: file, node: "main", label: `${file}:main` });
|
|
29
|
+
expect(resolveEvalRunTarget(dir)).toEqual({ agentFile: path.join(dir, "main.agency"), node: "main", label: `${path.join(dir, "main.agency")}:main` });
|
|
30
|
+
});
|
|
31
|
+
});
|
|
32
|
+
describe("resolveEvalTarget", () => {
|
|
33
|
+
it("resolves --agent into a file target", () => {
|
|
34
|
+
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "target-"));
|
|
35
|
+
dirs.push(tmpDir);
|
|
36
|
+
const file = path.join(tmpDir, "a.agency");
|
|
37
|
+
fs.writeFileSync(file, "node main(task: string) {}\n");
|
|
38
|
+
expect(resolveEvalTarget({ agent: `${file}:main` }))
|
|
39
|
+
.toEqual({ kind: "file", agentFile: file, node: "main", label: `${file}:main` });
|
|
40
|
+
});
|
|
41
|
+
it("resolves --agent-cmd into a command target with the placeholder intact", () => {
|
|
42
|
+
expect(resolveEvalTarget({ agentCmd: `agency agent -p -- {task}` }))
|
|
43
|
+
.toEqual({ kind: "command", tokens: ["agency", "agent", "-p", "--", "{task}"], label: "agency agent -p -- {task}" });
|
|
44
|
+
});
|
|
45
|
+
it("rejects both, neither, and a command without {task}", () => {
|
|
46
|
+
expect(() => resolveEvalTarget({ agent: "a.agency", agentCmd: "x {task}" })).toThrow(/exactly one of/);
|
|
47
|
+
expect(() => resolveEvalTarget({})).toThrow(/exactly one of/);
|
|
48
|
+
expect(() => resolveEvalTarget({ agentCmd: "agency agent -p hello" })).toThrow(/\{task\}/);
|
|
49
|
+
});
|
|
50
|
+
});
|
|
51
|
+
describe("assertEvalEntryNodeTakesOneParameter", () => {
|
|
52
|
+
function agentWith(source) {
|
|
53
|
+
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "agent-shape-"));
|
|
54
|
+
dirs.push(tmpDir);
|
|
55
|
+
const file = path.join(tmpDir, "agent.agency");
|
|
56
|
+
fs.writeFileSync(file, source);
|
|
57
|
+
return file;
|
|
58
|
+
}
|
|
59
|
+
it("accepts a one-parameter node, default value or not", () => {
|
|
60
|
+
expect(() => assertEvalEntryNodeTakesOneParameter(agentWith("node main(task: string) {}\n"), "main")).not.toThrow();
|
|
61
|
+
expect(() => assertEvalEntryNodeTakesOneParameter(agentWith('node main(task: string = "") {}\n'), "main")).not.toThrow();
|
|
62
|
+
});
|
|
63
|
+
it("rejects a zero-parameter node with the add-one hint", () => {
|
|
64
|
+
expect(() => assertEvalEntryNodeTakesOneParameter(agentWith("node main() {}\n"), "main"))
|
|
65
|
+
.toThrow(/takes none — add one/);
|
|
66
|
+
});
|
|
67
|
+
it("rejects a two-parameter node, naming the parameters", () => {
|
|
68
|
+
expect(() => assertEvalEntryNodeTakesOneParameter(agentWith("node main(a: string, b: string) {}\n"), "main"))
|
|
69
|
+
.toThrow(/takes 2 \(a, b\)/);
|
|
70
|
+
});
|
|
71
|
+
it("stays quiet on unparseable files and absent nodes — compile/run report those better", () => {
|
|
72
|
+
expect(() => assertEvalEntryNodeTakesOneParameter(agentWith("this is not agency"), "main")).not.toThrow();
|
|
73
|
+
expect(() => assertEvalEntryNodeTakesOneParameter(agentWith("node other(x: string) {}\n"), "main")).not.toThrow();
|
|
74
|
+
expect(() => assertEvalEntryNodeTakesOneParameter(path.join(os.tmpdir(), "does-not-exist.agency"), "main")).not.toThrow();
|
|
75
|
+
});
|
|
76
|
+
});
|
|
@@ -725,7 +725,7 @@ async function __oneShotAgent_impl(target, prompt) {
|
|
|
725
725
|
__stack.locals.__scrutinee_5 = __stack.locals.result;
|
|
726
726
|
});
|
|
727
727
|
await runner2.step(2, async (runner3) => {
|
|
728
|
-
__stack.locals.__hoist_0 = await isSuccess(__stack.locals.
|
|
728
|
+
__stack.locals.__hoist_0 = await isSuccess(__stack.locals.result);
|
|
729
729
|
if (hasInterrupts(__stack.locals.__hoist_0)) {
|
|
730
730
|
await getRuntimeContext().ctx.pendingPromises.awaitAll();
|
|
731
731
|
runner3.halt(__stack.locals.__hoist_0);
|
|
@@ -741,7 +741,7 @@ async function __oneShotAgent_impl(target, prompt) {
|
|
|
741
741
|
condition: async () => __stack.locals.__hoist_0,
|
|
742
742
|
body: async (runner3) => {
|
|
743
743
|
await runner3.step(0, async (runner4) => {
|
|
744
|
-
__stack.locals.r = __stack.locals.
|
|
744
|
+
__stack.locals.r = __stack.locals.result.value;
|
|
745
745
|
});
|
|
746
746
|
await runner3.step(1, async (runner4) => {
|
|
747
747
|
__stack.locals.__armval_3 = __stack.locals.r;
|
|
@@ -754,7 +754,7 @@ async function __oneShotAgent_impl(target, prompt) {
|
|
|
754
754
|
}
|
|
755
755
|
], async (runner3) => {
|
|
756
756
|
await runner3.step(3, async (runner4) => {
|
|
757
|
-
__stack.locals.__hoist_1 = await isFailure(__stack.locals.
|
|
757
|
+
__stack.locals.__hoist_1 = await isFailure(__stack.locals.result);
|
|
758
758
|
if (hasInterrupts(__stack.locals.__hoist_1)) {
|
|
759
759
|
await getRuntimeContext().ctx.pendingPromises.awaitAll();
|
|
760
760
|
runner4.halt(__stack.locals.__hoist_1);
|
|
@@ -770,7 +770,7 @@ async function __oneShotAgent_impl(target, prompt) {
|
|
|
770
770
|
condition: async () => __stack.locals.__hoist_1,
|
|
771
771
|
body: async (runner4) => {
|
|
772
772
|
await runner4.step(0, async (runner5) => {
|
|
773
|
-
__stack.locals.f = __stack.locals.
|
|
773
|
+
__stack.locals.f = __stack.locals.result.error;
|
|
774
774
|
});
|
|
775
775
|
await runner4.step(1, async (runner5) => {
|
|
776
776
|
__stack.locals.__armval_4 = await __call(formatTurnFailure, {
|
|
@@ -1588,6 +1588,10 @@ async function main({ messages, callbacks } = {}) {
|
|
|
1588
1588
|
}
|
|
1589
1589
|
const __mainNodeParams = [];
|
|
1590
1590
|
if (__process.argv[1] === fileURLToPath(import.meta.url)) {
|
|
1591
|
+
if (__process.argv.length > 2) {
|
|
1592
|
+
console.error(`main() takes 0 argument(s) but got ${__process.argv.length - 2}; extra: ${__process.argv.slice(2).join(" ")}`);
|
|
1593
|
+
__process.exit(1);
|
|
1594
|
+
}
|
|
1591
1595
|
try {
|
|
1592
1596
|
const initialState = {
|
|
1593
1597
|
messages: new ThreadStore(),
|
|
@@ -2038,7 +2038,7 @@ async function __runTurn_impl(agentName, userMsg) {
|
|
|
2038
2038
|
__stack.locals.__scrutinee_14 = __stack.locals.result;
|
|
2039
2039
|
});
|
|
2040
2040
|
await runner2.step(4, async (runner3) => {
|
|
2041
|
-
__stack.locals.__hoist_2 = await isSuccess(__stack.locals.
|
|
2041
|
+
__stack.locals.__hoist_2 = await isSuccess(__stack.locals.result);
|
|
2042
2042
|
if (hasInterrupts(__stack.locals.__hoist_2)) {
|
|
2043
2043
|
await getRuntimeContext().ctx.pendingPromises.awaitAll();
|
|
2044
2044
|
runner3.halt(__stack.locals.__hoist_2);
|
|
@@ -2054,7 +2054,7 @@ async function __runTurn_impl(agentName, userMsg) {
|
|
|
2054
2054
|
condition: async () => __stack.locals.__hoist_2,
|
|
2055
2055
|
body: async (runner3) => {
|
|
2056
2056
|
await runner3.step(0, async (runner4) => {
|
|
2057
|
-
__stack.locals.reply = __stack.locals.
|
|
2057
|
+
__stack.locals.reply = __stack.locals.result.value;
|
|
2058
2058
|
});
|
|
2059
2059
|
await runner3.step(1, async (runner4) => {
|
|
2060
2060
|
__stack.locals.__armval_13 = __stack.locals.reply;
|
|
@@ -2067,7 +2067,7 @@ async function __runTurn_impl(agentName, userMsg) {
|
|
|
2067
2067
|
}
|
|
2068
2068
|
], async (runner3) => {
|
|
2069
2069
|
await runner3.step(3, async (runner4) => {
|
|
2070
|
-
__stack.locals.__hoist_3 = await isFailure(__stack.locals.
|
|
2070
|
+
__stack.locals.__hoist_3 = await isFailure(__stack.locals.result);
|
|
2071
2071
|
if (hasInterrupts(__stack.locals.__hoist_3)) {
|
|
2072
2072
|
await getRuntimeContext().ctx.pendingPromises.awaitAll();
|
|
2073
2073
|
runner4.halt(__stack.locals.__hoist_3);
|
|
@@ -2083,7 +2083,7 @@ async function __runTurn_impl(agentName, userMsg) {
|
|
|
2083
2083
|
condition: async () => __stack.locals.__hoist_3,
|
|
2084
2084
|
body: async (runner4) => {
|
|
2085
2085
|
await runner4.step(0, async (runner5) => {
|
|
2086
|
-
__stack.locals.f = __stack.locals.
|
|
2086
|
+
__stack.locals.f = __stack.locals.result.error;
|
|
2087
2087
|
});
|
|
2088
2088
|
await runner4.step(1, async (runner5) => {
|
|
2089
2089
|
__stack.locals.dimension = `cost`;
|
|
@@ -851,7 +851,7 @@ async function __getPolicyForAgent_impl(_policyArg, interactive) {
|
|
|
851
851
|
__stack.locals.__scrutinee_7 = __stack.locals.loaded;
|
|
852
852
|
});
|
|
853
853
|
await runner3.step(2, async (runner4) => {
|
|
854
|
-
__stack.locals.__hoist_6 = await isSuccess(__stack.locals.
|
|
854
|
+
__stack.locals.__hoist_6 = await isSuccess(__stack.locals.loaded);
|
|
855
855
|
if (hasInterrupts(__stack.locals.__hoist_6)) {
|
|
856
856
|
await getRuntimeContext().ctx.pendingPromises.awaitAll();
|
|
857
857
|
runner4.halt(__stack.locals.__hoist_6);
|
|
@@ -867,7 +867,7 @@ async function __getPolicyForAgent_impl(_policyArg, interactive) {
|
|
|
867
867
|
condition: async () => __stack.locals.__hoist_6,
|
|
868
868
|
body: async (runner4) => {
|
|
869
869
|
await runner4.step(0, async (runner5) => {
|
|
870
|
-
__stack.locals._ = __stack.locals.
|
|
870
|
+
__stack.locals._ = __stack.locals.loaded.value;
|
|
871
871
|
});
|
|
872
872
|
await runner4.step(1, async (runner5) => {
|
|
873
873
|
__stack.locals.__armval_6 = __stack.args._policyArg;
|
|
@@ -880,7 +880,7 @@ async function __getPolicyForAgent_impl(_policyArg, interactive) {
|
|
|
880
880
|
}
|
|
881
881
|
], async (runner4) => {
|
|
882
882
|
await runner4.step(3, async (runner5) => {
|
|
883
|
-
__stack.locals.__hoist_7 = await isFailure(__stack.locals.
|
|
883
|
+
__stack.locals.__hoist_7 = await isFailure(__stack.locals.loaded);
|
|
884
884
|
if (hasInterrupts(__stack.locals.__hoist_7)) {
|
|
885
885
|
await getRuntimeContext().ctx.pendingPromises.awaitAll();
|
|
886
886
|
runner5.halt(__stack.locals.__hoist_7);
|
|
@@ -896,7 +896,7 @@ async function __getPolicyForAgent_impl(_policyArg, interactive) {
|
|
|
896
896
|
condition: async () => __stack.locals.__hoist_7,
|
|
897
897
|
body: async (runner5) => {
|
|
898
898
|
await runner5.step(0, async (runner6) => {
|
|
899
|
-
__stack.locals.f = __stack.locals.
|
|
899
|
+
__stack.locals.f = __stack.locals.loaded.error;
|
|
900
900
|
});
|
|
901
901
|
await runner5.step(1, async (runner6) => {
|
|
902
902
|
__stack.locals.__hoist_8 = await __call(builtinPolicyNames, {
|
|
@@ -762,6 +762,10 @@ async function createMeals({ messages, callbacks } = {}) {
|
|
|
762
762
|
}
|
|
763
763
|
const __createMealsNodeParams = [];
|
|
764
764
|
if (__process.argv[1] === fileURLToPath(import.meta.url)) {
|
|
765
|
+
if (__process.argv.length > 2) {
|
|
766
|
+
console.error(`main() takes 0 argument(s) but got ${__process.argv.length - 2}; extra: ${__process.argv.slice(2).join(" ")}`);
|
|
767
|
+
__process.exit(1);
|
|
768
|
+
}
|
|
765
769
|
try {
|
|
766
770
|
const initialState = {
|
|
767
771
|
messages: new ThreadStore(),
|
|
@@ -314,12 +314,16 @@ async function main(goal, output, expected, { messages, callbacks } = {}) {
|
|
|
314
314
|
}
|
|
315
315
|
const __mainNodeParams = ["goal", "output", "expected"];
|
|
316
316
|
if (__process.argv[1] === fileURLToPath(import.meta.url)) {
|
|
317
|
+
if (__process.argv.length > 5) {
|
|
318
|
+
console.error(`main() takes 3 argument(s) but got ${__process.argv.length - 2}; extra: ${__process.argv.slice(5).join(" ")}`);
|
|
319
|
+
__process.exit(1);
|
|
320
|
+
}
|
|
317
321
|
try {
|
|
318
322
|
const initialState = {
|
|
319
323
|
messages: new ThreadStore(),
|
|
320
324
|
data: {}
|
|
321
325
|
};
|
|
322
|
-
const __result = await main(initialState);
|
|
326
|
+
const __result = await main(__process.argv[2], __process.argv[3], __process.argv[4], initialState);
|
|
323
327
|
await resolveCliInterrupts(__result, respondToInterrupts);
|
|
324
328
|
} catch (__error) {
|
|
325
329
|
reportBudgetExceededAndExit(__error);
|
|
@@ -2067,6 +2067,10 @@ async function main({ messages, callbacks } = {}) {
|
|
|
2067
2067
|
}
|
|
2068
2068
|
const __mainNodeParams = [];
|
|
2069
2069
|
if (__process.argv[1] === fileURLToPath(import.meta.url)) {
|
|
2070
|
+
if (__process.argv.length > 2) {
|
|
2071
|
+
console.error(`main() takes 0 argument(s) but got ${__process.argv.length - 2}; extra: ${__process.argv.slice(2).join(" ")}`);
|
|
2072
|
+
__process.exit(1);
|
|
2073
|
+
}
|
|
2070
2074
|
try {
|
|
2071
2075
|
const initialState = {
|
|
2072
2076
|
messages: new ThreadStore(),
|
|
@@ -802,6 +802,10 @@ async function end({ messages, callbacks } = {}) {
|
|
|
802
802
|
}
|
|
803
803
|
const __endNodeParams = [];
|
|
804
804
|
if (__process.argv[1] === fileURLToPath(import.meta.url)) {
|
|
805
|
+
if (__process.argv.length > 2) {
|
|
806
|
+
console.error(`main() takes 0 argument(s) but got ${__process.argv.length - 2}; extra: ${__process.argv.slice(2).join(" ")}`);
|
|
807
|
+
__process.exit(1);
|
|
808
|
+
}
|
|
805
809
|
try {
|
|
806
810
|
const initialState = {
|
|
807
811
|
messages: new ThreadStore(),
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { AgencyProgram } from "../types.js";
|
|
2
|
+
/** One file of an agent's .agency import closure, parsed exactly once.
|
|
3
|
+
* Two consumers: seeding reads only `absoluteFile`; the optimizer's target
|
|
4
|
+
* discovery reads `source` (content hashes for safe writeback) and
|
|
5
|
+
* `program` (finding optimize-marked declarations). */
|
|
6
|
+
export type ParsedSourceFile = {
|
|
7
|
+
absoluteFile: string;
|
|
8
|
+
source: string;
|
|
9
|
+
program: AgencyProgram;
|
|
10
|
+
};
|
|
11
|
+
/** Walk the local .agency import closure of `absoluteEntryFile`, parsing each
|
|
12
|
+
* file once. The single parse pass target discovery and seeding both share. */
|
|
13
|
+
export declare function walkAgencyClosure(absoluteEntryFile: string): ParsedSourceFile[];
|
|
14
|
+
/** Absolute base directory a closure's relative paths hang off: the files'
|
|
15
|
+
* common ancestor, unless that lies inside the current working directory, in
|
|
16
|
+
* which case cwd wins. */
|
|
17
|
+
export declare function closureBaseDir(absoluteFiles: string[]): string;
|
|
18
|
+
/** The entry file's full local closure: every .agency file the walk reaches,
|
|
19
|
+
* plus the transitive local TS/JS files their interop imports need. baseDir
|
|
20
|
+
* is computed from the .agency files only, so adding TS deps can never
|
|
21
|
+
* shift it. */
|
|
22
|
+
export declare function agentClosure(entryFile: string): {
|
|
23
|
+
baseDir: string;
|
|
24
|
+
files: string[];
|
|
25
|
+
};
|
|
26
|
+
export declare function agentClosureBaseDir(entryFile: string): string;
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import path from "path";
|
|
3
|
+
import { buildSync } from "esbuild";
|
|
4
|
+
import { agencyImportTargets, nonAgencyLocalImportTargets, resolveAgencyImportPath } from "../importPaths.js";
|
|
5
|
+
import { parseAgency } from "../parser.js";
|
|
6
|
+
/** Walk the local .agency import closure of `absoluteEntryFile`, parsing each
|
|
7
|
+
* file once. The single parse pass target discovery and seeding both share. */
|
|
8
|
+
export function walkAgencyClosure(absoluteEntryFile) {
|
|
9
|
+
const parsedFiles = [];
|
|
10
|
+
visitFile(absoluteEntryFile, {}, parsedFiles);
|
|
11
|
+
return parsedFiles;
|
|
12
|
+
}
|
|
13
|
+
function visitFile(absoluteFile, visited, parsedFiles) {
|
|
14
|
+
const canonicalFile = fs.realpathSync(absoluteFile);
|
|
15
|
+
if (visited[canonicalFile])
|
|
16
|
+
return;
|
|
17
|
+
visited[canonicalFile] = true;
|
|
18
|
+
const source = fs.readFileSync(canonicalFile, "utf8");
|
|
19
|
+
const parseResult = parseAgency(source, {}, false);
|
|
20
|
+
if (!parseResult.success) {
|
|
21
|
+
throw new Error(`Failed to parse ${canonicalFile}: ${parseResult.message ?? "parse error"}`);
|
|
22
|
+
}
|
|
23
|
+
parsedFiles.push({ absoluteFile: canonicalFile, source, program: parseResult.result });
|
|
24
|
+
for (const importPath of agencyImportTargets(parseResult.result, { localOnly: true })) {
|
|
25
|
+
visitFile(resolveAgencyImportPath(importPath, canonicalFile), visited, parsedFiles);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
/** Absolute base directory a closure's relative paths hang off: the files'
|
|
29
|
+
* common ancestor, unless that lies inside the current working directory, in
|
|
30
|
+
* which case cwd wins. */
|
|
31
|
+
export function closureBaseDir(absoluteFiles) {
|
|
32
|
+
const ancestor = commonAncestor(absoluteFiles.map((file) => path.dirname(file)));
|
|
33
|
+
const cwd = fs.realpathSync(process.cwd());
|
|
34
|
+
return isInsideOrSame(ancestor, cwd) ? cwd : ancestor;
|
|
35
|
+
}
|
|
36
|
+
function commonAncestor(paths) {
|
|
37
|
+
if (paths.length === 0)
|
|
38
|
+
return process.cwd();
|
|
39
|
+
const [first, ...rest] = paths.map((candidate) => path.resolve(candidate).split(path.sep));
|
|
40
|
+
const prefix = [];
|
|
41
|
+
for (let index = 0; index < first.length; index += 1) {
|
|
42
|
+
const segment = first[index];
|
|
43
|
+
if (rest.some((candidate) => candidate[index] !== segment))
|
|
44
|
+
break;
|
|
45
|
+
prefix.push(segment);
|
|
46
|
+
}
|
|
47
|
+
const joined = prefix.join(path.sep);
|
|
48
|
+
return joined === "" ? path.sep : joined;
|
|
49
|
+
}
|
|
50
|
+
function isInsideOrSame(candidate, parent) {
|
|
51
|
+
const relative = path.relative(parent, path.resolve(candidate));
|
|
52
|
+
return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative));
|
|
53
|
+
}
|
|
54
|
+
/** The entry file's full local closure: every .agency file the walk reaches,
|
|
55
|
+
* plus the transitive local TS/JS files their interop imports need. baseDir
|
|
56
|
+
* is computed from the .agency files only, so adding TS deps can never
|
|
57
|
+
* shift it. */
|
|
58
|
+
export function agentClosure(entryFile) {
|
|
59
|
+
const absoluteEntryFile = fs.realpathSync(path.resolve(entryFile));
|
|
60
|
+
const parsedFiles = walkAgencyClosure(absoluteEntryFile);
|
|
61
|
+
const agencyFiles = parsedFiles.map((parsed) => parsed.absoluteFile);
|
|
62
|
+
const interopEntries = parsedFiles.flatMap((parsed) => nonAgencyLocalImportTargets(parsed.program).map((specifier) => resolveInteropEntry(path.dirname(parsed.absoluteFile), specifier)));
|
|
63
|
+
const interopFiles = interopEntries.flatMap((interopEntry) => transitiveTsFiles(interopEntry));
|
|
64
|
+
const files = uniqueSorted([...agencyFiles, ...interopFiles].sort());
|
|
65
|
+
return { baseDir: closureBaseDir(agencyFiles), files };
|
|
66
|
+
}
|
|
67
|
+
/** `./greet.js` on disk may be greet.ts (the guide says to write .js even for
|
|
68
|
+
* TypeScript sources). Prefer the file that exists; .ts when both do not. */
|
|
69
|
+
function resolveInteropEntry(fromDir, specifier) {
|
|
70
|
+
const asWritten = path.resolve(fromDir, specifier);
|
|
71
|
+
if (fs.existsSync(asWritten))
|
|
72
|
+
return asWritten;
|
|
73
|
+
const asTs = asWritten.replace(/\.js$/, ".ts");
|
|
74
|
+
if (fs.existsSync(asTs))
|
|
75
|
+
return asTs;
|
|
76
|
+
return asWritten; // let esbuild produce the resolution error, which names the importer
|
|
77
|
+
}
|
|
78
|
+
/** The transitive local file set of one TS/JS entry, from esbuild's metafile.
|
|
79
|
+
* Bare (package) imports stay external, so only relative files appear. */
|
|
80
|
+
function transitiveTsFiles(interopEntry) {
|
|
81
|
+
const result = buildSync({
|
|
82
|
+
entryPoints: [interopEntry],
|
|
83
|
+
bundle: true,
|
|
84
|
+
write: false,
|
|
85
|
+
metafile: true,
|
|
86
|
+
packages: "external",
|
|
87
|
+
logLevel: "silent",
|
|
88
|
+
});
|
|
89
|
+
return Object.keys(result.metafile.inputs).map((inputPath) => path.resolve(inputPath));
|
|
90
|
+
}
|
|
91
|
+
/** Remove duplicates from a sorted list (equal items are adjacent). Dupes
|
|
92
|
+
* arise when two .agency files import the same TS helper, or two TS entries
|
|
93
|
+
* share transitive files. */
|
|
94
|
+
function uniqueSorted(sorted) {
|
|
95
|
+
return sorted.filter((item, index) => index === 0 || item !== sorted[index - 1]);
|
|
96
|
+
}
|
|
97
|
+
export function agentClosureBaseDir(entryFile) {
|
|
98
|
+
return agentClosure(entryFile).baseDir;
|
|
99
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import * as fs from "fs";
|
|
2
|
+
import * as os from "os";
|
|
3
|
+
import * as path from "path";
|
|
4
|
+
import { describe, expect, it } from "vitest";
|
|
5
|
+
import { agentClosure, agentClosureBaseDir } from "./closure.js";
|
|
6
|
+
describe("agentClosure", () => {
|
|
7
|
+
function closureProject() {
|
|
8
|
+
const projectDir = fs.mkdtempSync(path.join(os.tmpdir(), "closure-"));
|
|
9
|
+
fs.mkdirSync(path.join(projectDir, "lib"), { recursive: true });
|
|
10
|
+
fs.writeFileSync(path.join(projectDir, "lib", "helper.agency"), "export def helper(): string { return \"hi\" }\n");
|
|
11
|
+
fs.writeFileSync(path.join(projectDir, "agent.agency"), "import { helper } from \"./lib/helper.agency\"\nnode main() {}\n");
|
|
12
|
+
fs.writeFileSync(path.join(projectDir, "unrelated.agency"), "node main() {}\n");
|
|
13
|
+
return projectDir;
|
|
14
|
+
}
|
|
15
|
+
it("returns the entry plus every transitively imported .agency file, and agrees with agentClosureBaseDir", () => {
|
|
16
|
+
const projectDir = closureProject();
|
|
17
|
+
const closure = agentClosure(path.join(projectDir, "agent.agency"));
|
|
18
|
+
const rels = closure.files.map((file) => path.relative(closure.baseDir, file)).sort();
|
|
19
|
+
expect(rels).toEqual(["agent.agency", path.join("lib", "helper.agency")]);
|
|
20
|
+
expect(closure.baseDir).toBe(agentClosureBaseDir(path.join(projectDir, "agent.agency")));
|
|
21
|
+
fs.rmSync(projectDir, { recursive: true, force: true });
|
|
22
|
+
});
|
|
23
|
+
it("includes TypeScript interop files, transitively, resolving ./x.js to x.ts", () => {
|
|
24
|
+
const projectDir = closureProject();
|
|
25
|
+
// agent imports ./greet.js (the interop form); greet.ts exists and imports ./util.ts
|
|
26
|
+
fs.writeFileSync(path.join(projectDir, "agent.agency"), "import { helper } from \"./lib/helper.agency\"\nimport { greet } from \"./greet.js\"\nnode main() {}\n");
|
|
27
|
+
fs.writeFileSync(path.join(projectDir, "greet.ts"), "import { upper } from \"./util.js\";\nexport function greet(name: string): string { return upper(name); }\n");
|
|
28
|
+
fs.writeFileSync(path.join(projectDir, "util.ts"), "export function upper(value: string): string { return value.toUpperCase(); }\n");
|
|
29
|
+
const closure = agentClosure(path.join(projectDir, "agent.agency"));
|
|
30
|
+
const rels = closure.files.map((file) => path.relative(closure.baseDir, file)).sort();
|
|
31
|
+
expect(rels).toEqual(["agent.agency", "greet.ts", path.join("lib", "helper.agency"), "util.ts"]);
|
|
32
|
+
fs.rmSync(projectDir, { recursive: true, force: true });
|
|
33
|
+
});
|
|
34
|
+
it("leaves bare package imports external (no node_modules files in the closure)", () => {
|
|
35
|
+
const projectDir = closureProject();
|
|
36
|
+
fs.writeFileSync(path.join(projectDir, "agent.agency"), "import { z } from \"./schema.js\"\nnode main() {}\n");
|
|
37
|
+
fs.writeFileSync(path.join(projectDir, "schema.ts"), "import { z } from \"zod\";\nexport { z };\n");
|
|
38
|
+
const closure = agentClosure(path.join(projectDir, "agent.agency"));
|
|
39
|
+
const rels = closure.files.map((file) => path.relative(closure.baseDir, file)).sort();
|
|
40
|
+
expect(rels).toEqual(["agent.agency", "schema.ts"]);
|
|
41
|
+
fs.rmSync(projectDir, { recursive: true, force: true });
|
|
42
|
+
});
|
|
43
|
+
});
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { AgencyProgram } from "../types.js";
|
|
2
|
+
/** The agency-file imports a program declares: import-node statements plus
|
|
3
|
+
* `import ... from "<agency path>"` statements. Filters out npm/Node module
|
|
4
|
+
* imports — see compile.ts's getAllImports for the unfiltered walk. */
|
|
5
|
+
export declare function getImports(program: AgencyProgram): string[];
|
|
6
|
+
export type AnyImport = {
|
|
7
|
+
path: string;
|
|
8
|
+
kind: "module" | "node";
|
|
9
|
+
};
|
|
10
|
+
export declare function getAllImports(program: AgencyProgram): AnyImport[];
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { isAgencyImport } from "../importPaths.js";
|
|
2
|
+
/** The agency-file imports a program declares: import-node statements plus
|
|
3
|
+
* `import ... from "<agency path>"` statements. Filters out npm/Node module
|
|
4
|
+
* imports — see compile.ts's getAllImports for the unfiltered walk. */
|
|
5
|
+
export function getImports(program) {
|
|
6
|
+
const toolAndNodeImports = program.nodes
|
|
7
|
+
.filter((node) => node.type === "importNodeStatement")
|
|
8
|
+
.map((node) => node.agencyFile.trim());
|
|
9
|
+
// this makes compile() try to parse non-agency files
|
|
10
|
+
const importStatements = program.nodes
|
|
11
|
+
.filter((node) => node.type === "importStatement" && isAgencyImport(node.modulePath))
|
|
12
|
+
.map((node) => node.modulePath.trim());
|
|
13
|
+
return [...toolAndNodeImports, ...importStatements];
|
|
14
|
+
}
|
|
15
|
+
export function getAllImports(program) {
|
|
16
|
+
return program.nodes.flatMap((node) => {
|
|
17
|
+
if (node.type === "importStatement") {
|
|
18
|
+
return [{ path: node.modulePath.trim(), kind: "module" }];
|
|
19
|
+
}
|
|
20
|
+
if (node.type === "importNodeStatement") {
|
|
21
|
+
return [{ path: node.agencyFile.trim(), kind: "node" }];
|
|
22
|
+
}
|
|
23
|
+
return [];
|
|
24
|
+
});
|
|
25
|
+
}
|
|
@@ -44,7 +44,6 @@ export declare class AgencyGenerator {
|
|
|
44
44
|
protected importAttachedComments: Map<ImportStatement | ImportNodeStatement, AgencyNode[]>;
|
|
45
45
|
protected functionDefinitions: Record<string, FunctionDefinition>;
|
|
46
46
|
protected currentScope: Scope[];
|
|
47
|
-
protected program: AgencyProgram | null;
|
|
48
47
|
protected agencyConfig: AgencyConfig;
|
|
49
48
|
private indentLevel;
|
|
50
49
|
private indentSize;
|
|
@@ -201,6 +200,20 @@ export declare class AgencyGenerator {
|
|
|
201
200
|
protected processCodeLiteral(node: CodeLiteral): string;
|
|
202
201
|
protected asyncAwaitPrefix(code: string, async?: boolean): string;
|
|
203
202
|
protected processMatchBlock(node: MatchBlock): string;
|
|
203
|
+
/** Whether a one-statement arm body may print inline (`=> stmt`)
|
|
204
|
+
* rather than as a `{ ... }` block.
|
|
205
|
+
*
|
|
206
|
+
* The author's form wins: an arm written as a block prints as a block
|
|
207
|
+
* (`blockBody`, set by the parser), an arm written inline prints
|
|
208
|
+
* inline. The shape tests below only decide for cases with NO
|
|
209
|
+
* recorded form — ASTs built programmatically — where inline is only
|
|
210
|
+
* safe for the shapes the single-statement arm grammar accepts;
|
|
211
|
+
* anything else must take block form or the printed arm will not
|
|
212
|
+
* re-parse (#708). */
|
|
213
|
+
protected armPrintsInline(caseNode: {
|
|
214
|
+
body: AgencyNode[];
|
|
215
|
+
blockBody?: true;
|
|
216
|
+
}): boolean;
|
|
204
217
|
protected processForLoop(node: ForLoop): string;
|
|
205
218
|
protected processWhileLoop(node: WhileLoop): string;
|
|
206
219
|
protected processIfElse(node: IfElse): string;
|