agent-bober 0.15.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +20 -0
- package/.claude-plugin/plugin.json +2 -2
- package/CHANGELOG.md +30 -0
- package/README.md +127 -3
- package/agents/bober-architect.md +34 -0
- package/agents/bober-code-reviewer.md +2 -0
- package/agents/bober-curator.md +12 -0
- package/agents/bober-documenter.md +129 -0
- package/agents/bober-evaluator.md +46 -0
- package/agents/bober-generator.md +12 -0
- package/agents/bober-planner.md +8 -1
- package/dist/cli/commands/graph.js +3 -3
- package/dist/cli/commands/graph.js.map +1 -1
- package/dist/cli/commands/init.js +4 -0
- package/dist/cli/commands/init.js.map +1 -1
- package/dist/cli/commands/memory.d.ts +14 -0
- package/dist/cli/commands/memory.d.ts.map +1 -0
- package/dist/cli/commands/memory.js +132 -0
- package/dist/cli/commands/memory.js.map +1 -0
- package/dist/cli/index.js +6 -0
- package/dist/cli/index.js.map +1 -1
- package/dist/config/defaults.d.ts.map +1 -1
- package/dist/config/defaults.js +14 -3
- package/dist/config/defaults.js.map +1 -1
- package/dist/config/loader.d.ts.map +1 -1
- package/dist/config/loader.js +7 -0
- package/dist/config/loader.js.map +1 -1
- package/dist/config/role-providers.d.ts +29 -0
- package/dist/config/role-providers.d.ts.map +1 -0
- package/dist/config/role-providers.js +115 -0
- package/dist/config/role-providers.js.map +1 -0
- package/dist/config/schema.d.ts +383 -14
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +42 -0
- package/dist/config/schema.js.map +1 -1
- package/dist/contracts/eval-result.d.ts +112 -56
- package/dist/contracts/eval-result.d.ts.map +1 -1
- package/dist/contracts/eval-result.js +3 -0
- package/dist/contracts/eval-result.js.map +1 -1
- package/dist/contracts/sprint-contract.d.ts +30 -30
- package/dist/fleet/aggregator.d.ts +5 -0
- package/dist/fleet/aggregator.d.ts.map +1 -0
- package/dist/fleet/aggregator.js +39 -0
- package/dist/fleet/aggregator.js.map +1 -0
- package/dist/fleet/child-config.d.ts +12 -0
- package/dist/fleet/child-config.d.ts.map +1 -0
- package/dist/fleet/child-config.js +38 -0
- package/dist/fleet/child-config.js.map +1 -0
- package/dist/fleet/coordinator.d.ts +25 -0
- package/dist/fleet/coordinator.d.ts.map +1 -0
- package/dist/fleet/coordinator.js +40 -0
- package/dist/fleet/coordinator.js.map +1 -0
- package/dist/fleet/index.d.ts +40 -0
- package/dist/fleet/index.d.ts.map +1 -0
- package/dist/fleet/index.js +117 -0
- package/dist/fleet/index.js.map +1 -0
- package/dist/fleet/manifest.d.ts +51 -0
- package/dist/fleet/manifest.d.ts.map +1 -0
- package/dist/fleet/manifest.js +32 -0
- package/dist/fleet/manifest.js.map +1 -0
- package/dist/fleet/reporter.d.ts +32 -0
- package/dist/fleet/reporter.d.ts.map +1 -0
- package/dist/fleet/reporter.js +71 -0
- package/dist/fleet/reporter.js.map +1 -0
- package/dist/fleet/runner.d.ts +48 -0
- package/dist/fleet/runner.d.ts.map +1 -0
- package/dist/fleet/runner.js +104 -0
- package/dist/fleet/runner.js.map +1 -0
- package/dist/fleet/scaffolder.d.ts +12 -0
- package/dist/fleet/scaffolder.d.ts.map +1 -0
- package/dist/fleet/scaffolder.js +82 -0
- package/dist/fleet/scaffolder.js.map +1 -0
- package/dist/fleet/types.d.ts +21 -0
- package/dist/fleet/types.d.ts.map +1 -0
- package/dist/fleet/types.js +2 -0
- package/dist/fleet/types.js.map +1 -0
- package/dist/graph/cli.d.ts +6 -2
- package/dist/graph/cli.d.ts.map +1 -1
- package/dist/graph/cli.js +53 -12
- package/dist/graph/cli.js.map +1 -1
- package/dist/graph/pipeline-lifecycle.d.ts +9 -0
- package/dist/graph/pipeline-lifecycle.d.ts.map +1 -1
- package/dist/graph/pipeline-lifecycle.js +12 -0
- package/dist/graph/pipeline-lifecycle.js.map +1 -1
- package/dist/graph/preflight-injector.d.ts +14 -0
- package/dist/graph/preflight-injector.d.ts.map +1 -1
- package/dist/graph/preflight-injector.js +84 -4
- package/dist/graph/preflight-injector.js.map +1 -1
- package/dist/incident/types.d.ts +24 -24
- package/dist/mcp/tools/graph-schemas.d.ts +7 -7
- package/dist/mcp/tools/init.d.ts.map +1 -1
- package/dist/mcp/tools/init.js +2 -0
- package/dist/mcp/tools/init.js.map +1 -1
- package/dist/orchestrator/agent-loader.d.ts.map +1 -1
- package/dist/orchestrator/agent-loader.js +15 -1
- package/dist/orchestrator/agent-loader.js.map +1 -1
- package/dist/orchestrator/agentic-loop.d.ts +51 -0
- package/dist/orchestrator/agentic-loop.d.ts.map +1 -1
- package/dist/orchestrator/agentic-loop.js +123 -4
- package/dist/orchestrator/agentic-loop.js.map +1 -1
- package/dist/orchestrator/arch-lenses.d.ts +7 -0
- package/dist/orchestrator/arch-lenses.d.ts.map +1 -0
- package/dist/orchestrator/arch-lenses.js +22 -0
- package/dist/orchestrator/arch-lenses.js.map +1 -0
- package/dist/orchestrator/architect-agent.d.ts +16 -0
- package/dist/orchestrator/architect-agent.d.ts.map +1 -1
- package/dist/orchestrator/architect-agent.js +509 -1
- package/dist/orchestrator/architect-agent.js.map +1 -1
- package/dist/orchestrator/curator-agent.js +1 -1
- package/dist/orchestrator/curator-agent.js.map +1 -1
- package/dist/orchestrator/deploy/types.d.ts +2 -2
- package/dist/orchestrator/documenter-agent.d.ts +57 -0
- package/dist/orchestrator/documenter-agent.d.ts.map +1 -0
- package/dist/orchestrator/documenter-agent.js +195 -0
- package/dist/orchestrator/documenter-agent.js.map +1 -0
- package/dist/orchestrator/environment.d.ts +45 -0
- package/dist/orchestrator/environment.d.ts.map +1 -0
- package/dist/orchestrator/environment.js +151 -0
- package/dist/orchestrator/environment.js.map +1 -0
- package/dist/orchestrator/eval-lenses.d.ts +7 -0
- package/dist/orchestrator/eval-lenses.d.ts.map +1 -0
- package/dist/orchestrator/eval-lenses.js +19 -0
- package/dist/orchestrator/eval-lenses.js.map +1 -0
- package/dist/orchestrator/eval-persist.d.ts +25 -0
- package/dist/orchestrator/eval-persist.d.ts.map +1 -0
- package/dist/orchestrator/eval-persist.js +74 -0
- package/dist/orchestrator/eval-persist.js.map +1 -0
- package/dist/orchestrator/evaluator-agent.d.ts +23 -0
- package/dist/orchestrator/evaluator-agent.d.ts.map +1 -1
- package/dist/orchestrator/evaluator-agent.js +60 -3
- package/dist/orchestrator/evaluator-agent.js.map +1 -1
- package/dist/orchestrator/generator-agent.d.ts.map +1 -1
- package/dist/orchestrator/generator-agent.js +32 -0
- package/dist/orchestrator/generator-agent.js.map +1 -1
- package/dist/orchestrator/memory/distill.d.ts +60 -0
- package/dist/orchestrator/memory/distill.d.ts.map +1 -0
- package/dist/orchestrator/memory/distill.js +177 -0
- package/dist/orchestrator/memory/distill.js.map +1 -0
- package/dist/orchestrator/memory/eval-source.d.ts +20 -0
- package/dist/orchestrator/memory/eval-source.d.ts.map +1 -0
- package/dist/orchestrator/memory/eval-source.js +88 -0
- package/dist/orchestrator/memory/eval-source.js.map +1 -0
- package/dist/orchestrator/memory/retrieve.d.ts +45 -0
- package/dist/orchestrator/memory/retrieve.d.ts.map +1 -0
- package/dist/orchestrator/memory/retrieve.js +102 -0
- package/dist/orchestrator/memory/retrieve.js.map +1 -0
- package/dist/orchestrator/model-resolver.d.ts.map +1 -1
- package/dist/orchestrator/model-resolver.js +12 -0
- package/dist/orchestrator/model-resolver.js.map +1 -1
- package/dist/orchestrator/pipeline.d.ts +10 -0
- package/dist/orchestrator/pipeline.d.ts.map +1 -1
- package/dist/orchestrator/pipeline.js +111 -3
- package/dist/orchestrator/pipeline.js.map +1 -1
- package/dist/orchestrator/planner-agent.d.ts +22 -1
- package/dist/orchestrator/planner-agent.d.ts.map +1 -1
- package/dist/orchestrator/planner-agent.js +160 -4
- package/dist/orchestrator/planner-agent.js.map +1 -1
- package/dist/orchestrator/research-agent.js +2 -2
- package/dist/orchestrator/research-agent.js.map +1 -1
- package/dist/orchestrator/tools/handlers.d.ts +14 -0
- package/dist/orchestrator/tools/handlers.d.ts.map +1 -1
- package/dist/orchestrator/tools/handlers.js +29 -4
- package/dist/orchestrator/tools/handlers.js.map +1 -1
- package/dist/orchestrator/tools/schemas.js +5 -5
- package/dist/orchestrator/tools/schemas.js.map +1 -1
- package/dist/orchestrator/workflow/args-builder.d.ts +35 -0
- package/dist/orchestrator/workflow/args-builder.d.ts.map +1 -0
- package/dist/orchestrator/workflow/args-builder.js +142 -0
- package/dist/orchestrator/workflow/args-builder.js.map +1 -0
- package/dist/orchestrator/workflow/budget.d.ts +57 -0
- package/dist/orchestrator/workflow/budget.d.ts.map +1 -0
- package/dist/orchestrator/workflow/budget.js +80 -0
- package/dist/orchestrator/workflow/budget.js.map +1 -0
- package/dist/orchestrator/workflow/conformance.d.ts +27 -0
- package/dist/orchestrator/workflow/conformance.d.ts.map +1 -0
- package/dist/orchestrator/workflow/conformance.js +111 -0
- package/dist/orchestrator/workflow/conformance.js.map +1 -0
- package/dist/orchestrator/workflow/eligibility.d.ts +8 -0
- package/dist/orchestrator/workflow/eligibility.d.ts.map +1 -0
- package/dist/orchestrator/workflow/eligibility.js +10 -0
- package/dist/orchestrator/workflow/eligibility.js.map +1 -0
- package/dist/orchestrator/workflow/engine.d.ts +10 -0
- package/dist/orchestrator/workflow/engine.d.ts.map +1 -0
- package/dist/orchestrator/workflow/engine.js +2 -0
- package/dist/orchestrator/workflow/engine.js.map +1 -0
- package/dist/orchestrator/workflow/errors.d.ts +13 -0
- package/dist/orchestrator/workflow/errors.d.ts.map +1 -0
- package/dist/orchestrator/workflow/errors.js +26 -0
- package/dist/orchestrator/workflow/errors.js.map +1 -0
- package/dist/orchestrator/workflow/flusher.d.ts +19 -0
- package/dist/orchestrator/workflow/flusher.d.ts.map +1 -0
- package/dist/orchestrator/workflow/flusher.js +81 -0
- package/dist/orchestrator/workflow/flusher.js.map +1 -0
- package/dist/orchestrator/workflow/interpreter.d.ts +48 -0
- package/dist/orchestrator/workflow/interpreter.d.ts.map +1 -0
- package/dist/orchestrator/workflow/interpreter.js +92 -0
- package/dist/orchestrator/workflow/interpreter.js.map +1 -0
- package/dist/orchestrator/workflow/pure-sprint.d.ts +65 -0
- package/dist/orchestrator/workflow/pure-sprint.d.ts.map +1 -0
- package/dist/orchestrator/workflow/pure-sprint.js +82 -0
- package/dist/orchestrator/workflow/pure-sprint.js.map +1 -0
- package/dist/orchestrator/workflow/reconciler.d.ts +15 -0
- package/dist/orchestrator/workflow/reconciler.d.ts.map +1 -0
- package/dist/orchestrator/workflow/reconciler.js +65 -0
- package/dist/orchestrator/workflow/reconciler.js.map +1 -0
- package/dist/orchestrator/workflow/resume-cursor.d.ts +10 -0
- package/dist/orchestrator/workflow/resume-cursor.d.ts.map +1 -0
- package/dist/orchestrator/workflow/resume-cursor.js +25 -0
- package/dist/orchestrator/workflow/resume-cursor.js.map +1 -0
- package/dist/orchestrator/workflow/retry.d.ts +50 -0
- package/dist/orchestrator/workflow/retry.d.ts.map +1 -0
- package/dist/orchestrator/workflow/retry.js +100 -0
- package/dist/orchestrator/workflow/retry.js.map +1 -0
- package/dist/orchestrator/workflow/scheduler.d.ts +87 -0
- package/dist/orchestrator/workflow/scheduler.d.ts.map +1 -0
- package/dist/orchestrator/workflow/scheduler.js +158 -0
- package/dist/orchestrator/workflow/scheduler.js.map +1 -0
- package/dist/orchestrator/workflow/selector.d.ts +26 -0
- package/dist/orchestrator/workflow/selector.d.ts.map +1 -0
- package/dist/orchestrator/workflow/selector.js +54 -0
- package/dist/orchestrator/workflow/selector.js.map +1 -0
- package/dist/orchestrator/workflow/synthesizer.d.ts +52 -0
- package/dist/orchestrator/workflow/synthesizer.d.ts.map +1 -0
- package/dist/orchestrator/workflow/synthesizer.js +75 -0
- package/dist/orchestrator/workflow/synthesizer.js.map +1 -0
- package/dist/orchestrator/workflow/ts-engine.d.ts +13 -0
- package/dist/orchestrator/workflow/ts-engine.d.ts.map +1 -0
- package/dist/orchestrator/workflow/ts-engine.js +14 -0
- package/dist/orchestrator/workflow/ts-engine.js.map +1 -0
- package/dist/orchestrator/workflow/types.d.ts +55 -0
- package/dist/orchestrator/workflow/types.d.ts.map +1 -0
- package/dist/orchestrator/workflow/types.js +3 -0
- package/dist/orchestrator/workflow/types.js.map +1 -0
- package/dist/orchestrator/workflow/workflow-engine.d.ts +31 -0
- package/dist/orchestrator/workflow/workflow-engine.d.ts.map +1 -0
- package/dist/orchestrator/workflow/workflow-engine.js +70 -0
- package/dist/orchestrator/workflow/workflow-engine.js.map +1 -0
- package/dist/providers/anthropic.d.ts.map +1 -1
- package/dist/providers/anthropic.js +49 -6
- package/dist/providers/anthropic.js.map +1 -1
- package/dist/providers/claude-code.d.ts +44 -0
- package/dist/providers/claude-code.d.ts.map +1 -0
- package/dist/providers/claude-code.js +143 -0
- package/dist/providers/claude-code.js.map +1 -0
- package/dist/providers/factory.d.ts +16 -2
- package/dist/providers/factory.d.ts.map +1 -1
- package/dist/providers/factory.js +66 -12
- package/dist/providers/factory.js.map +1 -1
- package/dist/providers/google.d.ts.map +1 -1
- package/dist/providers/google.js +27 -3
- package/dist/providers/google.js.map +1 -1
- package/dist/providers/index.d.ts +3 -1
- package/dist/providers/index.d.ts.map +1 -1
- package/dist/providers/index.js +3 -1
- package/dist/providers/index.js.map +1 -1
- package/dist/providers/openai.d.ts.map +1 -1
- package/dist/providers/openai.js +24 -3
- package/dist/providers/openai.js.map +1 -1
- package/dist/providers/preflight.d.ts +22 -0
- package/dist/providers/preflight.d.ts.map +1 -0
- package/dist/providers/preflight.js +54 -0
- package/dist/providers/preflight.js.map +1 -0
- package/dist/providers/structured.d.ts +130 -0
- package/dist/providers/structured.d.ts.map +1 -0
- package/dist/providers/structured.js +205 -0
- package/dist/providers/structured.js.map +1 -0
- package/dist/providers/types.d.ts +28 -0
- package/dist/providers/types.d.ts.map +1 -1
- package/dist/state/history-rotation.d.ts +17 -0
- package/dist/state/history-rotation.d.ts.map +1 -0
- package/dist/state/history-rotation.js +84 -0
- package/dist/state/history-rotation.js.map +1 -0
- package/dist/state/history.d.ts +16 -4
- package/dist/state/history.d.ts.map +1 -1
- package/dist/state/history.js +62 -20
- package/dist/state/history.js.map +1 -1
- package/dist/state/index.d.ts +1 -1
- package/dist/state/index.d.ts.map +1 -1
- package/dist/state/index.js +1 -1
- package/dist/state/index.js.map +1 -1
- package/dist/state/memory.d.ts +60 -0
- package/dist/state/memory.d.ts.map +1 -0
- package/dist/state/memory.js +242 -0
- package/dist/state/memory.js.map +1 -0
- package/hooks/hooks.json +12 -2
- package/package.json +9 -5
- package/scripts/spike-claude-code-provider.mjs +66 -0
- package/scripts/spike-deepseek.mjs +63 -0
- package/scripts/sync-targets.json +12 -0
- package/scripts/update-all.mjs +255 -0
- package/skills/bober.architect/SKILL.md +13 -0
- package/skills/bober.architect/references/arch-lens-panel.md +126 -0
- package/skills/bober.eval/SKILL.md +9 -0
- package/skills/bober.eval/references/lens-panel.md +115 -0
- package/skills/bober.plan/SKILL.md +6 -0
- package/skills/bober.run/SKILL.md +23 -4
- package/skills/bober.run/references/lens-panel.md +115 -0
- package/skills/bober.sprint/SKILL.md +44 -2
- package/skills/bober.sprint/references/lens-panel.md +115 -0
- package/skills/shared/arch-lens-panel.md +126 -0
- package/skills/shared/lens-panel.md +115 -0
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { BoberConfig } from "../config/schema.js";
|
|
2
2
|
import type { ContextHandoff } from "./context-handoff.js";
|
|
3
|
+
import type { EvalResult } from "../contracts/eval-result.js";
|
|
3
4
|
import type { EvaluationRunResult } from "../evaluators/registry.js";
|
|
4
5
|
export type { EvaluationRunResult } from "../evaluators/registry.js";
|
|
5
6
|
/**
|
|
@@ -15,4 +16,26 @@ export type { EvaluationRunResult } from "../evaluators/registry.js";
|
|
|
15
16
|
* @returns A combined EvaluationRunResult.
|
|
16
17
|
*/
|
|
17
18
|
export declare function runEvaluatorAgent(handoff: ContextHandoff, projectRoot: string, config: BoberConfig): Promise<EvaluationRunResult>;
|
|
19
|
+
/**
|
|
20
|
+
* Run the agent-based qualitative evaluation using a multi-turn agentic
|
|
21
|
+
* loop with bash, read_file, glob, and grep tools.
|
|
22
|
+
*
|
|
23
|
+
* When panel.enabled is true and >=2 lenses are configured, fans out
|
|
24
|
+
* one judge call per lens (bounded by maxConcurrent) and reconciles
|
|
25
|
+
* via majority vote into a single evaluator='panel' EvalResult.
|
|
26
|
+
* Otherwise runs the existing single judge call (byte-identical off path).
|
|
27
|
+
*
|
|
28
|
+
* The evaluator can run commands, take screenshots, inspect code, start
|
|
29
|
+
* dev servers, and curl endpoints — but CANNOT write or edit files.
|
|
30
|
+
*/
|
|
31
|
+
/**
|
|
32
|
+
* Side-effect-free lens-evaluation core. Runs the judge call(s) and returns the
|
|
33
|
+
* per-lens verdicts — a single verdict when the panel is off, one per lens
|
|
34
|
+
* (bounded concurrency) when on. Writes NOTHING to `.bober/` (no history append,
|
|
35
|
+
* no reconcile), so the workflow interpreter's `evaluate` dep can call it while
|
|
36
|
+
* the flusher remains the sole clock/commit source. The persisting wrapper
|
|
37
|
+
* {@link runAgentEvaluation} layers the history append + reconcile on top for
|
|
38
|
+
* the TS pipeline.
|
|
39
|
+
*/
|
|
40
|
+
export declare function evaluateLenses(handoff: ContextHandoff, programmaticResults: EvalResult[], projectRoot: string, config: BoberConfig): Promise<EvalResult[]>;
|
|
18
41
|
//# sourceMappingURL=evaluator-agent.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evaluator-agent.d.ts","sourceRoot":"","sources":["../../src/orchestrator/evaluator-agent.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"evaluator-agent.d.ts","sourceRoot":"","sources":["../../src/orchestrator/evaluator-agent.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAE3D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAK9D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAgBrE,YAAY,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAQrE;;;;;;;;;;;GAWG;AACH,wBAAsB,iBAAiB,CACrC,OAAO,EAAE,cAAc,EACvB,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,WAAW,GAClB,OAAO,CAAC,mBAAmB,CAAC,CA8E9B;AAID;;;;;;;;;;;GAWG;AACH;;;;;;;;GAQG;AACH,wBAAsB,cAAc,CAClC,OAAO,EAAE,cAAc,EACvB,mBAAmB,EAAE,UAAU,EAAE,EACjC,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,WAAW,GAClB,OAAO,CAAC,UAAU,EAAE,CAAC,CAQvB"}
|
|
@@ -10,8 +10,12 @@ import { runAgenticLoop } from "./agentic-loop.js";
|
|
|
10
10
|
import { PreflightContextInjector } from "../graph/preflight-injector.js";
|
|
11
11
|
import { graphPipelineLifecycle } from "../graph/pipeline-lifecycle.js";
|
|
12
12
|
import { emit } from "../telemetry/emit.js";
|
|
13
|
+
import { appendHistory } from "../state/history.js";
|
|
14
|
+
import { reconcile } from "./workflow/reconciler.js";
|
|
15
|
+
import { mapBounded } from "./workflow/scheduler.js";
|
|
16
|
+
import { resolveLensFocus } from "./eval-lenses.js";
|
|
13
17
|
// ── Constants ──────────────────────────────────────────────────────
|
|
14
|
-
const EVALUATOR_MAX_TURNS =
|
|
18
|
+
const EVALUATOR_MAX_TURNS = 100;
|
|
15
19
|
// ── Main ───────────────────────────────────────────────────────────
|
|
16
20
|
/**
|
|
17
21
|
* Run the evaluator agent, combining programmatic evaluation (plugins)
|
|
@@ -83,10 +87,58 @@ export async function runEvaluatorAgent(handoff, projectRoot, config) {
|
|
|
83
87
|
* Run the agent-based qualitative evaluation using a multi-turn agentic
|
|
84
88
|
* loop with bash, read_file, glob, and grep tools.
|
|
85
89
|
*
|
|
90
|
+
* When panel.enabled is true and >=2 lenses are configured, fans out
|
|
91
|
+
* one judge call per lens (bounded by maxConcurrent) and reconciles
|
|
92
|
+
* via majority vote into a single evaluator='panel' EvalResult.
|
|
93
|
+
* Otherwise runs the existing single judge call (byte-identical off path).
|
|
94
|
+
*
|
|
86
95
|
* The evaluator can run commands, take screenshots, inspect code, start
|
|
87
96
|
* dev servers, and curl endpoints — but CANNOT write or edit files.
|
|
88
97
|
*/
|
|
98
|
+
/**
|
|
99
|
+
* Side-effect-free lens-evaluation core. Runs the judge call(s) and returns the
|
|
100
|
+
* per-lens verdicts — a single verdict when the panel is off, one per lens
|
|
101
|
+
* (bounded concurrency) when on. Writes NOTHING to `.bober/` (no history append,
|
|
102
|
+
* no reconcile), so the workflow interpreter's `evaluate` dep can call it while
|
|
103
|
+
* the flusher remains the sole clock/commit source. The persisting wrapper
|
|
104
|
+
* {@link runAgentEvaluation} layers the history append + reconcile on top for
|
|
105
|
+
* the TS pipeline.
|
|
106
|
+
*/
|
|
107
|
+
export async function evaluateLenses(handoff, programmaticResults, projectRoot, config) {
|
|
108
|
+
const panel = config.evaluator.panel;
|
|
109
|
+
if (!panel.enabled || panel.lenses.length < 2) {
|
|
110
|
+
return [await runSingleLensEval(handoff, programmaticResults, projectRoot, config)];
|
|
111
|
+
}
|
|
112
|
+
return mapBounded(panel.lenses, panel.maxConcurrent, (lens) => runSingleLensEval(handoff, programmaticResults, projectRoot, config, lens));
|
|
113
|
+
}
|
|
89
114
|
async function runAgentEvaluation(handoff, programmaticResults, projectRoot, config) {
|
|
115
|
+
const panel = config.evaluator.panel;
|
|
116
|
+
// Side-effect-free core: run the judge call(s) → per-lens verdicts.
|
|
117
|
+
const lensResults = await evaluateLenses(handoff, programmaticResults, projectRoot, config);
|
|
118
|
+
if (!panel.enabled || panel.lenses.length < 2) {
|
|
119
|
+
// Off path — single judge call, byte-identical to the original behavior.
|
|
120
|
+
return lensResults[0];
|
|
121
|
+
}
|
|
122
|
+
const contractId = handoff.currentContract?.contractId ?? "unknown";
|
|
123
|
+
// C3 — per-lens verdict telemetry (PANEL path only; index-aligned with panel.lenses).
|
|
124
|
+
for (let i = 0; i < panel.lenses.length; i++) {
|
|
125
|
+
await appendHistory(projectRoot, {
|
|
126
|
+
timestamp: new Date().toISOString(),
|
|
127
|
+
event: "eval-lens-verdict",
|
|
128
|
+
phase: "evaluating",
|
|
129
|
+
sprintId: contractId,
|
|
130
|
+
details: { lens: panel.lenses[i], passed: lensResults[i].passed },
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
return reconcile(contractId, 1, lensResults, new Date().toISOString());
|
|
134
|
+
}
|
|
135
|
+
// ── Single-lens evaluation ──────────────────────────────────────────
|
|
136
|
+
/**
|
|
137
|
+
* Run a single judge call. When `lens` is provided, a focus block is
|
|
138
|
+
* appended to the user message; when undefined the prompt is byte-identical
|
|
139
|
+
* to the original single-judge behavior (C2).
|
|
140
|
+
*/
|
|
141
|
+
async function runSingleLensEval(handoff, programmaticResults, projectRoot, config, lens) {
|
|
90
142
|
const timestamp = new Date().toISOString();
|
|
91
143
|
try {
|
|
92
144
|
const model = resolveModel(config.evaluator.model);
|
|
@@ -158,12 +210,17 @@ Your final response must contain ONLY a JSON object matching this schema (no mar
|
|
|
158
210
|
"feedback": "Actionable feedback for the generator if anything needs fixing",
|
|
159
211
|
"timestamp": "${timestamp}"
|
|
160
212
|
}`;
|
|
213
|
+
// When a lens is provided, append a focus block (on path only).
|
|
214
|
+
// When lens is undefined the prompt is byte-identical to the original (C2).
|
|
215
|
+
const lensBlock = lens
|
|
216
|
+
? `\n\n## Evaluation Lens: ${lens}\n${resolveLensFocus(lens)}`
|
|
217
|
+
: "";
|
|
161
218
|
// Pre-flight graph context injection (ADR-9): prepend graph context to userMessage.
|
|
162
219
|
// On failure or timeout, userMessage is returned unchanged (spawn not blocked).
|
|
163
220
|
const graphClient = graphPipelineLifecycle.getGraphClient();
|
|
164
221
|
const preflightInjector = new PreflightContextInjector(graphClient, config.graph);
|
|
165
|
-
const enhancedMessage = await preflightInjector.inject("evaluator", handoff.currentContract ?? null, userMessage
|
|
166
|
-
logger.info(`Calling evaluator model (${config.evaluator.model} → ${model})...`);
|
|
222
|
+
const enhancedMessage = await preflightInjector.inject("evaluator", handoff.currentContract ?? null, `${userMessage}${lensBlock}`, { baselineSha: "HEAD~1" });
|
|
223
|
+
logger.info(`Calling evaluator model (${config.evaluator.model} → ${model})${lens ? ` [lens: ${lens}]` : ""}...`);
|
|
167
224
|
const result = await runAgenticLoop({
|
|
168
225
|
client,
|
|
169
226
|
model,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evaluator-agent.js","sourceRoot":"","sources":["../../src/orchestrator/evaluator-agent.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAExD,OAAO,EACL,qBAAqB,EACrB,aAAa,GACd,MAAM,2BAA2B,CAAC;AAEnC,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACjF,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,wBAAwB,EAAE,MAAM,gCAAgC,CAAC;AAC1E,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AACxE,OAAO,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAI5C,sEAAsE;AAEtE,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAE/B,sEAAsE;AAEtE;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,OAAuB,EACvB,WAAmB,EACnB,MAAmB;IAEnB,MAAM,QAAQ,GAAG,OAAO,CAAC,eAAe,CAAC;IACzC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACpE,CAAC;IAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,UAAU,CAAC;IAErC,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,eAAe,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;IACzD,qEAAqE;IACrE,KAAK,IAAI,CAAC,WAAW,EAAE,MAAM,EAAE,aAAa,EAAE,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,CAAC;IAEhG,gEAAgE;IAChE,MAAM,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG,MAAM,qBAAqB,CAAC,MAAM,CAAC,CAAC;IAErD,IAAI,YAAsB,CAAC;IAC3B,IAAI,CAAC;QACH,YAAY;YACV,OAAO,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC;gBAC7B,CAAC,CAAC,OAAO,CAAC,YAAY;gBACtB,CAAC,CAAC,MAAM,eAAe,CAAC,WAAW,CAAC,CAAC;IAC3C,CAAC;IAAC,MAAM,CAAC;QACP,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IACtC,CAAC;IAED,MAAM,gBAAgB,GAAG,MAAM,aAAa,CAC1C,QAAQ,EACR,WAAW,EACX,MAAM,EACN,QAAQ,EACR,YAAY,CACb,CAAC;IAEF,KAAK,MAAM,MAAM,IAAI,gBAAgB,CAAC,OAAO,EAAE,CAAC;QAC9C,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QAC7C,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,KAAK,MAAM,CAAC,SAAS,KAAK,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,2EAA2E;IAC3E,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;IAC3C,MAAM,WAAW,GAAG,MAAM,kBAAkB,CAC1C,OAAO,EACP,gBAAgB,CAAC,OAAO,EACxB,WAAW,EACX,MAAM,CACP,CAAC;IAEF,qBAAqB;IACrB,MAAM,UAAU,GAAG,CAAC,GAAG,gBAAgB,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAE9D,MAAM,aAAa,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IACtE,MAAM,QAAQ,GACZ,aAAa,CAAC,MAAM,GAAG,CAAC;QACtB,CAAC,CAAC,IAAI,CAAC,KAAK,CACR,aAAa,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;YACvD,aAAa,CAAC,MAAM,CACvB;QACH,CAAC,CAAC,CAAC,CAAC;IAER,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAC9D,MAAM,YAAY,GAAG;QACnB,wBAAwB,WAAW,IAAI,UAAU,CAAC,MAAM,oBAAoB;QAC5E,UAAU,QAAQ,MAAM;KACzB,CAAC;IAEF,MAAM,UAAU,GAAwB;QACtC,MAAM,EAAE,gBAAgB,CAAC,MAAM,IAAI,WAAW,CAAC,MAAM;QACrD,KAAK,EAAE,QAAQ;QACf,OAAO,EAAE,UAAU;QACnB,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;QAChC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACpC,CAAC;IAEF,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC5D,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,cAAc,WAAW,EAAE,CAAC,CAAC;IAErD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,sEAAsE;AAEtE;;;;;;GAMG;AACH,KAAK,UAAU,kBAAkB,CAC/B,OAAuB,EACvB,mBAAiC,EACjC,WAAmB,EACnB,MAAmB;IAEnB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAE3C,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAEnD,2EAA2E;QAC3E,8EAA8E;QAC9E,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;QACzC,MAAM,SAAS,GAAG,UAAU,CAAC,YAAY,KAAK,OAAO,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QACnF,MAAM,OAAO,GAAG,gBAAgB,CAAC,WAAW,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,IAAI,SAAS,CAAC,CAAC;QAC/F,yEAAyE;QACzE,MAAM,YAAY,GAAG,MAAM,oBAAoB,CAAC,WAAW,EAAE,iBAAiB,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;QAEzG,MAAM,MAAM,GAAG,YAAY,CACzB,MAAM,CAAC,SAAS,CAAC,QAAQ,IAAI,IAAI,EACjC,MAAM,CAAC,SAAS,CAAC,QAAQ,IAAI,IAAI,EACjC,MAAM,CAAC,SAAS,CAAC,cAAc,EAC/B,MAAM,CAAC,SAAS,CAAC,KAAK,CACvB,CAAC;QACF,MAAM,WAAW,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;QAE9C,0CAA0C;QAC1C,MAAM,mBAAmB,GAAG,mBAAmB;aAC5C,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACT,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YAC/E,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;gBAC5B,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC1C,CAAC;YACD,KAAK,MAAM,MAAM,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;gBAC/B,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;oBACnB,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI;wBACrB,CAAC,CAAC,OAAO,MAAM,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE;wBAC3E,CAAC,CAAC,EAAE,CAAC;oBACP,KAAK,CAAC,IAAI,CACR,MAAM,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC,OAAO,GAAG,GAAG,EAAE,CAC/D,CAAC;gBACJ,CAAC;YACH,CAAC;YACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC,CAAC;aACD,IAAI,CAAC,MAAM,CAAC,CAAC;QAEhB,MAAM,WAAW,GAAG;EACtB,WAAW;;;EAGX,WAAW;;;EAGX,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAgCH,SAAS;EACzB,CAAC;QAEC,oFAAoF;QACpF,gFAAgF;QAChF,MAAM,WAAW,GAAG,sBAAsB,CAAC,cAAc,EAAE,CAAC;QAC5D,MAAM,iBAAiB,GAAG,IAAI,wBAAwB,CAAC,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QAClF,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAAC,MAAM,CACpD,WAAW,EACX,OAAO,CAAC,eAAe,IAAI,IAAI,EAC/B,WAAW,EACX,EAAE,WAAW,EAAE,QAAQ,EAAE,CAC1B,CAAC;QAEF,MAAM,CAAC,IAAI,CACT,4BAA4B,MAAM,CAAC,SAAS,CAAC,KAAK,MAAM,KAAK,MAAM,CACpE,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC;YAClC,MAAM;YACN,KAAK;YACL,YAAY;YACZ,WAAW,EAAE,eAAe;YAC5B,KAAK,EAAE,OAAO,CAAC,OAAO;YACtB,YAAY,EAAE,OAAO,CAAC,QAAQ;YAC9B,QAAQ,EAAE,mBAAmB;YAC7B,SAAS,EAAE,KAAK;YAChB,SAAS,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;gBACzB,MAAM,GAAG,GAAG,KAAgC,CAAC;gBAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;gBACnD,MAAM,CAAC,KAAK,CAAC,iBAAiB,IAAI,IAAI,QAAQ,GAAG,CAAC,CAAC;YACrD,CAAC;SACF,CAAC,CAAC;QAEH,MAAM,CAAC,KAAK,CACV,0BAA0B,MAAM,CAAC,SAAS,WAAW,MAAM,CAAC,WAAW,CAAC,MAAM,cAAc,CAC7F,CAAC;QAEF,OAAO,eAAe,CAAC,MAAM,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IACtD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,IAAI,CACT,4BAA4B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAC/E,CAAC;QAEF,OAAO;YACL,SAAS,EAAE,kBAAkB;YAC7B,MAAM,EAAE,IAAI,EAAE,oCAAoC;YAClD,KAAK,EAAE,SAAS;YAChB,OAAO,EAAE,EAAE;YACX,OAAO,EAAE,0CAA0C;YACnD,QAAQ,EAAE,2BAA2B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;YACvF,SAAS;SACV,CAAC;IACJ,CAAC;AACH,CAAC;AAED,sEAAsE;AAEtE;;GAEG;AACH,SAAS,eAAe,CAAC,IAAY,EAAE,iBAAyB;IAC9D,IAAI,MAAe,CAAC;IAEpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,UAAU,GAAG,uCAAuC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtE,IAAI,UAAU,EAAE,CAAC;YACf,IAAI,CAAC;gBACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YAC5C,CAAC;YAAC,MAAM,CAAC;gBACP,eAAe;YACjB,CAAC;QACH,CAAC;QAED,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YACvC,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,QAAQ,GAAG,UAAU,EAAE,CAAC;gBAC/C,IAAI,CAAC;oBACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC;gBAC5D,CAAC;gBAAC,MAAM,CAAC;oBACP,eAAe;gBACjB,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QAC5D,MAAM,GAAG,GAAG,MAAiC,CAAC;QAE9C,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC;YACxC,CAAC,CAAE,GAAG,CAAC,OAAqB;iBACvB,MAAM,CACL,CAAC,CAAC,EAAgC,EAAE,CAClC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,CACtC;iBACA,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACX,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,IAAI,SAAS,CAAC;gBAC3C,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;gBACzB,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC;gBAChC,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC,QAAQ,CAC9C,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CACnB;oBACC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;oBACpB,CAAC,CAAC,MAAM,CAAiC;gBAC3C,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvD,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACxD,CAAC,CAAC;YACP,CAAC,CAAC,EAAE,CAAC;QAEP,OAAO;YACL,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,SAAS,IAAI,kBAAkB,CAAC;YACtD,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC;YAC3B,KAAK,EAAE,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;YAC5D,OAAO;YACP,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO,IAAI,sBAAsB,CAAC;YACtD,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,IAAI,uBAAuB,CAAC;YACzD,SAAS,EACP,OAAO,GAAG,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,iBAAiB;SACxE,CAAC;IACJ,CAAC;IAED,OAAO;QACL,SAAS,EAAE,kBAAkB;QAC7B,MAAM,EAAE,KAAK;QACb,OAAO,EAAE,EAAE;QACX,OAAO,EAAE,4CAA4C;QACrD,QAAQ,EAAE,kBAAkB,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;QAChD,SAAS,EAAE,iBAAiB;KAC7B,CAAC;AACJ,CAAC"}
|
|
1
|
+
{"version":3,"file":"evaluator-agent.js","sourceRoot":"","sources":["../../src/orchestrator/evaluator-agent.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAExD,OAAO,EACL,qBAAqB,EACrB,aAAa,GACd,MAAM,2BAA2B,CAAC;AAEnC,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACjF,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,wBAAwB,EAAE,MAAM,gCAAgC,CAAC;AAC1E,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AACxE,OAAO,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAIpD,sEAAsE;AAEtE,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAEhC,sEAAsE;AAEtE;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,OAAuB,EACvB,WAAmB,EACnB,MAAmB;IAEnB,MAAM,QAAQ,GAAG,OAAO,CAAC,eAAe,CAAC;IACzC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACpE,CAAC;IAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,UAAU,CAAC;IAErC,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,eAAe,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;IACzD,qEAAqE;IACrE,KAAK,IAAI,CAAC,WAAW,EAAE,MAAM,EAAE,aAAa,EAAE,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,CAAC;IAEhG,gEAAgE;IAChE,MAAM,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG,MAAM,qBAAqB,CAAC,MAAM,CAAC,CAAC;IAErD,IAAI,YAAsB,CAAC;IAC3B,IAAI,CAAC;QACH,YAAY;YACV,OAAO,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC;gBAC7B,CAAC,CAAC,OAAO,CAAC,YAAY;gBACtB,CAAC,CAAC,MAAM,eAAe,CAAC,WAAW,CAAC,CAAC;IAC3C,CAAC;IAAC,MAAM,CAAC;QACP,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IACtC,CAAC;IAED,MAAM,gBAAgB,GAAG,MAAM,aAAa,CAC1C,QAAQ,EACR,WAAW,EACX,MAAM,EACN,QAAQ,EACR,YAAY,CACb,CAAC;IAEF,KAAK,MAAM,MAAM,IAAI,gBAAgB,CAAC,OAAO,EAAE,CAAC;QAC9C,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QAC7C,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,KAAK,MAAM,CAAC,SAAS,KAAK,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,2EAA2E;IAC3E,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;IAC3C,MAAM,WAAW,GAAG,MAAM,kBAAkB,CAC1C,OAAO,EACP,gBAAgB,CAAC,OAAO,EACxB,WAAW,EACX,MAAM,CACP,CAAC;IAEF,qBAAqB;IACrB,MAAM,UAAU,GAAG,CAAC,GAAG,gBAAgB,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAE9D,MAAM,aAAa,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IACtE,MAAM,QAAQ,GACZ,aAAa,CAAC,MAAM,GAAG,CAAC;QACtB,CAAC,CAAC,IAAI,CAAC,KAAK,CACR,aAAa,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;YACvD,aAAa,CAAC,MAAM,CACvB;QACH,CAAC,CAAC,CAAC,CAAC;IAER,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAC9D,MAAM,YAAY,GAAG;QACnB,wBAAwB,WAAW,IAAI,UAAU,CAAC,MAAM,oBAAoB;QAC5E,UAAU,QAAQ,MAAM;KACzB,CAAC;IAEF,MAAM,UAAU,GAAwB;QACtC,MAAM,EAAE,gBAAgB,CAAC,MAAM,IAAI,WAAW,CAAC,MAAM;QACrD,KAAK,EAAE,QAAQ;QACf,OAAO,EAAE,UAAU;QACnB,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;QAChC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACpC,CAAC;IAEF,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC5D,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,cAAc,WAAW,EAAE,CAAC,CAAC;IAErD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,sEAAsE;AAEtE;;;;;;;;;;;GAWG;AACH;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,OAAuB,EACvB,mBAAiC,EACjC,WAAmB,EACnB,MAAmB;IAEnB,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC;IACrC,IAAI,CAAC,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9C,OAAO,CAAC,MAAM,iBAAiB,CAAC,OAAO,EAAE,mBAAmB,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC;IACtF,CAAC;IACD,OAAO,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,aAAa,EAAE,CAAC,IAAI,EAAE,EAAE,CAC5D,iBAAiB,CAAC,OAAO,EAAE,mBAAmB,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,CAAC,CAC3E,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,kBAAkB,CAC/B,OAAuB,EACvB,mBAAiC,EACjC,WAAmB,EACnB,MAAmB;IAEnB,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC;IAErC,oEAAoE;IACpE,MAAM,WAAW,GAAG,MAAM,cAAc,CAAC,OAAO,EAAE,mBAAmB,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;IAE5F,IAAI,CAAC,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9C,yEAAyE;QACzE,OAAO,WAAW,CAAC,CAAC,CAAC,CAAC;IACxB,CAAC;IAED,MAAM,UAAU,GAAG,OAAO,CAAC,eAAe,EAAE,UAAU,IAAI,SAAS,CAAC;IAEpE,sFAAsF;IACtF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7C,MAAM,aAAa,CAAC,WAAW,EAAE;YAC/B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,KAAK,EAAE,mBAAmB;YAC1B,KAAK,EAAE,YAAY;YACnB,QAAQ,EAAE,UAAU;YACpB,OAAO,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE;SAClE,CAAC,CAAC;IACL,CAAC;IAED,OAAO,SAAS,CAAC,UAAU,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;AACzE,CAAC;AAED,uEAAuE;AAEvE;;;;GAIG;AACH,KAAK,UAAU,iBAAiB,CAC9B,OAAuB,EACvB,mBAAiC,EACjC,WAAmB,EACnB,MAAmB,EACnB,IAAa;IAEb,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAE3C,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAEnD,2EAA2E;QAC3E,8EAA8E;QAC9E,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;QACzC,MAAM,SAAS,GAAG,UAAU,CAAC,YAAY,KAAK,OAAO,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QACnF,MAAM,OAAO,GAAG,gBAAgB,CAAC,WAAW,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,IAAI,SAAS,CAAC,CAAC;QAC/F,yEAAyE;QACzE,MAAM,YAAY,GAAG,MAAM,oBAAoB,CAAC,WAAW,EAAE,iBAAiB,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;QAEzG,MAAM,MAAM,GAAG,YAAY,CACzB,MAAM,CAAC,SAAS,CAAC,QAAQ,IAAI,IAAI,EACjC,MAAM,CAAC,SAAS,CAAC,QAAQ,IAAI,IAAI,EACjC,MAAM,CAAC,SAAS,CAAC,cAAc,EAC/B,MAAM,CAAC,SAAS,CAAC,KAAK,CACvB,CAAC;QACF,MAAM,WAAW,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;QAE9C,0CAA0C;QAC1C,MAAM,mBAAmB,GAAG,mBAAmB;aAC5C,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACT,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YAC/E,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;gBAC5B,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC1C,CAAC;YACD,KAAK,MAAM,MAAM,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;gBAC/B,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;oBACnB,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI;wBACrB,CAAC,CAAC,OAAO,MAAM,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE;wBAC3E,CAAC,CAAC,EAAE,CAAC;oBACP,KAAK,CAAC,IAAI,CACR,MAAM,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC,OAAO,GAAG,GAAG,EAAE,CAC/D,CAAC;gBACJ,CAAC;YACH,CAAC;YACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC,CAAC;aACD,IAAI,CAAC,MAAM,CAAC,CAAC;QAEhB,MAAM,WAAW,GAAG;EACtB,WAAW;;;EAGX,WAAW;;;EAGX,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAgCH,SAAS;EACzB,CAAC;QAEC,gEAAgE;QAChE,4EAA4E;QAC5E,MAAM,SAAS,GAAG,IAAI;YACpB,CAAC,CAAC,2BAA2B,IAAI,KAAK,gBAAgB,CAAC,IAAI,CAAC,EAAE;YAC9D,CAAC,CAAC,EAAE,CAAC;QAEP,oFAAoF;QACpF,gFAAgF;QAChF,MAAM,WAAW,GAAG,sBAAsB,CAAC,cAAc,EAAE,CAAC;QAC5D,MAAM,iBAAiB,GAAG,IAAI,wBAAwB,CAAC,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QAClF,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAAC,MAAM,CACpD,WAAW,EACX,OAAO,CAAC,eAAe,IAAI,IAAI,EAC/B,GAAG,WAAW,GAAG,SAAS,EAAE,EAC5B,EAAE,WAAW,EAAE,QAAQ,EAAE,CAC1B,CAAC;QAEF,MAAM,CAAC,IAAI,CACT,4BAA4B,MAAM,CAAC,SAAS,CAAC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,CACrG,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC;YAClC,MAAM;YACN,KAAK;YACL,YAAY;YACZ,WAAW,EAAE,eAAe;YAC5B,KAAK,EAAE,OAAO,CAAC,OAAO;YACtB,YAAY,EAAE,OAAO,CAAC,QAAQ;YAC9B,QAAQ,EAAE,mBAAmB;YAC7B,SAAS,EAAE,KAAK;YAChB,SAAS,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;gBACzB,MAAM,GAAG,GAAG,KAAgC,CAAC;gBAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;gBACnD,MAAM,CAAC,KAAK,CAAC,iBAAiB,IAAI,IAAI,QAAQ,GAAG,CAAC,CAAC;YACrD,CAAC;SACF,CAAC,CAAC;QAEH,MAAM,CAAC,KAAK,CACV,0BAA0B,MAAM,CAAC,SAAS,WAAW,MAAM,CAAC,WAAW,CAAC,MAAM,cAAc,CAC7F,CAAC;QAEF,OAAO,eAAe,CAAC,MAAM,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IACtD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,IAAI,CACT,4BAA4B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAC/E,CAAC;QAEF,OAAO;YACL,SAAS,EAAE,kBAAkB;YAC7B,MAAM,EAAE,IAAI,EAAE,oCAAoC;YAClD,KAAK,EAAE,SAAS;YAChB,OAAO,EAAE,EAAE;YACX,OAAO,EAAE,0CAA0C;YACnD,QAAQ,EAAE,2BAA2B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;YACvF,SAAS;SACV,CAAC;IACJ,CAAC;AACH,CAAC;AAED,sEAAsE;AAEtE;;GAEG;AACH,SAAS,eAAe,CAAC,IAAY,EAAE,iBAAyB;IAC9D,IAAI,MAAe,CAAC;IAEpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,UAAU,GAAG,uCAAuC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtE,IAAI,UAAU,EAAE,CAAC;YACf,IAAI,CAAC;gBACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YAC5C,CAAC;YAAC,MAAM,CAAC;gBACP,eAAe;YACjB,CAAC;QACH,CAAC;QAED,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YACvC,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,QAAQ,GAAG,UAAU,EAAE,CAAC;gBAC/C,IAAI,CAAC;oBACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC;gBAC5D,CAAC;gBAAC,MAAM,CAAC;oBACP,eAAe;gBACjB,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QAC5D,MAAM,GAAG,GAAG,MAAiC,CAAC;QAE9C,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC;YACxC,CAAC,CAAE,GAAG,CAAC,OAAqB;iBACvB,MAAM,CACL,CAAC,CAAC,EAAgC,EAAE,CAClC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,CACtC;iBACA,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACX,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,IAAI,SAAS,CAAC;gBAC3C,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;gBACzB,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC;gBAChC,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC,QAAQ,CAC9C,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CACnB;oBACC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;oBACpB,CAAC,CAAC,MAAM,CAAiC;gBAC3C,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvD,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACxD,CAAC,CAAC;YACP,CAAC,CAAC,EAAE,CAAC;QAEP,OAAO;YACL,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,SAAS,IAAI,kBAAkB,CAAC;YACtD,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC;YAC3B,KAAK,EAAE,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;YAC5D,OAAO;YACP,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO,IAAI,sBAAsB,CAAC;YACtD,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,IAAI,uBAAuB,CAAC;YACzD,SAAS,EACP,OAAO,GAAG,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,iBAAiB;SACxE,CAAC;IACJ,CAAC;IAED,OAAO;QACL,SAAS,EAAE,kBAAkB;QAC7B,MAAM,EAAE,KAAK;QACb,OAAO,EAAE,EAAE;QACX,OAAO,EAAE,4CAA4C;QACrD,QAAQ,EAAE,kBAAkB,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;QAChD,SAAS,EAAE,iBAAiB;KAC7B,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"generator-agent.d.ts","sourceRoot":"","sources":["../../src/orchestrator/generator-agent.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAc3D,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,yCAAyC;IACzC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,sCAAsC;IACtC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,mBAAmB;IACnB,KAAK,CAAC,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC;CACvD;AAID;;;;;;;;GAQG;AACH,wBAAsB,YAAY,CAChC,OAAO,EAAE,cAAc,EACvB,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,WAAW,GAClB,OAAO,CAAC,eAAe,CAAC,
|
|
1
|
+
{"version":3,"file":"generator-agent.d.ts","sourceRoot":"","sources":["../../src/orchestrator/generator-agent.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAc3D,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,yCAAyC;IACzC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,sCAAsC;IACtC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,mBAAmB;IACnB,KAAK,CAAC,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC;CACvD;AAID;;;;;;;;GAQG;AACH,wBAAsB,YAAY,CAChC,OAAO,EAAE,cAAc,EACvB,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,WAAW,GAClB,OAAO,CAAC,eAAe,CAAC,CA4G1B"}
|
|
@@ -89,10 +89,42 @@ When you are done, your final response must contain ONLY a JSON object with this
|
|
|
89
89
|
const inputStr = JSON.stringify(inp).slice(0, 120);
|
|
90
90
|
logger.debug(` [generator] ${name}(${inputStr})`);
|
|
91
91
|
},
|
|
92
|
+
// DeepSeek-style models sometimes narrate ("let me write the files...") and
|
|
93
|
+
// stop without calling any tool, which would end the sprint with no work
|
|
94
|
+
// done. Treat a tool-less turn as complete ONLY if it carries the JSON
|
|
95
|
+
// report; otherwise nudge the model to actually call its tools.
|
|
96
|
+
completionCheck: (text) => looksLikeGeneratorReport(text),
|
|
97
|
+
nudgeMessage: "You stopped without calling any tool and without producing the final " +
|
|
98
|
+
"JSON report. If implementation work remains, CALL write_file / edit_file " +
|
|
99
|
+
"/ your other tools NOW to make the changes — do not just describe them. " +
|
|
100
|
+
"Only once everything is implemented and verified, output ONLY the final " +
|
|
101
|
+
"JSON report object described in the instructions.",
|
|
102
|
+
maxNudges: 3,
|
|
92
103
|
});
|
|
93
104
|
logger.debug(`Generator completed in ${result.turnsUsed} turns (${result.toolsCalled.length} tool calls)`);
|
|
94
105
|
return parseGeneratorResult(result.finalText, filesWritten, result);
|
|
95
106
|
}
|
|
107
|
+
/**
|
|
108
|
+
* True when the text contains a JSON object that looks like the generator's
|
|
109
|
+
* completion report (has a "status" or "success" field). Used as the agentic
|
|
110
|
+
* loop's completion predicate so a tool-less "I'll write the files" narration
|
|
111
|
+
* is treated as incomplete (and nudged) rather than as a finished sprint.
|
|
112
|
+
*/
|
|
113
|
+
function looksLikeGeneratorReport(text) {
|
|
114
|
+
const start = text.indexOf("{");
|
|
115
|
+
const end = text.lastIndexOf("}");
|
|
116
|
+
if (start === -1 || end <= start)
|
|
117
|
+
return false;
|
|
118
|
+
try {
|
|
119
|
+
const obj = JSON.parse(text.slice(start, end + 1));
|
|
120
|
+
return (typeof obj === "object" &&
|
|
121
|
+
obj !== null &&
|
|
122
|
+
("status" in obj || "success" in obj));
|
|
123
|
+
}
|
|
124
|
+
catch {
|
|
125
|
+
return false;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
96
128
|
// ── JSON parser ────────────────────────────────────────────────────
|
|
97
129
|
/**
|
|
98
130
|
* Parse the generator response text into a GeneratorResult.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"generator-agent.js","sourceRoot":"","sources":["../../src/orchestrator/generator-agent.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACjF,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,wBAAwB,EAAE,MAAM,gCAAgC,CAAC;AAC1E,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AACxE,OAAO,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAiB5C,sEAAsE;AAEtE;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,OAAuB,EACvB,WAAmB,EACnB,MAAmB;IAEnB,MAAM,UAAU,GAAG,OAAO,CAAC,eAAe,EAAE,UAAU,IAAI,SAAS,CAAC;IACpE,MAAM,KAAK,GAAG,OAAO,CAAC,eAAe,EAAE,KAAK,IAAI,SAAS,CAAC;IAE1D,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,eAAe,KAAK,EAAE,CAAC,CAAC;IAClD,qEAAqE;IACrE,KAAK,IAAI,CAAC,WAAW,EAAE,MAAM,EAAE,aAAa,EAAE,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC,CAAC;IAEtF,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG,MAAM,CAAC,SAAS,CAAC,iBAAiB,CAAC;IAEpD,sEAAsE;IACtE,wDAAwD;IACxD,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IACzC,MAAM,SAAS,GAAG,UAAU,CAAC,YAAY,KAAK,OAAO,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IACnF,MAAM,OAAO,GAAG,gBAAgB,CAAC,WAAW,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,IAAI,SAAS,CAAC,CAAC;IAC/F,yEAAyE;IACzE,MAAM,YAAY,GAAG,MAAM,oBAAoB,CAAC,WAAW,EAAE,iBAAiB,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;IAEzG,MAAM,MAAM,GAAG,YAAY,CACzB,MAAM,CAAC,SAAS,CAAC,QAAQ,IAAI,IAAI,EACjC,MAAM,CAAC,SAAS,CAAC,QAAQ,IAAI,IAAI,EACjC,MAAM,CAAC,SAAS,CAAC,cAAc,EAC/B,MAAM,CAAC,SAAS,CAAC,KAAK,CACvB,CAAC;IACF,MAAM,WAAW,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAE9C,sDAAsD;IACtD,MAAM,YAAY,GAAG,oBAAoB,UAAU,cAAc,CAAC;IAElE,MAAM,WAAW,GAAG;EACpB,WAAW;;;EAGX,WAAW;EACX,YAAY,CAAC,CAAC,CAAC,uEAAuE,YAAY,mMAAmM,CAAC,CAAC,CAAC,EAAE;;;EAG1S,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,+BAA+B,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;;;;mBAI5E,UAAU;;;;;;;;;;;;EAY3B,CAAC;IAED,oFAAoF;IACpF,gFAAgF;IAChF,MAAM,WAAW,GAAG,sBAAsB,CAAC,cAAc,EAAE,CAAC;IAC5D,MAAM,iBAAiB,GAAG,IAAI,wBAAwB,CAAC,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;IAClF,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAAC,MAAM,CACpD,WAAW,EACX,OAAO,CAAC,eAAe,IAAI,IAAI,EAC/B,WAAW,CACZ,CAAC;IAEF,MAAM,CAAC,IAAI,CAAC,4BAA4B,MAAM,CAAC,SAAS,CAAC,KAAK,MAAM,KAAK,MAAM,CAAC,CAAC;IAEjF,kDAAkD;IAClD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;IAEvC,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC;QAClC,MAAM;QACN,KAAK;QACL,YAAY;QACZ,WAAW,EAAE,eAAe;QAC5B,KAAK,EAAE,OAAO,CAAC,OAAO;QACtB,YAAY,EAAE,OAAO,CAAC,QAAQ;QAC9B,QAAQ;QACR,SAAS,EAAE,KAAK;QAChB,SAAS,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;YACzB,MAAM,GAAG,GAAG,KAAgC,CAAC;YAC7C,IAAI,IAAI,KAAK,YAAY,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;gBAClD,MAAM,IAAI,GAAG,GAAG,CAAC,SAAmB,CAAC;gBACrC,IAAI,IAAI;oBAAE,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACnC,CAAC;YACD,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YACnD,MAAM,CAAC,KAAK,CAAC,iBAAiB,IAAI,IAAI,QAAQ,GAAG,CAAC,CAAC;QACrD,CAAC;
|
|
1
|
+
{"version":3,"file":"generator-agent.js","sourceRoot":"","sources":["../../src/orchestrator/generator-agent.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACjF,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,wBAAwB,EAAE,MAAM,gCAAgC,CAAC;AAC1E,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AACxE,OAAO,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAiB5C,sEAAsE;AAEtE;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,OAAuB,EACvB,WAAmB,EACnB,MAAmB;IAEnB,MAAM,UAAU,GAAG,OAAO,CAAC,eAAe,EAAE,UAAU,IAAI,SAAS,CAAC;IACpE,MAAM,KAAK,GAAG,OAAO,CAAC,eAAe,EAAE,KAAK,IAAI,SAAS,CAAC;IAE1D,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,eAAe,KAAK,EAAE,CAAC,CAAC;IAClD,qEAAqE;IACrE,KAAK,IAAI,CAAC,WAAW,EAAE,MAAM,EAAE,aAAa,EAAE,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC,CAAC;IAEtF,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG,MAAM,CAAC,SAAS,CAAC,iBAAiB,CAAC;IAEpD,sEAAsE;IACtE,wDAAwD;IACxD,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IACzC,MAAM,SAAS,GAAG,UAAU,CAAC,YAAY,KAAK,OAAO,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IACnF,MAAM,OAAO,GAAG,gBAAgB,CAAC,WAAW,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,IAAI,SAAS,CAAC,CAAC;IAC/F,yEAAyE;IACzE,MAAM,YAAY,GAAG,MAAM,oBAAoB,CAAC,WAAW,EAAE,iBAAiB,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;IAEzG,MAAM,MAAM,GAAG,YAAY,CACzB,MAAM,CAAC,SAAS,CAAC,QAAQ,IAAI,IAAI,EACjC,MAAM,CAAC,SAAS,CAAC,QAAQ,IAAI,IAAI,EACjC,MAAM,CAAC,SAAS,CAAC,cAAc,EAC/B,MAAM,CAAC,SAAS,CAAC,KAAK,CACvB,CAAC;IACF,MAAM,WAAW,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAE9C,sDAAsD;IACtD,MAAM,YAAY,GAAG,oBAAoB,UAAU,cAAc,CAAC;IAElE,MAAM,WAAW,GAAG;EACpB,WAAW;;;EAGX,WAAW;EACX,YAAY,CAAC,CAAC,CAAC,uEAAuE,YAAY,mMAAmM,CAAC,CAAC,CAAC,EAAE;;;EAG1S,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,+BAA+B,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;;;;mBAI5E,UAAU;;;;;;;;;;;;EAY3B,CAAC;IAED,oFAAoF;IACpF,gFAAgF;IAChF,MAAM,WAAW,GAAG,sBAAsB,CAAC,cAAc,EAAE,CAAC;IAC5D,MAAM,iBAAiB,GAAG,IAAI,wBAAwB,CAAC,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;IAClF,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAAC,MAAM,CACpD,WAAW,EACX,OAAO,CAAC,eAAe,IAAI,IAAI,EAC/B,WAAW,CACZ,CAAC;IAEF,MAAM,CAAC,IAAI,CAAC,4BAA4B,MAAM,CAAC,SAAS,CAAC,KAAK,MAAM,KAAK,MAAM,CAAC,CAAC;IAEjF,kDAAkD;IAClD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;IAEvC,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC;QAClC,MAAM;QACN,KAAK;QACL,YAAY;QACZ,WAAW,EAAE,eAAe;QAC5B,KAAK,EAAE,OAAO,CAAC,OAAO;QACtB,YAAY,EAAE,OAAO,CAAC,QAAQ;QAC9B,QAAQ;QACR,SAAS,EAAE,KAAK;QAChB,SAAS,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;YACzB,MAAM,GAAG,GAAG,KAAgC,CAAC;YAC7C,IAAI,IAAI,KAAK,YAAY,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;gBAClD,MAAM,IAAI,GAAG,GAAG,CAAC,SAAmB,CAAC;gBACrC,IAAI,IAAI;oBAAE,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACnC,CAAC;YACD,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YACnD,MAAM,CAAC,KAAK,CAAC,iBAAiB,IAAI,IAAI,QAAQ,GAAG,CAAC,CAAC;QACrD,CAAC;QACD,4EAA4E;QAC5E,yEAAyE;QACzE,uEAAuE;QACvE,gEAAgE;QAChE,eAAe,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,wBAAwB,CAAC,IAAI,CAAC;QACzD,YAAY,EACV,uEAAuE;YACvE,2EAA2E;YAC3E,0EAA0E;YAC1E,0EAA0E;YAC1E,mDAAmD;QACrD,SAAS,EAAE,CAAC;KACb,CAAC,CAAC;IAEH,MAAM,CAAC,KAAK,CACV,0BAA0B,MAAM,CAAC,SAAS,WAAW,MAAM,CAAC,WAAW,CAAC,MAAM,cAAc,CAC7F,CAAC;IAEF,OAAO,oBAAoB,CAAC,MAAM,CAAC,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;AACtE,CAAC;AAED;;;;;GAKG;AACH,SAAS,wBAAwB,CAAC,IAAY;IAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAChC,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,KAAK,KAAK,CAAC,CAAC,IAAI,GAAG,IAAI,KAAK;QAAE,OAAO,KAAK,CAAC;IAC/C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,GAAG,CAAC,CAAC,CAAY,CAAC;QAC9D,OAAO,CACL,OAAO,GAAG,KAAK,QAAQ;YACvB,GAAG,KAAK,IAAI;YACZ,CAAC,QAAQ,IAAI,GAAG,IAAI,SAAS,IAAI,GAAG,CAAC,CACtC,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,sEAAsE;AAEtE;;GAEG;AACH,SAAS,oBAAoB,CAC3B,IAAY,EACZ,YAAyB,EACzB,UAA8G;IAE9G,IAAI,MAAe,CAAC;IAEpB,mBAAmB;IACnB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,sCAAsC;QACtC,MAAM,UAAU,GAAG,uCAAuC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtE,IAAI,UAAU,EAAE,CAAC;YACf,IAAI,CAAC;gBACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YAC5C,CAAC;YAAC,MAAM,CAAC;gBACP,eAAe;YACjB,CAAC;QACH,CAAC;QAED,sBAAsB;QACtB,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YACvC,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,QAAQ,GAAG,UAAU,EAAE,CAAC;gBAC/C,IAAI,CAAC;oBACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC;gBAC5D,CAAC;gBAAC,MAAM,CAAC;oBACP,0BAA0B;gBAC5B,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,IACE,MAAM;QACN,OAAO,MAAM,KAAK,QAAQ;QAC1B,MAAM,KAAK,IAAI;QACf,CAAC,SAAS,IAAI,MAAM,IAAI,QAAQ,IAAI,MAAM,CAAC,EAC3C,CAAC;QACD,MAAM,GAAG,GAAG,MAAiC,CAAC;QAE9C,0EAA0E;QAC1E,MAAM,OAAO,GACX,SAAS,IAAI,GAAG;YACd,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC;YACtB,CAAC,CAAC,GAAG,CAAC,MAAM,KAAK,UAAU,CAAC;QAEhC,0DAA0D;QAC1D,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;YACjD,CAAC,CAAE,GAAG,CAAC,YAA0B;iBAC5B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;gBACT,IAAI,OAAO,CAAC,KAAK,QAAQ;oBAAE,OAAO,CAAC,CAAC;gBACpC,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,IAAI,MAAM,IAAI,CAAC;oBACpD,OAAO,MAAM,CAAE,CAA6B,CAAC,IAAI,CAAC,CAAC;gBACrD,OAAO,IAAI,CAAC;YACd,CAAC,CAAC;iBACD,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC;YAC3C,CAAC,CAAC,EAAE,CAAC;QAEP,MAAM,QAAQ,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,YAAY,EAAE,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;QAEjE,OAAO;YACL,OAAO;YACP,KAAK,EACH,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ;gBAC3B,CAAC,CAAC,GAAG,CAAC,KAAK;gBACX,CAAC,CAAC,oBAAoB;YAC1B,YAAY,EAAE,QAAQ;YACtB,UAAU,EACR,OAAO,GAAG,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;YACjE,SAAS,EAAE,UAAU,CAAC,SAAS;YAC/B,WAAW,EAAE,UAAU,CAAC,WAAW;YACnC,KAAK,EAAE,UAAU,CAAC,KAAK;SACxB,CAAC;IACJ,CAAC;IAED,+DAA+D;IAC/D,IAAI,YAAY,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QAC1B,OAAO;YACL,OAAO,EAAE,IAAI;YACb,KAAK,EAAE,mBAAmB,YAAY,CAAC,IAAI,0DAA0D,CAAC,GAAG,YAAY,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YACnI,YAAY,EAAE,CAAC,GAAG,YAAY,CAAC;YAC/B,SAAS,EAAE,UAAU,CAAC,SAAS;YAC/B,WAAW,EAAE,UAAU,CAAC,WAAW;YACnC,KAAK,EAAE,UAAU,CAAC,KAAK;SACxB,CAAC;IACJ,CAAC;IAED,OAAO;QACL,OAAO,EAAE,KAAK;QACd,KAAK,EAAE,oDAAoD,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;QAC/E,YAAY,EAAE,EAAE;QAChB,SAAS,EAAE,UAAU,CAAC,SAAS;QAC/B,WAAW,EAAE,UAAU,CAAC,WAAW;QACnC,KAAK,EAAE,UAAU,CAAC,KAAK;KACxB,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PURE deterministic distillation of sprint outcomes into LessonEntry records.
|
|
3
|
+
*
|
|
4
|
+
* PURE — must not import from ../providers; no network, no Date.now(), no side effects,
|
|
5
|
+
* no filesystem access. createdAt is stamped at PERSIST TIME by the CLI handler, not here.
|
|
6
|
+
* lessonId is a sha256 content-hash of category+tags+sourceEntryRefs — never derived from time.
|
|
7
|
+
*
|
|
8
|
+
* IMPORTANT — this distills from the data shapes the REAL pipeline actually produces, NOT
|
|
9
|
+
* an invented vocabulary. The three signals are:
|
|
10
|
+
* (a) recurring failed-criterion categories — from eval results' criteriaResults[].result==="fail",
|
|
11
|
+
* grouped by the criterion's verificationMethod (resolved from the owning contract).
|
|
12
|
+
* (b) repeated failing eval strategies — from eval results' strategyResults[].result==="fail",
|
|
13
|
+
* grouped by strategy name.
|
|
14
|
+
* (c) sprints that needed rework — from contract.iterationHistory entries whose
|
|
15
|
+
* result==="fail" (real shape: { iteration, evalId, result }), reinforced by history
|
|
16
|
+
* entries with phase==="rework" && event==="evaluation-failed".
|
|
17
|
+
*
|
|
18
|
+
* The eval-result inputs are read LENIENTLY (see DistillableEval) because the on-disk
|
|
19
|
+
* .bober/eval-results/*.json files and the compiled EvalResultSchema use different shapes;
|
|
20
|
+
* every field is optional and defensively narrowed.
|
|
21
|
+
*/
|
|
22
|
+
import type { HistoryEntry } from "../../state/history.js";
|
|
23
|
+
import type { SprintContract } from "../../contracts/sprint-contract.js";
|
|
24
|
+
import type { LessonEntry } from "../../state/memory.js";
|
|
25
|
+
/**
|
|
26
|
+
* Lenient, structurally-typed view of an eval result for distillation.
|
|
27
|
+
*
|
|
28
|
+
* Unifies BOTH the on-disk .bober/eval-results/*.json shape
|
|
29
|
+
* ({ overallResult, strategyResults, criteriaResults }) AND the compiled
|
|
30
|
+
* EvalResultSchema shape ({ passed, criteriaResults }). Every field is optional;
|
|
31
|
+
* distill never assumes a field is present.
|
|
32
|
+
*/
|
|
33
|
+
export interface DistillableEval {
|
|
34
|
+
evalId?: string;
|
|
35
|
+
contractId?: string;
|
|
36
|
+
iteration?: number;
|
|
37
|
+
/** on-disk variant: "pass" | "fail" */
|
|
38
|
+
overallResult?: string;
|
|
39
|
+
/** compiled-schema variant */
|
|
40
|
+
passed?: boolean;
|
|
41
|
+
criteriaResults?: Array<{
|
|
42
|
+
criterionId?: string;
|
|
43
|
+
result?: string;
|
|
44
|
+
verificationMethod?: string;
|
|
45
|
+
}>;
|
|
46
|
+
strategyResults?: Array<{
|
|
47
|
+
strategy?: string;
|
|
48
|
+
result?: string;
|
|
49
|
+
}>;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Pure, side-effect-free distillation of sprint outcomes into LessonEntry records.
|
|
53
|
+
*
|
|
54
|
+
* Derives lessons from structured fields only (see file header for the three signals).
|
|
55
|
+
* The returned array is sorted by lessonId for byte-identical output across calls.
|
|
56
|
+
* The `createdAt` field is set to SENTINEL_CREATED_AT — the CLI handler MUST
|
|
57
|
+
* replace it with the real wall-clock ISO string before persisting.
|
|
58
|
+
*/
|
|
59
|
+
export declare function distill(history: HistoryEntry[], contracts: SprintContract[], evalResults?: DistillableEval[]): LessonEntry[];
|
|
60
|
+
//# sourceMappingURL=distill.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"distill.d.ts","sourceRoot":"","sources":["../../../src/orchestrator/memory/distill.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAIH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oCAAoC,CAAC;AACzE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAezD;;;;;;;GAOG;AACH,MAAM,WAAW,eAAe;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,eAAe,CAAC,EAAE,KAAK,CAAC;QACtB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,kBAAkB,CAAC,EAAE,MAAM,CAAC;KAC7B,CAAC,CAAC;IACH,eAAe,CAAC,EAAE,KAAK,CAAC;QACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;CACJ;AA2CD;;;;;;;GAOG;AACH,wBAAgB,OAAO,CACrB,OAAO,EAAE,YAAY,EAAE,EACvB,SAAS,EAAE,cAAc,EAAE,EAC3B,WAAW,GAAE,eAAe,EAAO,GAClC,WAAW,EAAE,CAgIf"}
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PURE deterministic distillation of sprint outcomes into LessonEntry records.
|
|
3
|
+
*
|
|
4
|
+
* PURE — must not import from ../providers; no network, no Date.now(), no side effects,
|
|
5
|
+
* no filesystem access. createdAt is stamped at PERSIST TIME by the CLI handler, not here.
|
|
6
|
+
* lessonId is a sha256 content-hash of category+tags+sourceEntryRefs — never derived from time.
|
|
7
|
+
*
|
|
8
|
+
* IMPORTANT — this distills from the data shapes the REAL pipeline actually produces, NOT
|
|
9
|
+
* an invented vocabulary. The three signals are:
|
|
10
|
+
* (a) recurring failed-criterion categories — from eval results' criteriaResults[].result==="fail",
|
|
11
|
+
* grouped by the criterion's verificationMethod (resolved from the owning contract).
|
|
12
|
+
* (b) repeated failing eval strategies — from eval results' strategyResults[].result==="fail",
|
|
13
|
+
* grouped by strategy name.
|
|
14
|
+
* (c) sprints that needed rework — from contract.iterationHistory entries whose
|
|
15
|
+
* result==="fail" (real shape: { iteration, evalId, result }), reinforced by history
|
|
16
|
+
* entries with phase==="rework" && event==="evaluation-failed".
|
|
17
|
+
*
|
|
18
|
+
* The eval-result inputs are read LENIENTLY (see DistillableEval) because the on-disk
|
|
19
|
+
* .bober/eval-results/*.json files and the compiled EvalResultSchema use different shapes;
|
|
20
|
+
* every field is optional and defensively narrowed.
|
|
21
|
+
*/
|
|
22
|
+
import { createHash } from "node:crypto";
|
|
23
|
+
// ── Constants ───────────────────────────────────────────────────────
|
|
24
|
+
/** A sprint with at least this many failed iterations is flagged as needing rework. */
|
|
25
|
+
const REWORK_THRESHOLD = 1;
|
|
26
|
+
/** createdAt sentinel used internally so distill() returns LessonEntry-shaped objects.
|
|
27
|
+
* The CLI handler MUST overwrite this with the real wall-clock ISO string before
|
|
28
|
+
* calling appendLesson — this value is intentionally an epoch so that if it ever
|
|
29
|
+
* leaks to persistence it is obviously a placeholder. */
|
|
30
|
+
const SENTINEL_CREATED_AT = "1970-01-01T00:00:00.000Z";
|
|
31
|
+
// ── Helpers ──────────────────────────────────────────────────────────
|
|
32
|
+
/** Narrow an unknown value to a string-keyed record. */
|
|
33
|
+
function isRecord(v) {
|
|
34
|
+
return typeof v === "object" && v !== null && !Array.isArray(v);
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Derive a deterministic 16-char hex lessonId from the failure signature.
|
|
38
|
+
* Sorts tags and refs before hashing so ordering does not affect the id.
|
|
39
|
+
*/
|
|
40
|
+
function lessonIdFromSignature(category, tags, refs) {
|
|
41
|
+
const canonical = JSON.stringify({
|
|
42
|
+
category,
|
|
43
|
+
tags: [...tags].sort(),
|
|
44
|
+
refs: [...refs].sort(),
|
|
45
|
+
});
|
|
46
|
+
return createHash("sha256").update(canonical).digest("hex").slice(0, 16);
|
|
47
|
+
}
|
|
48
|
+
/** Map occurrence count to severity band. */
|
|
49
|
+
function severityFor(occurrences) {
|
|
50
|
+
if (occurrences >= 5)
|
|
51
|
+
return "high";
|
|
52
|
+
if (occurrences >= 3)
|
|
53
|
+
return "warn";
|
|
54
|
+
return "info";
|
|
55
|
+
}
|
|
56
|
+
// ── Core ─────────────────────────────────────────────────────────────
|
|
57
|
+
/**
|
|
58
|
+
* Pure, side-effect-free distillation of sprint outcomes into LessonEntry records.
|
|
59
|
+
*
|
|
60
|
+
* Derives lessons from structured fields only (see file header for the three signals).
|
|
61
|
+
* The returned array is sorted by lessonId for byte-identical output across calls.
|
|
62
|
+
* The `createdAt` field is set to SENTINEL_CREATED_AT — the CLI handler MUST
|
|
63
|
+
* replace it with the real wall-clock ISO string before persisting.
|
|
64
|
+
*/
|
|
65
|
+
export function distill(history, contracts, evalResults = []) {
|
|
66
|
+
const groups = new Map();
|
|
67
|
+
function upsertGroup(category, tags, summary, ref) {
|
|
68
|
+
const sortedTags = [...tags].sort();
|
|
69
|
+
const key = `${category}|${sortedTags.join(",")}`;
|
|
70
|
+
let group = groups.get(key);
|
|
71
|
+
if (!group) {
|
|
72
|
+
group = { category, tags: sortedTags, summary, sourceEntryRefs: new Set() };
|
|
73
|
+
groups.set(key, group);
|
|
74
|
+
}
|
|
75
|
+
group.sourceEntryRefs.add(ref);
|
|
76
|
+
}
|
|
77
|
+
// Lookup: contractId -> (criterionId -> verificationMethod), resolved from the contract.
|
|
78
|
+
const vmByContractCriterion = new Map();
|
|
79
|
+
for (const c of contracts) {
|
|
80
|
+
const m = new Map();
|
|
81
|
+
for (const sc of c.successCriteria ?? []) {
|
|
82
|
+
if (sc.criterionId && sc.verificationMethod) {
|
|
83
|
+
m.set(sc.criterionId, sc.verificationMethod);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
vmByContractCriterion.set(c.contractId, m);
|
|
87
|
+
}
|
|
88
|
+
// ── (a) failed-criterion categories & (b) failing eval strategies ──────
|
|
89
|
+
for (const ev of evalResults) {
|
|
90
|
+
const evRef = ev.evalId ?? ev.contractId ?? "unknown-eval";
|
|
91
|
+
// (b) repeated failing eval strategies (only present in the richer on-disk shape).
|
|
92
|
+
for (const s of ev.strategyResults ?? []) {
|
|
93
|
+
if (s?.result === "fail" && typeof s.strategy === "string" && s.strategy.length > 0) {
|
|
94
|
+
const category = `eval-strategy-failure:${s.strategy}`;
|
|
95
|
+
const tags = [`strategy:${s.strategy}`];
|
|
96
|
+
const summary = `Eval strategy '${s.strategy}' failed across sprints — recurring failing gate`;
|
|
97
|
+
upsertGroup(category, tags, summary, evRef);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
// (a) recurring failed-criterion categories, grouped by the criterion's verificationMethod.
|
|
101
|
+
for (const cr of ev.criteriaResults ?? []) {
|
|
102
|
+
if (cr?.result === "fail" && typeof cr.criterionId === "string" && cr.criterionId.length > 0) {
|
|
103
|
+
const resolvedVm = (ev.contractId && vmByContractCriterion.get(ev.contractId)?.get(cr.criterionId)) ||
|
|
104
|
+
cr.verificationMethod ||
|
|
105
|
+
"unknown";
|
|
106
|
+
const category = `failed-criterion:${resolvedVm}`;
|
|
107
|
+
const tags = [`verificationMethod:${resolvedVm}`];
|
|
108
|
+
const summary = `Success criteria verified by '${resolvedVm}' failed across sprints`;
|
|
109
|
+
upsertGroup(category, tags, summary, `${evRef}:${cr.criterionId}`);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
// ── (c) sprints that needed rework (from contract iterationHistory) ──────
|
|
114
|
+
// Track which sprints we already counted from iterationHistory so the history
|
|
115
|
+
// fallback below does not double-count the same sprint.
|
|
116
|
+
const sprintsCountedFromContracts = new Set();
|
|
117
|
+
for (const contract of contracts) {
|
|
118
|
+
const iters = Array.isArray(contract.iterationHistory)
|
|
119
|
+
? contract.iterationHistory
|
|
120
|
+
: [];
|
|
121
|
+
const failedRefs = [];
|
|
122
|
+
for (const it of iters) {
|
|
123
|
+
if (isRecord(it) && it["result"] === "fail") {
|
|
124
|
+
const n = typeof it["iteration"] === "number" ? it["iteration"] : "?";
|
|
125
|
+
failedRefs.push(`${contract.contractId}:iteration-${n}`);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
if (failedRefs.length >= REWORK_THRESHOLD) {
|
|
129
|
+
sprintsCountedFromContracts.add(contract.contractId);
|
|
130
|
+
const tags = ["phase:rework", `sprintId:${contract.contractId}`];
|
|
131
|
+
const summary = `Sprint '${contract.contractId}' needed ${failedRefs.length} rework iteration(s) before passing`;
|
|
132
|
+
for (const ref of failedRefs) {
|
|
133
|
+
upsertGroup("sprint-rework", tags, summary, ref);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
// ── (c, fallback) rework signal straight from history events ────────────
|
|
138
|
+
// The real pipeline writes { event:"evaluation-failed", phase:"rework", sprintId, details:{iteration} }.
|
|
139
|
+
// Only used for sprints not already accounted for via iterationHistory (avoids double-count).
|
|
140
|
+
for (const entry of history) {
|
|
141
|
+
if (entry.phase === "rework" && entry.event === "evaluation-failed") {
|
|
142
|
+
const sid = entry.sprintId ?? "unknown";
|
|
143
|
+
if (sprintsCountedFromContracts.has(sid))
|
|
144
|
+
continue;
|
|
145
|
+
const iter = entry.details["iteration"];
|
|
146
|
+
const refSuffix = typeof iter === "number" ? `iteration-${iter}` : entry.timestamp;
|
|
147
|
+
const tags = ["phase:rework", `sprintId:${sid}`];
|
|
148
|
+
const summary = `Sprint '${sid}' had a failed evaluation requiring rework`;
|
|
149
|
+
upsertGroup("sprint-rework", tags, summary, `${sid}:${refSuffix}`);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
// ── Build LessonEntry array ──────────────────────────────────────────
|
|
153
|
+
const lessons = [];
|
|
154
|
+
for (const group of groups.values()) {
|
|
155
|
+
const refs = [...group.sourceEntryRefs].sort();
|
|
156
|
+
// Enforce non-empty sourceEntryRefs (schema invariant).
|
|
157
|
+
if (refs.length === 0)
|
|
158
|
+
continue;
|
|
159
|
+
const occurrences = refs.length;
|
|
160
|
+
const lessonId = lessonIdFromSignature(group.category, group.tags, refs);
|
|
161
|
+
const severity = severityFor(occurrences);
|
|
162
|
+
lessons.push({
|
|
163
|
+
lessonId,
|
|
164
|
+
createdAt: SENTINEL_CREATED_AT,
|
|
165
|
+
category: group.category,
|
|
166
|
+
tags: group.tags,
|
|
167
|
+
summary: group.summary,
|
|
168
|
+
occurrences,
|
|
169
|
+
severity,
|
|
170
|
+
sourceEntryRefs: refs,
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
// Sort by lessonId for deterministic, byte-identical output.
|
|
174
|
+
lessons.sort((a, b) => a.lessonId.localeCompare(b.lessonId));
|
|
175
|
+
return lessons;
|
|
176
|
+
}
|
|
177
|
+
//# sourceMappingURL=distill.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"distill.js","sourceRoot":"","sources":["../../../src/orchestrator/memory/distill.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAMzC,uEAAuE;AAEvE,uFAAuF;AACvF,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAE3B;;;0DAG0D;AAC1D,MAAM,mBAAmB,GAAG,0BAA0B,CAAC;AAuCvD,wEAAwE;AAExE,wDAAwD;AACxD,SAAS,QAAQ,CAAC,CAAU;IAC1B,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AAClE,CAAC;AAED;;;GAGG;AACH,SAAS,qBAAqB,CAC5B,QAAgB,EAChB,IAAc,EACd,IAAc;IAEd,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QAC/B,QAAQ;QACR,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,EAAE;QACtB,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,EAAE;KACvB,CAAC,CAAC;IACH,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC3E,CAAC;AAED,6CAA6C;AAC7C,SAAS,WAAW,CAAC,WAAmB;IACtC,IAAI,WAAW,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC;IACpC,IAAI,WAAW,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC;IACpC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,wEAAwE;AAExE;;;;;;;GAOG;AACH,MAAM,UAAU,OAAO,CACrB,OAAuB,EACvB,SAA2B,EAC3B,cAAiC,EAAE;IAEnC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAwB,CAAC;IAE/C,SAAS,WAAW,CAClB,QAAgB,EAChB,IAAc,EACd,OAAe,EACf,GAAW;QAEX,MAAM,UAAU,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,GAAG,GAAG,GAAG,QAAQ,IAAI,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAClD,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,KAAK,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,eAAe,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;YAC5E,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACzB,CAAC;QACD,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACjC,CAAC;IAED,yFAAyF;IACzF,MAAM,qBAAqB,GAAG,IAAI,GAAG,EAA+B,CAAC;IACrE,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;QAC1B,MAAM,CAAC,GAAG,IAAI,GAAG,EAAkB,CAAC;QACpC,KAAK,MAAM,EAAE,IAAI,CAAC,CAAC,eAAe,IAAI,EAAE,EAAE,CAAC;YACzC,IAAI,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,kBAAkB,EAAE,CAAC;gBAC5C,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC;YAC/C,CAAC;QACH,CAAC;QACD,qBAAqB,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IAC7C,CAAC;IAED,0EAA0E;IAC1E,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,EAAE,CAAC,MAAM,IAAI,EAAE,CAAC,UAAU,IAAI,cAAc,CAAC;QAE3D,mFAAmF;QACnF,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,eAAe,IAAI,EAAE,EAAE,CAAC;YACzC,IAAI,CAAC,EAAE,MAAM,KAAK,MAAM,IAAI,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpF,MAAM,QAAQ,GAAG,yBAAyB,CAAC,CAAC,QAAQ,EAAE,CAAC;gBACvD,MAAM,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;gBACxC,MAAM,OAAO,GAAG,kBAAkB,CAAC,CAAC,QAAQ,kDAAkD,CAAC;gBAC/F,WAAW,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC;QAED,4FAA4F;QAC5F,KAAK,MAAM,EAAE,IAAI,EAAE,CAAC,eAAe,IAAI,EAAE,EAAE,CAAC;YAC1C,IAAI,EAAE,EAAE,MAAM,KAAK,MAAM,IAAI,OAAO,EAAE,CAAC,WAAW,KAAK,QAAQ,IAAI,EAAE,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7F,MAAM,UAAU,GACd,CAAC,EAAE,CAAC,UAAU,IAAI,qBAAqB,CAAC,GAAG,CAAC,EAAE,CAAC,UAAU,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC;oBAChF,EAAE,CAAC,kBAAkB;oBACrB,SAAS,CAAC;gBACZ,MAAM,QAAQ,GAAG,oBAAoB,UAAU,EAAE,CAAC;gBAClD,MAAM,IAAI,GAAG,CAAC,sBAAsB,UAAU,EAAE,CAAC,CAAC;gBAClD,MAAM,OAAO,GAAG,iCAAiC,UAAU,yBAAyB,CAAC;gBACrF,WAAW,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,KAAK,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;YACrE,CAAC;QACH,CAAC;IACH,CAAC;IAED,4EAA4E;IAC5E,8EAA8E;IAC9E,wDAAwD;IACxD,MAAM,2BAA2B,GAAG,IAAI,GAAG,EAAU,CAAC;IACtD,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC;YACpD,CAAC,CAAC,QAAQ,CAAC,gBAAgB;YAC3B,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,UAAU,GAAa,EAAE,CAAC;QAChC,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;YACvB,IAAI,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,KAAK,MAAM,EAAE,CAAC;gBAC5C,MAAM,CAAC,GAAG,OAAO,EAAE,CAAC,WAAW,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;gBACtE,UAAU,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,UAAU,cAAc,CAAC,EAAE,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC;QACD,IAAI,UAAU,CAAC,MAAM,IAAI,gBAAgB,EAAE,CAAC;YAC1C,2BAA2B,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;YACrD,MAAM,IAAI,GAAG,CAAC,cAAc,EAAE,YAAY,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YACjE,MAAM,OAAO,GAAG,WAAW,QAAQ,CAAC,UAAU,YAAY,UAAU,CAAC,MAAM,qCAAqC,CAAC;YACjH,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;gBAC7B,WAAW,CAAC,eAAe,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC;YACnD,CAAC;QACH,CAAC;IACH,CAAC;IAED,2EAA2E;IAC3E,yGAAyG;IACzG,8FAA8F;IAC9F,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,KAAK,CAAC,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,KAAK,KAAK,mBAAmB,EAAE,CAAC;YACpE,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ,IAAI,SAAS,CAAC;YACxC,IAAI,2BAA2B,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YACnD,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YACxC,MAAM,SAAS,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC;YACnF,MAAM,IAAI,GAAG,CAAC,cAAc,EAAE,YAAY,GAAG,EAAE,CAAC,CAAC;YACjD,MAAM,OAAO,GAAG,WAAW,GAAG,4CAA4C,CAAC;YAC3E,WAAW,CAAC,eAAe,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,GAAG,IAAI,SAAS,EAAE,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;IAED,wEAAwE;IACxE,MAAM,OAAO,GAAkB,EAAE,CAAC;IAElC,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;QACpC,MAAM,IAAI,GAAG,CAAC,GAAG,KAAK,CAAC,eAAe,CAAC,CAAC,IAAI,EAAE,CAAC;QAC/C,wDAAwD;QACxD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAEhC,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC;QAChC,MAAM,QAAQ,GAAG,qBAAqB,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACzE,MAAM,QAAQ,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;QAE1C,OAAO,CAAC,IAAI,CAAC;YACX,QAAQ;YACR,SAAS,EAAE,mBAAmB;YAC9B,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,WAAW;YACX,QAAQ;YACR,eAAe,EAAE,IAAI;SACtB,CAAC,CAAC;IACL,CAAC;IAED,6DAA6D;IAC7D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;IAE7D,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lenient loader for .bober/eval-results/*.json, used to feed distill().
|
|
3
|
+
*
|
|
4
|
+
* This is the IMPURE counterpart to the pure distill() function: it touches the
|
|
5
|
+
* filesystem. It deliberately does NOT validate against the compiled EvalResultSchema,
|
|
6
|
+
* because the on-disk eval-result files use a richer shape
|
|
7
|
+
* ({ overallResult, strategyResults, criteriaResults }) than the compiled schema
|
|
8
|
+
* ({ passed, details, criteriaResults? }). We read every file leniently and project
|
|
9
|
+
* only the fields distill consumes, so distillation is robust to schema drift.
|
|
10
|
+
*/
|
|
11
|
+
import type { DistillableEval } from "./distill.js";
|
|
12
|
+
/**
|
|
13
|
+
* Load all eval results from .bober/eval-results/, projected onto DistillableEval.
|
|
14
|
+
*
|
|
15
|
+
* Returns entries sorted by filename for deterministic distill input. A missing
|
|
16
|
+
* directory yields an empty array; malformed JSON files are skipped silently
|
|
17
|
+
* (mirrors listContracts' skip-on-bad-file behaviour).
|
|
18
|
+
*/
|
|
19
|
+
export declare function loadEvalResults(projectRoot: string): Promise<DistillableEval[]>;
|
|
20
|
+
//# sourceMappingURL=eval-source.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eval-source.d.ts","sourceRoot":"","sources":["../../../src/orchestrator/memory/eval-source.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAKH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAgDpD;;;;;;GAMG;AACH,wBAAsB,eAAe,CACnC,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,eAAe,EAAE,CAAC,CA0B5B"}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lenient loader for .bober/eval-results/*.json, used to feed distill().
|
|
3
|
+
*
|
|
4
|
+
* This is the IMPURE counterpart to the pure distill() function: it touches the
|
|
5
|
+
* filesystem. It deliberately does NOT validate against the compiled EvalResultSchema,
|
|
6
|
+
* because the on-disk eval-result files use a richer shape
|
|
7
|
+
* ({ overallResult, strategyResults, criteriaResults }) than the compiled schema
|
|
8
|
+
* ({ passed, details, criteriaResults? }). We read every file leniently and project
|
|
9
|
+
* only the fields distill consumes, so distillation is robust to schema drift.
|
|
10
|
+
*/
|
|
11
|
+
import { readFile, readdir } from "node:fs/promises";
|
|
12
|
+
import { join } from "node:path";
|
|
13
|
+
const EVAL_RESULTS_DIR = ".bober/eval-results";
|
|
14
|
+
function evalResultsDir(projectRoot) {
|
|
15
|
+
return join(projectRoot, EVAL_RESULTS_DIR);
|
|
16
|
+
}
|
|
17
|
+
/** Narrow an unknown value to a string-keyed record. */
|
|
18
|
+
function isRecord(v) {
|
|
19
|
+
return typeof v === "object" && v !== null && !Array.isArray(v);
|
|
20
|
+
}
|
|
21
|
+
/** Project a raw parsed JSON object onto the lenient DistillableEval shape. */
|
|
22
|
+
function toDistillable(raw) {
|
|
23
|
+
if (!isRecord(raw))
|
|
24
|
+
return null;
|
|
25
|
+
const out = {};
|
|
26
|
+
if (typeof raw["evalId"] === "string")
|
|
27
|
+
out.evalId = raw["evalId"];
|
|
28
|
+
if (typeof raw["contractId"] === "string")
|
|
29
|
+
out.contractId = raw["contractId"];
|
|
30
|
+
if (typeof raw["iteration"] === "number")
|
|
31
|
+
out.iteration = raw["iteration"];
|
|
32
|
+
if (typeof raw["overallResult"] === "string")
|
|
33
|
+
out.overallResult = raw["overallResult"];
|
|
34
|
+
if (typeof raw["passed"] === "boolean")
|
|
35
|
+
out.passed = raw["passed"];
|
|
36
|
+
if (Array.isArray(raw["criteriaResults"])) {
|
|
37
|
+
out.criteriaResults = raw["criteriaResults"]
|
|
38
|
+
.filter(isRecord)
|
|
39
|
+
.map((c) => ({
|
|
40
|
+
criterionId: typeof c["criterionId"] === "string" ? c["criterionId"] : undefined,
|
|
41
|
+
result: typeof c["result"] === "string" ? c["result"] : undefined,
|
|
42
|
+
verificationMethod: typeof c["verificationMethod"] === "string" ? c["verificationMethod"] : undefined,
|
|
43
|
+
}));
|
|
44
|
+
}
|
|
45
|
+
if (Array.isArray(raw["strategyResults"])) {
|
|
46
|
+
out.strategyResults = raw["strategyResults"]
|
|
47
|
+
.filter(isRecord)
|
|
48
|
+
.map((s) => ({
|
|
49
|
+
strategy: typeof s["strategy"] === "string" ? s["strategy"] : undefined,
|
|
50
|
+
result: typeof s["result"] === "string" ? s["result"] : undefined,
|
|
51
|
+
}));
|
|
52
|
+
}
|
|
53
|
+
return out;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Load all eval results from .bober/eval-results/, projected onto DistillableEval.
|
|
57
|
+
*
|
|
58
|
+
* Returns entries sorted by filename for deterministic distill input. A missing
|
|
59
|
+
* directory yields an empty array; malformed JSON files are skipped silently
|
|
60
|
+
* (mirrors listContracts' skip-on-bad-file behaviour).
|
|
61
|
+
*/
|
|
62
|
+
export async function loadEvalResults(projectRoot) {
|
|
63
|
+
const dir = evalResultsDir(projectRoot);
|
|
64
|
+
let entries;
|
|
65
|
+
try {
|
|
66
|
+
entries = await readdir(dir);
|
|
67
|
+
}
|
|
68
|
+
catch {
|
|
69
|
+
// Directory doesn't exist yet — no eval results.
|
|
70
|
+
return [];
|
|
71
|
+
}
|
|
72
|
+
const jsonFiles = entries.filter((f) => f.endsWith(".json")).sort();
|
|
73
|
+
const results = [];
|
|
74
|
+
for (const file of jsonFiles) {
|
|
75
|
+
try {
|
|
76
|
+
const content = await readFile(join(dir, file), "utf-8");
|
|
77
|
+
const parsed = JSON.parse(content);
|
|
78
|
+
const projected = toDistillable(parsed);
|
|
79
|
+
if (projected)
|
|
80
|
+
results.push(projected);
|
|
81
|
+
}
|
|
82
|
+
catch {
|
|
83
|
+
// Skip malformed files.
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
return results;
|
|
87
|
+
}
|
|
88
|
+
//# sourceMappingURL=eval-source.js.map
|