orizu 0.5.19 → 0.5.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -0
- package/dist/archive-artifacts-cli.d.ts +50 -0
- package/dist/archive-artifacts-cli.d.ts.map +1 -0
- package/dist/artifact-archive.d.ts +48 -0
- package/dist/artifact-archive.d.ts.map +1 -0
- package/dist/artifact-pull.d.ts +22 -0
- package/dist/artifact-pull.d.ts.map +1 -0
- package/dist/banner.d.ts +3 -0
- package/dist/banner.d.ts.map +1 -0
- package/dist/connectors-cli.d.ts +31 -0
- package/dist/connectors-cli.d.ts.map +1 -0
- package/dist/credentials.d.ts +46 -0
- package/dist/credentials.d.ts.map +1 -0
- package/dist/csv.d.ts +49 -0
- package/dist/csv.d.ts.map +1 -0
- package/dist/dataset-download.d.ts +6 -0
- package/dist/dataset-download.d.ts.map +1 -0
- package/dist/dataset-edit-rows.d.ts +10 -0
- package/dist/dataset-edit-rows.d.ts.map +1 -0
- package/dist/dataset-upload-snapshot.d.ts +13 -0
- package/dist/dataset-upload-snapshot.d.ts.map +1 -0
- package/dist/daytona-slice-rehearsal.d.ts +89 -0
- package/dist/daytona-slice-rehearsal.d.ts.map +1 -0
- package/dist/daytona-slice.d.ts +106 -0
- package/dist/daytona-slice.d.ts.map +1 -0
- package/dist/deploy-key-merge-core.d.ts +121 -0
- package/dist/deploy-key-merge-core.d.ts.map +1 -0
- package/dist/diff-comments-cli.d.ts +12 -0
- package/dist/diff-comments-cli.d.ts.map +1 -0
- package/dist/diff-comments-cli.js +21 -2
- package/dist/egress-allowlist-cli.d.ts +23 -0
- package/dist/egress-allowlist-cli.d.ts.map +1 -0
- package/dist/egress-policy.d.ts +108 -0
- package/dist/egress-policy.d.ts.map +1 -0
- package/dist/error-response.d.ts +9 -0
- package/dist/error-response.d.ts.map +1 -0
- package/dist/file-parser.d.ts +5 -0
- package/dist/file-parser.d.ts.map +1 -0
- package/dist/gepa-engine-dispatch.d.ts +9 -0
- package/dist/gepa-engine-dispatch.d.ts.map +1 -0
- package/dist/gepa-engine-dispatch.js +307 -0
- package/dist/gepa-python-command.d.ts +6 -0
- package/dist/gepa-python-command.d.ts.map +1 -0
- package/dist/gepa-python-command.js +26 -0
- package/dist/gepa-python-paths.d.ts +6 -0
- package/dist/gepa-python-paths.d.ts.map +1 -0
- package/dist/gepa-python-paths.js +42 -0
- package/dist/git-credential.d.ts +69 -0
- package/dist/git-credential.d.ts.map +1 -0
- package/dist/git-recovery-environment.d.ts +31 -0
- package/dist/git-recovery-environment.d.ts.map +1 -0
- package/dist/github-setup.d.ts +122 -0
- package/dist/github-setup.d.ts.map +1 -0
- package/dist/global-flags.d.ts +12 -0
- package/dist/global-flags.d.ts.map +1 -0
- package/dist/help.d.ts +46 -0
- package/dist/help.d.ts.map +1 -0
- package/dist/help.js +93 -25
- package/dist/hosted-boot.d.ts +273 -0
- package/dist/hosted-boot.d.ts.map +1 -0
- package/dist/hosted-bootstrap-rehearsal.d.ts +86 -0
- package/dist/hosted-bootstrap-rehearsal.d.ts.map +1 -0
- package/dist/hosted-bootstrap.d.ts +188 -0
- package/dist/hosted-bootstrap.d.ts.map +1 -0
- package/dist/hosted-broker-server.d.ts +50 -0
- package/dist/hosted-broker-server.d.ts.map +1 -0
- package/dist/hosted-git-http-server.d.ts +36 -0
- package/dist/hosted-git-http-server.d.ts.map +1 -0
- package/dist/hosted-harness-claude.d.ts +195 -0
- package/dist/hosted-harness-claude.d.ts.map +1 -0
- package/dist/hosted-harness-opencode.d.ts +227 -0
- package/dist/hosted-harness-opencode.d.ts.map +1 -0
- package/dist/hosted-harness.d.ts +118 -0
- package/dist/hosted-harness.d.ts.map +1 -0
- package/dist/hosted-harvest.d.ts +68 -0
- package/dist/hosted-harvest.d.ts.map +1 -0
- package/dist/hosted-headless.d.ts +72 -0
- package/dist/hosted-headless.d.ts.map +1 -0
- package/dist/hosted-loop-lifecycle.d.ts +206 -0
- package/dist/hosted-loop-lifecycle.d.ts.map +1 -0
- package/dist/hosted-loop.d.ts +170 -0
- package/dist/hosted-loop.d.ts.map +1 -0
- package/dist/hosted-run-event-sink.d.ts +222 -0
- package/dist/hosted-run-event-sink.d.ts.map +1 -0
- package/dist/hosted-runtime-assets.d.ts +186 -0
- package/dist/hosted-runtime-assets.d.ts.map +1 -0
- package/dist/hosted-session-cli.d.ts +258 -0
- package/dist/hosted-session-cli.d.ts.map +1 -0
- package/dist/hosted-skill-staging.d.ts +73 -0
- package/dist/hosted-skill-staging.d.ts.map +1 -0
- package/dist/http.d.ts +8 -0
- package/dist/http.d.ts.map +1 -0
- package/dist/index.d.ts +22 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +119 -129
- package/dist/instruction-set-gepa-launch.d.ts +3 -0
- package/dist/instruction-set-gepa-launch.d.ts.map +1 -0
- package/dist/instruction-set-gepa-launch.js +15 -0
- package/dist/instruction-set-loader.d.ts +6 -0
- package/dist/instruction-set-loader.d.ts.map +1 -0
- package/dist/instruction-set-loader.js +59 -0
- package/dist/instruction-set-manifest.d.ts +14 -0
- package/dist/instruction-set-manifest.d.ts.map +1 -0
- package/dist/instruction-set-manifest.js +162 -0
- package/dist/instruction-sets-cli.d.ts +40 -0
- package/dist/instruction-sets-cli.d.ts.map +1 -0
- package/dist/instruction-sets-cli.js +361 -0
- package/dist/json-response.d.ts +4 -0
- package/dist/json-response.d.ts.map +1 -0
- package/dist/jsonl-stream.d.ts +8 -0
- package/dist/jsonl-stream.d.ts.map +1 -0
- package/dist/kill-switch-cli.d.ts +30 -0
- package/dist/kill-switch-cli.d.ts.map +1 -0
- package/dist/loopback-serve.d.ts +25 -0
- package/dist/loopback-serve.d.ts.map +1 -0
- package/dist/manifests-cli.d.ts +22 -0
- package/dist/manifests-cli.d.ts.map +1 -0
- package/dist/markdown-report-input.d.ts +7 -0
- package/dist/markdown-report-input.d.ts.map +1 -0
- package/dist/markdown-report-input.js +15 -1
- package/dist/merge-job.d.ts +155 -0
- package/dist/merge-job.d.ts.map +1 -0
- package/dist/model-configs-cli.d.ts +19 -0
- package/dist/model-configs-cli.d.ts.map +1 -0
- package/dist/model-configs-cli.js +104 -0
- package/dist/optimization-export-cli.d.ts +19 -0
- package/dist/optimization-export-cli.d.ts.map +1 -0
- package/dist/optimizations-list-cli.d.ts +20 -0
- package/dist/optimizations-list-cli.d.ts.map +1 -0
- package/dist/optimizations-promote-cli.d.ts +8 -0
- package/dist/optimizations-promote-cli.d.ts.map +1 -0
- package/dist/optimizations-promote-cli.js +34 -0
- package/dist/option-validation.d.ts +15 -0
- package/dist/option-validation.d.ts.map +1 -0
- package/dist/preview-runtime.d.ts +63 -0
- package/dist/preview-runtime.d.ts.map +1 -0
- package/dist/prompt-draft-push.d.ts +60 -0
- package/dist/prompt-draft-push.d.ts.map +1 -0
- package/dist/prompt-label-cli.d.ts +11 -0
- package/dist/prompt-label-cli.d.ts.map +1 -0
- package/dist/prompt-label-cli.js +29 -0
- package/dist/prompt-length-wire.d.ts +28 -0
- package/dist/prompt-length-wire.d.ts.map +1 -0
- package/dist/prompt-length-wire.js +2 -2
- package/dist/prompt-report-cli.d.ts +10 -0
- package/dist/prompt-report-cli.d.ts.map +1 -0
- package/dist/prompt-report-cli.js +2 -2
- package/dist/prompt-summary-table.d.ts +15 -0
- package/dist/prompt-summary-table.d.ts.map +1 -0
- package/dist/prompt-summary-table.js +6 -0
- package/dist/registration-tag-warning.d.ts +32 -0
- package/dist/registration-tag-warning.d.ts.map +1 -0
- package/dist/report-comments-cli.d.ts +16 -0
- package/dist/report-comments-cli.d.ts.map +1 -0
- package/dist/report-markdown-blankness.d.ts +2 -0
- package/dist/report-markdown-blankness.d.ts.map +1 -0
- package/dist/report-markdown-blankness.js +6 -0
- package/dist/report-replacement-warning.d.ts +17 -0
- package/dist/report-replacement-warning.d.ts.map +1 -0
- package/dist/report-replacement-warning.js +31 -0
- package/dist/repository-backup-core.d.ts +225 -0
- package/dist/repository-backup-core.d.ts.map +1 -0
- package/dist/repository-recovery-manifest.d.ts +83 -0
- package/dist/repository-recovery-manifest.d.ts.map +1 -0
- package/dist/run-event-digest.d.ts +74 -0
- package/dist/run-event-digest.d.ts.map +1 -0
- package/dist/runner-dir-verify.d.ts +66 -0
- package/dist/runner-dir-verify.d.ts.map +1 -0
- package/dist/runner-instruction-set-contract.d.ts +24 -0
- package/dist/runner-instruction-set-contract.d.ts.map +1 -0
- package/dist/runner-instruction-set-contract.js +50 -0
- package/dist/sandbox-hygiene.d.ts +45 -0
- package/dist/sandbox-hygiene.d.ts.map +1 -0
- package/dist/sandbox-provider.d.ts +193 -0
- package/dist/sandbox-provider.d.ts.map +1 -0
- package/dist/scorer-draft-push.d.ts +50 -0
- package/dist/scorer-draft-push.d.ts.map +1 -0
- package/dist/scorers-cli.d.ts +20 -0
- package/dist/scorers-cli.d.ts.map +1 -0
- package/dist/scorers-cli.js +22 -0
- package/dist/scorers-verify-parity-bridge.d.ts +16 -0
- package/dist/scorers-verify-parity-bridge.d.ts.map +1 -0
- package/dist/scorers-verify-parity-bridge.js +90 -0
- package/dist/scorers-verify-parity.d.ts +81 -0
- package/dist/scorers-verify-parity.d.ts.map +1 -0
- package/dist/scorers-verify-parity.js +502 -0
- package/dist/scores-accept-cli.d.ts +21 -0
- package/dist/scores-accept-cli.d.ts.map +1 -0
- package/dist/secret-redaction.d.ts +47 -0
- package/dist/secret-redaction.d.ts.map +1 -0
- package/dist/session-timeline.d.ts +54 -0
- package/dist/session-timeline.d.ts.map +1 -0
- package/dist/setup-prompt.d.ts +5 -0
- package/dist/setup-prompt.d.ts.map +1 -0
- package/dist/skill-installer.d.ts +68 -0
- package/dist/skill-installer.d.ts.map +1 -0
- package/dist/skilled-proposer-launch.d.ts +7 -0
- package/dist/skilled-proposer-launch.d.ts.map +1 -0
- package/dist/skilled-proposer-launch.js +60 -0
- package/dist/smoke.d.ts +102 -0
- package/dist/smoke.d.ts.map +1 -0
- package/dist/start-intent.d.ts +12 -0
- package/dist/start-intent.d.ts.map +1 -0
- package/dist/task-assignment-manifest.d.ts +15 -0
- package/dist/task-assignment-manifest.d.ts.map +1 -0
- package/dist/task-create-error.d.ts +18 -0
- package/dist/task-create-error.d.ts.map +1 -0
- package/dist/team-connectors-cli.d.ts +29 -0
- package/dist/team-connectors-cli.d.ts.map +1 -0
- package/dist/types.d.ts +39 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/vercel-sandbox-provider.d.ts +156 -0
- package/dist/vercel-sandbox-provider.d.ts.map +1 -0
- package/dist/workbench-cli.d.ts +233 -0
- package/dist/workbench-cli.d.ts.map +1 -0
- package/dist/workspace-sync.d.ts +207 -0
- package/dist/workspace-sync.d.ts.map +1 -0
- package/dist/workspace.d.ts +92 -0
- package/dist/workspace.d.ts.map +1 -0
- package/dist/zip-draft-push.d.ts +55 -0
- package/dist/zip-draft-push.d.ts.map +1 -0
- package/package.json +15 -2
- package/requirements/README.md +138 -0
- package/requirements/skilled-proposer.in +66 -0
- package/requirements/skilled-proposer.lock +265 -0
- package/scripts/build-cli-dist.mjs +30 -0
- package/scripts/ensure-skilled-proposer-venv.mjs +5 -0
- package/scripts/orizu-source-checkout.d.mts +1 -0
- package/scripts/orizu-source-checkout.mjs +10 -0
- package/scripts/refresh-skilled-proposer-lock.mjs +256 -0
- package/scripts/vendor-gepa-python.mjs +187 -4
- package/src/diff-comments-cli.ts +41 -3
- package/src/gepa-engine-dispatch.ts +340 -0
- package/src/gepa-python-command.ts +31 -0
- package/src/gepa-python-paths.ts +56 -0
- package/src/help.ts +95 -25
- package/src/index.ts +131 -166
- package/src/instruction-set-gepa-launch.ts +28 -0
- package/src/instruction-set-loader.ts +37 -0
- package/src/instruction-set-manifest.ts +120 -0
- package/src/instruction-sets-cli.ts +306 -0
- package/src/markdown-report-input.ts +19 -1
- package/src/model-configs-cli.ts +119 -0
- package/src/optimizations-promote-cli.ts +41 -0
- package/src/prompt-label-cli.ts +55 -0
- package/src/prompt-length-wire.ts +1 -1
- package/src/prompt-report-cli.ts +2 -2
- package/src/prompt-summary-table.ts +12 -0
- package/src/report-markdown-blankness.ts +6 -0
- package/src/report-replacement-warning.ts +35 -0
- package/src/runner-instruction-set-contract.ts +64 -0
- package/src/scorers-cli.ts +40 -0
- package/src/scorers-verify-parity-bridge.ts +90 -0
- package/src/scorers-verify-parity.ts +564 -0
- package/src/skilled-proposer-launch.ts +89 -0
- package/src/skilled-proposer-venv-manager.mjs +757 -0
- package/tsconfig.json +2 -0
- package/vendor/gepa-python/LICENSE +7 -0
- package/vendor/gepa-python/manifest.json +107 -0
- package/vendor/gepa-python/src/gepa/__init__.py +22 -0
- package/vendor/gepa-python/src/gepa/adapters/__init__.py +0 -0
- package/vendor/gepa-python/src/gepa/adapters/anymaths_adapter/__init__.py +1 -0
- package/vendor/gepa-python/src/gepa/adapters/anymaths_adapter/anymaths_adapter.py +174 -0
- package/vendor/gepa-python/src/gepa/adapters/confidence_adapter/__init__.py +26 -0
- package/vendor/gepa-python/src/gepa/adapters/confidence_adapter/confidence_adapter.py +618 -0
- package/vendor/gepa-python/src/gepa/adapters/confidence_adapter/scoring.py +206 -0
- package/vendor/gepa-python/src/gepa/adapters/default_adapter/__init__.py +0 -0
- package/vendor/gepa-python/src/gepa/adapters/default_adapter/default_adapter.py +206 -0
- package/vendor/gepa-python/src/gepa/adapters/dspy_adapter/__init__.py +0 -0
- package/vendor/gepa-python/src/gepa/adapters/dspy_adapter/dspy_adapter.py +525 -0
- package/vendor/gepa-python/src/gepa/adapters/dspy_adapter/instruction_proposal.py +500 -0
- package/vendor/gepa-python/src/gepa/adapters/dspy_full_program_adapter/__init__.py +6 -0
- package/vendor/gepa-python/src/gepa/adapters/dspy_full_program_adapter/dspy_program_proposal_signature.py +137 -0
- package/vendor/gepa-python/src/gepa/adapters/dspy_full_program_adapter/full_program_adapter.py +269 -0
- package/vendor/gepa-python/src/gepa/adapters/generic_rag_adapter/__init__.py +56 -0
- package/vendor/gepa-python/src/gepa/adapters/generic_rag_adapter/evaluation_metrics.py +226 -0
- package/vendor/gepa-python/src/gepa/adapters/generic_rag_adapter/generic_rag_adapter.py +496 -0
- package/vendor/gepa-python/src/gepa/adapters/generic_rag_adapter/rag_pipeline.py +238 -0
- package/vendor/gepa-python/src/gepa/adapters/generic_rag_adapter/vector_store_interface.py +212 -0
- package/vendor/gepa-python/src/gepa/adapters/generic_rag_adapter/vector_stores/__init__.py +2 -0
- package/vendor/gepa-python/src/gepa/adapters/generic_rag_adapter/vector_stores/chroma_store.py +196 -0
- package/vendor/gepa-python/src/gepa/adapters/generic_rag_adapter/vector_stores/lancedb_store.py +422 -0
- package/vendor/gepa-python/src/gepa/adapters/generic_rag_adapter/vector_stores/milvus_store.py +409 -0
- package/vendor/gepa-python/src/gepa/adapters/generic_rag_adapter/vector_stores/qdrant_store.py +368 -0
- package/vendor/gepa-python/src/gepa/adapters/generic_rag_adapter/vector_stores/weaviate_store.py +418 -0
- package/vendor/gepa-python/src/gepa/adapters/langchain_adapter/__init__.py +11 -0
- package/vendor/gepa-python/src/gepa/adapters/langchain_adapter/langchain_adapter.py +261 -0
- package/vendor/gepa-python/src/gepa/adapters/mcp_adapter/__init__.py +37 -0
- package/vendor/gepa-python/src/gepa/adapters/mcp_adapter/mcp_adapter.py +693 -0
- package/vendor/gepa-python/src/gepa/adapters/mcp_adapter/mcp_client.py +364 -0
- package/vendor/gepa-python/src/gepa/adapters/optimize_anything_adapter/optimize_anything_adapter.py +879 -0
- package/vendor/gepa-python/src/gepa/adapters/terminal_bench_adapter/__init__.py +0 -0
- package/vendor/gepa-python/src/gepa/adapters/terminal_bench_adapter/terminal_bench_adapter.py +217 -0
- package/vendor/gepa-python/src/gepa/api.py +468 -0
- package/vendor/gepa-python/src/gepa/core/__init__.py +0 -0
- package/vendor/gepa-python/src/gepa/core/adapter.py +227 -0
- package/vendor/gepa-python/src/gepa/core/callbacks.py +551 -0
- package/vendor/gepa-python/src/gepa/core/data_loader.py +74 -0
- package/vendor/gepa-python/src/gepa/core/engine.py +1025 -0
- package/vendor/gepa-python/src/gepa/core/result.py +271 -0
- package/vendor/gepa-python/src/gepa/core/state.py +706 -0
- package/vendor/gepa-python/src/gepa/examples/__init__.py +0 -0
- package/vendor/gepa-python/src/gepa/examples/aime.py +24 -0
- package/vendor/gepa-python/src/gepa/examples/anymaths-bench/eval_default.py +111 -0
- package/vendor/gepa-python/src/gepa/examples/anymaths-bench/train_anymaths.py +177 -0
- package/vendor/gepa-python/src/gepa/examples/mcp_adapter/__init__.py +4 -0
- package/vendor/gepa-python/src/gepa/examples/mcp_adapter/mcp_optimization_example.py +456 -0
- package/vendor/gepa-python/src/gepa/examples/rag_adapter/__init__.py +9 -0
- package/vendor/gepa-python/src/gepa/examples/rag_adapter/rag_optimization.py +824 -0
- package/vendor/gepa-python/src/gepa/examples/terminal-bench/train_terminus.py +161 -0
- package/vendor/gepa-python/src/gepa/gepa_utils.py +117 -0
- package/vendor/gepa-python/src/gepa/gskill/__init__.py +1 -0
- package/vendor/gepa-python/src/gepa/gskill/gskill/__init__.py +0 -0
- package/vendor/gepa-python/src/gepa/gskill/gskill/cost_tracker.py +194 -0
- package/vendor/gepa-python/src/gepa/gskill/gskill/evaluate/__init__.py +0 -0
- package/vendor/gepa-python/src/gepa/gskill/gskill/evaluate/claude_code.py +1117 -0
- package/vendor/gepa-python/src/gepa/gskill/gskill/evaluate/claude_code_skills.py +613 -0
- package/vendor/gepa-python/src/gepa/gskill/gskill/evaluate/mini_swe_agent.py +440 -0
- package/vendor/gepa-python/src/gepa/gskill/gskill/experiment_logger.py +367 -0
- package/vendor/gepa-python/src/gepa/gskill/gskill/swe_fitness_fn.py +209 -0
- package/vendor/gepa-python/src/gepa/gskill/gskill/swe_harness.py +352 -0
- package/vendor/gepa-python/src/gepa/gskill/gskill/train_optimize_anything.py +830 -0
- package/vendor/gepa-python/src/gepa/image.py +104 -0
- package/vendor/gepa-python/src/gepa/lm.py +261 -0
- package/vendor/gepa-python/src/gepa/logging/__init__.py +0 -0
- package/vendor/gepa-python/src/gepa/logging/experiment_tracker.py +457 -0
- package/vendor/gepa-python/src/gepa/logging/logger.py +75 -0
- package/vendor/gepa-python/src/gepa/logging/utils.py +130 -0
- package/vendor/gepa-python/src/gepa/optimize_anything.py +1701 -0
- package/vendor/gepa-python/src/gepa/proposer/__init__.py +0 -0
- package/vendor/gepa-python/src/gepa/proposer/base.py +53 -0
- package/vendor/gepa-python/src/gepa/proposer/merge.py +403 -0
- package/vendor/gepa-python/src/gepa/proposer/reflective_mutation/__init__.py +0 -0
- package/vendor/gepa-python/src/gepa/proposer/reflective_mutation/base.py +64 -0
- package/vendor/gepa-python/src/gepa/proposer/reflective_mutation/reflection_lm.py +178 -0
- package/vendor/gepa-python/src/gepa/proposer/reflective_mutation/reflective_mutation.py +641 -0
- package/vendor/gepa-python/src/gepa/py.typed +0 -0
- package/vendor/gepa-python/src/gepa/strategies/__init__.py +0 -0
- package/vendor/gepa-python/src/gepa/strategies/acceptance.py +66 -0
- package/vendor/gepa-python/src/gepa/strategies/batch_sampler.py +103 -0
- package/vendor/gepa-python/src/gepa/strategies/candidate_selector.py +82 -0
- package/vendor/gepa-python/src/gepa/strategies/component_selector.py +36 -0
- package/vendor/gepa-python/src/gepa/strategies/eval_policy.py +64 -0
- package/vendor/gepa-python/src/gepa/strategies/instruction_proposal.py +153 -0
- package/vendor/gepa-python/src/gepa/strategies/proposal_sampling.py +128 -0
- package/vendor/gepa-python/src/gepa/strategies/proposal_selection.py +84 -0
- package/vendor/gepa-python/src/gepa/utils/__init__.py +63 -0
- package/vendor/gepa-python/src/gepa/utils/code_execution.py +626 -0
- package/vendor/gepa-python/src/gepa/utils/stdio_capture.py +145 -0
- package/vendor/gepa-python/src/gepa/utils/stop_condition.py +230 -0
- package/vendor/gepa-python/src/gepa/visualization.py +359 -0
- package/vendor/orizu-gepa/pyproject.toml +18 -0
- package/vendor/orizu-gepa/src/orizu_gepa_connector/__init__.py +9 -0
- package/vendor/orizu-gepa/src/orizu_gepa_connector/__main__.py +5 -0
- package/vendor/orizu-gepa/src/orizu_gepa_connector/adapter.py +138 -0
- package/vendor/orizu-gepa/src/orizu_gepa_connector/callbacks.py +666 -0
- package/vendor/orizu-gepa/src/orizu_gepa_connector/cli.py +7 -0
- package/vendor/orizu-gepa/src/orizu_gepa_connector/engine.py +269 -0
- package/vendor/orizu-gepa/src/orizu_gepa_connector/instruction_set_loader.py +52 -0
- package/vendor/orizu-gepa/src/orizu_gepa_connector/preflight.py +64 -0
- package/vendor/orizu-gepa/src/orizu_gepa_connector/reflection.py +186 -0
- package/vendor/orizu-gepa/src/orizu_gepa_connector/runtime.py +721 -0
- package/vendor/orizu-gepa/src/orizu_gepa_connector/skilled_proposer_bridge.py +381 -0
- package/vendor/orizu-gepa/src/orizu_gepa_connector/stop_conditions.py +55 -0
- package/vendor/orizu-gepa/src/orizu_gepa_connector/translator.py +203 -0
- package/vendor/orizu-gepa-python/pyproject.toml +1 -1
- package/vendor/orizu-gepa-python/src/orizu_gepa/cli.py +1 -0
- package/vendor/orizu-gepa-python/src/orizu_gepa/client.py +52 -7
- package/vendor/orizu-gepa-python/src/orizu_gepa/local_log.py +52 -0
- package/vendor/orizu-gepa-python/src/orizu_gepa/optimizer.py +10 -5
- package/vendor/orizu-gepa-python/src/orizu_gepa/reflection.py +323 -68
- package/vendor/orizu-gepa-python/src/orizu_gepa/runner.py +207 -13
- package/vendor/skills/orizu-cli/SKILL.md +3 -1
- package/vendor/skills/orizu-cli/references/cli-reference.md +85 -1
- package/vendor/skills/orizu-cli/references/migrate-existing-gepa-setup.md +303 -0
- package/vendor/skills/orizu-cli/references/optimization-reports.md +1 -1
- package/vendor/skills/orizu-cli/references/optimization-with-gepa.md +26 -1
- package/vendor/skills/orizu-cli/references/prompt-control-plane.md +113 -10
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# Copyright (c) 2025 Lakshya A Agrawal and the GEPA contributors
|
|
2
|
+
# https://github.com/gepa-ai/gepa
|
|
3
|
+
|
|
4
|
+
from typing import Protocol, runtime_checkable
|
|
5
|
+
|
|
6
|
+
from gepa.core.state import GEPAState
|
|
7
|
+
from gepa.proposer.base import CandidateProposal
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@runtime_checkable
|
|
11
|
+
class AcceptanceCriterion(Protocol):
|
|
12
|
+
"""Decides whether a proposed candidate should be accepted based on subsample evaluation results.
|
|
13
|
+
|
|
14
|
+
The ``should_accept`` method receives:
|
|
15
|
+
|
|
16
|
+
- ``proposal``: the full ``CandidateProposal`` containing:
|
|
17
|
+
|
|
18
|
+
- ``eval_before`` / ``eval_after``: ``SubsampleEvaluation`` objects with
|
|
19
|
+
per-example ``scores``, ``outputs``, ``objective_scores``, and ``trajectories``.
|
|
20
|
+
- ``subsample_scores_before`` / ``subsample_scores_after``: shorthand score lists.
|
|
21
|
+
- ``candidate``: the proposed candidate text.
|
|
22
|
+
- ``parent_program_ids``: indices of parent candidates.
|
|
23
|
+
- ``metadata``: free-form dict with LM prompts and raw outputs.
|
|
24
|
+
|
|
25
|
+
Implementations may additionally define an optional
|
|
26
|
+
``reject_reason(proposal, state) -> str`` method; when present, the engine
|
|
27
|
+
uses it to produce an informative log message for rejected proposals
|
|
28
|
+
instead of the generic fallback.
|
|
29
|
+
|
|
30
|
+
- ``state``: the full ``GEPAState``, giving access to all existing candidates,
|
|
31
|
+
validation scores, the Pareto frontier, iteration count, etc.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
def should_accept(self, proposal: CandidateProposal, state: GEPAState) -> bool:
|
|
35
|
+
"""Return ``True`` if the proposed candidate should be accepted.
|
|
36
|
+
|
|
37
|
+
Args:
|
|
38
|
+
proposal: The full proposal including evaluation data, candidate, and metadata.
|
|
39
|
+
state: The current optimization state.
|
|
40
|
+
"""
|
|
41
|
+
...
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class StrictImprovementAcceptance:
|
|
45
|
+
"""Accept only if the sum of new subsample scores is strictly greater than the old sum.
|
|
46
|
+
|
|
47
|
+
This is the default acceptance criterion used by GEPA.
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
def should_accept(self, proposal: CandidateProposal, state: GEPAState) -> bool:
|
|
51
|
+
old_sum = sum(proposal.subsample_scores_before or [])
|
|
52
|
+
new_sum = sum(proposal.subsample_scores_after or [])
|
|
53
|
+
return new_sum > old_sum
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class ImprovementOrEqualAcceptance:
|
|
57
|
+
"""Accept if the sum of new subsample scores is greater than or equal to the old sum.
|
|
58
|
+
|
|
59
|
+
Useful when you want to allow lateral moves that don't improve the score but
|
|
60
|
+
may explore different regions of the solution space.
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
def should_accept(self, proposal: CandidateProposal, state: GEPAState) -> bool:
|
|
64
|
+
old_sum = sum(proposal.subsample_scores_before or [])
|
|
65
|
+
new_sum = sum(proposal.subsample_scores_after or [])
|
|
66
|
+
return new_sum >= old_sum
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
# Copyright (c) 2025 Lakshya A Agrawal and the GEPA contributors
|
|
2
|
+
# https://github.com/gepa-ai/gepa
|
|
3
|
+
|
|
4
|
+
import random
|
|
5
|
+
from collections import Counter
|
|
6
|
+
from typing import Protocol
|
|
7
|
+
|
|
8
|
+
from gepa.core.adapter import DataInst
|
|
9
|
+
from gepa.core.data_loader import DataId, DataLoader
|
|
10
|
+
from gepa.core.state import GEPAState
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class BatchSampler(Protocol[DataId, DataInst]):
|
|
14
|
+
"""Yields the minibatch of trainset ids to propose from.
|
|
15
|
+
|
|
16
|
+
Multi-proposal sampling strategies call ``next_minibatch_ids`` once per
|
|
17
|
+
task within a single iteration (``state.i`` unchanged between calls).
|
|
18
|
+
Implementations should return a *different* minibatch on each repeated
|
|
19
|
+
call within an iteration, so parallel proposal tasks don't all share one
|
|
20
|
+
minibatch.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
def next_minibatch_ids(self, loader: DataLoader[DataId, DataInst], state: GEPAState) -> list[DataId]: ...
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class EpochShuffledBatchSampler(BatchSampler[DataId, DataInst]):
|
|
27
|
+
"""
|
|
28
|
+
Mirrors the original batching logic:
|
|
29
|
+
- Shuffle ids each epoch
|
|
30
|
+
- Pad to minibatch size with least frequent ids
|
|
31
|
+
- Deterministic via state.rng1
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
def __init__(self, minibatch_size: int, rng: random.Random | None = None):
|
|
35
|
+
self.minibatch_size = minibatch_size
|
|
36
|
+
self.shuffled_ids: list[DataId] = []
|
|
37
|
+
self.epoch = -1
|
|
38
|
+
self.id_freqs = Counter()
|
|
39
|
+
self.last_trainset_size = 0
|
|
40
|
+
self._current_iteration: int | None = None
|
|
41
|
+
self._calls_in_iteration = 0
|
|
42
|
+
if rng is None:
|
|
43
|
+
self.rng = random.Random(0)
|
|
44
|
+
else:
|
|
45
|
+
self.rng = rng
|
|
46
|
+
|
|
47
|
+
def _update_shuffled(self, loader: DataLoader[DataId, DataInst]):
|
|
48
|
+
all_ids = list(loader.all_ids())
|
|
49
|
+
trainset_size = len(loader)
|
|
50
|
+
self.last_trainset_size = trainset_size
|
|
51
|
+
|
|
52
|
+
if trainset_size == 0:
|
|
53
|
+
self.shuffled_ids = []
|
|
54
|
+
self.id_freqs = Counter()
|
|
55
|
+
return
|
|
56
|
+
|
|
57
|
+
self.shuffled_ids = list(all_ids)
|
|
58
|
+
self.rng.shuffle(self.shuffled_ids)
|
|
59
|
+
self.id_freqs = Counter(self.shuffled_ids)
|
|
60
|
+
|
|
61
|
+
mod = trainset_size % self.minibatch_size
|
|
62
|
+
num_to_pad = (self.minibatch_size - mod) if mod != 0 else 0
|
|
63
|
+
if num_to_pad > 0:
|
|
64
|
+
for _ in range(num_to_pad):
|
|
65
|
+
selected_id = self.id_freqs.most_common()[::-1][0][0]
|
|
66
|
+
self.shuffled_ids.append(selected_id)
|
|
67
|
+
self.id_freqs[selected_id] += 1
|
|
68
|
+
|
|
69
|
+
def next_minibatch_ids(self, loader: DataLoader[DataId, DataInst], state: GEPAState) -> list[DataId]:
|
|
70
|
+
trainset_size = len(loader)
|
|
71
|
+
if trainset_size == 0:
|
|
72
|
+
raise ValueError("Cannot sample a minibatch from an empty loader.")
|
|
73
|
+
|
|
74
|
+
# Repeated calls within one iteration (multi-proposal sampling
|
|
75
|
+
# strategies request one minibatch per task) advance one chunk each,
|
|
76
|
+
# so tasks in the same iteration get distinct minibatches. The first
|
|
77
|
+
# call of an iteration is unchanged from the classic behavior. Chunks
|
|
78
|
+
# wrap around, so distinctness holds only while the iteration's call
|
|
79
|
+
# count stays within len(shuffled_ids) / minibatch_size.
|
|
80
|
+
if state.i == self._current_iteration:
|
|
81
|
+
self._calls_in_iteration += 1
|
|
82
|
+
else:
|
|
83
|
+
self._current_iteration = state.i
|
|
84
|
+
self._calls_in_iteration = 0
|
|
85
|
+
|
|
86
|
+
base_idx = state.i * self.minibatch_size
|
|
87
|
+
curr_epoch = 0 if self.epoch == -1 else base_idx // max(len(self.shuffled_ids), 1)
|
|
88
|
+
|
|
89
|
+
needs_refresh = not self.shuffled_ids or trainset_size != self.last_trainset_size or curr_epoch > self.epoch
|
|
90
|
+
if needs_refresh:
|
|
91
|
+
self.epoch = curr_epoch
|
|
92
|
+
self._update_shuffled(loader)
|
|
93
|
+
|
|
94
|
+
assert len(self.shuffled_ids) >= self.minibatch_size
|
|
95
|
+
assert len(self.shuffled_ids) % self.minibatch_size == 0
|
|
96
|
+
|
|
97
|
+
# The epoch bookkeeping above uses the un-offset base_idx (constant
|
|
98
|
+
# within an iteration), so repeat calls never trigger a reshuffle and
|
|
99
|
+
# the shuffle sequence stays identical to the single-call path.
|
|
100
|
+
base_idx = (base_idx + self._calls_in_iteration * self.minibatch_size) % len(self.shuffled_ids)
|
|
101
|
+
end_idx = base_idx + self.minibatch_size
|
|
102
|
+
assert end_idx <= len(self.shuffled_ids)
|
|
103
|
+
return self.shuffled_ids[base_idx:end_idx]
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
# Copyright (c) 2025 Lakshya A Agrawal and the GEPA contributors
|
|
2
|
+
# https://github.com/gepa-ai/gepa
|
|
3
|
+
|
|
4
|
+
import random
|
|
5
|
+
|
|
6
|
+
from gepa.core.state import GEPAState
|
|
7
|
+
from gepa.gepa_utils import idxmax, select_program_candidate_from_pareto_front
|
|
8
|
+
from gepa.proposer.reflective_mutation.base import CandidateSelector
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class ParetoCandidateSelector(CandidateSelector):
|
|
12
|
+
def __init__(self, rng: random.Random | None):
|
|
13
|
+
if rng is None:
|
|
14
|
+
self.rng = random.Random(0)
|
|
15
|
+
else:
|
|
16
|
+
self.rng = rng
|
|
17
|
+
|
|
18
|
+
def select_candidate_idx(self, state: GEPAState) -> int:
|
|
19
|
+
assert len(state.program_full_scores_val_set) == len(state.program_candidates)
|
|
20
|
+
return select_program_candidate_from_pareto_front(
|
|
21
|
+
state.get_pareto_front_mapping(),
|
|
22
|
+
state.per_program_tracked_scores,
|
|
23
|
+
self.rng,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class CurrentBestCandidateSelector(CandidateSelector):
|
|
28
|
+
def __init__(self):
|
|
29
|
+
pass
|
|
30
|
+
|
|
31
|
+
def select_candidate_idx(self, state: GEPAState) -> int:
|
|
32
|
+
assert len(state.program_full_scores_val_set) == len(state.program_candidates)
|
|
33
|
+
return idxmax(state.program_full_scores_val_set)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class EpsilonGreedyCandidateSelector(CandidateSelector):
|
|
37
|
+
def __init__(self, epsilon: float, rng: random.Random | None):
|
|
38
|
+
assert 0.0 <= epsilon <= 1.0
|
|
39
|
+
self.epsilon = epsilon
|
|
40
|
+
if rng is None:
|
|
41
|
+
self.rng = random.Random(0)
|
|
42
|
+
else:
|
|
43
|
+
self.rng = rng
|
|
44
|
+
|
|
45
|
+
def select_candidate_idx(self, state: GEPAState) -> int:
|
|
46
|
+
assert len(state.program_full_scores_val_set) == len(state.program_candidates)
|
|
47
|
+
if self.rng.random() < self.epsilon:
|
|
48
|
+
return self.rng.randint(0, len(state.program_candidates) - 1)
|
|
49
|
+
else:
|
|
50
|
+
return idxmax(state.program_full_scores_val_set)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class TopKParetoCandidateSelector(CandidateSelector):
|
|
54
|
+
"""Pareto selection restricted to the top K programs by aggregate score."""
|
|
55
|
+
|
|
56
|
+
def __init__(self, k: int, rng: random.Random | None):
|
|
57
|
+
assert k > 0
|
|
58
|
+
self.k = k
|
|
59
|
+
if rng is None:
|
|
60
|
+
self.rng = random.Random(0)
|
|
61
|
+
else:
|
|
62
|
+
self.rng = rng
|
|
63
|
+
|
|
64
|
+
def select_candidate_idx(self, state: GEPAState) -> int:
|
|
65
|
+
assert len(state.program_full_scores_val_set) == len(state.program_candidates)
|
|
66
|
+
# Get top K program indices by aggregate score
|
|
67
|
+
scores = state.per_program_tracked_scores
|
|
68
|
+
top_k_indices = set(sorted(range(len(scores)), key=lambda i: scores[i], reverse=True)[: self.k])
|
|
69
|
+
|
|
70
|
+
# Filter pareto front mapping to only include top K programs
|
|
71
|
+
pareto_mapping = state.get_pareto_front_mapping()
|
|
72
|
+
filtered_mapping = {}
|
|
73
|
+
for key, prog_set in pareto_mapping.items():
|
|
74
|
+
filtered = prog_set & top_k_indices
|
|
75
|
+
if filtered:
|
|
76
|
+
filtered_mapping[key] = filtered
|
|
77
|
+
|
|
78
|
+
if not filtered_mapping:
|
|
79
|
+
# Fallback: pick the best program overall
|
|
80
|
+
return idxmax(scores)
|
|
81
|
+
|
|
82
|
+
return select_program_candidate_from_pareto_front(filtered_mapping, scores, self.rng)
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# Copyright (c) 2025 Lakshya A Agrawal and the GEPA contributors
|
|
2
|
+
# https://github.com/gepa-ai/gepa
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
from gepa.core.adapter import Trajectory
|
|
6
|
+
from gepa.core.state import GEPAState
|
|
7
|
+
from gepa.proposer.reflective_mutation.base import ReflectionComponentSelector
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class RoundRobinReflectionComponentSelector(ReflectionComponentSelector):
|
|
11
|
+
def __call__(
|
|
12
|
+
self,
|
|
13
|
+
state: GEPAState,
|
|
14
|
+
trajectories: list[Trajectory],
|
|
15
|
+
subsample_scores: list[float],
|
|
16
|
+
candidate_idx: int,
|
|
17
|
+
candidate: dict[str, str],
|
|
18
|
+
) -> list[str]:
|
|
19
|
+
pid = state.named_predictor_id_to_update_next_for_program_candidate[candidate_idx]
|
|
20
|
+
state.named_predictor_id_to_update_next_for_program_candidate[candidate_idx] = (pid + 1) % len(
|
|
21
|
+
state.list_of_named_predictors
|
|
22
|
+
)
|
|
23
|
+
name = state.list_of_named_predictors[pid]
|
|
24
|
+
return [name]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class AllReflectionComponentSelector(ReflectionComponentSelector):
|
|
28
|
+
def __call__(
|
|
29
|
+
self,
|
|
30
|
+
state: GEPAState,
|
|
31
|
+
trajectories: list[Trajectory],
|
|
32
|
+
subsample_scores: list[float],
|
|
33
|
+
candidate_idx: int,
|
|
34
|
+
candidate: dict[str, str],
|
|
35
|
+
) -> list[str]:
|
|
36
|
+
return list(candidate.keys())
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""Validation evaluation policy protocols and helpers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from abc import abstractmethod
|
|
6
|
+
from typing import Protocol, runtime_checkable
|
|
7
|
+
|
|
8
|
+
from gepa.core.data_loader import DataId, DataInst, DataLoader
|
|
9
|
+
from gepa.core.state import GEPAState, ProgramIdx
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@runtime_checkable
|
|
13
|
+
class EvaluationPolicy(Protocol[DataId, DataInst]): # type: ignore
|
|
14
|
+
"""Strategy for choosing validation ids to evaluate and identifying best programs for validation instances."""
|
|
15
|
+
|
|
16
|
+
@abstractmethod
|
|
17
|
+
def get_eval_batch(
|
|
18
|
+
self, loader: DataLoader[DataId, DataInst], state: GEPAState, target_program_idx: ProgramIdx | None = None
|
|
19
|
+
) -> list[DataId]:
|
|
20
|
+
"""Select examples for evaluation for a program"""
|
|
21
|
+
...
|
|
22
|
+
|
|
23
|
+
@abstractmethod
|
|
24
|
+
def get_best_program(self, state: GEPAState) -> ProgramIdx:
|
|
25
|
+
"""Return "best" program given all validation results so far across candidates"""
|
|
26
|
+
...
|
|
27
|
+
|
|
28
|
+
@abstractmethod
|
|
29
|
+
def get_valset_score(self, program_idx: ProgramIdx, state: GEPAState) -> float:
|
|
30
|
+
"""Return the score of the program on the valset"""
|
|
31
|
+
...
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class FullEvaluationPolicy(EvaluationPolicy[DataId, DataInst]):
|
|
35
|
+
"""Policy that evaluates all validation instances every time."""
|
|
36
|
+
|
|
37
|
+
def get_eval_batch(
|
|
38
|
+
self, loader: DataLoader[DataId, DataInst], state: GEPAState, target_program_idx: ProgramIdx | None = None
|
|
39
|
+
) -> list[DataId]:
|
|
40
|
+
"""Always return the full ordered list of validation ids."""
|
|
41
|
+
return list(loader.all_ids())
|
|
42
|
+
|
|
43
|
+
def get_best_program(self, state: GEPAState) -> ProgramIdx:
|
|
44
|
+
"""Pick the program whose evaluated validation scores achieve the highest average."""
|
|
45
|
+
best_idx, best_score, best_coverage = -1, float("-inf"), -1
|
|
46
|
+
for program_idx, scores in enumerate(state.prog_candidate_val_subscores):
|
|
47
|
+
coverage = len(scores)
|
|
48
|
+
avg = sum(scores.values()) / coverage if coverage else float("-inf")
|
|
49
|
+
if avg > best_score or (avg == best_score and coverage > best_coverage):
|
|
50
|
+
best_score = avg
|
|
51
|
+
best_idx = program_idx
|
|
52
|
+
best_coverage = coverage
|
|
53
|
+
return best_idx
|
|
54
|
+
|
|
55
|
+
def get_valset_score(self, program_idx: ProgramIdx, state: GEPAState) -> float:
|
|
56
|
+
"""Return the score of the program on the valset"""
|
|
57
|
+
return state.get_program_average_val_subset(program_idx)[0]
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
__all__ = [
|
|
61
|
+
"DataLoader",
|
|
62
|
+
"EvaluationPolicy",
|
|
63
|
+
"FullEvaluationPolicy",
|
|
64
|
+
]
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
# Copyright (c) 2025 Lakshya A Agrawal and the GEPA contributors
|
|
2
|
+
# https://github.com/gepa-ai/gepa
|
|
3
|
+
|
|
4
|
+
import re
|
|
5
|
+
from collections.abc import Mapping, Sequence
|
|
6
|
+
from typing import Any, ClassVar
|
|
7
|
+
|
|
8
|
+
from gepa.image import Image
|
|
9
|
+
from gepa.proposer.reflective_mutation.base import Signature
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class InstructionProposalSignature(Signature):
|
|
13
|
+
default_prompt_template = """I provided an assistant with the following instructions to perform a task for me:
|
|
14
|
+
```
|
|
15
|
+
<curr_param>
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
The following are examples of different task inputs provided to the assistant along with the assistant's response for each of them, and some feedback on how the assistant's response could be better:
|
|
19
|
+
```
|
|
20
|
+
<side_info>
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
Your task is to write a new instruction for the assistant.
|
|
24
|
+
|
|
25
|
+
Read the inputs carefully and identify the input format and infer detailed task description about the task I wish to solve with the assistant.
|
|
26
|
+
|
|
27
|
+
Read all the assistant responses and the corresponding feedback. Identify all niche and domain specific factual information about the task and include it in the instruction, as a lot of it may not be available to the assistant in the future. The assistant may have utilized a generalizable strategy to solve the task, if so, include that in the instruction as well.
|
|
28
|
+
|
|
29
|
+
Provide the new instructions within ``` blocks."""
|
|
30
|
+
|
|
31
|
+
input_keys: ClassVar[list[str]] = ["current_instruction_doc", "dataset_with_feedback", "prompt_template"]
|
|
32
|
+
output_keys: ClassVar[list[str]] = ["new_instruction"]
|
|
33
|
+
|
|
34
|
+
@classmethod
|
|
35
|
+
def validate_prompt_template(cls, prompt_template: str | None) -> None:
|
|
36
|
+
if prompt_template is None:
|
|
37
|
+
return
|
|
38
|
+
missing_placeholders = [
|
|
39
|
+
placeholder for placeholder in ("<curr_param>", "<side_info>") if placeholder not in prompt_template
|
|
40
|
+
]
|
|
41
|
+
if missing_placeholders:
|
|
42
|
+
raise ValueError(f"Missing placeholder(s) in prompt template: {', '.join(missing_placeholders)}")
|
|
43
|
+
|
|
44
|
+
@classmethod
|
|
45
|
+
def prompt_renderer(cls, input_dict: Mapping[str, Any]) -> str | list[dict[str, Any]]:
|
|
46
|
+
current_instruction = input_dict.get("current_instruction_doc")
|
|
47
|
+
if not isinstance(current_instruction, str):
|
|
48
|
+
raise TypeError("current_instruction_doc must be a string")
|
|
49
|
+
|
|
50
|
+
dataset = input_dict.get("dataset_with_feedback")
|
|
51
|
+
if not isinstance(dataset, Sequence) or isinstance(dataset, str | bytes):
|
|
52
|
+
raise TypeError("dataset_with_feedback must be a sequence of records")
|
|
53
|
+
|
|
54
|
+
def format_samples(samples: Sequence[Mapping[str, Any]]) -> tuple[str, list[Image]]:
|
|
55
|
+
"""Render samples as markdown, extracting any Image objects.
|
|
56
|
+
|
|
57
|
+
Returns:
|
|
58
|
+
A tuple of (formatted_text, collected_images). Image objects
|
|
59
|
+
are replaced with ``[IMAGE-N]`` placeholders in the text.
|
|
60
|
+
"""
|
|
61
|
+
collected_images: list[Image] = []
|
|
62
|
+
|
|
63
|
+
def render_value(value: Any, level: int = 3) -> str:
|
|
64
|
+
# level controls markdown header depth (###, ####, etc.)
|
|
65
|
+
if isinstance(value, Image):
|
|
66
|
+
collected_images.append(value)
|
|
67
|
+
return f"[IMAGE-{len(collected_images)} — see visual content]\n\n"
|
|
68
|
+
elif isinstance(value, dict):
|
|
69
|
+
s = ""
|
|
70
|
+
for k, v in value.items():
|
|
71
|
+
s += f"{'#' * level} {k}\n"
|
|
72
|
+
s += render_value(v, min(level + 1, 6))
|
|
73
|
+
if not value:
|
|
74
|
+
s += "\n"
|
|
75
|
+
return s
|
|
76
|
+
elif isinstance(value, list | tuple):
|
|
77
|
+
s = ""
|
|
78
|
+
for i, item in enumerate(value):
|
|
79
|
+
s += f"{'#' * level} Item {i + 1}\n"
|
|
80
|
+
s += render_value(item, min(level + 1, 6))
|
|
81
|
+
if not value:
|
|
82
|
+
s += "\n"
|
|
83
|
+
return s
|
|
84
|
+
else:
|
|
85
|
+
return f"{str(value).strip()}\n\n"
|
|
86
|
+
|
|
87
|
+
def convert_sample_to_markdown(sample: Mapping[str, Any], examplenum: int) -> str:
|
|
88
|
+
s = f"# Example {examplenum}\n"
|
|
89
|
+
for key, val in sample.items():
|
|
90
|
+
s += f"## {key}\n"
|
|
91
|
+
s += render_value(val, level=3)
|
|
92
|
+
return s
|
|
93
|
+
|
|
94
|
+
text = "\n\n".join(convert_sample_to_markdown(sample, i + 1) for i, sample in enumerate(samples))
|
|
95
|
+
return text, collected_images
|
|
96
|
+
|
|
97
|
+
prompt_template = input_dict.get("prompt_template")
|
|
98
|
+
if prompt_template is None:
|
|
99
|
+
prompt_template = cls.default_prompt_template
|
|
100
|
+
|
|
101
|
+
cls.validate_prompt_template(prompt_template)
|
|
102
|
+
|
|
103
|
+
formatted_text, images = format_samples(dataset)
|
|
104
|
+
|
|
105
|
+
if images:
|
|
106
|
+
formatted_text = (
|
|
107
|
+
f"The evaluation data below includes visual content ({len(images)} image(s)). "
|
|
108
|
+
"Analyze both the text and images when suggesting improvements.\n\n" + formatted_text
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
prompt = prompt_template.replace("<curr_param>", current_instruction)
|
|
112
|
+
prompt = prompt.replace("<side_info>", formatted_text)
|
|
113
|
+
|
|
114
|
+
# When images are present, return an OpenAI-compatible multimodal
|
|
115
|
+
# messages list so the reflection LM receives the images inline.
|
|
116
|
+
if images:
|
|
117
|
+
content: list[dict[str, Any]] = [{"type": "text", "text": prompt}]
|
|
118
|
+
for img in images:
|
|
119
|
+
content.append(img.to_openai_content_part())
|
|
120
|
+
return [{"role": "user", "content": content}]
|
|
121
|
+
|
|
122
|
+
return prompt
|
|
123
|
+
|
|
124
|
+
@classmethod
|
|
125
|
+
def output_extractor(cls, lm_out: str) -> dict[str, str]:
|
|
126
|
+
def extract_instruction_text() -> str:
|
|
127
|
+
# Find the first and last backtick positions (if any)
|
|
128
|
+
start = lm_out.find("```") + 3
|
|
129
|
+
end = lm_out.rfind("```")
|
|
130
|
+
|
|
131
|
+
# Handle if the first and last backticks are the same or overlap
|
|
132
|
+
if start >= end:
|
|
133
|
+
# Handle incomplete blocks
|
|
134
|
+
stripped = lm_out.strip()
|
|
135
|
+
if stripped.startswith("```"):
|
|
136
|
+
# Remove opening ``` and optional language specifier
|
|
137
|
+
match = re.match(r"^```\S*\n?", lm_out)
|
|
138
|
+
if match:
|
|
139
|
+
return lm_out[match.end() :].strip()
|
|
140
|
+
elif stripped.endswith("```"):
|
|
141
|
+
# Remove closing ```
|
|
142
|
+
return stripped[:-3].strip()
|
|
143
|
+
return stripped
|
|
144
|
+
|
|
145
|
+
# Skip optional language specifier
|
|
146
|
+
content = lm_out[start:end]
|
|
147
|
+
match = re.match(r"^\S*\n", content)
|
|
148
|
+
if match:
|
|
149
|
+
content = content[match.end() :]
|
|
150
|
+
|
|
151
|
+
return content.strip()
|
|
152
|
+
|
|
153
|
+
return {"new_instruction": extract_instruction_text()}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
# Copyright (c) 2025 Lakshya A Agrawal and the GEPA contributors
|
|
2
|
+
# https://github.com/gepa-ai/gepa
|
|
3
|
+
|
|
4
|
+
"""Sampling strategies for selecting (parent, minibatch) pairs each iteration."""
|
|
5
|
+
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from typing import Generic, Protocol
|
|
8
|
+
|
|
9
|
+
from gepa.core.data_loader import DataId, DataInst, DataLoader
|
|
10
|
+
from gepa.core.state import GEPAState
|
|
11
|
+
from gepa.proposer.reflective_mutation.base import CandidateSelector
|
|
12
|
+
from gepa.strategies.batch_sampler import BatchSampler
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass
|
|
16
|
+
class ProposalTask(Generic[DataId, DataInst]):
|
|
17
|
+
"""One (parent, minibatch) pair to propose from."""
|
|
18
|
+
|
|
19
|
+
parent_idx: int
|
|
20
|
+
parent_candidate: dict[str, str]
|
|
21
|
+
minibatch_ids: list[DataId]
|
|
22
|
+
minibatch: list[DataInst]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class SamplingStrategy(Protocol[DataId, DataInst]):
|
|
26
|
+
"""Generates one or more (parent, minibatch) tasks per iteration.
|
|
27
|
+
|
|
28
|
+
The default (``SingleMutationSampling``) produces exactly one task,
|
|
29
|
+
matching GEPA's original sequential behavior. Swap in a different
|
|
30
|
+
strategy to propose multiple candidates per iteration.
|
|
31
|
+
|
|
32
|
+
Note on trainset size: multi-task strategies draw ``n`` minibatches from
|
|
33
|
+
the epoch-shuffled sampler per iteration. When
|
|
34
|
+
``n * minibatch_size > len(trainset)`` the sampler wraps around within the
|
|
35
|
+
iteration, so some tasks will receive repeated minibatches. This is safe
|
|
36
|
+
but reduces the diversity benefit — prefer ``n <= len(trainset) //
|
|
37
|
+
minibatch_size``.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
def sample_tasks(
|
|
41
|
+
self,
|
|
42
|
+
state: GEPAState,
|
|
43
|
+
candidate_selector: CandidateSelector,
|
|
44
|
+
batch_sampler: BatchSampler[DataId, DataInst],
|
|
45
|
+
trainset: DataLoader[DataId, DataInst],
|
|
46
|
+
) -> list[ProposalTask[DataId, DataInst]]: ...
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class SingleMutationSampling(SamplingStrategy):
|
|
50
|
+
"""1 parent, 1 mutation — default GEPA behavior."""
|
|
51
|
+
|
|
52
|
+
def sample_tasks(
|
|
53
|
+
self,
|
|
54
|
+
state: GEPAState,
|
|
55
|
+
candidate_selector: CandidateSelector,
|
|
56
|
+
batch_sampler: BatchSampler,
|
|
57
|
+
trainset: DataLoader,
|
|
58
|
+
) -> list[ProposalTask]:
|
|
59
|
+
parent_idx = candidate_selector.select_candidate_idx(state)
|
|
60
|
+
mb_ids = batch_sampler.next_minibatch_ids(trainset, state)
|
|
61
|
+
return [ProposalTask(parent_idx, state.program_candidates[parent_idx], mb_ids, trainset.fetch(mb_ids))]
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class SameParentSampling(SamplingStrategy):
|
|
65
|
+
"""Best-of-N on the same parent with different minibatches."""
|
|
66
|
+
|
|
67
|
+
def __init__(self, n: int):
|
|
68
|
+
self.n = n
|
|
69
|
+
|
|
70
|
+
def sample_tasks(
|
|
71
|
+
self,
|
|
72
|
+
state: GEPAState,
|
|
73
|
+
candidate_selector: CandidateSelector,
|
|
74
|
+
batch_sampler: BatchSampler,
|
|
75
|
+
trainset: DataLoader,
|
|
76
|
+
) -> list[ProposalTask]:
|
|
77
|
+
parent_idx = candidate_selector.select_candidate_idx(state)
|
|
78
|
+
parent = state.program_candidates[parent_idx]
|
|
79
|
+
tasks = []
|
|
80
|
+
for _ in range(self.n):
|
|
81
|
+
mb_ids = batch_sampler.next_minibatch_ids(trainset, state)
|
|
82
|
+
tasks.append(ProposalTask(parent_idx, parent, mb_ids, trainset.fetch(mb_ids)))
|
|
83
|
+
return tasks
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class IndependentSampling(SamplingStrategy):
|
|
87
|
+
"""N different parents, each with their own minibatch."""
|
|
88
|
+
|
|
89
|
+
def __init__(self, n: int):
|
|
90
|
+
self.n = n
|
|
91
|
+
|
|
92
|
+
def sample_tasks(
|
|
93
|
+
self,
|
|
94
|
+
state: GEPAState,
|
|
95
|
+
candidate_selector: CandidateSelector,
|
|
96
|
+
batch_sampler: BatchSampler,
|
|
97
|
+
trainset: DataLoader,
|
|
98
|
+
) -> list[ProposalTask]:
|
|
99
|
+
tasks = []
|
|
100
|
+
for _ in range(self.n):
|
|
101
|
+
parent_idx = candidate_selector.select_candidate_idx(state)
|
|
102
|
+
mb_ids = batch_sampler.next_minibatch_ids(trainset, state)
|
|
103
|
+
tasks.append(ProposalTask(parent_idx, state.program_candidates[parent_idx], mb_ids, trainset.fetch(mb_ids)))
|
|
104
|
+
return tasks
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
class PxNSampling(SamplingStrategy):
|
|
108
|
+
"""P parents x N mutations each = P*N total tasks."""
|
|
109
|
+
|
|
110
|
+
def __init__(self, p: int, n: int):
|
|
111
|
+
self.p = p
|
|
112
|
+
self.n = n
|
|
113
|
+
|
|
114
|
+
def sample_tasks(
|
|
115
|
+
self,
|
|
116
|
+
state: GEPAState,
|
|
117
|
+
candidate_selector: CandidateSelector,
|
|
118
|
+
batch_sampler: BatchSampler,
|
|
119
|
+
trainset: DataLoader,
|
|
120
|
+
) -> list[ProposalTask]:
|
|
121
|
+
tasks = []
|
|
122
|
+
for _ in range(self.p):
|
|
123
|
+
parent_idx = candidate_selector.select_candidate_idx(state)
|
|
124
|
+
parent = state.program_candidates[parent_idx]
|
|
125
|
+
for _ in range(self.n):
|
|
126
|
+
mb_ids = batch_sampler.next_minibatch_ids(trainset, state)
|
|
127
|
+
tasks.append(ProposalTask(parent_idx, parent, mb_ids, trainset.fetch(mb_ids)))
|
|
128
|
+
return tasks
|