@warlock.js/ai 4.4.0 → 4.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +63 -52
- package/cjs/index.cjs +240 -14147
- package/cjs/src-Bmajk4Qg.cjs +30 -0
- package/cjs/src-OZyDYHxm.cjs +25089 -0
- package/cjs/src-OZyDYHxm.cjs.map +1 -0
- package/esm/agent/agent-config.type.d.mts +119 -1
- package/esm/agent/agent-config.type.d.mts.map +1 -1
- package/esm/agent/agent-input-builder.mjs +27 -6
- package/esm/agent/agent-input-builder.mjs.map +1 -1
- package/esm/agent/agent.d.mts +40 -1
- package/esm/agent/agent.d.mts.map +1 -1
- package/esm/agent/agent.mjs +328 -36
- package/esm/agent/agent.mjs.map +1 -1
- package/esm/agent/index.d.mts +2 -1
- package/esm/agent/index.mjs +1 -0
- package/esm/agent/judge-config.type.d.mts +33 -0
- package/esm/agent/judge-config.type.d.mts.map +1 -0
- package/esm/agent/judge-config.type.mjs +13 -0
- package/esm/agent/judge-config.type.mjs.map +1 -0
- package/esm/agent/signature.mjs +57 -0
- package/esm/agent/signature.mjs.map +1 -0
- package/esm/agent/snapshot.mjs +101 -0
- package/esm/agent/snapshot.mjs.map +1 -0
- package/esm/ai-openai/src/embedder.mjs +4 -0
- package/esm/ai-openai/src/image.mjs +5 -0
- package/esm/ai-openai/src/index.mjs +7 -0
- package/esm/ai-openai/src/model.mjs +5 -0
- package/esm/ai-openai/src/sdk.mjs +9 -0
- package/esm/ai-openai/src/speech.mjs +5 -0
- package/esm/ai-openai/src/transcription.mjs +6 -0
- package/esm/ai-openai/src/utils/index.mjs +5 -0
- package/esm/ai-openai/src/utils/to-openai-messages.mjs +3 -0
- package/esm/ai-openai/src/utils/to-openai-tools.mjs +3 -0
- package/esm/ai-openai/src/utils/wrap-openai-error.mjs +4 -0
- package/esm/ai.d.mts +164 -53
- package/esm/ai.d.mts.map +1 -1
- package/esm/ai.mjs +79 -8
- package/esm/ai.mjs.map +1 -1
- package/esm/batch/batch.d.mts.map +1 -1
- package/esm/batch/batch.mjs +21 -1
- package/esm/batch/batch.mjs.map +1 -1
- package/esm/batch/batch.type.d.mts +11 -4
- package/esm/batch/batch.type.d.mts.map +1 -1
- package/esm/config.d.mts +39 -3
- package/esm/config.d.mts.map +1 -1
- package/esm/config.mjs +26 -2
- package/esm/config.mjs.map +1 -1
- package/esm/contracts/agent/agent-options.type.d.mts +33 -5
- package/esm/contracts/agent/agent-options.type.d.mts.map +1 -1
- package/esm/contracts/agent/agent-snapshot.type.d.mts +90 -0
- package/esm/contracts/agent/agent-snapshot.type.d.mts.map +1 -0
- package/esm/contracts/agent/agent.contract.d.mts +29 -1
- package/esm/contracts/agent/agent.contract.d.mts.map +1 -1
- package/esm/contracts/agent/eval.type.d.mts +43 -2
- package/esm/contracts/agent/eval.type.d.mts.map +1 -1
- package/esm/contracts/agent/index.d.mts +2 -1
- package/esm/contracts/attachment-policy.type.d.mts +51 -0
- package/esm/contracts/attachment-policy.type.d.mts.map +1 -0
- package/esm/contracts/attachment.type.d.mts +16 -7
- package/esm/contracts/attachment.type.d.mts.map +1 -1
- package/esm/contracts/content-part.type.d.mts +14 -5
- package/esm/contracts/content-part.type.d.mts.map +1 -1
- package/esm/contracts/events/supervisor-events.type.d.mts.map +1 -1
- package/esm/contracts/image-model.contract.d.mts +156 -0
- package/esm/contracts/image-model.contract.d.mts.map +1 -0
- package/esm/contracts/index.d.mts +13 -6
- package/esm/contracts/model.contract.d.mts +11 -8
- package/esm/contracts/model.contract.d.mts.map +1 -1
- package/esm/contracts/orchestrator/index.d.mts +1 -0
- package/esm/contracts/orchestrator/orchestrator-config.type.d.mts +31 -2
- package/esm/contracts/orchestrator/orchestrator-config.type.d.mts.map +1 -1
- package/esm/contracts/orchestrator/session-lock.contract.d.mts +47 -0
- package/esm/contracts/orchestrator/session-lock.contract.d.mts.map +1 -0
- package/esm/contracts/planner/index.d.mts +3 -2
- package/esm/contracts/planner/planner-config.type.d.mts +61 -0
- package/esm/contracts/planner/planner-config.type.d.mts.map +1 -1
- package/esm/contracts/planner/planner-execute-options.type.d.mts +60 -1
- package/esm/contracts/planner/planner-execute-options.type.d.mts.map +1 -1
- package/esm/contracts/planner/planner-result.type.d.mts +8 -0
- package/esm/contracts/planner/planner-result.type.d.mts.map +1 -1
- package/esm/contracts/planner/planner-snapshot.type.d.mts +77 -0
- package/esm/contracts/planner/planner-snapshot.type.d.mts.map +1 -0
- package/esm/contracts/planner/planner.contract.d.mts +21 -1
- package/esm/contracts/planner/planner.contract.d.mts.map +1 -1
- package/esm/contracts/result/agent-result.type.d.mts +6 -4
- package/esm/contracts/result/agent-result.type.d.mts.map +1 -1
- package/esm/contracts/result/base-report.type.d.mts +21 -3
- package/esm/contracts/result/base-report.type.d.mts.map +1 -1
- package/esm/contracts/result/base-report.type.mjs.map +1 -1
- package/esm/contracts/result/execution-report.type.d.mts +53 -1
- package/esm/contracts/result/execution-report.type.d.mts.map +1 -1
- package/esm/contracts/result/supervisor-result.type.d.mts +14 -2
- package/esm/contracts/result/supervisor-result.type.d.mts.map +1 -1
- package/esm/contracts/result/workflow-result.type.d.mts +2 -1
- package/esm/contracts/result/workflow-result.type.d.mts.map +1 -1
- package/esm/contracts/sdk-adapter.contract.d.mts +37 -0
- package/esm/contracts/sdk-adapter.contract.d.mts.map +1 -1
- package/esm/contracts/speech-model.contract.d.mts +97 -0
- package/esm/contracts/speech-model.contract.d.mts.map +1 -0
- package/esm/contracts/supervisor/supervisor-config.type.d.mts +29 -0
- package/esm/contracts/supervisor/supervisor-config.type.d.mts.map +1 -1
- package/esm/contracts/system-prompt.contract.d.mts +103 -1
- package/esm/contracts/system-prompt.contract.d.mts.map +1 -1
- package/esm/contracts/team/index.d.mts +1 -0
- package/esm/contracts/team/team-config.type.d.mts +127 -0
- package/esm/contracts/team/team-config.type.d.mts.map +1 -0
- package/esm/contracts/tool.contract.d.mts +4 -2
- package/esm/contracts/tool.contract.d.mts.map +1 -1
- package/esm/contracts/transcription-model.contract.d.mts +101 -0
- package/esm/contracts/transcription-model.contract.d.mts.map +1 -0
- package/esm/contracts/workflow/step.contract.d.mts +30 -5
- package/esm/contracts/workflow/step.contract.d.mts.map +1 -1
- package/esm/contracts/workflow/workflow.contract.d.mts +16 -0
- package/esm/contracts/workflow/workflow.contract.d.mts.map +1 -1
- package/esm/errors/agent-drift-error.d.mts +32 -0
- package/esm/errors/agent-drift-error.d.mts.map +1 -0
- package/esm/errors/agent-drift-error.mjs +31 -0
- package/esm/errors/agent-drift-error.mjs.map +1 -0
- package/esm/errors/error-code.type.d.mts +1 -1
- package/esm/errors/index.d.mts +3 -0
- package/esm/errors/index.mjs +3 -0
- package/esm/errors/outbound-policy-error.d.mts +27 -0
- package/esm/errors/outbound-policy-error.d.mts.map +1 -0
- package/esm/errors/outbound-policy-error.mjs +32 -0
- package/esm/errors/outbound-policy-error.mjs.map +1 -0
- package/esm/errors/planner-drift-error.d.mts +34 -0
- package/esm/errors/planner-drift-error.d.mts.map +1 -0
- package/esm/errors/planner-drift-error.mjs +33 -0
- package/esm/errors/planner-drift-error.mjs.map +1 -0
- package/esm/eval/dataset.d.mts +28 -0
- package/esm/eval/dataset.d.mts.map +1 -0
- package/esm/eval/dataset.mjs +112 -0
- package/esm/eval/dataset.mjs.map +1 -0
- package/esm/eval/dataset.type.d.mts +53 -0
- package/esm/eval/dataset.type.d.mts.map +1 -0
- package/esm/eval/eval-runner.d.mts.map +1 -1
- package/esm/eval/eval-runner.mjs +16 -2
- package/esm/eval/eval-runner.mjs.map +1 -1
- package/esm/eval/index.d.mts +20 -1
- package/esm/eval/index.d.mts.map +1 -1
- package/esm/eval/index.mjs +20 -2
- package/esm/eval/index.mjs.map +1 -1
- package/esm/eval/regression.d.mts +30 -0
- package/esm/eval/regression.d.mts.map +1 -0
- package/esm/eval/regression.mjs +51 -0
- package/esm/eval/regression.mjs.map +1 -0
- package/esm/eval/report-json.d.mts +30 -0
- package/esm/eval/report-json.d.mts.map +1 -0
- package/esm/eval/report-json.mjs +33 -0
- package/esm/eval/report-json.mjs.map +1 -0
- package/esm/eval/report-junit.d.mts +22 -0
- package/esm/eval/report-junit.d.mts.map +1 -0
- package/esm/eval/report-junit.mjs +60 -0
- package/esm/eval/report-junit.mjs.map +1 -0
- package/esm/guard/contracts/guard-options.type.d.mts +165 -0
- package/esm/guard/contracts/guard-options.type.d.mts.map +1 -0
- package/esm/guard/contracts/guardrail.contract.d.mts +78 -0
- package/esm/guard/contracts/guardrail.contract.d.mts.map +1 -0
- package/esm/guard/contracts/index.d.mts +4 -0
- package/esm/guard/contracts/openai-client.contract.d.mts +51 -0
- package/esm/guard/contracts/openai-client.contract.d.mts.map +1 -0
- package/esm/guard/contracts/verdict.type.d.mts +76 -0
- package/esm/guard/contracts/verdict.type.d.mts.map +1 -0
- package/esm/guard/detectors/index.d.mts +4 -0
- package/esm/guard/detectors/index.mjs +6 -0
- package/esm/guard/detectors/injection.d.mts +34 -0
- package/esm/guard/detectors/injection.d.mts.map +1 -0
- package/esm/guard/detectors/injection.mjs +254 -0
- package/esm/guard/detectors/injection.mjs.map +1 -0
- package/esm/guard/detectors/moderation.d.mts +32 -0
- package/esm/guard/detectors/moderation.d.mts.map +1 -0
- package/esm/guard/detectors/moderation.mjs +134 -0
- package/esm/guard/detectors/moderation.mjs.map +1 -0
- package/esm/guard/detectors/pii.d.mts +41 -0
- package/esm/guard/detectors/pii.d.mts.map +1 -0
- package/esm/guard/detectors/pii.mjs +199 -0
- package/esm/guard/detectors/pii.mjs.map +1 -0
- package/esm/guard/detectors/topic.d.mts +29 -0
- package/esm/guard/detectors/topic.d.mts.map +1 -0
- package/esm/guard/detectors/topic.mjs +99 -0
- package/esm/guard/detectors/topic.mjs.map +1 -0
- package/esm/guard/errors.d.mts +12 -0
- package/esm/guard/errors.d.mts.map +1 -0
- package/esm/guard/errors.mjs +18 -0
- package/esm/guard/errors.mjs.map +1 -0
- package/esm/guard/guard.d.mts +67 -0
- package/esm/guard/guard.d.mts.map +1 -0
- package/esm/guard/guard.mjs +209 -0
- package/esm/guard/guard.mjs.map +1 -0
- package/esm/guard/guardrail.d.mts +39 -0
- package/esm/guard/guardrail.d.mts.map +1 -0
- package/esm/guard/guardrail.mjs +22 -0
- package/esm/guard/guardrail.mjs.map +1 -0
- package/esm/human/contracts/approval.type.d.mts +154 -0
- package/esm/human/contracts/approval.type.d.mts.map +1 -0
- package/esm/human/contracts/human-approval.type.d.mts +38 -0
- package/esm/human/contracts/human-approval.type.d.mts.map +1 -0
- package/esm/human/contracts/index.d.mts +4 -0
- package/esm/human/contracts/interrupt-store.contract.d.mts +112 -0
- package/esm/human/contracts/interrupt-store.contract.d.mts.map +1 -0
- package/esm/human/contracts/resume.type.d.mts +77 -0
- package/esm/human/contracts/resume.type.d.mts.map +1 -0
- package/esm/human/errors.d.mts +84 -0
- package/esm/human/errors.d.mts.map +1 -0
- package/esm/human/errors.mjs +60 -0
- package/esm/human/errors.mjs.map +1 -0
- package/esm/human/human-approval.d.mts +57 -0
- package/esm/human/human-approval.d.mts.map +1 -0
- package/esm/human/human-approval.mjs +170 -0
- package/esm/human/human-approval.mjs.map +1 -0
- package/esm/human/policy.d.mts +55 -0
- package/esm/human/policy.d.mts.map +1 -0
- package/esm/human/policy.mjs +67 -0
- package/esm/human/policy.mjs.map +1 -0
- package/esm/human/register.mjs +37 -0
- package/esm/human/register.mjs.map +1 -0
- package/esm/human/resume-seed.mjs +53 -0
- package/esm/human/resume-seed.mjs.map +1 -0
- package/esm/human/resume.d.mts +54 -0
- package/esm/human/resume.d.mts.map +1 -0
- package/esm/human/resume.mjs +121 -0
- package/esm/human/resume.mjs.map +1 -0
- package/esm/human/stores/index.mjs +5 -0
- package/esm/human/stores/memory.d.mts +30 -0
- package/esm/human/stores/memory.d.mts.map +1 -0
- package/esm/human/stores/memory.mjs +91 -0
- package/esm/human/stores/memory.mjs.map +1 -0
- package/esm/human/stores/pg.d.mts +59 -0
- package/esm/human/stores/pg.d.mts.map +1 -0
- package/esm/human/stores/pg.mjs +220 -0
- package/esm/human/stores/pg.mjs.map +1 -0
- package/esm/human/stores/redis.d.mts +56 -0
- package/esm/human/stores/redis.d.mts.map +1 -0
- package/esm/human/stores/redis.mjs +201 -0
- package/esm/human/stores/redis.mjs.map +1 -0
- package/esm/image/image-cost.d.mts +32 -0
- package/esm/image/image-cost.d.mts.map +1 -0
- package/esm/image/image-cost.mjs +55 -0
- package/esm/image/image-cost.mjs.map +1 -0
- package/esm/image/image.d.mts +92 -0
- package/esm/image/image.d.mts.map +1 -0
- package/esm/image/image.mjs +113 -0
- package/esm/image/image.mjs.map +1 -0
- package/esm/image/index.mjs +4 -0
- package/esm/index.d.mts +137 -24
- package/esm/index.mjs +101 -11
- package/esm/middleware/builtins/budget.mjs +6 -2
- package/esm/middleware/builtins/budget.mjs.map +1 -1
- package/esm/middleware/utils/extract-user-text.d.mts +8 -1
- package/esm/middleware/utils/extract-user-text.d.mts.map +1 -1
- package/esm/middleware/utils/extract-user-text.mjs +8 -1
- package/esm/middleware/utils/extract-user-text.mjs.map +1 -1
- package/esm/mock/index.d.mts +3 -0
- package/esm/mock/index.mjs +3 -0
- package/esm/mock/mock-config.type.d.mts +22 -0
- package/esm/mock/mock-config.type.d.mts.map +1 -1
- package/esm/mock/mock-image-model.d.mts +41 -0
- package/esm/mock/mock-image-model.d.mts.map +1 -0
- package/esm/mock/mock-image-model.mjs +52 -0
- package/esm/mock/mock-image-model.mjs.map +1 -0
- package/esm/mock/mock-sdk.d.mts +7 -1
- package/esm/mock/mock-sdk.d.mts.map +1 -1
- package/esm/mock/mock-sdk.mjs +27 -0
- package/esm/mock/mock-sdk.mjs.map +1 -1
- package/esm/mock/mock-speech-model.d.mts +31 -0
- package/esm/mock/mock-speech-model.d.mts.map +1 -0
- package/esm/mock/mock-speech-model.mjs +39 -0
- package/esm/mock/mock-speech-model.mjs.map +1 -0
- package/esm/mock/mock-transcription-model.d.mts +32 -0
- package/esm/mock/mock-transcription-model.d.mts.map +1 -0
- package/esm/mock/mock-transcription-model.mjs +36 -0
- package/esm/mock/mock-transcription-model.mjs.map +1 -0
- package/esm/object-stream/index.d.mts +2 -0
- package/esm/object-stream/index.mjs +4 -0
- package/esm/object-stream/parse-partial-json.d.mts +22 -0
- package/esm/object-stream/parse-partial-json.d.mts.map +1 -0
- package/esm/object-stream/parse-partial-json.mjs +78 -0
- package/esm/object-stream/parse-partial-json.mjs.map +1 -0
- package/esm/object-stream/stream-object.d.mts +68 -0
- package/esm/object-stream/stream-object.d.mts.map +1 -0
- package/esm/object-stream/stream-object.mjs +104 -0
- package/esm/object-stream/stream-object.mjs.map +1 -0
- package/esm/observe/index.mjs +4 -0
- package/esm/observe/observer-registry.d.mts +30 -0
- package/esm/observe/observer-registry.d.mts.map +1 -0
- package/esm/observe/observer-registry.mjs +51 -0
- package/esm/observe/observer-registry.mjs.map +1 -0
- package/esm/observe/observer.contract.d.mts +40 -0
- package/esm/observe/observer.contract.d.mts.map +1 -0
- package/esm/observe/resolve-observers.d.mts +40 -0
- package/esm/observe/resolve-observers.d.mts.map +1 -0
- package/esm/observe/resolve-observers.mjs +73 -0
- package/esm/observe/resolve-observers.mjs.map +1 -0
- package/esm/orchestrator/execution.d.mts.map +1 -1
- package/esm/orchestrator/execution.mjs +5 -2
- package/esm/orchestrator/execution.mjs.map +1 -1
- package/esm/orchestrator/index.d.mts +1 -0
- package/esm/orchestrator/index.mjs +1 -0
- package/esm/orchestrator/orchestrator.d.mts.map +1 -1
- package/esm/orchestrator/orchestrator.mjs +39 -6
- package/esm/orchestrator/orchestrator.mjs.map +1 -1
- package/esm/orchestrator/session-lock.d.mts +25 -0
- package/esm/orchestrator/session-lock.d.mts.map +1 -0
- package/esm/orchestrator/session-lock.mjs +83 -0
- package/esm/orchestrator/session-lock.mjs.map +1 -0
- package/esm/planner/dag-scheduler.mjs +97 -0
- package/esm/planner/dag-scheduler.mjs.map +1 -0
- package/esm/planner/plan-prompt.d.mts +1 -1
- package/esm/planner/plan-prompt.d.mts.map +1 -1
- package/esm/planner/plan-prompt.mjs +2 -1
- package/esm/planner/plan-prompt.mjs.map +1 -1
- package/esm/planner/planner-run.d.mts +8 -0
- package/esm/planner/planner-run.d.mts.map +1 -1
- package/esm/planner/planner-run.mjs +457 -31
- package/esm/planner/planner-run.mjs.map +1 -1
- package/esm/planner/planner.d.mts.map +1 -1
- package/esm/planner/planner.mjs +26 -2
- package/esm/planner/planner.mjs.map +1 -1
- package/esm/planner/snapshot.mjs +95 -0
- package/esm/planner/snapshot.mjs.map +1 -0
- package/esm/prompt/errors.d.mts +57 -0
- package/esm/prompt/errors.d.mts.map +1 -0
- package/esm/prompt/errors.mjs +73 -0
- package/esm/prompt/errors.mjs.map +1 -0
- package/esm/prompt/index.d.mts +3 -0
- package/esm/prompt/index.mjs +4 -0
- package/esm/prompt/prompt-langfuse-sync.mjs +104 -0
- package/esm/prompt/prompt-langfuse-sync.mjs.map +1 -0
- package/esm/prompt/prompt-langfuse-sync.type.d.mts +32 -0
- package/esm/prompt/prompt-langfuse-sync.type.d.mts.map +1 -0
- package/esm/prompt/prompt-validate.mjs +170 -0
- package/esm/prompt/prompt-validate.mjs.map +1 -0
- package/esm/prompt/prompt.d.mts +54 -0
- package/esm/prompt/prompt.d.mts.map +1 -0
- package/esm/prompt/prompt.mjs +218 -0
- package/esm/prompt/prompt.mjs.map +1 -0
- package/esm/prompt/prompt.type.d.mts +174 -0
- package/esm/prompt/prompt.type.d.mts.map +1 -0
- package/esm/prompts/index.d.mts +3 -0
- package/esm/prompts/index.mjs +3 -0
- package/esm/prompts/prompts-manager.contract.d.mts +154 -0
- package/esm/prompts/prompts-manager.contract.d.mts.map +1 -0
- package/esm/prompts/prompts-manager.d.mts +38 -0
- package/esm/prompts/prompts-manager.d.mts.map +1 -0
- package/esm/prompts/prompts-manager.mjs +410 -0
- package/esm/prompts/prompts-manager.mjs.map +1 -0
- package/esm/prompts/prompts-manager.type.d.mts +172 -0
- package/esm/prompts/prompts-manager.type.d.mts.map +1 -0
- package/esm/prompts/prompts-validate.mjs +200 -0
- package/esm/prompts/prompts-validate.mjs.map +1 -0
- package/esm/rag/as-tool.mjs +48 -0
- package/esm/rag/as-tool.mjs.map +1 -0
- package/esm/rag/chunk/chunk.d.mts +24 -0
- package/esm/rag/chunk/chunk.d.mts.map +1 -0
- package/esm/rag/chunk/chunk.mjs +44 -0
- package/esm/rag/chunk/chunk.mjs.map +1 -0
- package/esm/rag/chunk/fixed.mjs +32 -0
- package/esm/rag/chunk/fixed.mjs.map +1 -0
- package/esm/rag/chunk/markdown.mjs +75 -0
- package/esm/rag/chunk/markdown.mjs.map +1 -0
- package/esm/rag/chunk/recursive.mjs +132 -0
- package/esm/rag/chunk/recursive.mjs.map +1 -0
- package/esm/rag/chunk/sentence.mjs +73 -0
- package/esm/rag/chunk/sentence.mjs.map +1 -0
- package/esm/rag/contracts/chunk-options.type.d.mts +35 -0
- package/esm/rag/contracts/chunk-options.type.d.mts.map +1 -0
- package/esm/rag/contracts/citation.type.d.mts +35 -0
- package/esm/rag/contracts/citation.type.d.mts.map +1 -0
- package/esm/rag/contracts/index.d.mts +4 -0
- package/esm/rag/contracts/rag-config.type.d.mts +68 -0
- package/esm/rag/contracts/rag-config.type.d.mts.map +1 -0
- package/esm/rag/contracts/rag-document.type.d.mts +21 -0
- package/esm/rag/contracts/rag-document.type.d.mts.map +1 -0
- package/esm/rag/hybrid/bm25.d.mts +23 -0
- package/esm/rag/hybrid/bm25.d.mts.map +1 -0
- package/esm/rag/hybrid/bm25.mjs +51 -0
- package/esm/rag/hybrid/bm25.mjs.map +1 -0
- package/esm/rag/hybrid/hybrid-rank.d.mts +33 -0
- package/esm/rag/hybrid/hybrid-rank.d.mts.map +1 -0
- package/esm/rag/hybrid/hybrid-rank.mjs +29 -0
- package/esm/rag/hybrid/hybrid-rank.mjs.map +1 -0
- package/esm/rag/hybrid/rrf.d.mts +25 -0
- package/esm/rag/hybrid/rrf.d.mts.map +1 -0
- package/esm/rag/hybrid/rrf.mjs +30 -0
- package/esm/rag/hybrid/rrf.mjs.map +1 -0
- package/esm/rag/index.d.mts +22 -0
- package/esm/rag/index.mjs +18 -0
- package/esm/rag/loaders/errors.d.mts +19 -0
- package/esm/rag/loaders/errors.d.mts.map +1 -0
- package/esm/rag/loaders/errors.mjs +25 -0
- package/esm/rag/loaders/errors.mjs.map +1 -0
- package/esm/rag/loaders/index.mjs +7 -0
- package/esm/rag/loaders/load-html.d.mts +26 -0
- package/esm/rag/loaders/load-html.d.mts.map +1 -0
- package/esm/rag/loaders/load-html.mjs +138 -0
- package/esm/rag/loaders/load-html.mjs.map +1 -0
- package/esm/rag/loaders/load-pdf.d.mts +38 -0
- package/esm/rag/loaders/load-pdf.d.mts.map +1 -0
- package/esm/rag/loaders/load-pdf.mjs +150 -0
- package/esm/rag/loaders/load-pdf.mjs.map +1 -0
- package/esm/rag/loaders/load-text.d.mts +47 -0
- package/esm/rag/loaders/load-text.d.mts.map +1 -0
- package/esm/rag/loaders/load-text.mjs +60 -0
- package/esm/rag/loaders/load-text.mjs.map +1 -0
- package/esm/rag/loaders/load-web.d.mts +42 -0
- package/esm/rag/loaders/load-web.d.mts.map +1 -0
- package/esm/rag/loaders/load-web.mjs +89 -0
- package/esm/rag/loaders/load-web.mjs.map +1 -0
- package/esm/rag/loaders/loader.type.d.mts +89 -0
- package/esm/rag/loaders/loader.type.d.mts.map +1 -0
- package/esm/rag/rag.d.mts +38 -0
- package/esm/rag/rag.d.mts.map +1 -0
- package/esm/rag/rag.mjs +126 -0
- package/esm/rag/rag.mjs.map +1 -0
- package/esm/rag/rerank/keyword-reranker.d.mts +32 -0
- package/esm/rag/rerank/keyword-reranker.d.mts.map +1 -0
- package/esm/rag/rerank/keyword-reranker.mjs +58 -0
- package/esm/rag/rerank/keyword-reranker.mjs.map +1 -0
- package/esm/rag/rerank/llm-reranker.d.mts +36 -0
- package/esm/rag/rerank/llm-reranker.d.mts.map +1 -0
- package/esm/rag/rerank/llm-reranker.mjs +85 -0
- package/esm/rag/rerank/llm-reranker.mjs.map +1 -0
- package/esm/rag/rerank/reranker.contract.d.mts +28 -0
- package/esm/rag/rerank/reranker.contract.d.mts.map +1 -0
- package/esm/rag/retrieve.mjs +68 -0
- package/esm/rag/retrieve.mjs.map +1 -0
- package/esm/rag/store/cache-vector-store.d.mts +27 -0
- package/esm/rag/store/cache-vector-store.d.mts.map +1 -0
- package/esm/rag/store/cache-vector-store.mjs +48 -0
- package/esm/rag/store/cache-vector-store.mjs.map +1 -0
- package/esm/rag/store/pg-vector-store.d.mts +139 -0
- package/esm/rag/store/pg-vector-store.d.mts.map +1 -0
- package/esm/rag/store/pg-vector-store.mjs +328 -0
- package/esm/rag/store/pg-vector-store.mjs.map +1 -0
- package/esm/rag/store/vector-store.contract.d.mts +38 -0
- package/esm/rag/store/vector-store.contract.d.mts.map +1 -0
- package/esm/rag/transforms/multi-query.d.mts +27 -0
- package/esm/rag/transforms/multi-query.d.mts.map +1 -0
- package/esm/rag/transforms/multi-query.mjs +41 -0
- package/esm/rag/transforms/multi-query.mjs.map +1 -0
- package/esm/security/index.mjs +5 -0
- package/esm/security/outbound-policy.d.mts +46 -0
- package/esm/security/outbound-policy.d.mts.map +1 -0
- package/esm/security/outbound-policy.mjs +187 -0
- package/esm/security/outbound-policy.mjs.map +1 -0
- package/esm/security/outbound-policy.type.d.mts +74 -0
- package/esm/security/outbound-policy.type.d.mts.map +1 -0
- package/esm/security/private-ip.d.mts +15 -0
- package/esm/security/private-ip.d.mts.map +1 -0
- package/esm/security/private-ip.mjs +48 -0
- package/esm/security/private-ip.mjs.map +1 -0
- package/esm/security/redact.d.mts +59 -0
- package/esm/security/redact.d.mts.map +1 -0
- package/esm/security/redact.mjs +122 -0
- package/esm/security/redact.mjs.map +1 -0
- package/esm/serve/serve.d.mts +50 -0
- package/esm/serve/serve.d.mts.map +1 -0
- package/esm/serve/serve.mjs +90 -0
- package/esm/serve/serve.mjs.map +1 -0
- package/esm/serve/sse.d.mts +20 -0
- package/esm/serve/sse.d.mts.map +1 -0
- package/esm/serve/sse.mjs +25 -0
- package/esm/serve/sse.mjs.map +1 -0
- package/esm/serve/stream-to-sse.d.mts +29 -0
- package/esm/serve/stream-to-sse.d.mts.map +1 -0
- package/esm/serve/stream-to-sse.mjs +37 -0
- package/esm/serve/stream-to-sse.mjs.map +1 -0
- package/esm/skills/catalog.d.mts +49 -0
- package/esm/skills/catalog.d.mts.map +1 -0
- package/esm/skills/catalog.mjs +140 -0
- package/esm/skills/catalog.mjs.map +1 -0
- package/esm/skills/contracts/skill-record.type.d.mts +37 -0
- package/esm/skills/contracts/skill-record.type.d.mts.map +1 -0
- package/esm/skills/contracts/skills-config.type.d.mts +108 -0
- package/esm/skills/contracts/skills-config.type.d.mts.map +1 -0
- package/esm/skills/contracts/skills-store.contract.d.mts +28 -0
- package/esm/skills/contracts/skills-store.contract.d.mts.map +1 -0
- package/esm/skills/contracts/skills.contract.d.mts +43 -0
- package/esm/skills/contracts/skills.contract.d.mts.map +1 -0
- package/esm/skills/index.d.mts +16 -0
- package/esm/skills/index.mjs +14 -0
- package/esm/skills/load-skill-tool.d.mts +38 -0
- package/esm/skills/load-skill-tool.d.mts.map +1 -0
- package/esm/skills/load-skill-tool.mjs +65 -0
- package/esm/skills/load-skill-tool.mjs.map +1 -0
- package/esm/skills/review-gate.d.mts +33 -0
- package/esm/skills/review-gate.d.mts.map +1 -0
- package/esm/skills/review-gate.mjs +60 -0
- package/esm/skills/review-gate.mjs.map +1 -0
- package/esm/skills/save-skill-tool.d.mts +39 -0
- package/esm/skills/save-skill-tool.d.mts.map +1 -0
- package/esm/skills/save-skill-tool.mjs +65 -0
- package/esm/skills/save-skill-tool.mjs.map +1 -0
- package/esm/skills/skills.d.mts +33 -0
- package/esm/skills/skills.d.mts.map +1 -0
- package/esm/skills/skills.mjs +109 -0
- package/esm/skills/skills.mjs.map +1 -0
- package/esm/skills/sources/directory-source.d.mts +19 -0
- package/esm/skills/sources/directory-source.d.mts.map +1 -0
- package/esm/skills/sources/directory-source.mjs +108 -0
- package/esm/skills/sources/directory-source.mjs.map +1 -0
- package/esm/skills/sources/index.d.mts +18 -0
- package/esm/skills/sources/index.d.mts.map +1 -0
- package/esm/skills/sources/index.mjs +27 -0
- package/esm/skills/sources/index.mjs.map +1 -0
- package/esm/skills/sources/parse-frontmatter.d.mts +27 -0
- package/esm/skills/sources/parse-frontmatter.d.mts.map +1 -0
- package/esm/skills/sources/parse-frontmatter.mjs +46 -0
- package/esm/skills/sources/parse-frontmatter.mjs.map +1 -0
- package/esm/skills/sources/store-source.d.mts +14 -0
- package/esm/skills/sources/store-source.d.mts.map +1 -0
- package/esm/skills/sources/store-source.mjs +15 -0
- package/esm/skills/sources/store-source.mjs.map +1 -0
- package/esm/skills/sources/url-source.d.mts +29 -0
- package/esm/skills/sources/url-source.d.mts.map +1 -0
- package/esm/skills/sources/url-source.mjs +117 -0
- package/esm/skills/sources/url-source.mjs.map +1 -0
- package/esm/skills/store/mock-skills-store.d.mts +57 -0
- package/esm/skills/store/mock-skills-store.d.mts.map +1 -0
- package/esm/skills/store/mock-skills-store.mjs +100 -0
- package/esm/skills/store/mock-skills-store.mjs.map +1 -0
- package/esm/skills/store/procedural-skill-store.d.mts +30 -0
- package/esm/skills/store/procedural-skill-store.d.mts.map +1 -0
- package/esm/skills/store/procedural-skill-store.mjs +125 -0
- package/esm/skills/store/procedural-skill-store.mjs.map +1 -0
- package/esm/speech/index.mjs +3 -0
- package/esm/speech/speech.d.mts +65 -0
- package/esm/speech/speech.d.mts.map +1 -0
- package/esm/speech/speech.mjs +123 -0
- package/esm/speech/speech.mjs.map +1 -0
- package/esm/supervisor/as-tool.mjs +2 -2
- package/esm/supervisor/as-tool.mjs.map +1 -1
- package/esm/supervisor/entries.mjs +2 -2
- package/esm/supervisor/entries.mjs.map +1 -1
- package/esm/supervisor/execution.d.mts.map +1 -1
- package/esm/supervisor/execution.mjs +31 -28
- package/esm/supervisor/execution.mjs.map +1 -1
- package/esm/supervisor/supervisor.d.mts.map +1 -1
- package/esm/supervisor/supervisor.mjs +8 -3
- package/esm/supervisor/supervisor.mjs.map +1 -1
- package/esm/system-prompt/index.d.mts +4 -0
- package/esm/system-prompt/system-prompt.d.mts +68 -4
- package/esm/system-prompt/system-prompt.d.mts.map +1 -1
- package/esm/system-prompt/system-prompt.mjs +89 -5
- package/esm/system-prompt/system-prompt.mjs.map +1 -1
- package/esm/team/gates.mjs +48 -0
- package/esm/team/gates.mjs.map +1 -0
- package/esm/team/index.d.mts +1 -0
- package/esm/team/index.mjs +3 -0
- package/esm/team/team.d.mts +42 -0
- package/esm/team/team.d.mts.map +1 -0
- package/esm/team/team.mjs +94 -0
- package/esm/team/team.mjs.map +1 -0
- package/esm/tool/executable-as-tool.d.mts.map +1 -1
- package/esm/tool/executable-as-tool.mjs +2 -2
- package/esm/tool/executable-as-tool.mjs.map +1 -1
- package/esm/tool/tool.d.mts.map +1 -1
- package/esm/tool/tool.mjs +2 -2
- package/esm/tool/tool.mjs.map +1 -1
- package/esm/transcribe/audio-input.d.mts +47 -0
- package/esm/transcribe/audio-input.d.mts.map +1 -0
- package/esm/transcribe/audio-input.mjs +84 -0
- package/esm/transcribe/audio-input.mjs.map +1 -0
- package/esm/transcribe/index.mjs +4 -0
- package/esm/transcribe/transcribe.d.mts +64 -0
- package/esm/transcribe/transcribe.d.mts.map +1 -0
- package/esm/transcribe/transcribe.mjs +128 -0
- package/esm/transcribe/transcribe.mjs.map +1 -0
- package/esm/utils/compute-cost.d.mts +17 -1
- package/esm/utils/compute-cost.d.mts.map +1 -1
- package/esm/utils/compute-cost.mjs +26 -1
- package/esm/utils/compute-cost.mjs.map +1 -1
- package/esm/utils/extract-json-lenient.d.mts +42 -0
- package/esm/utils/extract-json-lenient.d.mts.map +1 -0
- package/esm/utils/extract-json-lenient.mjs +97 -0
- package/esm/utils/extract-json-lenient.mjs.map +1 -0
- package/esm/utils/index.d.mts +4 -2
- package/esm/utils/index.mjs +3 -1
- package/esm/utils/json-schema.d.mts +1 -1
- package/esm/utils/prepare-attachment-part.d.mts +10 -1
- package/esm/utils/prepare-attachment-part.d.mts.map +1 -1
- package/esm/utils/prepare-attachment-part.mjs +103 -11
- package/esm/utils/prepare-attachment-part.mjs.map +1 -1
- package/esm/utils/resolve-attachment.d.mts +4 -3
- package/esm/utils/resolve-attachment.d.mts.map +1 -1
- package/esm/utils/resolve-attachment.mjs +4 -3
- package/esm/utils/resolve-attachment.mjs.map +1 -1
- package/esm/utils/run-context.d.mts +94 -0
- package/esm/utils/run-context.d.mts.map +1 -0
- package/esm/utils/run-context.mjs +98 -0
- package/esm/utils/run-context.mjs.map +1 -0
- package/esm/vcr/cassette-io.mjs +57 -0
- package/esm/vcr/cassette-io.mjs.map +1 -0
- package/esm/vcr/errors.d.mts +42 -0
- package/esm/vcr/errors.d.mts.map +1 -0
- package/esm/vcr/errors.mjs +37 -0
- package/esm/vcr/errors.mjs.map +1 -0
- package/esm/vcr/hash-request.d.mts +28 -0
- package/esm/vcr/hash-request.d.mts.map +1 -0
- package/esm/vcr/hash-request.mjs +118 -0
- package/esm/vcr/hash-request.mjs.map +1 -0
- package/esm/vcr/index.d.mts +4 -0
- package/esm/vcr/index.mjs +5 -0
- package/esm/vcr/vcr.d.mts +32 -0
- package/esm/vcr/vcr.d.mts.map +1 -0
- package/esm/vcr/vcr.mjs +248 -0
- package/esm/vcr/vcr.mjs.map +1 -0
- package/esm/vcr/vcr.type.d.mts +118 -0
- package/esm/vcr/vcr.type.d.mts.map +1 -0
- package/esm/workflow/as-tool.mjs +2 -2
- package/esm/workflow/as-tool.mjs.map +1 -1
- package/esm/workflow/engine.mjs +1 -0
- package/esm/workflow/engine.mjs.map +1 -1
- package/esm/workflow/step-runner.mjs +19 -20
- package/esm/workflow/step-runner.mjs.map +1 -1
- package/esm/workflow/workflow.d.mts.map +1 -1
- package/esm/workflow/workflow.mjs +8 -3
- package/esm/workflow/workflow.mjs.map +1 -1
- package/llms-full.txt +2572 -204
- package/llms.txt +21 -4
- package/package.json +7 -3
- package/skills/README.md +44 -4
- package/skills/ai-dx-helpers/SKILL.md +2 -2
- package/skills/approve-tool-calls/SKILL.md +134 -0
- package/skills/attach-ai-middleware/SKILL.md +1 -1
- package/skills/detect-and-redact-pii/SKILL.md +104 -0
- package/skills/durable-agent-runs/SKILL.md +135 -0
- package/skills/durable-resume/SKILL.md +128 -0
- package/skills/escalate-block-to-human/SKILL.md +85 -0
- package/skills/eval-datasets-and-ci/SKILL.md +117 -0
- package/skills/generate-images/SKILL.md +138 -0
- package/skills/generate-speech/SKILL.md +139 -0
- package/skills/guard-input-output/SKILL.md +117 -0
- package/skills/manage-prompts/SKILL.md +186 -0
- package/skills/observe-ai-flows/SKILL.md +94 -0
- package/skills/rag-loaders-and-stores/SKILL.md +164 -0
- package/skills/record-replay-llm/SKILL.md +92 -0
- package/skills/run-ai-agent/SKILL.md +26 -1
- package/skills/run-ai-rag/SKILL.md +139 -0
- package/skills/run-ai-team/SKILL.md +107 -0
- package/skills/run-orchestrator/SKILL.md +2 -0
- package/skills/run-planner/SKILL.md +73 -8
- package/skills/run-supervisor/SKILL.md +20 -1
- package/skills/transcribe-audio/SKILL.md +157 -0
- package/skills/use-runtime-skills/SKILL.md +106 -0
- package/skills/write-system-prompt/SKILL.md +30 -1
- package/cjs/index.cjs.map +0 -1
- package/esm/contracts/result/index.d.mts +0 -15
package/llms-full.txt
CHANGED
|
@@ -28,7 +28,7 @@ ai.orchestrator() → durable session — state/history/resume [shipped]
|
|
|
28
28
|
|
|
29
29
|
Each primitive is an escape hatch to the next level of complexity. Users start low, graduate upward only when needed. Every primitive returns the same result envelope — canonical destructure `{ data, error, usage, report }` (the shared `BaseResult` guarantees `usage` + optional `error`; each primitive adds `data` + `report`). Workflows, supervisors, and orchestrators expose `.asTool()` so an agent can call them inside its tool loop; raw executables also auto-adapt when dropped into an agent's `tools: []`. Compose freely.
|
|
30
30
|
|
|
31
|
-
Beyond the ladder: `ai.planner()` (LLM-generated plans), `ai.memory()` (working + semantic
|
|
31
|
+
Beyond the ladder: `ai.planner()` (LLM-generated plans), `ai.memory()` (working + semantic recall), `ai.batch()` / `ai.fallbackModel()` / `ai.router()` / `ai.fanOut()` (DX helpers), `agent.eval()` (scoring), and the `ai.checkpoint.*` / `ai.snapshot.*` orchestrator stores.
|
|
32
32
|
|
|
33
33
|
## Foundations
|
|
34
34
|
|
|
@@ -66,14 +66,14 @@ console.log(text, usage.total, report.duration);
|
|
|
66
66
|
|
|
67
67
|
| If the task is about… | Load |
|
|
68
68
|
| --- | --- |
|
|
69
|
-
| `ai.agent({...})` — single-LLM-turn primitive, structured output, streaming, attachments | [`@warlock.js/ai/run-ai-agent/SKILL.md`](@warlock.js/ai/run-ai-agent/SKILL.md) |
|
|
69
|
+
| `ai.agent({...})` — single-LLM-turn primitive, structured output, streaming, attachments, `spawnSubAgent` | [`@warlock.js/ai/run-ai-agent/SKILL.md`](@warlock.js/ai/run-ai-agent/SKILL.md) |
|
|
70
70
|
| `ai.tool({...})` — typed validated functions the model can call | [`@warlock.js/ai/define-ai-tool/SKILL.md`](@warlock.js/ai/define-ai-tool/SKILL.md) |
|
|
71
71
|
| `ai.systemPrompt()` / `ai.persona()` / `ai.instruction()` — composable prompts with placeholders | [`@warlock.js/ai/write-system-prompt/SKILL.md`](@warlock.js/ai/write-system-prompt/SKILL.md) |
|
|
72
72
|
| `ai.workflow({...})` — durable resumable pipelines with steps, routing, retry | [`@warlock.js/ai/run-ai-workflow/SKILL.md`](@warlock.js/ai/run-ai-workflow/SKILL.md) |
|
|
73
73
|
| `ai.supervisor({...})` — multi-intent routing, fan-out, evaluate loops | [`@warlock.js/ai/run-supervisor/SKILL.md`](@warlock.js/ai/run-supervisor/SKILL.md) |
|
|
74
74
|
| `ai.orchestrator({...})` — durable stateful sessions, drift, compaction, resume | [`@warlock.js/ai/run-orchestrator/SKILL.md`](@warlock.js/ai/run-orchestrator/SKILL.md) |
|
|
75
75
|
| `ai.planner({...})` — LLM-generated plans over registered capabilities | [`@warlock.js/ai/run-planner/SKILL.md`](@warlock.js/ai/run-planner/SKILL.md) |
|
|
76
|
-
| `ai.memory({...})` — working + semantic
|
|
76
|
+
| `ai.memory({...})` — working + semantic recall for agents / sessions | [`@warlock.js/ai/use-ai-memory/SKILL.md`](@warlock.js/ai/use-ai-memory/SKILL.md) |
|
|
77
77
|
| `ai.checkpoint.*` / `ai.snapshot.*` — orchestrator session + run stores | [`@warlock.js/ai/manage-ai-stores/SKILL.md`](@warlock.js/ai/manage-ai-stores/SKILL.md) |
|
|
78
78
|
| DX helpers — `batch` / `fallbackModel` / `eval` + matchers / SLO contracts / `fromFile` | [`@warlock.js/ai/ai-dx-helpers/SKILL.md`](@warlock.js/ai/ai-dx-helpers/SKILL.md) |
|
|
79
79
|
| `sdk.embedder({...})` — text-to-vector for RAG tools, vector ingest | [`@warlock.js/ai/embed-text/SKILL.md`](@warlock.js/ai/embed-text/SKILL.md) |
|
|
@@ -156,8 +156,8 @@ A drop-in `ModelContract` that wraps an ordered list and advances to the next mo
|
|
|
156
156
|
|
|
157
157
|
```ts
|
|
158
158
|
const model = ai.fallbackModel([
|
|
159
|
-
|
|
160
|
-
|
|
159
|
+
openai.model({ name: "gpt-4o" }),
|
|
160
|
+
anthropic.model({ name: "claude-3-5-sonnet" }),
|
|
161
161
|
]);
|
|
162
162
|
|
|
163
163
|
const agent = ai.agent({ model }); // hand it anywhere a model goes
|
|
@@ -298,6 +298,144 @@ const concierge = ai.agent({
|
|
|
298
298
|
- [`@warlock.js/ai/pick-ai-provider/SKILL.md`](@warlock.js/ai/pick-ai-provider/SKILL.md) — `fallbackModel` wraps these adapters; cost-truth tokens
|
|
299
299
|
|
|
300
300
|
|
|
301
|
+
## approve-tool-calls `@warlock.js/ai/approve-tool-calls/SKILL.md`
|
|
302
|
+
|
|
303
|
+
---
|
|
304
|
+
name: approve-tool-calls
|
|
305
|
+
description: 'Gate an agent''s tool calls behind a human with `ai.human.approval(options)` (the `tool.before` approval-gate middleware) — ships in @warlock.js/ai core. Triggers: `ai.human.approval`, `humanApproval`, `HumanApprovalOptions`, `ApprovalRequest`, `ApprovalDecision`, `ApprovalHandler`, `InterruptPolicy`, `evaluatePolicy`, `ApprovalRejectedError`, `policy: { type: "allowlist" | "denylist" | "predicate" }`, decision `{ type: "approve" | "reject" | "edit" }`; ''human in the loop'', ''approve a tool call before it runs'', ''ask a human before the agent sends/charges/deletes'', ''pause before a dangerous tool'', ''let an operator edit the tool args'', ''reject a tool call with a reason the model can self-correct from''. Typical import `import { ai } from "@warlock.js/ai"`. Skip: persisting the request and resuming hours later out-of-process — `@warlock.js/ai/durable-resume/SKILL.md`; the agent/middleware/tool primitives themselves — `@warlock.js/ai`.'
|
|
306
|
+
---
|
|
307
|
+
|
|
308
|
+
# Approve tool calls — the human-in-the-loop gate
|
|
309
|
+
|
|
310
|
+
`ai.human.approval(options)` returns an `AgentMiddleware` with **one** hook — `tool.before` — that pauses *before a specific tool call* and routes it to a human. The human can **approve** (run the real tool unchanged), **reject** (the model sees a typed error and self-corrects), or **edit** (run the tool with replaced args). Every call the policy doesn't gate passes through untouched.
|
|
311
|
+
|
|
312
|
+
`ai.human.*` ships natively on the shared `ai` object from `@warlock.js/ai` core — no extra import or registration step. The named `humanApproval` export is the same factory.
|
|
313
|
+
|
|
314
|
+
```ts
|
|
315
|
+
import { ai } from "@warlock.js/ai";
|
|
316
|
+
import { OpenAISDK } from "@warlock.js/ai-openai";
|
|
317
|
+
|
|
318
|
+
const openai = new OpenAISDK({ apiKey: process.env.OPENAI_API_KEY! });
|
|
319
|
+
|
|
320
|
+
const support = ai.agent({
|
|
321
|
+
model: openai.model({ name: "gpt-4o" }),
|
|
322
|
+
tools: [refundCustomer, lookupOrder],
|
|
323
|
+
middleware: [
|
|
324
|
+
ai.human.approval({
|
|
325
|
+
policy: { type: "allowlist", tools: ["refundCustomer"], tags: () => ["money"] },
|
|
326
|
+
// SSE / CLI handler resolves when the operator rules:
|
|
327
|
+
handler: async (req) => ui.prompt(req), // → { type: "approve" } | { type: "reject", reason } | { type: "edit", args }
|
|
328
|
+
}),
|
|
329
|
+
],
|
|
330
|
+
});
|
|
331
|
+
|
|
332
|
+
await support.execute("Refund order #4821"); // pauses at refundCustomer, awaits the operator
|
|
333
|
+
```
|
|
334
|
+
|
|
335
|
+
## The three decisions
|
|
336
|
+
|
|
337
|
+
A handler turns an `ApprovalRequest` into an `ApprovalDecision` — a union discriminated by `type` (never `kind`):
|
|
338
|
+
|
|
339
|
+
| `decision.type` | Effect | What the model sees next |
|
|
340
|
+
|---|---|---|
|
|
341
|
+
| `"approve"` | The real tool runs with the model's original args. | The tool's normal result. |
|
|
342
|
+
| `"reject"` | Short-circuits an `ApprovalRejectedError` carrying `reason`. | `{ error }` on the next trip — it can self-correct. |
|
|
343
|
+
| `"edit"` | The reviewer's `args` replace the model's; the real tool then runs. | The tool's result for the edited args. |
|
|
344
|
+
|
|
345
|
+
```ts
|
|
346
|
+
type ApprovalDecision =
|
|
347
|
+
| { type: "approve" }
|
|
348
|
+
| { type: "reject"; reason: string }
|
|
349
|
+
| { type: "edit"; args: unknown; reason?: string };
|
|
350
|
+
```
|
|
351
|
+
|
|
352
|
+
An `edit` still goes through the tool's own Standard-Schema validation — if the replaced args don't fit the schema, the tool surfaces a validation error on `result.error` and the model self-corrects. No special-casing.
|
|
353
|
+
|
|
354
|
+
## The interrupt policy — which calls need a human
|
|
355
|
+
|
|
356
|
+
`policy` decides *which* tool calls are gated. It's a union keyed on `type`:
|
|
357
|
+
|
|
358
|
+
| `policy.type` | Gates a call when… | Tags |
|
|
359
|
+
|---|---|---|
|
|
360
|
+
| `"allowlist"` | the tool name **is** in `tools`. | optional `tags(toolName)` callback |
|
|
361
|
+
| `"denylist"` | the tool name is **not** in `tools` (gate everything else). | optional `tags(toolName)` callback |
|
|
362
|
+
| `"predicate"` | `requiresApproval(ctx)` returns a truthy value. | a returned `string[]` doubles as the tags |
|
|
363
|
+
|
|
364
|
+
```ts
|
|
365
|
+
// Allowlist — only refunds need sign-off:
|
|
366
|
+
{ type: "allowlist", tools: ["refundCustomer"], tags: () => ["money"] }
|
|
367
|
+
|
|
368
|
+
// Denylist — everything except read-only lookups needs sign-off:
|
|
369
|
+
{ type: "denylist", tools: ["lookupOrder", "searchCatalog"] }
|
|
370
|
+
|
|
371
|
+
// Predicate — args-aware: only large refunds, tagged for the reviewer UI:
|
|
372
|
+
{
|
|
373
|
+
type: "predicate",
|
|
374
|
+
requiresApproval: (ctx) =>
|
|
375
|
+
ctx.toolName === "refundCustomer" && (ctx.args as { amount: number }).amount > 100
|
|
376
|
+
? ["money", "high-value"]
|
|
377
|
+
: false,
|
|
378
|
+
}
|
|
379
|
+
```
|
|
380
|
+
|
|
381
|
+
The predicate sees a read-only `PolicyContext` — `toolName`, `toolDescription`, `args` (the model's exact input), `agentName`, `tripIndex`, `sessionId`. Return `false` (or an empty array) to skip approval; `true` or a non-empty `string[]` to require it. The `string[]` becomes `request.context.tags`, surfaced verbatim to the reviewer so a UI can group or prioritize.
|
|
382
|
+
|
|
383
|
+
Compose with `forTool(names, mw)` from `@warlock.js/ai` for static, name-based scoping and let `policy` be the dynamic, args-aware layer on top.
|
|
384
|
+
|
|
385
|
+
`evaluatePolicy(policy, context)` is the exported, pure core if you want to reuse the gate decision outside the middleware (it never throws, does no IO, and returns `{ requiresApproval, tags? }`).
|
|
386
|
+
|
|
387
|
+
## The request a reviewer rules on
|
|
388
|
+
|
|
389
|
+
For a gated call the middleware builds an `ApprovalRequest` and hands it to your `handler`:
|
|
390
|
+
|
|
391
|
+
```ts
|
|
392
|
+
interface ApprovalRequest {
|
|
393
|
+
interruptId: string; // stable id; durable mode keys the store on it
|
|
394
|
+
toolName: string;
|
|
395
|
+
toolDescription?: string;
|
|
396
|
+
args: unknown; // the model's exact args
|
|
397
|
+
context: {
|
|
398
|
+
agentName: string;
|
|
399
|
+
tripIndex: number;
|
|
400
|
+
sessionId?: string;
|
|
401
|
+
originalInput?: string; // the run's prompt (used by durable re-run)
|
|
402
|
+
tags?: string[]; // from the policy match
|
|
403
|
+
};
|
|
404
|
+
requestedAt: string; // ISO-8601
|
|
405
|
+
}
|
|
406
|
+
```
|
|
407
|
+
|
|
408
|
+
The handler runs in one of two modes that share this one signature:
|
|
409
|
+
|
|
410
|
+
- **interactive** — return the decision (or a promise of it); the hook `await`s it in-process. The whole agent run stays on the stack — no store needed. This skill.
|
|
411
|
+
- **durable** — persist the request and `throw` to suspend, resuming from another process later. See [`durable-resume/SKILL.md`](@warlock.js/ai/durable-resume/SKILL.md).
|
|
412
|
+
|
|
413
|
+
## It never throws out of the pipeline
|
|
414
|
+
|
|
415
|
+
The middleware is a harness, not a detector — every outcome (skip, approve, reject, edit) returns normally. A `reject` does **not** throw out of `execute()`: it short-circuits a failed `ToolInvokeResult` carrying an `ApprovalRejectedError`, so the error rides `result.error` like every other `AIError` and `agent.execute()` still never throws.
|
|
416
|
+
|
|
417
|
+
```ts
|
|
418
|
+
const result = await support.execute("Refund order #4821");
|
|
419
|
+
|
|
420
|
+
if (result.error instanceof ApprovalRejectedError) {
|
|
421
|
+
logAudit(`${result.error.toolName} rejected: ${result.error.reason}`);
|
|
422
|
+
}
|
|
423
|
+
```
|
|
424
|
+
|
|
425
|
+
Only a *handler bug* — a non-sentinel throw from your handler — propagates, and even then the agent dispatch funnels it onto `result.error` rather than crashing the run. The gate never swallows a bug into a silent approval.
|
|
426
|
+
|
|
427
|
+
## Edge cases
|
|
428
|
+
|
|
429
|
+
- **Duplicate middleware name.** Middleware names are validated unique per agent. The default name is `"human-approval"`, so two approval middlewares on one agent need distinct `name`s.
|
|
430
|
+
- **Silent tools.** A `silent`-mode tool's result isn't fed back to the model, but approval still runs (we gate *before* dispatch). A `reject` on a silent tool writes a tool message that's harmless but unread.
|
|
431
|
+
- **Abort during an interactive await.** `ctx.signal` is in scope; honor it in a long-running handler so a cancelled run rejects rather than hanging.
|
|
432
|
+
|
|
433
|
+
## See also
|
|
434
|
+
|
|
435
|
+
- [`@warlock.js/ai/durable-resume/SKILL.md`](@warlock.js/ai/durable-resume/SKILL.md) — persist the request, resume out-of-process hours later via `ai.human.resume(...)` and the `InterruptStore`.
|
|
436
|
+
- `@warlock.js/ai` — the `ai.agent(...)`, `AgentMiddleware`, `tool.before`, and `ToolInvokeResult` primitives this gate wraps.
|
|
437
|
+
|
|
438
|
+
|
|
301
439
|
## attach-ai-middleware `@warlock.js/ai/attach-ai-middleware/SKILL.md`
|
|
302
440
|
|
|
303
441
|
---
|
|
@@ -307,7 +445,7 @@ description: 'Wire agent middleware — ai.middleware.budget (token / USD caps +
|
|
|
307
445
|
|
|
308
446
|
# Middleware — agent-level pipeline
|
|
309
447
|
|
|
310
|
-
Cross-cutting concerns wrapped around an agent run at three granularities
|
|
448
|
+
Cross-cutting concerns wrapped around an agent run at three agent-level granularities (`execute`, `trip`, `tool`), plus a `supervisor` level that wraps a whole supervisor run (see below). One middleware = one object. Ships with `budget`, `guardrail`, and `semanticCache` built-ins.
|
|
311
449
|
|
|
312
450
|
## Install order at a glance
|
|
313
451
|
|
|
@@ -705,6 +843,385 @@ Workflow errors surface as `ToolExecutionError` with `cause` pointing at the ori
|
|
|
705
843
|
- [`@warlock.js/ai/run-ai-workflow/SKILL.md`](@warlock.js/ai/run-ai-workflow/SKILL.md) — `workflow.asTool()` composition
|
|
706
844
|
|
|
707
845
|
|
|
846
|
+
## detect-and-redact-pii `@warlock.js/ai/detect-and-redact-pii/SKILL.md`
|
|
847
|
+
|
|
848
|
+
---
|
|
849
|
+
name: detect-and-redact-pii
|
|
850
|
+
description: 'Detect and redact PII (and run model-graded moderation) with @warlock.js/ai-guard detectors — `ai.guardrail.pii(...)` and the optional `ai.guardrail.moderation(...)` peer. Triggers: `ai.guardrail.pii`, `piiDetector`, `PiiDetectorOptions`, `PiiCategory`, `mask`, `{label}`, `dictionary`, `onMatch`, `ai.guardrail.moderation`, `openAiModeration`, `OpenAiModerationOptions`, `blockOn`, `omni-moderation-latest`; ''redact PII from model output'', ''mask SSN / credit card / email / phone / IP'', ''stop PII leaking into a tool call'', ''scrub sensitive data'', ''add OpenAI moderation'', ''block violent / self-harm content''; typical import `import "@warlock.js/ai-guard"` (registers `ai.guardrail.pii` / `.moderation`) or `import { pii, moderation } from "@warlock.js/ai-guard"`. Skip: composing the guard / wiring it into an agent — `@warlock.js/ai-guard/guard-input-output/SKILL.md`; routing a block to a human — `@warlock.js/ai-guard/escalate-block-to-human/SKILL.md`.'
|
|
851
|
+
---
|
|
852
|
+
|
|
853
|
+
# Detect and redact PII (and moderate)
|
|
854
|
+
|
|
855
|
+
`ai.guardrail.pii(...)` is a **zero-dependency** detector — regex + exact-string matching, no runtime peer. Pass it into any phase array of `ai.guardrail({ ... })`.
|
|
856
|
+
|
|
857
|
+
```ts
|
|
858
|
+
import { ai } from "@warlock.js/ai";
|
|
859
|
+
import "@warlock.js/ai-guard";
|
|
860
|
+
|
|
861
|
+
const policy = ai.guardrail({
|
|
862
|
+
output: [ai.guardrail.pii({ onMatch: "redact", mask: "[REDACTED:{label}]" })],
|
|
863
|
+
});
|
|
864
|
+
```
|
|
865
|
+
|
|
866
|
+
## Categories — `detect`
|
|
867
|
+
|
|
868
|
+
Scans for these `PiiCategory` values; `detect` narrows the set (default: all). Each pattern is **linear** (anchored, no nested quantifiers) — safe against catastrophic backtracking.
|
|
869
|
+
|
|
870
|
+
| Category | Matches |
|
|
871
|
+
|---|---|
|
|
872
|
+
| `ssn` | US Social Security numbers |
|
|
873
|
+
| `email` | email addresses |
|
|
874
|
+
| `phone` | phone numbers |
|
|
875
|
+
| `credit-card` | credit-card numbers |
|
|
876
|
+
| `ipv4` | IPv4 addresses |
|
|
877
|
+
|
|
878
|
+
```ts
|
|
879
|
+
ai.guardrail.pii({ detect: ["ssn", "credit-card"] }); // scan only these two
|
|
880
|
+
```
|
|
881
|
+
|
|
882
|
+
Add `dictionary` for extra exact-string terms (internal codenames, customer IDs) treated as PII alongside the built-in regexes:
|
|
883
|
+
|
|
884
|
+
```ts
|
|
885
|
+
ai.guardrail.pii({ dictionary: ["PROJECT-ORION", "ACME-INTERNAL"] });
|
|
886
|
+
```
|
|
887
|
+
|
|
888
|
+
## Action — `onMatch`
|
|
889
|
+
|
|
890
|
+
`onMatch` is `"redact" | "block" | "flag"`, default **`"redact"`**:
|
|
891
|
+
|
|
892
|
+
- **`redact`** — replace each match with the `mask` and continue (output phase only — see below).
|
|
893
|
+
- **`block`** — reject the trip / tool call with a `GuardrailViolationError`.
|
|
894
|
+
- **`flag`** — allow but record the matches into `ctx.state` for a downstream observer.
|
|
895
|
+
|
|
896
|
+
## The `mask` template
|
|
897
|
+
|
|
898
|
+
On `redact`, each match is replaced by `mask`. The `{label}` token is substituted with the matched category, so a redacted SSN becomes `[REDACTED:ssn]`:
|
|
899
|
+
|
|
900
|
+
```ts
|
|
901
|
+
ai.guardrail.pii({ onMatch: "redact", mask: "[REDACTED:{label}]" });
|
|
902
|
+
// "My SSN is 123-45-6789" -> "My SSN is [REDACTED:ssn]"
|
|
903
|
+
```
|
|
904
|
+
|
|
905
|
+
Omit `mask` to use the default fixed placeholder.
|
|
906
|
+
|
|
907
|
+
## Where redaction actually applies
|
|
908
|
+
|
|
909
|
+
Redaction only rewrites-and-continues where the pipeline seam supports it:
|
|
910
|
+
|
|
911
|
+
- **Output (`output: [...]`)** — works. `trip.after` returns a replacement `ModelResponse` with the scrubbed `content`. This is the primary PII-redaction use case.
|
|
912
|
+
- **Input (`input: [...]`)** — a `redact` verdict **downgrades to `block`**. The core `trip.before` hook can only short-circuit, not rewrite-and-continue, so the un-redacted prompt can't be threaded back.
|
|
913
|
+
- **Tool (`tool: [...]`)** — a `redact` verdict **downgrades to `block`** (`reason: "tool-arg-redaction-unsupported"`), because silently rewriting tool arguments changes the call's side-effects.
|
|
914
|
+
|
|
915
|
+
So: **redact on output, block on input/tool.**
|
|
916
|
+
|
|
917
|
+
```ts
|
|
918
|
+
const policy = ai.guardrail({
|
|
919
|
+
output: [ai.guardrail.pii({ onMatch: "redact", mask: "[REDACTED:{label}]" })], // scrub the answer
|
|
920
|
+
tool: [ai.guardrail.pii({ onMatch: "block" })], // refuse to leak into tools
|
|
921
|
+
toolNames: ["send_email"],
|
|
922
|
+
});
|
|
923
|
+
```
|
|
924
|
+
|
|
925
|
+
## Optional moderation peer — `ai.guardrail.moderation`
|
|
926
|
+
|
|
927
|
+
For model-graded content (violence, self-harm, hate) beyond regex, the optional `moderation` detector calls OpenAI's moderation endpoint. The `openai` SDK is an **optional lazy peer** — importing `@warlock.js/ai-guard` never forces it to resolve; the detector throws a curated install string on first `check()` when the peer is absent (mirrors ai-panoptic's lazy Langfuse exporter).
|
|
928
|
+
|
|
929
|
+
```ts
|
|
930
|
+
const policy = ai.guardrail({
|
|
931
|
+
output: [
|
|
932
|
+
ai.guardrail.moderation({ blockOn: ["violence", "self-harm"] }),
|
|
933
|
+
],
|
|
934
|
+
});
|
|
935
|
+
```
|
|
936
|
+
|
|
937
|
+
- `blockOn` — categories that escalate to `block`; every other flagged category produces a `flag`. Omit to `flag` on any category.
|
|
938
|
+
- `model` — defaults to `"omni-moderation-latest"`.
|
|
939
|
+
- `apiKey` — defaults to `OPENAI_API_KEY`.
|
|
940
|
+
- `client` — pass a pre-built OpenAI-compatible client to bypass the lazy import entirely (the bring-your-own-client / test escape hatch).
|
|
941
|
+
|
|
942
|
+
Install the peer only when you use this detector:
|
|
943
|
+
|
|
944
|
+
```bash
|
|
945
|
+
npm install openai
|
|
946
|
+
```
|
|
947
|
+
|
|
948
|
+
## See also
|
|
949
|
+
|
|
950
|
+
- [`@warlock.js/ai-guard/guard-input-output/SKILL.md`](@warlock.js/ai-guard/guard-input-output/SKILL.md) — composing the guard, the verdict model, phases, and `toolNames` scoping.
|
|
951
|
+
- [`@warlock.js/ai-guard/escalate-block-to-human/SKILL.md`](@warlock.js/ai-guard/escalate-block-to-human/SKILL.md) — escalating a hard `block` to a human-review surface.
|
|
952
|
+
|
|
953
|
+
|
|
954
|
+
## durable-agent-runs `@warlock.js/ai/durable-agent-runs/SKILL.md`
|
|
955
|
+
|
|
956
|
+
---
|
|
957
|
+
name: durable-agent-runs
|
|
958
|
+
description: 'Mid-run crash-resume for agents AND planners — opt in with durable: { store, deleteOnComplete? } on the config, pass a stable runId to execute(), and call agent.resume(runId) / planner.resume(runId) after a crash to continue from the last settled trip / plan node. Reuses the ai.snapshot.{memory,pg,redis} stores; checkpoints per-trip (agent) / per-node (planner); completed trips + nodes never re-run their tools and usage is never double-counted; a drifted definition throws AgentDriftError / PlannerDriftError (bypass with { force: true }). Triggers: `durable`, `agent.resume`, `planner.resume`, `resume(runId)`, `runId`, `AgentSnapshot`, `PlannerSnapshot`, `AgentSnapshotStatus`, `PlannerSnapshotStatus`, `AgentDriftError`, `PlannerDriftError`, `computeAgentSignature`, `agent.signature`, `deleteOnComplete`, `defaultSnapshotStore`, `ai.snapshot.pg`, `ai.snapshot.memory`, `SnapshotStore`, `force: true`; ''resume an agent after a crash'', ''durable agent run'', ''continue a planner from where it crashed'', ''checkpoint agent state'', ''idempotent tool re-run on resume'', ''signature drift on resume''; typical import `import { ai } from "@warlock.js/ai"`. Skip: durable human-in-the-loop approval resume (ai.human.resume of a PendingInterrupt) — `@warlock.js/ai/durable-resume/SKILL.md`; supervisor/workflow iterate-mid-turn snapshot resume + the store contracts themselves — `@warlock.js/ai/manage-ai-stores/SKILL.md`; competing libs `temporal`, `inngest`, `restate`.'
|
|
959
|
+
---
|
|
960
|
+
|
|
961
|
+
# Durable agent + planner runs — resume from the last checkpoint
|
|
962
|
+
|
|
963
|
+
Opt-in mid-run crash-resume for the two long-running primitives. Turn it on, give the run a stable `runId`, and after a process crash `resume(runId)` re-hydrates the persisted state and continues from where it stopped — never re-issuing a settled trip's model call or re-invoking a completed node's capability.
|
|
964
|
+
|
|
965
|
+
> **Not the same as [[durable-resume]].** That skill is `ai.human.resume(interruptId, decision)` — resuming a **gated tool call** hours later after a human rules (a `PendingInterrupt` in an `InterruptStore`). *This* skill is **crash-resume of an in-flight run** (an `AgentSnapshot` / `PlannerSnapshot` in a `SnapshotStore`): the process died mid-run, you restart, and continue the same trip / plan. Different trigger (a crash, not a human), different store, different verb (`agent.resume` / `planner.resume`, not `ai.human.resume`).
|
|
966
|
+
|
|
967
|
+
## Opt in — `durable` on the config
|
|
968
|
+
|
|
969
|
+
```ts
|
|
970
|
+
import { ai } from "@warlock.js/ai";
|
|
971
|
+
|
|
972
|
+
const writer = ai.agent({
|
|
973
|
+
name: "writer",
|
|
974
|
+
model,
|
|
975
|
+
tools: [searchTool, draftTool],
|
|
976
|
+
durable: {
|
|
977
|
+
store: ai.snapshot.pg({ client: pgPool }), // reuses the ai.snapshot.* stores
|
|
978
|
+
deleteOnComplete: false, // default — keep for the completed-run short-circuit + audit
|
|
979
|
+
},
|
|
980
|
+
});
|
|
981
|
+
```
|
|
982
|
+
|
|
983
|
+
`durable` shape (identical on the agent and planner config):
|
|
984
|
+
|
|
985
|
+
- **`store?`** — a `SnapshotStore`. Falls back to `ai.config({ defaultSnapshotStore })`. When neither resolves, snapshot writes **silently skip** and `resume()` throws.
|
|
986
|
+
- **`deleteOnComplete?`** — drop the snapshot once the run completes successfully. Default `false`.
|
|
987
|
+
|
|
988
|
+
**Absent `durable` ⇒ zero behavior change** — the loop starts at trip 0 / the first node, never writes a snapshot, and runs byte-for-byte as before.
|
|
989
|
+
|
|
990
|
+
## Run with a stable `runId`, then resume
|
|
991
|
+
|
|
992
|
+
The `runId` is the store key. Pass a stable one to `execute()` (or read the generated one off `result.report.runId`) so a later `resume()` can find the snapshot:
|
|
993
|
+
|
|
994
|
+
```ts
|
|
995
|
+
const result = await writer.execute("research X", { runId: "run-42" });
|
|
996
|
+
|
|
997
|
+
// ...process crashes mid-run, restarts...
|
|
998
|
+
|
|
999
|
+
const recovered = await writer.resume("run-42");
|
|
1000
|
+
// continues from the next unsettled trip; `recovered.report.status === "completed"`
|
|
1001
|
+
```
|
|
1002
|
+
|
|
1003
|
+
Planners are the mirror image — `durable` on the config, `runId` on `execute(goal)`, `planner.resume(runId)`:
|
|
1004
|
+
|
|
1005
|
+
```ts
|
|
1006
|
+
const research = ai.planner({
|
|
1007
|
+
name: "research-assistant",
|
|
1008
|
+
model,
|
|
1009
|
+
capabilities: [{ name: "search", executable: searchAgent }, { name: "write", executable: writerAgent }],
|
|
1010
|
+
durable: { store: ai.snapshot.pg({ client: pgPool }) },
|
|
1011
|
+
});
|
|
1012
|
+
|
|
1013
|
+
const first = await research.execute("compare A vs B", { runId: "plan-7" });
|
|
1014
|
+
// ...crash...
|
|
1015
|
+
const done = await research.resume("plan-7");
|
|
1016
|
+
```
|
|
1017
|
+
|
|
1018
|
+
## Checkpoint granularity
|
|
1019
|
+
|
|
1020
|
+
| Primitive | Written | Contains | Resume continues at |
|
|
1021
|
+
|---|---|---|---|
|
|
1022
|
+
| **agent** | after every settled **trip** (`runTrip` end) | `messages`, `trips`, `toolCalls`, `usage`, resolved `systemPrompt` / `responseSchema`, `signature`, `status` | `trips.length` (the next trip index) |
|
|
1023
|
+
| **planner** | after every settled **plan node** (`executeStep` end) | the frozen `plan`, `executedSteps` ledger, `usage`, child `children` reports, `replanCount`, `signature`, `status` | the unfinished frontier (from `executedSteps`) |
|
|
1024
|
+
|
|
1025
|
+
The write happens only where the persisted arrays are mutually consistent — for the agent, after every tool a trip requested has been dispatched and its result appended. A crash **mid-trip** loses only that in-flight trip (never checkpointed), which the resume re-issues cleanly. The planner **never re-calls the planning LLM** on resume — the plan is frozen on the first run; re-asking would burn tokens and risk a plan that no longer matches the ledger. Every field on both snapshots is JSON-serializable, so they round-trip through any `ai.snapshot.{memory,pg,redis}` backend verbatim.
|
|
1026
|
+
|
|
1027
|
+
## Idempotency — what does and doesn't re-run
|
|
1028
|
+
|
|
1029
|
+
```ts
|
|
1030
|
+
// Completed run: resume is a no-op that re-returns the stored result.
|
|
1031
|
+
const again = await writer.resume("run-42"); // runs nothing when status === "completed"
|
|
1032
|
+
```
|
|
1033
|
+
|
|
1034
|
+
- **Completed trips / nodes never re-run their tools.** On agent resume, `trips.length` is the starting trip index — earlier trips' model calls are not replayed and their tool dispatches are not re-invoked. On planner resume, a completed node's capability dispatch is skipped (the sequential skip-guard / DAG re-seed derive the completed set from `executedSteps`).
|
|
1035
|
+
- **Usage is never double-counted.** The running `usage` total is restored from the snapshot; only the newly-executed trips / nodes add to it.
|
|
1036
|
+
- **Caveat — a crash MID-trip re-runs that trip's tools.** The in-flight trip was never checkpointed, so on resume its tools fire again. **Side-effectful tools (charging a card, sending an email) must be idempotent** — the same caller-responsibility boundary the supervisor and workflow primitives document. Guard them with your own dedupe key (e.g. `${runId}:${toolCallId}`).
|
|
1037
|
+
|
|
1038
|
+
## Drift — definition changed since the snapshot
|
|
1039
|
+
|
|
1040
|
+
Every agent / planner carries a structural `signature` (`agent.signature` — computed at factory time by `computeAgentSignature`), stamped on each snapshot. `resume()` compares the stored signature against the current definition; a mismatch throws before executing anything:
|
|
1041
|
+
|
|
1042
|
+
- **agent** covers: model name + provider, sorted tool names, `maxTrips`, whether a default `output` schema is set, `version`. It does **not** cover system-prompt text, middleware, per-event handlers, placeholders, or `modelOptions` — runtime knobs that don't change a resumable run's shape.
|
|
1043
|
+
- **planner** covers: name + ordered capability names. A mid-run **re-plan is NOT drift** (the plan changed, not the definition); `replanCount` is persisted so the replan budget survives a resume.
|
|
1044
|
+
|
|
1045
|
+
```ts
|
|
1046
|
+
import { AgentDriftError } from "@warlock.js/ai";
|
|
1047
|
+
|
|
1048
|
+
try {
|
|
1049
|
+
await writer.resume("run-42");
|
|
1050
|
+
} catch (error) {
|
|
1051
|
+
if (error instanceof AgentDriftError) {
|
|
1052
|
+
// The definition changed (a tool was added, the model swapped). Either roll the
|
|
1053
|
+
// definition back, or — only when you've verified the change is snapshot-safe:
|
|
1054
|
+
await writer.resume("run-42", { force: true }); // bypasses the drift check
|
|
1055
|
+
}
|
|
1056
|
+
}
|
|
1057
|
+
```
|
|
1058
|
+
|
|
1059
|
+
`{ force: true }` is the escape hatch (mirror `PlannerDriftError` for planners). `resume()` also throws `AgentExecutionError` / `PlannerFailedError` when no store is configured or no snapshot exists for the `runId`.
|
|
1060
|
+
|
|
1061
|
+
## Pattern — a boot-drain resume loop
|
|
1062
|
+
|
|
1063
|
+
On restart, resume every run the store still has in flight. Snapshots carry a `status` (`"running" | "completed" | "cancelled" | "failed"`), so you only resume the live ones:
|
|
1064
|
+
|
|
1065
|
+
```ts
|
|
1066
|
+
const store = ai.snapshot.pg({ client: pgPool });
|
|
1067
|
+
const runIds = (await store.list?.()) ?? [];
|
|
1068
|
+
|
|
1069
|
+
for (const runId of runIds) {
|
|
1070
|
+
const snapshot = await store.load(runId);
|
|
1071
|
+
if (snapshot?.status === "running") {
|
|
1072
|
+
await writer.resume(runId); // completed/failed snapshots short-circuit or re-throw — skip them
|
|
1073
|
+
}
|
|
1074
|
+
}
|
|
1075
|
+
```
|
|
1076
|
+
|
|
1077
|
+
Pair `deleteOnComplete: true` with this loop when you don't need the completed-run audit trail — the store then holds only genuinely-unfinished runs, so the drain never touches settled ones.
|
|
1078
|
+
|
|
1079
|
+
## Cost + testing
|
|
1080
|
+
|
|
1081
|
+
- **Checkpointing cost is one store write per settled trip / node** — a `JSONB` upsert on `pg`, an in-process `Map` set on `memory`. A failed checkpoint is surfaced via logs, not thrown: it loses resume-ability from that point but never breaks an otherwise-healthy run.
|
|
1082
|
+
- **Resume saves the tokens of every settled trip / node** — their model calls are not replayed. A completed-run resume spends nothing (it rebuilds the result from the snapshot). The planning LLM is never re-called on planner resume.
|
|
1083
|
+
- **Test with `ai.snapshot.memory()`.** Drive `execute(input, { runId })` against a flaky model that throws once, assert the tool spy was called once, flip the failure off, `resume(runId)`, and assert (a) `status === "completed"`, (b) the tool spy count is unchanged (no re-invoke), and (c) `usage.total` counts each trip's tokens exactly once. Drift is testable by mutating the definition (add a tool) between `execute` and `resume` and asserting `AgentDriftError` — then `{ force: true }` proceeds.
|
|
1084
|
+
|
|
1085
|
+
## See also
|
|
1086
|
+
|
|
1087
|
+
- [[handle-ai-errors]] — the typed `AgentDriftError` / `PlannerDriftError` / `AgentExecutionError` / `PlannerFailedError` and how `result.error` surfaces a failed run.
|
|
1088
|
+
- [[manage-ai-stores]] — the `ai.snapshot.{memory,pg,redis}()` factories, the `SnapshotStore` contract, dev-owned `pg` / `redis` clients, and never-auto-migrated `schema()`.
|
|
1089
|
+
- [[persist-ai-data]] — supervisor / workflow snapshot resume (the sibling `iterate`-style durability) and the SnapshotStore migration notes.
|
|
1090
|
+
- [[durable-resume]] — the OTHER resume: `ai.human.resume` of a gated tool call (human-in-the-loop), not a crash.
|
|
1091
|
+
|
|
1092
|
+
|
|
1093
|
+
## durable-resume `@warlock.js/ai/durable-resume/SKILL.md`
|
|
1094
|
+
|
|
1095
|
+
---
|
|
1096
|
+
name: durable-resume
|
|
1097
|
+
description: 'Persist a gated tool call and resume it from another process hours later — ships in @warlock.js/ai core: `ai.human.resume(interruptId, decision, options)`, the `InterruptStore` (`ai.human.interrupt.{memory,pg,redis}()`), `PendingInterrupt`, and the `InterruptSuspendedError` suspend sentinel. Triggers: `ai.human.resume`, `resume(interruptId, decision)`, `InterruptStore`, `ai.human.interrupt.memory`, `ai.human.interrupt.pg`, `ai.human.interrupt.redis`, `interruptMemory`, `interruptPg`, `interruptRedis`, `PendingInterrupt`, `InterruptSuspendedError`, `ResumeOptions`, `ResumeResult`, `PgClientLike`, `RedisClientLike`; ''approve hours later from a webhook'', ''persist the approval request and resume in another process'', ''durable human-in-the-loop'', ''store the interrupt in Postgres/Redis'', ''re-run the agent turn once the human approves''. Typical import `import { ai, InterruptSuspendedError } from "@warlock.js/ai"`. Skip: the in-process await gate and the policy/decision shapes — `@warlock.js/ai/approve-tool-calls/SKILL.md`.'
|
|
1098
|
+
---
|
|
1099
|
+
|
|
1100
|
+
# Durable resume — persist the interrupt, approve from another process
|
|
1101
|
+
|
|
1102
|
+
Interactive approval `await`s the operator in-process. **Durable** approval is for when the reviewer rules out-of-band — a Slack button, a webhook, hours later, in a different process. The flow: the handler **persists** the request to an `InterruptStore` and **throws** `InterruptSuspendedError` to suspend the run; the caller surfaces the `interruptId`; later, `ai.human.resume(interruptId, decision, { store })` applies the ruling.
|
|
1103
|
+
|
|
1104
|
+
> **v1 durable resume re-runs the turn** with the decision pre-seeded — it does **not** rehydrate an in-flight supervisor mid-call (that's the deferred v2 lift). Re-running is idempotent because the prompt and the seeded decision fully determine the gated call's outcome.
|
|
1105
|
+
|
|
1106
|
+
## Process A — suspend and surface the id
|
|
1107
|
+
|
|
1108
|
+
```ts
|
|
1109
|
+
import { ai, InterruptSuspendedError } from "@warlock.js/ai";
|
|
1110
|
+
|
|
1111
|
+
const store = ai.human.interrupt.memory(); // swap for pg / redis in production
|
|
1112
|
+
|
|
1113
|
+
const agent = ai.agent({
|
|
1114
|
+
model,
|
|
1115
|
+
tools: [deleteAccount],
|
|
1116
|
+
middleware: [
|
|
1117
|
+
ai.human.approval({
|
|
1118
|
+
policy: { type: "predicate", requiresApproval: (c) => c.toolName === "deleteAccount" },
|
|
1119
|
+
store,
|
|
1120
|
+
handler: async (req) => {
|
|
1121
|
+
// 1. persist the pending interrupt
|
|
1122
|
+
await store.save({
|
|
1123
|
+
interruptId: req.interruptId,
|
|
1124
|
+
request: req,
|
|
1125
|
+
status: "pending",
|
|
1126
|
+
savedAt: new Date().toISOString(),
|
|
1127
|
+
});
|
|
1128
|
+
// 2. notify the reviewer out-of-band
|
|
1129
|
+
await slack.postApproval(req);
|
|
1130
|
+
// 3. suspend the run — the middleware recognizes its OWN sentinel
|
|
1131
|
+
throw new InterruptSuspendedError("Awaiting human approval", {
|
|
1132
|
+
interruptId: req.interruptId,
|
|
1133
|
+
});
|
|
1134
|
+
},
|
|
1135
|
+
}),
|
|
1136
|
+
],
|
|
1137
|
+
});
|
|
1138
|
+
|
|
1139
|
+
const result = await agent.execute("Delete account #88");
|
|
1140
|
+
|
|
1141
|
+
// execute() never throws — the suspend rides result.error:
|
|
1142
|
+
if (result.error instanceof InterruptSuspendedError) {
|
|
1143
|
+
return { status: "awaiting-approval", interruptId: result.error.interruptId };
|
|
1144
|
+
}
|
|
1145
|
+
```
|
|
1146
|
+
|
|
1147
|
+
The middleware catches the **sentinel** (`instanceof InterruptSuspendedError`) and short-circuits a failed `ToolInvokeResult` carrying it, so `error.interruptId` is on `result.error`. Hand that id to the reviewer.
|
|
1148
|
+
|
|
1149
|
+
## Process B — resume hours later
|
|
1150
|
+
|
|
1151
|
+
```ts
|
|
1152
|
+
import { ai } from "@warlock.js/ai";
|
|
1153
|
+
|
|
1154
|
+
// Re-run the turn with the decision pre-seeded:
|
|
1155
|
+
const outcome = await ai.human.resume(
|
|
1156
|
+
interruptId,
|
|
1157
|
+
{ type: "edit", args: { confirm: true } },
|
|
1158
|
+
{ store, agent },
|
|
1159
|
+
);
|
|
1160
|
+
|
|
1161
|
+
if (outcome.type === "applied" && outcome.result) {
|
|
1162
|
+
console.log(outcome.result.text); // the re-run completed with the ruling applied
|
|
1163
|
+
}
|
|
1164
|
+
```
|
|
1165
|
+
|
|
1166
|
+
`ai.human.resume(interruptId, decision, options)` loads the `PendingInterrupt`, validates the decision shape, deletes the record, and — when an `agent` is supplied — re-executes the original prompt with the decision **pre-seeded** so the gated tool call resolves to the ruling instead of pausing again. The prompt comes from `request.context.originalInput`; pass `options.input` to override (e.g. to append the reviewer's note), and `options.executeOptions` to forward history / output schema / signal to the re-run.
|
|
1167
|
+
|
|
1168
|
+
### Two resume shapes
|
|
1169
|
+
|
|
1170
|
+
| Shape | Pass | Behavior |
|
|
1171
|
+
|---|---|---|
|
|
1172
|
+
| **re-run** | `{ store, agent }` | Loads, deletes, re-executes the turn; `ResultResult.result` carries the `AgentResult`. |
|
|
1173
|
+
| **apply-only** | `{ store }` (no `agent`) | Loads, validates, deletes; returns `{ type: "applied", decision }` for a caller-owned re-drive (custom transport). No turn re-run. |
|
|
1174
|
+
|
|
1175
|
+
### Idempotent by construction
|
|
1176
|
+
|
|
1177
|
+
```ts
|
|
1178
|
+
type ResumeResult =
|
|
1179
|
+
| { type: "applied"; interruptId: string; decision: ApprovalDecision; result?: AgentResult }
|
|
1180
|
+
| { type: "already-resolved"; interruptId: string };
|
|
1181
|
+
```
|
|
1182
|
+
|
|
1183
|
+
A second resume of an already-resolved (deleted) or never-raised interrupt returns `{ type: "already-resolved" }` — it never double-applies the decision or re-runs the turn. The record is deleted **before** the re-run, so even a re-run that itself raises a fresh interrupt can't collide with the one being resolved. A malformed decision (`reject` with no `reason`, `edit` with no `args`, an unknown `type`) throws a `TypeError` loudly rather than silently mis-driving the re-run.
|
|
1184
|
+
|
|
1185
|
+
## The `InterruptStore`
|
|
1186
|
+
|
|
1187
|
+
`ai.human.interrupt.{memory,pg,redis}()` build the store. The contract mirrors `@warlock.js/ai`'s `CheckpointStore` / `SnapshotStore` — `save` / `load` / `delete` / optional `list(prefix?)` / `schema()` — so a consumer already running an orchestrator can reuse the **same** pool for the interrupt table.
|
|
1188
|
+
|
|
1189
|
+
| Factory | Backing | Deps |
|
|
1190
|
+
|---|---|---|
|
|
1191
|
+
| `ai.human.interrupt.memory()` | process-local `Map` | none — zero runtime deps |
|
|
1192
|
+
| `ai.human.interrupt.pg(options)` | one Postgres row per interrupt, keyed by `interrupt_id` | lazily imports the optional `pg` peer |
|
|
1193
|
+
| `ai.human.interrupt.redis(options)` | one namespaced JSON value + a self-maintained id index | lazily imports the optional `redis` peer |
|
|
1194
|
+
|
|
1195
|
+
```ts
|
|
1196
|
+
// Memory — dev / tests / single-process:
|
|
1197
|
+
const store = ai.human.interrupt.memory();
|
|
1198
|
+
|
|
1199
|
+
// Postgres — pass a live pool (core never imports pg in that case):
|
|
1200
|
+
import { Pool } from "pg";
|
|
1201
|
+
const pool = new Pool({ connectionString: process.env.DATABASE_URL });
|
|
1202
|
+
const store = ai.human.interrupt.pg({ client: pool });
|
|
1203
|
+
// Once, via your migration tool — the framework never auto-migrates:
|
|
1204
|
+
// await pool.query(store.schema());
|
|
1205
|
+
|
|
1206
|
+
// …or let the store build its own pool (lazily import("pg")):
|
|
1207
|
+
const store = ai.human.interrupt.pg({ connectionString: process.env.DATABASE_URL });
|
|
1208
|
+
|
|
1209
|
+
// Redis — pass a connected client, or a url:
|
|
1210
|
+
const store = ai.human.interrupt.redis({ url: process.env.REDIS_URL });
|
|
1211
|
+
```
|
|
1212
|
+
|
|
1213
|
+
### Optional peers are lazy
|
|
1214
|
+
|
|
1215
|
+
`pg` and `redis` are **optional** peer dependencies — neither is a hard dependency. The driver is imported only inside the store that needs it, and only when you pass a `connectionString` / `url` (passing a live `client` imports nothing). If the driver is absent, a **curated install string** surfaces on first use, never a raw module-resolution stack trace at import — so a memory-only consumer always loads cleanly. `PgClientLike` / `RedisClientLike` are structural interfaces, so any compatible pool/client satisfies them.
|
|
1216
|
+
|
|
1217
|
+
`schema()` returns the reference DDL for the Postgres store (run it through your migration tool once) and an empty string for memory / redis, so callers treat `schema()` uniformly across drivers.
|
|
1218
|
+
|
|
1219
|
+
## See also
|
|
1220
|
+
|
|
1221
|
+
- [`@warlock.js/ai/approve-tool-calls/SKILL.md`](@warlock.js/ai/approve-tool-calls/SKILL.md) — the gate itself: the interrupt policy, the approve / reject / edit decision union, and the interactive (in-process await) handler.
|
|
1222
|
+
- `@warlock.js/ai` — the `CheckpointStore` / `SnapshotStore` the `InterruptStore` mirrors, and the `ai.agent(...)` re-run target.
|
|
1223
|
+
|
|
1224
|
+
|
|
708
1225
|
## embed-text `@warlock.js/ai/embed-text/SKILL.md`
|
|
709
1226
|
|
|
710
1227
|
---
|
|
@@ -814,21 +1331,637 @@ No built-in vector store. Bring your own (pgvector / Qdrant / Pinecone / Chroma
|
|
|
814
1331
|
- [`@warlock.js/cascade/search-by-vector/SKILL.md`](@warlock.js/cascade/search-by-vector/SKILL.md) — cascade `similarTo` query method
|
|
815
1332
|
|
|
816
1333
|
|
|
817
|
-
##
|
|
1334
|
+
## escalate-block-to-human `@warlock.js/ai/escalate-block-to-human/SKILL.md`
|
|
818
1335
|
|
|
819
1336
|
---
|
|
820
|
-
name:
|
|
821
|
-
description: '
|
|
1337
|
+
name: escalate-block-to-human
|
|
1338
|
+
description: 'Route a hard guardrail block to a human-review surface with @warlock.js/ai-guard — the `escalation.onBlock` seam and an `escalate: true` verdict. Triggers: `escalation`, `onBlock`, `GuardrailEscalation`, `GuardrailBlockEvent`, `escalate: true`, `{ type: "block", escalate: true }`, ''escalate a block to a human'', ''human review queue for guardrail'', ''page an operator on a guardrail block'', ''human-in-the-loop guardrail'', ''compose a block with a review surface'', ''custom detector that escalates''; typical import `import "@warlock.js/ai-guard"` then `ai.guardrail({ escalation: { onBlock } })`. Skip: composing the guard / phases / verdict model — `@warlock.js/ai-guard/guard-input-output/SKILL.md`; PII/moderation detectors — `@warlock.js/ai-guard/detect-and-redact-pii/SKILL.md`; durable suspend/resume human-step machinery (deferred) — not in this package.'
|
|
822
1339
|
---
|
|
823
1340
|
|
|
824
|
-
#
|
|
825
|
-
|
|
826
|
-
Every error surfaced by `@warlock.js/ai` and every adapter package is an `AIError` subclass with a stable `code`. The base extends platform `Error`; it does NOT extend `HttpError`. Plain `Error` never leaks.
|
|
1341
|
+
# Escalate a block to a human
|
|
827
1342
|
|
|
828
|
-
|
|
1343
|
+
A `block` verdict can carry `escalate: true`. When it does, the guard `await`s your `escalation.onBlock(...)` handler **before** throwing the `GuardrailViolationError` — the seam to a human-review queue, an operator page, or any out-of-band approval surface.
|
|
829
1344
|
|
|
830
|
-
|
|
831
|
-
|
|
1345
|
+
```ts
|
|
1346
|
+
import { ai } from "@warlock.js/ai";
|
|
1347
|
+
import "@warlock.js/ai-guard";
|
|
1348
|
+
|
|
1349
|
+
const policy = ai.guardrail({
|
|
1350
|
+
output: [ai.guardrail.moderation({ blockOn: ["self-harm"] })],
|
|
1351
|
+
escalation: {
|
|
1352
|
+
async onBlock(event) {
|
|
1353
|
+
await reviewQueue.enqueue({
|
|
1354
|
+
phase: event.phase, // "input" | "output" | "tool"
|
|
1355
|
+
reason: event.reason, // the detector's human-readable reason
|
|
1356
|
+
});
|
|
1357
|
+
},
|
|
1358
|
+
},
|
|
1359
|
+
});
|
|
1360
|
+
|
|
1361
|
+
const agent = ai.agent({ model, middleware: [policy] });
|
|
1362
|
+
```
|
|
1363
|
+
|
|
1364
|
+
## When `onBlock` fires
|
|
1365
|
+
|
|
1366
|
+
`onBlock` fires **only** for a verdict of `{ type: "block", escalate: true }` — not for an ordinary `block`, and never for `allow` / `redact` / `flag`. It is **awaited before** the `GuardrailViolationError` is thrown, so your handler runs to completion (enqueue succeeds, the page is sent) before the error surfaces on `result.error`. The run still aborts: escalation is a *signal*, not a recovery — `execute()` returns with `result.error` populated as usual.
|
|
1367
|
+
|
|
1368
|
+
## The `GuardrailBlockEvent` payload
|
|
1369
|
+
|
|
1370
|
+
`onBlock(event)` receives:
|
|
1371
|
+
|
|
1372
|
+
| Field | Type | Meaning |
|
|
1373
|
+
|---|---|---|
|
|
1374
|
+
| `phase` | `"input" \| "output" \| "tool"` | where the block fired |
|
|
1375
|
+
| `reason` | `string` | the detector's human-readable reason |
|
|
1376
|
+
| `matches` | `readonly GuardrailMatch[] \| undefined` | what tripped the rule (rule id, span, label), when reported |
|
|
1377
|
+
| `ctx` | `MiddlewareTripContext` | the live trip context — `state`, `messages`, `agent`, `model`, `signal` |
|
|
1378
|
+
|
|
1379
|
+
`ctx` lets the handler enrich the review item with run context (session id from `ctx.state`, the offending messages, etc.).
|
|
1380
|
+
|
|
1381
|
+
## Producing an escalating verdict
|
|
1382
|
+
|
|
1383
|
+
The built-in detectors return ordinary `block` verdicts (no `escalate`). To escalate, author a tiny custom `GuardrailDetector` that sets `escalate: true` on its `block`:
|
|
1384
|
+
|
|
1385
|
+
```ts
|
|
1386
|
+
import type { GuardrailDetector } from "@warlock.js/ai-guard";
|
|
1387
|
+
|
|
1388
|
+
const wirePolicy: GuardrailDetector = {
|
|
1389
|
+
name: "wire-transfer",
|
|
1390
|
+
check(text) {
|
|
1391
|
+
if (/wire \$?\d{5,}/i.test(text)) {
|
|
1392
|
+
return {
|
|
1393
|
+
type: "block",
|
|
1394
|
+
reason: "large wire transfer requires human approval",
|
|
1395
|
+
escalate: true, // <- routes through escalation.onBlock
|
|
1396
|
+
matches: [{ rule: "wire-transfer.large", label: "wire" }],
|
|
1397
|
+
};
|
|
1398
|
+
}
|
|
1399
|
+
|
|
1400
|
+
return { type: "allow" };
|
|
1401
|
+
},
|
|
1402
|
+
};
|
|
1403
|
+
|
|
1404
|
+
const policy = ai.guardrail({
|
|
1405
|
+
tool: [wirePolicy],
|
|
1406
|
+
toolNames: ["initiate_transfer"],
|
|
1407
|
+
escalation: { async onBlock(e) { await approvals.request(e); } },
|
|
1408
|
+
});
|
|
1409
|
+
```
|
|
1410
|
+
|
|
1411
|
+
A `check()` may be sync or async (async = call an external service); the guard awaits either.
|
|
1412
|
+
|
|
1413
|
+
## A plain callback by design
|
|
1414
|
+
|
|
1415
|
+
`escalation.onBlock` is a **plain callback** — `ai-guard` takes **no** dependency on the deferred durable human-step machinery (suspend/resume). The callback is the decoupling seam: inside it you wire your own review queue, and (where your stack supports it) a `workflow.resume(...)` loop. This package only emits the *signal*; it does not own durable suspension. When the typed human-step handoff ships, `onBlock` upgrades to it without a breaking change here.
|
|
1416
|
+
|
|
1417
|
+
## See also
|
|
1418
|
+
|
|
1419
|
+
- [`@warlock.js/ai-guard/guard-input-output/SKILL.md`](@warlock.js/ai-guard/guard-input-output/SKILL.md) — composing the guard, the phases, the verdict model, and how a `block` surfaces on `result.error`.
|
|
1420
|
+
- [`@warlock.js/ai-guard/detect-and-redact-pii/SKILL.md`](@warlock.js/ai-guard/detect-and-redact-pii/SKILL.md) — the `pii` detector and the optional `moderation` peer that commonly drives an escalation.
|
|
1421
|
+
|
|
1422
|
+
|
|
1423
|
+
## eval-datasets-and-ci `@warlock.js/ai/eval-datasets-and-ci/SKILL.md`
|
|
1424
|
+
|
|
1425
|
+
---
|
|
1426
|
+
name: eval-datasets-and-ci
|
|
1427
|
+
description: 'Datasets + regression-gated eval CI with ai.dataset({...}) feeding agent.eval({cases,baseline,tolerance}). Covers the immutable filterable/shardable dataset (cases / fromFile JSONL), DatasetEntry tags, EvalReport.regression (regressed/added/removed/passed) against a baseline, and the ai.eval reporters toJUnit / toJSON / fromJSON for CI artifacts + committed baselines. Triggers: `ai.dataset`, `DatasetContract`, `DatasetEntry`, `DatasetOptions`, `dataset.filter`, `dataset.shard`, `fromFile`, `agent.eval`, `EvalOptions`, `EvalReport`, `EvalCaseResult`, `EvalRegression`, `baseline`, `tolerance`, `ai.eval.toJUnit`, `ai.eval.toJSON`, `ai.eval.fromJSON`, `diff`, JSONL; ''eval dataset from a JSONL file'', ''shard an eval suite across CI jobs'', ''fail CI on an eval regression'', ''emit a JUnit report'', ''snapshot an eval baseline''; typical import `import { ai } from "@warlock.js/ai"`. Skip: the scorers + LLM-as-judge + Vitest matchers themselves — `@warlock.js/ai/ai-dx-helpers/SKILL.md` (registerAiMatchers / ai.eval.exact|contains|predicate|judge); record/replay of model calls for deterministic tests — `@warlock.js/ai/record-replay-llm/SKILL.md`; competing libs `promptfoo`, `braintrust`.'
|
|
1428
|
+
---
|
|
1429
|
+
|
|
1430
|
+
# `ai.dataset()` + `agent.eval()` regression CI
|
|
1431
|
+
|
|
1432
|
+
Turn a corpus of cases into a regression-gated CI signal. `ai.dataset(...)` wraps cases into an immutable, filterable, shardable collection; `agent.eval({ cases, baseline, tolerance })` runs them, scores them, and diffs against a prior report; the `ai.eval.*` reporters serialize the result for CI ingestion and tomorrow's baseline.
|
|
1433
|
+
|
|
1434
|
+
> This skill is the **dataset + CI** layer. The scorers, LLM-as-judge config, and Vitest matchers live in [`@warlock.js/ai/ai-dx-helpers/SKILL.md`](@warlock.js/ai/ai-dx-helpers/SKILL.md); `agent.eval`'s core scoring loop is in [`@warlock.js/ai/run-ai-agent/SKILL.md`](@warlock.js/ai/run-ai-agent/SKILL.md).
|
|
1435
|
+
|
|
1436
|
+
## `ai.dataset()` — immutable, filterable, shardable
|
|
1437
|
+
|
|
1438
|
+
```ts
|
|
1439
|
+
import { ai } from "@warlock.js/ai";
|
|
1440
|
+
|
|
1441
|
+
const ds = ai.dataset({
|
|
1442
|
+
name: "support",
|
|
1443
|
+
cases: [{ name: "greeting", input: "hi", expected: "Hello" }], // inline entries
|
|
1444
|
+
fromFile: "./eval/support.jsonl", // JSONL read ONCE, synchronously, at construction
|
|
1445
|
+
});
|
|
1446
|
+
|
|
1447
|
+
ds.name; // "support"
|
|
1448
|
+
ds.cases; // DatasetEntry[] (inline first, then file entries appended)
|
|
1449
|
+
```
|
|
1450
|
+
|
|
1451
|
+
- A `DatasetEntry` is an `EvalCase` plus optional `tags?: string[]` for filtering / sharding.
|
|
1452
|
+
- **`fromFile`** reads a JSONL file (one JSON object per line; blank lines skipped) synchronously at construction — mirroring `SystemPrompt.fromFile`. A malformed line throws an **`InvalidRequestError` naming the 1-based line number**; a missing/unreadable path throws too. `cases` and `fromFile` combine (file entries append after inline).
|
|
1453
|
+
|
|
1454
|
+
### `filter` / `shard` — derive new datasets
|
|
1455
|
+
|
|
1456
|
+
```ts
|
|
1457
|
+
const smoke = ds.filter((entry) => entry.tags?.includes("smoke"));
|
|
1458
|
+
const shard = ds.shard(0, 4); // first of four parallel CI shards
|
|
1459
|
+
```
|
|
1460
|
+
|
|
1461
|
+
Both return a **fresh dataset sharing nothing mutable**. `shard(index, total)` is deterministic round-robin by position: every entry lands in exactly one shard, so the union of all `total` shards reproduces the full list with no gaps or overlaps. `shard` validates its args (positive integer `total`, `index` in `[0, total)`) and throws `InvalidRequestError` otherwise.
|
|
1462
|
+
|
|
1463
|
+
## `agent.eval({ cases })` — accepts a dataset directly
|
|
1464
|
+
|
|
1465
|
+
```ts
|
|
1466
|
+
const report = await myAgent.eval({
|
|
1467
|
+
cases: ds, // a DatasetContract OR a raw EvalCase[]
|
|
1468
|
+
scorers: [ai.eval.contains()],
|
|
1469
|
+
});
|
|
1470
|
+
|
|
1471
|
+
expect(report.passed).toBe(true); // true only when EVERY case passed
|
|
1472
|
+
```
|
|
1473
|
+
|
|
1474
|
+
The runner reads `.cases` off a dataset. Full `EvalReport`: `{ agentName, total, passedCount, failedCount, passRate, meanScore, passed, cases, duration, regression? }`. Each `EvalCaseResult` carries the case, the full `AgentResult`, every scorer's `scores`, the mean `score`, `passed`, and `duration`.
|
|
1475
|
+
|
|
1476
|
+
## Regression gating — `baseline` + `tolerance`
|
|
1477
|
+
|
|
1478
|
+
```ts
|
|
1479
|
+
import { readFile, writeFile } from "node:fs/promises";
|
|
1480
|
+
|
|
1481
|
+
const baseline = ai.eval.fromJSON(await readFile("./eval/baseline.json", "utf8"));
|
|
1482
|
+
|
|
1483
|
+
const report = await myAgent.eval({
|
|
1484
|
+
cases: ds,
|
|
1485
|
+
scorers: [ai.eval.exact()],
|
|
1486
|
+
baseline, // a prior EvalReport to diff against
|
|
1487
|
+
tolerance: 0.05, // max allowed per-case score DROP before it regresses. default 0 (any drop)
|
|
1488
|
+
});
|
|
1489
|
+
|
|
1490
|
+
if (report.regression && !report.regression.passed) {
|
|
1491
|
+
console.error("Regressed:", report.regression.regressed); // [{ name, before, after }]
|
|
1492
|
+
process.exit(1);
|
|
1493
|
+
}
|
|
1494
|
+
```
|
|
1495
|
+
|
|
1496
|
+
When `baseline` is set the report carries a `regression` block (`EvalRegression`), joining cases by `name`:
|
|
1497
|
+
|
|
1498
|
+
- **`regressed`** — `[{ name, before, after }]` for cases whose new score fell more than `tolerance` below baseline.
|
|
1499
|
+
- **`added`** / **`removed`** — case names present in only one report. Adding or dropping a case **never fails the gate by itself**.
|
|
1500
|
+
- **`passed`** — `true` when `regressed` is empty.
|
|
1501
|
+
|
|
1502
|
+
The pure `diff(report, baseline, tolerance)` function (exported as `diff`) is the same logic, decoupled from the runner — depends only on the two reports and the tolerance, mutates neither.
|
|
1503
|
+
|
|
1504
|
+
## CI reporters — `ai.eval.toJUnit` / `toJSON` / `fromJSON`
|
|
1505
|
+
|
|
1506
|
+
Pure functions over a finished `EvalReport`:
|
|
1507
|
+
|
|
1508
|
+
```ts
|
|
1509
|
+
// JUnit-XML artifact for CI ingestion — one <testsuite> (the agent), one <testcase> per case,
|
|
1510
|
+
// a <failure> on each non-passing case (joined scorer reasons), times in SECONDS.
|
|
1511
|
+
await writeFile("./report.junit.xml", ai.eval.toJUnit(report));
|
|
1512
|
+
|
|
1513
|
+
// Round-trippable snapshot — today's report becomes tomorrow's baseline.
|
|
1514
|
+
await writeFile("./eval/baseline.json", ai.eval.toJSON(report));
|
|
1515
|
+
const restored = ai.eval.fromJSON(await readFile("./eval/baseline.json", "utf8"));
|
|
1516
|
+
```
|
|
1517
|
+
|
|
1518
|
+
`toJSON`/`fromJSON` preserve `result` payloads, per-case `scores`, timings, and any attached `regression` block, so a parsed report drives regression diffing exactly as the in-memory one. `toJUnit` hand-emits XML (no `xml` dependency) and entity-escapes every dynamic value.
|
|
1519
|
+
|
|
1520
|
+
## Typical CI shard job
|
|
1521
|
+
|
|
1522
|
+
```ts
|
|
1523
|
+
const shard = ai.dataset({ name: "support", fromFile: "./eval/support.jsonl" })
|
|
1524
|
+
.shard(Number(process.env.SHARD_INDEX), Number(process.env.SHARD_TOTAL));
|
|
1525
|
+
|
|
1526
|
+
const report = await agent.eval({
|
|
1527
|
+
cases: shard,
|
|
1528
|
+
scorers: [ai.eval.contains()],
|
|
1529
|
+
baseline: ai.eval.fromJSON(await readFile("./eval/baseline.json", "utf8")),
|
|
1530
|
+
tolerance: 0.05,
|
|
1531
|
+
});
|
|
1532
|
+
|
|
1533
|
+
await writeFile(`./out/report-${process.env.SHARD_INDEX}.junit.xml`, ai.eval.toJUnit(report));
|
|
1534
|
+
if (report.regression && !report.regression.passed) process.exit(1);
|
|
1535
|
+
```
|
|
1536
|
+
|
|
1537
|
+
## See also
|
|
1538
|
+
|
|
1539
|
+
- [`@warlock.js/ai/ai-dx-helpers/SKILL.md`](@warlock.js/ai/ai-dx-helpers/SKILL.md) — `ai.eval.{exact,contains,predicate,judge}` scorers + Vitest matchers
|
|
1540
|
+
- [`@warlock.js/ai/run-ai-agent/SKILL.md`](@warlock.js/ai/run-ai-agent/SKILL.md) — `agent.eval` scoring loop, `EvalCase` / `EvalJudge`
|
|
1541
|
+
- [`@warlock.js/ai/record-replay-llm/SKILL.md`](@warlock.js/ai/record-replay-llm/SKILL.md) — `ai.vcr` for deterministic, offline eval runs
|
|
1542
|
+
|
|
1543
|
+
|
|
1544
|
+
## generate-images `@warlock.js/ai/generate-images/SKILL.md`
|
|
1545
|
+
|
|
1546
|
+
---
|
|
1547
|
+
name: generate-images
|
|
1548
|
+
description: 'Text-to-image via ai.image({ model: sdk.image({ name }), prompt }) — the image-OUTPUT verb (Theme I), returning the uniform never-throws { data, error, usage, report } envelope with cost-truth + panoptic observation. Models come from an adapter''s image() factory: OpenAI gpt-image-* (token-metered) / dall-e-* (per-image), Google imagen-* (per-image). Result images are a discriminated GeneratedImage = { type: "base64" } | { type: "url" }. Triggers: `ai.image`, `sdk.image`, `openai.image`, `google.image`, `ImageModelContract`, `GeneratedImage`, `ImageModelPricing`; ''generate an image'', ''text to image'', ''gpt-image'', ''dall-e'', ''imagen'', ''product thumbnail'', ''image output''; typical import `import { ai } from "@warlock.js/ai"` + `import { OpenAISDK } from "@warlock.js/ai-openai"`. Skip: image INPUT / vision attachments to a chat agent — `@warlock.js/ai/run-ai-agent/SKILL.md`; embeddings — `@warlock.js/ai/embed-text/SKILL.md`; competing libs raw `openai.images.generate`, `langchain` image tools.'
|
|
1549
|
+
---
|
|
1550
|
+
|
|
1551
|
+
# Generate images — the image-output verb (`ai.image`)
|
|
1552
|
+
|
|
1553
|
+
`ai.image()` is the output counterpart to `ai.agent` for the image modality (the first verb of the output-modality track, Theme I). Prompt-in / images-out, wrapped in the same uniform result contract every executable returns — so it slots into cost dashboards and panoptic traces exactly like an agent run.
|
|
1554
|
+
|
|
1555
|
+
This is image **output** (generation). For image/PDF/audio **input** to a chat agent (vision), see [`@warlock.js/ai/run-ai-agent/SKILL.md`](@warlock.js/ai/run-ai-agent/SKILL.md).
|
|
1556
|
+
|
|
1557
|
+
## Shape
|
|
1558
|
+
|
|
1559
|
+
```ts
|
|
1560
|
+
// 1. Build an image model from an adapter's image() factory.
|
|
1561
|
+
const model = openai.image({ name: "gpt-image-1" }); // ImageModelContract
|
|
1562
|
+
|
|
1563
|
+
// 2. Run the verb — never throws; failures land on result.error.
|
|
1564
|
+
const { data, error, usage, report } = await ai.image({ model, prompt: "a red bicycle" });
|
|
1565
|
+
|
|
1566
|
+
if (error) {
|
|
1567
|
+
console.warn(error.code); // typed AIError (auth / rate-limit / content-filter / …)
|
|
1568
|
+
} else {
|
|
1569
|
+
for (const img of data.images) { // GeneratedImage[]
|
|
1570
|
+
if (img.type === "base64") save(Buffer.from(img.base64, "base64"), img.mediaType);
|
|
1571
|
+
else download(img.url);
|
|
1572
|
+
}
|
|
1573
|
+
}
|
|
1574
|
+
```
|
|
1575
|
+
|
|
1576
|
+
`ImageModelContract` mirrors `EmbedderContract` — a peer primitive on the SDK adapter, produced by the optional `image?()` factory. An adapter without an image API simply doesn't define `image()`, so `ai.anthropic.image(...)` is a **compile-time** error, not a silent runtime failure.
|
|
1577
|
+
|
|
1578
|
+
## The result envelope
|
|
1579
|
+
|
|
1580
|
+
```ts
|
|
1581
|
+
type ImageResult = {
|
|
1582
|
+
type: "image";
|
|
1583
|
+
data?: { images: GeneratedImage[] }; // undefined on failure
|
|
1584
|
+
error?: AIError; // undefined on success — NEVER thrown
|
|
1585
|
+
usage: Usage; // tokens (gpt-image) + cost when priced
|
|
1586
|
+
report: ImageReport; // type:"image", model, imageCount, lineage
|
|
1587
|
+
};
|
|
1588
|
+
|
|
1589
|
+
type GeneratedImage =
|
|
1590
|
+
| { type: "base64"; base64: string; mediaType: string; revisedPrompt?: string }
|
|
1591
|
+
| { type: "url"; url: string; mediaType?: string; revisedPrompt?: string };
|
|
1592
|
+
```
|
|
1593
|
+
|
|
1594
|
+
## Generation options (provider-neutral)
|
|
1595
|
+
|
|
1596
|
+
```ts
|
|
1597
|
+
await ai.image({
|
|
1598
|
+
model,
|
|
1599
|
+
prompt: "an isometric office desk, soft studio lighting",
|
|
1600
|
+
count: 2, // n images
|
|
1601
|
+
size: "1024x1024", // OpenAI WxH (also resolves perImageBySize pricing)
|
|
1602
|
+
quality: "high", // OpenAI quality tier
|
|
1603
|
+
aspectRatio: "16:9", // Imagen ratio
|
|
1604
|
+
negativePrompt: "blurry, watermark", // Imagen
|
|
1605
|
+
format: "png", // output container hint
|
|
1606
|
+
signal, // AbortSignal
|
|
1607
|
+
observe: collector, // route the report to an Observer (panoptic), like agents
|
|
1608
|
+
sessionId: "checkout-123",
|
|
1609
|
+
options: { background: "transparent" }, // provider-specific passthrough
|
|
1610
|
+
});
|
|
1611
|
+
```
|
|
1612
|
+
|
|
1613
|
+
Each adapter maps the options its API supports and ignores the rest. `options` is the escape hatch for provider-specific knobs (OpenAI `background`, DALL·E `responseFormat: "url"`, Imagen `imageSize` / `personGeneration`).
|
|
1614
|
+
|
|
1615
|
+
## OpenAI — gpt-image (token-metered) + DALL·E (per-image)
|
|
1616
|
+
|
|
1617
|
+
```ts
|
|
1618
|
+
import { OpenAISDK } from "@warlock.js/ai-openai";
|
|
1619
|
+
|
|
1620
|
+
const openai = new OpenAISDK({ apiKey: process.env.OPENAI_API_KEY! });
|
|
1621
|
+
|
|
1622
|
+
// gpt-image-1 always returns base64 bytes; priced per TOKEN.
|
|
1623
|
+
const gpt = openai.image({ name: "gpt-image-1", pricing: { input: 5, output: 40 } });
|
|
1624
|
+
|
|
1625
|
+
// DALL·E 3 — per-image pricing; defaults to base64 (opt into url with options).
|
|
1626
|
+
const dalle = openai.image({ name: "dall-e-3", pricing: { perImage: 0.04 } });
|
|
1627
|
+
```
|
|
1628
|
+
|
|
1629
|
+
A non-image model id (`openai.image({ name: "gpt-4o" })`) throws `InvalidRequestError` **at construction** — fail fast, like the embedder/vision guards.
|
|
1630
|
+
|
|
1631
|
+
## Google — Imagen (per-image)
|
|
1632
|
+
|
|
1633
|
+
```ts
|
|
1634
|
+
import { GoogleSDK } from "@warlock.js/ai-google";
|
|
1635
|
+
|
|
1636
|
+
const google = new GoogleSDK({ apiKey: process.env.GEMINI_API_KEY! });
|
|
1637
|
+
const imagen = google.image({ name: "imagen-4.0-generate-001", pricing: { perImage: 0.04 } });
|
|
1638
|
+
|
|
1639
|
+
const { data } = await ai.image({ model: imagen, prompt: "a watercolor lighthouse at dawn", aspectRatio: "3:4" });
|
|
1640
|
+
```
|
|
1641
|
+
|
|
1642
|
+
Imagen returns base64 bytes (no hosted URL). When every candidate is safety-filtered, `ai.image` surfaces a typed `ContentFilterError` on `result.error`.
|
|
1643
|
+
|
|
1644
|
+
## Cost-truth — one rollup, two metering models
|
|
1645
|
+
|
|
1646
|
+
`ai.image` fills `usage.cost` (a `ModelPricing`-shaped USD breakdown) so image spend folds into the **same** `Usage.cost` rollup as text — no second accounting path:
|
|
1647
|
+
|
|
1648
|
+
- **Token-metered** (gpt-image-1): `{ input, output }` USD-per-1M-tokens → standard `computeCost` against the returned token usage.
|
|
1649
|
+
- **Per-image** (DALL·E, Imagen): `{ perImage }` (or `perImageBySize["1792x1024"]`) × image count → `cost.output`.
|
|
1650
|
+
|
|
1651
|
+
Unpriced model → `usage.cost` stays `undefined` (honest "cost unknown", never a false zero). A pre-priced adapter response is honored, not overwritten.
|
|
1652
|
+
|
|
1653
|
+
## Pattern — catalog thumbnail in a workflow `run` step
|
|
1654
|
+
|
|
1655
|
+
```ts
|
|
1656
|
+
ai.step({
|
|
1657
|
+
name: "thumbnail",
|
|
1658
|
+
run: async (ctx) => {
|
|
1659
|
+
const { data, error } = await ai.image({
|
|
1660
|
+
model: openai.image({ name: "gpt-image-1" }),
|
|
1661
|
+
prompt: `product photo, white background: ${ctx.steps.extract.output.title}`,
|
|
1662
|
+
size: "1024x1024",
|
|
1663
|
+
});
|
|
1664
|
+
if (error) throw error; // step retry/backoff handles transient provider faults
|
|
1665
|
+
ctx.state.thumb = data.images[0];
|
|
1666
|
+
},
|
|
1667
|
+
});
|
|
1668
|
+
```
|
|
1669
|
+
|
|
1670
|
+
## Observability
|
|
1671
|
+
|
|
1672
|
+
The completed `ImageReport` routes to any registered `Observer` (panoptic, OTel, …) through the shared `observe` seam — pass `observe: true` (global), an `Observer` object (flow-local), or rely on observe-all. Cost + latency attribute to `report.model` for free. See [`@warlock.js/ai/observe-ai-flows/SKILL.md`](@warlock.js/ai/observe-ai-flows/SKILL.md).
|
|
1673
|
+
|
|
1674
|
+
## Testing
|
|
1675
|
+
|
|
1676
|
+
`MockSDK({ imageResponses, imagePricing }).image({ name })` returns a deterministic `MockImageModel` — no HTTP. Script images/usage/errors and inspect `model.calls`.
|
|
1677
|
+
|
|
1678
|
+
```ts
|
|
1679
|
+
import { MockSDK } from "@warlock.js/ai";
|
|
1680
|
+
|
|
1681
|
+
const mock = MockSDK({ imageResponses: [{}], imagePricing: { perImage: 0.04 } });
|
|
1682
|
+
const { data, usage } = await ai.image({ model: mock.image({ name: "mock-image" }), prompt: "x" });
|
|
1683
|
+
```
|
|
1684
|
+
|
|
1685
|
+
|
|
1686
|
+
## generate-speech `@warlock.js/ai/generate-speech/SKILL.md`
|
|
1687
|
+
|
|
1688
|
+
---
|
|
1689
|
+
name: generate-speech
|
|
1690
|
+
description: 'Text-to-speech via ai.speech({ model: sdk.speech({ name }), text }) — the audio-OUTPUT verb (Theme I), returning the uniform never-throws { data, error, usage, report } envelope with cost-truth + panoptic observation. Models come from an adapter''s speech() factory: OpenAI tts-1 / tts-1-hd (per-character) or gpt-4o-mini-tts (per-token). Synthesized audio is a discriminated GeneratedAudio = { type: "base64"; base64; mediaType }. Options: voice / format / speed / instructions / signal. Triggers: `ai.speech`, `sdk.speech`, `openai.speech`, `SpeechModelContract`, `GeneratedAudio`, `SpeechModelPricing`, `SpeechOptions`, `MockSpeechModel`; ''text to speech'', ''TTS'', ''synthesize voice'', ''read this aloud'', ''tts-1'', ''gpt-4o-mini-tts'', ''voice narration'', ''audio output'', ''speak this text''; typical import `import { ai } from "@warlock.js/ai"` + `import { OpenAISDK } from "@warlock.js/ai-openai"`. Skip: speech-to-text / transcribing a voice note — [[transcribe-audio]]; image OUTPUT — [[generate-images]]; competing libs raw `openai.audio.speech.create`, `elevenlabs` SDK.'
|
|
1691
|
+
---
|
|
1692
|
+
|
|
1693
|
+
# Generate speech — the text-to-speech verb (`ai.speech`)
|
|
1694
|
+
|
|
1695
|
+
`ai.speech()` is the audio-output counterpart to `ai.image()` on the output-modality track (Theme I). Text-in / audio-out, wrapped in the same uniform result contract every executable returns — so a synthesized voicemail slots into cost dashboards and panoptic traces exactly like an agent run.
|
|
1696
|
+
|
|
1697
|
+
This is audio **output** (TTS). For audio **input** (speech-to-text on a WhatsApp voice note or a meeting recording), see [[transcribe-audio]].
|
|
1698
|
+
|
|
1699
|
+
## Shape
|
|
1700
|
+
|
|
1701
|
+
```ts
|
|
1702
|
+
// 1. Build a speech model from an adapter's speech() factory.
|
|
1703
|
+
const model = openai.speech({ name: "tts-1", voice: "alloy" }); // SpeechModelContract
|
|
1704
|
+
|
|
1705
|
+
// 2. Run the verb — never throws; failures land on result.error.
|
|
1706
|
+
const { data, error, usage, report } = await ai.speech({ model, text: "Your order has shipped." });
|
|
1707
|
+
|
|
1708
|
+
if (error) {
|
|
1709
|
+
console.warn(error.code); // typed AIError (auth / rate-limit / content-filter / …)
|
|
1710
|
+
} else {
|
|
1711
|
+
const { base64, mediaType } = data.audio; // GeneratedAudio (always base64 today)
|
|
1712
|
+
await fs.writeFile("ship.mp3", Buffer.from(base64, "base64"));
|
|
1713
|
+
}
|
|
1714
|
+
```
|
|
1715
|
+
|
|
1716
|
+
`SpeechModelContract` mirrors `EmbedderContract` / `ImageModelContract` — a peer primitive produced by the adapter's optional `speech?()` factory. An adapter without a TTS API simply doesn't define `speech()`, so calling it is a **compile-time** error, not a silent runtime failure. A non-TTS model id (`openai.speech({ name: "gpt-4o" })`) throws `InvalidRequestError` **at construction** — fail fast, like the embedder/image guards.
|
|
1717
|
+
|
|
1718
|
+
## The result envelope
|
|
1719
|
+
|
|
1720
|
+
```ts
|
|
1721
|
+
type SpeechResult = {
|
|
1722
|
+
type: "speech";
|
|
1723
|
+
data?: { audio: GeneratedAudio }; // undefined on failure
|
|
1724
|
+
error?: AIError; // undefined on success — NEVER thrown
|
|
1725
|
+
usage: Usage; // tokens (gpt-4o-mini-tts) + cost when priced
|
|
1726
|
+
report: SpeechReport; // type:"speech", model, characters, lineage
|
|
1727
|
+
};
|
|
1728
|
+
|
|
1729
|
+
type GeneratedAudio = {
|
|
1730
|
+
type: "base64";
|
|
1731
|
+
base64: string; // base64-encoded audio bytes
|
|
1732
|
+
mediaType: string; // IANA type, e.g. "audio/mpeg", "audio/wav"
|
|
1733
|
+
};
|
|
1734
|
+
```
|
|
1735
|
+
|
|
1736
|
+
`GeneratedAudio` is a discriminated union with a single `base64` variant today — the union leaves room for a future hosted-`url` variant without a breaking change, so always branch on `audio.type` rather than assuming `base64`.
|
|
1737
|
+
|
|
1738
|
+
## Generation options (provider-neutral)
|
|
1739
|
+
|
|
1740
|
+
```ts
|
|
1741
|
+
await ai.speech({
|
|
1742
|
+
model,
|
|
1743
|
+
text: "Welcome aboard. Let's get you set up.",
|
|
1744
|
+
voice: "verse", // voice id/name; overrides the model's default
|
|
1745
|
+
format: "wav", // "mp3" | "opus" | "aac" | "flac" | "wav" | "pcm"
|
|
1746
|
+
speed: 1.25, // playback multiplier (OpenAI 0.25–4.0)
|
|
1747
|
+
instructions: "calm, warm", // tone/delivery steering (gpt-4o-mini-tts only)
|
|
1748
|
+
signal, // AbortSignal
|
|
1749
|
+
observe: collector, // route the report to an Observer (panoptic), like agents
|
|
1750
|
+
sessionId: "onboarding-42", // group into a session for flat cost/trace queries
|
|
1751
|
+
options: { /* provider passthrough */ },
|
|
1752
|
+
});
|
|
1753
|
+
```
|
|
1754
|
+
|
|
1755
|
+
Each adapter maps the options its API supports and forwards `options` verbatim. On OpenAI the container defaults to `mp3` (→ `audio/mpeg`); `speed` and `instructions` are only sent when set, and the default voice is `alloy` when neither the call nor the model config supplies one.
|
|
1756
|
+
|
|
1757
|
+
## OpenAI — tts-1 (per-character) + gpt-4o-mini-tts (per-token)
|
|
1758
|
+
|
|
1759
|
+
```ts
|
|
1760
|
+
import { ai } from "@warlock.js/ai";
|
|
1761
|
+
import { OpenAISDK } from "@warlock.js/ai-openai";
|
|
1762
|
+
|
|
1763
|
+
const openai = new OpenAISDK({ apiKey: process.env.OPENAI_API_KEY! });
|
|
1764
|
+
|
|
1765
|
+
// tts-1 / tts-1-hd — billed per INPUT CHARACTER.
|
|
1766
|
+
const classic = openai.speech({ name: "tts-1", voice: "alloy", pricing: { perMillionCharacters: 15 } });
|
|
1767
|
+
|
|
1768
|
+
// gpt-4o-mini-tts — billed per TOKEN like a chat model; supports `instructions`.
|
|
1769
|
+
const steered = openai.speech({ name: "gpt-4o-mini-tts", pricing: { input: 0.6, output: 12 } });
|
|
1770
|
+
|
|
1771
|
+
const { data } = await ai.speech({ model: steered, text: "Read this warmly.", instructions: "gentle" });
|
|
1772
|
+
```
|
|
1773
|
+
|
|
1774
|
+
## Cost-truth — one rollup, two metering models
|
|
1775
|
+
|
|
1776
|
+
`ai.speech` fills `usage.cost` (a USD breakdown) so TTS spend folds into the **same** `Usage.cost` rollup as text — no second accounting path:
|
|
1777
|
+
|
|
1778
|
+
- **Per-character** (`tts-1` / `tts-1-hd`): `{ perMillionCharacters }` × `report.characters` → `cost.input`. The Speech API reports no token usage, so `usage` tokens stay `{ 0, 0, 0 }` and spend is priced entirely from the input character count.
|
|
1779
|
+
- **Token-metered** (`gpt-4o-mini-tts`): `{ input, output }` USD-per-1M-tokens → standard `computeCost` against the returned token usage.
|
|
1780
|
+
|
|
1781
|
+
Per-character wins when both are set. An unpriced model leaves `usage.cost` **`undefined`** (honest "cost unknown", never a false zero); a pre-priced adapter response is honored, not overwritten.
|
|
1782
|
+
|
|
1783
|
+
## Pattern — order-confirmation voice line in a workflow `run` step
|
|
1784
|
+
|
|
1785
|
+
```ts
|
|
1786
|
+
ai.step({
|
|
1787
|
+
name: "voiceLine",
|
|
1788
|
+
run: async (ctx) => {
|
|
1789
|
+
const { data, error } = await ai.speech({
|
|
1790
|
+
model: openai.speech({ name: "tts-1", voice: "alloy" }),
|
|
1791
|
+
text: `Order ${ctx.steps.order.output.id} confirmed. Thank you!`,
|
|
1792
|
+
format: "mp3",
|
|
1793
|
+
});
|
|
1794
|
+
if (error) throw error; // step retry/backoff handles transient provider faults
|
|
1795
|
+
ctx.state.audio = data.audio; // { type:"base64", base64, mediaType:"audio/mpeg" }
|
|
1796
|
+
},
|
|
1797
|
+
});
|
|
1798
|
+
```
|
|
1799
|
+
|
|
1800
|
+
## Observability
|
|
1801
|
+
|
|
1802
|
+
The completed `SpeechReport` (with `report.characters` and cost/latency attributed to `report.model`) routes to any registered `Observer` (panoptic, OTel, …) through the shared `observe` seam — pass `observe: true` (global), an `Observer` object (flow-local), or rely on observe-all. See [[observe-ai-flows]]. Provider faults surface as typed `AIError`s on `result.error`; see [[handle-ai-errors]].
|
|
1803
|
+
|
|
1804
|
+
## Testing
|
|
1805
|
+
|
|
1806
|
+
`MockSpeechModel(name, responses, pricing?)` is a deterministic `SpeechModelContract` double — no HTTP. Script audio/usage/errors and inspect `model.calls`. `MockSDK({ speechResponses, speechPricing }).speech({ name })` wires the same double behind a full adapter.
|
|
1807
|
+
|
|
1808
|
+
```ts
|
|
1809
|
+
import { MockSpeechModel } from "@warlock.js/ai";
|
|
1810
|
+
import { speech } from "@warlock.js/ai";
|
|
1811
|
+
|
|
1812
|
+
const model = new MockSpeechModel("tts-1", [{}], { perMillionCharacters: 15 });
|
|
1813
|
+
const { data, usage } = await speech({ model, text: "abcdefghij" }); // 10 chars
|
|
1814
|
+
// data.audio → { type:"base64", base64:"AAAA", mediaType:"audio/mpeg" }
|
|
1815
|
+
// usage.cost.input → (10 * 15) / 1_000_000
|
|
1816
|
+
// model.calls[0] records { text, options } for assertions
|
|
1817
|
+
```
|
|
1818
|
+
|
|
1819
|
+
Scripting `[{ error: new ProviderRateLimitError("slow down") }]` drives the never-throws path — `result.error` is the typed error and `result.data` is `undefined`.
|
|
1820
|
+
|
|
1821
|
+
## See also
|
|
1822
|
+
|
|
1823
|
+
- [[transcribe-audio]] — the inverse verb (`ai.transcribe`), audio → text
|
|
1824
|
+
- [[generate-images]] — the sibling image-output verb (`ai.image`)
|
|
1825
|
+
- [[observe-ai-flows]] — routing the `SpeechReport` to panoptic / OTel
|
|
1826
|
+
- [[handle-ai-errors]] — the typed `AIError` taxonomy on `result.error`
|
|
1827
|
+
|
|
1828
|
+
|
|
1829
|
+
## guard-input-output `@warlock.js/ai/guard-input-output/SKILL.md`
|
|
1830
|
+
|
|
1831
|
+
---
|
|
1832
|
+
name: guard-input-output
|
|
1833
|
+
description: 'Build the composed guardrail middleware with @warlock.js/ai-guard and wire it into an agent — `ai.guardrail({ input, output, tool, toolNames, escalation })`. Triggers: `ai.guardrail`, `guard`, `GuardOptions`, `GuardrailVerdict`, `GuardrailDetector`, `GuardrailPhase`, `GuardrailMatch`, `GuardrailViolationError`, `ai.guardrail.topic`, `ai.guardrail.injection`, `topicFilter`, `injectionDetector`, `toolNames`, `forTool`; ''add a guardrail to my agent'', ''block prompt injection'', ''filter banned topics'', ''guard agent input and output'', ''stop the model leaking data into a tool call'', ''scope a detector to one tool''; typical import `import "@warlock.js/ai-guard"` (registers `ai.guardrail`) or `import { guard } from "@warlock.js/ai-guard"`. Skip: PII detection/redaction specifically — `@warlock.js/ai-guard/detect-and-redact-pii/SKILL.md`; routing a block to a human — `@warlock.js/ai-guard/escalate-block-to-human/SKILL.md`; the core middleware pipeline / hook contract — `@warlock.js/ai/run-ai-agent/SKILL.md`.'
|
|
1834
|
+
---
|
|
1835
|
+
|
|
1836
|
+
# Guard agent input, output, and tool args
|
|
1837
|
+
|
|
1838
|
+
`ai.guardrail(...)` is a **middleware factory**. It produces one `AgentMiddleware` that runs your detectors at three hook points and maps each verdict onto the agent pipeline's existing throw / return / record mechanics. Importing the package registers the verb (and its attached detector factories) on the shared `ai` namespace:
|
|
1839
|
+
|
|
1840
|
+
```ts
|
|
1841
|
+
import { ai } from "@warlock.js/ai";
|
|
1842
|
+
import "@warlock.js/ai-guard"; // registers ai.guardrail + ai.guardrail.pii/.topic/.injection/.moderation
|
|
1843
|
+
|
|
1844
|
+
const policy = ai.guardrail({
|
|
1845
|
+
name: "compliance",
|
|
1846
|
+
input: [ai.guardrail.injection({ onMatch: "block" })],
|
|
1847
|
+
output: [ai.guardrail.topic({ deny: [/medical advice/i, "diagnosis"], onMatch: "block" })],
|
|
1848
|
+
});
|
|
1849
|
+
|
|
1850
|
+
const agent = ai.agent({ model, middleware: [policy] });
|
|
1851
|
+
```
|
|
1852
|
+
|
|
1853
|
+
A named-export form is available for callers who prefer not to rely on the augmented namespace:
|
|
1854
|
+
|
|
1855
|
+
```ts
|
|
1856
|
+
import { guard, topic, injection } from "@warlock.js/ai-guard";
|
|
1857
|
+
const policy = guard({ input: [injection({ onMatch: "block" })] });
|
|
1858
|
+
```
|
|
1859
|
+
|
|
1860
|
+
## The three phases
|
|
1861
|
+
|
|
1862
|
+
| Phase | Hook | Inspected text | Set with |
|
|
1863
|
+
|---|---|---|---|
|
|
1864
|
+
| **input** | `trip.before` | the outbound prompt (`extractUserText(ctx.messages)`) | `input: [...]` |
|
|
1865
|
+
| **output** | `trip.after` | `response.content` | `output: [...]` |
|
|
1866
|
+
| **tool** | `tool.before` | `JSON.stringify(toolArgs)` | `tool: [...]` |
|
|
1867
|
+
|
|
1868
|
+
Each phase array runs its detectors in **registration order**; the first non-`allow` verdict decides the action for that phase (short-circuit). A phase you don't configure is inert — a guard with no detectors is a no-op middleware.
|
|
1869
|
+
|
|
1870
|
+
## The verdict model
|
|
1871
|
+
|
|
1872
|
+
A detector inspects text and returns a `GuardrailVerdict`, discriminated by `type` (never `kind`):
|
|
1873
|
+
|
|
1874
|
+
| `type` | Effect |
|
|
1875
|
+
|---|---|
|
|
1876
|
+
| `allow` | Pass to the next detector. |
|
|
1877
|
+
| `redact` | Rewrite the inspected text and continue — **output phase only** (see limitation below). |
|
|
1878
|
+
| `block` | Short-circuit with the existing `GuardrailViolationError`. |
|
|
1879
|
+
| `flag` | Pass, but append a `FlagRecord` into `ctx.state` under `<name>.flags` for a downstream observer (panoptic, the caller). |
|
|
1880
|
+
|
|
1881
|
+
`agent.execute()` **never throws** — a `block` surfaces on `result.error` as a `GuardrailViolationError`, exactly like every other `AIError`. Branch on it after the run:
|
|
1882
|
+
|
|
1883
|
+
```ts
|
|
1884
|
+
const result = await agent.execute(userInput);
|
|
1885
|
+
|
|
1886
|
+
if (result.error instanceof ai.errors.GuardrailViolationError) {
|
|
1887
|
+
// result.error.phase is "input" | "output" | "tool"
|
|
1888
|
+
// result.error.reason / result.error.guardrail carry the detail
|
|
1889
|
+
}
|
|
1890
|
+
```
|
|
1891
|
+
|
|
1892
|
+
## Built-in detectors
|
|
1893
|
+
|
|
1894
|
+
Three zero-dependency detectors ship (a fourth, `moderation`, is an optional `openai` peer — see [`detect-and-redact-pii/SKILL.md`](@warlock.js/ai-guard/detect-and-redact-pii/SKILL.md)):
|
|
1895
|
+
|
|
1896
|
+
- **`ai.guardrail.injection(options?)`** — jailbreak / prompt-injection marker phrases. Extra `markers` (string | RegExp); `onMatch` defaults to `"flag"`, callers commonly use `"block"` on input.
|
|
1897
|
+
- **`ai.guardrail.topic(options)`** — `deny` (string substring | RegExp) and/or `allow` (allow-list miss triggers `onMatch`). `onMatch` is `"block" | "flag"`, default `"block"`.
|
|
1898
|
+
- **`ai.guardrail.pii(options?)`** — PII regex + dictionary (its own skill).
|
|
1899
|
+
|
|
1900
|
+
```ts
|
|
1901
|
+
const policy = ai.guardrail({
|
|
1902
|
+
input: [
|
|
1903
|
+
ai.guardrail.injection({ onMatch: "block", markers: ["ignore previous instructions"] }),
|
|
1904
|
+
ai.guardrail.topic({ deny: ["competitor-name"], onMatch: "block" }),
|
|
1905
|
+
],
|
|
1906
|
+
});
|
|
1907
|
+
```
|
|
1908
|
+
|
|
1909
|
+
## Scope tool detectors to specific tools
|
|
1910
|
+
|
|
1911
|
+
`tool` detectors fire on **every** tool call by default. Set `toolNames` to scope them — the whole middleware is wrapped with the core `forTool(toolNames, mw)` helper so the `tool` hooks fire only for those names; `input` / `output` (`trip`) hooks are unaffected:
|
|
1912
|
+
|
|
1913
|
+
```ts
|
|
1914
|
+
const policy = ai.guardrail({
|
|
1915
|
+
tool: [ai.guardrail.pii({ onMatch: "block" })], // stop PII reaching the tool
|
|
1916
|
+
toolNames: ["send_email", "post_webhook"], // ...only for these tools
|
|
1917
|
+
});
|
|
1918
|
+
|
|
1919
|
+
const agent = ai.agent({ model, tools: [sendEmail, postWebhook, lookup], middleware: [policy] });
|
|
1920
|
+
// `lookup` runs unguarded; `send_email` / `post_webhook` block on PII in their args.
|
|
1921
|
+
```
|
|
1922
|
+
|
|
1923
|
+
A `block` from `tool.before` aborts that tool dispatch and surfaces on `result.error` with `phase: "tool"` — the agent run itself still never crashes.
|
|
1924
|
+
|
|
1925
|
+
## Install order
|
|
1926
|
+
|
|
1927
|
+
A guard is a normal `AgentMiddleware`; registration order is execution order (`before` top-down, `after` bottom-up). The canonical order is `[cache, budget, guardrail, observability]`. A `semanticCache` that short-circuits `trip.before` runs *before* the guard — a cached response then skips the **output** detectors, so place the guard before the cache if you don't trust cached contents.
|
|
1928
|
+
|
|
1929
|
+
## Input-redaction limitation (v1)
|
|
1930
|
+
|
|
1931
|
+
The core `trip.before` hook can only **short-circuit** (return a `ModelResponse`); it cannot rewrite the outbound prompt and continue. So:
|
|
1932
|
+
|
|
1933
|
+
- **Input detectors are `block` / `flag` only.** A `redact` verdict on an input detector is treated as a `block` rather than silently passing an un-redacted prompt.
|
|
1934
|
+
- **Output redaction works** — `trip.after` returns a replacement `ModelResponse` with the rewritten `content`.
|
|
1935
|
+
- **Tool-arg `redact` is also withheld** — it downgrades to a `block` (`reason: "tool-arg-redaction-unsupported"`), because silently rewriting tool arguments changes the call's side-effects unpredictably.
|
|
1936
|
+
|
|
1937
|
+
Lifting the input limitation needs a small, non-breaking core affordance and is deferred.
|
|
1938
|
+
|
|
1939
|
+
## Failure isolation
|
|
1940
|
+
|
|
1941
|
+
A detector's `check()` **rejecting** is an infrastructure fault, not a content violation — it is recorded as a `flag` (`<detector>.error`) into `ctx.state` and the fold **continues** (fail-open). A moderation-API outage degrades to missing annotation, never a failed agent run.
|
|
1942
|
+
|
|
1943
|
+
## See also
|
|
1944
|
+
|
|
1945
|
+
- [`@warlock.js/ai-guard/detect-and-redact-pii/SKILL.md`](@warlock.js/ai-guard/detect-and-redact-pii/SKILL.md) — the `pii` detector (detect/redact/block), the `mask` template, and the optional `moderation` peer.
|
|
1946
|
+
- [`@warlock.js/ai-guard/escalate-block-to-human/SKILL.md`](@warlock.js/ai-guard/escalate-block-to-human/SKILL.md) — routing a `block` to a human-review surface via `escalation.onBlock`.
|
|
1947
|
+
- [`@warlock.js/ai/run-ai-agent/SKILL.md`](@warlock.js/ai/run-ai-agent/SKILL.md) — running the agent, the middleware pipeline, and the `GuardrailViolationError` on `result.error`.
|
|
1948
|
+
|
|
1949
|
+
|
|
1950
|
+
## handle-ai-errors `@warlock.js/ai/handle-ai-errors/SKILL.md`
|
|
1951
|
+
|
|
1952
|
+
---
|
|
1953
|
+
name: handle-ai-errors
|
|
1954
|
+
description: 'Typed AIError hierarchy with stable code strings + coarse category for retry-policy dispatch. execute() never throws — errors surface via result.error (the sole exception: OrchestratorConfigError throws at construction). Triggers: `AIError`, `ProviderRateLimitError`, `ProviderAuthError`, `ContextLengthExceededError`, `ContentFilterError`, `SchemaValidationError`, `ToolExecutionError`, `WorkflowDriftError`, `SupervisorDriftError`, `SupervisorFailedError`, `SupervisorRoutingError`, `OrchestratorFailedError`, `OrchestratorDriftError`, `OrchestratorConfigError`, `OrchestratorCancelledError`, `PlannerFailedError`, `PlannerPlanInvalidError`, `PlannerCancelledError`, `BudgetExceededError`, `GuardrailViolationError`, `error.code`, `error.category`; ''handle ai error'', ''retry on rate limit'', ''branch on error code'', ''ORCHESTRATOR_DRIFT'', ''PLANNER_PLAN_INVALID'', ''build fallback ladder''; typical import `import { AIError } from "@warlock.js/ai"`. Skip: log surfacing — `@warlock.js/ai/log-ai-calls/SKILL.md`; native `try / catch` on raw `openai`.'
|
|
1955
|
+
---
|
|
1956
|
+
|
|
1957
|
+
# Typed errors — `AIError` hierarchy
|
|
1958
|
+
|
|
1959
|
+
Every error surfaced by `@warlock.js/ai` and every adapter package is an `AIError` subclass with a stable `code`. The base extends platform `Error`; it does NOT extend `HttpError`. Plain `Error` never leaks.
|
|
1960
|
+
|
|
1961
|
+
## Two invariants
|
|
1962
|
+
|
|
1963
|
+
1. **`execute()` never throws.** Every `agent.execute()` / `workflow.execute()` resolves with a well-formed result. Failures funnel into `result.error`. Same for `stream.result`.
|
|
1964
|
+
2. **Every error is an `AIError`.** Both core and adapter packages funnel everything through `AIError` subclasses. Branch on `error.code` (stable string) or `instanceof`.
|
|
832
1965
|
|
|
833
1966
|
## Dispatch pattern
|
|
834
1967
|
|
|
@@ -1197,109 +2330,397 @@ description: 'Durable orchestrator stores — ai.checkpoint.{memory,pg,redis}()
|
|
|
1197
2330
|
```ts
|
|
1198
2331
|
import { ai } from "@warlock.js/ai";
|
|
1199
2332
|
|
|
1200
|
-
const orch = ai.orchestrator({
|
|
1201
|
-
name: "support",
|
|
1202
|
-
intents,
|
|
1203
|
-
route,
|
|
1204
|
-
iterate: true,
|
|
1205
|
-
checkpointStore: ai.checkpoint.pg({ client: pgPool }),
|
|
1206
|
-
snapshotStore: ai.snapshot.pg({ client: pgPool }), // a single pg.Pool backs both
|
|
1207
|
-
});
|
|
1208
|
-
```
|
|
2333
|
+
const orch = ai.orchestrator({
|
|
2334
|
+
name: "support",
|
|
2335
|
+
intents,
|
|
2336
|
+
route,
|
|
2337
|
+
iterate: true,
|
|
2338
|
+
checkpointStore: ai.checkpoint.pg({ client: pgPool }),
|
|
2339
|
+
snapshotStore: ai.snapshot.pg({ client: pgPool }), // a single pg.Pool backs both
|
|
2340
|
+
});
|
|
2341
|
+
```
|
|
2342
|
+
|
|
2343
|
+
### Global defaults
|
|
2344
|
+
|
|
2345
|
+
```ts
|
|
2346
|
+
ai.config({
|
|
2347
|
+
defaultCheckpointStore: ai.checkpoint.memory(),
|
|
2348
|
+
defaultSnapshotStore: ai.snapshot.memory(),
|
|
2349
|
+
});
|
|
2350
|
+
```
|
|
2351
|
+
|
|
2352
|
+
Resolution: explicit `checkpointStore` / `snapshotStore` on the config wins, else the matching `ai.config({ default… })`, else undefined. `iterate: true` with no snapshot store resolvable throws `OrchestratorConfigError` at construction.
|
|
2353
|
+
|
|
2354
|
+
## The three drivers
|
|
2355
|
+
|
|
2356
|
+
| Driver | Client | Durable | Cross-process | Fits |
|
|
2357
|
+
|---|---|---|---|---|
|
|
2358
|
+
| `memory()` | none | ❌ | ❌ | dev / tests / single-process; no resume across restarts |
|
|
2359
|
+
| `pg({ client, table?, ttl? })` | dev-supplied `pg.Pool`/`Client` | ✅ | ✅ | production with Postgres |
|
|
2360
|
+
| `redis({ client, prefix?, ttl? })` | dev-supplied `redis` client | ✅ | ✅ | production with Redis |
|
|
2361
|
+
|
|
2362
|
+
`@warlock.js/ai` takes **NO peer dependency** on `pg` or `redis` — you install the client, build it, and pass it in via `{ client }` (anything matching `PgClientLike` / `RedisClientLike`). The store never opens or closes the connection. A single `pg.Pool` can back the cache, the checkpoint store, and the snapshot store at once.
|
|
2363
|
+
|
|
2364
|
+
```ts
|
|
2365
|
+
import { Pool } from "pg";
|
|
2366
|
+
const pool = new Pool({ connectionString: process.env.DATABASE_URL });
|
|
2367
|
+
|
|
2368
|
+
ai.checkpoint.pg({ client: pool, table: "warlock_orchestrator_sessions", ttl: 86_400 });
|
|
2369
|
+
ai.snapshot.pg({ client: pool, table: "warlock_supervisor_snapshots" });
|
|
2370
|
+
|
|
2371
|
+
// redis
|
|
2372
|
+
ai.checkpoint.redis({ client: redisClient, prefix: "warlock:orchestrator", ttl: 86_400 });
|
|
2373
|
+
ai.snapshot.redis({ client: redisClient, prefix: "warlock:snapshot" });
|
|
2374
|
+
```
|
|
2375
|
+
|
|
2376
|
+
Table / prefix names must be safe SQL identifiers (`[A-Za-z_][A-Za-z0-9_]*`) — interpolated into DDL/DML, so anything outside that subset is rejected. Defaults: pg checkpoint table `warlock_orchestrator_sessions`, pg snapshot table `warlock_supervisor_snapshots`, redis prefix `warlock:orchestrator`.
|
|
2377
|
+
|
|
2378
|
+
## Schema is NEVER auto-migrated
|
|
2379
|
+
|
|
2380
|
+
The framework never creates or alters tables. Each pg store exposes `schema()` returning the reference DDL — run it through YOUR migration tool once before use:
|
|
2381
|
+
|
|
2382
|
+
```ts
|
|
2383
|
+
const store = ai.checkpoint.pg({ client: pool });
|
|
2384
|
+
await pool.query(store.schema()); // once, via your migration tooling
|
|
2385
|
+
```
|
|
2386
|
+
|
|
2387
|
+
The memory and redis drivers return an empty `schema()` string (no backing table), so callers can treat `schema()` uniformly.
|
|
2388
|
+
|
|
2389
|
+
## `CheckpointRecord` — the persisted row
|
|
2390
|
+
|
|
2391
|
+
```ts
|
|
2392
|
+
type CheckpointRecord = {
|
|
2393
|
+
orchestrator_name: string; // PK segment 1
|
|
2394
|
+
session_id: string; // PK segment 2
|
|
2395
|
+
turn_index: number; // PK segment 3 — highest is live
|
|
2396
|
+
state: unknown; // post-merge session accumulator (TState)
|
|
2397
|
+
last_route: string | string[] | null;
|
|
2398
|
+
signature: string; // drift fingerprint at write time
|
|
2399
|
+
version: string | null; // config.version tag — metadata only
|
|
2400
|
+
summarized_through: number | null; // exclusive turn index compaction reached
|
|
2401
|
+
lock_acquired_at: string | null; // compaction lock metadata
|
|
2402
|
+
lock_expires_at: string | null;
|
|
2403
|
+
saved_at: string; // ISO write timestamp
|
|
2404
|
+
};
|
|
2405
|
+
```
|
|
2406
|
+
|
|
2407
|
+
Append-only from v1 — `save()` never overwrites a prior `turn_index`. `load(name, sessionId)` returns the latest row (highest `turn_index`), or `undefined` for a session the store has never seen.
|
|
2408
|
+
|
|
2409
|
+
## Store contract methods
|
|
2410
|
+
|
|
2411
|
+
Both stores: `load(...)`, `save(...)`, `delete(...)`, optional `list(...)`, `schema()`.
|
|
2412
|
+
|
|
2413
|
+
- **`list(orchestratorName, prefix?)`** (checkpoint) / **`list(prefix?)`** (snapshot) — enumerate session/run ids for the production boot-drain loop. Optional: stores that can't enumerate omit it.
|
|
2414
|
+
- The orchestrator's **`keepSnapshots`** retention policy lives on the orchestrator config, NOT the store — the orchestrator calls the pg store's `prune()` after a successful `save` when `keepSnapshots` is a finite number; `"all"` skips pruning.
|
|
2415
|
+
|
|
2416
|
+
## Boot-drain pattern
|
|
2417
|
+
|
|
2418
|
+
On startup, resume any session whose last turn was interrupted mid-flight:
|
|
2419
|
+
|
|
2420
|
+
```ts
|
|
2421
|
+
const sessions = await checkpointStore.list?.(orch.name) ?? [];
|
|
2422
|
+
for (const sessionId of sessions) {
|
|
2423
|
+
await orch.resume(sessionId); // null when nothing in flight — harmless
|
|
2424
|
+
}
|
|
2425
|
+
```
|
|
2426
|
+
|
|
2427
|
+
## Distinct from `@warlock.js/cache` snapshot resume
|
|
2428
|
+
|
|
2429
|
+
A bare `ai.supervisor()` / `ai.workflow()` uses a `snapshotStore` for `resume(runId)`. That `SnapshotStore` was promoted from the historical `@warlock.js/cache` `CacheDriver` path. ⚠ The CacheDriver overload is deprecated for one minor — new code wires `ai.snapshot.*` stores. See [`@warlock.js/ai/persist-ai-data/SKILL.md`](@warlock.js/ai/persist-ai-data/SKILL.md) for the supervisor/workflow side and the cache-backed semantic cache.
|
|
2430
|
+
|
|
2431
|
+
## See also
|
|
2432
|
+
|
|
2433
|
+
- [`@warlock.js/ai/run-orchestrator/SKILL.md`](@warlock.js/ai/run-orchestrator/SKILL.md) — the consumer of these stores
|
|
2434
|
+
- [`@warlock.js/ai/persist-ai-data/SKILL.md`](@warlock.js/ai/persist-ai-data/SKILL.md) — supervisor/workflow snapshot resume + the SnapshotStore migration
|
|
2435
|
+
- [`@warlock.js/ai/handle-ai-errors/SKILL.md`](@warlock.js/ai/handle-ai-errors/SKILL.md) — `OrchestratorDriftError` / `OrchestratorConfigError`
|
|
2436
|
+
|
|
2437
|
+
|
|
2438
|
+
## manage-prompts `@warlock.js/ai/manage-prompts/SKILL.md`
|
|
2439
|
+
|
|
2440
|
+
---
|
|
2441
|
+
name: manage-prompts
|
|
2442
|
+
description: 'Unified prompt registry — ai.prompts: one process-wide store of named, versioned systemPrompt(...) builders keyed by name@version. Register by giving a prompt a meta.name (auto-registers), resolve by get(name) / resolve(name, versionOrTag, placeholders) / the inline name@selector form, bulk-register with define(name, versions), pin tags with tag(name, tag, version), compare with diff(name, from, to), round-trip with export() / import(snapshot), and quality-check with a unified validate(target, options) (deterministic missing-placeholder check + optional Nova-safe LLM-as-judge with verdict caching). Compose registered prompts into new ones with systemPrompt().merge(name, { fromVersion }) — provenance recorded in meta.composedFrom. ai.prompt is now a thin FACADE over ai.prompts (BREAKING vs the old standalone registry). Triggers: `ai.prompts`, `ai.prompt`, `PromptsManagerContract`, `PromptsManagerEntry`, `SystemPromptContract`, `SystemPromptMeta`, `SystemPromptMergeOptions`, `PromptsValidateOptions`, `PromptValidationResult`, `PromptValidateTarget`, `PromptTemplateVersion`, `PromptDiff`, `ExportedRegistry`, `defaultPromptsManager`, `prompts()`, `promptKey`, `meta`, `name`, `version`, `composedFrom`, `fromVersion`, `register`, `create`, `get`, `has`, `list`, `versions`, `resolve`, `define`, `tag`, `validate`, `diff`, `export`, `import`, `merge`, `judge`, `judgeCache`; ''register a prompt by name'', ''resolve a prompt by name@version or tag'', ''pin a production tag to a prompt version'', ''diff two prompt versions'', ''export / import the prompt registry'', ''validate a prompt for missing placeholders'', ''merge a registered prompt into another''; typical import `import { ai } from "@warlock.js/ai"`. Skip: composing a single prompt from persona + instruction blocks (the builder itself) — `@warlock.js/ai/write-system-prompt/SKILL.md`; runtime loadable skill bodies — `@warlock.js/ai/use-runtime-skills/SKILL.md`; eval scoring of agent outputs — `@warlock.js/ai/eval-datasets-and-ci/SKILL.md`; competing libs `langfuse` (direct), `promptfoo`.'
|
|
2443
|
+
---
|
|
2444
|
+
|
|
2445
|
+
# `ai.prompts` — the unified prompt registry
|
|
2446
|
+
|
|
2447
|
+
`ai.prompts` is ONE process-wide registry of named, versioned `systemPrompt(...)` builders keyed by `name@version`. A `systemPrompt(input, { name })` (or any `.meta({ name })` rename) auto-registers here; `ai.prompts.get(name)` / `.resolve(name)` read them back; `systemPrompt().merge(name)` folds a registered prompt into a new one. There is exactly **one storage shape** behind the whole prompt surface — a `SystemPromptContract` keyed by `name@version` — and `ai.prompt(...)` is now a thin facade over it (see the migration note below).
|
|
2448
|
+
|
|
2449
|
+
```ts
|
|
2450
|
+
import { ai } from "@warlock.js/ai";
|
|
2451
|
+
|
|
2452
|
+
// Register: any named systemPrompt auto-registers in ai.prompts.
|
|
2453
|
+
ai.systemPrompt("You are support for {{product}}.", { name: "support" });
|
|
2454
|
+
|
|
2455
|
+
// Resolve back — latest version, or a version / pinned tag.
|
|
2456
|
+
ai.prompts.get("support"); // → the SystemPromptContract
|
|
2457
|
+
ai.prompts.resolve("support", undefined, { product: "Warlock" }); // → final string
|
|
2458
|
+
```
|
|
2459
|
+
|
|
2460
|
+
`ai.prompts` is the process-wide default (`defaultPromptsManager()`). For an **isolated** registry (parallel test suites, multi-tenant apps) call the `prompts()` factory — same `PromptsManagerContract`, its own store, no global side effects.
|
|
2461
|
+
|
|
2462
|
+
## Identity — `SystemPromptMeta` (`meta.name` / `version` / `description` / `required` / `composedFrom`)
|
|
2463
|
+
|
|
2464
|
+
A prompt's identity rides on its `meta`. Read it with the no-argument accessor; update it immutably with the one-argument form:
|
|
2465
|
+
|
|
2466
|
+
```ts
|
|
2467
|
+
const base = ai.systemPrompt("You are support.", {
|
|
2468
|
+
name: "support",
|
|
2469
|
+
version: "1",
|
|
2470
|
+
description: "Tier-1 support persona.",
|
|
2471
|
+
required: ["product"],
|
|
2472
|
+
});
|
|
2473
|
+
|
|
2474
|
+
base.meta(); // → { name: "support", version: "1", description, required }
|
|
2475
|
+
const v2 = base.meta({ version: "2" }); // new builder, shallow-merged meta; original untouched
|
|
2476
|
+
```
|
|
2477
|
+
|
|
2478
|
+
- **`name`** — when present, the prompt auto-registers in `ai.prompts` under `name@version`. Anonymous prompts (no `name`) are never registered.
|
|
2479
|
+
- **`version`** — free-form label (`"1"`, `"2025-draft"`). Defaults to the **next integer** for that name when omitted.
|
|
2480
|
+
- **`description`** — human-readable purpose (carried through `export`).
|
|
2481
|
+
- **`required`** — placeholder keys callers must supply; `validate()` reads them.
|
|
2482
|
+
- **`composedFrom`** — deterministic source labels a prompt was merged from (e.g. `["base@2", "global@1"]`). No random suffixes — the same merge always yields the same labels.
|
|
2483
|
+
|
|
2484
|
+
## Register / resolve — `register` / `get` / `resolve` / `has` / `list` / `versions`
|
|
2485
|
+
|
|
2486
|
+
```ts
|
|
2487
|
+
const registry = ai.prompts; // or prompts() for an isolated one
|
|
2488
|
+
|
|
2489
|
+
registry.register(ai.systemPrompt("You are support.", { name: "support" }));
|
|
2490
|
+
registry.versions("support"); // ["1"] — version derived as next integer
|
|
2491
|
+
|
|
2492
|
+
registry.get("support"); // latest SystemPromptContract
|
|
2493
|
+
registry.get("support@1"); // inline name@selector
|
|
2494
|
+
registry.resolve("support", "1", { product: "Warlock" }); // pick version + render in one call
|
|
2495
|
+
|
|
2496
|
+
registry.has("support"); // boolean
|
|
2497
|
+
registry.list(); // every registered name, first-seen order
|
|
2498
|
+
```
|
|
2499
|
+
|
|
2500
|
+
- **Version selection** — `get(name)` / `resolve(name)` return the **latest** by insertion order; pass a version label, a pinned tag, or fold it into the first arg as `name@selector` (`get("support@1")`, `resolve("support@production")`).
|
|
2501
|
+
- **Duplicates** — re-registering the same `name@version` throws `InvalidRequestError` **unless** the content is byte-identical (idempotent re-registration is a no-op).
|
|
2502
|
+
- **Unknown name / version / tag** → `InvalidRequestError`.
|
|
2503
|
+
- `register()` throws if the prompt has no `meta.name`.
|
|
2504
|
+
|
|
2505
|
+
## `create()` — build + register in one entry point
|
|
2506
|
+
|
|
2507
|
+
`ai.prompts.create(input?, meta?)` is a documented alias of `ai.systemPrompt(...)` — identical input forms (no arg → empty builder; a string → one instruction; an array of blocks → verbatim). Pass `meta.name` to auto-register, so authoring and lookup read side-by-side:
|
|
2508
|
+
|
|
2509
|
+
```ts
|
|
2510
|
+
ai.prompts.create("You are support for {{product}}.", { name: "support" });
|
|
2511
|
+
ai.prompts.resolve("support", undefined, { product: "Warlock" });
|
|
2512
|
+
```
|
|
2513
|
+
|
|
2514
|
+
## `define()` — bulk-register many versions
|
|
2515
|
+
|
|
2516
|
+
```ts
|
|
2517
|
+
ai.prompts.define("agent", [
|
|
2518
|
+
{ version: "1", template: "You are v1." },
|
|
2519
|
+
{ version: "2", template: [ai.persona("You are Alex."), ai.instruction("Be concise.")] },
|
|
2520
|
+
]);
|
|
2521
|
+
```
|
|
2522
|
+
|
|
2523
|
+
A `PromptTemplateVersion`'s `template` is a raw string (wrapped into one instruction block) or an explicit ordered block list (verbatim). Versions register **oldest-first** in array order; the same duplicate / idempotency rule applies per `name@version`. Returns the manager for chaining.
|
|
2524
|
+
|
|
2525
|
+
## `tag()` — pin a moving label to a version
|
|
2526
|
+
|
|
2527
|
+
```ts
|
|
2528
|
+
ai.prompts.tag("agent", "production", "2"); // pin "production" → version 2
|
|
2529
|
+
|
|
2530
|
+
ai.prompts.get("agent", "production"); // resolves through the tag
|
|
2531
|
+
ai.prompts.resolve("agent", "production");
|
|
2532
|
+
ai.prompts.get("agent@production"); // inline form
|
|
2533
|
+
```
|
|
2534
|
+
|
|
2535
|
+
Re-pinning an existing tag moves it. An unknown name / version throws `InvalidRequestError`. Tags survive `export` / `import`.
|
|
2536
|
+
|
|
2537
|
+
## `validate()` — unified deterministic + optional LLM-judge
|
|
2538
|
+
|
|
2539
|
+
```ts
|
|
2540
|
+
const report = await ai.prompts.validate("support", {
|
|
2541
|
+
placeholders: { product: "Warlock" }, // values you intend to supply
|
|
2542
|
+
declare: ["language"], // extra keys to treat as known
|
|
2543
|
+
judge: judgeModel, // optional — turns on the LLM-as-judge pass
|
|
2544
|
+
});
|
|
2545
|
+
|
|
2546
|
+
report.ok; // true iff no required placeholder is missing (DETERMINISTIC verdict alone)
|
|
2547
|
+
report.missing; // placeholder keys referenced with no default, unsupplied, undeclared
|
|
2548
|
+
report.score; // 0..1 — present ONLY when a judge ran and produced a usable verdict
|
|
2549
|
+
report.issues; // advisory judge reasons / a degrade note — present only when a judge was supplied
|
|
2550
|
+
```
|
|
2551
|
+
|
|
2552
|
+
- **Always** runs the deterministic check: every `{{key}}` with no inline default that is neither supplied (`placeholders`), declared (`declare`), nor in the prompt's `meta.required` lands in `missing`; `ok` is `true` iff `missing` is empty.
|
|
2553
|
+
- **`judge`** adds a **Nova-safe** LLM-as-judge quality pass — it **never throws** and degrades to an `issues` note (leaving `score` undefined) on failure, so a flaky judge can **never flip `ok`**.
|
|
2554
|
+
- **`target`** is a registered name (or `name@selector`), a `SystemPromptContract` instance, or a raw prompt string.
|
|
2555
|
+
- **`judgeCache`** (per-call or via the `prompts({ judgeCache })` factory option) memoizes judge verdicts by a content hash of the resolved body + the judge model id — a structural `{ get, set }` subset of `@warlock.js/cache`'s `CacheDriver`, so the cache package stays a strictly **optional** peer.
|
|
2556
|
+
|
|
2557
|
+
`systemPrompt().validate(options?)` is the per-builder sugar — `ai.prompts.validate(this, options)` under the hood, same result shape.
|
|
2558
|
+
|
|
2559
|
+
## `diff()` — block-level version diff
|
|
2560
|
+
|
|
2561
|
+
```ts
|
|
2562
|
+
const diff = ai.prompts.diff("agent", "1", "2");
|
|
2563
|
+
|
|
2564
|
+
diff.identical; // true when both versions have identical blocks in identical order
|
|
2565
|
+
diff.added; // blocks in `to` not at the same position in `from`
|
|
2566
|
+
diff.removed; // blocks in `from` not at the same position in `to`
|
|
2567
|
+
diff.changed; // [{ from, to }] — same position, type/text changed
|
|
2568
|
+
```
|
|
2569
|
+
|
|
2570
|
+
Blocks are matched **positionally**. Unknown name / version → `InvalidRequestError`.
|
|
2571
|
+
|
|
2572
|
+
## `export()` / `import()` — portable JSON round-trip
|
|
2573
|
+
|
|
2574
|
+
```ts
|
|
2575
|
+
const snapshot = ai.prompts.export(); // ExportedRegistry — every name, version, pinned tag, description/required
|
|
2576
|
+
otherRegistry.import(snapshot); // rehydrate (same duplicate / idempotency rule; tags restored)
|
|
2577
|
+
```
|
|
2578
|
+
|
|
2579
|
+
Each version flattens to `{ type, text }` blocks so the registry round-trips without live builder instances — commit a snapshot, ship it, restore it elsewhere.
|
|
2580
|
+
|
|
2581
|
+
## Compose registered prompts — `systemPrompt().merge(name, { fromVersion })`
|
|
2582
|
+
|
|
2583
|
+
`merge` folds another prompt's blocks into a new builder (persona **replaces**, instructions **append**) and records `meta.composedFrom`:
|
|
2584
|
+
|
|
2585
|
+
```ts
|
|
2586
|
+
ai.systemPrompt("Always answer in {{language|English}}.", { name: "global", version: "1" });
|
|
2587
|
+
|
|
2588
|
+
const supportPrompt = ai.systemPrompt("You are support for {{product}}.")
|
|
2589
|
+
.merge("global", { fromVersion: "1" }); // fold the registered prompt by name
|
|
2590
|
+
|
|
2591
|
+
supportPrompt.meta()?.composedFrom; // ["…", "global@1"] — deterministic provenance
|
|
2592
|
+
```
|
|
2593
|
+
|
|
2594
|
+
`merge` accepts three source forms: a pre-built block, another `SystemPromptContract`, or a **registered name** resolved from `ai.prompts` (latest version unless `options.fromVersion` selects another — an unknown name / version throws `InvalidRequestError`).
|
|
2595
|
+
|
|
2596
|
+
## `ai.prompt(...)` — now a thin facade (⚠ breaking vs the old registry)
|
|
2597
|
+
|
|
2598
|
+
`ai.prompt` has **two** call forms, both backed by the unified manager — there is no longer a separate prompt store:
|
|
2599
|
+
|
|
2600
|
+
```ts
|
|
2601
|
+
// (a) Resolve a globally-registered prompt from ai.prompts by name.
|
|
2602
|
+
ai.systemPrompt("You are support.", { name: "support" });
|
|
2603
|
+
const sp = ai.prompt("support"); // → SystemPromptContract (latest)
|
|
2604
|
+
const v1 = ai.prompt("support", "1"); // → a specific version / pinned tag
|
|
2605
|
+
|
|
2606
|
+
// (b) Build an ISOLATED legacy-shaped registry (PromptRegistryContract).
|
|
2607
|
+
const reg = ai.prompt({
|
|
2608
|
+
prompts: [{ name: "summarizer", versions: [{ version: "1", template: "Summarize: {{text}}" }] }],
|
|
2609
|
+
});
|
|
2610
|
+
const resolved = reg.resolve("summarizer", { placeholders: { text } });
|
|
2611
|
+
resolved.toSystemPrompt(); // drop-in for ai.agent({ systemPrompt })
|
|
2612
|
+
```
|
|
2613
|
+
|
|
2614
|
+
**⚠ Migration.** Before unification, `ai.prompt(...)` only built a standalone, self-contained registry with its **own private** storage. It now:
|
|
2615
|
+
|
|
2616
|
+
1. Adds the **string overload** `ai.prompt(name, versionOrTag?)` → resolves from the shared `ai.prompts` manager. (New capability — `ai.prompt("x")` used to be a type error.)
|
|
2617
|
+
2. Backs the **options form** (`ai.prompt({ ... })` → `PromptRegistryContract`) by an internal `PromptsManagerContract`, so its storage shape and validation primitives are now the unified ones. The legacy method surface (`register` / `add` / `versions` / `resolve` / `validate` / `sync` + the `{ score, notes }` report shape) is **unchanged**, and each `ai.prompt({ ... })` call still returns its **own isolated** registry — no shared global state.
|
|
2618
|
+
|
|
2619
|
+
If you only ever called `ai.prompt({ ... })` and used the returned registry, **no code change is needed**. The new behavior is additive: prefer `ai.prompts` (the unified manager) for new code; reach for `ai.prompt({ ... })` only when you want the legacy `ResolvedPrompt` / `toSystemPrompt()` ergonomics or the optional Langfuse sync. The legacy facade's reference — `register` / `add` / `resolve(name, { version, placeholders })` / `validate` (`{ score, notes }`) / `sync()` (lazy `langfuse` peer) — is documented inline in `src/prompt/prompt.ts`.
|
|
2620
|
+
|
|
2621
|
+
## See also
|
|
2622
|
+
|
|
2623
|
+
- [`@warlock.js/ai/write-system-prompt/SKILL.md`](@warlock.js/ai/write-system-prompt/SKILL.md) — the `systemPrompt()` / `persona()` / `instruction()` builder, `.meta()`, and `merge()` this registry stores and composes
|
|
2624
|
+
- [`@warlock.js/ai/eval-datasets-and-ci/SKILL.md`](@warlock.js/ai/eval-datasets-and-ci/SKILL.md) — the eval `judge` scorer `validate()`'s LLM pass reuses
|
|
2625
|
+
- [`@warlock.js/ai/run-ai-agent/SKILL.md`](@warlock.js/ai/run-ai-agent/SKILL.md) — wiring a resolved prompt into an agent, plus the judge-safe agent preset (`ai.agent.judge`)
|
|
2626
|
+
|
|
2627
|
+
|
|
2628
|
+
## observe-ai-flows `@warlock.js/ai/observe-ai-flows/SKILL.md`
|
|
2629
|
+
|
|
2630
|
+
---
|
|
2631
|
+
name: observe-ai-flows
|
|
2632
|
+
description: 'The core Observer seam — a generic, tool-agnostic observability hook every flow routes its completed ExecutionReport through. Covers the per-flow `observe?: boolean | Observer` option on ai.agent / workflow / supervisor / team, the global registry (registerObserver / getObservers / setObserveAll / isObserveAll / clearObservers), resolveObservers / notifyObservers resolution, the opt-in AgentConfig.captureMessages → AgentReport.messages full-history capture, the onConfigApplied dependency-inversion seam, and that @warlock.js/ai-panoptic is the batteries-included Observer. Triggers: `Observer`, `observe`, `registerObserver`, `getObservers`, `setObserveAll`, `isObserveAll`, `clearObservers`, `resolveObservers`, `notifyObservers`, `FlowObserveOption`, `ExecutionReport`, `captureMessages`, `AgentReport.messages`, `CapturedMessage`, `onConfigApplied`, `observeAll`; ''observe an agent run'', ''send finished reports to a collector'', ''capture the full message history'', ''observe every flow by default'', ''wire panoptic / tracing''; typical import `import { ai, registerObserver } from "@warlock.js/ai"`. Skip: structured logging of events — `@warlock.js/ai/log-ai-calls/SKILL.md`; reading the report tree shape (trips / children) — `@warlock.js/ai/run-ai-agent/SKILL.md`; per-call cost / usage rollup — `@warlock.js/ai/handle-ai-errors/SKILL.md`. The batteries-included Observer is the `@warlock.js/ai-panoptic` package.'
|
|
2633
|
+
---
|
|
2634
|
+
|
|
2635
|
+
# The `Observer` seam — generic, tool-agnostic observability
|
|
1209
2636
|
|
|
1210
|
-
|
|
2637
|
+
Core defines a structural `Observer` and a tiny registry; it never imports any observability package (panoptic, OTel, Langfuse, …). A flow that resolves to "observed" hands its completed `ExecutionReport` to every registered observer. An observability tool **implements `Observer` and registers itself**, so `observe: true` / observe-all route reports without coupling core to the tool — the dependency inversion that keeps the two sides decoupled.
|
|
1211
2638
|
|
|
1212
2639
|
```ts
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
});
|
|
2640
|
+
export interface Observer {
|
|
2641
|
+
collect(report: ExecutionReport): void | Promise<void>;
|
|
2642
|
+
}
|
|
1217
2643
|
```
|
|
1218
2644
|
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
## The three drivers
|
|
2645
|
+
`collect` may be sync or async — the flow awaits it. A throw is **swallowed** by the flow (never breaks the run), mirroring the existing `onUsage` / `onComplete` hooks.
|
|
1222
2646
|
|
|
1223
|
-
|
|
1224
|
-
|---|---|---|---|---|
|
|
1225
|
-
| `memory()` | none | ❌ | ❌ | dev / tests / single-process; no resume across restarts |
|
|
1226
|
-
| `pg({ client, table?, ttl? })` | dev-supplied `pg.Pool`/`Client` | ✅ | ✅ | production with Postgres |
|
|
1227
|
-
| `redis({ client, prefix?, ttl? })` | dev-supplied `redis` client | ✅ | ✅ | production with Redis |
|
|
2647
|
+
## Per-flow `observe` option
|
|
1228
2648
|
|
|
1229
|
-
|
|
2649
|
+
`observe?: boolean | Observer` (`FlowObserveOption`) is accepted on **`ai.agent`, `ai.workflow`, `ai.supervisor`, and `ai.team`** (a team forwards it verbatim to the supervisor it desugars into):
|
|
1230
2650
|
|
|
1231
2651
|
```ts
|
|
1232
|
-
|
|
1233
|
-
const pool = new Pool({ connectionString: process.env.DATABASE_URL });
|
|
1234
|
-
|
|
1235
|
-
ai.checkpoint.pg({ client: pool, table: "warlock_orchestrator_sessions", ttl: 86_400 });
|
|
1236
|
-
ai.snapshot.pg({ client: pool, table: "warlock_supervisor_snapshots" });
|
|
2652
|
+
const collector: Observer = { collect(report) { exporter.send(report); } };
|
|
1237
2653
|
|
|
1238
|
-
//
|
|
1239
|
-
ai.
|
|
1240
|
-
ai.
|
|
2654
|
+
ai.agent({ model, observe: true }); // → the globally registered observers, even if observe-all is off
|
|
2655
|
+
ai.agent({ model, observe: false }); // → opt out entirely, even when observe-all is on
|
|
2656
|
+
ai.agent({ model, observe: collector }); // → a flow-LOCAL collector; only this flow's report, only to it
|
|
2657
|
+
ai.agent({ model }); // → undefined: follow the global observe-all flag
|
|
1241
2658
|
```
|
|
1242
2659
|
|
|
1243
|
-
|
|
2660
|
+
Resolution (`resolveObservers(observe)`):
|
|
1244
2661
|
|
|
1245
|
-
|
|
2662
|
+
- `false` → `[]` (opted out).
|
|
2663
|
+
- `true` → the globally registered observers.
|
|
2664
|
+
- an `Observer` object → just that one (flow-local; the global observers are skipped).
|
|
2665
|
+
- `undefined` → the global observers when observe-all is on, otherwise `[]`.
|
|
1246
2666
|
|
|
1247
|
-
|
|
2667
|
+
`notifyObservers(observe, report)` routes a completed report to each resolved observer, awaiting each `collect` (so async exporters finish before the flow returns) and swallowing any throw. The object form is typed as the structural `Observer` (NOT a panoptic-specific type), so a panoptic flow-local collector — which implements `Observer` — can be passed directly.
|
|
2668
|
+
|
|
2669
|
+
## The global registry
|
|
1248
2670
|
|
|
1249
2671
|
```ts
|
|
1250
|
-
|
|
1251
|
-
|
|
2672
|
+
import {
|
|
2673
|
+
registerObserver, getObservers, setObserveAll, isObserveAll, clearObservers,
|
|
2674
|
+
} from "@warlock.js/ai";
|
|
2675
|
+
|
|
2676
|
+
registerObserver(collector); // an observability tool registers ONE collector when its config is applied
|
|
2677
|
+
getObservers(); // read-only snapshot of the registered observers (do not mutate)
|
|
2678
|
+
|
|
2679
|
+
setObserveAll(true); // "observe every flow by default" — flows without their own `observe` get observed
|
|
2680
|
+
isObserveAll(); // read the flag (default false — opt-in observability)
|
|
2681
|
+
|
|
2682
|
+
clearObservers(); // test-only: reset observers + the observe-all flag for spec isolation
|
|
1252
2683
|
```
|
|
1253
2684
|
|
|
1254
|
-
|
|
2685
|
+
`observeAll` defaults to `false` (opt-in). A flow that never sets `observe` is observed **only** when observe-all is on; individual flows still opt out with `observe: false`.
|
|
1255
2686
|
|
|
1256
|
-
##
|
|
2687
|
+
## Full-history capture — `captureMessages` → `AgentReport.messages`
|
|
2688
|
+
|
|
2689
|
+
Off by default. When `ai.agent({ captureMessages: true })` is set, the agent normalizes the real assembled turn array onto `AgentReport.messages` as a `CapturedMessage[]`:
|
|
1257
2690
|
|
|
1258
2691
|
```ts
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
session_id: string; // PK segment 2
|
|
1262
|
-
turn_index: number; // PK segment 3 — highest is live
|
|
1263
|
-
state: unknown; // post-merge session accumulator (TState)
|
|
1264
|
-
last_route: string | string[] | null;
|
|
1265
|
-
signature: string; // drift fingerprint at write time
|
|
1266
|
-
version: string | null; // config.version tag — metadata only
|
|
1267
|
-
summarized_through: number | null; // exclusive turn index compaction reached
|
|
1268
|
-
lock_acquired_at: string | null; // compaction lock metadata
|
|
1269
|
-
lock_expires_at: string | null;
|
|
1270
|
-
saved_at: string; // ISO write timestamp
|
|
1271
|
-
};
|
|
2692
|
+
const { report } = await ai.agent({ model, tools, captureMessages: true }).execute("Go");
|
|
2693
|
+
report.messages; // CapturedMessage[] — every role (system/user/assistant/tool), every trip
|
|
1272
2694
|
```
|
|
1273
2695
|
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
## Store contract methods
|
|
2696
|
+
A `CapturedMessage` is a JSON-safe projection: `{ role, content, toolCalls?, toolCallId? }` — `content` is always a string (tool results stringified), assistant turns that triggered tools carry `toolCalls`, tool-result turns carry the `toolCallId` they answer. Unlike `trips[].input` (which stubs non-first trips with `"[tool results]"`), this preserves the **real** turn array. Omitted ⇒ the field is **absent** and the report is byte-for-byte as before. Opt-in because messages can be large and sensitive (full prompts, tool inputs/outputs) — and **required for panoptic full-history capture**.
|
|
1277
2697
|
|
|
1278
|
-
|
|
2698
|
+
## Callback sub-agents nest in the report tree
|
|
1279
2699
|
|
|
1280
|
-
|
|
1281
|
-
- The orchestrator's **`keepSnapshots`** retention policy lives on the orchestrator config, NOT the store — the orchestrator calls the pg store's `prune()` after a successful `save` when `keepSnapshots` is a finite number; `"all"` skips pruning.
|
|
2700
|
+
The `ExecutionReport` an observer receives reflects **full** lineage: a supervisor / team / orchestrator callback that calls `agent.execute()` directly auto-nests `callback → agent → tool` (via an ambient `RunFrame`), so usage / cost roll up and panoptic renders the sub-agent under its callback instead of as a lone `$0` span. No observer-side change is needed — the tree arrives already nested. A team's root span carries `type: "team"` (a first-class `ReportType`, not `"supervisor"`), so observers can distinguish, group, and label team runs as their own type. See [`@warlock.js/ai/run-supervisor/SKILL.md`](@warlock.js/ai/run-supervisor/SKILL.md).
|
|
1282
2701
|
|
|
1283
|
-
##
|
|
2702
|
+
## The config seam — `onConfigApplied`
|
|
1284
2703
|
|
|
1285
|
-
|
|
2704
|
+
An observability tool reacts to its own augmented config slot without core importing it. Core lets tools attach an opaque slot (e.g. `panoptic?`) via declaration merging on `AIConfig`, then fires registered listeners after each `ai.config(...)` merge:
|
|
1286
2705
|
|
|
1287
2706
|
```ts
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
2707
|
+
import { onConfigApplied, getAIConfig } from "@warlock.js/ai";
|
|
2708
|
+
|
|
2709
|
+
onConfigApplied((config) => applyPanopticConfig(config.panoptic)); // react on every config merge
|
|
2710
|
+
applyPanopticConfig(getAIConfig().panoptic); // catch a pre-set config
|
|
1292
2711
|
```
|
|
1293
2712
|
|
|
1294
|
-
|
|
2713
|
+
A misbehaving listener's throw is swallowed (same swallow-on-throw discipline as the observer hooks). This mirrors the `Observer` registry's dependency inversion: a tool flips `setObserveAll(true)` and calls `registerObserver(...)` from inside its `onConfigApplied` listener.
|
|
1295
2714
|
|
|
1296
|
-
|
|
2715
|
+
## The batteries-included Observer
|
|
2716
|
+
|
|
2717
|
+
`@warlock.js/ai-panoptic` is the shipped, full-featured `Observer` — install it, configure it via `ai.config({ panoptic })`, and it registers its collector + (optionally) flips observe-all for you. Core stays dependency-free; this skill documents the seam panoptic plugs into.
|
|
1297
2718
|
|
|
1298
2719
|
## See also
|
|
1299
2720
|
|
|
1300
|
-
- [`@warlock.js/ai/run-
|
|
1301
|
-
- [`@warlock.js/ai/
|
|
1302
|
-
- [`@warlock.js/ai/
|
|
2721
|
+
- [`@warlock.js/ai/run-ai-agent/SKILL.md`](@warlock.js/ai/run-ai-agent/SKILL.md) — the `AgentReport` / `ExecutionReport` tree (`trips`, `children`) an observer receives
|
|
2722
|
+
- [`@warlock.js/ai/log-ai-calls/SKILL.md`](@warlock.js/ai/log-ai-calls/SKILL.md) — event-level structured logging (vs. report-level observation)
|
|
2723
|
+
- [`@warlock.js/ai/run-ai-team/SKILL.md`](@warlock.js/ai/run-ai-team/SKILL.md) — a team inherits `observe` through the supervisor it forwards to
|
|
1303
2724
|
|
|
1304
2725
|
|
|
1305
2726
|
## persist-ai-data `@warlock.js/ai/persist-ai-data/SKILL.md`
|
|
@@ -1646,91 +3067,355 @@ type ModelPricing = {
|
|
|
1646
3067
|
};
|
|
1647
3068
|
```
|
|
1648
3069
|
|
|
1649
|
-
Configure it and every report carries `Usage.cost` — a per-channel breakdown captured at emit time as a historical fact (stored reports stay accurate after the upstream table changes):
|
|
3070
|
+
Configure it and every report carries `Usage.cost` — a per-channel breakdown captured at emit time as a historical fact (stored reports stay accurate after the upstream table changes):
|
|
3071
|
+
|
|
3072
|
+
```ts
|
|
3073
|
+
const { usage } = await ai.agent({ model: openai.model({ name: "gpt-4o-mini" }) }).execute("hi");
|
|
3074
|
+
|
|
3075
|
+
usage.cost; // { input, output, cachedInput?, cachedOutput? } — USD per channel
|
|
3076
|
+
// single scalar total: sum the populated fields, treating undefined as 0.
|
|
3077
|
+
```
|
|
3078
|
+
|
|
3079
|
+
`usage.cost` is `undefined` when no pricing is available — honest absence over false zero. Aggregators merge only defined fields, so one unpriced child never erases a priced sibling's cost.
|
|
3080
|
+
|
|
3081
|
+
### Token channels (`Usage`) — what each adapter reports
|
|
3082
|
+
|
|
3083
|
+
Beyond `input` / `output` / `total`, `Usage` carries optional sub-channels (undefined when the provider doesn't meter them):
|
|
3084
|
+
|
|
3085
|
+
| Channel | Meaning | Provider source |
|
|
3086
|
+
|---|---|---|
|
|
3087
|
+
| `cachedTokens` | subset of `input` served from prompt cache (READ hit) | OpenAI `prompt_tokens_details.cached_tokens`, Anthropic `cache_read_input_tokens` |
|
|
3088
|
+
| `cacheWriteTokens` | input tokens WRITTEN to the cache this call | Anthropic `cache_creation_input_tokens` (OpenAI does not write-bill) |
|
|
3089
|
+
| `reasoningTokens` | subset of `output` for internal reasoning/thinking | OpenAI `completion_tokens_details.reasoning_tokens`, Anthropic extended-thinking |
|
|
3090
|
+
|
|
3091
|
+
### Driving cache + reasoning per call
|
|
3092
|
+
|
|
3093
|
+
`ModelCallOptions` exposes vendor-neutral controls the agent forwards only when `capabilities` allows:
|
|
3094
|
+
|
|
3095
|
+
```ts
|
|
3096
|
+
await model.complete(messages, {
|
|
3097
|
+
reasoning: { effort: "high", maxTokens: 8_000 }, // effort → OpenAI reasoning_effort; maxTokens → Anthropic thinking budget
|
|
3098
|
+
cacheControl: { breakpoints: 1 }, // WRITE breakpoint → Anthropic cache_control markers
|
|
3099
|
+
});
|
|
3100
|
+
```
|
|
3101
|
+
|
|
3102
|
+
Read-side cache accounting (`Usage.cachedTokens`) works WITHOUT `cacheControl` — it only controls WRITE placement. Adapters whose `capabilities.reasoning` / `.promptCaching` is absent ignore these rather than forwarding unsupported params.
|
|
3103
|
+
|
|
3104
|
+
## Embeddings
|
|
3105
|
+
|
|
3106
|
+
OpenAI ships the first embedder:
|
|
3107
|
+
|
|
3108
|
+
```ts
|
|
3109
|
+
const embedder = openai.embedder({ name: "text-embedding-3-small" });
|
|
3110
|
+
const { vector } = await embedder.embed("Hello, world.");
|
|
3111
|
+
```
|
|
3112
|
+
|
|
3113
|
+
See [`@warlock.js/ai/embed-text/SKILL.md`](@warlock.js/ai/embed-text/SKILL.md).
|
|
3114
|
+
|
|
3115
|
+
## Multi-provider apps
|
|
3116
|
+
|
|
3117
|
+
Pattern: one SDK instance per provider, mix at the call site:
|
|
3118
|
+
|
|
3119
|
+
```ts
|
|
3120
|
+
const openai = new OpenAISDK({ apiKey: process.env.OPENAI_API_KEY! });
|
|
3121
|
+
const openrouter = new OpenAISDK({
|
|
3122
|
+
apiKey: process.env.OPENROUTER_API_KEY!,
|
|
3123
|
+
baseURL: "https://openrouter.ai/api/v1",
|
|
3124
|
+
provider: "openrouter",
|
|
3125
|
+
});
|
|
3126
|
+
|
|
3127
|
+
const fastAgent = ai.agent({ model: openai.model({ name: "gpt-4o-mini" }) });
|
|
3128
|
+
const claudeAgent = ai.agent({ model: openrouter.model({ name: "anthropic/claude-3.5-sonnet" }) });
|
|
3129
|
+
```
|
|
3130
|
+
|
|
3131
|
+
Reports label per-agent provider correctly. Pricing applies per SDK instance.
|
|
3132
|
+
|
|
3133
|
+
## When the adapter changes
|
|
3134
|
+
|
|
3135
|
+
If you switch providers mid-project (e.g. OpenAI → Anthropic):
|
|
3136
|
+
|
|
3137
|
+
1. The agent factory call signature stays the same — `ai.agent({ model: <newSdk>.model({...}) })`.
|
|
3138
|
+
2. Capabilities matter — if the new model doesn't support `structuredOutput` natively, fall back to the soft "respond in JSON only" instruction (framework handles it).
|
|
3139
|
+
3. Errors stay typed — `ProviderAuthError`, `ContextLengthExceededError`, etc. are adapter-agnostic.
|
|
3140
|
+
4. Pricing matrix needs updating per the new provider's rates.
|
|
3141
|
+
|
|
3142
|
+
## See also
|
|
3143
|
+
|
|
3144
|
+
- [`@warlock.js/ai-openai/setup-openai/SKILL.md`](@warlock.js/ai-openai/setup-openai/SKILL.md) — full OpenAI adapter docs
|
|
3145
|
+
- [`@warlock.js/ai/run-ai-agent/SKILL.md`](@warlock.js/ai/run-ai-agent/SKILL.md) — model passed into `ai.agent({...})`
|
|
3146
|
+
- [`@warlock.js/ai/embed-text/SKILL.md`](@warlock.js/ai/embed-text/SKILL.md) — embedder primitive on the SDK
|
|
3147
|
+
- [`@warlock.js/ai/handle-ai-errors/SKILL.md`](@warlock.js/ai/handle-ai-errors/SKILL.md) — adapter error categorization
|
|
3148
|
+
|
|
3149
|
+
|
|
3150
|
+
## rag-loaders-and-stores `@warlock.js/ai/rag-loaders-and-stores/SKILL.md`
|
|
3151
|
+
|
|
3152
|
+
---
|
|
3153
|
+
name: rag-loaders-and-stores
|
|
3154
|
+
description: 'Turn any source into a RagDocument and index it in a production vector store — the document loaders ai.rag.loadText / loadHtml / loadWeb (SSRF-safe via guardedFetch) / loadPdf (lazy pdf-parse peer), plus the swappable stores ai.rag.pgVectorStore({client}) (pgvector + ensureSchema DDL + hnsw/ivfflat index) and ai.rag.cacheVectorStore(driver), both satisfying VectorStoreContract (upsert / query / removeNamespace). Loaders return the exact RagDocument[] that kb.index() consumes — no adapter. Triggers: `ai.rag.loadText`, `ai.rag.loadHtml`, `ai.rag.loadWeb`, `ai.rag.loadPdf`, `loadText`, `loadHtml`, `loadWeb`, `loadPdf`, `ai.rag.pgVectorStore`, `ai.rag.cacheVectorStore`, `pgVectorStore`, `cacheVectorStore`, `VectorStore`, `PgVectorStoreOptions`, `PgVectorStoreInstance`, `ensureSchema`, `schema()`, `RagLoaderResult`, `LoadWebOptions`, `LoadPdfOptions`, `perPage`, `OutboundPolicy`, `guardedFetch`, `hnsw`, `ivfflat`, `pgvector`, `dimensions`, `PgClientLike`, `PDF_PARSE_INSTALL_INSTRUCTIONS`; ''load a website into a knowledge base'', ''index a PDF for RAG'', ''strip HTML to text for embedding'', ''pgvector store for RAG'', ''SSRF-safe document fetch'', ''one document per PDF page'', ''swap the vector store''; typical import `import { ai } from "@warlock.js/ai"`. Skip: the chunk → embed → retrieve → rerank → cite pipeline that consumes these — `@warlock.js/ai/run-ai-rag/SKILL.md`; the raw embedder primitive — `@warlock.js/ai/embed-text/SKILL.md`; cache similarity internals — `@warlock.js/cache/use-cache-similarity/SKILL.md`; competing libs `langchain` loaders, `llamaindex` readers.'
|
|
3155
|
+
---
|
|
3156
|
+
|
|
3157
|
+
# RAG loaders + vector stores — source → `RagDocument` → durable store
|
|
3158
|
+
|
|
3159
|
+
Two feature groups that bracket `ai.rag()`: **loaders** turn a source (string, raw HTML, a URL, or PDF bytes) into the exact `RagDocument[]` shape `kb.index()` consumes, and **stores** are the swappable backends that hold the embeddings. Both live on the `ai.rag.*` namespace — present the moment `@warlock.js/ai` is imported, no side-effect import, no module augmentation.
|
|
3160
|
+
|
|
3161
|
+
## Contract — what each side produces / satisfies
|
|
3162
|
+
|
|
3163
|
+
Every loader returns `RagLoaderResult` — a plain `RagDocument[]` — so a load feeds `index()` with no adapter and callers never branch on arity (one page ⇒ 1 doc, a per-page PDF ⇒ N docs):
|
|
3164
|
+
|
|
3165
|
+
```ts
|
|
3166
|
+
type RagLoaderResult = RagDocument[];
|
|
3167
|
+
type RagDocument = { id: string; text: string; metadata?: Record<string, unknown>; tags?: string[] };
|
|
3168
|
+
```
|
|
3169
|
+
|
|
3170
|
+
Every store satisfies the three-method `VectorStore` contract (a thin narrowing of the cache `similar()` surface — NOT a new engine):
|
|
3171
|
+
|
|
3172
|
+
```ts
|
|
3173
|
+
interface VectorStore {
|
|
3174
|
+
upsert(key: string, value: unknown, vector: number[], tags?: string[]): Promise<void>;
|
|
3175
|
+
query<T>(vector: number[], options: { topK: number; threshold?: number; tags?: string[] }): Promise<{ key: string; value: T; score: number }[]>;
|
|
3176
|
+
removeNamespace(namespace: string): Promise<void>;
|
|
3177
|
+
}
|
|
3178
|
+
```
|
|
3179
|
+
|
|
3180
|
+
## Loaders
|
|
3181
|
+
|
|
3182
|
+
| Loader | Input | Deps | Emits |
|
|
3183
|
+
|---|---|---|---|
|
|
3184
|
+
| `ai.rag.loadText(input, opts?)` | `string` \| `{ id, text }` \| array of either | none | one doc per non-empty item |
|
|
3185
|
+
| `ai.rag.loadHtml(html, opts?)` | raw HTML string | none (regex strip) | one doc, `metadata.title` from `<title>` |
|
|
3186
|
+
| `ai.rag.loadWeb(url, opts?)` | absolute URL | none (uses core `guardedFetch`) | one doc, SSRF-safe fetch |
|
|
3187
|
+
| `ai.rag.loadPdf(bytes, opts?)` | `Buffer` \| `ArrayBuffer` \| `Uint8Array` | lazy `pdf-parse` peer | one doc, or one per page with `perPage: true` |
|
|
3188
|
+
|
|
3189
|
+
Shared options (`RagLoaderOptions`): `id` (source id — falls back to the URL for web, `"document"` otherwise), `metadata` (merged **over** the loader-derived keys, so an explicit `metadata.title` always wins), and `tags` (applied to every chunk for `retrieve({ tags })` filtering). Loader-derived keys: `source`, `loader` (`"text" | "html" | "web" | "pdf"`), plus `title` / `page` / `pageCount` / `contentType` where determinable. Empty / whitespace-only / all-markup inputs emit **no** document — never a no-op record for `index()` to skip.
|
|
3190
|
+
|
|
3191
|
+
```ts
|
|
3192
|
+
import { ai } from "@warlock.js/ai";
|
|
3193
|
+
|
|
3194
|
+
// Bare string, or many records → many distinctly-identified docs:
|
|
3195
|
+
await kb.index(ai.rag.loadText([
|
|
3196
|
+
{ id: "faq-billing", text: "…", metadata: { section: "billing" } },
|
|
3197
|
+
{ id: "faq-shipping", text: "…" },
|
|
3198
|
+
]));
|
|
3199
|
+
|
|
3200
|
+
// Raw HTML → readable text (scripts/styles dropped, entities decoded):
|
|
3201
|
+
await kb.index(ai.rag.loadHtml(rawHtml, { id: "landing", tags: ["marketing"] }));
|
|
3202
|
+
```
|
|
3203
|
+
|
|
3204
|
+
### `loadWeb` is SSRF-safe — never a raw `fetch`
|
|
3205
|
+
|
|
3206
|
+
Every request goes through core's `guardedFetch` under an `OutboundPolicy`. The strict defaults (https-only, private-IP-deny on, 10s timeout, 5 MiB cap) apply even when you pass no `policy`, so an untuned call is already hardened. Tighten it per call:
|
|
3207
|
+
|
|
3208
|
+
```ts
|
|
3209
|
+
await kb.index(await ai.rag.loadWeb("https://docs.example.com/guide", {
|
|
3210
|
+
policy: { hostAllowlist: ["docs.example.com"], maxBytes: 2_000_000, timeoutMs: 5_000 },
|
|
3211
|
+
tags: ["docs"],
|
|
3212
|
+
}));
|
|
3213
|
+
```
|
|
3214
|
+
|
|
3215
|
+
HTML responses run through the same tag-strip pass as `loadHtml`; non-HTML text (`text/plain`, markdown) is used verbatim. `metadata.source` is the resolved URL, `metadata.contentType` the server-reported type. A non-OK response, a policy block, a timeout, or an over-cap body throws `OutboundPolicyError`.
|
|
3216
|
+
|
|
3217
|
+
### `loadPdf` — lazy optional peer, page-precise citations
|
|
3218
|
+
|
|
3219
|
+
`pdf-parse` is an **optional** peer, dynamic-imported on the FIRST `loadPdf` call — importing `@warlock.js/ai` never forces it. When it is absent, the curated `PDF_PARSE_INSTALL_INSTRUCTIONS` string is thrown as a plain `Error` (a missing infra peer, not a content problem), never a raw module-resolution stack trace.
|
|
3220
|
+
|
|
3221
|
+
```ts
|
|
3222
|
+
import { readFile } from "node:fs/promises";
|
|
3223
|
+
|
|
3224
|
+
// Whole PDF → one doc carrying metadata.pageCount:
|
|
3225
|
+
await kb.index(await ai.rag.loadPdf(await readFile("manual.pdf"), { id: "manual" }));
|
|
3226
|
+
|
|
3227
|
+
// One doc per page → citations stay page-precise (id suffixed `#p<n>`, metadata.page set):
|
|
3228
|
+
await kb.index(await ai.rag.loadPdf(bytes, { id: "manual", perPage: true }));
|
|
3229
|
+
```
|
|
3230
|
+
|
|
3231
|
+
An image-only / scanned page has no text layer and is dropped, so a fully-scanned PDF yields zero docs (nothing to embed).
|
|
3232
|
+
|
|
3233
|
+
## Stores
|
|
3234
|
+
|
|
3235
|
+
### `ai.rag.cacheVectorStore(driver)` — adapt any `@warlock.js/cache` driver
|
|
3236
|
+
|
|
3237
|
+
The cache driver **is** the vector store — `upsert → set({ vector, tags })`, `query → similar()`, `removeNamespace → removeNamespace()`. A driver without similarity support throws `CacheUnsupportedError` unchanged (pointing you at the `pg` / `redis` cache drivers).
|
|
3238
|
+
|
|
3239
|
+
```ts
|
|
3240
|
+
import { MemoryCacheDriver } from "@warlock.js/cache";
|
|
3241
|
+
|
|
3242
|
+
const store = ai.rag.cacheVectorStore(new MemoryCacheDriver()); // dev / tests
|
|
3243
|
+
```
|
|
3244
|
+
|
|
3245
|
+
### `ai.rag.pgVectorStore(options)` — production pgvector
|
|
3246
|
+
|
|
3247
|
+
One durable row per chunk keyed by the pipeline's dotted key, the chunk payload in a `JSONB` `value` column, the embedding in a pgvector `vector` column. Pass a live pool (`{ client }` — `@warlock.js/ai` imports **nothing**) or a `{ connectionString }` and let the store lazily `import("pg")` (the optional peer; curated install string on first use if absent). Exactly one of the two is required.
|
|
3248
|
+
|
|
3249
|
+
```ts
|
|
3250
|
+
type PgVectorStoreOptions = {
|
|
3251
|
+
client?: PgClientLike; // a pg.Pool / pg.Client — only `query` is ever called
|
|
3252
|
+
connectionString?: string; // else the store builds its own Pool lazily
|
|
3253
|
+
table?: string; // default "warlock_ai_rag_vectors"; must be a safe identifier
|
|
3254
|
+
dimensions?: number; // vector(N) width in the DDL, default 1536
|
|
3255
|
+
index?: "hnsw" | "ivfflat" | "none"; // ANN strategy, default "hnsw"
|
|
3256
|
+
ivfflatLists?: number; // ivfflat only, default 100
|
|
3257
|
+
};
|
|
3258
|
+
```
|
|
3259
|
+
|
|
3260
|
+
`schema()` (alias `ensureSchema()`) returns the reference migration DDL — `CREATE EXTENSION vector`, the table, a GIN index on `tags`, and the chosen ANN index (`USING hnsw (embedding vector_cosine_ops)`). It **only returns the string**; the framework never auto-migrates — you run it once through your own tool. Index and query MUST use the same embedding model: the `vector(N)` width is fixed at table-creation time from `dimensions`.
|
|
3261
|
+
|
|
3262
|
+
```ts
|
|
3263
|
+
import { Pool } from "pg";
|
|
3264
|
+
|
|
3265
|
+
const pool = new Pool({ connectionString: process.env.DATABASE_URL });
|
|
3266
|
+
const store = ai.rag.pgVectorStore({ client: pool, dimensions: 1536, index: "hnsw" });
|
|
3267
|
+
|
|
3268
|
+
// Once, via your migration tooling — never auto-run:
|
|
3269
|
+
await pool.query(store.ensureSchema());
|
|
3270
|
+
```
|
|
3271
|
+
|
|
3272
|
+
`query()` runs the cosine floor (`threshold`) and `tags` overlap filter **in SQL** (a below-floor row never crosses the wire), orders by `embedding <=> $vec`, caps at `topK`, and maps the pgvector distance back to a `[0,1]` cosine-similarity `score` — the same scale the cache store emits. `removeNamespace()` is a prefix DELETE that escapes `_` / `%` so dropping `ai.rag.docs` never also catches `ai.rag.docs2`.
|
|
3273
|
+
|
|
3274
|
+
## Pattern — a knowledge base from a website, backed by pgvector
|
|
3275
|
+
|
|
3276
|
+
```ts
|
|
3277
|
+
import { Pool } from "pg";
|
|
3278
|
+
import { ai } from "@warlock.js/ai";
|
|
3279
|
+
import { OpenAISDK } from "@warlock.js/ai-openai";
|
|
3280
|
+
|
|
3281
|
+
const openai = new OpenAISDK({ apiKey: process.env.OPENAI_API_KEY! });
|
|
3282
|
+
const pool = new Pool({ connectionString: process.env.DATABASE_URL });
|
|
3283
|
+
|
|
3284
|
+
const store = ai.rag.pgVectorStore({ client: pool, dimensions: 1536 });
|
|
3285
|
+
await pool.query(store.ensureSchema()); // once at boot / migration
|
|
3286
|
+
|
|
3287
|
+
const kb = ai.rag({
|
|
3288
|
+
name: "docs",
|
|
3289
|
+
embedder: openai.embedder({ name: "text-embedding-3-small" }), // 1536 dims — matches the DDL
|
|
3290
|
+
store,
|
|
3291
|
+
});
|
|
3292
|
+
|
|
3293
|
+
// Crawl a few pages (SSRF-safe) + a spec PDF into the same namespace:
|
|
3294
|
+
for (const url of ["https://docs.example.com/intro", "https://docs.example.com/config"]) {
|
|
3295
|
+
await kb.index(await ai.rag.loadWeb(url, { policy: { hostAllowlist: ["docs.example.com"] }, tags: ["docs"] }));
|
|
3296
|
+
}
|
|
3297
|
+
await kb.index(await ai.rag.loadPdf(await readFile("spec.pdf"), { id: "spec", perPage: true, tags: ["spec"] }));
|
|
3298
|
+
|
|
3299
|
+
// Now retrieve — every hit's citation traces back to its source URL / page:
|
|
3300
|
+
const { chunks } = await kb.retrieve("how do I configure caching?", { topK: 4, tags: ["docs"] });
|
|
3301
|
+
```
|
|
3302
|
+
|
|
3303
|
+
The `embedder`'s `dimensions` MUST equal the store's `dimensions` — a mismatch is a runtime insert failure at the pgvector column, not a type error.
|
|
3304
|
+
|
|
3305
|
+
## Cost + testing
|
|
3306
|
+
|
|
3307
|
+
- **Loaders are cheap.** `loadText` / `loadHtml` are zero-dependency string passes; `loadWeb` costs one guarded HTTP round-trip; `loadPdf` costs the `pdf-parse` parse. **None embed** — embedding cost lands entirely in `kb.index()` (batched, 96 texts per `embedMany` call). The token spend is per chunk, so `perPage` PDFs and finer chunking mean more, smaller vectors.
|
|
3308
|
+
- **`pgVectorStore` construction is synchronous and does no I/O** — the `pg` import + pool build are deferred to first `query`/`upsert`. Table-name validation (`/^[A-Za-z_][A-Za-z0-9_]*$/`) throws at construction, so a `table: "bad; DROP TABLE x"` fails fast.
|
|
3309
|
+
- **Unit-test loaders with fixtures** (a stubbed `policy.fetch` for `loadWeb`, `vi.mock("pdf-parse")` for `loadPdf` — the literal specifier is mockable). Test stores against a `FakePgClient` implementing `{ query }`, or `cacheVectorStore(new MemoryCacheDriver())` for a real end-to-end index/retrieve with no external service.
|
|
3310
|
+
|
|
3311
|
+
## See also
|
|
3312
|
+
|
|
3313
|
+
- [[run-ai-rag]] — the chunk → embed → retrieve → rerank → cite pipeline that **consumes** these loaders and stores (`ai.rag({ embedder, store })`, `index()` / `retrieve()`).
|
|
3314
|
+
- [[embed-text]] — the `sdk.embedder` primitive whose `dimensions` must match the store's `vector(N)` width.
|
|
3315
|
+
- [`@warlock.js/cache/use-cache-similarity/SKILL.md`](@warlock.js/cache/use-cache-similarity/SKILL.md) — the cache driver `cacheVectorStore` adapts.
|
|
3316
|
+
|
|
3317
|
+
|
|
3318
|
+
## record-replay-llm `@warlock.js/ai/record-replay-llm/SKILL.md`
|
|
3319
|
+
|
|
3320
|
+
---
|
|
3321
|
+
name: record-replay-llm
|
|
3322
|
+
description: 'Deterministic, offline LLM tests with ai.vcr(model,{path,mode}) — a record/replay decorator over ANY ModelContract that intercepts only complete()/stream(), delegates name/provider/capabilities/pricing to the inner model, and hashes each request against a JSON cassette on disk. Covers the three modes (record / replay / auto), the cassette format, save(), VcrCassetteMissError, streaming round-trip, hashOptions, and composing below fallbackModel. Triggers: `ai.vcr`, `vcr`, `VcrModel`, `VcrOptions`, `VcrMode`, `Cassette`, `CassetteEntry`, `VcrCassetteMissError`, `hashRequest`, `DEFAULT_HASH_OPTIONS`, `mode`, `path`, `hashOptions`, `save`, `cassette`, record, replay, cassette; ''record LLM responses for tests'', ''replay model calls offline in CI'', ''deterministic agent test without hitting the provider'', ''cassette for model calls''; typical import `import { ai } from "@warlock.js/ai"`. Skip: eval scoring + regression gating — `@warlock.js/ai/eval-datasets-and-ci/SKILL.md`; the Vitest matchers + mockRouter — `@warlock.js/ai/ai-dx-helpers/SKILL.md`; choosing a provider adapter — `@warlock.js/ai/pick-ai-provider/SKILL.md`; competing libs `nock`, `polly.js`.'
|
|
3323
|
+
---
|
|
3324
|
+
|
|
3325
|
+
# `ai.vcr()` — record / replay any model
|
|
3326
|
+
|
|
3327
|
+
`ai.vcr(model, { path, mode })` wraps any `ModelContract` in a record/replay decorator backed by a JSON cassette on disk. It intercepts only `complete()` / `stream()` — the single seam every agent trip funnels through — and delegates `name`, `provider`, `capabilities`, and `pricing` to the inner model untouched, so cost accounting and capability detection are unchanged. Depends only on `ModelContract`, so it works with **any** adapter.
|
|
3328
|
+
|
|
3329
|
+
## Shape
|
|
3330
|
+
|
|
3331
|
+
```ts
|
|
3332
|
+
import { ai } from "@warlock.js/ai";
|
|
3333
|
+
import { OpenAISDK } from "@warlock.js/ai-openai";
|
|
3334
|
+
|
|
3335
|
+
const openai = new OpenAISDK({ apiKey: process.env.OPENAI_API_KEY! });
|
|
3336
|
+
const live = openai.model({ name: "gpt-4o-mini" });
|
|
1650
3337
|
|
|
1651
|
-
|
|
1652
|
-
|
|
3338
|
+
const model = ai.vcr(live, {
|
|
3339
|
+
path: "./cassettes/support.json", // cassette file (JSON); read on construct, written on save()
|
|
3340
|
+
mode: "auto", // "record" | "replay" | "auto" (default)
|
|
3341
|
+
});
|
|
1653
3342
|
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
```
|
|
3343
|
+
const agent = ai.agent({ model, systemPrompt: "..." });
|
|
3344
|
+
const result = await agent.execute("hi");
|
|
1657
3345
|
|
|
1658
|
-
|
|
3346
|
+
await model.save(); // first run records; later runs replay deterministically
|
|
3347
|
+
```
|
|
1659
3348
|
|
|
1660
|
-
|
|
3349
|
+
The returned `VcrModel` is a `ModelContract` plus `save(): Promise<void>` and a readonly `cassette` (exposed for assertions).
|
|
1661
3350
|
|
|
1662
|
-
|
|
3351
|
+
## Modes — `VcrMode`
|
|
1663
3352
|
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
| `cacheWriteTokens` | input tokens WRITTEN to the cache this call | Anthropic `cache_creation_input_tokens` (OpenAI does not write-bill) |
|
|
1668
|
-
| `reasoningTokens` | subset of `output` for internal reasoning/thinking | OpenAI `completion_tokens_details.reasoning_tokens`, Anthropic extended-thinking |
|
|
3353
|
+
- **`record`** — always calls the inner model and appends a cassette entry. Never replays (the in-memory cassette starts empty, so a record run never accidentally replays a stale entry). Use to (re)capture a fresh cassette.
|
|
3354
|
+
- **`replay`** — never calls the inner model. A cassette hit returns the stored response / re-yields its chunks / re-throws its error; a **miss throws `VcrCassetteMissError`** — never a silent live call. Use in CI for deterministic, offline tests.
|
|
3355
|
+
- **`auto`** (default) — replay on a hit, record on a miss. The friendliest mode for local dev: records once, replays thereafter.
|
|
1669
3356
|
|
|
1670
|
-
|
|
3357
|
+
## Request hashing — `hashOptions`
|
|
1671
3358
|
|
|
1672
|
-
|
|
3359
|
+
On each call VCR computes a stable hash over `{ messages, picked options }` and looks for a matching `CassetteEntry`. The hashed option fields default to:
|
|
1673
3360
|
|
|
1674
3361
|
```ts
|
|
1675
|
-
|
|
1676
|
-
reasoning: { effort: "high", maxTokens: 8_000 }, // effort → OpenAI reasoning_effort; maxTokens → Anthropic thinking budget
|
|
1677
|
-
cacheControl: { breakpoints: 1 }, // WRITE breakpoint → Anthropic cache_control markers
|
|
1678
|
-
});
|
|
3362
|
+
["temperature", "maxTokens", "responseSchema", "tools", "reasoning"]
|
|
1679
3363
|
```
|
|
1680
3364
|
|
|
1681
|
-
|
|
3365
|
+
(`DEFAULT_HASH_OPTIONS`). `signal` and unknown provider keys are **always excluded**, so an otherwise-identical logical call still matches. `tools` are hashed by name + description + input-schema shape, not object identity. Override `hashOptions` to widen / narrow what counts as "the same request". `hashRequest(messages, options, hashOptions)` is exported for direct use.
|
|
1682
3366
|
|
|
1683
|
-
##
|
|
3367
|
+
## Cassette format
|
|
1684
3368
|
|
|
1685
|
-
|
|
3369
|
+
A `Cassette` is `{ version: 1, model, provider, entries: CassetteEntry[] }`. Each `CassetteEntry` is `{ requestHash, request: { messages, options? }, ... }` where **exactly one** of `response` / `chunks` / `error` is populated — mirroring the three outcomes of a model call (non-streaming reply, streamed chunk list, or a thrown provider error). The full `request` is stored verbatim for human readability and so the cassette can be re-hashed if the hashing format ever changes.
|
|
3370
|
+
|
|
3371
|
+
## Streaming round-trip
|
|
1686
3372
|
|
|
1687
3373
|
```ts
|
|
1688
|
-
const
|
|
1689
|
-
|
|
3374
|
+
for await (const chunk of model.stream(messages)) {
|
|
3375
|
+
// record mode: buffers each chunk into entry.chunks[] while re-emitting
|
|
3376
|
+
// replay mode: re-yields the stored chunks in order (delta / tool-call / done sequence)
|
|
3377
|
+
}
|
|
3378
|
+
await model.save();
|
|
1690
3379
|
```
|
|
1691
3380
|
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
## Multi-provider apps
|
|
3381
|
+
Recorded chunks reproduce the exact `delta` / `tool-call` / `done` sequence on replay; a recorded error is re-thrown.
|
|
1695
3382
|
|
|
1696
|
-
|
|
3383
|
+
## `VcrCassetteMissError`
|
|
1697
3384
|
|
|
1698
3385
|
```ts
|
|
1699
|
-
|
|
1700
|
-
const openrouter = new OpenAISDK({
|
|
1701
|
-
apiKey: process.env.OPENROUTER_API_KEY!,
|
|
1702
|
-
baseURL: "https://openrouter.ai/api/v1",
|
|
1703
|
-
provider: "openrouter",
|
|
1704
|
-
});
|
|
3386
|
+
import { VcrCassetteMissError } from "@warlock.js/ai";
|
|
1705
3387
|
|
|
1706
|
-
|
|
1707
|
-
|
|
3388
|
+
try {
|
|
3389
|
+
await vcrModel.complete(messages);
|
|
3390
|
+
} catch (error) {
|
|
3391
|
+
if (error instanceof VcrCassetteMissError) {
|
|
3392
|
+
console.error("Re-record the cassette:", error.path, error.requestHash);
|
|
3393
|
+
}
|
|
3394
|
+
}
|
|
1708
3395
|
```
|
|
1709
3396
|
|
|
1710
|
-
|
|
3397
|
+
Thrown only in `replay` mode on a miss (code `"VCR_CASSETTE_MISS"`). It carries the looked-up `requestHash` and the cassette `path` so a failing CI run names exactly which call was not recorded. Extends `AIError` directly (not `ProviderError`) — a miss is a harness/config failure, not a provider failure. **The whole point:** `replay` never falls back to a live call, which would silently re-introduce non-determinism into a test that asked for the opposite. Re-record by running once in `record` / `auto`.
|
|
1711
3398
|
|
|
1712
|
-
##
|
|
3399
|
+
## `save()` — flush new entries
|
|
1713
3400
|
|
|
1714
|
-
|
|
3401
|
+
`save()` writes newly recorded entries to `path`. It is a **no-op when nothing was recorded** (pure replay, or a record/auto run that only hit cached entries), so calling it unconditionally is safe.
|
|
1715
3402
|
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
4. Pricing matrix needs updating per the new provider's rates.
|
|
3403
|
+
## Composition
|
|
3404
|
+
|
|
3405
|
+
VCR composes **below** `ai.fallbackModel` and works with any adapter. Wrap the live model in `vcr(...)`, then pass it anywhere a `ModelContract` is accepted (agent, planner, reranker, judge).
|
|
1720
3406
|
|
|
1721
3407
|
## See also
|
|
1722
3408
|
|
|
1723
|
-
- [`@warlock.js/ai
|
|
1724
|
-
- [`@warlock.js/ai/
|
|
1725
|
-
- [`@warlock.js/ai/
|
|
1726
|
-
- [`@warlock.js/ai/handle-ai-errors/SKILL.md`](@warlock.js/ai/handle-ai-errors/SKILL.md) — adapter error categorization
|
|
3409
|
+
- [`@warlock.js/ai/eval-datasets-and-ci/SKILL.md`](@warlock.js/ai/eval-datasets-and-ci/SKILL.md) — pair a cassette with a dataset for fully offline eval CI
|
|
3410
|
+
- [`@warlock.js/ai/ai-dx-helpers/SKILL.md`](@warlock.js/ai/ai-dx-helpers/SKILL.md) — `ai.mockRouter` + Vitest matchers for the rest of the test surface
|
|
3411
|
+
- [`@warlock.js/ai/pick-ai-provider/SKILL.md`](@warlock.js/ai/pick-ai-provider/SKILL.md) — the adapters whose models VCR wraps
|
|
1727
3412
|
|
|
1728
3413
|
|
|
1729
3414
|
## run-ai-agent `@warlock.js/ai/run-ai-agent/SKILL.md`
|
|
1730
3415
|
|
|
1731
3416
|
---
|
|
1732
3417
|
name: run-ai-agent
|
|
1733
|
-
description: 'Build agents with ai.agent({...}) — the single-LLM-turn primitive. Covers execute / stream, attachments, structured output, placeholders, events, agent.eval scoring, and auto-adapting raw executables in tools:[]. Triggers: `ai.agent`, `agent.execute`, `agent.stream`, `agent.eval`, `AgentResult`, `AgentReport`, `AgentToolEntry`, `streamingToolGuard`, `attachments`, `repair`, `maxTrips`, `sessionId`; ''run an agent'', ''stream an agent response'', ''structured output schema'', ''pass image to agent'', ''evaluate an agent'', ''put a supervisor in tools'', ''cancel an agent run''; typical import `import { ai } from "@warlock.js/ai"`. Skip: tool definition — `@warlock.js/ai/define-ai-tool/SKILL.md`; workflows — `@warlock.js/ai/run-ai-workflow/SKILL.md`; eval matchers / batch / fallback detail — `@warlock.js/ai/ai-dx-helpers/SKILL.md`; competing libs `langchain`, `ai` (Vercel), raw `openai`.'
|
|
3418
|
+
description: 'Build agents with ai.agent({...}) — the single-LLM-turn primitive. Covers execute / stream, attachments, structured output, placeholders, events, agent.eval scoring, the judge-safe preset for resilient LLM-as-judge / verdict classifiers (ai.agent.judge / judge: true — lenient JSON parse + repair + never-throw, for Nova-class models), and auto-adapting raw executables in tools:[]. Triggers: `ai.agent`, `ai.agent.judge`, `agent.execute`, `agent.stream`, `agent.eval`, `AgentResult`, `AgentReport`, `AgentToolEntry`, `JudgeConfig`, `JudgeAgentConfig`, `judge`, `repairAttempts`, `streamingToolGuard`, `attachments`, `repair`, `maxTrips`, `sessionId`, `spawnSubAgent`, `SpawnSubAgentSpec`; ''run an agent'', ''stream an agent response'', ''structured output schema'', ''pass image to agent'', ''evaluate an agent'', ''LLM-as-judge that survives malformed JSON'', ''grade with a Nova model without crashing'', ''put a supervisor in tools'', ''cancel an agent run'', ''spawn a one-shot sub-agent with a per-task budget''; typical import `import { ai } from "@warlock.js/ai"`. Skip: tool definition — `@warlock.js/ai/define-ai-tool/SKILL.md`; workflows — `@warlock.js/ai/run-ai-workflow/SKILL.md`; eval matchers / batch / fallback detail — `@warlock.js/ai/ai-dx-helpers/SKILL.md`; competing libs `langchain`, `ai` (Vercel), raw `openai`.'
|
|
1734
3419
|
---
|
|
1735
3420
|
|
|
1736
3421
|
# `ai.agent()` — single-turn primitive
|
|
@@ -1913,6 +3598,31 @@ await myAgent.execute(input, {
|
|
|
1913
3598
|
|
|
1914
3599
|
Disabled by default. Each repair attempt counts against `maxTrips`.
|
|
1915
3600
|
|
|
3601
|
+
## `judge` preset — resilient LLM-as-judge / verdict classifiers
|
|
3602
|
+
|
|
3603
|
+
For graders and verdict classifiers running on models that emit **corrupted** structured output — notably the Amazon Nova family, which wraps verdicts in fenced ` ```json ` blocks, prepends prose, or trails commentary — set `judge: true` (or a `JudgeConfig`). It turns on three behaviors at once:
|
|
3604
|
+
|
|
3605
|
+
```ts
|
|
3606
|
+
const grader = ai.agent.judge({
|
|
3607
|
+
model: nova.model({ name: "amazon.nova-pro-v1:0" }),
|
|
3608
|
+
systemPrompt: "Grade the answer. Respond with JSON only.",
|
|
3609
|
+
output: verdictSchema,
|
|
3610
|
+
});
|
|
3611
|
+
|
|
3612
|
+
const result = await grader.execute(prompt);
|
|
3613
|
+
if (result.error) {
|
|
3614
|
+
// graceful default — the judge couldn't produce a clean verdict
|
|
3615
|
+
}
|
|
3616
|
+
```
|
|
3617
|
+
|
|
3618
|
+
1. **Repair** — a couple of re-ask attempts by default (`repairAttempts`, defaults to `2`; bounded by `maxTrips`) when the verdict fails to parse / validate. The caller's per-call `options.repair` still wins.
|
|
3619
|
+
2. **Lenient verdict parsing** — extracts the first balanced JSON object / array (tolerating fenced blocks + surrounding prose) instead of the strict parser.
|
|
3620
|
+
3. **Never throws on a parse miss** — even an unparseable verdict yields a well-formed result (`result.error` populated, `result.data` undefined), so a flaky judge degrades instead of crashing the flow.
|
|
3621
|
+
|
|
3622
|
+
`ai.agent.judge(config, judge?)` is sugar for `ai.agent({ ...config, judge })`; the bare `ai.agent({ judge: true })` option does the same. `judge: {}` ≡ `judge: true` (every field falls back to its resilient default); `judge: { repairAttempts: 0 }` keeps the lenient parser + never-throw guarantee but disables repair.
|
|
3623
|
+
|
|
3624
|
+
**Trade-off — resilience over strictness.** The lenient parse can recover JSON the strict parser would (correctly) reject — leave `judge` **off** for normal structured output, where a hard parse failure is a useful signal. Off by default; omitting it parses strictly and never auto-enables repair, byte-for-byte as before. (This is the same Nova-safe judge the unified prompt `validate()` uses — see [`@warlock.js/ai/manage-prompts/SKILL.md`](@warlock.js/ai/manage-prompts/SKILL.md).)
|
|
3625
|
+
|
|
1916
3626
|
## Pattern — image attachments
|
|
1917
3627
|
|
|
1918
3628
|
```ts
|
|
@@ -1930,102 +3640,374 @@ attachments: [
|
|
|
1930
3640
|
];
|
|
1931
3641
|
```
|
|
1932
3642
|
|
|
1933
|
-
Model must declare `capabilities.vision`. OpenAI adapter auto-infers from name; override with `openai.model({ name, vision: true })`.
|
|
3643
|
+
Model must declare `capabilities.vision`. OpenAI adapter auto-infers from name; override with `openai.model({ name, vision: true })`.
|
|
3644
|
+
|
|
3645
|
+
## Pattern — streaming
|
|
3646
|
+
|
|
3647
|
+
```ts
|
|
3648
|
+
const stream = myAgent.stream(input);
|
|
3649
|
+
|
|
3650
|
+
for await (const event of stream) {
|
|
3651
|
+
if (event.type === "agent.trip.streaming") {
|
|
3652
|
+
process.stdout.write(event.delta);
|
|
3653
|
+
}
|
|
3654
|
+
}
|
|
3655
|
+
|
|
3656
|
+
const result = await stream.result;
|
|
3657
|
+
```
|
|
3658
|
+
|
|
3659
|
+
Or use `.on({ "agent.trip.streaming": ..., "agent.completed": ..., "agent.error": ... })` alongside iteration.
|
|
3660
|
+
|
|
3661
|
+
## Pattern — cancellation
|
|
3662
|
+
|
|
3663
|
+
```ts
|
|
3664
|
+
const ctrl = new AbortController();
|
|
3665
|
+
const resultPromise = myAgent.execute(input, { signal: ctrl.signal });
|
|
3666
|
+
|
|
3667
|
+
setTimeout(() => ctrl.abort("too slow"), 30_000);
|
|
3668
|
+
|
|
3669
|
+
const { error, report } = await resultPromise;
|
|
3670
|
+
if (report.status === "cancelled") {
|
|
3671
|
+
// error is an AgentCancelledError (code "AGENT_CANCELLED",
|
|
3672
|
+
// category "cancelled") carrying `cancelledAt` + `reason`
|
|
3673
|
+
}
|
|
3674
|
+
```
|
|
3675
|
+
|
|
3676
|
+
Between-trip abort is guaranteed. Mid-trip best-effort.
|
|
3677
|
+
|
|
3678
|
+
## Events — dot-notation + 3-tier subscription
|
|
3679
|
+
|
|
3680
|
+
- `agent.starting`, `agent.trip.started`, `agent.trip.streaming`, `agent.trip.completed`
|
|
3681
|
+
- `agent.tool.calling`, `agent.tool.called`, `agent.tool.failed`
|
|
3682
|
+
- `agent.completed`, `agent.error`
|
|
3683
|
+
|
|
3684
|
+
Three subscription tiers — fire in order **factory → instance → per-call**:
|
|
3685
|
+
|
|
3686
|
+
```ts
|
|
3687
|
+
ai.agent({ model, on: { "agent.starting": () => metrics.inc("agent.runs") } });
|
|
3688
|
+
|
|
3689
|
+
const unsubscribe = myAgent.on("agent.error", ({ error }) => logger.error(error));
|
|
3690
|
+
|
|
3691
|
+
await myAgent.execute("go", {
|
|
3692
|
+
on: { "agent.trip.completed": ({ trip }) => console.log(trip.duration) },
|
|
3693
|
+
});
|
|
3694
|
+
```
|
|
3695
|
+
|
|
3696
|
+
Every event payload carries `runId` and `rootRunId`. Same identity fields ride on stream events.
|
|
3697
|
+
|
|
3698
|
+
## `tools: []` — auto-adapt executables
|
|
3699
|
+
|
|
3700
|
+
Each `tools` entry is either a built `ToolContract` (from `ai.tool(...)` or an explicit `.asTool(...)`) OR a **raw executable primitive** (`AgentContract` / `WorkflowInstance` / `SupervisorContract` / orchestrator) — auto-adapted into a `ToolContract` at factory time. The manifest is derived from the executable's `name` + `description` + (optional) `inputSchema`; dispatch flows through its `execute()`.
|
|
3701
|
+
|
|
3702
|
+
```ts
|
|
3703
|
+
const concierge = ai.agent({
|
|
3704
|
+
model,
|
|
3705
|
+
tools: [billingWorkflow, supportSupervisor, lookupTool], // no .asTool() needed
|
|
3706
|
+
});
|
|
3707
|
+
```
|
|
3708
|
+
|
|
3709
|
+
`.asTool()` still works and takes precedence when you need a custom name / schema per use. A supervisor/orchestrator needs `inputSchema` on its config to drop straight into `tools: []`. See [`@warlock.js/ai/define-ai-tool/SKILL.md`](@warlock.js/ai/define-ai-tool/SKILL.md).
|
|
3710
|
+
|
|
3711
|
+
## `agent.eval(options)` — score the agent against a suite
|
|
3712
|
+
|
|
3713
|
+
```ts
|
|
3714
|
+
const report = await myAgent.eval({
|
|
3715
|
+
cases: [
|
|
3716
|
+
{ name: "capital", input: "Capital of Egypt?", expected: "Cairo" },
|
|
3717
|
+
{ name: "tone", input: "Comfort an upset user." }, // judge-scored
|
|
3718
|
+
],
|
|
3719
|
+
scorers: [ai.eval.contains()], // default for cases w/o their own
|
|
3720
|
+
judge: { agent: judgeAgent, rubric: "Score 1.0 only if empathetic." }, // LLM-as-judge fallback
|
|
3721
|
+
passThreshold: 0.5, // default
|
|
3722
|
+
});
|
|
3723
|
+
|
|
3724
|
+
expect(report.passed).toBe(true); // true only when EVERY case passed
|
|
3725
|
+
```
|
|
3726
|
+
|
|
3727
|
+
Each case runs through `execute(input)`; scorer precedence is per-case `scorers` → suite `scorers` → synthesized `judge` (throws at author time if a case resolves none). Built-in scorers on `ai.eval.*`: `exact()`, `contains()`, `predicate(fn)`, `judge(config)`. Full coverage — plus the Vitest matchers (`registerAiMatchers` / `toRouteTo` / `toConverge` / `toPassStep` / `toOutputShape`) — in [`@warlock.js/ai/ai-dx-helpers/SKILL.md`](@warlock.js/ai/ai-dx-helpers/SKILL.md).
|
|
3728
|
+
|
|
3729
|
+
## `ai.spawnSubAgent()` — one-shot delegation with a budget
|
|
3730
|
+
|
|
3731
|
+
`ai.spawnSubAgent(spec)` is a thin wrapper over this same `ai.agent()`: it builds a fresh agent from the spec, optionally attaches a `budget` middleware, runs the `task` once, and returns the `AgentResult`. Not a sandbox or a separate runtime — a spawn is an ordinary new agent (empty conversation, its own tools/prompt). It is a **general** primitive: usable inside a tool, a workflow or planner step, a supervisor intent, or hand-rolled orchestration — it is NOT planner-specific (the planner engine never calls it).
|
|
3732
|
+
|
|
3733
|
+
```ts
|
|
3734
|
+
import { ai } from "@warlock.js/ai";
|
|
3735
|
+
|
|
3736
|
+
const result = await ai.spawnSubAgent({
|
|
3737
|
+
name: "extract-entities",
|
|
3738
|
+
model,
|
|
3739
|
+
task: "Pull every company name from this article: ...",
|
|
3740
|
+
budget: { maxCostUSD: 0.05 }, // per-task spend cap — aborts when crossed
|
|
3741
|
+
output: companiesSchema,
|
|
3742
|
+
});
|
|
3743
|
+
```
|
|
3744
|
+
|
|
3745
|
+
The one field a bare agent config doesn't surface ergonomically is `budget` (`BudgetOptions` — `maxTokens` / `maxCostUSD`), equivalent to `ai.agent({ middleware: [ai.middleware.budget(...)] })` but promoted to a first-class spec field so a delegated subtask can't overrun its cap (distinct from `maxTrips`, which caps round-trips, not spend). The surface is **narrower** than `agent.execute()`: one-shot, with no `history`, `placeholders`, per-call events, or `repair`. The spawned `report` slots under the caller's `report.children[]`, so cost and traces roll up uniformly. Reach for it when you want a named single-use delegation with a hard spend cap; otherwise just build an `ai.agent()` and call it.
|
|
3746
|
+
|
|
3747
|
+
## When NOT to use this primitive
|
|
3748
|
+
|
|
3749
|
+
- Multi-step pipeline with a fixed shape → [`@warlock.js/ai/run-ai-workflow/SKILL.md`](@warlock.js/ai/run-ai-workflow/SKILL.md)
|
|
3750
|
+
- Multi-agent routing with iteration → [`@warlock.js/ai/run-supervisor/SKILL.md`](@warlock.js/ai/run-supervisor/SKILL.md)
|
|
3751
|
+
|
|
3752
|
+
## See also
|
|
3753
|
+
|
|
3754
|
+
- [`@warlock.js/ai/define-ai-tool/SKILL.md`](@warlock.js/ai/define-ai-tool/SKILL.md) — tool wiring + schema validation
|
|
3755
|
+
- [`@warlock.js/ai/write-system-prompt/SKILL.md`](@warlock.js/ai/write-system-prompt/SKILL.md) — persona / instruction builders
|
|
3756
|
+
- [`@warlock.js/ai/handle-ai-errors/SKILL.md`](@warlock.js/ai/handle-ai-errors/SKILL.md) — `AIError` hierarchy
|
|
3757
|
+
|
|
3758
|
+
|
|
3759
|
+
## run-ai-rag `@warlock.js/ai/run-ai-rag/SKILL.md`
|
|
3760
|
+
|
|
3761
|
+
---
|
|
3762
|
+
name: run-ai-rag
|
|
3763
|
+
description: 'Retrieval-augmented generation with ai.rag({...}) — a chunk → embed → vector-store → retrieve → rerank → cite pipeline that reuses ai.embedder + a @warlock.js/cache CacheDriver. Covers index() / retrieve() / clear() / asTool(), chunking strategies (recursive | markdown | sentence | fixed), Citation / RetrievedChunk provenance, and the opt-in rerankers ai.rag.keywordReranker / ai.rag.llmReranker. Triggers: `ai.rag`, `rag.index`, `rag.retrieve`, `rag.clear`, `rag.asTool`, `RagConfig`, `RagDocument`, `RetrieveOptions`, `RetrieveResult`, `RetrievedChunk`, `Citation`, `ChunkOptions`, `ChunkType`, `ai.rag.keywordReranker`, `ai.rag.llmReranker`, `cacheVectorStore`, `VectorStore`, `topK`, `threshold`, `candidates`; ''build a knowledge base'', ''retrieve relevant chunks for a query'', ''cite the source of an answer'', ''chunk markdown for embedding'', ''rerank retrieval results'', ''expose retrieval as a tool''; typical import `import { ai } from "@warlock.js/ai"`. Skip: raw single-string embedding — `@warlock.js/ai/embed-text/SKILL.md`; exact + vector LLM-response cache — `@warlock.js/ai/attach-ai-middleware/SKILL.md` (ai.middleware.semanticCache); tool wiring — `@warlock.js/ai/define-ai-tool/SKILL.md`; competing libs `langchain`, `llamaindex`.'
|
|
3764
|
+
---
|
|
3765
|
+
|
|
3766
|
+
# `ai.rag()` — chunk → embed → retrieve → rerank → cite
|
|
3767
|
+
|
|
3768
|
+
A self-contained retrieval pipeline. It reuses the embedder you already have (`provider.embedder(...)`), a `@warlock.js/cache` vector-capable `CacheDriver` as the store, and the composite-as-tool engine for `asTool()`. Zero new dependencies. `ai.rag` is a native core verb — present the moment `@warlock.js/ai` is imported (no module augmentation, no side-effect import).
|
|
3769
|
+
|
|
3770
|
+
## Factory shape
|
|
3771
|
+
|
|
3772
|
+
```ts
|
|
3773
|
+
import { ai } from "@warlock.js/ai";
|
|
3774
|
+
import { MemoryCacheDriver } from "@warlock.js/cache";
|
|
3775
|
+
import { OpenAISDK } from "@warlock.js/ai-openai";
|
|
3776
|
+
|
|
3777
|
+
const openai = new OpenAISDK({ apiKey: process.env.OPENAI_API_KEY! });
|
|
3778
|
+
|
|
3779
|
+
const kb = ai.rag({
|
|
3780
|
+
name: "docs", // default "rag"
|
|
3781
|
+
embedder: openai.embedder({ name: "text-embedding-3-small" }), // REQUIRED
|
|
3782
|
+
store: new MemoryCacheDriver(), // or ai.config({ defaultStore })
|
|
3783
|
+
namespace: "ai.rag.docs", // default `ai.rag.<name>`
|
|
3784
|
+
chunk: { type: "markdown", size: 800, overlap: 120 }, // index() defaults
|
|
3785
|
+
reranker: ai.rag.keywordReranker(), // OFF by default (cosine-only)
|
|
3786
|
+
retrieve: { topK: 4, threshold: 0.5 }, // default retrieval knobs
|
|
3787
|
+
});
|
|
3788
|
+
```
|
|
3789
|
+
|
|
3790
|
+
Resolution is **loud at construction** (mirrors `ai.memory`):
|
|
3791
|
+
|
|
3792
|
+
- `embedder` is **required** — a provider with no embedder must be caught here, not at first `index()`.
|
|
3793
|
+
- `store` falls back to `ai.config({ defaultStore })`; if neither resolves, the factory throws.
|
|
3794
|
+
|
|
3795
|
+
## Surface — `Rag`
|
|
3796
|
+
|
|
3797
|
+
```ts
|
|
3798
|
+
interface Rag {
|
|
3799
|
+
readonly name: string;
|
|
3800
|
+
index(docs: RagDocument[], chunk?: ChunkOptions): Promise<{ chunks: number }>;
|
|
3801
|
+
retrieve(query: string, options?: RetrieveOptions): Promise<RetrieveResult>;
|
|
3802
|
+
clear(): Promise<void>;
|
|
3803
|
+
asTool(options?: RagAsToolOptions): ToolContract<{ query: string }, RetrieveResult>;
|
|
3804
|
+
}
|
|
3805
|
+
```
|
|
3806
|
+
|
|
3807
|
+
## `index()` — chunk, embed (batched), store
|
|
3808
|
+
|
|
3809
|
+
```ts
|
|
3810
|
+
await kb.index([
|
|
3811
|
+
{ id: "guide", text: longMarkdown, metadata: { url: "/guide" }, tags: ["frontend"] },
|
|
3812
|
+
{ id: "faq", text: faqText },
|
|
3813
|
+
]);
|
|
3814
|
+
```
|
|
3815
|
+
|
|
3816
|
+
A `RagDocument` is `{ id, text, metadata?, tags? }` — **you** load + parse documents to text (loaders are out of scope for v1). Each doc is split into chunks, embedded in sub-batches of 96 texts per `embedMany()` call (so one giant doc never blows the provider's per-request cap), and upserted. Returns the chunk count written. Empty / whitespace-only documents yield zero chunks — nothing is embedded. The per-call `chunk` arg overrides `config.chunk` for that index.
|
|
3817
|
+
|
|
3818
|
+
## Chunking — `ChunkOptions`
|
|
3819
|
+
|
|
3820
|
+
All sizing is in **characters** (tokenizer-free; the embedder owns token counting). `chunk(text, options)` is also exported standalone.
|
|
3821
|
+
|
|
3822
|
+
```ts
|
|
3823
|
+
type ChunkType = "recursive" | "sentence" | "fixed" | "markdown";
|
|
3824
|
+
|
|
3825
|
+
{
|
|
3826
|
+
type?: ChunkType, // default "recursive"
|
|
3827
|
+
size?: number, // target chars per chunk, default 1000
|
|
3828
|
+
overlap?: number, // chars carried between adjacent chunks, default 200
|
|
3829
|
+
separators?: string[], // recursive only; default ["\n\n", "\n", ". ", " ", ""]
|
|
3830
|
+
}
|
|
3831
|
+
```
|
|
3832
|
+
|
|
3833
|
+
- **`recursive`** (default) — separator-aware greedy packing, largest unit first.
|
|
3834
|
+
- **`markdown`** — heading/section-aware, then recursive within each section.
|
|
3835
|
+
- **`sentence`** — packs whole sentences up to `size`.
|
|
3836
|
+
- **`fixed`** — back-to-back character windows.
|
|
3837
|
+
|
|
3838
|
+
Every chunk records its exact `[start, end)` span in the original text, so a `Citation.span` is precise.
|
|
3839
|
+
|
|
3840
|
+
## `retrieve()` — embed query, fetch, rerank, slice, cite
|
|
3841
|
+
|
|
3842
|
+
```ts
|
|
3843
|
+
const { query, chunks } = await kb.retrieve("how do I configure caching?", {
|
|
3844
|
+
topK: 4, // returned AFTER reranking. default 5
|
|
3845
|
+
threshold: 0.5, // cosine floor at the store stage. default 0.5
|
|
3846
|
+
candidates: 16, // pool fetched before rerank. default topK * 4 (clamped >= topK)
|
|
3847
|
+
tags: ["frontend"], // restrict to chunks whose source had one of these tags
|
|
3848
|
+
});
|
|
3849
|
+
|
|
3850
|
+
for (const hit of chunks) {
|
|
3851
|
+
console.log(hit.score, hit.text);
|
|
3852
|
+
console.log(hit.citation.sourceId, hit.citation.chunkIndex, hit.citation.span);
|
|
3853
|
+
}
|
|
3854
|
+
```
|
|
3855
|
+
|
|
3856
|
+
`retrieve()` is **return-only** — it never auto-injects into a prompt. The caller formats the cited chunks (or uses `asTool()` for the agent loop). A `RetrievedChunk` carries `{ text, score, citation }`; the `Citation` is `{ sourceId, chunkIndex, span, score, metadata? }`. The reranker is **OFF by default** (cosine ranking only) unless `config.reranker` is set.
|
|
3857
|
+
|
|
3858
|
+
## Rerankers — opt-in, on `ai.rag.*`
|
|
3859
|
+
|
|
3860
|
+
Both are exposed as namespaced helpers on the factory (`ai.rag.keywordReranker`, `ai.rag.llmReranker`).
|
|
3861
|
+
|
|
3862
|
+
```ts
|
|
3863
|
+
// Zero-dependency lexical reranker (BM25-lite keyword overlap).
|
|
3864
|
+
ai.rag.keywordReranker({ weight: 0.5 }); // weight in [0,1]; 1 = pure keyword, 0 = keep cosine
|
|
3865
|
+
|
|
3866
|
+
// Model-backed reranker — one or more model calls per retrieval.
|
|
3867
|
+
ai.rag.llmReranker({ model: openai.model({ name: "gpt-4o-mini" }), batchSize: 10 });
|
|
3868
|
+
```
|
|
3869
|
+
|
|
3870
|
+
- **`keywordReranker`** — blends lexical query-term overlap with the original cosine score by `weight`; ties keep the incoming cosine order. Costs nothing beyond string splits. Reach for it when embedding-only ranking buries a keyword-rich chunk.
|
|
3871
|
+
- **`llmReranker`** — asks an LLM to grade each over-fetched candidate `0..1` and sorts by that. Candidates the model fails to score keep their cosine score, so a garbled reply degrades gracefully. Opt in only when precision beats latency/cost. Both implement the `RagReranker` contract, so you can write your own.
|
|
3872
|
+
|
|
3873
|
+
## `asTool()` — drop retrieval into an agent's `tools: []`
|
|
3874
|
+
|
|
3875
|
+
```ts
|
|
3876
|
+
const agent = ai.agent({
|
|
3877
|
+
model: openai.model({ name: "gpt-4o" }),
|
|
3878
|
+
tools: [kb.asTool({ name: "search_docs", retrieve: { topK: 6 } })],
|
|
3879
|
+
});
|
|
3880
|
+
```
|
|
3881
|
+
|
|
3882
|
+
Input is `{ query: string }`; output is the `RetrieveResult`. Default tool name is `retrieve_<rag.name>`; `description` and a per-tool `retrieve` override are optional. Built via the same composite-as-tool engine every other primitive uses.
|
|
3883
|
+
|
|
3884
|
+
## `clear()`
|
|
3885
|
+
|
|
3886
|
+
```ts
|
|
3887
|
+
await kb.clear(); // drops every entry written under this rag's namespace
|
|
3888
|
+
```
|
|
3889
|
+
|
|
3890
|
+
## Advanced
|
|
3891
|
+
|
|
3892
|
+
- `cacheVectorStore(driver)` + the `VectorStore` contract are exported for swapping in a custom store.
|
|
3893
|
+
- A stored chunk's namespaced key is `${namespace}.${sourceId}.${chunkIndex}`.
|
|
3894
|
+
|
|
3895
|
+
## See also
|
|
3896
|
+
|
|
3897
|
+
- [`@warlock.js/ai/embed-text/SKILL.md`](@warlock.js/ai/embed-text/SKILL.md) — the `sdk.embedder` primitive this consumes
|
|
3898
|
+
- [`@warlock.js/ai/define-ai-tool/SKILL.md`](@warlock.js/ai/define-ai-tool/SKILL.md) — what `asTool()` produces
|
|
3899
|
+
- [`@warlock.js/ai/run-ai-agent/SKILL.md`](@warlock.js/ai/run-ai-agent/SKILL.md) — wiring the retrieval tool into an agent
|
|
3900
|
+
|
|
3901
|
+
|
|
3902
|
+
## run-ai-team `@warlock.js/ai/run-ai-team/SKILL.md`
|
|
3903
|
+
|
|
3904
|
+
---
|
|
3905
|
+
name: run-ai-team
|
|
3906
|
+
description: 'Manager-led multi-agent teams with ai.team({...}) — transparent sugar over ai.supervisor that maps a manager → route/router, members → intents, and a gate → evaluate, returning a REAL SupervisorContract (no new loop, no new contract). Covers the built-in gate strings "quality" (review-then-fix) and "verify" (test-then-fix), a custom gate function, role mapping (roles / gateKey), and the verbatim supervisor pass-throughs (goal / output / state / maxIterations / snapshotStore / on / observe). Triggers: `ai.team`, `TeamConfig`, `TeamGate`, `TeamGateFn`, `TeamMemberValue`, `manager`, `members`, `gate`, `roles`, `gateKey`, `buildQualityGate`, `buildVerifyGate`, `SupervisorContract`, `ReportType`; ''build a team of agents'', ''manager that delegates to members'', ''review then fix loop'', ''test then fix loop'', ''quality gate for a multi-agent run'', ''report type team''; typical import `import { ai } from "@warlock.js/ai"`. Skip: routing one input to a fixed roster directly — `@warlock.js/ai/run-supervisor/SKILL.md` (team is sugar over it); durable cross-turn sessions — `@warlock.js/ai/run-orchestrator/SKILL.md`; LLM-generated plans — `@warlock.js/ai/run-planner/SKILL.md`; competing libs `crewai`, `autogen`.'
|
|
3907
|
+
---
|
|
3908
|
+
|
|
3909
|
+
# `ai.team()` — manager + members + a quality gate
|
|
3910
|
+
|
|
3911
|
+
`ai.team(config)` is **thin, transparent sugar over `ai.supervisor`**. It builds a `SupervisorConfig` from the team-shaped config, calls `supervisor(...)`, and returns the **unchanged** `SupervisorContract<TOutput>` — the exact object `ai.supervisor` returns. So `ctx.intents.<member>.execute()`, `.asTool()`, `.resume()`, snapshots, and events all stay intact. `team()` owns no loop of its own.
|
|
3912
|
+
|
|
3913
|
+
The mapping:
|
|
3914
|
+
|
|
3915
|
+
| team field | becomes supervisor field |
|
|
3916
|
+
| --- | --- |
|
|
3917
|
+
| `manager` | `route` (deterministic `{ route }`) XOR `router` (an agent / `RouterEntry`) |
|
|
3918
|
+
| `members` | `intents` |
|
|
3919
|
+
| `gate` | `evaluate` |
|
|
1934
3920
|
|
|
1935
|
-
|
|
3921
|
+
Everything else passes through 1:1 — the sole exception is the report/result `type`, which is stamped `"team"` (see [Pass-throughs](#pass-throughs-verbatim-supervisor-semantics)).
|
|
3922
|
+
|
|
3923
|
+
## Shape
|
|
1936
3924
|
|
|
1937
3925
|
```ts
|
|
1938
|
-
|
|
3926
|
+
import { ai } from "@warlock.js/ai";
|
|
3927
|
+
import { v } from "@warlock.js/seal";
|
|
1939
3928
|
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
}
|
|
1944
|
-
}
|
|
3929
|
+
const codeTeam = ai.team({
|
|
3930
|
+
name: "code-team",
|
|
3931
|
+
goal: "Ship a tested module that passes review.",
|
|
3932
|
+
manager: techLeadRouter, // an agent / RouterEntry → router; or { route } → deterministic
|
|
3933
|
+
members: { builder, reviewer, fixer }, // role-name → agent | workflow
|
|
3934
|
+
gate: "quality", // "quality" | "verify" | (ctx) => EvaluateResult
|
|
3935
|
+
output: v.object({ code: v.string() }),
|
|
3936
|
+
maxIterations: 6, // default 10 (supervisor's)
|
|
3937
|
+
});
|
|
1945
3938
|
|
|
1946
|
-
const
|
|
3939
|
+
const { data, report } = await codeTeam.execute("Build a debounce<T> utility.");
|
|
1947
3940
|
```
|
|
1948
3941
|
|
|
1949
|
-
|
|
3942
|
+
A `member` is an `AgentContract` or a `WorkflowInstance` (the `TeamMemberValue` union — the autocomplete-friendly common case; callback / full-entry intent shapes still work when forwarded). The keys are both the role names the manager routes to AND the keys `ctx.intents.<role>` exposes (the supervisor escape hatch is preserved).
|
|
1950
3943
|
|
|
1951
|
-
##
|
|
3944
|
+
## The manager — `route` XOR `router`
|
|
1952
3945
|
|
|
1953
3946
|
```ts
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
setTimeout(() => ctrl.abort("too slow"), 30_000);
|
|
3947
|
+
// LLM-driven manager: an agent (or RouterEntry) → becomes SupervisorConfig.router
|
|
3948
|
+
manager: techLeadRouter
|
|
1958
3949
|
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
// error is an AgentCancelledError (code "AGENT_CANCELLED",
|
|
1962
|
-
// category "cancelled") carrying `cancelledAt` + `reason`
|
|
1963
|
-
}
|
|
3950
|
+
// Deterministic manager: { route } → becomes SupervisorConfig.route
|
|
3951
|
+
manager: { route: (ctx) => (ctx.iteration === 0 ? "builder" : "reviewer") }
|
|
1964
3952
|
```
|
|
1965
3953
|
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
## Events — dot-notation + 3-tier subscription
|
|
3954
|
+
Exactly one form is forwarded — mutually exclusive, mirroring the supervisor's own `router` XOR `route` rule. A malformed manager surfaces the existing `SupervisorFailedError` downstream.
|
|
1969
3955
|
|
|
1970
|
-
|
|
1971
|
-
- `agent.tool.calling`, `agent.tool.called`, `agent.tool.failed`
|
|
1972
|
-
- `agent.completed`, `agent.error`
|
|
3956
|
+
## Gates — `"quality"` | `"verify"` | a function
|
|
1973
3957
|
|
|
1974
|
-
|
|
3958
|
+
A `gate` string selects a pre-built `evaluate` strategy; both desugar to a concrete `evaluate` callback that leans entirely on the already-shipped `EvaluateResult` semantics (`satisfied` terminates, `reassignTo` re-dispatches the fixer, `feedback` threads forward) — **no new termination or loop code**.
|
|
1975
3959
|
|
|
1976
|
-
|
|
1977
|
-
ai.agent({ model, on: { "agent.starting": () => metrics.inc("agent.runs") } });
|
|
3960
|
+
### `gate: "quality"` — review-then-fix
|
|
1978
3961
|
|
|
1979
|
-
|
|
3962
|
+
After each iteration's members settle and merge into supervisor `state`, the gate reads `state.approved` (the `gateKey`, default `"approved"`). If truthy → `{ satisfied: true }`; otherwise → `{ reassignTo: "fixer", feedback: String(state.notes ?? "") }`. The reviewer's feedback (`state.notes`) threads into the next iteration.
|
|
1980
3963
|
|
|
1981
|
-
|
|
1982
|
-
on: { "agent.trip.completed": ({ trip }) => console.log(trip.duration) },
|
|
1983
|
-
});
|
|
1984
|
-
```
|
|
3964
|
+
### `gate: "verify"` — test-then-fix
|
|
1985
3965
|
|
|
1986
|
-
|
|
3966
|
+
Identical shape but keyed on the tester's pass/fail slice `state.passed` (default `gateKey`) rather than a subjective score. On failure it re-dispatches the fixer; there is no feedback channel for a pass/fail signal, so none is threaded.
|
|
1987
3967
|
|
|
1988
|
-
|
|
3968
|
+
> The named member whose `output` schema writes the gate slice must produce a boolean into `gateKey`.
|
|
1989
3969
|
|
|
1990
|
-
|
|
3970
|
+
### A custom gate (full escape hatch)
|
|
1991
3971
|
|
|
1992
3972
|
```ts
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
}
|
|
3973
|
+
gate: (ctx) => {
|
|
3974
|
+
if (ctx.state.score >= 0.9) return { satisfied: true };
|
|
3975
|
+
return { reassignTo: "fixer", feedback: ctx.state.review };
|
|
3976
|
+
}
|
|
1997
3977
|
```
|
|
1998
3978
|
|
|
1999
|
-
|
|
3979
|
+
Supplying a `TeamGateFn` instead of a string opts out of the sugar entirely while keeping the rest of `team()`'s wiring — it forwards straight to `SupervisorConfig.evaluate` with zero wrapping.
|
|
2000
3980
|
|
|
2001
|
-
##
|
|
3981
|
+
## Role mapping — `roles` + `gateKey`
|
|
3982
|
+
|
|
3983
|
+
The string gates default to canonical role names. Override when your `members` keys differ:
|
|
2002
3984
|
|
|
2003
3985
|
```ts
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
passThreshold: 0.5, // default
|
|
3986
|
+
ai.team({
|
|
3987
|
+
name: "qa-team",
|
|
3988
|
+
manager,
|
|
3989
|
+
members: { author, critic, patcher },
|
|
3990
|
+
gate: "quality",
|
|
3991
|
+
roles: { reviewer: "critic", fixer: "patcher" }, // map gate roles → your member keys
|
|
3992
|
+
gateKey: "ok", // state slice the gate reads
|
|
2012
3993
|
});
|
|
2013
|
-
|
|
2014
|
-
expect(report.passed).toBe(true); // true only when EVERY case passed
|
|
2015
3994
|
```
|
|
2016
3995
|
|
|
2017
|
-
|
|
3996
|
+
**Construction-time validation:** when the gate is a string, the resolved `fixer` (and, for `"quality"`, the `reviewer`) role is checked against `members`. A missing role throws an authoring-style `SupervisorFailedError` (`context: { authoring: true }`) immediately — rather than silently starving until `maxIterations`.
|
|
2018
3997
|
|
|
2019
|
-
##
|
|
3998
|
+
## Pass-throughs (verbatim supervisor semantics)
|
|
2020
3999
|
|
|
2021
|
-
|
|
2022
|
-
|
|
4000
|
+
`goal`, `output`, `state`, `maxIterations`, `snapshotStore`, `on`, `observe`, and `version` are forwarded unchanged. Because the returned object IS a supervisor, observability rides the same generic `Observer` seam every other flow uses (see `observe-ai-flows`), and snapshot resume works exactly as on a bare supervisor.
|
|
4001
|
+
|
|
4002
|
+
The one behavioural difference from a bare supervisor: a team stamps **`type: "team"`** on both its report (a first-class `ReportType`, was `"supervisor"`) and its result, so Panoptic and any `Observer` can distinguish, group, filter, and label team runs as their own type rather than folding them into plain supervisor runs. Everything else passes through 1:1.
|
|
4003
|
+
|
|
4004
|
+
A member callback that calls `agent.execute()` **directly** still nests `member → agent → tool` under the member span with usage rolled up — the same ambient-`RunFrame` auto-nesting as a bare supervisor. See [`@warlock.js/ai/run-supervisor/SKILL.md`](@warlock.js/ai/run-supervisor/SKILL.md).
|
|
2023
4005
|
|
|
2024
4006
|
## See also
|
|
2025
4007
|
|
|
2026
|
-
- [`@warlock.js/ai/
|
|
2027
|
-
- [`@warlock.js/ai/
|
|
2028
|
-
- [`@warlock.js/ai/
|
|
4008
|
+
- [`@warlock.js/ai/run-supervisor/SKILL.md`](@warlock.js/ai/run-supervisor/SKILL.md) — the primitive team desugars into (intents, route/router, evaluate, ctx.intents)
|
|
4009
|
+
- [`@warlock.js/ai/run-orchestrator/SKILL.md`](@warlock.js/ai/run-orchestrator/SKILL.md) — wrap a team in durable cross-turn session state
|
|
4010
|
+
- [`@warlock.js/ai/observe-ai-flows/SKILL.md`](@warlock.js/ai/observe-ai-flows/SKILL.md) — the `observe` seam a team inherits
|
|
2029
4011
|
|
|
2030
4012
|
|
|
2031
4013
|
## run-ai-workflow `@warlock.js/ai/run-ai-workflow/SKILL.md`
|
|
@@ -2346,7 +4328,7 @@ await supportBot.execute(input, {
|
|
|
2346
4328
|
state: { tier: "gold" }, // partial patch shallow-merged into loaded state
|
|
2347
4329
|
context: { userId, db }, // request-scoped, never persisted
|
|
2348
4330
|
signal: AbortSignal.timeout(60_000),
|
|
2349
|
-
on: { "orchestrator.turn.
|
|
4331
|
+
on: { "orchestrator.turn.awaiting-input": (e) => log(e) }, // tier-3 per-call handlers
|
|
2350
4332
|
force: false, // bypass drift check for this call
|
|
2351
4333
|
});
|
|
2352
4334
|
```
|
|
@@ -2361,7 +4343,7 @@ await supportBot.execute(input, {
|
|
|
2361
4343
|
6. **persist** — append a checkpoint row for the settled turn, then prune to `keepSnapshots` (`orchestrator.checkpoint.persisted`).
|
|
2362
4344
|
7. **compaction** — fire the post-turn compaction trigger if configured (`orchestrator.compaction.suggested` / `.applied`).
|
|
2363
4345
|
|
|
2364
|
-
|
|
4346
|
+
A clean turn ends with `orchestrator.turn.awaiting-input` (the session stays open for the next user turn); `orchestrator.turn.failed` and `orchestrator.turn.cancelled` end error / cancelled turns. (`orchestrator.turn.completed` is defined on the event map, but the v1 lifecycle maps a clean completion to `awaiting-input`, so it isn't emitted on the normal path — subscribe to `awaiting-input` for "turn done".)
|
|
2365
4347
|
|
|
2366
4348
|
## `OrchestratorResult` — read the report
|
|
2367
4349
|
|
|
@@ -2380,6 +4362,8 @@ result.compaction; // CompactionResult when a turn compacted (and
|
|
|
2380
4362
|
|
|
2381
4363
|
`report.children[]` carries ONLY the current turn's dispatched primitive reports. Full session history lives on `report.turns[]` — a `children[]` walker will NOT reach prior turns (intentional). Child `supervisor.*` / `agent.*` events bubble up unmodified under their own identity.
|
|
2382
4364
|
|
|
4365
|
+
A turn callback that calls `agent.execute()` **directly** (not via `ctx.run` / `ctx.intents`) still nests `callback → agent → tool` inside the turn's report tree, with usage rolled up and the session's `sessionId` stamped onto the captured subtree — an ambient `RunFrame` handles the self-attach. See [`@warlock.js/ai/run-supervisor/SKILL.md`](@warlock.js/ai/run-supervisor/SKILL.md).
|
|
4366
|
+
|
|
2383
4367
|
**`awaiting-input` is the only non-terminal status across the unified result tree.** Code branching on `status === "completed"` MUST explicitly handle `"awaiting-input"` as a session-continues path, not a failure.
|
|
2384
4368
|
|
|
2385
4369
|
## `iterate` — single dispatch vs. internal supervisor
|
|
@@ -2496,7 +4480,7 @@ await orch.execute(input, { sessionId, history, on: { "orchestrator.drift.checke
|
|
|
2496
4480
|
|
|
2497
4481
|
---
|
|
2498
4482
|
name: run-planner
|
|
2499
|
-
description: 'Goal-driven planning with ai.planner({...}) — an LLM GENERATES an ordered execution plan over your registered capabilities (agents / workflows / supervisors / tools), then the planner EXECUTES
|
|
4483
|
+
description: 'Goal-driven planning with ai.planner({...}) — an LLM GENERATES an ordered execution plan over your registered capabilities (agents / workflows / supervisors / tools), then the planner EXECUTES it, threading each step output into the next, and returns the unified {data, report, usage, error} envelope with report.type "planner". Supports DAG scheduling (dag:true + maxConcurrency off dependsOn), adaptive re-planning (replan:{maxReplans} + the onStep continue/abort/replan directive), and plan-only / approval (mode:"plan-only" → status "awaiting-approval" → approvedPlan). A plan step may delegate via ai.spawnSubAgent({...}) — a GENERAL one-shot-agent helper covered in `@warlock.js/ai/run-ai-agent/SKILL.md`; it is not planner-specific. Triggers: `ai.planner`, `planner.execute`, `spawnSubAgent`, `PlannerConfig`, `PlannerCapability`, `PlannerResult`, `PlannerReport`, `PlannerPlan`, `PlannerStep`, `PlannerStepDirective`, `PlannerPlanInvalidError`, `maxSteps`, `dag`, `maxConcurrency`, `dependsOn`, `replan`, `onStep`, `mode`, `approvedPlan`, `awaiting-approval`, `report.plan`, `report.executedSteps`; ''let the model plan the steps'', ''dynamic plan from a goal'', ''run independent steps in parallel'', ''re-plan when a step fails'', ''generate a plan for approval before running it''; typical import `import { ai } from "@warlock.js/ai"`. Skip: a FIXED known pipeline — `@warlock.js/ai/run-ai-workflow/SKILL.md`; routing one input to a specialist each turn — `@warlock.js/ai/run-supervisor/SKILL.md`; a single model + tools call — `@warlock.js/ai/run-ai-agent/SKILL.md`; competing libs `langgraph`, `crewai`.'
|
|
2500
4484
|
---
|
|
2501
4485
|
|
|
2502
4486
|
# `ai.planner()` — LLM-generated, then executed, plans
|
|
@@ -2508,16 +4492,19 @@ A planner turns a free-form **goal** into an ordered **plan** the LLM writes its
|
|
|
2508
4492
|
- **`agent`** — one model + tools, single task. No multi-step decomposition.
|
|
2509
4493
|
- **`workflow`** — a FIXED pipeline you author by hand (`steps: [...]`). The steps are known at design time.
|
|
2510
4494
|
- **`supervisor`** — routes one input to the right specialist each turn; loops on a quality verdict.
|
|
2511
|
-
- **`planner`** — the steps are NOT known in advance. The LLM generates the ordered plan from the goal, then the planner executes it.
|
|
4495
|
+
- **`planner`** — the steps are NOT known in advance. The LLM generates the ordered plan from the goal, then the planner executes it. Sequential by default; opt into **DAG** scheduling, **adaptive re-planning**, and **plan-only / approval** as needed (below).
|
|
2512
4496
|
|
|
2513
4497
|
## Shape
|
|
2514
4498
|
|
|
2515
4499
|
```ts
|
|
2516
4500
|
import { ai } from "@warlock.js/ai";
|
|
4501
|
+
import { OpenAISDK } from "@warlock.js/ai-openai";
|
|
4502
|
+
|
|
4503
|
+
const openai = new OpenAISDK({ apiKey: process.env.OPENAI_API_KEY! });
|
|
2517
4504
|
|
|
2518
4505
|
const research = ai.planner({
|
|
2519
4506
|
name: "research-assistant",
|
|
2520
|
-
model:
|
|
4507
|
+
model: openai.model({ name: "gpt-4o" }), // the plan-GENERATION brain
|
|
2521
4508
|
capabilities: [
|
|
2522
4509
|
{ name: "search", description: "Search the web for sources", executable: searchAgent },
|
|
2523
4510
|
{ name: "summarize", description: "Summarize text into bullet points", executable: summarizer },
|
|
@@ -2538,19 +4525,81 @@ for (const step of report.executedSteps) { // forensic, in execution order
|
|
|
2538
4525
|
- `planner` lets you bring your own fully-configured planning agent (custom prompt, middleware). The planner injects the plan schema as that agent's per-call `output`.
|
|
2539
4526
|
- A `capability` is `{ name, description, executable }`. The `name` is what the LLM references per step; the `description` is what it reads to pick. `executable` is any `ExecutableContract` (agent / workflow / supervisor / tool).
|
|
2540
4527
|
|
|
2541
|
-
## Execution model
|
|
4528
|
+
## Execution model
|
|
2542
4529
|
|
|
2543
|
-
1. **Generate** — the planning agent is asked for a `{ steps, summary? }` plan via a generated schema whose `capability` field is an `enum` of your capability names.
|
|
2544
|
-
2. **Execute** — steps run **strictly in array order
|
|
4530
|
+
1. **Generate** — the planning agent is asked for a `{ steps, summary? }` plan via a generated schema whose `capability` field is an `enum` of your capability names. Each `PlannerStep` is `{ capability, input, id?, reason?, dependsOn? }`.
|
|
4531
|
+
2. **Execute** — by default steps run **strictly in array order**; each completed step's output is threaded into the next step's input as "Context from earlier steps". (Set `dag: true` to schedule on `dependsOn` instead — below.)
|
|
2545
4532
|
3. **Finalize** — when `output` is set (factory or per-call), the LAST completed step's structured output is validated into `result.data`. A capability that should feed typed output to the planner's `output` should declare its own `output` schema (the planner reads `data`, falling back to an agent's raw `text`).
|
|
2546
4533
|
|
|
2547
|
-
`report.type === "planner"`; `report.children[]` carries every dispatched capability report (plus the planning trip), with usage rolled up. Lazy capability loading is **deferred** — every capability is fully constructed up front.
|
|
4534
|
+
`report.type === "planner"`; `report.children[]` carries every dispatched capability report (plus the planning trip), with usage rolled up. `report.executedSteps` is the authoritative per-step record (`PlannerStepSnapshot[]`). Lazy capability loading is **deferred** — every capability is fully constructed up front.
|
|
4535
|
+
|
|
4536
|
+
## DAG scheduling — `dag: true` + `maxConcurrency`
|
|
4537
|
+
|
|
4538
|
+
Run independent steps in parallel instead of array-order:
|
|
4539
|
+
|
|
4540
|
+
```ts
|
|
4541
|
+
const research = ai.planner({
|
|
4542
|
+
name: "research",
|
|
4543
|
+
model,
|
|
4544
|
+
capabilities,
|
|
4545
|
+
dag: true, // build a DAG from each step's `id` / `dependsOn`
|
|
4546
|
+
maxConcurrency: 4, // max steps in flight at once. default 4
|
|
4547
|
+
});
|
|
4548
|
+
```
|
|
4549
|
+
|
|
4550
|
+
With `dag: true` the planner builds a DAG from step `id` / `dependsOn`, runs each **ready level concurrently** (up to `maxConcurrency`), and feeds each step **only its dependencies' outputs** (not the whole prior transcript). A **cycle** or a `dependsOn` naming an unknown step raises a typed `PlannerPlanInvalidError` **before any step runs**. Default `false` ⇒ the strict array-order loop, byte-for-byte unchanged (where `dependsOn` is advisory-only metadata).
|
|
4551
|
+
|
|
4552
|
+
## Adaptive re-planning — `replan: { maxReplans }` + `onStep`
|
|
4553
|
+
|
|
4554
|
+
When set, a **failed step** (or a `replan` verdict from the `onStep` hook) **revises the REMAINING plan** instead of aborting — re-asking the planning agent for a fresh plan seeded with the executed-step digest plus the feedback. Bounded by `maxReplans`; on exhaustion the run ends with the last failure.
|
|
4555
|
+
|
|
4556
|
+
```ts
|
|
4557
|
+
const planner = ai.planner({
|
|
4558
|
+
name: "adaptive",
|
|
4559
|
+
model,
|
|
4560
|
+
capabilities,
|
|
4561
|
+
replan: { maxReplans: 2 },
|
|
4562
|
+
});
|
|
4563
|
+
|
|
4564
|
+
await planner.execute(goal, {
|
|
4565
|
+
onStep: (snapshot, plan) => {
|
|
4566
|
+
// fired after EACH step settles (both the sequential and the DAG path)
|
|
4567
|
+
if (snapshot.status === "completed" && looksWrong(snapshot.output)) {
|
|
4568
|
+
return { type: "replan", feedback: "The summary missed the pricing section." };
|
|
4569
|
+
}
|
|
4570
|
+
// return nothing / { type: "continue" } to proceed; { type: "abort" } to stop
|
|
4571
|
+
},
|
|
4572
|
+
});
|
|
4573
|
+
```
|
|
4574
|
+
|
|
4575
|
+
The `onStep` directive (`PlannerStepDirective`):
|
|
4576
|
+
|
|
4577
|
+
- `{ type: "continue" }` (or returning nothing) — proceed.
|
|
4578
|
+
- `{ type: "abort" }` — stop; remaining steps recorded `skipped` (exactly as a failure aborts).
|
|
4579
|
+
- `{ type: "replan"; feedback }` — re-plan the remainder, seeded with the digest + `feedback`. **A `replan` directive with no `replan` config is treated as `continue`** (no-op). Default off ⇒ a failure aborts exactly as before.
|
|
4580
|
+
|
|
4581
|
+
## Plan-only / approval — `mode: "plan-only"` + `approvedPlan`
|
|
4582
|
+
|
|
4583
|
+
Generate (and validate) a plan, return it for human sign-off, then execute the approved plan in a follow-up call:
|
|
4584
|
+
|
|
4585
|
+
```ts
|
|
4586
|
+
// 1. Generate WITHOUT executing.
|
|
4587
|
+
const draft = await planner.execute(goal, { mode: "plan-only" });
|
|
4588
|
+
// draft.report.status === "awaiting-approval"; draft.plan carries the generated PlannerPlan.
|
|
4589
|
+
|
|
4590
|
+
// 2. (human reviews draft.plan) ... then execute it verbatim.
|
|
4591
|
+
const final = await planner.execute(goal, { approvedPlan: draft.plan! });
|
|
4592
|
+
```
|
|
4593
|
+
|
|
4594
|
+
- `mode: "plan-only"` generates + validates the plan and returns **without executing** — `report.status === "awaiting-approval"` (a planner-specific NON-terminal status) and `result.plan` carries the generated plan.
|
|
4595
|
+
- `approvedPlan` executes that exact plan, **skipping plan generation entirely**. It is still validated against the **live** capabilities, so a stale plan naming a capability the planner no longer has surfaces a `PlannerPlanInvalidError`.
|
|
4596
|
+
- `mode: "plan-only"` **with** `approvedPlan` is contradictory — `approvedPlan` wins (the plan executes).
|
|
2548
4597
|
|
|
2549
4598
|
## Failure + cancellation
|
|
2550
4599
|
|
|
2551
4600
|
`execute()` never throws — failures surface on `result.error`:
|
|
2552
4601
|
|
|
2553
|
-
- **`PlannerPlanInvalidError`** (`PLANNER_PLAN_INVALID`, category `schema`) — empty plan
|
|
4602
|
+
- **`PlannerPlanInvalidError`** (`PLANNER_PLAN_INVALID`, category `schema`) — empty plan, a step naming an unknown capability, a DAG cycle, a `dependsOn` naming an unknown step, a stale `approvedPlan`, or a final-output validation failure.
|
|
2554
4603
|
- **`PlannerCancelledError`** (`PLANNER_CANCELLED`, category `cancelled`) — the `AbortSignal` fired. `report.status === "cancelled"`, `report.cancelledAt` set; remaining steps are `skipped`.
|
|
2555
4604
|
- A child capability's own error (agent / tool / provider) flows through unchanged on the failing step's snapshot and as `result.error`. The planner stops at the first failed step and marks the rest `skipped`.
|
|
2556
4605
|
- **`PlannerFailedError`** is the base for the `PLANNER_*` family.
|
|
@@ -2568,7 +4617,7 @@ Use `MockSDK` for the planning model — script the plan as a JSON string matchi
|
|
|
2568
4617
|
|
|
2569
4618
|
---
|
|
2570
4619
|
name: run-supervisor
|
|
2571
|
-
description: 'Multi-intent routing with ai.supervisor({...}) — classifier (iter-0 dispatch), router agent OR route callback, intents as agents / workflows / callbacks, fan-out, evaluate quality loop, ack receptionist, supervisor-level middleware. Triggers: `ai.supervisor`, `ai.router`, `ai.fanOut`, `supervisor.execute`, `supervisor.resume`, `intents`, `router`, `route`, `classifier`, `evaluate`, `ack`, `artifactsSchema`, `middleware`, `END`, `ctx.intents.X.execute`; ''route one input across specialists'', ''multi-intent dispatch'', ''fan-out then evaluate'', ''classifier then router'', ''supervisor middleware'', ''self-consistency / voting''; typical import `import { ai } from "@warlock.js/ai"`. Skip: durable multi-turn sessions — `@warlock.js/ai/run-orchestrator/SKILL.md`; fixed pipelines — `@warlock.js/ai/run-ai-workflow/SKILL.md`; single agent — `@warlock.js/ai/run-ai-agent/SKILL.md`; competing libs `langgraph`, `crewai`.'
|
|
4620
|
+
description: 'Multi-intent routing with ai.supervisor({...}) — classifier (iter-0 dispatch), router agent OR route callback, intents as agents / workflows / callbacks, fan-out, evaluate quality loop, ack receptionist, supervisor-level middleware. A callback that calls agent.execute() directly auto-nests agent → tool under the callback span (ambient RunFrame) with usage / cost rolled up — same for team members and orchestrator turns. Triggers: `ai.supervisor`, `ai.router`, `ai.fanOut`, `supervisor.execute`, `supervisor.resume`, `intents`, `router`, `route`, `classifier`, `evaluate`, `ack`, `artifactsSchema`, `middleware`, `END`, `ctx.intents.X.execute`, `ctx.run`, `RunFrame`, `callback span`, `children`, `parentRunId`, `rootRunId`, `trace nesting`, `sub-agent`; ''route one input across specialists'', ''multi-intent dispatch'', ''fan-out then evaluate'', ''classifier then router'', ''supervisor middleware'', ''self-consistency / voting'', ''why is my callback agent not nested / cost is $0'', ''nest a sub-agent under a callback''; typical import `import { ai } from "@warlock.js/ai"`. Skip: durable multi-turn sessions — `@warlock.js/ai/run-orchestrator/SKILL.md`; fixed pipelines — `@warlock.js/ai/run-ai-workflow/SKILL.md`; single agent — `@warlock.js/ai/run-ai-agent/SKILL.md`; competing libs `langgraph`, `crewai`.'
|
|
2572
4621
|
---
|
|
2573
4622
|
|
|
2574
4623
|
# `ai.supervisor()` — multi-intent routing
|
|
@@ -2835,6 +4884,25 @@ intents: {
|
|
|
2835
4884
|
|
|
2836
4885
|
Cycle protection: per-branch call stack. Re-entry on same intent → `SUPERVISOR_DISPATCH_CYCLE`.
|
|
2837
4886
|
|
|
4887
|
+
### Sub-agent trace nesting — `agent.execute()` inside a callback auto-nests
|
|
4888
|
+
|
|
4889
|
+
A callback that calls `agent.execute()` (or `team` member / `orchestrator` turn callback) **directly** — not through `ctx.run(agent)` / `ctx.intents.X.execute()` — still nests under its enclosing span. An ambient async-local `RunFrame` lets the agent self-attach to the callback's `children[]`, so the report tree is `callback → agent → tool` with usage / cost **rolled up** (no `$0` lone callback span, no manual id threading):
|
|
4890
|
+
|
|
4891
|
+
```ts
|
|
4892
|
+
ai.supervisor({
|
|
4893
|
+
intents: {
|
|
4894
|
+
delegate: async (ctx) => {
|
|
4895
|
+
const result = await worker.execute(String(ctx.input)); // direct call — still nested
|
|
4896
|
+
return { reply: result.text };
|
|
4897
|
+
},
|
|
4898
|
+
},
|
|
4899
|
+
route: (ctx) => (ctx.iteration === 0 ? "delegate" : END),
|
|
4900
|
+
});
|
|
4901
|
+
// report → callback("delegate") → agent("worker") → tool("echo"); usage flows up to the root.
|
|
4902
|
+
```
|
|
4903
|
+
|
|
4904
|
+
Same behavior across `ai.supervisor`, `ai.team` (member callbacks), and `ai.orchestrator` (turn callbacks) — and `sessionId` propagates onto the captured subtree. `ctx.run(agent)` is captured **exactly once** (the explicit path does not double-count via the ambient frame), and a standalone `agent.execute()` **outside** any callback keeps its own self-root (no frame leakage). This is what an `Observer` / panoptic sees — see [`@warlock.js/ai/observe-ai-flows/SKILL.md`](@warlock.js/ai/observe-ai-flows/SKILL.md).
|
|
4905
|
+
|
|
2838
4906
|
## Per-call options
|
|
2839
4907
|
|
|
2840
4908
|
```ts
|
|
@@ -2922,6 +4990,167 @@ const escalationAgent = ai.agent({ model, tools: [supportTool] });
|
|
|
2922
4990
|
- [`@warlock.js/ai/define-ai-tool/SKILL.md`](@warlock.js/ai/define-ai-tool/SKILL.md) — tool artifacts side-channel
|
|
2923
4991
|
|
|
2924
4992
|
|
|
4993
|
+
## transcribe-audio `@warlock.js/ai/transcribe-audio/SKILL.md`
|
|
4994
|
+
|
|
4995
|
+
---
|
|
4996
|
+
name: transcribe-audio
|
|
4997
|
+
description: 'Speech-to-text via ai.transcribe({ model: sdk.transcribe({ name }), audio }) — the audio-INPUT verb (Theme I), returning the uniform never-throws { data, error, usage, report } envelope with cost-truth + panoptic observation. Feed it an AudioInput = { base64; mediaType; filename? } — build one with ai.audioFromFile(path) (reads disk, infers media type incl. WhatsApp .ogg/.opus) or ai.audioFromBuffer(bytes, mediaType). Models: OpenAI whisper-1 (verbose_json, per-minute, segments + durationSeconds) or gpt-4o-transcribe (json, per-token). Triggers: `ai.transcribe`, `ai.audioFromFile`, `ai.audioFromBuffer`, `sdk.transcribe`, `openai.transcribe`, `TranscriptionModelContract`, `AudioInput`, `TranscriptionSegment`, `MockTranscriptionModel`; ''speech to text'', ''transcribe audio'', ''voice note to text'', ''WhatsApp voice message'', ''whisper'', ''gpt-4o-transcribe'', ''subtitle segments'', ''audio input''; typical import `import { ai } from "@warlock.js/ai"` + `import { OpenAISDK } from "@warlock.js/ai-openai"`. Skip: text-to-speech / synthesizing a voice — [[generate-speech]]; competing libs raw `openai.audio.transcriptions.create`, `whisper.cpp`.'
|
|
4998
|
+
---
|
|
4999
|
+
|
|
5000
|
+
# Transcribe audio — the speech-to-text verb (`ai.transcribe`)
|
|
5001
|
+
|
|
5002
|
+
`ai.transcribe()` is the inverse of `ai.speech()` on the modality track (Theme I). Audio-in / text-out, wrapped in the same uniform result contract every executable returns — so transcribing a support voicemail slots into cost dashboards and panoptic traces exactly like an agent run.
|
|
5003
|
+
|
|
5004
|
+
**Extracting text from an audio file NEEDS AI** — that is the `ai.transcribe` step. The file handling (`ai.audioFromFile` / `ai.audioFromBuffer`) is pure, non-AI **utility** that just packages bytes into an `AudioInput`; it does no I/O to a provider on its own.
|
|
5005
|
+
|
|
5006
|
+
This is audio **input** (STT). For audio **output** (synthesizing a voice line), see [[generate-speech]].
|
|
5007
|
+
|
|
5008
|
+
## Shape — WhatsApp voice note → text, end to end
|
|
5009
|
+
|
|
5010
|
+
```ts
|
|
5011
|
+
import { ai } from "@warlock.js/ai";
|
|
5012
|
+
import { OpenAISDK } from "@warlock.js/ai-openai";
|
|
5013
|
+
|
|
5014
|
+
const openai = new OpenAISDK({ apiKey: process.env.OPENAI_API_KEY! });
|
|
5015
|
+
|
|
5016
|
+
// audioFromFile reads the file + infers the media type from the extension.
|
|
5017
|
+
// .ogg / .opus (Android WhatsApp) and .m4a (iOS) are recognized out of the box.
|
|
5018
|
+
const audio = await ai.audioFromFile("./voice-note.ogg");
|
|
5019
|
+
|
|
5020
|
+
const { data, error } = await ai.transcribe({
|
|
5021
|
+
model: openai.transcribe({ name: "whisper-1" }),
|
|
5022
|
+
audio,
|
|
5023
|
+
language: "en", // BCP-47 hint — improves accuracy + latency
|
|
5024
|
+
});
|
|
5025
|
+
|
|
5026
|
+
if (error) console.warn(error.code); // typed AIError
|
|
5027
|
+
else console.log(data.text); // the transcript
|
|
5028
|
+
```
|
|
5029
|
+
|
|
5030
|
+
`TranscriptionModelContract` mirrors `SpeechModelContract` — a peer primitive produced by the adapter's optional `transcribe?()` factory. A non-STT model id (`openai.transcribe({ name: "gpt-4o" })`) throws `InvalidRequestError` **at construction** — fail fast, like the speech/embedder guards.
|
|
5031
|
+
|
|
5032
|
+
## The `AudioInput` shape + the two builders
|
|
5033
|
+
|
|
5034
|
+
```ts
|
|
5035
|
+
type AudioInput = {
|
|
5036
|
+
base64: string; // base64-encoded audio bytes
|
|
5037
|
+
mediaType: string; // IANA type, e.g. "audio/ogg", "audio/mpeg"
|
|
5038
|
+
filename?: string; // helps providers infer the codec from the extension
|
|
5039
|
+
};
|
|
5040
|
+
|
|
5041
|
+
// From a file on disk — reads + infers media type (override for extensionless files).
|
|
5042
|
+
const fromDisk = await ai.audioFromFile("./meeting.m4a");
|
|
5043
|
+
const forced = await ai.audioFromFile("./blob", { mediaType: "audio/ogg" });
|
|
5044
|
+
|
|
5045
|
+
// From bytes you already hold (an upload buffer, a downloaded blob) — no I/O, no AI.
|
|
5046
|
+
const fromBytes = ai.audioFromBuffer(uploadBuffer, "audio/ogg", "note.ogg");
|
|
5047
|
+
```
|
|
5048
|
+
|
|
5049
|
+
Keeping `AudioInput` as inlined base64 + explicit media type makes the verb provider-neutral and serializable — there is no `fs` coupling in core, so the same request can cross a queue or an RPC boundary.
|
|
5050
|
+
|
|
5051
|
+
## The result envelope
|
|
5052
|
+
|
|
5053
|
+
```ts
|
|
5054
|
+
type TranscriptionResult = {
|
|
5055
|
+
type: "transcription";
|
|
5056
|
+
data?: {
|
|
5057
|
+
text: string; // full transcript
|
|
5058
|
+
segments?: TranscriptionSegment[]; // timestamped, in verbose mode
|
|
5059
|
+
}; // undefined on failure
|
|
5060
|
+
error?: AIError; // undefined on success — NEVER thrown
|
|
5061
|
+
usage: Usage; // tokens (gpt-4o-transcribe) + cost when priced
|
|
5062
|
+
report: TranscriptionReport; // type:"transcription", model, durationSeconds, lineage
|
|
5063
|
+
};
|
|
5064
|
+
|
|
5065
|
+
type TranscriptionSegment = { text: string; start?: number; end?: number };
|
|
5066
|
+
```
|
|
5067
|
+
|
|
5068
|
+
`segments` and `report.durationSeconds` appear only when the provider returns them (whisper's `verbose_json` mode). Use segments to build subtitles or to jump-to-timestamp in a player.
|
|
5069
|
+
|
|
5070
|
+
## Transcribe options (provider-neutral)
|
|
5071
|
+
|
|
5072
|
+
```ts
|
|
5073
|
+
await ai.transcribe({
|
|
5074
|
+
model,
|
|
5075
|
+
audio,
|
|
5076
|
+
language: "en", // BCP-47 hint
|
|
5077
|
+
prompt: "Names: Acme, Zoë", // priming — spelling / style hints
|
|
5078
|
+
format: "verbose_json", // response-format override (segments + duration)
|
|
5079
|
+
signal, // AbortSignal
|
|
5080
|
+
observe: collector, // route the report to an Observer (panoptic)
|
|
5081
|
+
sessionId: "ticket-88", // group into a session for flat cost/trace queries
|
|
5082
|
+
options: { /* provider passthrough */ },
|
|
5083
|
+
});
|
|
5084
|
+
```
|
|
5085
|
+
|
|
5086
|
+
## OpenAI — whisper-1 (per-minute) + gpt-4o-transcribe (per-token)
|
|
5087
|
+
|
|
5088
|
+
```ts
|
|
5089
|
+
// whisper-1 — defaults to verbose_json → segments + duration; billed PER MINUTE.
|
|
5090
|
+
const whisper = openai.transcribe({ name: "whisper-1", pricing: { perMinute: 0.006 } });
|
|
5091
|
+
|
|
5092
|
+
// gpt-4o-transcribe — defaults to json; billed PER TOKEN like a chat model.
|
|
5093
|
+
const gpt = openai.transcribe({ name: "gpt-4o-transcribe", pricing: { input: 2.5, output: 10 } });
|
|
5094
|
+
|
|
5095
|
+
const { data, usage } = await ai.transcribe({ model: whisper, audio });
|
|
5096
|
+
// data.segments → [{ text, start, end }, …]; usage.cost from report.durationSeconds
|
|
5097
|
+
```
|
|
5098
|
+
|
|
5099
|
+
The adapter wraps the base64 bytes in an uploadable via the SDK's `toFile`, using `audio.filename` (or `"audio"`) and `audio.mediaType` so the codec is declared correctly.
|
|
5100
|
+
|
|
5101
|
+
## Cost-truth — one rollup, two metering models
|
|
5102
|
+
|
|
5103
|
+
`ai.transcribe` fills `usage.cost` so STT spend folds into the **same** `Usage.cost` rollup as text:
|
|
5104
|
+
|
|
5105
|
+
- **Per-minute** (`whisper-1`): `{ perMinute }` × `(durationSeconds / 60)` → `cost.input`. If the provider didn't report a duration, cost stays **`undefined`** (no guessing).
|
|
5106
|
+
- **Token-metered** (`gpt-4o-transcribe`): `{ input, output }` USD-per-1M-tokens → standard `computeCost` against the returned token usage.
|
|
5107
|
+
|
|
5108
|
+
Per-minute wins when both are set; an unpriced model leaves `usage.cost` **`undefined`** (honest "cost unknown", never a false zero).
|
|
5109
|
+
|
|
5110
|
+
## Pattern — inbound voice-message webhook
|
|
5111
|
+
|
|
5112
|
+
```ts
|
|
5113
|
+
const stt = openai.transcribe({ name: "whisper-1" });
|
|
5114
|
+
|
|
5115
|
+
async function onVoiceMessage(buffer: Buffer, mediaType: string) {
|
|
5116
|
+
const audio = ai.audioFromBuffer(buffer, mediaType, "inbound.ogg");
|
|
5117
|
+
const { data, error } = await ai.transcribe({ model: stt, audio, sessionId: "inbox" });
|
|
5118
|
+
|
|
5119
|
+
if (error) return replyWith("Sorry, I couldn't understand that audio.");
|
|
5120
|
+
return routeToAgent(data.text); // hand the transcript to an ai.agent for a reply
|
|
5121
|
+
}
|
|
5122
|
+
```
|
|
5123
|
+
|
|
5124
|
+
## Observability
|
|
5125
|
+
|
|
5126
|
+
The completed `TranscriptionReport` (with `report.durationSeconds` and cost/latency attributed to `report.model`) routes to any registered `Observer` (panoptic, OTel, …) through the shared `observe` seam — `observe: true` (global), an `Observer` (flow-local), or observe-all. See [[observe-ai-flows]]. Provider faults surface as typed `AIError`s on `result.error`; see [[handle-ai-errors]].
|
|
5127
|
+
|
|
5128
|
+
## Testing
|
|
5129
|
+
|
|
5130
|
+
`MockTranscriptionModel(name, responses, pricing?)` is a deterministic `TranscriptionModelContract` double — no HTTP. Script text/segments/duration/usage/errors and inspect `model.calls`. `MockSDK({ transcriptionResponses, transcriptionPricing }).transcribe({ name })` wires the same double behind a full adapter.
|
|
5131
|
+
|
|
5132
|
+
```ts
|
|
5133
|
+
import { MockTranscriptionModel, transcribe } from "@warlock.js/ai";
|
|
5134
|
+
|
|
5135
|
+
const AUDIO = { base64: "QUJD", mediaType: "audio/mpeg", filename: "clip.mp3" };
|
|
5136
|
+
|
|
5137
|
+
const model = new MockTranscriptionModel("whisper-1", [{ durationSeconds: 120 }], { perMinute: 0.006 });
|
|
5138
|
+
const { data, usage, report } = await transcribe({ model, audio: AUDIO });
|
|
5139
|
+
// data.text → "mock transcript"
|
|
5140
|
+
// usage.cost.input → (120 / 60) * 0.006 report.durationSeconds → 120
|
|
5141
|
+
// model.calls[0] records { audio, options } for assertions
|
|
5142
|
+
```
|
|
5143
|
+
|
|
5144
|
+
Scripting `[{ error: new ProviderRateLimitError("slow down") }]` drives the never-throws path — `result.error` is the typed error and `result.data` is `undefined`.
|
|
5145
|
+
|
|
5146
|
+
## See also
|
|
5147
|
+
|
|
5148
|
+
- [[generate-speech]] — the inverse verb (`ai.speech`), text → audio
|
|
5149
|
+
- [[generate-images]] — the sibling image-output verb (`ai.image`)
|
|
5150
|
+
- [[observe-ai-flows]] — routing the `TranscriptionReport` to panoptic / OTel
|
|
5151
|
+
- [[handle-ai-errors]] — the typed `AIError` taxonomy on `result.error`
|
|
5152
|
+
|
|
5153
|
+
|
|
2925
5154
|
## use-ai-memory `@warlock.js/ai/use-ai-memory/SKILL.md`
|
|
2926
5155
|
|
|
2927
5156
|
---
|
|
@@ -3050,11 +5279,121 @@ Drivers without similarity support throw `CacheUnsupportedError` from `set({ vec
|
|
|
3050
5279
|
- [`@warlock.js/cache/cache-basics/SKILL.md`](@warlock.js/cache/cache-basics/SKILL.md) — vector driver catalog
|
|
3051
5280
|
|
|
3052
5281
|
|
|
5282
|
+
## use-runtime-skills `@warlock.js/ai/use-runtime-skills/SKILL.md`
|
|
5283
|
+
|
|
5284
|
+
---
|
|
5285
|
+
name: use-runtime-skills
|
|
5286
|
+
description: 'Progressive-disclosure agent skills with ai.skills({...}) and the first-class `skills` option on ai.agent — an always-injected cheap metadata catalog plus an on-demand loadSkill tool, backed by directory / url / store sources. Covers inject ("all" | {select:"semantic",topK,embedder}), maxLoadsPerRun, scope tags, the MockSkillsStore, semantic preload, and the inert-by-default Phase-2 self-authoring (saveSkill + default-DENY review gate → promote). Triggers: `ai.skills`, `SkillsConfig`, `SkillsContract`, `SkillSource`, `SkillInjectMode`, `SkillRecord`, `SkillCatalogEntry`, `loadSkill`, `loadSkillTool`, `saveSkill`, `saveSkillTool`, `SkillReviewGate`, `runReviewGate`, `MockSkillsStore`, `proceduralSkillStore`, `maxLoadsPerRun`, `inject`, `scope`, `review`, the agent `skills:` option; ''give an agent loadable skills'', ''progressive disclosure of instructions'', ''catalog of skills the model pulls on demand'', ''semantic preload of skill bodies'', ''let an agent author and review a skill''; typical import `import { ai } from "@warlock.js/ai"`. Skip: composing static system prompts — `@warlock.js/ai/write-system-prompt/SKILL.md`; durable agent memory tiers — `@warlock.js/ai/use-ai-memory/SKILL.md`; defining callable tools — `@warlock.js/ai/define-ai-tool/SKILL.md`.'
|
|
5287
|
+
---
|
|
5288
|
+
|
|
5289
|
+
# `ai.skills()` — runtime skills with progressive disclosure
|
|
5290
|
+
|
|
5291
|
+
A **skill is text injected into an agent's context — it never runs code.** `ai.skills(config)` builds a `SkillsContract`: the mechanism behind the first-class `skills` agent option. The agent always injects a cheap **metadata catalog** (one line per in-scope skill) and registers a `loadSkill` tool so the model pulls a skill's full **body** only when it needs it (progressive disclosure). Bodies are withheld until loaded — keeping context lean.
|
|
5292
|
+
|
|
5293
|
+
## The first-class agent option (the supported way)
|
|
5294
|
+
|
|
5295
|
+
```ts
|
|
5296
|
+
import { ai } from "@warlock.js/ai";
|
|
5297
|
+
|
|
5298
|
+
const agent = ai.agent({
|
|
5299
|
+
model: openai.model({ name: "gpt-4o" }),
|
|
5300
|
+
systemPrompt: "You are a build assistant.",
|
|
5301
|
+
skills: { // a SkillsConfig OR an ai.skills(...) instance
|
|
5302
|
+
name: "build-skills",
|
|
5303
|
+
sources: [{ type: "directory", path: "./agent-skills" }],
|
|
5304
|
+
},
|
|
5305
|
+
});
|
|
5306
|
+
```
|
|
5307
|
+
|
|
5308
|
+
When `skills` is set the agent owns the runtime flow at execute time: it **prepends the always-injected catalog** (and, under `inject`, the preloaded bodies) in front of your system prompt, auto-registers `loadSkill` (plus `saveSkill` only when a `review` gate is configured), and threads the run id so `maxLoadsPerRun` is enforced per execution. **Omitted ⇒ no skills behavior; the agent runs byte-for-byte as today.** The option accepts a raw `SkillsConfig` (the agent passes it to `skills()` for you) or a pre-built `SkillsContract`.
|
|
5309
|
+
|
|
5310
|
+
## Factory config — `SkillsConfig`
|
|
5311
|
+
|
|
5312
|
+
```ts
|
|
5313
|
+
const lib = ai.skills({
|
|
5314
|
+
name: "build-skills", // surfaced in analytics + the catalog block
|
|
5315
|
+
sources: [{ type: "directory", path: "./agent-skills" }], // >= 1; later source wins on name clash
|
|
5316
|
+
inject: { select: "semantic", topK: 2, embedder }, // body-injection policy (see below)
|
|
5317
|
+
maxLoadsPerRun: 4, // cap on loadSkill calls per run. default 5
|
|
5318
|
+
scope: { tags: ["frontend"] }, // only skills whose tags intersect are catalogued
|
|
5319
|
+
review: { approve, store }, // Phase 2 — absent ⇒ saveSkill is NOT exposed
|
|
5320
|
+
analytics: (event) => track(event), // optional efficacy sink (errors swallowed)
|
|
5321
|
+
});
|
|
5322
|
+
```
|
|
5323
|
+
|
|
5324
|
+
### Sources — `SkillSource` (discriminated by `type`, never `kind`)
|
|
5325
|
+
|
|
5326
|
+
- `{ type: "directory", path }` — reads `path/<folder>/SKILL.md` off disk (lazy `node:fs/promises`).
|
|
5327
|
+
- `{ type: "url", url, headers? }` — `fetch()`es a JSON manifest of skills.
|
|
5328
|
+
- `{ type: "store", store }` — any `SkillsStoreContract`, e.g. `MockSkillsStore`.
|
|
5329
|
+
|
|
5330
|
+
Sources merge in order; a later source wins on a name collision.
|
|
5331
|
+
|
|
5332
|
+
### Injection — `inject` (`SkillInjectMode`)
|
|
5333
|
+
|
|
5334
|
+
The metadata catalog is **always** injected (it's cheap). `inject` controls whether any **bodies** are auto-injected up front:
|
|
5335
|
+
|
|
5336
|
+
- **omitted** (default) — inject NO bodies; the model pulls them via `loadSkill`. Pure progressive disclosure.
|
|
5337
|
+
- `"all"` — inject every body up front (small libraries only).
|
|
5338
|
+
- `{ select: "semantic", topK, embedder?, threshold? }` — embed the run input, rank the catalog by cosine similarity, inject the top-`topK` bodies. Needs an embedder (passed here, or lazily auto-resolved).
|
|
5339
|
+
|
|
5340
|
+
## `SkillsContract` surface
|
|
5341
|
+
|
|
5342
|
+
```ts
|
|
5343
|
+
interface SkillsContract {
|
|
5344
|
+
readonly name: string;
|
|
5345
|
+
catalog(scopeInput?: string): Promise<SkillCatalogEntry[]>; // cheap metadata, body omitted
|
|
5346
|
+
catalogPrompt(scopeInput?: string): Promise<string>; // catalog rendered as a system block
|
|
5347
|
+
preload(input: string): Promise<SkillRecord[]>; // bodies per `inject`; [] when omitted
|
|
5348
|
+
tools(runId?: string): AgentToolEntry<any, any>[]; // loadSkill always; saveSkill iff review
|
|
5349
|
+
}
|
|
5350
|
+
```
|
|
5351
|
+
|
|
5352
|
+
A `SkillCatalogEntry` is `Pick<SkillRecord, "name"|"description"|"version"|"tags"|"type">` — the **structural omission of `body`** is the type-level guarantee the catalog never carries skill bodies. A `SkillRecord` adds the full `body` plus `type: "authored" | "promoted" | "candidate"`.
|
|
5353
|
+
|
|
5354
|
+
## `maxLoadsPerRun` — a budget, not a throw
|
|
5355
|
+
|
|
5356
|
+
`loadSkill` calls are capped per run (default 5). Exhaustion is an **error RESULT the model self-corrects from**, never a throw — the tool returns `{ error }` and the loop continues. `runId` scopes both the budget and analytics correlation.
|
|
5357
|
+
|
|
5358
|
+
## Stores
|
|
5359
|
+
|
|
5360
|
+
```ts
|
|
5361
|
+
import { ai, MockSkillsStore } from "@warlock.js/ai";
|
|
5362
|
+
|
|
5363
|
+
const store = new MockSkillsStore([
|
|
5364
|
+
{ name: "scaffold", description: "Scaffold a form", version: 1, body: "...", type: "authored" },
|
|
5365
|
+
]);
|
|
5366
|
+
const lib = ai.skills({ name: "build", sources: [{ type: "store", store }] });
|
|
5367
|
+
```
|
|
5368
|
+
|
|
5369
|
+
`MockSkillsStore` is an in-memory `SkillsStoreContract` that ships with the package (construct via `new` — it is a concrete test/utility store, not a factory-fronted primitive). It holds the latest record per name, filters out `candidate`s from `list()` / `load()`, and exposes `saveCandidate` / `promote`. `proceduralSkillStore` is also exported (unifies proven procedural memories with named skills).
|
|
5370
|
+
|
|
5371
|
+
## Phase 2 — self-authoring (inert by default)
|
|
5372
|
+
|
|
5373
|
+
Self-authoring is **gated and OFF unless a `review` gate is wired**:
|
|
5374
|
+
|
|
5375
|
+
- Without `review`, the `saveSkill` tool is **never registered** — a candidate can never be written, let alone injected.
|
|
5376
|
+
- With `review: { approve, store }`, `saveSkill` writes an **INERT** `type: "candidate"` (`version: 0`), filtered out of every catalog/load until promoted.
|
|
5377
|
+
- The `SkillReviewGate.approve(candidate)` is **default-DENY**: only `{ approve: true }` promotes the candidate to a new audited version (`promote` → `type: "promoted"`, `version + 1`). Anything else — `{ approve: false }`, a malformed result, or a **throw** (fail-closed) — keeps it inert. `runReviewGate(candidate, gate, emit?)` runs this and never throws (a throwing gate is a denial), emitting `promoted` / `denied` analytics events.
|
|
5378
|
+
|
|
5379
|
+
The three interchangeable approve shapes — a policy fn, a validator agent, a human callback — all reduce to one `Promise<{ approve: boolean; reason? }>`.
|
|
5380
|
+
|
|
5381
|
+
## Analytics
|
|
5382
|
+
|
|
5383
|
+
The optional `analytics` sink fires `catalogued` / `loaded` / `used` / `saved` / `promoted` / `denied` events `{ type, skill, version, runId?, outcome? }`. Errors from the sink are swallowed (mirroring the agent's `onUsage` / `onComplete`), so analytics never crash a run.
|
|
5384
|
+
|
|
5385
|
+
## See also
|
|
5386
|
+
|
|
5387
|
+
- [`@warlock.js/ai/write-system-prompt/SKILL.md`](@warlock.js/ai/write-system-prompt/SKILL.md) — static persona / instruction blocks (vs. dynamic loaded skills)
|
|
5388
|
+
- [`@warlock.js/ai/use-ai-memory/SKILL.md`](@warlock.js/ai/use-ai-memory/SKILL.md) — the procedural memory tier `proceduralSkillStore` unifies with
|
|
5389
|
+
- [`@warlock.js/ai/run-ai-agent/SKILL.md`](@warlock.js/ai/run-ai-agent/SKILL.md) — the agent the `skills` option attaches to
|
|
5390
|
+
|
|
5391
|
+
|
|
3053
5392
|
## write-system-prompt `@warlock.js/ai/write-system-prompt/SKILL.md`
|
|
3054
5393
|
|
|
3055
5394
|
---
|
|
3056
5395
|
name: write-system-prompt
|
|
3057
|
-
description: 'Compose system prompts via ai.systemPrompt() / ai.persona() / ai.instruction() — immutable builders with {{placeholder}} substitution, plus ai.systemPrompt.fromFile(path) to seed from a file read once at construction. Triggers: `ai.systemPrompt`, `ai.systemPrompt.fromFile`, `ai.persona`, `ai.instruction`, `SystemPromptBlockContract`, `PersonaContract`, `InstructionContract`, `placeholders`, `{{placeholder|default}}`, `InvalidRequestError`; ''write a system prompt'', ''compose persona + instructions'', ''prompt from a file'', ''per-call prompt override'', ''mustache placeholder''; typical import `import { ai } from "@warlock.js/ai"`. Skip: agent factory wiring — `@warlock.js/ai/run-ai-agent/SKILL.md`; competing libs `langchain` `PromptTemplate`, raw f-strings.'
|
|
5396
|
+
description: 'Compose system prompts via ai.systemPrompt() / ai.persona() / ai.instruction() — immutable builders with {{placeholder}} substitution, plus ai.systemPrompt.fromFile(path) to seed from a file read once at construction. Carry identity with .meta({ name, version, description, required }) (a name auto-registers in ai.prompts) and compose with merge(...blocks) / merge(contract) / merge(name, { fromVersion }) (provenance in meta.composedFrom). Triggers: `ai.systemPrompt`, `ai.systemPrompt.fromFile`, `ai.persona`, `ai.instruction`, `SystemPromptBlockContract`, `SystemPromptContract`, `SystemPromptMeta`, `SystemPromptMergeOptions`, `PersonaContract`, `InstructionContract`, `meta`, `merge`, `composedFrom`, `fromVersion`, `placeholders`, `{{placeholder|default}}`, `InvalidRequestError`; ''write a system prompt'', ''compose persona + instructions'', ''prompt from a file'', ''name and version a prompt'', ''merge prompts together'', ''per-call prompt override'', ''mustache placeholder''; typical import `import { ai } from "@warlock.js/ai"`. Skip: the named/versioned prompt registry (register / resolve / tag / diff / export / validate) — `@warlock.js/ai/manage-prompts/SKILL.md`; agent factory wiring — `@warlock.js/ai/run-ai-agent/SKILL.md`; competing libs `langchain` `PromptTemplate`, raw f-strings.'
|
|
3058
5397
|
---
|
|
3059
5398
|
|
|
3060
5399
|
# System prompts — immutable builders
|
|
@@ -3161,6 +5500,34 @@ ai.agent({ model, systemPrompt: prompt, placeholders: { language: "Arabic" } });
|
|
|
3161
5500
|
|
|
3162
5501
|
Substitution works on the **rendered** concatenation of every block, so `{{key}}` inside a persona and inside an instruction both resolve against the same placeholder bag.
|
|
3163
5502
|
|
|
5503
|
+
## Identity + composition — `.meta()` and `merge()`
|
|
5504
|
+
|
|
5505
|
+
A prompt carries optional `SystemPromptMeta` — `{ name?, version?, description?, required?, composedFrom? }`. Read it with the no-argument accessor; update it immutably with the one-argument form. **Giving a prompt a `name` auto-registers it in the `ai.prompts` registry** (keyed by `name@version`):
|
|
5506
|
+
|
|
5507
|
+
```ts
|
|
5508
|
+
const base = ai.systemPrompt("You are support.", { name: "support", version: "1" });
|
|
5509
|
+
base.meta(); // → { name: "support", version: "1" }
|
|
5510
|
+
const v2 = base.meta({ version: "2" }); // new builder; original untouched; re-registers under support@2
|
|
5511
|
+
```
|
|
5512
|
+
|
|
5513
|
+
`merge(...)` folds blocks from another source into a **new** builder — a persona **replaces**, instructions **append**:
|
|
5514
|
+
|
|
5515
|
+
```ts
|
|
5516
|
+
// (a) N pre-built blocks in one call
|
|
5517
|
+
const p = ai.systemPrompt().merge(ai.persona("You are Alex."), ai.instruction("Be concise."));
|
|
5518
|
+
|
|
5519
|
+
// (b) another prompt contract — its blocks fold in; meta.composedFrom records provenance
|
|
5520
|
+
const merged = ai.systemPrompt("Be terse.").merge(otherPrompt);
|
|
5521
|
+
merged.meta()?.composedFrom; // deterministic source labels, e.g. ["base@2"]
|
|
5522
|
+
|
|
5523
|
+
// (c) a registered prompt resolved from ai.prompts by name (latest, or a pinned fromVersion)
|
|
5524
|
+
const composed = ai.systemPrompt("You are support.").merge("global", { fromVersion: "1" });
|
|
5525
|
+
```
|
|
5526
|
+
|
|
5527
|
+
The name / contract / registry-name forms are the registry's composition surface — full coverage (register / resolve / version / tag / diff / validate) in [`@warlock.js/ai/manage-prompts/SKILL.md`](@warlock.js/ai/manage-prompts/SKILL.md).
|
|
5528
|
+
|
|
5529
|
+
`.validate(options?)` is per-builder sugar over `ai.prompts.validate(this, options)` — the deterministic missing-placeholder check plus an optional Nova-safe LLM-judge.
|
|
5530
|
+
|
|
3164
5531
|
## Per-call overrides
|
|
3165
5532
|
|
|
3166
5533
|
Replace the agent's system prompt for a single run:
|
|
@@ -3192,6 +5559,7 @@ Three distinct prompts, one common foundation. Base is immutable — safe to sha
|
|
|
3192
5559
|
|
|
3193
5560
|
## See also
|
|
3194
5561
|
|
|
5562
|
+
- [`@warlock.js/ai/manage-prompts/SKILL.md`](@warlock.js/ai/manage-prompts/SKILL.md) — the `ai.prompts` registry these named prompts auto-register into (resolve / version / tag / diff / export / validate)
|
|
3195
5563
|
- [`@warlock.js/ai/run-ai-agent/SKILL.md`](@warlock.js/ai/run-ai-agent/SKILL.md) — `systemPrompt` on factory + per-call override
|
|
3196
5564
|
- [`@warlock.js/ai/run-ai-workflow/SKILL.md`](@warlock.js/ai/run-ai-workflow/SKILL.md) — per-step agent references inherit their own system prompt
|
|
3197
5565
|
|