@warlock.js/ai 4.4.0 → 4.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -55
- package/cjs/index.cjs +221 -14147
- package/cjs/src-C02yzsLs.cjs +22991 -0
- package/cjs/src-C02yzsLs.cjs.map +1 -0
- package/cjs/src-DFibP2FQ.cjs +30 -0
- package/esm/agent/agent-config.type.d.mts +90 -1
- package/esm/agent/agent-config.type.d.mts.map +1 -1
- package/esm/agent/agent-input-builder.mjs +27 -6
- package/esm/agent/agent-input-builder.mjs.map +1 -1
- package/esm/agent/agent.d.mts +40 -1
- package/esm/agent/agent.d.mts.map +1 -1
- package/esm/agent/agent.mjs +203 -30
- package/esm/agent/agent.mjs.map +1 -1
- package/esm/agent/index.d.mts +2 -1
- package/esm/agent/index.mjs +1 -0
- package/esm/agent/judge-config.type.d.mts +33 -0
- package/esm/agent/judge-config.type.d.mts.map +1 -0
- package/esm/agent/judge-config.type.mjs +13 -0
- package/esm/agent/judge-config.type.mjs.map +1 -0
- package/esm/ai-openai/src/embedder.mjs +4 -0
- package/esm/ai-openai/src/index.mjs +4 -0
- package/esm/ai-openai/src/model.mjs +5 -0
- package/esm/ai-openai/src/sdk.mjs +6 -0
- package/esm/ai-openai/src/utils/index.mjs +4 -0
- package/esm/ai-openai/src/utils/to-openai-tools.mjs +3 -0
- package/esm/ai-openai/src/utils/wrap-openai-error.mjs +4 -0
- package/esm/ai.d.mts +119 -53
- package/esm/ai.d.mts.map +1 -1
- package/esm/ai.mjs +43 -8
- package/esm/ai.mjs.map +1 -1
- package/esm/batch/batch.d.mts.map +1 -1
- package/esm/batch/batch.mjs +21 -1
- package/esm/batch/batch.mjs.map +1 -1
- package/esm/batch/batch.type.d.mts +11 -4
- package/esm/batch/batch.type.d.mts.map +1 -1
- package/esm/config.d.mts +39 -3
- package/esm/config.d.mts.map +1 -1
- package/esm/config.mjs +26 -2
- package/esm/config.mjs.map +1 -1
- package/esm/contracts/agent/agent-options.type.d.mts +11 -3
- package/esm/contracts/agent/agent-options.type.d.mts.map +1 -1
- package/esm/contracts/agent/eval.type.d.mts +43 -2
- package/esm/contracts/agent/eval.type.d.mts.map +1 -1
- package/esm/contracts/attachment-policy.type.d.mts +51 -0
- package/esm/contracts/attachment-policy.type.d.mts.map +1 -0
- package/esm/contracts/attachment.type.d.mts +16 -7
- package/esm/contracts/attachment.type.d.mts.map +1 -1
- package/esm/contracts/content-part.type.d.mts +14 -5
- package/esm/contracts/content-part.type.d.mts.map +1 -1
- package/esm/contracts/events/supervisor-events.type.d.mts.map +1 -1
- package/esm/contracts/index.d.mts +6 -4
- package/esm/contracts/model.contract.d.mts +11 -8
- package/esm/contracts/model.contract.d.mts.map +1 -1
- package/esm/contracts/orchestrator/index.d.mts +1 -0
- package/esm/contracts/orchestrator/orchestrator-config.type.d.mts +31 -2
- package/esm/contracts/orchestrator/orchestrator-config.type.d.mts.map +1 -1
- package/esm/contracts/orchestrator/session-lock.contract.d.mts +47 -0
- package/esm/contracts/orchestrator/session-lock.contract.d.mts.map +1 -0
- package/esm/contracts/planner/index.d.mts +1 -1
- package/esm/contracts/planner/planner-config.type.d.mts +31 -0
- package/esm/contracts/planner/planner-config.type.d.mts.map +1 -1
- package/esm/contracts/planner/planner-execute-options.type.d.mts +48 -1
- package/esm/contracts/planner/planner-execute-options.type.d.mts.map +1 -1
- package/esm/contracts/planner/planner-result.type.d.mts +8 -0
- package/esm/contracts/planner/planner-result.type.d.mts.map +1 -1
- package/esm/contracts/planner/planner.contract.d.mts +1 -1
- package/esm/contracts/result/agent-result.type.d.mts +6 -4
- package/esm/contracts/result/agent-result.type.d.mts.map +1 -1
- package/esm/contracts/result/base-report.type.d.mts +21 -3
- package/esm/contracts/result/base-report.type.d.mts.map +1 -1
- package/esm/contracts/result/base-report.type.mjs.map +1 -1
- package/esm/contracts/result/execution-report.type.d.mts +53 -1
- package/esm/contracts/result/execution-report.type.d.mts.map +1 -1
- package/esm/contracts/result/supervisor-result.type.d.mts +14 -2
- package/esm/contracts/result/supervisor-result.type.d.mts.map +1 -1
- package/esm/contracts/result/workflow-result.type.d.mts +2 -1
- package/esm/contracts/result/workflow-result.type.d.mts.map +1 -1
- package/esm/contracts/supervisor/supervisor-config.type.d.mts +29 -0
- package/esm/contracts/supervisor/supervisor-config.type.d.mts.map +1 -1
- package/esm/contracts/system-prompt.contract.d.mts +103 -1
- package/esm/contracts/system-prompt.contract.d.mts.map +1 -1
- package/esm/contracts/team/index.d.mts +1 -0
- package/esm/contracts/team/team-config.type.d.mts +127 -0
- package/esm/contracts/team/team-config.type.d.mts.map +1 -0
- package/esm/contracts/tool.contract.d.mts +4 -2
- package/esm/contracts/tool.contract.d.mts.map +1 -1
- package/esm/contracts/workflow/step.contract.d.mts +30 -5
- package/esm/contracts/workflow/step.contract.d.mts.map +1 -1
- package/esm/contracts/workflow/workflow.contract.d.mts +16 -0
- package/esm/contracts/workflow/workflow.contract.d.mts.map +1 -1
- package/esm/errors/error-code.type.d.mts +1 -1
- package/esm/errors/index.d.mts +1 -0
- package/esm/errors/index.mjs +1 -0
- package/esm/errors/outbound-policy-error.d.mts +27 -0
- package/esm/errors/outbound-policy-error.d.mts.map +1 -0
- package/esm/errors/outbound-policy-error.mjs +32 -0
- package/esm/errors/outbound-policy-error.mjs.map +1 -0
- package/esm/eval/dataset.d.mts +28 -0
- package/esm/eval/dataset.d.mts.map +1 -0
- package/esm/eval/dataset.mjs +112 -0
- package/esm/eval/dataset.mjs.map +1 -0
- package/esm/eval/dataset.type.d.mts +53 -0
- package/esm/eval/dataset.type.d.mts.map +1 -0
- package/esm/eval/eval-runner.d.mts.map +1 -1
- package/esm/eval/eval-runner.mjs +16 -2
- package/esm/eval/eval-runner.mjs.map +1 -1
- package/esm/eval/index.d.mts +20 -1
- package/esm/eval/index.d.mts.map +1 -1
- package/esm/eval/index.mjs +20 -2
- package/esm/eval/index.mjs.map +1 -1
- package/esm/eval/regression.d.mts +30 -0
- package/esm/eval/regression.d.mts.map +1 -0
- package/esm/eval/regression.mjs +51 -0
- package/esm/eval/regression.mjs.map +1 -0
- package/esm/eval/report-json.d.mts +30 -0
- package/esm/eval/report-json.d.mts.map +1 -0
- package/esm/eval/report-json.mjs +33 -0
- package/esm/eval/report-json.mjs.map +1 -0
- package/esm/eval/report-junit.d.mts +22 -0
- package/esm/eval/report-junit.d.mts.map +1 -0
- package/esm/eval/report-junit.mjs +60 -0
- package/esm/eval/report-junit.mjs.map +1 -0
- package/esm/guard/contracts/guard-options.type.d.mts +165 -0
- package/esm/guard/contracts/guard-options.type.d.mts.map +1 -0
- package/esm/guard/contracts/guardrail.contract.d.mts +78 -0
- package/esm/guard/contracts/guardrail.contract.d.mts.map +1 -0
- package/esm/guard/contracts/index.d.mts +4 -0
- package/esm/guard/contracts/openai-client.contract.d.mts +51 -0
- package/esm/guard/contracts/openai-client.contract.d.mts.map +1 -0
- package/esm/guard/contracts/verdict.type.d.mts +76 -0
- package/esm/guard/contracts/verdict.type.d.mts.map +1 -0
- package/esm/guard/detectors/index.d.mts +4 -0
- package/esm/guard/detectors/index.mjs +6 -0
- package/esm/guard/detectors/injection.d.mts +34 -0
- package/esm/guard/detectors/injection.d.mts.map +1 -0
- package/esm/guard/detectors/injection.mjs +254 -0
- package/esm/guard/detectors/injection.mjs.map +1 -0
- package/esm/guard/detectors/moderation.d.mts +32 -0
- package/esm/guard/detectors/moderation.d.mts.map +1 -0
- package/esm/guard/detectors/moderation.mjs +134 -0
- package/esm/guard/detectors/moderation.mjs.map +1 -0
- package/esm/guard/detectors/pii.d.mts +41 -0
- package/esm/guard/detectors/pii.d.mts.map +1 -0
- package/esm/guard/detectors/pii.mjs +199 -0
- package/esm/guard/detectors/pii.mjs.map +1 -0
- package/esm/guard/detectors/topic.d.mts +29 -0
- package/esm/guard/detectors/topic.d.mts.map +1 -0
- package/esm/guard/detectors/topic.mjs +99 -0
- package/esm/guard/detectors/topic.mjs.map +1 -0
- package/esm/guard/errors.d.mts +12 -0
- package/esm/guard/errors.d.mts.map +1 -0
- package/esm/guard/errors.mjs +18 -0
- package/esm/guard/errors.mjs.map +1 -0
- package/esm/guard/guard.d.mts +67 -0
- package/esm/guard/guard.d.mts.map +1 -0
- package/esm/guard/guard.mjs +209 -0
- package/esm/guard/guard.mjs.map +1 -0
- package/esm/guard/guardrail.d.mts +39 -0
- package/esm/guard/guardrail.d.mts.map +1 -0
- package/esm/guard/guardrail.mjs +22 -0
- package/esm/guard/guardrail.mjs.map +1 -0
- package/esm/human/contracts/approval.type.d.mts +154 -0
- package/esm/human/contracts/approval.type.d.mts.map +1 -0
- package/esm/human/contracts/human-approval.type.d.mts +38 -0
- package/esm/human/contracts/human-approval.type.d.mts.map +1 -0
- package/esm/human/contracts/index.d.mts +4 -0
- package/esm/human/contracts/interrupt-store.contract.d.mts +112 -0
- package/esm/human/contracts/interrupt-store.contract.d.mts.map +1 -0
- package/esm/human/contracts/resume.type.d.mts +77 -0
- package/esm/human/contracts/resume.type.d.mts.map +1 -0
- package/esm/human/errors.d.mts +84 -0
- package/esm/human/errors.d.mts.map +1 -0
- package/esm/human/errors.mjs +60 -0
- package/esm/human/errors.mjs.map +1 -0
- package/esm/human/human-approval.d.mts +57 -0
- package/esm/human/human-approval.d.mts.map +1 -0
- package/esm/human/human-approval.mjs +170 -0
- package/esm/human/human-approval.mjs.map +1 -0
- package/esm/human/policy.d.mts +55 -0
- package/esm/human/policy.d.mts.map +1 -0
- package/esm/human/policy.mjs +67 -0
- package/esm/human/policy.mjs.map +1 -0
- package/esm/human/register.mjs +37 -0
- package/esm/human/register.mjs.map +1 -0
- package/esm/human/resume-seed.mjs +53 -0
- package/esm/human/resume-seed.mjs.map +1 -0
- package/esm/human/resume.d.mts +54 -0
- package/esm/human/resume.d.mts.map +1 -0
- package/esm/human/resume.mjs +121 -0
- package/esm/human/resume.mjs.map +1 -0
- package/esm/human/stores/index.mjs +5 -0
- package/esm/human/stores/memory.d.mts +30 -0
- package/esm/human/stores/memory.d.mts.map +1 -0
- package/esm/human/stores/memory.mjs +91 -0
- package/esm/human/stores/memory.mjs.map +1 -0
- package/esm/human/stores/pg.d.mts +59 -0
- package/esm/human/stores/pg.d.mts.map +1 -0
- package/esm/human/stores/pg.mjs +220 -0
- package/esm/human/stores/pg.mjs.map +1 -0
- package/esm/human/stores/redis.d.mts +56 -0
- package/esm/human/stores/redis.d.mts.map +1 -0
- package/esm/human/stores/redis.mjs +201 -0
- package/esm/human/stores/redis.mjs.map +1 -0
- package/esm/index.d.mts +113 -22
- package/esm/index.mjs +82 -11
- package/esm/middleware/builtins/budget.mjs +6 -2
- package/esm/middleware/builtins/budget.mjs.map +1 -1
- package/esm/middleware/utils/extract-user-text.d.mts +8 -1
- package/esm/middleware/utils/extract-user-text.d.mts.map +1 -1
- package/esm/middleware/utils/extract-user-text.mjs +8 -1
- package/esm/middleware/utils/extract-user-text.mjs.map +1 -1
- package/esm/object-stream/index.d.mts +2 -0
- package/esm/object-stream/index.mjs +4 -0
- package/esm/object-stream/parse-partial-json.d.mts +22 -0
- package/esm/object-stream/parse-partial-json.d.mts.map +1 -0
- package/esm/object-stream/parse-partial-json.mjs +78 -0
- package/esm/object-stream/parse-partial-json.mjs.map +1 -0
- package/esm/object-stream/stream-object.d.mts +68 -0
- package/esm/object-stream/stream-object.d.mts.map +1 -0
- package/esm/object-stream/stream-object.mjs +104 -0
- package/esm/object-stream/stream-object.mjs.map +1 -0
- package/esm/observe/index.mjs +4 -0
- package/esm/observe/observer-registry.d.mts +30 -0
- package/esm/observe/observer-registry.d.mts.map +1 -0
- package/esm/observe/observer-registry.mjs +51 -0
- package/esm/observe/observer-registry.mjs.map +1 -0
- package/esm/observe/observer.contract.d.mts +40 -0
- package/esm/observe/observer.contract.d.mts.map +1 -0
- package/esm/observe/resolve-observers.d.mts +40 -0
- package/esm/observe/resolve-observers.d.mts.map +1 -0
- package/esm/observe/resolve-observers.mjs +73 -0
- package/esm/observe/resolve-observers.mjs.map +1 -0
- package/esm/orchestrator/execution.d.mts.map +1 -1
- package/esm/orchestrator/execution.mjs +5 -2
- package/esm/orchestrator/execution.mjs.map +1 -1
- package/esm/orchestrator/index.d.mts +1 -0
- package/esm/orchestrator/index.mjs +1 -0
- package/esm/orchestrator/orchestrator.d.mts.map +1 -1
- package/esm/orchestrator/orchestrator.mjs +39 -6
- package/esm/orchestrator/orchestrator.mjs.map +1 -1
- package/esm/orchestrator/session-lock.d.mts +25 -0
- package/esm/orchestrator/session-lock.d.mts.map +1 -0
- package/esm/orchestrator/session-lock.mjs +83 -0
- package/esm/orchestrator/session-lock.mjs.map +1 -0
- package/esm/planner/dag-scheduler.mjs +97 -0
- package/esm/planner/dag-scheduler.mjs.map +1 -0
- package/esm/planner/plan-prompt.d.mts +1 -1
- package/esm/planner/plan-prompt.d.mts.map +1 -1
- package/esm/planner/plan-prompt.mjs +2 -1
- package/esm/planner/plan-prompt.mjs.map +1 -1
- package/esm/planner/planner-run.d.mts.map +1 -1
- package/esm/planner/planner-run.mjs +300 -29
- package/esm/planner/planner-run.mjs.map +1 -1
- package/esm/planner/planner.mjs +1 -1
- package/esm/planner/planner.mjs.map +1 -1
- package/esm/prompt/errors.d.mts +57 -0
- package/esm/prompt/errors.d.mts.map +1 -0
- package/esm/prompt/errors.mjs +73 -0
- package/esm/prompt/errors.mjs.map +1 -0
- package/esm/prompt/index.d.mts +3 -0
- package/esm/prompt/index.mjs +4 -0
- package/esm/prompt/prompt-langfuse-sync.mjs +104 -0
- package/esm/prompt/prompt-langfuse-sync.mjs.map +1 -0
- package/esm/prompt/prompt-langfuse-sync.type.d.mts +32 -0
- package/esm/prompt/prompt-langfuse-sync.type.d.mts.map +1 -0
- package/esm/prompt/prompt-validate.mjs +170 -0
- package/esm/prompt/prompt-validate.mjs.map +1 -0
- package/esm/prompt/prompt.d.mts +54 -0
- package/esm/prompt/prompt.d.mts.map +1 -0
- package/esm/prompt/prompt.mjs +218 -0
- package/esm/prompt/prompt.mjs.map +1 -0
- package/esm/prompt/prompt.type.d.mts +174 -0
- package/esm/prompt/prompt.type.d.mts.map +1 -0
- package/esm/prompts/index.d.mts +3 -0
- package/esm/prompts/index.mjs +3 -0
- package/esm/prompts/prompts-manager.contract.d.mts +154 -0
- package/esm/prompts/prompts-manager.contract.d.mts.map +1 -0
- package/esm/prompts/prompts-manager.d.mts +38 -0
- package/esm/prompts/prompts-manager.d.mts.map +1 -0
- package/esm/prompts/prompts-manager.mjs +410 -0
- package/esm/prompts/prompts-manager.mjs.map +1 -0
- package/esm/prompts/prompts-manager.type.d.mts +172 -0
- package/esm/prompts/prompts-manager.type.d.mts.map +1 -0
- package/esm/prompts/prompts-validate.mjs +200 -0
- package/esm/prompts/prompts-validate.mjs.map +1 -0
- package/esm/rag/as-tool.mjs +48 -0
- package/esm/rag/as-tool.mjs.map +1 -0
- package/esm/rag/chunk/chunk.d.mts +24 -0
- package/esm/rag/chunk/chunk.d.mts.map +1 -0
- package/esm/rag/chunk/chunk.mjs +44 -0
- package/esm/rag/chunk/chunk.mjs.map +1 -0
- package/esm/rag/chunk/fixed.mjs +32 -0
- package/esm/rag/chunk/fixed.mjs.map +1 -0
- package/esm/rag/chunk/markdown.mjs +75 -0
- package/esm/rag/chunk/markdown.mjs.map +1 -0
- package/esm/rag/chunk/recursive.mjs +132 -0
- package/esm/rag/chunk/recursive.mjs.map +1 -0
- package/esm/rag/chunk/sentence.mjs +73 -0
- package/esm/rag/chunk/sentence.mjs.map +1 -0
- package/esm/rag/contracts/chunk-options.type.d.mts +35 -0
- package/esm/rag/contracts/chunk-options.type.d.mts.map +1 -0
- package/esm/rag/contracts/citation.type.d.mts +35 -0
- package/esm/rag/contracts/citation.type.d.mts.map +1 -0
- package/esm/rag/contracts/index.d.mts +4 -0
- package/esm/rag/contracts/rag-config.type.d.mts +68 -0
- package/esm/rag/contracts/rag-config.type.d.mts.map +1 -0
- package/esm/rag/contracts/rag-document.type.d.mts +21 -0
- package/esm/rag/contracts/rag-document.type.d.mts.map +1 -0
- package/esm/rag/hybrid/bm25.d.mts +23 -0
- package/esm/rag/hybrid/bm25.d.mts.map +1 -0
- package/esm/rag/hybrid/bm25.mjs +51 -0
- package/esm/rag/hybrid/bm25.mjs.map +1 -0
- package/esm/rag/hybrid/hybrid-rank.d.mts +33 -0
- package/esm/rag/hybrid/hybrid-rank.d.mts.map +1 -0
- package/esm/rag/hybrid/hybrid-rank.mjs +29 -0
- package/esm/rag/hybrid/hybrid-rank.mjs.map +1 -0
- package/esm/rag/hybrid/rrf.d.mts +25 -0
- package/esm/rag/hybrid/rrf.d.mts.map +1 -0
- package/esm/rag/hybrid/rrf.mjs +30 -0
- package/esm/rag/hybrid/rrf.mjs.map +1 -0
- package/esm/rag/index.d.mts +15 -0
- package/esm/rag/index.mjs +11 -0
- package/esm/rag/rag.d.mts +38 -0
- package/esm/rag/rag.d.mts.map +1 -0
- package/esm/rag/rag.mjs +126 -0
- package/esm/rag/rag.mjs.map +1 -0
- package/esm/rag/rerank/keyword-reranker.d.mts +32 -0
- package/esm/rag/rerank/keyword-reranker.d.mts.map +1 -0
- package/esm/rag/rerank/keyword-reranker.mjs +58 -0
- package/esm/rag/rerank/keyword-reranker.mjs.map +1 -0
- package/esm/rag/rerank/llm-reranker.d.mts +36 -0
- package/esm/rag/rerank/llm-reranker.d.mts.map +1 -0
- package/esm/rag/rerank/llm-reranker.mjs +85 -0
- package/esm/rag/rerank/llm-reranker.mjs.map +1 -0
- package/esm/rag/rerank/reranker.contract.d.mts +28 -0
- package/esm/rag/rerank/reranker.contract.d.mts.map +1 -0
- package/esm/rag/retrieve.mjs +68 -0
- package/esm/rag/retrieve.mjs.map +1 -0
- package/esm/rag/store/cache-vector-store.d.mts +27 -0
- package/esm/rag/store/cache-vector-store.d.mts.map +1 -0
- package/esm/rag/store/cache-vector-store.mjs +48 -0
- package/esm/rag/store/cache-vector-store.mjs.map +1 -0
- package/esm/rag/store/vector-store.contract.d.mts +38 -0
- package/esm/rag/store/vector-store.contract.d.mts.map +1 -0
- package/esm/rag/transforms/multi-query.d.mts +27 -0
- package/esm/rag/transforms/multi-query.d.mts.map +1 -0
- package/esm/rag/transforms/multi-query.mjs +41 -0
- package/esm/rag/transforms/multi-query.mjs.map +1 -0
- package/esm/security/index.mjs +5 -0
- package/esm/security/outbound-policy.d.mts +46 -0
- package/esm/security/outbound-policy.d.mts.map +1 -0
- package/esm/security/outbound-policy.mjs +187 -0
- package/esm/security/outbound-policy.mjs.map +1 -0
- package/esm/security/outbound-policy.type.d.mts +74 -0
- package/esm/security/outbound-policy.type.d.mts.map +1 -0
- package/esm/security/private-ip.d.mts +15 -0
- package/esm/security/private-ip.d.mts.map +1 -0
- package/esm/security/private-ip.mjs +48 -0
- package/esm/security/private-ip.mjs.map +1 -0
- package/esm/security/redact.d.mts +59 -0
- package/esm/security/redact.d.mts.map +1 -0
- package/esm/security/redact.mjs +122 -0
- package/esm/security/redact.mjs.map +1 -0
- package/esm/serve/serve.d.mts +50 -0
- package/esm/serve/serve.d.mts.map +1 -0
- package/esm/serve/serve.mjs +90 -0
- package/esm/serve/serve.mjs.map +1 -0
- package/esm/serve/sse.d.mts +20 -0
- package/esm/serve/sse.d.mts.map +1 -0
- package/esm/serve/sse.mjs +25 -0
- package/esm/serve/sse.mjs.map +1 -0
- package/esm/serve/stream-to-sse.d.mts +29 -0
- package/esm/serve/stream-to-sse.d.mts.map +1 -0
- package/esm/serve/stream-to-sse.mjs +37 -0
- package/esm/serve/stream-to-sse.mjs.map +1 -0
- package/esm/skills/catalog.d.mts +49 -0
- package/esm/skills/catalog.d.mts.map +1 -0
- package/esm/skills/catalog.mjs +140 -0
- package/esm/skills/catalog.mjs.map +1 -0
- package/esm/skills/contracts/skill-record.type.d.mts +37 -0
- package/esm/skills/contracts/skill-record.type.d.mts.map +1 -0
- package/esm/skills/contracts/skills-config.type.d.mts +108 -0
- package/esm/skills/contracts/skills-config.type.d.mts.map +1 -0
- package/esm/skills/contracts/skills-store.contract.d.mts +28 -0
- package/esm/skills/contracts/skills-store.contract.d.mts.map +1 -0
- package/esm/skills/contracts/skills.contract.d.mts +43 -0
- package/esm/skills/contracts/skills.contract.d.mts.map +1 -0
- package/esm/skills/index.d.mts +16 -0
- package/esm/skills/index.mjs +14 -0
- package/esm/skills/load-skill-tool.d.mts +38 -0
- package/esm/skills/load-skill-tool.d.mts.map +1 -0
- package/esm/skills/load-skill-tool.mjs +65 -0
- package/esm/skills/load-skill-tool.mjs.map +1 -0
- package/esm/skills/review-gate.d.mts +33 -0
- package/esm/skills/review-gate.d.mts.map +1 -0
- package/esm/skills/review-gate.mjs +60 -0
- package/esm/skills/review-gate.mjs.map +1 -0
- package/esm/skills/save-skill-tool.d.mts +39 -0
- package/esm/skills/save-skill-tool.d.mts.map +1 -0
- package/esm/skills/save-skill-tool.mjs +65 -0
- package/esm/skills/save-skill-tool.mjs.map +1 -0
- package/esm/skills/skills.d.mts +33 -0
- package/esm/skills/skills.d.mts.map +1 -0
- package/esm/skills/skills.mjs +109 -0
- package/esm/skills/skills.mjs.map +1 -0
- package/esm/skills/sources/directory-source.d.mts +19 -0
- package/esm/skills/sources/directory-source.d.mts.map +1 -0
- package/esm/skills/sources/directory-source.mjs +108 -0
- package/esm/skills/sources/directory-source.mjs.map +1 -0
- package/esm/skills/sources/index.d.mts +18 -0
- package/esm/skills/sources/index.d.mts.map +1 -0
- package/esm/skills/sources/index.mjs +27 -0
- package/esm/skills/sources/index.mjs.map +1 -0
- package/esm/skills/sources/parse-frontmatter.d.mts +27 -0
- package/esm/skills/sources/parse-frontmatter.d.mts.map +1 -0
- package/esm/skills/sources/parse-frontmatter.mjs +46 -0
- package/esm/skills/sources/parse-frontmatter.mjs.map +1 -0
- package/esm/skills/sources/store-source.d.mts +14 -0
- package/esm/skills/sources/store-source.d.mts.map +1 -0
- package/esm/skills/sources/store-source.mjs +15 -0
- package/esm/skills/sources/store-source.mjs.map +1 -0
- package/esm/skills/sources/url-source.d.mts +29 -0
- package/esm/skills/sources/url-source.d.mts.map +1 -0
- package/esm/skills/sources/url-source.mjs +117 -0
- package/esm/skills/sources/url-source.mjs.map +1 -0
- package/esm/skills/store/mock-skills-store.d.mts +57 -0
- package/esm/skills/store/mock-skills-store.d.mts.map +1 -0
- package/esm/skills/store/mock-skills-store.mjs +100 -0
- package/esm/skills/store/mock-skills-store.mjs.map +1 -0
- package/esm/skills/store/procedural-skill-store.d.mts +30 -0
- package/esm/skills/store/procedural-skill-store.d.mts.map +1 -0
- package/esm/skills/store/procedural-skill-store.mjs +125 -0
- package/esm/skills/store/procedural-skill-store.mjs.map +1 -0
- package/esm/supervisor/as-tool.mjs +2 -2
- package/esm/supervisor/as-tool.mjs.map +1 -1
- package/esm/supervisor/execution.d.mts.map +1 -1
- package/esm/supervisor/execution.mjs +31 -28
- package/esm/supervisor/execution.mjs.map +1 -1
- package/esm/supervisor/supervisor.d.mts.map +1 -1
- package/esm/supervisor/supervisor.mjs +8 -3
- package/esm/supervisor/supervisor.mjs.map +1 -1
- package/esm/system-prompt/index.d.mts +4 -0
- package/esm/system-prompt/system-prompt.d.mts +68 -4
- package/esm/system-prompt/system-prompt.d.mts.map +1 -1
- package/esm/system-prompt/system-prompt.mjs +89 -5
- package/esm/system-prompt/system-prompt.mjs.map +1 -1
- package/esm/team/gates.mjs +48 -0
- package/esm/team/gates.mjs.map +1 -0
- package/esm/team/index.d.mts +1 -0
- package/esm/team/index.mjs +3 -0
- package/esm/team/team.d.mts +42 -0
- package/esm/team/team.d.mts.map +1 -0
- package/esm/team/team.mjs +94 -0
- package/esm/team/team.mjs.map +1 -0
- package/esm/tool/executable-as-tool.d.mts.map +1 -1
- package/esm/tool/executable-as-tool.mjs +2 -2
- package/esm/tool/executable-as-tool.mjs.map +1 -1
- package/esm/tool/tool.d.mts.map +1 -1
- package/esm/tool/tool.mjs +2 -2
- package/esm/tool/tool.mjs.map +1 -1
- package/esm/utils/compute-cost.d.mts +17 -1
- package/esm/utils/compute-cost.d.mts.map +1 -1
- package/esm/utils/compute-cost.mjs +26 -1
- package/esm/utils/compute-cost.mjs.map +1 -1
- package/esm/utils/extract-json-lenient.d.mts +42 -0
- package/esm/utils/extract-json-lenient.d.mts.map +1 -0
- package/esm/utils/extract-json-lenient.mjs +97 -0
- package/esm/utils/extract-json-lenient.mjs.map +1 -0
- package/esm/utils/index.d.mts +4 -2
- package/esm/utils/index.mjs +3 -1
- package/esm/utils/json-schema.d.mts +1 -1
- package/esm/utils/prepare-attachment-part.d.mts +10 -1
- package/esm/utils/prepare-attachment-part.d.mts.map +1 -1
- package/esm/utils/prepare-attachment-part.mjs +103 -11
- package/esm/utils/prepare-attachment-part.mjs.map +1 -1
- package/esm/utils/resolve-attachment.d.mts +4 -3
- package/esm/utils/resolve-attachment.d.mts.map +1 -1
- package/esm/utils/resolve-attachment.mjs +4 -3
- package/esm/utils/resolve-attachment.mjs.map +1 -1
- package/esm/utils/run-context.d.mts +94 -0
- package/esm/utils/run-context.d.mts.map +1 -0
- package/esm/utils/run-context.mjs +98 -0
- package/esm/utils/run-context.mjs.map +1 -0
- package/esm/vcr/cassette-io.mjs +57 -0
- package/esm/vcr/cassette-io.mjs.map +1 -0
- package/esm/vcr/errors.d.mts +42 -0
- package/esm/vcr/errors.d.mts.map +1 -0
- package/esm/vcr/errors.mjs +37 -0
- package/esm/vcr/errors.mjs.map +1 -0
- package/esm/vcr/hash-request.d.mts +28 -0
- package/esm/vcr/hash-request.d.mts.map +1 -0
- package/esm/vcr/hash-request.mjs +118 -0
- package/esm/vcr/hash-request.mjs.map +1 -0
- package/esm/vcr/index.d.mts +4 -0
- package/esm/vcr/index.mjs +5 -0
- package/esm/vcr/vcr.d.mts +32 -0
- package/esm/vcr/vcr.d.mts.map +1 -0
- package/esm/vcr/vcr.mjs +248 -0
- package/esm/vcr/vcr.mjs.map +1 -0
- package/esm/vcr/vcr.type.d.mts +118 -0
- package/esm/vcr/vcr.type.d.mts.map +1 -0
- package/esm/workflow/as-tool.mjs +2 -2
- package/esm/workflow/as-tool.mjs.map +1 -1
- package/esm/workflow/engine.mjs +1 -0
- package/esm/workflow/engine.mjs.map +1 -1
- package/esm/workflow/step-runner.mjs +19 -20
- package/esm/workflow/step-runner.mjs.map +1 -1
- package/esm/workflow/workflow.d.mts.map +1 -1
- package/esm/workflow/workflow.mjs +8 -3
- package/esm/workflow/workflow.mjs.map +1 -1
- package/llms-full.txt +1683 -68
- package/llms.txt +16 -4
- package/package.json +7 -3
- package/skills/README.md +40 -4
- package/skills/ai-dx-helpers/SKILL.md +2 -2
- package/skills/approve-tool-calls/SKILL.md +134 -0
- package/skills/attach-ai-middleware/SKILL.md +1 -1
- package/skills/detect-and-redact-pii/SKILL.md +104 -0
- package/skills/durable-resume/SKILL.md +128 -0
- package/skills/escalate-block-to-human/SKILL.md +85 -0
- package/skills/eval-datasets-and-ci/SKILL.md +117 -0
- package/skills/guard-input-output/SKILL.md +117 -0
- package/skills/manage-prompts/SKILL.md +186 -0
- package/skills/observe-ai-flows/SKILL.md +94 -0
- package/skills/record-replay-llm/SKILL.md +92 -0
- package/skills/run-ai-agent/SKILL.md +26 -1
- package/skills/run-ai-rag/SKILL.md +139 -0
- package/skills/run-ai-team/SKILL.md +107 -0
- package/skills/run-orchestrator/SKILL.md +2 -0
- package/skills/run-planner/SKILL.md +73 -8
- package/skills/run-supervisor/SKILL.md +20 -1
- package/skills/use-runtime-skills/SKILL.md +106 -0
- package/skills/write-system-prompt/SKILL.md +30 -1
- package/cjs/index.cjs.map +0 -1
- package/esm/contracts/result/index.d.mts +0 -15
package/llms-full.txt
CHANGED
|
@@ -28,7 +28,7 @@ ai.orchestrator() → durable session — state/history/resume [shipped]
|
|
|
28
28
|
|
|
29
29
|
Each primitive is an escape hatch to the next level of complexity. Users start low, graduate upward only when needed. Every primitive returns the same result envelope — canonical destructure `{ data, error, usage, report }` (the shared `BaseResult` guarantees `usage` + optional `error`; each primitive adds `data` + `report`). Workflows, supervisors, and orchestrators expose `.asTool()` so an agent can call them inside its tool loop; raw executables also auto-adapt when dropped into an agent's `tools: []`. Compose freely.
|
|
30
30
|
|
|
31
|
-
Beyond the ladder: `ai.planner()` (LLM-generated plans), `ai.memory()` (working + semantic
|
|
31
|
+
Beyond the ladder: `ai.planner()` (LLM-generated plans), `ai.memory()` (working + semantic recall), `ai.batch()` / `ai.fallbackModel()` / `ai.router()` / `ai.fanOut()` (DX helpers), `agent.eval()` (scoring), and the `ai.checkpoint.*` / `ai.snapshot.*` orchestrator stores.
|
|
32
32
|
|
|
33
33
|
## Foundations
|
|
34
34
|
|
|
@@ -66,14 +66,14 @@ console.log(text, usage.total, report.duration);
|
|
|
66
66
|
|
|
67
67
|
| If the task is about… | Load |
|
|
68
68
|
| --- | --- |
|
|
69
|
-
| `ai.agent({...})` — single-LLM-turn primitive, structured output, streaming, attachments | [`@warlock.js/ai/run-ai-agent/SKILL.md`](@warlock.js/ai/run-ai-agent/SKILL.md) |
|
|
69
|
+
| `ai.agent({...})` — single-LLM-turn primitive, structured output, streaming, attachments, `spawnSubAgent` | [`@warlock.js/ai/run-ai-agent/SKILL.md`](@warlock.js/ai/run-ai-agent/SKILL.md) |
|
|
70
70
|
| `ai.tool({...})` — typed validated functions the model can call | [`@warlock.js/ai/define-ai-tool/SKILL.md`](@warlock.js/ai/define-ai-tool/SKILL.md) |
|
|
71
71
|
| `ai.systemPrompt()` / `ai.persona()` / `ai.instruction()` — composable prompts with placeholders | [`@warlock.js/ai/write-system-prompt/SKILL.md`](@warlock.js/ai/write-system-prompt/SKILL.md) |
|
|
72
72
|
| `ai.workflow({...})` — durable resumable pipelines with steps, routing, retry | [`@warlock.js/ai/run-ai-workflow/SKILL.md`](@warlock.js/ai/run-ai-workflow/SKILL.md) |
|
|
73
73
|
| `ai.supervisor({...})` — multi-intent routing, fan-out, evaluate loops | [`@warlock.js/ai/run-supervisor/SKILL.md`](@warlock.js/ai/run-supervisor/SKILL.md) |
|
|
74
74
|
| `ai.orchestrator({...})` — durable stateful sessions, drift, compaction, resume | [`@warlock.js/ai/run-orchestrator/SKILL.md`](@warlock.js/ai/run-orchestrator/SKILL.md) |
|
|
75
75
|
| `ai.planner({...})` — LLM-generated plans over registered capabilities | [`@warlock.js/ai/run-planner/SKILL.md`](@warlock.js/ai/run-planner/SKILL.md) |
|
|
76
|
-
| `ai.memory({...})` — working + semantic
|
|
76
|
+
| `ai.memory({...})` — working + semantic recall for agents / sessions | [`@warlock.js/ai/use-ai-memory/SKILL.md`](@warlock.js/ai/use-ai-memory/SKILL.md) |
|
|
77
77
|
| `ai.checkpoint.*` / `ai.snapshot.*` — orchestrator session + run stores | [`@warlock.js/ai/manage-ai-stores/SKILL.md`](@warlock.js/ai/manage-ai-stores/SKILL.md) |
|
|
78
78
|
| DX helpers — `batch` / `fallbackModel` / `eval` + matchers / SLO contracts / `fromFile` | [`@warlock.js/ai/ai-dx-helpers/SKILL.md`](@warlock.js/ai/ai-dx-helpers/SKILL.md) |
|
|
79
79
|
| `sdk.embedder({...})` — text-to-vector for RAG tools, vector ingest | [`@warlock.js/ai/embed-text/SKILL.md`](@warlock.js/ai/embed-text/SKILL.md) |
|
|
@@ -156,8 +156,8 @@ A drop-in `ModelContract` that wraps an ordered list and advances to the next mo
|
|
|
156
156
|
|
|
157
157
|
```ts
|
|
158
158
|
const model = ai.fallbackModel([
|
|
159
|
-
|
|
160
|
-
|
|
159
|
+
openai.model({ name: "gpt-4o" }),
|
|
160
|
+
anthropic.model({ name: "claude-3-5-sonnet" }),
|
|
161
161
|
]);
|
|
162
162
|
|
|
163
163
|
const agent = ai.agent({ model }); // hand it anywhere a model goes
|
|
@@ -298,6 +298,144 @@ const concierge = ai.agent({
|
|
|
298
298
|
- [`@warlock.js/ai/pick-ai-provider/SKILL.md`](@warlock.js/ai/pick-ai-provider/SKILL.md) — `fallbackModel` wraps these adapters; cost-truth tokens
|
|
299
299
|
|
|
300
300
|
|
|
301
|
+
## approve-tool-calls `@warlock.js/ai/approve-tool-calls/SKILL.md`
|
|
302
|
+
|
|
303
|
+
---
|
|
304
|
+
name: approve-tool-calls
|
|
305
|
+
description: 'Gate an agent''s tool calls behind a human with `ai.human.approval(options)` (the `tool.before` approval-gate middleware) — ships in @warlock.js/ai core. Triggers: `ai.human.approval`, `humanApproval`, `HumanApprovalOptions`, `ApprovalRequest`, `ApprovalDecision`, `ApprovalHandler`, `InterruptPolicy`, `evaluatePolicy`, `ApprovalRejectedError`, `policy: { type: "allowlist" | "denylist" | "predicate" }`, decision `{ type: "approve" | "reject" | "edit" }`; ''human in the loop'', ''approve a tool call before it runs'', ''ask a human before the agent sends/charges/deletes'', ''pause before a dangerous tool'', ''let an operator edit the tool args'', ''reject a tool call with a reason the model can self-correct from''. Typical import `import { ai } from "@warlock.js/ai"`. Skip: persisting the request and resuming hours later out-of-process — `@warlock.js/ai/durable-resume/SKILL.md`; the agent/middleware/tool primitives themselves — `@warlock.js/ai`.'
|
|
306
|
+
---
|
|
307
|
+
|
|
308
|
+
# Approve tool calls — the human-in-the-loop gate
|
|
309
|
+
|
|
310
|
+
`ai.human.approval(options)` returns an `AgentMiddleware` with **one** hook — `tool.before` — that pauses *before a specific tool call* and routes it to a human. The human can **approve** (run the real tool unchanged), **reject** (the model sees a typed error and self-corrects), or **edit** (run the tool with replaced args). Every call the policy doesn't gate passes through untouched.
|
|
311
|
+
|
|
312
|
+
`ai.human.*` ships natively on the shared `ai` object from `@warlock.js/ai` core — no extra import or registration step. The named `humanApproval` export is the same factory.
|
|
313
|
+
|
|
314
|
+
```ts
|
|
315
|
+
import { ai } from "@warlock.js/ai";
|
|
316
|
+
import { OpenAISDK } from "@warlock.js/ai-openai";
|
|
317
|
+
|
|
318
|
+
const openai = new OpenAISDK({ apiKey: process.env.OPENAI_API_KEY! });
|
|
319
|
+
|
|
320
|
+
const support = ai.agent({
|
|
321
|
+
model: openai.model({ name: "gpt-4o" }),
|
|
322
|
+
tools: [refundCustomer, lookupOrder],
|
|
323
|
+
middleware: [
|
|
324
|
+
ai.human.approval({
|
|
325
|
+
policy: { type: "allowlist", tools: ["refundCustomer"], tags: () => ["money"] },
|
|
326
|
+
// SSE / CLI handler resolves when the operator rules:
|
|
327
|
+
handler: async (req) => ui.prompt(req), // → { type: "approve" } | { type: "reject", reason } | { type: "edit", args }
|
|
328
|
+
}),
|
|
329
|
+
],
|
|
330
|
+
});
|
|
331
|
+
|
|
332
|
+
await support.execute("Refund order #4821"); // pauses at refundCustomer, awaits the operator
|
|
333
|
+
```
|
|
334
|
+
|
|
335
|
+
## The three decisions
|
|
336
|
+
|
|
337
|
+
A handler turns an `ApprovalRequest` into an `ApprovalDecision` — a union discriminated by `type` (never `kind`):
|
|
338
|
+
|
|
339
|
+
| `decision.type` | Effect | What the model sees next |
|
|
340
|
+
|---|---|---|
|
|
341
|
+
| `"approve"` | The real tool runs with the model's original args. | The tool's normal result. |
|
|
342
|
+
| `"reject"` | Short-circuits an `ApprovalRejectedError` carrying `reason`. | `{ error }` on the next trip — it can self-correct. |
|
|
343
|
+
| `"edit"` | The reviewer's `args` replace the model's; the real tool then runs. | The tool's result for the edited args. |
|
|
344
|
+
|
|
345
|
+
```ts
|
|
346
|
+
type ApprovalDecision =
|
|
347
|
+
| { type: "approve" }
|
|
348
|
+
| { type: "reject"; reason: string }
|
|
349
|
+
| { type: "edit"; args: unknown; reason?: string };
|
|
350
|
+
```
|
|
351
|
+
|
|
352
|
+
An `edit` still goes through the tool's own Standard-Schema validation — if the replaced args don't fit the schema, the tool surfaces a validation error on `result.error` and the model self-corrects. No special-casing.
|
|
353
|
+
|
|
354
|
+
## The interrupt policy — which calls need a human
|
|
355
|
+
|
|
356
|
+
`policy` decides *which* tool calls are gated. It's a union keyed on `type`:
|
|
357
|
+
|
|
358
|
+
| `policy.type` | Gates a call when… | Tags |
|
|
359
|
+
|---|---|---|
|
|
360
|
+
| `"allowlist"` | the tool name **is** in `tools`. | optional `tags(toolName)` callback |
|
|
361
|
+
| `"denylist"` | the tool name is **not** in `tools` (gate everything else). | optional `tags(toolName)` callback |
|
|
362
|
+
| `"predicate"` | `requiresApproval(ctx)` returns a truthy value. | a returned `string[]` doubles as the tags |
|
|
363
|
+
|
|
364
|
+
```ts
|
|
365
|
+
// Allowlist — only refunds need sign-off:
|
|
366
|
+
{ type: "allowlist", tools: ["refundCustomer"], tags: () => ["money"] }
|
|
367
|
+
|
|
368
|
+
// Denylist — everything except read-only lookups needs sign-off:
|
|
369
|
+
{ type: "denylist", tools: ["lookupOrder", "searchCatalog"] }
|
|
370
|
+
|
|
371
|
+
// Predicate — args-aware: only large refunds, tagged for the reviewer UI:
|
|
372
|
+
{
|
|
373
|
+
type: "predicate",
|
|
374
|
+
requiresApproval: (ctx) =>
|
|
375
|
+
ctx.toolName === "refundCustomer" && (ctx.args as { amount: number }).amount > 100
|
|
376
|
+
? ["money", "high-value"]
|
|
377
|
+
: false,
|
|
378
|
+
}
|
|
379
|
+
```
|
|
380
|
+
|
|
381
|
+
The predicate sees a read-only `PolicyContext` — `toolName`, `toolDescription`, `args` (the model's exact input), `agentName`, `tripIndex`, `sessionId`. Return `false` (or an empty array) to skip approval; `true` or a non-empty `string[]` to require it. The `string[]` becomes `request.context.tags`, surfaced verbatim to the reviewer so a UI can group or prioritize.
|
|
382
|
+
|
|
383
|
+
Compose with `forTool(names, mw)` from `@warlock.js/ai` for static, name-based scoping and let `policy` be the dynamic, args-aware layer on top.
|
|
384
|
+
|
|
385
|
+
`evaluatePolicy(policy, context)` is the exported, pure core if you want to reuse the gate decision outside the middleware (it never throws, does no IO, and returns `{ requiresApproval, tags? }`).
|
|
386
|
+
|
|
387
|
+
## The request a reviewer rules on
|
|
388
|
+
|
|
389
|
+
For a gated call the middleware builds an `ApprovalRequest` and hands it to your `handler`:
|
|
390
|
+
|
|
391
|
+
```ts
|
|
392
|
+
interface ApprovalRequest {
|
|
393
|
+
interruptId: string; // stable id; durable mode keys the store on it
|
|
394
|
+
toolName: string;
|
|
395
|
+
toolDescription?: string;
|
|
396
|
+
args: unknown; // the model's exact args
|
|
397
|
+
context: {
|
|
398
|
+
agentName: string;
|
|
399
|
+
tripIndex: number;
|
|
400
|
+
sessionId?: string;
|
|
401
|
+
originalInput?: string; // the run's prompt (used by durable re-run)
|
|
402
|
+
tags?: string[]; // from the policy match
|
|
403
|
+
};
|
|
404
|
+
requestedAt: string; // ISO-8601
|
|
405
|
+
}
|
|
406
|
+
```
|
|
407
|
+
|
|
408
|
+
The handler runs in one of two modes that share this one signature:
|
|
409
|
+
|
|
410
|
+
- **interactive** — return the decision (or a promise of it); the hook `await`s it in-process. The whole agent run stays on the stack — no store needed. This skill.
|
|
411
|
+
- **durable** — persist the request and `throw` to suspend, resuming from another process later. See [`durable-resume/SKILL.md`](@warlock.js/ai/durable-resume/SKILL.md).
|
|
412
|
+
|
|
413
|
+
## It never throws out of the pipeline
|
|
414
|
+
|
|
415
|
+
The middleware is a harness, not a detector — every outcome (skip, approve, reject, edit) returns normally. A `reject` does **not** throw out of `execute()`: it short-circuits a failed `ToolInvokeResult` carrying an `ApprovalRejectedError`, so the error rides `result.error` like every other `AIError` and `agent.execute()` still never throws.
|
|
416
|
+
|
|
417
|
+
```ts
|
|
418
|
+
const result = await support.execute("Refund order #4821");
|
|
419
|
+
|
|
420
|
+
if (result.error instanceof ApprovalRejectedError) {
|
|
421
|
+
logAudit(`${result.error.toolName} rejected: ${result.error.reason}`);
|
|
422
|
+
}
|
|
423
|
+
```
|
|
424
|
+
|
|
425
|
+
Only a *handler bug* — a non-sentinel throw from your handler — propagates, and even then the agent dispatch funnels it onto `result.error` rather than crashing the run. The gate never swallows a bug into a silent approval.
|
|
426
|
+
|
|
427
|
+
## Edge cases
|
|
428
|
+
|
|
429
|
+
- **Duplicate middleware name.** Middleware names are validated unique per agent. The default name is `"human-approval"`, so two approval middlewares on one agent need distinct `name`s.
|
|
430
|
+
- **Silent tools.** A `silent`-mode tool's result isn't fed back to the model, but approval still runs (we gate *before* dispatch). A `reject` on a silent tool writes a tool message that's harmless but unread.
|
|
431
|
+
- **Abort during an interactive await.** `ctx.signal` is in scope; honor it in a long-running handler so a cancelled run rejects rather than hanging.
|
|
432
|
+
|
|
433
|
+
## See also
|
|
434
|
+
|
|
435
|
+
- [`@warlock.js/ai/durable-resume/SKILL.md`](@warlock.js/ai/durable-resume/SKILL.md) — persist the request, resume out-of-process hours later via `ai.human.resume(...)` and the `InterruptStore`.
|
|
436
|
+
- `@warlock.js/ai` — the `ai.agent(...)`, `AgentMiddleware`, `tool.before`, and `ToolInvokeResult` primitives this gate wraps.
|
|
437
|
+
|
|
438
|
+
|
|
301
439
|
## attach-ai-middleware `@warlock.js/ai/attach-ai-middleware/SKILL.md`
|
|
302
440
|
|
|
303
441
|
---
|
|
@@ -307,7 +445,7 @@ description: 'Wire agent middleware — ai.middleware.budget (token / USD caps +
|
|
|
307
445
|
|
|
308
446
|
# Middleware — agent-level pipeline
|
|
309
447
|
|
|
310
|
-
Cross-cutting concerns wrapped around an agent run at three granularities
|
|
448
|
+
Cross-cutting concerns wrapped around an agent run at three agent-level granularities (`execute`, `trip`, `tool`), plus a `supervisor` level that wraps a whole supervisor run (see below). One middleware = one object. Ships with `budget`, `guardrail`, and `semanticCache` built-ins.
|
|
311
449
|
|
|
312
450
|
## Install order at a glance
|
|
313
451
|
|
|
@@ -705,6 +843,246 @@ Workflow errors surface as `ToolExecutionError` with `cause` pointing at the ori
|
|
|
705
843
|
- [`@warlock.js/ai/run-ai-workflow/SKILL.md`](@warlock.js/ai/run-ai-workflow/SKILL.md) — `workflow.asTool()` composition
|
|
706
844
|
|
|
707
845
|
|
|
846
|
+
## detect-and-redact-pii `@warlock.js/ai/detect-and-redact-pii/SKILL.md`
|
|
847
|
+
|
|
848
|
+
---
|
|
849
|
+
name: detect-and-redact-pii
|
|
850
|
+
description: 'Detect and redact PII (and run model-graded moderation) with @warlock.js/ai-guard detectors — `ai.guardrail.pii(...)` and the optional `ai.guardrail.moderation(...)` peer. Triggers: `ai.guardrail.pii`, `piiDetector`, `PiiDetectorOptions`, `PiiCategory`, `mask`, `{label}`, `dictionary`, `onMatch`, `ai.guardrail.moderation`, `openAiModeration`, `OpenAiModerationOptions`, `blockOn`, `omni-moderation-latest`; ''redact PII from model output'', ''mask SSN / credit card / email / phone / IP'', ''stop PII leaking into a tool call'', ''scrub sensitive data'', ''add OpenAI moderation'', ''block violent / self-harm content''; typical import `import "@warlock.js/ai-guard"` (registers `ai.guardrail.pii` / `.moderation`) or `import { pii, moderation } from "@warlock.js/ai-guard"`. Skip: composing the guard / wiring it into an agent — `@warlock.js/ai-guard/guard-input-output/SKILL.md`; routing a block to a human — `@warlock.js/ai-guard/escalate-block-to-human/SKILL.md`.'
|
|
851
|
+
---
|
|
852
|
+
|
|
853
|
+
# Detect and redact PII (and moderate)
|
|
854
|
+
|
|
855
|
+
`ai.guardrail.pii(...)` is a **zero-dependency** detector — regex + exact-string matching, no runtime peer. Pass it into any phase array of `ai.guardrail({ ... })`.
|
|
856
|
+
|
|
857
|
+
```ts
|
|
858
|
+
import { ai } from "@warlock.js/ai";
|
|
859
|
+
import "@warlock.js/ai-guard";
|
|
860
|
+
|
|
861
|
+
const policy = ai.guardrail({
|
|
862
|
+
output: [ai.guardrail.pii({ onMatch: "redact", mask: "[REDACTED:{label}]" })],
|
|
863
|
+
});
|
|
864
|
+
```
|
|
865
|
+
|
|
866
|
+
## Categories — `detect`
|
|
867
|
+
|
|
868
|
+
Scans for these `PiiCategory` values; `detect` narrows the set (default: all). Each pattern is **linear** (anchored, no nested quantifiers) — safe against catastrophic backtracking.
|
|
869
|
+
|
|
870
|
+
| Category | Matches |
|
|
871
|
+
|---|---|
|
|
872
|
+
| `ssn` | US Social Security numbers |
|
|
873
|
+
| `email` | email addresses |
|
|
874
|
+
| `phone` | phone numbers |
|
|
875
|
+
| `credit-card` | credit-card numbers |
|
|
876
|
+
| `ipv4` | IPv4 addresses |
|
|
877
|
+
|
|
878
|
+
```ts
|
|
879
|
+
ai.guardrail.pii({ detect: ["ssn", "credit-card"] }); // scan only these two
|
|
880
|
+
```
|
|
881
|
+
|
|
882
|
+
Add `dictionary` for extra exact-string terms (internal codenames, customer IDs) treated as PII alongside the built-in regexes:
|
|
883
|
+
|
|
884
|
+
```ts
|
|
885
|
+
ai.guardrail.pii({ dictionary: ["PROJECT-ORION", "ACME-INTERNAL"] });
|
|
886
|
+
```
|
|
887
|
+
|
|
888
|
+
## Action — `onMatch`
|
|
889
|
+
|
|
890
|
+
`onMatch` is `"redact" | "block" | "flag"`, default **`"redact"`**:
|
|
891
|
+
|
|
892
|
+
- **`redact`** — replace each match with the `mask` and continue (output phase only — see below).
|
|
893
|
+
- **`block`** — reject the trip / tool call with a `GuardrailViolationError`.
|
|
894
|
+
- **`flag`** — allow but record the matches into `ctx.state` for a downstream observer.
|
|
895
|
+
|
|
896
|
+
## The `mask` template
|
|
897
|
+
|
|
898
|
+
On `redact`, each match is replaced by `mask`. The `{label}` token is substituted with the matched category, so a redacted SSN becomes `[REDACTED:ssn]`:
|
|
899
|
+
|
|
900
|
+
```ts
|
|
901
|
+
ai.guardrail.pii({ onMatch: "redact", mask: "[REDACTED:{label}]" });
|
|
902
|
+
// "My SSN is 123-45-6789" -> "My SSN is [REDACTED:ssn]"
|
|
903
|
+
```
|
|
904
|
+
|
|
905
|
+
Omit `mask` to use the default fixed placeholder.
|
|
906
|
+
|
|
907
|
+
## Where redaction actually applies
|
|
908
|
+
|
|
909
|
+
Redaction only rewrites-and-continues where the pipeline seam supports it:
|
|
910
|
+
|
|
911
|
+
- **Output (`output: [...]`)** — works. `trip.after` returns a replacement `ModelResponse` with the scrubbed `content`. This is the primary PII-redaction use case.
|
|
912
|
+
- **Input (`input: [...]`)** — a `redact` verdict **downgrades to `block`**. The core `trip.before` hook can only short-circuit, not rewrite-and-continue, so the un-redacted prompt can't be threaded back.
|
|
913
|
+
- **Tool (`tool: [...]`)** — a `redact` verdict **downgrades to `block`** (`reason: "tool-arg-redaction-unsupported"`), because silently rewriting tool arguments changes the call's side-effects.
|
|
914
|
+
|
|
915
|
+
So: **redact on output, block on input/tool.**
|
|
916
|
+
|
|
917
|
+
```ts
|
|
918
|
+
const policy = ai.guardrail({
|
|
919
|
+
output: [ai.guardrail.pii({ onMatch: "redact", mask: "[REDACTED:{label}]" })], // scrub the answer
|
|
920
|
+
tool: [ai.guardrail.pii({ onMatch: "block" })], // refuse to leak into tools
|
|
921
|
+
toolNames: ["send_email"],
|
|
922
|
+
});
|
|
923
|
+
```
|
|
924
|
+
|
|
925
|
+
## Optional moderation peer — `ai.guardrail.moderation`
|
|
926
|
+
|
|
927
|
+
For model-graded content (violence, self-harm, hate) beyond regex, the optional `moderation` detector calls OpenAI's moderation endpoint. The `openai` SDK is an **optional lazy peer** — importing `@warlock.js/ai-guard` never forces it to resolve; the detector throws a curated install string on first `check()` when the peer is absent (mirrors ai-panoptic's lazy Langfuse exporter).
|
|
928
|
+
|
|
929
|
+
```ts
|
|
930
|
+
const policy = ai.guardrail({
|
|
931
|
+
output: [
|
|
932
|
+
ai.guardrail.moderation({ blockOn: ["violence", "self-harm"] }),
|
|
933
|
+
],
|
|
934
|
+
});
|
|
935
|
+
```
|
|
936
|
+
|
|
937
|
+
- `blockOn` — categories that escalate to `block`; every other flagged category produces a `flag`. Omit to `flag` on any category.
|
|
938
|
+
- `model` — defaults to `"omni-moderation-latest"`.
|
|
939
|
+
- `apiKey` — defaults to `OPENAI_API_KEY`.
|
|
940
|
+
- `client` — pass a pre-built OpenAI-compatible client to bypass the lazy import entirely (the bring-your-own-client / test escape hatch).
|
|
941
|
+
|
|
942
|
+
Install the peer only when you use this detector:
|
|
943
|
+
|
|
944
|
+
```bash
|
|
945
|
+
npm install openai
|
|
946
|
+
```
|
|
947
|
+
|
|
948
|
+
## See also
|
|
949
|
+
|
|
950
|
+
- [`@warlock.js/ai-guard/guard-input-output/SKILL.md`](@warlock.js/ai-guard/guard-input-output/SKILL.md) — composing the guard, the verdict model, phases, and `toolNames` scoping.
|
|
951
|
+
- [`@warlock.js/ai-guard/escalate-block-to-human/SKILL.md`](@warlock.js/ai-guard/escalate-block-to-human/SKILL.md) — escalating a hard `block` to a human-review surface.
|
|
952
|
+
|
|
953
|
+
|
|
954
|
+
## durable-resume `@warlock.js/ai/durable-resume/SKILL.md`
|
|
955
|
+
|
|
956
|
+
---
|
|
957
|
+
name: durable-resume
|
|
958
|
+
description: 'Persist a gated tool call and resume it from another process hours later — ships in @warlock.js/ai core: `ai.human.resume(interruptId, decision, options)`, the `InterruptStore` (`ai.human.interrupt.{memory,pg,redis}()`), `PendingInterrupt`, and the `InterruptSuspendedError` suspend sentinel. Triggers: `ai.human.resume`, `resume(interruptId, decision)`, `InterruptStore`, `ai.human.interrupt.memory`, `ai.human.interrupt.pg`, `ai.human.interrupt.redis`, `interruptMemory`, `interruptPg`, `interruptRedis`, `PendingInterrupt`, `InterruptSuspendedError`, `ResumeOptions`, `ResumeResult`, `PgClientLike`, `RedisClientLike`; ''approve hours later from a webhook'', ''persist the approval request and resume in another process'', ''durable human-in-the-loop'', ''store the interrupt in Postgres/Redis'', ''re-run the agent turn once the human approves''. Typical import `import { ai, InterruptSuspendedError } from "@warlock.js/ai"`. Skip: the in-process await gate and the policy/decision shapes — `@warlock.js/ai/approve-tool-calls/SKILL.md`.'
|
|
959
|
+
---
|
|
960
|
+
|
|
961
|
+
# Durable resume — persist the interrupt, approve from another process
|
|
962
|
+
|
|
963
|
+
Interactive approval `await`s the operator in-process. **Durable** approval is for when the reviewer rules out-of-band — a Slack button, a webhook, hours later, in a different process. The flow: the handler **persists** the request to an `InterruptStore` and **throws** `InterruptSuspendedError` to suspend the run; the caller surfaces the `interruptId`; later, `ai.human.resume(interruptId, decision, { store })` applies the ruling.
|
|
964
|
+
|
|
965
|
+
> **v1 durable resume re-runs the turn** with the decision pre-seeded — it does **not** rehydrate an in-flight supervisor mid-call (that's the deferred v2 lift). Re-running is idempotent because the prompt and the seeded decision fully determine the gated call's outcome.
|
|
966
|
+
|
|
967
|
+
## Process A — suspend and surface the id
|
|
968
|
+
|
|
969
|
+
```ts
|
|
970
|
+
import { ai, InterruptSuspendedError } from "@warlock.js/ai";
|
|
971
|
+
|
|
972
|
+
const store = ai.human.interrupt.memory(); // swap for pg / redis in production
|
|
973
|
+
|
|
974
|
+
const agent = ai.agent({
|
|
975
|
+
model,
|
|
976
|
+
tools: [deleteAccount],
|
|
977
|
+
middleware: [
|
|
978
|
+
ai.human.approval({
|
|
979
|
+
policy: { type: "predicate", requiresApproval: (c) => c.toolName === "deleteAccount" },
|
|
980
|
+
store,
|
|
981
|
+
handler: async (req) => {
|
|
982
|
+
// 1. persist the pending interrupt
|
|
983
|
+
await store.save({
|
|
984
|
+
interruptId: req.interruptId,
|
|
985
|
+
request: req,
|
|
986
|
+
status: "pending",
|
|
987
|
+
savedAt: new Date().toISOString(),
|
|
988
|
+
});
|
|
989
|
+
// 2. notify the reviewer out-of-band
|
|
990
|
+
await slack.postApproval(req);
|
|
991
|
+
// 3. suspend the run — the middleware recognizes its OWN sentinel
|
|
992
|
+
throw new InterruptSuspendedError("Awaiting human approval", {
|
|
993
|
+
interruptId: req.interruptId,
|
|
994
|
+
});
|
|
995
|
+
},
|
|
996
|
+
}),
|
|
997
|
+
],
|
|
998
|
+
});
|
|
999
|
+
|
|
1000
|
+
const result = await agent.execute("Delete account #88");
|
|
1001
|
+
|
|
1002
|
+
// execute() never throws — the suspend rides result.error:
|
|
1003
|
+
if (result.error instanceof InterruptSuspendedError) {
|
|
1004
|
+
return { status: "awaiting-approval", interruptId: result.error.interruptId };
|
|
1005
|
+
}
|
|
1006
|
+
```
|
|
1007
|
+
|
|
1008
|
+
The middleware catches the **sentinel** (`instanceof InterruptSuspendedError`) and short-circuits a failed `ToolInvokeResult` carrying it, so `error.interruptId` is on `result.error`. Hand that id to the reviewer.
|
|
1009
|
+
|
|
1010
|
+
## Process B — resume hours later
|
|
1011
|
+
|
|
1012
|
+
```ts
|
|
1013
|
+
import { ai } from "@warlock.js/ai";
|
|
1014
|
+
|
|
1015
|
+
// Re-run the turn with the decision pre-seeded:
|
|
1016
|
+
const outcome = await ai.human.resume(
|
|
1017
|
+
interruptId,
|
|
1018
|
+
{ type: "edit", args: { confirm: true } },
|
|
1019
|
+
{ store, agent },
|
|
1020
|
+
);
|
|
1021
|
+
|
|
1022
|
+
if (outcome.type === "applied" && outcome.result) {
|
|
1023
|
+
console.log(outcome.result.text); // the re-run completed with the ruling applied
|
|
1024
|
+
}
|
|
1025
|
+
```
|
|
1026
|
+
|
|
1027
|
+
`ai.human.resume(interruptId, decision, options)` loads the `PendingInterrupt`, validates the decision shape, deletes the record, and — when an `agent` is supplied — re-executes the original prompt with the decision **pre-seeded** so the gated tool call resolves to the ruling instead of pausing again. The prompt comes from `request.context.originalInput`; pass `options.input` to override (e.g. to append the reviewer's note), and `options.executeOptions` to forward history / output schema / signal to the re-run.
|
|
1028
|
+
|
|
1029
|
+
### Two resume shapes
|
|
1030
|
+
|
|
1031
|
+
| Shape | Pass | Behavior |
|
|
1032
|
+
|---|---|---|
|
|
1033
|
+
| **re-run** | `{ store, agent }` | Loads, deletes, re-executes the turn; `ResultResult.result` carries the `AgentResult`. |
|
|
1034
|
+
| **apply-only** | `{ store }` (no `agent`) | Loads, validates, deletes; returns `{ type: "applied", decision }` for a caller-owned re-drive (custom transport). No turn re-run. |
|
|
1035
|
+
|
|
1036
|
+
### Idempotent by construction
|
|
1037
|
+
|
|
1038
|
+
```ts
|
|
1039
|
+
type ResumeResult =
|
|
1040
|
+
| { type: "applied"; interruptId: string; decision: ApprovalDecision; result?: AgentResult }
|
|
1041
|
+
| { type: "already-resolved"; interruptId: string };
|
|
1042
|
+
```
|
|
1043
|
+
|
|
1044
|
+
A second resume of an already-resolved (deleted) or never-raised interrupt returns `{ type: "already-resolved" }` — it never double-applies the decision or re-runs the turn. The record is deleted **before** the re-run, so even a re-run that itself raises a fresh interrupt can't collide with the one being resolved. A malformed decision (`reject` with no `reason`, `edit` with no `args`, an unknown `type`) throws a `TypeError` loudly rather than silently mis-driving the re-run.
|
|
1045
|
+
|
|
1046
|
+
## The `InterruptStore`
|
|
1047
|
+
|
|
1048
|
+
`ai.human.interrupt.{memory,pg,redis}()` build the store. The contract mirrors `@warlock.js/ai`'s `CheckpointStore` / `SnapshotStore` — `save` / `load` / `delete` / optional `list(prefix?)` / `schema()` — so a consumer already running an orchestrator can reuse the **same** pool for the interrupt table.
|
|
1049
|
+
|
|
1050
|
+
| Factory | Backing | Deps |
|
|
1051
|
+
|---|---|---|
|
|
1052
|
+
| `ai.human.interrupt.memory()` | process-local `Map` | none — zero runtime deps |
|
|
1053
|
+
| `ai.human.interrupt.pg(options)` | one Postgres row per interrupt, keyed by `interrupt_id` | lazily imports the optional `pg` peer |
|
|
1054
|
+
| `ai.human.interrupt.redis(options)` | one namespaced JSON value + a self-maintained id index | lazily imports the optional `redis` peer |
|
|
1055
|
+
|
|
1056
|
+
```ts
|
|
1057
|
+
// Memory — dev / tests / single-process:
|
|
1058
|
+
const store = ai.human.interrupt.memory();
|
|
1059
|
+
|
|
1060
|
+
// Postgres — pass a live pool (core never imports pg in that case):
|
|
1061
|
+
import { Pool } from "pg";
|
|
1062
|
+
const pool = new Pool({ connectionString: process.env.DATABASE_URL });
|
|
1063
|
+
const store = ai.human.interrupt.pg({ client: pool });
|
|
1064
|
+
// Once, via your migration tool — the framework never auto-migrates:
|
|
1065
|
+
// await pool.query(store.schema());
|
|
1066
|
+
|
|
1067
|
+
// …or let the store build its own pool (lazily import("pg")):
|
|
1068
|
+
const store = ai.human.interrupt.pg({ connectionString: process.env.DATABASE_URL });
|
|
1069
|
+
|
|
1070
|
+
// Redis — pass a connected client, or a url:
|
|
1071
|
+
const store = ai.human.interrupt.redis({ url: process.env.REDIS_URL });
|
|
1072
|
+
```
|
|
1073
|
+
|
|
1074
|
+
### Optional peers are lazy
|
|
1075
|
+
|
|
1076
|
+
`pg` and `redis` are **optional** peer dependencies — neither is a hard dependency. The driver is imported only inside the store that needs it, and only when you pass a `connectionString` / `url` (passing a live `client` imports nothing). If the driver is absent, a **curated install string** surfaces on first use, never a raw module-resolution stack trace at import — so a memory-only consumer always loads cleanly. `PgClientLike` / `RedisClientLike` are structural interfaces, so any compatible pool/client satisfies them.
|
|
1077
|
+
|
|
1078
|
+
`schema()` returns the reference DDL for the Postgres store (run it through your migration tool once) and an empty string for memory / redis, so callers treat `schema()` uniformly across drivers.
|
|
1079
|
+
|
|
1080
|
+
## See also
|
|
1081
|
+
|
|
1082
|
+
- [`@warlock.js/ai/approve-tool-calls/SKILL.md`](@warlock.js/ai/approve-tool-calls/SKILL.md) — the gate itself: the interrupt policy, the approve / reject / edit decision union, and the interactive (in-process await) handler.
|
|
1083
|
+
- `@warlock.js/ai` — the `CheckpointStore` / `SnapshotStore` the `InterruptStore` mirrors, and the `ai.agent(...)` re-run target.
|
|
1084
|
+
|
|
1085
|
+
|
|
708
1086
|
## embed-text `@warlock.js/ai/embed-text/SKILL.md`
|
|
709
1087
|
|
|
710
1088
|
---
|
|
@@ -739,79 +1117,410 @@ if (typeof sdk.embedder === "function") {
|
|
|
739
1117
|
}
|
|
740
1118
|
```
|
|
741
1119
|
|
|
742
|
-
## OpenAI adapter — first implementation
|
|
1120
|
+
## OpenAI adapter — first implementation
|
|
1121
|
+
|
|
1122
|
+
```ts
|
|
1123
|
+
import { OpenAISDK } from "@warlock.js/ai-openai";
|
|
1124
|
+
|
|
1125
|
+
const openai = new OpenAISDK({ apiKey: process.env.OPENAI_API_KEY! });
|
|
1126
|
+
const embedder = openai.embedder({ name: "text-embedding-3-small" });
|
|
1127
|
+
|
|
1128
|
+
const one = await embedder.embed("Hello, world.");
|
|
1129
|
+
// { vector: number[], dimensions: number, usage: { promptTokens, totalTokens } }
|
|
1130
|
+
|
|
1131
|
+
const many = await embedder.embedMany(["foo", "bar", "baz"]);
|
|
1132
|
+
// { vectors: number[][], dimensions: number, usage: { promptTokens, totalTokens } }
|
|
1133
|
+
```
|
|
1134
|
+
|
|
1135
|
+
## Not wired into the agent loop
|
|
1136
|
+
|
|
1137
|
+
Embeddings are deliberately not automatic. Consumers obtain an embedder from the adapter and call it directly. Composes into:
|
|
1138
|
+
|
|
1139
|
+
- **Retrieval tools** the agent can call (RAG pattern).
|
|
1140
|
+
- **`run` steps** in a workflow (vector ingest, catalog item embedding).
|
|
1141
|
+
- **Query vectors** for `ai.middleware.semanticCache` — see [`@warlock.js/ai/attach-ai-middleware/SKILL.md`](@warlock.js/ai/attach-ai-middleware/SKILL.md).
|
|
1142
|
+
- **Cascade vector columns** for native pgvector search — see [`@warlock.js/cascade/search-by-vector/SKILL.md`](@warlock.js/cascade/search-by-vector/SKILL.md).
|
|
1143
|
+
- **Cache similarity retrieval** via `cache.set({ vector })` + `cache.similar(...)` — see [`@warlock.js/cache/use-cache-similarity/SKILL.md`](@warlock.js/cache/use-cache-similarity/SKILL.md).
|
|
1144
|
+
|
|
1145
|
+
## Usage example — workflow `run` step
|
|
1146
|
+
|
|
1147
|
+
```ts
|
|
1148
|
+
ai.step({
|
|
1149
|
+
name: "embed",
|
|
1150
|
+
run: async (ctx) => {
|
|
1151
|
+
const text = `${ctx.steps.extract.output.name} ${ctx.steps.extract.output.description}`;
|
|
1152
|
+
const { vector } = await embedder.embed(text);
|
|
1153
|
+
ctx.state.embedding = vector;
|
|
1154
|
+
},
|
|
1155
|
+
output: { extract: (ctx) => ({ dims: (ctx.state.embedding as number[]).length }) },
|
|
1156
|
+
});
|
|
1157
|
+
```
|
|
1158
|
+
|
|
1159
|
+
## Pattern — RAG tool
|
|
1160
|
+
|
|
1161
|
+
```ts
|
|
1162
|
+
import { v } from "@warlock.js/seal";
|
|
1163
|
+
|
|
1164
|
+
const searchKb = ai.tool({
|
|
1165
|
+
name: "searchKb",
|
|
1166
|
+
description: "Search the knowledge base for relevant passages.",
|
|
1167
|
+
input: v.object({ query: v.string(), k: v.number().optional() }),
|
|
1168
|
+
execute: async ({ query, k }) => {
|
|
1169
|
+
const { vector } = await embedder.embed(query);
|
|
1170
|
+
const hits = await vectorStore.query(vector, { topK: k ?? 5 });
|
|
1171
|
+
return hits.map((h) => ({ text: h.text, score: h.score, source: h.source }));
|
|
1172
|
+
},
|
|
1173
|
+
});
|
|
1174
|
+
|
|
1175
|
+
ai.agent({ model, tools: [searchKb] });
|
|
1176
|
+
```
|
|
1177
|
+
|
|
1178
|
+
## Dimensions
|
|
1179
|
+
|
|
1180
|
+
`embedder.dimensions` is `0` on a fresh embedder when no override is given — populated from the first embed call's response. Pre-seed via the adapter's `dimensions` config option when you need the value before the first call (e.g. to size a vector column in a migration schema).
|
|
1181
|
+
|
|
1182
|
+
## Retrieval is app-level
|
|
1183
|
+
|
|
1184
|
+
No built-in vector store. Bring your own (pgvector / Qdrant / Pinecone / Chroma / cache's `similar()`) and wrap it in an `ai.tool({...})`.
|
|
1185
|
+
|
|
1186
|
+
## See also
|
|
1187
|
+
|
|
1188
|
+
- [`@warlock.js/ai/run-ai-agent/SKILL.md`](@warlock.js/ai/run-ai-agent/SKILL.md) — composing embedders into tools
|
|
1189
|
+
- [`@warlock.js/ai/run-ai-workflow/SKILL.md`](@warlock.js/ai/run-ai-workflow/SKILL.md) — embeddings inside `run` steps
|
|
1190
|
+
- [`@warlock.js/ai/persist-ai-data/SKILL.md`](@warlock.js/ai/persist-ai-data/SKILL.md) — performance guidance on vector storage
|
|
1191
|
+
- [`@warlock.js/cache/use-cache-similarity/SKILL.md`](@warlock.js/cache/use-cache-similarity/SKILL.md) — cache as a vector store
|
|
1192
|
+
- [`@warlock.js/cascade/search-by-vector/SKILL.md`](@warlock.js/cascade/search-by-vector/SKILL.md) — cascade `similarTo` query method
|
|
1193
|
+
|
|
1194
|
+
|
|
1195
|
+
## escalate-block-to-human `@warlock.js/ai/escalate-block-to-human/SKILL.md`
|
|
1196
|
+
|
|
1197
|
+
---
|
|
1198
|
+
name: escalate-block-to-human
|
|
1199
|
+
description: 'Route a hard guardrail block to a human-review surface with @warlock.js/ai-guard — the `escalation.onBlock` seam and an `escalate: true` verdict. Triggers: `escalation`, `onBlock`, `GuardrailEscalation`, `GuardrailBlockEvent`, `escalate: true`, `{ type: "block", escalate: true }`, ''escalate a block to a human'', ''human review queue for guardrail'', ''page an operator on a guardrail block'', ''human-in-the-loop guardrail'', ''compose a block with a review surface'', ''custom detector that escalates''; typical import `import "@warlock.js/ai-guard"` then `ai.guardrail({ escalation: { onBlock } })`. Skip: composing the guard / phases / verdict model — `@warlock.js/ai-guard/guard-input-output/SKILL.md`; PII/moderation detectors — `@warlock.js/ai-guard/detect-and-redact-pii/SKILL.md`; durable suspend/resume human-step machinery (deferred) — not in this package.'
|
|
1200
|
+
---
|
|
1201
|
+
|
|
1202
|
+
# Escalate a block to a human
|
|
1203
|
+
|
|
1204
|
+
A `block` verdict can carry `escalate: true`. When it does, the guard `await`s your `escalation.onBlock(...)` handler **before** throwing the `GuardrailViolationError` — the seam to a human-review queue, an operator page, or any out-of-band approval surface.
|
|
1205
|
+
|
|
1206
|
+
```ts
|
|
1207
|
+
import { ai } from "@warlock.js/ai";
|
|
1208
|
+
import "@warlock.js/ai-guard";
|
|
1209
|
+
|
|
1210
|
+
const policy = ai.guardrail({
|
|
1211
|
+
output: [ai.guardrail.moderation({ blockOn: ["self-harm"] })],
|
|
1212
|
+
escalation: {
|
|
1213
|
+
async onBlock(event) {
|
|
1214
|
+
await reviewQueue.enqueue({
|
|
1215
|
+
phase: event.phase, // "input" | "output" | "tool"
|
|
1216
|
+
reason: event.reason, // the detector's human-readable reason
|
|
1217
|
+
});
|
|
1218
|
+
},
|
|
1219
|
+
},
|
|
1220
|
+
});
|
|
1221
|
+
|
|
1222
|
+
const agent = ai.agent({ model, middleware: [policy] });
|
|
1223
|
+
```
|
|
1224
|
+
|
|
1225
|
+
## When `onBlock` fires
|
|
1226
|
+
|
|
1227
|
+
`onBlock` fires **only** for a verdict of `{ type: "block", escalate: true }` — not for an ordinary `block`, and never for `allow` / `redact` / `flag`. It is **awaited before** the `GuardrailViolationError` is thrown, so your handler runs to completion (enqueue succeeds, the page is sent) before the error surfaces on `result.error`. The run still aborts: escalation is a *signal*, not a recovery — `execute()` returns with `result.error` populated as usual.
|
|
1228
|
+
|
|
1229
|
+
## The `GuardrailBlockEvent` payload
|
|
1230
|
+
|
|
1231
|
+
`onBlock(event)` receives:
|
|
1232
|
+
|
|
1233
|
+
| Field | Type | Meaning |
|
|
1234
|
+
|---|---|---|
|
|
1235
|
+
| `phase` | `"input" \| "output" \| "tool"` | where the block fired |
|
|
1236
|
+
| `reason` | `string` | the detector's human-readable reason |
|
|
1237
|
+
| `matches` | `readonly GuardrailMatch[] \| undefined` | what tripped the rule (rule id, span, label), when reported |
|
|
1238
|
+
| `ctx` | `MiddlewareTripContext` | the live trip context — `state`, `messages`, `agent`, `model`, `signal` |
|
|
1239
|
+
|
|
1240
|
+
`ctx` lets the handler enrich the review item with run context (session id from `ctx.state`, the offending messages, etc.).
|
|
1241
|
+
|
|
1242
|
+
## Producing an escalating verdict
|
|
1243
|
+
|
|
1244
|
+
The built-in detectors return ordinary `block` verdicts (no `escalate`). To escalate, author a tiny custom `GuardrailDetector` that sets `escalate: true` on its `block`:
|
|
1245
|
+
|
|
1246
|
+
```ts
|
|
1247
|
+
import type { GuardrailDetector } from "@warlock.js/ai-guard";
|
|
1248
|
+
|
|
1249
|
+
const wirePolicy: GuardrailDetector = {
|
|
1250
|
+
name: "wire-transfer",
|
|
1251
|
+
check(text) {
|
|
1252
|
+
if (/wire \$?\d{5,}/i.test(text)) {
|
|
1253
|
+
return {
|
|
1254
|
+
type: "block",
|
|
1255
|
+
reason: "large wire transfer requires human approval",
|
|
1256
|
+
escalate: true, // <- routes through escalation.onBlock
|
|
1257
|
+
matches: [{ rule: "wire-transfer.large", label: "wire" }],
|
|
1258
|
+
};
|
|
1259
|
+
}
|
|
1260
|
+
|
|
1261
|
+
return { type: "allow" };
|
|
1262
|
+
},
|
|
1263
|
+
};
|
|
1264
|
+
|
|
1265
|
+
const policy = ai.guardrail({
|
|
1266
|
+
tool: [wirePolicy],
|
|
1267
|
+
toolNames: ["initiate_transfer"],
|
|
1268
|
+
escalation: { async onBlock(e) { await approvals.request(e); } },
|
|
1269
|
+
});
|
|
1270
|
+
```
|
|
1271
|
+
|
|
1272
|
+
A `check()` may be sync or async (async = call an external service); the guard awaits either.
|
|
1273
|
+
|
|
1274
|
+
## A plain callback by design
|
|
1275
|
+
|
|
1276
|
+
`escalation.onBlock` is a **plain callback** — `ai-guard` takes **no** dependency on the deferred durable human-step machinery (suspend/resume). The callback is the decoupling seam: inside it you wire your own review queue, and (where your stack supports it) a `workflow.resume(...)` loop. This package only emits the *signal*; it does not own durable suspension. When the typed human-step handoff ships, `onBlock` upgrades to it without a breaking change here.
|
|
1277
|
+
|
|
1278
|
+
## See also
|
|
1279
|
+
|
|
1280
|
+
- [`@warlock.js/ai-guard/guard-input-output/SKILL.md`](@warlock.js/ai-guard/guard-input-output/SKILL.md) — composing the guard, the phases, the verdict model, and how a `block` surfaces on `result.error`.
|
|
1281
|
+
- [`@warlock.js/ai-guard/detect-and-redact-pii/SKILL.md`](@warlock.js/ai-guard/detect-and-redact-pii/SKILL.md) — the `pii` detector and the optional `moderation` peer that commonly drives an escalation.
|
|
1282
|
+
|
|
1283
|
+
|
|
1284
|
+
## eval-datasets-and-ci `@warlock.js/ai/eval-datasets-and-ci/SKILL.md`
|
|
1285
|
+
|
|
1286
|
+
---
|
|
1287
|
+
name: eval-datasets-and-ci
|
|
1288
|
+
description: 'Datasets + regression-gated eval CI with ai.dataset({...}) feeding agent.eval({cases,baseline,tolerance}). Covers the immutable filterable/shardable dataset (cases / fromFile JSONL), DatasetEntry tags, EvalReport.regression (regressed/added/removed/passed) against a baseline, and the ai.eval reporters toJUnit / toJSON / fromJSON for CI artifacts + committed baselines. Triggers: `ai.dataset`, `DatasetContract`, `DatasetEntry`, `DatasetOptions`, `dataset.filter`, `dataset.shard`, `fromFile`, `agent.eval`, `EvalOptions`, `EvalReport`, `EvalCaseResult`, `EvalRegression`, `baseline`, `tolerance`, `ai.eval.toJUnit`, `ai.eval.toJSON`, `ai.eval.fromJSON`, `diff`, JSONL; ''eval dataset from a JSONL file'', ''shard an eval suite across CI jobs'', ''fail CI on an eval regression'', ''emit a JUnit report'', ''snapshot an eval baseline''; typical import `import { ai } from "@warlock.js/ai"`. Skip: the scorers + LLM-as-judge + Vitest matchers themselves — `@warlock.js/ai/ai-dx-helpers/SKILL.md` (registerAiMatchers / ai.eval.exact|contains|predicate|judge); record/replay of model calls for deterministic tests — `@warlock.js/ai/record-replay-llm/SKILL.md`; competing libs `promptfoo`, `braintrust`.'
|
|
1289
|
+
---
|
|
1290
|
+
|
|
1291
|
+
# `ai.dataset()` + `agent.eval()` regression CI
|
|
1292
|
+
|
|
1293
|
+
Turn a corpus of cases into a regression-gated CI signal. `ai.dataset(...)` wraps cases into an immutable, filterable, shardable collection; `agent.eval({ cases, baseline, tolerance })` runs them, scores them, and diffs against a prior report; the `ai.eval.*` reporters serialize the result for CI ingestion and tomorrow's baseline.
|
|
1294
|
+
|
|
1295
|
+
> This skill is the **dataset + CI** layer. The scorers, LLM-as-judge config, and Vitest matchers live in [`@warlock.js/ai/ai-dx-helpers/SKILL.md`](@warlock.js/ai/ai-dx-helpers/SKILL.md); `agent.eval`'s core scoring loop is in [`@warlock.js/ai/run-ai-agent/SKILL.md`](@warlock.js/ai/run-ai-agent/SKILL.md).
|
|
1296
|
+
|
|
1297
|
+
## `ai.dataset()` — immutable, filterable, shardable
|
|
1298
|
+
|
|
1299
|
+
```ts
|
|
1300
|
+
import { ai } from "@warlock.js/ai";
|
|
1301
|
+
|
|
1302
|
+
const ds = ai.dataset({
|
|
1303
|
+
name: "support",
|
|
1304
|
+
cases: [{ name: "greeting", input: "hi", expected: "Hello" }], // inline entries
|
|
1305
|
+
fromFile: "./eval/support.jsonl", // JSONL read ONCE, synchronously, at construction
|
|
1306
|
+
});
|
|
1307
|
+
|
|
1308
|
+
ds.name; // "support"
|
|
1309
|
+
ds.cases; // DatasetEntry[] (inline first, then file entries appended)
|
|
1310
|
+
```
|
|
1311
|
+
|
|
1312
|
+
- A `DatasetEntry` is an `EvalCase` plus optional `tags?: string[]` for filtering / sharding.
|
|
1313
|
+
- **`fromFile`** reads a JSONL file (one JSON object per line; blank lines skipped) synchronously at construction — mirroring `SystemPrompt.fromFile`. A malformed line throws an **`InvalidRequestError` naming the 1-based line number**; a missing/unreadable path throws too. `cases` and `fromFile` combine (file entries append after inline).
|
|
1314
|
+
|
|
1315
|
+
### `filter` / `shard` — derive new datasets
|
|
1316
|
+
|
|
1317
|
+
```ts
|
|
1318
|
+
const smoke = ds.filter((entry) => entry.tags?.includes("smoke"));
|
|
1319
|
+
const shard = ds.shard(0, 4); // first of four parallel CI shards
|
|
1320
|
+
```
|
|
1321
|
+
|
|
1322
|
+
Both return a **fresh dataset sharing nothing mutable**. `shard(index, total)` is deterministic round-robin by position: every entry lands in exactly one shard, so the union of all `total` shards reproduces the full list with no gaps or overlaps. `shard` validates its args (positive integer `total`, `index` in `[0, total)`) and throws `InvalidRequestError` otherwise.
|
|
1323
|
+
|
|
1324
|
+
## `agent.eval({ cases })` — accepts a dataset directly
|
|
1325
|
+
|
|
1326
|
+
```ts
|
|
1327
|
+
const report = await myAgent.eval({
|
|
1328
|
+
cases: ds, // a DatasetContract OR a raw EvalCase[]
|
|
1329
|
+
scorers: [ai.eval.contains()],
|
|
1330
|
+
});
|
|
1331
|
+
|
|
1332
|
+
expect(report.passed).toBe(true); // true only when EVERY case passed
|
|
1333
|
+
```
|
|
1334
|
+
|
|
1335
|
+
The runner reads `.cases` off a dataset. Full `EvalReport`: `{ agentName, total, passedCount, failedCount, passRate, meanScore, passed, cases, duration, regression? }`. Each `EvalCaseResult` carries the case, the full `AgentResult`, every scorer's `scores`, the mean `score`, `passed`, and `duration`.
|
|
1336
|
+
|
|
1337
|
+
## Regression gating — `baseline` + `tolerance`
|
|
1338
|
+
|
|
1339
|
+
```ts
|
|
1340
|
+
import { readFile, writeFile } from "node:fs/promises";
|
|
1341
|
+
|
|
1342
|
+
const baseline = ai.eval.fromJSON(await readFile("./eval/baseline.json", "utf8"));
|
|
1343
|
+
|
|
1344
|
+
const report = await myAgent.eval({
|
|
1345
|
+
cases: ds,
|
|
1346
|
+
scorers: [ai.eval.exact()],
|
|
1347
|
+
baseline, // a prior EvalReport to diff against
|
|
1348
|
+
tolerance: 0.05, // max allowed per-case score DROP before it regresses. default 0 (any drop)
|
|
1349
|
+
});
|
|
1350
|
+
|
|
1351
|
+
if (report.regression && !report.regression.passed) {
|
|
1352
|
+
console.error("Regressed:", report.regression.regressed); // [{ name, before, after }]
|
|
1353
|
+
process.exit(1);
|
|
1354
|
+
}
|
|
1355
|
+
```
|
|
1356
|
+
|
|
1357
|
+
When `baseline` is set the report carries a `regression` block (`EvalRegression`), joining cases by `name`:
|
|
1358
|
+
|
|
1359
|
+
- **`regressed`** — `[{ name, before, after }]` for cases whose new score fell more than `tolerance` below baseline.
|
|
1360
|
+
- **`added`** / **`removed`** — case names present in only one report. Adding or dropping a case **never fails the gate by itself**.
|
|
1361
|
+
- **`passed`** — `true` when `regressed` is empty.
|
|
1362
|
+
|
|
1363
|
+
The pure `diff(report, baseline, tolerance)` function (exported as `diff`) is the same logic, decoupled from the runner — depends only on the two reports and the tolerance, mutates neither.
|
|
1364
|
+
|
|
1365
|
+
## CI reporters — `ai.eval.toJUnit` / `toJSON` / `fromJSON`
|
|
1366
|
+
|
|
1367
|
+
Pure functions over a finished `EvalReport`:
|
|
1368
|
+
|
|
1369
|
+
```ts
|
|
1370
|
+
// JUnit-XML artifact for CI ingestion — one <testsuite> (the agent), one <testcase> per case,
|
|
1371
|
+
// a <failure> on each non-passing case (joined scorer reasons), times in SECONDS.
|
|
1372
|
+
await writeFile("./report.junit.xml", ai.eval.toJUnit(report));
|
|
1373
|
+
|
|
1374
|
+
// Round-trippable snapshot — today's report becomes tomorrow's baseline.
|
|
1375
|
+
await writeFile("./eval/baseline.json", ai.eval.toJSON(report));
|
|
1376
|
+
const restored = ai.eval.fromJSON(await readFile("./eval/baseline.json", "utf8"));
|
|
1377
|
+
```
|
|
1378
|
+
|
|
1379
|
+
`toJSON`/`fromJSON` preserve `result` payloads, per-case `scores`, timings, and any attached `regression` block, so a parsed report drives regression diffing exactly as the in-memory one. `toJUnit` hand-emits XML (no `xml` dependency) and entity-escapes every dynamic value.
|
|
1380
|
+
|
|
1381
|
+
## Typical CI shard job
|
|
1382
|
+
|
|
1383
|
+
```ts
|
|
1384
|
+
const shard = ai.dataset({ name: "support", fromFile: "./eval/support.jsonl" })
|
|
1385
|
+
.shard(Number(process.env.SHARD_INDEX), Number(process.env.SHARD_TOTAL));
|
|
1386
|
+
|
|
1387
|
+
const report = await agent.eval({
|
|
1388
|
+
cases: shard,
|
|
1389
|
+
scorers: [ai.eval.contains()],
|
|
1390
|
+
baseline: ai.eval.fromJSON(await readFile("./eval/baseline.json", "utf8")),
|
|
1391
|
+
tolerance: 0.05,
|
|
1392
|
+
});
|
|
1393
|
+
|
|
1394
|
+
await writeFile(`./out/report-${process.env.SHARD_INDEX}.junit.xml`, ai.eval.toJUnit(report));
|
|
1395
|
+
if (report.regression && !report.regression.passed) process.exit(1);
|
|
1396
|
+
```
|
|
1397
|
+
|
|
1398
|
+
## See also
|
|
1399
|
+
|
|
1400
|
+
- [`@warlock.js/ai/ai-dx-helpers/SKILL.md`](@warlock.js/ai/ai-dx-helpers/SKILL.md) — `ai.eval.{exact,contains,predicate,judge}` scorers + Vitest matchers
|
|
1401
|
+
- [`@warlock.js/ai/run-ai-agent/SKILL.md`](@warlock.js/ai/run-ai-agent/SKILL.md) — `agent.eval` scoring loop, `EvalCase` / `EvalJudge`
|
|
1402
|
+
- [`@warlock.js/ai/record-replay-llm/SKILL.md`](@warlock.js/ai/record-replay-llm/SKILL.md) — `ai.vcr` for deterministic, offline eval runs
|
|
1403
|
+
|
|
1404
|
+
|
|
1405
|
+
## guard-input-output `@warlock.js/ai/guard-input-output/SKILL.md`
|
|
1406
|
+
|
|
1407
|
+
---
|
|
1408
|
+
name: guard-input-output
|
|
1409
|
+
description: 'Build the composed guardrail middleware with @warlock.js/ai-guard and wire it into an agent — `ai.guardrail({ input, output, tool, toolNames, escalation })`. Triggers: `ai.guardrail`, `guard`, `GuardOptions`, `GuardrailVerdict`, `GuardrailDetector`, `GuardrailPhase`, `GuardrailMatch`, `GuardrailViolationError`, `ai.guardrail.topic`, `ai.guardrail.injection`, `topicFilter`, `injectionDetector`, `toolNames`, `forTool`; ''add a guardrail to my agent'', ''block prompt injection'', ''filter banned topics'', ''guard agent input and output'', ''stop the model leaking data into a tool call'', ''scope a detector to one tool''; typical import `import "@warlock.js/ai-guard"` (registers `ai.guardrail`) or `import { guard } from "@warlock.js/ai-guard"`. Skip: PII detection/redaction specifically — `@warlock.js/ai-guard/detect-and-redact-pii/SKILL.md`; routing a block to a human — `@warlock.js/ai-guard/escalate-block-to-human/SKILL.md`; the core middleware pipeline / hook contract — `@warlock.js/ai/run-ai-agent/SKILL.md`.'
|
|
1410
|
+
---
|
|
1411
|
+
|
|
1412
|
+
# Guard agent input, output, and tool args
|
|
1413
|
+
|
|
1414
|
+
`ai.guardrail(...)` is a **middleware factory**. It produces one `AgentMiddleware` that runs your detectors at three hook points and maps each verdict onto the agent pipeline's existing throw / return / record mechanics. Importing the package registers the verb (and its attached detector factories) on the shared `ai` namespace:
|
|
1415
|
+
|
|
1416
|
+
```ts
|
|
1417
|
+
import { ai } from "@warlock.js/ai";
|
|
1418
|
+
import "@warlock.js/ai-guard"; // registers ai.guardrail + ai.guardrail.pii/.topic/.injection/.moderation
|
|
1419
|
+
|
|
1420
|
+
const policy = ai.guardrail({
|
|
1421
|
+
name: "compliance",
|
|
1422
|
+
input: [ai.guardrail.injection({ onMatch: "block" })],
|
|
1423
|
+
output: [ai.guardrail.topic({ deny: [/medical advice/i, "diagnosis"], onMatch: "block" })],
|
|
1424
|
+
});
|
|
1425
|
+
|
|
1426
|
+
const agent = ai.agent({ model, middleware: [policy] });
|
|
1427
|
+
```
|
|
1428
|
+
|
|
1429
|
+
A named-export form is available for callers who prefer not to rely on the augmented namespace:
|
|
1430
|
+
|
|
1431
|
+
```ts
|
|
1432
|
+
import { guard, topic, injection } from "@warlock.js/ai-guard";
|
|
1433
|
+
const policy = guard({ input: [injection({ onMatch: "block" })] });
|
|
1434
|
+
```
|
|
1435
|
+
|
|
1436
|
+
## The three phases
|
|
1437
|
+
|
|
1438
|
+
| Phase | Hook | Inspected text | Set with |
|
|
1439
|
+
|---|---|---|---|
|
|
1440
|
+
| **input** | `trip.before` | the outbound prompt (`extractUserText(ctx.messages)`) | `input: [...]` |
|
|
1441
|
+
| **output** | `trip.after` | `response.content` | `output: [...]` |
|
|
1442
|
+
| **tool** | `tool.before` | `JSON.stringify(toolArgs)` | `tool: [...]` |
|
|
1443
|
+
|
|
1444
|
+
Each phase array runs its detectors in **registration order**; the first non-`allow` verdict decides the action for that phase (short-circuit). A phase you don't configure is inert — a guard with no detectors is a no-op middleware.
|
|
1445
|
+
|
|
1446
|
+
## The verdict model
|
|
743
1447
|
|
|
744
|
-
|
|
745
|
-
import { OpenAISDK } from "@warlock.js/ai-openai";
|
|
1448
|
+
A detector inspects text and returns a `GuardrailVerdict`, discriminated by `type` (never `kind`):
|
|
746
1449
|
|
|
747
|
-
|
|
748
|
-
|
|
1450
|
+
| `type` | Effect |
|
|
1451
|
+
|---|---|
|
|
1452
|
+
| `allow` | Pass to the next detector. |
|
|
1453
|
+
| `redact` | Rewrite the inspected text and continue — **output phase only** (see limitation below). |
|
|
1454
|
+
| `block` | Short-circuit with the existing `GuardrailViolationError`. |
|
|
1455
|
+
| `flag` | Pass, but append a `FlagRecord` into `ctx.state` under `<name>.flags` for a downstream observer (panoptic, the caller). |
|
|
749
1456
|
|
|
750
|
-
|
|
751
|
-
// { vector: number[], dimensions: number, usage: { promptTokens, totalTokens } }
|
|
1457
|
+
`agent.execute()` **never throws** — a `block` surfaces on `result.error` as a `GuardrailViolationError`, exactly like every other `AIError`. Branch on it after the run:
|
|
752
1458
|
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
```
|
|
1459
|
+
```ts
|
|
1460
|
+
const result = await agent.execute(userInput);
|
|
756
1461
|
|
|
757
|
-
|
|
1462
|
+
if (result.error instanceof ai.errors.GuardrailViolationError) {
|
|
1463
|
+
// result.error.phase is "input" | "output" | "tool"
|
|
1464
|
+
// result.error.reason / result.error.guardrail carry the detail
|
|
1465
|
+
}
|
|
1466
|
+
```
|
|
758
1467
|
|
|
759
|
-
|
|
1468
|
+
## Built-in detectors
|
|
760
1469
|
|
|
761
|
-
-
|
|
762
|
-
- **`run` steps** in a workflow (vector ingest, catalog item embedding).
|
|
763
|
-
- **Query vectors** for `ai.middleware.semanticCache` — see [`@warlock.js/ai/attach-ai-middleware/SKILL.md`](@warlock.js/ai/attach-ai-middleware/SKILL.md).
|
|
764
|
-
- **Cascade vector columns** for native pgvector search — see [`@warlock.js/cascade/search-by-vector/SKILL.md`](@warlock.js/cascade/search-by-vector/SKILL.md).
|
|
765
|
-
- **Cache similarity retrieval** via `cache.set({ vector })` + `cache.similar(...)` — see [`@warlock.js/cache/use-cache-similarity/SKILL.md`](@warlock.js/cache/use-cache-similarity/SKILL.md).
|
|
1470
|
+
Three zero-dependency detectors ship (a fourth, `moderation`, is an optional `openai` peer — see [`detect-and-redact-pii/SKILL.md`](@warlock.js/ai-guard/detect-and-redact-pii/SKILL.md)):
|
|
766
1471
|
|
|
767
|
-
|
|
1472
|
+
- **`ai.guardrail.injection(options?)`** — jailbreak / prompt-injection marker phrases. Extra `markers` (string | RegExp); `onMatch` defaults to `"flag"`, callers commonly use `"block"` on input.
|
|
1473
|
+
- **`ai.guardrail.topic(options)`** — `deny` (string substring | RegExp) and/or `allow` (allow-list miss triggers `onMatch`). `onMatch` is `"block" | "flag"`, default `"block"`.
|
|
1474
|
+
- **`ai.guardrail.pii(options?)`** — PII regex + dictionary (its own skill).
|
|
768
1475
|
|
|
769
1476
|
```ts
|
|
770
|
-
ai.
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
ctx.state.embedding = vector;
|
|
776
|
-
},
|
|
777
|
-
output: { extract: (ctx) => ({ dims: (ctx.state.embedding as number[]).length }) },
|
|
1477
|
+
const policy = ai.guardrail({
|
|
1478
|
+
input: [
|
|
1479
|
+
ai.guardrail.injection({ onMatch: "block", markers: ["ignore previous instructions"] }),
|
|
1480
|
+
ai.guardrail.topic({ deny: ["competitor-name"], onMatch: "block" }),
|
|
1481
|
+
],
|
|
778
1482
|
});
|
|
779
1483
|
```
|
|
780
1484
|
|
|
781
|
-
##
|
|
1485
|
+
## Scope tool detectors to specific tools
|
|
782
1486
|
|
|
783
|
-
|
|
784
|
-
import { v } from "@warlock.js/seal";
|
|
1487
|
+
`tool` detectors fire on **every** tool call by default. Set `toolNames` to scope them — the whole middleware is wrapped with the core `forTool(toolNames, mw)` helper so the `tool` hooks fire only for those names; `input` / `output` (`trip`) hooks are unaffected:
|
|
785
1488
|
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
execute: async ({ query, k }) => {
|
|
791
|
-
const { vector } = await embedder.embed(query);
|
|
792
|
-
const hits = await vectorStore.query(vector, { topK: k ?? 5 });
|
|
793
|
-
return hits.map((h) => ({ text: h.text, score: h.score, source: h.source }));
|
|
794
|
-
},
|
|
1489
|
+
```ts
|
|
1490
|
+
const policy = ai.guardrail({
|
|
1491
|
+
tool: [ai.guardrail.pii({ onMatch: "block" })], // stop PII reaching the tool
|
|
1492
|
+
toolNames: ["send_email", "post_webhook"], // ...only for these tools
|
|
795
1493
|
});
|
|
796
1494
|
|
|
797
|
-
ai.agent({ model, tools: [
|
|
1495
|
+
const agent = ai.agent({ model, tools: [sendEmail, postWebhook, lookup], middleware: [policy] });
|
|
1496
|
+
// `lookup` runs unguarded; `send_email` / `post_webhook` block on PII in their args.
|
|
798
1497
|
```
|
|
799
1498
|
|
|
800
|
-
|
|
1499
|
+
A `block` from `tool.before` aborts that tool dispatch and surfaces on `result.error` with `phase: "tool"` — the agent run itself still never crashes.
|
|
801
1500
|
|
|
802
|
-
|
|
1501
|
+
## Install order
|
|
803
1502
|
|
|
804
|
-
|
|
1503
|
+
A guard is a normal `AgentMiddleware`; registration order is execution order (`before` top-down, `after` bottom-up). The canonical order is `[cache, budget, guardrail, observability]`. A `semanticCache` that short-circuits `trip.before` runs *before* the guard — a cached response then skips the **output** detectors, so place the guard before the cache if you don't trust cached contents.
|
|
805
1504
|
|
|
806
|
-
|
|
1505
|
+
## Input-redaction limitation (v1)
|
|
1506
|
+
|
|
1507
|
+
The core `trip.before` hook can only **short-circuit** (return a `ModelResponse`); it cannot rewrite the outbound prompt and continue. So:
|
|
1508
|
+
|
|
1509
|
+
- **Input detectors are `block` / `flag` only.** A `redact` verdict on an input detector is treated as a `block` rather than silently passing an un-redacted prompt.
|
|
1510
|
+
- **Output redaction works** — `trip.after` returns a replacement `ModelResponse` with the rewritten `content`.
|
|
1511
|
+
- **Tool-arg `redact` is also withheld** — it downgrades to a `block` (`reason: "tool-arg-redaction-unsupported"`), because silently rewriting tool arguments changes the call's side-effects unpredictably.
|
|
1512
|
+
|
|
1513
|
+
Lifting the input limitation needs a small, non-breaking core affordance and is deferred.
|
|
1514
|
+
|
|
1515
|
+
## Failure isolation
|
|
1516
|
+
|
|
1517
|
+
A detector's `check()` **rejecting** is an infrastructure fault, not a content violation — it is recorded as a `flag` (`<detector>.error`) into `ctx.state` and the fold **continues** (fail-open). A moderation-API outage degrades to missing annotation, never a failed agent run.
|
|
807
1518
|
|
|
808
1519
|
## See also
|
|
809
1520
|
|
|
810
|
-
- [`@warlock.js/ai/
|
|
811
|
-
- [`@warlock.js/ai/
|
|
812
|
-
- [`@warlock.js/ai/
|
|
813
|
-
- [`@warlock.js/cache/use-cache-similarity/SKILL.md`](@warlock.js/cache/use-cache-similarity/SKILL.md) — cache as a vector store
|
|
814
|
-
- [`@warlock.js/cascade/search-by-vector/SKILL.md`](@warlock.js/cascade/search-by-vector/SKILL.md) — cascade `similarTo` query method
|
|
1521
|
+
- [`@warlock.js/ai-guard/detect-and-redact-pii/SKILL.md`](@warlock.js/ai-guard/detect-and-redact-pii/SKILL.md) — the `pii` detector (detect/redact/block), the `mask` template, and the optional `moderation` peer.
|
|
1522
|
+
- [`@warlock.js/ai-guard/escalate-block-to-human/SKILL.md`](@warlock.js/ai-guard/escalate-block-to-human/SKILL.md) — routing a `block` to a human-review surface via `escalation.onBlock`.
|
|
1523
|
+
- [`@warlock.js/ai/run-ai-agent/SKILL.md`](@warlock.js/ai/run-ai-agent/SKILL.md) — running the agent, the middleware pipeline, and the `GuardrailViolationError` on `result.error`.
|
|
815
1524
|
|
|
816
1525
|
|
|
817
1526
|
## handle-ai-errors `@warlock.js/ai/handle-ai-errors/SKILL.md`
|
|
@@ -1302,6 +2011,294 @@ A bare `ai.supervisor()` / `ai.workflow()` uses a `snapshotStore` for `resume(ru
|
|
|
1302
2011
|
- [`@warlock.js/ai/handle-ai-errors/SKILL.md`](@warlock.js/ai/handle-ai-errors/SKILL.md) — `OrchestratorDriftError` / `OrchestratorConfigError`
|
|
1303
2012
|
|
|
1304
2013
|
|
|
2014
|
+
## manage-prompts `@warlock.js/ai/manage-prompts/SKILL.md`
|
|
2015
|
+
|
|
2016
|
+
---
|
|
2017
|
+
name: manage-prompts
|
|
2018
|
+
description: 'Unified prompt registry — ai.prompts: one process-wide store of named, versioned systemPrompt(...) builders keyed by name@version. Register by giving a prompt a meta.name (auto-registers), resolve by get(name) / resolve(name, versionOrTag, placeholders) / the inline name@selector form, bulk-register with define(name, versions), pin tags with tag(name, tag, version), compare with diff(name, from, to), round-trip with export() / import(snapshot), and quality-check with a unified validate(target, options) (deterministic missing-placeholder check + optional Nova-safe LLM-as-judge with verdict caching). Compose registered prompts into new ones with systemPrompt().merge(name, { fromVersion }) — provenance recorded in meta.composedFrom. ai.prompt is now a thin FACADE over ai.prompts (BREAKING vs the old standalone registry). Triggers: `ai.prompts`, `ai.prompt`, `PromptsManagerContract`, `PromptsManagerEntry`, `SystemPromptContract`, `SystemPromptMeta`, `SystemPromptMergeOptions`, `PromptsValidateOptions`, `PromptValidationResult`, `PromptValidateTarget`, `PromptTemplateVersion`, `PromptDiff`, `ExportedRegistry`, `defaultPromptsManager`, `prompts()`, `promptKey`, `meta`, `name`, `version`, `composedFrom`, `fromVersion`, `register`, `create`, `get`, `has`, `list`, `versions`, `resolve`, `define`, `tag`, `validate`, `diff`, `export`, `import`, `merge`, `judge`, `judgeCache`; ''register a prompt by name'', ''resolve a prompt by name@version or tag'', ''pin a production tag to a prompt version'', ''diff two prompt versions'', ''export / import the prompt registry'', ''validate a prompt for missing placeholders'', ''merge a registered prompt into another''; typical import `import { ai } from "@warlock.js/ai"`. Skip: composing a single prompt from persona + instruction blocks (the builder itself) — `@warlock.js/ai/write-system-prompt/SKILL.md`; runtime loadable skill bodies — `@warlock.js/ai/use-runtime-skills/SKILL.md`; eval scoring of agent outputs — `@warlock.js/ai/eval-datasets-and-ci/SKILL.md`; competing libs `langfuse` (direct), `promptfoo`.'
|
|
2019
|
+
---
|
|
2020
|
+
|
|
2021
|
+
# `ai.prompts` — the unified prompt registry
|
|
2022
|
+
|
|
2023
|
+
`ai.prompts` is ONE process-wide registry of named, versioned `systemPrompt(...)` builders keyed by `name@version`. A `systemPrompt(input, { name })` (or any `.meta({ name })` rename) auto-registers here; `ai.prompts.get(name)` / `.resolve(name)` read them back; `systemPrompt().merge(name)` folds a registered prompt into a new one. There is exactly **one storage shape** behind the whole prompt surface — a `SystemPromptContract` keyed by `name@version` — and `ai.prompt(...)` is now a thin facade over it (see the migration note below).
|
|
2024
|
+
|
|
2025
|
+
```ts
|
|
2026
|
+
import { ai } from "@warlock.js/ai";
|
|
2027
|
+
|
|
2028
|
+
// Register: any named systemPrompt auto-registers in ai.prompts.
|
|
2029
|
+
ai.systemPrompt("You are support for {{product}}.", { name: "support" });
|
|
2030
|
+
|
|
2031
|
+
// Resolve back — latest version, or a version / pinned tag.
|
|
2032
|
+
ai.prompts.get("support"); // → the SystemPromptContract
|
|
2033
|
+
ai.prompts.resolve("support", undefined, { product: "Warlock" }); // → final string
|
|
2034
|
+
```
|
|
2035
|
+
|
|
2036
|
+
`ai.prompts` is the process-wide default (`defaultPromptsManager()`). For an **isolated** registry (parallel test suites, multi-tenant apps) call the `prompts()` factory — same `PromptsManagerContract`, its own store, no global side effects.
|
|
2037
|
+
|
|
2038
|
+
## Identity — `SystemPromptMeta` (`meta.name` / `version` / `description` / `required` / `composedFrom`)
|
|
2039
|
+
|
|
2040
|
+
A prompt's identity rides on its `meta`. Read it with the no-argument accessor; update it immutably with the one-argument form:
|
|
2041
|
+
|
|
2042
|
+
```ts
|
|
2043
|
+
const base = ai.systemPrompt("You are support.", {
|
|
2044
|
+
name: "support",
|
|
2045
|
+
version: "1",
|
|
2046
|
+
description: "Tier-1 support persona.",
|
|
2047
|
+
required: ["product"],
|
|
2048
|
+
});
|
|
2049
|
+
|
|
2050
|
+
base.meta(); // → { name: "support", version: "1", description, required }
|
|
2051
|
+
const v2 = base.meta({ version: "2" }); // new builder, shallow-merged meta; original untouched
|
|
2052
|
+
```
|
|
2053
|
+
|
|
2054
|
+
- **`name`** — when present, the prompt auto-registers in `ai.prompts` under `name@version`. Anonymous prompts (no `name`) are never registered.
|
|
2055
|
+
- **`version`** — free-form label (`"1"`, `"2025-draft"`). Defaults to the **next integer** for that name when omitted.
|
|
2056
|
+
- **`description`** — human-readable purpose (carried through `export`).
|
|
2057
|
+
- **`required`** — placeholder keys callers must supply; `validate()` reads them.
|
|
2058
|
+
- **`composedFrom`** — deterministic source labels a prompt was merged from (e.g. `["base@2", "global@1"]`). No random suffixes — the same merge always yields the same labels.
|
|
2059
|
+
|
|
2060
|
+
## Register / resolve — `register` / `get` / `resolve` / `has` / `list` / `versions`
|
|
2061
|
+
|
|
2062
|
+
```ts
|
|
2063
|
+
const registry = ai.prompts; // or prompts() for an isolated one
|
|
2064
|
+
|
|
2065
|
+
registry.register(ai.systemPrompt("You are support.", { name: "support" }));
|
|
2066
|
+
registry.versions("support"); // ["1"] — version derived as next integer
|
|
2067
|
+
|
|
2068
|
+
registry.get("support"); // latest SystemPromptContract
|
|
2069
|
+
registry.get("support@1"); // inline name@selector
|
|
2070
|
+
registry.resolve("support", "1", { product: "Warlock" }); // pick version + render in one call
|
|
2071
|
+
|
|
2072
|
+
registry.has("support"); // boolean
|
|
2073
|
+
registry.list(); // every registered name, first-seen order
|
|
2074
|
+
```
|
|
2075
|
+
|
|
2076
|
+
- **Version selection** — `get(name)` / `resolve(name)` return the **latest** by insertion order; pass a version label, a pinned tag, or fold it into the first arg as `name@selector` (`get("support@1")`, `resolve("support@production")`).
|
|
2077
|
+
- **Duplicates** — re-registering the same `name@version` throws `InvalidRequestError` **unless** the content is byte-identical (idempotent re-registration is a no-op).
|
|
2078
|
+
- **Unknown name / version / tag** → `InvalidRequestError`.
|
|
2079
|
+
- `register()` throws if the prompt has no `meta.name`.
|
|
2080
|
+
|
|
2081
|
+
## `create()` — build + register in one entry point
|
|
2082
|
+
|
|
2083
|
+
`ai.prompts.create(input?, meta?)` is a documented alias of `ai.systemPrompt(...)` — identical input forms (no arg → empty builder; a string → one instruction; an array of blocks → verbatim). Pass `meta.name` to auto-register, so authoring and lookup read side-by-side:
|
|
2084
|
+
|
|
2085
|
+
```ts
|
|
2086
|
+
ai.prompts.create("You are support for {{product}}.", { name: "support" });
|
|
2087
|
+
ai.prompts.resolve("support", undefined, { product: "Warlock" });
|
|
2088
|
+
```
|
|
2089
|
+
|
|
2090
|
+
## `define()` — bulk-register many versions
|
|
2091
|
+
|
|
2092
|
+
```ts
|
|
2093
|
+
ai.prompts.define("agent", [
|
|
2094
|
+
{ version: "1", template: "You are v1." },
|
|
2095
|
+
{ version: "2", template: [ai.persona("You are Alex."), ai.instruction("Be concise.")] },
|
|
2096
|
+
]);
|
|
2097
|
+
```
|
|
2098
|
+
|
|
2099
|
+
A `PromptTemplateVersion`'s `template` is a raw string (wrapped into one instruction block) or an explicit ordered block list (verbatim). Versions register **oldest-first** in array order; the same duplicate / idempotency rule applies per `name@version`. Returns the manager for chaining.
|
|
2100
|
+
|
|
2101
|
+
## `tag()` — pin a moving label to a version
|
|
2102
|
+
|
|
2103
|
+
```ts
|
|
2104
|
+
ai.prompts.tag("agent", "production", "2"); // pin "production" → version 2
|
|
2105
|
+
|
|
2106
|
+
ai.prompts.get("agent", "production"); // resolves through the tag
|
|
2107
|
+
ai.prompts.resolve("agent", "production");
|
|
2108
|
+
ai.prompts.get("agent@production"); // inline form
|
|
2109
|
+
```
|
|
2110
|
+
|
|
2111
|
+
Re-pinning an existing tag moves it. An unknown name / version throws `InvalidRequestError`. Tags survive `export` / `import`.
|
|
2112
|
+
|
|
2113
|
+
## `validate()` — unified deterministic + optional LLM-judge
|
|
2114
|
+
|
|
2115
|
+
```ts
|
|
2116
|
+
const report = await ai.prompts.validate("support", {
|
|
2117
|
+
placeholders: { product: "Warlock" }, // values you intend to supply
|
|
2118
|
+
declare: ["language"], // extra keys to treat as known
|
|
2119
|
+
judge: judgeModel, // optional — turns on the LLM-as-judge pass
|
|
2120
|
+
});
|
|
2121
|
+
|
|
2122
|
+
report.ok; // true iff no required placeholder is missing (DETERMINISTIC verdict alone)
|
|
2123
|
+
report.missing; // placeholder keys referenced with no default, unsupplied, undeclared
|
|
2124
|
+
report.score; // 0..1 — present ONLY when a judge ran and produced a usable verdict
|
|
2125
|
+
report.issues; // advisory judge reasons / a degrade note — present only when a judge was supplied
|
|
2126
|
+
```
|
|
2127
|
+
|
|
2128
|
+
- **Always** runs the deterministic check: every `{{key}}` with no inline default that is neither supplied (`placeholders`), declared (`declare`), nor in the prompt's `meta.required` lands in `missing`; `ok` is `true` iff `missing` is empty.
|
|
2129
|
+
- **`judge`** adds a **Nova-safe** LLM-as-judge quality pass — it **never throws** and degrades to an `issues` note (leaving `score` undefined) on failure, so a flaky judge can **never flip `ok`**.
|
|
2130
|
+
- **`target`** is a registered name (or `name@selector`), a `SystemPromptContract` instance, or a raw prompt string.
|
|
2131
|
+
- **`judgeCache`** (per-call or via the `prompts({ judgeCache })` factory option) memoizes judge verdicts by a content hash of the resolved body + the judge model id — a structural `{ get, set }` subset of `@warlock.js/cache`'s `CacheDriver`, so the cache package stays a strictly **optional** peer.
|
|
2132
|
+
|
|
2133
|
+
`systemPrompt().validate(options?)` is the per-builder sugar — `ai.prompts.validate(this, options)` under the hood, same result shape.
|
|
2134
|
+
|
|
2135
|
+
## `diff()` — block-level version diff
|
|
2136
|
+
|
|
2137
|
+
```ts
|
|
2138
|
+
const diff = ai.prompts.diff("agent", "1", "2");
|
|
2139
|
+
|
|
2140
|
+
diff.identical; // true when both versions have identical blocks in identical order
|
|
2141
|
+
diff.added; // blocks in `to` not at the same position in `from`
|
|
2142
|
+
diff.removed; // blocks in `from` not at the same position in `to`
|
|
2143
|
+
diff.changed; // [{ from, to }] — same position, type/text changed
|
|
2144
|
+
```
|
|
2145
|
+
|
|
2146
|
+
Blocks are matched **positionally**. Unknown name / version → `InvalidRequestError`.
|
|
2147
|
+
|
|
2148
|
+
## `export()` / `import()` — portable JSON round-trip
|
|
2149
|
+
|
|
2150
|
+
```ts
|
|
2151
|
+
const snapshot = ai.prompts.export(); // ExportedRegistry — every name, version, pinned tag, description/required
|
|
2152
|
+
otherRegistry.import(snapshot); // rehydrate (same duplicate / idempotency rule; tags restored)
|
|
2153
|
+
```
|
|
2154
|
+
|
|
2155
|
+
Each version flattens to `{ type, text }` blocks so the registry round-trips without live builder instances — commit a snapshot, ship it, restore it elsewhere.
|
|
2156
|
+
|
|
2157
|
+
## Compose registered prompts — `systemPrompt().merge(name, { fromVersion })`
|
|
2158
|
+
|
|
2159
|
+
`merge` folds another prompt's blocks into a new builder (persona **replaces**, instructions **append**) and records `meta.composedFrom`:
|
|
2160
|
+
|
|
2161
|
+
```ts
|
|
2162
|
+
ai.systemPrompt("Always answer in {{language|English}}.", { name: "global", version: "1" });
|
|
2163
|
+
|
|
2164
|
+
const supportPrompt = ai.systemPrompt("You are support for {{product}}.")
|
|
2165
|
+
.merge("global", { fromVersion: "1" }); // fold the registered prompt by name
|
|
2166
|
+
|
|
2167
|
+
supportPrompt.meta()?.composedFrom; // ["…", "global@1"] — deterministic provenance
|
|
2168
|
+
```
|
|
2169
|
+
|
|
2170
|
+
`merge` accepts three source forms: a pre-built block, another `SystemPromptContract`, or a **registered name** resolved from `ai.prompts` (latest version unless `options.fromVersion` selects another — an unknown name / version throws `InvalidRequestError`).
|
|
2171
|
+
|
|
2172
|
+
## `ai.prompt(...)` — now a thin facade (⚠ breaking vs the old registry)
|
|
2173
|
+
|
|
2174
|
+
`ai.prompt` has **two** call forms, both backed by the unified manager — there is no longer a separate prompt store:
|
|
2175
|
+
|
|
2176
|
+
```ts
|
|
2177
|
+
// (a) Resolve a globally-registered prompt from ai.prompts by name.
|
|
2178
|
+
ai.systemPrompt("You are support.", { name: "support" });
|
|
2179
|
+
const sp = ai.prompt("support"); // → SystemPromptContract (latest)
|
|
2180
|
+
const v1 = ai.prompt("support", "1"); // → a specific version / pinned tag
|
|
2181
|
+
|
|
2182
|
+
// (b) Build an ISOLATED legacy-shaped registry (PromptRegistryContract).
|
|
2183
|
+
const reg = ai.prompt({
|
|
2184
|
+
prompts: [{ name: "summarizer", versions: [{ version: "1", template: "Summarize: {{text}}" }] }],
|
|
2185
|
+
});
|
|
2186
|
+
const resolved = reg.resolve("summarizer", { placeholders: { text } });
|
|
2187
|
+
resolved.toSystemPrompt(); // drop-in for ai.agent({ systemPrompt })
|
|
2188
|
+
```
|
|
2189
|
+
|
|
2190
|
+
**⚠ Migration.** Before unification, `ai.prompt(...)` only built a standalone, self-contained registry with its **own private** storage. It now:
|
|
2191
|
+
|
|
2192
|
+
1. Adds the **string overload** `ai.prompt(name, versionOrTag?)` → resolves from the shared `ai.prompts` manager. (New capability — `ai.prompt("x")` used to be a type error.)
|
|
2193
|
+
2. Backs the **options form** (`ai.prompt({ ... })` → `PromptRegistryContract`) by an internal `PromptsManagerContract`, so its storage shape and validation primitives are now the unified ones. The legacy method surface (`register` / `add` / `versions` / `resolve` / `validate` / `sync` + the `{ score, notes }` report shape) is **unchanged**, and each `ai.prompt({ ... })` call still returns its **own isolated** registry — no shared global state.
|
|
2194
|
+
|
|
2195
|
+
If you only ever called `ai.prompt({ ... })` and used the returned registry, **no code change is needed**. The new behavior is additive: prefer `ai.prompts` (the unified manager) for new code; reach for `ai.prompt({ ... })` only when you want the legacy `ResolvedPrompt` / `toSystemPrompt()` ergonomics or the optional Langfuse sync. The legacy facade's reference — `register` / `add` / `resolve(name, { version, placeholders })` / `validate` (`{ score, notes }`) / `sync()` (lazy `langfuse` peer) — is documented inline in `src/prompt/prompt.ts`.
|
|
2196
|
+
|
|
2197
|
+
## See also
|
|
2198
|
+
|
|
2199
|
+
- [`@warlock.js/ai/write-system-prompt/SKILL.md`](@warlock.js/ai/write-system-prompt/SKILL.md) — the `systemPrompt()` / `persona()` / `instruction()` builder, `.meta()`, and `merge()` this registry stores and composes
|
|
2200
|
+
- [`@warlock.js/ai/eval-datasets-and-ci/SKILL.md`](@warlock.js/ai/eval-datasets-and-ci/SKILL.md) — the eval `judge` scorer `validate()`'s LLM pass reuses
|
|
2201
|
+
- [`@warlock.js/ai/run-ai-agent/SKILL.md`](@warlock.js/ai/run-ai-agent/SKILL.md) — wiring a resolved prompt into an agent, plus the judge-safe agent preset (`ai.agent.judge`)
|
|
2202
|
+
|
|
2203
|
+
|
|
2204
|
+
## observe-ai-flows `@warlock.js/ai/observe-ai-flows/SKILL.md`
|
|
2205
|
+
|
|
2206
|
+
---
|
|
2207
|
+
name: observe-ai-flows
|
|
2208
|
+
description: 'The core Observer seam — a generic, tool-agnostic observability hook every flow routes its completed ExecutionReport through. Covers the per-flow `observe?: boolean | Observer` option on ai.agent / workflow / supervisor / team, the global registry (registerObserver / getObservers / setObserveAll / isObserveAll / clearObservers), resolveObservers / notifyObservers resolution, the opt-in AgentConfig.captureMessages → AgentReport.messages full-history capture, the onConfigApplied dependency-inversion seam, and that @warlock.js/ai-panoptic is the batteries-included Observer. Triggers: `Observer`, `observe`, `registerObserver`, `getObservers`, `setObserveAll`, `isObserveAll`, `clearObservers`, `resolveObservers`, `notifyObservers`, `FlowObserveOption`, `ExecutionReport`, `captureMessages`, `AgentReport.messages`, `CapturedMessage`, `onConfigApplied`, `observeAll`; ''observe an agent run'', ''send finished reports to a collector'', ''capture the full message history'', ''observe every flow by default'', ''wire panoptic / tracing''; typical import `import { ai, registerObserver } from "@warlock.js/ai"`. Skip: structured logging of events — `@warlock.js/ai/log-ai-calls/SKILL.md`; reading the report tree shape (trips / children) — `@warlock.js/ai/run-ai-agent/SKILL.md`; per-call cost / usage rollup — `@warlock.js/ai/handle-ai-errors/SKILL.md`. The batteries-included Observer is the `@warlock.js/ai-panoptic` package.'
|
|
2209
|
+
---
|
|
2210
|
+
|
|
2211
|
+
# The `Observer` seam — generic, tool-agnostic observability
|
|
2212
|
+
|
|
2213
|
+
Core defines a structural `Observer` and a tiny registry; it never imports any observability package (panoptic, OTel, Langfuse, …). A flow that resolves to "observed" hands its completed `ExecutionReport` to every registered observer. An observability tool **implements `Observer` and registers itself**, so `observe: true` / observe-all route reports without coupling core to the tool — the dependency inversion that keeps the two sides decoupled.
|
|
2214
|
+
|
|
2215
|
+
```ts
|
|
2216
|
+
export interface Observer {
|
|
2217
|
+
collect(report: ExecutionReport): void | Promise<void>;
|
|
2218
|
+
}
|
|
2219
|
+
```
|
|
2220
|
+
|
|
2221
|
+
`collect` may be sync or async — the flow awaits it. A throw is **swallowed** by the flow (never breaks the run), mirroring the existing `onUsage` / `onComplete` hooks.
|
|
2222
|
+
|
|
2223
|
+
## Per-flow `observe` option
|
|
2224
|
+
|
|
2225
|
+
`observe?: boolean | Observer` (`FlowObserveOption`) is accepted on **`ai.agent`, `ai.workflow`, `ai.supervisor`, and `ai.team`** (a team forwards it verbatim to the supervisor it desugars into):
|
|
2226
|
+
|
|
2227
|
+
```ts
|
|
2228
|
+
const collector: Observer = { collect(report) { exporter.send(report); } };
|
|
2229
|
+
|
|
2230
|
+
ai.agent({ model, observe: true }); // → the globally registered observers, even if observe-all is off
|
|
2231
|
+
ai.agent({ model, observe: false }); // → opt out entirely, even when observe-all is on
|
|
2232
|
+
ai.agent({ model, observe: collector }); // → a flow-LOCAL collector; only this flow's report, only to it
|
|
2233
|
+
ai.agent({ model }); // → undefined: follow the global observe-all flag
|
|
2234
|
+
```
|
|
2235
|
+
|
|
2236
|
+
Resolution (`resolveObservers(observe)`):
|
|
2237
|
+
|
|
2238
|
+
- `false` → `[]` (opted out).
|
|
2239
|
+
- `true` → the globally registered observers.
|
|
2240
|
+
- an `Observer` object → just that one (flow-local; the global observers are skipped).
|
|
2241
|
+
- `undefined` → the global observers when observe-all is on, otherwise `[]`.
|
|
2242
|
+
|
|
2243
|
+
`notifyObservers(observe, report)` routes a completed report to each resolved observer, awaiting each `collect` (so async exporters finish before the flow returns) and swallowing any throw. The object form is typed as the structural `Observer` (NOT a panoptic-specific type), so a panoptic flow-local collector — which implements `Observer` — can be passed directly.
|
|
2244
|
+
|
|
2245
|
+
## The global registry
|
|
2246
|
+
|
|
2247
|
+
```ts
|
|
2248
|
+
import {
|
|
2249
|
+
registerObserver, getObservers, setObserveAll, isObserveAll, clearObservers,
|
|
2250
|
+
} from "@warlock.js/ai";
|
|
2251
|
+
|
|
2252
|
+
registerObserver(collector); // an observability tool registers ONE collector when its config is applied
|
|
2253
|
+
getObservers(); // read-only snapshot of the registered observers (do not mutate)
|
|
2254
|
+
|
|
2255
|
+
setObserveAll(true); // "observe every flow by default" — flows without their own `observe` get observed
|
|
2256
|
+
isObserveAll(); // read the flag (default false — opt-in observability)
|
|
2257
|
+
|
|
2258
|
+
clearObservers(); // test-only: reset observers + the observe-all flag for spec isolation
|
|
2259
|
+
```
|
|
2260
|
+
|
|
2261
|
+
`observeAll` defaults to `false` (opt-in). A flow that never sets `observe` is observed **only** when observe-all is on; individual flows still opt out with `observe: false`.
|
|
2262
|
+
|
|
2263
|
+
## Full-history capture — `captureMessages` → `AgentReport.messages`
|
|
2264
|
+
|
|
2265
|
+
Off by default. When `ai.agent({ captureMessages: true })` is set, the agent normalizes the real assembled turn array onto `AgentReport.messages` as a `CapturedMessage[]`:
|
|
2266
|
+
|
|
2267
|
+
```ts
|
|
2268
|
+
const { report } = await ai.agent({ model, tools, captureMessages: true }).execute("Go");
|
|
2269
|
+
report.messages; // CapturedMessage[] — every role (system/user/assistant/tool), every trip
|
|
2270
|
+
```
|
|
2271
|
+
|
|
2272
|
+
A `CapturedMessage` is a JSON-safe projection: `{ role, content, toolCalls?, toolCallId? }` — `content` is always a string (tool results stringified), assistant turns that triggered tools carry `toolCalls`, tool-result turns carry the `toolCallId` they answer. Unlike `trips[].input` (which stubs non-first trips with `"[tool results]"`), this preserves the **real** turn array. Omitted ⇒ the field is **absent** and the report is byte-for-byte as before. Opt-in because messages can be large and sensitive (full prompts, tool inputs/outputs) — and **required for panoptic full-history capture**.
|
|
2273
|
+
|
|
2274
|
+
## Callback sub-agents nest in the report tree
|
|
2275
|
+
|
|
2276
|
+
The `ExecutionReport` an observer receives reflects **full** lineage: a supervisor / team / orchestrator callback that calls `agent.execute()` directly auto-nests `callback → agent → tool` (via an ambient `RunFrame`), so usage / cost roll up and panoptic renders the sub-agent under its callback instead of as a lone `$0` span. No observer-side change is needed — the tree arrives already nested. A team's root span carries `type: "team"` (a first-class `ReportType`, not `"supervisor"`), so observers can distinguish, group, and label team runs as their own type. See [`@warlock.js/ai/run-supervisor/SKILL.md`](@warlock.js/ai/run-supervisor/SKILL.md).
|
|
2277
|
+
|
|
2278
|
+
## The config seam — `onConfigApplied`
|
|
2279
|
+
|
|
2280
|
+
An observability tool reacts to its own augmented config slot without core importing it. Core lets tools attach an opaque slot (e.g. `panoptic?`) via declaration merging on `AIConfig`, then fires registered listeners after each `ai.config(...)` merge:
|
|
2281
|
+
|
|
2282
|
+
```ts
|
|
2283
|
+
import { onConfigApplied, getAIConfig } from "@warlock.js/ai";
|
|
2284
|
+
|
|
2285
|
+
onConfigApplied((config) => applyPanopticConfig(config.panoptic)); // react on every config merge
|
|
2286
|
+
applyPanopticConfig(getAIConfig().panoptic); // catch a pre-set config
|
|
2287
|
+
```
|
|
2288
|
+
|
|
2289
|
+
A misbehaving listener's throw is swallowed (same swallow-on-throw discipline as the observer hooks). This mirrors the `Observer` registry's dependency inversion: a tool flips `setObserveAll(true)` and calls `registerObserver(...)` from inside its `onConfigApplied` listener.
|
|
2290
|
+
|
|
2291
|
+
## The batteries-included Observer
|
|
2292
|
+
|
|
2293
|
+
`@warlock.js/ai-panoptic` is the shipped, full-featured `Observer` — install it, configure it via `ai.config({ panoptic })`, and it registers its collector + (optionally) flips observe-all for you. Core stays dependency-free; this skill documents the seam panoptic plugs into.
|
|
2294
|
+
|
|
2295
|
+
## See also
|
|
2296
|
+
|
|
2297
|
+
- [`@warlock.js/ai/run-ai-agent/SKILL.md`](@warlock.js/ai/run-ai-agent/SKILL.md) — the `AgentReport` / `ExecutionReport` tree (`trips`, `children`) an observer receives
|
|
2298
|
+
- [`@warlock.js/ai/log-ai-calls/SKILL.md`](@warlock.js/ai/log-ai-calls/SKILL.md) — event-level structured logging (vs. report-level observation)
|
|
2299
|
+
- [`@warlock.js/ai/run-ai-team/SKILL.md`](@warlock.js/ai/run-ai-team/SKILL.md) — a team inherits `observe` through the supervisor it forwards to
|
|
2300
|
+
|
|
2301
|
+
|
|
1305
2302
|
## persist-ai-data `@warlock.js/ai/persist-ai-data/SKILL.md`
|
|
1306
2303
|
|
|
1307
2304
|
---
|
|
@@ -1726,11 +2723,107 @@ If you switch providers mid-project (e.g. OpenAI → Anthropic):
|
|
|
1726
2723
|
- [`@warlock.js/ai/handle-ai-errors/SKILL.md`](@warlock.js/ai/handle-ai-errors/SKILL.md) — adapter error categorization
|
|
1727
2724
|
|
|
1728
2725
|
|
|
2726
|
+
## record-replay-llm `@warlock.js/ai/record-replay-llm/SKILL.md`
|
|
2727
|
+
|
|
2728
|
+
---
|
|
2729
|
+
name: record-replay-llm
|
|
2730
|
+
description: 'Deterministic, offline LLM tests with ai.vcr(model,{path,mode}) — a record/replay decorator over ANY ModelContract that intercepts only complete()/stream(), delegates name/provider/capabilities/pricing to the inner model, and hashes each request against a JSON cassette on disk. Covers the three modes (record / replay / auto), the cassette format, save(), VcrCassetteMissError, streaming round-trip, hashOptions, and composing below fallbackModel. Triggers: `ai.vcr`, `vcr`, `VcrModel`, `VcrOptions`, `VcrMode`, `Cassette`, `CassetteEntry`, `VcrCassetteMissError`, `hashRequest`, `DEFAULT_HASH_OPTIONS`, `mode`, `path`, `hashOptions`, `save`, `cassette`, record, replay, cassette; ''record LLM responses for tests'', ''replay model calls offline in CI'', ''deterministic agent test without hitting the provider'', ''cassette for model calls''; typical import `import { ai } from "@warlock.js/ai"`. Skip: eval scoring + regression gating — `@warlock.js/ai/eval-datasets-and-ci/SKILL.md`; the Vitest matchers + mockRouter — `@warlock.js/ai/ai-dx-helpers/SKILL.md`; choosing a provider adapter — `@warlock.js/ai/pick-ai-provider/SKILL.md`; competing libs `nock`, `polly.js`.'
|
|
2731
|
+
---
|
|
2732
|
+
|
|
2733
|
+
# `ai.vcr()` — record / replay any model
|
|
2734
|
+
|
|
2735
|
+
`ai.vcr(model, { path, mode })` wraps any `ModelContract` in a record/replay decorator backed by a JSON cassette on disk. It intercepts only `complete()` / `stream()` — the single seam every agent trip funnels through — and delegates `name`, `provider`, `capabilities`, and `pricing` to the inner model untouched, so cost accounting and capability detection are unchanged. Depends only on `ModelContract`, so it works with **any** adapter.
|
|
2736
|
+
|
|
2737
|
+
## Shape
|
|
2738
|
+
|
|
2739
|
+
```ts
|
|
2740
|
+
import { ai } from "@warlock.js/ai";
|
|
2741
|
+
import { OpenAISDK } from "@warlock.js/ai-openai";
|
|
2742
|
+
|
|
2743
|
+
const openai = new OpenAISDK({ apiKey: process.env.OPENAI_API_KEY! });
|
|
2744
|
+
const live = openai.model({ name: "gpt-4o-mini" });
|
|
2745
|
+
|
|
2746
|
+
const model = ai.vcr(live, {
|
|
2747
|
+
path: "./cassettes/support.json", // cassette file (JSON); read on construct, written on save()
|
|
2748
|
+
mode: "auto", // "record" | "replay" | "auto" (default)
|
|
2749
|
+
});
|
|
2750
|
+
|
|
2751
|
+
const agent = ai.agent({ model, systemPrompt: "..." });
|
|
2752
|
+
const result = await agent.execute("hi");
|
|
2753
|
+
|
|
2754
|
+
await model.save(); // first run records; later runs replay deterministically
|
|
2755
|
+
```
|
|
2756
|
+
|
|
2757
|
+
The returned `VcrModel` is a `ModelContract` plus `save(): Promise<void>` and a readonly `cassette` (exposed for assertions).
|
|
2758
|
+
|
|
2759
|
+
## Modes — `VcrMode`
|
|
2760
|
+
|
|
2761
|
+
- **`record`** — always calls the inner model and appends a cassette entry. Never replays (the in-memory cassette starts empty, so a record run never accidentally replays a stale entry). Use to (re)capture a fresh cassette.
|
|
2762
|
+
- **`replay`** — never calls the inner model. A cassette hit returns the stored response / re-yields its chunks / re-throws its error; a **miss throws `VcrCassetteMissError`** — never a silent live call. Use in CI for deterministic, offline tests.
|
|
2763
|
+
- **`auto`** (default) — replay on a hit, record on a miss. The friendliest mode for local dev: records once, replays thereafter.
|
|
2764
|
+
|
|
2765
|
+
## Request hashing — `hashOptions`
|
|
2766
|
+
|
|
2767
|
+
On each call VCR computes a stable hash over `{ messages, picked options }` and looks for a matching `CassetteEntry`. The hashed option fields default to:
|
|
2768
|
+
|
|
2769
|
+
```ts
|
|
2770
|
+
["temperature", "maxTokens", "responseSchema", "tools", "reasoning"]
|
|
2771
|
+
```
|
|
2772
|
+
|
|
2773
|
+
(`DEFAULT_HASH_OPTIONS`). `signal` and unknown provider keys are **always excluded**, so an otherwise-identical logical call still matches. `tools` are hashed by name + description + input-schema shape, not object identity. Override `hashOptions` to widen / narrow what counts as "the same request". `hashRequest(messages, options, hashOptions)` is exported for direct use.
|
|
2774
|
+
|
|
2775
|
+
## Cassette format
|
|
2776
|
+
|
|
2777
|
+
A `Cassette` is `{ version: 1, model, provider, entries: CassetteEntry[] }`. Each `CassetteEntry` is `{ requestHash, request: { messages, options? }, ... }` where **exactly one** of `response` / `chunks` / `error` is populated — mirroring the three outcomes of a model call (non-streaming reply, streamed chunk list, or a thrown provider error). The full `request` is stored verbatim for human readability and so the cassette can be re-hashed if the hashing format ever changes.
|
|
2778
|
+
|
|
2779
|
+
## Streaming round-trip
|
|
2780
|
+
|
|
2781
|
+
```ts
|
|
2782
|
+
for await (const chunk of model.stream(messages)) {
|
|
2783
|
+
// record mode: buffers each chunk into entry.chunks[] while re-emitting
|
|
2784
|
+
// replay mode: re-yields the stored chunks in order (delta / tool-call / done sequence)
|
|
2785
|
+
}
|
|
2786
|
+
await model.save();
|
|
2787
|
+
```
|
|
2788
|
+
|
|
2789
|
+
Recorded chunks reproduce the exact `delta` / `tool-call` / `done` sequence on replay; a recorded error is re-thrown.
|
|
2790
|
+
|
|
2791
|
+
## `VcrCassetteMissError`
|
|
2792
|
+
|
|
2793
|
+
```ts
|
|
2794
|
+
import { VcrCassetteMissError } from "@warlock.js/ai";
|
|
2795
|
+
|
|
2796
|
+
try {
|
|
2797
|
+
await vcrModel.complete(messages);
|
|
2798
|
+
} catch (error) {
|
|
2799
|
+
if (error instanceof VcrCassetteMissError) {
|
|
2800
|
+
console.error("Re-record the cassette:", error.path, error.requestHash);
|
|
2801
|
+
}
|
|
2802
|
+
}
|
|
2803
|
+
```
|
|
2804
|
+
|
|
2805
|
+
Thrown only in `replay` mode on a miss (code `"VCR_CASSETTE_MISS"`). It carries the looked-up `requestHash` and the cassette `path` so a failing CI run names exactly which call was not recorded. Extends `AIError` directly (not `ProviderError`) — a miss is a harness/config failure, not a provider failure. **The whole point:** `replay` never falls back to a live call, which would silently re-introduce non-determinism into a test that asked for the opposite. Re-record by running once in `record` / `auto`.
|
|
2806
|
+
|
|
2807
|
+
## `save()` — flush new entries
|
|
2808
|
+
|
|
2809
|
+
`save()` writes newly recorded entries to `path`. It is a **no-op when nothing was recorded** (pure replay, or a record/auto run that only hit cached entries), so calling it unconditionally is safe.
|
|
2810
|
+
|
|
2811
|
+
## Composition
|
|
2812
|
+
|
|
2813
|
+
VCR composes **below** `ai.fallbackModel` and works with any adapter. Wrap the live model in `vcr(...)`, then pass it anywhere a `ModelContract` is accepted (agent, planner, reranker, judge).
|
|
2814
|
+
|
|
2815
|
+
## See also
|
|
2816
|
+
|
|
2817
|
+
- [`@warlock.js/ai/eval-datasets-and-ci/SKILL.md`](@warlock.js/ai/eval-datasets-and-ci/SKILL.md) — pair a cassette with a dataset for fully offline eval CI
|
|
2818
|
+
- [`@warlock.js/ai/ai-dx-helpers/SKILL.md`](@warlock.js/ai/ai-dx-helpers/SKILL.md) — `ai.mockRouter` + Vitest matchers for the rest of the test surface
|
|
2819
|
+
- [`@warlock.js/ai/pick-ai-provider/SKILL.md`](@warlock.js/ai/pick-ai-provider/SKILL.md) — the adapters whose models VCR wraps
|
|
2820
|
+
|
|
2821
|
+
|
|
1729
2822
|
## run-ai-agent `@warlock.js/ai/run-ai-agent/SKILL.md`
|
|
1730
2823
|
|
|
1731
2824
|
---
|
|
1732
2825
|
name: run-ai-agent
|
|
1733
|
-
description: 'Build agents with ai.agent({...}) — the single-LLM-turn primitive. Covers execute / stream, attachments, structured output, placeholders, events, agent.eval scoring, and auto-adapting raw executables in tools:[]. Triggers: `ai.agent`, `agent.execute`, `agent.stream`, `agent.eval`, `AgentResult`, `AgentReport`, `AgentToolEntry`, `streamingToolGuard`, `attachments`, `repair`, `maxTrips`, `sessionId`; ''run an agent'', ''stream an agent response'', ''structured output schema'', ''pass image to agent'', ''evaluate an agent'', ''put a supervisor in tools'', ''cancel an agent run''; typical import `import { ai } from "@warlock.js/ai"`. Skip: tool definition — `@warlock.js/ai/define-ai-tool/SKILL.md`; workflows — `@warlock.js/ai/run-ai-workflow/SKILL.md`; eval matchers / batch / fallback detail — `@warlock.js/ai/ai-dx-helpers/SKILL.md`; competing libs `langchain`, `ai` (Vercel), raw `openai`.'
|
|
2826
|
+
description: 'Build agents with ai.agent({...}) — the single-LLM-turn primitive. Covers execute / stream, attachments, structured output, placeholders, events, agent.eval scoring, the judge-safe preset for resilient LLM-as-judge / verdict classifiers (ai.agent.judge / judge: true — lenient JSON parse + repair + never-throw, for Nova-class models), and auto-adapting raw executables in tools:[]. Triggers: `ai.agent`, `ai.agent.judge`, `agent.execute`, `agent.stream`, `agent.eval`, `AgentResult`, `AgentReport`, `AgentToolEntry`, `JudgeConfig`, `JudgeAgentConfig`, `judge`, `repairAttempts`, `streamingToolGuard`, `attachments`, `repair`, `maxTrips`, `sessionId`, `spawnSubAgent`, `SpawnSubAgentSpec`; ''run an agent'', ''stream an agent response'', ''structured output schema'', ''pass image to agent'', ''evaluate an agent'', ''LLM-as-judge that survives malformed JSON'', ''grade with a Nova model without crashing'', ''put a supervisor in tools'', ''cancel an agent run'', ''spawn a one-shot sub-agent with a per-task budget''; typical import `import { ai } from "@warlock.js/ai"`. Skip: tool definition — `@warlock.js/ai/define-ai-tool/SKILL.md`; workflows — `@warlock.js/ai/run-ai-workflow/SKILL.md`; eval matchers / batch / fallback detail — `@warlock.js/ai/ai-dx-helpers/SKILL.md`; competing libs `langchain`, `ai` (Vercel), raw `openai`.'
|
|
1734
2827
|
---
|
|
1735
2828
|
|
|
1736
2829
|
# `ai.agent()` — single-turn primitive
|
|
@@ -1911,7 +3004,32 @@ await myAgent.execute(input, {
|
|
|
1911
3004
|
});
|
|
1912
3005
|
```
|
|
1913
3006
|
|
|
1914
|
-
Disabled by default. Each repair attempt counts against `maxTrips`.
|
|
3007
|
+
Disabled by default. Each repair attempt counts against `maxTrips`.
|
|
3008
|
+
|
|
3009
|
+
## `judge` preset — resilient LLM-as-judge / verdict classifiers
|
|
3010
|
+
|
|
3011
|
+
For graders and verdict classifiers running on models that emit **corrupted** structured output — notably the Amazon Nova family, which wraps verdicts in fenced ` ```json ` blocks, prepends prose, or trails commentary — set `judge: true` (or a `JudgeConfig`). It turns on three behaviors at once:
|
|
3012
|
+
|
|
3013
|
+
```ts
|
|
3014
|
+
const grader = ai.agent.judge({
|
|
3015
|
+
model: nova.model({ name: "amazon.nova-pro-v1:0" }),
|
|
3016
|
+
systemPrompt: "Grade the answer. Respond with JSON only.",
|
|
3017
|
+
output: verdictSchema,
|
|
3018
|
+
});
|
|
3019
|
+
|
|
3020
|
+
const result = await grader.execute(prompt);
|
|
3021
|
+
if (result.error) {
|
|
3022
|
+
// graceful default — the judge couldn't produce a clean verdict
|
|
3023
|
+
}
|
|
3024
|
+
```
|
|
3025
|
+
|
|
3026
|
+
1. **Repair** — a couple of re-ask attempts by default (`repairAttempts`, defaults to `2`; bounded by `maxTrips`) when the verdict fails to parse / validate. The caller's per-call `options.repair` still wins.
|
|
3027
|
+
2. **Lenient verdict parsing** — extracts the first balanced JSON object / array (tolerating fenced blocks + surrounding prose) instead of the strict parser.
|
|
3028
|
+
3. **Never throws on a parse miss** — even an unparseable verdict yields a well-formed result (`result.error` populated, `result.data` undefined), so a flaky judge degrades instead of crashing the flow.
|
|
3029
|
+
|
|
3030
|
+
`ai.agent.judge(config, judge?)` is sugar for `ai.agent({ ...config, judge })`; the bare `ai.agent({ judge: true })` option does the same. `judge: {}` ≡ `judge: true` (every field falls back to its resilient default); `judge: { repairAttempts: 0 }` keeps the lenient parser + never-throw guarantee but disables repair.
|
|
3031
|
+
|
|
3032
|
+
**Trade-off — resilience over strictness.** The lenient parse can recover JSON the strict parser would (correctly) reject — leave `judge` **off** for normal structured output, where a hard parse failure is a useful signal. Off by default; omitting it parses strictly and never auto-enables repair, byte-for-byte as before. (This is the same Nova-safe judge the unified prompt `validate()` uses — see [`@warlock.js/ai/manage-prompts/SKILL.md`](@warlock.js/ai/manage-prompts/SKILL.md).)
|
|
1915
3033
|
|
|
1916
3034
|
## Pattern — image attachments
|
|
1917
3035
|
|
|
@@ -2016,6 +3134,24 @@ expect(report.passed).toBe(true); // true only when EVERY case passed
|
|
|
2016
3134
|
|
|
2017
3135
|
Each case runs through `execute(input)`; scorer precedence is per-case `scorers` → suite `scorers` → synthesized `judge` (throws at author time if a case resolves none). Built-in scorers on `ai.eval.*`: `exact()`, `contains()`, `predicate(fn)`, `judge(config)`. Full coverage — plus the Vitest matchers (`registerAiMatchers` / `toRouteTo` / `toConverge` / `toPassStep` / `toOutputShape`) — in [`@warlock.js/ai/ai-dx-helpers/SKILL.md`](@warlock.js/ai/ai-dx-helpers/SKILL.md).
|
|
2018
3136
|
|
|
3137
|
+
## `ai.spawnSubAgent()` — one-shot delegation with a budget
|
|
3138
|
+
|
|
3139
|
+
`ai.spawnSubAgent(spec)` is a thin wrapper over this same `ai.agent()`: it builds a fresh agent from the spec, optionally attaches a `budget` middleware, runs the `task` once, and returns the `AgentResult`. Not a sandbox or a separate runtime — a spawn is an ordinary new agent (empty conversation, its own tools/prompt). It is a **general** primitive: usable inside a tool, a workflow or planner step, a supervisor intent, or hand-rolled orchestration — it is NOT planner-specific (the planner engine never calls it).
|
|
3140
|
+
|
|
3141
|
+
```ts
|
|
3142
|
+
import { ai } from "@warlock.js/ai";
|
|
3143
|
+
|
|
3144
|
+
const result = await ai.spawnSubAgent({
|
|
3145
|
+
name: "extract-entities",
|
|
3146
|
+
model,
|
|
3147
|
+
task: "Pull every company name from this article: ...",
|
|
3148
|
+
budget: { maxCostUSD: 0.05 }, // per-task spend cap — aborts when crossed
|
|
3149
|
+
output: companiesSchema,
|
|
3150
|
+
});
|
|
3151
|
+
```
|
|
3152
|
+
|
|
3153
|
+
The one field a bare agent config doesn't surface ergonomically is `budget` (`BudgetOptions` — `maxTokens` / `maxCostUSD`), equivalent to `ai.agent({ middleware: [ai.middleware.budget(...)] })` but promoted to a first-class spec field so a delegated subtask can't overrun its cap (distinct from `maxTrips`, which caps round-trips, not spend). The surface is **narrower** than `agent.execute()`: one-shot, with no `history`, `placeholders`, per-call events, or `repair`. The spawned `report` slots under the caller's `report.children[]`, so cost and traces roll up uniformly. Reach for it when you want a named single-use delegation with a hard spend cap; otherwise just build an `ai.agent()` and call it.
|
|
3154
|
+
|
|
2019
3155
|
## When NOT to use this primitive
|
|
2020
3156
|
|
|
2021
3157
|
- Multi-step pipeline with a fixed shape → [`@warlock.js/ai/run-ai-workflow/SKILL.md`](@warlock.js/ai/run-ai-workflow/SKILL.md)
|
|
@@ -2028,6 +3164,260 @@ Each case runs through `execute(input)`; scorer precedence is per-case `scorers`
|
|
|
2028
3164
|
- [`@warlock.js/ai/handle-ai-errors/SKILL.md`](@warlock.js/ai/handle-ai-errors/SKILL.md) — `AIError` hierarchy
|
|
2029
3165
|
|
|
2030
3166
|
|
|
3167
|
+
## run-ai-rag `@warlock.js/ai/run-ai-rag/SKILL.md`
|
|
3168
|
+
|
|
3169
|
+
---
|
|
3170
|
+
name: run-ai-rag
|
|
3171
|
+
description: 'Retrieval-augmented generation with ai.rag({...}) — a chunk → embed → vector-store → retrieve → rerank → cite pipeline that reuses ai.embedder + a @warlock.js/cache CacheDriver. Covers index() / retrieve() / clear() / asTool(), chunking strategies (recursive | markdown | sentence | fixed), Citation / RetrievedChunk provenance, and the opt-in rerankers ai.rag.keywordReranker / ai.rag.llmReranker. Triggers: `ai.rag`, `rag.index`, `rag.retrieve`, `rag.clear`, `rag.asTool`, `RagConfig`, `RagDocument`, `RetrieveOptions`, `RetrieveResult`, `RetrievedChunk`, `Citation`, `ChunkOptions`, `ChunkType`, `ai.rag.keywordReranker`, `ai.rag.llmReranker`, `cacheVectorStore`, `VectorStore`, `topK`, `threshold`, `candidates`; ''build a knowledge base'', ''retrieve relevant chunks for a query'', ''cite the source of an answer'', ''chunk markdown for embedding'', ''rerank retrieval results'', ''expose retrieval as a tool''; typical import `import { ai } from "@warlock.js/ai"`. Skip: raw single-string embedding — `@warlock.js/ai/embed-text/SKILL.md`; exact + vector LLM-response cache — `@warlock.js/ai/attach-ai-middleware/SKILL.md` (ai.middleware.semanticCache); tool wiring — `@warlock.js/ai/define-ai-tool/SKILL.md`; competing libs `langchain`, `llamaindex`.'
|
|
3172
|
+
---
|
|
3173
|
+
|
|
3174
|
+
# `ai.rag()` — chunk → embed → retrieve → rerank → cite
|
|
3175
|
+
|
|
3176
|
+
A self-contained retrieval pipeline. It reuses the embedder you already have (`provider.embedder(...)`), a `@warlock.js/cache` vector-capable `CacheDriver` as the store, and the composite-as-tool engine for `asTool()`. Zero new dependencies. `ai.rag` is a native core verb — present the moment `@warlock.js/ai` is imported (no module augmentation, no side-effect import).
|
|
3177
|
+
|
|
3178
|
+
## Factory shape
|
|
3179
|
+
|
|
3180
|
+
```ts
|
|
3181
|
+
import { ai } from "@warlock.js/ai";
|
|
3182
|
+
import { MemoryCacheDriver } from "@warlock.js/cache";
|
|
3183
|
+
import { OpenAISDK } from "@warlock.js/ai-openai";
|
|
3184
|
+
|
|
3185
|
+
const openai = new OpenAISDK({ apiKey: process.env.OPENAI_API_KEY! });
|
|
3186
|
+
|
|
3187
|
+
const kb = ai.rag({
|
|
3188
|
+
name: "docs", // default "rag"
|
|
3189
|
+
embedder: openai.embedder({ name: "text-embedding-3-small" }), // REQUIRED
|
|
3190
|
+
store: new MemoryCacheDriver(), // or ai.config({ defaultStore })
|
|
3191
|
+
namespace: "ai.rag.docs", // default `ai.rag.<name>`
|
|
3192
|
+
chunk: { type: "markdown", size: 800, overlap: 120 }, // index() defaults
|
|
3193
|
+
reranker: ai.rag.keywordReranker(), // OFF by default (cosine-only)
|
|
3194
|
+
retrieve: { topK: 4, threshold: 0.5 }, // default retrieval knobs
|
|
3195
|
+
});
|
|
3196
|
+
```
|
|
3197
|
+
|
|
3198
|
+
Resolution is **loud at construction** (mirrors `ai.memory`):
|
|
3199
|
+
|
|
3200
|
+
- `embedder` is **required** — a provider with no embedder must be caught here, not at first `index()`.
|
|
3201
|
+
- `store` falls back to `ai.config({ defaultStore })`; if neither resolves, the factory throws.
|
|
3202
|
+
|
|
3203
|
+
## Surface — `Rag`
|
|
3204
|
+
|
|
3205
|
+
```ts
|
|
3206
|
+
interface Rag {
|
|
3207
|
+
readonly name: string;
|
|
3208
|
+
index(docs: RagDocument[], chunk?: ChunkOptions): Promise<{ chunks: number }>;
|
|
3209
|
+
retrieve(query: string, options?: RetrieveOptions): Promise<RetrieveResult>;
|
|
3210
|
+
clear(): Promise<void>;
|
|
3211
|
+
asTool(options?: RagAsToolOptions): ToolContract<{ query: string }, RetrieveResult>;
|
|
3212
|
+
}
|
|
3213
|
+
```
|
|
3214
|
+
|
|
3215
|
+
## `index()` — chunk, embed (batched), store
|
|
3216
|
+
|
|
3217
|
+
```ts
|
|
3218
|
+
await kb.index([
|
|
3219
|
+
{ id: "guide", text: longMarkdown, metadata: { url: "/guide" }, tags: ["frontend"] },
|
|
3220
|
+
{ id: "faq", text: faqText },
|
|
3221
|
+
]);
|
|
3222
|
+
```
|
|
3223
|
+
|
|
3224
|
+
A `RagDocument` is `{ id, text, metadata?, tags? }` — **you** load + parse documents to text (loaders are out of scope for v1). Each doc is split into chunks, embedded in sub-batches of 96 texts per `embedMany()` call (so one giant doc never blows the provider's per-request cap), and upserted. Returns the chunk count written. Empty / whitespace-only documents yield zero chunks — nothing is embedded. The per-call `chunk` arg overrides `config.chunk` for that index.
|
|
3225
|
+
|
|
3226
|
+
## Chunking — `ChunkOptions`
|
|
3227
|
+
|
|
3228
|
+
All sizing is in **characters** (tokenizer-free; the embedder owns token counting). `chunk(text, options)` is also exported standalone.
|
|
3229
|
+
|
|
3230
|
+
```ts
|
|
3231
|
+
type ChunkType = "recursive" | "sentence" | "fixed" | "markdown";
|
|
3232
|
+
|
|
3233
|
+
{
|
|
3234
|
+
type?: ChunkType, // default "recursive"
|
|
3235
|
+
size?: number, // target chars per chunk, default 1000
|
|
3236
|
+
overlap?: number, // chars carried between adjacent chunks, default 200
|
|
3237
|
+
separators?: string[], // recursive only; default ["\n\n", "\n", ". ", " ", ""]
|
|
3238
|
+
}
|
|
3239
|
+
```
|
|
3240
|
+
|
|
3241
|
+
- **`recursive`** (default) — separator-aware greedy packing, largest unit first.
|
|
3242
|
+
- **`markdown`** — heading/section-aware, then recursive within each section.
|
|
3243
|
+
- **`sentence`** — packs whole sentences up to `size`.
|
|
3244
|
+
- **`fixed`** — back-to-back character windows.
|
|
3245
|
+
|
|
3246
|
+
Every chunk records its exact `[start, end)` span in the original text, so a `Citation.span` is precise.
|
|
3247
|
+
|
|
3248
|
+
## `retrieve()` — embed query, fetch, rerank, slice, cite
|
|
3249
|
+
|
|
3250
|
+
```ts
|
|
3251
|
+
const { query, chunks } = await kb.retrieve("how do I configure caching?", {
|
|
3252
|
+
topK: 4, // returned AFTER reranking. default 5
|
|
3253
|
+
threshold: 0.5, // cosine floor at the store stage. default 0.5
|
|
3254
|
+
candidates: 16, // pool fetched before rerank. default topK * 4 (clamped >= topK)
|
|
3255
|
+
tags: ["frontend"], // restrict to chunks whose source had one of these tags
|
|
3256
|
+
});
|
|
3257
|
+
|
|
3258
|
+
for (const hit of chunks) {
|
|
3259
|
+
console.log(hit.score, hit.text);
|
|
3260
|
+
console.log(hit.citation.sourceId, hit.citation.chunkIndex, hit.citation.span);
|
|
3261
|
+
}
|
|
3262
|
+
```
|
|
3263
|
+
|
|
3264
|
+
`retrieve()` is **return-only** — it never auto-injects into a prompt. The caller formats the cited chunks (or uses `asTool()` for the agent loop). A `RetrievedChunk` carries `{ text, score, citation }`; the `Citation` is `{ sourceId, chunkIndex, span, score, metadata? }`. The reranker is **OFF by default** (cosine ranking only) unless `config.reranker` is set.
|
|
3265
|
+
|
|
3266
|
+
## Rerankers — opt-in, on `ai.rag.*`
|
|
3267
|
+
|
|
3268
|
+
Both are exposed as namespaced helpers on the factory (`ai.rag.keywordReranker`, `ai.rag.llmReranker`).
|
|
3269
|
+
|
|
3270
|
+
```ts
|
|
3271
|
+
// Zero-dependency lexical reranker (BM25-lite keyword overlap).
|
|
3272
|
+
ai.rag.keywordReranker({ weight: 0.5 }); // weight in [0,1]; 1 = pure keyword, 0 = keep cosine
|
|
3273
|
+
|
|
3274
|
+
// Model-backed reranker — one or more model calls per retrieval.
|
|
3275
|
+
ai.rag.llmReranker({ model: openai.model({ name: "gpt-4o-mini" }), batchSize: 10 });
|
|
3276
|
+
```
|
|
3277
|
+
|
|
3278
|
+
- **`keywordReranker`** — blends lexical query-term overlap with the original cosine score by `weight`; ties keep the incoming cosine order. Costs nothing beyond string splits. Reach for it when embedding-only ranking buries a keyword-rich chunk.
|
|
3279
|
+
- **`llmReranker`** — asks an LLM to grade each over-fetched candidate `0..1` and sorts by that. Candidates the model fails to score keep their cosine score, so a garbled reply degrades gracefully. Opt in only when precision beats latency/cost. Both implement the `RagReranker` contract, so you can write your own.
|
|
3280
|
+
|
|
3281
|
+
## `asTool()` — drop retrieval into an agent's `tools: []`
|
|
3282
|
+
|
|
3283
|
+
```ts
|
|
3284
|
+
const agent = ai.agent({
|
|
3285
|
+
model: openai.model({ name: "gpt-4o" }),
|
|
3286
|
+
tools: [kb.asTool({ name: "search_docs", retrieve: { topK: 6 } })],
|
|
3287
|
+
});
|
|
3288
|
+
```
|
|
3289
|
+
|
|
3290
|
+
Input is `{ query: string }`; output is the `RetrieveResult`. Default tool name is `retrieve_<rag.name>`; `description` and a per-tool `retrieve` override are optional. Built via the same composite-as-tool engine every other primitive uses.
|
|
3291
|
+
|
|
3292
|
+
## `clear()`
|
|
3293
|
+
|
|
3294
|
+
```ts
|
|
3295
|
+
await kb.clear(); // drops every entry written under this rag's namespace
|
|
3296
|
+
```
|
|
3297
|
+
|
|
3298
|
+
## Advanced
|
|
3299
|
+
|
|
3300
|
+
- `cacheVectorStore(driver)` + the `VectorStore` contract are exported for swapping in a custom store.
|
|
3301
|
+
- A stored chunk's namespaced key is `${namespace}.${sourceId}.${chunkIndex}`.
|
|
3302
|
+
|
|
3303
|
+
## See also
|
|
3304
|
+
|
|
3305
|
+
- [`@warlock.js/ai/embed-text/SKILL.md`](@warlock.js/ai/embed-text/SKILL.md) — the `sdk.embedder` primitive this consumes
|
|
3306
|
+
- [`@warlock.js/ai/define-ai-tool/SKILL.md`](@warlock.js/ai/define-ai-tool/SKILL.md) — what `asTool()` produces
|
|
3307
|
+
- [`@warlock.js/ai/run-ai-agent/SKILL.md`](@warlock.js/ai/run-ai-agent/SKILL.md) — wiring the retrieval tool into an agent
|
|
3308
|
+
|
|
3309
|
+
|
|
3310
|
+
## run-ai-team `@warlock.js/ai/run-ai-team/SKILL.md`
|
|
3311
|
+
|
|
3312
|
+
---
|
|
3313
|
+
name: run-ai-team
|
|
3314
|
+
description: 'Manager-led multi-agent teams with ai.team({...}) — transparent sugar over ai.supervisor that maps a manager → route/router, members → intents, and a gate → evaluate, returning a REAL SupervisorContract (no new loop, no new contract). Covers the built-in gate strings "quality" (review-then-fix) and "verify" (test-then-fix), a custom gate function, role mapping (roles / gateKey), and the verbatim supervisor pass-throughs (goal / output / state / maxIterations / snapshotStore / on / observe). Triggers: `ai.team`, `TeamConfig`, `TeamGate`, `TeamGateFn`, `TeamMemberValue`, `manager`, `members`, `gate`, `roles`, `gateKey`, `buildQualityGate`, `buildVerifyGate`, `SupervisorContract`, `ReportType`; ''build a team of agents'', ''manager that delegates to members'', ''review then fix loop'', ''test then fix loop'', ''quality gate for a multi-agent run'', ''report type team''; typical import `import { ai } from "@warlock.js/ai"`. Skip: routing one input to a fixed roster directly — `@warlock.js/ai/run-supervisor/SKILL.md` (team is sugar over it); durable cross-turn sessions — `@warlock.js/ai/run-orchestrator/SKILL.md`; LLM-generated plans — `@warlock.js/ai/run-planner/SKILL.md`; competing libs `crewai`, `autogen`.'
|
|
3315
|
+
---
|
|
3316
|
+
|
|
3317
|
+
# `ai.team()` — manager + members + a quality gate
|
|
3318
|
+
|
|
3319
|
+
`ai.team(config)` is **thin, transparent sugar over `ai.supervisor`**. It builds a `SupervisorConfig` from the team-shaped config, calls `supervisor(...)`, and returns the **unchanged** `SupervisorContract<TOutput>` — the exact object `ai.supervisor` returns. So `ctx.intents.<member>.execute()`, `.asTool()`, `.resume()`, snapshots, and events all stay intact. `team()` owns no loop of its own.
|
|
3320
|
+
|
|
3321
|
+
The mapping:
|
|
3322
|
+
|
|
3323
|
+
| team field | becomes supervisor field |
|
|
3324
|
+
| --- | --- |
|
|
3325
|
+
| `manager` | `route` (deterministic `{ route }`) XOR `router` (an agent / `RouterEntry`) |
|
|
3326
|
+
| `members` | `intents` |
|
|
3327
|
+
| `gate` | `evaluate` |
|
|
3328
|
+
|
|
3329
|
+
Everything else passes through 1:1 — the sole exception is the report/result `type`, which is stamped `"team"` (see [Pass-throughs](#pass-throughs-verbatim-supervisor-semantics)).
|
|
3330
|
+
|
|
3331
|
+
## Shape
|
|
3332
|
+
|
|
3333
|
+
```ts
|
|
3334
|
+
import { ai } from "@warlock.js/ai";
|
|
3335
|
+
import { v } from "@warlock.js/seal";
|
|
3336
|
+
|
|
3337
|
+
const codeTeam = ai.team({
|
|
3338
|
+
name: "code-team",
|
|
3339
|
+
goal: "Ship a tested module that passes review.",
|
|
3340
|
+
manager: techLeadRouter, // an agent / RouterEntry → router; or { route } → deterministic
|
|
3341
|
+
members: { builder, reviewer, fixer }, // role-name → agent | workflow
|
|
3342
|
+
gate: "quality", // "quality" | "verify" | (ctx) => EvaluateResult
|
|
3343
|
+
output: v.object({ code: v.string() }),
|
|
3344
|
+
maxIterations: 6, // default 10 (supervisor's)
|
|
3345
|
+
});
|
|
3346
|
+
|
|
3347
|
+
const { data, report } = await codeTeam.execute("Build a debounce<T> utility.");
|
|
3348
|
+
```
|
|
3349
|
+
|
|
3350
|
+
A `member` is an `AgentContract` or a `WorkflowInstance` (the `TeamMemberValue` union — the autocomplete-friendly common case; callback / full-entry intent shapes still work when forwarded). The keys are both the role names the manager routes to AND the keys `ctx.intents.<role>` exposes (the supervisor escape hatch is preserved).
|
|
3351
|
+
|
|
3352
|
+
## The manager — `route` XOR `router`
|
|
3353
|
+
|
|
3354
|
+
```ts
|
|
3355
|
+
// LLM-driven manager: an agent (or RouterEntry) → becomes SupervisorConfig.router
|
|
3356
|
+
manager: techLeadRouter
|
|
3357
|
+
|
|
3358
|
+
// Deterministic manager: { route } → becomes SupervisorConfig.route
|
|
3359
|
+
manager: { route: (ctx) => (ctx.iteration === 0 ? "builder" : "reviewer") }
|
|
3360
|
+
```
|
|
3361
|
+
|
|
3362
|
+
Exactly one form is forwarded — mutually exclusive, mirroring the supervisor's own `router` XOR `route` rule. A malformed manager surfaces the existing `SupervisorFailedError` downstream.
|
|
3363
|
+
|
|
3364
|
+
## Gates — `"quality"` | `"verify"` | a function
|
|
3365
|
+
|
|
3366
|
+
A `gate` string selects a pre-built `evaluate` strategy; both desugar to a concrete `evaluate` callback that leans entirely on the already-shipped `EvaluateResult` semantics (`satisfied` terminates, `reassignTo` re-dispatches the fixer, `feedback` threads forward) — **no new termination or loop code**.
|
|
3367
|
+
|
|
3368
|
+
### `gate: "quality"` — review-then-fix
|
|
3369
|
+
|
|
3370
|
+
After each iteration's members settle and merge into supervisor `state`, the gate reads `state.approved` (the `gateKey`, default `"approved"`). If truthy → `{ satisfied: true }`; otherwise → `{ reassignTo: "fixer", feedback: String(state.notes ?? "") }`. The reviewer's feedback (`state.notes`) threads into the next iteration.
|
|
3371
|
+
|
|
3372
|
+
### `gate: "verify"` — test-then-fix
|
|
3373
|
+
|
|
3374
|
+
Identical shape but keyed on the tester's pass/fail slice `state.passed` (default `gateKey`) rather than a subjective score. On failure it re-dispatches the fixer; there is no feedback channel for a pass/fail signal, so none is threaded.
|
|
3375
|
+
|
|
3376
|
+
> The named member whose `output` schema writes the gate slice must produce a boolean into `gateKey`.
|
|
3377
|
+
|
|
3378
|
+
### A custom gate (full escape hatch)
|
|
3379
|
+
|
|
3380
|
+
```ts
|
|
3381
|
+
gate: (ctx) => {
|
|
3382
|
+
if (ctx.state.score >= 0.9) return { satisfied: true };
|
|
3383
|
+
return { reassignTo: "fixer", feedback: ctx.state.review };
|
|
3384
|
+
}
|
|
3385
|
+
```
|
|
3386
|
+
|
|
3387
|
+
Supplying a `TeamGateFn` instead of a string opts out of the sugar entirely while keeping the rest of `team()`'s wiring — it forwards straight to `SupervisorConfig.evaluate` with zero wrapping.
|
|
3388
|
+
|
|
3389
|
+
## Role mapping — `roles` + `gateKey`
|
|
3390
|
+
|
|
3391
|
+
The string gates default to canonical role names. Override when your `members` keys differ:
|
|
3392
|
+
|
|
3393
|
+
```ts
|
|
3394
|
+
ai.team({
|
|
3395
|
+
name: "qa-team",
|
|
3396
|
+
manager,
|
|
3397
|
+
members: { author, critic, patcher },
|
|
3398
|
+
gate: "quality",
|
|
3399
|
+
roles: { reviewer: "critic", fixer: "patcher" }, // map gate roles → your member keys
|
|
3400
|
+
gateKey: "ok", // state slice the gate reads
|
|
3401
|
+
});
|
|
3402
|
+
```
|
|
3403
|
+
|
|
3404
|
+
**Construction-time validation:** when the gate is a string, the resolved `fixer` (and, for `"quality"`, the `reviewer`) role is checked against `members`. A missing role throws an authoring-style `SupervisorFailedError` (`context: { authoring: true }`) immediately — rather than silently starving until `maxIterations`.
|
|
3405
|
+
|
|
3406
|
+
## Pass-throughs (verbatim supervisor semantics)
|
|
3407
|
+
|
|
3408
|
+
`goal`, `output`, `state`, `maxIterations`, `snapshotStore`, `on`, `observe`, and `version` are forwarded unchanged. Because the returned object IS a supervisor, observability rides the same generic `Observer` seam every other flow uses (see `observe-ai-flows`), and snapshot resume works exactly as on a bare supervisor.
|
|
3409
|
+
|
|
3410
|
+
The one behavioural difference from a bare supervisor: a team stamps **`type: "team"`** on both its report (a first-class `ReportType`, was `"supervisor"`) and its result, so Panoptic and any `Observer` can distinguish, group, filter, and label team runs as their own type rather than folding them into plain supervisor runs. Everything else passes through 1:1.
|
|
3411
|
+
|
|
3412
|
+
A member callback that calls `agent.execute()` **directly** still nests `member → agent → tool` under the member span with usage rolled up — the same ambient-`RunFrame` auto-nesting as a bare supervisor. See [`@warlock.js/ai/run-supervisor/SKILL.md`](@warlock.js/ai/run-supervisor/SKILL.md).
|
|
3413
|
+
|
|
3414
|
+
## See also
|
|
3415
|
+
|
|
3416
|
+
- [`@warlock.js/ai/run-supervisor/SKILL.md`](@warlock.js/ai/run-supervisor/SKILL.md) — the primitive team desugars into (intents, route/router, evaluate, ctx.intents)
|
|
3417
|
+
- [`@warlock.js/ai/run-orchestrator/SKILL.md`](@warlock.js/ai/run-orchestrator/SKILL.md) — wrap a team in durable cross-turn session state
|
|
3418
|
+
- [`@warlock.js/ai/observe-ai-flows/SKILL.md`](@warlock.js/ai/observe-ai-flows/SKILL.md) — the `observe` seam a team inherits
|
|
3419
|
+
|
|
3420
|
+
|
|
2031
3421
|
## run-ai-workflow `@warlock.js/ai/run-ai-workflow/SKILL.md`
|
|
2032
3422
|
|
|
2033
3423
|
---
|
|
@@ -2346,7 +3736,7 @@ await supportBot.execute(input, {
|
|
|
2346
3736
|
state: { tier: "gold" }, // partial patch shallow-merged into loaded state
|
|
2347
3737
|
context: { userId, db }, // request-scoped, never persisted
|
|
2348
3738
|
signal: AbortSignal.timeout(60_000),
|
|
2349
|
-
on: { "orchestrator.turn.
|
|
3739
|
+
on: { "orchestrator.turn.awaiting-input": (e) => log(e) }, // tier-3 per-call handlers
|
|
2350
3740
|
force: false, // bypass drift check for this call
|
|
2351
3741
|
});
|
|
2352
3742
|
```
|
|
@@ -2361,7 +3751,7 @@ await supportBot.execute(input, {
|
|
|
2361
3751
|
6. **persist** — append a checkpoint row for the settled turn, then prune to `keepSnapshots` (`orchestrator.checkpoint.persisted`).
|
|
2362
3752
|
7. **compaction** — fire the post-turn compaction trigger if configured (`orchestrator.compaction.suggested` / `.applied`).
|
|
2363
3753
|
|
|
2364
|
-
|
|
3754
|
+
A clean turn ends with `orchestrator.turn.awaiting-input` (the session stays open for the next user turn); `orchestrator.turn.failed` and `orchestrator.turn.cancelled` end error / cancelled turns. (`orchestrator.turn.completed` is defined on the event map, but the v1 lifecycle maps a clean completion to `awaiting-input`, so it isn't emitted on the normal path — subscribe to `awaiting-input` for "turn done".)
|
|
2365
3755
|
|
|
2366
3756
|
## `OrchestratorResult` — read the report
|
|
2367
3757
|
|
|
@@ -2380,6 +3770,8 @@ result.compaction; // CompactionResult when a turn compacted (and
|
|
|
2380
3770
|
|
|
2381
3771
|
`report.children[]` carries ONLY the current turn's dispatched primitive reports. Full session history lives on `report.turns[]` — a `children[]` walker will NOT reach prior turns (intentional). Child `supervisor.*` / `agent.*` events bubble up unmodified under their own identity.
|
|
2382
3772
|
|
|
3773
|
+
A turn callback that calls `agent.execute()` **directly** (not via `ctx.run` / `ctx.intents`) still nests `callback → agent → tool` inside the turn's report tree, with usage rolled up and the session's `sessionId` stamped onto the captured subtree — an ambient `RunFrame` handles the self-attach. See [`@warlock.js/ai/run-supervisor/SKILL.md`](@warlock.js/ai/run-supervisor/SKILL.md).
|
|
3774
|
+
|
|
2383
3775
|
**`awaiting-input` is the only non-terminal status across the unified result tree.** Code branching on `status === "completed"` MUST explicitly handle `"awaiting-input"` as a session-continues path, not a failure.
|
|
2384
3776
|
|
|
2385
3777
|
## `iterate` — single dispatch vs. internal supervisor
|
|
@@ -2496,7 +3888,7 @@ await orch.execute(input, { sessionId, history, on: { "orchestrator.drift.checke
|
|
|
2496
3888
|
|
|
2497
3889
|
---
|
|
2498
3890
|
name: run-planner
|
|
2499
|
-
description: 'Goal-driven planning with ai.planner({...}) — an LLM GENERATES an ordered execution plan over your registered capabilities (agents / workflows / supervisors / tools), then the planner EXECUTES
|
|
3891
|
+
description: 'Goal-driven planning with ai.planner({...}) — an LLM GENERATES an ordered execution plan over your registered capabilities (agents / workflows / supervisors / tools), then the planner EXECUTES it, threading each step output into the next, and returns the unified {data, report, usage, error} envelope with report.type "planner". Supports DAG scheduling (dag:true + maxConcurrency off dependsOn), adaptive re-planning (replan:{maxReplans} + the onStep continue/abort/replan directive), and plan-only / approval (mode:"plan-only" → status "awaiting-approval" → approvedPlan). A plan step may delegate via ai.spawnSubAgent({...}) — a GENERAL one-shot-agent helper covered in `@warlock.js/ai/run-ai-agent/SKILL.md`; it is not planner-specific. Triggers: `ai.planner`, `planner.execute`, `spawnSubAgent`, `PlannerConfig`, `PlannerCapability`, `PlannerResult`, `PlannerReport`, `PlannerPlan`, `PlannerStep`, `PlannerStepDirective`, `PlannerPlanInvalidError`, `maxSteps`, `dag`, `maxConcurrency`, `dependsOn`, `replan`, `onStep`, `mode`, `approvedPlan`, `awaiting-approval`, `report.plan`, `report.executedSteps`; ''let the model plan the steps'', ''dynamic plan from a goal'', ''run independent steps in parallel'', ''re-plan when a step fails'', ''generate a plan for approval before running it''; typical import `import { ai } from "@warlock.js/ai"`. Skip: a FIXED known pipeline — `@warlock.js/ai/run-ai-workflow/SKILL.md`; routing one input to a specialist each turn — `@warlock.js/ai/run-supervisor/SKILL.md`; a single model + tools call — `@warlock.js/ai/run-ai-agent/SKILL.md`; competing libs `langgraph`, `crewai`.'
|
|
2500
3892
|
---
|
|
2501
3893
|
|
|
2502
3894
|
# `ai.planner()` — LLM-generated, then executed, plans
|
|
@@ -2508,16 +3900,19 @@ A planner turns a free-form **goal** into an ordered **plan** the LLM writes its
|
|
|
2508
3900
|
- **`agent`** — one model + tools, single task. No multi-step decomposition.
|
|
2509
3901
|
- **`workflow`** — a FIXED pipeline you author by hand (`steps: [...]`). The steps are known at design time.
|
|
2510
3902
|
- **`supervisor`** — routes one input to the right specialist each turn; loops on a quality verdict.
|
|
2511
|
-
- **`planner`** — the steps are NOT known in advance. The LLM generates the ordered plan from the goal, then the planner executes it.
|
|
3903
|
+
- **`planner`** — the steps are NOT known in advance. The LLM generates the ordered plan from the goal, then the planner executes it. Sequential by default; opt into **DAG** scheduling, **adaptive re-planning**, and **plan-only / approval** as needed (below).
|
|
2512
3904
|
|
|
2513
3905
|
## Shape
|
|
2514
3906
|
|
|
2515
3907
|
```ts
|
|
2516
3908
|
import { ai } from "@warlock.js/ai";
|
|
3909
|
+
import { OpenAISDK } from "@warlock.js/ai-openai";
|
|
3910
|
+
|
|
3911
|
+
const openai = new OpenAISDK({ apiKey: process.env.OPENAI_API_KEY! });
|
|
2517
3912
|
|
|
2518
3913
|
const research = ai.planner({
|
|
2519
3914
|
name: "research-assistant",
|
|
2520
|
-
model:
|
|
3915
|
+
model: openai.model({ name: "gpt-4o" }), // the plan-GENERATION brain
|
|
2521
3916
|
capabilities: [
|
|
2522
3917
|
{ name: "search", description: "Search the web for sources", executable: searchAgent },
|
|
2523
3918
|
{ name: "summarize", description: "Summarize text into bullet points", executable: summarizer },
|
|
@@ -2538,19 +3933,81 @@ for (const step of report.executedSteps) { // forensic, in execution order
|
|
|
2538
3933
|
- `planner` lets you bring your own fully-configured planning agent (custom prompt, middleware). The planner injects the plan schema as that agent's per-call `output`.
|
|
2539
3934
|
- A `capability` is `{ name, description, executable }`. The `name` is what the LLM references per step; the `description` is what it reads to pick. `executable` is any `ExecutableContract` (agent / workflow / supervisor / tool).
|
|
2540
3935
|
|
|
2541
|
-
## Execution model
|
|
3936
|
+
## Execution model
|
|
2542
3937
|
|
|
2543
|
-
1. **Generate** — the planning agent is asked for a `{ steps, summary? }` plan via a generated schema whose `capability` field is an `enum` of your capability names.
|
|
2544
|
-
2. **Execute** — steps run **strictly in array order
|
|
3938
|
+
1. **Generate** — the planning agent is asked for a `{ steps, summary? }` plan via a generated schema whose `capability` field is an `enum` of your capability names. Each `PlannerStep` is `{ capability, input, id?, reason?, dependsOn? }`.
|
|
3939
|
+
2. **Execute** — by default steps run **strictly in array order**; each completed step's output is threaded into the next step's input as "Context from earlier steps". (Set `dag: true` to schedule on `dependsOn` instead — below.)
|
|
2545
3940
|
3. **Finalize** — when `output` is set (factory or per-call), the LAST completed step's structured output is validated into `result.data`. A capability that should feed typed output to the planner's `output` should declare its own `output` schema (the planner reads `data`, falling back to an agent's raw `text`).
|
|
2546
3941
|
|
|
2547
|
-
`report.type === "planner"`; `report.children[]` carries every dispatched capability report (plus the planning trip), with usage rolled up. Lazy capability loading is **deferred** — every capability is fully constructed up front.
|
|
3942
|
+
`report.type === "planner"`; `report.children[]` carries every dispatched capability report (plus the planning trip), with usage rolled up. `report.executedSteps` is the authoritative per-step record (`PlannerStepSnapshot[]`). Lazy capability loading is **deferred** — every capability is fully constructed up front.
|
|
3943
|
+
|
|
3944
|
+
## DAG scheduling — `dag: true` + `maxConcurrency`
|
|
3945
|
+
|
|
3946
|
+
Run independent steps in parallel instead of array-order:
|
|
3947
|
+
|
|
3948
|
+
```ts
|
|
3949
|
+
const research = ai.planner({
|
|
3950
|
+
name: "research",
|
|
3951
|
+
model,
|
|
3952
|
+
capabilities,
|
|
3953
|
+
dag: true, // build a DAG from each step's `id` / `dependsOn`
|
|
3954
|
+
maxConcurrency: 4, // max steps in flight at once. default 4
|
|
3955
|
+
});
|
|
3956
|
+
```
|
|
3957
|
+
|
|
3958
|
+
With `dag: true` the planner builds a DAG from step `id` / `dependsOn`, runs each **ready level concurrently** (up to `maxConcurrency`), and feeds each step **only its dependencies' outputs** (not the whole prior transcript). A **cycle** or a `dependsOn` naming an unknown step raises a typed `PlannerPlanInvalidError` **before any step runs**. Default `false` ⇒ the strict array-order loop, byte-for-byte unchanged (where `dependsOn` is advisory-only metadata).
|
|
3959
|
+
|
|
3960
|
+
## Adaptive re-planning — `replan: { maxReplans }` + `onStep`
|
|
3961
|
+
|
|
3962
|
+
When set, a **failed step** (or a `replan` verdict from the `onStep` hook) **revises the REMAINING plan** instead of aborting — re-asking the planning agent for a fresh plan seeded with the executed-step digest plus the feedback. Bounded by `maxReplans`; on exhaustion the run ends with the last failure.
|
|
3963
|
+
|
|
3964
|
+
```ts
|
|
3965
|
+
const planner = ai.planner({
|
|
3966
|
+
name: "adaptive",
|
|
3967
|
+
model,
|
|
3968
|
+
capabilities,
|
|
3969
|
+
replan: { maxReplans: 2 },
|
|
3970
|
+
});
|
|
3971
|
+
|
|
3972
|
+
await planner.execute(goal, {
|
|
3973
|
+
onStep: (snapshot, plan) => {
|
|
3974
|
+
// fired after EACH step settles (both the sequential and the DAG path)
|
|
3975
|
+
if (snapshot.status === "completed" && looksWrong(snapshot.output)) {
|
|
3976
|
+
return { type: "replan", feedback: "The summary missed the pricing section." };
|
|
3977
|
+
}
|
|
3978
|
+
// return nothing / { type: "continue" } to proceed; { type: "abort" } to stop
|
|
3979
|
+
},
|
|
3980
|
+
});
|
|
3981
|
+
```
|
|
3982
|
+
|
|
3983
|
+
The `onStep` directive (`PlannerStepDirective`):
|
|
3984
|
+
|
|
3985
|
+
- `{ type: "continue" }` (or returning nothing) — proceed.
|
|
3986
|
+
- `{ type: "abort" }` — stop; remaining steps recorded `skipped` (exactly as a failure aborts).
|
|
3987
|
+
- `{ type: "replan"; feedback }` — re-plan the remainder, seeded with the digest + `feedback`. **A `replan` directive with no `replan` config is treated as `continue`** (no-op). Default off ⇒ a failure aborts exactly as before.
|
|
3988
|
+
|
|
3989
|
+
## Plan-only / approval — `mode: "plan-only"` + `approvedPlan`
|
|
3990
|
+
|
|
3991
|
+
Generate (and validate) a plan, return it for human sign-off, then execute the approved plan in a follow-up call:
|
|
3992
|
+
|
|
3993
|
+
```ts
|
|
3994
|
+
// 1. Generate WITHOUT executing.
|
|
3995
|
+
const draft = await planner.execute(goal, { mode: "plan-only" });
|
|
3996
|
+
// draft.report.status === "awaiting-approval"; draft.plan carries the generated PlannerPlan.
|
|
3997
|
+
|
|
3998
|
+
// 2. (human reviews draft.plan) ... then execute it verbatim.
|
|
3999
|
+
const final = await planner.execute(goal, { approvedPlan: draft.plan! });
|
|
4000
|
+
```
|
|
4001
|
+
|
|
4002
|
+
- `mode: "plan-only"` generates + validates the plan and returns **without executing** — `report.status === "awaiting-approval"` (a planner-specific NON-terminal status) and `result.plan` carries the generated plan.
|
|
4003
|
+
- `approvedPlan` executes that exact plan, **skipping plan generation entirely**. It is still validated against the **live** capabilities, so a stale plan naming a capability the planner no longer has surfaces a `PlannerPlanInvalidError`.
|
|
4004
|
+
- `mode: "plan-only"` **with** `approvedPlan` is contradictory — `approvedPlan` wins (the plan executes).
|
|
2548
4005
|
|
|
2549
4006
|
## Failure + cancellation
|
|
2550
4007
|
|
|
2551
4008
|
`execute()` never throws — failures surface on `result.error`:
|
|
2552
4009
|
|
|
2553
|
-
- **`PlannerPlanInvalidError`** (`PLANNER_PLAN_INVALID`, category `schema`) — empty plan
|
|
4010
|
+
- **`PlannerPlanInvalidError`** (`PLANNER_PLAN_INVALID`, category `schema`) — empty plan, a step naming an unknown capability, a DAG cycle, a `dependsOn` naming an unknown step, a stale `approvedPlan`, or a final-output validation failure.
|
|
2554
4011
|
- **`PlannerCancelledError`** (`PLANNER_CANCELLED`, category `cancelled`) — the `AbortSignal` fired. `report.status === "cancelled"`, `report.cancelledAt` set; remaining steps are `skipped`.
|
|
2555
4012
|
- A child capability's own error (agent / tool / provider) flows through unchanged on the failing step's snapshot and as `result.error`. The planner stops at the first failed step and marks the rest `skipped`.
|
|
2556
4013
|
- **`PlannerFailedError`** is the base for the `PLANNER_*` family.
|
|
@@ -2568,7 +4025,7 @@ Use `MockSDK` for the planning model — script the plan as a JSON string matchi
|
|
|
2568
4025
|
|
|
2569
4026
|
---
|
|
2570
4027
|
name: run-supervisor
|
|
2571
|
-
description: 'Multi-intent routing with ai.supervisor({...}) — classifier (iter-0 dispatch), router agent OR route callback, intents as agents / workflows / callbacks, fan-out, evaluate quality loop, ack receptionist, supervisor-level middleware. Triggers: `ai.supervisor`, `ai.router`, `ai.fanOut`, `supervisor.execute`, `supervisor.resume`, `intents`, `router`, `route`, `classifier`, `evaluate`, `ack`, `artifactsSchema`, `middleware`, `END`, `ctx.intents.X.execute`; ''route one input across specialists'', ''multi-intent dispatch'', ''fan-out then evaluate'', ''classifier then router'', ''supervisor middleware'', ''self-consistency / voting''; typical import `import { ai } from "@warlock.js/ai"`. Skip: durable multi-turn sessions — `@warlock.js/ai/run-orchestrator/SKILL.md`; fixed pipelines — `@warlock.js/ai/run-ai-workflow/SKILL.md`; single agent — `@warlock.js/ai/run-ai-agent/SKILL.md`; competing libs `langgraph`, `crewai`.'
|
|
4028
|
+
description: 'Multi-intent routing with ai.supervisor({...}) — classifier (iter-0 dispatch), router agent OR route callback, intents as agents / workflows / callbacks, fan-out, evaluate quality loop, ack receptionist, supervisor-level middleware. A callback that calls agent.execute() directly auto-nests agent → tool under the callback span (ambient RunFrame) with usage / cost rolled up — same for team members and orchestrator turns. Triggers: `ai.supervisor`, `ai.router`, `ai.fanOut`, `supervisor.execute`, `supervisor.resume`, `intents`, `router`, `route`, `classifier`, `evaluate`, `ack`, `artifactsSchema`, `middleware`, `END`, `ctx.intents.X.execute`, `ctx.run`, `RunFrame`, `callback span`, `children`, `parentRunId`, `rootRunId`, `trace nesting`, `sub-agent`; ''route one input across specialists'', ''multi-intent dispatch'', ''fan-out then evaluate'', ''classifier then router'', ''supervisor middleware'', ''self-consistency / voting'', ''why is my callback agent not nested / cost is $0'', ''nest a sub-agent under a callback''; typical import `import { ai } from "@warlock.js/ai"`. Skip: durable multi-turn sessions — `@warlock.js/ai/run-orchestrator/SKILL.md`; fixed pipelines — `@warlock.js/ai/run-ai-workflow/SKILL.md`; single agent — `@warlock.js/ai/run-ai-agent/SKILL.md`; competing libs `langgraph`, `crewai`.'
|
|
2572
4029
|
---
|
|
2573
4030
|
|
|
2574
4031
|
# `ai.supervisor()` — multi-intent routing
|
|
@@ -2835,6 +4292,25 @@ intents: {
|
|
|
2835
4292
|
|
|
2836
4293
|
Cycle protection: per-branch call stack. Re-entry on same intent → `SUPERVISOR_DISPATCH_CYCLE`.
|
|
2837
4294
|
|
|
4295
|
+
### Sub-agent trace nesting — `agent.execute()` inside a callback auto-nests
|
|
4296
|
+
|
|
4297
|
+
A callback that calls `agent.execute()` (or `team` member / `orchestrator` turn callback) **directly** — not through `ctx.run(agent)` / `ctx.intents.X.execute()` — still nests under its enclosing span. An ambient async-local `RunFrame` lets the agent self-attach to the callback's `children[]`, so the report tree is `callback → agent → tool` with usage / cost **rolled up** (no `$0` lone callback span, no manual id threading):
|
|
4298
|
+
|
|
4299
|
+
```ts
|
|
4300
|
+
ai.supervisor({
|
|
4301
|
+
intents: {
|
|
4302
|
+
delegate: async (ctx) => {
|
|
4303
|
+
const result = await worker.execute(String(ctx.input)); // direct call — still nested
|
|
4304
|
+
return { reply: result.text };
|
|
4305
|
+
},
|
|
4306
|
+
},
|
|
4307
|
+
route: (ctx) => (ctx.iteration === 0 ? "delegate" : END),
|
|
4308
|
+
});
|
|
4309
|
+
// report → callback("delegate") → agent("worker") → tool("echo"); usage flows up to the root.
|
|
4310
|
+
```
|
|
4311
|
+
|
|
4312
|
+
Same behavior across `ai.supervisor`, `ai.team` (member callbacks), and `ai.orchestrator` (turn callbacks) — and `sessionId` propagates onto the captured subtree. `ctx.run(agent)` is captured **exactly once** (the explicit path does not double-count via the ambient frame), and a standalone `agent.execute()` **outside** any callback keeps its own self-root (no frame leakage). This is what an `Observer` / panoptic sees — see [`@warlock.js/ai/observe-ai-flows/SKILL.md`](@warlock.js/ai/observe-ai-flows/SKILL.md).
|
|
4313
|
+
|
|
2838
4314
|
## Per-call options
|
|
2839
4315
|
|
|
2840
4316
|
```ts
|
|
@@ -3050,11 +4526,121 @@ Drivers without similarity support throw `CacheUnsupportedError` from `set({ vec
|
|
|
3050
4526
|
- [`@warlock.js/cache/cache-basics/SKILL.md`](@warlock.js/cache/cache-basics/SKILL.md) — vector driver catalog
|
|
3051
4527
|
|
|
3052
4528
|
|
|
4529
|
+
## use-runtime-skills `@warlock.js/ai/use-runtime-skills/SKILL.md`
|
|
4530
|
+
|
|
4531
|
+
---
|
|
4532
|
+
name: use-runtime-skills
|
|
4533
|
+
description: 'Progressive-disclosure agent skills with ai.skills({...}) and the first-class `skills` option on ai.agent — an always-injected cheap metadata catalog plus an on-demand loadSkill tool, backed by directory / url / store sources. Covers inject ("all" | {select:"semantic",topK,embedder}), maxLoadsPerRun, scope tags, the MockSkillsStore, semantic preload, and the inert-by-default Phase-2 self-authoring (saveSkill + default-DENY review gate → promote). Triggers: `ai.skills`, `SkillsConfig`, `SkillsContract`, `SkillSource`, `SkillInjectMode`, `SkillRecord`, `SkillCatalogEntry`, `loadSkill`, `loadSkillTool`, `saveSkill`, `saveSkillTool`, `SkillReviewGate`, `runReviewGate`, `MockSkillsStore`, `proceduralSkillStore`, `maxLoadsPerRun`, `inject`, `scope`, `review`, the agent `skills:` option; ''give an agent loadable skills'', ''progressive disclosure of instructions'', ''catalog of skills the model pulls on demand'', ''semantic preload of skill bodies'', ''let an agent author and review a skill''; typical import `import { ai } from "@warlock.js/ai"`. Skip: composing static system prompts — `@warlock.js/ai/write-system-prompt/SKILL.md`; durable agent memory tiers — `@warlock.js/ai/use-ai-memory/SKILL.md`; defining callable tools — `@warlock.js/ai/define-ai-tool/SKILL.md`.'
|
|
4534
|
+
---
|
|
4535
|
+
|
|
4536
|
+
# `ai.skills()` — runtime skills with progressive disclosure
|
|
4537
|
+
|
|
4538
|
+
A **skill is text injected into an agent's context — it never runs code.** `ai.skills(config)` builds a `SkillsContract`: the mechanism behind the first-class `skills` agent option. The agent always injects a cheap **metadata catalog** (one line per in-scope skill) and registers a `loadSkill` tool so the model pulls a skill's full **body** only when it needs it (progressive disclosure). Bodies are withheld until loaded — keeping context lean.
|
|
4539
|
+
|
|
4540
|
+
## The first-class agent option (the supported way)
|
|
4541
|
+
|
|
4542
|
+
```ts
|
|
4543
|
+
import { ai } from "@warlock.js/ai";
|
|
4544
|
+
|
|
4545
|
+
const agent = ai.agent({
|
|
4546
|
+
model: openai.model({ name: "gpt-4o" }),
|
|
4547
|
+
systemPrompt: "You are a build assistant.",
|
|
4548
|
+
skills: { // a SkillsConfig OR an ai.skills(...) instance
|
|
4549
|
+
name: "build-skills",
|
|
4550
|
+
sources: [{ type: "directory", path: "./agent-skills" }],
|
|
4551
|
+
},
|
|
4552
|
+
});
|
|
4553
|
+
```
|
|
4554
|
+
|
|
4555
|
+
When `skills` is set the agent owns the runtime flow at execute time: it **prepends the always-injected catalog** (and, under `inject`, the preloaded bodies) in front of your system prompt, auto-registers `loadSkill` (plus `saveSkill` only when a `review` gate is configured), and threads the run id so `maxLoadsPerRun` is enforced per execution. **Omitted ⇒ no skills behavior; the agent runs byte-for-byte as today.** The option accepts a raw `SkillsConfig` (the agent passes it to `skills()` for you) or a pre-built `SkillsContract`.
|
|
4556
|
+
|
|
4557
|
+
## Factory config — `SkillsConfig`
|
|
4558
|
+
|
|
4559
|
+
```ts
|
|
4560
|
+
const lib = ai.skills({
|
|
4561
|
+
name: "build-skills", // surfaced in analytics + the catalog block
|
|
4562
|
+
sources: [{ type: "directory", path: "./agent-skills" }], // >= 1; later source wins on name clash
|
|
4563
|
+
inject: { select: "semantic", topK: 2, embedder }, // body-injection policy (see below)
|
|
4564
|
+
maxLoadsPerRun: 4, // cap on loadSkill calls per run. default 5
|
|
4565
|
+
scope: { tags: ["frontend"] }, // only skills whose tags intersect are catalogued
|
|
4566
|
+
review: { approve, store }, // Phase 2 — absent ⇒ saveSkill is NOT exposed
|
|
4567
|
+
analytics: (event) => track(event), // optional efficacy sink (errors swallowed)
|
|
4568
|
+
});
|
|
4569
|
+
```
|
|
4570
|
+
|
|
4571
|
+
### Sources — `SkillSource` (discriminated by `type`, never `kind`)
|
|
4572
|
+
|
|
4573
|
+
- `{ type: "directory", path }` — reads `path/<folder>/SKILL.md` off disk (lazy `node:fs/promises`).
|
|
4574
|
+
- `{ type: "url", url, headers? }` — `fetch()`es a JSON manifest of skills.
|
|
4575
|
+
- `{ type: "store", store }` — any `SkillsStoreContract`, e.g. `MockSkillsStore`.
|
|
4576
|
+
|
|
4577
|
+
Sources merge in order; a later source wins on a name collision.
|
|
4578
|
+
|
|
4579
|
+
### Injection — `inject` (`SkillInjectMode`)
|
|
4580
|
+
|
|
4581
|
+
The metadata catalog is **always** injected (it's cheap). `inject` controls whether any **bodies** are auto-injected up front:
|
|
4582
|
+
|
|
4583
|
+
- **omitted** (default) — inject NO bodies; the model pulls them via `loadSkill`. Pure progressive disclosure.
|
|
4584
|
+
- `"all"` — inject every body up front (small libraries only).
|
|
4585
|
+
- `{ select: "semantic", topK, embedder?, threshold? }` — embed the run input, rank the catalog by cosine similarity, inject the top-`topK` bodies. Needs an embedder (passed here, or lazily auto-resolved).
|
|
4586
|
+
|
|
4587
|
+
## `SkillsContract` surface
|
|
4588
|
+
|
|
4589
|
+
```ts
|
|
4590
|
+
interface SkillsContract {
|
|
4591
|
+
readonly name: string;
|
|
4592
|
+
catalog(scopeInput?: string): Promise<SkillCatalogEntry[]>; // cheap metadata, body omitted
|
|
4593
|
+
catalogPrompt(scopeInput?: string): Promise<string>; // catalog rendered as a system block
|
|
4594
|
+
preload(input: string): Promise<SkillRecord[]>; // bodies per `inject`; [] when omitted
|
|
4595
|
+
tools(runId?: string): AgentToolEntry<any, any>[]; // loadSkill always; saveSkill iff review
|
|
4596
|
+
}
|
|
4597
|
+
```
|
|
4598
|
+
|
|
4599
|
+
A `SkillCatalogEntry` is `Pick<SkillRecord, "name"|"description"|"version"|"tags"|"type">` — the **structural omission of `body`** is the type-level guarantee the catalog never carries skill bodies. A `SkillRecord` adds the full `body` plus `type: "authored" | "promoted" | "candidate"`.
|
|
4600
|
+
|
|
4601
|
+
## `maxLoadsPerRun` — a budget, not a throw
|
|
4602
|
+
|
|
4603
|
+
`loadSkill` calls are capped per run (default 5). Exhaustion is an **error RESULT the model self-corrects from**, never a throw — the tool returns `{ error }` and the loop continues. `runId` scopes both the budget and analytics correlation.
|
|
4604
|
+
|
|
4605
|
+
## Stores
|
|
4606
|
+
|
|
4607
|
+
```ts
|
|
4608
|
+
import { ai, MockSkillsStore } from "@warlock.js/ai";
|
|
4609
|
+
|
|
4610
|
+
const store = new MockSkillsStore([
|
|
4611
|
+
{ name: "scaffold", description: "Scaffold a form", version: 1, body: "...", type: "authored" },
|
|
4612
|
+
]);
|
|
4613
|
+
const lib = ai.skills({ name: "build", sources: [{ type: "store", store }] });
|
|
4614
|
+
```
|
|
4615
|
+
|
|
4616
|
+
`MockSkillsStore` is an in-memory `SkillsStoreContract` that ships with the package (construct via `new` — it is a concrete test/utility store, not a factory-fronted primitive). It holds the latest record per name, filters out `candidate`s from `list()` / `load()`, and exposes `saveCandidate` / `promote`. `proceduralSkillStore` is also exported (unifies proven procedural memories with named skills).
|
|
4617
|
+
|
|
4618
|
+
## Phase 2 — self-authoring (inert by default)
|
|
4619
|
+
|
|
4620
|
+
Self-authoring is **gated and OFF unless a `review` gate is wired**:
|
|
4621
|
+
|
|
4622
|
+
- Without `review`, the `saveSkill` tool is **never registered** — a candidate can never be written, let alone injected.
|
|
4623
|
+
- With `review: { approve, store }`, `saveSkill` writes an **INERT** `type: "candidate"` (`version: 0`), filtered out of every catalog/load until promoted.
|
|
4624
|
+
- The `SkillReviewGate.approve(candidate)` is **default-DENY**: only `{ approve: true }` promotes the candidate to a new audited version (`promote` → `type: "promoted"`, `version + 1`). Anything else — `{ approve: false }`, a malformed result, or a **throw** (fail-closed) — keeps it inert. `runReviewGate(candidate, gate, emit?)` runs this and never throws (a throwing gate is a denial), emitting `promoted` / `denied` analytics events.
|
|
4625
|
+
|
|
4626
|
+
The three interchangeable approve shapes — a policy fn, a validator agent, a human callback — all reduce to one `Promise<{ approve: boolean; reason? }>`.
|
|
4627
|
+
|
|
4628
|
+
## Analytics
|
|
4629
|
+
|
|
4630
|
+
The optional `analytics` sink fires `catalogued` / `loaded` / `used` / `saved` / `promoted` / `denied` events `{ type, skill, version, runId?, outcome? }`. Errors from the sink are swallowed (mirroring the agent's `onUsage` / `onComplete`), so analytics never crash a run.
|
|
4631
|
+
|
|
4632
|
+
## See also
|
|
4633
|
+
|
|
4634
|
+
- [`@warlock.js/ai/write-system-prompt/SKILL.md`](@warlock.js/ai/write-system-prompt/SKILL.md) — static persona / instruction blocks (vs. dynamic loaded skills)
|
|
4635
|
+
- [`@warlock.js/ai/use-ai-memory/SKILL.md`](@warlock.js/ai/use-ai-memory/SKILL.md) — the procedural memory tier `proceduralSkillStore` unifies with
|
|
4636
|
+
- [`@warlock.js/ai/run-ai-agent/SKILL.md`](@warlock.js/ai/run-ai-agent/SKILL.md) — the agent the `skills` option attaches to
|
|
4637
|
+
|
|
4638
|
+
|
|
3053
4639
|
## write-system-prompt `@warlock.js/ai/write-system-prompt/SKILL.md`
|
|
3054
4640
|
|
|
3055
4641
|
---
|
|
3056
4642
|
name: write-system-prompt
|
|
3057
|
-
description: 'Compose system prompts via ai.systemPrompt() / ai.persona() / ai.instruction() — immutable builders with {{placeholder}} substitution, plus ai.systemPrompt.fromFile(path) to seed from a file read once at construction. Triggers: `ai.systemPrompt`, `ai.systemPrompt.fromFile`, `ai.persona`, `ai.instruction`, `SystemPromptBlockContract`, `PersonaContract`, `InstructionContract`, `placeholders`, `{{placeholder|default}}`, `InvalidRequestError`; ''write a system prompt'', ''compose persona + instructions'', ''prompt from a file'', ''per-call prompt override'', ''mustache placeholder''; typical import `import { ai } from "@warlock.js/ai"`. Skip: agent factory wiring — `@warlock.js/ai/run-ai-agent/SKILL.md`; competing libs `langchain` `PromptTemplate`, raw f-strings.'
|
|
4643
|
+
description: 'Compose system prompts via ai.systemPrompt() / ai.persona() / ai.instruction() — immutable builders with {{placeholder}} substitution, plus ai.systemPrompt.fromFile(path) to seed from a file read once at construction. Carry identity with .meta({ name, version, description, required }) (a name auto-registers in ai.prompts) and compose with merge(...blocks) / merge(contract) / merge(name, { fromVersion }) (provenance in meta.composedFrom). Triggers: `ai.systemPrompt`, `ai.systemPrompt.fromFile`, `ai.persona`, `ai.instruction`, `SystemPromptBlockContract`, `SystemPromptContract`, `SystemPromptMeta`, `SystemPromptMergeOptions`, `PersonaContract`, `InstructionContract`, `meta`, `merge`, `composedFrom`, `fromVersion`, `placeholders`, `{{placeholder|default}}`, `InvalidRequestError`; ''write a system prompt'', ''compose persona + instructions'', ''prompt from a file'', ''name and version a prompt'', ''merge prompts together'', ''per-call prompt override'', ''mustache placeholder''; typical import `import { ai } from "@warlock.js/ai"`. Skip: the named/versioned prompt registry (register / resolve / tag / diff / export / validate) — `@warlock.js/ai/manage-prompts/SKILL.md`; agent factory wiring — `@warlock.js/ai/run-ai-agent/SKILL.md`; competing libs `langchain` `PromptTemplate`, raw f-strings.'
|
|
3058
4644
|
---
|
|
3059
4645
|
|
|
3060
4646
|
# System prompts — immutable builders
|
|
@@ -3161,6 +4747,34 @@ ai.agent({ model, systemPrompt: prompt, placeholders: { language: "Arabic" } });
|
|
|
3161
4747
|
|
|
3162
4748
|
Substitution works on the **rendered** concatenation of every block, so `{{key}}` inside a persona and inside an instruction both resolve against the same placeholder bag.
|
|
3163
4749
|
|
|
4750
|
+
## Identity + composition — `.meta()` and `merge()`
|
|
4751
|
+
|
|
4752
|
+
A prompt carries optional `SystemPromptMeta` — `{ name?, version?, description?, required?, composedFrom? }`. Read it with the no-argument accessor; update it immutably with the one-argument form. **Giving a prompt a `name` auto-registers it in the `ai.prompts` registry** (keyed by `name@version`):
|
|
4753
|
+
|
|
4754
|
+
```ts
|
|
4755
|
+
const base = ai.systemPrompt("You are support.", { name: "support", version: "1" });
|
|
4756
|
+
base.meta(); // → { name: "support", version: "1" }
|
|
4757
|
+
const v2 = base.meta({ version: "2" }); // new builder; original untouched; re-registers under support@2
|
|
4758
|
+
```
|
|
4759
|
+
|
|
4760
|
+
`merge(...)` folds blocks from another source into a **new** builder — a persona **replaces**, instructions **append**:
|
|
4761
|
+
|
|
4762
|
+
```ts
|
|
4763
|
+
// (a) N pre-built blocks in one call
|
|
4764
|
+
const p = ai.systemPrompt().merge(ai.persona("You are Alex."), ai.instruction("Be concise."));
|
|
4765
|
+
|
|
4766
|
+
// (b) another prompt contract — its blocks fold in; meta.composedFrom records provenance
|
|
4767
|
+
const merged = ai.systemPrompt("Be terse.").merge(otherPrompt);
|
|
4768
|
+
merged.meta()?.composedFrom; // deterministic source labels, e.g. ["base@2"]
|
|
4769
|
+
|
|
4770
|
+
// (c) a registered prompt resolved from ai.prompts by name (latest, or a pinned fromVersion)
|
|
4771
|
+
const composed = ai.systemPrompt("You are support.").merge("global", { fromVersion: "1" });
|
|
4772
|
+
```
|
|
4773
|
+
|
|
4774
|
+
The name / contract / registry-name forms are the registry's composition surface — full coverage (register / resolve / version / tag / diff / validate) in [`@warlock.js/ai/manage-prompts/SKILL.md`](@warlock.js/ai/manage-prompts/SKILL.md).
|
|
4775
|
+
|
|
4776
|
+
`.validate(options?)` is per-builder sugar over `ai.prompts.validate(this, options)` — the deterministic missing-placeholder check plus an optional Nova-safe LLM-judge.
|
|
4777
|
+
|
|
3164
4778
|
## Per-call overrides
|
|
3165
4779
|
|
|
3166
4780
|
Replace the agent's system prompt for a single run:
|
|
@@ -3192,6 +4806,7 @@ Three distinct prompts, one common foundation. Base is immutable — safe to sha
|
|
|
3192
4806
|
|
|
3193
4807
|
## See also
|
|
3194
4808
|
|
|
4809
|
+
- [`@warlock.js/ai/manage-prompts/SKILL.md`](@warlock.js/ai/manage-prompts/SKILL.md) — the `ai.prompts` registry these named prompts auto-register into (resolve / version / tag / diff / export / validate)
|
|
3195
4810
|
- [`@warlock.js/ai/run-ai-agent/SKILL.md`](@warlock.js/ai/run-ai-agent/SKILL.md) — `systemPrompt` on factory + per-call override
|
|
3196
4811
|
- [`@warlock.js/ai/run-ai-workflow/SKILL.md`](@warlock.js/ai/run-ai-workflow/SKILL.md) — per-step agent references inherit their own system prompt
|
|
3197
4812
|
|