@warlock.js/core 4.8.0 → 4.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/esm/connectors/ai-connector.mjs +1 -1
- package/package.json +12 -9
- package/esm/ai/src/agent/agent-input-builder.mjs +0 -133
- package/esm/ai/src/agent/agent-input-builder.mjs.map +0 -1
- package/esm/ai/src/agent/agent-log-event.mjs +0 -101
- package/esm/ai/src/agent/agent-log-event.mjs.map +0 -1
- package/esm/ai/src/agent/agent-stream.mjs +0 -82
- package/esm/ai/src/agent/agent-stream.mjs.map +0 -1
- package/esm/ai/src/agent/agent-to-stream-event.mjs +0 -86
- package/esm/ai/src/agent/agent-to-stream-event.mjs.map +0 -1
- package/esm/ai/src/agent/agent.mjs +0 -1276
- package/esm/ai/src/agent/agent.mjs.map +0 -1
- package/esm/ai/src/agent/index.mjs +0 -5
- package/esm/ai/src/agent/json-stream-guard.mjs +0 -400
- package/esm/ai/src/agent/json-stream-guard.mjs.map +0 -1
- package/esm/ai/src/agent/signature.mjs +0 -57
- package/esm/ai/src/agent/signature.mjs.map +0 -1
- package/esm/ai/src/agent/snapshot.mjs +0 -101
- package/esm/ai/src/agent/snapshot.mjs.map +0 -1
- package/esm/ai/src/agent/spawn-sub-agent.mjs +0 -68
- package/esm/ai/src/agent/spawn-sub-agent.mjs.map +0 -1
- package/esm/ai/src/ai.mjs +0 -191
- package/esm/ai/src/ai.mjs.map +0 -1
- package/esm/ai/src/batch/batch.mjs +0 -198
- package/esm/ai/src/batch/batch.mjs.map +0 -1
- package/esm/ai/src/batch/index.mjs +0 -3
- package/esm/ai/src/batch/run-batch-item.mjs +0 -100
- package/esm/ai/src/batch/run-batch-item.mjs.map +0 -1
- package/esm/ai/src/batch/run-with-concurrency.mjs +0 -39
- package/esm/ai/src/batch/run-with-concurrency.mjs.map +0 -1
- package/esm/ai/src/checkpoint/memory.mjs +0 -0
- package/esm/ai/src/checkpoint/memory.mjs.map +0 -1
- package/esm/ai/src/checkpoint/pg.mjs +0 -265
- package/esm/ai/src/checkpoint/pg.mjs.map +0 -1
- package/esm/ai/src/checkpoint/redis.mjs +0 -200
- package/esm/ai/src/checkpoint/redis.mjs.map +0 -1
- package/esm/ai/src/config.mjs +0 -59
- package/esm/ai/src/config.mjs.map +0 -1
- package/esm/ai/src/contracts/end.type.mjs +0 -47
- package/esm/ai/src/contracts/end.type.mjs.map +0 -1
- package/esm/ai/src/errors/agent-cancelled-error.mjs +0 -43
- package/esm/ai/src/errors/agent-cancelled-error.mjs.map +0 -1
- package/esm/ai/src/errors/agent-drift-error.mjs +0 -31
- package/esm/ai/src/errors/agent-drift-error.mjs.map +0 -1
- package/esm/ai/src/errors/agent-execution-error.mjs +0 -35
- package/esm/ai/src/errors/agent-execution-error.mjs.map +0 -1
- package/esm/ai/src/errors/agent-max-trips-error.mjs +0 -41
- package/esm/ai/src/errors/agent-max-trips-error.mjs.map +0 -1
- package/esm/ai/src/errors/ai-error.mjs +0 -68
- package/esm/ai/src/errors/ai-error.mjs.map +0 -1
- package/esm/ai/src/errors/budget-exceeded-error.mjs +0 -31
- package/esm/ai/src/errors/budget-exceeded-error.mjs.map +0 -1
- package/esm/ai/src/errors/content-filter-error.mjs +0 -28
- package/esm/ai/src/errors/content-filter-error.mjs.map +0 -1
- package/esm/ai/src/errors/context-length-exceeded-error.mjs +0 -31
- package/esm/ai/src/errors/context-length-exceeded-error.mjs.map +0 -1
- package/esm/ai/src/errors/guardrail-violation-error.mjs +0 -41
- package/esm/ai/src/errors/guardrail-violation-error.mjs.map +0 -1
- package/esm/ai/src/errors/index.mjs +0 -40
- package/esm/ai/src/errors/invalid-request-error.mjs +0 -31
- package/esm/ai/src/errors/invalid-request-error.mjs.map +0 -1
- package/esm/ai/src/errors/max-iterations-error.mjs +0 -32
- package/esm/ai/src/errors/max-iterations-error.mjs.map +0 -1
- package/esm/ai/src/errors/max-steps-exceeded-error.mjs +0 -21
- package/esm/ai/src/errors/max-steps-exceeded-error.mjs.map +0 -1
- package/esm/ai/src/errors/orchestrator-cancelled-error.mjs +0 -31
- package/esm/ai/src/errors/orchestrator-cancelled-error.mjs.map +0 -1
- package/esm/ai/src/errors/orchestrator-config-error.mjs +0 -30
- package/esm/ai/src/errors/orchestrator-config-error.mjs.map +0 -1
- package/esm/ai/src/errors/orchestrator-drift-error.mjs +0 -37
- package/esm/ai/src/errors/orchestrator-drift-error.mjs.map +0 -1
- package/esm/ai/src/errors/orchestrator-failed-error.mjs +0 -36
- package/esm/ai/src/errors/orchestrator-failed-error.mjs.map +0 -1
- package/esm/ai/src/errors/outbound-policy-error.mjs +0 -32
- package/esm/ai/src/errors/outbound-policy-error.mjs.map +0 -1
- package/esm/ai/src/errors/planner-cancelled-error.mjs +0 -29
- package/esm/ai/src/errors/planner-cancelled-error.mjs.map +0 -1
- package/esm/ai/src/errors/planner-drift-error.mjs +0 -33
- package/esm/ai/src/errors/planner-drift-error.mjs.map +0 -1
- package/esm/ai/src/errors/planner-failed-error.mjs +0 -37
- package/esm/ai/src/errors/planner-failed-error.mjs.map +0 -1
- package/esm/ai/src/errors/planner-plan-invalid-error.mjs +0 -25
- package/esm/ai/src/errors/planner-plan-invalid-error.mjs.map +0 -1
- package/esm/ai/src/errors/prompt-refinement-error.mjs +0 -27
- package/esm/ai/src/errors/prompt-refinement-error.mjs.map +0 -1
- package/esm/ai/src/errors/provider-auth-error.mjs +0 -26
- package/esm/ai/src/errors/provider-auth-error.mjs.map +0 -1
- package/esm/ai/src/errors/provider-error.mjs +0 -42
- package/esm/ai/src/errors/provider-error.mjs.map +0 -1
- package/esm/ai/src/errors/provider-rate-limit-error.mjs +0 -31
- package/esm/ai/src/errors/provider-rate-limit-error.mjs.map +0 -1
- package/esm/ai/src/errors/provider-timeout-error.mjs +0 -26
- package/esm/ai/src/errors/provider-timeout-error.mjs.map +0 -1
- package/esm/ai/src/errors/quota-exceeded-error.mjs +0 -40
- package/esm/ai/src/errors/quota-exceeded-error.mjs.map +0 -1
- package/esm/ai/src/errors/routing-error.mjs +0 -23
- package/esm/ai/src/errors/routing-error.mjs.map +0 -1
- package/esm/ai/src/errors/schema-validation-error.mjs +0 -37
- package/esm/ai/src/errors/schema-validation-error.mjs.map +0 -1
- package/esm/ai/src/errors/step-failed-error.mjs +0 -19
- package/esm/ai/src/errors/step-failed-error.mjs.map +0 -1
- package/esm/ai/src/errors/supervisor-cancelled-error.mjs +0 -29
- package/esm/ai/src/errors/supervisor-cancelled-error.mjs.map +0 -1
- package/esm/ai/src/errors/supervisor-drift-error.mjs +0 -30
- package/esm/ai/src/errors/supervisor-drift-error.mjs.map +0 -1
- package/esm/ai/src/errors/supervisor-failed-error.mjs +0 -43
- package/esm/ai/src/errors/supervisor-failed-error.mjs.map +0 -1
- package/esm/ai/src/errors/supervisor-routing-error.mjs +0 -39
- package/esm/ai/src/errors/supervisor-routing-error.mjs.map +0 -1
- package/esm/ai/src/errors/tool-execution-error.mjs +0 -32
- package/esm/ai/src/errors/tool-execution-error.mjs.map +0 -1
- package/esm/ai/src/errors/workflow-cancelled-error.mjs +0 -23
- package/esm/ai/src/errors/workflow-cancelled-error.mjs.map +0 -1
- package/esm/ai/src/errors/workflow-drift-error.mjs +0 -24
- package/esm/ai/src/errors/workflow-drift-error.mjs.map +0 -1
- package/esm/ai/src/errors/workflow-error.mjs +0 -18
- package/esm/ai/src/errors/workflow-error.mjs.map +0 -1
- package/esm/ai/src/eval/dataset.mjs +0 -112
- package/esm/ai/src/eval/dataset.mjs.map +0 -1
- package/esm/ai/src/eval/eval-runner.mjs +0 -135
- package/esm/ai/src/eval/eval-runner.mjs.map +0 -1
- package/esm/ai/src/eval/index.mjs +0 -48
- package/esm/ai/src/eval/index.mjs.map +0 -1
- package/esm/ai/src/eval/judge-scorer.mjs +0 -87
- package/esm/ai/src/eval/judge-scorer.mjs.map +0 -1
- package/esm/ai/src/eval/regression.mjs +0 -51
- package/esm/ai/src/eval/regression.mjs.map +0 -1
- package/esm/ai/src/eval/report-json.mjs +0 -33
- package/esm/ai/src/eval/report-json.mjs.map +0 -1
- package/esm/ai/src/eval/report-junit.mjs +0 -60
- package/esm/ai/src/eval/report-junit.mjs.map +0 -1
- package/esm/ai/src/eval/scorers.mjs +0 -101
- package/esm/ai/src/eval/scorers.mjs.map +0 -1
- package/esm/ai/src/guard/detectors/index.mjs +0 -6
- package/esm/ai/src/guard/detectors/injection.mjs +0 -254
- package/esm/ai/src/guard/detectors/injection.mjs.map +0 -1
- package/esm/ai/src/guard/detectors/moderation.mjs +0 -134
- package/esm/ai/src/guard/detectors/moderation.mjs.map +0 -1
- package/esm/ai/src/guard/detectors/pii.mjs +0 -199
- package/esm/ai/src/guard/detectors/pii.mjs.map +0 -1
- package/esm/ai/src/guard/detectors/topic.mjs +0 -99
- package/esm/ai/src/guard/detectors/topic.mjs.map +0 -1
- package/esm/ai/src/guard/errors.mjs +0 -18
- package/esm/ai/src/guard/errors.mjs.map +0 -1
- package/esm/ai/src/guard/guard.mjs +0 -209
- package/esm/ai/src/guard/guard.mjs.map +0 -1
- package/esm/ai/src/guard/guardrail.mjs +0 -22
- package/esm/ai/src/guard/guardrail.mjs.map +0 -1
- package/esm/ai/src/human/errors.mjs +0 -60
- package/esm/ai/src/human/errors.mjs.map +0 -1
- package/esm/ai/src/human/human-approval.mjs +0 -170
- package/esm/ai/src/human/human-approval.mjs.map +0 -1
- package/esm/ai/src/human/policy.mjs +0 -67
- package/esm/ai/src/human/policy.mjs.map +0 -1
- package/esm/ai/src/human/register.mjs +0 -37
- package/esm/ai/src/human/register.mjs.map +0 -1
- package/esm/ai/src/human/resume-seed.mjs +0 -53
- package/esm/ai/src/human/resume-seed.mjs.map +0 -1
- package/esm/ai/src/human/resume.mjs +0 -121
- package/esm/ai/src/human/resume.mjs.map +0 -1
- package/esm/ai/src/human/stores/index.mjs +0 -5
- package/esm/ai/src/human/stores/memory.mjs +0 -91
- package/esm/ai/src/human/stores/memory.mjs.map +0 -1
- package/esm/ai/src/human/stores/pg.mjs +0 -220
- package/esm/ai/src/human/stores/pg.mjs.map +0 -1
- package/esm/ai/src/human/stores/redis.mjs +0 -201
- package/esm/ai/src/human/stores/redis.mjs.map +0 -1
- package/esm/ai/src/image/image-cost.mjs +0 -55
- package/esm/ai/src/image/image-cost.mjs.map +0 -1
- package/esm/ai/src/image/image.mjs +0 -112
- package/esm/ai/src/image/image.mjs.map +0 -1
- package/esm/ai/src/image/index.mjs +0 -4
- package/esm/ai/src/index.mjs +0 -196
- package/esm/ai/src/memory/derive-id.mjs +0 -24
- package/esm/ai/src/memory/derive-id.mjs.map +0 -1
- package/esm/ai/src/memory/episodic-memory.mjs +0 -106
- package/esm/ai/src/memory/episodic-memory.mjs.map +0 -1
- package/esm/ai/src/memory/memory.mjs +0 -166
- package/esm/ai/src/memory/memory.mjs.map +0 -1
- package/esm/ai/src/memory/procedural-memory.mjs +0 -103
- package/esm/ai/src/memory/procedural-memory.mjs.map +0 -1
- package/esm/ai/src/memory/semantic-memory.mjs +0 -80
- package/esm/ai/src/memory/semantic-memory.mjs.map +0 -1
- package/esm/ai/src/memory/working-memory.mjs +0 -62
- package/esm/ai/src/memory/working-memory.mjs.map +0 -1
- package/esm/ai/src/middleware/builtins/budget.mjs +0 -219
- package/esm/ai/src/middleware/builtins/budget.mjs.map +0 -1
- package/esm/ai/src/middleware/builtins/guardrail.mjs +0 -76
- package/esm/ai/src/middleware/builtins/guardrail.mjs.map +0 -1
- package/esm/ai/src/middleware/builtins/semantic-cache.mjs +0 -149
- package/esm/ai/src/middleware/builtins/semantic-cache.mjs.map +0 -1
- package/esm/ai/src/middleware/helpers/compose.mjs +0 -52
- package/esm/ai/src/middleware/helpers/compose.mjs.map +0 -1
- package/esm/ai/src/middleware/helpers/for-tool.mjs +0 -80
- package/esm/ai/src/middleware/helpers/for-tool.mjs.map +0 -1
- package/esm/ai/src/middleware/index.mjs +0 -10
- package/esm/ai/src/middleware/pipeline.mjs +0 -100
- package/esm/ai/src/middleware/pipeline.mjs.map +0 -1
- package/esm/ai/src/middleware/utils/extract-user-text.mjs +0 -47
- package/esm/ai/src/middleware/utils/extract-user-text.mjs.map +0 -1
- package/esm/ai/src/middleware/utils/namespaced-state.mjs +0 -60
- package/esm/ai/src/middleware/utils/namespaced-state.mjs.map +0 -1
- package/esm/ai/src/mock/index.mjs +0 -4
- package/esm/ai/src/mock/mock-agent.mjs +0 -3
- package/esm/ai/src/mock/mock-router.mjs +0 -58
- package/esm/ai/src/mock/mock-router.mjs.map +0 -1
- package/esm/ai/src/model/fallback-model.mjs +0 -218
- package/esm/ai/src/model/fallback-model.mjs.map +0 -1
- package/esm/ai/src/model/index.mjs +0 -3
- package/esm/ai/src/object-stream/index.mjs +0 -4
- package/esm/ai/src/object-stream/parse-partial-json.mjs +0 -78
- package/esm/ai/src/object-stream/parse-partial-json.mjs.map +0 -1
- package/esm/ai/src/object-stream/stream-object.mjs +0 -97
- package/esm/ai/src/object-stream/stream-object.mjs.map +0 -1
- package/esm/ai/src/observe/index.mjs +0 -4
- package/esm/ai/src/observe/observer-registry.mjs +0 -35
- package/esm/ai/src/observe/observer-registry.mjs.map +0 -1
- package/esm/ai/src/observe/resolve-observers.mjs +0 -73
- package/esm/ai/src/observe/resolve-observers.mjs.map +0 -1
- package/esm/ai/src/orchestrator/as-tool.mjs +0 -98
- package/esm/ai/src/orchestrator/as-tool.mjs.map +0 -1
- package/esm/ai/src/orchestrator/checkpoint.mjs +0 -75
- package/esm/ai/src/orchestrator/checkpoint.mjs.map +0 -1
- package/esm/ai/src/orchestrator/commands.mjs +0 -34
- package/esm/ai/src/orchestrator/commands.mjs.map +0 -1
- package/esm/ai/src/orchestrator/compaction.mjs +0 -206
- package/esm/ai/src/orchestrator/compaction.mjs.map +0 -1
- package/esm/ai/src/orchestrator/dispatch.mjs +0 -171
- package/esm/ai/src/orchestrator/dispatch.mjs.map +0 -1
- package/esm/ai/src/orchestrator/emitter.mjs +0 -85
- package/esm/ai/src/orchestrator/emitter.mjs.map +0 -1
- package/esm/ai/src/orchestrator/execution.mjs +0 -395
- package/esm/ai/src/orchestrator/execution.mjs.map +0 -1
- package/esm/ai/src/orchestrator/index.mjs +0 -11
- package/esm/ai/src/orchestrator/load.mjs +0 -49
- package/esm/ai/src/orchestrator/load.mjs.map +0 -1
- package/esm/ai/src/orchestrator/lock.mjs +0 -75
- package/esm/ai/src/orchestrator/lock.mjs.map +0 -1
- package/esm/ai/src/orchestrator/memory.mjs +0 -141
- package/esm/ai/src/orchestrator/memory.mjs.map +0 -1
- package/esm/ai/src/orchestrator/orchestrator-stream.mjs +0 -98
- package/esm/ai/src/orchestrator/orchestrator-stream.mjs.map +0 -1
- package/esm/ai/src/orchestrator/orchestrator.mjs +0 -206
- package/esm/ai/src/orchestrator/orchestrator.mjs.map +0 -1
- package/esm/ai/src/orchestrator/resume.mjs +0 -74
- package/esm/ai/src/orchestrator/resume.mjs.map +0 -1
- package/esm/ai/src/orchestrator/session-lock.mjs +0 -83
- package/esm/ai/src/orchestrator/session-lock.mjs.map +0 -1
- package/esm/ai/src/orchestrator/signature.mjs +0 -120
- package/esm/ai/src/orchestrator/signature.mjs.map +0 -1
- package/esm/ai/src/orchestrator/window.mjs +0 -56
- package/esm/ai/src/orchestrator/window.mjs.map +0 -1
- package/esm/ai/src/planner/dag-scheduler.mjs +0 -97
- package/esm/ai/src/planner/dag-scheduler.mjs.map +0 -1
- package/esm/ai/src/planner/index.mjs +0 -6
- package/esm/ai/src/planner/plan-prompt.mjs +0 -31
- package/esm/ai/src/planner/plan-prompt.mjs.map +0 -1
- package/esm/ai/src/planner/plan-schema.mjs +0 -120
- package/esm/ai/src/planner/plan-schema.mjs.map +0 -1
- package/esm/ai/src/planner/planner-run.mjs +0 -769
- package/esm/ai/src/planner/planner-run.mjs.map +0 -1
- package/esm/ai/src/planner/planner.mjs +0 -144
- package/esm/ai/src/planner/planner.mjs.map +0 -1
- package/esm/ai/src/planner/signature.mjs +0 -27
- package/esm/ai/src/planner/signature.mjs.map +0 -1
- package/esm/ai/src/planner/snapshot.mjs +0 -95
- package/esm/ai/src/planner/snapshot.mjs.map +0 -1
- package/esm/ai/src/prompt/errors.mjs +0 -73
- package/esm/ai/src/prompt/errors.mjs.map +0 -1
- package/esm/ai/src/prompt/index.mjs +0 -4
- package/esm/ai/src/prompt/prompt-langfuse-sync.mjs +0 -104
- package/esm/ai/src/prompt/prompt-langfuse-sync.mjs.map +0 -1
- package/esm/ai/src/prompt/prompt-validate.mjs +0 -170
- package/esm/ai/src/prompt/prompt-validate.mjs.map +0 -1
- package/esm/ai/src/prompt/prompt.mjs +0 -218
- package/esm/ai/src/prompt/prompt.mjs.map +0 -1
- package/esm/ai/src/prompts/index.mjs +0 -3
- package/esm/ai/src/prompts/prompts-manager.mjs +0 -410
- package/esm/ai/src/prompts/prompts-manager.mjs.map +0 -1
- package/esm/ai/src/prompts/prompts-validate.mjs +0 -0
- package/esm/ai/src/prompts/prompts-validate.mjs.map +0 -1
- package/esm/ai/src/rag/as-tool.mjs +0 -48
- package/esm/ai/src/rag/as-tool.mjs.map +0 -1
- package/esm/ai/src/rag/chunk/chunk.mjs +0 -44
- package/esm/ai/src/rag/chunk/chunk.mjs.map +0 -1
- package/esm/ai/src/rag/chunk/fixed.mjs +0 -32
- package/esm/ai/src/rag/chunk/fixed.mjs.map +0 -1
- package/esm/ai/src/rag/chunk/markdown.mjs +0 -75
- package/esm/ai/src/rag/chunk/markdown.mjs.map +0 -1
- package/esm/ai/src/rag/chunk/recursive.mjs +0 -132
- package/esm/ai/src/rag/chunk/recursive.mjs.map +0 -1
- package/esm/ai/src/rag/chunk/sentence.mjs +0 -73
- package/esm/ai/src/rag/chunk/sentence.mjs.map +0 -1
- package/esm/ai/src/rag/hybrid/bm25.mjs +0 -51
- package/esm/ai/src/rag/hybrid/bm25.mjs.map +0 -1
- package/esm/ai/src/rag/hybrid/hybrid-rank.mjs +0 -29
- package/esm/ai/src/rag/hybrid/hybrid-rank.mjs.map +0 -1
- package/esm/ai/src/rag/hybrid/rrf.mjs +0 -30
- package/esm/ai/src/rag/hybrid/rrf.mjs.map +0 -1
- package/esm/ai/src/rag/index.mjs +0 -18
- package/esm/ai/src/rag/loaders/errors.mjs +0 -25
- package/esm/ai/src/rag/loaders/errors.mjs.map +0 -1
- package/esm/ai/src/rag/loaders/index.mjs +0 -7
- package/esm/ai/src/rag/loaders/load-html.mjs +0 -138
- package/esm/ai/src/rag/loaders/load-html.mjs.map +0 -1
- package/esm/ai/src/rag/loaders/load-pdf.mjs +0 -150
- package/esm/ai/src/rag/loaders/load-pdf.mjs.map +0 -1
- package/esm/ai/src/rag/loaders/load-text.mjs +0 -60
- package/esm/ai/src/rag/loaders/load-text.mjs.map +0 -1
- package/esm/ai/src/rag/loaders/load-web.mjs +0 -89
- package/esm/ai/src/rag/loaders/load-web.mjs.map +0 -1
- package/esm/ai/src/rag/rag.mjs +0 -126
- package/esm/ai/src/rag/rag.mjs.map +0 -1
- package/esm/ai/src/rag/rerank/keyword-reranker.mjs +0 -58
- package/esm/ai/src/rag/rerank/keyword-reranker.mjs.map +0 -1
- package/esm/ai/src/rag/rerank/llm-reranker.mjs +0 -85
- package/esm/ai/src/rag/rerank/llm-reranker.mjs.map +0 -1
- package/esm/ai/src/rag/retrieve.mjs +0 -68
- package/esm/ai/src/rag/retrieve.mjs.map +0 -1
- package/esm/ai/src/rag/store/cache-vector-store.mjs +0 -48
- package/esm/ai/src/rag/store/cache-vector-store.mjs.map +0 -1
- package/esm/ai/src/rag/store/pg-vector-store.mjs +0 -328
- package/esm/ai/src/rag/store/pg-vector-store.mjs.map +0 -1
- package/esm/ai/src/rag/transforms/multi-query.mjs +0 -41
- package/esm/ai/src/rag/transforms/multi-query.mjs.map +0 -1
- package/esm/ai/src/security/index.mjs +0 -5
- package/esm/ai/src/security/outbound-policy.mjs +0 -187
- package/esm/ai/src/security/outbound-policy.mjs.map +0 -1
- package/esm/ai/src/security/private-ip.mjs +0 -48
- package/esm/ai/src/security/private-ip.mjs.map +0 -1
- package/esm/ai/src/security/redact.mjs +0 -60
- package/esm/ai/src/security/redact.mjs.map +0 -1
- package/esm/ai/src/serve/serve.mjs +0 -90
- package/esm/ai/src/serve/serve.mjs.map +0 -1
- package/esm/ai/src/serve/sse.mjs +0 -25
- package/esm/ai/src/serve/sse.mjs.map +0 -1
- package/esm/ai/src/serve/stream-to-sse.mjs +0 -37
- package/esm/ai/src/serve/stream-to-sse.mjs.map +0 -1
- package/esm/ai/src/skills/catalog.mjs +0 -140
- package/esm/ai/src/skills/catalog.mjs.map +0 -1
- package/esm/ai/src/skills/index.mjs +0 -11
- package/esm/ai/src/skills/load-skill-tool.mjs +0 -65
- package/esm/ai/src/skills/load-skill-tool.mjs.map +0 -1
- package/esm/ai/src/skills/save-skill-tool.mjs +0 -65
- package/esm/ai/src/skills/save-skill-tool.mjs.map +0 -1
- package/esm/ai/src/skills/skills.mjs +0 -109
- package/esm/ai/src/skills/skills.mjs.map +0 -1
- package/esm/ai/src/skills/sources/directory-source.mjs +0 -108
- package/esm/ai/src/skills/sources/directory-source.mjs.map +0 -1
- package/esm/ai/src/skills/sources/index.mjs +0 -27
- package/esm/ai/src/skills/sources/index.mjs.map +0 -1
- package/esm/ai/src/skills/sources/parse-frontmatter.mjs +0 -46
- package/esm/ai/src/skills/sources/parse-frontmatter.mjs.map +0 -1
- package/esm/ai/src/skills/sources/store-source.mjs +0 -15
- package/esm/ai/src/skills/sources/store-source.mjs.map +0 -1
- package/esm/ai/src/skills/sources/url-source.mjs +0 -117
- package/esm/ai/src/skills/sources/url-source.mjs.map +0 -1
- package/esm/ai/src/snapshot/memory.mjs +0 -81
- package/esm/ai/src/snapshot/memory.mjs.map +0 -1
- package/esm/ai/src/snapshot/pg.mjs +0 -146
- package/esm/ai/src/snapshot/pg.mjs.map +0 -1
- package/esm/ai/src/snapshot/redis.mjs +0 -101
- package/esm/ai/src/snapshot/redis.mjs.map +0 -1
- package/esm/ai/src/speech/index.mjs +0 -3
- package/esm/ai/src/speech/speech.mjs +0 -122
- package/esm/ai/src/speech/speech.mjs.map +0 -1
- package/esm/ai/src/supervisor/as-tool.mjs +0 -74
- package/esm/ai/src/supervisor/as-tool.mjs.map +0 -1
- package/esm/ai/src/supervisor/cancellation.mjs +0 -22
- package/esm/ai/src/supervisor/cancellation.mjs.map +0 -1
- package/esm/ai/src/supervisor/decide.mjs +0 -272
- package/esm/ai/src/supervisor/decide.mjs.map +0 -1
- package/esm/ai/src/supervisor/emitter.mjs +0 -47
- package/esm/ai/src/supervisor/emitter.mjs.map +0 -1
- package/esm/ai/src/supervisor/entries.mjs +0 -206
- package/esm/ai/src/supervisor/entries.mjs.map +0 -1
- package/esm/ai/src/supervisor/execution.mjs +0 -1921
- package/esm/ai/src/supervisor/execution.mjs.map +0 -1
- package/esm/ai/src/supervisor/fan-out.mjs +0 -65
- package/esm/ai/src/supervisor/fan-out.mjs.map +0 -1
- package/esm/ai/src/supervisor/index.mjs +0 -14
- package/esm/ai/src/supervisor/router-factory.mjs +0 -141
- package/esm/ai/src/supervisor/router-factory.mjs.map +0 -1
- package/esm/ai/src/supervisor/router-prompt.mjs +0 -52
- package/esm/ai/src/supervisor/router-prompt.mjs.map +0 -1
- package/esm/ai/src/supervisor/signature.mjs +0 -89
- package/esm/ai/src/supervisor/signature.mjs.map +0 -1
- package/esm/ai/src/supervisor/snapshot.mjs +0 -69
- package/esm/ai/src/supervisor/snapshot.mjs.map +0 -1
- package/esm/ai/src/supervisor/supervisor-stream.mjs +0 -89
- package/esm/ai/src/supervisor/supervisor-stream.mjs.map +0 -1
- package/esm/ai/src/supervisor/supervisor.mjs +0 -144
- package/esm/ai/src/supervisor/supervisor.mjs.map +0 -1
- package/esm/ai/src/system-prompt/index.mjs +0 -7
- package/esm/ai/src/system-prompt/instruction.mjs +0 -57
- package/esm/ai/src/system-prompt/instruction.mjs.map +0 -1
- package/esm/ai/src/system-prompt/persona.mjs +0 -57
- package/esm/ai/src/system-prompt/persona.mjs.map +0 -1
- package/esm/ai/src/system-prompt/refined-system-prompt.mjs +0 -461
- package/esm/ai/src/system-prompt/refined-system-prompt.mjs.map +0 -1
- package/esm/ai/src/system-prompt/render-placeholders.mjs +0 -58
- package/esm/ai/src/system-prompt/render-placeholders.mjs.map +0 -1
- package/esm/ai/src/system-prompt/system-prompt.mjs +0 -286
- package/esm/ai/src/system-prompt/system-prompt.mjs.map +0 -1
- package/esm/ai/src/team/gates.mjs +0 -48
- package/esm/ai/src/team/gates.mjs.map +0 -1
- package/esm/ai/src/team/index.mjs +0 -3
- package/esm/ai/src/team/team.mjs +0 -94
- package/esm/ai/src/team/team.mjs.map +0 -1
- package/esm/ai/src/tool/executable-as-tool.mjs +0 -81
- package/esm/ai/src/tool/executable-as-tool.mjs.map +0 -1
- package/esm/ai/src/tool/index.mjs +0 -4
- package/esm/ai/src/tool/tool.mjs +0 -185
- package/esm/ai/src/tool/tool.mjs.map +0 -1
- package/esm/ai/src/transcribe/audio-input.mjs +0 -84
- package/esm/ai/src/transcribe/audio-input.mjs.map +0 -1
- package/esm/ai/src/transcribe/index.mjs +0 -4
- package/esm/ai/src/transcribe/transcribe.mjs +0 -127
- package/esm/ai/src/transcribe/transcribe.mjs.map +0 -1
- package/esm/ai/src/utils/compute-cost.mjs +0 -112
- package/esm/ai/src/utils/compute-cost.mjs.map +0 -1
- package/esm/ai/src/utils/extract-json-lenient.mjs +0 -97
- package/esm/ai/src/utils/extract-json-lenient.mjs.map +0 -1
- package/esm/ai/src/utils/extract-json-payload.mjs +0 -39
- package/esm/ai/src/utils/extract-json-payload.mjs.map +0 -1
- package/esm/ai/src/utils/generate-run-id.mjs +0 -23
- package/esm/ai/src/utils/generate-run-id.mjs.map +0 -1
- package/esm/ai/src/utils/index.mjs +0 -12
- package/esm/ai/src/utils/json-schema.mjs +0 -89
- package/esm/ai/src/utils/json-schema.mjs.map +0 -1
- package/esm/ai/src/utils/prepare-attachment-part.mjs +0 -232
- package/esm/ai/src/utils/prepare-attachment-part.mjs.map +0 -1
- package/esm/ai/src/utils/resolve-attachment.mjs +0 -74
- package/esm/ai/src/utils/resolve-attachment.mjs.map +0 -1
- package/esm/ai/src/utils/run-context.mjs +0 -89
- package/esm/ai/src/utils/run-context.mjs.map +0 -1
- package/esm/ai/src/utils/safe-json-parse.mjs +0 -22
- package/esm/ai/src/utils/safe-json-parse.mjs.map +0 -1
- package/esm/ai/src/utils/stamp-report-lineage.mjs +0 -45
- package/esm/ai/src/utils/stamp-report-lineage.mjs.map +0 -1
- package/esm/ai/src/vcr/cassette-io.mjs +0 -57
- package/esm/ai/src/vcr/cassette-io.mjs.map +0 -1
- package/esm/ai/src/vcr/errors.mjs +0 -37
- package/esm/ai/src/vcr/errors.mjs.map +0 -1
- package/esm/ai/src/vcr/hash-request.mjs +0 -118
- package/esm/ai/src/vcr/hash-request.mjs.map +0 -1
- package/esm/ai/src/vcr/index.mjs +0 -5
- package/esm/ai/src/vcr/vcr.mjs +0 -248
- package/esm/ai/src/vcr/vcr.mjs.map +0 -1
- package/esm/ai/src/workflow/as-tool.mjs +0 -49
- package/esm/ai/src/workflow/as-tool.mjs.map +0 -1
- package/esm/ai/src/workflow/cancellation.mjs +0 -44
- package/esm/ai/src/workflow/cancellation.mjs.map +0 -1
- package/esm/ai/src/workflow/emitter.mjs +0 -40
- package/esm/ai/src/workflow/emitter.mjs.map +0 -1
- package/esm/ai/src/workflow/engine.mjs +0 -363
- package/esm/ai/src/workflow/engine.mjs.map +0 -1
- package/esm/ai/src/workflow/index.mjs +0 -5
- package/esm/ai/src/workflow/retry.mjs +0 -32
- package/esm/ai/src/workflow/retry.mjs.map +0 -1
- package/esm/ai/src/workflow/router.mjs +0 -56
- package/esm/ai/src/workflow/router.mjs.map +0 -1
- package/esm/ai/src/workflow/run-scoped-emitter.mjs +0 -32
- package/esm/ai/src/workflow/run-scoped-emitter.mjs.map +0 -1
- package/esm/ai/src/workflow/signature.mjs +0 -34
- package/esm/ai/src/workflow/signature.mjs.map +0 -1
- package/esm/ai/src/workflow/snapshot.mjs +0 -77
- package/esm/ai/src/workflow/snapshot.mjs.map +0 -1
- package/esm/ai/src/workflow/state.mjs +0 -37
- package/esm/ai/src/workflow/state.mjs.map +0 -1
- package/esm/ai/src/workflow/step-runner.mjs +0 -370
- package/esm/ai/src/workflow/step-runner.mjs.map +0 -1
- package/esm/ai/src/workflow/step.mjs +0 -37
- package/esm/ai/src/workflow/step.mjs.map +0 -1
- package/esm/ai/src/workflow/workflow.mjs +0 -122
- package/esm/ai/src/workflow/workflow.mjs.map +0 -1
- package/esm/ai-openai/src/embedder.mjs +0 -4
- package/esm/ai-openai/src/image.mjs +0 -5
- package/esm/ai-openai/src/index.mjs +0 -7
- package/esm/ai-openai/src/model.mjs +0 -5
- package/esm/ai-openai/src/sdk.mjs +0 -9
- package/esm/ai-openai/src/speech.mjs +0 -5
- package/esm/ai-openai/src/transcription.mjs +0 -6
- package/esm/ai-openai/src/utils/index.mjs +0 -5
- package/esm/ai-openai/src/utils/to-openai-messages.mjs +0 -3
- package/esm/ai-openai/src/utils/to-openai-tools.mjs +0 -3
- package/esm/ai-openai/src/utils/wrap-openai-error.mjs +0 -4
- package/esm/node_modules/langfuse/lib/index.mjs +0 -1827
- package/esm/node_modules/langfuse/lib/index.mjs.map +0 -1
- package/esm/node_modules/langfuse-core/lib/index.mjs +0 -1770
- package/esm/node_modules/langfuse-core/lib/index.mjs.map +0 -1
- package/esm/node_modules/mustache/mustache.mjs +0 -587
- package/esm/node_modules/mustache/mustache.mjs.map +0 -1
- package/esm/node_modules/openai/_vendor/partial-json-parser/parser.mjs +0 -180
- package/esm/node_modules/openai/_vendor/partial-json-parser/parser.mjs.map +0 -1
- package/esm/node_modules/openai/auth/workload-identity-auth.mjs +0 -80
- package/esm/node_modules/openai/auth/workload-identity-auth.mjs.map +0 -1
- package/esm/node_modules/openai/azure.mjs +0 -82
- package/esm/node_modules/openai/azure.mjs.map +0 -1
- package/esm/node_modules/openai/client.mjs +0 -638
- package/esm/node_modules/openai/client.mjs.map +0 -1
- package/esm/node_modules/openai/core/api-promise.mjs +0 -75
- package/esm/node_modules/openai/core/api-promise.mjs.map +0 -1
- package/esm/node_modules/openai/core/error.mjs +0 -110
- package/esm/node_modules/openai/core/error.mjs.map +0 -1
- package/esm/node_modules/openai/core/pagination.mjs +0 -160
- package/esm/node_modules/openai/core/pagination.mjs.map +0 -1
- package/esm/node_modules/openai/core/resource.mjs +0 -10
- package/esm/node_modules/openai/core/resource.mjs.map +0 -1
- package/esm/node_modules/openai/core/streaming.mjs +0 -238
- package/esm/node_modules/openai/core/streaming.mjs.map +0 -1
- package/esm/node_modules/openai/index.mjs +0 -8
- package/esm/node_modules/openai/internal/decoders/line.mjs +0 -91
- package/esm/node_modules/openai/internal/decoders/line.mjs.map +0 -1
- package/esm/node_modules/openai/internal/detect-platform.mjs +0 -123
- package/esm/node_modules/openai/internal/detect-platform.mjs.map +0 -1
- package/esm/node_modules/openai/internal/errors.mjs +0 -26
- package/esm/node_modules/openai/internal/errors.mjs.map +0 -1
- package/esm/node_modules/openai/internal/headers.mjs +0 -65
- package/esm/node_modules/openai/internal/headers.mjs.map +0 -1
- package/esm/node_modules/openai/internal/parse.mjs +0 -41
- package/esm/node_modules/openai/internal/parse.mjs.map +0 -1
- package/esm/node_modules/openai/internal/qs/formats.mjs +0 -12
- package/esm/node_modules/openai/internal/qs/formats.mjs.map +0 -1
- package/esm/node_modules/openai/internal/qs/stringify.mjs +0 -179
- package/esm/node_modules/openai/internal/qs/stringify.mjs.map +0 -1
- package/esm/node_modules/openai/internal/qs/utils.mjs +0 -65
- package/esm/node_modules/openai/internal/qs/utils.mjs.map +0 -1
- package/esm/node_modules/openai/internal/request-options.mjs +0 -11
- package/esm/node_modules/openai/internal/request-options.mjs.map +0 -1
- package/esm/node_modules/openai/internal/shims.mjs +0 -77
- package/esm/node_modules/openai/internal/shims.mjs.map +0 -1
- package/esm/node_modules/openai/internal/to-file.mjs +0 -63
- package/esm/node_modules/openai/internal/to-file.mjs.map +0 -1
- package/esm/node_modules/openai/internal/tslib.mjs +0 -16
- package/esm/node_modules/openai/internal/tslib.mjs.map +0 -1
- package/esm/node_modules/openai/internal/uploads.mjs +0 -94
- package/esm/node_modules/openai/internal/uploads.mjs.map +0 -1
- package/esm/node_modules/openai/internal/utils/base64.mjs +0 -22
- package/esm/node_modules/openai/internal/utils/base64.mjs.map +0 -1
- package/esm/node_modules/openai/internal/utils/bytes.mjs +0 -26
- package/esm/node_modules/openai/internal/utils/bytes.mjs.map +0 -1
- package/esm/node_modules/openai/internal/utils/env.mjs +0 -16
- package/esm/node_modules/openai/internal/utils/env.mjs.map +0 -1
- package/esm/node_modules/openai/internal/utils/log.mjs +0 -58
- package/esm/node_modules/openai/internal/utils/log.mjs.map +0 -1
- package/esm/node_modules/openai/internal/utils/path.mjs +0 -62
- package/esm/node_modules/openai/internal/utils/path.mjs.map +0 -1
- package/esm/node_modules/openai/internal/utils/query.mjs +0 -10
- package/esm/node_modules/openai/internal/utils/query.mjs.map +0 -1
- package/esm/node_modules/openai/internal/utils/sleep.mjs +0 -6
- package/esm/node_modules/openai/internal/utils/sleep.mjs.map +0 -1
- package/esm/node_modules/openai/internal/utils/uuid.mjs +0 -18
- package/esm/node_modules/openai/internal/utils/uuid.mjs.map +0 -1
- package/esm/node_modules/openai/internal/utils/values.mjs +0 -41
- package/esm/node_modules/openai/internal/utils/values.mjs.map +0 -1
- package/esm/node_modules/openai/lib/AbstractChatCompletionRunner.mjs +0 -247
- package/esm/node_modules/openai/lib/AbstractChatCompletionRunner.mjs.map +0 -1
- package/esm/node_modules/openai/lib/AssistantStream.mjs +0 -472
- package/esm/node_modules/openai/lib/AssistantStream.mjs.map +0 -1
- package/esm/node_modules/openai/lib/ChatCompletionRunner.mjs +0 -26
- package/esm/node_modules/openai/lib/ChatCompletionRunner.mjs.map +0 -1
- package/esm/node_modules/openai/lib/ChatCompletionStream.mjs +0 -423
- package/esm/node_modules/openai/lib/ChatCompletionStream.mjs.map +0 -1
- package/esm/node_modules/openai/lib/ChatCompletionStreamingRunner.mjs +0 -26
- package/esm/node_modules/openai/lib/ChatCompletionStreamingRunner.mjs.map +0 -1
- package/esm/node_modules/openai/lib/EventStream.mjs +0 -163
- package/esm/node_modules/openai/lib/EventStream.mjs.map +0 -1
- package/esm/node_modules/openai/lib/ResponsesParser.mjs +0 -94
- package/esm/node_modules/openai/lib/ResponsesParser.mjs.map +0 -1
- package/esm/node_modules/openai/lib/RunnableFunction.mjs +0 -8
- package/esm/node_modules/openai/lib/RunnableFunction.mjs.map +0 -1
- package/esm/node_modules/openai/lib/Util.mjs +0 -19
- package/esm/node_modules/openai/lib/Util.mjs.map +0 -1
- package/esm/node_modules/openai/lib/chatCompletionUtils.mjs +0 -11
- package/esm/node_modules/openai/lib/chatCompletionUtils.mjs.map +0 -1
- package/esm/node_modules/openai/lib/parser.mjs +0 -88
- package/esm/node_modules/openai/lib/parser.mjs.map +0 -1
- package/esm/node_modules/openai/lib/responses/ResponseStream.mjs +0 -231
- package/esm/node_modules/openai/lib/responses/ResponseStream.mjs.map +0 -1
- package/esm/node_modules/openai/resources/admin/admin.mjs +0 -15
- package/esm/node_modules/openai/resources/admin/admin.mjs.map +0 -1
- package/esm/node_modules/openai/resources/admin/organization/admin-api-keys.mjs +0 -81
- package/esm/node_modules/openai/resources/admin/organization/admin-api-keys.mjs.map +0 -1
- package/esm/node_modules/openai/resources/admin/organization/audit-logs.mjs +0 -31
- package/esm/node_modules/openai/resources/admin/organization/audit-logs.mjs.map +0 -1
- package/esm/node_modules/openai/resources/admin/organization/certificates.mjs +0 -153
- package/esm/node_modules/openai/resources/admin/organization/certificates.mjs.map +0 -1
- package/esm/node_modules/openai/resources/admin/organization/data-retention.mjs +0 -42
- package/esm/node_modules/openai/resources/admin/organization/data-retention.mjs.map +0 -1
- package/esm/node_modules/openai/resources/admin/organization/groups/groups.mjs +0 -106
- package/esm/node_modules/openai/resources/admin/organization/groups/groups.mjs.map +0 -1
- package/esm/node_modules/openai/resources/admin/organization/groups/roles.mjs +0 -88
- package/esm/node_modules/openai/resources/admin/organization/groups/roles.mjs.map +0 -1
- package/esm/node_modules/openai/resources/admin/organization/groups/users.mjs +0 -88
- package/esm/node_modules/openai/resources/admin/organization/groups/users.mjs.map +0 -1
- package/esm/node_modules/openai/resources/admin/organization/invites.mjs +0 -83
- package/esm/node_modules/openai/resources/admin/organization/invites.mjs.map +0 -1
- package/esm/node_modules/openai/resources/admin/organization/organization.mjs +0 -45
- package/esm/node_modules/openai/resources/admin/organization/organization.mjs.map +0 -1
- package/esm/node_modules/openai/resources/admin/organization/projects/api-keys.mjs +0 -72
- package/esm/node_modules/openai/resources/admin/organization/projects/api-keys.mjs.map +0 -1
- package/esm/node_modules/openai/resources/admin/organization/projects/certificates.mjs +0 -78
- package/esm/node_modules/openai/resources/admin/organization/projects/certificates.mjs.map +0 -1
- package/esm/node_modules/openai/resources/admin/organization/projects/data-retention.mjs +0 -46
- package/esm/node_modules/openai/resources/admin/organization/projects/data-retention.mjs.map +0 -1
- package/esm/node_modules/openai/resources/admin/organization/projects/groups/groups.mjs +0 -95
- package/esm/node_modules/openai/resources/admin/organization/projects/groups/groups.mjs.map +0 -1
- package/esm/node_modules/openai/resources/admin/organization/projects/groups/roles.mjs +0 -91
- package/esm/node_modules/openai/resources/admin/organization/projects/groups/roles.mjs.map +0 -1
- package/esm/node_modules/openai/resources/admin/organization/projects/hosted-tool-permissions.mjs +0 -45
- package/esm/node_modules/openai/resources/admin/organization/projects/hosted-tool-permissions.mjs.map +0 -1
- package/esm/node_modules/openai/resources/admin/organization/projects/model-permissions.mjs +0 -63
- package/esm/node_modules/openai/resources/admin/organization/projects/model-permissions.mjs.map +0 -1
- package/esm/node_modules/openai/resources/admin/organization/projects/projects.mjs +0 -137
- package/esm/node_modules/openai/resources/admin/organization/projects/projects.mjs.map +0 -1
- package/esm/node_modules/openai/resources/admin/organization/projects/rate-limits.mjs +0 -51
- package/esm/node_modules/openai/resources/admin/organization/projects/rate-limits.mjs.map +0 -1
- package/esm/node_modules/openai/resources/admin/organization/projects/roles.mjs +0 -108
- package/esm/node_modules/openai/resources/admin/organization/projects/roles.mjs.map +0 -1
- package/esm/node_modules/openai/resources/admin/organization/projects/service-accounts.mjs +0 -112
- package/esm/node_modules/openai/resources/admin/organization/projects/service-accounts.mjs.map +0 -1
- package/esm/node_modules/openai/resources/admin/organization/projects/spend-alerts.mjs +0 -106
- package/esm/node_modules/openai/resources/admin/organization/projects/spend-alerts.mjs.map +0 -1
- package/esm/node_modules/openai/resources/admin/organization/projects/users/roles.mjs +0 -91
- package/esm/node_modules/openai/resources/admin/organization/projects/users/roles.mjs.map +0 -1
- package/esm/node_modules/openai/resources/admin/organization/projects/users/users.mjs +0 -118
- package/esm/node_modules/openai/resources/admin/organization/projects/users/users.mjs.map +0 -1
- package/esm/node_modules/openai/resources/admin/organization/roles.mjs +0 -96
- package/esm/node_modules/openai/resources/admin/organization/roles.mjs.map +0 -1
- package/esm/node_modules/openai/resources/admin/organization/spend-alerts.mjs +0 -97
- package/esm/node_modules/openai/resources/admin/organization/spend-alerts.mjs.map +0 -1
- package/esm/node_modules/openai/resources/admin/organization/usage.mjs +0 -207
- package/esm/node_modules/openai/resources/admin/organization/usage.mjs.map +0 -1
- package/esm/node_modules/openai/resources/admin/organization/users/roles.mjs +0 -88
- package/esm/node_modules/openai/resources/admin/organization/users/roles.mjs.map +0 -1
- package/esm/node_modules/openai/resources/admin/organization/users/users.mjs +0 -82
- package/esm/node_modules/openai/resources/admin/organization/users/users.mjs.map +0 -1
- package/esm/node_modules/openai/resources/audio/audio.mjs +0 -21
- package/esm/node_modules/openai/resources/audio/audio.mjs.map +0 -1
- package/esm/node_modules/openai/resources/audio/speech.mjs +0 -39
- package/esm/node_modules/openai/resources/audio/speech.mjs.map +0 -1
- package/esm/node_modules/openai/resources/audio/transcriptions.mjs +0 -22
- package/esm/node_modules/openai/resources/audio/transcriptions.mjs.map +0 -1
- package/esm/node_modules/openai/resources/audio/translations.mjs +0 -21
- package/esm/node_modules/openai/resources/audio/translations.mjs.map +0 -1
- package/esm/node_modules/openai/resources/batches.mjs +0 -54
- package/esm/node_modules/openai/resources/batches.mjs.map +0 -1
- package/esm/node_modules/openai/resources/beta/assistants.mjs +0 -78
- package/esm/node_modules/openai/resources/beta/assistants.mjs.map +0 -1
- package/esm/node_modules/openai/resources/beta/beta.mjs +0 -24
- package/esm/node_modules/openai/resources/beta/beta.mjs.map +0 -1
- package/esm/node_modules/openai/resources/beta/chatkit/chatkit.mjs +0 -18
- package/esm/node_modules/openai/resources/beta/chatkit/chatkit.mjs.map +0 -1
- package/esm/node_modules/openai/resources/beta/chatkit/sessions.mjs +0 -49
- package/esm/node_modules/openai/resources/beta/chatkit/sessions.mjs.map +0 -1
- package/esm/node_modules/openai/resources/beta/chatkit/threads.mjs +0 -85
- package/esm/node_modules/openai/resources/beta/chatkit/threads.mjs.map +0 -1
- package/esm/node_modules/openai/resources/beta/realtime/realtime.mjs +0 -21
- package/esm/node_modules/openai/resources/beta/realtime/realtime.mjs.map +0 -1
- package/esm/node_modules/openai/resources/beta/realtime/sessions.mjs +0 -33
- package/esm/node_modules/openai/resources/beta/realtime/sessions.mjs.map +0 -1
- package/esm/node_modules/openai/resources/beta/realtime/transcription-sessions.mjs +0 -33
- package/esm/node_modules/openai/resources/beta/realtime/transcription-sessions.mjs.map +0 -1
- package/esm/node_modules/openai/resources/beta/threads/messages.mjs +0 -83
- package/esm/node_modules/openai/resources/beta/threads/messages.mjs.map +0 -1
- package/esm/node_modules/openai/resources/beta/threads/runs/runs.mjs +0 -183
- package/esm/node_modules/openai/resources/beta/threads/runs/runs.mjs.map +0 -1
- package/esm/node_modules/openai/resources/beta/threads/runs/steps.mjs +0 -45
- package/esm/node_modules/openai/resources/beta/threads/runs/steps.mjs.map +0 -1
- package/esm/node_modules/openai/resources/beta/threads/threads.mjs +0 -101
- package/esm/node_modules/openai/resources/beta/threads/threads.mjs.map +0 -1
- package/esm/node_modules/openai/resources/chat/chat.mjs +0 -15
- package/esm/node_modules/openai/resources/chat/chat.mjs.map +0 -1
- package/esm/node_modules/openai/resources/chat/completions/completions.mjs +0 -123
- package/esm/node_modules/openai/resources/chat/completions/completions.mjs.map +0 -1
- package/esm/node_modules/openai/resources/chat/completions/index.mjs +0 -7
- package/esm/node_modules/openai/resources/chat/completions/messages.mjs +0 -35
- package/esm/node_modules/openai/resources/chat/completions/messages.mjs.map +0 -1
- package/esm/node_modules/openai/resources/chat/index.mjs +0 -5
- package/esm/node_modules/openai/resources/completions.mjs +0 -20
- package/esm/node_modules/openai/resources/completions.mjs.map +0 -1
- package/esm/node_modules/openai/resources/containers/containers.mjs +0 -57
- package/esm/node_modules/openai/resources/containers/containers.mjs.map +0 -1
- package/esm/node_modules/openai/resources/containers/files/content.mjs +0 -23
- package/esm/node_modules/openai/resources/containers/files/content.mjs.map +0 -1
- package/esm/node_modules/openai/resources/containers/files/files.mjs +0 -63
- package/esm/node_modules/openai/resources/containers/files/files.mjs.map +0 -1
- package/esm/node_modules/openai/resources/conversations/conversations.mjs +0 -57
- package/esm/node_modules/openai/resources/conversations/conversations.mjs.map +0 -1
- package/esm/node_modules/openai/resources/conversations/items.mjs +0 -57
- package/esm/node_modules/openai/resources/conversations/items.mjs.map +0 -1
- package/esm/node_modules/openai/resources/embeddings.mjs +0 -48
- package/esm/node_modules/openai/resources/embeddings.mjs.map +0 -1
- package/esm/node_modules/openai/resources/evals/evals.mjs +0 -73
- package/esm/node_modules/openai/resources/evals/evals.mjs.map +0 -1
- package/esm/node_modules/openai/resources/evals/runs/output-items.mjs +0 -35
- package/esm/node_modules/openai/resources/evals/runs/output-items.mjs.map +0 -1
- package/esm/node_modules/openai/resources/evals/runs/runs.mjs +0 -72
- package/esm/node_modules/openai/resources/evals/runs/runs.mjs.map +0 -1
- package/esm/node_modules/openai/resources/files.mjs +0 -111
- package/esm/node_modules/openai/resources/files.mjs.map +0 -1
- package/esm/node_modules/openai/resources/fine-tuning/alpha/alpha.mjs +0 -15
- package/esm/node_modules/openai/resources/fine-tuning/alpha/alpha.mjs.map +0 -1
- package/esm/node_modules/openai/resources/fine-tuning/alpha/graders.mjs +0 -60
- package/esm/node_modules/openai/resources/fine-tuning/alpha/graders.mjs.map +0 -1
- package/esm/node_modules/openai/resources/fine-tuning/checkpoints/checkpoints.mjs +0 -15
- package/esm/node_modules/openai/resources/fine-tuning/checkpoints/checkpoints.mjs.map +0 -1
- package/esm/node_modules/openai/resources/fine-tuning/checkpoints/permissions.mjs +0 -102
- package/esm/node_modules/openai/resources/fine-tuning/checkpoints/permissions.mjs.map +0 -1
- package/esm/node_modules/openai/resources/fine-tuning/fine-tuning.mjs +0 -24
- package/esm/node_modules/openai/resources/fine-tuning/fine-tuning.mjs.map +0 -1
- package/esm/node_modules/openai/resources/fine-tuning/jobs/checkpoints.mjs +0 -34
- package/esm/node_modules/openai/resources/fine-tuning/jobs/checkpoints.mjs.map +0 -1
- package/esm/node_modules/openai/resources/fine-tuning/jobs/jobs.mjs +0 -148
- package/esm/node_modules/openai/resources/fine-tuning/jobs/jobs.mjs.map +0 -1
- package/esm/node_modules/openai/resources/fine-tuning/methods.mjs +0 -8
- package/esm/node_modules/openai/resources/fine-tuning/methods.mjs.map +0 -1
- package/esm/node_modules/openai/resources/graders/grader-models.mjs +0 -8
- package/esm/node_modules/openai/resources/graders/grader-models.mjs.map +0 -1
- package/esm/node_modules/openai/resources/graders/graders.mjs +0 -15
- package/esm/node_modules/openai/resources/graders/graders.mjs.map +0 -1
- package/esm/node_modules/openai/resources/images.mjs +0 -46
- package/esm/node_modules/openai/resources/images.mjs.map +0 -1
- package/esm/node_modules/openai/resources/index.mjs +0 -27
- package/esm/node_modules/openai/resources/models.mjs +0 -44
- package/esm/node_modules/openai/resources/models.mjs.map +0 -1
- package/esm/node_modules/openai/resources/moderations.mjs +0 -23
- package/esm/node_modules/openai/resources/moderations.mjs.map +0 -1
- package/esm/node_modules/openai/resources/realtime/calls.mjs +0 -79
- package/esm/node_modules/openai/resources/realtime/calls.mjs.map +0 -1
- package/esm/node_modules/openai/resources/realtime/client-secrets.mjs +0 -39
- package/esm/node_modules/openai/resources/realtime/client-secrets.mjs.map +0 -1
- package/esm/node_modules/openai/resources/realtime/realtime.mjs +0 -18
- package/esm/node_modules/openai/resources/realtime/realtime.mjs.map +0 -1
- package/esm/node_modules/openai/resources/responses/input-items.mjs +0 -31
- package/esm/node_modules/openai/resources/responses/input-items.mjs.map +0 -1
- package/esm/node_modules/openai/resources/responses/input-tokens.mjs +0 -27
- package/esm/node_modules/openai/resources/responses/input-tokens.mjs.map +0 -1
- package/esm/node_modules/openai/resources/responses/responses.mjs +0 -110
- package/esm/node_modules/openai/resources/responses/responses.mjs.map +0 -1
- package/esm/node_modules/openai/resources/skills/content.mjs +0 -22
- package/esm/node_modules/openai/resources/skills/content.mjs.map +0 -1
- package/esm/node_modules/openai/resources/skills/skills.mjs +0 -69
- package/esm/node_modules/openai/resources/skills/skills.mjs.map +0 -1
- package/esm/node_modules/openai/resources/skills/versions/content.mjs +0 -23
- package/esm/node_modules/openai/resources/skills/versions/content.mjs.map +0 -1
- package/esm/node_modules/openai/resources/skills/versions/versions.mjs +0 -58
- package/esm/node_modules/openai/resources/skills/versions/versions.mjs.map +0 -1
- package/esm/node_modules/openai/resources/uploads/parts.mjs +0 -34
- package/esm/node_modules/openai/resources/uploads/parts.mjs.map +0 -1
- package/esm/node_modules/openai/resources/uploads/uploads.mjs +0 -84
- package/esm/node_modules/openai/resources/uploads/uploads.mjs.map +0 -1
- package/esm/node_modules/openai/resources/vector-stores/file-batches.mjs +0 -126
- package/esm/node_modules/openai/resources/vector-stores/file-batches.mjs.map +0 -1
- package/esm/node_modules/openai/resources/vector-stores/files.mjs +0 -147
- package/esm/node_modules/openai/resources/vector-stores/files.mjs.map +0 -1
- package/esm/node_modules/openai/resources/vector-stores/vector-stores.mjs +0 -87
- package/esm/node_modules/openai/resources/vector-stores/vector-stores.mjs.map +0 -1
- package/esm/node_modules/openai/resources/videos.mjs +0 -115
- package/esm/node_modules/openai/resources/videos.mjs.map +0 -1
- package/esm/node_modules/openai/resources/webhooks/index.mjs +0 -3
- package/esm/node_modules/openai/resources/webhooks/webhooks.mjs +0 -69
- package/esm/node_modules/openai/resources/webhooks/webhooks.mjs.map +0 -1
- package/esm/node_modules/openai/resources/webhooks.mjs +0 -4
- package/esm/node_modules/openai/streaming.mjs +0 -3
- package/esm/node_modules/openai/version.mjs +0 -6
- package/esm/node_modules/openai/version.mjs.map +0 -1
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"rag.mjs","names":["chunkText","runRetrieve"],"sources":["../../../../../../../../../@warlock.js/ai/src/rag/rag.ts"],"sourcesContent":["import { resolveDefaultStore } from \"../config\";\nimport { chunk as chunkText } from \"./chunk/chunk\";\nimport type { ChunkOptions } from \"./contracts/chunk-options.type\";\nimport type { RetrieveOptions, RetrieveResult } from \"./contracts/citation.type\";\nimport type {\n Rag,\n RagAsToolOptions,\n RagConfig,\n} from \"./contracts/rag-config.type\";\nimport type { RagDocument } from \"./contracts/rag-document.type\";\nimport { ragAsTool } from \"./as-tool\";\nimport { retrieve as runRetrieve, type StoredChunk } from \"./retrieve\";\nimport { cacheVectorStore } from \"./store/cache-vector-store\";\nimport type { VectorStore } from \"./store/vector-store.contract\";\n\nconst DEFAULT_NAME = \"rag\";\nconst DEFAULT_NAMESPACE_PREFIX = \"ai.rag\";\n\n/**\n * Max chunk texts embedded per `embedder.embedMany()` call. One call is\n * one provider request, so a giant document is sub-batched to stay under\n * the provider's per-request token cap (the design's \"chunk larger than\n * provider per-request cap\" guard).\n */\nconst DEFAULT_MAX_BATCH = 96;\n\n/**\n * Create a RAG pipeline: **chunk → embed → vector store → retrieve →\n * rerank → cite**, reusing the app's `ai.embedder` for embedding, a\n * `@warlock.js/cache` `CacheDriver` as the vector store, and the\n * composite-as-tool engine to expose retrieval as a tool.\n *\n * Resolution is loud at construction (mirroring `memory()`):\n * - `embedder` is required — a provider with no embedder must be caught\n * here, not at first index.\n * - `store` falls back to `ai.config({ defaultStore })`; if neither\n * resolves, construction throws.\n *\n * `retrieve()` is return-only — it never auto-injects into a prompt; the\n * caller formats the cited chunks (or uses `asTool()` for the agent loop).\n * The reranker is OFF by default (cosine-only) unless `config.reranker`\n * is set.\n *\n * @example\n * import { ai } from \"@warlock.js/ai\";\n * import { MemoryCacheDriver } from \"@warlock.js/cache\";\n *\n * const kb = ai.rag({\n * name: \"docs\",\n * embedder: openai.embedder({ name: \"text-embedding-3-small\" }),\n * store: new MemoryCacheDriver(),\n * chunk: { type: \"markdown\", size: 800, overlap: 120 },\n * });\n *\n * await kb.index([{ id: \"guide\", text: longMarkdown, metadata: { url: \"/guide\" } }]);\n * const { chunks } = await kb.retrieve(\"how do I configure caching?\", { topK: 4 });\n */\nexport function rag(config: RagConfig): Rag {\n const name = config.name ?? DEFAULT_NAME;\n\n if (!config.embedder) {\n throw new Error(\n `rag(\"${name}\"): an \\`embedder\\` is required — pass one from a provider that supports embeddings (e.g. openai.embedder({ name: \"text-embedding-3-small\" }))`,\n );\n }\n\n const driver = config.store ?? resolveDefaultStore();\n\n if (!driver) {\n throw new Error(\n `rag(\"${name}\"): no store — pass \\`store\\` (a vector-capable @warlock.js/cache CacheDriver) or call \\`ai.config({ defaultStore })\\` at app boot before constructing the rag`,\n );\n }\n\n const store: VectorStore = cacheVectorStore(driver);\n const namespace = config.namespace ?? `${DEFAULT_NAMESPACE_PREFIX}.${name}`;\n const embedder = config.embedder;\n\n // Captured at first index for the dimension-mismatch guard in retrieve().\n let indexedDimensions: number | undefined;\n\n const instance: Rag = {\n name,\n\n async index(\n docs: RagDocument[],\n chunkOverride?: ChunkOptions,\n ): Promise<{ chunks: number }> {\n const chunkOptions = chunkOverride ?? config.chunk;\n\n // Ingestion guardrails (D5) — fail BEFORE any embedding spend.\n const limits = config.limits;\n if (limits?.maxDocuments !== undefined && docs.length > limits.maxDocuments) {\n throw new Error(\n `rag(\"${name}\"): index() received ${docs.length} documents, exceeding the configured maxDocuments of ${limits.maxDocuments}`,\n );\n }\n if (limits?.maxBytes !== undefined) {\n const totalBytes = docs.reduce(\n (sum, doc) => sum + Buffer.byteLength(doc.text ?? \"\"),\n 0,\n );\n if (totalBytes > limits.maxBytes) {\n throw new Error(\n `rag(\"${name}\"): index() received ${totalBytes} bytes of document text, exceeding the configured maxBytes of ${limits.maxBytes}`,\n );\n }\n }\n\n // Flatten every document into stored-chunk records + their texts,\n // preserving document order so a single batched embed maps back 1:1.\n const records: { key: string; value: StoredChunk; text: string; tags?: string[] }[] = [];\n\n for (const doc of docs) {\n const pieces = chunkText(doc.text, chunkOptions);\n\n for (const piece of pieces) {\n const value: StoredChunk = {\n sourceId: doc.id,\n chunkIndex: piece.index,\n span: piece.span,\n text: piece.text,\n metadata: doc.metadata,\n };\n\n records.push({\n key: keyFor(namespace, doc.id, piece.index),\n value,\n text: piece.text,\n tags: doc.tags,\n });\n }\n }\n\n // Empty / whitespace-only documents yield zero chunks — write\n // nothing and never embed an empty batch.\n if (records.length === 0) {\n return { chunks: 0 };\n }\n\n // Chunk cap (D5) — checked after chunking, still before embedding.\n if (limits?.maxChunks !== undefined && records.length > limits.maxChunks) {\n throw new Error(\n `rag(\"${name}\"): index() produced ${records.length} chunks, exceeding the configured maxChunks of ${limits.maxChunks}`,\n );\n }\n\n // Sub-batch the embed calls so one giant document does not blow the\n // provider's per-request token cap.\n for (let offset = 0; offset < records.length; offset += DEFAULT_MAX_BATCH) {\n const batch = records.slice(offset, offset + DEFAULT_MAX_BATCH);\n const { vectors, dimensions } = await embedder.embedMany(\n batch.map((record) => record.text),\n );\n\n if (indexedDimensions === undefined && dimensions !== 0) {\n indexedDimensions = dimensions;\n }\n\n await Promise.all(\n batch.map((record, position) =>\n store.upsert(record.key, record.value, vectors[position], record.tags),\n ),\n );\n }\n\n return { chunks: records.length };\n },\n\n async retrieve(query: string, options?: RetrieveOptions): Promise<RetrieveResult> {\n return runRetrieve(\n query,\n {\n embedder,\n store,\n namespace,\n reranker: config.reranker,\n defaults: config.retrieve,\n indexedDimensions,\n },\n options,\n );\n },\n\n async clear(): Promise<void> {\n await store.removeNamespace(namespace);\n },\n\n asTool(options?: RagAsToolOptions) {\n return ragAsTool(name, (query, retrieveOptions) => instance.retrieve(query, retrieveOptions), options);\n },\n };\n\n return instance;\n}\n\n/**\n * Namespaced key for a stored chunk. Uses the `.` separator (matching\n * `SemanticMemory.keyFor`) so namespace-prefix filtering on the returned\n * `hit.key` stays aligned with the cache's `parseKey` normalization.\n */\nfunction keyFor(namespace: string, sourceId: string, chunkIndex: number): string {\n return `${namespace}.${sourceId}.${chunkIndex}`;\n}\n"],"mappings":";;;;;;;AAeA,MAAM,eAAe;AACrB,MAAM,2BAA2B;;;;;;;AAQjC,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiC1B,SAAgB,IAAI,QAAwB;CAC1C,MAAM,OAAO,OAAO,QAAQ;CAE5B,IAAI,CAAC,OAAO,UACV,MAAM,IAAI,MACR,QAAQ,KAAK,+IACf;CAGF,MAAM,SAAS,OAAO,SAAS,oBAAoB;CAEnD,IAAI,CAAC,QACH,MAAM,IAAI,MACR,QAAQ,KAAK,+JACf;CAGF,MAAM,QAAqB,iBAAiB,MAAM;CAClD,MAAM,YAAY,OAAO,aAAa,GAAG,yBAAyB,GAAG;CACrE,MAAM,WAAW,OAAO;CAGxB,IAAI;CAEJ,MAAM,WAAgB;EACpB;EAEA,MAAM,MACJ,MACA,eAC6B;GAC7B,MAAM,eAAe,iBAAiB,OAAO;GAG7C,MAAM,SAAS,OAAO;GACtB,IAAI,QAAQ,iBAAiB,UAAa,KAAK,SAAS,OAAO,cAC7D,MAAM,IAAI,MACR,QAAQ,KAAK,uBAAuB,KAAK,OAAO,uDAAuD,OAAO,cAChH;GAEF,IAAI,QAAQ,aAAa,QAAW;IAClC,MAAM,aAAa,KAAK,QACrB,KAAK,QAAQ,MAAM,OAAO,WAAW,IAAI,QAAQ,EAAE,GACpD,CACF;IACA,IAAI,aAAa,OAAO,UACtB,MAAM,IAAI,MACR,QAAQ,KAAK,uBAAuB,WAAW,gEAAgE,OAAO,UACxH;GAEJ;GAIA,MAAM,UAAgF,CAAC;GAEvF,KAAK,MAAM,OAAO,MAAM;IACtB,MAAM,SAASA,MAAU,IAAI,MAAM,YAAY;IAE/C,KAAK,MAAM,SAAS,QAAQ;KAC1B,MAAM,QAAqB;MACzB,UAAU,IAAI;MACd,YAAY,MAAM;MAClB,MAAM,MAAM;MACZ,MAAM,MAAM;MACZ,UAAU,IAAI;KAChB;KAEA,QAAQ,KAAK;MACX,KAAK,OAAO,WAAW,IAAI,IAAI,MAAM,KAAK;MAC1C;MACA,MAAM,MAAM;MACZ,MAAM,IAAI;KACZ,CAAC;IACH;GACF;GAIA,IAAI,QAAQ,WAAW,GACrB,OAAO,EAAE,QAAQ,EAAE;GAIrB,IAAI,QAAQ,cAAc,UAAa,QAAQ,SAAS,OAAO,WAC7D,MAAM,IAAI,MACR,QAAQ,KAAK,uBAAuB,QAAQ,OAAO,iDAAiD,OAAO,WAC7G;GAKF,KAAK,IAAI,SAAS,GAAG,SAAS,QAAQ,QAAQ,UAAU,mBAAmB;IACzE,MAAM,QAAQ,QAAQ,MAAM,QAAQ,SAAS,iBAAiB;IAC9D,MAAM,EAAE,SAAS,eAAe,MAAM,SAAS,UAC7C,MAAM,KAAK,WAAW,OAAO,IAAI,CACnC;IAEA,IAAI,sBAAsB,UAAa,eAAe,GACpD,oBAAoB;IAGtB,MAAM,QAAQ,IACZ,MAAM,KAAK,QAAQ,aACjB,MAAM,OAAO,OAAO,KAAK,OAAO,OAAO,QAAQ,WAAW,OAAO,IAAI,CACvE,CACF;GACF;GAEA,OAAO,EAAE,QAAQ,QAAQ,OAAO;EAClC;EAEA,MAAM,SAAS,OAAe,SAAoD;GAChF,OAAOC,SACL,OACA;IACE;IACA;IACA;IACA,UAAU,OAAO;IACjB,UAAU,OAAO;IACjB;GACF,GACA,OACF;EACF;EAEA,MAAM,QAAuB;GAC3B,MAAM,MAAM,gBAAgB,SAAS;EACvC;EAEA,OAAO,SAA4B;GACjC,OAAO,UAAU,OAAO,OAAO,oBAAoB,SAAS,SAAS,OAAO,eAAe,GAAG,OAAO;EACvG;CACF;CAEA,OAAO;AACT;;;;;;AAOA,SAAS,OAAO,WAAmB,UAAkB,YAA4B;CAC/E,OAAO,GAAG,UAAU,GAAG,SAAS,GAAG;AACrC"}
|
|
@@ -1,58 +0,0 @@
|
|
|
1
|
-
//#region ../@warlock.js/ai/src/rag/rerank/keyword-reranker.ts
|
|
2
|
-
/** Splits text into lowercase alphanumeric terms. */
|
|
3
|
-
function tokenize(text) {
|
|
4
|
-
return text.toLowerCase().split(/[^a-z0-9]+/).filter((term) => term.length > 0);
|
|
5
|
-
}
|
|
6
|
-
/**
|
|
7
|
-
* Zero-dependency lexical reranker (a BM25-lite, IDF-free keyword overlap).
|
|
8
|
-
*
|
|
9
|
-
* For each candidate it computes the fraction of distinct query terms that
|
|
10
|
-
* appear in the chunk, blends that with the candidate's original cosine
|
|
11
|
-
* score by `weight`, and sorts descending. A pure-lexical pass costs
|
|
12
|
-
* nothing beyond string splits — no peer, no model — so it is the
|
|
13
|
-
* recommended opt-in reranker when an embedding-only ranking surfaces a
|
|
14
|
-
* keyword-rich chunk too low.
|
|
15
|
-
*
|
|
16
|
-
* Ties (equal blended score) preserve the incoming order, so the cosine
|
|
17
|
-
* ranking breaks ties deterministically.
|
|
18
|
-
*
|
|
19
|
-
* @example
|
|
20
|
-
* const kb = ai.rag({ embedder, store, reranker: ai.rag.keywordReranker() });
|
|
21
|
-
*/
|
|
22
|
-
function keywordReranker(options = {}) {
|
|
23
|
-
const weight = options.weight ?? .5;
|
|
24
|
-
return {
|
|
25
|
-
name: "keyword",
|
|
26
|
-
async rerank(query, candidates) {
|
|
27
|
-
if (candidates.length === 0) return [];
|
|
28
|
-
const queryTerms = new Set(tokenize(query));
|
|
29
|
-
if (queryTerms.size === 0) return [...candidates];
|
|
30
|
-
const scored = candidates.map((candidate, position) => {
|
|
31
|
-
const chunkTerms = new Set(tokenize(candidate.text));
|
|
32
|
-
let overlap = 0;
|
|
33
|
-
for (const term of queryTerms) if (chunkTerms.has(term)) overlap += 1;
|
|
34
|
-
return {
|
|
35
|
-
candidate,
|
|
36
|
-
blended: weight * (overlap / queryTerms.size) + (1 - weight) * candidate.score,
|
|
37
|
-
position
|
|
38
|
-
};
|
|
39
|
-
});
|
|
40
|
-
scored.sort((first, second) => {
|
|
41
|
-
if (second.blended !== first.blended) return second.blended - first.blended;
|
|
42
|
-
return first.position - second.position;
|
|
43
|
-
});
|
|
44
|
-
return scored.map((entry) => ({
|
|
45
|
-
...entry.candidate,
|
|
46
|
-
score: entry.blended,
|
|
47
|
-
citation: {
|
|
48
|
-
...entry.candidate.citation,
|
|
49
|
-
score: entry.blended
|
|
50
|
-
}
|
|
51
|
-
}));
|
|
52
|
-
}
|
|
53
|
-
};
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
//#endregion
|
|
57
|
-
export { keywordReranker };
|
|
58
|
-
//# sourceMappingURL=keyword-reranker.mjs.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"keyword-reranker.mjs","names":[],"sources":["../../../../../../../../../../@warlock.js/ai/src/rag/rerank/keyword-reranker.ts"],"sourcesContent":["import type { RetrievedChunk } from \"../contracts/citation.type\";\nimport type { RagReranker } from \"./reranker.contract\";\n\n/** Options for the {@link keywordReranker}. */\nexport type KeywordRerankerOptions = {\n /**\n * Weight of the lexical-overlap signal blended with the original cosine\n * score, in `[0, 1]`. `1` ranks purely by keyword overlap; `0` keeps the\n * cosine order. Default `0.5`.\n */\n weight?: number;\n};\n\n/** Splits text into lowercase alphanumeric terms. */\nfunction tokenize(text: string): string[] {\n return text\n .toLowerCase()\n .split(/[^a-z0-9]+/)\n .filter((term) => term.length > 0);\n}\n\n/**\n * Zero-dependency lexical reranker (a BM25-lite, IDF-free keyword overlap).\n *\n * For each candidate it computes the fraction of distinct query terms that\n * appear in the chunk, blends that with the candidate's original cosine\n * score by `weight`, and sorts descending. A pure-lexical pass costs\n * nothing beyond string splits — no peer, no model — so it is the\n * recommended opt-in reranker when an embedding-only ranking surfaces a\n * keyword-rich chunk too low.\n *\n * Ties (equal blended score) preserve the incoming order, so the cosine\n * ranking breaks ties deterministically.\n *\n * @example\n * const kb = ai.rag({ embedder, store, reranker: ai.rag.keywordReranker() });\n */\nexport function keywordReranker(options: KeywordRerankerOptions = {}): RagReranker {\n const weight = options.weight ?? 0.5;\n\n return {\n name: \"keyword\",\n async rerank(query: string, candidates: RetrievedChunk[]): Promise<RetrievedChunk[]> {\n if (candidates.length === 0) {\n return [];\n }\n\n const queryTerms = new Set(tokenize(query));\n\n if (queryTerms.size === 0) {\n return [...candidates];\n }\n\n const scored = candidates.map((candidate, position) => {\n const chunkTerms = new Set(tokenize(candidate.text));\n\n let overlap = 0;\n for (const term of queryTerms) {\n if (chunkTerms.has(term)) {\n overlap += 1;\n }\n }\n\n const lexical = overlap / queryTerms.size;\n const blended = weight * lexical + (1 - weight) * candidate.score;\n\n return { candidate, blended, position };\n });\n\n scored.sort((first, second) => {\n if (second.blended !== first.blended) {\n return second.blended - first.blended;\n }\n\n // Stable on ties: keep the incoming (cosine) order.\n return first.position - second.position;\n });\n\n return scored.map((entry) => ({\n ...entry.candidate,\n score: entry.blended,\n citation: { ...entry.candidate.citation, score: entry.blended },\n }));\n },\n };\n}\n"],"mappings":";;AAcA,SAAS,SAAS,MAAwB;CACxC,OAAO,KACJ,YAAY,CAAC,CACb,MAAM,YAAY,CAAC,CACnB,QAAQ,SAAS,KAAK,SAAS,CAAC;AACrC;;;;;;;;;;;;;;;;;AAkBA,SAAgB,gBAAgB,UAAkC,CAAC,GAAgB;CACjF,MAAM,SAAS,QAAQ,UAAU;CAEjC,OAAO;EACL,MAAM;EACN,MAAM,OAAO,OAAe,YAAyD;GACnF,IAAI,WAAW,WAAW,GACxB,OAAO,CAAC;GAGV,MAAM,aAAa,IAAI,IAAI,SAAS,KAAK,CAAC;GAE1C,IAAI,WAAW,SAAS,GACtB,OAAO,CAAC,GAAG,UAAU;GAGvB,MAAM,SAAS,WAAW,KAAK,WAAW,aAAa;IACrD,MAAM,aAAa,IAAI,IAAI,SAAS,UAAU,IAAI,CAAC;IAEnD,IAAI,UAAU;IACd,KAAK,MAAM,QAAQ,YACjB,IAAI,WAAW,IAAI,IAAI,GACrB,WAAW;IAOf,OAAO;KAAE;KAAW,SAFJ,UADA,UAAU,WAAW,SACD,IAAI,UAAU,UAAU;KAE/B;IAAS;GACxC,CAAC;GAED,OAAO,MAAM,OAAO,WAAW;IAC7B,IAAI,OAAO,YAAY,MAAM,SAC3B,OAAO,OAAO,UAAU,MAAM;IAIhC,OAAO,MAAM,WAAW,OAAO;GACjC,CAAC;GAED,OAAO,OAAO,KAAK,WAAW;IAC5B,GAAG,MAAM;IACT,OAAO,MAAM;IACb,UAAU;KAAE,GAAG,MAAM,UAAU;KAAU,OAAO,MAAM;IAAQ;GAChE,EAAE;EACJ;CACF;AACF"}
|
|
@@ -1,85 +0,0 @@
|
|
|
1
|
-
//#region ../@warlock.js/ai/src/rag/rerank/llm-reranker.ts
|
|
2
|
-
/**
|
|
3
|
-
* Build the scoring prompt — the model rates each candidate's relevance to
|
|
4
|
-
* the query on a `0..1` scale and replies with one `index: score` line per
|
|
5
|
-
* candidate. Kept terse and JSON-light so any chat model can answer.
|
|
6
|
-
*/
|
|
7
|
-
function buildPrompt(query, candidates) {
|
|
8
|
-
return [{
|
|
9
|
-
role: "system",
|
|
10
|
-
content: "You are a relevance grader. For each numbered passage, rate how well it answers the query on a scale from 0 (irrelevant) to 1 (fully relevant). Reply with ONLY a JSON array of objects like [{\"index\":0,\"score\":0.9}], one entry per passage, no prose."
|
|
11
|
-
}, {
|
|
12
|
-
role: "user",
|
|
13
|
-
content: `Query: ${query}\n\nPassages:\n${candidates.map((candidate, index) => `[${index}] ${candidate.text}`).join("\n\n")}`
|
|
14
|
-
}];
|
|
15
|
-
}
|
|
16
|
-
/**
|
|
17
|
-
* Parse the model's reply into a score map. Tolerant of surrounding prose:
|
|
18
|
-
* extracts the first JSON array and reads `{ index, score }` entries.
|
|
19
|
-
* Returns an empty map when nothing parseable is found, so the caller can
|
|
20
|
-
* fall back to the original order.
|
|
21
|
-
*/
|
|
22
|
-
function parseScores(reply) {
|
|
23
|
-
const scores = /* @__PURE__ */ new Map();
|
|
24
|
-
const start = reply.indexOf("[");
|
|
25
|
-
const end = reply.lastIndexOf("]");
|
|
26
|
-
if (start === -1 || end === -1 || end <= start) return scores;
|
|
27
|
-
let parsed;
|
|
28
|
-
try {
|
|
29
|
-
parsed = JSON.parse(reply.slice(start, end + 1));
|
|
30
|
-
} catch {
|
|
31
|
-
return scores;
|
|
32
|
-
}
|
|
33
|
-
if (!Array.isArray(parsed)) return scores;
|
|
34
|
-
for (const entry of parsed) if (entry && typeof entry.index === "number" && typeof entry.score === "number" && Number.isFinite(entry.score)) scores.set(entry.index, Math.max(0, Math.min(1, entry.score)));
|
|
35
|
-
return scores;
|
|
36
|
-
}
|
|
37
|
-
/**
|
|
38
|
-
* Optional model-backed reranker.
|
|
39
|
-
*
|
|
40
|
-
* Asks an LLM to grade each over-fetched candidate's relevance to the
|
|
41
|
-
* query on a `0..1` scale, then sorts descending by the model's score.
|
|
42
|
-
* Candidates the model does not score keep their original cosine score, so
|
|
43
|
-
* a partial/garbled reply degrades gracefully rather than dropping hits.
|
|
44
|
-
* Scoring is batched (`batchSize`) to bound prompt length.
|
|
45
|
-
*
|
|
46
|
-
* Unlike {@link keywordReranker}, this costs one or more model calls per
|
|
47
|
-
* retrieval — opt in only when precision matters more than latency/cost.
|
|
48
|
-
*
|
|
49
|
-
* @example
|
|
50
|
-
* const kb = ai.rag({
|
|
51
|
-
* embedder,
|
|
52
|
-
* store,
|
|
53
|
-
* reranker: ai.rag.llmReranker({ model: openai.model({ name: "gpt-4o-mini" }) }),
|
|
54
|
-
* });
|
|
55
|
-
*/
|
|
56
|
-
function llmReranker(options) {
|
|
57
|
-
const batchSize = options.batchSize ?? 10;
|
|
58
|
-
return {
|
|
59
|
-
name: "llm",
|
|
60
|
-
async rerank(query, candidates) {
|
|
61
|
-
if (candidates.length === 0) return [];
|
|
62
|
-
const rescored = [];
|
|
63
|
-
for (let offset = 0; offset < candidates.length; offset += batchSize) {
|
|
64
|
-
const batch = candidates.slice(offset, offset + batchSize);
|
|
65
|
-
const scores = parseScores((await options.model.complete(buildPrompt(query, batch))).content);
|
|
66
|
-
batch.forEach((candidate, index) => {
|
|
67
|
-
const score = scores.has(index) ? scores.get(index) : candidate.score;
|
|
68
|
-
rescored.push({
|
|
69
|
-
...candidate,
|
|
70
|
-
score,
|
|
71
|
-
citation: {
|
|
72
|
-
...candidate.citation,
|
|
73
|
-
score
|
|
74
|
-
}
|
|
75
|
-
});
|
|
76
|
-
});
|
|
77
|
-
}
|
|
78
|
-
return rescored.sort((first, second) => second.score - first.score);
|
|
79
|
-
}
|
|
80
|
-
};
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
//#endregion
|
|
84
|
-
export { llmReranker };
|
|
85
|
-
//# sourceMappingURL=llm-reranker.mjs.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"llm-reranker.mjs","names":[],"sources":["../../../../../../../../../../@warlock.js/ai/src/rag/rerank/llm-reranker.ts"],"sourcesContent":["import type { Message } from \"../../contracts/conversation-message.type\";\nimport type { ModelContract } from \"../../contracts/model.contract\";\nimport type { RetrievedChunk } from \"../contracts/citation.type\";\nimport type { RagReranker } from \"./reranker.contract\";\n\n/** Options for the {@link llmReranker}. */\nexport type LlmRerankerOptions = {\n /** The model used to score candidate relevance. Required. */\n model: ModelContract;\n /**\n * How many candidates to score per model call. Larger batches mean\n * fewer round-trips but a longer prompt. Default `10`.\n */\n batchSize?: number;\n};\n\n/**\n * A single relevance score the model returns for a candidate, in `[0, 1]`,\n * keyed by the candidate's position in the batch.\n */\ntype ScoreLine = {\n index: number;\n score: number;\n};\n\n/**\n * Build the scoring prompt — the model rates each candidate's relevance to\n * the query on a `0..1` scale and replies with one `index: score` line per\n * candidate. Kept terse and JSON-light so any chat model can answer.\n */\nfunction buildPrompt(query: string, candidates: RetrievedChunk[]): Message[] {\n const lines = candidates\n .map((candidate, index) => `[${index}] ${candidate.text}`)\n .join(\"\\n\\n\");\n\n return [\n {\n role: \"system\",\n content:\n \"You are a relevance grader. For each numbered passage, rate how well it answers the query on a scale from 0 (irrelevant) to 1 (fully relevant). Reply with ONLY a JSON array of objects like [{\\\"index\\\":0,\\\"score\\\":0.9}], one entry per passage, no prose.\",\n },\n {\n role: \"user\",\n content: `Query: ${query}\\n\\nPassages:\\n${lines}`,\n },\n ];\n}\n\n/**\n * Parse the model's reply into a score map. Tolerant of surrounding prose:\n * extracts the first JSON array and reads `{ index, score }` entries.\n * Returns an empty map when nothing parseable is found, so the caller can\n * fall back to the original order.\n */\nfunction parseScores(reply: string): Map<number, number> {\n const scores = new Map<number, number>();\n const start = reply.indexOf(\"[\");\n const end = reply.lastIndexOf(\"]\");\n\n if (start === -1 || end === -1 || end <= start) {\n return scores;\n }\n\n let parsed: unknown;\n try {\n parsed = JSON.parse(reply.slice(start, end + 1));\n } catch {\n return scores;\n }\n\n if (!Array.isArray(parsed)) {\n return scores;\n }\n\n for (const entry of parsed as ScoreLine[]) {\n if (\n entry &&\n typeof entry.index === \"number\" &&\n typeof entry.score === \"number\" &&\n Number.isFinite(entry.score)\n ) {\n scores.set(entry.index, Math.max(0, Math.min(1, entry.score)));\n }\n }\n\n return scores;\n}\n\n/**\n * Optional model-backed reranker.\n *\n * Asks an LLM to grade each over-fetched candidate's relevance to the\n * query on a `0..1` scale, then sorts descending by the model's score.\n * Candidates the model does not score keep their original cosine score, so\n * a partial/garbled reply degrades gracefully rather than dropping hits.\n * Scoring is batched (`batchSize`) to bound prompt length.\n *\n * Unlike {@link keywordReranker}, this costs one or more model calls per\n * retrieval — opt in only when precision matters more than latency/cost.\n *\n * @example\n * const kb = ai.rag({\n * embedder,\n * store,\n * reranker: ai.rag.llmReranker({ model: openai.model({ name: \"gpt-4o-mini\" }) }),\n * });\n */\nexport function llmReranker(options: LlmRerankerOptions): RagReranker {\n const batchSize = options.batchSize ?? 10;\n\n return {\n name: \"llm\",\n async rerank(query: string, candidates: RetrievedChunk[]): Promise<RetrievedChunk[]> {\n if (candidates.length === 0) {\n return [];\n }\n\n const rescored: RetrievedChunk[] = [];\n\n for (let offset = 0; offset < candidates.length; offset += batchSize) {\n const batch = candidates.slice(offset, offset + batchSize);\n const response = await options.model.complete(buildPrompt(query, batch));\n const scores = parseScores(response.content);\n\n batch.forEach((candidate, index) => {\n const score = scores.has(index) ? (scores.get(index) as number) : candidate.score;\n\n rescored.push({\n ...candidate,\n score,\n citation: { ...candidate.citation, score },\n });\n });\n }\n\n return rescored.sort((first, second) => second.score - first.score);\n },\n };\n}\n"],"mappings":";;;;;;AA8BA,SAAS,YAAY,OAAe,YAAyC;CAK3E,OAAO,CACL;EACE,MAAM;EACN,SACE;CACJ,GACA;EACE,MAAM;EACN,SAAS,UAAU,MAAM,iBAZf,WACX,KAAK,WAAW,UAAU,IAAI,MAAM,IAAI,UAAU,MAAM,CAAC,CACzD,KAAK,MAU0C;CAChD,CACF;AACF;;;;;;;AAQA,SAAS,YAAY,OAAoC;CACvD,MAAM,yBAAS,IAAI,IAAoB;CACvC,MAAM,QAAQ,MAAM,QAAQ,GAAG;CAC/B,MAAM,MAAM,MAAM,YAAY,GAAG;CAEjC,IAAI,UAAU,MAAM,QAAQ,MAAM,OAAO,OACvC,OAAO;CAGT,IAAI;CACJ,IAAI;EACF,SAAS,KAAK,MAAM,MAAM,MAAM,OAAO,MAAM,CAAC,CAAC;CACjD,QAAQ;EACN,OAAO;CACT;CAEA,IAAI,CAAC,MAAM,QAAQ,MAAM,GACvB,OAAO;CAGT,KAAK,MAAM,SAAS,QAClB,IACE,SACA,OAAO,MAAM,UAAU,YACvB,OAAO,MAAM,UAAU,YACvB,OAAO,SAAS,MAAM,KAAK,GAE3B,OAAO,IAAI,MAAM,OAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,MAAM,KAAK,CAAC,CAAC;CAIjE,OAAO;AACT;;;;;;;;;;;;;;;;;;;;AAqBA,SAAgB,YAAY,SAA0C;CACpE,MAAM,YAAY,QAAQ,aAAa;CAEvC,OAAO;EACL,MAAM;EACN,MAAM,OAAO,OAAe,YAAyD;GACnF,IAAI,WAAW,WAAW,GACxB,OAAO,CAAC;GAGV,MAAM,WAA6B,CAAC;GAEpC,KAAK,IAAI,SAAS,GAAG,SAAS,WAAW,QAAQ,UAAU,WAAW;IACpE,MAAM,QAAQ,WAAW,MAAM,QAAQ,SAAS,SAAS;IAEzD,MAAM,SAAS,aAAY,MADJ,QAAQ,MAAM,SAAS,YAAY,OAAO,KAAK,CAAC,EACpC,CAAC,OAAO;IAE3C,MAAM,SAAS,WAAW,UAAU;KAClC,MAAM,QAAQ,OAAO,IAAI,KAAK,IAAK,OAAO,IAAI,KAAK,IAAe,UAAU;KAE5E,SAAS,KAAK;MACZ,GAAG;MACH;MACA,UAAU;OAAE,GAAG,UAAU;OAAU;MAAM;KAC3C,CAAC;IACH,CAAC;GACH;GAEA,OAAO,SAAS,MAAM,OAAO,WAAW,OAAO,QAAQ,MAAM,KAAK;EACpE;CACF;AACF"}
|
|
@@ -1,68 +0,0 @@
|
|
|
1
|
-
//#region ../@warlock.js/ai/src/rag/retrieve.ts
|
|
2
|
-
/** Default number of chunks returned after reranking. */
|
|
3
|
-
const DEFAULT_TOP_K = 5;
|
|
4
|
-
/** Default cosine floor applied at the vector-store stage. */
|
|
5
|
-
const DEFAULT_THRESHOLD = .5;
|
|
6
|
-
/**
|
|
7
|
-
* The cite pipeline: embed the query → over-fetch candidates from the
|
|
8
|
-
* store → filter to this rag's namespace → map to {@link RetrievedChunk}s
|
|
9
|
-
* with a {@link Citation} → optionally rerank → slice `topK`.
|
|
10
|
-
*
|
|
11
|
-
* Behavior matches the design's failure modes:
|
|
12
|
-
* - No hits clearing the threshold → `{ query, chunks: [] }`, never throws.
|
|
13
|
-
* - Namespace-prefix filtering keeps two rags sharing one driver isolated.
|
|
14
|
-
* - A reranker that throws is caught; the raw cosine order is used instead.
|
|
15
|
-
* - A dimension mismatch (indexed with model A, queried with model B)
|
|
16
|
-
* throws a clear error rather than returning garbage hits.
|
|
17
|
-
*/
|
|
18
|
-
async function retrieve(query, deps, options = {}) {
|
|
19
|
-
const topK = options.topK ?? deps.defaults?.topK ?? 5;
|
|
20
|
-
const threshold = options.threshold ?? deps.defaults?.threshold ?? .5;
|
|
21
|
-
const tags = options.tags ?? deps.defaults?.tags;
|
|
22
|
-
const candidates = options.candidates ?? deps.defaults?.candidates ?? Math.max(topK * 4, topK);
|
|
23
|
-
const { vector, dimensions } = await deps.embedder.embed(query);
|
|
24
|
-
if (deps.indexedDimensions !== void 0 && dimensions !== 0 && deps.indexedDimensions !== 0 && dimensions !== deps.indexedDimensions) throw new Error(`rag.retrieve(): query embedder dimensions (${dimensions}) do not match the dimensions captured at index time (${deps.indexedDimensions}); index and query must use the same embedding model`);
|
|
25
|
-
const hits = await deps.store.query(vector, {
|
|
26
|
-
topK: candidates,
|
|
27
|
-
threshold,
|
|
28
|
-
tags
|
|
29
|
-
});
|
|
30
|
-
const prefix = `${deps.namespace}.`;
|
|
31
|
-
let retrieved = hits.filter((hit) => hit.key.startsWith(prefix)).map((hit) => toRetrievedChunk(hit.value, hit.score));
|
|
32
|
-
retrieved = await applyReranker(query, retrieved, deps.reranker);
|
|
33
|
-
return {
|
|
34
|
-
query,
|
|
35
|
-
chunks: retrieved.slice(0, topK)
|
|
36
|
-
};
|
|
37
|
-
}
|
|
38
|
-
/** Build a cited {@link RetrievedChunk} from a stored chunk + its cosine score. */
|
|
39
|
-
function toRetrievedChunk(stored, score) {
|
|
40
|
-
const citation = {
|
|
41
|
-
sourceId: stored.sourceId,
|
|
42
|
-
chunkIndex: stored.chunkIndex,
|
|
43
|
-
span: stored.span,
|
|
44
|
-
score,
|
|
45
|
-
metadata: stored.metadata
|
|
46
|
-
};
|
|
47
|
-
return {
|
|
48
|
-
text: stored.text,
|
|
49
|
-
score,
|
|
50
|
-
citation
|
|
51
|
-
};
|
|
52
|
-
}
|
|
53
|
-
/**
|
|
54
|
-
* Run the optional reranker, degrading to the raw cosine order if it
|
|
55
|
-
* throws — a flaky optional reranker must never fail the whole retrieval.
|
|
56
|
-
*/
|
|
57
|
-
async function applyReranker(query, candidates, reranker) {
|
|
58
|
-
if (!reranker) return candidates;
|
|
59
|
-
try {
|
|
60
|
-
return await reranker.rerank(query, candidates);
|
|
61
|
-
} catch {
|
|
62
|
-
return candidates;
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
//#endregion
|
|
67
|
-
export { retrieve };
|
|
68
|
-
//# sourceMappingURL=retrieve.mjs.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"retrieve.mjs","names":[],"sources":["../../../../../../../../../@warlock.js/ai/src/rag/retrieve.ts"],"sourcesContent":["import type { EmbedderContract } from \"../contracts/embedder.contract\";\nimport type {\n Citation,\n RetrievedChunk,\n RetrieveOptions,\n RetrieveResult,\n} from \"./contracts/citation.type\";\nimport type { RagReranker } from \"./rerank/reranker.contract\";\nimport type { VectorStore } from \"./store/vector-store.contract\";\n\n/** Default number of chunks returned after reranking. */\nexport const DEFAULT_TOP_K = 5;\n\n/** Default cosine floor applied at the vector-store stage. */\nexport const DEFAULT_THRESHOLD = 0.5;\n\n/**\n * Shape persisted per chunk in the vector store. The vector itself is held\n * by the driver's own index (passed via `set({ vector })`), so it is not\n * duplicated here.\n */\nexport type StoredChunk = {\n sourceId: string;\n chunkIndex: number;\n span: [start: number, end: number];\n text: string;\n metadata?: Record<string, unknown>;\n};\n\n/** Dependencies the retrieve pipeline needs, resolved once by `rag()`. */\nexport type RetrieveDeps = {\n embedder: EmbedderContract;\n store: VectorStore;\n /** Namespace prefix every stored key carries (e.g. `\"ai.rag.docs\"`). */\n namespace: string;\n /** Optional reranker; when absent the cosine order is kept. */\n reranker?: RagReranker;\n /** Pipeline-level retrieval defaults. */\n defaults?: RetrieveOptions;\n /**\n * Dimension count captured at first index for the mismatch guard. When\n * set, the query embedder's `dimensions` must equal it.\n */\n indexedDimensions?: number;\n};\n\n/**\n * The cite pipeline: embed the query → over-fetch candidates from the\n * store → filter to this rag's namespace → map to {@link RetrievedChunk}s\n * with a {@link Citation} → optionally rerank → slice `topK`.\n *\n * Behavior matches the design's failure modes:\n * - No hits clearing the threshold → `{ query, chunks: [] }`, never throws.\n * - Namespace-prefix filtering keeps two rags sharing one driver isolated.\n * - A reranker that throws is caught; the raw cosine order is used instead.\n * - A dimension mismatch (indexed with model A, queried with model B)\n * throws a clear error rather than returning garbage hits.\n */\nexport async function retrieve(\n query: string,\n deps: RetrieveDeps,\n options: RetrieveOptions = {},\n): Promise<RetrieveResult> {\n const topK = options.topK ?? deps.defaults?.topK ?? DEFAULT_TOP_K;\n const threshold = options.threshold ?? deps.defaults?.threshold ?? DEFAULT_THRESHOLD;\n const tags = options.tags ?? deps.defaults?.tags;\n const candidates = options.candidates ?? deps.defaults?.candidates ?? Math.max(topK * 4, topK);\n\n const { vector, dimensions } = await deps.embedder.embed(query);\n\n if (\n deps.indexedDimensions !== undefined &&\n dimensions !== 0 &&\n deps.indexedDimensions !== 0 &&\n dimensions !== deps.indexedDimensions\n ) {\n throw new Error(\n `rag.retrieve(): query embedder dimensions (${dimensions}) do not match the dimensions captured at index time (${deps.indexedDimensions}); index and query must use the same embedding model`,\n );\n }\n\n const hits = await deps.store.query<StoredChunk>(vector, {\n topK: candidates,\n threshold,\n tags,\n });\n\n const prefix = `${deps.namespace}.`;\n\n let retrieved: RetrievedChunk[] = hits\n .filter((hit) => hit.key.startsWith(prefix))\n .map((hit) => toRetrievedChunk(hit.value, hit.score));\n\n retrieved = await applyReranker(query, retrieved, deps.reranker);\n\n return { query, chunks: retrieved.slice(0, topK) };\n}\n\n/** Build a cited {@link RetrievedChunk} from a stored chunk + its cosine score. */\nfunction toRetrievedChunk(stored: StoredChunk, score: number): RetrievedChunk {\n const citation: Citation = {\n sourceId: stored.sourceId,\n chunkIndex: stored.chunkIndex,\n span: stored.span,\n score,\n metadata: stored.metadata,\n };\n\n return { text: stored.text, score, citation };\n}\n\n/**\n * Run the optional reranker, degrading to the raw cosine order if it\n * throws — a flaky optional reranker must never fail the whole retrieval.\n */\nasync function applyReranker(\n query: string,\n candidates: RetrievedChunk[],\n reranker: RagReranker | undefined,\n): Promise<RetrievedChunk[]> {\n if (!reranker) {\n return candidates;\n }\n\n try {\n return await reranker.rerank(query, candidates);\n } catch {\n // Logged at the call site in a richer build; here we degrade silently\n // to vector-only ranking rather than aborting the retrieval.\n return candidates;\n }\n}\n"],"mappings":";;AAWA,MAAa,gBAAgB;;AAG7B,MAAa,oBAAoB;;;;;;;;;;;;;AA4CjC,eAAsB,SACpB,OACA,MACA,UAA2B,CAAC,GACH;CACzB,MAAM,OAAO,QAAQ,QAAQ,KAAK,UAAU;CAC5C,MAAM,YAAY,QAAQ,aAAa,KAAK,UAAU;CACtD,MAAM,OAAO,QAAQ,QAAQ,KAAK,UAAU;CAC5C,MAAM,aAAa,QAAQ,cAAc,KAAK,UAAU,cAAc,KAAK,IAAI,OAAO,GAAG,IAAI;CAE7F,MAAM,EAAE,QAAQ,eAAe,MAAM,KAAK,SAAS,MAAM,KAAK;CAE9D,IACE,KAAK,sBAAsB,UAC3B,eAAe,KACf,KAAK,sBAAsB,KAC3B,eAAe,KAAK,mBAEpB,MAAM,IAAI,MACR,8CAA8C,WAAW,wDAAwD,KAAK,kBAAkB,qDAC1I;CAGF,MAAM,OAAO,MAAM,KAAK,MAAM,MAAmB,QAAQ;EACvD,MAAM;EACN;EACA;CACF,CAAC;CAED,MAAM,SAAS,GAAG,KAAK,UAAU;CAEjC,IAAI,YAA8B,KAC/B,QAAQ,QAAQ,IAAI,IAAI,WAAW,MAAM,CAAC,CAAC,CAC3C,KAAK,QAAQ,iBAAiB,IAAI,OAAO,IAAI,KAAK,CAAC;CAEtD,YAAY,MAAM,cAAc,OAAO,WAAW,KAAK,QAAQ;CAE/D,OAAO;EAAE;EAAO,QAAQ,UAAU,MAAM,GAAG,IAAI;CAAE;AACnD;;AAGA,SAAS,iBAAiB,QAAqB,OAA+B;CAC5E,MAAM,WAAqB;EACzB,UAAU,OAAO;EACjB,YAAY,OAAO;EACnB,MAAM,OAAO;EACb;EACA,UAAU,OAAO;CACnB;CAEA,OAAO;EAAE,MAAM,OAAO;EAAM;EAAO;CAAS;AAC9C;;;;;AAMA,eAAe,cACb,OACA,YACA,UAC2B;CAC3B,IAAI,CAAC,UACH,OAAO;CAGT,IAAI;EACF,OAAO,MAAM,SAAS,OAAO,OAAO,UAAU;CAChD,QAAQ;EAGN,OAAO;CACT;AACF"}
|
|
@@ -1,48 +0,0 @@
|
|
|
1
|
-
//#region ../@warlock.js/ai/src/rag/store/cache-vector-store.ts
|
|
2
|
-
/**
|
|
3
|
-
* Adapt any `@warlock.js/cache` `CacheDriver` to the {@link VectorStore}
|
|
4
|
-
* narrowing the RAG pipeline depends on. The cache driver IS the vector
|
|
5
|
-
* store — exactly as `SemanticMemory` and `semanticCache` already use it:
|
|
6
|
-
*
|
|
7
|
-
* - `upsert` → `driver.set(key, value, { vector, tags })`
|
|
8
|
-
* - `query` → `driver.similar<T>(vector, { topK, threshold, tags })`
|
|
9
|
-
* - `removeNamespace` → `driver.removeNamespace(namespace)`
|
|
10
|
-
*
|
|
11
|
-
* Drivers without similarity support throw `CacheUnsupportedError` from
|
|
12
|
-
* `set({ vector })` / `similar()`; the error surfaces unchanged so the
|
|
13
|
-
* caller sees the cache layer's own message (pointing at the `pg` /
|
|
14
|
-
* `redis` drivers for production-scale similarity).
|
|
15
|
-
*
|
|
16
|
-
* @example
|
|
17
|
-
* const store = cacheVectorStore(new MemoryCacheDriver());
|
|
18
|
-
* await store.upsert("ai.rag.docs.guide.0", { text: "…" }, vector);
|
|
19
|
-
* const hits = await store.query(queryVector, { topK: 5, threshold: 0.5 });
|
|
20
|
-
*/
|
|
21
|
-
function cacheVectorStore(driver) {
|
|
22
|
-
return {
|
|
23
|
-
async upsert(key, value, vector, tags) {
|
|
24
|
-
await driver.set(key, value, tags && tags.length > 0 ? {
|
|
25
|
-
vector,
|
|
26
|
-
tags
|
|
27
|
-
} : { vector });
|
|
28
|
-
},
|
|
29
|
-
async query(vector, options) {
|
|
30
|
-
return (await driver.similar(vector, {
|
|
31
|
-
topK: options.topK,
|
|
32
|
-
threshold: options.threshold,
|
|
33
|
-
tags: options.tags
|
|
34
|
-
})).map((hit) => ({
|
|
35
|
-
key: hit.key,
|
|
36
|
-
value: hit.value,
|
|
37
|
-
score: hit.score
|
|
38
|
-
}));
|
|
39
|
-
},
|
|
40
|
-
async removeNamespace(namespace) {
|
|
41
|
-
await driver.removeNamespace(namespace);
|
|
42
|
-
}
|
|
43
|
-
};
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
//#endregion
|
|
47
|
-
export { cacheVectorStore };
|
|
48
|
-
//# sourceMappingURL=cache-vector-store.mjs.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cache-vector-store.mjs","names":[],"sources":["../../../../../../../../../../@warlock.js/ai/src/rag/store/cache-vector-store.ts"],"sourcesContent":["import type { CacheDriver } from \"@warlock.js/cache\";\nimport type { VectorStore } from \"./vector-store.contract\";\n\n/**\n * Adapt any `@warlock.js/cache` `CacheDriver` to the {@link VectorStore}\n * narrowing the RAG pipeline depends on. The cache driver IS the vector\n * store — exactly as `SemanticMemory` and `semanticCache` already use it:\n *\n * - `upsert` → `driver.set(key, value, { vector, tags })`\n * - `query` → `driver.similar<T>(vector, { topK, threshold, tags })`\n * - `removeNamespace` → `driver.removeNamespace(namespace)`\n *\n * Drivers without similarity support throw `CacheUnsupportedError` from\n * `set({ vector })` / `similar()`; the error surfaces unchanged so the\n * caller sees the cache layer's own message (pointing at the `pg` /\n * `redis` drivers for production-scale similarity).\n *\n * @example\n * const store = cacheVectorStore(new MemoryCacheDriver());\n * await store.upsert(\"ai.rag.docs.guide.0\", { text: \"…\" }, vector);\n * const hits = await store.query(queryVector, { topK: 5, threshold: 0.5 });\n */\nexport function cacheVectorStore(driver: CacheDriver<any, any>): VectorStore {\n return {\n async upsert(\n key: string,\n value: unknown,\n vector: number[],\n tags?: string[],\n ): Promise<void> {\n await driver.set(key, value, tags && tags.length > 0 ? { vector, tags } : { vector });\n },\n\n async query<T>(\n vector: number[],\n options: { topK: number; threshold?: number; tags?: string[] },\n ): Promise<{ key: string; value: T; score: number }[]> {\n const hits = await driver.similar<T>(vector, {\n topK: options.topK,\n threshold: options.threshold,\n tags: options.tags,\n });\n\n return hits.map((hit: { key: string; value: T; score: number }) => ({\n key: hit.key,\n value: hit.value,\n score: hit.score,\n }));\n },\n\n async removeNamespace(namespace: string): Promise<void> {\n await driver.removeNamespace(namespace);\n },\n };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAsBA,SAAgB,iBAAiB,QAA4C;CAC3E,OAAO;EACL,MAAM,OACJ,KACA,OACA,QACA,MACe;GACf,MAAM,OAAO,IAAI,KAAK,OAAO,QAAQ,KAAK,SAAS,IAAI;IAAE;IAAQ;GAAK,IAAI,EAAE,OAAO,CAAC;EACtF;EAEA,MAAM,MACJ,QACA,SACqD;GAOrD,QAAO,MANY,OAAO,QAAW,QAAQ;IAC3C,MAAM,QAAQ;IACd,WAAW,QAAQ;IACnB,MAAM,QAAQ;GAChB,CAAC,EAEU,CAAC,KAAK,SAAmD;IAClE,KAAK,IAAI;IACT,OAAO,IAAI;IACX,OAAO,IAAI;GACb,EAAE;EACJ;EAEA,MAAM,gBAAgB,WAAkC;GACtD,MAAM,OAAO,gBAAgB,SAAS;EACxC;CACF;AACF"}
|
|
@@ -1,328 +0,0 @@
|
|
|
1
|
-
//#region ../@warlock.js/ai/src/rag/store/pg-vector-store.ts
|
|
2
|
-
/**
|
|
3
|
-
* Default backing table — provisions the store with no extra config when
|
|
4
|
-
* the dev runs {@link VectorStore.schema | ensureSchema} through their
|
|
5
|
-
* migration tool.
|
|
6
|
-
*/
|
|
7
|
-
const DEFAULT_TABLE = "warlock_ai_rag_vectors";
|
|
8
|
-
/** Default embedding width baked into the DDL (OpenAI `text-embedding-3-small`). */
|
|
9
|
-
const DEFAULT_DIMENSIONS = 1536;
|
|
10
|
-
/** Default `ivfflat` list count when that index strategy is chosen. */
|
|
11
|
-
const DEFAULT_IVFFLAT_LISTS = 100;
|
|
12
|
-
/**
|
|
13
|
-
* Allowed characters in a Postgres identifier (table name). The table name
|
|
14
|
-
* is interpolated into DDL/DML, so anything outside this conservative
|
|
15
|
-
* ASCII subset is rejected — interpolating an arbitrary string would be a
|
|
16
|
-
* SQL-injection footgun (mirrors the snapshot / human-interrupt pg stores
|
|
17
|
-
* and `@warlock.js/cache`'s `PgCacheDriver`).
|
|
18
|
-
*/
|
|
19
|
-
const SAFE_IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
20
|
-
/**
|
|
21
|
-
* Module specifier for the optional `pg` driver. Held in a `string`
|
|
22
|
-
* variable so the dynamic `import()` is not statically resolved at compile
|
|
23
|
-
* time — `pg` is an optional peer that need not be installed for this
|
|
24
|
-
* package to type-check or for a cache-only consumer to run.
|
|
25
|
-
*/
|
|
26
|
-
const PG_MODULE = "pg";
|
|
27
|
-
/**
|
|
28
|
-
* Curated install string surfaced (at use time) when a `connectionString`
|
|
29
|
-
* is configured but the optional `pg` driver is absent. Never thrown at
|
|
30
|
-
* import — a cache-only consumer must be able to load this module.
|
|
31
|
-
*/
|
|
32
|
-
const PG_INSTALL_INSTRUCTIONS = `
|
|
33
|
-
The @warlock.js/ai Postgres vector store requires the pg package and a
|
|
34
|
-
Postgres database with the pgvector extension. Install the driver with:
|
|
35
|
-
|
|
36
|
-
npm install pg
|
|
37
|
-
|
|
38
|
-
Or with your preferred package manager:
|
|
39
|
-
|
|
40
|
-
pnpm add pg
|
|
41
|
-
yarn add pg
|
|
42
|
-
`.trim();
|
|
43
|
-
/**
|
|
44
|
-
* Lazily import `pg` and return a `Pool` built from `connectionString`. A
|
|
45
|
-
* bare `catch` rethrows the curated install string — a missing optional
|
|
46
|
-
* peer surfaces as actionable guidance, never a raw resolution error.
|
|
47
|
-
*/
|
|
48
|
-
async function buildPgClient(connectionString) {
|
|
49
|
-
let sdk;
|
|
50
|
-
try {
|
|
51
|
-
sdk = await import(PG_MODULE);
|
|
52
|
-
} catch {
|
|
53
|
-
throw new Error(PG_INSTALL_INSTRUCTIONS);
|
|
54
|
-
}
|
|
55
|
-
return new sdk.Pool({ connectionString });
|
|
56
|
-
}
|
|
57
|
-
/**
|
|
58
|
-
* Serialize a JS `number[]` to the pgvector text literal: `[1,2,3]`.
|
|
59
|
-
* pgvector accepts a vector either as this bracketed literal or via a
|
|
60
|
-
* typed parameter; passing the literal string + an explicit `::vector`
|
|
61
|
-
* cast keeps the store driver-agnostic (no dependency on a registered
|
|
62
|
-
* `pg` type parser).
|
|
63
|
-
*
|
|
64
|
-
* Non-finite components (`NaN` / `±Infinity`) are rejected — pgvector
|
|
65
|
-
* stores only finite floats, and silently coercing them would corrupt the
|
|
66
|
-
* index. The check is cheap relative to the embed call that produced the
|
|
67
|
-
* vector.
|
|
68
|
-
*
|
|
69
|
-
* @example
|
|
70
|
-
* vectorLiteral([1, 0.5, -2]); // "[1,0.5,-2]"
|
|
71
|
-
*/
|
|
72
|
-
function vectorLiteral(vector) {
|
|
73
|
-
let literal = "[";
|
|
74
|
-
for (let index = 0; index < vector.length; index++) {
|
|
75
|
-
const component = vector[index];
|
|
76
|
-
if (!Number.isFinite(component)) throw new TypeError(`pgVectorStore: embedding component at index ${index} is not finite (${component}); pgvector stores only finite floats.`);
|
|
77
|
-
if (index > 0) literal += ",";
|
|
78
|
-
literal += String(component);
|
|
79
|
-
}
|
|
80
|
-
return literal + "]";
|
|
81
|
-
}
|
|
82
|
-
/**
|
|
83
|
-
* Coerce a `value` JSONB column back into the stored payload. node-postgres
|
|
84
|
-
* parses `JSONB` into a JS value already, but some pool wrappers hand back
|
|
85
|
-
* the raw string — be defensive across both (mirrors the snapshot store's
|
|
86
|
-
* `parsePayload`).
|
|
87
|
-
*/
|
|
88
|
-
function parseValue(value) {
|
|
89
|
-
if (typeof value === "string") return JSON.parse(value);
|
|
90
|
-
return value;
|
|
91
|
-
}
|
|
92
|
-
/**
|
|
93
|
-
* Coerce a pgvector cosine **distance** (`<=>`, in `[0, 2]`, 0 = identical)
|
|
94
|
-
* into the cosine **similarity** score the {@link VectorStore} contract
|
|
95
|
-
* declares (`[0, 1]`, 1 = identical). `pg` returns the computed distance
|
|
96
|
-
* column as a string for `double precision`; parse then map `1 - distance`,
|
|
97
|
-
* clamped to `[0, 1]` so a tiny floating-point overshoot never yields a
|
|
98
|
-
* score just outside the contract's range.
|
|
99
|
-
*/
|
|
100
|
-
function distanceToScore(distance) {
|
|
101
|
-
const score = 1 - (typeof distance === "string" ? Number(distance) : distance);
|
|
102
|
-
if (score < 0) return 0;
|
|
103
|
-
if (score > 1) return 1;
|
|
104
|
-
return score;
|
|
105
|
-
}
|
|
106
|
-
/**
|
|
107
|
-
* Postgres + pgvector-backed {@link VectorStore} — one durable row per
|
|
108
|
-
* indexed chunk, keyed by the RAG pipeline's dotted `key`
|
|
109
|
-
* (`ai.rag.<name>.<sourceId>.<chunkIndex>`), with the chunk payload in a
|
|
110
|
-
* `value` JSONB column and the embedding in a `vector` column.
|
|
111
|
-
*
|
|
112
|
-
* Owns: the three RAG vector operations against a pgvector index —
|
|
113
|
-
* `upsert` (INSERT … ON CONFLICT DO UPDATE), `query` (cosine
|
|
114
|
-
* `ORDER BY embedding <=> $vec` with a `threshold` floor + optional `tags`
|
|
115
|
-
* overlap filter, capped at `topK`), and `removeNamespace` (prefix DELETE).
|
|
116
|
-
* Also emits the reference DDL via {@link PgVectorStore.schema} (alias
|
|
117
|
-
* {@link PgVectorStore.ensureSchema}).
|
|
118
|
-
*
|
|
119
|
-
* Does NOT own: the connection lifecycle (a dev-supplied `client` is never
|
|
120
|
-
* closed; a store-built `Pool` from a `connectionString` is also left open
|
|
121
|
-
* for the process to reuse) or schema migration — the dev runs the DDL
|
|
122
|
-
* through their own tool; the framework never auto-migrates.
|
|
123
|
-
*
|
|
124
|
-
* Front it with the {@link pgVectorStore} factory — callers never `new` it.
|
|
125
|
-
*/
|
|
126
|
-
var PgVectorStore = class {
|
|
127
|
-
constructor(options) {
|
|
128
|
-
const table = options.table ?? DEFAULT_TABLE;
|
|
129
|
-
if (!SAFE_IDENTIFIER.test(table)) throw new TypeError(`pgVectorStore: invalid table name '${table}'. Allowed: [A-Za-z_][A-Za-z0-9_]*.`);
|
|
130
|
-
this.table = table;
|
|
131
|
-
this.dimensions = options.dimensions ?? DEFAULT_DIMENSIONS;
|
|
132
|
-
this.index = options.index ?? "hnsw";
|
|
133
|
-
this.ivfflatLists = options.ivfflatLists ?? DEFAULT_IVFFLAT_LISTS;
|
|
134
|
-
if (options.client) {
|
|
135
|
-
if (typeof options.client.query !== "function") throw new TypeError("pgVectorStore requires a 'client' option implementing { query(text, params) } — pass a pg.Pool or pg.Client.");
|
|
136
|
-
this.clientPromise = Promise.resolve(options.client);
|
|
137
|
-
return;
|
|
138
|
-
}
|
|
139
|
-
if (options.connectionString) {
|
|
140
|
-
this.clientPromise = buildPgClient(options.connectionString);
|
|
141
|
-
return;
|
|
142
|
-
}
|
|
143
|
-
throw new TypeError("pgVectorStore requires either a 'client' or a 'connectionString' option.");
|
|
144
|
-
}
|
|
145
|
-
/**
|
|
146
|
-
* Resolve the backing client, surfacing the lazy `pg` import's curated
|
|
147
|
-
* install string on the first call that needs it.
|
|
148
|
-
*/
|
|
149
|
-
client() {
|
|
150
|
-
return this.clientPromise;
|
|
151
|
-
}
|
|
152
|
-
/**
|
|
153
|
-
* Index `value` under `key` with its embedding `vector`. Upserts on the
|
|
154
|
-
* `key` primary key — re-indexing the same chunk overwrites its payload,
|
|
155
|
-
* embedding, and tags rather than appending a duplicate row. Optional
|
|
156
|
-
* `tags` ride a `text[]` column so {@link query} can restrict the
|
|
157
|
-
* candidate set with an array-overlap filter.
|
|
158
|
-
*
|
|
159
|
-
* The embedding is sent as a pgvector text literal (`$3`) cast to
|
|
160
|
-
* `::vector`, so the store needs no registered `pg` type parser. `tags`
|
|
161
|
-
* defaults to an empty array (never `NULL`) to keep the overlap filter's
|
|
162
|
-
* `&&` semantics simple.
|
|
163
|
-
*/
|
|
164
|
-
async upsert(key, value, vector, tags) {
|
|
165
|
-
await (await this.client()).query(`INSERT INTO ${this.table} (key, value, embedding, tags)
|
|
166
|
-
VALUES ($1, $2::jsonb, $3::vector, $4::text[])
|
|
167
|
-
ON CONFLICT (key) DO UPDATE
|
|
168
|
-
SET value = EXCLUDED.value,
|
|
169
|
-
embedding = EXCLUDED.embedding,
|
|
170
|
-
tags = EXCLUDED.tags`, [
|
|
171
|
-
key,
|
|
172
|
-
JSON.stringify(value),
|
|
173
|
-
vectorLiteral(vector),
|
|
174
|
-
tags ?? []
|
|
175
|
-
]);
|
|
176
|
-
}
|
|
177
|
-
/**
|
|
178
|
-
* Return the cosine-nearest rows to `vector`, mapped to the contract's
|
|
179
|
-
* `{ key, value, score }` shape. The SQL:
|
|
180
|
-
*
|
|
181
|
-
* - computes `embedding <=> $1::vector` (cosine **distance**) once, aliased
|
|
182
|
-
* `distance`, and `ORDER BY` it ascending (nearest first);
|
|
183
|
-
* - applies the `threshold` floor as `distance <= 1 - threshold`
|
|
184
|
-
* (similarity `>=` threshold), so the default `0.5` floor maps to a
|
|
185
|
-
* `<= 0.5` distance bound — the filter runs in SQL, not in JS, so a
|
|
186
|
-
* below-floor row never crosses the wire;
|
|
187
|
-
* - when `tags` are given, restricts to rows whose `tags` array overlaps
|
|
188
|
-
* the requested set via `tags && $tags::text[]` (one-of semantics,
|
|
189
|
-
* matching the cache store);
|
|
190
|
-
* - caps the result at `topK` with `LIMIT`.
|
|
191
|
-
*
|
|
192
|
-
* The returned `score` is `1 - distance`, clamped to `[0, 1]`, so callers
|
|
193
|
-
* see the same cosine-similarity scale the cache store emits.
|
|
194
|
-
*/
|
|
195
|
-
async query(vector, options) {
|
|
196
|
-
const client = await this.client();
|
|
197
|
-
const params = [vectorLiteral(vector), options.topK];
|
|
198
|
-
const conditions = [];
|
|
199
|
-
if (options.threshold !== void 0) {
|
|
200
|
-
params.push(1 - options.threshold);
|
|
201
|
-
conditions.push(`(embedding <=> $1::vector) <= $${params.length}`);
|
|
202
|
-
}
|
|
203
|
-
if (options.tags !== void 0 && options.tags.length > 0) {
|
|
204
|
-
params.push(options.tags);
|
|
205
|
-
conditions.push(`tags && $${params.length}::text[]`);
|
|
206
|
-
}
|
|
207
|
-
const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
208
|
-
const { rows } = await client.query(`SELECT key, value, (embedding <=> $1::vector) AS distance
|
|
209
|
-
FROM ${this.table}
|
|
210
|
-
${where}
|
|
211
|
-
ORDER BY embedding <=> $1::vector
|
|
212
|
-
LIMIT $2`, params);
|
|
213
|
-
return rows.map((row) => ({
|
|
214
|
-
key: row.key,
|
|
215
|
-
value: parseValue(row.value),
|
|
216
|
-
score: distanceToScore(row.distance)
|
|
217
|
-
}));
|
|
218
|
-
}
|
|
219
|
-
/**
|
|
220
|
-
* Drop every entry written under `namespace`. The RAG pipeline keys
|
|
221
|
-
* chunks as `<namespace>.<sourceId>.<chunkIndex>`, so a row belongs to
|
|
222
|
-
* the namespace when its `key` equals it OR begins with `<namespace>.`
|
|
223
|
-
* — deleting `ai.rag.docs` must not also catch `ai.rag.docs2`. The `_`
|
|
224
|
-
* and `%` LIKE wildcards in the namespace are escaped so a namespace
|
|
225
|
-
* that happens to contain them is matched literally.
|
|
226
|
-
*/
|
|
227
|
-
async removeNamespace(namespace) {
|
|
228
|
-
const client = await this.client();
|
|
229
|
-
const escaped = namespace.replace(/\\/g, "\\\\").replace(/_/g, "\\_").replace(/%/g, "\\%");
|
|
230
|
-
await client.query(`DELETE FROM ${this.table}
|
|
231
|
-
WHERE key = $1 OR key LIKE $2 ESCAPE '\\'`, [namespace, `${escaped}.%`]);
|
|
232
|
-
}
|
|
233
|
-
/**
|
|
234
|
-
* Return the reference migration DDL for this store's backing table,
|
|
235
|
-
* interpolating the configured table name, embedding width, and ANN
|
|
236
|
-
* index strategy. The dev runs it once through their migration tool —
|
|
237
|
-
* the framework never auto-migrates.
|
|
238
|
-
*
|
|
239
|
-
* The emitted statements:
|
|
240
|
-
* 1. `CREATE EXTENSION IF NOT EXISTS vector;` — enables pgvector (needs
|
|
241
|
-
* a superuser or a role with `CREATE` on the database the first time).
|
|
242
|
-
* 2. `CREATE TABLE IF NOT EXISTS <table> (key TEXT PRIMARY KEY, value
|
|
243
|
-
* JSONB NOT NULL, embedding vector(<dimensions>) NOT NULL, tags
|
|
244
|
-
* text[] NOT NULL DEFAULT '{}');`
|
|
245
|
-
* 3. A GIN index on `tags` so the array-overlap filter stays sargable.
|
|
246
|
-
* 4. The chosen ANN index over `embedding` using `vector_cosine_ops`:
|
|
247
|
-
* - `"hnsw"` → `USING hnsw (embedding vector_cosine_ops)`;
|
|
248
|
-
* - `"ivfflat"` → `USING ivfflat (embedding vector_cosine_ops)
|
|
249
|
-
* WITH (lists = <ivfflatLists>)`;
|
|
250
|
-
* - `"none"` → emitted as a comment (exact scan, no ANN index).
|
|
251
|
-
*
|
|
252
|
-
* @example
|
|
253
|
-
* const store = pgVectorStore({ client: pool, dimensions: 1536 });
|
|
254
|
-
* await pool.query(store.ensureSchema());
|
|
255
|
-
*/
|
|
256
|
-
schema() {
|
|
257
|
-
const lines = [
|
|
258
|
-
`CREATE EXTENSION IF NOT EXISTS vector;`,
|
|
259
|
-
`CREATE TABLE IF NOT EXISTS ${this.table} (`,
|
|
260
|
-
` key TEXT PRIMARY KEY,`,
|
|
261
|
-
` value JSONB NOT NULL,`,
|
|
262
|
-
` embedding vector(${this.dimensions}) NOT NULL,`,
|
|
263
|
-
` tags TEXT[] NOT NULL DEFAULT '{}'`,
|
|
264
|
-
`);`,
|
|
265
|
-
`CREATE INDEX IF NOT EXISTS idx_${this.table}_tags`,
|
|
266
|
-
` ON ${this.table} USING gin (tags);`
|
|
267
|
-
];
|
|
268
|
-
if (this.index === "hnsw") lines.push(`CREATE INDEX IF NOT EXISTS idx_${this.table}_embedding`, ` ON ${this.table} USING hnsw (embedding vector_cosine_ops);`);
|
|
269
|
-
else if (this.index === "ivfflat") lines.push(`CREATE INDEX IF NOT EXISTS idx_${this.table}_embedding`, ` ON ${this.table} USING ivfflat (embedding vector_cosine_ops)`, ` WITH (lists = ${this.ivfflatLists});`);
|
|
270
|
-
else lines.push(`-- No ANN index requested (index: "none"): cosine queries fall back`, `-- to an exact sequential scan, which is correct but linear in rows.`);
|
|
271
|
-
return lines.join("\n");
|
|
272
|
-
}
|
|
273
|
-
/**
|
|
274
|
-
* Alias for {@link PgVectorStore.schema} — reads more naturally in a
|
|
275
|
-
* migration script (`await pool.query(store.ensureSchema())`). Returns
|
|
276
|
-
* the identical DDL string; it does NOT execute anything against the
|
|
277
|
-
* database (the store never auto-migrates).
|
|
278
|
-
*/
|
|
279
|
-
ensureSchema() {
|
|
280
|
-
return this.schema();
|
|
281
|
-
}
|
|
282
|
-
};
|
|
283
|
-
/**
|
|
284
|
-
* Create a Postgres + pgvector-backed {@link VectorStore} for the RAG
|
|
285
|
-
* pipeline. Either pass a live `pg.Pool` / `pg.Client` (`{ client }`) —
|
|
286
|
-
* `@warlock.js/ai` never imports `pg` in that case — or a
|
|
287
|
-
* `{ connectionString }` and let the store lazily `import("pg")` to build
|
|
288
|
-
* its own pool. When `pg` is not installed, a curated install string
|
|
289
|
-
* surfaces on first use, never at import.
|
|
290
|
-
*
|
|
291
|
-
* Run {@link PgVectorStoreInstance.ensureSchema} through your migration
|
|
292
|
-
* tool once before use (it enables the `vector` extension, creates the
|
|
293
|
-
* table, and builds the tag + ANN indexes); the store never auto-migrates.
|
|
294
|
-
*
|
|
295
|
-
* Index and query MUST use the same embedding model — the `vector(N)`
|
|
296
|
-
* column width is fixed at table-creation time from `dimensions`.
|
|
297
|
-
*
|
|
298
|
-
* @example
|
|
299
|
-
* import { Pool } from "pg";
|
|
300
|
-
* import { ai } from "@warlock.js/ai";
|
|
301
|
-
*
|
|
302
|
-
* const pool = new Pool({ connectionString: process.env.DATABASE_URL });
|
|
303
|
-
* const store = ai.rag.pgVectorStore({ client: pool, dimensions: 1536 });
|
|
304
|
-
*
|
|
305
|
-
* // Once, via your migration tooling:
|
|
306
|
-
* // await pool.query(store.ensureSchema());
|
|
307
|
-
*
|
|
308
|
-
* const kb = ai.rag({
|
|
309
|
-
* name: "docs",
|
|
310
|
-
* embedder: openai.embedder({ name: "text-embedding-3-small" }),
|
|
311
|
-
* store,
|
|
312
|
-
* });
|
|
313
|
-
*
|
|
314
|
-
* @example
|
|
315
|
-
* // Let the store build its own pool from a connection string:
|
|
316
|
-
* const store = ai.rag.pgVectorStore({
|
|
317
|
-
* connectionString: process.env.DATABASE_URL,
|
|
318
|
-
* index: "ivfflat",
|
|
319
|
-
* ivfflatLists: 200,
|
|
320
|
-
* });
|
|
321
|
-
*/
|
|
322
|
-
function pgVectorStore(options) {
|
|
323
|
-
return new PgVectorStore(options);
|
|
324
|
-
}
|
|
325
|
-
|
|
326
|
-
//#endregion
|
|
327
|
-
export { pgVectorStore, vectorLiteral };
|
|
328
|
-
//# sourceMappingURL=pg-vector-store.mjs.map
|