@revealui/ai 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +22 -0
- package/LICENSE.commercial +112 -0
- package/README.md +314 -0
- package/dist/a2a/card.d.ts +26 -0
- package/dist/a2a/card.d.ts.map +1 -0
- package/dist/a2a/card.js +173 -0
- package/dist/a2a/handler.d.ts +26 -0
- package/dist/a2a/handler.d.ts.map +1 -0
- package/dist/a2a/handler.js +170 -0
- package/dist/a2a/index.d.ts +10 -0
- package/dist/a2a/index.d.ts.map +1 -0
- package/dist/a2a/index.js +9 -0
- package/dist/a2a/task-store.d.ts +42 -0
- package/dist/a2a/task-store.d.ts.map +1 -0
- package/dist/a2a/task-store.js +99 -0
- package/dist/audit/emitter.d.ts +34 -0
- package/dist/audit/emitter.d.ts.map +1 -0
- package/dist/audit/emitter.js +34 -0
- package/dist/audit/index.d.ts +44 -0
- package/dist/audit/index.d.ts.map +1 -0
- package/dist/audit/index.js +48 -0
- package/dist/audit/observer.d.ts +108 -0
- package/dist/audit/observer.d.ts.map +1 -0
- package/dist/audit/observer.js +271 -0
- package/dist/audit/policy.d.ts +70 -0
- package/dist/audit/policy.d.ts.map +1 -0
- package/dist/audit/policy.js +209 -0
- package/dist/audit/store.d.ts +42 -0
- package/dist/audit/store.d.ts.map +1 -0
- package/dist/audit/store.js +80 -0
- package/dist/audit/types.d.ts +169 -0
- package/dist/audit/types.d.ts.map +1 -0
- package/dist/audit/types.js +80 -0
- package/dist/client/hooks/index.d.ts +22 -0
- package/dist/client/hooks/index.d.ts.map +1 -0
- package/dist/client/hooks/index.js +21 -0
- package/dist/client/hooks/useAgentContext.d.ts +30 -0
- package/dist/client/hooks/useAgentContext.d.ts.map +1 -0
- package/dist/client/hooks/useAgentContext.js +161 -0
- package/dist/client/hooks/useAgentEvents.d.ts +126 -0
- package/dist/client/hooks/useAgentEvents.d.ts.map +1 -0
- package/dist/client/hooks/useAgentEvents.js +232 -0
- package/dist/client/hooks/useAgentStream.d.ts +44 -0
- package/dist/client/hooks/useAgentStream.d.ts.map +1 -0
- package/dist/client/hooks/useAgentStream.js +101 -0
- package/dist/client/hooks/useEpisodicMemory.d.ts +25 -0
- package/dist/client/hooks/useEpisodicMemory.d.ts.map +1 -0
- package/dist/client/hooks/useEpisodicMemory.js +174 -0
- package/dist/client/hooks/useWorkingMemory.d.ts +57 -0
- package/dist/client/hooks/useWorkingMemory.d.ts.map +1 -0
- package/dist/client/hooks/useWorkingMemory.js +276 -0
- package/dist/client/index.d.ts +14 -0
- package/dist/client/index.d.ts.map +1 -0
- package/dist/client/index.js +13 -0
- package/dist/embeddings/index.d.ts +51 -0
- package/dist/embeddings/index.d.ts.map +1 -0
- package/dist/embeddings/index.js +73 -0
- package/dist/index.d.ts +83 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +103 -0
- package/dist/inference/context-assembly.d.ts +27 -0
- package/dist/inference/context-assembly.d.ts.map +1 -0
- package/dist/inference/context-assembly.js +81 -0
- package/dist/inference/overflow-compressor.d.ts +17 -0
- package/dist/inference/overflow-compressor.d.ts.map +1 -0
- package/dist/inference/overflow-compressor.js +40 -0
- package/dist/inference/runRag.d.ts +35 -0
- package/dist/inference/runRag.d.ts.map +1 -0
- package/dist/inference/runRag.js +53 -0
- package/dist/ingestion/bm25.d.ts +29 -0
- package/dist/ingestion/bm25.d.ts.map +1 -0
- package/dist/ingestion/bm25.js +161 -0
- package/dist/ingestion/cms-indexer.d.ts +39 -0
- package/dist/ingestion/cms-indexer.d.ts.map +1 -0
- package/dist/ingestion/cms-indexer.js +74 -0
- package/dist/ingestion/file-parsers.d.ts +51 -0
- package/dist/ingestion/file-parsers.d.ts.map +1 -0
- package/dist/ingestion/file-parsers.js +247 -0
- package/dist/ingestion/hybrid-search.d.ts +22 -0
- package/dist/ingestion/hybrid-search.d.ts.map +1 -0
- package/dist/ingestion/hybrid-search.js +63 -0
- package/dist/ingestion/index.d.ts +9 -0
- package/dist/ingestion/index.d.ts.map +1 -0
- package/dist/ingestion/index.js +8 -0
- package/dist/ingestion/pipeline.d.ts +35 -0
- package/dist/ingestion/pipeline.d.ts.map +1 -0
- package/dist/ingestion/pipeline.js +114 -0
- package/dist/ingestion/rag-vector-service.d.ts +34 -0
- package/dist/ingestion/rag-vector-service.d.ts.map +1 -0
- package/dist/ingestion/rag-vector-service.js +98 -0
- package/dist/ingestion/reranker.d.ts +10 -0
- package/dist/ingestion/reranker.d.ts.map +1 -0
- package/dist/ingestion/reranker.js +41 -0
- package/dist/ingestion/text-splitter.d.ts +25 -0
- package/dist/ingestion/text-splitter.d.ts.map +1 -0
- package/dist/ingestion/text-splitter.js +119 -0
- package/dist/llm/cache-utils.d.ts +146 -0
- package/dist/llm/cache-utils.d.ts.map +1 -0
- package/dist/llm/cache-utils.js +204 -0
- package/dist/llm/client.d.ts +134 -0
- package/dist/llm/client.d.ts.map +1 -0
- package/dist/llm/client.js +497 -0
- package/dist/llm/key-validator.d.ts +25 -0
- package/dist/llm/key-validator.d.ts.map +1 -0
- package/dist/llm/key-validator.js +101 -0
- package/dist/llm/provider-health.d.ts +40 -0
- package/dist/llm/provider-health.d.ts.map +1 -0
- package/dist/llm/provider-health.js +97 -0
- package/dist/llm/providers/anthropic.d.ts +31 -0
- package/dist/llm/providers/anthropic.d.ts.map +1 -0
- package/dist/llm/providers/anthropic.js +248 -0
- package/dist/llm/providers/base.d.ts +111 -0
- package/dist/llm/providers/base.d.ts.map +1 -0
- package/dist/llm/providers/base.js +6 -0
- package/dist/llm/providers/groq.d.ts +23 -0
- package/dist/llm/providers/groq.d.ts.map +1 -0
- package/dist/llm/providers/groq.js +27 -0
- package/dist/llm/providers/ollama.d.ts +27 -0
- package/dist/llm/providers/ollama.d.ts.map +1 -0
- package/dist/llm/providers/ollama.js +48 -0
- package/dist/llm/providers/openai.d.ts +19 -0
- package/dist/llm/providers/openai.d.ts.map +1 -0
- package/dist/llm/providers/openai.js +245 -0
- package/dist/llm/providers/vultr.d.ts +18 -0
- package/dist/llm/providers/vultr.d.ts.map +1 -0
- package/dist/llm/providers/vultr.js +168 -0
- package/dist/llm/response-cache.d.ts +166 -0
- package/dist/llm/response-cache.d.ts.map +1 -0
- package/dist/llm/response-cache.js +233 -0
- package/dist/llm/semantic-cache.d.ts +179 -0
- package/dist/llm/semantic-cache.d.ts.map +1 -0
- package/dist/llm/semantic-cache.js +306 -0
- package/dist/llm/server.d.ts +14 -0
- package/dist/llm/server.d.ts.map +1 -0
- package/dist/llm/server.js +15 -0
- package/dist/llm/token-counter.d.ts +48 -0
- package/dist/llm/token-counter.d.ts.map +1 -0
- package/dist/llm/token-counter.js +77 -0
- package/dist/llm/workspace-provider-config.d.ts +38 -0
- package/dist/llm/workspace-provider-config.d.ts.map +1 -0
- package/dist/llm/workspace-provider-config.js +47 -0
- package/dist/memory/agent/context-manager.d.ts +148 -0
- package/dist/memory/agent/context-manager.d.ts.map +1 -0
- package/dist/memory/agent/context-manager.js +284 -0
- package/dist/memory/agent/index.d.ts +7 -0
- package/dist/memory/agent/index.d.ts.map +1 -0
- package/dist/memory/agent/index.js +6 -0
- package/dist/memory/crdt/index.d.ts +13 -0
- package/dist/memory/crdt/index.d.ts.map +1 -0
- package/dist/memory/crdt/index.js +12 -0
- package/dist/memory/crdt/lww-register.d.ts +108 -0
- package/dist/memory/crdt/lww-register.d.ts.map +1 -0
- package/dist/memory/crdt/lww-register.js +169 -0
- package/dist/memory/crdt/or-set.d.ts +141 -0
- package/dist/memory/crdt/or-set.d.ts.map +1 -0
- package/dist/memory/crdt/or-set.js +291 -0
- package/dist/memory/crdt/pn-counter.d.ts +116 -0
- package/dist/memory/crdt/pn-counter.d.ts.map +1 -0
- package/dist/memory/crdt/pn-counter.js +174 -0
- package/dist/memory/crdt/vector-clock.d.ts +115 -0
- package/dist/memory/crdt/vector-clock.d.ts.map +1 -0
- package/dist/memory/crdt/vector-clock.js +179 -0
- package/dist/memory/errors/index.d.ts +56 -0
- package/dist/memory/errors/index.d.ts.map +1 -0
- package/dist/memory/errors/index.js +85 -0
- package/dist/memory/index.d.ts +21 -0
- package/dist/memory/index.d.ts.map +1 -0
- package/dist/memory/index.js +20 -0
- package/dist/memory/persistence/crdt-persistence.d.ts +85 -0
- package/dist/memory/persistence/crdt-persistence.d.ts.map +1 -0
- package/dist/memory/persistence/crdt-persistence.js +204 -0
- package/dist/memory/persistence/index.d.ts +7 -0
- package/dist/memory/persistence/index.d.ts.map +1 -0
- package/dist/memory/persistence/index.js +6 -0
- package/dist/memory/preferences/index.d.ts +7 -0
- package/dist/memory/preferences/index.d.ts.map +1 -0
- package/dist/memory/preferences/index.js +6 -0
- package/dist/memory/preferences/user-preferences-manager.d.ts +133 -0
- package/dist/memory/preferences/user-preferences-manager.d.ts.map +1 -0
- package/dist/memory/preferences/user-preferences-manager.js +342 -0
- package/dist/memory/services/index.d.ts +8 -0
- package/dist/memory/services/index.d.ts.map +1 -0
- package/dist/memory/services/index.js +6 -0
- package/dist/memory/services/node-id-service.d.ts +75 -0
- package/dist/memory/services/node-id-service.d.ts.map +1 -0
- package/dist/memory/services/node-id-service.js +190 -0
- package/dist/memory/stores/episodic-memory.d.ts +182 -0
- package/dist/memory/stores/episodic-memory.d.ts.map +1 -0
- package/dist/memory/stores/episodic-memory.js +378 -0
- package/dist/memory/stores/index.d.ts +16 -0
- package/dist/memory/stores/index.d.ts.map +1 -0
- package/dist/memory/stores/index.js +15 -0
- package/dist/memory/stores/procedural-memory.d.ts +89 -0
- package/dist/memory/stores/procedural-memory.d.ts.map +1 -0
- package/dist/memory/stores/procedural-memory.js +152 -0
- package/dist/memory/stores/semantic-memory.d.ts +92 -0
- package/dist/memory/stores/semantic-memory.d.ts.map +1 -0
- package/dist/memory/stores/semantic-memory.js +155 -0
- package/dist/memory/stores/working-memory.d.ts +225 -0
- package/dist/memory/stores/working-memory.d.ts.map +1 -0
- package/dist/memory/stores/working-memory.js +336 -0
- package/dist/memory/utils/deep-clone.d.ts +10 -0
- package/dist/memory/utils/deep-clone.d.ts.map +1 -0
- package/dist/memory/utils/deep-clone.js +9 -0
- package/dist/memory/utils/index.d.ts +8 -0
- package/dist/memory/utils/index.d.ts.map +1 -0
- package/dist/memory/utils/index.js +7 -0
- package/dist/memory/utils/logger.d.ts +21 -0
- package/dist/memory/utils/logger.d.ts.map +1 -0
- package/dist/memory/utils/logger.js +62 -0
- package/dist/memory/utils/sql-helpers.d.ts +97 -0
- package/dist/memory/utils/sql-helpers.d.ts.map +1 -0
- package/dist/memory/utils/sql-helpers.js +214 -0
- package/dist/memory/utils/validation.d.ts +62 -0
- package/dist/memory/utils/validation.d.ts.map +1 -0
- package/dist/memory/utils/validation.js +244 -0
- package/dist/memory/vector/index.d.ts +12 -0
- package/dist/memory/vector/index.d.ts.map +1 -0
- package/dist/memory/vector/index.js +14 -0
- package/dist/memory/vector/vector-memory-service.d.ts +88 -0
- package/dist/memory/vector/vector-memory-service.d.ts.map +1 -0
- package/dist/memory/vector/vector-memory-service.js +335 -0
- package/dist/observability/logger.d.ts +79 -0
- package/dist/observability/logger.d.ts.map +1 -0
- package/dist/observability/logger.js +165 -0
- package/dist/observability/metrics.d.ts +43 -0
- package/dist/observability/metrics.d.ts.map +1 -0
- package/dist/observability/metrics.js +197 -0
- package/dist/observability/query.d.ts +150 -0
- package/dist/observability/query.d.ts.map +1 -0
- package/dist/observability/query.js +339 -0
- package/dist/observability/types.d.ts +140 -0
- package/dist/observability/types.d.ts.map +1 -0
- package/dist/observability/types.js +6 -0
- package/dist/orchestration/agent.d.ts +98 -0
- package/dist/orchestration/agent.d.ts.map +1 -0
- package/dist/orchestration/agent.js +6 -0
- package/dist/orchestration/defaults.d.ts +21 -0
- package/dist/orchestration/defaults.d.ts.map +1 -0
- package/dist/orchestration/defaults.js +22 -0
- package/dist/orchestration/memory-integration.d.ts +58 -0
- package/dist/orchestration/memory-integration.d.ts.map +1 -0
- package/dist/orchestration/memory-integration.js +130 -0
- package/dist/orchestration/orchestrator.d.ts +67 -0
- package/dist/orchestration/orchestrator.d.ts.map +1 -0
- package/dist/orchestration/orchestrator.js +174 -0
- package/dist/orchestration/runtime.d.ts +82 -0
- package/dist/orchestration/runtime.d.ts.map +1 -0
- package/dist/orchestration/runtime.js +251 -0
- package/dist/orchestration/streaming-runtime.d.ts +36 -0
- package/dist/orchestration/streaming-runtime.d.ts.map +1 -0
- package/dist/orchestration/streaming-runtime.js +175 -0
- package/dist/orchestration/ticket-agent.d.ts +70 -0
- package/dist/orchestration/ticket-agent.d.ts.map +1 -0
- package/dist/orchestration/ticket-agent.js +146 -0
- package/dist/skills/activation/index.d.ts +7 -0
- package/dist/skills/activation/index.d.ts.map +1 -0
- package/dist/skills/activation/index.js +6 -0
- package/dist/skills/activation/skill-activator.d.ts +68 -0
- package/dist/skills/activation/skill-activator.d.ts.map +1 -0
- package/dist/skills/activation/skill-activator.js +224 -0
- package/dist/skills/catalog/catalog-search.d.ts +55 -0
- package/dist/skills/catalog/catalog-search.d.ts.map +1 -0
- package/dist/skills/catalog/catalog-search.js +111 -0
- package/dist/skills/catalog/catalog-types.d.ts +81 -0
- package/dist/skills/catalog/catalog-types.d.ts.map +1 -0
- package/dist/skills/catalog/catalog-types.js +66 -0
- package/dist/skills/catalog/index.d.ts +9 -0
- package/dist/skills/catalog/index.d.ts.map +1 -0
- package/dist/skills/catalog/index.js +7 -0
- package/dist/skills/catalog/vercel-catalog.d.ts +42 -0
- package/dist/skills/catalog/vercel-catalog.d.ts.map +1 -0
- package/dist/skills/catalog/vercel-catalog.js +189 -0
- package/dist/skills/compat/index.d.ts +9 -0
- package/dist/skills/compat/index.d.ts.map +1 -0
- package/dist/skills/compat/index.js +8 -0
- package/dist/skills/compat/skill-enhancer.d.ts +37 -0
- package/dist/skills/compat/skill-enhancer.d.ts.map +1 -0
- package/dist/skills/compat/skill-enhancer.js +76 -0
- package/dist/skills/compat/tool-mapper.d.ts +61 -0
- package/dist/skills/compat/tool-mapper.d.ts.map +1 -0
- package/dist/skills/compat/tool-mapper.js +168 -0
- package/dist/skills/compat/vercel-compat.d.ts +33 -0
- package/dist/skills/compat/vercel-compat.d.ts.map +1 -0
- package/dist/skills/compat/vercel-compat.js +132 -0
- package/dist/skills/index.d.ts +40 -0
- package/dist/skills/index.d.ts.map +1 -0
- package/dist/skills/index.js +47 -0
- package/dist/skills/integration/agent-skill-provider.d.ts +94 -0
- package/dist/skills/integration/agent-skill-provider.d.ts.map +1 -0
- package/dist/skills/integration/agent-skill-provider.js +161 -0
- package/dist/skills/integration/index.d.ts +7 -0
- package/dist/skills/integration/index.d.ts.map +1 -0
- package/dist/skills/integration/index.js +6 -0
- package/dist/skills/loader/github-loader.d.ts +61 -0
- package/dist/skills/loader/github-loader.d.ts.map +1 -0
- package/dist/skills/loader/github-loader.js +176 -0
- package/dist/skills/loader/index.d.ts +10 -0
- package/dist/skills/loader/index.d.ts.map +1 -0
- package/dist/skills/loader/index.js +9 -0
- package/dist/skills/loader/local-loader.d.ts +56 -0
- package/dist/skills/loader/local-loader.d.ts.map +1 -0
- package/dist/skills/loader/local-loader.js +186 -0
- package/dist/skills/loader/vercel-loader.d.ts +64 -0
- package/dist/skills/loader/vercel-loader.d.ts.map +1 -0
- package/dist/skills/loader/vercel-loader.js +313 -0
- package/dist/skills/loader/vercel-types.d.ts +64 -0
- package/dist/skills/loader/vercel-types.d.ts.map +1 -0
- package/dist/skills/loader/vercel-types.js +55 -0
- package/dist/skills/parser/index.d.ts +7 -0
- package/dist/skills/parser/index.d.ts.map +1 -0
- package/dist/skills/parser/index.js +6 -0
- package/dist/skills/parser/skill-md-parser.d.ts +64 -0
- package/dist/skills/parser/skill-md-parser.d.ts.map +1 -0
- package/dist/skills/parser/skill-md-parser.js +242 -0
- package/dist/skills/registry/index.d.ts +7 -0
- package/dist/skills/registry/index.d.ts.map +1 -0
- package/dist/skills/registry/index.js +6 -0
- package/dist/skills/registry/skill-registry.d.ts +133 -0
- package/dist/skills/registry/skill-registry.d.ts.map +1 -0
- package/dist/skills/registry/skill-registry.js +373 -0
- package/dist/skills/types.d.ts +216 -0
- package/dist/skills/types.d.ts.map +1 -0
- package/dist/skills/types.js +176 -0
- package/dist/templates/agent-spec.d.ts +138 -0
- package/dist/templates/agent-spec.d.ts.map +1 -0
- package/dist/templates/agent-spec.js +138 -0
- package/dist/templates/index.d.ts +56 -0
- package/dist/templates/index.d.ts.map +1 -0
- package/dist/templates/index.js +58 -0
- package/dist/templates/prompt-spec.d.ts +140 -0
- package/dist/templates/prompt-spec.d.ts.map +1 -0
- package/dist/templates/prompt-spec.js +210 -0
- package/dist/templates/skill-spec.d.ts +106 -0
- package/dist/templates/skill-spec.d.ts.map +1 -0
- package/dist/templates/skill-spec.js +119 -0
- package/dist/tools/base.d.ts +74 -0
- package/dist/tools/base.d.ts.map +1 -0
- package/dist/tools/base.js +6 -0
- package/dist/tools/cms/collection-tools.d.ts +36 -0
- package/dist/tools/cms/collection-tools.d.ts.map +1 -0
- package/dist/tools/cms/collection-tools.js +178 -0
- package/dist/tools/cms/factory.d.ts +89 -0
- package/dist/tools/cms/factory.d.ts.map +1 -0
- package/dist/tools/cms/factory.js +462 -0
- package/dist/tools/cms/global-tools.d.ts +21 -0
- package/dist/tools/cms/global-tools.d.ts.map +1 -0
- package/dist/tools/cms/global-tools.js +92 -0
- package/dist/tools/cms/index.d.ts +11 -0
- package/dist/tools/cms/index.d.ts.map +1 -0
- package/dist/tools/cms/index.js +11 -0
- package/dist/tools/cms/media-tools.d.ts +31 -0
- package/dist/tools/cms/media-tools.d.ts.map +1 -0
- package/dist/tools/cms/media-tools.js +140 -0
- package/dist/tools/cms/user-tools.d.ts +31 -0
- package/dist/tools/cms/user-tools.d.ts.map +1 -0
- package/dist/tools/cms/user-tools.js +135 -0
- package/dist/tools/deduplicator.d.ts +19 -0
- package/dist/tools/deduplicator.d.ts.map +1 -0
- package/dist/tools/deduplicator.js +53 -0
- package/dist/tools/document-summarizer.d.ts +11 -0
- package/dist/tools/document-summarizer.d.ts.map +1 -0
- package/dist/tools/document-summarizer.js +82 -0
- package/dist/tools/mcp-adapter.d.ts +66 -0
- package/dist/tools/mcp-adapter.d.ts.map +1 -0
- package/dist/tools/mcp-adapter.js +152 -0
- package/dist/tools/memory/index.d.ts +3 -0
- package/dist/tools/memory/index.d.ts.map +1 -0
- package/dist/tools/memory/index.js +1 -0
- package/dist/tools/memory/store-memory.d.ts +39 -0
- package/dist/tools/memory/store-memory.d.ts.map +1 -0
- package/dist/tools/memory/store-memory.js +94 -0
- package/dist/tools/registry.d.ts +14 -0
- package/dist/tools/registry.d.ts.map +1 -0
- package/dist/tools/registry.js +48 -0
- package/dist/tools/ticket-tools.d.ts +31 -0
- package/dist/tools/ticket-tools.d.ts.map +1 -0
- package/dist/tools/ticket-tools.js +74 -0
- package/dist/tools/web/duck-duck-go.d.ts +52 -0
- package/dist/tools/web/duck-duck-go.d.ts.map +1 -0
- package/dist/tools/web/duck-duck-go.js +202 -0
- package/dist/tools/web/exa.d.ts +34 -0
- package/dist/tools/web/exa.d.ts.map +1 -0
- package/dist/tools/web/exa.js +80 -0
- package/dist/tools/web/index.d.ts +6 -0
- package/dist/tools/web/index.d.ts.map +1 -0
- package/dist/tools/web/index.js +4 -0
- package/dist/tools/web/scraper.d.ts +9 -0
- package/dist/tools/web/scraper.d.ts.map +1 -0
- package/dist/tools/web/scraper.js +118 -0
- package/dist/tools/web/tavily.d.ts +32 -0
- package/dist/tools/web/tavily.d.ts.map +1 -0
- package/dist/tools/web/tavily.js +73 -0
- package/dist/tools/web/types.d.ts +31 -0
- package/dist/tools/web/types.d.ts.map +1 -0
- package/dist/tools/web/types.js +9 -0
- package/package.json +143 -0
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* OpenAI Provider
|
|
3
|
+
*
|
|
4
|
+
* Implementation of LLMProvider for OpenAI API
|
|
5
|
+
*/
|
|
6
|
+
const authorizationHeader = 'Authorization';
|
|
7
|
+
const openAiOrganizationHeader = 'OpenAI-Organization';
|
|
8
|
+
const maxTokensKey = 'max_tokens';
|
|
9
|
+
const toolChoiceKey = 'tool_choice';
|
|
10
|
+
const toolCallsKey = 'tool_calls';
|
|
11
|
+
const toolCallIdKey = 'tool_call_id';
|
|
12
|
+
const finishReasonKey = 'finish_reason';
|
|
13
|
+
const promptTokensKey = 'prompt_tokens';
|
|
14
|
+
const completionTokensKey = 'completion_tokens';
|
|
15
|
+
const totalTokensKey = 'total_tokens';
|
|
16
|
+
const asRecord = (value) => {
|
|
17
|
+
if (!value || typeof value !== 'object') {
|
|
18
|
+
return undefined;
|
|
19
|
+
}
|
|
20
|
+
return value;
|
|
21
|
+
};
|
|
22
|
+
const isFunctionToolCall = (call) => {
|
|
23
|
+
const record = asRecord(call);
|
|
24
|
+
if (!record || record.type !== 'function' || typeof record.id !== 'string') {
|
|
25
|
+
return false;
|
|
26
|
+
}
|
|
27
|
+
const fn = asRecord(record.function);
|
|
28
|
+
return !!fn && typeof fn.name === 'string' && typeof fn.arguments === 'string';
|
|
29
|
+
};
|
|
30
|
+
export class OpenAIProvider {
|
|
31
|
+
config;
|
|
32
|
+
baseURL;
|
|
33
|
+
constructor(config) {
|
|
34
|
+
this.config = config;
|
|
35
|
+
this.baseURL = config.baseURL || 'https://api.openai.com/v1';
|
|
36
|
+
}
|
|
37
|
+
async chat(messages, options) {
|
|
38
|
+
const response = await fetch(`${this.baseURL}/chat/completions`, {
|
|
39
|
+
method: 'POST',
|
|
40
|
+
headers: {
|
|
41
|
+
'Content-Type': 'application/json',
|
|
42
|
+
[authorizationHeader]: `Bearer ${this.config.apiKey}`,
|
|
43
|
+
...(this.config.organization && {
|
|
44
|
+
[openAiOrganizationHeader]: this.config.organization,
|
|
45
|
+
}),
|
|
46
|
+
},
|
|
47
|
+
body: JSON.stringify({
|
|
48
|
+
model: this.config.model || 'gpt-4o-mini',
|
|
49
|
+
messages: this.formatMessages(messages),
|
|
50
|
+
temperature: options?.temperature ?? this.config.temperature ?? 0.7,
|
|
51
|
+
[maxTokensKey]: options?.maxTokens ?? this.config.maxTokens,
|
|
52
|
+
tools: options?.tools,
|
|
53
|
+
[toolChoiceKey]: options?.toolChoice,
|
|
54
|
+
}),
|
|
55
|
+
});
|
|
56
|
+
if (!response.ok) {
|
|
57
|
+
const errorPayload = (await response.json().catch(() => undefined));
|
|
58
|
+
const errorRecord = asRecord(errorPayload);
|
|
59
|
+
const errorDetail = asRecord(errorRecord?.error);
|
|
60
|
+
const errorMessage = errorDetail && typeof errorDetail.message === 'string'
|
|
61
|
+
? errorDetail.message
|
|
62
|
+
: response.statusText;
|
|
63
|
+
throw new Error(`OpenAI API error: ${errorMessage}`);
|
|
64
|
+
}
|
|
65
|
+
const data = (await response.json());
|
|
66
|
+
const choice = data.choices?.[0];
|
|
67
|
+
const choiceRecord = asRecord(choice);
|
|
68
|
+
const messageRecord = asRecord(choiceRecord?.message);
|
|
69
|
+
const rawToolCalls = messageRecord?.[toolCallsKey];
|
|
70
|
+
const toolCalls = Array.isArray(rawToolCalls)
|
|
71
|
+
? rawToolCalls.filter(isFunctionToolCall).map((tc) => ({
|
|
72
|
+
id: tc.id,
|
|
73
|
+
type: 'function',
|
|
74
|
+
function: {
|
|
75
|
+
name: tc.function.name,
|
|
76
|
+
arguments: tc.function.arguments,
|
|
77
|
+
},
|
|
78
|
+
}))
|
|
79
|
+
: undefined;
|
|
80
|
+
const finishReasonValue = choiceRecord?.[finishReasonKey];
|
|
81
|
+
const finishReason = typeof finishReasonValue === 'string' ? finishReasonValue : undefined;
|
|
82
|
+
const usageRecord = asRecord(data.usage);
|
|
83
|
+
const promptTokens = usageRecord && typeof usageRecord[promptTokensKey] === 'number'
|
|
84
|
+
? usageRecord[promptTokensKey]
|
|
85
|
+
: undefined;
|
|
86
|
+
const completionTokens = usageRecord && typeof usageRecord[completionTokensKey] === 'number'
|
|
87
|
+
? usageRecord[completionTokensKey]
|
|
88
|
+
: undefined;
|
|
89
|
+
const totalTokens = usageRecord && typeof usageRecord[totalTokensKey] === 'number'
|
|
90
|
+
? usageRecord[totalTokensKey]
|
|
91
|
+
: undefined;
|
|
92
|
+
return {
|
|
93
|
+
content: typeof messageRecord?.content === 'string' ? messageRecord.content : '',
|
|
94
|
+
role: 'assistant',
|
|
95
|
+
toolCalls,
|
|
96
|
+
finishReason,
|
|
97
|
+
usage: promptTokens !== undefined && completionTokens !== undefined && totalTokens !== undefined
|
|
98
|
+
? {
|
|
99
|
+
promptTokens,
|
|
100
|
+
completionTokens,
|
|
101
|
+
totalTokens,
|
|
102
|
+
}
|
|
103
|
+
: undefined,
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
async embed(text, options) {
|
|
107
|
+
const texts = Array.isArray(text) ? text : [text];
|
|
108
|
+
const model = options?.model || 'text-embedding-3-small';
|
|
109
|
+
const response = await fetch(`${this.baseURL}/embeddings`, {
|
|
110
|
+
method: 'POST',
|
|
111
|
+
headers: {
|
|
112
|
+
'Content-Type': 'application/json',
|
|
113
|
+
[authorizationHeader]: `Bearer ${this.config.apiKey}`,
|
|
114
|
+
...(this.config.organization && {
|
|
115
|
+
[openAiOrganizationHeader]: this.config.organization,
|
|
116
|
+
}),
|
|
117
|
+
},
|
|
118
|
+
body: JSON.stringify({
|
|
119
|
+
model,
|
|
120
|
+
input: texts,
|
|
121
|
+
}),
|
|
122
|
+
});
|
|
123
|
+
if (!response.ok) {
|
|
124
|
+
const errorPayload = (await response.json().catch(() => undefined));
|
|
125
|
+
const errorRecord = asRecord(errorPayload);
|
|
126
|
+
const errorDetail = asRecord(errorRecord?.error);
|
|
127
|
+
const errorMessage = errorDetail && typeof errorDetail.message === 'string'
|
|
128
|
+
? errorDetail.message
|
|
129
|
+
: response.statusText;
|
|
130
|
+
throw new Error(`OpenAI API error: ${errorMessage}`);
|
|
131
|
+
}
|
|
132
|
+
const data = (await response.json());
|
|
133
|
+
const embeddings = (data.data || []).map((item) => {
|
|
134
|
+
const vector = item.embedding || [];
|
|
135
|
+
return {
|
|
136
|
+
vector,
|
|
137
|
+
dimension: vector.length,
|
|
138
|
+
model: item.model || model,
|
|
139
|
+
};
|
|
140
|
+
});
|
|
141
|
+
return Array.isArray(text) ? embeddings : embeddings[0];
|
|
142
|
+
}
|
|
143
|
+
async *stream(messages, options) {
|
|
144
|
+
const response = await fetch(`${this.baseURL}/chat/completions`, {
|
|
145
|
+
method: 'POST',
|
|
146
|
+
headers: {
|
|
147
|
+
'Content-Type': 'application/json',
|
|
148
|
+
[authorizationHeader]: `Bearer ${this.config.apiKey}`,
|
|
149
|
+
...(this.config.organization && {
|
|
150
|
+
[openAiOrganizationHeader]: this.config.organization,
|
|
151
|
+
}),
|
|
152
|
+
},
|
|
153
|
+
body: JSON.stringify({
|
|
154
|
+
model: this.config.model || 'gpt-4o-mini',
|
|
155
|
+
messages: this.formatMessages(messages),
|
|
156
|
+
temperature: options?.temperature ?? this.config.temperature ?? 0.7,
|
|
157
|
+
[maxTokensKey]: options?.maxTokens ?? this.config.maxTokens,
|
|
158
|
+
tools: options?.tools,
|
|
159
|
+
stream: true,
|
|
160
|
+
}),
|
|
161
|
+
});
|
|
162
|
+
if (!response.ok) {
|
|
163
|
+
const errorPayload = (await response.json().catch(() => undefined));
|
|
164
|
+
const errorRecord = asRecord(errorPayload);
|
|
165
|
+
const errorDetail = asRecord(errorRecord?.error);
|
|
166
|
+
const errorMessage = errorDetail && typeof errorDetail.message === 'string'
|
|
167
|
+
? errorDetail.message
|
|
168
|
+
: response.statusText;
|
|
169
|
+
throw new Error(`OpenAI API error: ${errorMessage}`);
|
|
170
|
+
}
|
|
171
|
+
const reader = response.body?.getReader();
|
|
172
|
+
const decoder = new TextDecoder();
|
|
173
|
+
if (!reader) {
|
|
174
|
+
throw new Error('Response body is not readable');
|
|
175
|
+
}
|
|
176
|
+
let buffer = '';
|
|
177
|
+
while (true) {
|
|
178
|
+
const { done, value } = await reader.read();
|
|
179
|
+
if (done) {
|
|
180
|
+
yield { content: '', done: true };
|
|
181
|
+
break;
|
|
182
|
+
}
|
|
183
|
+
buffer += decoder.decode(value, { stream: true });
|
|
184
|
+
const lines = buffer.split('\n');
|
|
185
|
+
buffer = lines.pop() || '';
|
|
186
|
+
for (const line of lines) {
|
|
187
|
+
if (line.startsWith('data: ')) {
|
|
188
|
+
const data = line.slice(6);
|
|
189
|
+
if (data === '[DONE]') {
|
|
190
|
+
yield { content: '', done: true };
|
|
191
|
+
return;
|
|
192
|
+
}
|
|
193
|
+
try {
|
|
194
|
+
const parsed = JSON.parse(data);
|
|
195
|
+
const choice = parsed.choices?.[0];
|
|
196
|
+
const choiceRecord = asRecord(choice);
|
|
197
|
+
const deltaRecord = asRecord(choiceRecord?.delta);
|
|
198
|
+
if (deltaRecord) {
|
|
199
|
+
const deltaToolCalls = Array.isArray(deltaRecord[toolCallsKey])
|
|
200
|
+
? deltaRecord[toolCallsKey].filter(isFunctionToolCall).map((tc) => ({
|
|
201
|
+
id: tc.id,
|
|
202
|
+
type: 'function',
|
|
203
|
+
function: {
|
|
204
|
+
name: tc.function.name,
|
|
205
|
+
arguments: tc.function.arguments,
|
|
206
|
+
},
|
|
207
|
+
}))
|
|
208
|
+
: undefined;
|
|
209
|
+
yield {
|
|
210
|
+
content: typeof deltaRecord.content === 'string' ? deltaRecord.content : '',
|
|
211
|
+
done: false,
|
|
212
|
+
toolCalls: deltaToolCalls,
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
catch {
|
|
217
|
+
// Ignore parse errors for incomplete chunks
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
formatMessages(messages) {
|
|
224
|
+
return messages.map((msg) => {
|
|
225
|
+
const formatted = {
|
|
226
|
+
role: msg.role,
|
|
227
|
+
content: msg.content,
|
|
228
|
+
};
|
|
229
|
+
if (msg.name) {
|
|
230
|
+
formatted.name = msg.name;
|
|
231
|
+
}
|
|
232
|
+
if (msg.toolCalls) {
|
|
233
|
+
formatted[toolCallsKey] = msg.toolCalls.map((tc) => ({
|
|
234
|
+
id: tc.id,
|
|
235
|
+
type: tc.type,
|
|
236
|
+
function: tc.function,
|
|
237
|
+
}));
|
|
238
|
+
}
|
|
239
|
+
if (msg.toolCallId) {
|
|
240
|
+
formatted[toolCallIdKey] = msg.toolCallId;
|
|
241
|
+
}
|
|
242
|
+
return formatted;
|
|
243
|
+
});
|
|
244
|
+
}
|
|
245
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Vultr Inference Provider
|
|
3
|
+
*
|
|
4
|
+
* Implements the LLMProvider interface for Vultr Serverless Inference API
|
|
5
|
+
*/
|
|
6
|
+
import type { Embedding, LLMChatOptions, LLMChunk, LLMEmbedOptions, LLMProvider, LLMProviderConfig, LLMResponse, LLMStreamOptions, Message } from './base.js';
|
|
7
|
+
export interface VultrProviderConfig extends LLMProviderConfig {
|
|
8
|
+
subscriptionId?: string;
|
|
9
|
+
}
|
|
10
|
+
export declare class VultrProvider implements LLMProvider {
|
|
11
|
+
private config;
|
|
12
|
+
private baseURL;
|
|
13
|
+
constructor(config: VultrProviderConfig);
|
|
14
|
+
chat(messages: Message[], options?: LLMChatOptions): Promise<LLMResponse>;
|
|
15
|
+
embed(text: string | string[], options?: LLMEmbedOptions): Promise<Embedding | Embedding[]>;
|
|
16
|
+
stream(messages: Message[], options?: LLMStreamOptions): AsyncIterable<LLMChunk>;
|
|
17
|
+
}
|
|
18
|
+
//# sourceMappingURL=vultr.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vultr.d.ts","sourceRoot":"","sources":["../../../src/llm/providers/vultr.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EACV,SAAS,EACT,cAAc,EACd,QAAQ,EACR,eAAe,EACf,WAAW,EACX,iBAAiB,EACjB,WAAW,EACX,gBAAgB,EAChB,OAAO,EAER,MAAM,WAAW,CAAA;AAElB,MAAM,WAAW,mBAAoB,SAAQ,iBAAiB;IAC5D,cAAc,CAAC,EAAE,MAAM,CAAA;CACxB;AAgBD,qBAAa,aAAc,YAAW,WAAW;IAC/C,OAAO,CAAC,MAAM,CAAqB;IACnC,OAAO,CAAC,OAAO,CAAQ;gBAEX,MAAM,EAAE,mBAAmB;IAKjC,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,WAAW,CAAC;IA2DzE,KAAK,CACT,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,EACvB,OAAO,CAAC,EAAE,eAAe,GACxB,OAAO,CAAC,SAAS,GAAG,SAAS,EAAE,CAAC;IAuC5B,MAAM,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,aAAa,CAAC,QAAQ,CAAC;CAkExF"}
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Vultr Inference Provider
|
|
3
|
+
*
|
|
4
|
+
* Implements the LLMProvider interface for Vultr Serverless Inference API
|
|
5
|
+
*/
|
|
6
|
+
const authorizationHeader = 'Authorization';
|
|
7
|
+
const contentTypeHeader = 'Content-Type';
|
|
8
|
+
const asRecord = (v) => typeof v === 'object' && v !== null ? v : undefined;
|
|
9
|
+
const isFunctionToolCall = (call) => {
|
|
10
|
+
const r = asRecord(call);
|
|
11
|
+
if (!r)
|
|
12
|
+
return false;
|
|
13
|
+
return r.type === 'function' && typeof r.id === 'string';
|
|
14
|
+
};
|
|
15
|
+
export class VultrProvider {
|
|
16
|
+
config;
|
|
17
|
+
baseURL;
|
|
18
|
+
constructor(config) {
|
|
19
|
+
this.config = config;
|
|
20
|
+
this.baseURL = config.baseURL || 'https://api.vultrinference.com/v1';
|
|
21
|
+
}
|
|
22
|
+
async chat(messages, options) {
|
|
23
|
+
const body = {
|
|
24
|
+
model: this.config.model,
|
|
25
|
+
messages: messages.map((m) => ({ role: m.role, content: m.content, name: m.name })),
|
|
26
|
+
temperature: options?.temperature ?? this.config.temperature ?? 0.7,
|
|
27
|
+
max_tokens: options?.maxTokens ?? this.config.maxTokens,
|
|
28
|
+
tools: options?.tools,
|
|
29
|
+
};
|
|
30
|
+
const res = await fetch(`${this.baseURL}/chat/completions`, {
|
|
31
|
+
method: 'POST',
|
|
32
|
+
headers: {
|
|
33
|
+
[contentTypeHeader]: 'application/json',
|
|
34
|
+
[authorizationHeader]: `Bearer ${this.config.apiKey}`,
|
|
35
|
+
},
|
|
36
|
+
body: JSON.stringify(body),
|
|
37
|
+
});
|
|
38
|
+
if (!res.ok) {
|
|
39
|
+
const json = await res.json().catch(() => undefined);
|
|
40
|
+
const rec = asRecord(json);
|
|
41
|
+
const err = rec && typeof rec.error === 'string' ? rec.error : res.statusText;
|
|
42
|
+
throw new Error(`Vultr API error: ${err}`);
|
|
43
|
+
}
|
|
44
|
+
const data = (await res.json());
|
|
45
|
+
const choice = Array.isArray(data.choices) ? data.choices[0] : undefined;
|
|
46
|
+
const choiceRec = asRecord(choice);
|
|
47
|
+
const message = asRecord(choiceRec?.message);
|
|
48
|
+
// Parse tool calls if present
|
|
49
|
+
let toolCalls;
|
|
50
|
+
if (message && Array.isArray(message.tool_calls || message.toolCalls)) {
|
|
51
|
+
const raw = (message.tool_calls || message.toolCalls);
|
|
52
|
+
toolCalls = raw.filter(isFunctionToolCall).map((tc) => {
|
|
53
|
+
const r = asRecord(tc) ?? {};
|
|
54
|
+
const fn = asRecord(r.function);
|
|
55
|
+
return {
|
|
56
|
+
id: String(r.id),
|
|
57
|
+
type: 'function',
|
|
58
|
+
function: {
|
|
59
|
+
name: typeof fn?.name === 'string' ? fn?.name : '',
|
|
60
|
+
arguments: typeof fn?.arguments === 'string'
|
|
61
|
+
? fn?.arguments
|
|
62
|
+
: JSON.stringify(fn?.arguments ?? {}),
|
|
63
|
+
},
|
|
64
|
+
};
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
return {
|
|
68
|
+
content: typeof message?.content === 'string' ? message.content : '',
|
|
69
|
+
role: 'assistant',
|
|
70
|
+
toolCalls,
|
|
71
|
+
finishReason: undefined,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
async embed(text, options) {
|
|
75
|
+
// Vultr may support embeddings on some models. Try /embeddings endpoint if available.
|
|
76
|
+
const inputs = Array.isArray(text) ? text : [text];
|
|
77
|
+
const model = options?.model || this.config.model;
|
|
78
|
+
const res = await fetch(`${this.baseURL}/embeddings`, {
|
|
79
|
+
method: 'POST',
|
|
80
|
+
headers: {
|
|
81
|
+
[contentTypeHeader]: 'application/json',
|
|
82
|
+
[authorizationHeader]: `Bearer ${this.config.apiKey}`,
|
|
83
|
+
},
|
|
84
|
+
body: JSON.stringify({ model, input: inputs }),
|
|
85
|
+
});
|
|
86
|
+
if (!res.ok) {
|
|
87
|
+
const json = await res.json().catch(() => undefined);
|
|
88
|
+
const rec = asRecord(json);
|
|
89
|
+
// If embeddings not supported, throw a clear error
|
|
90
|
+
if (res.status === 404 || (rec?.error && typeof rec.error === 'string')) {
|
|
91
|
+
throw new Error('Vultr embeddings not available for this configuration');
|
|
92
|
+
}
|
|
93
|
+
throw new Error(`Vultr embeddings API error: ${res.statusText}`);
|
|
94
|
+
}
|
|
95
|
+
const data = (await res.json());
|
|
96
|
+
const items = Array.isArray(data.data) ? data.data : [];
|
|
97
|
+
const embeddings = items.map((it) => {
|
|
98
|
+
const rec = asRecord(it);
|
|
99
|
+
const vector = Array.isArray(rec?.embedding) ? rec?.embedding : [];
|
|
100
|
+
return {
|
|
101
|
+
vector,
|
|
102
|
+
dimension: vector.length,
|
|
103
|
+
model: typeof rec?.model === 'string' ? rec?.model : String(model),
|
|
104
|
+
};
|
|
105
|
+
});
|
|
106
|
+
return Array.isArray(text) ? embeddings : embeddings[0];
|
|
107
|
+
}
|
|
108
|
+
async *stream(messages, options) {
|
|
109
|
+
const body = {
|
|
110
|
+
model: this.config.model,
|
|
111
|
+
messages: messages.map((m) => ({ role: m.role, content: m.content, name: m.name })),
|
|
112
|
+
temperature: options?.temperature ?? this.config.temperature ?? 0.7,
|
|
113
|
+
max_tokens: options?.maxTokens ?? this.config.maxTokens,
|
|
114
|
+
stream: true,
|
|
115
|
+
};
|
|
116
|
+
const res = await fetch(`${this.baseURL}/chat/completions`, {
|
|
117
|
+
method: 'POST',
|
|
118
|
+
headers: {
|
|
119
|
+
[contentTypeHeader]: 'application/json',
|
|
120
|
+
[authorizationHeader]: `Bearer ${this.config.apiKey}`,
|
|
121
|
+
},
|
|
122
|
+
body: JSON.stringify(body),
|
|
123
|
+
});
|
|
124
|
+
if (!res.ok) {
|
|
125
|
+
const json = await res.json().catch(() => undefined);
|
|
126
|
+
const rec = asRecord(json);
|
|
127
|
+
const err = rec && typeof rec.error === 'string' ? rec.error : res.statusText;
|
|
128
|
+
throw new Error(`Vultr streaming error: ${err}`);
|
|
129
|
+
}
|
|
130
|
+
const reader = res.body?.getReader();
|
|
131
|
+
const decoder = new TextDecoder();
|
|
132
|
+
if (!reader) {
|
|
133
|
+
throw new Error('Response body is not readable');
|
|
134
|
+
}
|
|
135
|
+
let buffer = '';
|
|
136
|
+
while (true) {
|
|
137
|
+
const { done, value } = await reader.read();
|
|
138
|
+
if (done) {
|
|
139
|
+
yield { content: '', done: true };
|
|
140
|
+
break;
|
|
141
|
+
}
|
|
142
|
+
buffer += decoder.decode(value, { stream: true });
|
|
143
|
+
const lines = buffer.split('\n');
|
|
144
|
+
buffer = lines.pop() || '';
|
|
145
|
+
for (const line of lines) {
|
|
146
|
+
if (line.startsWith('data: ')) {
|
|
147
|
+
const data = line.slice(6);
|
|
148
|
+
if (data === '[DONE]') {
|
|
149
|
+
yield { content: '', done: true };
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
152
|
+
try {
|
|
153
|
+
const parsed = JSON.parse(data);
|
|
154
|
+
const choice = Array.isArray(parsed.choices) ? parsed.choices[0] : undefined;
|
|
155
|
+
// Vultr uses OpenAI-compatible SSE format: delta not message
|
|
156
|
+
const delta = asRecord(asRecord(choice)?.delta);
|
|
157
|
+
if (delta && typeof delta.content === 'string') {
|
|
158
|
+
yield { content: delta.content, done: false };
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
catch {
|
|
162
|
+
// ignore partial JSON
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
}
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Response Cache for LLM Providers
|
|
3
|
+
*
|
|
4
|
+
* Application-level caching for LLM responses. Works with any provider.
|
|
5
|
+
* Provides 100% cost savings on cache hits vs Anthropic's 90% token-level savings.
|
|
6
|
+
*
|
|
7
|
+
* Use cases:
|
|
8
|
+
* - Vultr (doesn't have prompt caching)
|
|
9
|
+
* - OpenAI (doesn't have prompt caching)
|
|
10
|
+
* - Any provider without built-in caching
|
|
11
|
+
* - Supplement to Anthropic prompt caching for exact duplicates
|
|
12
|
+
*
|
|
13
|
+
* Features:
|
|
14
|
+
* - LRU eviction policy
|
|
15
|
+
* - Configurable TTL (default: 5 minutes)
|
|
16
|
+
* - SHA-256 based cache keys
|
|
17
|
+
* - Hit/miss statistics
|
|
18
|
+
* - Memory-efficient storage
|
|
19
|
+
*/
|
|
20
|
+
import type { Message, ToolCall, ToolDefinition } from './providers/base.js';
|
|
21
|
+
export interface CachedResponse {
|
|
22
|
+
content: string;
|
|
23
|
+
role: 'assistant';
|
|
24
|
+
finishReason?: 'stop' | 'length' | 'tool_calls' | 'content_filter';
|
|
25
|
+
toolCalls?: ToolCall[];
|
|
26
|
+
timestamp: number;
|
|
27
|
+
usage?: {
|
|
28
|
+
promptTokens: number;
|
|
29
|
+
completionTokens: number;
|
|
30
|
+
totalTokens: number;
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
export interface ResponseCacheOptions {
|
|
34
|
+
/** Maximum number of cached responses (default: 1000) */
|
|
35
|
+
max?: number;
|
|
36
|
+
/** Time to live in milliseconds (default: 5 minutes) */
|
|
37
|
+
ttl?: number;
|
|
38
|
+
/** Enable cache statistics tracking (default: true) */
|
|
39
|
+
enableStats?: boolean;
|
|
40
|
+
}
|
|
41
|
+
export interface CacheStats {
|
|
42
|
+
hits: number;
|
|
43
|
+
misses: number;
|
|
44
|
+
evictions: number;
|
|
45
|
+
hitRate: number;
|
|
46
|
+
size: number;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Response cache for LLM completions
|
|
50
|
+
*
|
|
51
|
+
* @example
|
|
52
|
+
* ```typescript
|
|
53
|
+
* const cache = new ResponseCache({ max: 1000, ttl: 5 * 60 * 1000 })
|
|
54
|
+
*
|
|
55
|
+
* const key = cache.getCacheKey(messages, options)
|
|
56
|
+
* const cached = cache.get(key)
|
|
57
|
+
*
|
|
58
|
+
* if (cached) {
|
|
59
|
+
* return { ...cached, cached: true }
|
|
60
|
+
* }
|
|
61
|
+
*
|
|
62
|
+
* const response = await llm.chat(messages, options)
|
|
63
|
+
* cache.set(key, response)
|
|
64
|
+
* ```
|
|
65
|
+
*/
|
|
66
|
+
export declare class ResponseCache {
|
|
67
|
+
private cache;
|
|
68
|
+
private stats;
|
|
69
|
+
private enableStats;
|
|
70
|
+
constructor(options?: ResponseCacheOptions);
|
|
71
|
+
/**
|
|
72
|
+
* Generate cache key from messages and options
|
|
73
|
+
*
|
|
74
|
+
* Uses SHA-256 hash of JSON-serialized payload for:
|
|
75
|
+
* - Consistent key generation
|
|
76
|
+
* - Fixed-length keys (64 chars)
|
|
77
|
+
* - Security (prevents key enumeration)
|
|
78
|
+
*/
|
|
79
|
+
getCacheKey(messages: Message[], options?: {
|
|
80
|
+
temperature?: number;
|
|
81
|
+
maxTokens?: number;
|
|
82
|
+
tools?: ToolDefinition[];
|
|
83
|
+
model?: string;
|
|
84
|
+
}): string;
|
|
85
|
+
/**
|
|
86
|
+
* Get cached response
|
|
87
|
+
*
|
|
88
|
+
* @returns Cached response or undefined if not found/expired
|
|
89
|
+
*/
|
|
90
|
+
get(key: string): CachedResponse | undefined;
|
|
91
|
+
/**
|
|
92
|
+
* Store response in cache
|
|
93
|
+
*/
|
|
94
|
+
set(key: string, response: CachedResponse): void;
|
|
95
|
+
/**
|
|
96
|
+
* Check if key exists in cache (without updating stats)
|
|
97
|
+
*/
|
|
98
|
+
has(key: string): boolean;
|
|
99
|
+
/**
|
|
100
|
+
* Remove entry from cache
|
|
101
|
+
*/
|
|
102
|
+
delete(key: string): boolean;
|
|
103
|
+
/**
|
|
104
|
+
* Clear all cached responses
|
|
105
|
+
*/
|
|
106
|
+
clear(): void;
|
|
107
|
+
/**
|
|
108
|
+
* Get cache statistics
|
|
109
|
+
*/
|
|
110
|
+
getStats(): CacheStats;
|
|
111
|
+
/**
|
|
112
|
+
* Reset statistics
|
|
113
|
+
*/
|
|
114
|
+
resetStats(): void;
|
|
115
|
+
/**
|
|
116
|
+
* Get cache size (number of entries)
|
|
117
|
+
*/
|
|
118
|
+
get size(): number;
|
|
119
|
+
/**
|
|
120
|
+
* Get maximum cache size
|
|
121
|
+
*/
|
|
122
|
+
get maxSize(): number;
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* Get or create global response cache
|
|
126
|
+
*
|
|
127
|
+
* @example
|
|
128
|
+
* ```typescript
|
|
129
|
+
* const cache = getGlobalResponseCache()
|
|
130
|
+
* const stats = cache.getStats()
|
|
131
|
+
* // Hit rate: ${stats.hitRate}%
|
|
132
|
+
* ```
|
|
133
|
+
*/
|
|
134
|
+
export declare function getGlobalResponseCache(options?: ResponseCacheOptions): ResponseCache;
|
|
135
|
+
/**
|
|
136
|
+
* Clear global response cache
|
|
137
|
+
*/
|
|
138
|
+
export declare function clearGlobalResponseCache(): void;
|
|
139
|
+
/**
|
|
140
|
+
* Calculate cost savings from response caching
|
|
141
|
+
*
|
|
142
|
+
* @example
|
|
143
|
+
* ```typescript
|
|
144
|
+
* const stats = cache.getStats()
|
|
145
|
+
* const savings = calculateResponseCacheSavings(stats, {
|
|
146
|
+
* avgInputTokens: 3000,
|
|
147
|
+
* avgOutputTokens: 500,
|
|
148
|
+
* inputCostPerM: 3.0,
|
|
149
|
+
* outputCostPerM: 15.0,
|
|
150
|
+
* })
|
|
151
|
+
*
|
|
152
|
+
* // Saved: $${savings.totalSaved.toFixed(2)}
|
|
153
|
+
* // Avoided ${savings.tokensAvoided.toLocaleString()} tokens
|
|
154
|
+
* ```
|
|
155
|
+
*/
|
|
156
|
+
export declare function calculateResponseCacheSavings(stats: CacheStats, pricing: {
|
|
157
|
+
avgInputTokens: number;
|
|
158
|
+
avgOutputTokens: number;
|
|
159
|
+
inputCostPerM: number;
|
|
160
|
+
outputCostPerM: number;
|
|
161
|
+
}): {
|
|
162
|
+
totalSaved: number;
|
|
163
|
+
tokensAvoided: number;
|
|
164
|
+
requestsAvoided: number;
|
|
165
|
+
};
|
|
166
|
+
//# sourceMappingURL=response-cache.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"response-cache.d.ts","sourceRoot":"","sources":["../../src/llm/response-cache.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAIH,OAAO,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAA;AAE5E,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAA;IACf,IAAI,EAAE,WAAW,CAAA;IACjB,YAAY,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,YAAY,GAAG,gBAAgB,CAAA;IAClE,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAA;IACtB,SAAS,EAAE,MAAM,CAAA;IACjB,KAAK,CAAC,EAAE;QACN,YAAY,EAAE,MAAM,CAAA;QACpB,gBAAgB,EAAE,MAAM,CAAA;QACxB,WAAW,EAAE,MAAM,CAAA;KACpB,CAAA;CACF;AAED,MAAM,WAAW,oBAAoB;IACnC,yDAAyD;IACzD,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,wDAAwD;IACxD,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,uDAAuD;IACvD,WAAW,CAAC,EAAE,OAAO,CAAA;CACtB;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;IACf,IAAI,EAAE,MAAM,CAAA;CACb;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,aAAa;IACxB,OAAO,CAAC,KAAK,CAAkC;IAC/C,OAAO,CAAC,KAAK,CAAqD;IAClE,OAAO,CAAC,WAAW,CAAS;gBAEhB,OAAO,GAAE,oBAAyB;IAqB9C;;;;;;;OAOG;IACH,WAAW,CACT,QAAQ,EAAE,OAAO,EAAE,EACnB,OAAO,CAAC,EAAE;QACR,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,SAAS,CAAC,EAAE,MAAM,CAAA;QAClB,KAAK,CAAC,EAAE,cAAc,EAAE,CAAA;QACxB,KAAK,CAAC,EAAE,MAAM,CAAA;KACf,GACA,MAAM;IAqBT;;;;OAIG;IACH,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS;IAiB5C;;OAEG;IACH,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,cAAc,GAAG,IAAI;IAShD;;OAEG;IACH,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAIzB;;OAEG;IACH,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAI5B;;OAEG;IACH,KAAK,IAAI,IAAI;IASb;;OAEG;IACH,QAAQ,IAAI,UAAU;IAatB;;OAEG;IACH,UAAU,IAAI,IAAI;IAQlB;;OAEG;IACH,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED;;OAEG;IACH,IAAI,OAAO,IAAI,MAAM,CAEpB;CACF;AAQD;;;;;;;;;GASG;AACH,wBAAgB,sBAAsB,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,aAAa,CAKpF;AAED;;GAEG;AACH,wBAAgB,wBAAwB,IAAI,IAAI,CAI/C;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,6BAA6B,CAC3C,KAAK,EAAE,UAAU,EACjB,OAAO,EAAE;IACP,cAAc,EAAE,MAAM,CAAA;IACtB,eAAe,EAAE,MAAM,CAAA;IACvB,aAAa,EAAE,MAAM,CAAA;IACrB,cAAc,EAAE,MAAM,CAAA;CACvB,GACA;IACD,UAAU,EAAE,MAAM,CAAA;IAClB,aAAa,EAAE,MAAM,CAAA;IACrB,eAAe,EAAE,MAAM,CAAA;CACxB,CAcA"}
|