adaptive-memory-multi-model-router 2.16.0 → 2.16.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/adapters-ci.yml +142 -0
- package/.github/workflows/auto-submit-sitemap.yml +41 -0
- package/.github/workflows/ci.yml +2 -5
- package/.github/workflows/mcp-pypi-publish.yml +34 -0
- package/.github/workflows/pypi-publish.yml +146 -0
- package/.github/workflows/tmlpd-publish.yml +23 -0
- package/README.md +245 -148
- package/RELEASE_v2.16.0.md +149 -0
- package/TECHNICAL_README.md +253 -0
- package/adapters/README.md +36 -0
- package/adapters/__init__.py +25 -0
- package/adapters/a3m_adapter/__init__.py +51 -0
- package/adapters/a3m_adapter/adapter/__init__.py +22 -0
- package/adapters/a3m_adapter/adapter/autogen.py +169 -0
- package/adapters/a3m_adapter/adapter/config.py +100 -0
- package/adapters/a3m_adapter/adapter/haystack.py +197 -0
- package/adapters/a3m_adapter/adapter/langchain.py +155 -0
- package/adapters/a3m_adapter/adapter/langgraph.py +196 -0
- package/adapters/a3m_adapter/adapter/llamaindex.py +162 -0
- package/adapters/a3m_adapter/adapter/pinecone.py +217 -0
- package/adapters/a3m_adapter/adapter/vercel.py +188 -0
- package/adapters/a3m_adapter/tests/__init__.py +1 -0
- package/adapters/a3m_adapter/tests/test_adapters.py +118 -0
- package/adapters/a3m_adapter/tests/test_integration.py +80 -0
- package/adapters/requirements-dev.txt +6 -0
- package/adapters/requirements.txt +4 -0
- package/adapters/setup.py +23 -0
- package/demo.py +251 -0
- package/discoverability-diagnosis.md +280 -0
- package/dist/analytics/costAnalytics.d.ts.map +1 -1
- package/dist/benchmark/reproducible.d.ts.map +1 -1
- package/dist/cache/semanticCache.d.ts.map +1 -1
- package/dist/cli/setupWizard.d.ts +257 -50
- package/dist/cli/setupWizard.d.ts.map +1 -1
- package/dist/cli/setupWizard.js +419 -109
- package/dist/cli/setupWizard.js.map +1 -1
- package/dist/cli/tui.d.ts +6 -0
- package/dist/cli/tui.js +96 -67
- package/dist/cli/tui.js.map +1 -0
- package/dist/cli.js +9 -0
- package/dist/cost/budgetEnforcer.d.ts.map +1 -1
- package/dist/cost/costTracker.d.ts.map +1 -1
- package/dist/ensemble/multiRoundDialog.d.ts.map +1 -1
- package/dist/ensemble/shapleyValue.d.ts.map +1 -1
- package/dist/ensemble.d.ts +1 -1
- package/dist/ensemble.js +141 -0
- package/dist/integrations/langchainAdapter.d.ts.map +1 -1
- package/dist/integrations/langchainAdapter.js +3 -3
- package/dist/integrations/langchainAdapter.js.map +1 -1
- package/dist/integrations/oauth.d.ts.map +1 -1
- package/dist/integrations/scienceAdapter.d.ts.map +1 -1
- package/dist/memory/autoFetch.d.ts.map +1 -1
- package/dist/memory/hybridMemory.d.ts.map +1 -1
- package/dist/memory/memoryTree.d.ts.map +1 -1
- package/dist/memory/obsidianVault.d.ts.map +1 -1
- package/dist/memory/reasoningBank.d.ts.map +1 -1
- package/dist/observability/metrics.d.ts.map +1 -1
- package/dist/observability/tracer.d.ts.map +1 -1
- package/dist/providers/providerConfig.d.ts.map +1 -1
- package/dist/providers/providerConfig.js +32 -17
- package/dist/providers/providerConfig.js.map +1 -1
- package/dist/routing/advancedRouter.d.ts.map +1 -1
- package/dist/routing/advancedRouter.js +106 -14
- package/dist/routing/advancedRouter.js.map +1 -1
- package/dist/routing/providerHealth.d.ts.map +1 -1
- package/dist/routing/providerRetry.d.ts.map +1 -1
- package/dist/routing/shadowSampler.d.ts.map +1 -0
- package/dist/routing/shadowSampler.js.map +1 -1
- package/dist/security/guardrails.d.ts.map +1 -1
- package/dist/server/handlers/chatHandler.d.ts.map +1 -1
- package/dist/server/handlers/completionsHandler.d.ts.map +1 -1
- package/dist/server/handlers/embeddingsHandler.d.ts.map +1 -1
- package/dist/server/handlers/healthHandler.d.ts.map +1 -1
- package/dist/server/handlers/metricsHandler.d.ts.map +1 -1
- package/dist/server/handlers/modelsHandler.d.ts.map +1 -1
- package/dist/server/metrics.d.ts.map +1 -1
- package/dist/server/proxyServer.d.ts.map +1 -1
- package/dist/server/router.d.ts.map +1 -1
- package/dist/server/state.d.ts.map +1 -1
- package/dist/skills/__tests__/skill_manager.test.js +5 -265
- package/dist/skills/__tests__/skill_manager.test.js.map +1 -1
- package/dist/utils/tokenUtils.d.ts.map +1 -1
- package/docker-compose.yml +84 -60
- package/docs/ARTICLE_Biology_Inspired_Routing.md +208 -0
- package/docs/ARTICLE_Master.md +78 -0
- package/docs/ARTICLE_Master_CN.md +78 -0
- package/docs/ARTICLE_OpenRouter_Stripe.md +140 -0
- package/docs/DEVPTO_ARTICLE.md +84 -0
- package/docs/HUMAN_STYLE_GUIDE.md +75 -0
- package/docs/IMPRINT_PLAN.md +88 -0
- package/docs/OPENROUTER_ALTERNATIVE.md +184 -0
- package/docs/SOCIAL_CAMPAIGN.md +316 -0
- package/docs/anthropic.html +45 -0
- package/docs/best-llm-routers-2025.html +157 -0
- package/docs/cerebras.html +43 -0
- package/docs/cli-cheatsheet.md +286 -212
- package/docs/comparison.md +2 -2
- package/docs/deepseek.html +44 -0
- package/docs/google.html +47 -0
- package/docs/groq.html +44 -0
- package/docs/llms-full.txt +360 -138
- package/docs/llms.txt +70 -71
- package/docs/mistral.html +43 -0
- package/docs/ollama.html +50 -0
- package/docs/openai.html +57 -0
- package/docs/sitemap.xml +69 -57
- package/docs-site/blog/best-llm-routers-2025.html +157 -0
- package/docs-site/index.html +68 -9
- package/docs-site/providers/anthropic.html +45 -0
- package/docs-site/providers/cerebras.html +43 -0
- package/docs-site/providers/deepseek.html +44 -0
- package/docs-site/providers/google.html +47 -0
- package/docs-site/providers/groq.html +44 -0
- package/docs-site/providers/index.html +41 -0
- package/docs-site/providers/mistral.html +43 -0
- package/docs-site/providers/ollama.html +50 -0
- package/docs-site/providers/openai.html +57 -0
- package/docs-site/sitemap.xml +69 -0
- package/llms.txt +70 -62
- package/package.json +44 -182
- package/packages/agentkit-adapter/LICENSE +21 -0
- package/packages/agentkit-adapter/README.md +126 -0
- package/packages/agentkit-adapter/examples/agentkit-example.ts +139 -0
- package/packages/agentkit-adapter/package.json +57 -0
- package/packages/agentkit-adapter/src/adapter.ts +381 -0
- package/packages/agentkit-adapter/src/index.ts +36 -0
- package/packages/agentkit-adapter/src/types.ts +105 -0
- package/packages/agentkit-adapter/src/util.ts +13 -0
- package/packages/agentkit-adapter/tsconfig.json +22 -0
- package/prometheus.yml +8 -0
- package/python/README.md +35 -81
- package/python/a3m/__init__.py +32 -3
- package/python/a3m/adapters/__init__.py +21 -0
- package/python/a3m/adapters/langchain.py +190 -0
- package/python/a3m/adapters/llamaindex.py +249 -0
- package/python/a3m/adapters/qdrant.py +240 -0
- package/python/a3m/adapters/weaviate.py +263 -0
- package/python/a3m/client.py +5 -0
- package/python/build_verify.sh +32 -0
- package/python/mcp-server/README.md +172 -0
- package/python/mcp-server/a3m_mcp/__init__.py +25 -0
- package/python/mcp-server/a3m_mcp/__main__.py +15 -0
- package/python/mcp-server/a3m_mcp/server.py +205 -0
- package/python/mcp-server/pyproject.toml +24 -0
- package/python/pyproject.toml +60 -6
- package/python/setup.py +3 -28
- package/scripts/submit-sitemap.sh +52 -0
- package/src/__types__/registry.d.ts +14 -0
- package/src/cli/setupWizard.ts +443 -112
- package/src/cli/tui.ts +159 -0
- package/src/ensemble.ts +154 -1
- package/src/integrations/langchainAdapter.ts +2 -2
- package/src/providers/providerConfig.ts +32 -17
- package/src/providers/registry.js +27 -0
- package/src/routing/advancedRouter.ts +99 -14
- package/src/routing/shadowSampler.ts +1 -1
- package/test-install/package.json +12 -0
- package/tests/tsconfig.json +0 -1
- package/tmlpd-pi-extension/README.md +105 -44
- package/tmlpd-pi-extension/docs/demo.svg +33 -0
- package/tmlpd-pi-extension/package.json +35 -106
- package/tmlpd-pi-extension/src/tokenOptimization/contextStratifier.ts +163 -0
- package/tmlpd-pi-extension/src/tokenOptimization/fetchOnceLocal.ts +136 -0
- package/tmlpd-pi-extension/src/tokenOptimization/index.ts +197 -0
- package/tmlpd-pi-extension/src/tokenOptimization/interAgentCompression.ts +157 -0
- package/tmlpd-pi-extension/src/tokenOptimization/schemaContract.ts +101 -0
- package/tmlpd-pi-extension/src/tokenOptimization/semanticCache.ts +248 -0
- package/tmlpd-pi-extension/src/tokenOptimization/tokenAwareFallback.ts +192 -0
- package/tmlpd-pi-extension/test/verify.js +21 -0
- package/tsconfig.build.json +3 -2
- package/packages/a3m-vercel-ai/dist/a3m-language-model.d.ts +0 -12
- package/packages/a3m-vercel-ai/dist/a3m-language-model.d.ts.map +0 -1
- package/packages/a3m-vercel-ai/dist/a3m-language-model.js +0 -289
- package/packages/a3m-vercel-ai/dist/a3m-language-model.js.map +0 -1
- package/packages/a3m-vercel-ai/dist/index.d.ts +0 -82
- package/packages/a3m-vercel-ai/dist/index.d.ts.map +0 -1
- package/packages/a3m-vercel-ai/dist/index.js +0 -79
- package/packages/a3m-vercel-ai/dist/index.js.map +0 -1
- package/packages/a3m-vercel-ai/dist/types.d.ts +0 -97
- package/packages/a3m-vercel-ai/dist/types.d.ts.map +0 -1
- package/packages/a3m-vercel-ai/dist/types.js +0 -5
- package/packages/a3m-vercel-ai/dist/types.js.map +0 -1
- package/src/skills/__tests__/skill_manager.test.ts +0 -328
- package/tmlpd-pi-extension/dist/cache/prefixCache.d.ts +0 -114
- package/tmlpd-pi-extension/dist/cache/prefixCache.d.ts.map +0 -1
- package/tmlpd-pi-extension/dist/cache/prefixCache.js +0 -285
- package/tmlpd-pi-extension/dist/cache/prefixCache.js.map +0 -1
- package/tmlpd-pi-extension/dist/cache/responseCache.d.ts +0 -58
- package/tmlpd-pi-extension/dist/cache/responseCache.d.ts.map +0 -1
- package/tmlpd-pi-extension/dist/cache/responseCache.js +0 -153
- package/tmlpd-pi-extension/dist/cache/responseCache.js.map +0 -1
- package/tmlpd-pi-extension/dist/cli.js +0 -59
- package/tmlpd-pi-extension/dist/cost/costTracker.d.ts +0 -95
- package/tmlpd-pi-extension/dist/cost/costTracker.d.ts.map +0 -1
- package/tmlpd-pi-extension/dist/cost/costTracker.js +0 -240
- package/tmlpd-pi-extension/dist/cost/costTracker.js.map +0 -1
- package/tmlpd-pi-extension/dist/index.d.ts +0 -723
- package/tmlpd-pi-extension/dist/index.d.ts.map +0 -1
- package/tmlpd-pi-extension/dist/index.js +0 -239
- package/tmlpd-pi-extension/dist/index.js.map +0 -1
- package/tmlpd-pi-extension/dist/memory/episodicMemory.d.ts +0 -82
- package/tmlpd-pi-extension/dist/memory/episodicMemory.d.ts.map +0 -1
- package/tmlpd-pi-extension/dist/memory/episodicMemory.js +0 -145
- package/tmlpd-pi-extension/dist/memory/episodicMemory.js.map +0 -1
- package/tmlpd-pi-extension/dist/orchestration/haloOrchestrator.d.ts +0 -102
- package/tmlpd-pi-extension/dist/orchestration/haloOrchestrator.d.ts.map +0 -1
- package/tmlpd-pi-extension/dist/orchestration/haloOrchestrator.js +0 -207
- package/tmlpd-pi-extension/dist/orchestration/haloOrchestrator.js.map +0 -1
- package/tmlpd-pi-extension/dist/orchestration/mctsWorkflow.d.ts +0 -85
- package/tmlpd-pi-extension/dist/orchestration/mctsWorkflow.d.ts.map +0 -1
- package/tmlpd-pi-extension/dist/orchestration/mctsWorkflow.js +0 -210
- package/tmlpd-pi-extension/dist/orchestration/mctsWorkflow.js.map +0 -1
- package/tmlpd-pi-extension/dist/providers/localProvider.d.ts +0 -102
- package/tmlpd-pi-extension/dist/providers/localProvider.d.ts.map +0 -1
- package/tmlpd-pi-extension/dist/providers/localProvider.js +0 -338
- package/tmlpd-pi-extension/dist/providers/localProvider.js.map +0 -1
- package/tmlpd-pi-extension/dist/providers/registry.d.ts +0 -55
- package/tmlpd-pi-extension/dist/providers/registry.d.ts.map +0 -1
- package/tmlpd-pi-extension/dist/providers/registry.js +0 -138
- package/tmlpd-pi-extension/dist/providers/registry.js.map +0 -1
- package/tmlpd-pi-extension/dist/routing/advancedRouter.d.ts +0 -68
- package/tmlpd-pi-extension/dist/routing/advancedRouter.d.ts.map +0 -1
- package/tmlpd-pi-extension/dist/routing/advancedRouter.js +0 -332
- package/tmlpd-pi-extension/dist/routing/advancedRouter.js.map +0 -1
- package/tmlpd-pi-extension/dist/tools/tmlpdTools.d.ts +0 -101
- package/tmlpd-pi-extension/dist/tools/tmlpdTools.d.ts.map +0 -1
- package/tmlpd-pi-extension/dist/tools/tmlpdTools.js +0 -368
- package/tmlpd-pi-extension/dist/tools/tmlpdTools.js.map +0 -1
- package/tmlpd-pi-extension/dist/utils/batchProcessor.d.ts +0 -96
- package/tmlpd-pi-extension/dist/utils/batchProcessor.d.ts.map +0 -1
- package/tmlpd-pi-extension/dist/utils/batchProcessor.js +0 -170
- package/tmlpd-pi-extension/dist/utils/batchProcessor.js.map +0 -1
- package/tmlpd-pi-extension/dist/utils/compression.d.ts +0 -61
- package/tmlpd-pi-extension/dist/utils/compression.d.ts.map +0 -1
- package/tmlpd-pi-extension/dist/utils/compression.js +0 -281
- package/tmlpd-pi-extension/dist/utils/compression.js.map +0 -1
- package/tmlpd-pi-extension/dist/utils/reliability.d.ts +0 -74
- package/tmlpd-pi-extension/dist/utils/reliability.d.ts.map +0 -1
- package/tmlpd-pi-extension/dist/utils/reliability.js +0 -177
- package/tmlpd-pi-extension/dist/utils/reliability.js.map +0 -1
- package/tmlpd-pi-extension/dist/utils/speculativeDecoding.d.ts +0 -117
- package/tmlpd-pi-extension/dist/utils/speculativeDecoding.d.ts.map +0 -1
- package/tmlpd-pi-extension/dist/utils/speculativeDecoding.js +0 -246
- package/tmlpd-pi-extension/dist/utils/speculativeDecoding.js.map +0 -1
- package/tmlpd-pi-extension/dist/utils/tokenUtils.d.ts +0 -50
- package/tmlpd-pi-extension/dist/utils/tokenUtils.d.ts.map +0 -1
- package/tmlpd-pi-extension/dist/utils/tokenUtils.js +0 -124
- package/tmlpd-pi-extension/dist/utils/tokenUtils.js.map +0 -1
|
@@ -1,285 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
/**
|
|
3
|
-
* TMLPD Prefix Cache - RadixAttention Style
|
|
4
|
-
*
|
|
5
|
-
* Inspired by SGLang's RadixAttention (arXiv:2312.07104)
|
|
6
|
-
* Caches KV states for common prefixes (system prompts, etc.)
|
|
7
|
-
* 5-10x speedup for repeated prompt patterns
|
|
8
|
-
*/
|
|
9
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
-
exports.PrefixCache = void 0;
|
|
11
|
-
exports.createWarmedCache = createWarmedCache;
|
|
12
|
-
class PrefixCache {
|
|
13
|
-
entries = new Map();
|
|
14
|
-
access_order = []; // LRU tracking
|
|
15
|
-
max_entries;
|
|
16
|
-
max_memory_mb;
|
|
17
|
-
constructor(options) {
|
|
18
|
-
this.max_entries = options?.max_entries || 10000;
|
|
19
|
-
this.max_memory_mb = options?.max_memory_mb || 512;
|
|
20
|
-
}
|
|
21
|
-
/**
|
|
22
|
-
* Generate cache key from text prefix
|
|
23
|
-
*/
|
|
24
|
-
generateKey(text, model) {
|
|
25
|
-
// Simple hash for now - in production use SHA-256
|
|
26
|
-
const normalized = text.toLowerCase().trim().substring(0, 500);
|
|
27
|
-
const str = `${model || "default"}:${normalized}`;
|
|
28
|
-
let hash = 0;
|
|
29
|
-
for (let i = 0; i < str.length; i++) {
|
|
30
|
-
const char = str.charCodeAt(i);
|
|
31
|
-
hash = ((hash << 5) - hash) + char;
|
|
32
|
-
hash = hash & hash; // Convert to 32bit integer
|
|
33
|
-
}
|
|
34
|
-
return `pc_${Math.abs(hash).toString(16)}`;
|
|
35
|
-
}
|
|
36
|
-
/**
|
|
37
|
-
* Check if prefix is cached
|
|
38
|
-
*/
|
|
39
|
-
has(prefix, model) {
|
|
40
|
-
const key = this.generateKey(prefix, model);
|
|
41
|
-
return this.entries.has(key);
|
|
42
|
-
}
|
|
43
|
-
/**
|
|
44
|
-
* Get cached entry
|
|
45
|
-
*/
|
|
46
|
-
get(prefix, model) {
|
|
47
|
-
const key = this.generateKey(prefix, model);
|
|
48
|
-
const entry = this.entries.get(key);
|
|
49
|
-
if (entry) {
|
|
50
|
-
// Update LRU
|
|
51
|
-
this.updateLRU(key);
|
|
52
|
-
entry.hit_count++;
|
|
53
|
-
entry.last_used = Date.now();
|
|
54
|
-
}
|
|
55
|
-
return entry;
|
|
56
|
-
}
|
|
57
|
-
/**
|
|
58
|
-
* Store a new prefix with its KV state
|
|
59
|
-
*/
|
|
60
|
-
store(prefix, options) {
|
|
61
|
-
const key = this.generateKey(prefix, options?.model);
|
|
62
|
-
// Check if already exists
|
|
63
|
-
if (this.entries.has(key)) {
|
|
64
|
-
const existing = this.entries.get(key);
|
|
65
|
-
existing.hit_count++;
|
|
66
|
-
existing.last_used = Date.now();
|
|
67
|
-
return key;
|
|
68
|
-
}
|
|
69
|
-
// Estimate memory
|
|
70
|
-
const token_count = Math.ceil(prefix.split(/\s+/).length * 1.3);
|
|
71
|
-
const memory_bytes = token_count * 16 * 128 * 2; // Rough KV estimate
|
|
72
|
-
const memory_mb = memory_bytes / (1024 * 1024);
|
|
73
|
-
const entry = {
|
|
74
|
-
key,
|
|
75
|
-
prefix: prefix.substring(0, 1000), // Store truncated
|
|
76
|
-
kv_state: options?.kv_state,
|
|
77
|
-
response_hash: options?.response_hash,
|
|
78
|
-
hit_count: 1,
|
|
79
|
-
last_used: Date.now(),
|
|
80
|
-
token_count,
|
|
81
|
-
children: options?.children || new Map()
|
|
82
|
-
};
|
|
83
|
-
// Evict if necessary
|
|
84
|
-
while (this.entries.size >= this.max_entries || this.getMemoryUsage() + memory_mb > this.max_memory_mb) {
|
|
85
|
-
this.evictLRU();
|
|
86
|
-
}
|
|
87
|
-
this.entries.set(key, entry);
|
|
88
|
-
this.access_order.push(key);
|
|
89
|
-
return key;
|
|
90
|
-
}
|
|
91
|
-
/**
|
|
92
|
-
* Extend cached prefix with completion
|
|
93
|
-
*/
|
|
94
|
-
extend(prefix, completion, options) {
|
|
95
|
-
const prefix_key = this.generateKey(prefix, options?.model);
|
|
96
|
-
const parent = this.entries.get(prefix_key);
|
|
97
|
-
if (!parent) {
|
|
98
|
-
// No parent - just store completion as new entry
|
|
99
|
-
return this.store(completion, { model: options?.model });
|
|
100
|
-
}
|
|
101
|
-
// Create child entry for the extended sequence
|
|
102
|
-
const extended = prefix + completion;
|
|
103
|
-
const child_key = this.store(extended, { model: options?.model });
|
|
104
|
-
// Link child to parent
|
|
105
|
-
const completion_key = this.generateKey(completion);
|
|
106
|
-
parent.children.set(completion_key, child_key);
|
|
107
|
-
return child_key;
|
|
108
|
-
}
|
|
109
|
-
/**
|
|
110
|
-
* Find common prefix between two texts
|
|
111
|
-
*/
|
|
112
|
-
findCommonPrefix(text1, text2) {
|
|
113
|
-
const words1 = text1.split(/\s+/);
|
|
114
|
-
const words2 = text2.split(/\s+/);
|
|
115
|
-
let common_length = 0;
|
|
116
|
-
for (let i = 0; i < Math.min(words1.length, words2.length); i++) {
|
|
117
|
-
if (words1[i].toLowerCase() === words2[i].toLowerCase()) {
|
|
118
|
-
common_length = i + 1;
|
|
119
|
-
}
|
|
120
|
-
else {
|
|
121
|
-
break;
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
return words1.slice(0, common_length).join(" ");
|
|
125
|
-
}
|
|
126
|
-
/**
|
|
127
|
-
* Lookup with prefix matching
|
|
128
|
-
* Returns cached entry if any prefix is found
|
|
129
|
-
*/
|
|
130
|
-
lookup(text, model) {
|
|
131
|
-
// Try exact match first
|
|
132
|
-
const exact_key = this.generateKey(text, model);
|
|
133
|
-
if (this.entries.has(exact_key)) {
|
|
134
|
-
return { cached: true };
|
|
135
|
-
}
|
|
136
|
-
// Try progressively shorter prefixes
|
|
137
|
-
const words = text.split(/\s+/);
|
|
138
|
-
for (let len = words.length - 1; len >= 5; len--) { // Min 5 words
|
|
139
|
-
const prefix = words.slice(0, len).join(" ");
|
|
140
|
-
const key = this.generateKey(prefix, model);
|
|
141
|
-
if (this.entries.has(key)) {
|
|
142
|
-
const remaining = words.slice(len).join(" ");
|
|
143
|
-
return { cached: true, prefix, remaining };
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
return { cached: false };
|
|
147
|
-
}
|
|
148
|
-
/**
|
|
149
|
-
* Batch lookup for multiple texts
|
|
150
|
-
*/
|
|
151
|
-
lookupBatch(texts, model) {
|
|
152
|
-
return texts.map(t => this.lookup(t, model));
|
|
153
|
-
}
|
|
154
|
-
/**
|
|
155
|
-
* Get cache statistics
|
|
156
|
-
*/
|
|
157
|
-
getStats() {
|
|
158
|
-
const now = Date.now();
|
|
159
|
-
let oldest_age = 0;
|
|
160
|
-
let total_hits = 0;
|
|
161
|
-
for (const entry of this.entries.values()) {
|
|
162
|
-
total_hits += entry.hit_count;
|
|
163
|
-
const age = now - entry.last_used;
|
|
164
|
-
if (age > oldest_age)
|
|
165
|
-
oldest_age = age;
|
|
166
|
-
}
|
|
167
|
-
const total_requests = total_hits + this.entries.size; // Approximate
|
|
168
|
-
const hit_rate = total_requests > 0 ? total_hits / total_requests : 0;
|
|
169
|
-
return {
|
|
170
|
-
total_entries: this.entries.size,
|
|
171
|
-
total_hits: total_hits,
|
|
172
|
-
total_misses: this.entries.size, // Approximate
|
|
173
|
-
hit_rate,
|
|
174
|
-
memory_estimate_mb: this.getMemoryUsage(),
|
|
175
|
-
oldest_entry_age_ms: oldest_age
|
|
176
|
-
};
|
|
177
|
-
}
|
|
178
|
-
/**
|
|
179
|
-
* Get estimated memory usage
|
|
180
|
-
*/
|
|
181
|
-
getMemoryUsage() {
|
|
182
|
-
let total_bytes = 0;
|
|
183
|
-
for (const entry of this.entries.values()) {
|
|
184
|
-
// Base entry overhead
|
|
185
|
-
total_bytes += 200;
|
|
186
|
-
// Prefix text
|
|
187
|
-
total_bytes += entry.prefix.length * 2;
|
|
188
|
-
// KV state (if stored)
|
|
189
|
-
if (entry.kv_state) {
|
|
190
|
-
total_bytes += entry.kv_state.length;
|
|
191
|
-
}
|
|
192
|
-
// Children map
|
|
193
|
-
total_bytes += entry.children.size * 50;
|
|
194
|
-
}
|
|
195
|
-
return total_bytes / (1024 * 1024);
|
|
196
|
-
}
|
|
197
|
-
/**
|
|
198
|
-
* Update LRU order
|
|
199
|
-
*/
|
|
200
|
-
updateLRU(key) {
|
|
201
|
-
const index = this.access_order.indexOf(key);
|
|
202
|
-
if (index > -1) {
|
|
203
|
-
this.access_order.splice(index, 1);
|
|
204
|
-
}
|
|
205
|
-
this.access_order.push(key);
|
|
206
|
-
}
|
|
207
|
-
/**
|
|
208
|
-
* Evict least recently used entry
|
|
209
|
-
*/
|
|
210
|
-
evictLRU() {
|
|
211
|
-
if (this.access_order.length === 0)
|
|
212
|
-
return false;
|
|
213
|
-
const lru_key = this.access_order.shift();
|
|
214
|
-
const entry = this.entries.get(lru_key);
|
|
215
|
-
if (entry) {
|
|
216
|
-
// If has children, re-parent them
|
|
217
|
-
for (const [child_key, child_cache_key] of entry.children) {
|
|
218
|
-
const child = this.entries.get(child_cache_key);
|
|
219
|
-
if (child) {
|
|
220
|
-
// Promote child to standalone
|
|
221
|
-
this.access_order.push(child_cache_key);
|
|
222
|
-
}
|
|
223
|
-
}
|
|
224
|
-
this.entries.delete(lru_key);
|
|
225
|
-
return true;
|
|
226
|
-
}
|
|
227
|
-
return false;
|
|
228
|
-
}
|
|
229
|
-
/**
|
|
230
|
-
* Clear all cache
|
|
231
|
-
*/
|
|
232
|
-
clear() {
|
|
233
|
-
this.entries.clear();
|
|
234
|
-
this.access_order = [];
|
|
235
|
-
}
|
|
236
|
-
/**
|
|
237
|
-
* Invalidate entries matching pattern
|
|
238
|
-
*/
|
|
239
|
-
invalidate(pattern) {
|
|
240
|
-
let count = 0;
|
|
241
|
-
if (!pattern) {
|
|
242
|
-
// Clear all
|
|
243
|
-
count = this.entries.size;
|
|
244
|
-
this.clear();
|
|
245
|
-
return count;
|
|
246
|
-
}
|
|
247
|
-
// Pattern-based invalidation
|
|
248
|
-
for (const [key, entry] of this.entries) {
|
|
249
|
-
if (entry.prefix.includes(pattern)) {
|
|
250
|
-
this.entries.delete(key);
|
|
251
|
-
count++;
|
|
252
|
-
}
|
|
253
|
-
}
|
|
254
|
-
return count;
|
|
255
|
-
}
|
|
256
|
-
/**
|
|
257
|
-
* Warm up cache with common system prompts
|
|
258
|
-
*/
|
|
259
|
-
warmup(common_prefixes, model) {
|
|
260
|
-
for (const prefix of common_prefixes) {
|
|
261
|
-
this.store(prefix, { model });
|
|
262
|
-
}
|
|
263
|
-
console.log(`[PrefixCache] Warmed up with ${common_prefixes.length} common prefixes`);
|
|
264
|
-
}
|
|
265
|
-
}
|
|
266
|
-
exports.PrefixCache = PrefixCache;
|
|
267
|
-
// Common system prompts that benefit from prefix caching
|
|
268
|
-
const COMMON_SYSTEM_PROMPTS = [
|
|
269
|
-
"You are a helpful assistant.",
|
|
270
|
-
"You are a coding assistant. Help with programming tasks.",
|
|
271
|
-
"You are an expert data scientist.",
|
|
272
|
-
"You are a senior software engineer.",
|
|
273
|
-
"Analyze the following code and provide feedback.",
|
|
274
|
-
"Explain this concept in simple terms.",
|
|
275
|
-
"Write clean, well-documented code.",
|
|
276
|
-
"Think step by step and explain your reasoning."
|
|
277
|
-
];
|
|
278
|
-
exports.default = PrefixCache;
|
|
279
|
-
// Utility function for creating pre-warmed cache
|
|
280
|
-
function createWarmedCache() {
|
|
281
|
-
const cache = new PrefixCache({ max_entries: 5000 });
|
|
282
|
-
cache.warmup(COMMON_SYSTEM_PROMPTS);
|
|
283
|
-
return cache;
|
|
284
|
-
}
|
|
285
|
-
//# sourceMappingURL=prefixCache.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"prefixCache.js","sourceRoot":"","sources":["../../src/cache/prefixCache.ts"],"names":[],"mappings":";AAAA;;;;;;GAMG;;;AAkWH,8CAIC;AAhVD,MAAa,WAAW;IACd,OAAO,GAA4B,IAAI,GAAG,EAAE,CAAC;IAC7C,YAAY,GAAa,EAAE,CAAC,CAAE,eAAe;IAC7C,WAAW,CAAS;IACpB,aAAa,CAAS;IAE9B,YAAY,OAGX;QACC,IAAI,CAAC,WAAW,GAAG,OAAO,EAAE,WAAW,IAAI,KAAK,CAAC;QACjD,IAAI,CAAC,aAAa,GAAG,OAAO,EAAE,aAAa,IAAI,GAAG,CAAC;IACrD,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,IAAY,EAAE,KAAc;QAC9C,kDAAkD;QAClD,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC/D,MAAM,GAAG,GAAG,GAAG,KAAK,IAAI,SAAS,IAAI,UAAU,EAAE,CAAC;QAElD,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,MAAM,IAAI,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAC/B,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;YACnC,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAE,2BAA2B;QAClD,CAAC;QAED,OAAO,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,MAAc,EAAE,KAAc;QAChC,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAC5C,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,MAAc,EAAE,KAAc;QAChC,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEpC,IAAI,KAAK,EAAE,CAAC;YACV,aAAa;YACb,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACpB,KAAK,CAAC,SAAS,EAAE,CAAC;YAClB,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC/B,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACH,KAAK,CACH,MAAc,EACd,OAKC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;QAErD,0BAA0B;QAC1B,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC;YACxC,QAAQ,CAAC,SAAS,EAAE,CAAC;YACrB,QAAQ,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAChC,OAAO,GAAG,CAAC;QACb,CAAC;QAED,kBAAkB;QAClB,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;QAChE,MAAM,YAAY,GAAG,WAAW,GAAG,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC,CAAE,oBAAoB;QACtE,MAAM,SAAS,GAAG,YAAY,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;QAE/C,MAAM,KAAK,GAAe;YACxB,GAAG;YACH,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,EAAG,kBAAkB;YACtD,QAAQ,EAAE,OAAO,EAAE,QAAQ;YAC3B,aAAa,EAAE,OAAO,EAAE,aAAa;YACrC,SAAS,EAAE,CAAC;YACZ,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;YACrB,WAAW;YACX,QAAQ,EAAE,OAAO,EAAE,QAAQ,IAAI,IAAI,GAAG,EAAE;SACzC,CAAC;QAEF,qBAAqB;QACrB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,cAAc,EAAE,GAAG,SAAS,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YACvG,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClB,CAAC;QAED,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC7B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAE5B,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;OAEG;IACH,MAAM,CACJ,MAAc,EACd,UAAkB,EAClB,OAA4B;QAE5B,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAE5C,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,iDAAiD;YACjD,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3D,CAAC;QAED,+CAA+C;QAC/C,MAAM,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;QACrC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;QAElE,uBAAuB;QACvB,MAAM,cAAc,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QACpD,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;QAE/C,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACH,gBAAgB,CAAC,KAAa,EAAE,KAAa;QAC3C,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAClC,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAElC,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAChE,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;gBACxD,aAAa,GAAG,CAAC,GAAG,CAAC,CAAC;YACxB,CAAC;iBAAM,CAAC;gBACN,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAClD,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,IAAY,EAAE,KAAc;QACjC,wBAAwB;QACxB,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAChD,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YAChC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QAC1B,CAAC;QAED,qCAAqC;QACrC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAChC,KAAK,IAAI,GAAG,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAE,cAAc;YACjE,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;YAE5C,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC7C,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;YAC7C,CAAC;QACH,CAAC;QAED,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,KAAe,EAAE,KAAc;QACzC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;IAC/C,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC1C,UAAU,IAAI,KAAK,CAAC,SAAS,CAAC;YAC9B,MAAM,GAAG,GAAG,GAAG,GAAG,KAAK,CAAC,SAAS,CAAC;YAClC,IAAI,GAAG,GAAG,UAAU;gBAAE,UAAU,GAAG,GAAG,CAAC;QACzC,CAAC;QAED,MAAM,cAAc,GAAG,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAE,cAAc;QACtE,MAAM,QAAQ,GAAG,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC;QAEtE,OAAO;YACL,aAAa,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI;YAChC,UAAU,EAAE,UAAU;YACtB,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI,EAAG,cAAc;YAChD,QAAQ;YACR,kBAAkB,EAAE,IAAI,CAAC,cAAc,EAAE;YACzC,mBAAmB,EAAE,UAAU;SAChC,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,cAAc;QACpB,IAAI,WAAW,GAAG,CAAC,CAAC;QAEpB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC1C,sBAAsB;YACtB,WAAW,IAAI,GAAG,CAAC;YAEnB,cAAc;YACd,WAAW,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;YAEvC,uBAAuB;YACvB,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACnB,WAAW,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC;YACvC,CAAC;YAED,eAAe;YACf,WAAW,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,GAAG,EAAE,CAAC;QAC1C,CAAC;QAED,OAAO,WAAW,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;IACrC,CAAC;IAED;;OAEG;IACK,SAAS,CAAC,GAAW;QAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC7C,IAAI,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC;YACf,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACrC,CAAC;QACD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC9B,CAAC;IAED;;OAEG;IACK,QAAQ;QACd,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QAEjD,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,EAAG,CAAC;QAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAExC,IAAI,KAAK,EAAE,CAAC;YACV,kCAAkC;YAClC,KAAK,MAAM,CAAC,SAAS,EAAE,eAAe,CAAC,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;gBAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;gBAChD,IAAI,KAAK,EAAE,CAAC;oBACV,8BAA8B;oBAC9B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;gBAC1C,CAAC;YACH,CAAC;YAED,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC7B,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACrB,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,OAAgB;QACzB,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,YAAY;YACZ,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YAC1B,IAAI,CAAC,KAAK,EAAE,CAAC;YACb,OAAO,KAAK,CAAC;QACf,CAAC;QAED,6BAA6B;QAC7B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACxC,IAAI,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;gBACnC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACzB,KAAK,EAAE,CAAC;YACV,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,eAAyB,EAAE,KAAc;QAC9C,KAAK,MAAM,MAAM,IAAI,eAAe,EAAE,CAAC;YACrC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QAChC,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,gCAAgC,eAAe,CAAC,MAAM,kBAAkB,CAAC,CAAC;IACxF,CAAC;CACF;AA3TD,kCA2TC;AAED,yDAAyD;AACzD,MAAM,qBAAqB,GAAG;IAC5B,8BAA8B;IAC9B,0DAA0D;IAC1D,mCAAmC;IACnC,qCAAqC;IACrC,kDAAkD;IAClD,uCAAuC;IACvC,oCAAoC;IACpC,gDAAgD;CACjD,CAAC;AAEF,kBAAe,WAAW,CAAC;AAE3B,iDAAiD;AACjD,SAAgB,iBAAiB;IAC/B,MAAM,KAAK,GAAG,IAAI,WAAW,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;IACrD,KAAK,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC;IACpC,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -1,58 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* TMLPD Response Cache
|
|
3
|
-
*
|
|
4
|
-
* Caches LLM responses to avoid redundant API calls.
|
|
5
|
-
* Uses content hash for cache key and supports TTL.
|
|
6
|
-
*/
|
|
7
|
-
export interface CacheEntry {
|
|
8
|
-
content: string;
|
|
9
|
-
model: string;
|
|
10
|
-
provider: string;
|
|
11
|
-
tokens: number;
|
|
12
|
-
cost: number;
|
|
13
|
-
cached_at: number;
|
|
14
|
-
expires_at: number;
|
|
15
|
-
}
|
|
16
|
-
export interface CacheConfig {
|
|
17
|
-
enabled: boolean;
|
|
18
|
-
ttl_seconds: number;
|
|
19
|
-
max_entries: number;
|
|
20
|
-
cache_dir?: string;
|
|
21
|
-
}
|
|
22
|
-
export declare class ResponseCache {
|
|
23
|
-
private cache;
|
|
24
|
-
private config;
|
|
25
|
-
private hits;
|
|
26
|
-
private misses;
|
|
27
|
-
constructor(config?: Partial<CacheConfig>);
|
|
28
|
-
/**
|
|
29
|
-
* Generate cache key from prompt + model
|
|
30
|
-
*/
|
|
31
|
-
generateKey(prompt: string, model: string): string;
|
|
32
|
-
/**
|
|
33
|
-
* Get cached response if available and not expired
|
|
34
|
-
*/
|
|
35
|
-
get(prompt: string, model: string): CacheEntry | null;
|
|
36
|
-
/**
|
|
37
|
-
* Store response in cache
|
|
38
|
-
*/
|
|
39
|
-
set(prompt: string, model: string, response: Partial<CacheEntry>): void;
|
|
40
|
-
/**
|
|
41
|
-
* Invalidate cache for specific model or all
|
|
42
|
-
*/
|
|
43
|
-
invalidate(model?: string): void;
|
|
44
|
-
/**
|
|
45
|
-
* Get cache statistics
|
|
46
|
-
*/
|
|
47
|
-
getStats(): {
|
|
48
|
-
hits: number;
|
|
49
|
-
misses: number;
|
|
50
|
-
size: number;
|
|
51
|
-
hit_rate: number;
|
|
52
|
-
};
|
|
53
|
-
/**
|
|
54
|
-
* Evict oldest entry by cached_at timestamp
|
|
55
|
-
*/
|
|
56
|
-
private evictOldest;
|
|
57
|
-
}
|
|
58
|
-
//# sourceMappingURL=responseCache.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"responseCache.d.ts","sourceRoot":"","sources":["../../src/cache/responseCache.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,aAAa;IACxB,OAAO,CAAC,KAAK,CAAsC;IACnD,OAAO,CAAC,MAAM,CAAc;IAC5B,OAAO,CAAC,IAAI,CAAK;IACjB,OAAO,CAAC,MAAM,CAAK;gBAEP,MAAM,GAAE,OAAO,CAAC,WAAW,CAAM;IAS7C;;OAEG;IACH,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM;IAMlD;;OAEG;IACH,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,UAAU,GAAG,IAAI;IAsBrD;;OAEG;IACH,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,UAAU,CAAC,GAAG,IAAI;IAsBvE;;OAEG;IACH,UAAU,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI;IAYhC;;OAEG;IACH,QAAQ,IAAI;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE;IAU5E;;OAEG;IACH,OAAO,CAAC,WAAW;CAepB"}
|
|
@@ -1,153 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
/**
|
|
3
|
-
* TMLPD Response Cache
|
|
4
|
-
*
|
|
5
|
-
* Caches LLM responses to avoid redundant API calls.
|
|
6
|
-
* Uses content hash for cache key and supports TTL.
|
|
7
|
-
*/
|
|
8
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
9
|
-
if (k2 === undefined) k2 = k;
|
|
10
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
11
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
12
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
13
|
-
}
|
|
14
|
-
Object.defineProperty(o, k2, desc);
|
|
15
|
-
}) : (function(o, m, k, k2) {
|
|
16
|
-
if (k2 === undefined) k2 = k;
|
|
17
|
-
o[k2] = m[k];
|
|
18
|
-
}));
|
|
19
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
20
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
21
|
-
}) : function(o, v) {
|
|
22
|
-
o["default"] = v;
|
|
23
|
-
});
|
|
24
|
-
var __importStar = (this && this.__importStar) || (function () {
|
|
25
|
-
var ownKeys = function(o) {
|
|
26
|
-
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
27
|
-
var ar = [];
|
|
28
|
-
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
29
|
-
return ar;
|
|
30
|
-
};
|
|
31
|
-
return ownKeys(o);
|
|
32
|
-
};
|
|
33
|
-
return function (mod) {
|
|
34
|
-
if (mod && mod.__esModule) return mod;
|
|
35
|
-
var result = {};
|
|
36
|
-
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
37
|
-
__setModuleDefault(result, mod);
|
|
38
|
-
return result;
|
|
39
|
-
};
|
|
40
|
-
})();
|
|
41
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
42
|
-
exports.ResponseCache = void 0;
|
|
43
|
-
const crypto = __importStar(require("crypto"));
|
|
44
|
-
class ResponseCache {
|
|
45
|
-
cache = new Map();
|
|
46
|
-
config;
|
|
47
|
-
hits = 0;
|
|
48
|
-
misses = 0;
|
|
49
|
-
constructor(config = {}) {
|
|
50
|
-
this.config = {
|
|
51
|
-
enabled: config.enabled ?? true,
|
|
52
|
-
ttl_seconds: config.ttl_seconds ?? 3600, // 1 hour default
|
|
53
|
-
max_entries: config.max_entries ?? 1000,
|
|
54
|
-
cache_dir: config.cache_dir,
|
|
55
|
-
};
|
|
56
|
-
}
|
|
57
|
-
/**
|
|
58
|
-
* Generate cache key from prompt + model
|
|
59
|
-
*/
|
|
60
|
-
generateKey(prompt, model) {
|
|
61
|
-
const hash = crypto.createHash("sha256");
|
|
62
|
-
hash.update(prompt + "|" + model);
|
|
63
|
-
return hash.digest("hex").substring(0, 32);
|
|
64
|
-
}
|
|
65
|
-
/**
|
|
66
|
-
* Get cached response if available and not expired
|
|
67
|
-
*/
|
|
68
|
-
get(prompt, model) {
|
|
69
|
-
if (!this.config.enabled)
|
|
70
|
-
return null;
|
|
71
|
-
const key = this.generateKey(prompt, model);
|
|
72
|
-
const entry = this.cache.get(key);
|
|
73
|
-
if (!entry) {
|
|
74
|
-
this.misses++;
|
|
75
|
-
return null;
|
|
76
|
-
}
|
|
77
|
-
// Check expiration
|
|
78
|
-
if (Date.now() > entry.expires_at) {
|
|
79
|
-
this.cache.delete(key);
|
|
80
|
-
this.misses++;
|
|
81
|
-
return null;
|
|
82
|
-
}
|
|
83
|
-
this.hits++;
|
|
84
|
-
return entry;
|
|
85
|
-
}
|
|
86
|
-
/**
|
|
87
|
-
* Store response in cache
|
|
88
|
-
*/
|
|
89
|
-
set(prompt, model, response) {
|
|
90
|
-
if (!this.config.enabled)
|
|
91
|
-
return;
|
|
92
|
-
const key = this.generateKey(prompt, model);
|
|
93
|
-
const now = Date.now();
|
|
94
|
-
// Evict oldest if at capacity
|
|
95
|
-
if (this.cache.size >= this.config.max_entries) {
|
|
96
|
-
this.evictOldest();
|
|
97
|
-
}
|
|
98
|
-
this.cache.set(key, {
|
|
99
|
-
content: response.content ?? "",
|
|
100
|
-
model: response.model ?? model,
|
|
101
|
-
provider: response.provider ?? "unknown",
|
|
102
|
-
tokens: response.tokens ?? 0,
|
|
103
|
-
cost: response.cost ?? 0,
|
|
104
|
-
cached_at: now,
|
|
105
|
-
expires_at: now + this.config.ttl_seconds * 1000,
|
|
106
|
-
});
|
|
107
|
-
}
|
|
108
|
-
/**
|
|
109
|
-
* Invalidate cache for specific model or all
|
|
110
|
-
*/
|
|
111
|
-
invalidate(model) {
|
|
112
|
-
if (model) {
|
|
113
|
-
for (const [key, entry] of this.cache.entries()) {
|
|
114
|
-
if (entry.model.includes(model)) {
|
|
115
|
-
this.cache.delete(key);
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
}
|
|
119
|
-
else {
|
|
120
|
-
this.cache.clear();
|
|
121
|
-
}
|
|
122
|
-
}
|
|
123
|
-
/**
|
|
124
|
-
* Get cache statistics
|
|
125
|
-
*/
|
|
126
|
-
getStats() {
|
|
127
|
-
const total = this.hits + this.misses;
|
|
128
|
-
return {
|
|
129
|
-
hits: this.hits,
|
|
130
|
-
misses: this.misses,
|
|
131
|
-
size: this.cache.size,
|
|
132
|
-
hit_rate: total > 0 ? this.hits / total : 0,
|
|
133
|
-
};
|
|
134
|
-
}
|
|
135
|
-
/**
|
|
136
|
-
* Evict oldest entry by cached_at timestamp
|
|
137
|
-
*/
|
|
138
|
-
evictOldest() {
|
|
139
|
-
let oldestKey = null;
|
|
140
|
-
let oldestTime = Infinity;
|
|
141
|
-
for (const [key, entry] of this.cache.entries()) {
|
|
142
|
-
if (entry.cached_at < oldestTime) {
|
|
143
|
-
oldestTime = entry.cached_at;
|
|
144
|
-
oldestKey = key;
|
|
145
|
-
}
|
|
146
|
-
}
|
|
147
|
-
if (oldestKey) {
|
|
148
|
-
this.cache.delete(oldestKey);
|
|
149
|
-
}
|
|
150
|
-
}
|
|
151
|
-
}
|
|
152
|
-
exports.ResponseCache = ResponseCache;
|
|
153
|
-
//# sourceMappingURL=responseCache.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"responseCache.js","sourceRoot":"","sources":["../../src/cache/responseCache.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,+CAAiC;AAmBjC,MAAa,aAAa;IAChB,KAAK,GAA4B,IAAI,GAAG,EAAE,CAAC;IAC3C,MAAM,CAAc;IACpB,IAAI,GAAG,CAAC,CAAC;IACT,MAAM,GAAG,CAAC,CAAC;IAEnB,YAAY,SAA+B,EAAE;QAC3C,IAAI,CAAC,MAAM,GAAG;YACZ,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,IAAI;YAC/B,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,IAAI,EAAE,iBAAiB;YAC1D,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,IAAI;YACvC,SAAS,EAAE,MAAM,CAAC,SAAS;SAC5B,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,MAAc,EAAE,KAAa;QACvC,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,GAAG,GAAG,KAAK,CAAC,CAAC;QAClC,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,MAAc,EAAE,KAAa;QAC/B,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC;QAEtC,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAElC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,OAAO,IAAI,CAAC;QACd,CAAC;QAED,mBAAmB;QACnB,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,UAAU,EAAE,CAAC;YAClC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACvB,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,CAAC,IAAI,EAAE,CAAC;QACZ,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,MAAc,EAAE,KAAa,EAAE,QAA6B;QAC9D,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO;YAAE,OAAO;QAEjC,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAC5C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,8BAA8B;QAC9B,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YAC/C,IAAI,CAAC,WAAW,EAAE,CAAC;QACrB,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE;YAClB,OAAO,EAAE,QAAQ,CAAC,OAAO,IAAI,EAAE;YAC/B,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,KAAK;YAC9B,QAAQ,EAAE,QAAQ,CAAC,QAAQ,IAAI,SAAS;YACxC,MAAM,EAAE,QAAQ,CAAC,MAAM,IAAI,CAAC;YAC5B,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,CAAC;YACxB,SAAS,EAAE,GAAG;YACd,UAAU,EAAE,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,GAAG,IAAI;SACjD,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,KAAc;QACvB,IAAI,KAAK,EAAE,CAAC;YACV,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;gBAChD,IAAI,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;oBAChC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACzB,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC;QACtC,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI;YACrB,QAAQ,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;SAC5C,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,WAAW;QACjB,IAAI,SAAS,GAAkB,IAAI,CAAC;QACpC,IAAI,UAAU,GAAG,QAAQ,CAAC;QAE1B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YAChD,IAAI,KAAK,CAAC,SAAS,GAAG,UAAU,EAAE,CAAC;gBACjC,UAAU,GAAG,KAAK,CAAC,SAAS,CAAC;gBAC7B,SAAS,GAAG,GAAG,CAAC;YAClB,CAAC;QACH,CAAC;QAED,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;CACF;AAxHD,sCAwHC"}
|
|
@@ -1,59 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
/**
|
|
3
|
-
* TMLPD PI CLI
|
|
4
|
-
*/
|
|
5
|
-
|
|
6
|
-
const { createTMLPD } = require("./dist/index.js");
|
|
7
|
-
|
|
8
|
-
const args = process.argv.slice(2);
|
|
9
|
-
const command = args[0];
|
|
10
|
-
|
|
11
|
-
async function main() {
|
|
12
|
-
switch (command) {
|
|
13
|
-
case "execute": {
|
|
14
|
-
const tmlpd = createTMLPD();
|
|
15
|
-
const prompt = args.slice(1).join(" ");
|
|
16
|
-
const result = await tmlpd.execute(prompt);
|
|
17
|
-
console.log(JSON.stringify(result, null, 2));
|
|
18
|
-
break;
|
|
19
|
-
}
|
|
20
|
-
case "parallel": {
|
|
21
|
-
const tmlpd = createTMLPD();
|
|
22
|
-
const models = args.slice(1).join(" ").split(",").map(m => m.trim());
|
|
23
|
-
const prompt = "Compare these models";
|
|
24
|
-
const result = await tmlpd.executeParallel(prompt, models);
|
|
25
|
-
console.log(JSON.stringify(result, null, 2));
|
|
26
|
-
break;
|
|
27
|
-
}
|
|
28
|
-
case "cost": {
|
|
29
|
-
const tmlpd = createTMLPD();
|
|
30
|
-
const summary = tmlpd.getCostSummary();
|
|
31
|
-
console.log(JSON.stringify(summary, null, 2));
|
|
32
|
-
break;
|
|
33
|
-
}
|
|
34
|
-
case "status": {
|
|
35
|
-
const tmlpd = createTMLPD();
|
|
36
|
-
const status = tmlpd.getProviderStatus();
|
|
37
|
-
console.log(JSON.stringify(status, null, 2));
|
|
38
|
-
break;
|
|
39
|
-
}
|
|
40
|
-
case "cache": {
|
|
41
|
-
const tmlpd = createTMLPD();
|
|
42
|
-
const stats = tmlpd.getCacheStats();
|
|
43
|
-
console.log(JSON.stringify(stats, null, 2));
|
|
44
|
-
break;
|
|
45
|
-
}
|
|
46
|
-
default:
|
|
47
|
-
console.log(`TMLPD PI v1.0.0
|
|
48
|
-
|
|
49
|
-
Usage:
|
|
50
|
-
tmlpd-pi execute <prompt> Execute single prompt
|
|
51
|
-
tmlpd-pi parallel <models> Execute in parallel
|
|
52
|
-
tmlpd-pi cost Show cost summary
|
|
53
|
-
tmlpd-pi status Show provider status
|
|
54
|
-
tmlpd-pi cache Show cache stats
|
|
55
|
-
`);
|
|
56
|
-
}
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
main().catch(console.error);
|
|
@@ -1,95 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* TMLPD Cost Tracker
|
|
3
|
-
*
|
|
4
|
-
* Tracks real-time spending across all providers.
|
|
5
|
-
* Supports per-model budgets, spending alerts, and cost analysis.
|
|
6
|
-
*/
|
|
7
|
-
export interface BudgetConfig {
|
|
8
|
-
daily_limit?: number;
|
|
9
|
-
monthly_limit?: number;
|
|
10
|
-
per_model_limits?: Record<string, number>;
|
|
11
|
-
}
|
|
12
|
-
export interface CostAlert {
|
|
13
|
-
type: "daily" | "monthly" | "model" | "budget";
|
|
14
|
-
threshold: number;
|
|
15
|
-
current: number;
|
|
16
|
-
provider?: string;
|
|
17
|
-
model?: string;
|
|
18
|
-
}
|
|
19
|
-
export interface CostSnapshot {
|
|
20
|
-
provider: string;
|
|
21
|
-
model: string;
|
|
22
|
-
input_tokens: number;
|
|
23
|
-
output_tokens: number;
|
|
24
|
-
input_cost: number;
|
|
25
|
-
output_cost: number;
|
|
26
|
-
total_cost: number;
|
|
27
|
-
timestamp: number;
|
|
28
|
-
}
|
|
29
|
-
export interface CostSummary {
|
|
30
|
-
total_cost: number;
|
|
31
|
-
by_provider: Record<string, number>;
|
|
32
|
-
by_model: Record<string, number>;
|
|
33
|
-
daily_costs: Record<string, number>;
|
|
34
|
-
monthly_costs: Record<string, number>;
|
|
35
|
-
request_count: number;
|
|
36
|
-
token_count: {
|
|
37
|
-
input: number;
|
|
38
|
-
output: number;
|
|
39
|
-
};
|
|
40
|
-
average_cost_per_request: number;
|
|
41
|
-
}
|
|
42
|
-
export declare class CostTracker {
|
|
43
|
-
private history;
|
|
44
|
-
private budgets;
|
|
45
|
-
private alerts;
|
|
46
|
-
private alerts_callback;
|
|
47
|
-
private daily_reset;
|
|
48
|
-
private monthly_reset;
|
|
49
|
-
constructor(budgets?: BudgetConfig);
|
|
50
|
-
/**
|
|
51
|
-
* Calculate cost for a model based on tokens
|
|
52
|
-
*/
|
|
53
|
-
calculateCost(model: string, input_tokens: number, output_tokens: number): {
|
|
54
|
-
input: number;
|
|
55
|
-
output: number;
|
|
56
|
-
total: number;
|
|
57
|
-
};
|
|
58
|
-
/**
|
|
59
|
-
* Record a request's cost
|
|
60
|
-
*/
|
|
61
|
-
record(provider: string, model: string, input_tokens: number, output_tokens: number): CostSnapshot;
|
|
62
|
-
/**
|
|
63
|
-
* Check budgets and trigger alerts
|
|
64
|
-
*/
|
|
65
|
-
private checkBudgets;
|
|
66
|
-
/**
|
|
67
|
-
* Emit an alert via callback
|
|
68
|
-
*/
|
|
69
|
-
private emitAlert;
|
|
70
|
-
/**
|
|
71
|
-
* Register alert callback
|
|
72
|
-
*/
|
|
73
|
-
onAlert(callback: (alert: CostAlert) => void): void;
|
|
74
|
-
/**
|
|
75
|
-
* Get comprehensive cost summary
|
|
76
|
-
*/
|
|
77
|
-
getSummary(): CostSummary;
|
|
78
|
-
/**
|
|
79
|
-
* Get remaining budget
|
|
80
|
-
*/
|
|
81
|
-
getRemainingBudget(): {
|
|
82
|
-
daily: number | null;
|
|
83
|
-
monthly: number | null;
|
|
84
|
-
per_model: Record<string, number>;
|
|
85
|
-
};
|
|
86
|
-
/**
|
|
87
|
-
* Reset cost history
|
|
88
|
-
*/
|
|
89
|
-
reset(): void;
|
|
90
|
-
/**
|
|
91
|
-
* Export cost data for analysis
|
|
92
|
-
*/
|
|
93
|
-
export(): CostSnapshot[];
|
|
94
|
-
}
|
|
95
|
-
//# sourceMappingURL=costTracker.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"costTracker.d.ts","sourceRoot":"","sources":["../../src/cost/costTracker.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAoCH,MAAM,WAAW,YAAY;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC3C;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,OAAO,GAAG,SAAS,GAAG,OAAO,GAAG,QAAQ,CAAC;IAC/C,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpC,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/C,wBAAwB,EAAE,MAAM,CAAC;CAClC;AAED,qBAAa,WAAW;IACtB,OAAO,CAAC,OAAO,CAAsB;IACrC,OAAO,CAAC,OAAO,CAAe;IAC9B,OAAO,CAAC,MAAM,CAAmB;IACjC,OAAO,CAAC,eAAe,CAA6C;IACpE,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,aAAa,CAAS;gBAElB,OAAO,GAAE,YAAiB;IAOtC;;OAEG;IACH,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE;IAc3H;;OAEG;IACH,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,YAAY;IAkBlG;;OAEG;IACH,OAAO,CAAC,YAAY;IA8CpB;;OAEG;IACH,OAAO,CAAC,SAAS;IAiBjB;;OAEG;IACH,OAAO,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,SAAS,KAAK,IAAI,GAAG,IAAI;IAInD;;OAEG;IACH,UAAU,IAAI,WAAW;IAoDzB;;OAEG;IACH,kBAAkB,IAAI;QAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE;IAuBzG;;OAEG;IACH,KAAK,IAAI,IAAI;IAKb;;OAEG;IACH,MAAM,IAAI,YAAY,EAAE;CAGzB"}
|