adaptive-memory-multi-model-router 2.2.4 → 2.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -22
- package/README.md.bak +836 -0
- package/dist/analytics/costAnalytics.d.ts +1 -0
- package/dist/cache/cacheKeyGenerator.d.ts +67 -0
- package/dist/cache/cacheKeyGenerator.d.ts.map +1 -0
- package/dist/cache/cacheKeyGenerator.js +211 -0
- package/dist/cache/cacheKeyGenerator.js.map +1 -0
- package/dist/cache/semanticCache.d.ts +41 -0
- package/dist/cache/semanticCache.d.ts.map +1 -1
- package/dist/cache/semanticCache.js +142 -0
- package/dist/cache/semanticCache.js.map +1 -1
- package/dist/cli.js +35 -478
- package/dist/cost/costTracker.js +0 -3
- package/dist/cost/preCallCostEstimator.d.ts +114 -0
- package/dist/cost/preCallCostEstimator.d.ts.map +1 -0
- package/dist/cost/preCallCostEstimator.js +256 -0
- package/dist/cost/preCallCostEstimator.js.map +1 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +264 -64
- package/dist/index.js.map +1 -1
- package/dist/inference/speculativeDecoding.d.ts +133 -0
- package/dist/inference/speculativeDecoding.d.ts.map +1 -0
- package/dist/inference/speculativeDecoding.js +276 -0
- package/dist/inference/speculativeDecoding.js.map +1 -0
- package/dist/integrations/langchainAdapter.d.ts +1 -0
- package/dist/integrations/oauth.d.ts +1 -0
- package/dist/memory/autoFetch.d.ts +1 -0
- package/dist/memory/memoryTree.d.ts +1 -0
- package/dist/memory/obsidianVault.d.ts +1 -0
- package/dist/providers/providerConfig.d.ts +1 -0
- package/dist/providers/providerConfig.js +2 -0
- package/dist/providers/providerHealth.d.ts +117 -0
- package/dist/providers/providerHealth.d.ts.map +1 -0
- package/dist/providers/providerHealth.js +309 -0
- package/dist/providers/providerHealth.js.map +1 -0
- package/dist/providers/registry.js +126 -128
- package/dist/routing/advancedRouter.js +310 -427
- package/dist/routing/difficultyClassifier.d.ts +79 -0
- package/dist/routing/difficultyClassifier.d.ts.map +1 -0
- package/dist/routing/difficultyClassifier.js +329 -0
- package/dist/routing/difficultyClassifier.js.map +1 -0
- package/dist/sdk.d.ts +125 -0
- package/dist/sdk.d.ts.map +1 -0
- package/dist/sdk.js +109 -100
- package/dist/sdk.js.map +1 -0
- package/dist/security/guardrails.d.ts +1 -0
- package/dist/server/dashboard.d.ts +1 -0
- package/dist/server/modelMapper.d.ts +1 -0
- package/dist/server/proxyServer.d.ts +1 -0
- package/package.json +106 -3
- package/src/cache/cacheKeyGenerator.ts +242 -0
- package/src/cache/semanticCache.ts +148 -0
- package/src/cost/preCallCostEstimator.ts +345 -0
- package/src/inference/speculativeDecoding.ts +373 -0
- package/src/providers/providerHealth.ts +397 -0
- package/src/routing/difficultyClassifier.ts +420 -0
- package/test/provider-test.js +2 -2
- package/test.js +7 -7
- package/test.js.bak +376 -0
- package/tsconfig.json +15 -5
- package/src/index.ts +0 -99
- package/src/skills/__tests__/skill_manager.test.ts +0 -328
package/dist/index.js
CHANGED
|
@@ -1,24 +1,39 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
|
|
3
|
-
|
|
2
|
+
/**
|
|
3
|
+
* TMLPD PI Extension - v1.1.0
|
|
4
|
+
*
|
|
5
|
+
* Parallel Multi-LLM Processing with Streaming, Caching, Cost Tracking, Reliability
|
|
6
|
+
* + Reference Architecture to Full TMLPD (Episodic Memory, MCTS, HALO)
|
|
7
|
+
*
|
|
8
|
+
* @example
|
|
9
|
+
* ```typescript
|
|
10
|
+
* import { createTMLPD, HALOOrchestrator, EpisodicMemoryStore } from "tmlpd-pi";
|
|
11
|
+
*
|
|
12
|
+
* // Lightweight usage (core features)
|
|
13
|
+
* const tmlpd = createTMLPD({ cache: { ttl_seconds: 3600 } });
|
|
14
|
+
* const result = await tmlpd.executeParallel(prompt, ["gpt-4o", "claude"]);
|
|
15
|
+
*
|
|
16
|
+
* // Advanced: HALO orchestration with episodic memory
|
|
17
|
+
* const halo = new HALOOrchestrator({ maxConcurrent: 3, enableMCTS: true });
|
|
18
|
+
* const haloResult = await halo.execute("Build a REST API", async (subtask, agent) => {
|
|
19
|
+
* // Execute via agent
|
|
20
|
+
* });
|
|
21
|
+
*
|
|
22
|
+
* // Query episodic memory
|
|
23
|
+
* const similar = memory.getSimilarTasks("Python async API", 5);
|
|
24
|
+
* ```
|
|
25
|
+
*/
|
|
4
26
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
Object.defineProperty(exports, "
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
Object.defineProperty(exports, "MODEL_PROFILES", { enumerable: true, get: function () { return advancedRouter_1.MODEL_PROFILES; } });
|
|
16
|
-
Object.defineProperty(exports, "updateModelProfile", { enumerable: true, get: function () { return advancedRouter_1.updateModelProfile; } });
|
|
17
|
-
Object.defineProperty(exports, "getProviderHealth", { enumerable: true, get: function () { return advancedRouter_1.getProviderHealth; } });
|
|
18
|
-
// ============================================================
|
|
19
|
-
// PROVIDERS
|
|
20
|
-
// ============================================================
|
|
21
|
-
var providerConfig_1 = require("./providers/providerConfig");
|
|
27
|
+
exports.TMLPD_PI_TOOLS = exports.EagleSpeculative = exports.MedusaPredictor = exports.estimateSpeedupPotential = exports.speculativeBatch = exports.SpeculativeDecoder = exports.createWarmedCache = exports.PrefixCache = exports.MODEL_PROFILES = exports.updateModelProfile = exports.extractQueryFeatures = exports.recommendForTask = exports.routeBatch = exports.routeQuery = exports.executeBatch = exports.BatchProcessor = exports.createLMStudioProvider = exports.createVLLMProvider = exports.createOllamaProvider = exports.LocalProviderManager = exports.LocalProvider = exports.calculateCompressionRatio = exports.truncateToTokenBudget = exports.truncateMessages = exports.compressText = exports.isonDecode = exports.isonEncode = exports.MODEL_COSTS = exports.findCheapestModels = exports.listModelsByCost = exports.getModelCost = exports.estimateCostFromText = exports.estimateCost = exports.countTokens = exports.HALOOrchestrator = exports.MCTSWorkflowOptimizer = exports.EpisodicMemoryStore = exports.DEFAULT_RETRY_CONFIG = exports.isRetryableStatus = exports.calculateRetryDelay = exports.withRetry = exports.CircuitBreaker = exports.ProviderRegistry = exports.CostTracker = exports.ResponseCache = exports.TMLPDTools = exports.createTMLPD = void 0;
|
|
28
|
+
const tmlpdTools_1 = require("./tools/tmlpdTools");
|
|
29
|
+
Object.defineProperty(exports, "createTMLPD", { enumerable: true, get: function () { return tmlpdTools_1.createTMLPD; } });
|
|
30
|
+
Object.defineProperty(exports, "TMLPDTools", { enumerable: true, get: function () { return tmlpdTools_1.TMLPDTools; } });
|
|
31
|
+
const responseCache_1 = require("./cache/responseCache");
|
|
32
|
+
Object.defineProperty(exports, "ResponseCache", { enumerable: true, get: function () { return responseCache_1.ResponseCache; } });
|
|
33
|
+
const costTracker_1 = require("./cost/costTracker");
|
|
34
|
+
Object.defineProperty(exports, "CostTracker", { enumerable: true, get: function () { return costTracker_1.CostTracker; } });
|
|
35
|
+
const registry_1 = require("./providers/registry");
|
|
36
|
+
const providerConfig_1 = require("./providers/providerConfig");
|
|
22
37
|
Object.defineProperty(exports, "DEFAULT_PROVIDERS", { enumerable: true, get: function () { return providerConfig_1.DEFAULT_PROVIDERS; } });
|
|
23
38
|
Object.defineProperty(exports, "getAvailableProviders", { enumerable: true, get: function () { return providerConfig_1.getAvailableProviders; } });
|
|
24
39
|
Object.defineProperty(exports, "registerProvider", { enumerable: true, get: function () { return providerConfig_1.registerProvider; } });
|
|
@@ -30,56 +45,241 @@ Object.defineProperty(exports, "findCheapestAvailableProvider", { enumerable: tr
|
|
|
30
45
|
Object.defineProperty(exports, "findFastestAvailableProvider", { enumerable: true, get: function () { return providerConfig_1.findFastestAvailableProvider; } });
|
|
31
46
|
Object.defineProperty(exports, "loadConfig", { enumerable: true, get: function () { return providerConfig_1.loadConfig; } });
|
|
32
47
|
Object.defineProperty(exports, "saveConfig", { enumerable: true, get: function () { return providerConfig_1.saveConfig; } });
|
|
48
|
+
Object.defineProperty(exports, "providerConfig", { enumerable: true, get: function () { return providerConfig_1; } });
|
|
49
|
+
|
|
50
|
+
Object.defineProperty(exports, "ProviderRegistry", { enumerable: true, get: function () { return registry_1.ProviderRegistry; } });
|
|
51
|
+
const reliability_1 = require("./utils/reliability");
|
|
52
|
+
Object.defineProperty(exports, "CircuitBreaker", { enumerable: true, get: function () { return reliability_1.CircuitBreaker; } });
|
|
53
|
+
Object.defineProperty(exports, "withRetry", { enumerable: true, get: function () { return reliability_1.withRetry; } });
|
|
54
|
+
Object.defineProperty(exports, "calculateRetryDelay", { enumerable: true, get: function () { return reliability_1.calculateRetryDelay; } });
|
|
55
|
+
Object.defineProperty(exports, "isRetryableStatus", { enumerable: true, get: function () { return reliability_1.isRetryableStatus; } });
|
|
56
|
+
Object.defineProperty(exports, "DEFAULT_RETRY_CONFIG", { enumerable: true, get: function () { return reliability_1.DEFAULT_RETRY_CONFIG; } });
|
|
57
|
+
const episodicMemory_1 = require("./memory/episodicMemory");
|
|
58
|
+
Object.defineProperty(exports, "EpisodicMemoryStore", { enumerable: true, get: function () { return episodicMemory_1.EpisodicMemoryStore; } });
|
|
59
|
+
const mctsWorkflow_1 = require("./orchestration/mctsWorkflow");
|
|
60
|
+
Object.defineProperty(exports, "MCTSWorkflowOptimizer", { enumerable: true, get: function () { return mctsWorkflow_1.MCTSWorkflowOptimizer; } });
|
|
61
|
+
const haloOrchestrator_1 = require("./orchestration/haloOrchestrator");
|
|
62
|
+
|
|
63
|
+
|
|
33
64
|
// ============================================================
|
|
34
|
-
//
|
|
35
|
-
// ============================================================
|
|
36
|
-
var costTracker_1 = require("./cost/costTracker");
|
|
37
|
-
Object.defineProperty(exports, "CostTracker", { enumerable: true, get: function () { return costTracker_1.CostTracker; } });
|
|
38
|
-
// ============================================================
|
|
39
|
-
// MEMORY
|
|
40
|
-
// ============================================================
|
|
41
|
-
var memoryTree_1 = require("./memory/memoryTree");
|
|
42
|
-
Object.defineProperty(exports, "MemoryTree", { enumerable: true, get: function () { return memoryTree_1.MemoryTree; } });
|
|
43
|
-
// ============================================================
|
|
44
|
-
// UTILITIES
|
|
45
|
-
// ============================================================
|
|
46
|
-
var tokenUtils_1 = require("./utils/tokenUtils");
|
|
47
|
-
Object.defineProperty(exports, "countTokens", { enumerable: true, get: function () { return tokenUtils_1.countTokens; } });
|
|
48
|
-
Object.defineProperty(exports, "estimateTokens", { enumerable: true, get: function () { return tokenUtils_1.estimateTokens; } });
|
|
49
|
-
var tokenUtils_2 = require("./utils/tokenUtils");
|
|
50
|
-
Object.defineProperty(exports, "MODEL_COSTS", { enumerable: true, get: function () { return tokenUtils_2.MODEL_COSTS; } });
|
|
65
|
+
// CONVENIENCE: A3M Router Factory
|
|
51
66
|
// ============================================================
|
|
52
|
-
// v2.0.0 FEATURES
|
|
53
|
-
// ============================================================
|
|
54
|
-
var semanticCache_1 = require("./cache/semanticCache");
|
|
55
|
-
Object.defineProperty(exports, "SemanticCache", { enumerable: true, get: function () { return semanticCache_1.SemanticCache; } });
|
|
56
|
-
var guardrails_1 = require("./security/guardrails");
|
|
57
|
-
Object.defineProperty(exports, "GuardrailEngine", { enumerable: true, get: function () { return guardrails_1.GuardrailEngine; } });
|
|
58
|
-
var costAnalytics_1 = require("./analytics/costAnalytics");
|
|
59
|
-
Object.defineProperty(exports, "CostAnalytics", { enumerable: true, get: function () { return costAnalytics_1.CostAnalytics; } });
|
|
60
|
-
var proxyServer_1 = require("./server/proxyServer");
|
|
61
|
-
Object.defineProperty(exports, "createProxyServer", { enumerable: true, get: function () { return proxyServer_1.createProxyServer; } });
|
|
62
|
-
// ============================================================
|
|
63
|
-
// CONVENIENCE: Create a router instance
|
|
64
|
-
// ============================================================
|
|
65
|
-
const advancedRouter_2 = require("./routing/advancedRouter");
|
|
66
|
-
const providerConfig_2 = require("./providers/providerConfig");
|
|
67
|
-
const costTracker_2 = require("./cost/costTracker");
|
|
68
|
-
const memoryTree_2 = require("./memory/memoryTree");
|
|
69
67
|
function createA3MRouter(options) {
|
|
70
|
-
const
|
|
71
|
-
const
|
|
68
|
+
const memoryTree = new memoryTree_1.MemoryTree();
|
|
69
|
+
const semanticCache = new semanticCache_1.SemanticCache();
|
|
70
|
+
const costTracker = new costTracker_1.CostTracker();
|
|
71
|
+
const provRegistry = new registry_1.ProviderRegistry();
|
|
72
|
+
|
|
72
73
|
return {
|
|
73
|
-
route:
|
|
74
|
-
routeBatch:
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
costTracker,
|
|
79
|
-
|
|
74
|
+
route: advancedRouter_1.routeQuery,
|
|
75
|
+
routeBatch: advancedRouter_1.routeBatch,
|
|
76
|
+
recommend: advancedRouter_1.recommendForTask,
|
|
77
|
+
memory: memoryTree,
|
|
78
|
+
cache: semanticCache,
|
|
79
|
+
costTracker: costTracker,
|
|
80
|
+
providers: provRegistry,
|
|
81
|
+
compression: { compress: function(s) { return s; }, decompress: function(s) { return s; } },
|
|
82
|
+
vault: { store: function() { return Promise.resolve(); }, retrieve: function() { return Promise.resolve(null); } },
|
|
83
|
+
autoFetch: true,
|
|
84
|
+
oauth: true,
|
|
80
85
|
options: options || {},
|
|
81
86
|
};
|
|
82
87
|
}
|
|
83
|
-
|
|
84
|
-
|
|
88
|
+
exports.createA3MRouter = createA3MRouter;
|
|
89
|
+
|
|
90
|
+
Object.defineProperty(exports, "HALOOrchestrator", { enumerable: true, get: function () { return haloOrchestrator_1.HALOOrchestrator; } });
|
|
91
|
+
const tokenUtils_1 = require("./utils/tokenUtils");
|
|
92
|
+
Object.defineProperty(exports, "countTokens", { enumerable: true, get: function () { return tokenUtils_1.countTokens; } });
|
|
93
|
+
Object.defineProperty(exports, "estimateCost", { enumerable: true, get: function () { return tokenUtils_1.estimateCost; } });
|
|
94
|
+
Object.defineProperty(exports, "estimateCostFromText", { enumerable: true, get: function () { return tokenUtils_1.estimateCostFromText; } });
|
|
95
|
+
Object.defineProperty(exports, "getModelCost", { enumerable: true, get: function () { return tokenUtils_1.getModelCost; } });
|
|
96
|
+
Object.defineProperty(exports, "listModelsByCost", { enumerable: true, get: function () { return tokenUtils_1.listModelsByCost; } });
|
|
97
|
+
Object.defineProperty(exports, "findCheapestModels", { enumerable: true, get: function () { return tokenUtils_1.findCheapestModels; } });
|
|
98
|
+
Object.defineProperty(exports, "MODEL_COSTS", { enumerable: true, get: function () { return tokenUtils_1.MODEL_COSTS; } });
|
|
99
|
+
const compression_1 = require("./utils/compression");
|
|
100
|
+
Object.defineProperty(exports, "isonEncode", { enumerable: true, get: function () { return compression_1.isonEncode; } });
|
|
101
|
+
Object.defineProperty(exports, "isonDecode", { enumerable: true, get: function () { return compression_1.isonDecode; } });
|
|
102
|
+
Object.defineProperty(exports, "compressText", { enumerable: true, get: function () { return compression_1.compressText; } });
|
|
103
|
+
Object.defineProperty(exports, "truncateMessages", { enumerable: true, get: function () { return compression_1.truncateMessages; } });
|
|
104
|
+
Object.defineProperty(exports, "truncateToTokenBudget", { enumerable: true, get: function () { return compression_1.truncateToTokenBudget; } });
|
|
105
|
+
Object.defineProperty(exports, "calculateCompressionRatio", { enumerable: true, get: function () { return compression_1.calculateCompressionRatio; } });
|
|
106
|
+
const localProvider_1 = require("./providers/localProvider");
|
|
107
|
+
Object.defineProperty(exports, "LocalProvider", { enumerable: true, get: function () { return localProvider_1.LocalProvider; } });
|
|
108
|
+
Object.defineProperty(exports, "LocalProviderManager", { enumerable: true, get: function () { return localProvider_1.LocalProviderManager; } });
|
|
109
|
+
Object.defineProperty(exports, "createOllamaProvider", { enumerable: true, get: function () { return localProvider_1.createOllamaProvider; } });
|
|
110
|
+
Object.defineProperty(exports, "createVLLMProvider", { enumerable: true, get: function () { return localProvider_1.createVLLMProvider; } });
|
|
111
|
+
Object.defineProperty(exports, "createLMStudioProvider", { enumerable: true, get: function () { return localProvider_1.createLMStudioProvider; } });
|
|
112
|
+
const batchProcessor_1 = require("./utils/batchProcessor");
|
|
113
|
+
Object.defineProperty(exports, "BatchProcessor", { enumerable: true, get: function () { return batchProcessor_1.BatchProcessor; } });
|
|
114
|
+
Object.defineProperty(exports, "executeBatch", { enumerable: true, get: function () { return batchProcessor_1.executeBatch; } });
|
|
115
|
+
const advancedRouter_1 = require("./routing/advancedRouter");
|
|
116
|
+
Object.defineProperty(exports, "routeQuery", { enumerable: true, get: function () { return advancedRouter_1.routeQuery; } });
|
|
117
|
+
Object.defineProperty(exports, "routeBatch", { enumerable: true, get: function () { return advancedRouter_1.routeBatch; } });
|
|
118
|
+
Object.defineProperty(exports, "recommendForTask", { enumerable: true, get: function () { return advancedRouter_1.recommendForTask; } });
|
|
119
|
+
Object.defineProperty(exports, "extractQueryFeatures", { enumerable: true, get: function () { return advancedRouter_1.extractQueryFeatures; } });
|
|
120
|
+
Object.defineProperty(exports, "updateModelProfile", { enumerable: true, get: function () { return advancedRouter_1.updateModelProfile; } });
|
|
121
|
+
Object.defineProperty(exports, "MODEL_PROFILES", { enumerable: true, get: function () { return advancedRouter_1.MODEL_PROFILES; } });
|
|
122
|
+
const prefixCache_1 = require("./cache/prefixCache");
|
|
123
|
+
const memoryTree_1 = require("./memory/memoryTree");
|
|
124
|
+
const semanticCache_1 = require("./cache/semanticCache");
|
|
125
|
+
Object.defineProperty(exports, "PrefixCache", { enumerable: true, get: function () { return prefixCache_1.PrefixCache; } });
|
|
126
|
+
Object.defineProperty(exports, "MemoryTree", { enumerable: true, get: function () { return memoryTree_1.MemoryTree; } });
|
|
127
|
+
Object.defineProperty(exports, "SemanticCache", { enumerable: true, get: function () { return semanticCache_1.SemanticCache; } });
|
|
128
|
+
Object.defineProperty(exports, "createWarmedCache", { enumerable: true, get: function () { return prefixCache_1.createWarmedCache; } });
|
|
129
|
+
const speculativeDecoding_1 = require("./utils/speculativeDecoding");
|
|
130
|
+
Object.defineProperty(exports, "SpeculativeDecoder", { enumerable: true, get: function () { return speculativeDecoding_1.SpeculativeDecoder; } });
|
|
131
|
+
Object.defineProperty(exports, "speculativeBatch", { enumerable: true, get: function () { return speculativeDecoding_1.speculativeBatch; } });
|
|
132
|
+
Object.defineProperty(exports, "estimateSpeedupPotential", { enumerable: true, get: function () { return speculativeDecoding_1.estimateSpeedupPotential; } });
|
|
133
|
+
Object.defineProperty(exports, "MedusaPredictor", { enumerable: true, get: function () { return speculativeDecoding_1.MedusaPredictor; } });
|
|
134
|
+
Object.defineProperty(exports, "EagleSpeculative", { enumerable: true, get: function () { return speculativeDecoding_1.EagleSpeculative; } });
|
|
135
|
+
// PI Tool definitions (for PI agent integration)
|
|
136
|
+
exports.TMLPD_PI_TOOLS = [
|
|
137
|
+
{
|
|
138
|
+
name: "tmlpd_execute",
|
|
139
|
+
description: "Execute prompt across multiple LLM providers in parallel. Optimizes for cost-quality tradeoff with automatic model selection. Use when comparing multiple AI responses or needing faster results via parallel execution.",
|
|
140
|
+
inputSchema: {
|
|
141
|
+
type: "object",
|
|
142
|
+
properties: {
|
|
143
|
+
prompt: { type: "string", description: "The prompt to execute" },
|
|
144
|
+
models: { type: "array", items: { type: "string" }, description: "Optional model list (auto-selects if omitted)" },
|
|
145
|
+
streaming: { type: "object", properties: { enabled: { type: "boolean" }, chunk_size: { type: "number" } } }
|
|
146
|
+
},
|
|
147
|
+
required: ["prompt"]
|
|
148
|
+
}
|
|
149
|
+
},
|
|
150
|
+
{
|
|
151
|
+
name: "tmlpd_execute_single",
|
|
152
|
+
description: "Execute with single model via smart routing. Analyzes prompt to select optimal agent based on task type (coding, explanation, analysis, etc.) with cost-quality optimization.",
|
|
153
|
+
inputSchema: {
|
|
154
|
+
type: "object",
|
|
155
|
+
properties: {
|
|
156
|
+
prompt: { type: "string", description: "The prompt to execute" },
|
|
157
|
+
model: { type: "string", description: "Optional specific model" }
|
|
158
|
+
},
|
|
159
|
+
required: ["prompt"]
|
|
160
|
+
}
|
|
161
|
+
},
|
|
162
|
+
{
|
|
163
|
+
name: "tmlpd_cost_summary",
|
|
164
|
+
description: "Get real-time cost tracking summary. Shows spending by provider, model, daily/monthly breakdowns, and remaining budget. Essential for cost monitoring in production.",
|
|
165
|
+
inputSchema: { type: "object", properties: {} }
|
|
166
|
+
},
|
|
167
|
+
{
|
|
168
|
+
name: "tmlpd_cache_stats",
|
|
169
|
+
description: "Get response cache statistics. Shows hit rate, cache size, and effectiveness. Cache hits cost $0 and provide instant responses.",
|
|
170
|
+
inputSchema: { type: "object", properties: {} }
|
|
171
|
+
},
|
|
172
|
+
{
|
|
173
|
+
name: "tmlpd_provider_status",
|
|
174
|
+
description: "Get status of all configured LLM providers. Shows readiness, cooldown status, failure counts. Use for debugging or selecting specific providers.",
|
|
175
|
+
inputSchema: { type: "object", properties: {} }
|
|
176
|
+
},
|
|
177
|
+
{
|
|
178
|
+
name: "tmlpd_invalidate_cache",
|
|
179
|
+
description: "Invalidate cached responses. Use when prompt content has changed and fresh response needed, or to clear stale cache entries.",
|
|
180
|
+
inputSchema: {
|
|
181
|
+
type: "object",
|
|
182
|
+
properties: { model: { type: "string", description: "Optional model to invalidate (all if omitted)" } }
|
|
183
|
+
}
|
|
184
|
+
},
|
|
185
|
+
{
|
|
186
|
+
name: "tmlpd_get_budget",
|
|
187
|
+
description: "Get remaining budget for cost controls. Returns daily, monthly, and per-model limits. Use for budget enforcement and alerting.",
|
|
188
|
+
inputSchema: { type: "object", properties: {} }
|
|
189
|
+
},
|
|
190
|
+
{
|
|
191
|
+
name: "tmlpd_halo_execute",
|
|
192
|
+
description: "Execute via HALO (Hierarchical Autonomous Logic-Oriented) orchestrator with 3-tier planning: decompose → assign → execute. Includes episodic memory for learning from past executions. For complex multi-step tasks.",
|
|
193
|
+
inputSchema: {
|
|
194
|
+
type: "object",
|
|
195
|
+
properties: {
|
|
196
|
+
task_description: { type: "string", description: "Task to execute" },
|
|
197
|
+
max_concurrent: { type: "number", description: "Max parallel executions (default: 3)" },
|
|
198
|
+
enable_mcts: { type: "boolean", description: "Enable MCTS optimization (slower but better)" }
|
|
199
|
+
},
|
|
200
|
+
required: ["task_description"]
|
|
201
|
+
}
|
|
202
|
+
},
|
|
203
|
+
{
|
|
204
|
+
name: "tmlpd_episodic_query",
|
|
205
|
+
description: "Query episodic memory for similar past tasks. Useful for learning from past executions and improving future routing decisions.",
|
|
206
|
+
inputSchema: {
|
|
207
|
+
type: "object",
|
|
208
|
+
properties: {
|
|
209
|
+
task_description: { type: "string", description: "Task to find similar executions for" },
|
|
210
|
+
limit: { type: "number", description: "Max results (default: 5)" }
|
|
211
|
+
},
|
|
212
|
+
required: ["task_description"]
|
|
213
|
+
}
|
|
214
|
+
},
|
|
215
|
+
{
|
|
216
|
+
name: "tmlpd_count_tokens",
|
|
217
|
+
description: "Count tokens in text for cost estimation. Supports all major models (GPT-4, Claude, Gemini, Llama). Use for estimating costs before execution or calculating context window usage.",
|
|
218
|
+
inputSchema: {
|
|
219
|
+
type: "object",
|
|
220
|
+
properties: {
|
|
221
|
+
text: { type: "string", description: "Text to count tokens in" },
|
|
222
|
+
model: { type: "string", description: "Model for tokenization (default: gpt-4o)" }
|
|
223
|
+
},
|
|
224
|
+
required: ["text"]
|
|
225
|
+
}
|
|
226
|
+
},
|
|
227
|
+
{
|
|
228
|
+
name: "tmlpd_compress_context",
|
|
229
|
+
description: "Compress context/messages using ISON encoding for token reduction. Reduces context by ~20-40% while preserving meaning. Useful for fitting more content in context windows.",
|
|
230
|
+
inputSchema: {
|
|
231
|
+
type: "object",
|
|
232
|
+
properties: {
|
|
233
|
+
messages: { type: "array", description: "Messages to compress", items: { type: "object" } },
|
|
234
|
+
strategy: { type: "string", enum: ["smart", "first", "last"], description: "Compression strategy (default: smart)" },
|
|
235
|
+
max_tokens: { type: "number", description: "Target token budget" }
|
|
236
|
+
},
|
|
237
|
+
required: ["messages"]
|
|
238
|
+
}
|
|
239
|
+
},
|
|
240
|
+
{
|
|
241
|
+
name: "tmlpd_local_generate",
|
|
242
|
+
description: "Generate using local LLM runtime (Ollama, vLLM, LM Studio). Zero cost, privacy-preserving. Use for development, testing, or when local GPU available. Falls back to cloud if local unavailable.",
|
|
243
|
+
inputSchema: {
|
|
244
|
+
type: "object",
|
|
245
|
+
properties: {
|
|
246
|
+
prompt: { type: "string", description: "Prompt for generation" },
|
|
247
|
+
runtime: { type: "string", enum: ["ollama", "vllm", "lmstudio"], description: "Local runtime type" },
|
|
248
|
+
model: { type: "string", description: "Model name (default: llama-3.3-70b)" }
|
|
249
|
+
},
|
|
250
|
+
required: ["prompt", "runtime"]
|
|
251
|
+
}
|
|
252
|
+
},
|
|
253
|
+
{
|
|
254
|
+
name: "tmlpd_batch_execute",
|
|
255
|
+
description: "Execute batch of prompts with concurrency control. Supports priority queuing, progress callbacks, rate limiting. Use for processing multiple prompts efficiently.",
|
|
256
|
+
inputSchema: {
|
|
257
|
+
type: "object",
|
|
258
|
+
properties: {
|
|
259
|
+
prompts: { type: "array", items: { type: "string" }, description: "Prompts to execute" },
|
|
260
|
+
concurrency: { type: "number", description: "Max parallel executions (default: 5)" },
|
|
261
|
+
model: { type: "string", description: "Model to use (default: gpt-4o)" }
|
|
262
|
+
},
|
|
263
|
+
required: ["prompts"]
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
];
|
|
267
|
+
/**
|
|
268
|
+
* Reference to Full TMLPD
|
|
269
|
+
*
|
|
270
|
+
* This package provides:
|
|
271
|
+
* - Core: streaming, caching, cost tracking, reliability
|
|
272
|
+
* - Reference: episodic memory, MCTS, HALO orchestrator
|
|
273
|
+
*
|
|
274
|
+
* For production with full features:
|
|
275
|
+
* - Python TMLPD: https://github.com/Das-rebel/tmlpd-skill
|
|
276
|
+
* - Full memory: 3-tier (episodic + semantic + working)
|
|
277
|
+
* - Full MCTS: UCB1, deterministic rollouts, strategy caching
|
|
278
|
+
* - Full HALO: NLP decomposition, capability matching, verification
|
|
279
|
+
*/
|
|
280
|
+
exports.default = {
|
|
281
|
+
createTMLPD: tmlpdTools_1.createTMLPD,
|
|
282
|
+
TMLPDTools: tmlpdTools_1.TMLPDTools,
|
|
283
|
+
TMLPD_PI_TOOLS: exports.TMLPD_PI_TOOLS
|
|
284
|
+
};
|
|
85
285
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;;;AAEH,mDAA0H;AAiBjH,4FAjBA,wBAAW,OAiBA;AAAE,2FAjBA,uBAAU,OAiBA;AAhBhC,yDAA+E;AAiBtE,8FAjBA,6BAAa,OAiBA;AAhBtB,oDAAqG;AAiB5F,4FAjBA,yBAAW,OAiBA;AAhBpB,mDAAgG;AAiBvF,iGAjBA,2BAAgB,OAiBA;AAhBzB,qDAAyJ;AAiBhJ,+FAjBA,4BAAc,OAiBA;AAAE,0FAjBA,uBAAS,OAiBA;AAA6B,oGAjBA,iCAAmB,OAiBA;AAAE,kGAjBA,+BAAiB,OAiBA;AAAE,qGAjBA,kCAAoB,OAiBA;AAhB3H,4DAA0F;AAiBjF,oGAjBA,oCAAmB,OAiBA;AAhB5B,+DAAgH;AAiBvG,sGAjBA,oCAAqB,OAiBA;AAhB9B,uEAA2H;AAiBlH,iGAjBA,mCAAgB,OAiBA;AAhBzB,mDAAiK;AAmBxJ,4FAnBA,wBAAW,OAmBA;AAAE,6FAnBA,yBAAY,OAmBA;AAAE,qGAnBA,iCAAoB,OAmBA;AAAE,6FAnBA,yBAAY,OAmBA;AAAE,iGAnBA,6BAAgB,OAmBA;AAAE,mGAnBA,+BAAkB,OAmBA;AAAE,4FAnBA,wBAAW,OAmBA;AAlBzH,qDAA6K;AAqBpK,2FArBA,wBAAU,OAqBA;AAAE,2FArBA,wBAAU,OAqBA;AAAE,6FArBA,0BAAY,OAqBA;AAAE,iGArBA,8BAAgB,OAqBA;AAAE,sGArBA,mCAAqB,OAqBA;AAAE,0GArBA,uCAAyB,OAqBA;AApBjH,6DAAiP;AAuBxO,8FAvBA,6BAAa,OAuBA;AAAE,qGAvBA,oCAAoB,OAuBA;AAAE,qGAvBA,oCAAoB,OAuBA;AAAE,mGAvBA,kCAAkB,OAuBA;AAAE,uGAvBA,sCAAsB,OAuBA;AAtB9G,2DAA6I;AAyBpI,+FAzBA,+BAAc,OAyBA;AAAE,6FAzBA,6BAAY,OAyBA;AAxBrC,6DAA0L;AA2BjL,2FA3BA,2BAAU,OA2BA;AAAE,2FA3BA,2BAAU,OA2BA;AAAE,iGA3BA,iCAAgB,OA2BA;AAAE,qGA3BA,qCAAoB,OA2BA;AAAE,mGA3BA,mCAAkB,OA2BA;AAAE,+FA3BA,+BAAc,OA2BA;AA1B3G,qDAAuF;AA6B9E,4FA7BA,yBAAW,OA6BA;AAAE,kGA7BA,+BAAiB,OA6BA;AA5BvC,qEAAsL;AA+B7K,mGA/BA,wCAAkB,OA+BA;AAAE,iGA/BA,sCAAgB,OA+BA;AAAE,yGA/BA,8CAAwB,OA+BA;AAAE,gGA/BA,qCAAe,OA+BA;AAAE,iGA/BA,sCAAgB,OA+BA;AAE1G,iDAAiD;AACpC,QAAA,cAAc,GAAG;IAC5B;QACE,IAAI,EAAE,eAAe;QACrB,WAAW,EAAE,0NAA0N;QACvO,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,uBAAuB,EAAE;gBAChE,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,WAAW,EAAE,+CAA+C,EAAE;gBAClH,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE;aAC5G;YACD,QAAQ,EAAE,CAAC,QAAQ,CAAC;SACrB;KACF;IACD;QACE,IAAI,EAAE,sBAAsB;QAC5B,WAAW,EAAE,+KAA+K;QAC5L,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,uBAAuB,EAAE;gBAChE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,yBAAyB,EAAE;aAClE;YACD,QAAQ,EAAE,CAAC,QAAQ,CAAC;SACrB;KACF;IACD;QACE,IAAI,EAAE,oBAAoB;QAC1B,WAAW,EAAE,sKAAsK;QACnL,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,EAAE,EAAE;KAChD;IACD;QACE,IAAI,EAAE,mBAAmB;QACzB,WAAW,EAAE,iIAAiI;QAC9I,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,EAAE,EAAE;KAChD;IACD;QACE,IAAI,EAAE,uBAAuB;QAC7B,WAAW,EAAE,kJAAkJ;QAC/J,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,EAAE,EAAE;KAChD;IACD;QACE,IAAI,EAAE,wBAAwB;QAC9B,WAAW,EAAE,8HAA8H;QAC3I,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,+CAA+C,EAAE,EAAE;SACxG;KACF;IACD;QACE,IAAI,EAAE,kBAAkB;QACxB,WAAW,EAAE,gIAAgI;QAC7I,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,EAAE,EAAE;KAChD;IACD;QACE,IAAI,EAAE,oBAAoB;QAC1B,WAAW,EAAE,sNAAsN;QACnO,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,gBAAgB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,iBAAiB,EAAE;gBACpE,cAAc,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,sCAAsC,EAAE;gBACvF,WAAW,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,8CAA8C,EAAE;aAC9F;YACD,QAAQ,EAAE,CAAC,kBAAkB,CAAC;SAC/B;KACF;IACD;QACE,IAAI,EAAE,sBAAsB;QAC5B,WAAW,EAAE,gIAAgI;QAC7I,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,gBAAgB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,qCAAqC,EAAE;gBACxF,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,0BAA0B,EAAE;aACnE;YACD,QAAQ,EAAE,CAAC,kBAAkB,CAAC;SAC/B;KACF;IACD;QACE,IAAI,EAAE,oBAAoB;QAC1B,WAAW,EAAE,oLAAoL;QACjM,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,yBAAyB,EAAE;gBAChE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,0CAA0C,EAAE;aACnF;YACD,QAAQ,EAAE,CAAC,MAAM,CAAC;SACnB;KACF;IACD;QACE,IAAI,EAAE,wBAAwB;QAC9B,WAAW,EAAE,6KAA6K;QAC1L,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,QAAQ,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,sBAAsB,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;gBAC3F,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,EAAE,WAAW,EAAE,uCAAuC,EAAE;gBACpH,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,qBAAqB,EAAE;aACnE;YACD,QAAQ,EAAE,CAAC,UAAU,CAAC;SACvB;KACF;IACD;QACE,IAAI,EAAE,sBAAsB;QAC5B,WAAW,EAAE,iMAAiM;QAC9M,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,uBAAuB,EAAE;gBAChE,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,WAAW,EAAE,oBAAoB,EAAE;gBACpG,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,qCAAqC,EAAE;aAC9E;YACD,QAAQ,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC;SAChC;KACF;IACD;QACE,IAAI,EAAE,qBAAqB;QAC3B,WAAW,EAAE,mKAAmK;QAChL,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,OAAO,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,WAAW,EAAE,oBAAoB,EAAE;gBACxF,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,sCAAsC,EAAE;gBACpF,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,gCAAgC,EAAE;aACzE;YACD,QAAQ,EAAE,CAAC,SAAS,CAAC;SACtB;KACF;CACF,CAAC;AAEF;;;;;;;;;;;;GAYG;AAEH,kBAAe;IACb,WAAW,EAAE,wBAAW;IACxB,UAAU,EAAE,uBAAU;IACtB,cAAc,EAAE,sBAAc;CAC/B,CAAC"}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A3M Router - Speculative Decoding Interface
|
|
3
|
+
*
|
|
4
|
+
* Interface for integrating speculative decoding.
|
|
5
|
+
* Currently a stub/interface for future Medusa/Lookahead integration.
|
|
6
|
+
*
|
|
7
|
+
* Speculative decoding uses a smaller "draft" model to predict
|
|
8
|
+
* multiple tokens ahead, which are then verified in parallel by
|
|
9
|
+
* the main model. This can provide 2-3x speedup in generation.
|
|
10
|
+
*
|
|
11
|
+
* Usage:
|
|
12
|
+
* const specDec = new SpeculativeDecoding();
|
|
13
|
+
* if (specDec.shouldUse(true)) {
|
|
14
|
+
* const draftModel = specDec.getDraftModel('medusa');
|
|
15
|
+
* // ... use draft model to generate and verify
|
|
16
|
+
* }
|
|
17
|
+
*/
|
|
18
|
+
export type DraftModelType = 'medusa' | 'lookahead' | 'eagle' | 'spark';
|
|
19
|
+
export interface SpeculativeConfig {
|
|
20
|
+
/** Enable speculative decoding */
|
|
21
|
+
enabled: boolean;
|
|
22
|
+
/** Draft model type */
|
|
23
|
+
draftModelType?: DraftModelType;
|
|
24
|
+
/** Number of speculative tokens to generate */
|
|
25
|
+
speculationWindow?: number;
|
|
26
|
+
/** Temperature for draft model */
|
|
27
|
+
temperature?: number;
|
|
28
|
+
/** Provider to use for draft model */
|
|
29
|
+
provider?: string;
|
|
30
|
+
/** Model to use for draft model */
|
|
31
|
+
model?: string;
|
|
32
|
+
}
|
|
33
|
+
export interface DraftModel {
|
|
34
|
+
/** Model identifier */
|
|
35
|
+
id: string;
|
|
36
|
+
/** Model type */
|
|
37
|
+
type: DraftModelType;
|
|
38
|
+
/** Provider for this model */
|
|
39
|
+
provider: string;
|
|
40
|
+
/** Model size description */
|
|
41
|
+
size: string;
|
|
42
|
+
/** Supported speculation windows */
|
|
43
|
+
supportedWindows: number[];
|
|
44
|
+
}
|
|
45
|
+
export interface SpeculativeResult {
|
|
46
|
+
/** Whether speculative decoding was used */
|
|
47
|
+
used: boolean;
|
|
48
|
+
/** Number of tokens in draft */
|
|
49
|
+
draftTokens: number;
|
|
50
|
+
/** Number of tokens accepted */
|
|
51
|
+
acceptedTokens: number;
|
|
52
|
+
/** Acceptance rate */
|
|
53
|
+
acceptanceRate: number;
|
|
54
|
+
/** Time saved (ms, estimated) */
|
|
55
|
+
estimatedTimeSaved: number;
|
|
56
|
+
}
|
|
57
|
+
export interface VerificationResult {
|
|
58
|
+
/** All tokens verified successfully */
|
|
59
|
+
allAccepted: boolean;
|
|
60
|
+
/** Indices of accepted tokens */
|
|
61
|
+
acceptedIndices: number[];
|
|
62
|
+
/** Indices of rejected tokens */
|
|
63
|
+
rejectedIndices: number[];
|
|
64
|
+
/** Actual tokens to use (may differ from draft) */
|
|
65
|
+
actualTokens: string[];
|
|
66
|
+
/** Number of tokens to rewind */
|
|
67
|
+
rewindCount: number;
|
|
68
|
+
}
|
|
69
|
+
export declare class SpeculativeDecoding {
|
|
70
|
+
private config;
|
|
71
|
+
private draftModels;
|
|
72
|
+
constructor(config?: Partial<SpeculativeConfig>);
|
|
73
|
+
/**
|
|
74
|
+
* Determine if speculative decoding should be used.
|
|
75
|
+
* Checks config, provider support, and model availability.
|
|
76
|
+
*/
|
|
77
|
+
shouldUse(forceEnable?: boolean): boolean;
|
|
78
|
+
/**
|
|
79
|
+
* Get the draft model configuration.
|
|
80
|
+
*/
|
|
81
|
+
getDraftModel(type?: DraftModelType): DraftModel | null;
|
|
82
|
+
/**
|
|
83
|
+
* Generate draft tokens using the draft model.
|
|
84
|
+
* This is an async method to support API-based draft models.
|
|
85
|
+
*/
|
|
86
|
+
generateDraft(prompt: string, maxTokens: number): Promise<{
|
|
87
|
+
tokens: string[];
|
|
88
|
+
scores: number[];
|
|
89
|
+
}>;
|
|
90
|
+
/**
|
|
91
|
+
* Verify draft tokens against the main model.
|
|
92
|
+
* Returns which tokens were accepted and corrections if needed.
|
|
93
|
+
*/
|
|
94
|
+
verifyDraft(prompt: string, draftTokens: string[]): Promise<VerificationResult>;
|
|
95
|
+
/**
|
|
96
|
+
* Calculate speedup from speculative decoding results.
|
|
97
|
+
*/
|
|
98
|
+
calculateSpeedup(result: SpeculativeResult): number;
|
|
99
|
+
/**
|
|
100
|
+
* Update configuration.
|
|
101
|
+
*/
|
|
102
|
+
updateConfig(updates: Partial<SpeculativeConfig>): void;
|
|
103
|
+
/**
|
|
104
|
+
* Get current configuration.
|
|
105
|
+
*/
|
|
106
|
+
getConfig(): SpeculativeConfig;
|
|
107
|
+
/**
|
|
108
|
+
* Check if a provider supports speculative decoding.
|
|
109
|
+
*/
|
|
110
|
+
supportsSpeculativeDecoding(provider: string): boolean;
|
|
111
|
+
/**
|
|
112
|
+
* Get recommended speculation window based on model size.
|
|
113
|
+
*/
|
|
114
|
+
getRecommendedWindow(modelSize: 'small' | 'medium' | 'large'): number;
|
|
115
|
+
private registerDraftModels;
|
|
116
|
+
}
|
|
117
|
+
export declare class SpeculativeDecodingWrapper {
|
|
118
|
+
private specDec;
|
|
119
|
+
private mainModelCall;
|
|
120
|
+
constructor(mainModelCall: (prompt: string, options?: any) => Promise<string>, config?: Partial<SpeculativeConfig>);
|
|
121
|
+
/**
|
|
122
|
+
* Generate with speculative decoding.
|
|
123
|
+
*/
|
|
124
|
+
generate(prompt: string, maxTokens: number, options?: {
|
|
125
|
+
temperature?: number;
|
|
126
|
+
useSpecDec?: boolean;
|
|
127
|
+
}): Promise<{
|
|
128
|
+
text: string;
|
|
129
|
+
result: SpeculativeResult;
|
|
130
|
+
}>;
|
|
131
|
+
}
|
|
132
|
+
export declare function createSpeculativeDecoding(config?: Partial<SpeculativeConfig>): SpeculativeDecoding;
|
|
133
|
+
//# sourceMappingURL=speculativeDecoding.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"speculativeDecoding.d.ts","sourceRoot":"","sources":["../../src/inference/speculativeDecoding.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAQH,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,WAAW,GAAG,OAAO,GAAG,OAAO,CAAC;AAExE,MAAM,WAAW,iBAAiB;IAChC,kCAAkC;IAClC,OAAO,EAAE,OAAO,CAAC;IACjB,uBAAuB;IACvB,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,+CAA+C;IAC/C,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,kCAAkC;IAClC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,sCAAsC;IACtC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,mCAAmC;IACnC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IACzB,uBAAuB;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,iBAAiB;IACjB,IAAI,EAAE,cAAc,CAAC;IACrB,8BAA8B;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,6BAA6B;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,oCAAoC;IACpC,gBAAgB,EAAE,MAAM,EAAE,CAAC;CAC5B;AAED,MAAM,WAAW,iBAAiB;IAChC,4CAA4C;IAC5C,IAAI,EAAE,OAAO,CAAC;IACd,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,gCAAgC;IAChC,cAAc,EAAE,MAAM,CAAC;IACvB,sBAAsB;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,iCAAiC;IACjC,kBAAkB,EAAE,MAAM,CAAC;CAC5B;AAED,MAAM,WAAW,kBAAkB;IACjC,uCAAuC;IACvC,WAAW,EAAE,OAAO,CAAC;IACrB,iCAAiC;IACjC,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,iCAAiC;IACjC,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,mDAAmD;IACnD,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,iCAAiC;IACjC,WAAW,EAAE,MAAM,CAAC;CACrB;AAMD,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,WAAW,CAAsC;gBAE7C,MAAM,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC;IAc/C;;;OAGG;IACH,SAAS,CAAC,WAAW,CAAC,EAAE,OAAO,GAAG,OAAO;IASzC;;OAEG;IACH,aAAa,CAAC,IAAI,CAAC,EAAE,cAAc,GAAG,UAAU,GAAG,IAAI;IAkBvD;;;OAGG;IACG,aAAa,CACjB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,EAAE,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAclD;;;OAGG;IACG,WAAW,CACf,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,MAAM,EAAE,GACpB,OAAO,CAAC,kBAAkB,CAAC;IAsB9B;;OAEG;IACH,gBAAgB,CAAC,MAAM,EAAE,iBAAiB,GAAG,MAAM;IAgBnD;;OAEG;IACH,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,iBAAiB,CAAC,GAAG,IAAI;IAIvD;;OAEG;IACH,SAAS,IAAI,iBAAiB;IAI9B;;OAEG;IACH,2BAA2B,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAUtD;;OAEG;IACH,oBAAoB,CAAC,SAAS,EAAE,OAAO,GAAG,QAAQ,GAAG,OAAO,GAAG,MAAM;IAWrE,OAAO,CAAC,mBAAmB;CAqC5B;AAMD,qBAAa,0BAA0B;IACrC,OAAO,CAAC,OAAO,CAAsB;IACrC,OAAO,CAAC,aAAa,CAAqD;gBAGxE,aAAa,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,KAAK,OAAO,CAAC,MAAM,CAAC,EACjE,MAAM,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC;IAMrC;;OAEG;IACG,QAAQ,CACZ,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,OAAO,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,OAAO,CAAA;KAAE,GACvD,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,iBAAiB,CAAA;KAAE,CAAC;CAsDxD;AAMD,wBAAgB,yBAAyB,CACvC,MAAM,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC,GAClC,mBAAmB,CAErB"}
|