@goodandready/dsh-model-sync 0.5.15 → 0.5.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/README.ru.md +5 -0
- package/README.zh.md +5 -0
- package/lib/adapter-registry.js +4 -2
- package/lib/client.js +214 -6
- package/lib/http.js +81 -1
- package/lib/models.js +206 -6
- package/lib/pricing-database.js +150 -17
- package/lib/provider-service.js +76 -3
- package/lib/reconcile.js +12 -2
- package/lib/scheduler.js +7 -0
- package/lib/specs-database.js +223 -0
- package/lib/synchronizer-prober.js +117 -35
- package/lib/synchronizer.js +25 -1
- package/package.json +1 -1
package/lib/scheduler.js
CHANGED
|
@@ -24,6 +24,7 @@ export function createSyncScheduler(sync, {
|
|
|
24
24
|
} = {}) {
|
|
25
25
|
let timer = null
|
|
26
26
|
let running = false
|
|
27
|
+
let lastPricingSyncAt = 0
|
|
27
28
|
const state = new Map()
|
|
28
29
|
|
|
29
30
|
function stop() {
|
|
@@ -82,6 +83,12 @@ export function createSyncScheduler(sync, {
|
|
|
82
83
|
async function tick() {
|
|
83
84
|
const config = getConfig() ?? {}
|
|
84
85
|
if (config.enabled === false || config.scheduleEnabled === false || running) return
|
|
86
|
+
if (typeof sync.syncPricing === 'function' && (nowFn() - lastPricingSyncAt > 24 * 60 * 60 * 1000 || lastPricingSyncAt === 0)) {
|
|
87
|
+
lastPricingSyncAt = nowFn()
|
|
88
|
+
sync.syncPricing().catch((err) => {
|
|
89
|
+
logger.warn?.('[dsh-model-sync] background pricing sync failed: ' + errorMessage(err))
|
|
90
|
+
})
|
|
91
|
+
}
|
|
85
92
|
resetStates(config)
|
|
86
93
|
const rows = scheduleRows(config)
|
|
87
94
|
if (rows.length === 0) return
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
function cleanId(id) {
|
|
2
|
+
if (typeof id !== 'string') return ''
|
|
3
|
+
return id
|
|
4
|
+
.toLowerCase()
|
|
5
|
+
.trim()
|
|
6
|
+
.replace(/^models\//, '')
|
|
7
|
+
.replace(/:latest$/, '')
|
|
8
|
+
.replace(/-latest$/, '')
|
|
9
|
+
.replace(/-(202\d{5}|202\d-\d{2}-\d{2}|\d{4})$/, '')
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export const REFERENCE_SPECS = Object.freeze({
|
|
13
|
+
openai: {
|
|
14
|
+
'gpt-4o': { contextWindow: 128000, maxOutputTokens: 16384 },
|
|
15
|
+
'gpt-4o-mini': { contextWindow: 128000, maxOutputTokens: 16384 },
|
|
16
|
+
'o1': {
|
|
17
|
+
contextWindow: 200000,
|
|
18
|
+
maxOutputTokens: 100000,
|
|
19
|
+
reasoningConfig: { supportsReasoning: true, paramFormat: 'effort', defaultEffort: 'medium', supportedEfforts: ['low', 'medium', 'high'] },
|
|
20
|
+
compat: { supportsTemperature: false, supportsMaxTokens: false },
|
|
21
|
+
},
|
|
22
|
+
'o1-preview': {
|
|
23
|
+
contextWindow: 128000,
|
|
24
|
+
maxOutputTokens: 32768,
|
|
25
|
+
reasoningConfig: { supportsReasoning: true, paramFormat: 'effort', defaultEffort: 'medium', supportedEfforts: ['low', 'medium', 'high'] },
|
|
26
|
+
compat: { supportsTemperature: false, supportsMaxTokens: false },
|
|
27
|
+
},
|
|
28
|
+
'o1-mini': {
|
|
29
|
+
contextWindow: 128000,
|
|
30
|
+
maxOutputTokens: 65536,
|
|
31
|
+
reasoningConfig: { supportsReasoning: true, paramFormat: 'effort', defaultEffort: 'medium', supportedEfforts: ['low', 'medium', 'high'] },
|
|
32
|
+
compat: { supportsTemperature: false, supportsMaxTokens: false },
|
|
33
|
+
},
|
|
34
|
+
'o3-mini': {
|
|
35
|
+
contextWindow: 200000,
|
|
36
|
+
maxOutputTokens: 100000,
|
|
37
|
+
reasoningConfig: { supportsReasoning: true, paramFormat: 'effort', defaultEffort: 'medium', supportedEfforts: ['low', 'medium', 'high'] },
|
|
38
|
+
compat: { supportsTemperature: false, supportsMaxTokens: false },
|
|
39
|
+
},
|
|
40
|
+
'gpt-4-turbo': { contextWindow: 128000, maxOutputTokens: 4096 },
|
|
41
|
+
'gpt-4': { contextWindow: 8192, maxOutputTokens: 4096 },
|
|
42
|
+
'gpt-3.5-turbo': { contextWindow: 16385, maxOutputTokens: 4096 },
|
|
43
|
+
'chatgpt-4o-latest': { contextWindow: 128000, maxOutputTokens: 16384 },
|
|
44
|
+
},
|
|
45
|
+
anthropic: {
|
|
46
|
+
'claude-3-7-sonnet': {
|
|
47
|
+
contextWindow: 200000,
|
|
48
|
+
maxOutputTokens: 64000,
|
|
49
|
+
reasoningConfig: { supportsReasoning: true, paramFormat: 'budget_tokens', defaultBudget: 4096, maxBudget: 64000 },
|
|
50
|
+
},
|
|
51
|
+
'claude-3-5-sonnet': { contextWindow: 200000, maxOutputTokens: 8192 },
|
|
52
|
+
'claude-3-5-haiku': { contextWindow: 200000, maxOutputTokens: 8192 },
|
|
53
|
+
'claude-3-opus': { contextWindow: 200000, maxOutputTokens: 4096 },
|
|
54
|
+
'claude-3-haiku': { contextWindow: 200000, maxOutputTokens: 4096 },
|
|
55
|
+
},
|
|
56
|
+
deepseek: {
|
|
57
|
+
'deepseek-chat': { contextWindow: 64000, maxOutputTokens: 8192 },
|
|
58
|
+
'deepseek-reasoner': {
|
|
59
|
+
contextWindow: 64000,
|
|
60
|
+
maxOutputTokens: 8192,
|
|
61
|
+
reasoningConfig: { supportsReasoning: true, paramFormat: 'boolean' },
|
|
62
|
+
},
|
|
63
|
+
'deepseek-coder': { contextWindow: 16384, maxOutputTokens: 4096 },
|
|
64
|
+
},
|
|
65
|
+
google: {
|
|
66
|
+
'gemini-2.0-flash': { contextWindow: 1048576, maxOutputTokens: 8192 },
|
|
67
|
+
'gemini-2.0-flash-thinking-exp': {
|
|
68
|
+
contextWindow: 1048576,
|
|
69
|
+
maxOutputTokens: 65536,
|
|
70
|
+
reasoningConfig: { supportsReasoning: true, paramFormat: 'budget_tokens', defaultBudget: 8192, maxBudget: 65536 },
|
|
71
|
+
},
|
|
72
|
+
'gemini-2.0-flash-lite': { contextWindow: 1048576, maxOutputTokens: 8192 },
|
|
73
|
+
'gemini-1.5-pro': { contextWindow: 2097152, maxOutputTokens: 8192 },
|
|
74
|
+
'gemini-1.5-flash': { contextWindow: 1048576, maxOutputTokens: 8192 },
|
|
75
|
+
'gemini-1.5-flash-8b': { contextWindow: 1048576, maxOutputTokens: 8192 },
|
|
76
|
+
},
|
|
77
|
+
groq: {
|
|
78
|
+
'llama-3.3-70b-versatile': { contextWindow: 131072, maxOutputTokens: 32768 },
|
|
79
|
+
'llama-3.3-70b-specdec': { contextWindow: 8192, maxOutputTokens: 8192 },
|
|
80
|
+
'llama-3.1-70b-versatile': { contextWindow: 131072, maxOutputTokens: 32768 },
|
|
81
|
+
'llama-3.1-8b-instant': { contextWindow: 131072, maxOutputTokens: 8192 },
|
|
82
|
+
'llama3-70b-8192': { contextWindow: 8192, maxOutputTokens: 8192 },
|
|
83
|
+
'llama3-8b-8192': { contextWindow: 8192, maxOutputTokens: 8192 },
|
|
84
|
+
'mixtral-8x7b-32768': { contextWindow: 32768, maxOutputTokens: 32768 },
|
|
85
|
+
'gemma2-9b-it': { contextWindow: 8192, maxOutputTokens: 8192 },
|
|
86
|
+
'deepseek-r1-distill-llama-70b': {
|
|
87
|
+
contextWindow: 131072,
|
|
88
|
+
maxOutputTokens: 32768,
|
|
89
|
+
reasoningConfig: { supportsReasoning: true, paramFormat: 'budget_tokens', defaultBudget: 4096, maxBudget: 32768 },
|
|
90
|
+
},
|
|
91
|
+
'deepseek-r1-distill-qwen-32b': {
|
|
92
|
+
contextWindow: 131072,
|
|
93
|
+
maxOutputTokens: 32768,
|
|
94
|
+
reasoningConfig: { supportsReasoning: true, paramFormat: 'budget_tokens', defaultBudget: 4096, maxBudget: 32768 },
|
|
95
|
+
},
|
|
96
|
+
'qwen-2.5-32b': { contextWindow: 131072, maxOutputTokens: 8192 },
|
|
97
|
+
'qwen-2.5-coder-32b': { contextWindow: 131072, maxOutputTokens: 8192 },
|
|
98
|
+
'qwen-qwq-32b': {
|
|
99
|
+
contextWindow: 131072,
|
|
100
|
+
maxOutputTokens: 32768,
|
|
101
|
+
reasoningConfig: { supportsReasoning: true, paramFormat: 'budget_tokens', defaultBudget: 4096, maxBudget: 32768 },
|
|
102
|
+
},
|
|
103
|
+
},
|
|
104
|
+
xai: {
|
|
105
|
+
'grok-2': { contextWindow: 131072, maxOutputTokens: 8192 },
|
|
106
|
+
'grok-2-mini': { contextWindow: 131072, maxOutputTokens: 8192 },
|
|
107
|
+
'grok-2-vision': { contextWindow: 131072, maxOutputTokens: 8192 },
|
|
108
|
+
'grok-beta': { contextWindow: 131072, maxOutputTokens: 8192 },
|
|
109
|
+
},
|
|
110
|
+
mistral: {
|
|
111
|
+
'mistral-large': { contextWindow: 128000, maxOutputTokens: 8192 },
|
|
112
|
+
'mistral-small': { contextWindow: 128000, maxOutputTokens: 8192 },
|
|
113
|
+
'codestral': { contextWindow: 256000, maxOutputTokens: 8192 },
|
|
114
|
+
'ministral-8b': { contextWindow: 128000, maxOutputTokens: 8192 },
|
|
115
|
+
'ministral-3b': { contextWindow: 128000, maxOutputTokens: 8192 },
|
|
116
|
+
'pixtral-large': { contextWindow: 128000, maxOutputTokens: 8192 },
|
|
117
|
+
'pixtral-12b': { contextWindow: 128000, maxOutputTokens: 8192 },
|
|
118
|
+
},
|
|
119
|
+
})
|
|
120
|
+
|
|
121
|
+
export function lookupReferenceSpec(provider, modelId) {
|
|
122
|
+
if (typeof modelId !== 'string' || !modelId.trim()) return undefined
|
|
123
|
+
const provKey = String(provider || '').toLowerCase().trim()
|
|
124
|
+
const rawId = modelId.toLowerCase().trim()
|
|
125
|
+
const cleaned = cleanId(rawId)
|
|
126
|
+
|
|
127
|
+
// 1. Direct provider match
|
|
128
|
+
const table = REFERENCE_SPECS[provKey]
|
|
129
|
+
if (table) {
|
|
130
|
+
if (table[rawId]) return { ...table[rawId] }
|
|
131
|
+
if (table[cleaned]) return { ...table[cleaned] }
|
|
132
|
+
return undefined
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// 2. Slash prefix (e.g. "anthropic/claude-3-7-sonnet" or "openai/o3-mini")
|
|
136
|
+
const slashIdx = rawId.indexOf('/')
|
|
137
|
+
if (slashIdx > 0) {
|
|
138
|
+
const subProv = rawId.slice(0, slashIdx)
|
|
139
|
+
const subId = rawId.slice(slashIdx + 1)
|
|
140
|
+
const subCleaned = cleanId(subId)
|
|
141
|
+
const subTable = REFERENCE_SPECS[subProv]
|
|
142
|
+
if (subTable) {
|
|
143
|
+
if (subTable[subId]) return { ...subTable[subId] }
|
|
144
|
+
if (subTable[subCleaned]) return { ...subTable[subCleaned] }
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// 3. Fallback across all tables for aggregators (OpenRouter, ClineBot, CommandCode)
|
|
149
|
+
for (const [, t] of Object.entries(REFERENCE_SPECS)) {
|
|
150
|
+
if (t[rawId]) return { ...t[rawId] }
|
|
151
|
+
if (t[cleaned]) return { ...t[cleaned] }
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
return undefined
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
export function lookupReferenceLimits(provider, modelId) {
|
|
158
|
+
const spec = lookupReferenceSpec(provider, modelId)
|
|
159
|
+
if (!spec) return undefined
|
|
160
|
+
const res = {}
|
|
161
|
+
if (spec.contextWindow !== undefined) res.contextWindow = spec.contextWindow
|
|
162
|
+
if (spec.maxOutputTokens !== undefined) res.maxOutputTokens = spec.maxOutputTokens
|
|
163
|
+
return Object.keys(res).length > 0 ? res : undefined
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
export function lookupReferenceReasoning(provider, modelId) {
|
|
167
|
+
const spec = lookupReferenceSpec(provider, modelId)
|
|
168
|
+
return spec?.reasoningConfig ? structuredClone(spec.reasoningConfig) : undefined
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
export function lookupReferenceCompat(provider, modelId) {
|
|
172
|
+
const spec = lookupReferenceSpec(provider, modelId)
|
|
173
|
+
return spec?.compat ? structuredClone(spec.compat) : undefined
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
export function clampOutputTokens(modelOrLimit, requestedTokens) {
|
|
177
|
+
const limit = typeof modelOrLimit === 'number'
|
|
178
|
+
? modelOrLimit
|
|
179
|
+
: Number(modelOrLimit?.maxOutputTokens ?? modelOrLimit?.maxTokens)
|
|
180
|
+
if (!Number.isFinite(limit) || limit <= 0) {
|
|
181
|
+
return { clamped: false, value: requestedTokens, limit: undefined }
|
|
182
|
+
}
|
|
183
|
+
const requested = Number(requestedTokens)
|
|
184
|
+
if (!Number.isFinite(requested) || requested <= 0) {
|
|
185
|
+
return { clamped: false, value: requestedTokens, limit }
|
|
186
|
+
}
|
|
187
|
+
if (requested > limit) {
|
|
188
|
+
return { clamped: true, value: limit, original: requested, limit }
|
|
189
|
+
}
|
|
190
|
+
return { clamped: false, value: requested, limit }
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
export function adaptReasoningPayload(model, userOptions = {}) {
|
|
194
|
+
const warnings = []
|
|
195
|
+
const payload = {}
|
|
196
|
+
const config = model?.reasoningConfig
|
|
197
|
+
const format = config?.paramFormat || 'none'
|
|
198
|
+
const compat = model?.compat || {}
|
|
199
|
+
|
|
200
|
+
if (format === 'effort') {
|
|
201
|
+
const effort = userOptions.reasoningEffort || config?.defaultEffort || 'medium'
|
|
202
|
+
payload.reasoning_effort = effort
|
|
203
|
+
if (compat.supportsTemperature === false && userOptions.temperature !== undefined && userOptions.temperature !== 1) {
|
|
204
|
+
warnings.push(`temperature ${userOptions.temperature} is not supported by model ${model.id}; omitted`)
|
|
205
|
+
}
|
|
206
|
+
if (compat.supportsMaxTokens === false && userOptions.maxTokens !== undefined) {
|
|
207
|
+
payload.max_completion_tokens = userOptions.maxTokens
|
|
208
|
+
}
|
|
209
|
+
} else if (format === 'budget_tokens') {
|
|
210
|
+
const maxBudget = config?.maxBudget || 64000
|
|
211
|
+
const defaultBudget = config?.defaultBudget || 4096
|
|
212
|
+
const budget = Math.min(Math.max(1, Number(userOptions.thinkingBudget || defaultBudget)), maxBudget)
|
|
213
|
+
if (model?.provider === 'google' || String(model?.id || '').startsWith('gemini')) {
|
|
214
|
+
payload.thinking_config = { thinking_budget: budget }
|
|
215
|
+
} else {
|
|
216
|
+
payload.thinking = { type: 'enabled', budget_tokens: budget }
|
|
217
|
+
}
|
|
218
|
+
} else if (format === 'boolean') {
|
|
219
|
+
payload.thinking = true
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
return { payload, warnings, format }
|
|
223
|
+
}
|
|
@@ -1,3 +1,31 @@
|
|
|
1
|
+
const SPARKLINE_BARS = [' ', '▂', '▃', '▄', '▅', '▆', '▇', '█']
|
|
2
|
+
|
|
3
|
+
export function generateSparkline(values) {
|
|
4
|
+
if (!Array.isArray(values) || values.length === 0) return ''
|
|
5
|
+
const nums = values.filter((v) => typeof v === 'number' && Number.isFinite(v))
|
|
6
|
+
if (nums.length === 0) return ''
|
|
7
|
+
if (nums.length === 1) return '▄'
|
|
8
|
+
const min = Math.min(...nums)
|
|
9
|
+
const max = Math.max(...nums)
|
|
10
|
+
if (min === max) return '▄'.repeat(nums.length)
|
|
11
|
+
const range = max - min
|
|
12
|
+
return nums
|
|
13
|
+
.map((v) => {
|
|
14
|
+
const idx = Math.min(7, Math.max(0, Math.floor(((v - min) / range) * 7)))
|
|
15
|
+
return SPARKLINE_BARS[idx]
|
|
16
|
+
})
|
|
17
|
+
.join('')
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export function classifySpeedTier(latencyMs, isOk = true) {
|
|
21
|
+
if (!isOk) return 'degraded'
|
|
22
|
+
if (typeof latencyMs !== 'number' || !Number.isFinite(latencyMs) || latencyMs <= 0) return 'unknown'
|
|
23
|
+
if (latencyMs < 300) return 'lightning'
|
|
24
|
+
if (latencyMs < 1200) return 'fast'
|
|
25
|
+
if (latencyMs <= 3000) return 'moderate'
|
|
26
|
+
return 'slow'
|
|
27
|
+
}
|
|
28
|
+
|
|
1
29
|
export function createModelProber({
|
|
2
30
|
health,
|
|
3
31
|
getProviderCatalog = () => [],
|
|
@@ -5,8 +33,32 @@ export function createModelProber({
|
|
|
5
33
|
inferModel,
|
|
6
34
|
} = {}) {
|
|
7
35
|
const modelProbes = new Map()
|
|
36
|
+
const latencyHistory = new Map()
|
|
8
37
|
const probeFn = probeModel ?? inferModel
|
|
9
38
|
|
|
39
|
+
function recordLatencyEntry(provider, model, { latencyMs, ttftMs, ok, status, error }) {
|
|
40
|
+
const key = `${provider}:${model}`
|
|
41
|
+
modelProbes.set(key, {
|
|
42
|
+
latencyMs,
|
|
43
|
+
ttftMs,
|
|
44
|
+
ok,
|
|
45
|
+
status,
|
|
46
|
+
timestamp: Date.now(),
|
|
47
|
+
})
|
|
48
|
+
|
|
49
|
+
const hist = latencyHistory.get(key) || []
|
|
50
|
+
hist.push({
|
|
51
|
+
timestamp: Date.now(),
|
|
52
|
+
latencyMs,
|
|
53
|
+
ttftMs,
|
|
54
|
+
ok,
|
|
55
|
+
status,
|
|
56
|
+
...(error ? { error } : {}),
|
|
57
|
+
})
|
|
58
|
+
if (hist.length > 10) hist.shift()
|
|
59
|
+
latencyHistory.set(key, hist)
|
|
60
|
+
}
|
|
61
|
+
|
|
10
62
|
async function tryModel({ provider, model, signal } = {}) {
|
|
11
63
|
if (typeof provider !== 'string' || !provider) throw new Error('provider is required')
|
|
12
64
|
if (typeof model !== 'string' || !model) throw new Error('model is required')
|
|
@@ -21,15 +73,11 @@ export function createModelProber({
|
|
|
21
73
|
model,
|
|
22
74
|
ok: false,
|
|
23
75
|
latencyMs: 0,
|
|
76
|
+
ttftMs: 0,
|
|
24
77
|
status: 'model_not_found',
|
|
25
78
|
error: `model "${model}" not found in provider catalog`,
|
|
26
79
|
}
|
|
27
|
-
|
|
28
|
-
latencyMs: 0,
|
|
29
|
-
ok: false,
|
|
30
|
-
status: 'model_not_found',
|
|
31
|
-
timestamp: Date.now(),
|
|
32
|
-
})
|
|
80
|
+
recordLatencyEntry(provider, model, probeResult)
|
|
33
81
|
return probeResult
|
|
34
82
|
}
|
|
35
83
|
|
|
@@ -38,6 +86,9 @@ export function createModelProber({
|
|
|
38
86
|
try {
|
|
39
87
|
const probeRes = await probeFn({ provider, model, signal })
|
|
40
88
|
const latencyMs = Math.max(1, Date.now() - start)
|
|
89
|
+
const ttftMs = typeof probeRes?.ttftMs === 'number'
|
|
90
|
+
? probeRes.ttftMs
|
|
91
|
+
: (typeof probeRes?.metrics?.ttft === 'number' ? probeRes.metrics.ttft : Math.round(latencyMs * 0.45))
|
|
41
92
|
const isOk = probeRes?.ok !== false
|
|
42
93
|
const status = isOk ? 'ok' : (probeRes?.status || 'error')
|
|
43
94
|
const errorMsg = isOk ? undefined : (probeRes?.error || 'inference probe failed')
|
|
@@ -46,33 +97,26 @@ export function createModelProber({
|
|
|
46
97
|
model,
|
|
47
98
|
ok: isOk,
|
|
48
99
|
latencyMs,
|
|
100
|
+
ttftMs,
|
|
49
101
|
status,
|
|
50
102
|
...(errorMsg ? { error: errorMsg } : {}),
|
|
51
103
|
}
|
|
52
|
-
|
|
53
|
-
latencyMs,
|
|
54
|
-
ok: isOk,
|
|
55
|
-
status,
|
|
56
|
-
timestamp: Date.now(),
|
|
57
|
-
})
|
|
104
|
+
recordLatencyEntry(provider, model, probeResult)
|
|
58
105
|
return probeResult
|
|
59
106
|
} catch (err) {
|
|
60
107
|
const latencyMs = Math.max(1, Date.now() - start)
|
|
108
|
+
const ttftMs = Math.round(latencyMs * 0.45)
|
|
61
109
|
const status = err?.status ? `http_${err.status}` : (err?.code || 'error')
|
|
62
110
|
const probeResult = {
|
|
63
111
|
provider,
|
|
64
112
|
model,
|
|
65
113
|
ok: false,
|
|
66
114
|
latencyMs,
|
|
115
|
+
ttftMs,
|
|
67
116
|
status,
|
|
68
117
|
error: err?.message || 'inference probe failed',
|
|
69
118
|
}
|
|
70
|
-
|
|
71
|
-
latencyMs,
|
|
72
|
-
ok: false,
|
|
73
|
-
status,
|
|
74
|
-
timestamp: Date.now(),
|
|
75
|
-
})
|
|
119
|
+
recordLatencyEntry(provider, model, probeResult)
|
|
76
120
|
return probeResult
|
|
77
121
|
}
|
|
78
122
|
}
|
|
@@ -82,6 +126,7 @@ export function createModelProber({
|
|
|
82
126
|
const entry = healthResult?.results?.find((r) => r.provider === provider)
|
|
83
127
|
const providerOk = entry?.status === 'ok'
|
|
84
128
|
const latencyMs = Math.max(1, Date.now() - start)
|
|
129
|
+
const ttftMs = Math.round(latencyMs * 0.45)
|
|
85
130
|
const status = !providerOk ? (entry?.status ?? 'error') : 'ok'
|
|
86
131
|
const errorMsg = !providerOk ? (entry?.message || entry?.status || 'probe failed') : undefined
|
|
87
132
|
|
|
@@ -90,17 +135,12 @@ export function createModelProber({
|
|
|
90
135
|
model,
|
|
91
136
|
ok: providerOk,
|
|
92
137
|
latencyMs,
|
|
138
|
+
ttftMs,
|
|
93
139
|
status,
|
|
94
140
|
...(errorMsg ? { error: errorMsg } : {}),
|
|
95
141
|
}
|
|
96
142
|
|
|
97
|
-
|
|
98
|
-
latencyMs,
|
|
99
|
-
ok: providerOk,
|
|
100
|
-
status,
|
|
101
|
-
timestamp: Date.now(),
|
|
102
|
-
})
|
|
103
|
-
|
|
143
|
+
recordLatencyEntry(provider, model, probeResult)
|
|
104
144
|
return probeResult
|
|
105
145
|
}
|
|
106
146
|
|
|
@@ -110,7 +150,6 @@ export function createModelProber({
|
|
|
110
150
|
const list = [...new Set(models.filter((m) => typeof m === 'string' && m.trim()).map((m) => m.trim()))].slice(0, 50)
|
|
111
151
|
|
|
112
152
|
if (typeof probeFn === 'function') {
|
|
113
|
-
// Execute true model-specific probes with bounded concurrency
|
|
114
153
|
const limit = Math.max(1, Math.min(10, Math.floor(Number(concurrency) || 4)))
|
|
115
154
|
const results = new Array(list.length)
|
|
116
155
|
let cursor = 0
|
|
@@ -131,7 +170,6 @@ export function createModelProber({
|
|
|
131
170
|
}
|
|
132
171
|
}
|
|
133
172
|
|
|
134
|
-
// If no probeFn provided, use provider health once (backwards compatibility)
|
|
135
173
|
const start = Date.now()
|
|
136
174
|
const healthResult = await health?.({ provider, signal })
|
|
137
175
|
const entry = healthResult?.results?.find((r) => r.provider === provider)
|
|
@@ -139,6 +177,7 @@ export function createModelProber({
|
|
|
139
177
|
const catalog = typeof getProviderCatalog === 'function' ? getProviderCatalog(provider) : []
|
|
140
178
|
const hasCatalog = Array.isArray(catalog) && catalog.length > 0
|
|
141
179
|
const baseLatency = Math.max(1, Date.now() - start)
|
|
180
|
+
const ttftMs = Math.round(baseLatency * 0.45)
|
|
142
181
|
|
|
143
182
|
const results = list.map((model) => {
|
|
144
183
|
const inCatalog = !hasCatalog || catalog.some((m) => (typeof m === 'string' ? m : m?.id) === model)
|
|
@@ -153,17 +192,12 @@ export function createModelProber({
|
|
|
153
192
|
model,
|
|
154
193
|
ok,
|
|
155
194
|
latencyMs: baseLatency,
|
|
195
|
+
ttftMs,
|
|
156
196
|
status,
|
|
157
197
|
...(errorMsg ? { error: errorMsg } : {}),
|
|
158
198
|
}
|
|
159
199
|
|
|
160
|
-
|
|
161
|
-
latencyMs: baseLatency,
|
|
162
|
-
ok,
|
|
163
|
-
status,
|
|
164
|
-
timestamp: Date.now(),
|
|
165
|
-
})
|
|
166
|
-
|
|
200
|
+
recordLatencyEntry(provider, model, itemResult)
|
|
167
201
|
return itemResult
|
|
168
202
|
})
|
|
169
203
|
|
|
@@ -179,9 +213,57 @@ export function createModelProber({
|
|
|
179
213
|
return Object.fromEntries(modelProbes)
|
|
180
214
|
}
|
|
181
215
|
|
|
216
|
+
function getLatencyHistory(provider, model) {
|
|
217
|
+
if (provider && model) {
|
|
218
|
+
return latencyHistory.get(`${provider}:${model}`) || []
|
|
219
|
+
}
|
|
220
|
+
return Object.fromEntries(latencyHistory)
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
function getLatencyRadar({ provider } = {}) {
|
|
224
|
+
const radar = []
|
|
225
|
+
for (const [key, hist] of latencyHistory.entries()) {
|
|
226
|
+
if (!Array.isArray(hist) || hist.length === 0) continue
|
|
227
|
+
const [entryProv, ...rest] = key.split(':')
|
|
228
|
+
const entryModel = rest.join(':')
|
|
229
|
+
if (provider && entryProv !== provider) continue
|
|
230
|
+
|
|
231
|
+
const latest = hist[hist.length - 1]
|
|
232
|
+
const latencies = hist.map((h) => h.latencyMs)
|
|
233
|
+
const avgLatencyMs = Math.round(latencies.reduce((acc, v) => acc + v, 0) / latencies.length)
|
|
234
|
+
const sorted = [...latencies].sort((a, b) => a - b)
|
|
235
|
+
const p95Idx = Math.min(sorted.length - 1, Math.floor(sorted.length * 0.95))
|
|
236
|
+
const p95LatencyMs = sorted[p95Idx]
|
|
237
|
+
const sparkline = generateSparkline(latencies)
|
|
238
|
+
const tier = classifySpeedTier(latest.latencyMs, latest.ok)
|
|
239
|
+
|
|
240
|
+
radar.push({
|
|
241
|
+
provider: entryProv,
|
|
242
|
+
model: entryModel,
|
|
243
|
+
latencyMs: latest.latencyMs,
|
|
244
|
+
ttftMs: latest.ttftMs,
|
|
245
|
+
avgLatencyMs,
|
|
246
|
+
p95LatencyMs,
|
|
247
|
+
tier,
|
|
248
|
+
sparkline,
|
|
249
|
+
history: latencies,
|
|
250
|
+
ok: latest.ok,
|
|
251
|
+
status: latest.status,
|
|
252
|
+
lastChecked: latest.timestamp,
|
|
253
|
+
})
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
return {
|
|
257
|
+
updatedAt: Date.now(),
|
|
258
|
+
radar,
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
|
|
182
262
|
return {
|
|
183
263
|
tryModel,
|
|
184
264
|
batchTryModels,
|
|
185
265
|
getModelProbes,
|
|
266
|
+
getLatencyHistory,
|
|
267
|
+
getLatencyRadar,
|
|
186
268
|
}
|
|
187
|
-
}
|
|
269
|
+
}
|
package/lib/synchronizer.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import path from 'node:path'
|
|
2
|
+
import { fetchPricingUpdates, lookupReferencePricing, getDynamicPricingMap } from './pricing-database.js'
|
|
1
3
|
import { dispatchDiffNotifications, sendWebhookNotification } from "./diff-notifier.js"
|
|
2
4
|
import { listApiKeyProviders } from './inventory.js'
|
|
3
5
|
import { createProviderAdapterRegistry } from './adapter-registry.js'
|
|
@@ -583,6 +585,24 @@ export function createModelSynchronizer(ctx, { getConfig = () => ({}), saveConfi
|
|
|
583
585
|
}
|
|
584
586
|
return { provider, selectedModels, models: nextModels, availableModels: available, selectableModels: safeAvailable, applied: true }
|
|
585
587
|
}
|
|
588
|
+
async function syncPricing() {
|
|
589
|
+
const config = getConfig() ?? {}
|
|
590
|
+
const timeoutMs = config.requestTimeoutMs ?? 15000
|
|
591
|
+
const cachePath = path.join(process.cwd(), 'pricing-cache.json')
|
|
592
|
+
return await fetchPricingUpdates({ timeoutMs, fetchFn: fetchImpl, cachePath, logger: ctx?.logger })
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
function getPricingInfo({ provider, model } = {}) {
|
|
596
|
+
if (provider && model) {
|
|
597
|
+
return { provider, model, pricing: lookupReferencePricing(provider, model) }
|
|
598
|
+
}
|
|
599
|
+
const dynamic = Object.fromEntries(getDynamicPricingMap())
|
|
600
|
+
return {
|
|
601
|
+
dynamicCount: Object.keys(dynamic).length,
|
|
602
|
+
sample: dynamic,
|
|
603
|
+
}
|
|
604
|
+
}
|
|
605
|
+
|
|
586
606
|
const transfer = createConfigTransfer({
|
|
587
607
|
getConfig,
|
|
588
608
|
saveConfig,
|
|
@@ -613,6 +633,10 @@ export function createModelSynchronizer(ctx, { getConfig = () => ({}), saveConfi
|
|
|
613
633
|
notifications: notificationLedger,
|
|
614
634
|
updateNotification: updateNotificationState,
|
|
615
635
|
rollbackHistory,
|
|
636
|
+
syncPricing,
|
|
637
|
+
getPricingInfo,
|
|
638
|
+
getLatencyRadar: prober.getLatencyRadar,
|
|
639
|
+
getLatencyHistory: prober.getLatencyHistory,
|
|
616
640
|
testWebhook: async ({ url, secret, secretRef }) => {
|
|
617
641
|
let resolvedSecret = secret || ''
|
|
618
642
|
const targetRef = (secretRef || (typeof secret === 'string' && /^[A-Za-z_][A-Za-z0-9_]*$/.test(secret) ? secret : '')).trim()
|
|
@@ -629,6 +653,6 @@ export function createModelSynchronizer(ctx, { getConfig = () => ({}), saveConfi
|
|
|
629
653
|
}
|
|
630
654
|
return sendWebhookNotification(url, resolvedSecret, { event: 'model-sync.test', timestamp: Date.now(), test: true })
|
|
631
655
|
},
|
|
632
|
-
status: () => ({ running: Boolean(running), lastRun, lastHealth, lastReport: structuredClone(lastReport), notifications: notificationLedger(), history: history(), policies: listPolicies(), lifecycle: structuredClone(getConfig()?.modelLifecycle ?? {}), aliases: structuredClone(getConfig()?.aliases ?? {}) }),
|
|
656
|
+
status: () => ({ running: Boolean(running), lastRun, lastHealth, lastReport: structuredClone(lastReport), notifications: notificationLedger(), history: history(), policies: listPolicies(), lifecycle: structuredClone(getConfig()?.modelLifecycle ?? {}), aliases: structuredClone(getConfig()?.aliases ?? {}), latencyRadar: prober.getLatencyRadar() }),
|
|
633
657
|
})
|
|
634
658
|
}
|